@yumdee/mcp-studio-bench 0.1.0

package/LICENSE

@@ -0,0 +1,23 @@
+# License
+
+MIT License
+
+Copyright (c) 2024 yumdee-Mcp-studio Contributors
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -0,0 +1,22 @@
+# @yumdee/mcp-studio-bench
+
+Compliance & reliability scoring for MCP servers.
+
+## Features
+
+- Spec compliance checks (handshake format, error format, type consistency)
+- Reliability metrics (success rate, latency percentiles)
+- Replay-based scoring (no live network calls)
+- CI-gatable pass/fail gates (`bench check --min-score 80`)
+
+## Usage
+
+```bash
+mcp-studio-bench <server-name>
+mcp-studio-bench <session-id> --replay  # Replay a recorded session
+mcp-studio-bench check --min-score 80   # CI gate
+```
+
+## Development
+
+See [../../CONTRIBUTING.md](../../CONTRIBUTING.md)

package/dist/__tests__/bench.test.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=bench.test.d.ts.map

package/dist/__tests__/bench.test.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"bench.test.d.ts","sourceRoot":"","sources":["../../src/__tests__/bench.test.ts"],"names":[],"mappings":""}

package/dist/__tests__/bench.test.js

@@ -0,0 +1,92 @@
+import { describe, it, expect } from "vitest";
+import { createBenchmark } from "../index.js";
+describe("Benchmark & Compliance Scoring", () => {
+    const sampleSession = {
+        id: "b2c3d4e5-f6a7-4b8c-9d0e-1f2a3b4c5d6e",
+        serverInfo: {
+            name: "math-server",
+            version: "0.1.0",
+            transport: "stdio",
+        },
+        clientInfo: {
+            name: "yumdee-mcp-studio",
+            version: "0.1.0",
+        },
+        startedAt: new Date().toISOString(),
+        events: [
+            {
+                type: "request",
+                timestamp: new Date().toISOString(),
+                id: 1,
+                method: "initialize",
+                sentAtMs: 0,
+            },
+            {
+                type: "response",
+                timestamp: new Date().toISOString(),
+                id: 1,
+                result: { capabilities: {} },
+                receivedAtMs: 15,
+                latencyMs: 15,
+            },
+            {
+                type: "request",
+                timestamp: new Date().toISOString(),
+                id: 2,
+                method: "tools/call",
+                sentAtMs: 20,
+            },
+            {
+                type: "response",
+                timestamp: new Date().toISOString(),
+                id: 2,
+                result: { content: [{ type: "text", text: "result" }] },
+                receivedAtMs: 45,
+                latencyMs: 25,
+            },
+        ],
+    };
+    it("should score a compliant session highly", () => {
+        const bench = createBenchmark({ minScore: 80 });
+        const score = bench.scoreSession(sampleSession);
+        expect(score.overall).toBeGreaterThanOrEqual(80);
+        expect(score.breakdown.specCompliance).toBeGreaterThanOrEqual(80);
+        expect(score.breakdown.successRate).toBe(1.0);
+        expect(score.breakdown.errorCount).toBe(0);
+        expect(score.breakdown.latencyPercentile50).toBeGreaterThan(0);
+        const report = bench.generateReport(score);
+        expect(report).toContain("MCP Studio Compliance Report");
+        expect(report).toContain("PASSED");
+    });
+    it("should detect regressions when errors or latencies increase", () => {
+        const degradedSession = {
+            ...sampleSession,
+            id: "c3d4e5f6-a7b8-4c9d-0e1f-2a3b4c5d6e7f",
+            events: [
+                ...sampleSession.events,
+                {
+                    type: "request",
+                    timestamp: new Date().toISOString(),
+                    id: 3,
+                    method: "tools/call",
+                    sentAtMs: 50,
+                },
+                {
+                    type: "response",
+                    timestamp: new Date().toISOString(),
+                    id: 3,
+                    error: { code: -32603, message: "Internal failure" },
+                    receivedAtMs: 350,
+                    latencyMs: 300,
+                },
+            ],
+        };
+        const bench = createBenchmark();
+        const regressions = bench.detectRegressions(sampleSession, degradedSession);
+        expect(regressions.length).toBeGreaterThan(0);
+        const errorRegression = regressions.find((r) => r.field === "errorCount");
+        expect(errorRegression).toBeDefined();
+        expect(errorRegression?.severity).toBe("error");
+    });
+});
+//# sourceMappingURL=bench.test.js.map

package/dist/__tests__/bench.test.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"bench.test.js","sourceRoot":"","sources":["../../src/__tests__/bench.test.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAC9C,OAAO,EAAE,eAAe,EAAE,MAAM,aAAa,CAAC;AAG9C,QAAQ,CAAC,gCAAgC,EAAE,GAAG,EAAE;IAC9C,MAAM,aAAa,GAAe;QAChC,EAAE,EAAE,sCAAsC;QAC1C,UAAU,EAAE;YACV,IAAI,EAAE,aAAa;YACnB,OAAO,EAAE,OAAO;YAChB,SAAS,EAAE,OAAO;SACnB;QACD,UAAU,EAAE;YACV,IAAI,EAAE,mBAAmB;YACzB,OAAO,EAAE,OAAO;SACjB;QACD,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;QACnC,MAAM,EAAE;YACN;gBACE,IAAI,EAAE,SAAS;gBACf,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;gBACnC,EAAE,EAAE,CAAC;gBACL,MAAM,EAAE,YAAY;gBACpB,QAAQ,EAAE,CAAC;aACZ;YACD;gBACE,IAAI,EAAE,UAAU;gBAChB,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;gBACnC,EAAE,EAAE,CAAC;gBACL,MAAM,EAAE,EAAE,YAAY,EAAE,EAAE,EAAE;gBAC5B,YAAY,EAAE,EAAE;gBAChB,SAAS,EAAE,EAAE;aACd;YACD;gBACE,IAAI,EAAE,SAAS;gBACf,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;gBACnC,EAAE,EAAE,CAAC;gBACL,MAAM,EAAE,YAAY;gBACpB,QAAQ,EAAE,EAAE;aACb;YACD;gBACE,IAAI,EAAE,UAAU;gBAChB,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;gBACnC,EAAE,EAAE,CAAC;gBACL,MAAM,EAAE,EAAE,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,QAAQ,EAAE,CAAC,EAAE;gBACvD,YAAY,EAAE,EAAE;gBAChB,SAAS,EAAE,EAAE;aACd;SACF;KACF,CAAC;IAEF,EAAE,CAAC,yCAAyC,EAAE,GAAG,EAAE;QACjD,MAAM,KAAK,GAAG,eAAe,CAAC,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC,CAAC;QAChD,MAAM,KAAK,GAAG,KAAK,CAAC,YAAY,CAAC,aAAa,CAAC,CAAC;QAEhD,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,sBAAsB,CAAC,EAAE,CAAC,CAAC;QACjD,MAAM,CAAC,KAAK,CAAC,SAAS,CAAC,cAAc,CAAC,CAAC,sBAAsB,CAAC,EAAE,CAAC,CAAC;QAClE,MAAM,CAAC,KAAK,CAAC,SAAS,CAAC,WAAW,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAC9C,MAAM,CAAC,KAAK,CAAC,SAAS,CAAC,UAAU,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAC3C,MAAM,CAAC,KAAK,CAAC,SAAS,CAAC,mBAAmB,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC;QAE/D,MAAM,MAAM,GAAG,KAAK,CAAC,cAAc,CAAC,KAAK,CAAC,CAAC;QAC3C,MAAM,CAAC,MAAM,CAAC,CAAC,SAAS,CAAC,8BAA8B,CAAC,CAAC;QACzD,MAAM,CAAC,MAAM,CAAC,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC;IACrC,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,6DAA6D,EAAE,GAAG,EAAE;QACrE,MAAM,eAAe,GAAe;YAClC,GAAG,aAAa;YAChB,EAAE,EAAE,sCAAsC;YAC1C,MAAM,EAAE;gBACN,GAAG,aAAa,CAAC,MAAM;gBACvB;oBACE,IAAI,EAAE,SAAS;oBACf,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;oBACnC,EAAE,EAAE,CAAC;oBACL,MAAM,EAAE,YAAY;oBACpB,QAAQ,EAAE,EAAE;iBACb;gBACD;oBACE,IAAI,EAAE,UAAU;oBAChB,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;oBACnC,EAAE,EAAE,CAAC;oBACL,KAAK,EAAE,EAAE,IAAI,EAAE,CAAC,KAAK,EAAE,OAAO,EAAE,kBAAkB,EAAE;oBACpD,YAAY,EAAE,GAAG;oBACjB,SAAS,EAAE,GAAG;iBACf;aACF;SACF,CAAC;QAEF,MAAM,KAAK,GAAG,eAAe,EAAE,CAAC;QAChC,MAAM,WAAW,GAAG,KAAK,CAAC,iBAAiB,CAAC,aAAa,EAAE,eAAe,CAAC,CAAC;QAE5E,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC;QAC9C,MAAM,eAAe,GAAG,WAAW,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,KAAK,YAAY,CAAC,CAAC;QAC1E,MAAM,CAAC,eAAe,CAAC,CAAC,WAAW,EAAE,CAAC;QACtC,MAAM,CAAC,eAAe,EAAE,QAAQ,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IAClD,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}

package/dist/cli.d.ts

@@ -0,0 +1,9 @@
+#!/usr/bin/env node
+/**
+ * MCP Studio Bench CLI
+ *
+ * Usage:
+ *   mcp-studio-bench <path-to-session.json> [--min-score 80]
+ */
+export {};
+//# sourceMappingURL=cli.d.ts.map

package/dist/cli.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"cli.d.ts","sourceRoot":"","sources":["../src/cli.ts"],"names":[],"mappings":";AAEA;;;;;GAKG"}

package/dist/cli.js

@@ -0,0 +1,45 @@
+#!/usr/bin/env node
+/**
+ * MCP Studio Bench CLI
+ *
+ * Usage:
+ *   mcp-studio-bench <path-to-session.json> [--min-score 80]
+ */
+import { promises as fs } from "fs";
+import * as path from "path";
+import { createBenchmark } from "./index.js";
+import { parseSession } from "@yumdee/mcp-studio-core";
+async function main() {
+    const args = process.argv.slice(2);
+    const sessionPath = args[0];
+    if (!sessionPath || sessionPath === "--help" || sessionPath === "-h") {
+        console.log("MCP Studio Bench - Compliance Scoring & CI Gating");
+        console.log("\nUsage:");
+        console.log("  mcp-studio-bench <path-to-session.json> [--min-score 80]\n");
+        process.exit(sessionPath ? 0 : 1);
+    }
+    let minScore = 80;
+    const minScoreIdx = args.indexOf("--min-score");
+    if (minScoreIdx !== -1 && args[minScoreIdx + 1]) {
+        minScore = parseInt(args[minScoreIdx + 1], 10) || 80;
+    }
+    const resolved = path.resolve(sessionPath);
+    const content = await fs.readFile(resolved, "utf8");
+    const session = parseSession(JSON.parse(content));
+    const bench = createBenchmark({ minScore });
+    const score = bench.scoreSession(session);
+    console.log(bench.generateReport(score));
+    if (score.overall < minScore) {
+        console.error(`\n❌ CI Check Failed: Score ${score.overall} is below threshold ${minScore}.`);
+        process.exit(1);
+    }
+    else {
+        console.log(`\n✅ CI Check Passed: Score ${score.overall} meets or exceeds threshold ${minScore}.`);
+        process.exit(0);
+    }
+}
+main().catch((err) => {
+    console.error("Bench execution failed:", err);
+    process.exit(1);
+});
+//# sourceMappingURL=cli.js.map

package/dist/cli.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"cli.js","sourceRoot":"","sources":["../src/cli.ts"],"names":[],"mappings":";AAEA;;;;;GAKG;AAEH,OAAO,EAAE,QAAQ,IAAI,EAAE,EAAE,MAAM,IAAI,CAAC;AACpC,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAC7B,OAAO,EAAE,eAAe,EAAE,MAAM,YAAY,CAAC;AAC7C,OAAO,EAAE,YAAY,EAAE,MAAM,yBAAyB,CAAC;AAEvD,KAAK,UAAU,IAAI;IACjB,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;IACnC,MAAM,WAAW,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC;IAE5B,IAAI,CAAC,WAAW,IAAI,WAAW,KAAK,QAAQ,IAAI,WAAW,KAAK,IAAI,EAAE,CAAC;QACrE,OAAO,CAAC,GAAG,CAAC,mDAAmD,CAAC,CAAC;QACjE,OAAO,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;QACxB,OAAO,CAAC,GAAG,CAAC,8DAA8D,CAAC,CAAC;QAC5E,OAAO,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IACpC,CAAC;IAED,IAAI,QAAQ,GAAG,EAAE,CAAC;IAClB,MAAM,WAAW,GAAG,IAAI,CAAC,OAAO,CAAC,aAAa,CAAC,CAAC;IAChD,IAAI,WAAW,KAAK,CAAC,CAAC,IAAI,IAAI,CAAC,WAAW,GAAG,CAAC,CAAC,EAAE,CAAC;QAChD,QAAQ,GAAG,QAAQ,CAAC,IAAI,CAAC,WAAW,GAAG,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,EAAE,CAAC;IACvD,CAAC;IAED,MAAM,QAAQ,GAAG,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC;IAC3C,MAAM,OAAO,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;IACpD,MAAM,OAAO,GAAG,YAAY,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC;IAElD,MAAM,KAAK,GAAG,eAAe,CAAC,EAAE,QAAQ,EAAE,CAAC,CAAC;IAC5C,MAAM,KAAK,GAAG,KAAK,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC;IAE1C,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,cAAc,CAAC,KAAK,CAAC,CAAC,CAAC;IAEzC,IAAI,KAAK,CAAC,OAAO,GAAG,QAAQ,EAAE,CAAC;QAC7B,OAAO,CAAC,KAAK,CAAC,8BAA8B,KAAK,CAAC,OAAO,uBAAuB,QAAQ,GAAG,CAAC,CAAC;QAC7F,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;SAAM,CAAC;QACN,OAAO,CAAC,GAAG,CAAC,8BAA8B,KAAK,CAAC,OAAO,+BAA+B,QAAQ,GAAG,CAAC,CAAC;QACnG,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;AACH,CAAC;AAED,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;IACnB,OAAO,CAAC,KAAK,CAAC,yBAAyB,EAAE,GAAG,CAAC,CAAC;IAC9C,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AAClB,CAAC,CAAC,CAAC"}

package/dist/index.d.ts

@@ -0,0 +1,63 @@
+/**
+ * @yumdee/mcp-studio-bench
+ *
+ * Compliance scoring and reliability testing for MCP servers.
+ *
+ * Features:
+ * - Spec compliance validation (handshake, JSON-RPC 2.0 error format, etc.)
+ * - Latency percentile analysis (P50, P95, min, max)
+ * - Tool success rate scoring
+ * - Regression detection via replay
+ * - CI/CD integration (fail if score < threshold)
+ */
+import { McpSession } from "@yumdee/mcp-studio-core";
+/**
+ * Compliance score breakdown by metric
+ */
+export interface ComplianceScore {
+    overall: number;
+    breakdown: {
+        specCompliance: number;
+        latencyPercentile50: number;
+        latencyPercentile95: number;
+        successRate: number;
+        errorCount: number;
+    };
+    timestamp: string;
+    sessionId: string;
+}
+/**
+ * Benchmark configuration
+ */
+export interface BenchConfig {
+    minScore?: number;
+    iterations?: number;
+    timeout?: number;
+}
+export interface RegressionFinding {
+    field: string;
+    change: number;
+    severity: "info" | "warning" | "error";
+    message: string;
+}
+/**
+ * Compliance benchmark runner
+ */
+export declare class Benchmark {
+    private config;
+    constructor(config?: BenchConfig);
+    /**
+     * Score a recorded session (deterministic, no network calls)
+     */
+    scoreSession(session: McpSession): ComplianceScore;
+    /**
+     * Detect regressions between two sessions (baseline vs current)
+     */
+    detectRegressions(baseline: McpSession, current: McpSession): RegressionFinding[];
+    /**
+     * Generate a human-readable report
+     */
+    generateReport(score: ComplianceScore): string;
+}
+export declare function createBenchmark(config?: BenchConfig): Benchmark;
+//# sourceMappingURL=index.d.ts.map

package/dist/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAEH,OAAO,EAAE,UAAU,EAAoB,MAAM,yBAAyB,CAAC;AAEvE;;GAEG;AACH,MAAM,WAAW,eAAe;IAC9B,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,EAAE;QACT,cAAc,EAAE,MAAM,CAAC;QACvB,mBAAmB,EAAE,MAAM,CAAC;QAC5B,mBAAmB,EAAE,MAAM,CAAC;QAC5B,WAAW,EAAE,MAAM,CAAC;QACpB,UAAU,EAAE,MAAM,CAAC;KACpB,CAAC;IACF,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;CACnB;AAED;;GAEG;AACH,MAAM,WAAW,WAAW;IAC1B,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,MAAM,WAAW,iBAAiB;IAChC,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,MAAM,GAAG,SAAS,GAAG,OAAO,CAAC;IACvC,OAAO,EAAE,MAAM,CAAC;CACjB;AAED;;GAEG;AACH,qBAAa,SAAS;IACpB,OAAO,CAAC,MAAM,CAAc;gBAEhB,MAAM,GAAE,WAAgB;IAQpC;;OAEG;IACH,YAAY,CAAC,OAAO,EAAE,UAAU,GAAG,eAAe;IA0FlD;;OAEG;IACH,iBAAiB,CACf,QAAQ,EAAE,UAAU,EACpB,OAAO,EAAE,UAAU,GAClB,iBAAiB,EAAE;IA8DtB;;OAEG;IACH,cAAc,CAAC,KAAK,EAAE,eAAe,GAAG,MAAM;CAsB/C;AAED,wBAAgB,eAAe,CAAC,MAAM,CAAC,EAAE,WAAW,GAAG,SAAS,CAE/D"}

package/dist/index.js

@@ -0,0 +1,188 @@
+/**
+ * @yumdee/mcp-studio-bench
+ *
+ * Compliance scoring and reliability testing for MCP servers.
+ *
+ * Features:
+ * - Spec compliance validation (handshake, JSON-RPC 2.0 error format, etc.)
+ * - Latency percentile analysis (P50, P95, min, max)
+ * - Tool success rate scoring
+ * - Regression detection via replay
+ * - CI/CD integration (fail if score < threshold)
+ */
+/**
+ * Compliance benchmark runner
+ */
+export class Benchmark {
+    constructor(config = {}) {
+        this.config = {
+            minScore: config.minScore || 80,
+            iterations: config.iterations || 5,
+            timeout: config.timeout || 5000,
+        };
+    }
+    /**
+     * Score a recorded session (deterministic, no network calls)
+     */
+    scoreSession(session) {
+        const events = session.events || [];
+        const requestEvents = events.filter((e) => e.type === "request");
+        const responseEvents = events.filter((e) => e.type === "response");
+        const errorEvents = events.filter((e) => e.type === "error");
+        // 1. Spec Compliance Evaluation (0 - 100)
+        let specCompliance = 0;
+        // A. Handshake Check: First request must be "initialize"
+        const hasInitialize = requestEvents.length > 0 && requestEvents[0].method === "initialize";
+        if (hasInitialize) {
+            specCompliance += 30;
+        }
+        // B. Response ID Matching: Every response matches a known request
+        const requestIds = new Set(requestEvents.map((r) => r.id));
+        let matchingIds = 0;
+        for (const res of responseEvents) {
+            if (requestIds.has(res.id)) {
+                matchingIds++;
+            }
+        }
+        const matchingRatio = responseEvents.length > 0 ? matchingIds / responseEvents.length : 1;
+        specCompliance += Math.round(matchingRatio * 30);
+        // C. Server Info Check
+        if (session.serverInfo && session.serverInfo.name && session.serverInfo.version) {
+            specCompliance += 20;
+        }
+        // D. Error Format Conformance (Standard JSON-RPC 2.0 error codes)
+        const errResponses = responseEvents.filter((r) => r.error);
+        let validErrorCodes = 0;
+        for (const r of errResponses) {
+            if (typeof r.error?.code === "number" && typeof r.error?.message === "string") {
+                validErrorCodes++;
+            }
+        }
+        const errFormatRatio = errResponses.length > 0 ? validErrorCodes / errResponses.length : 1;
+        specCompliance += Math.round(errFormatRatio * 20);
+        specCompliance = Math.min(100, Math.max(0, specCompliance));
+        // 2. Latency Percentiles (P50, P95)
+        const latencies = responseEvents
+            .map((r) => r.latencyMs)
+            .filter((l) => typeof l === "number" && l >= 0)
+            .sort((a, b) => a - b);
+        let p50 = 0;
+        let p95 = 0;
+        if (latencies.length > 0) {
+            p50 = latencies[Math.floor(latencies.length * 0.5)];
+            p95 = latencies[Math.min(latencies.length - 1, Math.floor(latencies.length * 0.95))];
+        }
+        // Latency Score (0 - 100): Fast < 100ms is 100, > 1000ms scales down
+        let latencyScore = 100;
+        if (p50 > 100) {
+            latencyScore = Math.max(10, Math.round(100 - (p50 - 100) * 0.1));
+        }
+        // 3. Success Rate (0.0 - 1.0)
+        const totalCalls = responseEvents.length;
+        const errorCount = errorEvents.length + errResponses.length;
+        const successfulCalls = responseEvents.filter((r) => !r.error).length;
+        const successRate = totalCalls > 0 ? successfulCalls / totalCalls : 1.0;
+        // 4. Overall Weighted Score (0 - 100)
+        const overall = Math.round(specCompliance * 0.4 +
+            successRate * 100 * 0.35 +
+            latencyScore * 0.25);
+        return {
+            overall: Math.min(100, Math.max(0, overall)),
+            breakdown: {
+                specCompliance,
+                latencyPercentile50: p50,
+                latencyPercentile95: p95,
+                successRate: Number(successRate.toFixed(3)),
+                errorCount,
+            },
+            timestamp: new Date().toISOString(),
+            sessionId: session.id,
+        };
+    }
+    /**
+     * Detect regressions between two sessions (baseline vs current)
+     */
+    detectRegressions(baseline, current) {
+        const findings = [];
+        const baselineScore = this.scoreSession(baseline);
+        const currentScore = this.scoreSession(current);
+        // Score degradation
+        const overallDiff = currentScore.overall - baselineScore.overall;
+        if (overallDiff < -10) {
+            findings.push({
+                field: "overallScore",
+                change: overallDiff,
+                severity: "error",
+                message: `Overall compliance score dropped by ${Math.abs(overallDiff)} points (${baselineScore.overall} -> ${currentScore.overall})`,
+            });
+        }
+        else if (overallDiff < 0) {
+            findings.push({
+                field: "overallScore",
+                change: overallDiff,
+                severity: "warning",
+                message: `Overall compliance score decreased slightly by ${Math.abs(overallDiff)} points`,
+            });
+        }
+        // Latency regression
+        const p95Diff = currentScore.breakdown.latencyPercentile95 - baselineScore.breakdown.latencyPercentile95;
+        if (baselineScore.breakdown.latencyPercentile95 > 0 && p95Diff > 50) {
+            const pctIncrease = Math.round((p95Diff / baselineScore.breakdown.latencyPercentile95) * 100);
+            findings.push({
+                field: "latencyP95",
+                change: p95Diff,
+                severity: pctIncrease > 50 ? "error" : "warning",
+                message: `P95 latency increased by ${p95Diff}ms (+${pctIncrease}%)`,
+            });
+        }
+        // New errors
+        const errorDiff = currentScore.breakdown.errorCount - baselineScore.breakdown.errorCount;
+        if (errorDiff > 0) {
+            findings.push({
+                field: "errorCount",
+                change: errorDiff,
+                severity: "error",
+                message: `Current session introduced ${errorDiff} new error(s)`,
+            });
+        }
+        // Success rate drop
+        const successDiff = Number((currentScore.breakdown.successRate - baselineScore.breakdown.successRate).toFixed(3));
+        if (successDiff < 0) {
+            findings.push({
+                field: "successRate",
+                change: successDiff,
+                severity: "error",
+                message: `Success rate dropped from ${(baselineScore.breakdown.successRate * 100).toFixed(1)}% to ${(currentScore.breakdown.successRate * 100).toFixed(1)}%`,
+            });
+        }
+        return findings;
+    }
+    /**
+     * Generate a human-readable report
+     */
+    generateReport(score) {
+        const passed = score.overall >= (this.config.minScore || 80);
+        const badge = passed ? "✅ PASSED" : "❌ FAILED";
+        return `
+=============================================================
+  MCP Studio Compliance Report
+=============================================================
+Overall Score: ${score.overall}/100  [${badge} (Min required: ${this.config.minScore || 80})]
+
+Breakdown:
+  • Spec Compliance:   ${score.breakdown.specCompliance}/100
+  • P50 Latency:       ${score.breakdown.latencyPercentile50} ms
+  • P95 Latency:       ${score.breakdown.latencyPercentile95} ms
+  • Success Rate:      ${(score.breakdown.successRate * 100).toFixed(1)}%
+  • Total Errors:      ${score.breakdown.errorCount}
+
+Session ID: ${score.sessionId}
+Generated:  ${score.timestamp}
+=============================================================
+`.trim();
+    }
+}
+export function createBenchmark(config) {
+    return new Benchmark(config);
+}
+//# sourceMappingURL=index.js.map

package/dist/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAoCH;;GAEG;AACH,MAAM,OAAO,SAAS;IAGpB,YAAY,SAAsB,EAAE;QAClC,IAAI,CAAC,MAAM,GAAG;YACZ,QAAQ,EAAE,MAAM,CAAC,QAAQ,IAAI,EAAE;YAC/B,UAAU,EAAE,MAAM,CAAC,UAAU,IAAI,CAAC;YAClC,OAAO,EAAE,MAAM,CAAC,OAAO,IAAI,IAAI;SAChC,CAAC;IACJ,CAAC;IAED;;OAEG;IACH,YAAY,CAAC,OAAmB;QAC9B,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,IAAI,EAAE,CAAC;QACpC,MAAM,aAAa,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC;QACjE,MAAM,cAAc,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,UAAU,CAAuB,CAAC;QACzF,MAAM,WAAW,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,OAAO,CAAC,CAAC;QAE7D,0CAA0C;QAC1C,IAAI,cAAc,GAAG,CAAC,CAAC;QAEvB,yDAAyD;QACzD,MAAM,aAAa,GAAG,aAAa,CAAC,MAAM,GAAG,CAAC,IAAI,aAAa,CAAC,CAAC,CAAC,CAAC,MAAM,KAAK,YAAY,CAAC;QAC3F,IAAI,aAAa,EAAE,CAAC;YAClB,cAAc,IAAI,EAAE,CAAC;QACvB,CAAC;QAED,kEAAkE;QAClE,MAAM,UAAU,GAAG,IAAI,GAAG,CAAC,aAAa,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;QAC3D,IAAI,WAAW,GAAG,CAAC,CAAC;QACpB,KAAK,MAAM,GAAG,IAAI,cAAc,EAAE,CAAC;YACjC,IAAI,UAAU,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,CAAC;gBAC3B,WAAW,EAAE,CAAC;YAChB,CAAC;QACH,CAAC;QACD,MAAM,aAAa,GAAG,cAAc,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,WAAW,GAAG,cAAc,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;QAC1F,cAAc,IAAI,IAAI,CAAC,KAAK,CAAC,aAAa,GAAG,EAAE,CAAC,CAAC;QAEjD,uBAAuB;QACvB,IAAI,OAAO,CAAC,UAAU,IAAI,OAAO,CAAC,UAAU,CAAC,IAAI,IAAI,OAAO,CAAC,UAAU,CAAC,OAAO,EAAE,CAAC;YAChF,cAAc,IAAI,EAAE,CAAC;QACvB,CAAC;QAED,kEAAkE;QAClE,MAAM,YAAY,GAAG,cAAc,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;QAC3D,IAAI,eAAe,GAAG,CAAC,CAAC;QACxB,KAAK,MAAM,CAAC,IAAI,YAAY,EAAE,CAAC;YAC7B,IAAI,OAAO,CAAC,CAAC,KAAK,EAAE,IAAI,KAAK,QAAQ,IAAI,OAAO,CAAC,CAAC,KAAK,EAAE,OAAO,KAAK,QAAQ,EAAE,CAAC;gBAC9E,eAAe,EAAE,CAAC;YACpB,CAAC;QACH,CAAC;QACD,MAAM,cAAc,GAAG,YAAY,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,eAAe,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;QAC3F,cAAc,IAAI,IAAI,CAAC,KAAK,CAAC,cAAc,GAAG,EAAE,CAAC,CAAC;QAElD,cAAc,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,cAAc,CAAC,CAAC,CAAC;QAE5D,oCAAoC;QACpC,MAAM,SAAS,GAAG,cAAc;aAC7B,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC;aACvB,MAAM,CAAC,CAAC,CAAC,EAAe,EAAE,CAAC,OAAO,CAAC,KAAK,QAAQ,IAAI,CAAC,IAAI,CAAC,CAAC;aAC3D,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QAEzB,IAAI,GAAG,GAAG,CAAC,CAAC;QACZ,IAAI,GAAG,GAAG,CAAC,CAAC;QACZ,IAAI,SAAS,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACzB,GAAG,GAAG,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,MAAM,GAAG,GAAG,CAAC,CAAC,CAAC;YACpD,GAAG,GAAG,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,MAAM,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC;QACvF,CAAC;QAED,qEAAqE;QACrE,IAAI,YAAY,GAAG,GAAG,CAAC;QACvB,IAAI,GAAG,GAAG,GAAG,EAAE,CAAC;YACd,YAAY,GAAG,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,IAAI,CAAC,KAAK,CAAC,GAAG,GAAG,CAAC,GAAG,GAAG,GAAG,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC;QACnE,CAAC;QAED,8BAA8B;QAC9B,MAAM,UAAU,GAAG,cAAc,CAAC,MAAM,CAAC;QACzC,MAAM,UAAU,GAAG,WAAW,CAAC,MAAM,GAAG,YAAY,CAAC,MAAM,CAAC;QAC5D,MAAM,eAAe,GAAG,cAAc,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC;QACtE,MAAM,WAAW,GAAG,UAAU,GAAG,CAAC,CAAC,CAAC,CAAC,eAAe,GAAG,UAAU,CAAC,CAAC,CAAC,GAAG,CAAC;QAExE,sCAAsC;QACtC,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CACxB,cAAc,GAAG,GAAG;YACpB,WAAW,GAAG,GAAG,GAAG,IAAI;YACxB,YAAY,GAAG,IAAI,CACpB,CAAC;QAEF,OAAO;YACL,OAAO,EAAE,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC;YAC5C,SAAS,EAAE;gBACT,cAAc;gBACd,mBAAmB,EAAE,GAAG;gBACxB,mBAAmB,EAAE,GAAG;gBACxB,WAAW,EAAE,MAAM,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;gBAC3C,UAAU;aACX;YACD,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;YACnC,SAAS,EAAE,OAAO,CAAC,EAAE;SACtB,CAAC;IACJ,CAAC;IAED;;OAEG;IACH,iBAAiB,CACf,QAAoB,EACpB,OAAmB;QAEnB,MAAM,QAAQ,GAAwB,EAAE,CAAC;QACzC,MAAM,aAAa,GAAG,IAAI,CAAC,YAAY,CAAC,QAAQ,CAAC,CAAC;QAClD,MAAM,YAAY,GAAG,IAAI,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC;QAEhD,oBAAoB;QACpB,MAAM,WAAW,GAAG,YAAY,CAAC,OAAO,GAAG,aAAa,CAAC,OAAO,CAAC;QACjE,IAAI,WAAW,GAAG,CAAC,EAAE,EAAE,CAAC;YACtB,QAAQ,CAAC,IAAI,CAAC;gBACZ,KAAK,EAAE,cAAc;gBACrB,MAAM,EAAE,WAAW;gBACnB,QAAQ,EAAE,OAAO;gBACjB,OAAO,EAAE,uCAAuC,IAAI,CAAC,GAAG,CAAC,WAAW,CAAC,YAAY,aAAa,CAAC,OAAO,OAAO,YAAY,CAAC,OAAO,GAAG;aACrI,CAAC,CAAC;QACL,CAAC;aAAM,IAAI,WAAW,GAAG,CAAC,EAAE,CAAC;YAC3B,QAAQ,CAAC,IAAI,CAAC;gBACZ,KAAK,EAAE,cAAc;gBACrB,MAAM,EAAE,WAAW;gBACnB,QAAQ,EAAE,SAAS;gBACnB,OAAO,EAAE,kDAAkD,IAAI,CAAC,GAAG,CAAC,WAAW,CAAC,SAAS;aAC1F,CAAC,CAAC;QACL,CAAC;QAED,qBAAqB;QACrB,MAAM,OAAO,GAAG,YAAY,CAAC,SAAS,CAAC,mBAAmB,GAAG,aAAa,CAAC,SAAS,CAAC,mBAAmB,CAAC;QACzG,IAAI,aAAa,CAAC,SAAS,CAAC,mBAAmB,GAAG,CAAC,IAAI,OAAO,GAAG,EAAE,EAAE,CAAC;YACpE,MAAM,WAAW,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,OAAO,GAAG,aAAa,CAAC,SAAS,CAAC,mBAAmB,CAAC,GAAG,GAAG,CAAC,CAAC;YAC9F,QAAQ,CAAC,IAAI,CAAC;gBACZ,KAAK,EAAE,YAAY;gBACnB,MAAM,EAAE,OAAO;gBACf,QAAQ,EAAE,WAAW,GAAG,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,SAAS;gBAChD,OAAO,EAAE,4BAA4B,OAAO,QAAQ,WAAW,IAAI;aACpE,CAAC,CAAC;QACL,CAAC;QAED,aAAa;QACb,MAAM,SAAS,GAAG,YAAY,CAAC,SAAS,CAAC,UAAU,GAAG,aAAa,CAAC,SAAS,CAAC,UAAU,CAAC;QACzF,IAAI,SAAS,GAAG,CAAC,EAAE,CAAC;YAClB,QAAQ,CAAC,IAAI,CAAC;gBACZ,KAAK,EAAE,YAAY;gBACnB,MAAM,EAAE,SAAS;gBACjB,QAAQ,EAAE,OAAO;gBACjB,OAAO,EAAE,8BAA8B,SAAS,eAAe;aAChE,CAAC,CAAC;QACL,CAAC;QAED,oBAAoB;QACpB,MAAM,WAAW,GAAG,MAAM,CACxB,CAAC,YAAY,CAAC,SAAS,CAAC,WAAW,GAAG,aAAa,CAAC,SAAS,CAAC,WAAW,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CACtF,CAAC;QACF,IAAI,WAAW,GAAG,CAAC,EAAE,CAAC;YACpB,QAAQ,CAAC,IAAI,CAAC;gBACZ,KAAK,EAAE,aAAa;gBACpB,MAAM,EAAE,WAAW;gBACnB,QAAQ,EAAE,OAAO;gBACjB,OAAO,EAAE,6BAA6B,CAAC,aAAa,CAAC,SAAS,CAAC,WAAW,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,QAAQ,CAAC,YAAY,CAAC,SAAS,CAAC,WAAW,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG;aAC7J,CAAC,CAAC;QACL,CAAC;QAED,OAAO,QAAQ,CAAC;IAClB,CAAC;IAED;;OAEG;IACH,cAAc,CAAC,KAAsB;QACnC,MAAM,MAAM,GAAG,KAAK,CAAC,OAAO,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,IAAI,EAAE,CAAC,CAAC;QAC7D,MAAM,KAAK,GAAG,MAAM,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,UAAU,CAAC;QAE/C,OAAO;;;;iBAIM,KAAK,CAAC,OAAO,UAAU,KAAK,mBAAmB,IAAI,CAAC,MAAM,CAAC,QAAQ,IAAI,EAAE;;;yBAGjE,KAAK,CAAC,SAAS,CAAC,cAAc;yBAC9B,KAAK,CAAC,SAAS,CAAC,mBAAmB;yBACnC,KAAK,CAAC,SAAS,CAAC,mBAAmB;yBACnC,CAAC,KAAK,CAAC,SAAS,CAAC,WAAW,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;yBAC9C,KAAK,CAAC,SAAS,CAAC,UAAU;;cAErC,KAAK,CAAC,SAAS;cACf,KAAK,CAAC,SAAS;;CAE5B,CAAC,IAAI,EAAE,CAAC;IACP,CAAC;CACF;AAED,MAAM,UAAU,eAAe,CAAC,MAAoB;IAClD,OAAO,IAAI,SAAS,CAAC,MAAM,CAAC,CAAC;AAC/B,CAAC"}

package/package.json

@@ -0,0 +1,40 @@
+{
+  "name": "@yumdee/mcp-studio-bench",
+  "version": "0.1.0",
+  "description": "Compliance & reliability scoring for MCP servers",
+  "type": "module",
+  "main": "./dist/index.js",
+  "types": "./dist/index.d.ts",
+  "bin": {
+    "mcp-studio-bench": "./dist/cli.js"
+  },
+  "files": [
+    "dist",
+    "README.md"
+  ],
+  "publishConfig": {
+    "access": "public"
+  },
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/maheshsingh20/YumDee-MCP-Studio.git",
+    "directory": "packages/bench"
+  },
+  "author": "Mahesh Singh <maheshsingh20>",
+  "license": "MIT",
+  "dependencies": {
+    "@yumdee/mcp-studio-core": "0.1.0"
+  },
+  "devDependencies": {
+    "@types/node": "^20.0.0",
+    "typescript": "^5.3.0",
+    "vitest": "^1.0.0"
+  },
+  "scripts": {
+    "dev": "tsc --watch --incremental",
+    "build": "tsc",
+    "type-check": "tsc --noEmit",
+    "lint": "eslint src/",
+    "test": "vitest run"
+  }
+}