mcp-eval-gateway 1.0.0

package/dist/index.d.ts

@@ -0,0 +1,81 @@
+import { FetchLike, Transport } from '@modelcontextprotocol/sdk/shared/transport.js';
+import { ToolSet, LanguageModel } from 'ai';
+
+declare const EVALUATION_PROMPT = "You are an AI assistant with access to tools.\n\nWhen given a task, you MUST:\n1. Use the available tools to complete the task\n2. Provide summary of each step in your approach, wrapped in <summary> tags\n3. Provide feedback on the tools provided, wrapped in <feedback> tags\n4. Provide your final response, wrapped in <response> tags\n\nSummary Requirements:\n- In your <summary> tags, you must explain:\n  - The steps you took to complete the task\n  - Which tools you used, in what order, and why\n  - The inputs you provided to each tool\n  - The outputs you received from each tool\n  - A summary for how you arrived at the response\n\nFeedback Requirements:\n- In your <feedback> tags, provide constructive feedback on the tools:\n  - Comment on tool names: Are they clear and descriptive?\n  - Comment on input parameters: Are they well-documented? Are required vs optional parameters clear?\n  - Comment on descriptions: Do they accurately describe what the tool does?\n  - Comment on any errors encountered during tool usage: Did the tool fail to execute? Did the tool return too many tokens?\n  - Identify specific areas for improvement and explain WHY they would help\n  - Be specific and actionable in your suggestions\n\nResponse Requirements:\n- Your response should be concise and directly address what was asked\n- Always wrap your final response in <response> tags\n- If you cannot solve the task return <response>NOT_FOUND</response>\n- For numeric responses, provide just the number\n- For IDs, provide just the ID\n- For names or text, provide the exact text requested\n- Your response should go last";
+
+type InitEvalProjectOptions = {
+    dir?: string;
+};
+declare function initEvalProject(rootDir: string, options?: InitEvalProjectOptions): void;
+
+type ToolsFromMcpOptions = {
+    url: string | URL;
+    fetch?: FetchLike;
+    headers?: Record<string, string>;
+} | {
+    transport: Transport;
+};
+declare function toolsFromMcp(options: ToolsFromMcpOptions): Promise<{
+    tools: ToolSet;
+    close: () => Promise<void>;
+}>;
+
+declare function resolveModel(model: string): Promise<LanguageModel>;
+
+type EvalTask = {
+    name: string;
+    prompt: string;
+    expected: string;
+    required?: boolean;
+    setup?: () => void | Promise<void>;
+    scorer?: (actual: string | null, task: EvalTask) => number;
+};
+type ToolMetrics = Record<string, {
+    count: number;
+    durationsMs: number[];
+}>;
+type TaskResult = {
+    name: string;
+    prompt: string;
+    expected: string;
+    actual: string | null;
+    score: number;
+    passed: boolean;
+    required: boolean;
+    durationMs: number;
+    toolMetrics: ToolMetrics;
+    numToolCalls: number;
+    summary: string | null;
+    feedback: string | null;
+};
+type EvalRunResult = {
+    total: number;
+    correct: number;
+    accuracy: number;
+    results: TaskResult[];
+    report: string;
+};
+
+type RunEvalProjectOptions = {
+    dir?: string;
+    envFile?: string;
+    model?: string;
+};
+declare function runEvalProject(rootDir: string, options?: RunEvalProjectOptions): Promise<EvalRunResult>;
+
+type RunEvalsOptions = {
+    model: string | LanguageModel;
+    tools: ToolSet;
+    tasks: EvalTask[];
+    maxSteps?: number;
+    systemPrompt?: string;
+    scorer?: (actual: string | null, task: EvalTask) => number;
+};
+declare function runEvals(options: RunEvalsOptions): Promise<EvalRunResult>;
+
+declare function assertEvalResult(result: EvalRunResult, options?: {
+    threshold?: number;
+}): void;
+declare function writeGitHubSummary(result: EvalRunResult): boolean;
+
+export { EVALUATION_PROMPT, type EvalRunResult, type EvalTask, type InitEvalProjectOptions, type RunEvalProjectOptions, type TaskResult, type ToolMetrics, assertEvalResult, initEvalProject, resolveModel, runEvalProject, runEvals, toolsFromMcp, writeGitHubSummary };

package/dist/index.js

@@ -0,0 +1,21 @@
+import {
+  EVALUATION_PROMPT,
+  assertEvalResult,
+  initEvalProject,
+  resolveModel,
+  runEvalProject,
+  runEvals,
+  toolsFromMcp,
+  writeGitHubSummary
+} from "./chunk-6UIMILU4.js";
+export {
+  EVALUATION_PROMPT,
+  assertEvalResult,
+  initEvalProject,
+  resolveModel,
+  runEvalProject,
+  runEvals,
+  toolsFromMcp,
+  writeGitHubSummary
+};
+//# sourceMappingURL=index.js.map

package/dist/index.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":[],"sourcesContent":[],"mappings":"","names":[]}

package/package.json

@@ -0,0 +1,88 @@
+{
+  "name": "mcp-eval-gateway",
+  "version": "1.0.0",
+  "description": "Run LLM tool-use evaluations against MCP servers with the Vercel AI SDK",
+  "type": "module",
+  "main": "./dist/index.cjs",
+  "module": "./dist/index.js",
+  "types": "./dist/index.d.ts",
+  "exports": {
+    ".": {
+      "import": {
+        "types": "./dist/index.d.ts",
+        "default": "./dist/index.js"
+      },
+      "require": {
+        "types": "./dist/index.d.cts",
+        "default": "./dist/index.cjs"
+      }
+    }
+  },
+  "bin": {
+    "mcp-eval-gateway": "./dist/cli.js"
+  },
+  "files": [
+    "dist"
+  ],
+  "scripts": {
+    "test": "vitest run",
+    "build": "tsup",
+    "prepublishOnly": "npm run build"
+  },
+  "publishConfig": {
+    "access": "public"
+  },
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/guillegette/mcp-eval-gateway.git"
+  },
+  "keywords": [
+    "mcp",
+    "model context protocol",
+    "eval",
+    "evaluation",
+    "llm",
+    "ai-sdk"
+  ],
+  "author": "Guillermo Gette (https://github.com/guillegette)",
+  "bugs": {
+    "url": "https://github.com/guillegette/mcp-eval-gateway/issues"
+  },
+  "homepage": "https://github.com/guillegette/mcp-eval-gateway#readme",
+  "license": "MIT",
+  "engines": {
+    "node": ">=22"
+  },
+  "dependencies": {
+    "@modelcontextprotocol/sdk": "^1.30.0",
+    "ai": "^7.0.0",
+    "jiti": "^2.7.0",
+    "yaml": "^2.9.0"
+  },
+  "peerDependencies": {
+    "@ai-sdk/amazon-bedrock": "^5.0.0",
+    "@ai-sdk/anthropic": "^4.0.0",
+    "@ai-sdk/openai": "^4.0.0"
+  },
+  "peerDependenciesMeta": {
+    "@ai-sdk/amazon-bedrock": {
+      "optional": true
+    },
+    "@ai-sdk/anthropic": {
+      "optional": true
+    },
+    "@ai-sdk/openai": {
+      "optional": true
+    }
+  },
+  "devDependencies": {
+    "@ai-sdk/amazon-bedrock": "^5.0.66",
+    "@ai-sdk/anthropic": "^4.0.44",
+    "@ai-sdk/openai": "^4.0.50",
+    "@types/node": "^24.0.0",
+    "tsup": "^8.5.0",
+    "typescript": "^5.9.2",
+    "vitest": "^4.1.0",
+    "zod": "^4.0.0"
+  }
+}