@heilgar/pest-mcp 0.0.2

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 pest contributors
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -0,0 +1,11 @@
+# @heilgar/pest-mcp
+
+Prompt Evaluation & Scoring Toolkit - MCP server testing
+
+## Documentation
+
+For detailed documentation, visit [https://heilgar.github.io/pest/](https://heilgar.github.io/pest/)
+
+## License
+
+MIT

package/dist/chunk-CG5UCD4O.js

@@ -0,0 +1,76 @@
+// src/matchers.ts
+async function toExposeTools(received, names) {
+  const tools = await received.listTools();
+  const toolNames = tools.map((t) => t.name);
+  const missing = names.filter((n) => !toolNames.includes(n));
+  const pass = missing.length === 0;
+  return {
+    pass,
+    message: () => pass ? `Expected server NOT to expose tools [${names.join(", ")}], but it does.` : `Expected server to expose tools [${names.join(", ")}], but missing: [${missing.join(", ")}]. Found: [${toolNames.join(", ")}]`
+  };
+}
+async function toExposeTool(received, name) {
+  const tools = await received.listTools();
+  const toolNames = tools.map((t) => t.name);
+  const pass = toolNames.includes(name);
+  return {
+    pass,
+    message: () => pass ? `Expected server NOT to expose tool "${name}", but it does.` : `Expected server to expose tool "${name}". Found: [${toolNames.join(", ")}]`
+  };
+}
+async function toHaveValidToolSchemas(received) {
+  const tools = await received.listTools();
+  const invalid = [];
+  for (const tool of tools) {
+    const schema = tool.inputSchema;
+    if (!schema || typeof schema !== "object" || schema.type !== "object") {
+      invalid.push(tool.name);
+    }
+  }
+  const pass = invalid.length === 0;
+  return {
+    pass,
+    message: () => pass ? "Expected tool schemas NOT to be valid, but they are." : `Expected all tool schemas to be valid JSON Schema objects. Invalid: [${invalid.join(", ")}]`
+  };
+}
+async function toExposePrompts(received, names) {
+  const prompts = await received.listPrompts();
+  const promptNames = prompts.map((p) => p.name);
+  const missing = names.filter((n) => !promptNames.includes(n));
+  const pass = missing.length === 0;
+  return {
+    pass,
+    message: () => pass ? `Expected server NOT to expose prompts [${names.join(", ")}], but it does.` : `Expected server to expose prompts [${names.join(", ")}], but missing: [${missing.join(", ")}]. Found: [${promptNames.join(", ")}]`
+  };
+}
+async function toExposeResources(received, uris) {
+  const resources = await received.listResources();
+  const resourceUris = resources.map((r) => r.uri);
+  const missing = uris.filter((u) => !resourceUris.includes(u));
+  const pass = missing.length === 0;
+  return {
+    pass,
+    message: () => pass ? `Expected server NOT to expose resources [${uris.join(", ")}], but it does.` : `Expected server to expose resources [${uris.join(", ")}], but missing: [${missing.join(", ")}]. Found: [${resourceUris.join(", ")}]`
+  };
+}
+var mcpMatchers = {
+  async toExposeTools(received, names) {
+    return toExposeTools(received, names);
+  },
+  async toExposeTool(received, name) {
+    return toExposeTool(received, name);
+  },
+  async toHaveValidToolSchemas(received) {
+    return toHaveValidToolSchemas(received);
+  },
+  async toExposePrompts(received, names) {
+    return toExposePrompts(received, names);
+  },
+  async toExposeResources(received, uris) {
+    return toExposeResources(received, uris);
+  }
+};
+
+export {
+  mcpMatchers
+};

package/dist/chunk-FRYT6IAD.js

@@ -0,0 +1,219 @@
+// src/client.ts
+import { Client } from "@modelcontextprotocol/sdk/client/index.js";
+import { StdioClientTransport } from "@modelcontextprotocol/sdk/client/stdio.js";
+import { SSEClientTransport } from "@modelcontextprotocol/sdk/client/sse.js";
+import { StreamableHTTPClientTransport } from "@modelcontextprotocol/sdk/client/streamableHttp.js";
+var DEFAULT_TIMEOUT = 3e4;
+var McpClient = class _McpClient {
+  name;
+  client;
+  constructor(name, client) {
+    this.name = name;
+    this.client = client;
+  }
+  static async fromConfig(name, config, options) {
+    const timeout = options?.timeout ?? DEFAULT_TIMEOUT;
+    const client = new Client({ name: `pest-mcp-${name}`, version: "1.0.0" });
+    let transport;
+    if ("transport" in config && config.transport === "sse") {
+      transport = new SSEClientTransport(new URL(config.url));
+    } else if ("transport" in config && config.transport === "http") {
+      transport = new StreamableHTTPClientTransport(new URL(config.url));
+    } else {
+      transport = new StdioClientTransport({
+        command: config.command,
+        args: config.args,
+        env: config.env ? { ...process.env, ...config.env } : void 0
+      });
+    }
+    const connectPromise = client.connect(transport);
+    const timeoutPromise = new Promise(
+      (_, reject) => setTimeout(
+        () => reject(new Error(`MCP server "${name}" did not respond within ${timeout}ms`)),
+        timeout
+      )
+    );
+    await Promise.race([connectPromise, timeoutPromise]);
+    return new _McpClient(name, client);
+  }
+  async close() {
+    await this.client.close();
+  }
+  // --- Discovery ---
+  async listTools() {
+    const { tools } = await this.client.listTools();
+    return tools.map((t) => ({
+      name: t.name,
+      description: t.description,
+      inputSchema: t.inputSchema
+    }));
+  }
+  async listPrompts() {
+    const { prompts } = await this.client.listPrompts();
+    return prompts.map((p) => ({
+      name: p.name,
+      description: p.description,
+      arguments: p.arguments?.map((a) => ({
+        name: a.name,
+        description: a.description,
+        required: a.required
+      }))
+    }));
+  }
+  async listResources() {
+    const { resources } = await this.client.listResources();
+    return resources.map((r) => ({
+      uri: r.uri,
+      name: r.name,
+      description: r.description,
+      mimeType: r.mimeType
+    }));
+  }
+  // --- Execution ---
+  async callTool(name, args) {
+    const result = await this.client.callTool({ name, arguments: args });
+    return {
+      content: result.content ?? [],
+      structuredContent: result.structuredContent,
+      isError: result.isError
+    };
+  }
+  async getPrompt(name, args) {
+    const result = await this.client.getPrompt({ name, arguments: args });
+    return {
+      messages: result.messages.map((m) => {
+        const content = m.content;
+        return { role: m.role, content };
+      })
+    };
+  }
+  async readResource(uri) {
+    const result = await this.client.readResource({ uri });
+    return {
+      contents: result.contents.map((c) => ({
+        uri: c.uri,
+        text: "text" in c ? c.text : void 0,
+        blob: "blob" in c ? c.blob : void 0,
+        mimeType: c.mimeType
+      }))
+    };
+  }
+  // --- Conversion ---
+  async toPestTools() {
+    const tools = await this.listTools();
+    return tools.map((t) => ({
+      type: "function",
+      function: {
+        name: t.name,
+        description: t.description,
+        parameters: t.inputSchema
+      }
+    }));
+  }
+};
+
+// src/config.ts
+import { existsSync } from "fs";
+import { resolve } from "path";
+import { pathToFileURL } from "url";
+import { loadEnv } from "@heilgar/pest-core";
+import * as v from "valibot";
+var CONFIG_FILES = ["pest.config.ts", "pest.config.js", "pest.config.mjs"];
+var McpStdioServerSchema = v.object({
+  command: v.string(),
+  args: v.optional(v.array(v.string())),
+  env: v.optional(v.record(v.string(), v.string()))
+});
+var McpSseServerSchema = v.object({
+  transport: v.literal("sse"),
+  url: v.string()
+});
+var McpHttpServerSchema = v.object({
+  transport: v.literal("http"),
+  url: v.string()
+});
+var McpServerConfigSchema = v.union([
+  McpStdioServerSchema,
+  McpSseServerSchema,
+  McpHttpServerSchema
+]);
+var McpConfigSchema = v.object({
+  servers: v.record(v.string(), McpServerConfigSchema)
+});
+async function loadMcpConfig(cwd = process.cwd()) {
+  loadEnv(cwd);
+  let configPath;
+  for (const file of CONFIG_FILES) {
+    const candidate = resolve(cwd, file);
+    if (existsSync(candidate)) {
+      configPath = candidate;
+      break;
+    }
+  }
+  if (!configPath) {
+    throw new Error(
+      `No pest config found. Create one of: ${CONFIG_FILES.join(", ")}`
+    );
+  }
+  const configUrl = pathToFileURL(configPath).href;
+  let mod;
+  try {
+    mod = await import(configUrl);
+  } catch (err) {
+    if (configPath.endsWith(".ts") && err.code === "ERR_UNKNOWN_FILE_EXTENSION") {
+      throw new Error(
+        `Cannot import TypeScript config "${configPath}".
+Run with tsx (npx tsx ...) or use pest.config.js / pest.config.mjs instead.`
+      );
+    }
+    throw err;
+  }
+  const raw = mod.default ?? mod;
+  const mcpRaw = raw.mcp;
+  if (!mcpRaw) {
+    throw new Error(
+      'No "mcp" section found in pest config. Add mcp.servers to pest.config.ts.'
+    );
+  }
+  const result = v.safeParse(McpConfigSchema, mcpRaw);
+  if (!result.success) {
+    const issues = v.flatten(result.issues);
+    const messages = Object.entries(issues.nested ?? {}).map(([path, errors]) => `  ${path}: ${(errors ?? []).join(", ")}`).join("\n");
+    throw new Error(`Invalid MCP config:
+${messages}`);
+  }
+  return result.output;
+}
+var cache = /* @__PURE__ */ new Map();
+async function useMcpServer(name) {
+  const cached = cache.get(name);
+  if (cached) return cached;
+  const config = await loadMcpConfig();
+  const serverConfig = config.servers[name];
+  if (!serverConfig) {
+    throw new Error(
+      `MCP server "${name}" not found in pest.config.ts. Available: ${Object.keys(config.servers).join(", ")}`
+    );
+  }
+  const client = await McpClient.fromConfig(name, serverConfig);
+  cache.set(name, client);
+  return client;
+}
+async function closeAllMcpServers() {
+  const clients = [...cache.values()];
+  cache.clear();
+  await Promise.allSettled(clients.map((c) => c.close()));
+}
+process.on("beforeExit", () => {
+  if (cache.size > 0) {
+    closeAllMcpServers().catch(() => {
+    });
+  }
+});
+
+export {
+  McpClient,
+  loadMcpConfig,
+  useMcpServer,
+  closeAllMcpServers
+};

package/dist/index.d.ts

@@ -0,0 +1,126 @@
+import { ToolDefinition, SendAgenticOptions, Provider, PestResponse } from '@heilgar/pest-core';
+
+interface McpStdioServerConfig {
+    command: string;
+    args?: string[];
+    env?: Record<string, string>;
+}
+interface McpSseServerConfig {
+    transport: 'sse';
+    url: string;
+}
+interface McpHttpServerConfig {
+    transport: 'http';
+    url: string;
+}
+type McpServerConfig = McpStdioServerConfig | McpSseServerConfig | McpHttpServerConfig;
+interface McpTool {
+    name: string;
+    description?: string;
+    inputSchema: Record<string, unknown>;
+}
+interface McpToolResult {
+    content: Array<{
+        type: string;
+        text?: string;
+        data?: string;
+        mimeType?: string;
+    }>;
+    structuredContent?: Record<string, unknown>;
+    isError?: boolean;
+}
+interface McpPrompt {
+    name: string;
+    description?: string;
+    arguments?: Array<{
+        name: string;
+        description?: string;
+        required?: boolean;
+    }>;
+}
+interface McpPromptResult {
+    messages: Array<{
+        role: string;
+        content: {
+            type: string;
+            text: string;
+        };
+    }>;
+}
+interface McpResource {
+    uri: string;
+    name: string;
+    description?: string;
+    mimeType?: string;
+}
+interface McpResourceResult {
+    contents: Array<{
+        uri: string;
+        text?: string;
+        blob?: string;
+        mimeType?: string;
+    }>;
+}
+
+declare class McpClient {
+    readonly name: string;
+    private client;
+    private constructor();
+    static fromConfig(name: string, config: McpServerConfig, options?: {
+        timeout?: number;
+    }): Promise<McpClient>;
+    close(): Promise<void>;
+    listTools(): Promise<McpTool[]>;
+    listPrompts(): Promise<McpPrompt[]>;
+    listResources(): Promise<McpResource[]>;
+    callTool(name: string, args?: Record<string, unknown>): Promise<McpToolResult>;
+    getPrompt(name: string, args?: Record<string, string>): Promise<McpPromptResult>;
+    readResource(uri: string): Promise<McpResourceResult>;
+    toPestTools(): Promise<ToolDefinition[]>;
+}
+
+interface McpConfig {
+    servers: Record<string, McpServerConfig>;
+}
+/**
+ * Load the MCP config section from pest.config.ts.
+ * Loads the raw config file independently from core's loadConfig()
+ * because core's valibot schema strips unknown keys.
+ */
+declare function loadMcpConfig(cwd?: string): Promise<McpConfig>;
+/**
+ * Resolve an MCP server from pest.config.ts by name.
+ * Cached — subsequent calls return the same connection.
+ */
+declare function useMcpServer(name: string): Promise<McpClient>;
+/**
+ * Close all cached MCP connections.
+ * Call in afterAll() for reliable cleanup.
+ */
+declare function closeAllMcpServers(): Promise<void>;
+
+interface SendWithMcpOptions extends Omit<SendAgenticOptions, 'executor' | 'tools'> {
+    mcpServer: McpClient;
+    additionalTools?: ToolDefinition[];
+}
+/**
+ * Send a message to an LLM with MCP server tools auto-discovered
+ * and tool calls auto-routed to the MCP server.
+ *
+ * Returns a standard PestResponse so all existing pest matchers work.
+ */
+declare function sendWithMcp(provider: Provider, message: string, options: SendWithMcpOptions): Promise<PestResponse>;
+
+interface MatcherResult {
+    pass: boolean;
+    message: () => string;
+}
+declare const mcpMatchers: {
+    toExposeTools(received: McpClient, names: string[]): Promise<MatcherResult>;
+    toExposeTool(received: McpClient, name: string): Promise<MatcherResult>;
+    toHaveValidToolSchemas(received: McpClient): Promise<MatcherResult>;
+    toExposePrompts(received: McpClient, names: string[]): Promise<MatcherResult>;
+    toExposeResources(received: McpClient, uris: string[]): Promise<MatcherResult>;
+};
+
+export { McpClient, type McpConfig, type McpHttpServerConfig, type McpPrompt, type McpPromptResult, type McpResource, type McpResourceResult, type McpServerConfig, type McpSseServerConfig, type McpStdioServerConfig, type McpTool, type McpToolResult, type SendWithMcpOptions, closeAllMcpServers, loadMcpConfig, mcpMatchers, sendWithMcp, useMcpServer };

package/dist/index.js

@@ -0,0 +1,39 @@
+import {
+  McpClient,
+  closeAllMcpServers,
+  loadMcpConfig,
+  useMcpServer
+} from "./chunk-FRYT6IAD.js";
+import {
+  mcpMatchers
+} from "./chunk-CG5UCD4O.js";
+
+// src/send-mcp.ts
+import { sendAgentic } from "@heilgar/pest-core";
+async function sendWithMcp(provider, message, options) {
+  const { mcpServer, additionalTools, ...rest } = options;
+  const mcpTools = await mcpServer.toPestTools();
+  const mcpToolNames = new Set(mcpTools.map((t) => t.function.name));
+  const tools = [...mcpTools, ...additionalTools ?? []];
+  const executor = async (name, args) => {
+    if (mcpToolNames.has(name)) {
+      const result = await mcpServer.callTool(name, args);
+      const texts = result.content.filter((c) => c.type === "text" && c.text).map((c) => c.text);
+      return texts.join("\n") || JSON.stringify(result.content);
+    }
+    return "[]";
+  };
+  return sendAgentic(provider, message, {
+    ...rest,
+    tools,
+    executor
+  });
+}
+export {
+  McpClient,
+  closeAllMcpServers,
+  loadMcpConfig,
+  mcpMatchers,
+  sendWithMcp,
+  useMcpServer
+};

package/dist/qa.d.ts

@@ -0,0 +1,6 @@
+interface QaOptions {
+    verbose?: boolean;
+}
+declare function runMcpQa(serverName: string, options?: QaOptions): Promise<void>;
+
+export { runMcpQa };

package/dist/qa.js

@@ -0,0 +1,106 @@
+import {
+  McpClient,
+  loadMcpConfig
+} from "./chunk-FRYT6IAD.js";
+
+// src/qa.ts
+import { ansi as c } from "@heilgar/pest-core";
+function icon(status) {
+  if (status === "pass") return `${c.green}pass${c.reset}`;
+  if (status === "fail") return `${c.red}fail${c.reset}`;
+  return `${c.yellow}warn${c.reset}`;
+}
+async function runMcpQa(serverName, options) {
+  const config = await loadMcpConfig();
+  const serverConfig = config.servers[serverName];
+  if (!serverConfig) {
+    console.error(
+      `MCP server "${serverName}" not found in pest.config.ts. Available: ${Object.keys(config.servers).join(", ")}`
+    );
+    process.exit(1);
+  }
+  const command = "command" in serverConfig ? `${serverConfig.command} ${(serverConfig.args ?? []).join(" ")}`.trim() : serverConfig.url;
+  console.log(`
+pest qa \u2014 ${serverName} (${command})
+`);
+  const results = [];
+  let client;
+  try {
+    const start = performance.now();
+    client = await McpClient.fromConfig(serverName, serverConfig);
+    const elapsed = ((performance.now() - start) / 1e3).toFixed(1);
+    const transport = "command" in serverConfig ? "stdio" : serverConfig.transport;
+    results.push({ status: "pass", message: `Server started and connected via ${transport} (${elapsed}s)` });
+  } catch (err) {
+    results.push({ status: "fail", message: `Server failed to start: ${err}` });
+    for (const r of results) console.log(`  [${icon(r.status)}] ${r.message}`);
+    console.log(`
+  Result: 0 passed, 1 failed
+`);
+    process.exit(1);
+    return;
+  }
+  try {
+    try {
+      const tools = await client.listTools();
+      const names = tools.map((t) => t.name).join(", ");
+      results.push({ status: "pass", message: `tools/list: ${tools.length} tools (${names})` });
+      const invalid = tools.filter((t) => !t.inputSchema || t.inputSchema.type !== "object");
+      if (invalid.length === 0) {
+        results.push({ status: "pass", message: "Tool schemas valid" });
+      } else {
+        results.push({
+          status: "fail",
+          message: `Invalid tool schemas: ${invalid.map((t) => t.name).join(", ")}`
+        });
+      }
+    } catch (err) {
+      results.push({ status: "fail", message: `tools/list failed: ${err}` });
+    }
+    try {
+      const prompts = await client.listPrompts();
+      const names = prompts.map((p) => p.name).join(", ");
+      results.push({ status: "pass", message: `prompts/list: ${prompts.length} prompts${prompts.length > 0 ? ` (${names})` : ""}` });
+    } catch (err) {
+      results.push({ status: "warn", message: `prompts/list: ${err}` });
+    }
+    try {
+      const resources = await client.listResources();
+      results.push({ status: "pass", message: `resources/list: ${resources.length} resources` });
+      if (resources.length > 0 && options?.verbose) {
+        try {
+          const first = resources[0];
+          await client.readResource(first.uri);
+          results.push({ status: "pass", message: `resources/read: ${first.uri} readable` });
+        } catch (err) {
+          results.push({ status: "warn", message: `resources/read failed: ${err}` });
+        }
+      }
+    } catch (err) {
+      results.push({ status: "warn", message: `resources/list: ${err}` });
+    }
+    await client.close();
+    results.push({ status: "pass", message: "Server closed cleanly" });
+  } catch (err) {
+    results.push({ status: "fail", message: `Unexpected error: ${err}` });
+    try {
+      await client.close();
+    } catch {
+    }
+  }
+  for (const r of results) {
+    console.log(`  [${icon(r.status)}] ${r.message}`);
+  }
+  const passed = results.filter((r) => r.status === "pass").length;
+  const failed = results.filter((r) => r.status === "fail").length;
+  const warned = results.filter((r) => r.status === "warn").length;
+  console.log("");
+  const parts = [`${passed} passed`, `${failed} failed`];
+  if (warned > 0) parts.push(`${warned} warnings`);
+  console.log(`  Result: ${parts.join(", ")}
+`);
+  if (failed > 0) process.exit(1);
+}
+export {
+  runMcpQa
+};

package/dist/setup-jest.d.ts

@@ -0,0 +1,2 @@
+
+export {  }

package/dist/setup-jest.js

@@ -0,0 +1,6 @@
+import {
+  mcpMatchers
+} from "./chunk-CG5UCD4O.js";
+
+// src/setup-jest.ts
+expect.extend(mcpMatchers);

package/dist/setup-vitest.d.ts

@@ -0,0 +1,2 @@
+
+export {  }

package/dist/setup-vitest.js

@@ -0,0 +1,7 @@
+import {
+  mcpMatchers
+} from "./chunk-CG5UCD4O.js";
+
+// src/setup-vitest.ts
+import { expect } from "vitest";
+expect.extend(mcpMatchers);

package/package.json

@@ -0,0 +1,67 @@
+{
+  "name": "@heilgar/pest-mcp",
+  "version": "0.0.2",
+  "description": "Prompt Evaluation & Scoring Toolkit - MCP server testing",
+  "type": "module",
+  "main": "./dist/index.js",
+  "types": "./dist/index.d.ts",
+  "exports": {
+    ".": {
+      "import": "./dist/index.js",
+      "types": "./dist/index.d.ts"
+    },
+    "./setup/vitest": {
+      "import": "./dist/setup-vitest.js",
+      "types": "./dist/setup-vitest.d.ts"
+    },
+    "./setup/jest": {
+      "import": "./dist/setup-jest.js",
+      "types": "./dist/setup-jest.d.ts"
+    },
+    "./qa": {
+      "import": "./dist/qa.js",
+      "types": "./dist/qa.d.ts"
+    }
+  },
+  "files": [
+    "dist"
+  ],
+  "keywords": [
+    "llm",
+    "prompt",
+    "testing",
+    "mcp",
+    "model-context-protocol",
+    "evaluation",
+    "ai"
+  ],
+  "license": "MIT",
+  "dependencies": {
+    "@modelcontextprotocol/sdk": "^1.27.0",
+    "valibot": "^1.2.0",
+    "@heilgar/pest-core": "0.0.2"
+  },
+  "peerDependencies": {
+    "vitest": ">=2.0.0",
+    "jest": ">=29.0.0"
+  },
+  "peerDependenciesMeta": {
+    "vitest": {
+      "optional": true
+    },
+    "jest": {
+      "optional": true
+    }
+  },
+  "devDependencies": {
+    "@types/node": "^25.5.0",
+    "tsup": "^8.5.1",
+    "typescript": "^5.9.3",
+    "vitest": "^4.1.0"
+  },
+  "scripts": {
+    "build": "tsup",
+    "dev": "tsup --watch",
+    "lint": "tsc --noEmit"
+  }
+}