@gleanwork/mcp-server-tester 1.1.1 → 2.0.0-beta.1

package/dist/fixtures/mcp.d.ts

@@ -216,6 +216,77 @@ type RubricSpec = BuiltInRubric | {
 /** Valid LLM judge provider kinds. */
 type ProviderKind = 'anthropic' | 'vertex-anthropic' | 'anthropic-agent-sdk' | 'openai' | 'google';
 
+/**
+ * High-level API for interacting with MCP servers in tests
+ *
+ * This interface wraps the raw MCP Client with test-friendly methods
+ */
+interface MCPFixtureApi {
+    /**
+     * The underlying MCP client (for advanced usage)
+     */
+    client: Client;
+    /**
+     * Authentication type used for this test session
+     */
+    authType: AuthType;
+    /**
+     * Playwright project name for this test session
+     */
+    project?: string;
+    /**
+     * Lists all available tools from the MCP server
+     *
+     * @returns Array of tool definitions
+     */
+    listTools(): Promise<Array<Tool>>;
+    /**
+     * Calls a tool on the MCP server
+     *
+     * @param name - Tool name
+     * @param args - Tool arguments
+     * @returns Tool call result
+     */
+    callTool<TArgs extends Record<string, unknown> = Record<string, unknown>>(name: string, args: TArgs): Promise<CallToolResult>;
+    /**
+     * Gets information about the connected server
+     */
+    getServerInfo(): {
+        name?: string;
+        version?: string;
+    } | null;
+}
+
+/**
+ * Canonical type definitions for @gleanwork/mcp-server-tester
+ *
+ * This module is the single source of truth for shared types.
+ * All other modules should import from here rather than defining their own.
+ *
+ * @packageDocumentation
+ */
+/**
+ * Authentication type for MCP connections
+ *
+ * - 'oauth': Interactive OAuth 2.1 with PKCE (browser-based authentication)
+ * - 'api-token': Static API token (e.g., from a dashboard or environment variable)
+ * - 'none': No authentication
+ */
+type AuthType = 'oauth' | 'api-token' | 'none';
+
+interface HostEvent {
+    kind: 'tool_call' | 'skill' | 'command' | 'subagent';
+    source: 'mcp' | 'host';
+    name: string;
+    server?: string;
+    arguments?: Record<string, unknown>;
+    output?: string;
+    /** Explicit tool-result error status, absent when not observed. */
+    isError?: boolean;
+    rawName?: string;
+    id?: string;
+}
+
 /**
  * Tool call validators for mcp_host simulation results.
  *
@@ -226,6 +297,9 @@ type ProviderKind = 'anthropic' | 'vertex-anthropic' | 'anthropic-agent-sdk' | '
 interface ToolCallExpectation {
     calls: Array<{
         name: string;
+        kind?: HostEvent['kind'];
+        source?: HostEvent['source'];
+        server?: string;
         arguments?: Record<string, unknown>;
         required?: boolean;
     }>;
@@ -248,6 +322,8 @@ interface ToolCallCountOptions {
  * Options for the LLM judge matcher
  */
 interface JudgeMatcherOptions {
+    /** Plugin options, parsed by the registered judge's schema. */
+    options?: Record<string, unknown>;
     /** Reference response to compare against */
     reference?: unknown;
     /** Score threshold for passing (default: 0.7) */
@@ -261,7 +337,7 @@ interface JudgeMatcherOptions {
     /**
      * Name of a registered custom judge executor.
      * When set, the named judge handles the entire evaluation pipeline
-     * and its `pass` result is authoritative.
+     * and its score is compared with passingThreshold.
      */
     judge?: string;
 }
@@ -615,64 +691,6 @@ declare const expect: playwright_test.Expect<{
     toHaveToolCallCount: typeof toHaveToolCallCount;
 }>;
 
-/**
- * Canonical type definitions for @gleanwork/mcp-server-tester
- *
- * This module is the single source of truth for shared types.
- * All other modules should import from here rather than defining their own.
- *
- * @packageDocumentation
- */
-/**
- * Authentication type for MCP connections
- *
- * - 'oauth': Interactive OAuth 2.1 with PKCE (browser-based authentication)
- * - 'api-token': Static API token (e.g., from a dashboard or environment variable)
- * - 'none': No authentication
- */
-type AuthType = 'oauth' | 'api-token' | 'none';
-
-/**
- * High-level API for interacting with MCP servers in tests
- *
- * This interface wraps the raw MCP Client with test-friendly methods
- */
-interface MCPFixtureApi {
-    /**
-     * The underlying MCP client (for advanced usage)
-     */
-    client: Client;
-    /**
-     * Authentication type used for this test session
-     */
-    authType: AuthType;
-    /**
-     * Playwright project name for this test session
-     */
-    project?: string;
-    /**
-     * Lists all available tools from the MCP server
-     *
-     * @returns Array of tool definitions
-     */
-    listTools(): Promise<Array<Tool>>;
-    /**
-     * Calls a tool on the MCP server
-     *
-     * @param name - Tool name
-     * @param args - Tool arguments
-     * @returns Tool call result
-     */
-    callTool<TArgs extends Record<string, unknown> = Record<string, unknown>>(name: string, args: TArgs): Promise<CallToolResult>;
-    /**
-     * Gets information about the connected server
-     */
-    getServerInfo(): {
-        name?: string;
-        version?: string;
-    } | null;
-}
-
 /**
  * Internal fixture state for passing auth type between fixtures
  */

package/dist/fixtures/mcp.js

@@ -101,6 +101,9 @@ function extractText(response) {
     if (typeof r.text === "string") {
       return r.text;
     }
+    if (typeof r.response === "string") {
+      return r.response;
+    }
     return JSON.stringify(r);
   }
   if (typeof response === "number" || typeof response === "boolean" || typeof response === "bigint") {
@@ -416,7 +419,7 @@ function toRegExp(pattern, caseInsensitive) {
     if (caseInsensitive && !pattern.flags.includes("i")) {
       return new RegExp(pattern.source, pattern.flags + "i");
     }
-    return pattern;
+    return new RegExp(pattern.source, pattern.flags);
   }
   const flags = caseInsensitive ? "i" : "";
   return new RegExp(pattern, flags);
@@ -1176,21 +1179,109 @@ function createJudge(config = {}) {
       );
   }
 }
+var REGISTRY_STATE_KEY = /* @__PURE__ */ Symbol.for(
+  "mcp-server-tester.framework-registry-state"
+);
+var globalRegistry = globalThis;
+var existingRegistryState = globalRegistry[REGISTRY_STATE_KEY];
+var registryState = existingRegistryState ?? {
+  datasets: /* @__PURE__ */ new Map(),
+  hosts: /* @__PURE__ */ new Map(),
+  judges: /* @__PURE__ */ new Map(),
+  metrics: /* @__PURE__ */ new Map(),
+  resultStores: /* @__PURE__ */ new Map()
+};
+if (!existingRegistryState) globalRegistry[REGISTRY_STATE_KEY] = registryState;
+function createRegistry(kind, implementations) {
+  const typedImplementations = implementations;
+  return {
+    register(implementation) {
+      const existing = typedImplementations.get(implementation.name);
+      if (existing && existing !== implementation) {
+        throw new Error(
+          `${kind} "${implementation.name}" is already registered.`
+        );
+      }
+      typedImplementations.set(implementation.name, implementation);
+    },
+    get(name) {
+      const implementation = typedImplementations.get(name);
+      if (!implementation) {
+        const available = [...typedImplementations.keys()].sort().join(", ");
+        throw new Error(
+          `${kind} "${name}" is not registered.${available ? ` Available: ${available}.` : ""}`
+        );
+      }
+      return implementation;
+    },
+    list() {
+      return [...typedImplementations.values()].sort(
+        (a, b) => a.name.localeCompare(b.name)
+      );
+    },
+    clear() {
+      typedImplementations.clear();
+    }
+  };
+}
+createRegistry(
+  "Dataset source",
+  registryState.datasets
+);
+createRegistry("Host", registryState.hosts);
+var judges = createRegistry("Judge", registryState.judges);
+createRegistry(
+  "Metric",
+  registryState.metrics
+);
+createRegistry(
+  "Result store",
+  registryState.resultStores
+);
+var getJudge = (name) => judges.get(name);
+z.object({
+  iterations: z.number().int().positive().optional(),
+  maxCases: z.number().int().positive().optional(),
+  concurrency: z.number().int().positive().optional(),
+  filterTags: z.array(z.string().min(1)).optional()
+}).strict();
 
 // src/judge/judgeRegistry.ts
-var registry = /* @__PURE__ */ new Map();
 function getRegisteredJudge(name) {
-  const executor = registry.get(name);
-  if (!executor) {
-    const available = registry.size > 0 ? ` Available judges: ${[...registry.keys()].join(", ")}` : " No judges are registered.";
-    throw new Error(
-      `Judge "${name}" is not registered.${available} Register it with registerJudge() before tests run.`
-    );
+  try {
+    const judge = getJudge(name);
+    return judge.evaluate;
+  } catch (error) {
+    if (error instanceof Error && !error.message.includes("Available:")) {
+      throw new Error(`${error.message} No judges are registered.`);
+    }
+    throw error;
   }
-  return executor;
+}
+function getRegisteredJudgeOptions(name, options) {
+  const parsed = getJudge(name).schema.parse(options);
+  if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) {
+    throw new Error(`Judge "${name}" schema must return an options object.`);
+  }
+  return parsed;
 }
 
 // src/assertions/validators/judge.ts
+var judgeFrameworkOptionKeys = /* @__PURE__ */ new Set([
+  "options",
+  "judge",
+  "rubric",
+  "reference",
+  "threshold",
+  "reps",
+  "provider",
+  "model",
+  "apiKeyEnvVar",
+  "maxTokens",
+  "temperature",
+  "maxBudgetUsd",
+  "maxToolOutputSize"
+]);
 function computeStdDev(scores, mean) {
   if (scores.length <= 1) return 0;
   const variance = scores.reduce((sum, s) => sum + (s - mean) ** 2, 0) / scores.length;
@@ -1214,12 +1305,25 @@ async function validateJudge(response, config) {
   if (judgeName !== void 0) {
     try {
       const executor = getRegisteredJudge(judgeName);
-      const judgeResult = await executor(response, reference ?? void 0);
+      const options = getRegisteredJudgeOptions(
+        judgeName,
+        config.options ?? Object.fromEntries(
+          Object.entries(config).filter(
+            ([key]) => !judgeFrameworkOptionKeys.has(key)
+          )
+        )
+      );
+      const judgeResult = await executor(
+        response,
+        reference ?? void 0,
+        options
+      );
       const score = judgeResult.score;
       const passed = score >= threshold;
       return {
         pass: passed,
-        message: passed ? `Custom judge "${judgeName}" passed with score ${score.toFixed(2)}` : `Custom judge "${judgeName}" failed with score ${score.toFixed(2)} (threshold: ${threshold}). ${judgeResult.reasoning ?? ""}`
+        message: passed ? `Custom judge "${judgeName}" passed with score ${score.toFixed(2)}` : `Custom judge "${judgeName}" failed with score ${score.toFixed(2)} (threshold: ${threshold}). ${judgeResult.reasoning ?? ""}`,
+        details: { score, reasoning: judgeResult.reasoning }
       };
     } catch (err) {
       return {
@@ -1309,7 +1413,8 @@ async function runSingleJudge(received, rubric, options) {
     reps,
     provider,
     model,
-    judge
+    judge,
+    options: judgeOptions
   } = options;
   const validation = await validateJudge(received, {
     ...rubric !== void 0 && { rubric },
@@ -1318,7 +1423,8 @@ async function runSingleJudge(received, rubric, options) {
     ...reps !== void 0 && { reps },
     ...provider !== void 0 && { provider },
     ...model !== void 0 && { model },
-    ...judge !== void 0 && { judge }
+    ...judge !== void 0 && { judge },
+    ...judgeOptions !== void 0 && { options: judgeOptions }
   });
   return { pass: validation.pass, message: validation.message };
 }
@@ -1491,10 +1597,22 @@ function partialMatch(actual, expected) {
     return JSON.stringify(actualVal) === JSON.stringify(v);
   });
 }
+function matchesIdentity(call, expected) {
+  return (call.name === expected.name || call.server !== void 0 && `${call.server}.${call.name}` === expected.name) && (expected.kind === void 0 || (call.kind ?? "tool_call") === expected.kind) && (expected.source === void 0 || call.source === expected.source) && (expected.server === void 0 || call.server === expected.server);
+}
+function unverifiedEvidence(response) {
+  if (response.evidence === void 0 || response.evidence === "structured")
+    return void 0;
+  return {
+    pass: false,
+    message: `Host evidence is ${response.evidence}; structured tool evidence is required.`,
+    details: { evidence: response.evidence }
+  };
+}
 function findMatchingCall(actual, expected, startIndex = 0) {
   for (let i = startIndex; i < actual.length; i++) {
     const call = actual[i];
-    if (call.name !== expected.name) continue;
+    if (!matchesIdentity(call, expected)) continue;
     if (expected.arguments !== void 0 && !partialMatch(call.arguments ?? {}, expected.arguments)) {
       continue;
     }
@@ -1506,18 +1624,22 @@ function validateToolCalls(response, expectation) {
   if (!isSimulationResult(response)) {
     return {
       pass: false,
-      message: "toolsTriggered expectation requires mcp_host mode \u2014 response must be an MCPHostSimulationResult"
+      message: "toolsTriggered expectation requires a host simulation response with structured tool calls"
     };
   }
-  const actual = response.toolCalls;
+  const unverified = unverifiedEvidence(response);
+  if (unverified) return unverified;
+  const actual = response.events ?? response.toolCalls;
   const requiredCalls = expectation.calls.filter((c) => c.required !== false);
   const calledRequiredCount = requiredCalls.filter(
     (expected) => findMatchingCall(actual, expected) !== -1
   ).length;
   const recall = requiredCalls.length > 0 ? calledRequiredCount / requiredCalls.length : 1;
   const allowedNames = new Set(expectation.calls.map((c) => c.name));
-  const precision = actual.length > 0 ? actual.filter((c) => allowedNames.has(c.name)).length / actual.length : 1;
-  const metrics = { precision, recall };
+  const precision = actual.length > 0 ? actual.filter(
+    (call) => expectation.calls.some((expected) => matchesIdentity(call, expected))
+  ).length / actual.length : 1;
+  const metrics2 = { precision, recall };
   const order = expectation.order ?? "any";
   if (order === "strict") {
     let searchFrom = 0;
@@ -1532,7 +1654,7 @@ function validateToolCalls(response, expectation) {
               actual: actual.map((c) => c.name),
               expected: expected.name
             },
-            metrics
+            metrics: metrics2
           };
         }
       } else {
@@ -1552,13 +1674,15 @@ function validateToolCalls(response, expectation) {
             actual: actual.map((c) => c.name),
             expected: expected.name
           },
-          metrics
+          metrics: metrics2
         };
       }
     }
   }
   if (expectation.exclusive === true) {
-    const unexpected = actual.filter((c) => !allowedNames.has(c.name));
+    const unexpected = actual.filter(
+      (call) => !expectation.calls.some((expected) => matchesIdentity(call, expected))
+    );
     if (unexpected.length > 0) {
       const names = unexpected.map((c) => `'${c.name}'`).join(", ");
       return {
@@ -1568,20 +1692,24 @@ function validateToolCalls(response, expectation) {
           actual: actual.map((c) => c.name),
           unexpected: unexpected.map((c) => c.name)
         },
-        metrics
+        metrics: metrics2
       };
     }
   }
-  return { pass: true, message: "All tool call expectations met", metrics };
+  return { pass: true, message: "All tool call expectations met", metrics: metrics2 };
 }
 function validateToolCallCount(response, options) {
   if (!isSimulationResult(response)) {
     return {
       pass: false,
-      message: "toolCallCount expectation requires mcp_host mode \u2014 response must be an MCPHostSimulationResult"
+      message: "toolCallCount expectation requires a host simulation response with structured tool calls"
     };
   }
-  const count = response.toolCalls.length;
+  const unverified = unverifiedEvidence(response);
+  if (unverified) return unverified;
+  const count = (response.events ?? response.toolCalls).filter(
+    (call) => (call.kind ?? "tool_call") === "tool_call"
+  ).length;
   const { min, max, exact } = options;
   if (exact !== void 0 && count !== exact) {
     return {
@@ -1665,14 +1793,19 @@ var MCPClientCredentialsConfigSchema = z.object({
 });
 var MCPAuthConfigSchema = z.object({
   accessToken: z.string().optional(),
+  accessTokenEnv: z.string().min(1).optional(),
   oauth: MCPOAuthConfigSchema.optional(),
   clientCredentials: MCPClientCredentialsConfigSchema.optional()
 }).refine(
+  (data) => !(data.accessToken && data.accessTokenEnv),
+  "Cannot specify both accessToken and accessTokenEnv"
+).refine(
   (data) => !(data.accessToken && data.oauth),
   "Cannot specify both accessToken and oauth configuration"
 );
 var StdioConfigSchema = z.object({
   transport: z.literal("stdio"),
+  label: z.string().min(1).optional(),
   command: z.string().min(1, "command is required for stdio transport"),
   args: z.array(z.string()).optional(),
   cwd: z.string().optional(),
@@ -1688,6 +1821,7 @@ function isLocalhost(hostname) {
 }
 var HttpConfigSchema = z.object({
   transport: z.literal("http"),
+  label: z.string().min(1).optional(),
   serverUrl: z.string().url("serverUrl must be a valid URL").refine((url) => {
     let parsed;
     try {
@@ -1740,7 +1874,7 @@ var debugHttp = createDebug(`${NAMESPACE}:http`);
 
 // package.json
 var package_default = {
-  version: "1.1.1"};
+  version: "2.0.0-beta.1"};
 var debug = createDebug("mcp-server-tester:oauth-flow");
 async function generatePKCE() {
   const codeVerifier = oauth.generateRandomCodeVerifier();
@@ -2090,7 +2224,7 @@ async function createMCPClientForConfig(config, options) {
         debugHttp("Attempting transport: streamableHttp");
         const streamableTransport = new StreamableHTTPClientTransport(url, {
           requestInit,
-          authProvider: options?.authProvider
+          ...options?.authProvider ? { authProvider: options.authProvider } : {}
         });
         await client.connect(streamableTransport, connectOptions);
         debugClient("Connected via Streamable HTTP");
@@ -2104,7 +2238,7 @@ async function createMCPClientForConfig(config, options) {
         debugHttp("Attempting transport: sse");
         const sseTransport = new SSEClientTransport(url, {
           requestInit,
-          authProvider: options?.authProvider
+          ...options?.authProvider ? { authProvider: options.authProvider } : {}
         });
         await client.connect(sseTransport, connectOptions);
         debugClient("Connected via SSE");