@gleanwork/mcp-server-tester 1.0.0-beta.6 → 1.0.0-beta.8

package/dist/index.js

@@ -18,7 +18,7 @@ import { SSEClientTransport } from '@modelcontextprotocol/sdk/client/sse.js';
 import { ProxyAgent, Agent } from 'undici';
 import { readFileSync } from 'fs';
 import { query } from '@anthropic-ai/claude-agent-sdk';
-import { execFile } from 'child_process';
+import { execFile, spawn } from 'child_process';
 import { promisify } from 'util';
 
 var __defProp = Object.defineProperty;
@@ -3279,7 +3279,11 @@ async function performOAuthSetup(config) {
     const page = await context.newPage();
     page.setDefaultTimeout(timeoutMs);
     await page.goto(authorizationUrl.toString());
-    await completeLoginForm(page, config);
+    if ("customLoginFlow" in config && config.customLoginFlow) {
+      await config.customLoginFlow(page);
+    } else {
+      await completeLoginForm(page, config);
+    }
     await page.waitForURL(
       (url) => url.href.startsWith(redirectUri) && url.searchParams.has("code"),
       { timeout: timeoutMs }
@@ -4380,7 +4384,7 @@ function escapeHtml(text) {
 
 // package.json
 var package_default = {
-  version: "1.0.0-beta.6"};
+  version: "1.0.0-beta.8"};
 
 // src/mcp/clientFactory.ts
 function getRetryAfterDelayMs(err) {
@@ -5097,9 +5101,17 @@ function formatBytes(bytes) {
 function isSimulationResult(value) {
   return typeof value === "object" && value !== null && "success" in value && "toolCalls" in value && Array.isArray(value.toolCalls);
 }
+function isPatternMatcher(v) {
+  return typeof v === "object" && v !== null && "$pattern" in v && typeof v["$pattern"] === "string";
+}
 function partialMatch(actual, expected) {
   return Object.entries(expected).every(([k, v]) => {
     const actualVal = actual[k];
+    if (isPatternMatcher(v)) {
+      if (typeof actualVal !== "string") return false;
+      const re = new RegExp(v.$pattern, v.$flags);
+      return re.test(actualVal);
+    }
     if (typeof v === "object" && v !== null && typeof actualVal === "object" && actualVal !== null) {
       return partialMatch(
         actualVal,
@@ -5238,7 +5250,175 @@ var JudgeResponseSchema = z.object({
   reasoning: z.string()
 });
 
-// src/judge/claudeAgentJudge.ts
+// src/judge/anthropicJudge.ts
+function createAnthropicJudge(config = {}) {
+  const apiKeyEnvVar = config.apiKeyEnvVar ?? "ANTHROPIC_API_KEY";
+  const apiKey = process.env[apiKeyEnvVar];
+  if (!apiKey) {
+    throw new Error(
+      `Anthropic judge requires an API key. Set the ${apiKeyEnvVar} environment variable.`
+    );
+  }
+  const model = config.model ?? "claude-sonnet-4-20250514";
+  const maxTokens = config.maxTokens ?? 1e3;
+  const temperature = config.temperature ?? 0;
+  return {
+    async evaluate(candidate, reference, rubric) {
+      let anthropicModule;
+      try {
+        anthropicModule = await import('@anthropic-ai/sdk');
+      } catch (err) {
+        throw new Error(
+          `Anthropic judge requires the \`@anthropic-ai/sdk\` package. Install it with: npm install @anthropic-ai/sdk
+Original error: ${err instanceof Error ? err.message : String(err)}`
+        );
+      }
+      const client = new anthropicModule.default({ apiKey });
+      const prompt = buildJudgePrompt(candidate, reference, rubric);
+      const startTime = Date.now();
+      const response = await client.messages.create({
+        model,
+        max_tokens: maxTokens,
+        temperature,
+        system: 'You are an expert evaluator. Respond with valid JSON only: {"pass": true|false, "score": 0.0-1.0, "reasoning": "explanation"}',
+        messages: [{ role: "user", content: prompt }]
+      });
+      const durationMs = Date.now() - startTime;
+      const textBlock = response.content.find(
+        (b) => b.type === "text"
+      );
+      const text = textBlock?.text ?? "";
+      const parsed = parseJudgeResponse(text);
+      return {
+        pass: parsed.pass,
+        score: parsed.score,
+        reasoning: parsed.reasoning,
+        usage: {
+          inputTokens: response.usage?.input_tokens ?? 0,
+          outputTokens: response.usage?.output_tokens ?? 0,
+          totalCostUsd: 0,
+          durationMs
+        }
+      };
+    }
+  };
+}
+function buildJudgePrompt(candidate, reference, rubric) {
+  const candidateStr = typeof candidate === "string" ? candidate : JSON.stringify(candidate, null, 2);
+  const referenceStr = reference !== null && reference !== void 0 ? typeof reference === "string" ? reference : JSON.stringify(reference, null, 2) : null;
+  return `Rubric:
+${rubric}
+
+<candidate_response>
+${candidateStr}
+</candidate_response>
+
+<reference_answer>
+${referenceStr ?? "No reference provided."}
+</reference_answer>
+
+Evaluate and return JSON: {"pass": boolean, "score": number (0-1), "reasoning": string}`;
+}
+function parseJudgeResponse(text) {
+  const cleaned = text.replace(/```json\n?/g, "").replace(/```\n?/g, "").trim();
+  let parsed;
+  try {
+    parsed = JSON.parse(cleaned);
+  } catch {
+    throw new Error(`Failed to parse judge response as JSON: ${text}`);
+  }
+  const result = JudgeResponseSchema.safeParse(parsed);
+  if (!result.success) {
+    throw new Error(
+      `Judge returned invalid response. Expected {pass, score, reasoning} but got: ${cleaned.slice(0, 500)}
+Validation errors: ${JSON.stringify(result.error.issues)}`
+    );
+  }
+  return result.data;
+}
+
+// src/judge/vertexAnthropicJudge.ts
+function createVertexAnthropicJudge(config = {}) {
+  const model = config.model ?? "claude-sonnet-4-20250514";
+  const maxTokens = config.maxTokens ?? 1e3;
+  const temperature = config.temperature ?? 0;
+  return {
+    async evaluate(candidate, reference, rubric) {
+      let vertexModule;
+      try {
+        vertexModule = await import('@anthropic-ai/vertex-sdk');
+      } catch (err) {
+        throw new Error(
+          `Vertex Anthropic judge requires the \`@anthropic-ai/vertex-sdk\` package. Install it with: npm install @anthropic-ai/vertex-sdk
+Original error: ${err instanceof Error ? err.message : String(err)}`
+        );
+      }
+      const client = new vertexModule.AnthropicVertex({
+        projectId: process.env.GOOGLE_VERTEX_PROJECT ?? process.env.CLOUD_ML_PROJECT_ID,
+        region: process.env.GOOGLE_VERTEX_LOCATION ?? "us-east5"
+      });
+      const prompt = buildJudgePrompt2(candidate, reference, rubric);
+      const startTime = Date.now();
+      const response = await client.messages.create({
+        model,
+        max_tokens: maxTokens,
+        temperature,
+        system: 'You are an expert evaluator. Respond with valid JSON only: {"pass": true|false, "score": 0.0-1.0, "reasoning": "explanation"}',
+        messages: [{ role: "user", content: prompt }]
+      });
+      const durationMs = Date.now() - startTime;
+      const textBlock = response.content.find(
+        (b) => b.type === "text"
+      );
+      const text = textBlock?.text ?? "";
+      const parsed = parseJudgeResponse2(text);
+      return {
+        pass: parsed.pass,
+        score: parsed.score,
+        reasoning: parsed.reasoning,
+        usage: {
+          inputTokens: response.usage?.input_tokens ?? 0,
+          outputTokens: response.usage?.output_tokens ?? 0,
+          totalCostUsd: 0,
+          durationMs
+        }
+      };
+    }
+  };
+}
+function buildJudgePrompt2(candidate, reference, rubric) {
+  const candidateStr = typeof candidate === "string" ? candidate : JSON.stringify(candidate, null, 2);
+  const referenceStr = reference !== null && reference !== void 0 ? typeof reference === "string" ? reference : JSON.stringify(reference, null, 2) : null;
+  return `Rubric:
+${rubric}
+
+<candidate_response>
+${candidateStr}
+</candidate_response>
+
+<reference_answer>
+${referenceStr ?? "No reference provided."}
+</reference_answer>
+
+Evaluate and return JSON: {"pass": boolean, "score": number (0-1), "reasoning": string}`;
+}
+function parseJudgeResponse2(text) {
+  const cleaned = text.replace(/```json\n?/g, "").replace(/```\n?/g, "").trim();
+  let parsed;
+  try {
+    parsed = JSON.parse(cleaned);
+  } catch {
+    throw new Error(`Failed to parse judge response as JSON: ${text}`);
+  }
+  const result = JudgeResponseSchema.safeParse(parsed);
+  if (!result.success) {
+    throw new Error(
+      `Judge returned invalid response. Expected {pass, score, reasoning} but got: ${cleaned.slice(0, 500)}
+Validation errors: ${JSON.stringify(result.error.issues)}`
+    );
+  }
+  return result.data;
+}
 function createClaudeAgentJudge(config) {
   const model = config.model ?? "claude-sonnet-4-20250514";
   const maxBudgetUsd = config.maxBudgetUsd ?? 0.1;
@@ -5256,7 +5436,7 @@ function createClaudeAgentJudge(config) {
           exceedsMaxToolOutputSize: true
         };
       }
-      const prompt = buildJudgePrompt(candidate, reference, rubric);
+      const prompt = buildJudgePrompt3(candidate, reference, rubric);
       try {
         let resultMessage;
         for await (const message of query({
@@ -5288,7 +5468,7 @@ function createClaudeAgentJudge(config) {
           );
         }
         const responseText = resultMessage.result ?? "";
-        const parsed = parseJudgeResponse(responseText);
+        const parsed = parseJudgeResponse3(responseText);
         const usage = {
           inputTokens: resultMessage.usage?.input_tokens ?? 0,
           outputTokens: resultMessage.usage?.output_tokens ?? 0,
@@ -5317,7 +5497,7 @@ function createClaudeAgentJudge(config) {
 function buildSystemPrompt() {
   return 'You are an expert evaluator. Evaluate the candidate response based on the rubric provided. Respond ONLY with valid JSON in this exact format: {"pass": boolean, "score": number (0-1), "reasoning": string}. Do not include any other text, markdown formatting, or code blocks.';
 }
-function buildJudgePrompt(candidate, reference, rubric) {
+function buildJudgePrompt3(candidate, reference, rubric) {
   const candidateStr = typeof candidate === "string" ? candidate : JSON.stringify(candidate, null, 2);
   const referenceStr = reference !== null && reference !== void 0 ? typeof reference === "string" ? reference : JSON.stringify(reference, null, 2) : null;
   const parts = [];
@@ -5334,7 +5514,7 @@ function buildJudgePrompt(candidate, reference, rubric) {
   );
   return parts.join("");
 }
-function parseJudgeResponse(text) {
+function parseJudgeResponse3(text) {
   let jsonText = text.trim();
   if (jsonText.startsWith("```json")) {
     jsonText = jsonText.slice(7);
@@ -5391,7 +5571,7 @@ Original error: ${err instanceof Error ? err.message : String(err)}`
         );
       }
       const client = new openaiModule.default({ apiKey });
-      const prompt = buildJudgePrompt2(candidate, reference, rubric);
+      const prompt = buildJudgePrompt4(candidate, reference, rubric);
       const startTime = Date.now();
       const completion = await client.chat.completions.create({
         model,
@@ -5407,7 +5587,7 @@ Original error: ${err instanceof Error ? err.message : String(err)}`
       });
       const durationMs = Date.now() - startTime;
       const text = completion.choices[0]?.message.content ?? "";
-      const parsed = parseJudgeResponse2(text);
+      const parsed = parseJudgeResponse4(text);
       return {
         pass: parsed.pass,
         score: parsed.score,
@@ -5422,7 +5602,7 @@ Original error: ${err instanceof Error ? err.message : String(err)}`
     }
   };
 }
-function buildJudgePrompt2(candidate, reference, rubric) {
+function buildJudgePrompt4(candidate, reference, rubric) {
   const candidateStr = typeof candidate === "string" ? candidate : JSON.stringify(candidate, null, 2);
   const referenceStr = reference !== null && reference !== void 0 ? typeof reference === "string" ? reference : JSON.stringify(reference, null, 2) : null;
   return `Rubric:
@@ -5438,7 +5618,7 @@ ${referenceStr ?? "No reference provided."}
 
 Evaluate and return JSON: {"pass": boolean, "score": number (0-1), "reasoning": string}`;
 }
-function parseJudgeResponse2(text) {
+function parseJudgeResponse4(text) {
   const cleaned = text.replace(/```json\n?/g, "").replace(/```\n?/g, "").trim();
   let parsed;
   try {
@@ -5540,6 +5720,10 @@ function createJudge(config = {}) {
   const provider = config.provider ?? "anthropic";
   switch (provider) {
     case "anthropic":
+      return createAnthropicJudge(config);
+    case "vertex-anthropic":
+      return createVertexAnthropicJudge(config);
+    case "anthropic-agent-sdk":
       return createClaudeAgentJudge(config);
     case "openai":
       return createOpenAIJudge(config);
@@ -5550,6 +5734,34 @@ function createJudge(config = {}) {
   }
 }
 
+// src/judge/judgeRegistry.ts
+var registry = /* @__PURE__ */ new Map();
+function registerJudge(name15, executor) {
+  const existing = registry.get(name15);
+  if (existing !== void 0) {
+    if (existing === executor) {
+      return;
+    }
+    throw new Error(
+      `Judge "${name15}" is already registered with a different executor. Use clearJudgeRegistry() first if you need to replace it.`
+    );
+  }
+  registry.set(name15, executor);
+}
+function getRegisteredJudge(name15) {
+  const executor = registry.get(name15);
+  if (!executor) {
+    const available = registry.size > 0 ? ` Available judges: ${[...registry.keys()].join(", ")}` : " No judges are registered.";
+    throw new Error(
+      `Judge "${name15}" is not registered.${available} Register it with registerJudge() before tests run.`
+    );
+  }
+  return executor;
+}
+function clearJudgeRegistry() {
+  registry.clear();
+}
+
 // src/assertions/validators/judge.ts
 function computeStdDev(scores, mean) {
   if (scores.length <= 1) return 0;
@@ -5558,6 +5770,7 @@ function computeStdDev(scores, mean) {
 }
 async function validateJudge(response, config) {
   const {
+    judge: judgeName,
     rubric,
     reference,
     threshold = 0.7,
@@ -5570,6 +5783,29 @@ async function validateJudge(response, config) {
     maxBudgetUsd,
     maxToolOutputSize
   } = config;
+  if (judgeName !== void 0) {
+    try {
+      const executor = getRegisteredJudge(judgeName);
+      const judgeResult = await executor(response, reference ?? void 0);
+      const score = judgeResult.score;
+      const passed = score >= threshold;
+      return {
+        pass: passed,
+        message: passed ? `Custom judge "${judgeName}" passed with score ${score.toFixed(2)}` : `Custom judge "${judgeName}" failed with score ${score.toFixed(2)} (threshold: ${threshold}). ${judgeResult.reasoning ?? ""}`
+      };
+    } catch (err) {
+      return {
+        pass: false,
+        message: `Custom judge "${judgeName}" error: ${err instanceof Error ? err.message : String(err)}`
+      };
+    }
+  }
+  if (rubric === void 0) {
+    return {
+      pass: false,
+      message: 'Judge evaluation failed: either "judge" or "rubric" must be provided'
+    };
+  }
   const resolvedRubric = resolveRubric(rubric);
   const judgeConfig = {
     ...provider !== void 0 && { provider },
@@ -5616,11 +5852,17 @@ async function validateJudge(response, config) {
     return {
       pass: passed,
       message: passed ? `Judge passed with score ${meanScore.toFixed(2)}${repNote}` : `Judge failed with score ${meanScore.toFixed(2)} (threshold: ${threshold})${repNote}. ${lastReasoning ?? ""}`,
-      details: reps > 1 ? {
-        scores,
-        scoreStdDev: stdDev,
-        highVariance
-      } : void 0
+      details: {
+        score: meanScore,
+        reasoning: lastReasoning,
+        judgeProvider: provider ?? "anthropic",
+        judgeModel: model,
+        ...reps > 1 && {
+          scores,
+          scoreStdDev: stdDev,
+          highVariance
+        }
+      }
     };
   } catch (err) {
     return {
@@ -5999,31 +6241,68 @@ function toBeToolError(received, expected = true) {
 
 // src/assertions/matchers/toPassToolJudge.ts
 var DEFAULT_PASSING_THRESHOLD = 0.7;
-async function toPassToolJudge(received, rubric, options = {}) {
+async function runSingleJudge(received, rubric, options) {
   const {
     reference = null,
     passingThreshold = DEFAULT_PASSING_THRESHOLD,
     reps,
     provider,
-    model
+    model,
+    judge
   } = options;
   const validation = await validateJudge(received, {
-    rubric,
+    ...rubric !== void 0 && { rubric },
     reference: reference ?? void 0,
     threshold: passingThreshold,
     ...reps !== void 0 && { reps },
     ...provider !== void 0 && { provider },
-    ...model !== void 0 && { model }
+    ...model !== void 0 && { model },
+    ...judge !== void 0 && { judge }
   });
+  return { pass: validation.pass, message: validation.message };
+}
+async function toPassToolJudge(received, rubricOrOptions, maybeOptions) {
+  if (Array.isArray(rubricOrOptions)) {
+    const results = await Promise.all(
+      rubricOrOptions.map(async (judgeConfig) => {
+        const { rubric: r, ...opts } = judgeConfig;
+        return runSingleJudge(received, r, opts);
+      })
+    );
+    const allPassed = results.every((r) => r.pass);
+    const passCount = results.filter((r) => r.pass).length;
+    const summary = `${passCount}/${results.length} judges passed`;
+    const details = results.map((r) => r.message).join("\n");
+    if (this.isNot) {
+      return {
+        pass: !allPassed,
+        message: () => allPassed ? `Expected all judges to fail, but ${summary}` : `Judges failed as expected: ${summary}`
+      };
+    }
+    return {
+      pass: allPassed,
+      message: () => `${summary}
+${details}`
+    };
+  }
+  let rubric;
+  let options;
+  if (typeof rubricOrOptions === "string" || typeof rubricOrOptions === "object" && rubricOrOptions !== null && "text" in rubricOrOptions) {
+    rubric = rubricOrOptions;
+    options = maybeOptions ?? {};
+  } else {
+    options = rubricOrOptions;
+  }
+  const result = await runSingleJudge(received, rubric, options);
   if (this.isNot) {
     return {
-      pass: !validation.pass,
-      message: () => validation.pass ? `Expected judge evaluation to fail, but it passed` : `Judge evaluation failed as expected`
+      pass: !result.pass,
+      message: () => result.pass ? `Expected judge evaluation to fail, but it passed` : `Judge evaluation failed as expected`
     };
   }
   return {
-    pass: validation.pass,
-    message: () => validation.message
+    pass: result.pass,
+    message: () => result.message
   };
 }
 
@@ -6307,6 +6586,7 @@ function getAuthConfigFromEnv() {
   return void 0;
 }
 var MCPHostConfigSchema = z.object({
+  hostType: z.enum(["sdk", "cli", "browser", "desktop"]).optional(),
   provider: z.enum([
     "openai",
     "anthropic",
@@ -6317,12 +6597,18 @@ var MCPHostConfigSchema = z.object({
     "openrouter",
     "xai",
     "vertex-anthropic"
-  ]),
+  ]).optional(),
   apiKeyEnvVar: z.string().optional(),
   model: z.string().optional(),
   maxTokens: z.number().optional(),
   temperature: z.number().optional(),
-  maxToolCalls: z.number().optional()
+  maxToolCalls: z.number().optional(),
+  cli: z.object({
+    command: z.string(),
+    args: z.array(z.string()),
+    outputFormat: z.enum(["stream-json", "json"]).optional(),
+    timeout: z.number().optional()
+  }).optional()
 });
 var SnapshotSanitizerSchema = z.union([
   // Built-in sanitizers
@@ -6337,6 +6623,37 @@ var SnapshotSanitizerSchema = z.union([
     remove: z.array(z.string())
   })
 ]);
+var JudgeExpectConfigSchema = z.object({
+  judge: z.string().min(1).optional(),
+  rubric: z.union([
+    z.enum([
+      "correctness",
+      "completeness",
+      "groundedness",
+      "instruction-following",
+      "conciseness"
+    ]),
+    z.object({ text: z.string().min(1) })
+  ]).optional(),
+  reference: z.unknown().optional(),
+  threshold: z.number().min(0).max(1).optional(),
+  reps: z.number().int().min(1).optional(),
+  provider: z.enum([
+    "anthropic",
+    "vertex-anthropic",
+    "anthropic-agent-sdk",
+    "openai",
+    "google"
+  ]).optional(),
+  model: z.string().optional(),
+  apiKeyEnvVar: z.string().optional(),
+  maxTokens: z.number().int().positive().optional(),
+  temperature: z.number().min(0).max(1).optional(),
+  maxBudgetUsd: z.number().positive().optional(),
+  maxToolOutputSize: z.number().int().positive().optional()
+}).refine((data) => data.judge !== void 0 || data.rubric !== void 0, {
+  message: 'Either "judge" or "rubric" must be provided in passesJudge'
+});
 var EvalExpectBlockSchema = z.object({
   response: z.unknown().optional(),
   schema: z.string().optional(),
@@ -6345,28 +6662,7 @@ var EvalExpectBlockSchema = z.object({
   snapshot: z.string().optional(),
   snapshotSanitizers: z.array(SnapshotSanitizerSchema).optional(),
   isError: z.union([z.boolean(), z.string(), z.array(z.string())]).optional(),
-  passesJudge: z.object({
-    rubric: z.union([
-      z.enum([
-        "correctness",
-        "completeness",
-        "groundedness",
-        "instruction-following",
-        "conciseness"
-      ]),
-      z.object({ text: z.string().min(1) })
-    ]),
-    reference: z.unknown().optional(),
-    threshold: z.number().min(0).max(1).optional(),
-    reps: z.number().int().min(1).optional(),
-    provider: z.enum(["anthropic", "openai", "google"]).optional(),
-    model: z.string().optional(),
-    apiKeyEnvVar: z.string().optional(),
-    maxTokens: z.number().int().positive().optional(),
-    temperature: z.number().min(0).max(1).optional(),
-    maxBudgetUsd: z.number().positive().optional(),
-    maxToolOutputSize: z.number().int().positive().optional()
-  }).optional(),
+  passesJudge: z.union([JudgeExpectConfigSchema, z.array(JudgeExpectConfigSchema).min(1)]).optional(),
   responseSize: z.object({
     maxBytes: z.number().optional(),
     minBytes: z.number().optional()
@@ -6539,6 +6835,9 @@ function createVercelOrchestrator() {
       try {
         const { generateText, stepCountIs } = await import('ai');
         const { jsonSchema: jsonSchema2 } = await Promise.resolve().then(() => (init_dist3(), dist_exports));
+        if (!config.provider) {
+          throw new Error("provider is required for SDK host type");
+        }
         const modelId = config.model ?? defaultModel(config.provider);
         const model = await loadModel(config.provider, modelId);
         const mcpTools = await mcp.listTools();
@@ -6592,13 +6891,233 @@ function createVercelOrchestrator() {
         return {
           success: false,
           toolCalls: [],
-          error: enrichErrorMessage(err, config.provider)
+          error: enrichErrorMessage(err, config.provider ?? "unknown")
         };
       }
     }
   };
 }
 
+// src/evals/mcpHost/adapters/cli/parsers.ts
+function parseStreamJson(stdout) {
+  const lines = stdout.split("\n").filter((line) => line.trim().length > 0);
+  const toolCalls = [];
+  const textParts = [];
+  const conversationHistory = [];
+  for (const line of lines) {
+    let event;
+    try {
+      event = JSON.parse(line);
+    } catch {
+      continue;
+    }
+    if (event.type === "assistant" && event.message?.content) {
+      for (const block of event.message.content) {
+        if (block.type === "tool_use" && block.name) {
+          const rawName = block.name;
+          const mcpMatch = /^mcp__[^_]+__(.+)$/.exec(rawName);
+          toolCalls.push({
+            name: mcpMatch ? mcpMatch[1] : rawName,
+            arguments: block.input ?? {},
+            id: block.id
+          });
+        }
+        if (block.type === "text" && block.text) {
+          textParts.push(block.text);
+        }
+      }
+    }
+    if (event.type === "user" && event.message?.content) {
+      for (const block of event.message.content) {
+        if (block.type === "tool_result") {
+          const content = typeof block.content === "string" ? block.content : JSON.stringify(block.content);
+          conversationHistory.push({ role: "tool", content });
+        }
+      }
+    }
+    if (event.type === "result" && typeof event.result === "string") {
+      if (textParts.length === 0) {
+        textParts.push(event.result);
+      }
+    }
+    if (event.type === "result" && event.is_error === true) {
+      return {
+        success: false,
+        toolCalls,
+        error: typeof event.result === "string" ? event.result : "CLI host reported an error"
+      };
+    }
+  }
+  const response = textParts.join("");
+  if (response) {
+    conversationHistory.push({ role: "assistant", content: response });
+  }
+  return {
+    success: true,
+    toolCalls,
+    response: response || void 0,
+    conversationHistory: conversationHistory.length > 0 ? conversationHistory : void 0
+  };
+}
+function createJsonParser(paths) {
+  return (stdout) => {
+    const data = JSON.parse(stdout);
+    const rawToolCalls = getNestedValue(data, paths.toolCalls);
+    const toolCalls = Array.isArray(rawToolCalls) ? rawToolCalls.map((tc) => ({
+      name: typeof tc.name === "string" ? tc.name : "",
+      arguments: tc.arguments ?? tc.args ?? {}
+    })) : [];
+    const response = getNestedValue(data, paths.response);
+    const success = paths.success ? Boolean(getNestedValue(data, paths.success)) : true;
+    return {
+      success,
+      toolCalls,
+      response: typeof response === "string" ? response : void 0
+    };
+  };
+}
+function getNestedValue(obj, path3) {
+  return path3.split(".").reduce((current, key) => {
+    if (current !== null && typeof current === "object") {
+      return current[key];
+    }
+    return void 0;
+  }, obj);
+}
+
+// src/evals/mcpHost/adapters/cli/runner.ts
+var DEFAULT_TIMEOUT = 12e4;
+var MAX_BUFFER = 10 * 1024 * 1024;
+function getParser(format) {
+  switch (format ?? "stream-json") {
+    case "stream-json":
+      return parseStreamJson;
+    case "json":
+      return createJsonParser({
+        toolCalls: "toolCalls",
+        response: "response",
+        success: "success"
+      });
+  }
+}
+function interpolateArgs(args, scenario) {
+  return args.map((arg) => arg.replace(/\{\{scenario\}\}/g, scenario));
+}
+async function runCLIHost(cliConfig, scenario) {
+  const timeout = cliConfig.timeout ?? DEFAULT_TIMEOUT;
+  const args = interpolateArgs(cliConfig.args, scenario);
+  const startTime = Date.now();
+  let stdout;
+  try {
+    const result2 = await spawnProcess(cliConfig.command, args, { timeout });
+    stdout = result2.stdout;
+  } catch (err) {
+    const elapsed = Date.now() - startTime;
+    const message = err instanceof Error ? err.message : String(err);
+    if (message.includes("TIMEOUT") || message.includes("timed out")) {
+      return {
+        success: false,
+        toolCalls: [],
+        error: `CLI host timed out after ${elapsed}ms (limit: ${timeout}ms). Increase timeout via mcpHostConfig.cli.timeout.`
+      };
+    }
+    return {
+      success: false,
+      toolCalls: [],
+      error: `CLI host process failed: ${message}`
+    };
+  }
+  const parse = getParser(cliConfig.outputFormat);
+  let result;
+  try {
+    result = parse(stdout);
+  } catch (err) {
+    return {
+      success: false,
+      toolCalls: [],
+      error: `Failed to parse CLI host output: ${err instanceof Error ? err.message : String(err)}
+stdout (first 500 chars): ${stdout.slice(0, 500)}`
+    };
+  }
+  const validationError = validateSimulationResult(result);
+  if (validationError) {
+    return {
+      success: false,
+      toolCalls: [],
+      error: `CLI host returned invalid result: ${validationError}`
+    };
+  }
+  return result;
+}
+function validateSimulationResult(result) {
+  if (result === null || typeof result !== "object") {
+    return `Expected object, got ${typeof result}`;
+  }
+  const obj = result;
+  if (typeof obj.success !== "boolean") {
+    return `"success" must be a boolean, got ${typeof obj.success}`;
+  }
+  if (!Array.isArray(obj.toolCalls)) {
+    return `"toolCalls" must be an array, got ${typeof obj.toolCalls}`;
+  }
+  for (let i = 0; i < obj.toolCalls.length; i++) {
+    const tc = obj.toolCalls[i];
+    if (typeof tc.name !== "string") {
+      return `toolCalls[${i}].name must be a string, got ${typeof tc.name}`;
+    }
+    if (typeof tc.arguments !== "object" || tc.arguments === null) {
+      return `toolCalls[${i}].arguments must be an object, got ${typeof tc.arguments}`;
+    }
+  }
+  return null;
+}
+function spawnProcess(command, args, options) {
+  return new Promise((resolve2, reject) => {
+    const child = spawn(command, args, {
+      stdio: ["pipe", "pipe", "pipe"]
+    });
+    child.stdin.end();
+    const stdoutChunks = [];
+    const stderrChunks = [];
+    let totalBytes = 0;
+    child.stdout.on("data", (chunk) => {
+      totalBytes += chunk.length;
+      if (totalBytes <= MAX_BUFFER) {
+        stdoutChunks.push(chunk);
+      }
+    });
+    child.stderr.on("data", (chunk) => {
+      totalBytes += chunk.length;
+      if (totalBytes <= MAX_BUFFER) {
+        stderrChunks.push(chunk);
+      }
+    });
+    const timer = setTimeout(() => {
+      child.kill("SIGTERM");
+      reject(new Error(`Process timed out after ${options.timeout}ms`));
+    }, options.timeout);
+    child.on("error", (err) => {
+      clearTimeout(timer);
+      reject(err);
+    });
+    child.on("close", (code) => {
+      clearTimeout(timer);
+      const stdout = Buffer.concat(stdoutChunks).toString("utf-8");
+      const stderr = Buffer.concat(stderrChunks).toString("utf-8");
+      if (code !== 0) {
+        reject(
+          new Error(
+            `Command failed with exit code ${code ?? "null"}` + (stderr ? `
+stderr: ${stderr}` : "")
+          )
+        );
+        return;
+      }
+      resolve2({ stdout, stderr });
+    });
+  });
+}
+
 // src/evals/mcpHost/mcpHostSimulation.ts
 var vercelOrchestrator = createVercelOrchestrator();
 var allProviders = [
@@ -6616,6 +7135,25 @@ var simulatorRegistry = new Map(
   allProviders.map((p) => [p, vercelOrchestrator])
 );
 async function simulateMCPHost(mcp, scenario, config) {
+  const hostType = config.hostType ?? "sdk";
+  if (hostType === "cli") {
+    if (!config.cli) {
+      throw new Error(
+        `mcpHostConfig.cli is required when hostType is 'cli'. Provide { command } with a shell command containing {{scenario}}.`
+      );
+    }
+    return runCLIHost(config.cli, scenario);
+  }
+  if (hostType === "browser" || hostType === "desktop") {
+    throw new Error(
+      `Host type '${hostType}' is not yet implemented. Supported host types: 'sdk', 'cli'.`
+    );
+  }
+  if (!config.provider) {
+    throw new Error(
+      `mcpHostConfig.provider is required for 'sdk' host type. Supported: ${allProviders.join(", ")}`
+    );
+  }
   const simulator = simulatorRegistry.get(config.provider);
   if (!simulator) {
     throw new Error(
@@ -6807,17 +7345,39 @@ async function runExpectBlockValidations(expectBlock, response, config) {
     };
   }
   if (expectBlock.passesJudge !== void 0) {
-    const effectiveReps = expectBlock.passesJudge.reps ?? config.judgeReps ?? 1;
-    const effectiveReference = expectBlock.passesJudge.reference !== void 0 ? expectBlock.passesJudge.reference : config.canonicalAnswer;
-    const validation = await validateJudge(response, {
-      ...expectBlock.passesJudge,
-      reference: effectiveReference,
-      reps: effectiveReps
-    });
-    results.judge = {
-      pass: validation.pass,
-      details: validation.message
-    };
+    const judgeConfigs = Array.isArray(expectBlock.passesJudge) ? expectBlock.passesJudge : [expectBlock.passesJudge];
+    const judgeResultEntries = await Promise.all(
+      judgeConfigs.map(async (judgeConfig) => {
+        const effectiveReps = judgeConfig.reps ?? config.judgeReps ?? 1;
+        const effectiveReference = judgeConfig.reference !== void 0 ? judgeConfig.reference : config.canonicalAnswer;
+        const validation = await validateJudge(response, {
+          ...judgeConfig,
+          reference: effectiveReference,
+          reps: effectiveReps
+        });
+        const judgeName = judgeConfig.judge ?? (typeof judgeConfig.rubric === "string" ? judgeConfig.rubric : void 0);
+        return {
+          pass: validation.pass,
+          details: validation.message,
+          score: validation.details?.score,
+          reasoning: validation.details?.reasoning,
+          judgeName,
+          judgeProvider: validation.details?.judgeProvider,
+          judgeModel: validation.details?.judgeModel
+        };
+      })
+    );
+    if (judgeResultEntries.length === 1) {
+      results.judge = judgeResultEntries[0];
+    } else {
+      const allPassed = judgeResultEntries.every((r) => r.pass);
+      const passCount = judgeResultEntries.filter((r) => r.pass).length;
+      results.judge = {
+        pass: allPassed,
+        details: `${passCount}/${judgeResultEntries.length} judges passed`,
+        judgeResults: judgeResultEntries
+      };
+    }
   }
   if (expectBlock.snapshot !== void 0) {
     if (!config.playwrightExpect) {
@@ -6846,6 +7406,24 @@ async function runExpectBlockValidations(expectBlock, response, config) {
   }
   return { expectations: results, toolPrecision, toolRecall };
 }
+function buildRequest(evalCase) {
+  const request = {};
+  if (evalCase.description) request.description = evalCase.description;
+  if (evalCase.mode === "mcp_host") {
+    if (evalCase.scenario) request.scenario = evalCase.scenario;
+    if (evalCase.mcpHostConfig) {
+      request.mcpHostConfig = {
+        provider: evalCase.mcpHostConfig.provider,
+        ...evalCase.mcpHostConfig.model !== void 0 && {
+          model: evalCase.mcpHostConfig.model
+        }
+      };
+    }
+  } else {
+    if (evalCase.args) request.args = evalCase.args;
+  }
+  return request;
+}
 function isMCPHostSimulationResult(value) {
   return typeof value === "object" && value !== null && "success" in value && "toolCalls" in value && Array.isArray(value.toolCalls);
 }
@@ -6894,6 +7472,7 @@ async function runSingleIteration(evalCase, context, options) {
     toolName: evalCase.scenario != null ? "mcp_host" : evalCase.toolName ?? "unknown",
     source: "eval",
     pass: didCasePass(error, expectationResults),
+    request: buildRequest(evalCase),
     response,
     error,
     expectations: expectationResults,
@@ -7036,8 +7615,13 @@ async function runEvalDataset(options, context) {
   const casesToRun = filterTags && filterTags.length > 0 ? dataset.cases.filter((c) => c.tags?.some((t) => filterTags.includes(t))) : dataset.cases;
   const estimatedJudgeCalls = casesToRun.reduce((sum, c) => {
     const effectiveIterations = c.mode === "mcp_host" ? c.iterations ?? defaultLlmIterations ?? 1 : c.iterations ?? 1;
-    const judgeReps = c.expect?.passesJudge != null ? c.expect.passesJudge.reps ?? c.judgeReps ?? defaultJudgeReps ?? 1 : 0;
-    return sum + effectiveIterations * judgeReps;
+    if (c.expect?.passesJudge == null) return sum;
+    const judges = Array.isArray(c.expect.passesJudge) ? c.expect.passesJudge : [c.expect.passesJudge];
+    const totalReps = judges.reduce(
+      (r, j) => r + (j.reps ?? c.judgeReps ?? defaultJudgeReps ?? 1),
+      0
+    );
+    return sum + effectiveIterations * totalReps;
   }, 0);
   if (estimatedJudgeCalls > 50) {
     debugEval(
@@ -7385,6 +7969,6 @@ function formatCapabilities(capabilities) {
   return parts.length > 0 ? parts.join(", ") : "none declared";
 }
 
-export { BUILT_IN_RUBRICS, CLIOAuthClient, DiscoveryError, ENV_VAR_NAMES, EvalCaseSchema, EvalDatasetSchema, MCPConfigSchema, MCP_PROTOCOL_VERSION, PlaywrightOAuthClientProvider, SnapshotSanitizers, closeMCPClient, createJudge, createMCPClientForConfig, createMCPFixture, createTokenAuthHeaders, discoverAuthorizationServer, discoverProtectedResource, expect, extractText, getMissingDependencyMessage, getResponseSizeBytes, hasValidTokens, injectTokens, isBuiltInRubric, isHttpConfig, isProviderAvailable, isStdioConfig, isTokenExpired, isTokenExpiringSoon, loadBaseline, loadEvalDataset, loadEvalDatasetFromObject, loadTokens, loadTokensFromEnv, test2 as mcpAuthTest, normalizeToolResponse, normalizeWhitespace, performClientCredentialsFlow, performOAuthSetup, performOAuthSetupIfNeeded, resolveRubric, runConformanceChecks, runEvalCase, runEvalDataset, runServerComparison, saveBaseline, simulateMCPHost, test, validateAccessToken, validateError, validateEvalCase, validateEvalDataset, validateJudge, validateMCPConfig, validatePattern, validateResponse, validateSchema, validateSize, validateText, validateToolCallCount, validateToolCalls };
+export { BUILT_IN_RUBRICS, CLIOAuthClient, DiscoveryError, ENV_VAR_NAMES, EvalCaseSchema, EvalDatasetSchema, MCPConfigSchema, MCP_PROTOCOL_VERSION, PlaywrightOAuthClientProvider, SnapshotSanitizers, clearJudgeRegistry, closeMCPClient, createJudge, createMCPClientForConfig, createMCPFixture, createTokenAuthHeaders, discoverAuthorizationServer, discoverProtectedResource, expect, extractText, getMissingDependencyMessage, getRegisteredJudge, getResponseSizeBytes, hasValidTokens, injectTokens, isBuiltInRubric, isHttpConfig, isProviderAvailable, isStdioConfig, isTokenExpired, isTokenExpiringSoon, loadBaseline, loadEvalDataset, loadEvalDatasetFromObject, loadTokens, loadTokensFromEnv, test2 as mcpAuthTest, normalizeToolResponse, normalizeWhitespace, performClientCredentialsFlow, performOAuthSetup, performOAuthSetupIfNeeded, registerJudge, resolveRubric, runConformanceChecks, runEvalCase, runEvalDataset, runServerComparison, saveBaseline, simulateMCPHost, test, validateAccessToken, validateError, validateEvalCase, validateEvalDataset, validateJudge, validateMCPConfig, validatePattern, validateResponse, validateSchema, validateSize, validateText, validateToolCallCount, validateToolCalls };
 //# sourceMappingURL=index.js.map
 //# sourceMappingURL=index.js.map