@gleanwork/mcp-server-tester 1.1.0 → 1.1.1

package/dist/index.d.cts

@@ -1,5 +1,5 @@
-import { L as LLMProvider, M as MCPFixtureApi, a as MCPHostConfig, b as MCPHostSimulationResult, O as OAuthSetupConfig, V as ValidationResult, S as SchemaValidatorOptions, T as TextValidatorOptions, P as PatternValidatorOptions, c as SizeValidatorOptions, d as SnapshotSanitizer, R as RubricSpec, J as JudgeMatcherOptions, e as ToolPredicate, f as ToolCallExpectation, g as ToolCallCountOptions, A as AuthType, h as JudgeConfig, i as Judge } from './index-BcUWzQCx.cjs';
-export { B as BUILT_IN_RUBRICS, j as BuiltInRubric, k as BuiltInSanitizer, C as CLIConfig, l as CLIOAuthClient, m as CLIOAuthClientConfig, n as CLIOAuthResult, o as CLIOutputFormat, p as CaseComparisonResult, q as ClientCredentialsConfig, r as CompareEvalRunsOptions, s as ComparisonOutcome, t as ContentBlock, u as CreateMCPClientOptions, v as CustomJudgeExecutor, w as CustomJudgeResult, D as DiscoveryError, E as ENV_VAR_NAMES, x as EvalCase, y as EvalCaseComparison, z as EvalCaseComparisonOutcome, F as EvalCaseRequest, G as EvalCaseResult, H as EvalCaseSchema, I as EvalContext, K as EvalDataset, N as EvalDatasetSchema, Q as EvalExpectBlock, U as EvalExpectationResult, W as EvalMode, X as EvalResultStore, Y as EvalResultStoreConfig, Z as EvalResultStoreLike, _ as EvalRunComparisonLabels, $ as EvalRunComparisonResult, a0 as EvalRunMetadata, a1 as EvalRunnerOptions, a2 as EvalRunnerResult, a3 as ExpectationBreakdown, a4 as ExpectationResultMap, a5 as ExpectationType, a6 as ExperimentMetric, a7 as FieldRemovalSanitizer, a8 as FileEvalResultStore, a9 as FileEvalResultStoreConfig, aa as GCSEvalResultStore, ab as GCSEvalResultStoreConfig, ac as HostType, ad as HttpMCPConfig, ae as IterationResult, af as JudgeExpectConfig, ag as JudgeResult, ah as JudgeValidatorConfig, ai as LLMToolCall, aj as ListStoredArtifactsOptions, ak as LoadDatasetOptions, al as MCPAuthConfig, am as MCPAuthFixtures, an as MCPClientCredentialsConfig, ao as MCPConfig, ap as MCPConfigSchema, aq as MCPConformanceCheck, ar as MCPConformanceOptions, as as MCPConformanceRaw, at as MCPConformanceResult, au as MCPConformanceResultData, av as MCPEvalData, aw as MCPEvalHistoricalSummary, ax as MCPEvalReporterConfig, ay as MCPEvalRunData, az as MCPFixtureOptions, aA as MCPHostCapabilities, aB as MCPHostSimulator, aC as MCPOAuthConfig, aD as MCPServerCapabilitiesData, aE as MCP_PROTOCOL_VERSION, aF as NormalizedToolResponse, aG as PlaywrightOAuthClientProvider, aH as PlaywrightOAuthClientProviderConfig, aI as PredicateResult, aJ as ProposeVariantsContext, aK as ProtectedResourceDiscoveryResult, aL as ProtectedResourceMetadata, aM as ProviderKind, aN as RegexSanitizer, aO as ResultSource, aP as SaveBaselineOptions, aQ as SaveEvalRunComparisonOptions, aR as SaveServerComparisonOptions, aS as SchemaRegistry, aT as SerializedEvalDataset, aU as ServerComparisonOptions, aV as ServerComparisonResult, aW as SnapshotSanitizers, aX as StdioMCPConfig, aY as StoredArtifactKind, aZ as StoredArtifactSummary, a_ as StoredClientInfo, a$ as StoredEvalArtifact, b0 as StoredEvalArtifactMetadata, b1 as StoredEvalResultLoadOptions, b2 as StoredEvalResultRef, b3 as StoredEvalResultSaveOptions, b4 as StoredEvalRunRef, b5 as StoredOAuthState, b6 as StoredServerMetadata, b7 as StoredTokens, b8 as TokenResult, b9 as ToolMetadataOverride, ba as ToolOverrideVariant, bb as UsageMetrics, bc as VariantCandidateResult, bd as VariantExperimentOptions, be as VariantExperimentReason, bf as VariantExperimentResult, bg as VariantExperimentRound, bh as VariantImprovementProposal, bi as VariantRecommendation, bj as clearJudgeRegistry, bk as closeMCPClient, bl as compareEvalRuns, bm as createDefaultArtifactId, bn as createEvalResultStore, bo as createMCPClientForConfig, bp as createMCPFixture, bq as createStoredEvalArtifact, br as defaultEnvironmentMetadata, bs as discoverAuthorizationServer, bt as discoverProtectedResource, bu as extractText, bv as getRegisteredJudge, bw as hasValidTokens, bx as injectTokens, by as isBuiltInRubric, bz as isEvalResultStore, bA as isHttpConfig, bB as isStdioConfig, bC as loadBaseline, bD as loadEvalDataset, bE as loadEvalDatasetFromObject, bF as loadStoredEvalRunnerResult, bG as loadTokens, bH as loadTokensFromEnv, bI as mcpAuthTest, bJ as normalizeToolResponse, bK as performClientCredentialsFlow, bL as refreshAccessToken, bM as registerJudge, bN as resolveEvalResultStore, bO as resolveRubric, bP as runConformanceChecks, bQ as runEvalCase, bR as runEvalDataset, bS as runServerComparison, bT as runVariantExperiment, bU as saveBaseline, bV as saveEvalRunComparison, bW as saveServerComparison, bX as validateEvalCase, bY as validateEvalDataset, bZ as validateJudge, b_ as validateMCPConfig, b$ as validateToolCallCount, c0 as validateToolCalls } from './index-BcUWzQCx.cjs';
+import { L as LLMProvider, M as MCPFixtureApi, a as MCPHostConfig, b as MCPHostSimulationResult, O as OAuthSetupConfig, V as ValidationResult, S as SchemaValidatorOptions, T as TextValidatorOptions, P as PatternValidatorOptions, c as SizeValidatorOptions, d as SnapshotSanitizer, R as RubricSpec, J as JudgeMatcherOptions, e as ToolPredicate, f as ToolCallExpectation, g as ToolCallCountOptions, A as AuthType, h as JudgeConfig, i as Judge } from './index-BlqLvjnP.cjs';
+export { B as BUILT_IN_RUBRICS, j as BuiltInRubric, k as BuiltInSanitizer, C as CLIConfig, l as CLIOAuthClient, m as CLIOAuthClientConfig, n as CLIOAuthResult, o as CLIOutputFormat, p as CaseComparisonResult, q as ClientCredentialsConfig, r as CompareEvalRunsOptions, s as ComparisonOutcome, t as ContentBlock, u as CreateMCPClientOptions, v as CustomJudgeExecutor, w as CustomJudgeResult, D as DiscoveryError, E as ENV_VAR_NAMES, x as EvalCase, y as EvalCaseComparison, z as EvalCaseComparisonOutcome, F as EvalCaseRequest, G as EvalCaseResult, H as EvalCaseSchema, I as EvalContext, K as EvalDataset, N as EvalDatasetSchema, Q as EvalExpectBlock, U as EvalExpectationResult, W as EvalMode, X as EvalResultStore, Y as EvalResultStoreConfig, Z as EvalResultStoreLike, _ as EvalRunComparisonLabels, $ as EvalRunComparisonResult, a0 as EvalRunMetadata, a1 as EvalRunnerOptions, a2 as EvalRunnerResult, a3 as ExpectationBreakdown, a4 as ExpectationResultMap, a5 as ExpectationType, a6 as ExperimentMetric, a7 as FieldRemovalSanitizer, a8 as FileEvalResultStore, a9 as FileEvalResultStoreConfig, aa as GCSEvalResultStore, ab as GCSEvalResultStoreConfig, ac as HostType, ad as HttpMCPConfig, ae as IterationResult, af as JudgeExpectConfig, ag as JudgeResult, ah as JudgeValidatorConfig, ai as LLMToolCall, aj as ListStoredArtifactsOptions, ak as LoadDatasetOptions, al as MCPAuthConfig, am as MCPAuthFixtures, an as MCPClientCredentialsConfig, ao as MCPConfig, ap as MCPConfigSchema, aq as MCPConformanceCheck, ar as MCPConformanceOptions, as as MCPConformanceRaw, at as MCPConformanceResult, au as MCPConformanceResultData, av as MCPEvalData, aw as MCPEvalHistoricalSummary, ax as MCPEvalReporterConfig, ay as MCPEvalRunData, az as MCPFixtureOptions, aA as MCPHostCapabilities, aB as MCPHostSimulator, aC as MCPOAuthConfig, aD as MCPServerCapabilitiesData, aE as MCP_PROTOCOL_VERSION, aF as NormalizedToolResponse, aG as PlaywrightOAuthClientProvider, aH as PlaywrightOAuthClientProviderConfig, aI as PredicateResult, aJ as ProposeVariantsContext, aK as ProtectedResourceDiscoveryResult, aL as ProtectedResourceMetadata, aM as ProviderKind, aN as RegexSanitizer, aO as ResultSource, aP as SaveBaselineOptions, aQ as SaveEvalRunComparisonOptions, aR as SaveServerComparisonOptions, aS as SchemaRegistry, aT as SerializedEvalDataset, aU as ServerComparisonOptions, aV as ServerComparisonResult, aW as SnapshotSanitizers, aX as StdioMCPConfig, aY as StoredArtifactKind, aZ as StoredArtifactSummary, a_ as StoredClientInfo, a$ as StoredEvalArtifact, b0 as StoredEvalArtifactMetadata, b1 as StoredEvalResultLoadOptions, b2 as StoredEvalResultRef, b3 as StoredEvalResultSaveOptions, b4 as StoredEvalRunRef, b5 as StoredOAuthState, b6 as StoredServerMetadata, b7 as StoredTokens, b8 as TokenResult, b9 as ToolMetadataOverride, ba as ToolOverrideVariant, bb as UsageMetrics, bc as VariantCandidateResult, bd as VariantExperimentOptions, be as VariantExperimentReason, bf as VariantExperimentResult, bg as VariantExperimentRound, bh as VariantImprovementProposal, bi as VariantRecommendation, bj as clearJudgeRegistry, bk as closeMCPClient, bl as compareEvalRuns, bm as createDefaultArtifactId, bn as createEvalResultStore, bo as createMCPClientForConfig, bp as createMCPFixture, bq as createStoredEvalArtifact, br as defaultEnvironmentMetadata, bs as discoverAuthorizationServer, bt as discoverProtectedResource, bu as extractText, bv as getRegisteredJudge, bw as hasValidTokens, bx as injectTokens, by as isBuiltInRubric, bz as isEvalResultStore, bA as isHttpConfig, bB as isStdioConfig, bC as loadBaseline, bD as loadEvalDataset, bE as loadEvalDatasetFromObject, bF as loadStoredEvalRunnerResult, bG as loadTokens, bH as loadTokensFromEnv, bI as mcpAuthTest, bJ as normalizeToolResponse, bK as performClientCredentialsFlow, bL as refreshAccessToken, bM as registerJudge, bN as resolveEvalResultStore, bO as resolveRubric, bP as runConformanceChecks, bQ as runEvalCase, bR as runEvalDataset, bS as runServerComparison, bT as runVariantExperiment, bU as saveBaseline, bV as saveEvalRunComparison, bW as saveServerComparison, bX as validateEvalCase, bY as validateEvalDataset, bZ as validateJudge, b_ as validateMCPConfig, b$ as validateToolCallCount, c0 as validateToolCalls } from './index-BlqLvjnP.cjs';
 import { ZodType } from 'zod';
 import * as playwright_test from 'playwright/test';
 import { Client } from '@modelcontextprotocol/sdk/client/index.js';

package/dist/index.d.ts

@@ -1,5 +1,5 @@
-import { L as LLMProvider, M as MCPFixtureApi, a as MCPHostConfig, b as MCPHostSimulationResult, O as OAuthSetupConfig, V as ValidationResult, S as SchemaValidatorOptions, T as TextValidatorOptions, P as PatternValidatorOptions, c as SizeValidatorOptions, d as SnapshotSanitizer, R as RubricSpec, J as JudgeMatcherOptions, e as ToolPredicate, f as ToolCallExpectation, g as ToolCallCountOptions, A as AuthType, h as JudgeConfig, i as Judge } from './index-BcUWzQCx.js';
-export { B as BUILT_IN_RUBRICS, j as BuiltInRubric, k as BuiltInSanitizer, C as CLIConfig, l as CLIOAuthClient, m as CLIOAuthClientConfig, n as CLIOAuthResult, o as CLIOutputFormat, p as CaseComparisonResult, q as ClientCredentialsConfig, r as CompareEvalRunsOptions, s as ComparisonOutcome, t as ContentBlock, u as CreateMCPClientOptions, v as CustomJudgeExecutor, w as CustomJudgeResult, D as DiscoveryError, E as ENV_VAR_NAMES, x as EvalCase, y as EvalCaseComparison, z as EvalCaseComparisonOutcome, F as EvalCaseRequest, G as EvalCaseResult, H as EvalCaseSchema, I as EvalContext, K as EvalDataset, N as EvalDatasetSchema, Q as EvalExpectBlock, U as EvalExpectationResult, W as EvalMode, X as EvalResultStore, Y as EvalResultStoreConfig, Z as EvalResultStoreLike, _ as EvalRunComparisonLabels, $ as EvalRunComparisonResult, a0 as EvalRunMetadata, a1 as EvalRunnerOptions, a2 as EvalRunnerResult, a3 as ExpectationBreakdown, a4 as ExpectationResultMap, a5 as ExpectationType, a6 as ExperimentMetric, a7 as FieldRemovalSanitizer, a8 as FileEvalResultStore, a9 as FileEvalResultStoreConfig, aa as GCSEvalResultStore, ab as GCSEvalResultStoreConfig, ac as HostType, ad as HttpMCPConfig, ae as IterationResult, af as JudgeExpectConfig, ag as JudgeResult, ah as JudgeValidatorConfig, ai as LLMToolCall, aj as ListStoredArtifactsOptions, ak as LoadDatasetOptions, al as MCPAuthConfig, am as MCPAuthFixtures, an as MCPClientCredentialsConfig, ao as MCPConfig, ap as MCPConfigSchema, aq as MCPConformanceCheck, ar as MCPConformanceOptions, as as MCPConformanceRaw, at as MCPConformanceResult, au as MCPConformanceResultData, av as MCPEvalData, aw as MCPEvalHistoricalSummary, ax as MCPEvalReporterConfig, ay as MCPEvalRunData, az as MCPFixtureOptions, aA as MCPHostCapabilities, aB as MCPHostSimulator, aC as MCPOAuthConfig, aD as MCPServerCapabilitiesData, aE as MCP_PROTOCOL_VERSION, aF as NormalizedToolResponse, aG as PlaywrightOAuthClientProvider, aH as PlaywrightOAuthClientProviderConfig, aI as PredicateResult, aJ as ProposeVariantsContext, aK as ProtectedResourceDiscoveryResult, aL as ProtectedResourceMetadata, aM as ProviderKind, aN as RegexSanitizer, aO as ResultSource, aP as SaveBaselineOptions, aQ as SaveEvalRunComparisonOptions, aR as SaveServerComparisonOptions, aS as SchemaRegistry, aT as SerializedEvalDataset, aU as ServerComparisonOptions, aV as ServerComparisonResult, aW as SnapshotSanitizers, aX as StdioMCPConfig, aY as StoredArtifactKind, aZ as StoredArtifactSummary, a_ as StoredClientInfo, a$ as StoredEvalArtifact, b0 as StoredEvalArtifactMetadata, b1 as StoredEvalResultLoadOptions, b2 as StoredEvalResultRef, b3 as StoredEvalResultSaveOptions, b4 as StoredEvalRunRef, b5 as StoredOAuthState, b6 as StoredServerMetadata, b7 as StoredTokens, b8 as TokenResult, b9 as ToolMetadataOverride, ba as ToolOverrideVariant, bb as UsageMetrics, bc as VariantCandidateResult, bd as VariantExperimentOptions, be as VariantExperimentReason, bf as VariantExperimentResult, bg as VariantExperimentRound, bh as VariantImprovementProposal, bi as VariantRecommendation, bj as clearJudgeRegistry, bk as closeMCPClient, bl as compareEvalRuns, bm as createDefaultArtifactId, bn as createEvalResultStore, bo as createMCPClientForConfig, bp as createMCPFixture, bq as createStoredEvalArtifact, br as defaultEnvironmentMetadata, bs as discoverAuthorizationServer, bt as discoverProtectedResource, bu as extractText, bv as getRegisteredJudge, bw as hasValidTokens, bx as injectTokens, by as isBuiltInRubric, bz as isEvalResultStore, bA as isHttpConfig, bB as isStdioConfig, bC as loadBaseline, bD as loadEvalDataset, bE as loadEvalDatasetFromObject, bF as loadStoredEvalRunnerResult, bG as loadTokens, bH as loadTokensFromEnv, bI as mcpAuthTest, bJ as normalizeToolResponse, bK as performClientCredentialsFlow, bL as refreshAccessToken, bM as registerJudge, bN as resolveEvalResultStore, bO as resolveRubric, bP as runConformanceChecks, bQ as runEvalCase, bR as runEvalDataset, bS as runServerComparison, bT as runVariantExperiment, bU as saveBaseline, bV as saveEvalRunComparison, bW as saveServerComparison, bX as validateEvalCase, bY as validateEvalDataset, bZ as validateJudge, b_ as validateMCPConfig, b$ as validateToolCallCount, c0 as validateToolCalls } from './index-BcUWzQCx.js';
+import { L as LLMProvider, M as MCPFixtureApi, a as MCPHostConfig, b as MCPHostSimulationResult, O as OAuthSetupConfig, V as ValidationResult, S as SchemaValidatorOptions, T as TextValidatorOptions, P as PatternValidatorOptions, c as SizeValidatorOptions, d as SnapshotSanitizer, R as RubricSpec, J as JudgeMatcherOptions, e as ToolPredicate, f as ToolCallExpectation, g as ToolCallCountOptions, A as AuthType, h as JudgeConfig, i as Judge } from './index-BlqLvjnP.js';
+export { B as BUILT_IN_RUBRICS, j as BuiltInRubric, k as BuiltInSanitizer, C as CLIConfig, l as CLIOAuthClient, m as CLIOAuthClientConfig, n as CLIOAuthResult, o as CLIOutputFormat, p as CaseComparisonResult, q as ClientCredentialsConfig, r as CompareEvalRunsOptions, s as ComparisonOutcome, t as ContentBlock, u as CreateMCPClientOptions, v as CustomJudgeExecutor, w as CustomJudgeResult, D as DiscoveryError, E as ENV_VAR_NAMES, x as EvalCase, y as EvalCaseComparison, z as EvalCaseComparisonOutcome, F as EvalCaseRequest, G as EvalCaseResult, H as EvalCaseSchema, I as EvalContext, K as EvalDataset, N as EvalDatasetSchema, Q as EvalExpectBlock, U as EvalExpectationResult, W as EvalMode, X as EvalResultStore, Y as EvalResultStoreConfig, Z as EvalResultStoreLike, _ as EvalRunComparisonLabels, $ as EvalRunComparisonResult, a0 as EvalRunMetadata, a1 as EvalRunnerOptions, a2 as EvalRunnerResult, a3 as ExpectationBreakdown, a4 as ExpectationResultMap, a5 as ExpectationType, a6 as ExperimentMetric, a7 as FieldRemovalSanitizer, a8 as FileEvalResultStore, a9 as FileEvalResultStoreConfig, aa as GCSEvalResultStore, ab as GCSEvalResultStoreConfig, ac as HostType, ad as HttpMCPConfig, ae as IterationResult, af as JudgeExpectConfig, ag as JudgeResult, ah as JudgeValidatorConfig, ai as LLMToolCall, aj as ListStoredArtifactsOptions, ak as LoadDatasetOptions, al as MCPAuthConfig, am as MCPAuthFixtures, an as MCPClientCredentialsConfig, ao as MCPConfig, ap as MCPConfigSchema, aq as MCPConformanceCheck, ar as MCPConformanceOptions, as as MCPConformanceRaw, at as MCPConformanceResult, au as MCPConformanceResultData, av as MCPEvalData, aw as MCPEvalHistoricalSummary, ax as MCPEvalReporterConfig, ay as MCPEvalRunData, az as MCPFixtureOptions, aA as MCPHostCapabilities, aB as MCPHostSimulator, aC as MCPOAuthConfig, aD as MCPServerCapabilitiesData, aE as MCP_PROTOCOL_VERSION, aF as NormalizedToolResponse, aG as PlaywrightOAuthClientProvider, aH as PlaywrightOAuthClientProviderConfig, aI as PredicateResult, aJ as ProposeVariantsContext, aK as ProtectedResourceDiscoveryResult, aL as ProtectedResourceMetadata, aM as ProviderKind, aN as RegexSanitizer, aO as ResultSource, aP as SaveBaselineOptions, aQ as SaveEvalRunComparisonOptions, aR as SaveServerComparisonOptions, aS as SchemaRegistry, aT as SerializedEvalDataset, aU as ServerComparisonOptions, aV as ServerComparisonResult, aW as SnapshotSanitizers, aX as StdioMCPConfig, aY as StoredArtifactKind, aZ as StoredArtifactSummary, a_ as StoredClientInfo, a$ as StoredEvalArtifact, b0 as StoredEvalArtifactMetadata, b1 as StoredEvalResultLoadOptions, b2 as StoredEvalResultRef, b3 as StoredEvalResultSaveOptions, b4 as StoredEvalRunRef, b5 as StoredOAuthState, b6 as StoredServerMetadata, b7 as StoredTokens, b8 as TokenResult, b9 as ToolMetadataOverride, ba as ToolOverrideVariant, bb as UsageMetrics, bc as VariantCandidateResult, bd as VariantExperimentOptions, be as VariantExperimentReason, bf as VariantExperimentResult, bg as VariantExperimentRound, bh as VariantImprovementProposal, bi as VariantRecommendation, bj as clearJudgeRegistry, bk as closeMCPClient, bl as compareEvalRuns, bm as createDefaultArtifactId, bn as createEvalResultStore, bo as createMCPClientForConfig, bp as createMCPFixture, bq as createStoredEvalArtifact, br as defaultEnvironmentMetadata, bs as discoverAuthorizationServer, bt as discoverProtectedResource, bu as extractText, bv as getRegisteredJudge, bw as hasValidTokens, bx as injectTokens, by as isBuiltInRubric, bz as isEvalResultStore, bA as isHttpConfig, bB as isStdioConfig, bC as loadBaseline, bD as loadEvalDataset, bE as loadEvalDatasetFromObject, bF as loadStoredEvalRunnerResult, bG as loadTokens, bH as loadTokensFromEnv, bI as mcpAuthTest, bJ as normalizeToolResponse, bK as performClientCredentialsFlow, bL as refreshAccessToken, bM as registerJudge, bN as resolveEvalResultStore, bO as resolveRubric, bP as runConformanceChecks, bQ as runEvalCase, bR as runEvalDataset, bS as runServerComparison, bT as runVariantExperiment, bU as saveBaseline, bV as saveEvalRunComparison, bW as saveServerComparison, bX as validateEvalCase, bY as validateEvalDataset, bZ as validateJudge, b_ as validateMCPConfig, b$ as validateToolCallCount, c0 as validateToolCalls } from './index-BlqLvjnP.js';
 import { ZodType } from 'zod';
 import * as playwright_test from 'playwright/test';
 import { Client } from '@modelcontextprotocol/sdk/client/index.js';

package/dist/index.js

@@ -4398,7 +4398,7 @@ function escapeHtml(text) {
 
 // package.json
 var package_default = {
-  version: "1.1.0"};
+  version: "1.1.1"};
 
 // src/mcp/clientFactory.ts
 function getRetryAfterDelayMs(err) {
@@ -6811,8 +6811,8 @@ Hint: run \`getMissingDependencyMessage('${provider}')\` or check docs/mcp-host.
 Hint: check your API key environment variable (e.g. ANTHROPIC_API_KEY, GOOGLE_APPLICATION_CREDENTIALS).`;
   }
   if (raw.includes("404") || raw.includes("Not Found") || raw.toLowerCase().includes("model") && raw.toLowerCase().includes("not found")) {
-    return `MCP host simulation failed: model not found.
-Hint: check the model name format for your provider. For vertex-anthropic use 'claude-3-5-haiku@20241022' (with @).`;
+    return `MCP host simulation failed: ${raw}
+Hint: a 404 usually means the model id is wrong or retired, or a base-URL override (e.g. ANTHROPIC_BASE_URL / OPENAI_BASE_URL pointing at a gateway) is routing requests somewhere that doesn't serve this model. Verify the model id and that no unexpected *_BASE_URL is set.`;
   }
   if (raw.includes("ENOTFOUND") || raw.includes("fetch failed") || raw.includes("ECONNREFUSED")) {
     return `MCP host simulation failed: network error.
@@ -6910,12 +6910,18 @@ function createVercelOrchestrator() {
           tools[toolName] = {
             description: mcpTool.description ?? "",
             inputSchema: jsonSchema2(rawSchema),
-            execute: async (args) => {
+            execute: async (args, opts) => {
               const mcpStart = Date.now();
               const result2 = await mcp.callTool(toolName, args);
               mcpDurationMs += Date.now() - mcpStart;
-              allToolCalls.push({ name: toolName, arguments: args });
-              return extractText(result2);
+              const output = extractText(result2);
+              allToolCalls.push({
+                id: opts?.toolCallId,
+                name: toolName,
+                arguments: args,
+                output
+              });
+              return output;
             }
           };
         }
@@ -6937,10 +6943,17 @@ function createVercelOrchestrator() {
           totalCostUsd: 0,
           durationMs: llmDurationMs
         } : void 0;
-        const conversationHistory = (result.steps ?? []).map((step) => ({
-          role: step.toolCalls?.length > 0 ? "tool" : "assistant",
-          content: step.toolCalls?.length > 0 ? JSON.stringify(step.toolResults) : step.text ?? ""
-        }));
+        const conversationHistory = (result.steps ?? []).flatMap((step) => {
+          if (step.toolCalls?.length > 0) {
+            return step.toolCalls.map(
+              (tc) => ({
+                role: "tool",
+                toolCallId: tc.toolCallId
+              })
+            );
+          }
+          return step.text ? [{ role: "assistant", content: step.text }] : [];
+        });
         return {
           success: true,
           toolCalls: allToolCalls,
@@ -6996,7 +7009,13 @@ function parseStreamJson(stdout) {
       for (const block of event.message.content) {
         if (block.type === "tool_result") {
           const content = typeof block.content === "string" ? block.content : JSON.stringify(block.content);
-          conversationHistory.push({ role: "tool", content });
+          const call = block.tool_use_id ? toolCalls.find((tc) => tc.id === block.tool_use_id) : void 0;
+          if (call) {
+            call.output = content;
+            conversationHistory.push({ role: "tool", toolCallId: call.id });
+          } else {
+            conversationHistory.push({ role: "tool", content });
+          }
         }
       }
     }
@@ -7952,6 +7971,7 @@ async function getGitHash() {
   const result = await execFileNoThrow("git", ["rev-parse", "HEAD"]);
   return result.status === 0 ? result.stdout.trim() : void 0;
 }
+var warnedNoTestInfo = false;
 async function runEvalDataset(options, context) {
   const {
     dataset,
@@ -8122,7 +8142,8 @@ async function runEvalDataset(options, context) {
       contentType: "application/json",
       body: Buffer.from(JSON.stringify({ caseResults }))
     });
-  } else if (caseResults.length > 0) {
+  } else if (caseResults.length > 0 && !warnedNoTestInfo) {
+    warnedNoTestInfo = true;
     console.warn(
       "[mcp-server-tester] runEvalDataset: testInfo not provided \u2014 results will not appear in the MCP reporter.\nTo enable reporting, pass testInfo from the Playwright test function:\n  await runEvalDataset({ dataset }, { mcp, testInfo });"
     );
@@ -8402,9 +8423,13 @@ async function runVariantExperiment(options, context) {
   const maxRounds = options.maxRounds ?? 1;
   const minImprovement = options.minImprovement ?? 0;
   const allowRegressions = options.allowRegressions ?? false;
+  const internalContext = {
+    mcp: context.mcp,
+    expect: context.expect
+  };
   const baseline = await runEvalDataset(
     buildRunOptions(options, void 0),
-    context
+    internalContext
   );
   const baselineValue = readMetric(baseline, metric);
   if (baselineValue === void 0) {
@@ -8432,7 +8457,7 @@ async function runVariantExperiment(options, context) {
     for (const variant of variants) {
       const candidate = await scoreVariant(
         options,
-        context,
+        internalContext,
         baseline,
         baselineValue,
         metric,
@@ -8459,7 +8484,7 @@ async function runVariantExperiment(options, context) {
   const winner = bestSoFar;
   const proposalSource = winner ?? bestAttempted;
   const proposal = proposalSource ? buildProposal(metric, baselineValue, proposalSource, winner !== void 0) : void 0;
-  return {
+  const result = {
     metric,
     baseline,
     rounds,
@@ -8468,6 +8493,42 @@ async function runVariantExperiment(options, context) {
     converged: true,
     reason
   };
+  if (context.testInfo) {
+    const surfaceRun = winner?.result ?? bestAttempted?.result ?? baseline;
+    await context.testInfo.attach("mcp-test-results", {
+      contentType: "application/json",
+      body: Buffer.from(
+        JSON.stringify({ caseResults: surfaceRun.caseResults })
+      )
+    });
+    await context.testInfo.attach("mcp-variant-experiment", {
+      contentType: "application/json",
+      body: Buffer.from(
+        JSON.stringify(buildExperimentData(result, baselineValue))
+      )
+    });
+  }
+  return result;
+}
+function buildExperimentData(result, baselineValue) {
+  return {
+    metric: result.metric,
+    baselineValue,
+    bestValue: result.winner?.metricValue ?? result.proposal?.candidateValue ?? baselineValue,
+    rounds: result.rounds.map((round) => {
+      const best = round.best ?? round.candidates[0];
+      return {
+        round: round.round,
+        variantId: best?.variant.id ?? "(none)",
+        metricValue: best?.metricValue ?? baselineValue,
+        metricDelta: best?.metricDelta ?? 0,
+        disqualified: best?.disqualified ?? false
+      };
+    }),
+    winnerVariantId: result.winner?.variant.id,
+    recommendation: result.proposal?.recommendation,
+    reason: result.reason
+  };
 }
 async function gatherVariants(options, context) {
   if (context.round === 0 && options.variants && options.variants.length > 0) {