@gleanwork/mcp-server-tester 1.0.1 → 1.1.1

package/dist/index.js

@@ -1,7 +1,7 @@
 import * as fs from 'fs/promises';
-import { readFile, mkdir, writeFile } from 'fs/promises';
+import { readFile, mkdir, writeFile, readdir } from 'fs/promises';
 import * as path2 from 'path';
-import { dirname } from 'path';
+import { dirname, join } from 'path';
 import * as z4 from 'zod/v4';
 import { ZodFirstPartyTypeKind } from 'zod/v3';
 import { z } from 'zod';
@@ -3099,6 +3099,20 @@ var init_dist3 = __esm({
     };
   }
 });
+
+// src/assertions/validators/types.ts
+var SnapshotSanitizers = {
+  /** Replaces Unix timestamps (seconds and milliseconds) with a stable placeholder */
+  TIMESTAMP: "timestamp",
+  /** Replaces UUID v1-v5 strings with a stable placeholder */
+  UUID: "uuid",
+  /** Replaces ISO 8601 date/datetime strings with a stable placeholder */
+  ISO_DATE: "iso-date",
+  /** Replaces MongoDB ObjectId strings with a stable placeholder */
+  OBJECT_ID: "objectId",
+  /** Replaces JWT tokens with a stable placeholder */
+  JWT: "jwt"
+};
 var MCPHostCapabilitiesSchema = z.object({
   sampling: z.record(z.string(), z.unknown()).optional(),
   roots: z.object({
@@ -4384,7 +4398,7 @@ function escapeHtml(text) {
 
 // package.json
 var package_default = {
-  version: "1.0.1"};
+  version: "1.1.1"};
 
 // src/mcp/clientFactory.ts
 function getRetryAfterDelayMs(err) {
@@ -5908,20 +5922,6 @@ async function validateJudge(response, config) {
   }
 }
 
-// src/assertions/validators/types.ts
-var SnapshotSanitizers = {
-  /** Replaces Unix timestamps (seconds and milliseconds) with a stable placeholder */
-  TIMESTAMP: "timestamp",
-  /** Replaces UUID v1-v5 strings with a stable placeholder */
-  UUID: "uuid",
-  /** Replaces ISO 8601 date/datetime strings with a stable placeholder */
-  ISO_DATE: "iso-date",
-  /** Replaces MongoDB ObjectId strings with a stable placeholder */
-  OBJECT_ID: "objectId",
-  /** Replaces JWT tokens with a stable placeholder */
-  JWT: "jwt"
-};
-
 // src/mcp/fixtures/mcpFixture.ts
 var DEFAULT_CALL_TIMEOUT_MS = 3e4;
 function withCallTimeout(promise, ms, opName) {
@@ -6811,8 +6811,8 @@ Hint: run \`getMissingDependencyMessage('${provider}')\` or check docs/mcp-host.
 Hint: check your API key environment variable (e.g. ANTHROPIC_API_KEY, GOOGLE_APPLICATION_CREDENTIALS).`;
   }
   if (raw.includes("404") || raw.includes("Not Found") || raw.toLowerCase().includes("model") && raw.toLowerCase().includes("not found")) {
-    return `MCP host simulation failed: model not found.
-Hint: check the model name format for your provider. For vertex-anthropic use 'claude-3-5-haiku@20241022' (with @).`;
+    return `MCP host simulation failed: ${raw}
+Hint: a 404 usually means the model id is wrong or retired, or a base-URL override (e.g. ANTHROPIC_BASE_URL / OPENAI_BASE_URL pointing at a gateway) is routing requests somewhere that doesn't serve this model. Verify the model id and that no unexpected *_BASE_URL is set.`;
   }
   if (raw.includes("ENOTFOUND") || raw.includes("fetch failed") || raw.includes("ECONNREFUSED")) {
     return `MCP host simulation failed: network error.
@@ -6910,12 +6910,18 @@ function createVercelOrchestrator() {
           tools[toolName] = {
             description: mcpTool.description ?? "",
             inputSchema: jsonSchema2(rawSchema),
-            execute: async (args) => {
+            execute: async (args, opts) => {
               const mcpStart = Date.now();
               const result2 = await mcp.callTool(toolName, args);
               mcpDurationMs += Date.now() - mcpStart;
-              allToolCalls.push({ name: toolName, arguments: args });
-              return extractText(result2);
+              const output = extractText(result2);
+              allToolCalls.push({
+                id: opts?.toolCallId,
+                name: toolName,
+                arguments: args,
+                output
+              });
+              return output;
             }
           };
         }
@@ -6937,10 +6943,17 @@ function createVercelOrchestrator() {
           totalCostUsd: 0,
           durationMs: llmDurationMs
         } : void 0;
-        const conversationHistory = (result.steps ?? []).map((step) => ({
-          role: step.toolCalls?.length > 0 ? "tool" : "assistant",
-          content: step.toolCalls?.length > 0 ? JSON.stringify(step.toolResults) : step.text ?? ""
-        }));
+        const conversationHistory = (result.steps ?? []).flatMap((step) => {
+          if (step.toolCalls?.length > 0) {
+            return step.toolCalls.map(
+              (tc) => ({
+                role: "tool",
+                toolCallId: tc.toolCallId
+              })
+            );
+          }
+          return step.text ? [{ role: "assistant", content: step.text }] : [];
+        });
         return {
           success: true,
           toolCalls: allToolCalls,
@@ -6967,6 +6980,7 @@ function parseStreamJson(stdout) {
   const lines = stdout.split("\n").filter((line) => line.trim().length > 0);
   const toolCalls = [];
   const textParts = [];
+  let usage;
   const conversationHistory = [];
   for (const line of lines) {
     let event;
@@ -6995,20 +7009,38 @@ function parseStreamJson(stdout) {
       for (const block of event.message.content) {
         if (block.type === "tool_result") {
           const content = typeof block.content === "string" ? block.content : JSON.stringify(block.content);
-          conversationHistory.push({ role: "tool", content });
+          const call = block.tool_use_id ? toolCalls.find((tc) => tc.id === block.tool_use_id) : void 0;
+          if (call) {
+            call.output = content;
+            conversationHistory.push({ role: "tool", toolCallId: call.id });
+          } else {
+            conversationHistory.push({ role: "tool", content });
+          }
         }
       }
     }
-    if (event.type === "result" && typeof event.result === "string") {
-      if (textParts.length === 0) {
+    if (event.type === "result") {
+      if (typeof event.result === "string" && textParts.length === 0) {
         textParts.push(event.result);
       }
+      if (event.usage) {
+        usage = {
+          inputTokens: event.usage.input_tokens ?? 0,
+          outputTokens: event.usage.output_tokens ?? 0,
+          totalCostUsd: event.total_cost_usd ?? 0,
+          durationMs: event.duration_ms ?? 0,
+          durationApiMs: event.duration_api_ms,
+          cacheReadInputTokens: event.usage.cache_read_input_tokens,
+          cacheCreationInputTokens: event.usage.cache_creation_input_tokens
+        };
+      }
     }
     if (event.type === "result" && event.is_error === true) {
       return {
         success: false,
         toolCalls,
-        error: typeof event.result === "string" ? event.result : "CLI host reported an error"
+        error: typeof event.result === "string" ? event.result : "CLI host reported an error",
+        usage
       };
     }
   }
@@ -7020,7 +7052,8 @@ function parseStreamJson(stdout) {
     success: true,
     toolCalls,
     response: response || void 0,
-    conversationHistory: conversationHistory.length > 0 ? conversationHistory : void 0
+    conversationHistory: conversationHistory.length > 0 ? conversationHistory : void 0,
+    usage
   };
 }
 function createJsonParser(paths) {
@@ -7266,6 +7299,216 @@ function buildBaselinePassMap(baseline) {
   }
   return map;
 }
+var KIND_DIRS = {
+  "eval-runner-result": "eval-runs",
+  "reporter-run": "reporter-runs",
+  "eval-run-comparison": "comparisons/eval-runs",
+  "server-comparison": "comparisons/servers"
+};
+function createEvalResultStore(config) {
+  if (config.provider === "file") {
+    return new FileEvalResultStore(config);
+  }
+  return new GCSEvalResultStore(config);
+}
+function resolveEvalResultStore(store) {
+  return isEvalResultStore(store) ? store : createEvalResultStore(store);
+}
+function isEvalResultStore(value) {
+  return typeof value === "object" && value !== null && "saveArtifact" in value && "loadArtifact" in value && "loadLatestArtifact" in value && "listArtifacts" in value;
+}
+function createStoredEvalArtifact(options) {
+  const createdAt = options.createdAt ?? (/* @__PURE__ */ new Date()).toISOString();
+  return {
+    schemaVersion: 1,
+    kind: options.kind,
+    id: options.id ?? createDefaultArtifactId(createdAt),
+    createdAt,
+    metadata: {
+      ...defaultEnvironmentMetadata(),
+      ...options.metadata ?? {}
+    },
+    data: options.data
+  };
+}
+function createDefaultArtifactId(timestamp = (/* @__PURE__ */ new Date()).toISOString()) {
+  const safeTimestamp = timestamp.replace(/[:.]/g, "-");
+  const runNumber = process.env.GITHUB_RUN_NUMBER;
+  const sha = process.env.GITHUB_SHA?.slice(0, 12);
+  const suffix = runNumber ?? sha;
+  return suffix ? `${safeTimestamp}-${suffix}` : safeTimestamp;
+}
+function defaultEnvironmentMetadata() {
+  return {
+    ...process.env.GITHUB_SHA !== void 0 && {
+      gitHash: process.env.GITHUB_SHA
+    },
+    ...process.env.GITHUB_REF_NAME !== void 0 && {
+      branch: process.env.GITHUB_REF_NAME
+    },
+    ...process.env.GITHUB_RUN_NUMBER !== void 0 && {
+      runNumber: process.env.GITHUB_RUN_NUMBER
+    },
+    ...process.env.GITHUB_EVENT_NAME !== void 0 && {
+      trigger: process.env.GITHUB_EVENT_NAME
+    }
+  };
+}
+var FileEvalResultStore = class {
+  dir;
+  constructor(config) {
+    this.dir = config.dir;
+  }
+  async saveArtifact(artifact) {
+    const artifactDir = join(this.dir, KIND_DIRS[artifact.kind]);
+    await mkdir(artifactDir, { recursive: true });
+    const serialized = JSON.stringify(artifact, null, 2);
+    await writeFile(
+      join(artifactDir, `${artifact.id}.json`),
+      serialized,
+      "utf8"
+    );
+    await writeFile(join(artifactDir, "latest.json"), serialized, "utf8");
+  }
+  async loadArtifact(kind, id) {
+    const raw = await readFile(
+      join(this.dir, KIND_DIRS[kind], `${id}.json`),
+      "utf8"
+    );
+    return JSON.parse(raw);
+  }
+  async loadLatestArtifact(kind) {
+    try {
+      const raw = await readFile(
+        join(this.dir, KIND_DIRS[kind], "latest.json"),
+        "utf8"
+      );
+      return JSON.parse(raw);
+    } catch (error) {
+      if (isMissingFileError(error)) {
+        return null;
+      }
+      throw error;
+    }
+  }
+  async listArtifacts(kind, options = {}) {
+    let files;
+    try {
+      files = await readdir(join(this.dir, KIND_DIRS[kind]));
+    } catch (error) {
+      if (isMissingFileError(error)) {
+        return [];
+      }
+      throw error;
+    }
+    const summaries = await Promise.all(
+      files.filter((f) => f.endsWith(".json") && f !== "latest.json").map(async (file) => {
+        const raw = await readFile(
+          join(this.dir, KIND_DIRS[kind], file),
+          "utf8"
+        );
+        return toSummary(JSON.parse(raw));
+      })
+    );
+    return summaries.sort((a, b) => b.createdAt.localeCompare(a.createdAt)).slice(0, options.limit);
+  }
+};
+var GCSEvalResultStore = class {
+  bucketName;
+  prefix;
+  storage;
+  constructor(config) {
+    this.bucketName = config.bucket;
+    this.prefix = trimSlashes(config.prefix ?? "");
+  }
+  async saveArtifact(artifact) {
+    const bucket = await this.getBucket();
+    const serialized = JSON.stringify(artifact, null, 2);
+    await bucket.file(
+      this.objectPath(
+        artifact.kind,
+        `${encodeURIComponent(artifact.id)}.json`
+      )
+    ).save(serialized, {
+      contentType: "application/json",
+      resumable: false,
+      validation: false
+    });
+    await bucket.file(this.objectPath(artifact.kind, "latest.json")).save(serialized, {
+      contentType: "application/json",
+      resumable: false,
+      validation: false
+    });
+  }
+  async loadArtifact(kind, id) {
+    const bucket = await this.getBucket();
+    const file = bucket.file(
+      this.objectPath(kind, `${encodeURIComponent(id)}.json`)
+    );
+    const [buffer] = await file.download();
+    return JSON.parse(buffer.toString("utf8"));
+  }
+  async loadLatestArtifact(kind) {
+    const bucket = await this.getBucket();
+    const file = bucket.file(this.objectPath(kind, "latest.json"));
+    const [exists] = await file.exists();
+    if (!exists) return null;
+    const [buffer] = await file.download();
+    return JSON.parse(buffer.toString("utf8"));
+  }
+  async listArtifacts(kind, options = {}) {
+    const bucket = await this.getBucket();
+    const [files] = await bucket.getFiles({
+      prefix: this.objectPath(kind, "")
+    });
+    const summaries = await Promise.all(
+      files.filter(
+        (f) => f.name.endsWith(".json") && !f.name.endsWith("/latest.json")
+      ).map(async (file) => {
+        const [buffer] = await file.download();
+        return toSummary(
+          JSON.parse(buffer.toString("utf8"))
+        );
+      })
+    );
+    return summaries.sort((a, b) => b.createdAt.localeCompare(a.createdAt)).slice(0, options.limit);
+  }
+  async getBucket() {
+    if (!this.storage) {
+      const moduleName = "@google-cloud/storage";
+      let Storage;
+      try {
+        const mod = await import(moduleName);
+        Storage = mod.Storage;
+      } catch (error) {
+        throw new Error(
+          `GCS result storage requires the optional \`@google-cloud/storage\` package. Install it and authenticate with Application Default Credentials via GOOGLE_APPLICATION_CREDENTIALS.
+Original error: ${error instanceof Error ? error.message : String(error)}`
+        );
+      }
+      this.storage = new Storage();
+    }
+    return this.storage.bucket(this.bucketName);
+  }
+  objectPath(kind, filename) {
+    const parts = [this.prefix, KIND_DIRS[kind], filename].filter(Boolean);
+    return parts.join("/");
+  }
+};
+function toSummary(artifact) {
+  return {
+    kind: artifact.kind,
+    id: artifact.id,
+    createdAt: artifact.createdAt,
+    metadata: artifact.metadata
+  };
+}
+function trimSlashes(value) {
+  return value.replace(/^\/+|\/+$/g, "");
+}
+function isMissingFileError(error) {
+  return typeof error === "object" && error !== null && "code" in error && error.code === "ENOENT";
+}
 var execFileAsync = promisify(execFile);
 async function execFileNoThrow(file, args) {
   try {
@@ -7312,6 +7555,41 @@ function sumUsage(a, b) {
 }
 
 // src/evals/evalRunner.ts
+function createToolOverrideMCP(mcp, variant) {
+  return {
+    ...mcp,
+    async listTools() {
+      const tools = await mcp.listTools();
+      const knownToolNames = new Set(tools.map((tool2) => tool2.name));
+      const unknownToolNames = Object.keys(variant.tools).filter(
+        (name15) => !knownToolNames.has(name15)
+      );
+      if (unknownToolNames.length > 0) {
+        throw new Error(
+          `[mcp-server-tester] toolOverrides variant "${variant.id}" references unknown tool(s): ` + unknownToolNames.join(", ")
+        );
+      }
+      return tools.map((tool2) => {
+        const override = variant.tools[tool2.name];
+        if (!override) {
+          return tool2;
+        }
+        return {
+          ...tool2,
+          ...override.description !== void 0 && {
+            description: override.description
+          },
+          ...override.inputSchema !== void 0 && {
+            inputSchema: override.inputSchema
+          }
+        };
+      });
+    },
+    async callTool(name15, args) {
+      return mcp.callTool(name15, args);
+    }
+  };
+}
 async function executeToolCall(evalCase, mcp) {
   const mode = evalCase.mode || "direct";
   try {
@@ -7496,9 +7774,12 @@ async function runExpectBlockValidations(expectBlock, response, config) {
   }
   return { expectations: results, toolPrecision, toolRecall };
 }
-function buildRequest(evalCase) {
+function buildRequest(evalCase, toolOverrideVariantId) {
   const request = {};
   if (evalCase.description) request.description = evalCase.description;
+  if (toolOverrideVariantId !== void 0) {
+    request.toolOverrideVariantId = toolOverrideVariantId;
+  }
   if (evalCase.mode === "mcp_host") {
     if (evalCase.scenario) request.scenario = evalCase.scenario;
     if (evalCase.mcpHostConfig) {
@@ -7563,7 +7844,7 @@ async function runSingleIteration(evalCase, context, options) {
     toolName: evalCase.scenario != null ? "mcp_host" : evalCase.toolName ?? "unknown",
     source: "eval",
     pass: didCasePass(error, expectationResults),
-    request: buildRequest(evalCase),
+    request: buildRequest(evalCase, options.toolOverrideVariantId),
     response,
     error,
     expectations: expectationResults,
@@ -7642,7 +7923,8 @@ async function runEvalCase(evalCase, context, options = {}) {
     authType: context.mcp.authType,
     project: context.mcp.project,
     durationMs: 0,
-    tags: evalCase.tags
+    tags: evalCase.tags,
+    request: buildRequest(evalCase, options.toolOverrideVariantId)
   };
   const totalHostUsage = iterationResults.reduce(
     (acc, r) => sumUsage(acc, r.hostUsage),
@@ -7689,6 +7971,7 @@ async function getGitHash() {
   const result = await execFileNoThrow("git", ["rev-parse", "HEAD"]);
   return result.status === 0 ? result.stdout.trim() : void 0;
 }
+var warnedNoTestInfo = false;
 async function runEvalDataset(options, context) {
   const {
     dataset,
@@ -7701,11 +7984,15 @@ async function runEvalDataset(options, context) {
     filterTags,
     saveResultsTo,
     omitResponsesFromBaseline = true,
+    redactStoredResponses,
+    resultStore,
     baselineResultsFrom,
+    toolOverrides,
     mcpHostModel,
     judgeModel
   } = options;
   const startTime = Date.now();
+  const effectiveContext = toolOverrides ? { ...context, mcp: createToolOverrideMCP(context.mcp, toolOverrides) } : context;
   const allSchemas = {
     ...dataset.schemas,
     ...schemas
@@ -7737,9 +8024,10 @@ async function runEvalDataset(options, context) {
       }
     }
     const effectiveCase = withIterations.judgeReps === void 0 && defaultJudgeReps !== void 0 ? { ...withIterations, judgeReps: defaultJudgeReps } : withIterations;
-    const result2 = await runEvalCase(effectiveCase, context, {
+    const result2 = await runEvalCase(effectiveCase, effectiveContext, {
       datasetName: dataset.name,
-      schemas: allSchemas
+      schemas: allSchemas,
+      toolOverrideVariantId: toolOverrides?.id
     });
     if (onCaseComplete) {
       await onCaseComplete(result2);
@@ -7764,6 +8052,9 @@ async function runEvalDataset(options, context) {
     gitHash,
     timestamp: (/* @__PURE__ */ new Date()).toISOString(),
     packageVersion: package_default.version,
+    ...toolOverrides !== void 0 && {
+      toolOverrideVariantId: toolOverrides.id
+    },
     ...mcpHostModel !== void 0 && { mcpHostModel },
     ...judgeModel !== void 0 && { judgeModel }
   };
@@ -7782,7 +8073,7 @@ async function runEvalDataset(options, context) {
   };
   if (baselineResultsFrom) {
     try {
-      const baseline = await loadBaseline(baselineResultsFrom);
+      const baseline = typeof baselineResultsFrom === "string" ? await loadBaseline(baselineResultsFrom) : await loadStoredBaseline(baselineResultsFrom, resultStore);
       const baselinePassRate = baseline.total > 0 ? baseline.passed / baseline.total : 0;
       const baselineMap = buildBaselinePassMap(baseline);
       const currentCaseIds = result.caseResults.map((cr) => cr.id);
@@ -7810,7 +8101,7 @@ async function runEvalDataset(options, context) {
       result.deltaPassRate = result.total > 0 ? result.passed / result.total - baselinePassRate : 0;
     } catch (err) {
       console.warn(
-        `[mcp-server-tester] Could not load baseline from ${baselineResultsFrom}: ${err instanceof Error ? err.message : String(err)}`
+        `[mcp-server-tester] Could not load baseline from ${formatBaselineRef(baselineResultsFrom)}: ${err instanceof Error ? err.message : String(err)}`
       );
     }
   }
@@ -7825,22 +8116,84 @@ async function runEvalDataset(options, context) {
     result.datasetToolF1 = avgPrec + avgRecall > 0 ? 2 * avgPrec * avgRecall / (avgPrec + avgRecall) : 0;
   }
   if (saveResultsTo) {
-    await saveBaseline(result, saveResultsTo, {
-      omitResponses: omitResponsesFromBaseline
-    });
+    if (typeof saveResultsTo === "string") {
+      await saveBaseline(result, saveResultsTo, {
+        omitResponses: omitResponsesFromBaseline
+      });
+    } else {
+      await saveStoredEvalResult(result, saveResultsTo, {
+        resultStore,
+        omitResponses: redactStoredResponses ?? true,
+        metadata: {
+          datasetName: dataset.name,
+          ...toolOverrides?.id !== void 0 && {
+            toolOverrideVariantId: toolOverrides.id
+          },
+          ...mcpHostModel !== void 0 && { mcpHostModel },
+          ...judgeModel !== void 0 && { judgeModel },
+          ...gitHash !== void 0 && { gitHash },
+          packageVersion: package_default.version
+        }
+      });
+    }
   }
   if (context.testInfo) {
     await context.testInfo.attach("mcp-test-results", {
       contentType: "application/json",
       body: Buffer.from(JSON.stringify({ caseResults }))
     });
-  } else if (caseResults.length > 0) {
+  } else if (caseResults.length > 0 && !warnedNoTestInfo) {
+    warnedNoTestInfo = true;
     console.warn(
       "[mcp-server-tester] runEvalDataset: testInfo not provided \u2014 results will not appear in the MCP reporter.\nTo enable reporting, pass testInfo from the Playwright test function:\n  await runEvalDataset({ dataset }, { mcp, testInfo });"
     );
   }
   return result;
 }
+async function loadStoredBaseline(baselineResultsFrom, resultStore) {
+  if (!resultStore) {
+    throw new Error("resultStore is required for store-backed baselines");
+  }
+  const store = resolveEvalResultStore(resultStore);
+  const artifact = baselineResultsFrom.ref === "latest" ? await store.loadLatestArtifact("eval-runner-result") : await store.loadArtifact(
+    "eval-runner-result",
+    baselineResultsFrom.ref.id
+  );
+  if (!artifact) {
+    throw new Error("No latest eval run artifact found");
+  }
+  return artifact.data;
+}
+async function saveStoredEvalResult(result, saveResultsTo, options) {
+  if (!options.resultStore) {
+    throw new Error("resultStore is required for store-backed saves");
+  }
+  const store = resolveEvalResultStore(options.resultStore);
+  const data = options.omitResponses ? omitResponsesFromResult(result) : result;
+  const id = saveResultsTo.ref && saveResultsTo.ref !== "latest" ? saveResultsTo.ref.id : void 0;
+  await store.saveArtifact(
+    createStoredEvalArtifact({
+      kind: "eval-runner-result",
+      id,
+      data,
+      metadata: options.metadata
+    })
+  );
+}
+function omitResponsesFromResult(result) {
+  return {
+    ...result,
+    caseResults: result.caseResults.map(
+      ({ response: _response, ...rest }) => rest
+    )
+  };
+}
+function formatBaselineRef(baselineResultsFrom) {
+  if (typeof baselineResultsFrom === "string") {
+    return baselineResultsFrom;
+  }
+  return baselineResultsFrom.ref === "latest" ? "resultStore latest" : `resultStore ${baselineResultsFrom.ref.id}`;
+}
 
 // src/evals/serverComparison.ts
 async function runServerComparison(options, contextA, contextB) {
@@ -7882,7 +8235,7 @@ async function runServerComparison(options, contextA, contextB) {
   }
   const total = cases.length;
   const decidedCases = aWins + bWins + ties;
-  return {
+  const comparison = {
     dataset: options.dataset.name,
     total,
     aWins,
@@ -7899,6 +8252,366 @@ async function runServerComparison(options, contextA, contextB) {
     serverBResult: resultB,
     durationMs: Date.now() - startTime
   };
+  if (options.comparisonStore) {
+    await saveServerComparison({
+      store: options.comparisonStore,
+      comparison,
+      id: options.comparisonId,
+      metadata: {
+        datasetName: options.dataset.name,
+        ...options.comparisonMetadata ?? {}
+      },
+      redactStoredResponses: options.redactStoredResponses
+    });
+  }
+  return comparison;
+}
+async function saveServerComparison(options) {
+  const store = resolveEvalResultStore(options.store);
+  const data = options.redactStoredResponses ? redactResponses(options.comparison) : options.comparison;
+  const artifact = createStoredEvalArtifact({
+    kind: "server-comparison",
+    id: options.id,
+    data,
+    metadata: {
+      datasetName: options.comparison.dataset,
+      ...options.metadata ?? {}
+    }
+  });
+  await store.saveArtifact(artifact);
+  return artifact;
+}
+function redactResponses(value) {
+  return JSON.parse(
+    JSON.stringify(
+      value,
+      (key, currentValue) => key === "response" ? void 0 : currentValue
+    )
+  );
+}
+
+// src/evals/evalRunComparison.ts
+function compareEvalRuns(options) {
+  const { baseline, candidate, labels } = options;
+  const candidateMap = new Map(
+    candidate.caseResults.map((result) => [result.id, result])
+  );
+  const cases = [];
+  const seenIds = /* @__PURE__ */ new Set();
+  for (const baselineCase of baseline.caseResults) {
+    seenIds.add(baselineCase.id);
+    const candidateCase = candidateMap.get(baselineCase.id);
+    if (!candidateCase) {
+      cases.push({
+        id: baselineCase.id,
+        outcome: "MISSING_FROM_CANDIDATE",
+        baseline: baselineCase
+      });
+      continue;
+    }
+    cases.push({
+      id: baselineCase.id,
+      outcome: compareCaseOutcome(baselineCase.pass, candidateCase.pass),
+      baseline: baselineCase,
+      candidate: candidateCase
+    });
+  }
+  for (const candidateCase of candidate.caseResults) {
+    if (seenIds.has(candidateCase.id)) {
+      continue;
+    }
+    cases.push({
+      id: candidateCase.id,
+      outcome: "MISSING_FROM_BASELINE",
+      candidate: candidateCase
+    });
+  }
+  const baselinePassRate = passRate(baseline);
+  const candidatePassRate = passRate(candidate);
+  return {
+    baselineLabel: labels?.baseline ?? "baseline",
+    candidateLabel: labels?.candidate ?? candidate.metadata?.toolOverrideVariantId ?? "candidate",
+    baselinePassRate,
+    candidatePassRate,
+    deltaPassRate: candidatePassRate - baselinePassRate,
+    ...metricDelta(
+      "ToolPrecision",
+      baseline.datasetToolPrecision,
+      candidate.datasetToolPrecision
+    ),
+    ...metricDelta(
+      "ToolRecall",
+      baseline.datasetToolRecall,
+      candidate.datasetToolRecall
+    ),
+    ...metricDelta("ToolF1", baseline.datasetToolF1, candidate.datasetToolF1),
+    cases,
+    improvedCases: cases.filter((c) => c.outcome === "IMPROVED"),
+    regressedCases: cases.filter((c) => c.outcome === "REGRESSED"),
+    unchangedPasses: cases.filter((c) => c.outcome === "UNCHANGED_PASS"),
+    unchangedFailures: cases.filter((c) => c.outcome === "UNCHANGED_FAIL"),
+    missingFromBaseline: cases.filter(
+      (c) => c.outcome === "MISSING_FROM_BASELINE"
+    ),
+    missingFromCandidate: cases.filter(
+      (c) => c.outcome === "MISSING_FROM_CANDIDATE"
+    )
+  };
+}
+async function loadStoredEvalRunnerResult(storeLike, ref) {
+  const store = resolveEvalResultStore(storeLike);
+  const artifact = ref === "latest" ? await store.loadLatestArtifact("eval-runner-result") : await store.loadArtifact(
+    "eval-runner-result",
+    ref.id
+  );
+  if (!artifact) {
+    throw new Error("No latest eval run artifact found");
+  }
+  return artifact;
+}
+async function saveEvalRunComparison(options) {
+  const store = resolveEvalResultStore(options.store);
+  const data = options.redactStoredResponses ? redactResponses2(options.comparison) : options.comparison;
+  const artifact = createStoredEvalArtifact({
+    kind: "eval-run-comparison",
+    id: options.id,
+    data,
+    metadata: {
+      labels: {
+        baseline: options.comparison.baselineLabel,
+        candidate: options.comparison.candidateLabel
+      },
+      ...options.metadata ?? {}
+    }
+  });
+  await store.saveArtifact(artifact);
+  return artifact;
+}
+function compareCaseOutcome(baselinePass, candidatePass) {
+  if (!baselinePass && candidatePass) return "IMPROVED";
+  if (baselinePass && !candidatePass) return "REGRESSED";
+  return baselinePass ? "UNCHANGED_PASS" : "UNCHANGED_FAIL";
+}
+function passRate(result) {
+  return result.total > 0 ? result.passed / result.total : 0;
+}
+function metricDelta(name15, baselineValue, candidateValue) {
+  const result = {};
+  if (baselineValue !== void 0) {
+    result[`baseline${name15}`] = baselineValue;
+  }
+  if (candidateValue !== void 0) {
+    result[`candidate${name15}`] = candidateValue;
+  }
+  if (baselineValue !== void 0 && candidateValue !== void 0) {
+    result[`delta${name15}`] = candidateValue - baselineValue;
+  }
+  return result;
+}
+function redactResponses2(value) {
+  return JSON.parse(
+    JSON.stringify(
+      value,
+      (key, currentValue) => key === "response" ? void 0 : currentValue
+    )
+  );
+}
+
+// src/evals/variantExperiment.ts
+async function runVariantExperiment(options, context) {
+  const metric = options.metric ?? "passRate";
+  const maxRounds = options.maxRounds ?? 1;
+  const minImprovement = options.minImprovement ?? 0;
+  const allowRegressions = options.allowRegressions ?? false;
+  const internalContext = {
+    mcp: context.mcp,
+    expect: context.expect
+  };
+  const baseline = await runEvalDataset(
+    buildRunOptions(options, void 0),
+    internalContext
+  );
+  const baselineValue = readMetric(baseline, metric);
+  if (baselineValue === void 0) {
+    throw new Error(
+      `Metric '${metric}' is unavailable: the dataset produced no tool precision/recall data. Add mcp_host cases with toolsTriggered expectations, or use metric 'passRate'.`
+    );
+  }
+  const rounds = [];
+  let bestSoFar;
+  let bestAttempted;
+  let reason = "max-rounds";
+  for (let round = 0; round < maxRounds; round++) {
+    const variants = await gatherVariants(options, {
+      round,
+      baseline,
+      metric,
+      history: rounds,
+      bestSoFar
+    });
+    if (variants.length === 0) {
+      reason = round === 0 ? "no-variants" : "no-improvement";
+      break;
+    }
+    const candidates = [];
+    for (const variant of variants) {
+      const candidate = await scoreVariant(
+        options,
+        internalContext,
+        baseline,
+        baselineValue,
+        metric,
+        allowRegressions,
+        variant
+      );
+      candidates.push(candidate);
+      bestAttempted = pickBetter(bestAttempted, candidate, true);
+    }
+    const roundBest = candidates.reduce(
+      (best, candidate) => pickBetter(best, candidate, false),
+      void 0
+    );
+    rounds.push({ round, candidates, best: roundBest });
+    if (roundBest) {
+      const improvement = roundBest.metricValue - (bestSoFar?.metricValue ?? baselineValue);
+      bestSoFar = pickBetter(bestSoFar, roundBest, false);
+      if (improvement < minImprovement) {
+        reason = "no-improvement";
+        break;
+      }
+    }
+  }
+  const winner = bestSoFar;
+  const proposalSource = winner ?? bestAttempted;
+  const proposal = proposalSource ? buildProposal(metric, baselineValue, proposalSource, winner !== void 0) : void 0;
+  const result = {
+    metric,
+    baseline,
+    rounds,
+    winner,
+    proposal,
+    converged: true,
+    reason
+  };
+  if (context.testInfo) {
+    const surfaceRun = winner?.result ?? bestAttempted?.result ?? baseline;
+    await context.testInfo.attach("mcp-test-results", {
+      contentType: "application/json",
+      body: Buffer.from(
+        JSON.stringify({ caseResults: surfaceRun.caseResults })
+      )
+    });
+    await context.testInfo.attach("mcp-variant-experiment", {
+      contentType: "application/json",
+      body: Buffer.from(
+        JSON.stringify(buildExperimentData(result, baselineValue))
+      )
+    });
+  }
+  return result;
+}
+function buildExperimentData(result, baselineValue) {
+  return {
+    metric: result.metric,
+    baselineValue,
+    bestValue: result.winner?.metricValue ?? result.proposal?.candidateValue ?? baselineValue,
+    rounds: result.rounds.map((round) => {
+      const best = round.best ?? round.candidates[0];
+      return {
+        round: round.round,
+        variantId: best?.variant.id ?? "(none)",
+        metricValue: best?.metricValue ?? baselineValue,
+        metricDelta: best?.metricDelta ?? 0,
+        disqualified: best?.disqualified ?? false
+      };
+    }),
+    winnerVariantId: result.winner?.variant.id,
+    recommendation: result.proposal?.recommendation,
+    reason: result.reason
+  };
+}
+async function gatherVariants(options, context) {
+  if (context.round === 0 && options.variants && options.variants.length > 0) {
+    return options.variants;
+  }
+  if (options.proposeVariants) {
+    return options.proposeVariants(context);
+  }
+  return [];
+}
+async function scoreVariant(options, context, baseline, baselineValue, metric, allowRegressions, variant) {
+  const result = await runEvalDataset(
+    buildRunOptions(options, variant),
+    context
+  );
+  const comparison = compareEvalRuns({
+    baseline,
+    candidate: result,
+    labels: { candidate: variant.id }
+  });
+  const metricValue = readMetric(result, metric) ?? baselineValue;
+  const disqualified = !allowRegressions && comparison.regressedCases.length > 0;
+  return {
+    variant,
+    result,
+    comparison,
+    metricValue,
+    metricDelta: metricValue - baselineValue,
+    disqualified
+  };
+}
+function pickBetter(incumbent, challenger, includeDisqualified) {
+  if (!includeDisqualified && challenger.disqualified) {
+    return incumbent;
+  }
+  if (!incumbent) {
+    return challenger;
+  }
+  return challenger.metricValue > incumbent.metricValue ? challenger : incumbent;
+}
+function buildProposal(metric, baselineValue, source, isWinner) {
+  let recommendation;
+  if (isWinner) {
+    recommendation = source.metricDelta > 0 ? "apply" : "inconclusive";
+  } else {
+    recommendation = source.disqualified ? "reject" : "inconclusive";
+  }
+  return {
+    variantId: source.variant.id,
+    metric,
+    baselineValue,
+    candidateValue: source.metricValue,
+    delta: source.metricDelta,
+    toolChanges: source.variant.tools,
+    improvedCaseIds: source.comparison.improvedCases.map((c) => c.id),
+    regressedCaseIds: source.comparison.regressedCases.map((c) => c.id),
+    recommendation
+  };
+}
+function readMetric(result, metric) {
+  switch (metric) {
+    case "passRate":
+      return result.total > 0 ? result.passed / result.total : 0;
+    case "toolF1":
+      return result.datasetToolF1;
+    case "toolPrecision":
+      return result.datasetToolPrecision;
+    case "toolRecall":
+      return result.datasetToolRecall;
+  }
+}
+function buildRunOptions(options, toolOverrides) {
+  return {
+    dataset: options.dataset,
+    toolOverrides,
+    defaultLlmIterations: options.defaultLlmIterations,
+    defaultJudgeReps: options.defaultJudgeReps,
+    concurrency: options.concurrency,
+    filterTags: options.filterTags,
+    schemas: options.schemas,
+    mcpHostModel: options.mcpHostModel,
+    judgeModel: options.judgeModel
+  };
 }
 
 // src/spec/conformanceChecks.ts
@@ -8072,6 +8785,6 @@ function formatCapabilities(capabilities) {
   return parts.length > 0 ? parts.join(", ") : "none declared";
 }
 
-export { BUILT_IN_RUBRICS, CLIOAuthClient, DiscoveryError, ENV_VAR_NAMES, EvalCaseSchema, EvalDatasetSchema, MCPConfigSchema, MCP_PROTOCOL_VERSION, PlaywrightOAuthClientProvider, SnapshotSanitizers, clearJudgeRegistry, closeMCPClient, createJudge, createMCPClientForConfig, createMCPFixture, createTokenAuthHeaders, discoverAuthorizationServer, discoverProtectedResource, expect, extractText, getMissingDependencyMessage, getRegisteredJudge, getResponseSizeBytes, hasValidTokens, injectTokens, isBuiltInRubric, isHttpConfig, isProviderAvailable, isStdioConfig, isTokenExpired, isTokenExpiringSoon, loadBaseline, loadEvalDataset, loadEvalDatasetFromObject, loadTokens, loadTokensFromEnv, test2 as mcpAuthTest, normalizeToolResponse, normalizeWhitespace, performClientCredentialsFlow, performOAuthSetup, performOAuthSetupIfNeeded, refreshAccessToken, registerJudge, resolveRubric, runConformanceChecks, runEvalCase, runEvalDataset, runServerComparison, saveBaseline, simulateMCPHost, test, validateAccessToken, validateError, validateEvalCase, validateEvalDataset, validateJudge, validateMCPConfig, validatePattern, validateResponse, validateSchema, validateSize, validateText, validateToolCallCount, validateToolCalls };
+export { BUILT_IN_RUBRICS, CLIOAuthClient, DiscoveryError, ENV_VAR_NAMES, EvalCaseSchema, EvalDatasetSchema, FileEvalResultStore, GCSEvalResultStore, MCPConfigSchema, MCP_PROTOCOL_VERSION, PlaywrightOAuthClientProvider, SnapshotSanitizers, clearJudgeRegistry, closeMCPClient, compareEvalRuns, createDefaultArtifactId, createEvalResultStore, createJudge, createMCPClientForConfig, createMCPFixture, createStoredEvalArtifact, createTokenAuthHeaders, defaultEnvironmentMetadata, discoverAuthorizationServer, discoverProtectedResource, expect, extractText, getMissingDependencyMessage, getRegisteredJudge, getResponseSizeBytes, hasValidTokens, injectTokens, isBuiltInRubric, isEvalResultStore, isHttpConfig, isProviderAvailable, isStdioConfig, isTokenExpired, isTokenExpiringSoon, loadBaseline, loadEvalDataset, loadEvalDatasetFromObject, loadStoredEvalRunnerResult, loadTokens, loadTokensFromEnv, test2 as mcpAuthTest, normalizeToolResponse, normalizeWhitespace, performClientCredentialsFlow, performOAuthSetup, performOAuthSetupIfNeeded, refreshAccessToken, registerJudge, resolveEvalResultStore, resolveRubric, runConformanceChecks, runEvalCase, runEvalDataset, runServerComparison, runVariantExperiment, saveBaseline, saveEvalRunComparison, saveServerComparison, simulateMCPHost, test, validateAccessToken, validateError, validateEvalCase, validateEvalDataset, validateJudge, validateMCPConfig, validatePattern, validateResponse, validateSchema, validateSize, validateText, validateToolCallCount, validateToolCalls };
 //# sourceMappingURL=index.js.map
 //# sourceMappingURL=index.js.map