@gleanwork/mcp-server-tester 1.0.1 → 1.1.1

package/dist/index.cjs

@@ -3126,6 +3126,20 @@ var init_dist3 = __esm({
     };
   }
 });
+
+// src/assertions/validators/types.ts
+var SnapshotSanitizers = {
+  /** Replaces Unix timestamps (seconds and milliseconds) with a stable placeholder */
+  TIMESTAMP: "timestamp",
+  /** Replaces UUID v1-v5 strings with a stable placeholder */
+  UUID: "uuid",
+  /** Replaces ISO 8601 date/datetime strings with a stable placeholder */
+  ISO_DATE: "iso-date",
+  /** Replaces MongoDB ObjectId strings with a stable placeholder */
+  OBJECT_ID: "objectId",
+  /** Replaces JWT tokens with a stable placeholder */
+  JWT: "jwt"
+};
 var MCPHostCapabilitiesSchema = zod.z.object({
   sampling: zod.z.record(zod.z.string(), zod.z.unknown()).optional(),
   roots: zod.z.object({
@@ -4411,7 +4425,7 @@ function escapeHtml(text) {
 
 // package.json
 var package_default = {
-  version: "1.0.1"};
+  version: "1.1.1"};
 
 // src/mcp/clientFactory.ts
 function getRetryAfterDelayMs(err) {
@@ -5935,20 +5949,6 @@ async function validateJudge(response, config) {
   }
 }
 
-// src/assertions/validators/types.ts
-var SnapshotSanitizers = {
-  /** Replaces Unix timestamps (seconds and milliseconds) with a stable placeholder */
-  TIMESTAMP: "timestamp",
-  /** Replaces UUID v1-v5 strings with a stable placeholder */
-  UUID: "uuid",
-  /** Replaces ISO 8601 date/datetime strings with a stable placeholder */
-  ISO_DATE: "iso-date",
-  /** Replaces MongoDB ObjectId strings with a stable placeholder */
-  OBJECT_ID: "objectId",
-  /** Replaces JWT tokens with a stable placeholder */
-  JWT: "jwt"
-};
-
 // src/mcp/fixtures/mcpFixture.ts
 var DEFAULT_CALL_TIMEOUT_MS = 3e4;
 function withCallTimeout(promise, ms, opName) {
@@ -6838,8 +6838,8 @@ Hint: run \`getMissingDependencyMessage('${provider}')\` or check docs/mcp-host.
 Hint: check your API key environment variable (e.g. ANTHROPIC_API_KEY, GOOGLE_APPLICATION_CREDENTIALS).`;
   }
   if (raw.includes("404") || raw.includes("Not Found") || raw.toLowerCase().includes("model") && raw.toLowerCase().includes("not found")) {
-    return `MCP host simulation failed: model not found.
-Hint: check the model name format for your provider. For vertex-anthropic use 'claude-3-5-haiku@20241022' (with @).`;
+    return `MCP host simulation failed: ${raw}
+Hint: a 404 usually means the model id is wrong or retired, or a base-URL override (e.g. ANTHROPIC_BASE_URL / OPENAI_BASE_URL pointing at a gateway) is routing requests somewhere that doesn't serve this model. Verify the model id and that no unexpected *_BASE_URL is set.`;
   }
   if (raw.includes("ENOTFOUND") || raw.includes("fetch failed") || raw.includes("ECONNREFUSED")) {
     return `MCP host simulation failed: network error.
@@ -6937,12 +6937,18 @@ function createVercelOrchestrator() {
           tools[toolName] = {
             description: mcpTool.description ?? "",
             inputSchema: jsonSchema2(rawSchema),
-            execute: async (args) => {
+            execute: async (args, opts) => {
               const mcpStart = Date.now();
               const result2 = await mcp.callTool(toolName, args);
               mcpDurationMs += Date.now() - mcpStart;
-              allToolCalls.push({ name: toolName, arguments: args });
-              return extractText(result2);
+              const output = extractText(result2);
+              allToolCalls.push({
+                id: opts?.toolCallId,
+                name: toolName,
+                arguments: args,
+                output
+              });
+              return output;
             }
           };
         }
@@ -6964,10 +6970,17 @@ function createVercelOrchestrator() {
           totalCostUsd: 0,
           durationMs: llmDurationMs
         } : void 0;
-        const conversationHistory = (result.steps ?? []).map((step) => ({
-          role: step.toolCalls?.length > 0 ? "tool" : "assistant",
-          content: step.toolCalls?.length > 0 ? JSON.stringify(step.toolResults) : step.text ?? ""
-        }));
+        const conversationHistory = (result.steps ?? []).flatMap((step) => {
+          if (step.toolCalls?.length > 0) {
+            return step.toolCalls.map(
+              (tc) => ({
+                role: "tool",
+                toolCallId: tc.toolCallId
+              })
+            );
+          }
+          return step.text ? [{ role: "assistant", content: step.text }] : [];
+        });
         return {
           success: true,
           toolCalls: allToolCalls,
@@ -6994,6 +7007,7 @@ function parseStreamJson(stdout) {
   const lines = stdout.split("\n").filter((line) => line.trim().length > 0);
   const toolCalls = [];
   const textParts = [];
+  let usage;
   const conversationHistory = [];
   for (const line of lines) {
     let event;
@@ -7022,20 +7036,38 @@ function parseStreamJson(stdout) {
       for (const block of event.message.content) {
         if (block.type === "tool_result") {
           const content = typeof block.content === "string" ? block.content : JSON.stringify(block.content);
-          conversationHistory.push({ role: "tool", content });
+          const call = block.tool_use_id ? toolCalls.find((tc) => tc.id === block.tool_use_id) : void 0;
+          if (call) {
+            call.output = content;
+            conversationHistory.push({ role: "tool", toolCallId: call.id });
+          } else {
+            conversationHistory.push({ role: "tool", content });
+          }
         }
       }
     }
-    if (event.type === "result" && typeof event.result === "string") {
-      if (textParts.length === 0) {
+    if (event.type === "result") {
+      if (typeof event.result === "string" && textParts.length === 0) {
         textParts.push(event.result);
       }
+      if (event.usage) {
+        usage = {
+          inputTokens: event.usage.input_tokens ?? 0,
+          outputTokens: event.usage.output_tokens ?? 0,
+          totalCostUsd: event.total_cost_usd ?? 0,
+          durationMs: event.duration_ms ?? 0,
+          durationApiMs: event.duration_api_ms,
+          cacheReadInputTokens: event.usage.cache_read_input_tokens,
+          cacheCreationInputTokens: event.usage.cache_creation_input_tokens
+        };
+      }
     }
     if (event.type === "result" && event.is_error === true) {
       return {
         success: false,
         toolCalls,
-        error: typeof event.result === "string" ? event.result : "CLI host reported an error"
+        error: typeof event.result === "string" ? event.result : "CLI host reported an error",
+        usage
       };
     }
   }
@@ -7047,7 +7079,8 @@ function parseStreamJson(stdout) {
     success: true,
     toolCalls,
     response: response || void 0,
-    conversationHistory: conversationHistory.length > 0 ? conversationHistory : void 0
+    conversationHistory: conversationHistory.length > 0 ? conversationHistory : void 0,
+    usage
   };
 }
 function createJsonParser(paths) {
@@ -7293,6 +7326,216 @@ function buildBaselinePassMap(baseline) {
   }
   return map;
 }
+var KIND_DIRS = {
+  "eval-runner-result": "eval-runs",
+  "reporter-run": "reporter-runs",
+  "eval-run-comparison": "comparisons/eval-runs",
+  "server-comparison": "comparisons/servers"
+};
+function createEvalResultStore(config) {
+  if (config.provider === "file") {
+    return new FileEvalResultStore(config);
+  }
+  return new GCSEvalResultStore(config);
+}
+function resolveEvalResultStore(store) {
+  return isEvalResultStore(store) ? store : createEvalResultStore(store);
+}
+function isEvalResultStore(value) {
+  return typeof value === "object" && value !== null && "saveArtifact" in value && "loadArtifact" in value && "loadLatestArtifact" in value && "listArtifacts" in value;
+}
+function createStoredEvalArtifact(options) {
+  const createdAt = options.createdAt ?? (/* @__PURE__ */ new Date()).toISOString();
+  return {
+    schemaVersion: 1,
+    kind: options.kind,
+    id: options.id ?? createDefaultArtifactId(createdAt),
+    createdAt,
+    metadata: {
+      ...defaultEnvironmentMetadata(),
+      ...options.metadata ?? {}
+    },
+    data: options.data
+  };
+}
+function createDefaultArtifactId(timestamp = (/* @__PURE__ */ new Date()).toISOString()) {
+  const safeTimestamp = timestamp.replace(/[:.]/g, "-");
+  const runNumber = process.env.GITHUB_RUN_NUMBER;
+  const sha = process.env.GITHUB_SHA?.slice(0, 12);
+  const suffix = runNumber ?? sha;
+  return suffix ? `${safeTimestamp}-${suffix}` : safeTimestamp;
+}
+function defaultEnvironmentMetadata() {
+  return {
+    ...process.env.GITHUB_SHA !== void 0 && {
+      gitHash: process.env.GITHUB_SHA
+    },
+    ...process.env.GITHUB_REF_NAME !== void 0 && {
+      branch: process.env.GITHUB_REF_NAME
+    },
+    ...process.env.GITHUB_RUN_NUMBER !== void 0 && {
+      runNumber: process.env.GITHUB_RUN_NUMBER
+    },
+    ...process.env.GITHUB_EVENT_NAME !== void 0 && {
+      trigger: process.env.GITHUB_EVENT_NAME
+    }
+  };
+}
+var FileEvalResultStore = class {
+  dir;
+  constructor(config) {
+    this.dir = config.dir;
+  }
+  async saveArtifact(artifact) {
+    const artifactDir = path2.join(this.dir, KIND_DIRS[artifact.kind]);
+    await fs$1.mkdir(artifactDir, { recursive: true });
+    const serialized = JSON.stringify(artifact, null, 2);
+    await fs$1.writeFile(
+      path2.join(artifactDir, `${artifact.id}.json`),
+      serialized,
+      "utf8"
+    );
+    await fs$1.writeFile(path2.join(artifactDir, "latest.json"), serialized, "utf8");
+  }
+  async loadArtifact(kind, id) {
+    const raw = await fs$1.readFile(
+      path2.join(this.dir, KIND_DIRS[kind], `${id}.json`),
+      "utf8"
+    );
+    return JSON.parse(raw);
+  }
+  async loadLatestArtifact(kind) {
+    try {
+      const raw = await fs$1.readFile(
+        path2.join(this.dir, KIND_DIRS[kind], "latest.json"),
+        "utf8"
+      );
+      return JSON.parse(raw);
+    } catch (error) {
+      if (isMissingFileError(error)) {
+        return null;
+      }
+      throw error;
+    }
+  }
+  async listArtifacts(kind, options = {}) {
+    let files;
+    try {
+      files = await fs$1.readdir(path2.join(this.dir, KIND_DIRS[kind]));
+    } catch (error) {
+      if (isMissingFileError(error)) {
+        return [];
+      }
+      throw error;
+    }
+    const summaries = await Promise.all(
+      files.filter((f) => f.endsWith(".json") && f !== "latest.json").map(async (file) => {
+        const raw = await fs$1.readFile(
+          path2.join(this.dir, KIND_DIRS[kind], file),
+          "utf8"
+        );
+        return toSummary(JSON.parse(raw));
+      })
+    );
+    return summaries.sort((a, b) => b.createdAt.localeCompare(a.createdAt)).slice(0, options.limit);
+  }
+};
+var GCSEvalResultStore = class {
+  bucketName;
+  prefix;
+  storage;
+  constructor(config) {
+    this.bucketName = config.bucket;
+    this.prefix = trimSlashes(config.prefix ?? "");
+  }
+  async saveArtifact(artifact) {
+    const bucket = await this.getBucket();
+    const serialized = JSON.stringify(artifact, null, 2);
+    await bucket.file(
+      this.objectPath(
+        artifact.kind,
+        `${encodeURIComponent(artifact.id)}.json`
+      )
+    ).save(serialized, {
+      contentType: "application/json",
+      resumable: false,
+      validation: false
+    });
+    await bucket.file(this.objectPath(artifact.kind, "latest.json")).save(serialized, {
+      contentType: "application/json",
+      resumable: false,
+      validation: false
+    });
+  }
+  async loadArtifact(kind, id) {
+    const bucket = await this.getBucket();
+    const file = bucket.file(
+      this.objectPath(kind, `${encodeURIComponent(id)}.json`)
+    );
+    const [buffer] = await file.download();
+    return JSON.parse(buffer.toString("utf8"));
+  }
+  async loadLatestArtifact(kind) {
+    const bucket = await this.getBucket();
+    const file = bucket.file(this.objectPath(kind, "latest.json"));
+    const [exists] = await file.exists();
+    if (!exists) return null;
+    const [buffer] = await file.download();
+    return JSON.parse(buffer.toString("utf8"));
+  }
+  async listArtifacts(kind, options = {}) {
+    const bucket = await this.getBucket();
+    const [files] = await bucket.getFiles({
+      prefix: this.objectPath(kind, "")
+    });
+    const summaries = await Promise.all(
+      files.filter(
+        (f) => f.name.endsWith(".json") && !f.name.endsWith("/latest.json")
+      ).map(async (file) => {
+        const [buffer] = await file.download();
+        return toSummary(
+          JSON.parse(buffer.toString("utf8"))
+        );
+      })
+    );
+    return summaries.sort((a, b) => b.createdAt.localeCompare(a.createdAt)).slice(0, options.limit);
+  }
+  async getBucket() {
+    if (!this.storage) {
+      const moduleName = "@google-cloud/storage";
+      let Storage;
+      try {
+        const mod = await import(moduleName);
+        Storage = mod.Storage;
+      } catch (error) {
+        throw new Error(
+          `GCS result storage requires the optional \`@google-cloud/storage\` package. Install it and authenticate with Application Default Credentials via GOOGLE_APPLICATION_CREDENTIALS.
+Original error: ${error instanceof Error ? error.message : String(error)}`
+        );
+      }
+      this.storage = new Storage();
+    }
+    return this.storage.bucket(this.bucketName);
+  }
+  objectPath(kind, filename) {
+    const parts = [this.prefix, KIND_DIRS[kind], filename].filter(Boolean);
+    return parts.join("/");
+  }
+};
+function toSummary(artifact) {
+  return {
+    kind: artifact.kind,
+    id: artifact.id,
+    createdAt: artifact.createdAt,
+    metadata: artifact.metadata
+  };
+}
+function trimSlashes(value) {
+  return value.replace(/^\/+|\/+$/g, "");
+}
+function isMissingFileError(error) {
+  return typeof error === "object" && error !== null && "code" in error && error.code === "ENOENT";
+}
 var execFileAsync = util.promisify(child_process.execFile);
 async function execFileNoThrow(file, args) {
   try {
@@ -7339,6 +7582,41 @@ function sumUsage(a, b) {
 }
 
 // src/evals/evalRunner.ts
+function createToolOverrideMCP(mcp, variant) {
+  return {
+    ...mcp,
+    async listTools() {
+      const tools = await mcp.listTools();
+      const knownToolNames = new Set(tools.map((tool2) => tool2.name));
+      const unknownToolNames = Object.keys(variant.tools).filter(
+        (name15) => !knownToolNames.has(name15)
+      );
+      if (unknownToolNames.length > 0) {
+        throw new Error(
+          `[mcp-server-tester] toolOverrides variant "${variant.id}" references unknown tool(s): ` + unknownToolNames.join(", ")
+        );
+      }
+      return tools.map((tool2) => {
+        const override = variant.tools[tool2.name];
+        if (!override) {
+          return tool2;
+        }
+        return {
+          ...tool2,
+          ...override.description !== void 0 && {
+            description: override.description
+          },
+          ...override.inputSchema !== void 0 && {
+            inputSchema: override.inputSchema
+          }
+        };
+      });
+    },
+    async callTool(name15, args) {
+      return mcp.callTool(name15, args);
+    }
+  };
+}
 async function executeToolCall(evalCase, mcp) {
   const mode = evalCase.mode || "direct";
   try {
@@ -7523,9 +7801,12 @@ async function runExpectBlockValidations(expectBlock, response, config) {
   }
   return { expectations: results, toolPrecision, toolRecall };
 }
-function buildRequest(evalCase) {
+function buildRequest(evalCase, toolOverrideVariantId) {
   const request = {};
   if (evalCase.description) request.description = evalCase.description;
+  if (toolOverrideVariantId !== void 0) {
+    request.toolOverrideVariantId = toolOverrideVariantId;
+  }
   if (evalCase.mode === "mcp_host") {
     if (evalCase.scenario) request.scenario = evalCase.scenario;
     if (evalCase.mcpHostConfig) {
@@ -7590,7 +7871,7 @@ async function runSingleIteration(evalCase, context, options) {
     toolName: evalCase.scenario != null ? "mcp_host" : evalCase.toolName ?? "unknown",
     source: "eval",
     pass: didCasePass(error, expectationResults),
-    request: buildRequest(evalCase),
+    request: buildRequest(evalCase, options.toolOverrideVariantId),
     response,
     error,
     expectations: expectationResults,
@@ -7669,7 +7950,8 @@ async function runEvalCase(evalCase, context, options = {}) {
     authType: context.mcp.authType,
     project: context.mcp.project,
     durationMs: 0,
-    tags: evalCase.tags
+    tags: evalCase.tags,
+    request: buildRequest(evalCase, options.toolOverrideVariantId)
   };
   const totalHostUsage = iterationResults.reduce(
     (acc, r) => sumUsage(acc, r.hostUsage),
@@ -7716,6 +7998,7 @@ async function getGitHash() {
   const result = await execFileNoThrow("git", ["rev-parse", "HEAD"]);
   return result.status === 0 ? result.stdout.trim() : void 0;
 }
+var warnedNoTestInfo = false;
 async function runEvalDataset(options, context) {
   const {
     dataset,
@@ -7728,11 +8011,15 @@ async function runEvalDataset(options, context) {
     filterTags,
     saveResultsTo,
     omitResponsesFromBaseline = true,
+    redactStoredResponses,
+    resultStore,
     baselineResultsFrom,
+    toolOverrides,
     mcpHostModel,
     judgeModel
   } = options;
   const startTime = Date.now();
+  const effectiveContext = toolOverrides ? { ...context, mcp: createToolOverrideMCP(context.mcp, toolOverrides) } : context;
   const allSchemas = {
     ...dataset.schemas,
     ...schemas
@@ -7764,9 +8051,10 @@ async function runEvalDataset(options, context) {
       }
     }
     const effectiveCase = withIterations.judgeReps === void 0 && defaultJudgeReps !== void 0 ? { ...withIterations, judgeReps: defaultJudgeReps } : withIterations;
-    const result2 = await runEvalCase(effectiveCase, context, {
+    const result2 = await runEvalCase(effectiveCase, effectiveContext, {
       datasetName: dataset.name,
-      schemas: allSchemas
+      schemas: allSchemas,
+      toolOverrideVariantId: toolOverrides?.id
     });
     if (onCaseComplete) {
       await onCaseComplete(result2);
@@ -7791,6 +8079,9 @@ async function runEvalDataset(options, context) {
     gitHash,
     timestamp: (/* @__PURE__ */ new Date()).toISOString(),
     packageVersion: package_default.version,
+    ...toolOverrides !== void 0 && {
+      toolOverrideVariantId: toolOverrides.id
+    },
     ...mcpHostModel !== void 0 && { mcpHostModel },
     ...judgeModel !== void 0 && { judgeModel }
   };
@@ -7809,7 +8100,7 @@ async function runEvalDataset(options, context) {
   };
   if (baselineResultsFrom) {
     try {
-      const baseline = await loadBaseline(baselineResultsFrom);
+      const baseline = typeof baselineResultsFrom === "string" ? await loadBaseline(baselineResultsFrom) : await loadStoredBaseline(baselineResultsFrom, resultStore);
       const baselinePassRate = baseline.total > 0 ? baseline.passed / baseline.total : 0;
       const baselineMap = buildBaselinePassMap(baseline);
       const currentCaseIds = result.caseResults.map((cr) => cr.id);
@@ -7837,7 +8128,7 @@ async function runEvalDataset(options, context) {
       result.deltaPassRate = result.total > 0 ? result.passed / result.total - baselinePassRate : 0;
     } catch (err) {
       console.warn(
-        `[mcp-server-tester] Could not load baseline from ${baselineResultsFrom}: ${err instanceof Error ? err.message : String(err)}`
+        `[mcp-server-tester] Could not load baseline from ${formatBaselineRef(baselineResultsFrom)}: ${err instanceof Error ? err.message : String(err)}`
       );
     }
   }
@@ -7852,22 +8143,84 @@ async function runEvalDataset(options, context) {
     result.datasetToolF1 = avgPrec + avgRecall > 0 ? 2 * avgPrec * avgRecall / (avgPrec + avgRecall) : 0;
   }
   if (saveResultsTo) {
-    await saveBaseline(result, saveResultsTo, {
-      omitResponses: omitResponsesFromBaseline
-    });
+    if (typeof saveResultsTo === "string") {
+      await saveBaseline(result, saveResultsTo, {
+        omitResponses: omitResponsesFromBaseline
+      });
+    } else {
+      await saveStoredEvalResult(result, saveResultsTo, {
+        resultStore,
+        omitResponses: redactStoredResponses ?? true,
+        metadata: {
+          datasetName: dataset.name,
+          ...toolOverrides?.id !== void 0 && {
+            toolOverrideVariantId: toolOverrides.id
+          },
+          ...mcpHostModel !== void 0 && { mcpHostModel },
+          ...judgeModel !== void 0 && { judgeModel },
+          ...gitHash !== void 0 && { gitHash },
+          packageVersion: package_default.version
+        }
+      });
+    }
   }
   if (context.testInfo) {
     await context.testInfo.attach("mcp-test-results", {
       contentType: "application/json",
       body: Buffer.from(JSON.stringify({ caseResults }))
     });
-  } else if (caseResults.length > 0) {
+  } else if (caseResults.length > 0 && !warnedNoTestInfo) {
+    warnedNoTestInfo = true;
     console.warn(
       "[mcp-server-tester] runEvalDataset: testInfo not provided \u2014 results will not appear in the MCP reporter.\nTo enable reporting, pass testInfo from the Playwright test function:\n  await runEvalDataset({ dataset }, { mcp, testInfo });"
     );
   }
   return result;
 }
+async function loadStoredBaseline(baselineResultsFrom, resultStore) {
+  if (!resultStore) {
+    throw new Error("resultStore is required for store-backed baselines");
+  }
+  const store = resolveEvalResultStore(resultStore);
+  const artifact = baselineResultsFrom.ref === "latest" ? await store.loadLatestArtifact("eval-runner-result") : await store.loadArtifact(
+    "eval-runner-result",
+    baselineResultsFrom.ref.id
+  );
+  if (!artifact) {
+    throw new Error("No latest eval run artifact found");
+  }
+  return artifact.data;
+}
+async function saveStoredEvalResult(result, saveResultsTo, options) {
+  if (!options.resultStore) {
+    throw new Error("resultStore is required for store-backed saves");
+  }
+  const store = resolveEvalResultStore(options.resultStore);
+  const data = options.omitResponses ? omitResponsesFromResult(result) : result;
+  const id = saveResultsTo.ref && saveResultsTo.ref !== "latest" ? saveResultsTo.ref.id : void 0;
+  await store.saveArtifact(
+    createStoredEvalArtifact({
+      kind: "eval-runner-result",
+      id,
+      data,
+      metadata: options.metadata
+    })
+  );
+}
+function omitResponsesFromResult(result) {
+  return {
+    ...result,
+    caseResults: result.caseResults.map(
+      ({ response: _response, ...rest }) => rest
+    )
+  };
+}
+function formatBaselineRef(baselineResultsFrom) {
+  if (typeof baselineResultsFrom === "string") {
+    return baselineResultsFrom;
+  }
+  return baselineResultsFrom.ref === "latest" ? "resultStore latest" : `resultStore ${baselineResultsFrom.ref.id}`;
+}
 
 // src/evals/serverComparison.ts
 async function runServerComparison(options, contextA, contextB) {
@@ -7909,7 +8262,7 @@ async function runServerComparison(options, contextA, contextB) {
   }
   const total = cases.length;
   const decidedCases = aWins + bWins + ties;
-  return {
+  const comparison = {
     dataset: options.dataset.name,
     total,
     aWins,
@@ -7926,6 +8279,366 @@ async function runServerComparison(options, contextA, contextB) {
     serverBResult: resultB,
     durationMs: Date.now() - startTime
   };
+  if (options.comparisonStore) {
+    await saveServerComparison({
+      store: options.comparisonStore,
+      comparison,
+      id: options.comparisonId,
+      metadata: {
+        datasetName: options.dataset.name,
+        ...options.comparisonMetadata ?? {}
+      },
+      redactStoredResponses: options.redactStoredResponses
+    });
+  }
+  return comparison;
+}
+async function saveServerComparison(options) {
+  const store = resolveEvalResultStore(options.store);
+  const data = options.redactStoredResponses ? redactResponses(options.comparison) : options.comparison;
+  const artifact = createStoredEvalArtifact({
+    kind: "server-comparison",
+    id: options.id,
+    data,
+    metadata: {
+      datasetName: options.comparison.dataset,
+      ...options.metadata ?? {}
+    }
+  });
+  await store.saveArtifact(artifact);
+  return artifact;
+}
+function redactResponses(value) {
+  return JSON.parse(
+    JSON.stringify(
+      value,
+      (key, currentValue) => key === "response" ? void 0 : currentValue
+    )
+  );
+}
+
+// src/evals/evalRunComparison.ts
+function compareEvalRuns(options) {
+  const { baseline, candidate, labels } = options;
+  const candidateMap = new Map(
+    candidate.caseResults.map((result) => [result.id, result])
+  );
+  const cases = [];
+  const seenIds = /* @__PURE__ */ new Set();
+  for (const baselineCase of baseline.caseResults) {
+    seenIds.add(baselineCase.id);
+    const candidateCase = candidateMap.get(baselineCase.id);
+    if (!candidateCase) {
+      cases.push({
+        id: baselineCase.id,
+        outcome: "MISSING_FROM_CANDIDATE",
+        baseline: baselineCase
+      });
+      continue;
+    }
+    cases.push({
+      id: baselineCase.id,
+      outcome: compareCaseOutcome(baselineCase.pass, candidateCase.pass),
+      baseline: baselineCase,
+      candidate: candidateCase
+    });
+  }
+  for (const candidateCase of candidate.caseResults) {
+    if (seenIds.has(candidateCase.id)) {
+      continue;
+    }
+    cases.push({
+      id: candidateCase.id,
+      outcome: "MISSING_FROM_BASELINE",
+      candidate: candidateCase
+    });
+  }
+  const baselinePassRate = passRate(baseline);
+  const candidatePassRate = passRate(candidate);
+  return {
+    baselineLabel: labels?.baseline ?? "baseline",
+    candidateLabel: labels?.candidate ?? candidate.metadata?.toolOverrideVariantId ?? "candidate",
+    baselinePassRate,
+    candidatePassRate,
+    deltaPassRate: candidatePassRate - baselinePassRate,
+    ...metricDelta(
+      "ToolPrecision",
+      baseline.datasetToolPrecision,
+      candidate.datasetToolPrecision
+    ),
+    ...metricDelta(
+      "ToolRecall",
+      baseline.datasetToolRecall,
+      candidate.datasetToolRecall
+    ),
+    ...metricDelta("ToolF1", baseline.datasetToolF1, candidate.datasetToolF1),
+    cases,
+    improvedCases: cases.filter((c) => c.outcome === "IMPROVED"),
+    regressedCases: cases.filter((c) => c.outcome === "REGRESSED"),
+    unchangedPasses: cases.filter((c) => c.outcome === "UNCHANGED_PASS"),
+    unchangedFailures: cases.filter((c) => c.outcome === "UNCHANGED_FAIL"),
+    missingFromBaseline: cases.filter(
+      (c) => c.outcome === "MISSING_FROM_BASELINE"
+    ),
+    missingFromCandidate: cases.filter(
+      (c) => c.outcome === "MISSING_FROM_CANDIDATE"
+    )
+  };
+}
+async function loadStoredEvalRunnerResult(storeLike, ref) {
+  const store = resolveEvalResultStore(storeLike);
+  const artifact = ref === "latest" ? await store.loadLatestArtifact("eval-runner-result") : await store.loadArtifact(
+    "eval-runner-result",
+    ref.id
+  );
+  if (!artifact) {
+    throw new Error("No latest eval run artifact found");
+  }
+  return artifact;
+}
+async function saveEvalRunComparison(options) {
+  const store = resolveEvalResultStore(options.store);
+  const data = options.redactStoredResponses ? redactResponses2(options.comparison) : options.comparison;
+  const artifact = createStoredEvalArtifact({
+    kind: "eval-run-comparison",
+    id: options.id,
+    data,
+    metadata: {
+      labels: {
+        baseline: options.comparison.baselineLabel,
+        candidate: options.comparison.candidateLabel
+      },
+      ...options.metadata ?? {}
+    }
+  });
+  await store.saveArtifact(artifact);
+  return artifact;
+}
+function compareCaseOutcome(baselinePass, candidatePass) {
+  if (!baselinePass && candidatePass) return "IMPROVED";
+  if (baselinePass && !candidatePass) return "REGRESSED";
+  return baselinePass ? "UNCHANGED_PASS" : "UNCHANGED_FAIL";
+}
+function passRate(result) {
+  return result.total > 0 ? result.passed / result.total : 0;
+}
+function metricDelta(name15, baselineValue, candidateValue) {
+  const result = {};
+  if (baselineValue !== void 0) {
+    result[`baseline${name15}`] = baselineValue;
+  }
+  if (candidateValue !== void 0) {
+    result[`candidate${name15}`] = candidateValue;
+  }
+  if (baselineValue !== void 0 && candidateValue !== void 0) {
+    result[`delta${name15}`] = candidateValue - baselineValue;
+  }
+  return result;
+}
+function redactResponses2(value) {
+  return JSON.parse(
+    JSON.stringify(
+      value,
+      (key, currentValue) => key === "response" ? void 0 : currentValue
+    )
+  );
+}
+
+// src/evals/variantExperiment.ts
+async function runVariantExperiment(options, context) {
+  const metric = options.metric ?? "passRate";
+  const maxRounds = options.maxRounds ?? 1;
+  const minImprovement = options.minImprovement ?? 0;
+  const allowRegressions = options.allowRegressions ?? false;
+  const internalContext = {
+    mcp: context.mcp,
+    expect: context.expect
+  };
+  const baseline = await runEvalDataset(
+    buildRunOptions(options, void 0),
+    internalContext
+  );
+  const baselineValue = readMetric(baseline, metric);
+  if (baselineValue === void 0) {
+    throw new Error(
+      `Metric '${metric}' is unavailable: the dataset produced no tool precision/recall data. Add mcp_host cases with toolsTriggered expectations, or use metric 'passRate'.`
+    );
+  }
+  const rounds = [];
+  let bestSoFar;
+  let bestAttempted;
+  let reason = "max-rounds";
+  for (let round = 0; round < maxRounds; round++) {
+    const variants = await gatherVariants(options, {
+      round,
+      baseline,
+      metric,
+      history: rounds,
+      bestSoFar
+    });
+    if (variants.length === 0) {
+      reason = round === 0 ? "no-variants" : "no-improvement";
+      break;
+    }
+    const candidates = [];
+    for (const variant of variants) {
+      const candidate = await scoreVariant(
+        options,
+        internalContext,
+        baseline,
+        baselineValue,
+        metric,
+        allowRegressions,
+        variant
+      );
+      candidates.push(candidate);
+      bestAttempted = pickBetter(bestAttempted, candidate, true);
+    }
+    const roundBest = candidates.reduce(
+      (best, candidate) => pickBetter(best, candidate, false),
+      void 0
+    );
+    rounds.push({ round, candidates, best: roundBest });
+    if (roundBest) {
+      const improvement = roundBest.metricValue - (bestSoFar?.metricValue ?? baselineValue);
+      bestSoFar = pickBetter(bestSoFar, roundBest, false);
+      if (improvement < minImprovement) {
+        reason = "no-improvement";
+        break;
+      }
+    }
+  }
+  const winner = bestSoFar;
+  const proposalSource = winner ?? bestAttempted;
+  const proposal = proposalSource ? buildProposal(metric, baselineValue, proposalSource, winner !== void 0) : void 0;
+  const result = {
+    metric,
+    baseline,
+    rounds,
+    winner,
+    proposal,
+    converged: true,
+    reason
+  };
+  if (context.testInfo) {
+    const surfaceRun = winner?.result ?? bestAttempted?.result ?? baseline;
+    await context.testInfo.attach("mcp-test-results", {
+      contentType: "application/json",
+      body: Buffer.from(
+        JSON.stringify({ caseResults: surfaceRun.caseResults })
+      )
+    });
+    await context.testInfo.attach("mcp-variant-experiment", {
+      contentType: "application/json",
+      body: Buffer.from(
+        JSON.stringify(buildExperimentData(result, baselineValue))
+      )
+    });
+  }
+  return result;
+}
+function buildExperimentData(result, baselineValue) {
+  return {
+    metric: result.metric,
+    baselineValue,
+    bestValue: result.winner?.metricValue ?? result.proposal?.candidateValue ?? baselineValue,
+    rounds: result.rounds.map((round) => {
+      const best = round.best ?? round.candidates[0];
+      return {
+        round: round.round,
+        variantId: best?.variant.id ?? "(none)",
+        metricValue: best?.metricValue ?? baselineValue,
+        metricDelta: best?.metricDelta ?? 0,
+        disqualified: best?.disqualified ?? false
+      };
+    }),
+    winnerVariantId: result.winner?.variant.id,
+    recommendation: result.proposal?.recommendation,
+    reason: result.reason
+  };
+}
+async function gatherVariants(options, context) {
+  if (context.round === 0 && options.variants && options.variants.length > 0) {
+    return options.variants;
+  }
+  if (options.proposeVariants) {
+    return options.proposeVariants(context);
+  }
+  return [];
+}
+async function scoreVariant(options, context, baseline, baselineValue, metric, allowRegressions, variant) {
+  const result = await runEvalDataset(
+    buildRunOptions(options, variant),
+    context
+  );
+  const comparison = compareEvalRuns({
+    baseline,
+    candidate: result,
+    labels: { candidate: variant.id }
+  });
+  const metricValue = readMetric(result, metric) ?? baselineValue;
+  const disqualified = !allowRegressions && comparison.regressedCases.length > 0;
+  return {
+    variant,
+    result,
+    comparison,
+    metricValue,
+    metricDelta: metricValue - baselineValue,
+    disqualified
+  };
+}
+function pickBetter(incumbent, challenger, includeDisqualified) {
+  if (!includeDisqualified && challenger.disqualified) {
+    return incumbent;
+  }
+  if (!incumbent) {
+    return challenger;
+  }
+  return challenger.metricValue > incumbent.metricValue ? challenger : incumbent;
+}
+function buildProposal(metric, baselineValue, source, isWinner) {
+  let recommendation;
+  if (isWinner) {
+    recommendation = source.metricDelta > 0 ? "apply" : "inconclusive";
+  } else {
+    recommendation = source.disqualified ? "reject" : "inconclusive";
+  }
+  return {
+    variantId: source.variant.id,
+    metric,
+    baselineValue,
+    candidateValue: source.metricValue,
+    delta: source.metricDelta,
+    toolChanges: source.variant.tools,
+    improvedCaseIds: source.comparison.improvedCases.map((c) => c.id),
+    regressedCaseIds: source.comparison.regressedCases.map((c) => c.id),
+    recommendation
+  };
+}
+function readMetric(result, metric) {
+  switch (metric) {
+    case "passRate":
+      return result.total > 0 ? result.passed / result.total : 0;
+    case "toolF1":
+      return result.datasetToolF1;
+    case "toolPrecision":
+      return result.datasetToolPrecision;
+    case "toolRecall":
+      return result.datasetToolRecall;
+  }
+}
+function buildRunOptions(options, toolOverrides) {
+  return {
+    dataset: options.dataset,
+    toolOverrides,
+    defaultLlmIterations: options.defaultLlmIterations,
+    defaultJudgeReps: options.defaultJudgeReps,
+    concurrency: options.concurrency,
+    filterTags: options.filterTags,
+    schemas: options.schemas,
+    mcpHostModel: options.mcpHostModel,
+    judgeModel: options.judgeModel
+  };
 }
 
 // src/spec/conformanceChecks.ts
@@ -8105,15 +8818,22 @@ exports.DiscoveryError = DiscoveryError;
 exports.ENV_VAR_NAMES = ENV_VAR_NAMES;
 exports.EvalCaseSchema = EvalCaseSchema;
 exports.EvalDatasetSchema = EvalDatasetSchema;
+exports.FileEvalResultStore = FileEvalResultStore;
+exports.GCSEvalResultStore = GCSEvalResultStore;
 exports.MCPConfigSchema = MCPConfigSchema;
 exports.MCP_PROTOCOL_VERSION = MCP_PROTOCOL_VERSION;
 exports.SnapshotSanitizers = SnapshotSanitizers;
 exports.clearJudgeRegistry = clearJudgeRegistry;
 exports.closeMCPClient = closeMCPClient;
+exports.compareEvalRuns = compareEvalRuns;
+exports.createDefaultArtifactId = createDefaultArtifactId;
+exports.createEvalResultStore = createEvalResultStore;
 exports.createJudge = createJudge;
 exports.createMCPClientForConfig = createMCPClientForConfig;
 exports.createMCPFixture = createMCPFixture;
+exports.createStoredEvalArtifact = createStoredEvalArtifact;
 exports.createTokenAuthHeaders = createTokenAuthHeaders;
+exports.defaultEnvironmentMetadata = defaultEnvironmentMetadata;
 exports.discoverAuthorizationServer = discoverAuthorizationServer;
 exports.discoverProtectedResource = discoverProtectedResource;
 exports.expect = expect;
@@ -8124,6 +8844,7 @@ exports.getResponseSizeBytes = getResponseSizeBytes;
 exports.hasValidTokens = hasValidTokens;
 exports.injectTokens = injectTokens;
 exports.isBuiltInRubric = isBuiltInRubric;
+exports.isEvalResultStore = isEvalResultStore;
 exports.isHttpConfig = isHttpConfig;
 exports.isProviderAvailable = isProviderAvailable;
 exports.isStdioConfig = isStdioConfig;
@@ -8132,6 +8853,7 @@ exports.isTokenExpiringSoon = isTokenExpiringSoon;
 exports.loadBaseline = loadBaseline;
 exports.loadEvalDataset = loadEvalDataset;
 exports.loadEvalDatasetFromObject = loadEvalDatasetFromObject;
+exports.loadStoredEvalRunnerResult = loadStoredEvalRunnerResult;
 exports.loadTokens = loadTokens;
 exports.loadTokensFromEnv = loadTokensFromEnv;
 exports.mcpAuthTest = test2;
@@ -8142,12 +8864,16 @@ exports.performOAuthSetup = performOAuthSetup;
 exports.performOAuthSetupIfNeeded = performOAuthSetupIfNeeded;
 exports.refreshAccessToken = refreshAccessToken;
 exports.registerJudge = registerJudge;
+exports.resolveEvalResultStore = resolveEvalResultStore;
 exports.resolveRubric = resolveRubric;
 exports.runConformanceChecks = runConformanceChecks;
 exports.runEvalCase = runEvalCase;
 exports.runEvalDataset = runEvalDataset;
 exports.runServerComparison = runServerComparison;
+exports.runVariantExperiment = runVariantExperiment;
 exports.saveBaseline = saveBaseline;
+exports.saveEvalRunComparison = saveEvalRunComparison;
+exports.saveServerComparison = saveServerComparison;
 exports.simulateMCPHost = simulateMCPHost;
 exports.test = test;
 exports.validateAccessToken = validateAccessToken;