@gleanwork/mcp-server-tester 1.1.0 → 1.1.1

package/dist/{index-BcUWzQCx.d.cts → index-BlqLvjnP.d.cts}

@@ -2087,6 +2087,8 @@ interface LLMToolCall {
     arguments: Record<string, unknown>;
     /** Optional tool call ID (for tracking) */
     id?: string;
+    /** Tool result text, when the host surfaces it (paired to this call) */
+    output?: string;
 }
 /**
  * Result from an MCP host simulation
@@ -2102,10 +2104,17 @@ interface MCPHostSimulationResult {
     error?: string;
     /** The scenario prompt that was given to the LLM */
     scenario?: string;
-    /** The conversation turns for attribution analysis */
+    /**
+     * The conversation turns for attribution analysis.
+     *
+     * Tool turns reference their call via `toolCallId` rather than inlining the
+     * (potentially large) result — hydrate the output from the matching
+     * `toolCalls[]` entry. `content` holds assistant/user text.
+     */
     conversationHistory?: Array<{
         role: 'user' | 'assistant' | 'tool';
-        content: string;
+        content?: string;
+        toolCallId?: string;
     }>;
     /**
      * Milliseconds spent waiting for LLM responses
@@ -3272,6 +3281,37 @@ interface MCPEvalRunData {
      * Server capabilities discovered via listTools (optional)
      */
     serverCapabilities?: MCPServerCapabilitiesData[];
+    /**
+     * Summary of a tool-metadata variant experiment (runVariantExperiment),
+     * present when the run was produced by one. The `results` above reflect the
+     * winning variant; this records how the experiment got there.
+     */
+    variantExperiment?: MCPVariantExperimentData;
+}
+/**
+ * Compact summary of a `runVariantExperiment` run, for the reporter UI.
+ */
+interface MCPVariantExperimentData {
+    /** Metric optimized: passRate | toolF1 | toolPrecision | toolRecall. */
+    metric: string;
+    /** Baseline metric value (0-1), before any variant. */
+    baselineValue: number;
+    /** Best metric value achieved (the winner, or best attempt) (0-1). */
+    bestValue: number;
+    /** The best candidate from each round, in order. */
+    rounds: Array<{
+        round: number;
+        variantId: string;
+        metricValue: number;
+        metricDelta: number;
+        disqualified: boolean;
+    }>;
+    /** Winning variant id, if a non-regressing candidate beat the baseline. */
+    winnerVariantId?: string;
+    /** apply | reject | inconclusive */
+    recommendation?: string;
+    /** Why the experiment stopped. */
+    reason: string;
 }
 /**
  * Historical summary for trend charts

package/dist/{index-BcUWzQCx.d.ts → index-BlqLvjnP.d.ts}

@@ -2087,6 +2087,8 @@ interface LLMToolCall {
     arguments: Record<string, unknown>;
     /** Optional tool call ID (for tracking) */
     id?: string;
+    /** Tool result text, when the host surfaces it (paired to this call) */
+    output?: string;
 }
 /**
  * Result from an MCP host simulation
@@ -2102,10 +2104,17 @@ interface MCPHostSimulationResult {
     error?: string;
     /** The scenario prompt that was given to the LLM */
     scenario?: string;
-    /** The conversation turns for attribution analysis */
+    /**
+     * The conversation turns for attribution analysis.
+     *
+     * Tool turns reference their call via `toolCallId` rather than inlining the
+     * (potentially large) result — hydrate the output from the matching
+     * `toolCalls[]` entry. `content` holds assistant/user text.
+     */
     conversationHistory?: Array<{
         role: 'user' | 'assistant' | 'tool';
-        content: string;
+        content?: string;
+        toolCallId?: string;
     }>;
     /**
      * Milliseconds spent waiting for LLM responses
@@ -3272,6 +3281,37 @@ interface MCPEvalRunData {
      * Server capabilities discovered via listTools (optional)
      */
     serverCapabilities?: MCPServerCapabilitiesData[];
+    /**
+     * Summary of a tool-metadata variant experiment (runVariantExperiment),
+     * present when the run was produced by one. The `results` above reflect the
+     * winning variant; this records how the experiment got there.
+     */
+    variantExperiment?: MCPVariantExperimentData;
+}
+/**
+ * Compact summary of a `runVariantExperiment` run, for the reporter UI.
+ */
+interface MCPVariantExperimentData {
+    /** Metric optimized: passRate | toolF1 | toolPrecision | toolRecall. */
+    metric: string;
+    /** Baseline metric value (0-1), before any variant. */
+    baselineValue: number;
+    /** Best metric value achieved (the winner, or best attempt) (0-1). */
+    bestValue: number;
+    /** The best candidate from each round, in order. */
+    rounds: Array<{
+        round: number;
+        variantId: string;
+        metricValue: number;
+        metricDelta: number;
+        disqualified: boolean;
+    }>;
+    /** Winning variant id, if a non-regressing candidate beat the baseline. */
+    winnerVariantId?: string;
+    /** apply | reject | inconclusive */
+    recommendation?: string;
+    /** Why the experiment stopped. */
+    reason: string;
 }
 /**
  * Historical summary for trend charts

package/dist/index.cjs

@@ -4425,7 +4425,7 @@ function escapeHtml(text) {
 
 // package.json
 var package_default = {
-  version: "1.1.0"};
+  version: "1.1.1"};
 
 // src/mcp/clientFactory.ts
 function getRetryAfterDelayMs(err) {
@@ -6838,8 +6838,8 @@ Hint: run \`getMissingDependencyMessage('${provider}')\` or check docs/mcp-host.
 Hint: check your API key environment variable (e.g. ANTHROPIC_API_KEY, GOOGLE_APPLICATION_CREDENTIALS).`;
   }
   if (raw.includes("404") || raw.includes("Not Found") || raw.toLowerCase().includes("model") && raw.toLowerCase().includes("not found")) {
-    return `MCP host simulation failed: model not found.
-Hint: check the model name format for your provider. For vertex-anthropic use 'claude-3-5-haiku@20241022' (with @).`;
+    return `MCP host simulation failed: ${raw}
+Hint: a 404 usually means the model id is wrong or retired, or a base-URL override (e.g. ANTHROPIC_BASE_URL / OPENAI_BASE_URL pointing at a gateway) is routing requests somewhere that doesn't serve this model. Verify the model id and that no unexpected *_BASE_URL is set.`;
   }
   if (raw.includes("ENOTFOUND") || raw.includes("fetch failed") || raw.includes("ECONNREFUSED")) {
     return `MCP host simulation failed: network error.
@@ -6937,12 +6937,18 @@ function createVercelOrchestrator() {
           tools[toolName] = {
             description: mcpTool.description ?? "",
             inputSchema: jsonSchema2(rawSchema),
-            execute: async (args) => {
+            execute: async (args, opts) => {
               const mcpStart = Date.now();
               const result2 = await mcp.callTool(toolName, args);
               mcpDurationMs += Date.now() - mcpStart;
-              allToolCalls.push({ name: toolName, arguments: args });
-              return extractText(result2);
+              const output = extractText(result2);
+              allToolCalls.push({
+                id: opts?.toolCallId,
+                name: toolName,
+                arguments: args,
+                output
+              });
+              return output;
             }
           };
         }
@@ -6964,10 +6970,17 @@ function createVercelOrchestrator() {
           totalCostUsd: 0,
           durationMs: llmDurationMs
         } : void 0;
-        const conversationHistory = (result.steps ?? []).map((step) => ({
-          role: step.toolCalls?.length > 0 ? "tool" : "assistant",
-          content: step.toolCalls?.length > 0 ? JSON.stringify(step.toolResults) : step.text ?? ""
-        }));
+        const conversationHistory = (result.steps ?? []).flatMap((step) => {
+          if (step.toolCalls?.length > 0) {
+            return step.toolCalls.map(
+              (tc) => ({
+                role: "tool",
+                toolCallId: tc.toolCallId
+              })
+            );
+          }
+          return step.text ? [{ role: "assistant", content: step.text }] : [];
+        });
         return {
           success: true,
           toolCalls: allToolCalls,
@@ -7023,7 +7036,13 @@ function parseStreamJson(stdout) {
       for (const block of event.message.content) {
         if (block.type === "tool_result") {
           const content = typeof block.content === "string" ? block.content : JSON.stringify(block.content);
-          conversationHistory.push({ role: "tool", content });
+          const call = block.tool_use_id ? toolCalls.find((tc) => tc.id === block.tool_use_id) : void 0;
+          if (call) {
+            call.output = content;
+            conversationHistory.push({ role: "tool", toolCallId: call.id });
+          } else {
+            conversationHistory.push({ role: "tool", content });
+          }
         }
       }
     }
@@ -7979,6 +7998,7 @@ async function getGitHash() {
   const result = await execFileNoThrow("git", ["rev-parse", "HEAD"]);
   return result.status === 0 ? result.stdout.trim() : void 0;
 }
+var warnedNoTestInfo = false;
 async function runEvalDataset(options, context) {
   const {
     dataset,
@@ -8149,7 +8169,8 @@ async function runEvalDataset(options, context) {
       contentType: "application/json",
       body: Buffer.from(JSON.stringify({ caseResults }))
     });
-  } else if (caseResults.length > 0) {
+  } else if (caseResults.length > 0 && !warnedNoTestInfo) {
+    warnedNoTestInfo = true;
     console.warn(
       "[mcp-server-tester] runEvalDataset: testInfo not provided \u2014 results will not appear in the MCP reporter.\nTo enable reporting, pass testInfo from the Playwright test function:\n  await runEvalDataset({ dataset }, { mcp, testInfo });"
     );
@@ -8429,9 +8450,13 @@ async function runVariantExperiment(options, context) {
   const maxRounds = options.maxRounds ?? 1;
   const minImprovement = options.minImprovement ?? 0;
   const allowRegressions = options.allowRegressions ?? false;
+  const internalContext = {
+    mcp: context.mcp,
+    expect: context.expect
+  };
   const baseline = await runEvalDataset(
     buildRunOptions(options, void 0),
-    context
+    internalContext
   );
   const baselineValue = readMetric(baseline, metric);
   if (baselineValue === void 0) {
@@ -8459,7 +8484,7 @@ async function runVariantExperiment(options, context) {
     for (const variant of variants) {
       const candidate = await scoreVariant(
         options,
-        context,
+        internalContext,
         baseline,
         baselineValue,
         metric,
@@ -8486,7 +8511,7 @@ async function runVariantExperiment(options, context) {
   const winner = bestSoFar;
   const proposalSource = winner ?? bestAttempted;
   const proposal = proposalSource ? buildProposal(metric, baselineValue, proposalSource, winner !== void 0) : void 0;
-  return {
+  const result = {
     metric,
     baseline,
     rounds,
@@ -8495,6 +8520,42 @@ async function runVariantExperiment(options, context) {
     converged: true,
     reason
   };
+  if (context.testInfo) {
+    const surfaceRun = winner?.result ?? bestAttempted?.result ?? baseline;
+    await context.testInfo.attach("mcp-test-results", {
+      contentType: "application/json",
+      body: Buffer.from(
+        JSON.stringify({ caseResults: surfaceRun.caseResults })
+      )
+    });
+    await context.testInfo.attach("mcp-variant-experiment", {
+      contentType: "application/json",
+      body: Buffer.from(
+        JSON.stringify(buildExperimentData(result, baselineValue))
+      )
+    });
+  }
+  return result;
+}
+function buildExperimentData(result, baselineValue) {
+  return {
+    metric: result.metric,
+    baselineValue,
+    bestValue: result.winner?.metricValue ?? result.proposal?.candidateValue ?? baselineValue,
+    rounds: result.rounds.map((round) => {
+      const best = round.best ?? round.candidates[0];
+      return {
+        round: round.round,
+        variantId: best?.variant.id ?? "(none)",
+        metricValue: best?.metricValue ?? baselineValue,
+        metricDelta: best?.metricDelta ?? 0,
+        disqualified: best?.disqualified ?? false
+      };
+    }),
+    winnerVariantId: result.winner?.variant.id,
+    recommendation: result.proposal?.recommendation,
+    reason: result.reason
+  };
 }
 async function gatherVariants(options, context) {
   if (context.round === 0 && options.variants && options.variants.length > 0) {