@gleanwork/mcp-server-tester 1.0.0-beta.4 → 1.0.0-beta.6

package/dist/index.cjs

@@ -4407,7 +4407,7 @@ function escapeHtml(text) {
 
 // package.json
 var package_default = {
-  version: "1.0.0-beta.4"};
+  version: "1.0.0-beta.6"};
 
 // src/mcp/clientFactory.ts
 function getRetryAfterDelayMs(err) {
@@ -6669,9 +6669,16 @@ function getMissingDependencyMessage(provider) {
   const pkg = packageMap[provider];
   return pkg ? `${String(provider)} provider requires: ${pkg}` : `Unknown provider: ${String(provider)}`;
 }
-async function saveBaseline(result, filePath) {
+async function saveBaseline(result, filePath, options = {}) {
+  const { omitResponses = true } = options;
+  const toSave = omitResponses ? {
+    ...result,
+    caseResults: result.caseResults.map(
+      ({ response: _response, ...rest }) => rest
+    )
+  } : result;
   await fs$1.mkdir(path2.dirname(filePath), { recursive: true });
-  await fs$1.writeFile(filePath, JSON.stringify(result, null, 2), "utf8");
+  await fs$1.writeFile(filePath, JSON.stringify(toSave, null, 2), "utf8");
 }
 async function loadBaseline(filePath) {
   const raw = await fs$1.readFile(filePath, "utf8");
@@ -6911,7 +6918,7 @@ async function runSingleIteration(evalCase, context, options) {
   return {
     id: evalCase.id,
     datasetName: options.datasetName ?? "single-case",
-    toolName: evalCase.toolName ?? evalCase.scenario ?? "unknown",
+    toolName: evalCase.scenario != null ? "mcp_host" : evalCase.toolName ?? "unknown",
     source: "eval",
     pass: didCasePass(error, expectationResults),
     response,
@@ -6939,7 +6946,8 @@ function isInfrastructureError(err) {
   } else {
     return false;
   }
-  return name15 === "AbortError" || msg.includes("econnreset") || msg.includes("etimedout") || msg.includes("econnrefused") || msg.includes("rate limit") || msg.includes("429") || msg.includes("503") || msg.includes("network") || code.includes("econnreset") || code.includes("etimedout") || code.includes("econnrefused");
+  return name15 === "AbortError" || msg.includes("econnreset") || msg.includes("etimedout") || msg.includes("econnrefused") || msg.includes("rate limit") || msg.includes("429") || msg.includes("503") || msg.includes("network") || // Prompt/context overflow — LLM couldn't run, not a tool discoverability failure
+  msg.includes("prompt is too long") || msg.includes("context length exceeded") || msg.includes("maximum context length") || msg.includes("context_length_exceeded") || msg.includes("tokens > ") || code.includes("econnreset") || code.includes("etimedout") || code.includes("econnrefused");
 }
 async function runEvalCase(evalCase, context, options = {}) {
   const iterations = evalCase.iterations ?? 1;
@@ -6957,7 +6965,8 @@ async function runEvalCase(evalCase, context, options = {}) {
         pass: result.pass,
         durationMs: result.durationMs,
         error: result.error,
-        isInfrastructureError: infraError
+        isInfrastructureError: infraError,
+        mcpHostTrace: result.mcpHostTrace
       });
     } catch (err) {
       const errorMessage = err instanceof Error ? err.message : String(err);
@@ -6980,7 +6989,7 @@ async function runEvalCase(evalCase, context, options = {}) {
   const baseResult = lastResult ?? {
     id: evalCase.id,
     datasetName: options.datasetName ?? "single-case",
-    toolName: evalCase.toolName ?? evalCase.scenario ?? "unknown",
+    toolName: evalCase.scenario != null ? "mcp_host" : evalCase.toolName ?? "unknown",
     source: "eval",
     pass: false,
     error: iterationResults[0]?.error,
@@ -6994,12 +7003,25 @@ async function runEvalCase(evalCase, context, options = {}) {
     ...baseResult,
     pass: assertionPassRate >= threshold,
     assertionPassRate,
+    assertionPassRateCI: wilsonCI(passCount, assertionResults.length),
     infrastructureErrorRate,
     iterationResults,
     infrastructureErrorCount: infraErrors.length,
     durationMs: iterationResults.reduce((sum, r) => sum + r.durationMs, 0)
   };
 }
+function wilsonCI(k, n) {
+  if (n < 2) return void 0;
+  const z5 = 1.96;
+  const z22 = z5 * z5;
+  const \u00F1 = n + z22;
+  const p\u0303 = (k + z22 / 2) / \u00F1;
+  const margin = z5 * Math.sqrt(p\u0303 * (1 - p\u0303) / \u00F1);
+  return {
+    lower: Math.max(0, p\u0303 - margin),
+    upper: Math.min(1, p\u0303 + margin)
+  };
+}
 async function runWithConcurrency(tasks, limit) {
   const results = new Array(tasks.length);
   let index = 0;
@@ -7028,6 +7050,7 @@ async function runEvalDataset(options, context) {
     onCaseComplete,
     filterTags,
     saveResultsTo,
+    omitResponsesFromBaseline = true,
     baselineResultsFrom,
     mcpHostModel,
     judgeModel
@@ -7142,7 +7165,9 @@ async function runEvalDataset(options, context) {
     result.datasetToolF1 = avgPrec + avgRecall > 0 ? 2 * avgPrec * avgRecall / (avgPrec + avgRecall) : 0;
   }
   if (saveResultsTo) {
-    await saveBaseline(result, saveResultsTo);
+    await saveBaseline(result, saveResultsTo, {
+      omitResponses: omitResponsesFromBaseline
+    });
   }
   if (context.testInfo) {
     await context.testInfo.attach("mcp-test-results", {