@gleanwork/mcp-server-tester 1.0.0-beta.5 → 1.0.0-beta.6

package/dist/index.cjs

@@ -4407,7 +4407,7 @@ function escapeHtml(text) {
 
 // package.json
 var package_default = {
-  version: "1.0.0-beta.5"};
+  version: "1.0.0-beta.6"};
 
 // src/mcp/clientFactory.ts
 function getRetryAfterDelayMs(err) {
@@ -6669,9 +6669,16 @@ function getMissingDependencyMessage(provider) {
   const pkg = packageMap[provider];
   return pkg ? `${String(provider)} provider requires: ${pkg}` : `Unknown provider: ${String(provider)}`;
 }
-async function saveBaseline(result, filePath) {
+async function saveBaseline(result, filePath, options = {}) {
+  const { omitResponses = true } = options;
+  const toSave = omitResponses ? {
+    ...result,
+    caseResults: result.caseResults.map(
+      ({ response: _response, ...rest }) => rest
+    )
+  } : result;
   await fs$1.mkdir(path2.dirname(filePath), { recursive: true });
-  await fs$1.writeFile(filePath, JSON.stringify(result, null, 2), "utf8");
+  await fs$1.writeFile(filePath, JSON.stringify(toSave, null, 2), "utf8");
 }
 async function loadBaseline(filePath) {
   const raw = await fs$1.readFile(filePath, "utf8");
@@ -6939,7 +6946,8 @@ function isInfrastructureError(err) {
   } else {
     return false;
   }
-  return name15 === "AbortError" || msg.includes("econnreset") || msg.includes("etimedout") || msg.includes("econnrefused") || msg.includes("rate limit") || msg.includes("429") || msg.includes("503") || msg.includes("network") || code.includes("econnreset") || code.includes("etimedout") || code.includes("econnrefused");
+  return name15 === "AbortError" || msg.includes("econnreset") || msg.includes("etimedout") || msg.includes("econnrefused") || msg.includes("rate limit") || msg.includes("429") || msg.includes("503") || msg.includes("network") || // Prompt/context overflow — LLM couldn't run, not a tool discoverability failure
+  msg.includes("prompt is too long") || msg.includes("context length exceeded") || msg.includes("maximum context length") || msg.includes("context_length_exceeded") || msg.includes("tokens > ") || code.includes("econnreset") || code.includes("etimedout") || code.includes("econnrefused");
 }
 async function runEvalCase(evalCase, context, options = {}) {
   const iterations = evalCase.iterations ?? 1;
@@ -6957,7 +6965,8 @@ async function runEvalCase(evalCase, context, options = {}) {
         pass: result.pass,
         durationMs: result.durationMs,
         error: result.error,
-        isInfrastructureError: infraError
+        isInfrastructureError: infraError,
+        mcpHostTrace: result.mcpHostTrace
       });
     } catch (err) {
       const errorMessage = err instanceof Error ? err.message : String(err);
@@ -7041,6 +7050,7 @@ async function runEvalDataset(options, context) {
     onCaseComplete,
     filterTags,
     saveResultsTo,
+    omitResponsesFromBaseline = true,
     baselineResultsFrom,
     mcpHostModel,
     judgeModel
@@ -7155,7 +7165,9 @@ async function runEvalDataset(options, context) {
     result.datasetToolF1 = avgPrec + avgRecall > 0 ? 2 * avgPrec * avgRecall / (avgPrec + avgRecall) : 0;
   }
   if (saveResultsTo) {
-    await saveBaseline(result, saveResultsTo);
+    await saveBaseline(result, saveResultsTo, {
+      omitResponses: omitResponsesFromBaseline
+    });
   }
   if (context.testInfo) {
     await context.testInfo.attach("mcp-test-results", {