@gleanwork/mcp-server-tester 1.0.0-beta.8 → 1.0.1-beta.0

package/dist/index.cjs

@@ -4411,7 +4411,7 @@ function escapeHtml(text) {
 
 // package.json
 var package_default = {
-  version: "1.0.0-beta.8"};
+  version: "1.0.1-beta.0"};
 
 // src/mcp/clientFactory.ts
 function getRetryAfterDelayMs(err) {
@@ -4630,6 +4630,17 @@ async function createMCPClientForConfig(config, options) {
 }
 async function closeMCPClient(client) {
   try {
+    const transport = client.transport;
+    if (transport instanceof streamableHttp_js.StreamableHTTPClientTransport) {
+      try {
+        await transport.terminateSession();
+      } catch (sessionError) {
+        debugClient(
+          "Error terminating session: %s",
+          sessionError instanceof Error ? sessionError.message : String(sessionError)
+        );
+      }
+    }
     await client.close();
   } catch (error) {
     debugClient(
@@ -4858,11 +4869,13 @@ function validateSchema(response, schema, options = {}) {
   } catch (error) {
     const zodError = error;
     const issues = formatZodIssues(zodError);
+    const text = stringifyResponse(response);
     return {
       pass: false,
       message: `Response does not match schema: ${issues}`,
       details: {
-        issues: zodError.issues
+        issues: zodError.issues,
+        textPreview: truncateForDisplay2(text)
       }
     };
   }
@@ -4915,6 +4928,12 @@ function formatZodIssues(error) {
   });
   return issues.join("; ");
 }
+function truncateForDisplay2(str, maxLength = 200) {
+  if (str.length <= maxLength) {
+    return str;
+  }
+  return str.slice(0, maxLength) + "... (truncated)";
+}
 
 // src/assertions/validators/text.ts
 function validateText(response, expected, options = {}) {
@@ -4941,11 +4960,11 @@ function validateText(response, expected, options = {}) {
     details: {
       missing,
       textLength: text.length,
-      textPreview: truncateForDisplay2(text)
+      textPreview: truncateForDisplay3(text)
     }
   };
 }
-function truncateForDisplay2(str, maxLength = 200) {
+function truncateForDisplay3(str, maxLength = 200) {
   if (str.length <= maxLength) {
     return str;
   }
@@ -4977,7 +4996,7 @@ function validatePattern(response, patterns, options = {}) {
     details: {
       unmatched,
       textLength: text.length,
-      textPreview: truncateForDisplay3(text)
+      textPreview: truncateForDisplay4(text)
     }
   };
 }
@@ -4997,7 +5016,7 @@ function patternToString(pattern) {
   }
   return `/${pattern}/`;
 }
-function truncateForDisplay3(str, maxLength = 200) {
+function truncateForDisplay4(str, maxLength = 200) {
   if (str.length <= maxLength) {
     return str;
   }
@@ -5020,7 +5039,7 @@ function validateError(response, expected = true) {
         pass: false,
         message: "Expected an error response but got success",
         details: {
-          textPreview: truncateForDisplay4(extractText2(response))
+          textPreview: truncateForDisplay5(extractText2(response))
         }
       };
     } else {
@@ -5032,7 +5051,7 @@ function validateError(response, expected = true) {
       }
       return {
         pass: false,
-        message: `Expected a success response but got error: "${truncateForDisplay4(errorMessage)}"`,
+        message: `Expected a success response but got error: "${truncateForDisplay5(errorMessage)}"`,
         details: {
           errorMessage
         }
@@ -5045,7 +5064,7 @@ function validateError(response, expected = true) {
       pass: false,
       message: `Expected an error containing "${expectedMessages[0]}" but got success`,
       details: {
-        textPreview: truncateForDisplay4(extractText2(response))
+        textPreview: truncateForDisplay5(extractText2(response))
       }
     };
   }
@@ -5067,7 +5086,7 @@ function validateError(response, expected = true) {
     }
   };
 }
-function truncateForDisplay4(str, maxLength = 200) {
+function truncateForDisplay5(str, maxLength = 200) {
   if (str.length <= maxLength) {
     return str;
   }
@@ -5185,6 +5204,10 @@ function validateToolCalls(response, expectation) {
           return {
             pass: false,
             message: `Expected tool '${expected.name}' to be called in sequence (starting from position ${searchFrom}), but it was not found`,
+            details: {
+              actual: actual.map((c) => c.name),
+              expected: expected.name
+            },
             metrics
           };
         }
@@ -5201,6 +5224,10 @@ function validateToolCalls(response, expectation) {
         return {
           pass: false,
           message: `Expected tool '${expected.name}'${argsNote} to be called, but it was not`,
+          details: {
+            actual: actual.map((c) => c.name),
+            expected: expected.name
+          },
           metrics
         };
       }
@@ -5213,6 +5240,10 @@ function validateToolCalls(response, expectation) {
       return {
         pass: false,
         message: `Unexpected tool calls: ${names}. Only ${[...allowedNames].map((n) => `'${n}'`).join(", ")} are allowed`,
+        details: {
+          actual: actual.map((c) => c.name),
+          unexpected: unexpected.map((c) => c.name)
+        },
         metrics
       };
     }
@@ -5231,19 +5262,22 @@ function validateToolCallCount(response, options) {
   if (exact !== void 0 && count !== exact) {
     return {
       pass: false,
-      message: `Expected exactly ${exact} tool call(s), but got ${count}`
+      message: `Expected exactly ${exact} tool call(s), but got ${count}`,
+      details: { actual: count, expected: exact }
     };
   }
   if (min !== void 0 && count < min) {
     return {
       pass: false,
-      message: `Expected at least ${min} tool call(s), but got ${count}`
+      message: `Expected at least ${min} tool call(s), but got ${count}`,
+      details: { actual: count, min }
     };
   }
   if (max !== void 0 && count > max) {
     return {
       pass: false,
-      message: `Expected at most ${max} tool call(s), but got ${count}`
+      message: `Expected at most ${max} tool call(s), but got ${count}`,
+      details: { actual: count, max }
     };
   }
   return {
@@ -5757,7 +5791,9 @@ function createJudge(config = {}) {
     case "google":
       return createGoogleJudge(config);
     default:
-      throw new Error(`Unsupported LLM provider: ${String(provider)}`);
+      throw new Error(
+        `Unsupported LLM provider: ${String(provider)}. Valid providers: 'anthropic', 'vertex-anthropic', 'anthropic-agent-sdk', 'openai', 'google'`
+      );
   }
 }
 
@@ -6082,12 +6118,19 @@ function toMatchToolResponse(received, expected) {
 // src/assertions/matchers/toMatchToolSchema.ts
 function toMatchToolSchema(received, schema, options = {}) {
   const result = validateSchema(received, schema, options);
+  const preview = result.details?.textPreview;
   return {
     pass: result.pass,
     message: () => {
       if (this.isNot) {
         return result.pass ? "Expected response NOT to match schema, but it did" : result.message;
       }
+      if (!result.pass && preview) {
+        return `${result.message}
+
+Actual response (truncated):
+${preview}`;
+      }
       return result.message;
     }
   };
@@ -6096,6 +6139,7 @@ function toMatchToolSchema(received, schema, options = {}) {
 // src/assertions/matchers/toContainToolText.ts
 function toContainToolText(received, expected, options = {}) {
   const result = validateText(received, expected, options);
+  const preview = result.details?.textPreview;
   return {
     pass: result.pass,
     message: () => {
@@ -6103,6 +6147,12 @@ function toContainToolText(received, expected, options = {}) {
         const expectedStr = Array.isArray(expected) ? expected.map((s) => `"${s}"`).join(", ") : `"${expected}"`;
         return result.pass ? `Expected response NOT to contain ${expectedStr}, but it did` : result.message;
       }
+      if (!result.pass && preview) {
+        return `${result.message}
+
+Actual response (truncated):
+${preview}`;
+      }
       return result.message;
     }
   };
@@ -6111,12 +6161,19 @@ function toContainToolText(received, expected, options = {}) {
 // src/assertions/matchers/toMatchToolPattern.ts
 function toMatchToolPattern(received, patterns, options = {}) {
   const result = validatePattern(received, patterns, options);
+  const preview = result.details?.textPreview;
   return {
     pass: result.pass,
     message: () => {
       if (this.isNot) {
         return result.pass ? "Expected response NOT to match pattern(s), but it did" : result.message;
       }
+      if (!result.pass && preview) {
+        return `${result.message}
+
+Actual response (truncated):
+${preview}`;
+      }
       return result.message;
     }
   };
@@ -6901,6 +6958,12 @@ function createVercelOrchestrator() {
         });
         const totalDurationMs = Date.now() - llmStart;
         const llmDurationMs = totalDurationMs - mcpDurationMs;
+        const hostUsage = result.usage ? {
+          inputTokens: result.usage.promptTokens ?? 0,
+          outputTokens: result.usage.completionTokens ?? 0,
+          totalCostUsd: 0,
+          durationMs: llmDurationMs
+        } : void 0;
         const conversationHistory = (result.steps ?? []).map((step) => ({
           role: step.toolCalls?.length > 0 ? "tool" : "assistant",
           content: step.toolCalls?.length > 0 ? JSON.stringify(step.toolResults) : step.text ?? ""
@@ -6912,7 +6975,8 @@ function createVercelOrchestrator() {
           scenario,
           llmDurationMs,
           mcpDurationMs,
-          conversationHistory
+          conversationHistory,
+          usage: hostUsage
         };
       } catch (err) {
         return {
@@ -6930,6 +6994,7 @@ function parseStreamJson(stdout) {
   const lines = stdout.split("\n").filter((line) => line.trim().length > 0);
   const toolCalls = [];
   const textParts = [];
+  let usage;
   const conversationHistory = [];
   for (const line of lines) {
     let event;
@@ -6962,16 +7027,28 @@ function parseStreamJson(stdout) {
         }
       }
     }
-    if (event.type === "result" && typeof event.result === "string") {
-      if (textParts.length === 0) {
+    if (event.type === "result") {
+      if (typeof event.result === "string" && textParts.length === 0) {
         textParts.push(event.result);
       }
+      if (event.usage) {
+        usage = {
+          inputTokens: event.usage.input_tokens ?? 0,
+          outputTokens: event.usage.output_tokens ?? 0,
+          totalCostUsd: event.total_cost_usd ?? 0,
+          durationMs: event.duration_ms ?? 0,
+          durationApiMs: event.duration_api_ms,
+          cacheReadInputTokens: event.usage.cache_read_input_tokens,
+          cacheCreationInputTokens: event.usage.cache_creation_input_tokens
+        };
+      }
     }
     if (event.type === "result" && event.is_error === true) {
       return {
         success: false,
         toolCalls,
-        error: typeof event.result === "string" ? event.result : "CLI host reported an error"
+        error: typeof event.result === "string" ? event.result : "CLI host reported an error",
+        usage
       };
     }
   }
@@ -6983,7 +7060,8 @@ function parseStreamJson(stdout) {
     success: true,
     toolCalls,
     response: response || void 0,
-    conversationHistory: conversationHistory.length > 0 ? conversationHistory : void 0
+    conversationHistory: conversationHistory.length > 0 ? conversationHistory : void 0,
+    usage
   };
 }
 function createJsonParser(paths) {
@@ -7248,6 +7326,32 @@ async function execFileNoThrow(file, args) {
   }
 }
 
+// src/utils/usageUtils.ts
+function optionalSum(a, b) {
+  if (a === void 0 && b === void 0) return void 0;
+  return (a ?? 0) + (b ?? 0);
+}
+function sumUsage(a, b) {
+  if (!a && !b) return void 0;
+  if (!a) return b ? { ...b } : void 0;
+  if (!b) return { ...a };
+  return {
+    inputTokens: a.inputTokens + b.inputTokens,
+    outputTokens: a.outputTokens + b.outputTokens,
+    totalCostUsd: a.totalCostUsd + b.totalCostUsd,
+    durationMs: a.durationMs + b.durationMs,
+    durationApiMs: optionalSum(a.durationApiMs, b.durationApiMs),
+    cacheReadInputTokens: optionalSum(
+      a.cacheReadInputTokens,
+      b.cacheReadInputTokens
+    ),
+    cacheCreationInputTokens: optionalSum(
+      a.cacheCreationInputTokens,
+      b.cacheCreationInputTokens
+    )
+  };
+}
+
 // src/evals/evalRunner.ts
 async function executeToolCall(evalCase, mcp) {
   const mode = evalCase.mode || "direct";
@@ -7493,6 +7597,7 @@ async function runSingleIteration(evalCase, context, options) {
       };
     }
   }
+  const hostUsage = isMCPHostSimulationResult(response) && response.usage ? response.usage : void 0;
   return {
     id: evalCase.id,
     datasetName: options.datasetName ?? "single-case",
@@ -7509,7 +7614,8 @@ async function runSingleIteration(evalCase, context, options) {
     tags: evalCase.tags,
     toolPrecision,
     toolRecall,
-    mcpHostTrace
+    mcpHostTrace,
+    hostUsage
   };
 }
 function isInfrastructureError(err) {
@@ -7525,7 +7631,7 @@ function isInfrastructureError(err) {
   } else {
     return false;
   }
-  return name15 === "AbortError" || msg.includes("econnreset") || msg.includes("etimedout") || msg.includes("econnrefused") || msg.includes("rate limit") || msg.includes("429") || msg.includes("503") || msg.includes("network") || // Prompt/context overflow — LLM couldn't run, not a tool discoverability failure
+  return name15?.toLowerCase() === "aborterror" || msg.includes("econnreset") || msg.includes("etimedout") || msg.includes("econnrefused") || msg.includes("rate limit") || msg.includes("429") || msg.includes("503") || msg.includes("network") || // Prompt/context overflow — LLM couldn't run, not a tool discoverability failure
   msg.includes("prompt is too long") || msg.includes("context length exceeded") || msg.includes("maximum context length") || msg.includes("context_length_exceeded") || msg.includes("tokens > ") || code.includes("econnreset") || code.includes("etimedout") || code.includes("econnrefused");
 }
 async function runEvalCase(evalCase, context, options = {}) {
@@ -7545,7 +7651,8 @@ async function runEvalCase(evalCase, context, options = {}) {
         durationMs: result.durationMs,
         error: result.error,
         isInfrastructureError: infraError,
-        mcpHostTrace: result.mcpHostTrace
+        mcpHostTrace: result.mcpHostTrace,
+        hostUsage: result.hostUsage
       });
     } catch (err) {
       const errorMessage = err instanceof Error ? err.message : String(err);
@@ -7578,6 +7685,10 @@ async function runEvalCase(evalCase, context, options = {}) {
     durationMs: 0,
     tags: evalCase.tags
   };
+  const totalHostUsage = iterationResults.reduce(
+    (acc, r) => sumUsage(acc, r.hostUsage),
+    void 0
+  );
   return {
     ...baseResult,
     pass: assertionPassRate >= threshold,
@@ -7586,7 +7697,8 @@ async function runEvalCase(evalCase, context, options = {}) {
     infrastructureErrorRate,
     iterationResults,
     infrastructureErrorCount: infraErrors.length,
-    durationMs: iterationResults.reduce((sum, r) => sum + r.durationMs, 0)
+    durationMs: iterationResults.reduce((sum, r) => sum + r.durationMs, 0),
+    hostUsage: totalHostUsage
   };
 }
 function wilsonCI(k, n) {
@@ -7696,13 +7808,18 @@ async function runEvalDataset(options, context) {
     ...mcpHostModel !== void 0 && { mcpHostModel },
     ...judgeModel !== void 0 && { judgeModel }
   };
+  const runHostUsage = caseResults.reduce(
+    (acc, r) => sumUsage(acc, r.hostUsage),
+    void 0
+  );
   const result = {
     total,
     passed,
     failed: total - passed,
     caseResults,
     durationMs: Date.now() - startTime,
-    metadata
+    metadata,
+    totalHostUsage: runHostUsage
   };
   if (baselineResultsFrom) {
     try {
@@ -8037,6 +8154,7 @@ exports.normalizeWhitespace = normalizeWhitespace;
 exports.performClientCredentialsFlow = performClientCredentialsFlow;
 exports.performOAuthSetup = performOAuthSetup;
 exports.performOAuthSetupIfNeeded = performOAuthSetupIfNeeded;
+exports.refreshAccessToken = refreshAccessToken;
 exports.registerJudge = registerJudge;
 exports.resolveRubric = resolveRubric;
 exports.runConformanceChecks = runConformanceChecks;