@gleanwork/mcp-server-tester 1.0.0-beta.1 → 1.0.0-beta.3

package/dist/index.d.cts

@@ -2486,6 +2486,14 @@ declare function toMatchToolPattern(this: {
 /**
  * Creates the toMatchToolSnapshot matcher function
  *
+ * @remarks
+ * **Requires Playwright test context.** This matcher calls `expect(content).toMatchSnapshot()`
+ * internally, which only works inside a Playwright test (i.e., when `testInfo` is available).
+ * Calling it outside a Playwright test will throw a cryptic context error.
+ *
+ * To test sanitizer logic without a Playwright context, use the exported `applySanitizers`
+ * function directly.
+ *
  * Note: This is an async matcher that uses Playwright's snapshot testing.
  */
 declare function toMatchToolSnapshot(this: {
@@ -2896,7 +2904,7 @@ interface EvalCase {
     metadata?: Record<string, unknown>;
     /**
      * Number of times to run this case and compute an accuracy score.
-     * When > 1, `EvalCaseResult.accuracy` is populated and `pass` is determined
+     * When > 1, `EvalCaseResult.assertionPassRate` is populated and `pass` is determined
      * by `accuracyThreshold` rather than a single run.
      * @default 1
      */
@@ -4282,12 +4290,6 @@ interface EvalCaseResult {
      * Only present when the case was run with `iterations > 1`.
      */
     infrastructureErrorRate?: number;
-    /**
-     * Accuracy score (0–1) across all iterations.
-     * Alias for `assertionPassRate`. Only present when the case was run with `iterations > 1`.
-     * @deprecated Use `assertionPassRate` for clarity; this field is kept for backward compatibility.
-     */
-    accuracy?: number;
     /**
      * Per-iteration pass/fail breakdown.
      * Only present when the case was run with `iterations > 1`.

package/dist/index.d.ts

@@ -2486,6 +2486,14 @@ declare function toMatchToolPattern(this: {
 /**
  * Creates the toMatchToolSnapshot matcher function
  *
+ * @remarks
+ * **Requires Playwright test context.** This matcher calls `expect(content).toMatchSnapshot()`
+ * internally, which only works inside a Playwright test (i.e., when `testInfo` is available).
+ * Calling it outside a Playwright test will throw a cryptic context error.
+ *
+ * To test sanitizer logic without a Playwright context, use the exported `applySanitizers`
+ * function directly.
+ *
  * Note: This is an async matcher that uses Playwright's snapshot testing.
  */
 declare function toMatchToolSnapshot(this: {
@@ -2896,7 +2904,7 @@ interface EvalCase {
     metadata?: Record<string, unknown>;
     /**
      * Number of times to run this case and compute an accuracy score.
-     * When > 1, `EvalCaseResult.accuracy` is populated and `pass` is determined
+     * When > 1, `EvalCaseResult.assertionPassRate` is populated and `pass` is determined
      * by `accuracyThreshold` rather than a single run.
      * @default 1
      */
@@ -4282,12 +4290,6 @@ interface EvalCaseResult {
      * Only present when the case was run with `iterations > 1`.
      */
     infrastructureErrorRate?: number;
-    /**
-     * Accuracy score (0–1) across all iterations.
-     * Alias for `assertionPassRate`. Only present when the case was run with `iterations > 1`.
-     * @deprecated Use `assertionPassRate` for clarity; this field is kept for backward compatibility.
-     */
-    accuracy?: number;
     /**
      * Per-iteration pass/fail breakdown.
      * Only present when the case was run with `iterations > 1`.

package/dist/index.js

@@ -4380,7 +4380,7 @@ function escapeHtml(text) {
 
 // package.json
 var package_default = {
-  version: "1.0.0-beta.1"};
+  version: "1.0.0-beta.3"};
 
 // src/mcp/clientFactory.ts
 function getRetryAfterDelayMs(err) {
@@ -4471,7 +4471,10 @@ async function createMCPClientForConfig(config, options) {
       validatedConfig.connectTimeoutMs !== void 0 ? { timeout: validatedConfig.connectTimeoutMs } : void 0
     );
   } else if (isHttpConfig(validatedConfig)) {
-    const headers = { ...validatedConfig.headers };
+    const headers = {
+      "User-Agent": `@gleanwork/mcp-server-tester/${package_default.version}`,
+      ...validatedConfig.headers
+    };
     if (validatedConfig.auth?.clientCredentials && !options?.authProvider) {
       const ccConfig = validatedConfig.auth.clientCredentials;
       const clientId = ccConfig.clientId ?? process.env["MCP_CLIENT_ID"];
@@ -5888,7 +5891,18 @@ function applySanitizers(value, sanitizers) {
       continue;
     }
     if (isRegexSanitizer(sanitizer)) {
-      const pattern = sanitizer.pattern instanceof RegExp ? sanitizer.pattern : new RegExp(sanitizer.pattern, "g");
+      let pattern;
+      if (sanitizer.pattern instanceof RegExp) {
+        pattern = sanitizer.pattern;
+      } else {
+        try {
+          pattern = new RegExp(sanitizer.pattern, "g");
+        } catch {
+          throw new Error(
+            `toMatchToolSnapshot: invalid regex pattern "${sanitizer.pattern}" in sanitizer`
+          );
+        }
+      }
       const replacement = sanitizer.replacement ?? "[SANITIZED]";
       result = result.replace(pattern, replacement);
       continue;
@@ -6913,7 +6927,6 @@ async function runEvalCase(evalCase, context, options = {}) {
   const passCount = assertionResults.filter((r) => r.pass).length;
   const assertionPassRate = assertionResults.length > 0 ? passCount / assertionResults.length : 0;
   const infrastructureErrorRate = infraErrors.length / iterations;
-  const accuracy = assertionPassRate;
   const threshold = evalCase.accuracyThreshold ?? 1;
   const baseResult = lastResult ?? {
     id: evalCase.id,
@@ -6930,10 +6943,9 @@ async function runEvalCase(evalCase, context, options = {}) {
   };
   return {
     ...baseResult,
-    pass: accuracy >= threshold,
+    pass: assertionPassRate >= threshold,
     assertionPassRate,
     infrastructureErrorRate,
-    accuracy,
     iterationResults,
     infrastructureErrorCount: infraErrors.length,
     durationMs: iterationResults.reduce((sum, r) => sum + r.durationMs, 0)
@@ -6991,9 +7003,9 @@ async function runEvalDataset(options, context) {
     const withIterations = evalCase.mode === "llm_host" && evalCase.iterations === void 0 && defaultLlmIterations !== void 0 ? { ...evalCase, iterations: defaultLlmIterations } : evalCase;
     if (evalCase.mode === "llm_host") {
       const effectiveIterations = withIterations.iterations ?? 1;
-      if (effectiveIterations < 10) {
+      if (effectiveIterations > 1 && effectiveIterations < 10) {
         console.warn(
-          `[mcp-server-tester] Eval case "${evalCase.id}" uses llm_host mode with only ${effectiveIterations} iteration(s). The evals guide recommends >= 10 iterations. See docs/evals-guide.md for guidance on statistical reliability.`
+          `[mcp-server-tester] Eval case "${evalCase.id}": running ${effectiveIterations} iterations in llm_host mode may not be statistically reliable. Consider using 10+ iterations for accuracy measurements you can trust.`
         );
       }
     }
@@ -7041,6 +7053,16 @@ async function runEvalDataset(options, context) {
       const baseline = await loadBaseline(baselineResultsFrom);
       const baselinePassRate = baseline.total > 0 ? baseline.passed / baseline.total : 0;
       const baselineMap = buildBaselinePassMap(baseline);
+      const currentCaseIds = result.caseResults.map((cr) => cr.id);
+      const unmatchedCount = currentCaseIds.filter(
+        (id) => !baselineMap.has(id)
+      ).length;
+      const unmatchedRatio = currentCaseIds.length > 0 ? unmatchedCount / currentCaseIds.length : 0;
+      if (unmatchedRatio > 0.2) {
+        console.warn(
+          `[mcp-server-tester] Baseline comparison: ${unmatchedCount} of ${currentCaseIds.length} cases (${Math.round(unmatchedRatio * 100)}%) have no baseline entry. This may indicate the dataset structure has changed. Results for unmatched cases cannot be compared.`
+        );
+      }
       for (const cr of result.caseResults) {
         const baselinePass = baselineMap.get(cr.id);
         if (baselinePass !== void 0) {