@gleanwork/mcp-server-tester 0.12.0

package/dist/reporters/mcpReporter.js

@@ -0,0 +1,297 @@
+import { mkdir, cp, writeFile, readdir, readFile, unlink } from 'fs/promises';
+import { join, dirname, resolve } from 'path';
+import { fileURLToPath } from 'url';
+
+// src/reporters/mcpReporter.ts
+var MCPReporter = class {
+  config;
+  startTime = 0;
+  allResults = [];
+  conformanceChecks = [];
+  serverCapabilities = [];
+  constructor(options = {}) {
+    this.config = {
+      outputDir: options.outputDir ?? ".mcp-test-results",
+      autoOpen: options.autoOpen ?? true,
+      historyLimit: options.historyLimit ?? 10,
+      quiet: options.quiet ?? false,
+      includeAutoTracking: options.includeAutoTracking ?? true
+    };
+  }
+  log(message) {
+    if (!this.config.quiet) {
+      console.log(message);
+    }
+  }
+  logError(message, error) {
+    if (!this.config.quiet) {
+      console.error(message, error ?? "");
+    }
+  }
+  async onBegin(_config, _suite) {
+    this.startTime = Date.now();
+    this.allResults = [];
+    this.conformanceChecks = [];
+    this.serverCapabilities = [];
+    await mkdir(this.config.outputDir, { recursive: true });
+  }
+  async onTestEnd(test, result) {
+    const evalAttachment = result.attachments.find(
+      (a) => a.name === "mcp-test-results" && a.contentType === "application/json"
+    );
+    let hasEvalDataset = false;
+    if (evalAttachment && evalAttachment.body) {
+      try {
+        const evalResults = JSON.parse(
+          evalAttachment.body.toString("utf-8")
+        );
+        this.allResults.push(...evalResults.caseResults);
+        hasEvalDataset = true;
+      } catch (error) {
+        this.logError(
+          `[MCP Reporter] Failed to parse eval results from test "${test.title}":`,
+          error
+        );
+      }
+    }
+    const conformanceAttachment = result.attachments.find(
+      (a) => a.name === "mcp-conformance-checks" && a.contentType === "application/json"
+    );
+    if (conformanceAttachment && conformanceAttachment.body) {
+      try {
+        const conformanceData = JSON.parse(
+          conformanceAttachment.body.toString("utf-8")
+        );
+        if (Array.isArray(conformanceData.checks)) {
+          this.conformanceChecks.push({
+            testTitle: test.title,
+            pass: conformanceData.pass,
+            checks: conformanceData.checks,
+            serverInfo: conformanceData.serverInfo,
+            toolCount: conformanceData.toolCount,
+            authType: conformanceData.authType,
+            project: conformanceData.project
+          });
+        }
+      } catch (error) {
+        this.logError(
+          `[MCP Reporter] Failed to parse conformance check attachment for "${test.title}":`,
+          error
+        );
+      }
+    }
+    const listToolsAttachment = result.attachments.find(
+      (a) => a.name === "mcp-list-tools" && a.contentType === "application/json"
+    );
+    if (listToolsAttachment && listToolsAttachment.body) {
+      try {
+        const listToolsData = JSON.parse(
+          listToolsAttachment.body.toString("utf-8")
+        );
+        if (Array.isArray(listToolsData.tools)) {
+          this.serverCapabilities.push({
+            testTitle: test.title,
+            tools: listToolsData.tools,
+            toolCount: listToolsData.toolCount
+            // Note: authType and project are available from the mcp fixture
+            // but not currently included in the listTools attachment
+          });
+        }
+      } catch (error) {
+        this.logError(
+          `[MCP Reporter] Failed to parse mcp-list-tools attachment for "${test.title}":`,
+          error
+        );
+      }
+    }
+    if (hasEvalDataset || !this.config.includeAutoTracking) {
+      return;
+    }
+    const mcpCallAttachments = result.attachments.filter(
+      (a) => a.name && a.name.startsWith("mcp-call-") && a.contentType === "application/json"
+    );
+    for (const attachment of mcpCallAttachments) {
+      if (!attachment.body) continue;
+      try {
+        const callData = JSON.parse(attachment.body.toString("utf-8"));
+        const suiteName = test.parent?.title || "Uncategorized Tests";
+        const testPassed = result.status === "passed";
+        const syntheticResult = {
+          id: test.title,
+          datasetName: suiteName,
+          toolName: callData.toolName,
+          source: "test",
+          pass: testPassed,
+          response: callData.result,
+          error: !testPassed ? "Test failed" : void 0,
+          expectations: {},
+          authType: callData.authType,
+          project: callData.project,
+          durationMs: callData.durationMs
+        };
+        this.allResults.push(syntheticResult);
+      } catch (error) {
+        this.logError(
+          `[MCP Reporter] Failed to parse MCP call attachment "${attachment.name}":`,
+          error
+        );
+      }
+    }
+  }
+  async onEnd(_result) {
+    const endTime = Date.now();
+    const durationMs = endTime - this.startTime;
+    if (this.allResults.length === 0) {
+      this.log("[MCP Reporter] No MCP eval results found in test run");
+      return;
+    }
+    const runData = this.buildRunData(durationMs);
+    const historical = await this.loadHistoricalData();
+    historical.push({
+      timestamp: runData.timestamp,
+      total: runData.metrics.total,
+      passed: runData.metrics.passed,
+      failed: runData.metrics.failed,
+      passRate: runData.metrics.passRate,
+      durationMs: runData.durationMs
+    });
+    await this.saveRunData(runData);
+    await this.cleanupOldRuns();
+    const reportDir = join(this.config.outputDir, "latest");
+    await this.generateReport(runData, historical, reportDir);
+    const reportPath = join(reportDir, "index.html");
+    this.log(`
+[MCP Reporter] Report generated: ${reportPath}`);
+    this.log(
+      `[MCP Reporter] Results: ${runData.metrics.passed}/${runData.metrics.total} passed (${(runData.metrics.passRate * 100).toFixed(1)}%)`
+    );
+    if (this.config.autoOpen && !process.env.CI) {
+      await this.openReport(reportPath);
+    }
+  }
+  async generateReport(runData, historical, outputDir) {
+    const __filename2 = fileURLToPath(import.meta.url);
+    const __dirname2 = dirname(__filename2);
+    const uiDistPath = join(__dirname2, "ui-dist");
+    await mkdir(outputDir, { recursive: true });
+    await cp(uiDistPath, outputDir, { recursive: true, force: true });
+    const dataScript = `window.MCP_EVAL_DATA = ${JSON.stringify(
+      {
+        runData,
+        historical
+      },
+      null,
+      2
+    )};`;
+    await writeFile(join(outputDir, "data.js"), dataScript, "utf-8");
+  }
+  buildRunData(durationMs) {
+    const total = this.allResults.length;
+    const datasetBreakdown = {};
+    const expectationBreakdown = {
+      exact: 0,
+      schema: 0,
+      textContains: 0,
+      regex: 0,
+      snapshot: 0,
+      judge: 0,
+      error: 0,
+      size: 0
+    };
+    let passed = 0;
+    for (const r of this.allResults) {
+      if (r.pass) passed++;
+      const datasetName = r.datasetName || "Unknown Dataset";
+      datasetBreakdown[datasetName] = (datasetBreakdown[datasetName] || 0) + 1;
+      if (r.expectations.exact) expectationBreakdown.exact++;
+      if (r.expectations.schema) expectationBreakdown.schema++;
+      if (r.expectations.textContains) expectationBreakdown.textContains++;
+      if (r.expectations.regex) expectationBreakdown.regex++;
+      if (r.expectations.snapshot) expectationBreakdown.snapshot++;
+      if (r.expectations.judge) expectationBreakdown.judge++;
+      if (r.expectations.error) expectationBreakdown.error++;
+    }
+    const failed = total - passed;
+    return {
+      timestamp: (/* @__PURE__ */ new Date()).toISOString(),
+      durationMs,
+      environment: {
+        ci: !!process.env.CI,
+        node: process.version,
+        platform: process.platform
+      },
+      metrics: {
+        total,
+        passed,
+        failed,
+        passRate: passed / total,
+        datasetBreakdown,
+        expectationBreakdown
+      },
+      results: this.allResults,
+      conformanceChecks: this.conformanceChecks.length > 0 ? this.conformanceChecks : void 0,
+      serverCapabilities: this.serverCapabilities.length > 0 ? this.serverCapabilities : void 0
+    };
+  }
+  async loadHistoricalData() {
+    try {
+      const files = await readdir(this.config.outputDir);
+      const runFiles = files.filter((f) => f.startsWith("run-") && f.endsWith(".json")).sort().slice(-(this.config.historyLimit - 1));
+      const historical = [];
+      for (const file of runFiles) {
+        try {
+          const content = await readFile(
+            join(this.config.outputDir, file),
+            "utf-8"
+          );
+          const runData = JSON.parse(content);
+          historical.push({
+            timestamp: runData.timestamp,
+            total: runData.metrics.total,
+            passed: runData.metrics.passed,
+            failed: runData.metrics.failed,
+            passRate: runData.metrics.passRate,
+            durationMs: runData.durationMs
+          });
+        } catch (error) {
+          this.logError(`[MCP Reporter] Failed to load ${file}:`, error);
+        }
+      }
+      return historical;
+    } catch {
+      return [];
+    }
+  }
+  async saveRunData(runData) {
+    const filename = `run-${runData.timestamp.replace(/:/g, "-")}.json`;
+    const filepath = join(this.config.outputDir, filename);
+    await writeFile(filepath, JSON.stringify(runData, null, 2), "utf-8");
+  }
+  async cleanupOldRuns() {
+    try {
+      const files = await readdir(this.config.outputDir);
+      const runFiles = files.filter((f) => f.startsWith("run-") && f.endsWith(".json")).sort().reverse();
+      const toDelete = runFiles.slice(this.config.historyLimit);
+      for (const file of toDelete) {
+        await unlink(join(this.config.outputDir, file));
+      }
+    } catch (error) {
+      this.logError("[MCP Reporter] Failed to cleanup old runs:", error);
+    }
+  }
+  async openReport(reportPath) {
+    try {
+      const { default: open } = await import('open');
+      const absolutePath = resolve(reportPath);
+      await open(absolutePath);
+      this.log("[MCP Reporter] Opened report in browser");
+    } catch (error) {
+      this.logError("[MCP Reporter] Failed to open report:", error);
+      this.log(`[MCP Reporter] Open manually: file://${resolve(reportPath)}`);
+    }
+  }
+};
+
+export { MCPReporter as default };
+//# sourceMappingURL=mcpReporter.js.map
+//# sourceMappingURL=mcpReporter.js.map

package/dist/reporters/mcpReporter.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../src/reporters/mcpReporter.ts"],"names":["__filename","__dirname"],"mappings":";;;;;AAyCA,IAAqB,cAArB,MAAqD;AAAA,EAC3C,MAAA;AAAA,EACA,SAAA,GAAoB,CAAA;AAAA,EACpB,aAAoC,EAAC;AAAA,EACrC,oBAAqD,EAAC;AAAA,EACtD,qBAAuD,EAAC;AAAA,EAEhE,WAAA,CAAY,OAAA,GAAiC,EAAC,EAAG;AAC/C,IAAA,IAAA,CAAK,MAAA,GAAS;AAAA,MACZ,SAAA,EAAW,QAAQ,SAAA,IAAa,mBAAA;AAAA,MAChC,QAAA,EAAU,QAAQ,QAAA,IAAY,IAAA;AAAA,MAC9B,YAAA,EAAc,QAAQ,YAAA,IAAgB,EAAA;AAAA,MACtC,KAAA,EAAO,QAAQ,KAAA,IAAS,KAAA;AAAA,MACxB,mBAAA,EAAqB,QAAQ,mBAAA,IAAuB;AAAA,KACtD;AAAA,EACF;AAAA,EAEQ,IAAI,OAAA,EAAuB;AACjC,IAAA,IAAI,CAAC,IAAA,CAAK,MAAA,CAAO,KAAA,EAAO;AACtB,MAAA,OAAA,CAAQ,IAAI,OAAO,CAAA;AAAA,IACrB;AAAA,EACF;AAAA,EAEQ,QAAA,CAAS,SAAiB,KAAA,EAAuB;AACvD,IAAA,IAAI,CAAC,IAAA,CAAK,MAAA,CAAO,KAAA,EAAO;AACtB,MAAA,OAAA,CAAQ,KAAA,CAAM,OAAA,EAAS,KAAA,IAAS,EAAE,CAAA;AAAA,IACpC;AAAA,EACF;AAAA,EAEA,MAAM,OAAA,CAAQ,OAAA,EAAqB,MAAA,EAA8B;AAC/D,IAAA,IAAA,CAAK,SAAA,GAAY,KAAK,GAAA,EAAI;AAC1B,IAAA,IAAA,CAAK,aAAa,EAAC;AACnB,IAAA,IAAA,CAAK,oBAAoB,EAAC;AAC1B,IAAA,IAAA,CAAK,qBAAqB,EAAC;AAG3B,IAAA,MAAM,MAAM,IAAA,CAAK,MAAA,CAAO,WAAW,EAAE,SAAA,EAAW,MAAM,CAAA;AAAA,EACxD;AAAA,EAEA,MAAM,SAAA,CAAU,IAAA,EAAgB,MAAA,EAAmC;AAEjE,IAAA,MAAM,cAAA,GAAiB,OAAO,WAAA,CAAY,IAAA;AAAA,MACxC,CAAC,CAAA,KACC,CAAA,CAAE,IAAA,KAAS,kBAAA,IAAsB,EAAE,WAAA,KAAgB;AAAA,KACvD;AAEA,IAAA,IAAI,cAAA,GAAiB,KAAA;AAErB,IAAA,IAAI,cAAA,IAAkB,eAAe,IAAA,EAAM;AACzC,MAAA,IAAI;AACF,QAAA,MAAM,cAAc,IAAA,CAAK,KAAA;AAAA,UACvB,cAAA,CAAe,IAAA,CAAK,QAAA,CAAS,OAAO;AAAA,SACtC;AAMA,QAAA,IAAA,CAAK,UAAA,CAAW,IAAA,CAAK,GAAG,WAAA,CAAY,WAAW,CAAA;AAC/C,QAAA,cAAA,GAAiB,IAAA;AAAA,MACnB,SAAS,KAAA,EAAO;AACd,QAAA,IAAA,CAAK,QAAA;AAAA,UACH,CAAA,uDAAA,EAA0D,KAAK,KAAK,CAAA,EAAA,CAAA;AAAA,UACpE;AAAA,SACF;AAAA,MACF;AAAA,IACF;AAIA,IAAA,MAAM,qBAAA,GAAwB,OAAO,WAAA,CAAY,IAAA;AAAA,MAC/C,CAAC,CAAA,KACC,CAAA,CAAE,IAAA,KAAS,wBAAA,IACX,EAAE,WAAA,KAAgB;AAAA,KACtB;AAEA,IAAA,IAAI,qBAAA,IAAyB,sBAAsB,IAAA,EAAM;AACvD,MAAA,IAAI;AACF,QAAA,MAAM,kBAAkB,IAAA,CAAK,KAAA;AAAA,UAC3B,qBAAA,CAAsB,IAAA,CAAK,QAAA,CAAS,OAAO;AAAA,SAC7C;AAWA,QAAA,IAAI,KAAA,CAAM,OAAA,CAAQ,eAAA,CAAgB,MAAM,CAAA,EAAG;AACzC,UAAA,IAAA,CAAK,kBAAkB,IAAA,CAAK;AAAA,YAC1B,WAAW,IAAA,CAAK,KAAA;AAAA,YAChB,MAAM,eAAA,CAAgB,IAAA;AAAA,YACtB,QAAQ,eAAA,CAAgB,MAAA;AAAA,YACxB,YAAY,eAAA,CAAgB,UAAA;AAAA,YAC5B,WAAW,eAAA,CAAgB,SAAA;AAAA,YAC3B,UAAU,eAAA,CAAgB,QAAA;AAAA,YAC1B,SAAS,eAAA,CAAgB;AAAA,WAC1B,CAAA;AAAA,QACH;AAAA,MACF,SAAS,KAAA,EAAO;AACd,QAAA,IAAA,CAAK,QAAA;AAAA,UACH,CAAA,iEAAA,EAAoE,KAAK,KAAK,CAAA,EAAA,CAAA;AAAA,UAC9E;AAAA,SACF;AAAA,MACF;AAAA,IACF;AAIA,IAAA,MAAM,mBAAA,GAAsB,OAAO,WAAA,CAAY,IAAA;AAAA,MAC7C,CAAC,CAAA,KAAM,CAAA,CAAE,IAAA,KAAS,gBAAA,IAAoB,EAAE,WAAA,KAAgB;AAAA,KAC1D;AAEA,IAAA,IAAI,mBAAA,IAAuB,oBAAoB,IAAA,EAAM;AACnD,MAAA,IAAI;AACF,QAAA,MAAM,gBAAgB,IAAA,CAAK,KAAA;AAAA,UACzB,mBAAA,CAAoB,IAAA,CAAK,QAAA,CAAS,OAAO;AAAA,SAC3C;AAOA,QAAA,IAAI,KAAA,CAAM,OAAA,CAAQ,aAAA,CAAc,KAAK,CAAA,EAAG;AACtC,UAAA,IAAA,CAAK,mBAAmB,IAAA,CAAK;AAAA,YAC3B,WAAW,IAAA,CAAK,KAAA;AAAA,YAChB,OAAO,aAAA,CAAc,KAAA;AAAA,YACrB,WAAW,aAAA,CAAc;AAAA;AAAA;AAAA,WAG1B,CAAA;AAAA,QACH;AAAA,MACF,SAAS,KAAA,EAAO;AACd,QAAA,IAAA,CAAK,QAAA;AAAA,UACH,CAAA,8DAAA,EAAiE,KAAK,KAAK,CAAA,EAAA,CAAA;AAAA,UAC3E;AAAA,SACF;AAAA,MACF;AAAA,IACF;AAOA,IAAA,IAAI,cAAA,IAAkB,CAAC,IAAA,CAAK,MAAA,CAAO,mBAAA,EAAqB;AACtD,MAAA;AAAA,IACF;AAEA,IAAA,MAAM,kBAAA,GAAqB,OAAO,WAAA,CAAY,MAAA;AAAA,MAC5C,CAAC,CAAA,KACC,CAAA,CAAE,IAAA,IACF,CAAA,CAAE,KAAK,UAAA,CAAW,WAAW,CAAA,IAC7B,CAAA,CAAE,WAAA,KAAgB;AAAA,KACtB;AAEA,IAAA,KAAA,MAAW,cAAc,kBAAA,EAAoB;AAC3C,MAAA,IAAI,CAAC,WAAW,IAAA,EAAM;AAEtB,MAAA,IAAI;AAEF,QAAA,MAAM,WAAW,IAAA,CAAK,KAAA,CAAM,WAAW,IAAA,CAAK,QAAA,CAAS,OAAO,CAAC,CAAA;AAW7D,QAAA,MAAM,SAAA,GAAY,IAAA,CAAK,MAAA,EAAQ,KAAA,IAAS,qBAAA;AACxC,QAAA,MAAM,UAAA,GAAa,OAAO,MAAA,KAAW,QAAA;AAErC,QAAA,MAAM,eAAA,GAAkC;AAAA,UACtC,IAAI,IAAA,CAAK,KAAA;AAAA,UACT,WAAA,EAAa,SAAA;AAAA,UACb,UAAU,QAAA,CAAS,QAAA;AAAA,UACnB,MAAA,EAAQ,MAAA;AAAA,UACR,IAAA,EAAM,UAAA;AAAA,UACN,UAAU,QAAA,CAAS,MAAA;AAAA,UACnB,KAAA,EAAO,CAAC,UAAA,GAAa,aAAA,GAAgB,KAAA,CAAA;AAAA,UACrC,cAAc,EAAC;AAAA,UACf,UAAU,QAAA,CAAS,QAAA;AAAA,UACnB,SAAS,QAAA,CAAS,OAAA;AAAA,UAClB,YAAY,QAAA,CAAS;AAAA,SACvB;AAEA,QAAA,IAAA,CAAK,UAAA,CAAW,KAAK,eAAe,CAAA;AAAA,MACtC,SAAS,KAAA,EAAO;AACd,QAAA,IAAA,CAAK,QAAA;AAAA,UACH,CAAA,oDAAA,EAAuD,WAAW,IAAI,CAAA,EAAA,CAAA;AAAA,UACtE;AAAA,SACF;AAAA,MACF;AAAA,IACF;AAAA,EACF;AAAA,EAEA,MAAM,MAAM,OAAA,EAAoC;AAC9C,IAAA,MAAM,OAAA,GAAU,KAAK,GAAA,EAAI;AACzB,IAAA,MAAM,UAAA,GAAa,UAAU,IAAA,CAAK,SAAA;AAGlC,IAAA,IAAI,IAAA,CAAK,UAAA,CAAW,MAAA,KAAW,CAAA,EAAG;AAChC,MAAA,IAAA,CAAK,IAAI,sDAAsD,CAAA;AAC/D,MAAA;AAAA,IACF;AAGA,IAAA,MAAM,OAAA,GAAU,IAAA,CAAK,YAAA,CAAa,UAAU,CAAA;AAG5C,IAAA,MAAM,UAAA,GAAa,MAAM,IAAA,CAAK,kBAAA,EAAmB;AAGjD,IAAA,UAAA,CAAW,IAAA,CAAK;AAAA,MACd,WAAW,OAAA,CAAQ,SAAA;AAAA,MACnB,KAAA,EAAO,QAAQ,OAAA,CAAQ,KAAA;AAAA,MACvB,MAAA,EAAQ,QAAQ,OAAA,CAAQ,MAAA;AAAA,MACxB,MAAA,EAAQ,QAAQ,OAAA,CAAQ,MAAA;AAAA,MACxB,QAAA,EAAU,QAAQ,OAAA,CAAQ,QAAA;AAAA,MAC1B,YAAY,OAAA,CAAQ;AAAA,KACrB,CAAA;AAGD,IAAA,MAAM,IAAA,CAAK,YAAY,OAAO,CAAA;AAG9B,IAAA,MAAM,KAAK,cAAA,EAAe;AAG1B,IAAA,MAAM,SAAA,GAAY,IAAA,CAAK,IAAA,CAAK,MAAA,CAAO,WAAW,QAAQ,CAAA;AACtD,IAAA,MAAM,IAAA,CAAK,cAAA,CAAe,OAAA,EAAS,UAAA,EAAY,SAAS,CAAA;AAExD,IAAA,MAAM,UAAA,GAAa,IAAA,CAAK,SAAA,EAAW,YAAY,CAAA;AAC/C,IAAA,IAAA,CAAK,GAAA,CAAI;AAAA,iCAAA,EAAsC,UAAU,CAAA,CAAE,CAAA;AAC3D,IAAA,IAAA,CAAK,GAAA;AAAA,MACH,CAAA,wBAAA,EAA2B,OAAA,CAAQ,OAAA,CAAQ,MAAM,IAAI,OAAA,CAAQ,OAAA,CAAQ,KAAK,CAAA,SAAA,EAAA,CAAa,QAAQ,OAAA,CAAQ,QAAA,GAAW,GAAA,EAAK,OAAA,CAAQ,CAAC,CAAC,CAAA,EAAA;AAAA,KACnI;AAGA,IAAA,IAAI,KAAK,MAAA,CAAO,QAAA,IAAY,CAAC,OAAA,CAAQ,IAAI,EAAA,EAAI;AAC3C,MAAA,MAAM,IAAA,CAAK,WAAW,UAAU,CAAA;AAAA,IAClC;AAAA,EACF;AAAA,EAEA,MAAc,cAAA,CACZ,OAAA,EACA,UAAA,EACA,SAAA,EACe;AAGf,IAAA,MAAMA,WAAAA,GAAa,aAAA,CAAc,MAAA,CAAA,IAAA,CAAY,GAAG,CAAA;AAChD,IAAA,MAAMC,UAAAA,GAAY,QAAQD,WAAU,CAAA;AACpC,IAAA,MAAM,UAAA,GAAa,IAAA,CAAKC,UAAAA,EAAW,SAAS,CAAA;AAG5C,IAAA,MAAM,KAAA,CAAM,SAAA,EAAW,EAAE,SAAA,EAAW,MAAM,CAAA;AAC1C,IAAA,MAAM,EAAA,CAAG,YAAY,SAAA,EAAW,EAAE,WAAW,IAAA,EAAM,KAAA,EAAO,MAAM,CAAA;AAGhE,IAAA,MAAM,UAAA,GAAa,0BAA0B,IAAA,CAAK,SAAA;AAAA,MAChD;AAAA,QACE,OAAA;AAAA,QACA;AAAA,OACF;AAAA,MACA,IAAA;AAAA,MACA;AAAA,KACD,CAAA,CAAA,CAAA;AAED,IAAA,MAAM,UAAU,IAAA,CAAK,SAAA,EAAW,SAAS,CAAA,EAAG,YAAY,OAAO,CAAA;AAAA,EACjE;AAAA,EAEQ,aAAa,UAAA,EAAoC;AACvD,IAAA,MAAM,KAAA,GAAQ,KAAK,UAAA,CAAW,MAAA;AAC9B,IAAA,MAAM,mBAA2C,EAAC;AAClD,IAAA,MAAM,oBAAA,GAAuB;AAAA,MAC3B,KAAA,EAAO,CAAA;AAAA,MACP,MAAA,EAAQ,CAAA;AAAA,MACR,YAAA,EAAc,CAAA;AAAA,MACd,KAAA,EAAO,CAAA;AAAA,MACP,QAAA,EAAU,CAAA;AAAA,MACV,KAAA,EAAO,CAAA;AAAA,MACP,KAAA,EAAO,CAAA;AAAA,MACP,IAAA,EAAM;AAAA,KACR;AAEA,IAAA,IAAI,MAAA,GAAS,CAAA;AACb,IAAA,KAAA,MAAW,CAAA,IAAK,KAAK,UAAA,EAAY;AAC/B,MAAA,IAAI,EAAE,IAAA,EAAM,MAAA,EAAA;AAEZ,MAAA,MAAM,WAAA,GAAc,EAAE,WAAA,IAAe,iBAAA;AACrC,MAAA,gBAAA,CAAiB,WAAW,CAAA,GAAA,CAAK,gBAAA,CAAiB,WAAW,KAAK,CAAA,IAAK,CAAA;AAEvE,MAAA,IAAI,CAAA,CAAE,YAAA,CAAa,KAAA,EAAO,oBAAA,CAAqB,KAAA,EAAA;AAC/C,MAAA,IAAI,CAAA,CAAE,YAAA,CAAa,MAAA,EAAQ,oBAAA,CAAqB,MAAA,EAAA;AAChD,MAAA,IAAI,CAAA,CAAE,YAAA,CAAa,YAAA,EAAc,oBAAA,CAAqB,YAAA,EAAA;AACtD,MAAA,IAAI,CAAA,CAAE,YAAA,CAAa,KAAA,EAAO,oBAAA,CAAqB,KAAA,EAAA;AAC/C,MAAA,IAAI,CAAA,CAAE,YAAA,CAAa,QAAA,EAAU,oBAAA,CAAqB,QAAA,EAAA;AAClD,MAAA,IAAI,CAAA,CAAE,YAAA,CAAa,KAAA,EAAO,oBAAA,CAAqB,KAAA,EAAA;AAC/C,MAAA,IAAI,CAAA,CAAE,YAAA,CAAa,KAAA,EAAO,oBAAA,CAAqB,KAAA,EAAA;AAAA,IACjD;AAEA,IAAA,MAAM,SAAS,KAAA,GAAQ,MAAA;AAEvB,IAAA,OAAO;AAAA,MACL,SAAA,EAAA,iBAAW,IAAI,IAAA,EAAK,EAAE,WAAA,EAAY;AAAA,MAClC,UAAA;AAAA,MACA,WAAA,EAAa;AAAA,QACX,EAAA,EAAI,CAAC,CAAC,OAAA,CAAQ,GAAA,CAAI,EAAA;AAAA,QAClB,MAAM,OAAA,CAAQ,OAAA;AAAA,QACd,UAAU,OAAA,CAAQ;AAAA,OACpB;AAAA,MACA,OAAA,EAAS;AAAA,QACP,KAAA;AAAA,QACA,MAAA;AAAA,QACA,MAAA;AAAA,QACA,UAAU,MAAA,GAAS,KAAA;AAAA,QACnB,gBAAA;AAAA,QACA;AAAA,OACF;AAAA,MACA,SAAS,IAAA,CAAK,UAAA;AAAA,MACd,mBACE,IAAA,CAAK,iBAAA,CAAkB,MAAA,GAAS,CAAA,GAAI,KAAK,iBAAA,GAAoB,MAAA;AAAA,MAC/D,oBACE,IAAA,CAAK,kBAAA,CAAmB,MAAA,GAAS,CAAA,GAC7B,KAAK,kBAAA,GACL;AAAA,KACR;AAAA,EACF;AAAA,EAEA,MAAc,kBAAA,GAA+D;AAC3E,IAAA,IAAI;AACF,MAAA,MAAM,KAAA,GAAQ,MAAM,OAAA,CAAQ,IAAA,CAAK,OAAO,SAAS,CAAA;AACjD,MAAA,MAAM,QAAA,GAAW,MACd,MAAA,CAAO,CAAC,MAAM,CAAA,CAAE,UAAA,CAAW,MAAM,CAAA,IAAK,CAAA,CAAE,SAAS,OAAO,CAAC,EACzD,IAAA,EAAK,CACL,MAAM,EAAE,IAAA,CAAK,MAAA,CAAO,YAAA,GAAe,CAAA,CAAE,CAAA;AAExC,MAAA,MAAM,aAA8C,EAAC;AAErD,MAAA,KAAA,MAAW,QAAQ,QAAA,EAAU;AAC3B,QAAA,IAAI;AACF,UAAA,MAAM,UAAU,MAAM,QAAA;AAAA,YACpB,IAAA,CAAK,IAAA,CAAK,MAAA,CAAO,SAAA,EAAW,IAAI,CAAA;AAAA,YAChC;AAAA,WACF;AACA,UAAA,MAAM,OAAA,GAAU,IAAA,CAAK,KAAA,CAAM,OAAO,CAAA;AAElC,UAAA,UAAA,CAAW,IAAA,CAAK;AAAA,YACd,WAAW,OAAA,CAAQ,SAAA;AAAA,YACnB,KAAA,EAAO,QAAQ,OAAA,CAAQ,KAAA;AAAA,YACvB,MAAA,EAAQ,QAAQ,OAAA,CAAQ,MAAA;AAAA,YACxB,MAAA,EAAQ,QAAQ,OAAA,CAAQ,MAAA;AAAA,YACxB,QAAA,EAAU,QAAQ,OAAA,CAAQ,QAAA;AAAA,YAC1B,YAAY,OAAA,CAAQ;AAAA,WACrB,CAAA;AAAA,QACH,SAAS,KAAA,EAAO;AACd,UAAA,IAAA,CAAK,QAAA,CAAS,CAAA,8BAAA,EAAiC,IAAI,CAAA,CAAA,CAAA,EAAK,KAAK,CAAA;AAAA,QAC/D;AAAA,MACF;AAEA,MAAA,OAAO,UAAA;AAAA,IACT,CAAA,CAAA,MAAQ;AACN,MAAA,OAAO,EAAC;AAAA,IACV;AAAA,EACF;AAAA,EAEA,MAAc,YAAY,OAAA,EAAwC;AAChE,IAAA,MAAM,WAAW,CAAA,IAAA,EAAO,OAAA,CAAQ,UAAU,OAAA,CAAQ,IAAA,EAAM,GAAG,CAAC,CAAA,KAAA,CAAA;AAC5D,IAAA,MAAM,QAAA,GAAW,IAAA,CAAK,IAAA,CAAK,MAAA,CAAO,WAAW,QAAQ,CAAA;AAErD,IAAA,MAAM,SAAA,CAAU,UAAU,IAAA,CAAK,SAAA,CAAU,SAAS,IAAA,EAAM,CAAC,GAAG,OAAO,CAAA;AAAA,EACrE;AAAA,EAEA,MAAc,cAAA,GAAgC;AAC5C,IAAA,IAAI;AACF,MAAA,MAAM,KAAA,GAAQ,MAAM,OAAA,CAAQ,IAAA,CAAK,OAAO,SAAS,CAAA;AACjD,MAAA,MAAM,WAAW,KAAA,CACd,MAAA,CAAO,CAAC,CAAA,KAAM,EAAE,UAAA,CAAW,MAAM,CAAA,IAAK,CAAA,CAAE,SAAS,OAAO,CAAC,CAAA,CACzD,IAAA,GACA,OAAA,EAAQ;AAGX,MAAA,MAAM,QAAA,GAAW,QAAA,CAAS,KAAA,CAAM,IAAA,CAAK,OAAO,YAAY,CAAA;AAExD,MAAA,KAAA,MAAW,QAAQ,QAAA,EAAU;AAC3B,QAAA,MAAM,OAAO,IAAA,CAAK,IAAA,CAAK,MAAA,CAAO,SAAA,EAAW,IAAI,CAAC,CAAA;AAAA,MAChD;AAAA,IACF,SAAS,KAAA,EAAO;AACd,MAAA,IAAA,CAAK,QAAA,CAAS,8CAA8C,KAAK,CAAA;AAAA,IACnE;AAAA,EACF;AAAA,EAEA,MAAc,WAAW,UAAA,EAAmC;AAC1D,IAAA,IAAI;AAEF,MAAA,MAAM,EAAE,OAAA,EAAS,IAAA,EAAK,GAAI,MAAM,OAAO,MAAM,CAAA;AAC7C,MAAA,MAAM,YAAA,GAAe,QAAQ,UAAU,CAAA;AAEvC,MAAA,MAAM,KAAK,YAAY,CAAA;AACvB,MAAA,IAAA,CAAK,IAAI,yCAAyC,CAAA;AAAA,IACpD,SAAS,KAAA,EAAO;AACd,MAAA,IAAA,CAAK,QAAA,CAAS,yCAAyC,KAAK,CAAA;AAC5D,MAAA,IAAA,CAAK,GAAA,CAAI,CAAA,qCAAA,EAAwC,OAAA,CAAQ,UAAU,CAAC,CAAA,CAAE,CAAA;AAAA,IACxE;AAAA,EACF;AACF","file":"mcpReporter.js","sourcesContent":["import type {\n  Reporter,\n  FullConfig,\n  Suite,\n  TestCase,\n  TestResult,\n  FullResult,\n} from '@playwright/test/reporter';\nimport { mkdir, writeFile, readdir, readFile, unlink, cp } from 'fs/promises';\nimport { join, resolve, dirname } from 'path';\nimport { fileURLToPath } from 'url';\nimport type { MCPEvalReporterConfig } from './types.js';\nimport type { AuthType } from '../types/index.js';\nimport type {\n  MCPEvalRunData,\n  MCPEvalHistoricalSummary,\n  MCPConformanceResultData,\n  MCPServerCapabilitiesData,\n  EvalCaseResult,\n} from '../types/reporter.js';\nimport type { MCPConformanceCheck } from '../spec/conformanceChecks.js';\n\n/**\n * Custom Playwright reporter for MCP eval results\n *\n * Generates HTML reports with historical tracking and auto-opens in browser\n *\n * @example\n * ```typescript\n * // playwright.config.ts\n * export default defineConfig({\n *   reporter: [\n *     ['@gleanwork/mcp-server-tester/reporters/mcpReporter', {\n *       outputDir: '.mcp-test-results',\n *       autoOpen: true,\n *       historyLimit: 10\n *     }]\n *   ]\n * });\n * ```\n */\nexport default class MCPReporter implements Reporter {\n  private config: Required<MCPEvalReporterConfig>;\n  private startTime: number = 0;\n  private allResults: Array<EvalCaseResult> = [];\n  private conformanceChecks: Array<MCPConformanceResultData> = [];\n  private serverCapabilities: Array<MCPServerCapabilitiesData> = [];\n\n  constructor(options: MCPEvalReporterConfig = {}) {\n    this.config = {\n      outputDir: options.outputDir ?? '.mcp-test-results',\n      autoOpen: options.autoOpen ?? true,\n      historyLimit: options.historyLimit ?? 10,\n      quiet: options.quiet ?? false,\n      includeAutoTracking: options.includeAutoTracking ?? true,\n    };\n  }\n\n  private log(message: string): void {\n    if (!this.config.quiet) {\n      console.log(message);\n    }\n  }\n\n  private logError(message: string, error?: unknown): void {\n    if (!this.config.quiet) {\n      console.error(message, error ?? '');\n    }\n  }\n\n  async onBegin(_config: FullConfig, _suite: Suite): Promise<void> {\n    this.startTime = Date.now();\n    this.allResults = [];\n    this.conformanceChecks = [];\n    this.serverCapabilities = [];\n\n    // Ensure output directory exists\n    await mkdir(this.config.outputDir, { recursive: true });\n  }\n\n  async onTestEnd(test: TestCase, result: TestResult): Promise<void> {\n    // Strategy 1: Extract MCP eval results from runEvalDataset() attachments\n    const evalAttachment = result.attachments.find(\n      (a) =>\n        a.name === 'mcp-test-results' && a.contentType === 'application/json'\n    );\n\n    let hasEvalDataset = false;\n\n    if (evalAttachment && evalAttachment.body) {\n      try {\n        const evalResults = JSON.parse(\n          evalAttachment.body.toString('utf-8')\n        ) as {\n          caseResults: Array<EvalCaseResult>;\n        };\n\n        // Trust the data from the attachment - evalRunner now includes\n        // authType and project from the mcp fixture (Playwright is source of truth)\n        this.allResults.push(...evalResults.caseResults);\n        hasEvalDataset = true;\n      } catch (error) {\n        this.logError(\n          `[MCP Reporter] Failed to parse eval results from test \"${test.title}\":`,\n          error\n        );\n      }\n    }\n\n    // Strategy 2: Extract conformance check results\n    // These are created by runConformanceChecks() when testInfo is passed\n    const conformanceAttachment = result.attachments.find(\n      (a) =>\n        a.name === 'mcp-conformance-checks' &&\n        a.contentType === 'application/json'\n    );\n\n    if (conformanceAttachment && conformanceAttachment.body) {\n      try {\n        const conformanceData = JSON.parse(\n          conformanceAttachment.body.toString('utf-8')\n        ) as {\n          operation: string;\n          pass: boolean;\n          checks: MCPConformanceCheck[];\n          serverInfo?: { name?: string; version?: string };\n          toolCount: number;\n          authType?: AuthType;\n          project?: string;\n        };\n\n        // Only push if checks array is valid\n        if (Array.isArray(conformanceData.checks)) {\n          this.conformanceChecks.push({\n            testTitle: test.title,\n            pass: conformanceData.pass,\n            checks: conformanceData.checks,\n            serverInfo: conformanceData.serverInfo,\n            toolCount: conformanceData.toolCount,\n            authType: conformanceData.authType,\n            project: conformanceData.project,\n          });\n        }\n      } catch (error) {\n        this.logError(\n          `[MCP Reporter] Failed to parse conformance check attachment for \"${test.title}\":`,\n          error\n        );\n      }\n    }\n\n    // Strategy 2b: Extract server capabilities from mcp-list-tools attachments\n    // These are created by createMCPFixture().listTools()\n    const listToolsAttachment = result.attachments.find(\n      (a) => a.name === 'mcp-list-tools' && a.contentType === 'application/json'\n    );\n\n    if (listToolsAttachment && listToolsAttachment.body) {\n      try {\n        const listToolsData = JSON.parse(\n          listToolsAttachment.body.toString('utf-8')\n        ) as {\n          operation: string;\n          toolCount: number;\n          tools: Array<{ name: string; description?: string }>;\n        };\n\n        // Only push if tools array is valid\n        if (Array.isArray(listToolsData.tools)) {\n          this.serverCapabilities.push({\n            testTitle: test.title,\n            tools: listToolsData.tools,\n            toolCount: listToolsData.toolCount,\n            // Note: authType and project are available from the mcp fixture\n            // but not currently included in the listTools attachment\n          });\n        }\n      } catch (error) {\n        this.logError(\n          `[MCP Reporter] Failed to parse mcp-list-tools attachment for \"${test.title}\":`,\n          error\n        );\n      }\n    }\n\n    // Strategy 3: Extract MCP tool calls from auto-tracking attachments\n    // These are created by createMCPFixture()\n    // Skip if:\n    // - This test already has eval dataset results (to avoid duplicates)\n    // - Auto-tracking is disabled in config\n    if (hasEvalDataset || !this.config.includeAutoTracking) {\n      return;\n    }\n\n    const mcpCallAttachments = result.attachments.filter(\n      (a) =>\n        a.name &&\n        a.name.startsWith('mcp-call-') &&\n        a.contentType === 'application/json'\n    );\n\n    for (const attachment of mcpCallAttachments) {\n      if (!attachment.body) continue;\n\n      try {\n        // Attachment now includes authType and project from the mcp fixture\n        const callData = JSON.parse(attachment.body.toString('utf-8')) as {\n          operation: string;\n          toolName: string;\n          args: Record<string, unknown>;\n          result: unknown;\n          durationMs: number;\n          isError: boolean;\n          authType?: AuthType;\n          project?: string;\n        };\n\n        const suiteName = test.parent?.title || 'Uncategorized Tests';\n        const testPassed = result.status === 'passed';\n\n        const syntheticResult: EvalCaseResult = {\n          id: test.title,\n          datasetName: suiteName,\n          toolName: callData.toolName,\n          source: 'test',\n          pass: testPassed,\n          response: callData.result,\n          error: !testPassed ? 'Test failed' : undefined,\n          expectations: {},\n          authType: callData.authType,\n          project: callData.project,\n          durationMs: callData.durationMs,\n        };\n\n        this.allResults.push(syntheticResult);\n      } catch (error) {\n        this.logError(\n          `[MCP Reporter] Failed to parse MCP call attachment \"${attachment.name}\":`,\n          error\n        );\n      }\n    }\n  }\n\n  async onEnd(_result: FullResult): Promise<void> {\n    const endTime = Date.now();\n    const durationMs = endTime - this.startTime;\n\n    // Skip if no eval results collected\n    if (this.allResults.length === 0) {\n      this.log('[MCP Reporter] No MCP eval results found in test run');\n      return;\n    }\n\n    // Build run data\n    const runData = this.buildRunData(durationMs);\n\n    // Load historical data\n    const historical = await this.loadHistoricalData();\n\n    // Add current run to historical\n    historical.push({\n      timestamp: runData.timestamp,\n      total: runData.metrics.total,\n      passed: runData.metrics.passed,\n      failed: runData.metrics.failed,\n      passRate: runData.metrics.passRate,\n      durationMs: runData.durationMs,\n    });\n\n    // Save current run data\n    await this.saveRunData(runData);\n\n    // Clean up old runs\n    await this.cleanupOldRuns();\n\n    // Generate report using copy + inject pattern\n    const reportDir = join(this.config.outputDir, 'latest');\n    await this.generateReport(runData, historical, reportDir);\n\n    const reportPath = join(reportDir, 'index.html');\n    this.log(`\\n[MCP Reporter] Report generated: ${reportPath}`);\n    this.log(\n      `[MCP Reporter] Results: ${runData.metrics.passed}/${runData.metrics.total} passed (${(runData.metrics.passRate * 100).toFixed(1)}%)`\n    );\n\n    // Auto-open browser if configured and not in CI\n    if (this.config.autoOpen && !process.env.CI) {\n      await this.openReport(reportPath);\n    }\n  }\n\n  private async generateReport(\n    runData: MCPEvalRunData,\n    historical: Array<MCPEvalHistoricalSummary>,\n    outputDir: string\n  ): Promise<void> {\n    // Get the UI dist path (relative to this file)\n    // In ESM, we need to use import.meta.url\n    const __filename = fileURLToPath(import.meta.url);\n    const __dirname = dirname(__filename);\n    const uiDistPath = join(__dirname, 'ui-dist');\n\n    // Step 1: Copy pre-built UI template\n    await mkdir(outputDir, { recursive: true });\n    await cp(uiDistPath, outputDir, { recursive: true, force: true });\n\n    // Step 2: Inject test data as JavaScript\n    const dataScript = `window.MCP_EVAL_DATA = ${JSON.stringify(\n      {\n        runData,\n        historical,\n      },\n      null,\n      2\n    )};`;\n\n    await writeFile(join(outputDir, 'data.js'), dataScript, 'utf-8');\n  }\n\n  private buildRunData(durationMs: number): MCPEvalRunData {\n    const total = this.allResults.length;\n    const datasetBreakdown: Record<string, number> = {};\n    const expectationBreakdown = {\n      exact: 0,\n      schema: 0,\n      textContains: 0,\n      regex: 0,\n      snapshot: 0,\n      judge: 0,\n      error: 0,\n      size: 0,\n    };\n\n    let passed = 0;\n    for (const r of this.allResults) {\n      if (r.pass) passed++;\n\n      const datasetName = r.datasetName || 'Unknown Dataset';\n      datasetBreakdown[datasetName] = (datasetBreakdown[datasetName] || 0) + 1;\n\n      if (r.expectations.exact) expectationBreakdown.exact++;\n      if (r.expectations.schema) expectationBreakdown.schema++;\n      if (r.expectations.textContains) expectationBreakdown.textContains++;\n      if (r.expectations.regex) expectationBreakdown.regex++;\n      if (r.expectations.snapshot) expectationBreakdown.snapshot++;\n      if (r.expectations.judge) expectationBreakdown.judge++;\n      if (r.expectations.error) expectationBreakdown.error++;\n    }\n\n    const failed = total - passed;\n\n    return {\n      timestamp: new Date().toISOString(),\n      durationMs,\n      environment: {\n        ci: !!process.env.CI,\n        node: process.version,\n        platform: process.platform,\n      },\n      metrics: {\n        total,\n        passed,\n        failed,\n        passRate: passed / total,\n        datasetBreakdown,\n        expectationBreakdown,\n      },\n      results: this.allResults,\n      conformanceChecks:\n        this.conformanceChecks.length > 0 ? this.conformanceChecks : undefined,\n      serverCapabilities:\n        this.serverCapabilities.length > 0\n          ? this.serverCapabilities\n          : undefined,\n    };\n  }\n\n  private async loadHistoricalData(): Promise<Array<MCPEvalHistoricalSummary>> {\n    try {\n      const files = await readdir(this.config.outputDir);\n      const runFiles = files\n        .filter((f) => f.startsWith('run-') && f.endsWith('.json'))\n        .sort()\n        .slice(-(this.config.historyLimit - 1)); // Keep most recent, leave room for current run\n\n      const historical: Array<MCPEvalHistoricalSummary> = [];\n\n      for (const file of runFiles) {\n        try {\n          const content = await readFile(\n            join(this.config.outputDir, file),\n            'utf-8'\n          );\n          const runData = JSON.parse(content) as MCPEvalRunData;\n\n          historical.push({\n            timestamp: runData.timestamp,\n            total: runData.metrics.total,\n            passed: runData.metrics.passed,\n            failed: runData.metrics.failed,\n            passRate: runData.metrics.passRate,\n            durationMs: runData.durationMs,\n          });\n        } catch (error) {\n          this.logError(`[MCP Reporter] Failed to load ${file}:`, error);\n        }\n      }\n\n      return historical;\n    } catch {\n      return [];\n    }\n  }\n\n  private async saveRunData(runData: MCPEvalRunData): Promise<void> {\n    const filename = `run-${runData.timestamp.replace(/:/g, '-')}.json`;\n    const filepath = join(this.config.outputDir, filename);\n\n    await writeFile(filepath, JSON.stringify(runData, null, 2), 'utf-8');\n  }\n\n  private async cleanupOldRuns(): Promise<void> {\n    try {\n      const files = await readdir(this.config.outputDir);\n      const runFiles = files\n        .filter((f) => f.startsWith('run-') && f.endsWith('.json'))\n        .sort()\n        .reverse();\n\n      // Keep only historyLimit most recent runs\n      const toDelete = runFiles.slice(this.config.historyLimit);\n\n      for (const file of toDelete) {\n        await unlink(join(this.config.outputDir, file));\n      }\n    } catch (error) {\n      this.logError('[MCP Reporter] Failed to cleanup old runs:', error);\n    }\n  }\n\n  private async openReport(reportPath: string): Promise<void> {\n    try {\n      // Dynamic import to avoid bundling issues\n      const { default: open } = await import('open');\n      const absolutePath = resolve(reportPath);\n\n      await open(absolutePath);\n      this.log('[MCP Reporter] Opened report in browser');\n    } catch (error) {\n      this.logError('[MCP Reporter] Failed to open report:', error);\n      this.log(`[MCP Reporter] Open manually: file://${resolve(reportPath)}`);\n    }\n  }\n}\n"]}