agentdb 1.6.0 → 2.0.0-alpha.1

package/src/cli/lib/report-generator.ts

@@ -0,0 +1,455 @@
+/**
+ * Report generation in multiple formats
+ * Supports markdown, JSON, and HTML output
+ */
+
+import type { SimulationReport } from './simulation-runner.js';
+
+export class ReportGenerator {
+  /**
+   * Generate markdown report
+   */
+  async generateMarkdown(report: SimulationReport): Promise<string> {
+    const sections: string[] = [];
+
+    // Header
+    sections.push(`# AgentDB Simulation Report: ${report.scenarioId}`);
+    sections.push('');
+    sections.push(`**Generated**: ${new Date().toISOString()}`);
+    sections.push(`**Duration**: ${(report.totalDuration / 1000).toFixed(2)}s`);
+    sections.push(`**Iterations**: ${report.iterations.length}`);
+    sections.push('');
+
+    // Configuration
+    sections.push('## Configuration');
+    sections.push('');
+    sections.push('```json');
+    sections.push(JSON.stringify(report.config, null, 2));
+    sections.push('```');
+    sections.push('');
+
+    if (report.optimal) {
+      sections.push('✅ **Using optimal configuration**');
+    } else {
+      sections.push('⚠️ **Non-optimal configuration** (see warnings below)');
+    }
+    sections.push('');
+
+    // Summary
+    sections.push('## Summary');
+    sections.push('');
+    sections.push('| Metric | Value |');
+    sections.push('|--------|-------|');
+    sections.push(`| Success Rate | ${(report.summary.successRate * 100).toFixed(1)}% |`);
+    sections.push(`| Coherence Score | ${(report.coherenceScore * 100).toFixed(1)}% |`);
+    sections.push(`| Avg Latency (p50) | ${report.summary.avgLatencyUs.toFixed(2)}μs |`);
+    sections.push(`| Avg Recall@10 | ${(report.summary.avgRecall * 100).toFixed(1)}% |`);
+    sections.push(`| Avg QPS | ${report.summary.avgQps.toLocaleString()} |`);
+    sections.push(`| Avg Memory | ${report.summary.avgMemoryMB.toFixed(0)}MB |`);
+    sections.push('');
+
+    // Variance
+    sections.push('## Variance Analysis');
+    sections.push('');
+    sections.push('| Metric | Variance |');
+    sections.push('|--------|----------|');
+    sections.push(`| Latency | ${report.varianceMetrics.latencyVariance.toFixed(4)} |`);
+    sections.push(`| Recall | ${report.varianceMetrics.recallVariance.toFixed(6)} |`);
+    sections.push(`| QPS | ${report.varianceMetrics.qpsVariance.toFixed(2)} |`);
+    sections.push('');
+
+    // Iterations
+    sections.push('## Iteration Results');
+    sections.push('');
+    report.iterations.forEach((iter) => {
+      sections.push(`### Iteration ${iter.iteration}`);
+      sections.push('');
+      sections.push(`- **Status**: ${iter.success ? '✅ Success' : '❌ Failed'}`);
+      sections.push(`- **Duration**: ${(iter.duration / 1000).toFixed(2)}s`);
+      sections.push(`- **Timestamp**: ${iter.timestamp}`);
+
+      if (iter.success) {
+        sections.push('');
+        sections.push('**Metrics**:');
+        if (iter.metrics.latencyUs) {
+          sections.push(`- Latency p50: ${iter.metrics.latencyUs.p50.toFixed(2)}μs`);
+          sections.push(`- Latency p95: ${iter.metrics.latencyUs.p95.toFixed(2)}μs`);
+          sections.push(`- Latency p99: ${iter.metrics.latencyUs.p99.toFixed(2)}μs`);
+        }
+        if (iter.metrics.recallAtK) {
+          sections.push(`- Recall@10: ${(iter.metrics.recallAtK.k10 * 100).toFixed(1)}%`);
+          sections.push(`- Recall@50: ${(iter.metrics.recallAtK.k50 * 100).toFixed(1)}%`);
+          sections.push(`- Recall@100: ${(iter.metrics.recallAtK.k100 * 100).toFixed(1)}%`);
+        }
+        if (iter.metrics.qps) {
+          sections.push(`- QPS: ${iter.metrics.qps.toLocaleString()}`);
+        }
+        if (iter.metrics.memoryMB) {
+          sections.push(`- Memory: ${iter.metrics.memoryMB.toFixed(0)}MB`);
+        }
+      } else {
+        sections.push(`- **Error**: ${iter.error}`);
+      }
+      sections.push('');
+    });
+
+    // Warnings
+    if (report.warnings.length > 0) {
+      sections.push('## Warnings');
+      sections.push('');
+      report.warnings.forEach((warning) => {
+        sections.push(`- ⚠️ ${warning}`);
+      });
+      sections.push('');
+    }
+
+    // Footer
+    sections.push('---');
+    sections.push('');
+    sections.push('*Generated by AgentDB v2.0.0*');
+
+    return sections.join('\n');
+  }
+
+  /**
+   * Generate JSON report
+   */
+  async generateJSON(report: SimulationReport): Promise<string> {
+    return JSON.stringify(
+      {
+        ...report,
+        generatedAt: new Date().toISOString(),
+        version: '2.0.0',
+      },
+      null,
+      2
+    );
+  }
+
+  /**
+   * Generate HTML report
+   */
+  async generateHTML(report: SimulationReport): Promise<string> {
+    const html = `
+<!DOCTYPE html>
+<html lang="en">
+<head>
+  <meta charset="UTF-8">
+  <meta name="viewport" content="width=device-width, initial-scale=1.0">
+  <title>AgentDB Simulation Report: ${report.scenarioId}</title>
+  <style>
+    * {
+      margin: 0;
+      padding: 0;
+      box-sizing: border-box;
+    }
+    body {
+      font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, Oxygen, Ubuntu, Cantarell, sans-serif;
+      line-height: 1.6;
+      color: #333;
+      background: #f5f5f5;
+      padding: 20px;
+    }
+    .container {
+      max-width: 1200px;
+      margin: 0 auto;
+      background: white;
+      padding: 40px;
+      border-radius: 8px;
+      box-shadow: 0 2px 4px rgba(0,0,0,0.1);
+    }
+    h1 {
+      color: #2563eb;
+      margin-bottom: 20px;
+      padding-bottom: 10px;
+      border-bottom: 3px solid #2563eb;
+    }
+    h2 {
+      color: #1e40af;
+      margin-top: 30px;
+      margin-bottom: 15px;
+    }
+    h3 {
+      color: #3b82f6;
+      margin-top: 20px;
+      margin-bottom: 10px;
+    }
+    .meta {
+      background: #eff6ff;
+      padding: 15px;
+      border-radius: 4px;
+      margin-bottom: 20px;
+    }
+    .meta p {
+      margin: 5px 0;
+    }
+    .badge {
+      display: inline-block;
+      padding: 4px 12px;
+      border-radius: 12px;
+      font-size: 14px;
+      font-weight: 600;
+      margin: 10px 0;
+    }
+    .badge.success {
+      background: #d1fae5;
+      color: #065f46;
+    }
+    .badge.warning {
+      background: #fef3c7;
+      color: #92400e;
+    }
+    table {
+      width: 100%;
+      border-collapse: collapse;
+      margin: 20px 0;
+    }
+    th, td {
+      padding: 12px;
+      text-align: left;
+      border-bottom: 1px solid #e5e7eb;
+    }
+    th {
+      background: #f3f4f6;
+      font-weight: 600;
+      color: #1f2937;
+    }
+    tr:hover {
+      background: #f9fafb;
+    }
+    pre {
+      background: #1f2937;
+      color: #f9fafb;
+      padding: 15px;
+      border-radius: 4px;
+      overflow-x: auto;
+      margin: 10px 0;
+    }
+    .iteration {
+      background: #f9fafb;
+      padding: 15px;
+      border-radius: 4px;
+      margin: 15px 0;
+      border-left: 4px solid #3b82f6;
+    }
+    .metrics {
+      display: grid;
+      grid-template-columns: repeat(auto-fit, minmax(200px, 1fr));
+      gap: 15px;
+      margin: 15px 0;
+    }
+    .metric-card {
+      background: white;
+      padding: 15px;
+      border-radius: 4px;
+      border: 1px solid #e5e7eb;
+    }
+    .metric-card .label {
+      color: #6b7280;
+      font-size: 14px;
+      margin-bottom: 5px;
+    }
+    .metric-card .value {
+      font-size: 24px;
+      font-weight: 700;
+      color: #2563eb;
+    }
+    .warning-list {
+      background: #fef3c7;
+      padding: 15px;
+      border-radius: 4px;
+      border-left: 4px solid #f59e0b;
+    }
+    .warning-list li {
+      margin: 8px 0;
+      list-style-position: inside;
+    }
+  </style>
+</head>
+<body>
+  <div class="container">
+    <h1>AgentDB Simulation Report: ${report.scenarioId}</h1>
+
+    <div class="meta">
+      <p><strong>Generated:</strong> ${new Date().toISOString()}</p>
+      <p><strong>Duration:</strong> ${(report.totalDuration / 1000).toFixed(2)}s</p>
+      <p><strong>Iterations:</strong> ${report.iterations.length}</p>
+    </div>
+
+    ${
+      report.optimal
+        ? '<span class="badge success">✅ Optimal Configuration</span>'
+        : '<span class="badge warning">⚠️ Non-Optimal Configuration</span>'
+    }
+
+    <h2>Configuration</h2>
+    <pre>${JSON.stringify(report.config, null, 2)}</pre>
+
+    <h2>Summary</h2>
+    <div class="metrics">
+      <div class="metric-card">
+        <div class="label">Success Rate</div>
+        <div class="value">${(report.summary.successRate * 100).toFixed(1)}%</div>
+      </div>
+      <div class="metric-card">
+        <div class="label">Coherence Score</div>
+        <div class="value">${(report.coherenceScore * 100).toFixed(1)}%</div>
+      </div>
+      <div class="metric-card">
+        <div class="label">Avg Latency (p50)</div>
+        <div class="value">${report.summary.avgLatencyUs.toFixed(2)}μs</div>
+      </div>
+      <div class="metric-card">
+        <div class="label">Avg Recall@10</div>
+        <div class="value">${(report.summary.avgRecall * 100).toFixed(1)}%</div>
+      </div>
+      <div class="metric-card">
+        <div class="label">Avg QPS</div>
+        <div class="value">${report.summary.avgQps.toLocaleString()}</div>
+      </div>
+      <div class="metric-card">
+        <div class="label">Avg Memory</div>
+        <div class="value">${report.summary.avgMemoryMB.toFixed(0)}MB</div>
+      </div>
+    </div>
+
+    <h2>Variance Analysis</h2>
+    <table>
+      <thead>
+        <tr>
+          <th>Metric</th>
+          <th>Variance</th>
+        </tr>
+      </thead>
+      <tbody>
+        <tr>
+          <td>Latency</td>
+          <td>${report.varianceMetrics.latencyVariance.toFixed(4)}</td>
+        </tr>
+        <tr>
+          <td>Recall</td>
+          <td>${report.varianceMetrics.recallVariance.toFixed(6)}</td>
+        </tr>
+        <tr>
+          <td>QPS</td>
+          <td>${report.varianceMetrics.qpsVariance.toFixed(2)}</td>
+        </tr>
+      </tbody>
+    </table>
+
+    <h2>Iteration Results</h2>
+    ${report.iterations
+      .map(
+        (iter) => `
+      <div class="iteration">
+        <h3>Iteration ${iter.iteration}</h3>
+        <p><strong>Status:</strong> ${iter.success ? '✅ Success' : '❌ Failed'}</p>
+        <p><strong>Duration:</strong> ${(iter.duration / 1000).toFixed(2)}s</p>
+        <p><strong>Timestamp:</strong> ${iter.timestamp}</p>
+        ${
+          iter.success
+            ? `
+        <div class="metrics">
+          ${
+            iter.metrics.latencyUs
+              ? `
+          <div class="metric-card">
+            <div class="label">Latency p50</div>
+            <div class="value">${iter.metrics.latencyUs.p50.toFixed(2)}μs</div>
+          </div>
+          `
+              : ''
+          }
+          ${
+            iter.metrics.recallAtK
+              ? `
+          <div class="metric-card">
+            <div class="label">Recall@10</div>
+            <div class="value">${(iter.metrics.recallAtK.k10 * 100).toFixed(1)}%</div>
+          </div>
+          `
+              : ''
+          }
+          ${
+            iter.metrics.qps
+              ? `
+          <div class="metric-card">
+            <div class="label">QPS</div>
+            <div class="value">${iter.metrics.qps.toLocaleString()}</div>
+          </div>
+          `
+              : ''
+          }
+        </div>
+        `
+            : `<p><strong>Error:</strong> ${iter.error}</p>`
+        }
+      </div>
+    `
+      )
+      .join('')}
+
+    ${
+      report.warnings.length > 0
+        ? `
+    <h2>Warnings</h2>
+    <div class="warning-list">
+      <ul>
+        ${report.warnings.map((w) => `<li>⚠️ ${w}</li>`).join('')}
+      </ul>
+    </div>
+    `
+        : ''
+    }
+
+    <hr style="margin: 40px 0; border: none; border-top: 1px solid #e5e7eb;">
+    <p style="text-align: center; color: #6b7280; font-size: 14px;">
+      Generated by AgentDB v2.0.0 • ${new Date().toISOString()}
+    </p>
+  </div>
+</body>
+</html>
+    `.trim();
+
+    return html;
+  }
+
+  /**
+   * Save report to file
+   */
+  async saveReport(report: SimulationReport, outputPath: string, format: 'md' | 'json' | 'html'): Promise<string> {
+    const fs = await import('fs/promises');
+    const path = await import('path');
+
+    let content: string;
+    let extension: string;
+
+    switch (format) {
+      case 'md':
+        content = await this.generateMarkdown(report);
+        extension = 'md';
+        break;
+      case 'json':
+        content = await this.generateJSON(report);
+        extension = 'json';
+        break;
+      case 'html':
+        content = await this.generateHTML(report);
+        extension = 'html';
+        break;
+    }
+
+    // Ensure output directory exists
+    await fs.mkdir(outputPath, { recursive: true });
+
+    // Generate filename
+    const timestamp = new Date().toISOString().replace(/[:.]/g, '-');
+    const filename = `${report.scenarioId}-${timestamp}.${extension}`;
+    const fullPath = path.join(outputPath, filename);
+
+    // Write file
+    await fs.writeFile(fullPath, content, 'utf-8');
+
+    return fullPath;
+  }
+}