@vibecheckai/cli 3.1.8 → 3.2.0

package/bin/runners/lib/evidence-pack.js

@@ -0,0 +1,678 @@
+/**
+ * Evidence Pack Builder
+ * 
+ * Creates shareable "evidence packs" from vibecheck proof runs.
+ * Bundles videos, traces, screenshots, and findings into a single artifact.
+ * 
+ * Usage:
+ *   const { buildEvidencePack } = require('./lib/evidence-pack');
+ *   const pack = await buildEvidencePack(projectRoot, { includeVideos: true });
+ */
+
+"use strict";
+
+const fs = require("fs");
+const path = require("path");
+const crypto = require("crypto");
+const archiver = require("archiver");
+
+// ═══════════════════════════════════════════════════════════════════════════════
+// EVIDENCE PACK SCHEMA
+// ═══════════════════════════════════════════════════════════════════════════════
+
+/**
+ * Evidence Pack Structure:
+ * {
+ *   meta: { ... },
+ *   summary: { ... },
+ *   findings: [...],
+ *   allowlist: { ... },
+ *   artifacts: { ... }
+ * }
+ */
+
+const EVIDENCE_PACK_VERSION = "1.0.0";
+
+// ═══════════════════════════════════════════════════════════════════════════════
+// ALLOWLIST SCHEMA
+// ═══════════════════════════════════════════════════════════════════════════════
+
+/**
+ * Allowlist Entry Structure:
+ * {
+ *   id: string,
+ *   findingId: string,
+ *   pattern: string | RegExp,
+ *   reason: string,          // Why this is allowed
+ *   addedAt: ISO timestamp,
+ *   addedBy: string,         // user/ci/auto
+ *   expiresAt?: ISO timestamp,
+ *   scope: 'global' | 'file' | 'line',
+ *   file?: string,
+ *   lines?: [start, end]
+ * }
+ */
+
+const DEFAULT_ALLOWLIST_PATH = ".vibecheck/allowlist.json";
+
+function loadAllowlist(projectRoot) {
+  const allowlistPath = path.join(projectRoot, DEFAULT_ALLOWLIST_PATH);
+  try {
+    return JSON.parse(fs.readFileSync(allowlistPath, "utf8"));
+  } catch {
+    return { version: "1.0.0", entries: [], lastUpdated: null };
+  }
+}
+
+function saveAllowlist(projectRoot, allowlist) {
+  const allowlistPath = path.join(projectRoot, DEFAULT_ALLOWLIST_PATH);
+  fs.mkdirSync(path.dirname(allowlistPath), { recursive: true });
+  allowlist.lastUpdated = new Date().toISOString();
+  fs.writeFileSync(allowlistPath, JSON.stringify(allowlist, null, 2), "utf8");
+  return allowlistPath;
+}
+
+function addToAllowlist(projectRoot, entry) {
+  const allowlist = loadAllowlist(projectRoot);
+  
+  // Generate ID if not provided
+  if (!entry.id) {
+    entry.id = `AL_${crypto.randomBytes(4).toString("hex")}`;
+  }
+  
+  // Set defaults
+  entry.addedAt = entry.addedAt || new Date().toISOString();
+  entry.addedBy = entry.addedBy || "user";
+  entry.scope = entry.scope || "global";
+  
+  // Check for duplicates
+  const existing = allowlist.entries.find(e => 
+    e.pattern === entry.pattern && e.scope === entry.scope
+  );
+  
+  if (existing) {
+    // Update existing entry
+    Object.assign(existing, entry);
+  } else {
+    allowlist.entries.push(entry);
+  }
+  
+  saveAllowlist(projectRoot, allowlist);
+  return entry;
+}
+
+function isAllowlisted(finding, allowlist) {
+  if (!allowlist || !allowlist.entries) return false;
+  
+  const now = new Date();
+  
+  for (const entry of allowlist.entries) {
+    // Check expiration
+    if (entry.expiresAt && new Date(entry.expiresAt) < now) {
+      continue;
+    }
+    
+    // Check by finding ID
+    if (entry.findingId && entry.findingId === finding.id) {
+      return { allowed: true, reason: entry.reason, entry };
+    }
+    
+    // Check by pattern
+    if (entry.pattern) {
+      const pattern = typeof entry.pattern === 'string' 
+        ? new RegExp(entry.pattern, 'i')
+        : entry.pattern;
+      
+      // Match against finding title, page, or file
+      if (
+        pattern.test(finding.title || '') ||
+        pattern.test(finding.page || '') ||
+        pattern.test(finding.file || '') ||
+        (finding.evidence && finding.evidence.some(e => pattern.test(e.file || '')))
+      ) {
+        // Check scope
+        if (entry.scope === 'global') {
+          return { allowed: true, reason: entry.reason, entry };
+        }
+        if (entry.scope === 'file' && entry.file === finding.file) {
+          return { allowed: true, reason: entry.reason, entry };
+        }
+        if (entry.scope === 'line' && entry.file === finding.file) {
+          const findingLine = finding.line || (finding.evidence?.[0]?.line);
+          if (findingLine && entry.lines) {
+            const [start, end] = entry.lines;
+            if (findingLine >= start && findingLine <= end) {
+              return { allowed: true, reason: entry.reason, entry };
+            }
+          }
+        }
+      }
+    }
+  }
+  
+  return { allowed: false };
+}
+
+function filterByAllowlist(findings, allowlist) {
+  const filtered = [];
+  const allowed = [];
+  
+  for (const finding of findings) {
+    const result = isAllowlisted(finding, allowlist);
+    if (result.allowed) {
+      allowed.push({ 
+        finding, 
+        reason: result.reason,
+        entryId: result.entry?.id 
+      });
+    } else {
+      filtered.push(finding);
+    }
+  }
+  
+  return { filtered, allowed };
+}
+
+// ═══════════════════════════════════════════════════════════════════════════════
+// EVIDENCE STRUCTURE (What, Where, Why)
+// ═══════════════════════════════════════════════════════════════════════════════
+
+/**
+ * Enhanced Evidence Structure:
+ * {
+ *   what: string,           // What was found
+ *   where: {               // Where it was found
+ *     file: string,
+ *     line: number,
+ *     column?: number,
+ *     snippet?: string,
+ *     url?: string          // For runtime findings
+ *   },
+ *   why: string,            // Why this is a problem
+ *   confidence: number,     // 0.0 - 1.0
+ *   category: string,
+ *   severity: 'BLOCK' | 'WARN' | 'INFO',
+ *   fixHint?: string,
+ *   related?: Evidence[]
+ * }
+ */
+
+function enrichEvidence(finding) {
+  const evidence = {
+    what: finding.title || finding.message || 'Unknown finding',
+    where: {},
+    why: finding.reason || finding.why || 'Potential issue detected',
+    confidence: finding.confidence || 0.8,
+    category: finding.category || 'Unknown',
+    severity: finding.severity || 'WARN'
+  };
+  
+  // Populate where
+  if (finding.file) {
+    evidence.where.file = finding.file;
+  }
+  if (finding.line) {
+    evidence.where.line = finding.line;
+  }
+  if (finding.page || finding.url) {
+    evidence.where.url = finding.page || finding.url;
+  }
+  if (finding.snippet || finding.code) {
+    evidence.where.snippet = finding.snippet || finding.code;
+  }
+  if (finding.screenshot) {
+    evidence.where.screenshot = finding.screenshot;
+  }
+  
+  // Add legacy evidence array if present
+  if (finding.evidence && Array.isArray(finding.evidence)) {
+    evidence.where.file = evidence.where.file || finding.evidence[0]?.file;
+    evidence.where.line = evidence.where.line || finding.evidence[0]?.line;
+    evidence.where.snippet = evidence.where.snippet || finding.evidence[0]?.snippet;
+  }
+  
+  // Add fix hint
+  if (finding.fixHints && finding.fixHints.length > 0) {
+    evidence.fixHint = finding.fixHints[0];
+  } else if (finding.fix) {
+    evidence.fixHint = finding.fix;
+  }
+  
+  return evidence;
+}
+
+// ═══════════════════════════════════════════════════════════════════════════════
+// EVIDENCE PACK BUILDER
+// ═══════════════════════════════════════════════════════════════════════════════
+
+async function buildEvidencePack(projectRoot, options = {}) {
+  const {
+    includeVideos = true,
+    includeTraces = true,
+    includeScreenshots = true,
+    includeHar = true,
+    outputPath = null,
+    proveDir = null,
+    realityDir = null,
+    applyAllowlist = true
+  } = options;
+  
+  const outDir = path.join(projectRoot, ".vibecheck", "evidence-packs");
+  fs.mkdirSync(outDir, { recursive: true });
+  
+  // Load latest reports
+  const proveReportPath = proveDir 
+    ? path.join(proveDir, "prove_report.json")
+    : findLatestReport(projectRoot, "prove");
+  const realityReportPath = realityDir
+    ? path.join(realityDir, "reality_report.json")
+    : findLatestReport(projectRoot, "reality");
+  const shipReportPath = path.join(projectRoot, ".vibecheck", "last_ship.json");
+  
+  let proveReport = null;
+  let realityReport = null;
+  let shipReport = null;
+  
+  try { proveReport = JSON.parse(fs.readFileSync(proveReportPath, "utf8")); } catch {}
+  try { realityReport = JSON.parse(fs.readFileSync(realityReportPath, "utf8")); } catch {}
+  try { shipReport = JSON.parse(fs.readFileSync(shipReportPath, "utf8")); } catch {}
+  
+  if (!proveReport && !realityReport && !shipReport) {
+    throw new Error("No proof reports found. Run vibecheck prove or reality first.");
+  }
+  
+  // Load allowlist
+  const allowlist = applyAllowlist ? loadAllowlist(projectRoot) : null;
+  
+  // Collect all findings
+  let allFindings = [];
+  if (proveReport?.findings) allFindings.push(...proveReport.findings);
+  if (realityReport?.findings) allFindings.push(...realityReport.findings);
+  if (shipReport?.findings) allFindings.push(...shipReport.findings);
+  
+  // Dedupe findings by ID
+  const seenIds = new Set();
+  allFindings = allFindings.filter(f => {
+    if (seenIds.has(f.id)) return false;
+    seenIds.add(f.id);
+    return true;
+  });
+  
+  // Apply allowlist filtering
+  let findings = allFindings;
+  let allowedFindings = [];
+  if (allowlist) {
+    const result = filterByAllowlist(allFindings, allowlist);
+    findings = result.filtered;
+    allowedFindings = result.allowed;
+  }
+  
+  // Enrich evidence
+  const enrichedFindings = findings.map(enrichEvidence);
+  
+  // Build summary
+  const summary = {
+    totalFindings: allFindings.length,
+    filteredFindings: findings.length,
+    allowlistedCount: allowedFindings.length,
+    blocks: findings.filter(f => f.severity === 'BLOCK').length,
+    warns: findings.filter(f => f.severity === 'WARN').length,
+    verdict: proveReport?.verdict || shipReport?.meta?.verdict || 'UNKNOWN',
+    coverage: realityReport?.coverage || null
+  };
+  
+  // Collect artifacts
+  const artifacts = {
+    screenshots: [],
+    videos: [],
+    traces: [],
+    har: []
+  };
+  
+  // Find artifact directories
+  const realityBase = realityReportPath ? path.dirname(realityReportPath) : null;
+  
+  if (realityBase) {
+    // Screenshots
+    if (includeScreenshots) {
+      const shotsDir = path.join(realityBase, "screenshots");
+      if (fs.existsSync(shotsDir)) {
+        artifacts.screenshots = fs.readdirSync(shotsDir)
+          .filter(f => /\.(png|jpg|jpeg)$/i.test(f))
+          .map(f => path.join(shotsDir, f));
+      }
+    }
+    
+    // Videos
+    if (includeVideos) {
+      const videosDir = path.join(realityBase, "videos");
+      if (fs.existsSync(videosDir)) {
+        artifacts.videos = fs.readdirSync(videosDir)
+          .filter(f => /\.(webm|mp4)$/i.test(f))
+          .map(f => path.join(videosDir, f));
+      }
+    }
+    
+    // Traces
+    if (includeTraces) {
+      const tracesDir = path.join(realityBase, "traces");
+      if (fs.existsSync(tracesDir)) {
+        artifacts.traces = fs.readdirSync(tracesDir)
+          .filter(f => /\.zip$/i.test(f))
+          .map(f => path.join(tracesDir, f));
+      }
+    }
+    
+    // HAR
+    if (includeHar) {
+      const harDir = path.join(realityBase, "har");
+      if (fs.existsSync(harDir)) {
+        artifacts.har = fs.readdirSync(harDir)
+          .filter(f => /\.har$/i.test(f))
+          .map(f => path.join(harDir, f));
+      }
+    }
+  }
+  
+  // Build evidence pack manifest
+  const packId = `EP_${Date.now()}_${crypto.randomBytes(4).toString("hex")}`;
+  const manifest = {
+    id: packId,
+    version: EVIDENCE_PACK_VERSION,
+    meta: {
+      createdAt: new Date().toISOString(),
+      projectName: path.basename(projectRoot),
+      projectRoot: projectRoot,
+      proveReport: proveReportPath ? path.relative(projectRoot, proveReportPath) : null,
+      realityReport: realityReportPath ? path.relative(projectRoot, realityReportPath) : null,
+      shipReport: shipReportPath && fs.existsSync(shipReportPath) ? path.relative(projectRoot, shipReportPath) : null
+    },
+    summary,
+    findings: enrichedFindings,
+    allowlist: allowlist ? {
+      applied: true,
+      entriesCount: allowlist.entries?.length || 0,
+      allowedFindings: allowedFindings.map(a => ({
+        findingId: a.finding.id,
+        reason: a.reason
+      }))
+    } : { applied: false },
+    artifacts: {
+      screenshots: artifacts.screenshots.map(p => path.relative(projectRoot, p)),
+      videos: artifacts.videos.map(p => path.relative(projectRoot, p)),
+      traces: artifacts.traces.map(p => path.relative(projectRoot, p)),
+      har: artifacts.har.map(p => path.relative(projectRoot, p))
+    }
+  };
+  
+  // Write manifest
+  const manifestPath = path.join(outDir, `${packId}_manifest.json`);
+  fs.writeFileSync(manifestPath, JSON.stringify(manifest, null, 2), "utf8");
+  
+  // Create zip bundle if requested
+  let zipPath = null;
+  if (outputPath || artifacts.videos.length > 0 || artifacts.traces.length > 0) {
+    zipPath = outputPath || path.join(outDir, `${packId}.zip`);
+    await createEvidenceZip(projectRoot, manifest, artifacts, zipPath);
+  }
+  
+  return {
+    id: packId,
+    manifest,
+    manifestPath,
+    zipPath,
+    summary
+  };
+}
+
+async function createEvidenceZip(projectRoot, manifest, artifacts, outputPath) {
+  return new Promise((resolve, reject) => {
+    // Check if archiver is available, otherwise skip zip creation
+    let archiver;
+    try {
+      archiver = require("archiver");
+    } catch {
+      // archiver not installed, skip zip creation
+      resolve({ skipped: true, reason: "archiver not installed" });
+      return;
+    }
+    
+    const output = fs.createWriteStream(outputPath);
+    const archive = archiver("zip", { zlib: { level: 9 } });
+    
+    output.on("close", () => resolve({ path: outputPath, size: archive.pointer() }));
+    archive.on("error", reject);
+    
+    archive.pipe(output);
+    
+    // Add manifest
+    archive.append(JSON.stringify(manifest, null, 2), { name: "manifest.json" });
+    
+    // Add artifacts
+    for (const screenshot of artifacts.screenshots) {
+      if (fs.existsSync(screenshot)) {
+        archive.file(screenshot, { name: `screenshots/${path.basename(screenshot)}` });
+      }
+    }
+    
+    for (const video of artifacts.videos) {
+      if (fs.existsSync(video)) {
+        archive.file(video, { name: `videos/${path.basename(video)}` });
+      }
+    }
+    
+    for (const trace of artifacts.traces) {
+      if (fs.existsSync(trace)) {
+        archive.file(trace, { name: `traces/${path.basename(trace)}` });
+      }
+    }
+    
+    for (const har of artifacts.har) {
+      if (fs.existsSync(har)) {
+        archive.file(har, { name: `har/${path.basename(har)}` });
+      }
+    }
+    
+    archive.finalize();
+  });
+}
+
+function findLatestReport(projectRoot, type) {
+  const baseDir = path.join(projectRoot, ".vibecheck", type);
+  
+  // Check for latest.json pointer
+  const latestPath = path.join(baseDir, "latest.json");
+  try {
+    const latest = JSON.parse(fs.readFileSync(latestPath, "utf8"));
+    const reportPath = path.join(projectRoot, latest.latest, `${type}_report.json`);
+    if (fs.existsSync(reportPath)) return reportPath;
+  } catch {}
+  
+  // Fall back to scanning directories
+  try {
+    const dirs = fs.readdirSync(baseDir)
+      .filter(d => /^\d{8}_\d{6}$/.test(d))
+      .sort()
+      .reverse();
+    
+    for (const dir of dirs) {
+      const reportPath = path.join(baseDir, dir, `${type}_report.json`);
+      if (fs.existsSync(reportPath)) return reportPath;
+    }
+  } catch {}
+  
+  return null;
+}
+
+// ═══════════════════════════════════════════════════════════════════════════════
+// MARKDOWN REPORT GENERATOR
+// ═══════════════════════════════════════════════════════════════════════════════
+
+function generateMarkdownReport(pack) {
+  const { manifest, summary } = pack;
+  const verdict = summary.verdict;
+  const verdictEmoji = verdict === 'SHIP' ? '✅' : verdict === 'WARN' ? '⚠️' : '🛑';
+  
+  let md = `# Evidence Pack Report
+
+## ${verdictEmoji} Verdict: ${verdict}
+
+**Generated:** ${manifest.meta.createdAt}  
+**Project:** ${manifest.meta.projectName}
+
+---
+
+## Summary
+
+| Metric | Value |
+|--------|-------|
+| Total Findings | ${summary.totalFindings} |
+| Filtered (after allowlist) | ${summary.filteredFindings} |
+| Allowlisted | ${summary.allowlistedCount} |
+| **BLOCK** | ${summary.blocks} |
+| **WARN** | ${summary.warns} |
+
+`;
+
+  if (summary.coverage) {
+    md += `
+### Coverage
+
+- **Pages Hit:** ${summary.coverage.hit} / ${summary.coverage.total}
+- **Coverage:** ${summary.coverage.percent}%
+
+`;
+  }
+
+  if (manifest.findings.length > 0) {
+    md += `
+---
+
+## Findings
+
+`;
+    
+    for (const finding of manifest.findings.slice(0, 20)) {
+      const severityIcon = finding.severity === 'BLOCK' ? '🛑' : finding.severity === 'WARN' ? '⚠️' : 'ℹ️';
+      
+      md += `### ${severityIcon} ${finding.what}
+
+- **Category:** ${finding.category}
+- **Confidence:** ${Math.round(finding.confidence * 100)}%
+`;
+      
+      if (finding.where.file) {
+        md += `- **File:** \`${finding.where.file}\``;
+        if (finding.where.line) md += `:${finding.where.line}`;
+        md += '\n';
+      }
+      
+      if (finding.where.url) {
+        md += `- **URL:** ${finding.where.url}\n`;
+      }
+      
+      if (finding.where.screenshot) {
+        md += `- **Screenshot:** ${finding.where.screenshot}\n`;
+      }
+      
+      md += `
+**Why:** ${finding.why}
+`;
+      
+      if (finding.fixHint) {
+        md += `
+**Fix:** ${finding.fixHint}
+`;
+      }
+      
+      md += '\n';
+    }
+    
+    if (manifest.findings.length > 20) {
+      md += `\n_...and ${manifest.findings.length - 20} more findings. See manifest.json for full details._\n`;
+    }
+  }
+
+  if (manifest.artifacts.videos.length > 0 || manifest.artifacts.traces.length > 0) {
+    md += `
+---
+
+## Visual Artifacts
+
+`;
+    
+    if (manifest.artifacts.videos.length > 0) {
+      md += `### Videos
+
+`;
+      for (const video of manifest.artifacts.videos) {
+        md += `- [${path.basename(video)}](${video})\n`;
+      }
+    }
+    
+    if (manifest.artifacts.traces.length > 0) {
+      md += `
+### Traces
+
+View traces at [trace.playwright.dev](https://trace.playwright.dev)
+
+`;
+      for (const trace of manifest.artifacts.traces) {
+        md += `- [${path.basename(trace)}](${trace})\n`;
+      }
+    }
+  }
+
+  if (manifest.allowlist.applied && manifest.allowlist.allowedFindings.length > 0) {
+    md += `
+---
+
+## Allowlisted Findings
+
+The following findings were filtered by the allowlist:
+
+| Finding ID | Reason |
+|------------|--------|
+`;
+    
+    for (const allowed of manifest.allowlist.allowedFindings.slice(0, 10)) {
+      md += `| \`${allowed.findingId}\` | ${allowed.reason || 'No reason provided'} |\n`;
+    }
+    
+    if (manifest.allowlist.allowedFindings.length > 10) {
+      md += `\n_...and ${manifest.allowlist.allowedFindings.length - 10} more._\n`;
+    }
+  }
+
+  md += `
+---
+
+_Generated by vibecheck evidence-pack v${EVIDENCE_PACK_VERSION}_
+`;
+
+  return md;
+}
+
+// ═══════════════════════════════════════════════════════════════════════════════
+// EXPORTS
+// ═══════════════════════════════════════════════════════════════════════════════
+
+module.exports = {
+  // Core functions
+  buildEvidencePack,
+  generateMarkdownReport,
+  
+  // Allowlist functions
+  loadAllowlist,
+  saveAllowlist,
+  addToAllowlist,
+  isAllowlisted,
+  filterByAllowlist,
+  
+  // Evidence helpers
+  enrichEvidence,
+  
+  // Constants
+  EVIDENCE_PACK_VERSION,
+  DEFAULT_ALLOWLIST_PATH
+};