@vibecheckai/cli 3.9.1 → 4.0.0

package/mcp-server/proof-tools.js

@@ -1,571 +0,0 @@
-/**
- * Vibecheck MCP Proof Tools
- * 
- * Proof-specific tools for AI assistants:
- * - vibecheck.prove - Run full proof loop
- * - vibecheck.prove_status - Get current proof status
- * - vibecheck.get_evidence - Get evidence for specific finding
- * - vibecheck.check_flaky - Check if a finding is flaky
- * - vibecheck.allowlist_add - Add finding to allowlist with reason
- * - vibecheck.get_proof_graph - Get visual proof graph
- * - vibecheck.evidence_pack - Generate or retrieve evidence pack
- */
-
-import { execSync, spawn } from 'child_process';
-import fs from 'fs';
-import path from 'path';
-
-// ============================================================================
-// PROOF TOOL DEFINITIONS
-// ============================================================================
-
-export const PROOF_TOOLS = [
-  {
-    name: "vibecheck.prove",
-    description: "Run full proof loop: ctx → reality → ship → fix. Returns undeniable evidence that your app works. Use AFTER making changes to verify they're real.",
-    inputSchema: {
-      type: "object",
-      properties: {
-        url: { type: "string", description: "Base URL to test (e.g., http://localhost:3000)" },
-        projectPath: { type: "string", description: "Path to project root", default: "." },
-        auth: { type: "string", description: "Login credentials as email:password" },
-        skipFix: { type: "boolean", description: "Don't auto-fix, just diagnose", default: false },
-        stabilityRuns: { type: "number", description: "Number of runs for flakiness detection", default: 1 },
-        evidencePack: { type: "boolean", description: "Generate evidence pack", default: true }
-      },
-      required: ["url"]
-    }
-  },
-  {
-    name: "vibecheck.prove_status",
-    description: "Get the status and results of the last proof run. Returns verdict, findings, and artifact paths.",
-    inputSchema: {
-      type: "object",
-      properties: {
-        projectPath: { type: "string", description: "Path to project root", default: "." }
-      }
-    }
-  },
-  {
-    name: "vibecheck.get_evidence",
-    description: "Get detailed evidence for a specific finding including screenshots, video timestamps, and code context.",
-    inputSchema: {
-      type: "object",
-      properties: {
-        projectPath: { type: "string", description: "Path to project root", default: "." },
-        findingId: { type: "string", description: "Finding ID or fingerprint to get evidence for" }
-      },
-      required: ["findingId"]
-    }
-  },
-  {
-    name: "vibecheck.check_flaky",
-    description: "Check if a finding is flaky (inconsistent across runs). Returns stability data if available.",
-    inputSchema: {
-      type: "object",
-      properties: {
-        projectPath: { type: "string", description: "Path to project root", default: "." },
-        findingId: { type: "string", description: "Finding ID to check for flakiness" }
-      },
-      required: ["findingId"]
-    }
-  },
-  {
-    name: "vibecheck.allowlist_add",
-    description: "Add a finding to the allowlist with a reason. Use when a finding is acceptable and shouldn't block shipping.",
-    inputSchema: {
-      type: "object",
-      properties: {
-        projectPath: { type: "string", description: "Path to project root", default: "." },
-        findingId: { type: "string", description: "Finding ID or fingerprint to allowlist" },
-        reason: { type: "string", description: "Why this finding is acceptable" },
-        whyAllowed: { type: "string", description: "Detailed explanation of why this is allowed" },
-        expiresAt: { type: "string", description: "Optional expiration date (ISO format)" }
-      },
-      required: ["findingId", "reason", "whyAllowed"]
-    }
-  },
-  {
-    name: "vibecheck.get_proof_graph",
-    description: "Get the proof graph showing claims, evidence, and gaps. Visual representation of what's been verified.",
-    inputSchema: {
-      type: "object",
-      properties: {
-        projectPath: { type: "string", description: "Path to project root", default: "." },
-        format: { type: "string", enum: ["json", "mermaid"], description: "Output format", default: "json" }
-      }
-    }
-  },
-  {
-    name: "vibecheck.evidence_pack",
-    description: "Generate or retrieve the latest evidence pack with all proof artifacts (videos, traces, screenshots).",
-    inputSchema: {
-      type: "object",
-      properties: {
-        projectPath: { type: "string", description: "Path to project root", default: "." },
-        generate: { type: "boolean", description: "Generate new pack from latest results", default: false },
-        includeMedia: { type: "boolean", description: "Include video/screenshot paths", default: true }
-      }
-    }
-  }
-];
-
-// ============================================================================
-// TOOL HANDLERS
-// ============================================================================
-
-/**
- * Run full proof loop
- */
-async function handleProve(args) {
-  const { url, projectPath = '.', auth, skipFix = false, stabilityRuns = 1, evidencePack = true } = args;
-  
-  if (!url) {
-    return wrapResponse(null, { error: 'URL is required for prove command' });
-  }
-  
-  const cmdParts = ['vibecheck', 'prove', '--url', url, '--json'];
-  if (auth) cmdParts.push('--auth', auth);
-  if (skipFix) cmdParts.push('--skip-fix');
-  if (stabilityRuns > 1) cmdParts.push('--stability-runs', String(stabilityRuns));
-  if (evidencePack) cmdParts.push('--evidence-pack');
-  
-  try {
-    const result = execSync(cmdParts.join(' '), {
-      cwd: projectPath,
-      encoding: 'utf8',
-      timeout: 600000, // 10 minutes
-      maxBuffer: 50 * 1024 * 1024
-    });
-    
-    const parsed = JSON.parse(result);
-    return wrapResponse({
-      verdict: parsed.result?.finalVerdict || parsed.verdict,
-      findings: parsed.result?.findings || [],
-      coverage: parsed.result?.coverage,
-      duration: parsed.result?.duration,
-      artifacts: parsed.artifacts,
-      evidencePack: parsed.result?.evidencePackPath
-    }, {
-      evidence: (parsed.result?.findings || []).slice(0, 5).map(f => ({
-        file: f.file || f.page,
-        line: f.line,
-        snippet: f.title,
-        confidence: f.confidence || 0.9
-      }))
-    });
-  } catch (err) {
-    // Try to parse partial output
-    try {
-      const stdout = err.stdout?.toString() || '';
-      const lines = stdout.split('\n');
-      const jsonLine = lines.find(l => l.startsWith('{'));
-      if (jsonLine) {
-        const parsed = JSON.parse(jsonLine);
-        return wrapResponse(parsed, {
-          error: `Prove completed with exit code ${err.status}`
-        });
-      }
-    } catch {}
-    
-    return wrapResponse(null, { error: `Prove failed: ${err.message}` });
-  }
-}
-
-/**
- * Get last proof status
- */
-async function handleProveStatus(args) {
-  const { projectPath = '.' } = args;
-  
-  const reportPath = path.join(projectPath, '.vibecheck', 'prove', 'last_prove.json');
-  
-  if (!fs.existsSync(reportPath)) {
-    return wrapResponse(null, { error: 'No proof run found. Run vibecheck.prove first.' });
-  }
-  
-  try {
-    const report = JSON.parse(fs.readFileSync(reportPath, 'utf8'));
-    
-    // Check for evidence packs
-    const evidencePacksDir = path.join(projectPath, '.vibecheck', 'evidence-packs');
-    let latestPack = null;
-    if (fs.existsSync(evidencePacksDir)) {
-      const packs = fs.readdirSync(evidencePacksDir).sort().reverse();
-      if (packs.length > 0) {
-        latestPack = path.join(evidencePacksDir, packs[0]);
-      }
-    }
-    
-    return wrapResponse({
-      verdict: report.finalVerdict,
-      meta: report.meta,
-      timeline: report.timeline,
-      findingsCount: report.findings?.length || 0,
-      blockers: (report.findings || []).filter(f => f.severity === 'BLOCK').length,
-      warnings: (report.findings || []).filter(f => f.severity === 'WARN').length,
-      coverage: report.coverage,
-      artifacts: report.artifacts,
-      evidencePackPath: latestPack,
-      reportPath
-    });
-  } catch (err) {
-    return wrapResponse(null, { error: `Failed to read proof status: ${err.message}` });
-  }
-}
-
-/**
- * Get evidence for specific finding
- */
-async function handleGetEvidence(args) {
-  const { projectPath = '.', findingId } = args;
-  
-  // Load latest reports to find the finding
-  const reportPaths = [
-    path.join(projectPath, '.vibecheck', 'prove', 'last_prove.json'),
-    path.join(projectPath, '.vibecheck', 'reality', 'last_reality.json'),
-    path.join(projectPath, '.vibecheck', 'ship', 'last_ship.json')
-  ];
-  
-  let finding = null;
-  let source = null;
-  
-  for (const reportPath of reportPaths) {
-    if (fs.existsSync(reportPath)) {
-      try {
-        const report = JSON.parse(fs.readFileSync(reportPath, 'utf8'));
-        const findings = report.findings || [];
-        finding = findings.find(f => 
-          f.id === findingId || 
-          f.fingerprint === findingId ||
-          f.id?.includes(findingId) ||
-          f.fingerprint?.startsWith(findingId)
-        );
-        if (finding) {
-          source = reportPath;
-          break;
-        }
-      } catch {}
-    }
-  }
-  
-  if (!finding) {
-    return wrapResponse(null, { error: `Finding not found: ${findingId}` });
-  }
-  
-  // Build comprehensive evidence
-  const evidence = {
-    finding,
-    source,
-    what: {
-      type: finding.category,
-      title: finding.title,
-      detector: finding.detector || finding.category,
-      confidence: finding.confidence
-    },
-    where: {
-      file: finding.file,
-      line: finding.line,
-      page: finding.page,
-      url: finding.url
-    },
-    why: {
-      reason: finding.reason,
-      severity: finding.severity,
-      isBlocker: finding.severity === 'BLOCK'
-    },
-    artifacts: {
-      screenshot: finding.screenshot,
-      video: finding.video,
-      trace: finding.trace
-    },
-    stability: finding.stability
-  };
-  
-  return wrapResponse(evidence, {
-    evidence: [{
-      file: finding.file || finding.page,
-      line: finding.line,
-      snippet: finding.reason || finding.title,
-      confidence: finding.confidence || 0.9
-    }]
-  });
-}
-
-/**
- * Check if finding is flaky
- */
-async function handleCheckFlaky(args) {
-  const { projectPath = '.', findingId } = args;
-  
-  // Get the finding's stability data
-  const evidenceResult = await handleGetEvidence({ projectPath, findingId });
-  
-  if (!evidenceResult.ok) {
-    return evidenceResult;
-  }
-  
-  const finding = evidenceResult.data.finding;
-  const stability = finding.stability;
-  
-  if (!stability) {
-    return wrapResponse({
-      findingId,
-      hasStabilityData: false,
-      message: 'No stability data available. Run prove with --stability-runs > 1 to detect flakiness.'
-    });
-  }
-  
-  return wrapResponse({
-    findingId,
-    hasStabilityData: true,
-    isFlaky: stability.isFlaky,
-    occurrenceRate: stability.occurrenceRate,
-    flakinessScore: stability.flakinessScore,
-    appearedInRuns: stability.appearedInRuns,
-    totalRuns: stability.totalRuns,
-    recommendation: stability.isFlaky 
-      ? 'This finding is flaky - it may be due to timing issues or race conditions. Consider adding stability measures.'
-      : 'This finding is stable - it appears consistently across runs.'
-  });
-}
-
-/**
- * Add finding to allowlist
- */
-async function handleAllowlistAdd(args) {
-  const { projectPath = '.', findingId, reason, whyAllowed, expiresAt } = args;
-  
-  try {
-    // Load the unified allowlist module
-    const allowlistModule = require('../packages/cli/src/allowlist/unified-allowlist');
-    const { UnifiedAllowlist } = allowlistModule;
-    
-    const allowlist = new UnifiedAllowlist(projectPath);
-    
-    // Find the original finding to get its details
-    const evidenceResult = await handleGetEvidence({ projectPath, findingId });
-    
-    if (evidenceResult.ok && evidenceResult.data.finding) {
-      const finding = evidenceResult.data.finding;
-      
-      allowlist.addFinding(finding, {
-        reason,
-        whyAllowed,
-        approvedBy: 'AI Assistant (MCP)',
-        expiresAt
-      });
-    } else {
-      // Add with just the fingerprint
-      allowlist.add({
-        fingerprint: findingId,
-        category: 'other',
-        reason,
-        what: `Finding ${findingId}`,
-        whyAllowed,
-        approvedBy: 'AI Assistant (MCP)',
-        expiresAt
-      });
-    }
-    
-    allowlist.save();
-    
-    return wrapResponse({
-      success: true,
-      message: `Finding ${findingId} added to allowlist`,
-      reason,
-      whyAllowed,
-      expiresAt: expiresAt || 'Never'
-    });
-  } catch (err) {
-    return wrapResponse(null, { error: `Failed to add to allowlist: ${err.message}` });
-  }
-}
-
-/**
- * Get proof graph
- */
-async function handleGetProofGraph(args) {
-  const { projectPath = '.', format = 'json' } = args;
-  
-  const proofGraphPath = path.join(projectPath, '.vibecheck', 'proof-graph.json');
-  
-  if (!fs.existsSync(proofGraphPath)) {
-    // Try to find in evidence packs
-    const evidencePacksDir = path.join(projectPath, '.vibecheck', 'evidence-packs');
-    if (fs.existsSync(evidencePacksDir)) {
-      const packs = fs.readdirSync(evidencePacksDir).sort().reverse();
-      for (const pack of packs) {
-        const packGraphPath = path.join(evidencePacksDir, pack, 'proof-graph.json');
-        if (fs.existsSync(packGraphPath)) {
-          const graph = JSON.parse(fs.readFileSync(packGraphPath, 'utf8'));
-          return formatProofGraph(graph, format);
-        }
-      }
-    }
-    
-    return wrapResponse(null, { error: 'No proof graph found. Run vibecheck.prove first.' });
-  }
-  
-  const graph = JSON.parse(fs.readFileSync(proofGraphPath, 'utf8'));
-  return formatProofGraph(graph, format);
-}
-
-function formatProofGraph(graph, format) {
-  if (format === 'mermaid') {
-    // Convert to mermaid diagram
-    const lines = ['graph TD'];
-    
-    if (graph.verifiedClaims) {
-      for (const claim of graph.verifiedClaims) {
-        const claimId = `claim_${claim.id || Math.random().toString(36).slice(2, 8)}`;
-        lines.push(`    ${claimId}[${claim.claim || claim.title}]`);
-        lines.push(`    ${claimId} -->|verified| evidence_${claimId}`);
-        lines.push(`    evidence_${claimId}((Evidence))`);
-      }
-    }
-    
-    if (graph.gaps) {
-      for (const gap of graph.gaps) {
-        const gapId = `gap_${Math.random().toString(36).slice(2, 8)}`;
-        lines.push(`    ${gapId}[/${gap.description || gap}/]`);
-        lines.push(`    style ${gapId} fill:#f99`);
-      }
-    }
-    
-    return wrapResponse({
-      format: 'mermaid',
-      diagram: lines.join('\n')
-    });
-  }
-  
-  return wrapResponse({
-    format: 'json',
-    graph
-  });
-}
-
-/**
- * Generate or get evidence pack
- */
-async function handleEvidencePack(args) {
-  const { projectPath = '.', generate = false, includeMedia = true } = args;
-  
-  if (generate) {
-    try {
-      const evidencePackModule = require('../packages/cli/src/evidence/evidence-pack');
-      const { generateEvidencePack } = evidencePackModule;
-      
-      const result = await generateEvidencePack({
-        projectPath,
-        sourceType: 'prove',
-        includeVideos: includeMedia,
-        includeTraces: includeMedia,
-        includeScreenshots: includeMedia,
-        includeHar: true
-      });
-      
-      return wrapResponse({
-        generated: true,
-        packPath: result.packPath,
-        manifest: result.manifest
-      });
-    } catch (err) {
-      return wrapResponse(null, { error: `Failed to generate evidence pack: ${err.message}` });
-    }
-  }
-  
-  // Return latest evidence pack
-  const evidencePacksDir = path.join(projectPath, '.vibecheck', 'evidence-packs');
-  
-  if (!fs.existsSync(evidencePacksDir)) {
-    return wrapResponse(null, { 
-      error: 'No evidence packs found. Run vibecheck.prove with --evidence-pack or use generate: true.' 
-    });
-  }
-  
-  const packs = fs.readdirSync(evidencePacksDir).sort().reverse();
-  if (packs.length === 0) {
-    return wrapResponse(null, { error: 'No evidence packs found.' });
-  }
-  
-  const latestPackDir = path.join(evidencePacksDir, packs[0]);
-  const manifestPath = path.join(latestPackDir, 'manifest.json');
-  
-  if (!fs.existsSync(manifestPath)) {
-    return wrapResponse(null, { error: 'Evidence pack manifest not found.' });
-  }
-  
-  const manifest = JSON.parse(fs.readFileSync(manifestPath, 'utf8'));
-  
-  return wrapResponse({
-    generated: false,
-    packPath: latestPackDir,
-    manifest,
-    htmlViewer: fs.existsSync(path.join(latestPackDir, 'summary.html')) 
-      ? path.join(latestPackDir, 'summary.html') 
-      : null
-  });
-}
-
-// ============================================================================
-// RESPONSE WRAPPER
-// ============================================================================
-
-function wrapResponse(data, options = {}) {
-  const { evidence = [], cached = false, error = null } = options;
-  
-  if (error) {
-    return {
-      ok: false,
-      error: typeof error === 'string' ? error : error.message,
-      data: null,
-      evidence: [],
-      metadata: {
-        timestamp: new Date().toISOString(),
-        cached: false
-      }
-    };
-  }
-  
-  return {
-    ok: true,
-    data,
-    evidence: evidence.map(e => ({
-      file: e.file || null,
-      line: e.line || e.lines || null,
-      snippet: e.snippet || e.code || null,
-      confidence: e.confidence || 0.9,
-      reason: e.reason || null
-    })),
-    metadata: {
-      timestamp: new Date().toISOString(),
-      cached
-    }
-  };
-}
-
-// ============================================================================
-// HANDLER DISPATCH
-// ============================================================================
-
-export async function handleProofTool(toolName, args) {
-  const handlers = {
-    'vibecheck.prove': handleProve,
-    'vibecheck.prove_status': handleProveStatus,
-    'vibecheck.get_evidence': handleGetEvidence,
-    'vibecheck.check_flaky': handleCheckFlaky,
-    'vibecheck.allowlist_add': handleAllowlistAdd,
-    'vibecheck.get_proof_graph': handleGetProofGraph,
-    'vibecheck.evidence_pack': handleEvidencePack
-  };
-  
-  const handler = handlers[toolName];
-  if (!handler) {
-    return wrapResponse(null, { error: `Unknown proof tool: ${toolName}` });
-  }
-  
-  return handler(args);
-}
-
-export default { PROOF_TOOLS, handleProofTool };