@vibecheckai/cli 3.1.8 → 3.2.0

package/mcp-server/proof-tools.js

@@ -0,0 +1,571 @@
+/**
+ * Vibecheck MCP Proof Tools
+ * 
+ * Proof-specific tools for AI assistants:
+ * - vibecheck.prove - Run full proof loop
+ * - vibecheck.prove_status - Get current proof status
+ * - vibecheck.get_evidence - Get evidence for specific finding
+ * - vibecheck.check_flaky - Check if a finding is flaky
+ * - vibecheck.allowlist_add - Add finding to allowlist with reason
+ * - vibecheck.get_proof_graph - Get visual proof graph
+ * - vibecheck.evidence_pack - Generate or retrieve evidence pack
+ */
+
+import { execSync, spawn } from 'child_process';
+import fs from 'fs';
+import path from 'path';
+
+// ============================================================================
+// PROOF TOOL DEFINITIONS
+// ============================================================================
+
+export const PROOF_TOOLS = [
+  {
+    name: "vibecheck.prove",
+    description: "Run full proof loop: ctx → reality → ship → fix. Returns undeniable evidence that your app works. Use AFTER making changes to verify they're real.",
+    inputSchema: {
+      type: "object",
+      properties: {
+        url: { type: "string", description: "Base URL to test (e.g., http://localhost:3000)" },
+        projectPath: { type: "string", description: "Path to project root", default: "." },
+        auth: { type: "string", description: "Login credentials as email:password" },
+        skipFix: { type: "boolean", description: "Don't auto-fix, just diagnose", default: false },
+        stabilityRuns: { type: "number", description: "Number of runs for flakiness detection", default: 1 },
+        evidencePack: { type: "boolean", description: "Generate evidence pack", default: true }
+      },
+      required: ["url"]
+    }
+  },
+  {
+    name: "vibecheck.prove_status",
+    description: "Get the status and results of the last proof run. Returns verdict, findings, and artifact paths.",
+    inputSchema: {
+      type: "object",
+      properties: {
+        projectPath: { type: "string", description: "Path to project root", default: "." }
+      }
+    }
+  },
+  {
+    name: "vibecheck.get_evidence",
+    description: "Get detailed evidence for a specific finding including screenshots, video timestamps, and code context.",
+    inputSchema: {
+      type: "object",
+      properties: {
+        projectPath: { type: "string", description: "Path to project root", default: "." },
+        findingId: { type: "string", description: "Finding ID or fingerprint to get evidence for" }
+      },
+      required: ["findingId"]
+    }
+  },
+  {
+    name: "vibecheck.check_flaky",
+    description: "Check if a finding is flaky (inconsistent across runs). Returns stability data if available.",
+    inputSchema: {
+      type: "object",
+      properties: {
+        projectPath: { type: "string", description: "Path to project root", default: "." },
+        findingId: { type: "string", description: "Finding ID to check for flakiness" }
+      },
+      required: ["findingId"]
+    }
+  },
+  {
+    name: "vibecheck.allowlist_add",
+    description: "Add a finding to the allowlist with a reason. Use when a finding is acceptable and shouldn't block shipping.",
+    inputSchema: {
+      type: "object",
+      properties: {
+        projectPath: { type: "string", description: "Path to project root", default: "." },
+        findingId: { type: "string", description: "Finding ID or fingerprint to allowlist" },
+        reason: { type: "string", description: "Why this finding is acceptable" },
+        whyAllowed: { type: "string", description: "Detailed explanation of why this is allowed" },
+        expiresAt: { type: "string", description: "Optional expiration date (ISO format)" }
+      },
+      required: ["findingId", "reason", "whyAllowed"]
+    }
+  },
+  {
+    name: "vibecheck.get_proof_graph",
+    description: "Get the proof graph showing claims, evidence, and gaps. Visual representation of what's been verified.",
+    inputSchema: {
+      type: "object",
+      properties: {
+        projectPath: { type: "string", description: "Path to project root", default: "." },
+        format: { type: "string", enum: ["json", "mermaid"], description: "Output format", default: "json" }
+      }
+    }
+  },
+  {
+    name: "vibecheck.evidence_pack",
+    description: "Generate or retrieve the latest evidence pack with all proof artifacts (videos, traces, screenshots).",
+    inputSchema: {
+      type: "object",
+      properties: {
+        projectPath: { type: "string", description: "Path to project root", default: "." },
+        generate: { type: "boolean", description: "Generate new pack from latest results", default: false },
+        includeMedia: { type: "boolean", description: "Include video/screenshot paths", default: true }
+      }
+    }
+  }
+];
+
+// ============================================================================
+// TOOL HANDLERS
+// ============================================================================
+
+/**
+ * Run full proof loop
+ */
+async function handleProve(args) {
+  const { url, projectPath = '.', auth, skipFix = false, stabilityRuns = 1, evidencePack = true } = args;
+  
+  if (!url) {
+    return wrapResponse(null, { error: 'URL is required for prove command' });
+  }
+  
+  const cmdParts = ['vibecheck', 'prove', '--url', url, '--json'];
+  if (auth) cmdParts.push('--auth', auth);
+  if (skipFix) cmdParts.push('--skip-fix');
+  if (stabilityRuns > 1) cmdParts.push('--stability-runs', String(stabilityRuns));
+  if (evidencePack) cmdParts.push('--evidence-pack');
+  
+  try {
+    const result = execSync(cmdParts.join(' '), {
+      cwd: projectPath,
+      encoding: 'utf8',
+      timeout: 600000, // 10 minutes
+      maxBuffer: 50 * 1024 * 1024
+    });
+    
+    const parsed = JSON.parse(result);
+    return wrapResponse({
+      verdict: parsed.result?.finalVerdict || parsed.verdict,
+      findings: parsed.result?.findings || [],
+      coverage: parsed.result?.coverage,
+      duration: parsed.result?.duration,
+      artifacts: parsed.artifacts,
+      evidencePack: parsed.result?.evidencePackPath
+    }, {
+      evidence: (parsed.result?.findings || []).slice(0, 5).map(f => ({
+        file: f.file || f.page,
+        line: f.line,
+        snippet: f.title,
+        confidence: f.confidence || 0.9
+      }))
+    });
+  } catch (err) {
+    // Try to parse partial output
+    try {
+      const stdout = err.stdout?.toString() || '';
+      const lines = stdout.split('\n');
+      const jsonLine = lines.find(l => l.startsWith('{'));
+      if (jsonLine) {
+        const parsed = JSON.parse(jsonLine);
+        return wrapResponse(parsed, {
+          error: `Prove completed with exit code ${err.status}`
+        });
+      }
+    } catch {}
+    
+    return wrapResponse(null, { error: `Prove failed: ${err.message}` });
+  }
+}
+
+/**
+ * Get last proof status
+ */
+async function handleProveStatus(args) {
+  const { projectPath = '.' } = args;
+  
+  const reportPath = path.join(projectPath, '.vibecheck', 'prove', 'last_prove.json');
+  
+  if (!fs.existsSync(reportPath)) {
+    return wrapResponse(null, { error: 'No proof run found. Run vibecheck.prove first.' });
+  }
+  
+  try {
+    const report = JSON.parse(fs.readFileSync(reportPath, 'utf8'));
+    
+    // Check for evidence packs
+    const evidencePacksDir = path.join(projectPath, '.vibecheck', 'evidence-packs');
+    let latestPack = null;
+    if (fs.existsSync(evidencePacksDir)) {
+      const packs = fs.readdirSync(evidencePacksDir).sort().reverse();
+      if (packs.length > 0) {
+        latestPack = path.join(evidencePacksDir, packs[0]);
+      }
+    }
+    
+    return wrapResponse({
+      verdict: report.finalVerdict,
+      meta: report.meta,
+      timeline: report.timeline,
+      findingsCount: report.findings?.length || 0,
+      blockers: (report.findings || []).filter(f => f.severity === 'BLOCK').length,
+      warnings: (report.findings || []).filter(f => f.severity === 'WARN').length,
+      coverage: report.coverage,
+      artifacts: report.artifacts,
+      evidencePackPath: latestPack,
+      reportPath
+    });
+  } catch (err) {
+    return wrapResponse(null, { error: `Failed to read proof status: ${err.message}` });
+  }
+}
+
+/**
+ * Get evidence for specific finding
+ */
+async function handleGetEvidence(args) {
+  const { projectPath = '.', findingId } = args;
+  
+  // Load latest reports to find the finding
+  const reportPaths = [
+    path.join(projectPath, '.vibecheck', 'prove', 'last_prove.json'),
+    path.join(projectPath, '.vibecheck', 'reality', 'last_reality.json'),
+    path.join(projectPath, '.vibecheck', 'ship', 'last_ship.json')
+  ];
+  
+  let finding = null;
+  let source = null;
+  
+  for (const reportPath of reportPaths) {
+    if (fs.existsSync(reportPath)) {
+      try {
+        const report = JSON.parse(fs.readFileSync(reportPath, 'utf8'));
+        const findings = report.findings || [];
+        finding = findings.find(f => 
+          f.id === findingId || 
+          f.fingerprint === findingId ||
+          f.id?.includes(findingId) ||
+          f.fingerprint?.startsWith(findingId)
+        );
+        if (finding) {
+          source = reportPath;
+          break;
+        }
+      } catch {}
+    }
+  }
+  
+  if (!finding) {
+    return wrapResponse(null, { error: `Finding not found: ${findingId}` });
+  }
+  
+  // Build comprehensive evidence
+  const evidence = {
+    finding,
+    source,
+    what: {
+      type: finding.category,
+      title: finding.title,
+      detector: finding.detector || finding.category,
+      confidence: finding.confidence
+    },
+    where: {
+      file: finding.file,
+      line: finding.line,
+      page: finding.page,
+      url: finding.url
+    },
+    why: {
+      reason: finding.reason,
+      severity: finding.severity,
+      isBlocker: finding.severity === 'BLOCK'
+    },
+    artifacts: {
+      screenshot: finding.screenshot,
+      video: finding.video,
+      trace: finding.trace
+    },
+    stability: finding.stability
+  };
+  
+  return wrapResponse(evidence, {
+    evidence: [{
+      file: finding.file || finding.page,
+      line: finding.line,
+      snippet: finding.reason || finding.title,
+      confidence: finding.confidence || 0.9
+    }]
+  });
+}
+
+/**
+ * Check if finding is flaky
+ */
+async function handleCheckFlaky(args) {
+  const { projectPath = '.', findingId } = args;
+  
+  // Get the finding's stability data
+  const evidenceResult = await handleGetEvidence({ projectPath, findingId });
+  
+  if (!evidenceResult.ok) {
+    return evidenceResult;
+  }
+  
+  const finding = evidenceResult.data.finding;
+  const stability = finding.stability;
+  
+  if (!stability) {
+    return wrapResponse({
+      findingId,
+      hasStabilityData: false,
+      message: 'No stability data available. Run prove with --stability-runs > 1 to detect flakiness.'
+    });
+  }
+  
+  return wrapResponse({
+    findingId,
+    hasStabilityData: true,
+    isFlaky: stability.isFlaky,
+    occurrenceRate: stability.occurrenceRate,
+    flakinessScore: stability.flakinessScore,
+    appearedInRuns: stability.appearedInRuns,
+    totalRuns: stability.totalRuns,
+    recommendation: stability.isFlaky 
+      ? 'This finding is flaky - it may be due to timing issues or race conditions. Consider adding stability measures.'
+      : 'This finding is stable - it appears consistently across runs.'
+  });
+}
+
+/**
+ * Add finding to allowlist
+ */
+async function handleAllowlistAdd(args) {
+  const { projectPath = '.', findingId, reason, whyAllowed, expiresAt } = args;
+  
+  try {
+    // Load the unified allowlist module
+    const allowlistModule = require('../packages/cli/src/allowlist/unified-allowlist');
+    const { UnifiedAllowlist } = allowlistModule;
+    
+    const allowlist = new UnifiedAllowlist(projectPath);
+    
+    // Find the original finding to get its details
+    const evidenceResult = await handleGetEvidence({ projectPath, findingId });
+    
+    if (evidenceResult.ok && evidenceResult.data.finding) {
+      const finding = evidenceResult.data.finding;
+      
+      allowlist.addFinding(finding, {
+        reason,
+        whyAllowed,
+        approvedBy: 'AI Assistant (MCP)',
+        expiresAt
+      });
+    } else {
+      // Add with just the fingerprint
+      allowlist.add({
+        fingerprint: findingId,
+        category: 'other',
+        reason,
+        what: `Finding ${findingId}`,
+        whyAllowed,
+        approvedBy: 'AI Assistant (MCP)',
+        expiresAt
+      });
+    }
+    
+    allowlist.save();
+    
+    return wrapResponse({
+      success: true,
+      message: `Finding ${findingId} added to allowlist`,
+      reason,
+      whyAllowed,
+      expiresAt: expiresAt || 'Never'
+    });
+  } catch (err) {
+    return wrapResponse(null, { error: `Failed to add to allowlist: ${err.message}` });
+  }
+}
+
+/**
+ * Get proof graph
+ */
+async function handleGetProofGraph(args) {
+  const { projectPath = '.', format = 'json' } = args;
+  
+  const proofGraphPath = path.join(projectPath, '.vibecheck', 'proof-graph.json');
+  
+  if (!fs.existsSync(proofGraphPath)) {
+    // Try to find in evidence packs
+    const evidencePacksDir = path.join(projectPath, '.vibecheck', 'evidence-packs');
+    if (fs.existsSync(evidencePacksDir)) {
+      const packs = fs.readdirSync(evidencePacksDir).sort().reverse();
+      for (const pack of packs) {
+        const packGraphPath = path.join(evidencePacksDir, pack, 'proof-graph.json');
+        if (fs.existsSync(packGraphPath)) {
+          const graph = JSON.parse(fs.readFileSync(packGraphPath, 'utf8'));
+          return formatProofGraph(graph, format);
+        }
+      }
+    }
+    
+    return wrapResponse(null, { error: 'No proof graph found. Run vibecheck.prove first.' });
+  }
+  
+  const graph = JSON.parse(fs.readFileSync(proofGraphPath, 'utf8'));
+  return formatProofGraph(graph, format);
+}
+
+function formatProofGraph(graph, format) {
+  if (format === 'mermaid') {
+    // Convert to mermaid diagram
+    const lines = ['graph TD'];
+    
+    if (graph.verifiedClaims) {
+      for (const claim of graph.verifiedClaims) {
+        const claimId = `claim_${claim.id || Math.random().toString(36).slice(2, 8)}`;
+        lines.push(`    ${claimId}[${claim.claim || claim.title}]`);
+        lines.push(`    ${claimId} -->|verified| evidence_${claimId}`);
+        lines.push(`    evidence_${claimId}((Evidence))`);
+      }
+    }
+    
+    if (graph.gaps) {
+      for (const gap of graph.gaps) {
+        const gapId = `gap_${Math.random().toString(36).slice(2, 8)}`;
+        lines.push(`    ${gapId}[/${gap.description || gap}/]`);
+        lines.push(`    style ${gapId} fill:#f99`);
+      }
+    }
+    
+    return wrapResponse({
+      format: 'mermaid',
+      diagram: lines.join('\n')
+    });
+  }
+  
+  return wrapResponse({
+    format: 'json',
+    graph
+  });
+}
+
+/**
+ * Generate or get evidence pack
+ */
+async function handleEvidencePack(args) {
+  const { projectPath = '.', generate = false, includeMedia = true } = args;
+  
+  if (generate) {
+    try {
+      const evidencePackModule = require('../packages/cli/src/evidence/evidence-pack');
+      const { generateEvidencePack } = evidencePackModule;
+      
+      const result = await generateEvidencePack({
+        projectPath,
+        sourceType: 'prove',
+        includeVideos: includeMedia,
+        includeTraces: includeMedia,
+        includeScreenshots: includeMedia,
+        includeHar: true
+      });
+      
+      return wrapResponse({
+        generated: true,
+        packPath: result.packPath,
+        manifest: result.manifest
+      });
+    } catch (err) {
+      return wrapResponse(null, { error: `Failed to generate evidence pack: ${err.message}` });
+    }
+  }
+  
+  // Return latest evidence pack
+  const evidencePacksDir = path.join(projectPath, '.vibecheck', 'evidence-packs');
+  
+  if (!fs.existsSync(evidencePacksDir)) {
+    return wrapResponse(null, { 
+      error: 'No evidence packs found. Run vibecheck.prove with --evidence-pack or use generate: true.' 
+    });
+  }
+  
+  const packs = fs.readdirSync(evidencePacksDir).sort().reverse();
+  if (packs.length === 0) {
+    return wrapResponse(null, { error: 'No evidence packs found.' });
+  }
+  
+  const latestPackDir = path.join(evidencePacksDir, packs[0]);
+  const manifestPath = path.join(latestPackDir, 'manifest.json');
+  
+  if (!fs.existsSync(manifestPath)) {
+    return wrapResponse(null, { error: 'Evidence pack manifest not found.' });
+  }
+  
+  const manifest = JSON.parse(fs.readFileSync(manifestPath, 'utf8'));
+  
+  return wrapResponse({
+    generated: false,
+    packPath: latestPackDir,
+    manifest,
+    htmlViewer: fs.existsSync(path.join(latestPackDir, 'summary.html')) 
+      ? path.join(latestPackDir, 'summary.html') 
+      : null
+  });
+}
+
+// ============================================================================
+// RESPONSE WRAPPER
+// ============================================================================
+
+function wrapResponse(data, options = {}) {
+  const { evidence = [], cached = false, error = null } = options;
+  
+  if (error) {
+    return {
+      ok: false,
+      error: typeof error === 'string' ? error : error.message,
+      data: null,
+      evidence: [],
+      metadata: {
+        timestamp: new Date().toISOString(),
+        cached: false
+      }
+    };
+  }
+  
+  return {
+    ok: true,
+    data,
+    evidence: evidence.map(e => ({
+      file: e.file || null,
+      line: e.line || e.lines || null,
+      snippet: e.snippet || e.code || null,
+      confidence: e.confidence || 0.9,
+      reason: e.reason || null
+    })),
+    metadata: {
+      timestamp: new Date().toISOString(),
+      cached
+    }
+  };
+}
+
+// ============================================================================
+// HANDLER DISPATCH
+// ============================================================================
+
+export async function handleProofTool(toolName, args) {
+  const handlers = {
+    'vibecheck.prove': handleProve,
+    'vibecheck.prove_status': handleProveStatus,
+    'vibecheck.get_evidence': handleGetEvidence,
+    'vibecheck.check_flaky': handleCheckFlaky,
+    'vibecheck.allowlist_add': handleAllowlistAdd,
+    'vibecheck.get_proof_graph': handleGetProofGraph,
+    'vibecheck.evidence_pack': handleEvidencePack
+  };
+  
+  const handler = handlers[toolName];
+  if (!handler) {
+    return wrapResponse(null, { error: `Unknown proof tool: ${toolName}` });
+  }
+  
+  return handler(args);
+}
+
+export default { PROOF_TOOLS, handleProofTool };

package/mcp-server/tier-auth.js

@@ -41,7 +41,12 @@ export const TIERS = {
       mcpRateLimit: 10, // requests per minute
     },
     // MCP tools allowed on FREE
-    mcpTools: ['vibecheck.status', 'vibecheck.get_truthpack'],
+    mcpTools: [
+      'vibecheck.get_truthpack',
+      'vibecheck.validate_claim',
+      'vibecheck.compile_context',
+      'vibecheck.search_evidence',
+    ],
   },
   starter: {
     name: 'STARTER',
@@ -74,17 +79,17 @@ export const TIERS = {
       fixApplyPatches: false,
       mcpRateLimit: 60, // requests per minute
     },
-    // MCP tools allowed on STARTER (read-only safe tools)
+    // MCP tools allowed on STARTER (curated)
     mcpTools: [
-      'vibecheck.status',
-      'vibecheck.get_truthpack',
+      'vibecheck.ctx',
       'vibecheck.scan',
-      'vibecheck.list_routes',
-      'vibecheck.list_env',
-      'vibecheck.get_findings',
-      'vibecheck.contracts_diff',
+      'vibecheck.ship',
+      'vibecheck.get_truthpack',
       'vibecheck.validate_claim',
       'vibecheck.compile_context',
+      'vibecheck.search_evidence',
+      'vibecheck.find_counterexamples',
+      'vibecheck.check_invariants',
     ],
   },
   pro: {
@@ -114,19 +119,17 @@ export const TIERS = {
       fixApplyPatches: true,
       mcpRateLimit: -1, // unlimited
     },
-    // MCP tools allowed on PRO (full suite)
+    // MCP tools allowed on PRO (curated)
     mcpTools: [
-      // All STARTER tools plus...
-      'vibecheck.generate_mission',
-      'vibecheck.verify_patch',
-      'vibecheck.explain_evidence',
-      'vibecheck.fix',
-      'vibecheck.proof',
-      'vibecheck.prove',
+      'vibecheck.ctx',
+      'vibecheck.scan',
       'vibecheck.ship',
-      'vibecheck.reality',
-      'vibecheck.permissions',
-      'vibecheck.graph',
+      'vibecheck.get_truthpack',
+      'vibecheck.validate_claim',
+      'vibecheck.compile_context',
+      'vibecheck.search_evidence',
+      'vibecheck.find_counterexamples',
+      'vibecheck.check_invariants',
     ],
   },
   compliance: {