@vibecheckai/cli 3.4.0 → 3.5.1

package/mcp-server/__tests__/schemas.test.ts

@@ -0,0 +1,372 @@
+/**
+ * Schema Validation Tests
+ * 
+ * Ensures all canonical schemas are valid and work correctly.
+ */
+
+import { describe, it, expect, beforeAll } from 'vitest';
+import {
+  validateRunRequest,
+  validateFinding,
+  validateVerdict,
+  validateErrorEnvelope,
+  validateToolInput,
+  normalizeFindingId,
+  validateProjectPath,
+  ValidationError,
+} from '../lib/validator.js';
+
+describe('Schema Validation', () => {
+  describe('RunRequest Schema', () => {
+    it('should validate a valid run request', () => {
+      const request = {
+        tool: 'vibecheck.scan',
+        projectPath: '/home/user/project',
+        timeout: 60000,
+        cache: { mode: 'auto', maxAge: 300 },
+        options: { profile: 'quick' },
+      };
+
+      const result = validateRunRequest(request);
+      expect(result.valid).toBe(true);
+      expect(result.errors).toBeUndefined();
+    });
+
+    it('should reject invalid tool name format', () => {
+      const request = {
+        tool: 'invalid-tool', // Should be vibecheck.xxx
+        projectPath: '/home/user/project',
+      };
+
+      const result = validateRunRequest(request);
+      expect(result.valid).toBe(false);
+      expect(result.errors).toBeDefined();
+      expect(result.errors?.some((e: ValidationError) => e.path.includes('tool'))).toBe(true);
+    });
+
+    it('should require projectPath', () => {
+      const request = {
+        tool: 'vibecheck.scan',
+      };
+
+      const result = validateRunRequest(request);
+      expect(result.valid).toBe(false);
+    });
+
+    it('should validate timeout bounds', () => {
+      const tooShort = {
+        tool: 'vibecheck.scan',
+        projectPath: '/home/user/project',
+        timeout: 500, // Below minimum (1000)
+      };
+
+      const tooLong = {
+        tool: 'vibecheck.scan',
+        projectPath: '/home/user/project',
+        timeout: 700000, // Above maximum (600000)
+      };
+
+      expect(validateRunRequest(tooShort).valid).toBe(false);
+      expect(validateRunRequest(tooLong).valid).toBe(false);
+    });
+
+    it('should validate cache mode enum', () => {
+      const valid = {
+        tool: 'vibecheck.scan',
+        projectPath: '/home/user/project',
+        cache: { mode: 'force' },
+      };
+
+      const invalid = {
+        tool: 'vibecheck.scan',
+        projectPath: '/home/user/project',
+        cache: { mode: 'invalid' },
+      };
+
+      expect(validateRunRequest(valid).valid).toBe(true);
+      expect(validateRunRequest(invalid).valid).toBe(false);
+    });
+  });
+
+  describe('Finding Schema', () => {
+    it('should validate a complete finding', () => {
+      const finding = {
+        id: 'auth_gap-a1b2c3d4',
+        category: 'auth_gap',
+        severity: 'BLOCK',
+        title: 'Unprotected API endpoint',
+        description: 'The /api/admin endpoint has no authentication',
+        evidence: [
+          {
+            file: 'src/routes/admin.ts',
+            line: 42,
+            snippet: 'router.get("/admin", handler)',
+            reason: 'No auth middleware',
+          },
+        ],
+        fixHints: ['Add withAuth middleware'],
+        confidence: 0.95,
+        source: 'static',
+      };
+
+      const result = validateFinding(finding);
+      expect(result.valid).toBe(true);
+    });
+
+    it('should require evidence array with at least one item', () => {
+      const finding = {
+        id: 'auth_gap-a1b2c3d4',
+        category: 'auth_gap',
+        severity: 'BLOCK',
+        title: 'Missing auth',
+        evidence: [], // Empty array
+      };
+
+      const result = validateFinding(finding);
+      expect(result.valid).toBe(false);
+    });
+
+    it('should validate severity enum', () => {
+      const valid = {
+        id: 'test-12345678',
+        category: 'secrets',
+        severity: 'WARN',
+        title: 'Test',
+        evidence: [{ file: 'test.ts' }],
+      };
+
+      const invalid = {
+        ...valid,
+        severity: 'CRITICAL', // Not in enum
+      };
+
+      expect(validateFinding(valid).valid).toBe(true);
+      expect(validateFinding(invalid).valid).toBe(false);
+    });
+
+    it('should validate category enum', () => {
+      const validCategories = ['secrets', 'auth_gap', 'billing_bypass', 'dead_ui', 'fake_success'];
+      
+      for (const category of validCategories) {
+        const finding = {
+          id: `${category}-12345678`,
+          category,
+          severity: 'WARN',
+          title: 'Test',
+          evidence: [{ file: 'test.ts' }],
+        };
+        expect(validateFinding(finding).valid).toBe(true);
+      }
+    });
+
+    it('should validate finding ID format', () => {
+      const validIds = ['auth_gap-a1b2c3d4', 'secrets-12345678', 'dead_ui-abcdef12'];
+      const invalidIds = ['invalid', 'AUTH_GAP-123', 'auth_gap_123'];
+
+      for (const id of validIds) {
+        const finding = {
+          id,
+          category: 'secrets',
+          severity: 'WARN',
+          title: 'Test',
+          evidence: [{ file: 'test.ts' }],
+        };
+        expect(validateFinding(finding).valid).toBe(true);
+      }
+
+      for (const id of invalidIds) {
+        const finding = {
+          id,
+          category: 'secrets',
+          severity: 'WARN',
+          title: 'Test',
+          evidence: [{ file: 'test.ts' }],
+        };
+        expect(validateFinding(finding).valid).toBe(false);
+      }
+    });
+  });
+
+  describe('Verdict Schema', () => {
+    it('should validate a complete verdict', () => {
+      const verdict = {
+        verdict: 'SHIP',
+        score: 95,
+        grade: 'A',
+        summary: {
+          block: 0,
+          warn: 2,
+          info: 5,
+          total: 7,
+        },
+        findings: [],
+        meta: {
+          version: '3.0.0',
+          runId: 'run_abc123',
+          timestamp: '2024-01-15T10:30:00Z',
+          durationMs: 5000,
+        },
+      };
+
+      const result = validateVerdict(verdict);
+      expect(result.valid).toBe(true);
+    });
+
+    it('should validate verdict enum', () => {
+      const base = {
+        score: 50,
+        summary: { block: 0, warn: 0, info: 0, total: 0 },
+        findings: [],
+        meta: {
+          version: '3.0.0',
+          runId: 'run_123',
+          timestamp: '2024-01-15T10:30:00Z',
+          durationMs: 1000,
+        },
+      };
+
+      expect(validateVerdict({ ...base, verdict: 'SHIP' }).valid).toBe(true);
+      expect(validateVerdict({ ...base, verdict: 'WARN' }).valid).toBe(true);
+      expect(validateVerdict({ ...base, verdict: 'BLOCK' }).valid).toBe(true);
+      expect(validateVerdict({ ...base, verdict: 'MAYBE' }).valid).toBe(false);
+    });
+
+    it('should validate score bounds', () => {
+      const base = {
+        verdict: 'SHIP',
+        summary: { block: 0, warn: 0, info: 0, total: 0 },
+        findings: [],
+        meta: {
+          version: '3.0.0',
+          runId: 'run_123',
+          timestamp: '2024-01-15T10:30:00Z',
+          durationMs: 1000,
+        },
+      };
+
+      expect(validateVerdict({ ...base, score: 0 }).valid).toBe(true);
+      expect(validateVerdict({ ...base, score: 100 }).valid).toBe(true);
+      expect(validateVerdict({ ...base, score: -1 }).valid).toBe(false);
+      expect(validateVerdict({ ...base, score: 101 }).valid).toBe(false);
+    });
+  });
+
+  describe('ErrorEnvelope Schema', () => {
+    it('should validate a complete error envelope', () => {
+      const envelope = {
+        ok: false,
+        error: {
+          code: 'TIMEOUT',
+          message: 'Operation timed out after 60000ms',
+          retryable: true,
+          retryAfterMs: 5000,
+          userAction: 'Try again with a smaller scope',
+        },
+        requestId: 'req_abc123',
+        timestamp: '2024-01-15T10:30:00Z',
+      };
+
+      const result = validateErrorEnvelope(envelope);
+      expect(result.valid).toBe(true);
+    });
+
+    it('should require ok to be false', () => {
+      const envelope = {
+        ok: true, // Should be false
+        error: {
+          code: 'TIMEOUT',
+          message: 'Test',
+        },
+        requestId: 'req_123',
+        timestamp: '2024-01-15T10:30:00Z',
+      };
+
+      const result = validateErrorEnvelope(envelope);
+      expect(result.valid).toBe(false);
+    });
+
+    it('should validate error code enum', () => {
+      const validCodes = ['INVALID_INPUT', 'TIMEOUT', 'PATH_NOT_FOUND', 'TIER_REQUIRED'];
+      
+      for (const code of validCodes) {
+        const envelope = {
+          ok: false,
+          error: { code, message: 'Test' },
+          requestId: 'req_123',
+          timestamp: '2024-01-15T10:30:00Z',
+        };
+        expect(validateErrorEnvelope(envelope).valid).toBe(true);
+      }
+    });
+  });
+
+  describe('Tool Input Validation', () => {
+    it('should validate vibecheck.scan input', () => {
+      const valid = { profile: 'quick' };
+      const invalid = { profile: 'invalid_profile' };
+
+      expect(validateToolInput('vibecheck.scan', valid).valid).toBe(true);
+      expect(validateToolInput('vibecheck.scan', invalid).valid).toBe(false);
+    });
+
+    it('should validate vibecheck.reality required fields', () => {
+      const valid = { url: 'http://localhost:3000' };
+      const invalid = {}; // Missing required url
+
+      expect(validateToolInput('vibecheck.reality', valid).valid).toBe(true);
+      expect(validateToolInput('vibecheck.reality', invalid).valid).toBe(false);
+    });
+
+    it('should return error for unknown tool', () => {
+      const result = validateToolInput('vibecheck.unknown_tool', {});
+      expect(result.valid).toBe(false);
+      expect(result.errors?.some((e: ValidationError) => e.message.includes('Unknown tool'))).toBe(true);
+    });
+  });
+
+  describe('Finding ID Normalization', () => {
+    it('should create deterministic IDs', () => {
+      const evidence = { file: 'src/api.ts', line: 42 };
+      
+      const id1 = normalizeFindingId('auth_gap', evidence);
+      const id2 = normalizeFindingId('auth_gap', evidence);
+      
+      expect(id1).toBe(id2);
+      expect(id1).toMatch(/^auth_gap-[a-f0-9]{8}$/);
+    });
+
+    it('should produce different IDs for different evidence', () => {
+      const id1 = normalizeFindingId('auth_gap', { file: 'a.ts', line: 1 });
+      const id2 = normalizeFindingId('auth_gap', { file: 'b.ts', line: 1 });
+      const id3 = normalizeFindingId('auth_gap', { file: 'a.ts', line: 2 });
+
+      expect(id1).not.toBe(id2);
+      expect(id1).not.toBe(id3);
+      expect(id2).not.toBe(id3);
+    });
+  });
+
+  describe('Project Path Validation', () => {
+    it('should accept valid paths within workspace', () => {
+      const result = validateProjectPath('/home/user/project', '/home/user');
+      expect(result.valid).toBe(true);
+    });
+
+    it('should reject path traversal', () => {
+      const result = validateProjectPath('/home/user/../etc/passwd', '/home/user');
+      expect(result.valid).toBe(false);
+      expect(result.error).toContain('traversal');
+    });
+
+    it('should reject paths outside workspace', () => {
+      const result = validateProjectPath('/etc/passwd', '/home/user');
+      expect(result.valid).toBe(false);
+    });
+
+    it('should normalize paths', () => {
+      const result = validateProjectPath('project//src///file.ts');
+      expect(result.valid).toBe(true);
+      expect(result.normalized).toBe('project/src/file.ts');
+    });
+  });
+});

package/mcp-server/benchmarks/run-benchmarks.ts

@@ -0,0 +1,304 @@
+/**
+ * MCP Server Benchmarks
+ * 
+ * Measures performance for core tools.
+ * Shows cache benefit and correctness.
+ * 
+ * Usage:
+ *   npx tsx benchmarks/run-benchmarks.ts [projectPath]
+ * 
+ * TARGETS:
+ * - Warm runs 3-10x faster than cold
+ * - doctor warm <500ms (realistic <1.5s on Windows)
+ * - scan warm <3000ms
+ * - ship warm <2000ms
+ * - status warm <100ms
+ */
+
+import { performance } from 'perf_hooks';
+import { existsSync, mkdirSync, writeFileSync, rmSync } from 'fs';
+import { join } from 'path';
+import { tmpdir } from 'os';
+import { handleTool, ToolRequest } from '../handlers/tool-handler.js';
+import { getGlobalCache, initGlobalCache } from '../lib/cache.js';
+import { initLogger, LogLevel } from '../lib/logger.js';
+import { getMetricsCollector, initMetricsCollector } from '../lib/metrics.js';
+
+interface BenchmarkResult {
+  tool: string;
+  coldRun: number;
+  warmRun: number;
+  cacheHit: boolean;
+  speedup: number;
+  correct: boolean;
+  error?: string;
+}
+
+interface BenchmarkTargets {
+  warmTarget: number;
+  coldTarget: number;
+}
+
+const TARGETS: Record<string, BenchmarkTargets> = {
+  doctor: { warmTarget: 500, coldTarget: 3000 },
+  scan: { warmTarget: 3000, coldTarget: 15000 },
+  ship: { warmTarget: 2000, coldTarget: 10000 },
+  status: { warmTarget: 100, coldTarget: 1000 },
+};
+
+/**
+ * Create test fixture if needed
+ */
+function createTestFixture(): string {
+  const fixtureDir = join(tmpdir(), `vibecheck-bench-${Date.now()}`);
+  mkdirSync(fixtureDir, { recursive: true });
+  
+  // Create minimal project structure
+  writeFileSync(join(fixtureDir, 'package.json'), JSON.stringify({
+    name: 'benchmark-fixture',
+    version: '1.0.0',
+    dependencies: { express: '^4.18.0' },
+  }));
+  
+  mkdirSync(join(fixtureDir, 'src'));
+  writeFileSync(join(fixtureDir, 'src', 'index.ts'), `
+    import express from 'express';
+    const app = express();
+    app.get('/api/test', (req, res) => res.json({ ok: true }));
+    export default app;
+  `);
+  
+  mkdirSync(join(fixtureDir, '.vibecheck'), { recursive: true });
+  
+  return fixtureDir;
+}
+
+/**
+ * Run a single benchmark
+ */
+async function runBenchmark(
+  tool: string,
+  projectPath: string,
+  options?: Record<string, unknown>
+): Promise<BenchmarkResult> {
+  const cache = getGlobalCache();
+  
+  // Clear cache for cold run
+  cache.clear();
+
+  // Cold run
+  const coldStart = performance.now();
+  let coldResult: Awaited<ReturnType<typeof handleTool>>;
+  try {
+    coldResult = await handleTool({
+      tool: `vibecheck.${tool}`,
+      projectPath,
+      options,
+      cache: { mode: 'auto' },
+    });
+  } catch (e) {
+    return {
+      tool,
+      coldRun: -1,
+      warmRun: -1,
+      cacheHit: false,
+      speedup: 0,
+      correct: false,
+      error: (e as Error).message,
+    };
+  }
+  const coldTime = performance.now() - coldStart;
+
+  // Verify cold run correctness
+  const coldCorrect = coldResult.ok || 
+    (coldResult as { error?: { code: string } }).error?.code !== 'INTERNAL_ERROR';
+
+  // Warm run (should hit cache)
+  const warmStart = performance.now();
+  const warmResult = await handleTool({
+    tool: `vibecheck.${tool}`,
+    projectPath,
+    options,
+    cache: { mode: 'auto' },
+  });
+  const warmTime = performance.now() - warmStart;
+
+  const cacheHit = warmResult.ok && 
+    (warmResult as { meta?: { cached?: boolean } }).meta?.cached === true;
+  const speedup = coldTime / warmTime;
+
+  return {
+    tool,
+    coldRun: Math.round(coldTime),
+    warmRun: Math.round(warmTime),
+    cacheHit,
+    speedup: Math.round(speedup * 10) / 10,
+    correct: coldCorrect,
+  };
+}
+
+/**
+ * Print results table
+ */
+function printResults(benchmarks: BenchmarkResult[]): void {
+  console.log('');
+  console.log('╔══════════════════════════════════════════════════════════════════════════╗');
+  console.log('║                              RESULTS                                      ║');
+  console.log('╠═══════════════════════╦═════════╦═════════╦═════════╦═════════╦══════════╣');
+  console.log('║ Tool                  ║ Cold    ║ Warm    ║ Speedup ║ Cache   ║ Correct  ║');
+  console.log('╠═══════════════════════╬═════════╬═════════╬═════════╬═════════╬══════════╣');
+
+  for (const result of benchmarks) {
+    const tool = result.tool.padEnd(19);
+    const cold = result.coldRun >= 0 ? `${result.coldRun}ms`.padStart(7) : 'ERROR  ';
+    const warm = result.warmRun >= 0 ? `${result.warmRun}ms`.padStart(7) : 'ERROR  ';
+    const speedup = result.speedup > 0 ? `${result.speedup}x`.padStart(7) : 'N/A    ';
+    const cache = result.cacheHit ? '   ✓   ' : '   ✗   ';
+    const correct = result.correct ? '   ✓    ' : '   ✗    ';
+    
+    console.log(`║ ${tool} ║ ${cold} ║ ${warm} ║ ${speedup} ║${cache}║${correct}║`);
+  }
+
+  console.log('╚═══════════════════════╩═════════╩═════════╩═════════╩═════════╩══════════╝');
+}
+
+/**
+ * Print target checks
+ */
+function printTargetChecks(benchmarks: BenchmarkResult[]): void {
+  console.log('');
+  console.log('Target Checks:');
+  
+  let allPassed = true;
+  
+  for (const result of benchmarks) {
+    const targets = TARGETS[result.tool];
+    if (!targets) continue;
+    
+    const warmPass = result.warmRun >= 0 && result.warmRun <= targets.warmTarget;
+    const coldPass = result.coldRun >= 0 && result.coldRun <= targets.coldTarget;
+    
+    console.log(`  ${result.tool}:`);
+    console.log(`    warm <${targets.warmTarget}ms: ${warmPass ? '✓ PASS' : '✗ FAIL'} (${result.warmRun}ms)`);
+    console.log(`    cold <${targets.coldTarget}ms: ${coldPass ? '✓ PASS' : '✗ FAIL'} (${result.coldRun}ms)`);
+    
+    if (!warmPass || !coldPass) allPassed = false;
+  }
+  
+  // Speedup check
+  const avgSpeedup = benchmarks.filter(b => b.speedup > 0).reduce((sum, b) => sum + b.speedup, 0) / 
+                     benchmarks.filter(b => b.speedup > 0).length || 0;
+  const speedupPass = avgSpeedup >= 3;
+  
+  console.log(`  Average speedup >=3x: ${speedupPass ? '✓ PASS' : '✗ FAIL'} (${avgSpeedup.toFixed(1)}x)`);
+  
+  if (!speedupPass) allPassed = false;
+  
+  return;
+}
+
+/**
+ * Main benchmark runner
+ */
+async function main() {
+  // Initialize logging (quiet for benchmarks)
+  initLogger({ 
+    level: LogLevel.ERROR, 
+    enableConsole: false, 
+    enableFile: false 
+  });
+  
+  // Initialize cache and metrics
+  initGlobalCache({ maxMemoryEntries: 100, defaultTtl: 300 });
+  initMetricsCollector({ maxEntries: 1000 });
+  
+  // Get or create project path
+  let projectPath = process.argv[2];
+  let createdFixture = false;
+  
+  if (!projectPath || !existsSync(projectPath)) {
+    console.log('No valid project path provided. Creating test fixture...');
+    projectPath = createTestFixture();
+    createdFixture = true;
+  }
+  
+  console.log('╔══════════════════════════════════════════════════════════════╗');
+  console.log('║           Vibecheck MCP Server Benchmarks                     ║');
+  console.log('╠══════════════════════════════════════════════════════════════╣');
+  console.log(`║ Project: ${projectPath.slice(0, 50).padEnd(50)} ║`);
+  console.log('╚══════════════════════════════════════════════════════════════╝');
+  console.log('');
+
+  const benchmarks: BenchmarkResult[] = [];
+  const toolsToTest = ['doctor', 'scan', 'ship', 'status'];
+
+  for (const tool of toolsToTest) {
+    console.log(`Running vibecheck.${tool}...`);
+    try {
+      const options = tool === 'scan' ? { profile: 'quick' } : undefined;
+      const result = await runBenchmark(tool, projectPath, options);
+      benchmarks.push(result);
+      
+      if (result.error) {
+        console.log(`  Warning: ${result.error}`);
+      }
+    } catch (e) {
+      console.log(`  Error: ${e}`);
+      benchmarks.push({
+        tool,
+        coldRun: -1,
+        warmRun: -1,
+        cacheHit: false,
+        speedup: 0,
+        correct: false,
+        error: (e as Error).message,
+      });
+    }
+  }
+
+  // Print results
+  printResults(benchmarks);
+
+  // Summary
+  console.log('');
+  console.log('Summary:');
+  
+  const successful = benchmarks.filter(b => b.correct);
+  const withCache = benchmarks.filter(b => b.cacheHit);
+  const avgSpeedup = successful.filter(b => b.speedup > 0).reduce((sum, b) => sum + b.speedup, 0) / 
+                     successful.filter(b => b.speedup > 0).length || 0;
+
+  console.log(`  Tools tested: ${benchmarks.length}`);
+  console.log(`  Successful:   ${successful.length}/${benchmarks.length}`);
+  console.log(`  Cache hits:   ${withCache.length}/${benchmarks.length}`);
+  console.log(`  Avg speedup:  ${avgSpeedup.toFixed(1)}x`);
+
+  // Target checks
+  printTargetChecks(benchmarks);
+
+  // Metrics summary
+  const metrics = getMetricsCollector();
+  const summary = metrics.getSummary();
+  console.log('');
+  console.log('Metrics:');
+  console.log(`  Cache hit rate: ${(summary.cacheHitRate * 100).toFixed(1)}%`);
+
+  // Cleanup fixture if we created it
+  if (createdFixture) {
+    console.log('');
+    console.log('Cleaning up test fixture...');
+    rmSync(projectPath, { recursive: true, force: true });
+  }
+
+  // Exit code based on results
+  const allPassed = successful.length === benchmarks.length && 
+                    withCache.length >= Math.floor(benchmarks.length / 2) && 
+                    avgSpeedup >= 2;
+  
+  process.exit(allPassed ? 0 : 1);
+}
+
+main().catch(err => {
+  console.error('Benchmark failed:', err);
+  process.exit(1);
+});

package/mcp-server/examples/doctor.request.json

@@ -0,0 +1,14 @@
+{
+  "tool": "vibecheck.doctor",
+  "projectPath": "/home/user/my-project",
+  "requestId": "req_1a2b3c4d",
+  "timeout": 30000,
+  "cache": {
+    "mode": "auto",
+    "maxAge": 300
+  },
+  "options": {
+    "category": "all",
+    "fix": false
+  }
+}