@vibecheckai/cli 3.0.4 → 3.0.7

package/bin/runners/lib/schema-validator.js

@@ -0,0 +1,350 @@
+/**
+ * Schema Validation Utilities for vibecheck v2
+ * 
+ * Validates artifacts against JSON schemas (Draft 2020-12).
+ * Used for CI validation and ensuring deterministic output.
+ */
+
+"use strict";
+
+const fs = require("fs");
+const path = require("path");
+const crypto = require("crypto");
+
+const SCHEMA_VERSION = "2.0.0";
+
+/**
+ * Load a schema from the schemas directory
+ */
+function loadSchema(schemaName, schemasDir) {
+  const schemaPath = path.join(schemasDir, `${schemaName}.schema.json`);
+  if (!fs.existsSync(schemaPath)) {
+    return null;
+  }
+  return JSON.parse(fs.readFileSync(schemaPath, "utf8"));
+}
+
+/**
+ * Generate a stable fingerprint hash
+ */
+function generateFingerprint(data) {
+  const str = typeof data === "string" ? data : JSON.stringify(data);
+  return `sha256:${crypto.createHash("sha256").update(str).digest("hex")}`;
+}
+
+/**
+ * Generate a short fingerprint (12 chars)
+ */
+function shortFingerprint(data) {
+  const full = generateFingerprint(data);
+  return full.slice(0, 19); // sha256: + 12 chars
+}
+
+/**
+ * Create a v2-compliant Evidence object
+ */
+function createEvidence({
+  kind,
+  reason,
+  file = null,
+  lines = null,
+  snippet = null,
+  url = null,
+  httpStatus = null,
+  artifactPath = null,
+  requestId = null,
+  traceTimeRangeMs = null,
+}) {
+  const id = `E_${crypto.randomBytes(6).toString("hex").toUpperCase()}`;
+  
+  const evidence = {
+    id,
+    kind,
+    reason,
+  };
+
+  if (file) evidence.file = file;
+  if (lines) evidence.lines = lines;
+  if (snippet) evidence.snippetHash = generateFingerprint(snippet);
+  if (url) evidence.url = url;
+  if (httpStatus) evidence.httpStatus = httpStatus;
+  if (artifactPath) evidence.artifactPath = artifactPath;
+  if (requestId) evidence.requestId = requestId;
+  if (traceTimeRangeMs) evidence.traceTimeRangeMs = traceTimeRangeMs;
+
+  return evidence;
+}
+
+/**
+ * Create a v2-compliant Finding object
+ */
+function createFindingV2({
+  detectorId,
+  severity,
+  category,
+  scope,
+  title,
+  why,
+  confidence = "medium",
+  evidence = [],
+  fixHints = [],
+  related = [],
+  repro = null,
+}) {
+  // Generate stable fingerprint from detector + primary evidence
+  const fingerprintData = [
+    detectorId,
+    evidence[0]?.file,
+    evidence[0]?.lines,
+    title,
+  ].filter(Boolean).join("|");
+  
+  const fingerprint = generateFingerprint(fingerprintData);
+  const id = `F_${detectorId}_${fingerprint.slice(7, 19).toUpperCase()}`;
+
+  const finding = {
+    id,
+    fingerprint,
+    severity,
+    category,
+    scope,
+    title,
+    why,
+    confidence,
+    evidence,
+  };
+
+  if (fixHints.length > 0) finding.fixHints = fixHints.slice(0, 8);
+  if (related.length > 0) finding.related = related.slice(0, 12);
+  if (repro) finding.repro = repro;
+
+  return finding;
+}
+
+/**
+ * Validate a finding against the v2 schema
+ */
+function validateFindingV2(finding) {
+  const errors = [];
+
+  // Required fields
+  if (!finding.id || !/^F_[A-Z0-9_]+$/.test(finding.id)) {
+    errors.push(`Invalid id: ${finding.id} (must match F_[A-Z0-9_]+)`);
+  }
+  if (!finding.fingerprint || !/^(sha1|sha256):[a-f0-9]{40,64}$/.test(finding.fingerprint)) {
+    errors.push(`Invalid fingerprint: ${finding.fingerprint}`);
+  }
+  if (!["BLOCK", "WARN", "INFO"].includes(finding.severity)) {
+    errors.push(`Invalid severity: ${finding.severity}`);
+  }
+  if (!["DeadUI", "Routes", "AuthCoverage", "Env", "Billing", "Truth", "Entitlements", "Quality", "Drift"].includes(finding.category)) {
+    errors.push(`Invalid category: ${finding.category}`);
+  }
+  if (!["client", "server", "runtime", "contracts"].includes(finding.scope)) {
+    errors.push(`Invalid scope: ${finding.scope}`);
+  }
+  if (!finding.title || finding.title.length < 5) {
+    errors.push(`Title too short: ${finding.title}`);
+  }
+  if (!finding.why || finding.why.length < 10) {
+    errors.push(`Why too short: ${finding.why}`);
+  }
+  if (!["low", "medium", "high"].includes(finding.confidence)) {
+    errors.push(`Invalid confidence: ${finding.confidence}`);
+  }
+  if (!Array.isArray(finding.evidence) || finding.evidence.length < 1) {
+    errors.push("Evidence must have at least 1 item");
+  }
+
+  return {
+    valid: errors.length === 0,
+    errors,
+  };
+}
+
+/**
+ * Validate an evidence object against the v2 schema
+ */
+function validateEvidenceV2(evidence) {
+  const errors = [];
+
+  if (!evidence.id || !/^E_[A-Z0-9_]+$/.test(evidence.id)) {
+    errors.push(`Invalid id: ${evidence.id}`);
+  }
+  if (!["file", "url", "screenshot", "trace", "request", "console", "diff", "hash"].includes(evidence.kind)) {
+    errors.push(`Invalid kind: ${evidence.kind}`);
+  }
+  if (!evidence.reason || evidence.reason.length < 3) {
+    errors.push(`Reason too short: ${evidence.reason}`);
+  }
+  if (evidence.lines && !/^[0-9]+-[0-9]+$/.test(evidence.lines)) {
+    errors.push(`Invalid lines format: ${evidence.lines}`);
+  }
+  if (evidence.snippetHash && !/^(sha1|sha256):[a-f0-9]{40,64}$/.test(evidence.snippetHash)) {
+    errors.push(`Invalid snippetHash: ${evidence.snippetHash}`);
+  }
+
+  return {
+    valid: errors.length === 0,
+    errors,
+  };
+}
+
+/**
+ * Create a v2-compliant ship report
+ */
+function createShipReportV2({
+  projectPath,
+  findings,
+  truthpackPath,
+  truthpackHash,
+  realityPath = null,
+  realityHash = null,
+  contractsPaths = [],
+  policy = {},
+  proofGraph = null,
+}) {
+  const blocks = findings.filter(f => f.severity === "BLOCK").length;
+  const warns = findings.filter(f => f.severity === "WARN").length;
+  const infos = findings.filter(f => f.severity === "INFO").length;
+
+  const verdict = blocks > 0 ? "BLOCK" : warns > 0 ? "WARN" : "SHIP";
+  const exitCode = verdict === "SHIP" ? 0 : verdict === "WARN" ? 1 : 2;
+
+  // Top findings (blockers first, then warns)
+  const top = findings
+    .filter(f => f.severity === "BLOCK" || f.severity === "WARN")
+    .sort((a, b) => (a.severity === "BLOCK" ? -1 : 1))
+    .slice(0, 5)
+    .map(f => f.id);
+
+  const report = {
+    schemaVersion: SCHEMA_VERSION,
+    generatedAt: new Date().toISOString(),
+    vibecheckVersion: getVersion(),
+    projectFingerprint: generateProjectFingerprint(projectPath),
+
+    verdict,
+    exitCode,
+
+    policy: {
+      failOnWarn: policy.failOnWarn || false,
+      realityFreshnessMinutes: policy.realityFreshnessMinutes || 60,
+    },
+
+    inputs: {
+      truthpackPath,
+      truthpackHash,
+    },
+
+    summary: {
+      counts: { BLOCK: blocks, WARN: warns, INFO: infos },
+      top,
+    },
+
+    findings,
+  };
+
+  if (realityPath) {
+    report.inputs.realityPath = realityPath;
+    report.inputs.realityHash = realityHash;
+  }
+  if (contractsPaths.length > 0) {
+    report.inputs.contractsPaths = contractsPaths;
+  }
+  if (proofGraph) {
+    report.proofGraph = proofGraph;
+  }
+
+  return report;
+}
+
+/**
+ * Generate project fingerprint
+ */
+function generateProjectFingerprint(projectPath) {
+  const keyFiles = ["package.json", "pnpm-lock.yaml", "package-lock.json", "yarn.lock"];
+  const hashes = [];
+
+  for (const file of keyFiles) {
+    const filePath = path.join(projectPath, file);
+    if (fs.existsSync(filePath)) {
+      const content = fs.readFileSync(filePath, "utf8");
+      const hash = crypto.createHash("sha256").update(content).digest("hex").slice(0, 8);
+      hashes.push(`${file}:${hash}`);
+    }
+  }
+
+  // Git commit if available
+  try {
+    const { execSync } = require("child_process");
+    const commit = execSync("git rev-parse HEAD", { cwd: projectPath, encoding: "utf8" }).trim();
+    hashes.push(`git:${commit.slice(0, 8)}`);
+  } catch {}
+
+  return generateFingerprint(hashes.join("|"));
+}
+
+/**
+ * Get vibecheck version
+ */
+function getVersion() {
+  try {
+    const pkgPath = path.join(__dirname, "..", "..", "..", "package.json");
+    const pkg = JSON.parse(fs.readFileSync(pkgPath, "utf8"));
+    return pkg.version || "0.0.0";
+  } catch {
+    return "0.0.0";
+  }
+}
+
+/**
+ * Validate ship report against v2 schema
+ */
+function validateShipReportV2(report) {
+  const errors = [];
+
+  if (report.schemaVersion !== SCHEMA_VERSION) {
+    errors.push(`Invalid schemaVersion: ${report.schemaVersion}`);
+  }
+  if (!["SHIP", "WARN", "BLOCK"].includes(report.verdict)) {
+    errors.push(`Invalid verdict: ${report.verdict}`);
+  }
+  if (![0, 1, 2].includes(report.exitCode)) {
+    errors.push(`Invalid exitCode: ${report.exitCode}`);
+  }
+  if (!report.inputs?.truthpackPath) {
+    errors.push("Missing inputs.truthpackPath");
+  }
+  if (!report.summary?.counts) {
+    errors.push("Missing summary.counts");
+  }
+
+  // Validate each finding
+  for (const finding of report.findings || []) {
+    const result = validateFindingV2(finding);
+    if (!result.valid) {
+      errors.push(`Finding ${finding.id}: ${result.errors.join(", ")}`);
+    }
+  }
+
+  return {
+    valid: errors.length === 0,
+    errors,
+  };
+}
+
+module.exports = {
+  SCHEMA_VERSION,
+  loadSchema,
+  generateFingerprint,
+  shortFingerprint,
+  createEvidence,
+  createFindingV2,
+  validateFindingV2,
+  validateEvidenceV2,
+  createShipReportV2,
+  generateProjectFingerprint,
+  validateShipReportV2,
+  getVersion,
+};

package/bin/runners/lib/schemas/contracts.schema.json

@@ -0,0 +1,160 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "https://vibecheck.dev/schemas/contracts.schema.json",
+  "title": "Vibecheck Contracts",
+  "description": "Stable contracts extracted from truthpack for drift detection",
+  "type": "object",
+  "required": ["specVersion", "generatedAt", "projectRoot", "fingerprint", "routes", "env", "auth"],
+  "properties": {
+    "specVersion": {
+      "type": "string",
+      "const": "2.0"
+    },
+    "generatedAt": {
+      "type": "string",
+      "format": "date-time"
+    },
+    "projectRoot": {
+      "type": "string"
+    },
+    "fingerprint": {
+      "type": "string",
+      "pattern": "^sha256:[a-f0-9]{64}$",
+      "description": "Hash of contracts for quick comparison"
+    },
+    "routes": {
+      "$ref": "#/$defs/routesContract"
+    },
+    "env": {
+      "$ref": "#/$defs/envContract"
+    },
+    "auth": {
+      "$ref": "#/$defs/authContract"
+    },
+    "external": {
+      "$ref": "#/$defs/externalContract"
+    },
+    "artifacts": {
+      "type": "array",
+      "items": { "$ref": "#/$defs/artifact" }
+    }
+  },
+  "$defs": {
+    "routesContract": {
+      "type": "object",
+      "required": ["count", "fingerprint", "endpoints"],
+      "properties": {
+        "count": { "type": "integer", "minimum": 0 },
+        "fingerprint": {
+          "type": "string",
+          "pattern": "^sha256:[a-f0-9]{64}$"
+        },
+        "endpoints": {
+          "type": "array",
+          "items": {
+            "type": "object",
+            "required": ["method", "path", "framework"],
+            "properties": {
+              "method": {
+                "type": "string",
+                "enum": ["GET", "POST", "PUT", "PATCH", "DELETE", "HEAD", "OPTIONS", "ALL"]
+              },
+              "path": { "type": "string" },
+              "canonicalPath": { "type": "string" },
+              "framework": { "type": "string" },
+              "authRequired": { "type": "boolean" },
+              "source": {
+                "type": "string",
+                "enum": ["static", "runtime"]
+              }
+            }
+          }
+        },
+        "byFramework": {
+          "type": "object",
+          "additionalProperties": { "type": "integer" }
+        }
+      }
+    },
+    "envContract": {
+      "type": "object",
+      "required": ["count", "fingerprint", "vars"],
+      "properties": {
+        "count": { "type": "integer", "minimum": 0 },
+        "fingerprint": {
+          "type": "string",
+          "pattern": "^sha256:[a-f0-9]{64}$"
+        },
+        "vars": {
+          "type": "array",
+          "items": {
+            "type": "object",
+            "required": ["name"],
+            "properties": {
+              "name": { "type": "string" },
+              "required": { "type": "boolean" },
+              "isPublic": { "type": "boolean" },
+              "isSecret": { "type": "boolean" }
+            }
+          }
+        },
+        "baseUrlVar": { "type": ["string", "null"] }
+      }
+    },
+    "authContract": {
+      "type": "object",
+      "required": ["fingerprint", "protectedPatterns"],
+      "properties": {
+        "fingerprint": {
+          "type": "string",
+          "pattern": "^sha256:[a-f0-9]{64}$"
+        },
+        "provider": {
+          "type": ["string", "null"],
+          "enum": [null, "next-auth", "clerk", "supabase", "custom"]
+        },
+        "protectedPatterns": {
+          "type": "array",
+          "items": { "type": "string" }
+        },
+        "publicPatterns": {
+          "type": "array",
+          "items": { "type": "string" }
+        },
+        "middlewareHash": {
+          "type": ["string", "null"],
+          "pattern": "^sha256:[a-f0-9]{64}$"
+        }
+      }
+    },
+    "externalContract": {
+      "type": "object",
+      "properties": {
+        "fingerprint": {
+          "type": "string",
+          "pattern": "^sha256:[a-f0-9]{64}$"
+        },
+        "apis": {
+          "type": "array",
+          "items": {
+            "type": "object",
+            "required": ["host"],
+            "properties": {
+              "host": { "type": "string" },
+              "paths": { "type": "array", "items": { "type": "string" } },
+              "envVar": { "type": ["string", "null"] }
+            }
+          }
+        }
+      }
+    },
+    "artifact": {
+      "type": "object",
+      "required": ["path", "sha256"],
+      "properties": {
+        "path": { "type": "string" },
+        "sha256": { "type": "string", "pattern": "^[a-f0-9]{64}$" }
+      }
+    }
+  }
+}

package/bin/runners/lib/schemas/finding.schema.json

@@ -0,0 +1,100 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "https://vibecheck.dev/schemas/finding.schema.json",
+  "title": "Vibecheck Finding",
+  "description": "A single finding from static or runtime analysis",
+  "type": "object",
+  "required": ["id", "detectorId", "fingerprint", "severity", "category", "scope", "title", "confidence"],
+  "properties": {
+    "id": {
+      "type": "string",
+      "pattern": "^F_[A-Z0-9_]+_[A-F0-9]+$",
+      "description": "Unique finding ID: F_{DETECTOR}_{HASH}"
+    },
+    "detectorId": {
+      "type": "string",
+      "pattern": "^D_[A-Z0-9_]+$",
+      "description": "Detector that produced this finding"
+    },
+    "fingerprint": {
+      "type": "string",
+      "pattern": "^sha256:[a-f0-9]{64}$",
+      "description": "Stable hash for deduplication across runs"
+    },
+    "severity": {
+      "type": "string",
+      "enum": ["BLOCK", "WARN", "INFO"],
+      "description": "BLOCK stops ship, WARN allows with warning, INFO is informational"
+    },
+    "category": {
+      "type": "string",
+      "enum": ["Truth", "Routes", "Auth", "Env", "Billing", "DeadUI", "FakeSuccess", "UIMismatch", "ContractDrift", "Security", "Coverage"],
+      "description": "Finding category for grouping. Truth = truthpack/extraction issues, Routes = route matching, Auth = authentication, Env = environment vars, Billing = Stripe/payments, DeadUI = clicks with no effect, FakeSuccess = toast/UI mismatch with request, UIMismatch = UI/backend state mismatch, ContractDrift = contract changes, Security = security issues, Coverage = coverage threshold violations"
+    },
+    "scope": {
+      "type": "string",
+      "enum": ["client", "server", "runtime", "contracts", "shared"],
+      "description": "Where the finding applies"
+    },
+    "title": {
+      "type": "string",
+      "maxLength": 200,
+      "description": "Human-readable title"
+    },
+    "why": {
+      "type": "string",
+      "description": "Explanation of why this is a problem"
+    },
+    "confidence": {
+      "type": "string",
+      "enum": ["high", "medium", "low"],
+      "description": "Confidence level of detection"
+    },
+    "evidence": {
+      "type": "array",
+      "items": { "$ref": "#/$defs/evidence" },
+      "description": "Evidence supporting this finding"
+    },
+    "repro": {
+      "type": ["string", "null"],
+      "description": "How to reproduce this issue"
+    },
+    "related": {
+      "type": "array",
+      "items": { "type": "string" },
+      "description": "IDs of related findings"
+    },
+    "proofNode": {
+      "type": ["string", "null"],
+      "description": "ID of proof graph node if applicable"
+    },
+    "missionType": {
+      "type": ["string", "null"],
+      "description": "Mission type for fix generation"
+    }
+  },
+  "$defs": {
+    "evidence": {
+      "type": "object",
+      "required": ["id", "kind"],
+      "properties": {
+        "id": {
+          "type": "string",
+          "pattern": "^E_[A-F0-9]+$"
+        },
+        "kind": {
+          "type": "string",
+          "enum": ["file", "request", "runtime", "screenshot", "diff"]
+        },
+        "file": { "type": "string" },
+        "lines": { "type": "string", "pattern": "^\\d+-\\d+$" },
+        "snippetHash": { "type": "string", "pattern": "^sha256:[a-f0-9]+$" },
+        "reason": { "type": "string" },
+        "url": { "type": "string" },
+        "httpStatus": { "type": "integer" },
+        "screenshot": { "type": "string" },
+        "data": { "type": "object" }
+      }
+    }
+  }
+}