@vibecheckai/cli 3.0.4 → 3.0.7

package/bin/runners/lib/missions/templates.js

@@ -134,6 +134,51 @@ function templateForMissionType(type) {
         success: ["Dead UI findings disappear from ship (after running reality again)."]
       };
 
+    case "SYNC_CONTRACTS":
+      return {
+        intent: "Update contracts to match current code reality. Drift causes AI hallucinations.",
+        do: [
+          "Run 'vibecheck ctx sync' to regenerate contracts from truthpack.",
+          "Review the diff to ensure changes are intentional.",
+          "Commit updated contracts alongside code changes."
+        ],
+        dont: [
+          "Do not manually edit contract JSON files.",
+          "Do not ignore drift - it will cause AI to generate broken code."
+        ],
+        success: ["Contract drift findings disappear from ship."]
+      };
+
+    case "FIX_ROUTE_DRIFT":
+      return {
+        intent: "Align route contract with actual server routes.",
+        do: [
+          "If route was intentionally added: run 'vibecheck ctx sync'.",
+          "If route was accidentally removed: restore it or update client refs.",
+          "If client refs fake route: fix the client to use real routes."
+        ],
+        dont: [
+          "Do not invent routes to match client refs.",
+          "Do not remove routes without updating all client references."
+        ],
+        success: ["Route drift findings disappear."]
+      };
+
+    case "FIX_AUTH_DRIFT":
+      return {
+        intent: "Align auth contract with actual middleware patterns.",
+        do: [
+          "If auth pattern was intentionally changed: run 'vibecheck ctx sync'.",
+          "If auth was accidentally removed: RESTORE IT IMMEDIATELY (security risk).",
+          "Verify all sensitive routes are still protected."
+        ],
+        dont: [
+          "Do not remove auth patterns without security review.",
+          "Do not ignore auth drift - it may indicate a security regression."
+        ],
+        success: ["Auth drift findings disappear."]
+      };
+
     default:
       return {
         intent: "Fix the specific finding with smallest correct patch.",

package/bin/runners/lib/policy.js

@@ -0,0 +1,295 @@
+/**
+ * Policy Presets v2
+ * 
+ * Pre-configured policy settings so users don't have to bikeshed.
+ */
+
+"use strict";
+
+const fs = require("fs");
+const path = require("path");
+
+// =============================================================================
+// POLICY PRESETS
+// =============================================================================
+
+const POLICY_PRESETS = {
+  /**
+   * DEV: Warn-heavy, low thresholds
+   * For local development - informative but not blocking
+   */
+  dev: {
+    name: "dev",
+    description: "Development mode - warn-heavy, informative",
+    failOnWarn: false,
+    coverage: {
+      minActionCoverage: 0,
+      minRouteCoverage: 0,
+      requireAuthCoverage: false,
+    },
+    fastify: {
+      requireRuntimeDump: false,
+      allowStaticOnlyRoutes: true,
+    },
+    auth: {
+      requireVerifyAuth: false,
+      strictProtectedPatterns: false,
+    },
+    findings: {
+      // Downgrade some blockers to warnings in dev
+      downgradeToWarn: [
+        "D_ROUTE_MISSING",     // Missing routes less critical in dev
+        "D_CONTRACT_DRIFT",    // Drift expected during dev
+      ],
+    },
+    output: {
+      generateHTML: true,
+      verboseConsole: true,
+    },
+  },
+
+  /**
+   * CI: Strict enough to block bad PRs
+   * Default for CI pipelines
+   */
+  ci: {
+    name: "ci",
+    description: "CI mode - strict, blocks on real issues",
+    failOnWarn: false,
+    coverage: {
+      minActionCoverage: 0,       // Don't require coverage yet
+      minRouteCoverage: 0,
+      requireAuthCoverage: false,
+    },
+    fastify: {
+      requireRuntimeDump: false,   // Optional but recommended
+      allowStaticOnlyRoutes: true,
+    },
+    auth: {
+      requireVerifyAuth: false,    // Only if auth patterns exist
+      strictProtectedPatterns: true,
+    },
+    findings: {
+      downgradeToWarn: [],
+    },
+    output: {
+      generateHTML: true,
+      verboseConsole: false,
+    },
+  },
+
+  /**
+   * RELEASE: Production-ready checks
+   * Strictest mode for release gates
+   */
+  release: {
+    name: "release",
+    description: "Release mode - requires coverage if patterns exist",
+    failOnWarn: true,
+    coverage: {
+      minActionCoverage: 50,      // At least 50% of UI actions verified
+      minRouteCoverage: 80,       // At least 80% of routes covered
+      requireAuthCoverage: true,  // If auth patterns exist, verify them
+      minAuthCoverage: 80,
+    },
+    fastify: {
+      requireRuntimeDump: true,    // Must run runtime dump for Fastify
+      allowStaticOnlyRoutes: false,
+    },
+    auth: {
+      requireVerifyAuth: true,     // Must verify auth if patterns exist
+      strictProtectedPatterns: true,
+    },
+    findings: {
+      downgradeToWarn: [],
+      upgradeToBlock: [
+        "D_FAKE_SUCCESS",          // Fake success is blocking in release
+        "D_DEAD_CLICK",            // Dead clicks are blocking in release
+      ],
+    },
+    output: {
+      generateHTML: true,
+      verboseConsole: false,
+    },
+  },
+};
+
+// =============================================================================
+// POLICY RESOLUTION
+// =============================================================================
+
+/**
+ * Load policy from file or preset
+ */
+function loadPolicy(options = {}) {
+  const { 
+    preset = "ci", 
+    policyFile = null,
+    repoRoot = process.cwd(),
+    overrides = {},
+  } = options;
+
+  let policy;
+
+  // Try loading from file first
+  if (policyFile) {
+    const policyPath = path.isAbsolute(policyFile) 
+      ? policyFile 
+      : path.join(repoRoot, policyFile);
+    
+    if (fs.existsSync(policyPath)) {
+      try {
+        policy = JSON.parse(fs.readFileSync(policyPath, "utf8"));
+      } catch (e) {
+        console.warn(`Warning: Could not parse policy file ${policyPath}: ${e.message}`);
+      }
+    }
+  }
+
+  // Try loading from .vibecheck/policy.json
+  if (!policy) {
+    const defaultPolicyPath = path.join(repoRoot, ".vibecheck", "policy.json");
+    if (fs.existsSync(defaultPolicyPath)) {
+      try {
+        policy = JSON.parse(fs.readFileSync(defaultPolicyPath, "utf8"));
+      } catch (e) {
+        // Ignore parse errors for default file
+      }
+    }
+  }
+
+  // Fall back to preset
+  if (!policy) {
+    policy = POLICY_PRESETS[preset] || POLICY_PRESETS.ci;
+  }
+
+  // Apply overrides
+  return mergePolicy(policy, overrides);
+}
+
+/**
+ * Deep merge policy with overrides
+ */
+function mergePolicy(base, overrides) {
+  const merged = JSON.parse(JSON.stringify(base));
+
+  for (const [key, value] of Object.entries(overrides)) {
+    if (value === undefined) continue;
+    
+    if (typeof value === "object" && value !== null && !Array.isArray(value)) {
+      merged[key] = mergePolicy(merged[key] || {}, value);
+    } else {
+      merged[key] = value;
+    }
+  }
+
+  return merged;
+}
+
+/**
+ * Get effective thresholds from policy
+ */
+function getThresholdsFromPolicy(policy) {
+  return {
+    minActionCoverage: policy.coverage?.minActionCoverage ?? 0,
+    minRouteCoverage: policy.coverage?.minRouteCoverage ?? 0,
+    requireAuthCoverage: policy.coverage?.requireAuthCoverage ?? false,
+    minAuthCoverage: policy.coverage?.minAuthCoverage ?? 80,
+  };
+}
+
+/**
+ * Apply policy to findings (downgrades/upgrades)
+ */
+function applyPolicyToFindings(findings, policy) {
+  const downgrade = new Set(policy.findings?.downgradeToWarn || []);
+  const upgrade = new Set(policy.findings?.upgradeToBlock || []);
+
+  return findings.map(finding => {
+    if (downgrade.has(finding.detectorId) && finding.severity === "BLOCK") {
+      return {
+        ...finding,
+        severity: "WARN",
+        originalSeverity: "BLOCK",
+        policyDowngraded: true,
+      };
+    }
+    
+    if (upgrade.has(finding.detectorId) && finding.severity === "WARN") {
+      return {
+        ...finding,
+        severity: "BLOCK",
+        originalSeverity: "WARN",
+        policyUpgraded: true,
+      };
+    }
+    
+    return finding;
+  });
+}
+
+/**
+ * Check if auth verification is required based on policy and truthpack
+ */
+function requiresAuthVerification(policy, truthpack) {
+  if (!policy.auth?.requireVerifyAuth) return false;
+  
+  // Only require if auth patterns exist
+  const hasAuthPatterns = truthpack?.auth?.protectedPatterns?.length > 0;
+  return hasAuthPatterns;
+}
+
+/**
+ * Check if Fastify runtime dump is required
+ */
+function requiresFastifyRuntimeDump(policy, truthpack) {
+  if (!policy.fastify?.requireRuntimeDump) return false;
+  
+  // Only require if Fastify is detected
+  const hasFastify = truthpack?.stack?.fastify?.present;
+  return hasFastify;
+}
+
+/**
+ * Write policy file
+ */
+function writePolicyFile(repoRoot, policy) {
+  const dir = path.join(repoRoot, ".vibecheck");
+  fs.mkdirSync(dir, { recursive: true });
+
+  const policyPath = path.join(dir, "policy.json");
+  fs.writeFileSync(policyPath, JSON.stringify(policy, null, 2));
+
+  return policyPath;
+}
+
+/**
+ * Get preset names
+ */
+function getPresetNames() {
+  return Object.keys(POLICY_PRESETS);
+}
+
+/**
+ * Get preset by name
+ */
+function getPreset(name) {
+  return POLICY_PRESETS[name] || null;
+}
+
+// =============================================================================
+// EXPORTS
+// =============================================================================
+
+module.exports = {
+  POLICY_PRESETS,
+  loadPolicy,
+  mergePolicy,
+  getThresholdsFromPolicy,
+  applyPolicyToFindings,
+  requiresAuthVerification,
+  requiresFastifyRuntimeDump,
+  writePolicyFile,
+  getPresetNames,
+  getPreset,
+};

package/bin/runners/lib/reality/correlation-detectors.js

@@ -0,0 +1,359 @@
+/**
+ * Correlation Detectors v2
+ * 
+ * Detects mismatches between toast signals, UI changes, and network requests.
+ * These are the "fake success theatre" detectors.
+ */
+
+"use strict";
+
+const crypto = require("crypto");
+
+// =============================================================================
+// DETECTOR DEFINITIONS
+// =============================================================================
+
+const DETECTORS = {
+  D_CORR_012: {
+    id: "D_CORR_012",
+    name: "Success toast but no real change",
+    description: "Success toast appeared but UI didn't change meaningfully and no mutation succeeded",
+    severity: "BLOCK",
+    category: "FakeSuccess",
+  },
+  D_CORR_013: {
+    id: "D_CORR_013",
+    name: "Mutation succeeded but no UI feedback",
+    description: "Mutation request succeeded but no meaningful UI change or toast appeared",
+    severity: "WARN",
+    category: "DeadUI",
+  },
+  D_CORR_014: {
+    id: "D_CORR_014",
+    name: "Error toast with successful mutation",
+    description: "Error toast appeared but the mutation request succeeded",
+    severity: "WARN",
+    category: "UIMismatch",
+  },
+  D_CORR_015: {
+    id: "D_CORR_015",
+    name: "Click with no effect",
+    description: "Button/link clicked but no network request, no UI change, no toast",
+    severity: "BLOCK",
+    category: "DeadUI",
+  },
+  D_CORR_016: {
+    id: "D_CORR_016",
+    name: "Form submit with no validation feedback",
+    description: "Form submitted but no validation errors shown and no success indicator",
+    severity: "WARN",
+    category: "DeadUI",
+  },
+  D_CORR_017: {
+    id: "D_CORR_017",
+    name: "Optimistic update without server confirmation",
+    description: "UI updated immediately but server request failed or never completed",
+    severity: "BLOCK",
+    category: "FakeSuccess",
+  },
+  D_CORR_018: {
+    id: "D_CORR_018",
+    name: "Success toast before request completed",
+    description: "Success toast appeared before the mutation request finished",
+    severity: "BLOCK",
+    category: "FakeSuccess",
+  },
+};
+
+// =============================================================================
+// CORRELATION ANALYSIS
+// =============================================================================
+
+/**
+ * Analyze action outcomes and detect correlation issues
+ */
+function analyzeActionCorrelation(action, options = {}) {
+  const findings = [];
+  const {
+    uiChange = {},
+    signals = [],
+    requests = [],
+    actionType = "click",
+    actionLabel = "",
+  } = action;
+
+  const { meaningfulScoreThreshold = 0.6 } = options;
+
+  // Extract key data
+  const toastSuccess = signals.find(s => s.kind === "toast_success");
+  const toastError = signals.find(s => s.kind === "toast_error");
+  const toastAny = signals.find(s => s.kind?.startsWith("toast_"));
+
+  const mutationRequests = requests.filter(r => 
+    ["POST", "PUT", "PATCH", "DELETE"].includes(r.method?.toUpperCase())
+  );
+  const successfulMutations = mutationRequests.filter(r => r.status >= 200 && r.status < 300);
+  const failedMutations = mutationRequests.filter(r => r.status >= 400 || r.failed);
+
+  const hasMeaningfulChange = uiChange.meaningful === true || uiChange.score >= meaningfulScoreThreshold;
+  const hasNetworkActivity = requests.length > 0;
+  const hasMutation = mutationRequests.length > 0;
+
+  // D_CORR_012: Success toast but no real change
+  if (toastSuccess && !hasMeaningfulChange && successfulMutations.length === 0) {
+    findings.push(createFinding(DETECTORS.D_CORR_012, {
+      action,
+      evidence: {
+        toast: toastSuccess,
+        uiChangeScore: uiChange.score,
+        mutations: mutationRequests.length,
+        successfulMutations: successfulMutations.length,
+      },
+      reason: `Success toast "${toastSuccess.message?.slice(0, 50)}" appeared but UI score was ${uiChange.score?.toFixed(2)} and no mutation succeeded`,
+    }));
+  }
+
+  // D_CORR_013: Mutation succeeded but no UI feedback
+  if (successfulMutations.length > 0 && !hasMeaningfulChange && !toastAny) {
+    findings.push(createFinding(DETECTORS.D_CORR_013, {
+      action,
+      evidence: {
+        successfulMutations: successfulMutations.map(r => ({ method: r.method, url: r.url, status: r.status })),
+        uiChangeScore: uiChange.score,
+        hasToast: false,
+      },
+      reason: `Mutation ${successfulMutations[0]?.method} ${successfulMutations[0]?.url} succeeded (${successfulMutations[0]?.status}) but no UI feedback`,
+    }));
+  }
+
+  // D_CORR_014: Error toast with successful mutation
+  if (toastError && successfulMutations.length > 0 && failedMutations.length === 0) {
+    findings.push(createFinding(DETECTORS.D_CORR_014, {
+      action,
+      evidence: {
+        toast: toastError,
+        successfulMutations: successfulMutations.map(r => ({ method: r.method, url: r.url, status: r.status })),
+      },
+      reason: `Error toast "${toastError.message?.slice(0, 50)}" but mutation succeeded with ${successfulMutations[0]?.status}`,
+    }));
+  }
+
+  // D_CORR_015: Click with no effect
+  if (actionType === "click" && !hasNetworkActivity && !hasMeaningfulChange && !toastAny) {
+    // Only flag if it looks like an interactive element
+    const isInteractive = actionLabel && (
+      /save|submit|send|update|delete|create|add|remove/i.test(actionLabel)
+    );
+    
+    if (isInteractive) {
+      findings.push(createFinding(DETECTORS.D_CORR_015, {
+        action,
+        evidence: {
+          actionLabel,
+          uiChangeScore: uiChange.score,
+          requestCount: requests.length,
+          hasToast: !!toastAny,
+        },
+        reason: `Click on "${actionLabel}" had no network request, no UI change, no toast`,
+      }));
+    }
+  }
+
+  // D_CORR_017: Optimistic update without server confirmation
+  if (uiChange.meaningful && hasMutation && failedMutations.length > 0 && successfulMutations.length === 0) {
+    findings.push(createFinding(DETECTORS.D_CORR_017, {
+      action,
+      evidence: {
+        uiChangeReasons: uiChange.reasons,
+        failedMutations: failedMutations.map(r => ({ method: r.method, url: r.url, status: r.status })),
+      },
+      reason: `UI updated (${uiChange.reasons?.join(", ")}) but mutation failed with ${failedMutations[0]?.status}`,
+    }));
+  }
+
+  // D_CORR_018: Success toast before request completed
+  if (toastSuccess && mutationRequests.length > 0) {
+    const toastTime = toastSuccess.atMs;
+    const pendingAtToastTime = mutationRequests.filter(r => {
+      // Request was still pending when toast appeared
+      const requestEndTime = r.startTime + (r.duration || 0);
+      return r.startTime < toastTime && requestEndTime > toastTime;
+    });
+
+    if (pendingAtToastTime.length > 0) {
+      findings.push(createFinding(DETECTORS.D_CORR_018, {
+        action,
+        evidence: {
+          toast: toastSuccess,
+          toastTime,
+          pendingRequests: pendingAtToastTime.map(r => ({ 
+            method: r.method, 
+            url: r.url,
+            startTime: r.startTime,
+            duration: r.duration,
+          })),
+        },
+        reason: `Success toast appeared at ${toastTime}ms but request was still pending`,
+      }));
+    }
+  }
+
+  return findings;
+}
+
+/**
+ * Analyze multiple actions and aggregate findings
+ */
+function analyzeAllActions(actions, options = {}) {
+  const allFindings = [];
+
+  for (const action of actions) {
+    const findings = analyzeActionCorrelation(action, options);
+    allFindings.push(...findings);
+  }
+
+  // Dedupe by fingerprint
+  const seen = new Set();
+  const deduped = [];
+  for (const f of allFindings) {
+    if (!seen.has(f.fingerprint)) {
+      seen.add(f.fingerprint);
+      deduped.push(f);
+    }
+  }
+
+  return {
+    findings: deduped,
+    stats: {
+      totalActions: actions.length,
+      actionsWithFindings: new Set(allFindings.map(f => f.actionId)).size,
+      findingsByDetector: countByDetector(deduped),
+      findingsBySeverity: countBySeverity(deduped),
+    },
+  };
+}
+
+/**
+ * Create a finding from detector definition
+ */
+function createFinding(detector, data = {}) {
+  const { action, evidence, reason } = data;
+  
+  const fingerprint = generateFingerprint([
+    detector.id,
+    action?.id || action?.selector || "",
+    reason,
+  ]);
+
+  return {
+    id: `F_${detector.id}_${fingerprint.slice(0, 8)}`,
+    detectorId: detector.id,
+    fingerprint: `sha256:${fingerprint}`,
+    severity: detector.severity,
+    category: detector.category,
+    scope: "runtime",
+    title: detector.name,
+    why: reason || detector.description,
+    confidence: "high",
+    evidence: [
+      {
+        id: `E_${fingerprint.slice(0, 12)}`,
+        kind: "runtime",
+        reason,
+        data: evidence,
+      },
+    ],
+    actionId: action?.id,
+    repro: action?.selector ? `Click on "${action.selector}"` : null,
+  };
+}
+
+/**
+ * Generate deterministic fingerprint
+ */
+function generateFingerprint(parts) {
+  const content = parts.filter(Boolean).join("|");
+  return crypto.createHash("sha256").update(content).digest("hex");
+}
+
+/**
+ * Count findings by detector
+ */
+function countByDetector(findings) {
+  const counts = {};
+  for (const f of findings) {
+    counts[f.detectorId] = (counts[f.detectorId] || 0) + 1;
+  }
+  return counts;
+}
+
+/**
+ * Count findings by severity
+ */
+function countBySeverity(findings) {
+  const counts = { BLOCK: 0, WARN: 0, INFO: 0 };
+  for (const f of findings) {
+    counts[f.severity] = (counts[f.severity] || 0) + 1;
+  }
+  return counts;
+}
+
+/**
+ * Summarize action outcome for reporting
+ */
+function summarizeActionOutcome(action) {
+  const {
+    uiChange = {},
+    signals = [],
+    requests = [],
+    selector = "",
+    actionType = "click",
+  } = action;
+
+  const toasts = signals.filter(s => s.kind?.startsWith("toast_"));
+  const mutations = requests.filter(r => 
+    ["POST", "PUT", "PATCH", "DELETE"].includes(r.method?.toUpperCase())
+  );
+  const successfulMutations = mutations.filter(r => r.status >= 200 && r.status < 300);
+
+  let outcome = "unknown";
+  
+  if (successfulMutations.length > 0 && uiChange.meaningful) {
+    outcome = "success_confirmed";
+  } else if (successfulMutations.length > 0 && !uiChange.meaningful) {
+    outcome = "success_silent";
+  } else if (mutations.length > 0 && successfulMutations.length === 0) {
+    outcome = "mutation_failed";
+  } else if (toasts.some(t => t.kind === "toast_success")) {
+    outcome = successfulMutations.length > 0 ? "success_confirmed" : "toast_only";
+  } else if (toasts.some(t => t.kind === "toast_error")) {
+    outcome = "error_shown";
+  } else if (!uiChange.meaningful && requests.length === 0) {
+    outcome = "no_effect";
+  } else if (uiChange.meaningful) {
+    outcome = "ui_changed";
+  }
+
+  return {
+    actionType,
+    selector,
+    outcome,
+    uiChangeScore: uiChange.score,
+    uiChangeMeaningful: uiChange.meaningful,
+    uiChangeReasons: uiChange.reasons || [],
+    toastCount: toasts.length,
+    toastKinds: toasts.map(t => t.kind),
+    requestCount: requests.length,
+    mutationCount: mutations.length,
+    successfulMutationCount: successfulMutations.length,
+  };
+}
+
+module.exports = {
+  DETECTORS,
+  analyzeActionCorrelation,
+  analyzeAllActions,
+  createFinding,
+  summarizeActionOutcome,
+  generateFingerprint,
+};