@vibecheckai/cli 3.2.5 → 3.3.0

package/bin/runners/lib/reality/correlation-detectors.js

@@ -1,359 +1,359 @@
-/**
- * Correlation Detectors v2
- * 
- * Detects mismatches between toast signals, UI changes, and network requests.
- * These are the "fake success theatre" detectors.
- */
-
-"use strict";
-
-const crypto = require("crypto");
-
-// =============================================================================
-// DETECTOR DEFINITIONS
-// =============================================================================
-
-const DETECTORS = {
-  D_CORR_012: {
-    id: "D_CORR_012",
-    name: "Success toast but no real change",
-    description: "Success toast appeared but UI didn't change meaningfully and no mutation succeeded",
-    severity: "BLOCK",
-    category: "FakeSuccess",
-  },
-  D_CORR_013: {
-    id: "D_CORR_013",
-    name: "Mutation succeeded but no UI feedback",
-    description: "Mutation request succeeded but no meaningful UI change or toast appeared",
-    severity: "WARN",
-    category: "DeadUI",
-  },
-  D_CORR_014: {
-    id: "D_CORR_014",
-    name: "Error toast with successful mutation",
-    description: "Error toast appeared but the mutation request succeeded",
-    severity: "WARN",
-    category: "UIMismatch",
-  },
-  D_CORR_015: {
-    id: "D_CORR_015",
-    name: "Click with no effect",
-    description: "Button/link clicked but no network request, no UI change, no toast",
-    severity: "BLOCK",
-    category: "DeadUI",
-  },
-  D_CORR_016: {
-    id: "D_CORR_016",
-    name: "Form submit with no validation feedback",
-    description: "Form submitted but no validation errors shown and no success indicator",
-    severity: "WARN",
-    category: "DeadUI",
-  },
-  D_CORR_017: {
-    id: "D_CORR_017",
-    name: "Optimistic update without server confirmation",
-    description: "UI updated immediately but server request failed or never completed",
-    severity: "BLOCK",
-    category: "FakeSuccess",
-  },
-  D_CORR_018: {
-    id: "D_CORR_018",
-    name: "Success toast before request completed",
-    description: "Success toast appeared before the mutation request finished",
-    severity: "BLOCK",
-    category: "FakeSuccess",
-  },
-};
-
-// =============================================================================
-// CORRELATION ANALYSIS
-// =============================================================================
-
-/**
- * Analyze action outcomes and detect correlation issues
- */
-function analyzeActionCorrelation(action, options = {}) {
-  const findings = [];
-  const {
-    uiChange = {},
-    signals = [],
-    requests = [],
-    actionType = "click",
-    actionLabel = "",
-  } = action;
-
-  const { meaningfulScoreThreshold = 0.6 } = options;
-
-  // Extract key data
-  const toastSuccess = signals.find(s => s.kind === "toast_success");
-  const toastError = signals.find(s => s.kind === "toast_error");
-  const toastAny = signals.find(s => s.kind?.startsWith("toast_"));
-
-  const mutationRequests = requests.filter(r => 
-    ["POST", "PUT", "PATCH", "DELETE"].includes(r.method?.toUpperCase())
-  );
-  const successfulMutations = mutationRequests.filter(r => r.status >= 200 && r.status < 300);
-  const failedMutations = mutationRequests.filter(r => r.status >= 400 || r.failed);
-
-  const hasMeaningfulChange = uiChange.meaningful === true || uiChange.score >= meaningfulScoreThreshold;
-  const hasNetworkActivity = requests.length > 0;
-  const hasMutation = mutationRequests.length > 0;
-
-  // D_CORR_012: Success toast but no real change
-  if (toastSuccess && !hasMeaningfulChange && successfulMutations.length === 0) {
-    findings.push(createFinding(DETECTORS.D_CORR_012, {
-      action,
-      evidence: {
-        toast: toastSuccess,
-        uiChangeScore: uiChange.score,
-        mutations: mutationRequests.length,
-        successfulMutations: successfulMutations.length,
-      },
-      reason: `Success toast "${toastSuccess.message?.slice(0, 50)}" appeared but UI score was ${uiChange.score?.toFixed(2)} and no mutation succeeded`,
-    }));
-  }
-
-  // D_CORR_013: Mutation succeeded but no UI feedback
-  if (successfulMutations.length > 0 && !hasMeaningfulChange && !toastAny) {
-    findings.push(createFinding(DETECTORS.D_CORR_013, {
-      action,
-      evidence: {
-        successfulMutations: successfulMutations.map(r => ({ method: r.method, url: r.url, status: r.status })),
-        uiChangeScore: uiChange.score,
-        hasToast: false,
-      },
-      reason: `Mutation ${successfulMutations[0]?.method} ${successfulMutations[0]?.url} succeeded (${successfulMutations[0]?.status}) but no UI feedback`,
-    }));
-  }
-
-  // D_CORR_014: Error toast with successful mutation
-  if (toastError && successfulMutations.length > 0 && failedMutations.length === 0) {
-    findings.push(createFinding(DETECTORS.D_CORR_014, {
-      action,
-      evidence: {
-        toast: toastError,
-        successfulMutations: successfulMutations.map(r => ({ method: r.method, url: r.url, status: r.status })),
-      },
-      reason: `Error toast "${toastError.message?.slice(0, 50)}" but mutation succeeded with ${successfulMutations[0]?.status}`,
-    }));
-  }
-
-  // D_CORR_015: Click with no effect
-  if (actionType === "click" && !hasNetworkActivity && !hasMeaningfulChange && !toastAny) {
-    // Only flag if it looks like an interactive element
-    const isInteractive = actionLabel && (
-      /save|submit|send|update|delete|create|add|remove/i.test(actionLabel)
-    );
-    
-    if (isInteractive) {
-      findings.push(createFinding(DETECTORS.D_CORR_015, {
-        action,
-        evidence: {
-          actionLabel,
-          uiChangeScore: uiChange.score,
-          requestCount: requests.length,
-          hasToast: !!toastAny,
-        },
-        reason: `Click on "${actionLabel}" had no network request, no UI change, no toast`,
-      }));
-    }
-  }
-
-  // D_CORR_017: Optimistic update without server confirmation
-  if (uiChange.meaningful && hasMutation && failedMutations.length > 0 && successfulMutations.length === 0) {
-    findings.push(createFinding(DETECTORS.D_CORR_017, {
-      action,
-      evidence: {
-        uiChangeReasons: uiChange.reasons,
-        failedMutations: failedMutations.map(r => ({ method: r.method, url: r.url, status: r.status })),
-      },
-      reason: `UI updated (${uiChange.reasons?.join(", ")}) but mutation failed with ${failedMutations[0]?.status}`,
-    }));
-  }
-
-  // D_CORR_018: Success toast before request completed
-  if (toastSuccess && mutationRequests.length > 0) {
-    const toastTime = toastSuccess.atMs;
-    const pendingAtToastTime = mutationRequests.filter(r => {
-      // Request was still pending when toast appeared
-      const requestEndTime = r.startTime + (r.duration || 0);
-      return r.startTime < toastTime && requestEndTime > toastTime;
-    });
-
-    if (pendingAtToastTime.length > 0) {
-      findings.push(createFinding(DETECTORS.D_CORR_018, {
-        action,
-        evidence: {
-          toast: toastSuccess,
-          toastTime,
-          pendingRequests: pendingAtToastTime.map(r => ({ 
-            method: r.method, 
-            url: r.url,
-            startTime: r.startTime,
-            duration: r.duration,
-          })),
-        },
-        reason: `Success toast appeared at ${toastTime}ms but request was still pending`,
-      }));
-    }
-  }
-
-  return findings;
-}
-
-/**
- * Analyze multiple actions and aggregate findings
- */
-function analyzeAllActions(actions, options = {}) {
-  const allFindings = [];
-
-  for (const action of actions) {
-    const findings = analyzeActionCorrelation(action, options);
-    allFindings.push(...findings);
-  }
-
-  // Dedupe by fingerprint
-  const seen = new Set();
-  const deduped = [];
-  for (const f of allFindings) {
-    if (!seen.has(f.fingerprint)) {
-      seen.add(f.fingerprint);
-      deduped.push(f);
-    }
-  }
-
-  return {
-    findings: deduped,
-    stats: {
-      totalActions: actions.length,
-      actionsWithFindings: new Set(allFindings.map(f => f.actionId)).size,
-      findingsByDetector: countByDetector(deduped),
-      findingsBySeverity: countBySeverity(deduped),
-    },
-  };
-}
-
-/**
- * Create a finding from detector definition
- */
-function createFinding(detector, data = {}) {
-  const { action, evidence, reason } = data;
-  
-  const fingerprint = generateFingerprint([
-    detector.id,
-    action?.id || action?.selector || "",
-    reason,
-  ]);
-
-  return {
-    id: `F_${detector.id}_${fingerprint.slice(0, 8)}`,
-    detectorId: detector.id,
-    fingerprint: `sha256:${fingerprint}`,
-    severity: detector.severity,
-    category: detector.category,
-    scope: "runtime",
-    title: detector.name,
-    why: reason || detector.description,
-    confidence: "high",
-    evidence: [
-      {
-        id: `E_${fingerprint.slice(0, 12)}`,
-        kind: "runtime",
-        reason,
-        data: evidence,
-      },
-    ],
-    actionId: action?.id,
-    repro: action?.selector ? `Click on "${action.selector}"` : null,
-  };
-}
-
-/**
- * Generate deterministic fingerprint
- */
-function generateFingerprint(parts) {
-  const content = parts.filter(Boolean).join("|");
-  return crypto.createHash("sha256").update(content).digest("hex");
-}
-
-/**
- * Count findings by detector
- */
-function countByDetector(findings) {
-  const counts = {};
-  for (const f of findings) {
-    counts[f.detectorId] = (counts[f.detectorId] || 0) + 1;
-  }
-  return counts;
-}
-
-/**
- * Count findings by severity
- */
-function countBySeverity(findings) {
-  const counts = { BLOCK: 0, WARN: 0, INFO: 0 };
-  for (const f of findings) {
-    counts[f.severity] = (counts[f.severity] || 0) + 1;
-  }
-  return counts;
-}
-
-/**
- * Summarize action outcome for reporting
- */
-function summarizeActionOutcome(action) {
-  const {
-    uiChange = {},
-    signals = [],
-    requests = [],
-    selector = "",
-    actionType = "click",
-  } = action;
-
-  const toasts = signals.filter(s => s.kind?.startsWith("toast_"));
-  const mutations = requests.filter(r => 
-    ["POST", "PUT", "PATCH", "DELETE"].includes(r.method?.toUpperCase())
-  );
-  const successfulMutations = mutations.filter(r => r.status >= 200 && r.status < 300);
-
-  let outcome = "unknown";
-  
-  if (successfulMutations.length > 0 && uiChange.meaningful) {
-    outcome = "success_confirmed";
-  } else if (successfulMutations.length > 0 && !uiChange.meaningful) {
-    outcome = "success_silent";
-  } else if (mutations.length > 0 && successfulMutations.length === 0) {
-    outcome = "mutation_failed";
-  } else if (toasts.some(t => t.kind === "toast_success")) {
-    outcome = successfulMutations.length > 0 ? "success_confirmed" : "toast_only";
-  } else if (toasts.some(t => t.kind === "toast_error")) {
-    outcome = "error_shown";
-  } else if (!uiChange.meaningful && requests.length === 0) {
-    outcome = "no_effect";
-  } else if (uiChange.meaningful) {
-    outcome = "ui_changed";
-  }
-
-  return {
-    actionType,
-    selector,
-    outcome,
-    uiChangeScore: uiChange.score,
-    uiChangeMeaningful: uiChange.meaningful,
-    uiChangeReasons: uiChange.reasons || [],
-    toastCount: toasts.length,
-    toastKinds: toasts.map(t => t.kind),
-    requestCount: requests.length,
-    mutationCount: mutations.length,
-    successfulMutationCount: successfulMutations.length,
-  };
-}
-
-module.exports = {
-  DETECTORS,
-  analyzeActionCorrelation,
-  analyzeAllActions,
-  createFinding,
-  summarizeActionOutcome,
-  generateFingerprint,
-};
+/**
+ * Correlation Detectors v2
+ * 
+ * Detects mismatches between toast signals, UI changes, and network requests.
+ * These are the "fake success theatre" detectors.
+ */
+
+"use strict";
+
+const crypto = require("crypto");
+
+// =============================================================================
+// DETECTOR DEFINITIONS
+// =============================================================================
+
+const DETECTORS = {
+  D_CORR_012: {
+    id: "D_CORR_012",
+    name: "Success toast but no real change",
+    description: "Success toast appeared but UI didn't change meaningfully and no mutation succeeded",
+    severity: "BLOCK",
+    category: "FakeSuccess",
+  },
+  D_CORR_013: {
+    id: "D_CORR_013",
+    name: "Mutation succeeded but no UI feedback",
+    description: "Mutation request succeeded but no meaningful UI change or toast appeared",
+    severity: "WARN",
+    category: "DeadUI",
+  },
+  D_CORR_014: {
+    id: "D_CORR_014",
+    name: "Error toast with successful mutation",
+    description: "Error toast appeared but the mutation request succeeded",
+    severity: "WARN",
+    category: "UIMismatch",
+  },
+  D_CORR_015: {
+    id: "D_CORR_015",
+    name: "Click with no effect",
+    description: "Button/link clicked but no network request, no UI change, no toast",
+    severity: "BLOCK",
+    category: "DeadUI",
+  },
+  D_CORR_016: {
+    id: "D_CORR_016",
+    name: "Form submit with no validation feedback",
+    description: "Form submitted but no validation errors shown and no success indicator",
+    severity: "WARN",
+    category: "DeadUI",
+  },
+  D_CORR_017: {
+    id: "D_CORR_017",
+    name: "Optimistic update without server confirmation",
+    description: "UI updated immediately but server request failed or never completed",
+    severity: "BLOCK",
+    category: "FakeSuccess",
+  },
+  D_CORR_018: {
+    id: "D_CORR_018",
+    name: "Success toast before request completed",
+    description: "Success toast appeared before the mutation request finished",
+    severity: "BLOCK",
+    category: "FakeSuccess",
+  },
+};
+
+// =============================================================================
+// CORRELATION ANALYSIS
+// =============================================================================
+
+/**
+ * Analyze action outcomes and detect correlation issues
+ */
+function analyzeActionCorrelation(action, options = {}) {
+  const findings = [];
+  const {
+    uiChange = {},
+    signals = [],
+    requests = [],
+    actionType = "click",
+    actionLabel = "",
+  } = action;
+
+  const { meaningfulScoreThreshold = 0.6 } = options;
+
+  // Extract key data
+  const toastSuccess = signals.find(s => s.kind === "toast_success");
+  const toastError = signals.find(s => s.kind === "toast_error");
+  const toastAny = signals.find(s => s.kind?.startsWith("toast_"));
+
+  const mutationRequests = requests.filter(r => 
+    ["POST", "PUT", "PATCH", "DELETE"].includes(r.method?.toUpperCase())
+  );
+  const successfulMutations = mutationRequests.filter(r => r.status >= 200 && r.status < 300);
+  const failedMutations = mutationRequests.filter(r => r.status >= 400 || r.failed);
+
+  const hasMeaningfulChange = uiChange.meaningful === true || uiChange.score >= meaningfulScoreThreshold;
+  const hasNetworkActivity = requests.length > 0;
+  const hasMutation = mutationRequests.length > 0;
+
+  // D_CORR_012: Success toast but no real change
+  if (toastSuccess && !hasMeaningfulChange && successfulMutations.length === 0) {
+    findings.push(createFinding(DETECTORS.D_CORR_012, {
+      action,
+      evidence: {
+        toast: toastSuccess,
+        uiChangeScore: uiChange.score,
+        mutations: mutationRequests.length,
+        successfulMutations: successfulMutations.length,
+      },
+      reason: `Success toast "${toastSuccess.message?.slice(0, 50)}" appeared but UI score was ${uiChange.score?.toFixed(2)} and no mutation succeeded`,
+    }));
+  }
+
+  // D_CORR_013: Mutation succeeded but no UI feedback
+  if (successfulMutations.length > 0 && !hasMeaningfulChange && !toastAny) {
+    findings.push(createFinding(DETECTORS.D_CORR_013, {
+      action,
+      evidence: {
+        successfulMutations: successfulMutations.map(r => ({ method: r.method, url: r.url, status: r.status })),
+        uiChangeScore: uiChange.score,
+        hasToast: false,
+      },
+      reason: `Mutation ${successfulMutations[0]?.method} ${successfulMutations[0]?.url} succeeded (${successfulMutations[0]?.status}) but no UI feedback`,
+    }));
+  }
+
+  // D_CORR_014: Error toast with successful mutation
+  if (toastError && successfulMutations.length > 0 && failedMutations.length === 0) {
+    findings.push(createFinding(DETECTORS.D_CORR_014, {
+      action,
+      evidence: {
+        toast: toastError,
+        successfulMutations: successfulMutations.map(r => ({ method: r.method, url: r.url, status: r.status })),
+      },
+      reason: `Error toast "${toastError.message?.slice(0, 50)}" but mutation succeeded with ${successfulMutations[0]?.status}`,
+    }));
+  }
+
+  // D_CORR_015: Click with no effect
+  if (actionType === "click" && !hasNetworkActivity && !hasMeaningfulChange && !toastAny) {
+    // Only flag if it looks like an interactive element
+    const isInteractive = actionLabel && (
+      /save|submit|send|update|delete|create|add|remove/i.test(actionLabel)
+    );
+    
+    if (isInteractive) {
+      findings.push(createFinding(DETECTORS.D_CORR_015, {
+        action,
+        evidence: {
+          actionLabel,
+          uiChangeScore: uiChange.score,
+          requestCount: requests.length,
+          hasToast: !!toastAny,
+        },
+        reason: `Click on "${actionLabel}" had no network request, no UI change, no toast`,
+      }));
+    }
+  }
+
+  // D_CORR_017: Optimistic update without server confirmation
+  if (uiChange.meaningful && hasMutation && failedMutations.length > 0 && successfulMutations.length === 0) {
+    findings.push(createFinding(DETECTORS.D_CORR_017, {
+      action,
+      evidence: {
+        uiChangeReasons: uiChange.reasons,
+        failedMutations: failedMutations.map(r => ({ method: r.method, url: r.url, status: r.status })),
+      },
+      reason: `UI updated (${uiChange.reasons?.join(", ")}) but mutation failed with ${failedMutations[0]?.status}`,
+    }));
+  }
+
+  // D_CORR_018: Success toast before request completed
+  if (toastSuccess && mutationRequests.length > 0) {
+    const toastTime = toastSuccess.atMs;
+    const pendingAtToastTime = mutationRequests.filter(r => {
+      // Request was still pending when toast appeared
+      const requestEndTime = r.startTime + (r.duration || 0);
+      return r.startTime < toastTime && requestEndTime > toastTime;
+    });
+
+    if (pendingAtToastTime.length > 0) {
+      findings.push(createFinding(DETECTORS.D_CORR_018, {
+        action,
+        evidence: {
+          toast: toastSuccess,
+          toastTime,
+          pendingRequests: pendingAtToastTime.map(r => ({ 
+            method: r.method, 
+            url: r.url,
+            startTime: r.startTime,
+            duration: r.duration,
+          })),
+        },
+        reason: `Success toast appeared at ${toastTime}ms but request was still pending`,
+      }));
+    }
+  }
+
+  return findings;
+}
+
+/**
+ * Analyze multiple actions and aggregate findings
+ */
+function analyzeAllActions(actions, options = {}) {
+  const allFindings = [];
+
+  for (const action of actions) {
+    const findings = analyzeActionCorrelation(action, options);
+    allFindings.push(...findings);
+  }
+
+  // Dedupe by fingerprint
+  const seen = new Set();
+  const deduped = [];
+  for (const f of allFindings) {
+    if (!seen.has(f.fingerprint)) {
+      seen.add(f.fingerprint);
+      deduped.push(f);
+    }
+  }
+
+  return {
+    findings: deduped,
+    stats: {
+      totalActions: actions.length,
+      actionsWithFindings: new Set(allFindings.map(f => f.actionId)).size,
+      findingsByDetector: countByDetector(deduped),
+      findingsBySeverity: countBySeverity(deduped),
+    },
+  };
+}
+
+/**
+ * Create a finding from detector definition
+ */
+function createFinding(detector, data = {}) {
+  const { action, evidence, reason } = data;
+  
+  const fingerprint = generateFingerprint([
+    detector.id,
+    action?.id || action?.selector || "",
+    reason,
+  ]);
+
+  return {
+    id: `F_${detector.id}_${fingerprint.slice(0, 8)}`,
+    detectorId: detector.id,
+    fingerprint: `sha256:${fingerprint}`,
+    severity: detector.severity,
+    category: detector.category,
+    scope: "runtime",
+    title: detector.name,
+    why: reason || detector.description,
+    confidence: "high",
+    evidence: [
+      {
+        id: `E_${fingerprint.slice(0, 12)}`,
+        kind: "runtime",
+        reason,
+        data: evidence,
+      },
+    ],
+    actionId: action?.id,
+    repro: action?.selector ? `Click on "${action.selector}"` : null,
+  };
+}
+
+/**
+ * Generate deterministic fingerprint
+ */
+function generateFingerprint(parts) {
+  const content = parts.filter(Boolean).join("|");
+  return crypto.createHash("sha256").update(content).digest("hex");
+}
+
+/**
+ * Count findings by detector
+ */
+function countByDetector(findings) {
+  const counts = {};
+  for (const f of findings) {
+    counts[f.detectorId] = (counts[f.detectorId] || 0) + 1;
+  }
+  return counts;
+}
+
+/**
+ * Count findings by severity
+ */
+function countBySeverity(findings) {
+  const counts = { BLOCK: 0, WARN: 0, INFO: 0 };
+  for (const f of findings) {
+    counts[f.severity] = (counts[f.severity] || 0) + 1;
+  }
+  return counts;
+}
+
+/**
+ * Summarize action outcome for reporting
+ */
+function summarizeActionOutcome(action) {
+  const {
+    uiChange = {},
+    signals = [],
+    requests = [],
+    selector = "",
+    actionType = "click",
+  } = action;
+
+  const toasts = signals.filter(s => s.kind?.startsWith("toast_"));
+  const mutations = requests.filter(r => 
+    ["POST", "PUT", "PATCH", "DELETE"].includes(r.method?.toUpperCase())
+  );
+  const successfulMutations = mutations.filter(r => r.status >= 200 && r.status < 300);
+
+  let outcome = "unknown";
+  
+  if (successfulMutations.length > 0 && uiChange.meaningful) {
+    outcome = "success_confirmed";
+  } else if (successfulMutations.length > 0 && !uiChange.meaningful) {
+    outcome = "success_silent";
+  } else if (mutations.length > 0 && successfulMutations.length === 0) {
+    outcome = "mutation_failed";
+  } else if (toasts.some(t => t.kind === "toast_success")) {
+    outcome = successfulMutations.length > 0 ? "success_confirmed" : "toast_only";
+  } else if (toasts.some(t => t.kind === "toast_error")) {
+    outcome = "error_shown";
+  } else if (!uiChange.meaningful && requests.length === 0) {
+    outcome = "no_effect";
+  } else if (uiChange.meaningful) {
+    outcome = "ui_changed";
+  }
+
+  return {
+    actionType,
+    selector,
+    outcome,
+    uiChangeScore: uiChange.score,
+    uiChangeMeaningful: uiChange.meaningful,
+    uiChangeReasons: uiChange.reasons || [],
+    toastCount: toasts.length,
+    toastKinds: toasts.map(t => t.kind),
+    requestCount: requests.length,
+    mutationCount: mutations.length,
+    successfulMutationCount: successfulMutations.length,
+  };
+}
+
+module.exports = {
+  DETECTORS,
+  analyzeActionCorrelation,
+  analyzeAllActions,
+  createFinding,
+  summarizeActionOutcome,
+  generateFingerprint,
+};