@veraxhq/verax 0.1.0 → 0.2.0

package/src/cli/util/atomic-write.js

@@ -0,0 +1,37 @@
+import { writeFileSync, renameSync, mkdirSync } from 'fs';
+import { dirname } from 'path';
+
+/**
+ * Atomic write for JSON files
+ * Writes to a temp file and renames to prevent partial writes
+ */
+export function atomicWriteJson(filePath, data) {
+  const tempPath = `${filePath}.tmp`;
+  const dirPath = dirname(filePath);
+  
+  // Ensure directory exists
+  mkdirSync(dirPath, { recursive: true });
+  
+  // Write to temp file
+  writeFileSync(tempPath, JSON.stringify(data, null, 2), 'utf8');
+  
+  // Atomic rename
+  renameSync(tempPath, filePath);
+}
+
+/**
+ * Atomic write for text files (JSONL, logs, etc.)
+ */
+export function atomicWriteText(filePath, content) {
+  const tempPath = `${filePath}.tmp`;
+  const dirPath = dirname(filePath);
+  
+  // Ensure directory exists
+  mkdirSync(dirPath, { recursive: true });
+  
+  // Write to temp file
+  writeFileSync(tempPath, content, 'utf8');
+  
+  // Atomic rename
+  renameSync(tempPath, filePath);
+}

package/src/cli/util/detection-engine.js

@@ -0,0 +1,296 @@
+/**
+ * Detection Engine
+ * Compares learned expectations against observations to detect silent failures.
+ * Produces exactly one finding per expectation with deterministic confidence and impact.
+ */
+
+export async function detectFindings(learnData, observeData, projectPath, onProgress) {
+  const findings = [];
+  const stats = {
+    total: 0,
+    silentFailures: 0,
+    observed: 0,
+    coverageGaps: 0,
+    unproven: 0,
+    informational: 0,
+  };
+
+  const observationMap = indexObservations(observeData);
+  const expectations = learnData?.expectations || [];
+
+  const narration = [];
+
+  for (let i = 0; i < expectations.length; i++) {
+    const expectation = expectations[i];
+    const index = i + 1;
+
+    if (onProgress) {
+      onProgress({
+        event: 'detect:attempt',
+        index,
+        total: expectations.length,
+        expectationId: expectation.id,
+        type: expectation.type,
+        value: expectation?.promise?.value,
+      });
+    }
+
+    const observation = getObservationForExpectation(expectation, observationMap);
+    const finding = classifyExpectation(expectation, observation);
+
+    findings.push(finding);
+    stats.total++;
+    stats[findingStatKey(finding.classification)]++;
+
+    const icon = classificationIcon(finding.classification);
+    narration.push(`${icon} ${finding.classification.toUpperCase()} ${finding.promise?.value || ''}`.trim());
+
+    if (onProgress) {
+      onProgress({
+        event: 'detect:classified',
+        index,
+        classification: finding.classification,
+        impact: finding.impact,
+        confidence: finding.confidence,
+        expectationId: expectation.id,
+      });
+    }
+  }
+
+  // Terminal narration (one line per finding + summary)
+  narration.forEach((line) => console.log(line));
+  console.log(`SUMMARY findings=${findings.length} observed=${stats.observed} silent-failure=${stats.silentFailures} coverage-gap=${stats.coverageGaps} unproven=${stats.unproven}`);
+
+  // Emit completion event
+  if (onProgress) {
+    onProgress({
+      event: 'detect:completed',
+      total: findings.length,
+      stats,
+    });
+  }
+
+  return {
+    findings,
+    stats,
+    detectedAt: new Date().toISOString(),
+  };
+}
+
+function indexObservations(observeData) {
+  const map = new Map();
+  if (!observeData || !Array.isArray(observeData.observations)) return map;
+
+  observeData.observations.forEach((obs) => {
+    if (obs == null) return;
+    const keys = [];
+    if (obs.id) keys.push(obs.id);
+    if (obs.expectationId) keys.push(obs.expectationId);
+    keys.forEach((key) => {
+      if (!map.has(key)) map.set(key, obs);
+    });
+  });
+  return map;
+}
+
+function getObservationForExpectation(expectation, observationMap) {
+  if (!expectation || !expectation.id) return null;
+  return observationMap.get(expectation.id) || null;
+}
+
+function classificationIcon(classification) {
+  switch (classification) {
+    case 'observed':
+      return '✓';
+    case 'silent-failure':
+      return '✗';
+    case 'coverage-gap':
+      return '⚠';
+    case 'unproven':
+      return '⚠';
+    default:
+      return '•';
+  }
+}
+
+function findingStatKey(classification) {
+  switch (classification) {
+    case 'silent-failure':
+      return 'silentFailures';
+    case 'observed':
+      return 'observed';
+    case 'coverage-gap':
+      return 'coverageGaps';
+    case 'unproven':
+      return 'unproven';
+    default:
+      return 'informational';
+  }
+}
+
+/**
+ * Classify a single expectation according to deterministic rules.
+ */
+function classifyExpectation(expectation, observation) {
+  const finding = {
+    id: expectation.id,
+    type: expectation.type,
+    promise: expectation.promise,
+    source: expectation.source,
+    classification: 'informational',
+    impact: 'LOW',
+    confidence: 0,
+    evidence: [],
+    reason: null,
+  };
+
+  const attempted = Boolean(observation?.attempted);
+  const observed = observation?.observed === true;
+  const reason = observation?.reason || null;
+
+  const evidence = normalizeEvidence(observation?.evidenceFiles || []);
+  finding.evidence = evidence;
+
+  const evidenceSignals = analyzeEvidenceSignals(observation, evidence);
+
+  // 1) observed
+  if (observed) {
+    finding.classification = 'observed';
+    finding.reason = 'Expectation observed at runtime';
+    finding.impact = getImpact(expectation);
+    finding.confidence = 1.0;
+    return finding;
+  }
+
+  // 2) coverage-gap (not attempted or explicitly blocked)
+  if (!attempted || isSafetySkip(reason)) {
+    finding.classification = 'coverage-gap';
+    finding.reason = reason || 'No observation attempt recorded';
+    finding.impact = 'LOW';
+    finding.confidence = 0;
+    return finding;
+  }
+
+  // 3) silent-failure (attempted, observed === false, no safety skip)
+  if (attempted && observation?.observed === false && !isSafetySkip(reason)) {
+    finding.classification = 'silent-failure';
+    finding.reason = reason || 'Expected behavior not observed';
+    finding.impact = getImpact(expectation);
+    finding.confidence = calculateConfidence(expectation, evidenceSignals, 'silent-failure');
+    return finding;
+  }
+
+  // 4) unproven (attempted, ambiguous evidence)
+  if (attempted && !observed) {
+    finding.classification = 'unproven';
+    finding.reason = reason || 'Attempted but evidence insufficient';
+    finding.impact = 'MEDIUM';
+    finding.confidence = calculateConfidence(expectation, evidenceSignals, 'unproven');
+    return finding;
+  }
+
+  // 5) informational fallback
+  finding.classification = 'informational';
+  finding.reason = reason || 'No classification rule matched';
+  finding.impact = 'LOW';
+  finding.confidence = calculateConfidence(expectation, evidenceSignals, 'informational');
+  return finding;
+}
+
+function isSafetySkip(reason) {
+  if (!reason) return false;
+  const lower = reason.toLowerCase();
+  const safetyIndicators = ['blocked', 'timeout', 'not found', 'safety', 'permission', 'denied', 'captcha', 'forbidden'];
+  return safetyIndicators.some((indicator) => lower.includes(indicator));
+}
+
+/**
+ * Deterministic confidence calculation.
+ * Screenshots + network + DOM change => >=0.8
+ * Screenshots only => ~0.6
+ * Weak signals => <0.5
+ */
+function calculateConfidence(expectation, evidenceSignals, classification) {
+  if (classification === 'observed') return 1.0;
+  if (classification === 'coverage-gap') return 0;
+
+  const { hasScreenshots, hasNetworkLogs, hasDomChange } = evidenceSignals;
+
+  if (classification === 'silent-failure') {
+    if (hasScreenshots && hasNetworkLogs && hasDomChange) return 0.85;
+    if (hasScreenshots && hasNetworkLogs) return 0.75;
+    if (hasScreenshots && hasDomChange) return 0.7;
+    if (hasScreenshots) return 0.6;
+    if (hasNetworkLogs || hasDomChange) return 0.5;
+    return 0.4; // weak signals, attempted but no evidence
+  }
+
+  if (classification === 'unproven') {
+    if (hasScreenshots || hasNetworkLogs || hasDomChange) return 0.45;
+    return 0.3;
+  }
+
+  return 0.3;
+}
+
+function analyzeEvidenceSignals(observation, evidence) {
+  const hasScreenshots = evidence.some((e) => e.type === 'screenshot');
+  const hasNetworkLogs = evidence.some((e) => e.type === 'network-log');
+  const hasDomChange = Boolean(
+    observation?.domChanged ||
+      observation?.domChange === true ||
+      observation?.sensors?.uiSignals?.diff?.changed
+  );
+
+  return { hasScreenshots, hasNetworkLogs, hasDomChange };
+}
+
+function normalizeEvidence(evidenceFiles) {
+  if (!Array.isArray(evidenceFiles)) return [];
+  return evidenceFiles
+    .filter(Boolean)
+    .map((file) => ({
+      type: getEvidenceType(file),
+      path: file,
+      available: true,
+    }));
+}
+
+/**
+ * Impact classification per product spec.
+ */
+function getImpact(expectation) {
+  const type = expectation?.type;
+  const value = expectation?.promise?.value || '';
+  const valueStr = String(value).toLowerCase();
+
+  if (type === 'navigation') {
+    const primaryRoutes = ['/', '/home', '/about', '/contact', '/products', '/pricing', '/features', '/login', '/signup'];
+    if (primaryRoutes.includes(value)) return 'HIGH';
+    if (valueStr.includes('admin') || valueStr.includes('dashboard') || valueStr.includes('settings')) return 'MEDIUM';
+    if (valueStr.includes('privacy') || valueStr.includes('terms') || valueStr.includes('footer')) return 'LOW';
+    return 'MEDIUM';
+  }
+
+  if (type === 'network') {
+    if (valueStr.includes('/api/auth') || valueStr.includes('/api/payment') || valueStr.includes('/api/user')) return 'HIGH';
+    if (valueStr.includes('api.')) return 'MEDIUM';
+    if (valueStr.includes('/api/') && !valueStr.includes('/api/analytics')) return 'MEDIUM';
+    return 'LOW';
+  }
+
+  if (type === 'state') {
+    return 'MEDIUM';
+  }
+
+  return 'LOW';
+}
+
+function getEvidenceType(filename) {
+  if (!filename) return 'unknown';
+  const lower = filename.toLowerCase();
+  if (lower.endsWith('.png') || lower.endsWith('.jpg') || lower.endsWith('.jpeg')) return 'screenshot';
+  if (lower.endsWith('.json')) return 'network-log';
+  if (lower.endsWith('.log') || lower.endsWith('.txt')) return 'console-log';
+  return 'artifact';
+}

package/src/cli/util/env-url.js

@@ -0,0 +1,33 @@
+/**
+ * Attempt to infer URL from environment variables and common dev configs
+ */
+export function tryResolveUrlFromEnv() {
+  // Check common environment variables
+  const candidates = [
+    process.env.VERCEL_URL,
+    process.env.NEXT_PUBLIC_SITE_URL,
+    process.env.SITE_URL,
+    process.env.PUBLIC_URL,
+  ];
+  
+  for (const candidate of candidates) {
+    if (candidate) {
+      // Ensure it's a valid URL
+      if (candidate.startsWith('http://') || candidate.startsWith('https://')) {
+        return candidate;
+      }
+      // If it's a bare domain, assume https
+      if (candidate.includes('.') && !candidate.includes(' ')) {
+        return `https://${candidate}`;
+      }
+    }
+  }
+  
+  // Check for localhost/PORT
+  if (process.env.PORT) {
+    const port = process.env.PORT;
+    return `http://localhost:${port}`;
+  }
+  
+  return null;
+}

package/src/cli/util/errors.js

@@ -0,0 +1,44 @@
+/**
+ * CLI Error System
+ * Maps errors to exit codes:
+ * - 0: success (tool executed)
+ * - 2: internal crash
+ * - 64: invalid CLI usage
+ * - 65: invalid input data
+ */
+
+export class CLIError extends Error {
+  constructor(message, exitCode = 2) {
+    super(message);
+    this.name = 'CLIError';
+    this.exitCode = exitCode;
+  }
+}
+
+export class UsageError extends CLIError {
+  constructor(message) {
+    super(message, 64);
+    this.name = 'UsageError';
+  }
+}
+
+export class DataError extends CLIError {
+  constructor(message) {
+    super(message, 65);
+    this.name = 'DataError';
+  }
+}
+
+export class CrashError extends CLIError {
+  constructor(message) {
+    super(message, 2);
+    this.name = 'CrashError';
+  }
+}
+
+export function getExitCode(error) {
+  if (error instanceof CLIError) {
+    return error.exitCode;
+  }
+  return 2; // default to crash
+}

package/src/cli/util/events.js

@@ -0,0 +1,34 @@
+/**
+ * Event emitter for run progress
+ */
+export class RunEventEmitter {
+  constructor() {
+    this.events = [];
+    this.listeners = [];
+  }
+  
+  on(event, handler) {
+    this.listeners.push({ event, handler });
+  }
+  
+  emit(type, data = {}) {
+    const event = {
+      type,
+      timestamp: new Date().toISOString(),
+      ...data,
+    };
+    
+    this.events.push(event);
+    
+    // Call registered listeners
+    this.listeners.forEach(({ event: listenEvent, handler }) => {
+      if (listenEvent === type || listenEvent === '*') {
+        handler(event);
+      }
+    });
+  }
+  
+  getEvents() {
+    return this.events;
+  }
+}