@veraxhq/verax 0.2.1 → 0.4.0

package/src/verax/core/determinism/diff.js

@@ -0,0 +1,405 @@
+/**
+ * PHASE 18 — Determinism Diff Builder
+ * 
+ * Generates structured diffs between normalized artifacts from different runs.
+ */
+
+import { computeFindingIdentity as _computeFindingIdentity } from './finding-identity.js';
+
+/**
+ * PHASE 18: Diff reason codes
+ * PHASE 25: Extended with new reason codes
+ */
+export const DIFF_REASON = {
+  MISSING_ARTIFACT: 'DET_DIFF_MISSING_ARTIFACT',
+  SCHEMA_MISMATCH: 'DET_DIFF_SCHEMA_MISMATCH',
+  FINDING_ADDED: 'DET_DIFF_FINDING_ADDED',
+  FINDING_REMOVED: 'DET_DIFF_FINDING_REMOVED',
+  FINDING_STATUS_CHANGED: 'DET_DIFF_FINDING_STATUS_CHANGED',
+  FINDING_SEVERITY_CHANGED: 'DET_DIFF_FINDING_SEVERITY_CHANGED',
+  CONFIDENCE_CHANGED: 'DET_DIFF_CONFIDENCE_CHANGED',
+  CONFIDENCE_REASONS_CHANGED: 'DET_DIFF_CONFIDENCE_REASONS_CHANGED',
+  GUARDRAILS_CHANGED: 'DET_DIFF_GUARDRAILS_CHANGED',
+  EVIDENCE_COMPLETENESS_CHANGED: 'DET_DIFF_EVIDENCE_COMPLETENESS_CHANGED',
+  EVIDENCE_MISSING: 'DET_DIFF_EVIDENCE_MISSING',
+  OBSERVATION_COUNT_CHANGED: 'DET_DIFF_OBSERVATION_COUNT_CHANGED',
+  FIELD_VALUE_CHANGED: 'DET_DIFF_FIELD_VALUE_CHANGED',
+  RUN_FINGERPRINT_MISMATCH: 'DET_DIFF_RUN_FINGERPRINT_MISMATCH',
+};
+
+/**
+ * PHASE 18: Diff categories
+ */
+export const DIFF_CATEGORY = {
+  FINDINGS: 'FINDINGS',
+  EXPECTATIONS: 'EXPECTATIONS',
+  OBSERVATIONS: 'OBSERVATIONS',
+  EVIDENCE: 'EVIDENCE',
+  STATUS: 'STATUS',
+  ARTIFACTS: 'ARTIFACTS',
+};
+
+/**
+ * PHASE 18: Diff severity
+ */
+export const DIFF_SEVERITY = {
+  BLOCKER: 'BLOCKER',
+  WARN: 'WARN',
+  INFO: 'INFO',
+};
+
+function diffRunMeta(artifactA, artifactB) {
+  return diffGeneric(artifactA, artifactB, 'runMeta');
+}
+
+function diffDeterminismContract(artifactA, artifactB) {
+  return diffGeneric(artifactA, artifactB, 'determinismContract');
+}
+
+function diffReportArtifact(artifactA, artifactB, artifactName) {
+  return diffGeneric(artifactA, artifactB, artifactName);
+}
+
+/**
+ * PHASE 18: Diff artifacts
+ * 
+ * @param {Object} artifactA - First artifact (normalized)
+ * @param {Object} artifactB - Second artifact (normalized)
+ * @param {string} artifactName - Name of artifact
+ * @param {Map} findingIdentityMap - Map of finding identity to finding (for matching)
+ * @returns {Array} Array of diff objects
+ */
+export function diffArtifacts(artifactA, artifactB, artifactName, findingIdentityMap = null) {
+  const diffs = [];
+  
+  // Check if artifacts exist
+  if (!artifactA && artifactB) {
+    diffs.push({
+      category: DIFF_CATEGORY.ARTIFACTS,
+      severity: DIFF_SEVERITY.BLOCKER,
+      reasonCode: DIFF_REASON.MISSING_ARTIFACT,
+      message: `Artifact ${artifactName} missing in first run`,
+      artifact: artifactName,
+    });
+    return diffs;
+  }
+  
+  if (artifactA && !artifactB) {
+    diffs.push({
+      category: DIFF_CATEGORY.ARTIFACTS,
+      severity: DIFF_SEVERITY.BLOCKER,
+      reasonCode: DIFF_REASON.MISSING_ARTIFACT,
+      message: `Artifact ${artifactName} missing in second run`,
+      artifact: artifactName,
+    });
+    return diffs;
+  }
+  
+  if (!artifactA && !artifactB) {
+    return diffs; // Both missing, no diff
+  }
+  
+  // Artifact-specific diffing
+  if (artifactName === 'findings') {
+    diffs.push(...diffFindings(artifactA, artifactB, findingIdentityMap));
+  } else if (artifactName === 'runMeta') {
+    diffs.push(...diffRunMeta(artifactA, artifactB));
+  } else if (artifactName === 'determinismContract') {
+    diffs.push(...diffDeterminismContract(artifactA, artifactB));
+  } else if (artifactName === 'confidenceReport' || artifactName === 'guardrailsReport' || artifactName === 'evidenceIntent') {
+    diffs.push(...diffReportArtifact(artifactA, artifactB, artifactName));
+  } else {
+    // Generic diff for other artifacts
+    diffs.push(...diffGeneric(artifactA, artifactB, artifactName));
+  }
+  
+  return diffs;
+}
+
+/**
+ * Diff findings artifacts
+ */
+function diffFindings(artifactA, artifactB, findingIdentityMap) {
+  const diffs = [];
+  
+  const findingsA = artifactA.findings || [];
+  const findingsB = artifactB.findings || [];
+  
+  // Check counts
+  if (findingsA.length !== findingsB.length) {
+    diffs.push({
+      category: DIFF_CATEGORY.FINDINGS,
+      severity: DIFF_SEVERITY.BLOCKER,
+      reasonCode: DIFF_REASON.OBSERVATION_COUNT_CHANGED,
+      message: `Finding count changed: ${findingsA.length} → ${findingsB.length}`,
+      artifact: 'findings',
+      field: 'findings.length',
+      oldValue: findingsA.length,
+      newValue: findingsB.length,
+    });
+  }
+  
+  // Build identity maps if not provided
+  const mapA = new Map();
+  const mapB = new Map();
+  
+  for (const finding of findingsA) {
+    const identity = findingIdentityMap ? findingIdentityMap.get(finding) : computeFindingIdentitySimple(finding);
+    mapA.set(identity, finding);
+  }
+  
+  for (const finding of findingsB) {
+    const identity = findingIdentityMap ? findingIdentityMap.get(finding) : computeFindingIdentitySimple(finding);
+    mapB.set(identity, finding);
+  }
+  
+  // Find added findings
+  for (const [identity, finding] of mapB) {
+    if (!mapA.has(identity)) {
+      diffs.push({
+        category: DIFF_CATEGORY.FINDINGS,
+        severity: DIFF_SEVERITY.BLOCKER,
+        reasonCode: DIFF_REASON.FINDING_ADDED,
+        message: `Finding added: ${finding.type || 'unknown'}`,
+        artifact: 'findings',
+        findingIdentity: identity,
+        finding: finding,
+      });
+    }
+  }
+  
+  // Find removed findings
+  for (const [identity, finding] of mapA) {
+    if (!mapB.has(identity)) {
+      diffs.push({
+        category: DIFF_CATEGORY.FINDINGS,
+        severity: DIFF_SEVERITY.BLOCKER,
+        reasonCode: DIFF_REASON.FINDING_REMOVED,
+        message: `Finding removed: ${finding.type || 'unknown'}`,
+        artifact: 'findings',
+        findingIdentity: identity,
+        finding: finding,
+      });
+    }
+  }
+  
+  // Find changed findings
+  for (const [identity, findingA] of mapA) {
+    const findingB = mapB.get(identity);
+    if (findingB) {
+      diffs.push(...diffFinding(findingA, findingB, identity));
+    }
+  }
+  
+  return diffs;
+}
+
+/**
+ * Diff individual finding
+ */
+function diffFinding(findingA, findingB, identity) {
+  const diffs = [];
+  
+  // Check status/severity
+  const statusA = findingA.severity || findingA.status;
+  const statusB = findingB.severity || findingB.status;
+  if (statusA !== statusB) {
+    diffs.push({
+      category: DIFF_CATEGORY.FINDINGS,
+      severity: DIFF_SEVERITY.BLOCKER,
+      reasonCode: DIFF_REASON.FINDING_SEVERITY_CHANGED,
+      message: `Finding severity changed: ${statusA} → ${statusB}`,
+      artifact: 'findings',
+      findingIdentity: identity,
+      field: 'severity',
+      oldValue: statusA,
+      newValue: statusB,
+    });
+  }
+  
+  // Check confidence
+  const confA = findingA.confidence || 0;
+  const confB = findingB.confidence || 0;
+  if (Math.abs(confA - confB) > 0.001) {
+    diffs.push({
+      category: DIFF_CATEGORY.FINDINGS,
+      severity: DIFF_SEVERITY.WARN,
+      reasonCode: DIFF_REASON.CONFIDENCE_CHANGED,
+      message: `Finding confidence changed: ${confA.toFixed(3)} → ${confB.toFixed(3)}`,
+      artifact: 'findings',
+      findingIdentity: identity,
+      field: 'confidence',
+      oldValue: confA,
+      newValue: confB,
+    });
+  }
+  
+  // Check confidence reasons
+  const reasonsA = findingA.confidenceReasons || [];
+  const reasonsB = findingB.confidenceReasons || [];
+  if (JSON.stringify(reasonsA.sort()) !== JSON.stringify(reasonsB.sort())) {
+    diffs.push({
+      category: DIFF_CATEGORY.FINDINGS,
+      severity: DIFF_SEVERITY.WARN,
+      reasonCode: DIFF_REASON.CONFIDENCE_REASONS_CHANGED,
+      message: `Finding confidence reasons changed`,
+      artifact: 'findings',
+      findingIdentity: identity,
+      field: 'confidenceReasons',
+      oldValue: reasonsA,
+      newValue: reasonsB,
+    });
+  }
+  
+  // Check guardrails
+  const guardrailsA = findingA.guardrails;
+  const guardrailsB = findingB.guardrails;
+  if (guardrailsA || guardrailsB) {
+    if (!guardrailsA || !guardrailsB) {
+      diffs.push({
+        category: DIFF_CATEGORY.FINDINGS,
+        severity: DIFF_SEVERITY.WARN,
+        reasonCode: DIFF_REASON.GUARDRAILS_CHANGED,
+        message: `Finding guardrails presence changed`,
+        artifact: 'findings',
+        findingIdentity: identity,
+        field: 'guardrails',
+      });
+    } else if (guardrailsA.finalDecision !== guardrailsB.finalDecision) {
+      diffs.push({
+        category: DIFF_CATEGORY.FINDINGS,
+        severity: DIFF_SEVERITY.WARN,
+        reasonCode: DIFF_REASON.GUARDRAILS_CHANGED,
+        message: `Finding guardrails decision changed: ${guardrailsA.finalDecision} → ${guardrailsB.finalDecision}`,
+        artifact: 'findings',
+        findingIdentity: identity,
+        field: 'guardrails.finalDecision',
+        oldValue: guardrailsA.finalDecision,
+        newValue: guardrailsB.finalDecision,
+      });
+    }
+  }
+  
+  // Check evidence completeness
+  const evidenceA = findingA.evidenceCompleteness;
+  const evidenceB = findingB.evidenceCompleteness;
+  if (evidenceA || evidenceB) {
+    if (!evidenceA || !evidenceB) {
+      diffs.push({
+        category: DIFF_CATEGORY.EVIDENCE,
+        severity: DIFF_SEVERITY.BLOCKER,
+        reasonCode: DIFF_REASON.EVIDENCE_COMPLETENESS_CHANGED,
+        message: `Finding evidence completeness presence changed`,
+        artifact: 'findings',
+        findingIdentity: identity,
+        field: 'evidenceCompleteness',
+      });
+    } else if (evidenceA.isComplete !== evidenceB.isComplete) {
+      diffs.push({
+        category: DIFF_CATEGORY.EVIDENCE,
+        severity: DIFF_SEVERITY.BLOCKER,
+        reasonCode: DIFF_REASON.EVIDENCE_COMPLETENESS_CHANGED,
+        message: `Finding evidence completeness changed: ${evidenceA.isComplete} → ${evidenceB.isComplete}`,
+        artifact: 'findings',
+        findingIdentity: identity,
+        field: 'evidenceCompleteness.isComplete',
+        oldValue: evidenceA.isComplete,
+        newValue: evidenceB.isComplete,
+      });
+    }
+  }
+  
+  // Check evidencePackage presence
+  const evidencePackageA = findingA.evidencePackage;
+  const evidencePackageB = findingB.evidencePackage;
+  if (!evidencePackageA && evidencePackageB) {
+    diffs.push({
+      category: DIFF_CATEGORY.EVIDENCE,
+      severity: DIFF_SEVERITY.BLOCKER,
+      reasonCode: DIFF_REASON.EVIDENCE_MISSING,
+      message: `Finding evidence package missing in first run`,
+      artifact: 'findings',
+      findingIdentity: identity,
+      field: 'evidencePackage',
+    });
+  } else if (evidencePackageA && !evidencePackageB) {
+    diffs.push({
+      category: DIFF_CATEGORY.EVIDENCE,
+      severity: DIFF_SEVERITY.BLOCKER,
+      reasonCode: DIFF_REASON.EVIDENCE_MISSING,
+      message: `Finding evidence package missing in second run`,
+      artifact: 'findings',
+      findingIdentity: identity,
+      field: 'evidencePackage',
+    });
+  } else if (evidencePackageA && evidencePackageB) {
+    if (evidencePackageA.isComplete !== evidencePackageB.isComplete) {
+      diffs.push({
+        category: DIFF_CATEGORY.EVIDENCE,
+        severity: DIFF_SEVERITY.BLOCKER,
+        reasonCode: DIFF_REASON.EVIDENCE_COMPLETENESS_CHANGED,
+        message: `Evidence completeness changed: ${evidencePackageA.isComplete} → ${evidencePackageB.isComplete}`,
+        artifact: 'findings',
+        findingIdentity: identity,
+        field: 'evidencePackage.isComplete',
+        oldValue: evidencePackageA.isComplete,
+        newValue: evidencePackageB.isComplete,
+      });
+    }
+    const missingA = Array.isArray(evidencePackageA.missingEvidence) ? evidencePackageA.missingEvidence.sort() : [];
+    const missingB = Array.isArray(evidencePackageB.missingEvidence) ? evidencePackageB.missingEvidence.sort() : [];
+    if (missingA.join('|') !== missingB.join('|')) {
+      diffs.push({
+        category: DIFF_CATEGORY.EVIDENCE,
+        severity: DIFF_SEVERITY.BLOCKER,
+        reasonCode: DIFF_REASON.EVIDENCE_MISSING,
+        message: `Missing evidence changed`,
+        artifact: 'findings',
+        findingIdentity: identity,
+        field: 'evidencePackage.missingEvidence',
+        oldValue: missingA,
+        newValue: missingB,
+      });
+    }
+  }
+  
+  return diffs;
+}
+
+/**
+ * Diff generic artifacts
+ */
+function diffGeneric(artifactA, artifactB, artifactName) {
+  const diffs = [];
+  
+  // Simple deep comparison
+  const jsonA = JSON.stringify(artifactA, null, 2);
+  const jsonB = JSON.stringify(artifactB, null, 2);
+  
+  if (jsonA !== jsonB) {
+    diffs.push({
+      category: DIFF_CATEGORY.ARTIFACTS,
+      severity: DIFF_SEVERITY.WARN,
+      reasonCode: DIFF_REASON.FIELD_VALUE_CHANGED,
+      message: `Artifact ${artifactName} content changed`,
+      artifact: artifactName,
+    });
+  }
+  
+  return diffs;
+}
+
+/**
+ * Simple finding identity computation (fallback)
+ */
+function computeFindingIdentitySimple(finding) {
+  const parts = [
+    finding.type || 'unknown',
+    finding.interaction?.type || '',
+    finding.interaction?.selector || '',
+    finding.expectation?.targetPath || '',
+    finding.expectation?.urlPath || '',
+  ];
+  return parts.join('|');
+}
+

package/src/verax/core/determinism/engine.js

@@ -0,0 +1,222 @@
+/**
+ * PHASE 18 — Determinism Engine
+ * PHASE 21.2 — Determinism Truth Lock: Enforces HARD verdict
+ * 
+ * Runs the same scan multiple times and compares results for determinism.
+ * PHASE 21.2: Also checks DecisionRecorder for adaptive events that break determinism.
+ */
+
+import { readFileSync, existsSync } from 'fs';
+import { join as _join, resolve } from 'path';
+import { normalizeArtifact } from './normalize.js';
+import { diffArtifacts } from './diff.js';
+import { computeFindingIdentity } from './finding-identity.js';
+import { computeDeterminismVerdict, DETERMINISM_VERDICT } from './contract.js';
+import { DecisionRecorder } from '../determinism-model.js';
+
+/**
+ * PHASE 18: Determinism verdict (re-exported from contract for backward compatibility)
+ */
+export { DETERMINISM_VERDICT, DETERMINISM_REASON } from './contract.js';
+
+/**
+ * PHASE 18: Run determinism check
+ * 
+ * @param {Function} runFn - Function that executes a scan and returns artifact paths or in-memory artifacts
+ * @param {Object} options - Options
+ * @param {number} options.runs - Number of runs (default: 2)
+ * @param {Object} options.config - Configuration for runs
+ * @param {boolean} options.normalize - Whether to normalize artifacts (default: true)
+ * @returns {Promise<Object>} { verdict, summary, diffs, runsMeta }
+ */
+export async function runDeterminismCheck(runFn, options = { runs: 2, config: {}, normalize: true }) {
+  const { runs = 2, config = {}, normalize = true } = options;
+  
+  const runsMeta = [];
+  const runArtifacts = [];
+  
+  // Execute runs
+  for (let i = 0; i < runs; i++) {
+    const runResult = await runFn(config);
+    runsMeta.push({
+      runIndex: i + 1,
+      runId: runResult.runId || null,
+      timestamp: new Date().toISOString(),
+      artifactPaths: runResult.artifactPaths || {},
+      artifacts: runResult.artifacts || {},
+    });
+    
+    // Load artifacts if paths provided
+    const artifacts = {};
+    if (runResult.artifactPaths) {
+      for (const [key, path] of Object.entries(runResult.artifactPaths)) {
+        try {
+          const content = readFileSync(path, 'utf-8');
+  // @ts-expect-error - readFileSync with encoding returns string
+          artifacts[key] = JSON.parse(content);
+        } catch (error) {
+          // Artifact not found or invalid
+        }
+      }
+    } else if (runResult.artifacts) {
+      Object.assign(artifacts, runResult.artifacts);
+    }
+    
+    runArtifacts.push(artifacts);
+  }
+  
+  // Compare runs
+  const diffs = [];
+  const allArtifacts = new Set();
+  
+  // Collect all artifact names
+  for (const artifacts of runArtifacts) {
+    for (const key of Object.keys(artifacts)) {
+      allArtifacts.add(key);
+    }
+  }
+  
+  // Compare each artifact across runs
+  for (const artifactName of allArtifacts) {
+    const artifacts = runArtifacts.map(run => run[artifactName]);
+    
+    // Normalize if requested
+    const normalizedArtifacts = normalize
+      ? artifacts.map(art => art ? normalizeArtifact(artifactName, art) : null)
+      : artifacts;
+    
+    // Compare first run with all subsequent runs
+    for (let i = 1; i < normalizedArtifacts.length; i++) {
+      const artifactA = normalizedArtifacts[0];
+      const artifactB = normalizedArtifacts[i];
+      
+      // Build finding identity map for findings artifacts
+      let findingIdentityMap = null;
+      if (artifactName === 'findings' && artifactA && artifactB) {
+        findingIdentityMap = buildFindingIdentityMap(artifactA, artifactB);
+      }
+      
+      const artifactDiffs = diffArtifacts(artifactA, artifactB, artifactName, findingIdentityMap);
+      
+      // Add run context to diffs
+      for (const diff of artifactDiffs) {
+        diff.runA = 1;
+        diff.runB = i + 1;
+      }
+      
+      diffs.push(...artifactDiffs);
+    }
+  }
+  
+  // PHASE 21.2: Check for adaptive events in DecisionRecorder (if available)
+  // HARD RULE: If adaptive events exist → verdict MUST be NON_DETERMINISTIC
+  let adaptiveVerdict = null;
+  let adaptiveReasons = [];
+  let adaptiveEvents = [];
+  
+  // Try to load decisions.json from first run
+  if (runsMeta.length > 0 && runsMeta[0].artifactPaths?.runDir) {
+    const runDir = runsMeta[0].artifactPaths.runDir;
+    const decisionsPath = resolve(runDir, 'decisions.json');
+    
+    if (existsSync(decisionsPath)) {
+      try {
+  // @ts-expect-error - readFileSync with encoding returns string
+        const decisionsData = JSON.parse(readFileSync(decisionsPath, 'utf-8'));
+        const decisionRecorder = DecisionRecorder.fromExport(decisionsData);
+        const adaptiveCheck = computeDeterminismVerdict(decisionRecorder);
+        
+        adaptiveVerdict = adaptiveCheck.verdict;
+        adaptiveReasons = adaptiveCheck.reasons;
+        adaptiveEvents = adaptiveCheck.adaptiveEvents;
+      } catch (error) {
+        // Ignore errors reading decisions
+      }
+    }
+  }
+  
+  // PHASE 21.2: Determine verdict
+  // HARD RULE: If adaptive events exist → verdict MUST be NON_DETERMINISTIC (even if artifacts match)
+  const blockerDiffs = diffs.filter(d => d.severity === 'BLOCKER');
+  const artifactVerdict = blockerDiffs.length === 0 ? DETERMINISM_VERDICT.DETERMINISTIC : DETERMINISM_VERDICT.NON_DETERMINISTIC;
+  
+  // PHASE 21.2: Final verdict - adaptive events override artifact comparison
+  const verdict = (adaptiveVerdict === DETERMINISM_VERDICT.NON_DETERMINISTIC) 
+    ? DETERMINISM_VERDICT.NON_DETERMINISTIC 
+    : artifactVerdict;
+  
+  // Build summary
+  const summary = buildSummary(diffs, runsMeta);
+  
+  // PHASE 21.2: Include adaptive event information in summary
+  if (adaptiveVerdict === DETERMINISM_VERDICT.NON_DETERMINISTIC) {
+    summary.adaptiveEventsDetected = true;
+    summary.adaptiveEventCount = adaptiveEvents.length;
+    summary.adaptiveReasons = adaptiveReasons;
+  }
+  
+  return {
+    verdict,
+    summary,
+    diffs,
+    runsMeta,
+    // PHASE 21.2: Include adaptive event information
+    adaptiveVerdict,
+    adaptiveReasons,
+    adaptiveEvents
+  };
+}
+
+/**
+ * Build finding identity map for matching findings across runs
+ */
+function buildFindingIdentityMap(artifactA, artifactB) {
+  const map = new Map();
+  
+  const findingsA = artifactA.findings || [];
+  const findingsB = artifactB.findings || [];
+  
+  // Build identity for findings in both runs
+  for (const finding of [...findingsA, ...findingsB]) {
+    const identity = computeFindingIdentity(finding);
+    map.set(finding, identity);
+  }
+  
+  return map;
+}
+
+/**
+ * Build summary from diffs
+ */
+function buildSummary(diffs, _runsMeta) {
+  const blockerCount = diffs.filter(d => d.severity === 'BLOCKER').length;
+  const warnCount = diffs.filter(d => d.severity === 'WARN').length;
+  const infoCount = diffs.filter(d => d.severity === 'INFO').length;
+  
+  // Group by reason code
+  const reasonCounts = {};
+  for (const diff of diffs) {
+    const code = diff.reasonCode || 'UNKNOWN';
+    reasonCounts[code] = (reasonCounts[code] || 0) + 1;
+  }
+  
+  // Top reasons
+  const topReasons = Object.entries(reasonCounts)
+    .sort((a, b) => b[1] - a[1])
+    .slice(0, 5)
+    .map(([code, count]) => ({ code, count }));
+  
+  // Stability score (0..1)
+  const totalDiffs = diffs.length;
+  const stabilityScore = totalDiffs === 0 ? 1.0 : Math.max(0, 1.0 - (blockerCount * 0.5 + warnCount * 0.2 + infoCount * 0.1) / Math.max(1, totalDiffs));
+  
+  return {
+    totalDiffs,
+    blockerCount,
+    warnCount,
+    infoCount,
+    topReasons,
+    stabilityScore,
+  };
+}
+