outcome-cli 1.0.0

package/src/modules/omnibridge/verification/verification-engine.ts

@@ -0,0 +1,783 @@
+/**
+ * Verification Engine
+ *
+ * Provides hallucination detection, diff analysis for conflicts,
+ * and confidence scoring for agent verification.
+ *
+ * Requirements: 8.4, 8.5, 8.6
+ */
+
+import type {
+  ActionLogEntry,
+  VerificationProof,
+  DiffAnalysis,
+} from '../core/types.js';
+
+// =============================================================================
+// Types
+// =============================================================================
+
+/**
+ * Result of hallucination detection.
+ */
+export interface HallucinationDetectionResult {
+  /** Whether hallucination was detected */
+  isHallucination: boolean;
+  /** Confidence in the detection (0-1) */
+  confidence: number;
+  /** Reasons for the detection */
+  reasons: string[];
+  /** The claimed result that was analyzed */
+  claimedResult: unknown;
+  /** Summary of actions that were performed */
+  actionSummary: ActionSummary;
+}
+
+/**
+ * Summary of actions performed in a session.
+ */
+export interface ActionSummary {
+  /** Total number of actions */
+  totalActions: number;
+  /** Number of successful actions */
+  successfulActions: number;
+  /** Number of failed actions */
+  failedActions: number;
+  /** Number of extract actions */
+  extractActions: number;
+  /** Number of click actions */
+  clickActions: number;
+  /** Number of type actions */
+  typeActions: number;
+  /** Number of navigate actions */
+  navigateActions: number;
+  /** Unique intent IDs interacted with */
+  uniqueIntentIds: string[];
+}
+
+/**
+ * Result of confidence scoring.
+ */
+export interface ConfidenceScoreResult {
+  /** Overall confidence score (0-1) */
+  score: number;
+  /** Breakdown of confidence factors */
+  factors: ConfidenceFactor[];
+  /** Whether the result is considered reliable */
+  isReliable: boolean;
+  /** Threshold used for reliability determination */
+  reliabilityThreshold: number;
+}
+
+/**
+ * A factor contributing to confidence score.
+ */
+export interface ConfidenceFactor {
+  /** Name of the factor */
+  name: string;
+  /** Weight of this factor (0-1) */
+  weight: number;
+  /** Score for this factor (0-1) */
+  score: number;
+  /** Description of how this factor was calculated */
+  description: string;
+}
+
+/**
+ * Configuration for the Verification Engine.
+ */
+export interface VerificationEngineConfig {
+  /** Minimum confidence threshold for reliable results */
+  reliabilityThreshold: number;
+  /** Weight for action success rate in confidence calculation */
+  actionSuccessWeight: number;
+  /** Weight for extract action presence in confidence calculation */
+  extractPresenceWeight: number;
+  /** Weight for action-result alignment in confidence calculation */
+  alignmentWeight: number;
+  /** Weight for action completeness in confidence calculation */
+  completenessWeight: number;
+}
+
+/**
+ * Extended diff analysis with winner recommendation details.
+ */
+export interface ExtendedDiffAnalysis extends DiffAnalysis {
+  /** Detailed reasoning for the recommendation */
+  recommendationReason: string;
+  /** Confidence in the recommendation (0-1) */
+  recommendationConfidence: number;
+  /** Metrics for agent A */
+  agentAMetrics: AgentMetrics;
+  /** Metrics for agent B */
+  agentBMetrics: AgentMetrics;
+}
+
+/**
+ * Metrics for an agent's performance.
+ */
+export interface AgentMetrics {
+  /** Total actions performed */
+  totalActions: number;
+  /** Successful actions */
+  successfulActions: number;
+  /** Failed actions */
+  failedActions: number;
+  /** Success rate (0-1) */
+  successRate: number;
+  /** Number of extract actions */
+  extractActions: number;
+  /** Whether agent completed the task */
+  completedTask: boolean;
+}
+
+// =============================================================================
+// Default Configuration
+// =============================================================================
+
+const DEFAULT_CONFIG: VerificationEngineConfig = {
+  reliabilityThreshold: 0.7,
+  actionSuccessWeight: 0.3,
+  extractPresenceWeight: 0.25,
+  alignmentWeight: 0.3,
+  completenessWeight: 0.15,
+};
+
+// =============================================================================
+// Verification Engine Implementation
+// =============================================================================
+
+/**
+ * Verification Engine for detecting hallucinations and analyzing agent results.
+ */
+export class VerificationEngine {
+  private config: VerificationEngineConfig;
+
+  constructor(config: Partial<VerificationEngineConfig> = {}) {
+    this.config = { ...DEFAULT_CONFIG, ...config };
+  }
+
+  // ===========================================================================
+  // Hallucination Detection (Requirement 8.5)
+  // ===========================================================================
+
+  /**
+   * Detect if a claimed result is a hallucination.
+   *
+   * A hallucination is detected when:
+   * 1. Agent claims data but has no extract actions
+   * 2. Agent claims success but all actions failed
+   * 3. Claimed data fields don't correspond to any extracted intent IDs
+   * 4. Result complexity exceeds what actions could produce
+   *
+   * Requirements: 8.5
+   */
+  detectHallucination(
+    claimedResult: unknown,
+    actionLog: ActionLogEntry[]
+  ): HallucinationDetectionResult {
+    const reasons: string[] = [];
+    const actionSummary = this.summarizeActions(actionLog);
+
+    // Check 1: No actions at all but claiming a result
+    if (actionLog.length === 0 && claimedResult !== null && claimedResult !== undefined) {
+      reasons.push('Result claimed with no actions performed');
+    }
+
+    // Check 2: All actions failed but claiming success
+    if (actionSummary.totalActions > 0 && actionSummary.successfulActions === 0) {
+      if (this.isSuccessfulResult(claimedResult)) {
+        reasons.push('Success claimed but all actions failed');
+      }
+    }
+
+    // Check 3: Data claimed but no extract actions
+    if (this.hasDataClaim(claimedResult) && actionSummary.extractActions === 0) {
+      reasons.push('Data claimed but no extract actions performed');
+    }
+
+    // Check 4: Claimed data fields don't match extracted intent IDs
+    if (this.hasDataClaim(claimedResult)) {
+      const dataFieldMismatch = this.checkDataFieldMismatch(
+        claimedResult,
+        actionSummary.uniqueIntentIds
+      );
+      if (dataFieldMismatch) {
+        reasons.push('Claimed data fields do not correspond to extracted elements');
+      }
+    }
+
+    // Check 5: Result complexity exceeds action capability
+    if (this.isResultTooComplex(claimedResult, actionSummary)) {
+      reasons.push('Result complexity exceeds what actions could produce');
+    }
+
+    // Calculate confidence in the detection
+    const isHallucination = reasons.length > 0;
+    const confidence = this.calculateHallucinationConfidence(reasons, actionSummary);
+
+    return {
+      isHallucination,
+      confidence,
+      reasons,
+      claimedResult,
+      actionSummary,
+    };
+  }
+
+  /**
+   * Summarize actions from an action log.
+   */
+  private summarizeActions(actionLog: ActionLogEntry[]): ActionSummary {
+    const uniqueIntentIds = new Set<string>();
+
+    let successfulActions = 0;
+    let failedActions = 0;
+    let extractActions = 0;
+    let clickActions = 0;
+    let typeActions = 0;
+    let navigateActions = 0;
+
+    for (const entry of actionLog) {
+      uniqueIntentIds.add(entry.intentId);
+
+      if (entry.result === 'success') {
+        successfulActions++;
+      } else {
+        failedActions++;
+      }
+
+      switch (entry.action) {
+        case 'extract':
+          extractActions++;
+          break;
+        case 'click':
+          clickActions++;
+          break;
+        case 'type':
+          typeActions++;
+          break;
+        case 'navigate':
+          navigateActions++;
+          break;
+      }
+    }
+
+    return {
+      totalActions: actionLog.length,
+      successfulActions,
+      failedActions,
+      extractActions,
+      clickActions,
+      typeActions,
+      navigateActions,
+      uniqueIntentIds: Array.from(uniqueIntentIds),
+    };
+  }
+
+  /**
+   * Check if a result indicates success.
+   */
+  private isSuccessfulResult(result: unknown): boolean {
+    if (result === null || result === undefined) {
+      return false;
+    }
+
+    if (typeof result === 'object') {
+      const obj = result as Record<string, unknown>;
+
+      // Check for explicit success indicators
+      if ('success' in obj && obj.success === true) {
+        return true;
+      }
+
+      // Check for data presence (implies success)
+      if ('data' in obj && obj.data !== null && obj.data !== undefined) {
+        const data = obj.data;
+        if (Array.isArray(data) && data.length > 0) {
+          return true;
+        }
+        if (typeof data === 'object' && Object.keys(data as object).length > 0) {
+          return true;
+        }
+      }
+
+      // Check for non-empty result object
+      const nonMetaKeys = Object.keys(obj).filter(
+        (key) => !['metadata', 'verificationHash', 'confidence', 'error'].includes(key)
+      );
+      if (nonMetaKeys.length > 0) {
+        return true;
+      }
+    }
+
+    return false;
+  }
+
+  /**
+   * Check if a result claims to have data.
+   */
+  private hasDataClaim(result: unknown): boolean {
+    if (result === null || result === undefined) {
+      return false;
+    }
+
+    if (typeof result !== 'object') {
+      return false;
+    }
+
+    const obj = result as Record<string, unknown>;
+
+    // Check for explicit data field
+    if ('data' in obj) {
+      const data = obj.data;
+      if (Array.isArray(data) && data.length > 0) {
+        return true;
+      }
+      if (typeof data === 'object' && data !== null && Object.keys(data as object).length > 0) {
+        return true;
+      }
+    }
+
+    // Check for data-like fields (excluding metadata)
+    const dataFields = Object.keys(obj).filter(
+      (key) => !['metadata', 'verificationHash', 'confidence', 'error', 'success'].includes(key)
+    );
+
+    return dataFields.length > 0;
+  }
+
+  /**
+   * Check if claimed data fields mismatch with extracted intent IDs.
+   */
+  private checkDataFieldMismatch(
+    _result: unknown,
+    extractedIntentIds: string[]
+  ): boolean {
+    if (extractedIntentIds.length === 0) {
+      return true; // No extractions means any data claim is a mismatch
+    }
+
+    // If we have extract actions, we assume the data could be valid
+    // More sophisticated matching would require schema analysis
+    return false;
+  }
+
+  /**
+   * Check if result complexity exceeds what actions could produce.
+   */
+  private isResultTooComplex(result: unknown, summary: ActionSummary): boolean {
+    if (result === null || result === undefined) {
+      return false;
+    }
+
+    if (typeof result !== 'object') {
+      return false;
+    }
+
+    const obj = result as Record<string, unknown>;
+
+    // Count data items in result
+    let dataItemCount = 0;
+
+    if ('data' in obj && Array.isArray(obj.data)) {
+      dataItemCount = obj.data.length;
+    } else if ('data' in obj && typeof obj.data === 'object' && obj.data !== null) {
+      dataItemCount = Object.keys(obj.data as object).length;
+    }
+
+    // If claiming many data items but few extract actions, suspicious
+    // Allow some multiplier for batch extractions
+    const maxReasonableItems = Math.max(summary.extractActions * 10, 1);
+
+    return dataItemCount > maxReasonableItems && summary.extractActions < dataItemCount / 10;
+  }
+
+  /**
+   * Calculate confidence in hallucination detection.
+   */
+  private calculateHallucinationConfidence(
+    reasons: string[],
+    summary: ActionSummary
+  ): number {
+    if (reasons.length === 0) {
+      return 0; // No hallucination detected
+    }
+
+    // Base confidence from number of reasons
+    let confidence = Math.min(reasons.length * 0.3, 0.9);
+
+    // Increase confidence if no actions at all
+    if (summary.totalActions === 0) {
+      confidence = Math.max(confidence, 0.95);
+    }
+
+    // Increase confidence if all actions failed
+    if (summary.totalActions > 0 && summary.successfulActions === 0) {
+      confidence = Math.max(confidence, 0.85);
+    }
+
+    return Math.min(confidence, 1);
+  }
+
+  // ===========================================================================
+  // Diff Analysis for Conflicts (Requirement 8.4)
+  // ===========================================================================
+
+  /**
+   * Compare action logs of competing agents and recommend a winner.
+   *
+   * Requirements: 8.4
+   */
+  analyzeDiff(
+    agentALog: ActionLogEntry[],
+    agentBLog: ActionLogEntry[]
+  ): ExtendedDiffAnalysis {
+    // Find divergence point
+    const divergencePoint = this.findDivergencePoint(agentALog, agentBLog);
+
+    // Get paths from divergence
+    const agentAPath = agentALog.slice(divergencePoint);
+    const agentBPath = agentBLog.slice(divergencePoint);
+
+    // Calculate metrics for each agent
+    const agentAMetrics = this.calculateAgentMetrics(agentALog);
+    const agentBMetrics = this.calculateAgentMetrics(agentBLog);
+
+    // Determine recommendation
+    const { recommendation, reason, confidence } = this.determineWinner(
+      agentAMetrics,
+      agentBMetrics,
+      divergencePoint
+    );
+
+    return {
+      divergencePoint,
+      agentAPath,
+      agentBPath,
+      recommendation,
+      recommendationReason: reason,
+      recommendationConfidence: confidence,
+      agentAMetrics,
+      agentBMetrics,
+    };
+  }
+
+  /**
+   * Find the index where two action logs diverge.
+   */
+  private findDivergencePoint(
+    logA: ActionLogEntry[],
+    logB: ActionLogEntry[]
+  ): number {
+    const minLength = Math.min(logA.length, logB.length);
+
+    for (let i = 0; i < minLength; i++) {
+      if (!this.actionsMatch(logA[i], logB[i])) {
+        return i;
+      }
+    }
+
+    // If one log is longer, divergence is at the end of the shorter one
+    if (logA.length !== logB.length) {
+      return minLength;
+    }
+
+    // Logs are identical
+    return logA.length;
+  }
+
+  /**
+   * Check if two action entries match (ignoring timestamps).
+   */
+  private actionsMatch(a: ActionLogEntry, b: ActionLogEntry): boolean {
+    return (
+      a.action === b.action &&
+      a.intentId === b.intentId &&
+      a.value === b.value &&
+      a.result === b.result
+    );
+  }
+
+  /**
+   * Calculate metrics for an agent's action log.
+   */
+  private calculateAgentMetrics(log: ActionLogEntry[]): AgentMetrics {
+    const successfulActions = log.filter((e) => e.result === 'success').length;
+    const failedActions = log.filter((e) => e.result === 'failure').length;
+    const extractActions = log.filter((e) => e.action === 'extract').length;
+
+    const totalActions = log.length;
+    const successRate = totalActions > 0 ? successfulActions / totalActions : 0;
+
+    // Consider task completed if there are successful extract actions
+    const completedTask = extractActions > 0 && successfulActions > 0;
+
+    return {
+      totalActions,
+      successfulActions,
+      failedActions,
+      successRate,
+      extractActions,
+      completedTask,
+    };
+  }
+
+  /**
+   * Determine the winner based on metrics.
+   */
+  private determineWinner(
+    metricsA: AgentMetrics,
+    metricsB: AgentMetrics,
+    divergencePoint: number
+  ): { recommendation: 'agent_a' | 'agent_b' | 'tie' | 'both_invalid'; reason: string; confidence: number } {
+    // Both invalid if neither completed the task
+    if (!metricsA.completedTask && !metricsB.completedTask) {
+      // Check if both have only failures
+      if (metricsA.successfulActions === 0 && metricsB.successfulActions === 0) {
+        return {
+          recommendation: 'both_invalid',
+          reason: 'Both agents failed to complete any actions successfully',
+          confidence: 0.95,
+        };
+      }
+    }
+
+    // One completed, one didn't
+    if (metricsA.completedTask && !metricsB.completedTask) {
+      return {
+        recommendation: 'agent_a',
+        reason: 'Agent A completed the task while Agent B did not',
+        confidence: 0.9,
+      };
+    }
+    if (metricsB.completedTask && !metricsA.completedTask) {
+      return {
+        recommendation: 'agent_b',
+        reason: 'Agent B completed the task while Agent A did not',
+        confidence: 0.9,
+      };
+    }
+
+    // Both completed - compare success rates
+    const rateDiff = Math.abs(metricsA.successRate - metricsB.successRate);
+
+    if (rateDiff > 0.1) {
+      if (metricsA.successRate > metricsB.successRate) {
+        return {
+          recommendation: 'agent_a',
+          reason: `Agent A has higher success rate (${(metricsA.successRate * 100).toFixed(1)}% vs ${(metricsB.successRate * 100).toFixed(1)}%)`,
+          confidence: 0.7 + rateDiff * 0.2,
+        };
+      } else {
+        return {
+          recommendation: 'agent_b',
+          reason: `Agent B has higher success rate (${(metricsB.successRate * 100).toFixed(1)}% vs ${(metricsA.successRate * 100).toFixed(1)}%)`,
+          confidence: 0.7 + rateDiff * 0.2,
+        };
+      }
+    }
+
+    // Similar success rates - compare extract actions
+    if (metricsA.extractActions !== metricsB.extractActions) {
+      if (metricsA.extractActions > metricsB.extractActions) {
+        return {
+          recommendation: 'agent_a',
+          reason: `Agent A performed more data extractions (${metricsA.extractActions} vs ${metricsB.extractActions})`,
+          confidence: 0.65,
+        };
+      } else {
+        return {
+          recommendation: 'agent_b',
+          reason: `Agent B performed more data extractions (${metricsB.extractActions} vs ${metricsA.extractActions})`,
+          confidence: 0.65,
+        };
+      }
+    }
+
+    // Truly tied
+    return {
+      recommendation: 'tie',
+      reason: `Both agents performed similarly (diverged at action ${divergencePoint})`,
+      confidence: 0.5,
+    };
+  }
+
+  // ===========================================================================
+  // Confidence Scoring (Requirement 8.6)
+  // ===========================================================================
+
+  /**
+   * Calculate confidence score for a verification result.
+   *
+   * Requirements: 8.6
+   */
+  calculateConfidence(
+    claimedResult: unknown,
+    actionLog: ActionLogEntry[],
+    proof?: VerificationProof
+  ): ConfidenceScoreResult {
+    const factors: ConfidenceFactor[] = [];
+    const summary = this.summarizeActions(actionLog);
+
+    // Factor 1: Action success rate
+    const successRateScore = summary.totalActions > 0
+      ? summary.successfulActions / summary.totalActions
+      : 0;
+    factors.push({
+      name: 'Action Success Rate',
+      weight: this.config.actionSuccessWeight,
+      score: successRateScore,
+      description: `${summary.successfulActions}/${summary.totalActions} actions succeeded`,
+    });
+
+    // Factor 2: Extract action presence
+    const extractScore = summary.extractActions > 0 ? 1 : 0;
+    factors.push({
+      name: 'Extract Actions Present',
+      weight: this.config.extractPresenceWeight,
+      score: extractScore,
+      description: summary.extractActions > 0
+        ? `${summary.extractActions} extract actions performed`
+        : 'No extract actions performed',
+    });
+
+    // Factor 3: Action-result alignment
+    const alignmentScore = this.calculateAlignmentScore(claimedResult, summary);
+    factors.push({
+      name: 'Action-Result Alignment',
+      weight: this.config.alignmentWeight,
+      score: alignmentScore,
+      description: alignmentScore > 0.7
+        ? 'Result aligns well with actions'
+        : 'Result may not fully align with actions',
+    });
+
+    // Factor 4: Action completeness
+    const completenessScore = this.calculateCompletenessScore(summary);
+    factors.push({
+      name: 'Action Completeness',
+      weight: this.config.completenessWeight,
+      score: completenessScore,
+      description: completenessScore > 0.7
+        ? 'Action sequence appears complete'
+        : 'Action sequence may be incomplete',
+    });
+
+    // Calculate weighted score
+    const totalWeight = factors.reduce((sum, f) => sum + f.weight, 0);
+    const weightedSum = factors.reduce((sum, f) => sum + f.score * f.weight, 0);
+    const score = totalWeight > 0 ? weightedSum / totalWeight : 0;
+
+    // Adjust score based on proof if available
+    let finalScore = score;
+    if (proof) {
+      if (proof.resultMatchesActions) {
+        finalScore = Math.min(finalScore + 0.1, 1);
+      } else {
+        finalScore = Math.max(finalScore - 0.2, 0);
+      }
+    }
+
+    return {
+      score: finalScore,
+      factors,
+      isReliable: finalScore >= this.config.reliabilityThreshold,
+      reliabilityThreshold: this.config.reliabilityThreshold,
+    };
+  }
+
+  /**
+   * Calculate alignment score between result and actions.
+   */
+  private calculateAlignmentScore(_result: unknown, summary: ActionSummary): number {
+    // No result claimed - perfect alignment (nothing to misalign)
+    if (_result === null || _result === undefined) {
+      return 1;
+    }
+
+    // No actions but claiming result - poor alignment
+    if (summary.totalActions === 0) {
+      return 0;
+    }
+
+    // All failures but claiming success - poor alignment
+    if (summary.successfulActions === 0 && this.isSuccessfulResult(_result)) {
+      return 0;
+    }
+
+    // Data claimed without extract actions - poor alignment
+    if (this.hasDataClaim(_result) && summary.extractActions === 0) {
+      return 0.2;
+    }
+
+    // Has extract actions and data claim - good alignment
+    if (this.hasDataClaim(_result) && summary.extractActions > 0) {
+      return 0.9;
+    }
+
+    // Default moderate alignment
+    return 0.6;
+  }
+
+  /**
+   * Calculate completeness score for action sequence.
+   */
+  private calculateCompletenessScore(summary: ActionSummary): number {
+    // No actions - incomplete
+    if (summary.totalActions === 0) {
+      return 0;
+    }
+
+    // Check for typical complete sequence patterns
+    let score = 0.5; // Base score
+
+    // Has navigation (usually needed to start)
+    if (summary.navigateActions > 0) {
+      score += 0.15;
+    }
+
+    // Has interactions (clicks, types)
+    if (summary.clickActions > 0 || summary.typeActions > 0) {
+      score += 0.15;
+    }
+
+    // Has extractions (usually needed to get results)
+    if (summary.extractActions > 0) {
+      score += 0.2;
+    }
+
+    return Math.min(score, 1);
+  }
+
+  // ===========================================================================
+  // Utility Methods
+  // ===========================================================================
+
+  /**
+   * Get the current configuration.
+   */
+  getConfig(): VerificationEngineConfig {
+    return { ...this.config };
+  }
+
+  /**
+   * Update configuration.
+   */
+  updateConfig(config: Partial<VerificationEngineConfig>): void {
+    this.config = { ...this.config, ...config };
+  }
+}
+
+// =============================================================================
+// Factory Function
+// =============================================================================
+
+/**
+ * Create a new Verification Engine instance.
+ */
+export function createVerificationEngine(
+  config: Partial<VerificationEngineConfig> = {}
+): VerificationEngine {
+  return new VerificationEngine(config);
+}