outcome-cli 1.0.0

package/src/modules/omnibridge/__tests__/verification-engine.property.test.ts

@@ -0,0 +1,769 @@
+/**
+ * Verification Engine Property Tests
+ *
+ * Property-based tests for hallucination detection, diff analysis,
+ * and confidence scoring.
+ *
+ * Requirements: 8.4, 8.5, 8.6
+ */
+
+import { describe, test, expect, beforeEach } from 'vitest';
+import * as fc from 'fast-check';
+import {
+  VerificationEngine,
+  createVerificationEngine,
+} from '../verification/verification-engine.js';
+import type { ActionLogEntry } from '../core/types.js';
+
+// =============================================================================
+// Test Setup
+// =============================================================================
+
+// Note: Each property test creates its own verificationEngine instance to ensure isolation
+// The beforeEach instance is only used for tests that explicitly need shared state
+
+let verificationEngine: VerificationEngine;
+
+beforeEach(() => {
+  verificationEngine = createVerificationEngine({
+    reliabilityThreshold: 0.7,
+    actionSuccessWeight: 0.3,
+    extractPresenceWeight: 0.25,
+    alignmentWeight: 0.3,
+    completenessWeight: 0.15,
+  });
+});
+
+// =============================================================================
+// Arbitraries
+// =============================================================================
+
+/**
+ * Generate arbitrary session IDs.
+ */
+const sessionIdArbitrary = fc.stringMatching(/^session_[a-z0-9]{8,16}$/);
+
+/**
+ * Generate arbitrary intent IDs.
+ */
+const intentIdArbitrary = fc.stringMatching(
+  /^(ACTION|INPUT|DISPLAY|NAV)_ID:[A-Z_]{3,20}$/
+);
+
+/**
+ * Generate arbitrary action types.
+ */
+const actionTypeArbitrary = fc.constantFrom(
+  'click',
+  'type',
+  'navigate',
+  'wait',
+  'extract'
+) as fc.Arbitrary<'click' | 'type' | 'navigate' | 'wait' | 'extract'>;
+
+/**
+ * Generate arbitrary non-extract action types.
+ */
+const nonExtractActionTypeArbitrary = fc.constantFrom(
+  'click',
+  'type',
+  'navigate',
+  'wait'
+) as fc.Arbitrary<'click' | 'type' | 'navigate' | 'wait'>;
+
+/**
+ * Generate arbitrary action results.
+ */
+const actionResultArbitrary = fc.constantFrom('success', 'failure') as fc.Arbitrary<
+  'success' | 'failure'
+>;
+
+/**
+ * Generate arbitrary action log entries with timestamp.
+ */
+const actionLogEntryArbitrary = fc.record({
+  timestamp: fc.integer({ min: 1000000000000, max: 2000000000000 }),
+  sessionId: sessionIdArbitrary,
+  action: actionTypeArbitrary,
+  intentId: intentIdArbitrary,
+  value: fc.option(fc.string({ minLength: 0, maxLength: 100 }), { nil: undefined }),
+  result: actionResultArbitrary,
+});
+
+/**
+ * Generate arbitrary action log entries without extract actions.
+ */
+const nonExtractActionLogEntryArbitrary = fc.record({
+  timestamp: fc.integer({ min: 1000000000000, max: 2000000000000 }),
+  sessionId: sessionIdArbitrary,
+  action: nonExtractActionTypeArbitrary,
+  intentId: intentIdArbitrary,
+  value: fc.option(fc.string({ minLength: 0, maxLength: 100 }), { nil: undefined }),
+  result: actionResultArbitrary,
+});
+
+/**
+ * Generate arbitrary failed action log entries.
+ */
+const failedActionLogEntryArbitrary = fc.record({
+  timestamp: fc.integer({ min: 1000000000000, max: 2000000000000 }),
+  sessionId: sessionIdArbitrary,
+  action: actionTypeArbitrary,
+  intentId: intentIdArbitrary,
+  value: fc.option(fc.string({ minLength: 0, maxLength: 100 }), { nil: undefined }),
+  result: fc.constant('failure' as const),
+});
+
+/**
+ * Generate arbitrary claimed results with data.
+ */
+const claimedResultWithDataArbitrary = fc.record({
+  data: fc.record({
+    items: fc.array(fc.string({ minLength: 1, maxLength: 20 }), { minLength: 1, maxLength: 5 }),
+    total: fc.integer({ min: 1, max: 100 }),
+  }),
+  metadata: fc.record({
+    confidence: fc.float({ min: 0, max: 1 }),
+    executionTimeMs: fc.integer({ min: 0, max: 10000 }),
+    actionsPerformed: fc.integer({ min: 0, max: 100 }),
+  }),
+});
+
+/**
+ * Generate arbitrary successful action log entries with extract.
+ */
+const successfulExtractActionArbitrary = fc.record({
+  timestamp: fc.integer({ min: 1000000000000, max: 2000000000000 }),
+  sessionId: sessionIdArbitrary,
+  action: fc.constant('extract' as const),
+  intentId: intentIdArbitrary,
+  value: fc.option(fc.string({ minLength: 0, maxLength: 100 }), { nil: undefined }),
+  result: fc.constant('success' as const),
+});
+
+// =============================================================================
+// Property 19: Hallucination Detection
+// =============================================================================
+
+describe('Property 19: Hallucination Detection', () => {
+  /**
+   * **Feature: omnibridge, Property 19: Hallucination Detection**
+   *
+   * *For any* agent result where the claimed outcome cannot be derived from
+   * the recorded action sequence, the verification SHALL flag
+   * `isHallucination: true`.
+   *
+   * **Validates: Requirements 8.5**
+   */
+  test(
+    'Property 19: claimed data without extract actions is flagged as hallucination',
+    async () => {
+      await fc.assert(
+        fc.asyncProperty(
+          fc.array(nonExtractActionLogEntryArbitrary, { minLength: 1, maxLength: 10 }),
+          claimedResultWithDataArbitrary,
+          async (actionLog, claimedResult) => {
+            const result = verificationEngine.detectHallucination(
+              claimedResult,
+              actionLog
+            );
+
+            // Should be flagged as hallucination since no extract actions
+            expect(result.isHallucination).toBe(true);
+            expect(result.reasons.length).toBeGreaterThan(0);
+            expect(result.reasons).toContain(
+              'Data claimed but no extract actions performed'
+            );
+
+            return true;
+          }
+        ),
+        { numRuns: 100 }
+      );
+    }
+  );
+
+  /**
+   * Claimed data with extract actions is NOT flagged as hallucination.
+   */
+  test(
+    'claimed data with extract actions is NOT flagged as hallucination',
+    async () => {
+      await fc.assert(
+        fc.asyncProperty(
+          successfulExtractActionArbitrary,
+          fc.array(actionLogEntryArbitrary, { minLength: 0, maxLength: 5 }),
+          claimedResultWithDataArbitrary,
+          async (extractAction, otherActions, claimedResult) => {
+            // Combine extract action with other actions
+            const actionLog = [extractAction, ...otherActions];
+
+            const result = verificationEngine.detectHallucination(
+              claimedResult,
+              actionLog
+            );
+
+            // Should NOT be flagged for missing extract actions
+            expect(result.reasons).not.toContain(
+              'Data claimed but no extract actions performed'
+            );
+
+            return true;
+          }
+        ),
+        { numRuns: 100 }
+      );
+    }
+  );
+
+  /**
+   * All failed actions with success claim is flagged as hallucination.
+   */
+  test(
+    'all failed actions with success claim is flagged as hallucination',
+    async () => {
+      await fc.assert(
+        fc.asyncProperty(
+          fc.array(failedActionLogEntryArbitrary, { minLength: 1, maxLength: 10 }),
+          claimedResultWithDataArbitrary,
+          async (actionLog, claimedResult) => {
+            const result = verificationEngine.detectHallucination(
+              claimedResult,
+              actionLog
+            );
+
+            // Should be flagged as hallucination since all actions failed
+            expect(result.isHallucination).toBe(true);
+            expect(result.reasons).toContain(
+              'Success claimed but all actions failed'
+            );
+
+            return true;
+          }
+        ),
+        { numRuns: 100 }
+      );
+    }
+  );
+
+  /**
+   * No actions with claimed result is flagged as hallucination.
+   */
+  test(
+    'no actions with claimed result is flagged as hallucination',
+    async () => {
+      await fc.assert(
+        fc.asyncProperty(claimedResultWithDataArbitrary, async (claimedResult) => {
+          const result = verificationEngine.detectHallucination(claimedResult, []);
+
+          // Should be flagged as hallucination since no actions
+          expect(result.isHallucination).toBe(true);
+          expect(result.reasons).toContain(
+            'Result claimed with no actions performed'
+          );
+
+          return true;
+        }),
+        { numRuns: 100 }
+      );
+    }
+  );
+
+  /**
+   * Null claimed result is never flagged as hallucination.
+   */
+  test(
+    'null claimed result is never flagged as hallucination',
+    async () => {
+      await fc.assert(
+        fc.asyncProperty(
+          fc.array(actionLogEntryArbitrary, { minLength: 0, maxLength: 10 }),
+          async (actionLog) => {
+            const result = verificationEngine.detectHallucination(null, actionLog);
+
+            // Null result should never be a hallucination
+            expect(result.isHallucination).toBe(false);
+            expect(result.reasons.length).toBe(0);
+
+            return true;
+          }
+        ),
+        { numRuns: 100 }
+      );
+    }
+  );
+
+  /**
+   * Undefined claimed result is never flagged as hallucination.
+   */
+  test(
+    'undefined claimed result is never flagged as hallucination',
+    async () => {
+      await fc.assert(
+        fc.asyncProperty(
+          fc.array(actionLogEntryArbitrary, { minLength: 0, maxLength: 10 }),
+          async (actionLog) => {
+            const result = verificationEngine.detectHallucination(undefined, actionLog);
+
+            // Undefined result should never be a hallucination
+            expect(result.isHallucination).toBe(false);
+            expect(result.reasons.length).toBe(0);
+
+            return true;
+          }
+        ),
+        { numRuns: 100 }
+      );
+    }
+  );
+
+  /**
+   * Action summary is correctly calculated.
+   */
+  test(
+    'action summary is correctly calculated',
+    async () => {
+      await fc.assert(
+        fc.asyncProperty(
+          fc.array(actionLogEntryArbitrary, { minLength: 1, maxLength: 20 }),
+          async (actionLog) => {
+            const result = verificationEngine.detectHallucination(null, actionLog);
+
+            // Verify action summary
+            expect(result.actionSummary.totalActions).toBe(actionLog.length);
+
+            const expectedSuccessful = actionLog.filter(
+              (a) => a.result === 'success'
+            ).length;
+            expect(result.actionSummary.successfulActions).toBe(expectedSuccessful);
+
+            const expectedFailed = actionLog.filter(
+              (a) => a.result === 'failure'
+            ).length;
+            expect(result.actionSummary.failedActions).toBe(expectedFailed);
+
+            const expectedExtract = actionLog.filter(
+              (a) => a.action === 'extract'
+            ).length;
+            expect(result.actionSummary.extractActions).toBe(expectedExtract);
+
+            return true;
+          }
+        ),
+        { numRuns: 100 }
+      );
+    }
+  );
+
+  /**
+   * Hallucination confidence is between 0 and 1.
+   */
+  test(
+    'hallucination confidence is between 0 and 1',
+    async () => {
+      await fc.assert(
+        fc.asyncProperty(
+          fc.array(actionLogEntryArbitrary, { minLength: 0, maxLength: 10 }),
+          fc.oneof(
+            fc.constant(null),
+            fc.constant(undefined),
+            claimedResultWithDataArbitrary
+          ),
+          async (actionLog, claimedResult) => {
+            const result = verificationEngine.detectHallucination(
+              claimedResult,
+              actionLog
+            );
+
+            expect(result.confidence).toBeGreaterThanOrEqual(0);
+            expect(result.confidence).toBeLessThanOrEqual(1);
+
+            return true;
+          }
+        ),
+        { numRuns: 100 }
+      );
+    }
+  );
+});
+
+// =============================================================================
+// Diff Analysis Tests (Requirement 8.4)
+// =============================================================================
+
+describe('Diff Analysis for Conflicts', () => {
+  /**
+   * Divergence point is correctly identified.
+   */
+  test(
+    'divergence point is correctly identified',
+    async () => {
+      await fc.assert(
+        fc.asyncProperty(
+          fc.array(actionLogEntryArbitrary, { minLength: 2, maxLength: 10 }),
+          fc.integer({ min: 0, max: 9 }),
+          async (commonActions, divergeIndexRaw) => {
+            const divergeIndex = divergeIndexRaw % commonActions.length;
+
+            // Create two logs that share common prefix
+            const logA: ActionLogEntry[] = [];
+            const logB: ActionLogEntry[] = [];
+
+            for (let i = 0; i < divergeIndex; i++) {
+              logA.push(commonActions[i]);
+              logB.push(commonActions[i]);
+            }
+
+            // Add divergent actions
+            if (divergeIndex < commonActions.length) {
+              logA.push({
+                ...commonActions[divergeIndex],
+                result: 'success',
+              });
+              logB.push({
+                ...commonActions[divergeIndex],
+                result: 'failure',
+              });
+            }
+
+            const analysis = verificationEngine.analyzeDiff(logA, logB);
+
+            expect(analysis.divergencePoint).toBe(divergeIndex);
+
+            return true;
+          }
+        ),
+        { numRuns: 100 }
+      );
+    }
+  );
+
+  /**
+   * Identical logs have divergence at the end.
+   */
+  test(
+    'identical logs have divergence at the end',
+    async () => {
+      await fc.assert(
+        fc.asyncProperty(
+          fc.array(actionLogEntryArbitrary, { minLength: 1, maxLength: 10 }),
+          async (actions) => {
+            const analysis = verificationEngine.analyzeDiff(actions, actions);
+
+            // Divergence should be at the end for identical logs
+            expect(analysis.divergencePoint).toBe(actions.length);
+            
+            // Recommendation depends on whether agents completed the task
+            // If both have only failures, it's 'both_invalid'
+            // If both have successes, it's 'tie'
+            const hasSuccesses = actions.some(a => a.result === 'success');
+            const hasExtracts = actions.some(a => a.action === 'extract');
+            
+            if (!hasSuccesses) {
+              expect(analysis.recommendation).toBe('both_invalid');
+            } else if (hasExtracts && hasSuccesses) {
+              expect(analysis.recommendation).toBe('tie');
+            }
+            // Other cases may vary based on metrics
+
+            return true;
+          }
+        ),
+        { numRuns: 100 }
+      );
+    }
+  );
+
+  /**
+   * Agent with higher success rate is recommended.
+   */
+  test(
+    'agent with higher success rate is recommended',
+    async () => {
+      await fc.assert(
+        fc.asyncProperty(
+          fc.array(actionLogEntryArbitrary, { minLength: 3, maxLength: 10 }),
+          async (baseActions) => {
+            // Create logA with all successes
+            const logA: ActionLogEntry[] = baseActions.map((a) => ({
+              ...a,
+              result: 'success' as const,
+              action: 'extract' as const, // Ensure task completion
+            }));
+
+            // Create logB with all failures
+            const logB: ActionLogEntry[] = baseActions.map((a) => ({
+              ...a,
+              result: 'failure' as const,
+            }));
+
+            const analysis = verificationEngine.analyzeDiff(logA, logB);
+
+            // Agent A should be recommended (higher success rate)
+            expect(analysis.recommendation).toBe('agent_a');
+            expect(analysis.agentAMetrics.successRate).toBeGreaterThan(
+              analysis.agentBMetrics.successRate
+            );
+
+            return true;
+          }
+        ),
+        { numRuns: 100 }
+      );
+    }
+  );
+
+  /**
+   * Both invalid when neither completes task.
+   */
+  test(
+    'both invalid when neither has successful actions',
+    async () => {
+      await fc.assert(
+        fc.asyncProperty(
+          fc.array(failedActionLogEntryArbitrary, { minLength: 1, maxLength: 5 }),
+          fc.array(failedActionLogEntryArbitrary, { minLength: 1, maxLength: 5 }),
+          async (logA, logB) => {
+            const analysis = verificationEngine.analyzeDiff(logA, logB);
+
+            expect(analysis.recommendation).toBe('both_invalid');
+
+            return true;
+          }
+        ),
+        { numRuns: 100 }
+      );
+    }
+  );
+
+  /**
+   * Recommendation confidence is between 0 and 1.
+   */
+  test(
+    'recommendation confidence is between 0 and 1',
+    async () => {
+      await fc.assert(
+        fc.asyncProperty(
+          fc.array(actionLogEntryArbitrary, { minLength: 1, maxLength: 10 }),
+          fc.array(actionLogEntryArbitrary, { minLength: 1, maxLength: 10 }),
+          async (logA, logB) => {
+            const analysis = verificationEngine.analyzeDiff(logA, logB);
+
+            expect(analysis.recommendationConfidence).toBeGreaterThanOrEqual(0);
+            expect(analysis.recommendationConfidence).toBeLessThanOrEqual(1);
+
+            return true;
+          }
+        ),
+        { numRuns: 100 }
+      );
+    }
+  );
+});
+
+// =============================================================================
+// Confidence Scoring Tests (Requirement 8.6)
+// =============================================================================
+
+describe('Confidence Scoring', () => {
+  /**
+   * Confidence score is between 0 and 1.
+   */
+  test(
+    'confidence score is between 0 and 1',
+    async () => {
+      await fc.assert(
+        fc.asyncProperty(
+          fc.array(actionLogEntryArbitrary, { minLength: 0, maxLength: 10 }),
+          fc.oneof(
+            fc.constant(null),
+            fc.constant(undefined),
+            claimedResultWithDataArbitrary
+          ),
+          async (actionLog, claimedResult) => {
+            const result = verificationEngine.calculateConfidence(
+              claimedResult,
+              actionLog
+            );
+
+            expect(result.score).toBeGreaterThanOrEqual(0);
+            expect(result.score).toBeLessThanOrEqual(1);
+
+            return true;
+          }
+        ),
+        { numRuns: 100 }
+      );
+    }
+  );
+
+  /**
+   * Higher success rate leads to higher confidence.
+   */
+  test(
+    'higher success rate leads to higher confidence',
+    async () => {
+      await fc.assert(
+        fc.asyncProperty(
+          fc.array(actionLogEntryArbitrary, { minLength: 5, maxLength: 10 }),
+          async (baseActions) => {
+            // Create log with all successes
+            const successLog: ActionLogEntry[] = baseActions.map((a) => ({
+              ...a,
+              result: 'success' as const,
+            }));
+
+            // Create log with all failures
+            const failureLog: ActionLogEntry[] = baseActions.map((a) => ({
+              ...a,
+              result: 'failure' as const,
+            }));
+
+            const successResult = verificationEngine.calculateConfidence(
+              null,
+              successLog
+            );
+            const failureResult = verificationEngine.calculateConfidence(
+              null,
+              failureLog
+            );
+
+            // Success log should have higher confidence
+            expect(successResult.score).toBeGreaterThan(failureResult.score);
+
+            return true;
+          }
+        ),
+        { numRuns: 100 }
+      );
+    }
+  );
+
+  /**
+   * Extract actions increase confidence.
+   */
+  test(
+    'extract actions increase confidence',
+    async () => {
+      await fc.assert(
+        fc.asyncProperty(
+          fc.array(
+            fc.record({
+              timestamp: fc.integer({ min: 1000000000000, max: 2000000000000 }),
+              sessionId: sessionIdArbitrary,
+              action: fc.constant('click' as const),
+              intentId: intentIdArbitrary,
+              result: fc.constant('success' as const),
+            }),
+            { minLength: 3, maxLength: 5 }
+          ),
+          async (clickActions) => {
+            // Log without extract
+            const noExtractLog = clickActions;
+
+            // Log with extract
+            const withExtractLog: ActionLogEntry[] = [
+              ...clickActions,
+              {
+                timestamp: Date.now(),
+                sessionId: clickActions[0].sessionId,
+                action: 'extract' as const,
+                intentId: 'DISPLAY_ID:DATA',
+                result: 'success' as const,
+              },
+            ];
+
+            const noExtractResult = verificationEngine.calculateConfidence(
+              null,
+              noExtractLog
+            );
+            const withExtractResult = verificationEngine.calculateConfidence(
+              null,
+              withExtractLog
+            );
+
+            // With extract should have higher confidence
+            expect(withExtractResult.score).toBeGreaterThan(noExtractResult.score);
+
+            return true;
+          }
+        ),
+        { numRuns: 100 }
+      );
+    }
+  );
+
+  /**
+   * Reliability threshold is respected.
+   */
+  test(
+    'reliability threshold is respected',
+    async () => {
+      await fc.assert(
+        fc.asyncProperty(
+          fc.array(actionLogEntryArbitrary, { minLength: 1, maxLength: 10 }),
+          async (actionLog) => {
+            const result = verificationEngine.calculateConfidence(null, actionLog);
+
+            // isReliable should match threshold comparison
+            expect(result.isReliable).toBe(
+              result.score >= result.reliabilityThreshold
+            );
+
+            return true;
+          }
+        ),
+        { numRuns: 100 }
+      );
+    }
+  );
+
+  /**
+   * Confidence factors are all present.
+   */
+  test(
+    'confidence factors are all present',
+    async () => {
+      await fc.assert(
+        fc.asyncProperty(
+          fc.array(actionLogEntryArbitrary, { minLength: 1, maxLength: 10 }),
+          async (actionLog) => {
+            const result = verificationEngine.calculateConfidence(null, actionLog);
+
+            // Should have all expected factors
+            const factorNames = result.factors.map((f) => f.name);
+            expect(factorNames).toContain('Action Success Rate');
+            expect(factorNames).toContain('Extract Actions Present');
+            expect(factorNames).toContain('Action-Result Alignment');
+            expect(factorNames).toContain('Action Completeness');
+
+            // All factors should have valid scores
+            for (const factor of result.factors) {
+              expect(factor.score).toBeGreaterThanOrEqual(0);
+              expect(factor.score).toBeLessThanOrEqual(1);
+              expect(factor.weight).toBeGreaterThanOrEqual(0);
+              expect(factor.weight).toBeLessThanOrEqual(1);
+            }
+
+            return true;
+          }
+        ),
+        { numRuns: 100 }
+      );
+    }
+  );
+
+  /**
+   * Empty action log has low confidence.
+   */
+  test(
+    'empty action log has low confidence',
+    async () => {
+      const result = verificationEngine.calculateConfidence(null, []);
+
+      // Empty action log should have low confidence (below reliability threshold)
+      expect(result.score).toBeLessThan(result.reliabilityThreshold);
+      expect(result.isReliable).toBe(false);
+    }
+  );
+});