outcome-cli 1.0.0

package/src/league/scoreAgent.ts

@@ -0,0 +1,175 @@
+/**
+ * Score Agent - Agent scoring and winner determination
+ *
+ * Determines the winner of a league based on evaluation results and metrics.
+ *
+ * @module league/scoreAgent
+ * @see Requirements 4.5
+ */
+
+import type { EvaluationResult } from '../eval/evaluateOutcome.js';
+
+/**
+ * Metrics collected during an agent's run.
+ */
+export interface AgentMetrics {
+  /** Agent ID */
+  agentId: string;
+  /** Tokens spent */
+  tokensSpent: number;
+  /** Number of attempts made */
+  attempts: number;
+  /** Duration in milliseconds */
+  durationMs: number;
+}
+
+/**
+ * Score for an agent's performance.
+ */
+export interface AgentScore {
+  /** Agent ID */
+  agentId: string;
+  /** Whether the agent achieved success */
+  success: boolean;
+  /** Tokens spent */
+  cost: number;
+  /** Number of attempts made */
+  attempts: number;
+  /** Duration in milliseconds */
+  duration: number;
+  /** Timestamp when success was achieved (if applicable) */
+  successTimestamp?: number;
+}
+
+/**
+ * Scores an agent based on evaluation result and metrics.
+ *
+ * @param result - Evaluation result from the agent's artifact
+ * @param metrics - Metrics collected during the run
+ * @returns AgentScore for ranking
+ *
+ * @example
+ * const score = scoreAgent(evalResult, {
+ *   agentId: 'agent-1',
+ *   tokensSpent: 500,
+ *   attempts: 2,
+ *   durationMs: 5000
+ * });
+ *
+ * @see Requirements 4.5
+ */
+export function scoreAgent(
+  result: EvaluationResult,
+  metrics: AgentMetrics
+): AgentScore {
+  return {
+    agentId: metrics.agentId,
+    success: result.status === 'SUCCESS',
+    cost: metrics.tokensSpent,
+    attempts: metrics.attempts,
+    duration: metrics.durationMs,
+    successTimestamp: result.status === 'SUCCESS' ? Date.now() : undefined,
+  };
+}
+
+/**
+ * Determines the winner from a list of agent scores.
+ *
+ * Winner criteria (in order):
+ * 1. Must have success === true
+ * 2. Earliest successTimestamp wins (first to succeed)
+ *
+ * @param scores - Array of agent scores
+ * @returns Winning AgentScore or null if no winner
+ *
+ * @example
+ * const winner = determineWinner(scores);
+ * if (winner) {
+ *   console.log(`Winner: ${winner.agentId}`);
+ * }
+ *
+ * @see Requirements 4.5
+ */
+export function determineWinner(scores: AgentScore[]): AgentScore | null {
+  // Filter to only successful agents
+  const successfulAgents = scores.filter((s) => s.success);
+
+  if (successfulAgents.length === 0) {
+    return null;
+  }
+
+  // Sort by success timestamp (earliest first)
+  successfulAgents.sort((a, b) => {
+    const timeA = a.successTimestamp ?? Infinity;
+    const timeB = b.successTimestamp ?? Infinity;
+    return timeA - timeB;
+  });
+
+  return successfulAgents[0];
+}
+
+/**
+ * Ranks all agents by performance.
+ *
+ * Ranking criteria:
+ * 1. Success status (successful agents rank higher)
+ * 2. For successful: earlier success timestamp
+ * 3. For failed: fewer attempts, lower cost
+ *
+ * @param scores - Array of agent scores
+ * @returns Sorted array with best performer first
+ */
+export function rankAgents(scores: AgentScore[]): AgentScore[] {
+  return [...scores].sort((a, b) => {
+    // Successful agents rank higher
+    if (a.success && !b.success) return -1;
+    if (!a.success && b.success) return 1;
+
+    if (a.success && b.success) {
+      // Both successful: earlier timestamp wins
+      const timeA = a.successTimestamp ?? Infinity;
+      const timeB = b.successTimestamp ?? Infinity;
+      return timeA - timeB;
+    }
+
+    // Both failed: fewer attempts is better
+    if (a.attempts !== b.attempts) {
+      return a.attempts - b.attempts;
+    }
+
+    // Same attempts: lower cost is better
+    return a.cost - b.cost;
+  });
+}
+
+/**
+ * Calculates aggregate statistics for a league run.
+ *
+ * @param scores - Array of agent scores
+ * @returns Aggregate statistics
+ */
+export function calculateLeagueStats(scores: AgentScore[]): {
+  totalAgents: number;
+  successfulAgents: number;
+  failedAgents: number;
+  totalCost: number;
+  averageCost: number;
+  totalAttempts: number;
+  averageAttempts: number;
+} {
+  const totalAgents = scores.length;
+  const successfulAgents = scores.filter((s) => s.success).length;
+  const failedAgents = totalAgents - successfulAgents;
+  const totalCost = scores.reduce((sum, s) => sum + s.cost, 0);
+  const totalAttempts = scores.reduce((sum, s) => sum + s.attempts, 0);
+
+  return {
+    totalAgents,
+    successfulAgents,
+    failedAgents,
+    totalCost,
+    averageCost: totalAgents > 0 ? totalCost / totalAgents : 0,
+    totalAttempts,
+    averageAttempts: totalAgents > 0 ? totalAttempts / totalAgents : 0,
+  };
+}

package/src/modules/omnibridge/__tests__/.gitkeep

@@ -0,0 +1 @@
+# Tests - Unit tests and Property-based tests

package/src/modules/omnibridge/__tests__/auth-tunnel.property.test.ts

@@ -0,0 +1,524 @@
+/**
+ * Property-Based Tests for Auth Tunnel (MFA Relay)
+ *
+ * These tests validate the correctness properties defined in the design document.
+ * Each property test runs minimum 100 iterations with randomly generated inputs.
+ *
+ * **Property 11: MFA Timeout**
+ * **Validates: Requirements 5.6**
+ */
+
+import { describe, test, expect, beforeEach, afterEach, vi } from 'vitest';
+import * as fc from 'fast-check';
+import {
+  AuthTunnel,
+  createAuthTunnel,
+  MFA_TIMEOUT_MS,
+} from '../auth/auth-tunnel.js';
+import type { MFAChallenge, ShadowSession } from '../core/types.js';
+
+// =============================================================================
+// Test Setup
+// =============================================================================
+
+let authTunnel: AuthTunnel;
+
+beforeEach(() => {
+  vi.useFakeTimers();
+  authTunnel = createAuthTunnel({
+    mfaTimeoutMs: MFA_TIMEOUT_MS, // 5 minutes = 300000ms
+  });
+});
+
+afterEach(() => {
+  authTunnel.clear();
+  vi.useRealTimers();
+});
+
+// =============================================================================
+// Arbitraries (Test Data Generators)
+// =============================================================================
+
+/**
+ * Generate a valid session ID.
+ */
+const sessionIdArb = fc.stringMatching(/^shadow_[a-f0-9]{32}$/);
+
+/**
+ * Generate MFA challenge types.
+ */
+const mfaTypeArb: fc.Arbitrary<MFAChallenge['type']> = fc.constantFrom(
+  'sms',
+  'authenticator',
+  'push',
+  'email'
+);
+
+/**
+ * Generate a mock Shadow Session.
+ * Uses fixed timestamps relative to a base time to ensure deterministic test behavior.
+ */
+const BASE_TIME = 1700000000000; // Fixed base timestamp for deterministic tests
+
+const shadowSessionArb: fc.Arbitrary<ShadowSession> = fc.record({
+  id: sessionIdArb,
+  domain: fc.stringMatching(/^[a-z][a-z0-9-]{2,20}\.(com|org|net|io)$/),
+  status: fc.constant('active' as const),
+  createdAt: fc.integer({ min: BASE_TIME - 3600000, max: BASE_TIME }),
+  lastHeartbeat: fc.integer({ min: BASE_TIME - 60000, max: BASE_TIME }),
+  fingerprint: fc.record({
+    resolution: fc.record({
+      width: fc.integer({ min: 800, max: 3840 }),
+      height: fc.integer({ min: 600, max: 2160 }),
+    }),
+    userAgent: fc.constant('Mozilla/5.0 (Windows NT 10.0; Win64; x64) Chrome/120.0.0.0'),
+    fonts: fc.array(fc.string(), { minLength: 1, maxLength: 5 }),
+    gpuSignature: fc.string(),
+    timezone: fc.constant('America/New_York'),
+    language: fc.constant('en-US'),
+  }),
+});
+
+/**
+ * Generate page content with MFA keywords.
+ */
+const mfaPageContentArb = (type: MFAChallenge['type']): fc.Arbitrary<string> => {
+  const keywordsByType: Record<MFAChallenge['type'], string[]> = {
+    sms: ['Enter the verification code sent to your phone', 'SMS code', '6-digit code'],
+    authenticator: ['Enter the code from your authenticator app', 'Google Authenticator', 'TOTP'],
+    push: ['Approve the login request on your device', 'Push notification sent'],
+    email: ['Enter the code sent to your email', 'Email verification'],
+  };
+
+  return fc.constantFrom(...keywordsByType[type]).map((keyword) => {
+    return `<html><body><h1>Verification Required</h1><p>${keyword}</p><input type="text" name="code" /></body></html>`;
+  });
+};
+
+/**
+ * Generate valid MFA codes.
+ */
+const validMfaCodeArb = fc.stringMatching(/^\d{6}$/);
+
+/**
+ * Generate time delays in milliseconds.
+ */
+const timeDelayArb = fc.integer({ min: 0, max: MFA_TIMEOUT_MS + 60000 });
+
+// =============================================================================
+// Property Tests
+// =============================================================================
+
+describe('Auth Tunnel Property Tests', () => {
+  /**
+   * **Feature: omnibridge, Property 11: MFA Timeout**
+   *
+   * *For any* MFA challenge, if no response is received within 5 minutes (300 seconds),
+   * the Auth_Tunnel SHALL return a timeout error.
+   *
+   * **Validates: Requirements 5.6**
+   */
+  test('Property 11: MFA Timeout - challenges timeout after 5 minutes', async () => {
+    await fc.assert(
+      fc.asyncProperty(
+        shadowSessionArb,
+        mfaTypeArb,
+        async (session, mfaType) => {
+          // Generate page content for the MFA type
+          const pageContent = await fc.sample(mfaPageContentArb(mfaType), 1)[0];
+
+          // Detect MFA challenge
+          const challenge = await authTunnel.detectMFA(session, pageContent);
+
+          // If no challenge detected, skip this iteration
+          if (!challenge) {
+            return true;
+          }
+
+          // Request code (starts the timeout timer)
+          await authTunnel.requestCode(challenge);
+
+          // Verify challenge is pending
+          expect(authTunnel.getChallenge(challenge.id)).not.toBeNull();
+          expect(authTunnel.isTimedOut(challenge.id)).toBe(false);
+
+          // PROPERTY: Before timeout, challenge should be valid
+          const remainingBefore = authTunnel.getRemainingTime(challenge.id);
+          expect(remainingBefore).toBeGreaterThan(0);
+          expect(remainingBefore).toBeLessThanOrEqual(MFA_TIMEOUT_MS);
+
+          // Advance time past the timeout (5 minutes + 1 second)
+          vi.advanceTimersByTime(MFA_TIMEOUT_MS + 1000);
+
+          // PROPERTY: After 5 minutes, challenge should be timed out
+          expect(authTunnel.isTimedOut(challenge.id)).toBe(true);
+          expect(authTunnel.getRemainingTime(challenge.id)).toBe(0);
+
+          // PROPERTY: Attempting to inject code after timeout should fail
+          const result = await authTunnel.injectCode(challenge, '123456');
+          expect(result.success).toBe(false);
+          expect(result.error).toBeDefined();
+          expect(result.error?.code).toBe('timeout');
+
+          // Clean up for next iteration
+          authTunnel.clear();
+
+          return true;
+        }
+      ),
+      { numRuns: 100 }
+    );
+  });
+
+  /**
+   * Additional property: Challenges within timeout window should be valid.
+   */
+  test('Property 11a: Challenges within timeout window remain valid', async () => {
+    await fc.assert(
+      fc.asyncProperty(
+        shadowSessionArb,
+        mfaTypeArb,
+        fc.integer({ min: 1000, max: MFA_TIMEOUT_MS - 1000 }), // Time within window
+        async (session, mfaType, timeWithinWindow) => {
+          const pageContent = await fc.sample(mfaPageContentArb(mfaType), 1)[0];
+
+          const challenge = await authTunnel.detectMFA(session, pageContent);
+          if (!challenge) return true;
+
+          await authTunnel.requestCode(challenge);
+
+          // Advance time but stay within timeout window
+          vi.advanceTimersByTime(timeWithinWindow);
+
+          // PROPERTY: Challenge should still be valid
+          expect(authTunnel.isTimedOut(challenge.id)).toBe(false);
+          expect(authTunnel.getRemainingTime(challenge.id)).toBeGreaterThan(0);
+
+          // Challenge should still be retrievable
+          const retrievedChallenge = authTunnel.getChallenge(challenge.id);
+          expect(retrievedChallenge).not.toBeNull();
+          expect(retrievedChallenge?.id).toBe(challenge.id);
+
+          authTunnel.clear();
+          return true;
+        }
+      ),
+      { numRuns: 100 }
+    );
+  });
+
+  /**
+   * Additional property: Successful code injection before timeout should work.
+   */
+  test('Property 11b: Code injection before timeout succeeds', async () => {
+    await fc.assert(
+      fc.asyncProperty(
+        shadowSessionArb,
+        mfaTypeArb,
+        validMfaCodeArb,
+        fc.integer({ min: 0, max: MFA_TIMEOUT_MS - 10000 }), // Time before timeout
+        async (session, mfaType, code, timeBeforeTimeout) => {
+          const pageContent = await fc.sample(mfaPageContentArb(mfaType), 1)[0];
+
+          const challenge = await authTunnel.detectMFA(session, pageContent);
+          if (!challenge) return true;
+
+          await authTunnel.requestCode(challenge);
+
+          // Advance time but stay before timeout
+          vi.advanceTimersByTime(timeBeforeTimeout);
+
+          // PROPERTY: Code injection should succeed before timeout
+          const result = await authTunnel.injectCode(challenge, code);
+          expect(result.success).toBe(true);
+          expect(result.error).toBeUndefined();
+
+          // Challenge should be removed after successful injection
+          expect(authTunnel.getChallenge(challenge.id)).toBeNull();
+
+          authTunnel.clear();
+          return true;
+        }
+      ),
+      { numRuns: 100 }
+    );
+  });
+});
+
+// =============================================================================
+// Unit Tests for MFA Detection
+// =============================================================================
+
+describe('Auth Tunnel Unit Tests', () => {
+  test('detects SMS MFA challenge', async () => {
+    const session: ShadowSession = {
+      id: 'shadow_test123',
+      domain: 'example.com',
+      status: 'active',
+      createdAt: Date.now(),
+      lastHeartbeat: Date.now(),
+      fingerprint: {
+        resolution: { width: 1920, height: 1080 },
+        userAgent: 'Mozilla/5.0',
+        fonts: ['Arial'],
+        gpuSignature: 'NVIDIA',
+        timezone: 'America/New_York',
+        language: 'en-US',
+      },
+    };
+
+    const pageContent = `
+      <html>
+        <body>
+          <h1>Two-Factor Authentication</h1>
+          <p>We sent a verification code to your phone via SMS.</p>
+          <p>Enter the 6-digit code below:</p>
+          <input type="text" name="sms_code" placeholder="Enter code" />
+          <button type="submit">Verify</button>
+        </body>
+      </html>
+    `;
+
+    const challenge = await authTunnel.detectMFA(session, pageContent);
+
+    expect(challenge).not.toBeNull();
+    expect(challenge?.type).toBe('sms');
+    expect(challenge?.sessionId).toBe(session.id);
+    expect(challenge?.expiresAt).toBeGreaterThan(Date.now());
+  });
+
+  test('detects authenticator MFA challenge', async () => {
+    const session: ShadowSession = {
+      id: 'shadow_test456',
+      domain: 'example.com',
+      status: 'active',
+      createdAt: Date.now(),
+      lastHeartbeat: Date.now(),
+      fingerprint: {
+        resolution: { width: 1920, height: 1080 },
+        userAgent: 'Mozilla/5.0',
+        fonts: ['Arial'],
+        gpuSignature: 'NVIDIA',
+        timezone: 'America/New_York',
+        language: 'en-US',
+      },
+    };
+
+    const pageContent = `
+      <html>
+        <body>
+          <h1>Two-Factor Authentication</h1>
+          <p>Open your Google Authenticator app and enter the 6-digit code.</p>
+          <input type="text" name="totp_code" />
+          <button type="submit">Verify</button>
+        </body>
+      </html>
+    `;
+
+    const challenge = await authTunnel.detectMFA(session, pageContent);
+
+    expect(challenge).not.toBeNull();
+    expect(challenge?.type).toBe('authenticator');
+  });
+
+  test('detects push notification MFA challenge', async () => {
+    const session: ShadowSession = {
+      id: 'shadow_test789',
+      domain: 'example.com',
+      status: 'active',
+      createdAt: Date.now(),
+      lastHeartbeat: Date.now(),
+      fingerprint: {
+        resolution: { width: 1920, height: 1080 },
+        userAgent: 'Mozilla/5.0',
+        fonts: ['Arial'],
+        gpuSignature: 'NVIDIA',
+        timezone: 'America/New_York',
+        language: 'en-US',
+      },
+    };
+
+    const pageContent = `
+      <html>
+        <body>
+          <h1>Approve Login</h1>
+          <p>We sent a push notification to your device.</p>
+          <p>Tap to approve the login request.</p>
+          <p>Waiting for approval...</p>
+        </body>
+      </html>
+    `;
+
+    const challenge = await authTunnel.detectMFA(session, pageContent);
+
+    expect(challenge).not.toBeNull();
+    expect(challenge?.type).toBe('push');
+  });
+
+  test('detects email MFA challenge', async () => {
+    const session: ShadowSession = {
+      id: 'shadow_testabc',
+      domain: 'example.com',
+      status: 'active',
+      createdAt: Date.now(),
+      lastHeartbeat: Date.now(),
+      fingerprint: {
+        resolution: { width: 1920, height: 1080 },
+        userAgent: 'Mozilla/5.0',
+        fonts: ['Arial'],
+        gpuSignature: 'NVIDIA',
+        timezone: 'America/New_York',
+        language: 'en-US',
+      },
+    };
+
+    const pageContent = `
+      <html>
+        <body>
+          <h1>Email Verification</h1>
+          <p>We sent a verification email to your inbox.</p>
+          <p>Check your email and enter the email code below:</p>
+          <input type="text" name="email_code" />
+          <button type="submit">Verify</button>
+        </body>
+      </html>
+    `;
+
+    const challenge = await authTunnel.detectMFA(session, pageContent);
+
+    expect(challenge).not.toBeNull();
+    expect(challenge?.type).toBe('email');
+  });
+
+  test('returns null for non-MFA pages', async () => {
+    const session: ShadowSession = {
+      id: 'shadow_testxyz',
+      domain: 'example.com',
+      status: 'active',
+      createdAt: Date.now(),
+      lastHeartbeat: Date.now(),
+      fingerprint: {
+        resolution: { width: 1920, height: 1080 },
+        userAgent: 'Mozilla/5.0',
+        fonts: ['Arial'],
+        gpuSignature: 'NVIDIA',
+        timezone: 'America/New_York',
+        language: 'en-US',
+      },
+    };
+
+    const pageContent = `
+      <html>
+        <body>
+          <h1>Welcome to Our Website</h1>
+          <p>Browse our products and services.</p>
+          <a href="/products">View Products</a>
+        </body>
+      </html>
+    `;
+
+    const challenge = await authTunnel.detectMFA(session, pageContent);
+
+    expect(challenge).toBeNull();
+  });
+
+  test('MFA timeout constant is 5 minutes', () => {
+    expect(MFA_TIMEOUT_MS).toBe(300000); // 5 minutes in milliseconds
+  });
+
+  test('challenge expiration is set correctly', async () => {
+    const session: ShadowSession = {
+      id: 'shadow_exptest',
+      domain: 'example.com',
+      status: 'active',
+      createdAt: Date.now(),
+      lastHeartbeat: Date.now(),
+      fingerprint: {
+        resolution: { width: 1920, height: 1080 },
+        userAgent: 'Mozilla/5.0',
+        fonts: ['Arial'],
+        gpuSignature: 'NVIDIA',
+        timezone: 'America/New_York',
+        language: 'en-US',
+      },
+    };
+
+    const pageContent = 'Enter the verification code sent via SMS';
+    const beforeDetect = Date.now();
+
+    const challenge = await authTunnel.detectMFA(session, pageContent);
+
+    expect(challenge).not.toBeNull();
+    // Expiration should be approximately 5 minutes from now
+    expect(challenge!.expiresAt).toBeGreaterThanOrEqual(beforeDetect + MFA_TIMEOUT_MS);
+    expect(challenge!.expiresAt).toBeLessThanOrEqual(Date.now() + MFA_TIMEOUT_MS + 1000);
+  });
+
+  test('invalid code format is rejected', async () => {
+    const session: ShadowSession = {
+      id: 'shadow_codetest',
+      domain: 'example.com',
+      status: 'active',
+      createdAt: Date.now(),
+      lastHeartbeat: Date.now(),
+      fingerprint: {
+        resolution: { width: 1920, height: 1080 },
+        userAgent: 'Mozilla/5.0',
+        fonts: ['Arial'],
+        gpuSignature: 'NVIDIA',
+        timezone: 'America/New_York',
+        language: 'en-US',
+      },
+    };
+
+    const pageContent = 'Enter the verification code sent via SMS';
+    const challenge = await authTunnel.detectMFA(session, pageContent);
+
+    expect(challenge).not.toBeNull();
+    await authTunnel.requestCode(challenge!);
+
+    // Try invalid code formats
+    const invalidCodes = ['', '   ', 'abc', '12', '12345678901'];
+
+    for (const invalidCode of invalidCodes) {
+      const result = await authTunnel.injectCode(challenge!, invalidCode);
+      expect(result.success).toBe(false);
+      expect(result.error?.code).toBe('mfa_failed');
+    }
+  });
+
+  test('challenge can be cancelled', async () => {
+    const session: ShadowSession = {
+      id: 'shadow_canceltest',
+      domain: 'example.com',
+      status: 'active',
+      createdAt: Date.now(),
+      lastHeartbeat: Date.now(),
+      fingerprint: {
+        resolution: { width: 1920, height: 1080 },
+        userAgent: 'Mozilla/5.0',
+        fonts: ['Arial'],
+        gpuSignature: 'NVIDIA',
+        timezone: 'America/New_York',
+        language: 'en-US',
+      },
+    };
+
+    const pageContent = 'Enter the verification code sent via SMS';
+    const challenge = await authTunnel.detectMFA(session, pageContent);
+
+    expect(challenge).not.toBeNull();
+    await authTunnel.requestCode(challenge!);
+
+    // Challenge should be pending
+    expect(authTunnel.getPendingChallengeCount()).toBe(1);
+
+    // Cancel the challenge
+    const cancelled = authTunnel.cancelChallenge(challenge!.id);
+    expect(cancelled).toBe(true);
+
+    // Challenge should be removed
+    expect(authTunnel.getPendingChallengeCount()).toBe(0);
+    expect(authTunnel.getChallenge(challenge!.id)).toBeNull();
+  });
+});