outcome-cli 1.0.0

package/src/modules/omnibridge/__tests__/parallel-executor.property.test.ts

@@ -0,0 +1,671 @@
+/**
+ * Parallel Executor Property Tests
+ *
+ * Property-based tests for competition orchestration, identical starting states,
+ * and crash isolation.
+ *
+ * Requirements: 7.2, 7.6
+ */
+
+import { describe, test, expect, beforeEach, afterEach } from 'vitest';
+import * as fc from 'fast-check';
+import {
+  ParallelExecutor,
+  createParallelExecutor,
+  type CompetitionConfig,
+  type AgentExecutionTask,
+  type AgentExecutionResult,
+} from '../execution/parallel-executor.js';
+import { createSessionVault } from '../auth/session-vault.js';
+import { createShadowSessionOrchestrator } from '../execution/shadow-session.js';
+import type { SerializedSession, EncryptedBlob } from '../core/types.js';
+
+// =============================================================================
+// Test Setup
+// =============================================================================
+
+let executor: ParallelExecutor;
+
+beforeEach(() => {
+  const vault = createSessionVault();
+  const orchestrator = createShadowSessionOrchestrator({
+    vault,
+    heartbeatIntervalMs: 60000,
+    sessionTimeoutMs: 3600000,
+  });
+
+  executor = createParallelExecutor({
+    sessionOrchestrator: orchestrator,
+    sessionVault: vault,
+    maxConcurrentCompetitions: 10,
+    defaultTimeoutMs: 5000,
+  });
+});
+
+afterEach(async () => {
+  await executor.clear();
+});
+
+// =============================================================================
+// Arbitraries
+// =============================================================================
+
+/**
+ * Generate arbitrary bounty IDs.
+ */
+const bountyIdArbitrary = fc.stringMatching(/^bounty_[a-z0-9]{8,16}$/);
+
+/**
+ * Generate arbitrary domain names.
+ */
+const domainArbitrary = fc.stringMatching(/^[a-z][a-z0-9-]{2,20}\.(com|org|net|io)$/);
+
+/**
+ * Generate arbitrary agent counts (reasonable range for testing).
+ */
+const agentCountArbitrary = fc.integer({ min: 2, max: 10 });
+
+/**
+ * Generate arbitrary agent IDs.
+ */
+const agentIdArbitrary = fc.stringMatching(/^agent_[a-z0-9]{8,16}$/);
+
+/**
+ * Generate arbitrary session data.
+ */
+const sessionDataArbitrary = fc.record({
+  cookies: fc.string({ minLength: 0, maxLength: 100 }),
+  localStorage: fc.string({ minLength: 0, maxLength: 100 }),
+  sessionStorage: fc.string({ minLength: 0, maxLength: 100 }),
+});
+
+// =============================================================================
+// Property 15: Identical Starting States
+// =============================================================================
+
+describe('Property 15: Identical Starting States', () => {
+  /**
+   * **Feature: omnibridge, Property 15: Identical Starting States**
+   *
+   * *For any* Competition with N agents, all N Shadow_Sessions SHALL have
+   * byte-identical `startState` serializations.
+   *
+   * **Validates: Requirements 7.2**
+   */
+  test(
+    'Property 15: all agents in a competition have byte-identical starting states',
+    async () => {
+      await fc.assert(
+        fc.asyncProperty(
+          bountyIdArbitrary,
+          domainArbitrary,
+          agentCountArbitrary,
+          async (bountyId, domain, agentCount) => {
+            const config: CompetitionConfig = {
+              bountyId,
+              agentCount,
+              targetDomain: domain,
+            };
+
+            const result = await executor.createCompetition(config);
+
+            expect(result.success).toBe(true);
+            expect(result.competition).toBeDefined();
+
+            const competition = result.competition!;
+
+            // Verify we have the correct number of sessions
+            expect(competition.sessions.length).toBe(agentCount);
+
+            // Get the serialized start state
+            const startState = competition.startState;
+            const startStateStr = JSON.stringify(startState);
+
+            // Verify all sessions would start with identical state
+            // The startState is shared across all sessions
+            for (let i = 0; i < competition.sessions.length; i++) {
+              // Each session should reference the same startState
+              const sessionStartState = executor.serializeStartState(startState);
+              const sessionStartStateStr = JSON.stringify(sessionStartState);
+
+              // Byte-identical check
+              expect(sessionStartStateStr).toBe(startStateStr);
+            }
+
+            // Verify the areStatesIdentical helper works correctly
+            for (let i = 0; i < competition.sessions.length - 1; i++) {
+              const state1 = executor.serializeStartState(startState);
+              const state2 = executor.serializeStartState(startState);
+              expect(executor.areStatesIdentical(state1, state2)).toBe(true);
+            }
+
+            // Clean up
+            await executor.cleanupCompetition(competition.id);
+
+            return true;
+          }
+        ),
+        { numRuns: 100 }
+      );
+    }
+  );
+
+  /**
+   * Custom initial state should be preserved identically for all agents.
+   */
+  test(
+    'custom initial state is byte-identical for all agents',
+    async () => {
+      const vault = createSessionVault();
+
+      await fc.assert(
+        fc.asyncProperty(
+          bountyIdArbitrary,
+          domainArbitrary,
+          agentCountArbitrary,
+          sessionDataArbitrary,
+          async (bountyId, domain, agentCount, sessionData) => {
+            // Create a custom initial state
+            const customState: SerializedSession = {
+              cookies: vault.encrypt(sessionData.cookies),
+              localStorage: vault.encrypt(sessionData.localStorage),
+              sessionStorage: vault.encrypt(sessionData.sessionStorage),
+              expiresAt: Date.now() + 86400000,
+            };
+
+            const config: CompetitionConfig = {
+              bountyId,
+              agentCount,
+              targetDomain: domain,
+              initialState: customState,
+            };
+
+            const result = await executor.createCompetition(config);
+
+            expect(result.success).toBe(true);
+            expect(result.competition).toBeDefined();
+
+            const competition = result.competition!;
+
+            // The start state should be a serialized copy of the custom state
+            const startStateStr = JSON.stringify(competition.startState);
+            const customStateStr = JSON.stringify(customState);
+
+            // Should be identical
+            expect(startStateStr).toBe(customStateStr);
+
+            // All serialized copies should be identical
+            for (let i = 0; i < agentCount; i++) {
+              const copy = executor.serializeStartState(competition.startState);
+              expect(JSON.stringify(copy)).toBe(startStateStr);
+            }
+
+            // Clean up
+            await executor.cleanupCompetition(competition.id);
+
+            return true;
+          }
+        ),
+        { numRuns: 100 }
+      );
+    }
+  );
+
+  /**
+   * Sessions in a competition should all be isolated from each other.
+   */
+  test(
+    'all sessions in a competition are isolated',
+    async () => {
+      await fc.assert(
+        fc.asyncProperty(
+          bountyIdArbitrary,
+          domainArbitrary,
+          agentCountArbitrary,
+          async (bountyId, domain, agentCount) => {
+            const config: CompetitionConfig = {
+              bountyId,
+              agentCount,
+              targetDomain: domain,
+            };
+
+            const result = await executor.createCompetition(config);
+
+            expect(result.success).toBe(true);
+            expect(result.competition).toBeDefined();
+
+            const competition = result.competition!;
+            const orchestrator = executor.getSessionOrchestrator();
+
+            // Check all pairs of sessions are isolated
+            for (let i = 0; i < competition.sessions.length; i++) {
+              for (let j = i + 1; j < competition.sessions.length; j++) {
+                const session1 = competition.sessions[i];
+                const session2 = competition.sessions[j];
+
+                expect(orchestrator.areSessionsIsolated(session1.id, session2.id)).toBe(true);
+              }
+            }
+
+            // Clean up
+            await executor.cleanupCompetition(competition.id);
+
+            return true;
+          }
+        ),
+        { numRuns: 100 }
+      );
+    }
+  );
+});
+
+// =============================================================================
+// Property 17: Crash Isolation
+// =============================================================================
+
+describe('Property 17: Crash Isolation', () => {
+  /**
+   * **Feature: omnibridge, Property 17: Crash Isolation**
+   *
+   * *For any* Competition where agent A's session crashes,
+   * all other agents' sessions SHALL continue executing and return results independently.
+   *
+   * **Validates: Requirements 7.6**
+   */
+  test(
+    'Property 17: crashed session does not affect other sessions',
+    async () => {
+      await fc.assert(
+        fc.asyncProperty(
+          bountyIdArbitrary,
+          domainArbitrary,
+          fc.integer({ min: 3, max: 8 }), // Need at least 3 agents to test isolation
+          fc.integer({ min: 0, max: 7 }), // Index of agent to crash
+          async (bountyId, domain, agentCount, crashIndexRaw) => {
+            const crashIndex = crashIndexRaw % agentCount;
+
+            const config: CompetitionConfig = {
+              bountyId,
+              agentCount,
+              targetDomain: domain,
+            };
+
+            const result = await executor.createCompetition(config);
+
+            expect(result.success).toBe(true);
+            expect(result.competition).toBeDefined();
+
+            const competition = result.competition!;
+
+            // Create execution tasks - one will crash
+            const tasks: AgentExecutionTask[] = competition.sessions.map((session, index) => ({
+              agentId: `agent_${index}`,
+              sessionId: session.id,
+              execute: async (): Promise<AgentExecutionResult> => {
+                if (index === crashIndex) {
+                  // Simulate a crash
+                  throw new Error('Simulated crash');
+                }
+
+                // Simulate successful execution
+                return {
+                  agentId: `agent_${index}`,
+                  sessionId: session.id,
+                  status: 'completed',
+                  actionLog: [],
+                  executionTimeMs: 100,
+                  result: {
+                    data: { success: true },
+                    metadata: {
+                      confidence: 1.0,
+                      executionTimeMs: 100,
+                      actionsPerformed: 1,
+                      triangulationHeals: 0,
+                    },
+                    verificationHash: 'test_hash',
+                  },
+                };
+              },
+            }));
+
+            // Track results
+            const completedResults: string[] = [];
+            const failedResults: string[] = [];
+
+            // Execute all tasks
+            await executor.startAll(competition, tasks, {
+              timeoutMs: 5000,
+              onAgentComplete: (agentResult) => {
+                if (agentResult.status === 'completed') {
+                  completedResults.push(agentResult.agentId);
+                } else {
+                  failedResults.push(agentResult.agentId);
+                }
+              },
+            });
+
+            // Verify crash isolation:
+            // 1. The crashed agent should be marked as failed
+            expect(failedResults).toContain(`agent_${crashIndex}`);
+
+            // 2. All other agents should have completed successfully
+            for (let i = 0; i < agentCount; i++) {
+              if (i !== crashIndex) {
+                expect(completedResults).toContain(`agent_${i}`);
+              }
+            }
+
+            // 3. Total completed should be agentCount - 1
+            expect(completedResults.length).toBe(agentCount - 1);
+
+            // 4. Other sessions should not be affected by the crash
+            expect(executor.areOtherSessionsAffected(competition.id, competition.sessions[crashIndex].id)).toBe(false);
+
+            // Clean up
+            await executor.cleanupCompetition(competition.id);
+
+            return true;
+          }
+        ),
+        { numRuns: 100 }
+      );
+    }
+  );
+
+  /**
+   * Multiple crashes should not cascade to other agents.
+   */
+  test(
+    'multiple crashes do not cascade to other agents',
+    async () => {
+      await fc.assert(
+        fc.asyncProperty(
+          bountyIdArbitrary,
+          domainArbitrary,
+          fc.integer({ min: 5, max: 10 }), // Need enough agents
+          fc.integer({ min: 1, max: 3 }), // Number of crashes
+          async (bountyId, domain, agentCount, crashCount) => {
+            // Ensure we don't crash more agents than we have
+            const actualCrashCount = Math.min(crashCount, agentCount - 1);
+
+            const config: CompetitionConfig = {
+              bountyId,
+              agentCount,
+              targetDomain: domain,
+            };
+
+            const result = await executor.createCompetition(config);
+
+            expect(result.success).toBe(true);
+            expect(result.competition).toBeDefined();
+
+            const competition = result.competition!;
+
+            // Determine which agents will crash
+            const crashIndices = new Set<number>();
+            for (let i = 0; i < actualCrashCount; i++) {
+              crashIndices.add(i);
+            }
+
+            // Create execution tasks
+            const tasks: AgentExecutionTask[] = competition.sessions.map((session, index) => ({
+              agentId: `agent_${index}`,
+              sessionId: session.id,
+              execute: async (): Promise<AgentExecutionResult> => {
+                if (crashIndices.has(index)) {
+                  throw new Error(`Simulated crash for agent ${index}`);
+                }
+
+                return {
+                  agentId: `agent_${index}`,
+                  sessionId: session.id,
+                  status: 'completed',
+                  actionLog: [],
+                  executionTimeMs: 50,
+                  result: {
+                    data: { success: true },
+                    metadata: {
+                      confidence: 1.0,
+                      executionTimeMs: 50,
+                      actionsPerformed: 1,
+                      triangulationHeals: 0,
+                    },
+                    verificationHash: 'test_hash',
+                  },
+                };
+              },
+            }));
+
+            let completedCount = 0;
+            let failedCount = 0;
+
+            await executor.startAll(competition, tasks, {
+              timeoutMs: 5000,
+              onAgentComplete: (agentResult) => {
+                if (agentResult.status === 'completed') {
+                  completedCount++;
+                } else {
+                  failedCount++;
+                }
+              },
+            });
+
+            // Verify:
+            // 1. Exactly crashCount agents failed
+            expect(failedCount).toBe(actualCrashCount);
+
+            // 2. All other agents completed successfully
+            expect(completedCount).toBe(agentCount - actualCrashCount);
+
+            // Clean up
+            await executor.cleanupCompetition(competition.id);
+
+            return true;
+          }
+        ),
+        { numRuns: 100 }
+      );
+    }
+  );
+
+  /**
+   * Timeout should be treated as a crash and isolated.
+   */
+  test(
+    'timeout is isolated like a crash',
+    { timeout: 30000 },
+    async () => {
+      await fc.assert(
+        fc.asyncProperty(
+          bountyIdArbitrary,
+          domainArbitrary,
+          fc.integer({ min: 3, max: 6 }),
+          async (bountyId, domain, agentCount) => {
+            const config: CompetitionConfig = {
+              bountyId,
+              agentCount,
+              targetDomain: domain,
+            };
+
+            const result = await executor.createCompetition(config);
+
+            expect(result.success).toBe(true);
+            expect(result.competition).toBeDefined();
+
+            const competition = result.competition!;
+
+            // First agent will timeout, others complete quickly
+            const tasks: AgentExecutionTask[] = competition.sessions.map((session, index) => ({
+              agentId: `agent_${index}`,
+              sessionId: session.id,
+              execute: async (): Promise<AgentExecutionResult> => {
+                if (index === 0) {
+                  // Simulate a long-running task that will timeout
+                  await new Promise((resolve) => setTimeout(resolve, 500));
+                }
+
+                return {
+                  agentId: `agent_${index}`,
+                  sessionId: session.id,
+                  status: 'completed',
+                  actionLog: [],
+                  executionTimeMs: 10,
+                  result: {
+                    data: { success: true },
+                    metadata: {
+                      confidence: 1.0,
+                      executionTimeMs: 10,
+                      actionsPerformed: 1,
+                      triangulationHeals: 0,
+                    },
+                    verificationHash: 'test_hash',
+                  },
+                };
+              },
+            }));
+
+            let completedCount = 0;
+            let timeoutCount = 0;
+
+            await executor.startAll(competition, tasks, {
+              timeoutMs: 50, // Very short timeout to trigger timeout quickly
+              onAgentComplete: (agentResult) => {
+                if (agentResult.status === 'completed') {
+                  completedCount++;
+                } else if (agentResult.status === 'timeout') {
+                  timeoutCount++;
+                }
+              },
+            });
+
+            // Agent 0 should timeout
+            expect(timeoutCount).toBe(1);
+
+            // All other agents should complete
+            expect(completedCount).toBe(agentCount - 1);
+
+            // Clean up
+            await executor.cleanupCompetition(competition.id);
+
+            return true;
+          }
+        ),
+        { numRuns: 20 } // Reduced runs for timeout test
+      );
+    }
+  );
+});
+
+// =============================================================================
+// Additional Property Tests
+// =============================================================================
+
+describe('Competition Creation', () => {
+  /**
+   * Competition should create the correct number of sessions.
+   */
+  test(
+    'competition creates correct number of sessions',
+    async () => {
+      await fc.assert(
+        fc.asyncProperty(
+          bountyIdArbitrary,
+          domainArbitrary,
+          agentCountArbitrary,
+          async (bountyId, domain, agentCount) => {
+            const config: CompetitionConfig = {
+              bountyId,
+              agentCount,
+              targetDomain: domain,
+            };
+
+            const result = await executor.createCompetition(config);
+
+            expect(result.success).toBe(true);
+            expect(result.competition).toBeDefined();
+            expect(result.competition!.sessions.length).toBe(agentCount);
+
+            // Clean up
+            await executor.cleanupCompetition(result.competition!.id);
+
+            return true;
+          }
+        ),
+        { numRuns: 100 }
+      );
+    }
+  );
+
+  /**
+   * Competition with custom agent IDs should use those IDs.
+   */
+  test(
+    'competition uses provided agent IDs',
+    async () => {
+      await fc.assert(
+        fc.asyncProperty(
+          bountyIdArbitrary,
+          domainArbitrary,
+          fc.array(agentIdArbitrary, { minLength: 2, maxLength: 5 }),
+          async (bountyId, domain, agentIds) => {
+            // Ensure unique agent IDs
+            const uniqueAgentIds = [...new Set(agentIds)];
+            if (uniqueAgentIds.length < 2) {
+              return true; // Skip if not enough unique IDs
+            }
+
+            const config: CompetitionConfig = {
+              bountyId,
+              agentCount: uniqueAgentIds.length,
+              targetDomain: domain,
+              agentIds: uniqueAgentIds,
+            };
+
+            const result = await executor.createCompetition(config);
+
+            expect(result.success).toBe(true);
+            expect(result.competition).toBeDefined();
+            expect(result.competition!.sessions.length).toBe(uniqueAgentIds.length);
+
+            // Clean up
+            await executor.cleanupCompetition(result.competition!.id);
+
+            return true;
+          }
+        ),
+        { numRuns: 100 }
+      );
+    }
+  );
+
+  /**
+   * Invalid agent count should fail.
+   */
+  test(
+    'invalid agent count fails gracefully',
+    async () => {
+      await fc.assert(
+        fc.asyncProperty(
+          bountyIdArbitrary,
+          domainArbitrary,
+          fc.integer({ min: -10, max: 0 }),
+          async (bountyId, domain, invalidCount) => {
+            const config: CompetitionConfig = {
+              bountyId,
+              agentCount: invalidCount,
+              targetDomain: domain,
+            };
+
+            const result = await executor.createCompetition(config);
+
+            expect(result.success).toBe(false);
+            expect(result.error).toBeDefined();
+
+            return true;
+          }
+        ),
+        { numRuns: 100 }
+      );
+    }
+  );
+});