@elizaos/training 2.0.0-alpha.10

package/src/benchmark/__tests__/HeadToHead.test.ts

@@ -0,0 +1,126 @@
+import { describe, expect, it } from 'bun:test';
+import type { BenchmarkGameSnapshot } from '../BenchmarkDataGenerator';
+import { MetricsVisualizer } from '../MetricsVisualizer';
+import { SimulationEngine, type SimulationResult } from '../SimulationEngine';
+
+describe('Head-to-Head Benchmark Infrastructure', () => {
+  // 1. Test Simulation Engine PnL History Tracking
+  describe('SimulationEngine PnL History', () => {
+    it('should initialize with empty pnlHistory and return it after run()', async () => {
+      const mockSnapshot = {
+        id: 'test',
+        ticks: [],
+        initialState: {
+          predictionMarkets: [],
+          perpetualMarkets: [],
+          agents: [],
+        },
+        groundTruth: {
+          marketOutcomes: {},
+          priceHistory: {},
+          optimalActions: [],
+        },
+      } as unknown as BenchmarkGameSnapshot;
+
+      const engine = new SimulationEngine({
+        snapshot: mockSnapshot,
+        agentId: 'test-agent',
+        fastForward: true,
+      });
+
+      engine.initialize();
+      // Use public API - run() returns pnlHistory
+      const result = await engine.run();
+      expect(result.pnlHistory).toEqual([]);
+    });
+  });
+
+  // 2. Test MetricsVisualizer Logic
+  describe('MetricsVisualizer Comparison Logic', () => {
+    // Mock Result Helper
+    const createMockResult = (
+      id: string,
+      pnl: number,
+      history: number[]
+    ): SimulationResult => ({
+      id,
+      agentId: id,
+      benchmarkId: 'bench-1',
+      startTime: 0,
+      endTime: 1000,
+      ticksProcessed: history.length,
+      actions: [],
+      metrics: {
+        totalPnl: pnl,
+        predictionMetrics: {
+          accuracy: 0.5,
+          totalPositions: 0,
+          correctPredictions: 0,
+          incorrectPredictions: 0,
+          avgPnlPerPosition: 0,
+        },
+        perpMetrics: {
+          winRate: 0.5,
+          totalTrades: 0,
+          profitableTrades: 0,
+          avgPnlPerTrade: 0,
+          maxDrawdown: 0,
+        },
+        socialMetrics: {
+          postsCreated: 0,
+          groupsJoined: 0,
+          messagesReceived: 0,
+          reputationGained: 0,
+        },
+        timing: { totalDuration: 0, avgResponseTime: 0, maxResponseTime: 0 },
+        optimalityScore: 50,
+      },
+      trajectory: { states: [], actions: [], rewards: [], windowId: '' },
+      pnlHistory: history.map((val, idx) => ({ tick: idx, pnl: val })),
+    });
+
+    it('should correctly merge PnL histories of equal length', () => {
+      const baseline = createMockResult('baseline', 100, [10, 50, 100]);
+      const challenger = createMockResult('challenger', 200, [20, 100, 200]);
+
+      // Use public static method
+      const history = MetricsVisualizer.mergePnlHistory(baseline, challenger);
+
+      expect(history).toHaveLength(3);
+      expect(history[2]).toEqual({ tick: 2, baseline: 100, challenger: 200 });
+    });
+
+    it('should handle unequal history lengths (fill with final value)', () => {
+      // Baseline died early (e.g., bankruptcy or crash)
+      const baseline = createMockResult('baseline', -50, [10, -50]);
+      // Challenger kept going
+      const challenger = createMockResult('challenger', 100, [20, 60, 80, 100]);
+
+      const history = MetricsVisualizer.mergePnlHistory(baseline, challenger);
+
+      expect(history).toHaveLength(4); // Should match longest
+      // Tick 0
+      expect(history[0]).toEqual({ tick: 0, baseline: 10, challenger: 20 });
+      // Tick 1
+      expect(history[1]).toEqual({ tick: 1, baseline: -50, challenger: 60 });
+      // Tick 2 (Baseline stopped, should carry over -50)
+      expect(history[2]).toEqual({ tick: 2, baseline: -50, challenger: 80 });
+      // Tick 3
+      expect(history[3]).toEqual({ tick: 3, baseline: -50, challenger: 100 });
+    });
+
+    it('should generate ASCII chart string', () => {
+      const baseline = createMockResult('baseline', 100, [10, 100]);
+      const challenger = createMockResult('challenger', 200, [20, 200]);
+
+      const chart = MetricsVisualizer.generateAsciiComparison(
+        baseline,
+        challenger
+      );
+
+      expect(chart).toContain('HEAD-TO-HEAD RESULTS');
+      expect(chart).toContain('WINNER: Challenger');
+      expect(chart).toContain('Alpha Generated: +$100.00');
+    });
+  });
+});

package/src/benchmark/index.ts

@@ -0,0 +1,91 @@
+/**
+ * Benchmark Module
+ *
+ * Tools for evaluating agent performance through simulation.
+ */
+
+// Multi-archetype matchup benchmarking
+export type {
+  ArchetypeVsResult,
+  MatchupAgent,
+  MatchupAgentResult,
+  MatchupBenchmarkConfig,
+  MatchupBenchmarkResult,
+} from './ArchetypeMatchupBenchmark';
+export {
+  ArchetypeMatchupBenchmark,
+  runQuickMatchupBenchmark,
+} from './ArchetypeMatchupBenchmark';
+export type {
+  BenchmarkHistoryEntry,
+  ModelComparisonData,
+} from './BenchmarkChartGenerator';
+export { BenchmarkChartGenerator } from './BenchmarkChartGenerator';
+export type {
+  BenchmarkConfig,
+  BenchmarkGameSnapshot,
+  CausalEventType,
+  GroundTruth,
+  HiddenNarrativeFact,
+  ScheduledCausalEvent,
+  VolatilityBucket,
+} from './BenchmarkDataGenerator';
+export { BenchmarkDataGenerator, SeededRandom } from './BenchmarkDataGenerator';
+export { BenchmarkDataViewer } from './BenchmarkDataViewer';
+export type {
+  BenchmarkHistoryQuery,
+  BenchmarkResultInput,
+  BenchmarkTrendData,
+} from './BenchmarkHistoryService';
+export { BenchmarkHistoryService } from './BenchmarkHistoryService';
+export type {
+  BenchmarkComparisonResult,
+  BenchmarkRunConfig,
+} from './BenchmarkRunner';
+export { BenchmarkRunner } from './BenchmarkRunner';
+export * as BenchmarkValidator from './BenchmarkValidator';
+export type { FastEvalConfig, FastEvalResult } from './FastEvalRunner';
+export { FastEvalRunner } from './FastEvalRunner';
+export { MetricsValidator } from './MetricsValidator';
+export { MetricsVisualizer } from './MetricsVisualizer';
+export type {
+  AverageMetrics,
+  ModelBenchmarkOptions,
+  ModelBenchmarkResult,
+  ModelComparisonResult,
+} from './ModelBenchmarkService';
+export { ModelBenchmarkService } from './ModelBenchmarkService';
+export type { ModelConfig } from './ModelRegistry';
+export {
+  getBaselineModels,
+  getModelById,
+  getModelByModelId,
+  getModelDisplayName,
+  getModelsByProvider,
+  getModelsByTier,
+  MODEL_REGISTRY,
+  validateModelId,
+} from './ModelRegistry';
+// Shared utilities
+export {
+  type JsonValue,
+  parseSimulationMetrics,
+} from './parseSimulationMetrics';
+export {
+  createRulerContext,
+  extractMarketOutcomesFromBenchmark,
+  getHiddenEventsForTick,
+  getHiddenFactsForTick,
+  getTrueFacts,
+  scoreActionAgainstGroundTruth,
+  wasDecisionOptimal,
+} from './RulerBenchmarkIntegration';
+export { SimulationA2AInterface } from './SimulationA2AInterface';
+export type {
+  SimulationConfig,
+  SimulationMetrics,
+  SimulationResult,
+} from './SimulationEngine';
+export { SimulationEngine } from './SimulationEngine';
+export type { TaskRunnerConfig, TaskRunResult } from './TaskRunner';
+export { TaskRunner } from './TaskRunner';

package/src/benchmark/parseSimulationMetrics.ts

@@ -0,0 +1,124 @@
+/**
+ * Simulation Metrics Parser
+ *
+ * Shared utility for validating and parsing SimulationMetrics from JSON data.
+ * Used by ModelBenchmarkService and HuggingFaceModelUploader.
+ */
+
+import type { SimulationMetrics } from './SimulationEngine';
+
+/**
+ * JSON value type for parsing untyped data
+ */
+export type JsonValue =
+  | string
+  | number
+  | boolean
+  | null
+  | undefined
+  | JsonValue[]
+  | { [key: string]: JsonValue };
+
+/**
+ * Parse and validate SimulationMetrics from JSON data
+ *
+ * @param data - Raw JSON data to parse
+ * @returns Validated SimulationMetrics object
+ * @throws Error if data is invalid or missing required fields
+ */
+export function parseSimulationMetrics(data: JsonValue): SimulationMetrics {
+  if (typeof data !== 'object' || data === null) {
+    throw new Error('Invalid SimulationMetrics: expected object');
+  }
+
+  const metrics = data as Record<string, JsonValue>;
+
+  // Validate required fields
+  if (typeof metrics.totalPnl !== 'number') {
+    throw new Error('Invalid SimulationMetrics: totalPnl must be a number');
+  }
+
+  if (
+    typeof metrics.predictionMetrics !== 'object' ||
+    metrics.predictionMetrics === null
+  ) {
+    throw new Error(
+      'Invalid SimulationMetrics: predictionMetrics must be an object'
+    );
+  }
+
+  if (typeof metrics.perpMetrics !== 'object' || metrics.perpMetrics === null) {
+    throw new Error('Invalid SimulationMetrics: perpMetrics must be an object');
+  }
+
+  if (typeof metrics.optimalityScore !== 'number') {
+    throw new Error(
+      'Invalid SimulationMetrics: optimalityScore must be a number'
+    );
+  }
+
+  if (typeof metrics.timing !== 'object' || metrics.timing === null) {
+    throw new Error('Invalid SimulationMetrics: timing must be an object');
+  }
+
+  // Validate nested structures
+  const predictionMetrics = metrics.predictionMetrics as Record<
+    string,
+    JsonValue
+  >;
+  const perpMetrics = metrics.perpMetrics as Record<string, JsonValue>;
+  const timing = metrics.timing as Record<string, JsonValue>;
+
+  // Helper to safely get number or default
+  const getNumber = (obj: Record<string, JsonValue>, key: string): number => {
+    const val = obj[key];
+    return typeof val === 'number' ? val : 0;
+  };
+
+  // Parse socialMetrics if present
+  const socialMetricsData = metrics.socialMetrics;
+  const socialMetrics =
+    typeof socialMetricsData === 'object' && socialMetricsData !== null
+      ? (socialMetricsData as Record<string, JsonValue>)
+      : null;
+
+  return {
+    totalPnl: metrics.totalPnl as number,
+    predictionMetrics: {
+      totalPositions: getNumber(predictionMetrics, 'totalPositions'),
+      correctPredictions: getNumber(predictionMetrics, 'correctPredictions'),
+      incorrectPredictions: getNumber(
+        predictionMetrics,
+        'incorrectPredictions'
+      ),
+      accuracy: getNumber(predictionMetrics, 'accuracy'),
+      avgPnlPerPosition: getNumber(predictionMetrics, 'avgPnlPerPosition'),
+    },
+    perpMetrics: {
+      totalTrades: getNumber(perpMetrics, 'totalTrades'),
+      profitableTrades: getNumber(perpMetrics, 'profitableTrades'),
+      winRate: getNumber(perpMetrics, 'winRate'),
+      avgPnlPerTrade: getNumber(perpMetrics, 'avgPnlPerTrade'),
+      maxDrawdown: getNumber(perpMetrics, 'maxDrawdown'),
+    },
+    socialMetrics: socialMetrics
+      ? {
+          postsCreated: getNumber(socialMetrics, 'postsCreated'),
+          groupsJoined: getNumber(socialMetrics, 'groupsJoined'),
+          messagesReceived: getNumber(socialMetrics, 'messagesReceived'),
+          reputationGained: getNumber(socialMetrics, 'reputationGained'),
+        }
+      : {
+          postsCreated: 0,
+          groupsJoined: 0,
+          messagesReceived: 0,
+          reputationGained: 0,
+        },
+    timing: {
+      avgResponseTime: getNumber(timing, 'avgResponseTime'),
+      maxResponseTime: getNumber(timing, 'maxResponseTime'),
+      totalDuration: getNumber(timing, 'totalDuration'),
+    },
+    optimalityScore: metrics.optimalityScore as number,
+  };
+}

package/src/benchmark/simulation-types.ts

@@ -0,0 +1,78 @@
+import type { JsonValue } from '../adapter';
+
+export type AgentActionType =
+  | 'query_state'
+  | 'buy_prediction'
+  | 'sell_prediction'
+  | 'open_perp'
+  | 'close_perp'
+  | 'create_post'
+  | 'join_group'
+  | 'send_message';
+
+export interface AgentAction {
+  tick: number;
+  timestamp: number;
+  type: AgentActionType;
+  data: Record<string, JsonValue>;
+  /** How long agent took to respond in milliseconds */
+  duration: number;
+  correctness?: {
+    /** Prediction market correctness tracking */
+    predictionCorrect?: boolean;
+    actualOutcome?: boolean;
+    predictedOutcome?: boolean;
+
+    /** Perpetual trade correctness tracking */
+    perpCorrect?: boolean;
+    sentimentAtTrade?: number;
+    priceChange?: number;
+    expectedDirection?: 'up' | 'down';
+
+    /** Sentiment analysis accuracy tracking */
+    sentimentAccuracy?: number;
+    sentimentAtTime?: number;
+    actualSentiment?: number;
+  };
+}
+
+export interface SimulationMetrics {
+  /** Total P&L from all positions */
+  totalPnl: number;
+
+  /** Prediction market metrics */
+  predictionMetrics: {
+    totalPositions: number;
+    correctPredictions: number;
+    incorrectPredictions: number;
+    accuracy: number;
+    avgPnlPerPosition: number;
+  };
+
+  /** Perpetual trading metrics */
+  perpMetrics: {
+    totalTrades: number;
+    profitableTrades: number;
+    winRate: number;
+    avgPnlPerTrade: number;
+    maxDrawdown: number;
+  };
+
+  /** Social metrics */
+  socialMetrics: {
+    postsCreated: number;
+    groupsJoined: number;
+    messagesReceived: number;
+    reputationGained: number;
+  };
+
+  /** Timing metrics */
+  timing: {
+    avgResponseTime: number;
+    maxResponseTime: number;
+    totalDuration: number;
+  };
+
+  /** Compared to optimal actions */
+  optimalityScore: number; // 0-100, how close to optimal
+}