@elizaos/training 2.0.0-alpha.11

package/src/benchmark/index.ts

@@ -0,0 +1,89 @@
+/**
+ * Benchmark Module
+ *
+ * Tools for evaluating agent performance through simulation.
+ */
+
+// Multi-archetype matchup benchmarking
+export type {
+  ArchetypeVsResult,
+  MatchupAgent,
+  MatchupAgentResult,
+  MatchupBenchmarkConfig,
+  MatchupBenchmarkResult,
+} from './ArchetypeMatchupBenchmark';
+export {
+  ArchetypeMatchupBenchmark,
+  runQuickMatchupBenchmark,
+} from './ArchetypeMatchupBenchmark';
+export type {
+  BenchmarkHistoryEntry,
+  ModelComparisonData,
+} from './BenchmarkChartGenerator';
+export { BenchmarkChartGenerator } from './BenchmarkChartGenerator';
+export type {
+  BenchmarkConfig,
+  BenchmarkGameSnapshot,
+  CausalEventType,
+  GroundTruth,
+  HiddenNarrativeFact,
+  ScheduledCausalEvent,
+  VolatilityBucket,
+} from './BenchmarkDataGenerator';
+export { BenchmarkDataGenerator, SeededRandom } from './BenchmarkDataGenerator';
+export { BenchmarkDataViewer } from './BenchmarkDataViewer';
+export type {
+  BenchmarkHistoryQuery,
+  BenchmarkResultInput,
+  BenchmarkTrendData,
+} from './BenchmarkHistoryService';
+export { BenchmarkHistoryService } from './BenchmarkHistoryService';
+export type {
+  BenchmarkComparisonResult,
+  BenchmarkRunConfig,
+} from './BenchmarkRunner';
+export { BenchmarkRunner } from './BenchmarkRunner';
+export { BenchmarkValidator } from './BenchmarkValidator';
+export type { FastEvalConfig, FastEvalResult } from './FastEvalRunner';
+export { FastEvalRunner } from './FastEvalRunner';
+export { MetricsValidator } from './MetricsValidator';
+export { MetricsVisualizer } from './MetricsVisualizer';
+export type {
+  AverageMetrics,
+  ModelBenchmarkOptions,
+  ModelBenchmarkResult,
+  ModelComparisonResult,
+} from './ModelBenchmarkService';
+export { ModelBenchmarkService } from './ModelBenchmarkService';
+export type { ModelConfig } from './ModelRegistry';
+export {
+  getBaselineModels,
+  getModelById,
+  getModelByModelId,
+  getModelDisplayName,
+  getModelsByProvider,
+  getModelsByTier,
+  MODEL_REGISTRY,
+  validateModelId,
+} from './ModelRegistry';
+// Shared utilities
+export {
+  type JsonValue,
+  parseSimulationMetrics,
+} from './parseSimulationMetrics';
+export {
+  createRulerContext,
+  extractMarketOutcomesFromBenchmark,
+  getHiddenEventsForTick,
+  getHiddenFactsForTick,
+  getTrueFacts,
+  scoreActionAgainstGroundTruth,
+  wasDecisionOptimal,
+} from './RulerBenchmarkIntegration';
+export { SimulationA2AInterface } from './SimulationA2AInterface';
+export type {
+  SimulationConfig,
+  SimulationMetrics,
+  SimulationResult,
+} from './SimulationEngine';
+export { SimulationEngine } from './SimulationEngine';

package/src/benchmark/parseSimulationMetrics.ts

@@ -0,0 +1,124 @@
+/**
+ * Simulation Metrics Parser
+ *
+ * Shared utility for validating and parsing SimulationMetrics from JSON data.
+ * Used by ModelBenchmarkService and HuggingFaceModelUploader.
+ */
+
+import type { SimulationMetrics } from './SimulationEngine';
+
+/**
+ * JSON value type for parsing untyped data
+ */
+export type JsonValue =
+  | string
+  | number
+  | boolean
+  | null
+  | undefined
+  | JsonValue[]
+  | { [key: string]: JsonValue };
+
+/**
+ * Parse and validate SimulationMetrics from JSON data
+ *
+ * @param data - Raw JSON data to parse
+ * @returns Validated SimulationMetrics object
+ * @throws Error if data is invalid or missing required fields
+ */
+export function parseSimulationMetrics(data: JsonValue): SimulationMetrics {
+  if (typeof data !== 'object' || data === null) {
+    throw new Error('Invalid SimulationMetrics: expected object');
+  }
+
+  const metrics = data as Record<string, JsonValue>;
+
+  // Validate required fields
+  if (typeof metrics.totalPnl !== 'number') {
+    throw new Error('Invalid SimulationMetrics: totalPnl must be a number');
+  }
+
+  if (
+    typeof metrics.predictionMetrics !== 'object' ||
+    metrics.predictionMetrics === null
+  ) {
+    throw new Error(
+      'Invalid SimulationMetrics: predictionMetrics must be an object'
+    );
+  }
+
+  if (typeof metrics.perpMetrics !== 'object' || metrics.perpMetrics === null) {
+    throw new Error('Invalid SimulationMetrics: perpMetrics must be an object');
+  }
+
+  if (typeof metrics.optimalityScore !== 'number') {
+    throw new Error(
+      'Invalid SimulationMetrics: optimalityScore must be a number'
+    );
+  }
+
+  if (typeof metrics.timing !== 'object' || metrics.timing === null) {
+    throw new Error('Invalid SimulationMetrics: timing must be an object');
+  }
+
+  // Validate nested structures
+  const predictionMetrics = metrics.predictionMetrics as Record<
+    string,
+    JsonValue
+  >;
+  const perpMetrics = metrics.perpMetrics as Record<string, JsonValue>;
+  const timing = metrics.timing as Record<string, JsonValue>;
+
+  // Helper to safely get number or default
+  const getNumber = (obj: Record<string, JsonValue>, key: string): number => {
+    const val = obj[key];
+    return typeof val === 'number' ? val : 0;
+  };
+
+  // Parse socialMetrics if present
+  const socialMetricsData = metrics.socialMetrics;
+  const socialMetrics =
+    typeof socialMetricsData === 'object' && socialMetricsData !== null
+      ? (socialMetricsData as Record<string, JsonValue>)
+      : null;
+
+  return {
+    totalPnl: metrics.totalPnl as number,
+    predictionMetrics: {
+      totalPositions: getNumber(predictionMetrics, 'totalPositions'),
+      correctPredictions: getNumber(predictionMetrics, 'correctPredictions'),
+      incorrectPredictions: getNumber(
+        predictionMetrics,
+        'incorrectPredictions'
+      ),
+      accuracy: getNumber(predictionMetrics, 'accuracy'),
+      avgPnlPerPosition: getNumber(predictionMetrics, 'avgPnlPerPosition'),
+    },
+    perpMetrics: {
+      totalTrades: getNumber(perpMetrics, 'totalTrades'),
+      profitableTrades: getNumber(perpMetrics, 'profitableTrades'),
+      winRate: getNumber(perpMetrics, 'winRate'),
+      avgPnlPerTrade: getNumber(perpMetrics, 'avgPnlPerTrade'),
+      maxDrawdown: getNumber(perpMetrics, 'maxDrawdown'),
+    },
+    socialMetrics: socialMetrics
+      ? {
+          postsCreated: getNumber(socialMetrics, 'postsCreated'),
+          groupsJoined: getNumber(socialMetrics, 'groupsJoined'),
+          messagesReceived: getNumber(socialMetrics, 'messagesReceived'),
+          reputationGained: getNumber(socialMetrics, 'reputationGained'),
+        }
+      : {
+          postsCreated: 0,
+          groupsJoined: 0,
+          messagesReceived: 0,
+          reputationGained: 0,
+        },
+    timing: {
+      avgResponseTime: getNumber(timing, 'avgResponseTime'),
+      maxResponseTime: getNumber(timing, 'maxResponseTime'),
+      totalDuration: getNumber(timing, 'totalDuration'),
+    },
+    optimalityScore: metrics.optimalityScore as number,
+  };
+}

package/src/benchmark/simulation-types.ts

@@ -0,0 +1,78 @@
+import type { JsonValue } from '../adapter';
+
+export type AgentActionType =
+  | 'query_state'
+  | 'buy_prediction'
+  | 'sell_prediction'
+  | 'open_perp'
+  | 'close_perp'
+  | 'create_post'
+  | 'join_group'
+  | 'send_message';
+
+export interface AgentAction {
+  tick: number;
+  timestamp: number;
+  type: AgentActionType;
+  data: Record<string, JsonValue>;
+  /** How long agent took to respond in milliseconds */
+  duration: number;
+  correctness?: {
+    /** Prediction market correctness tracking */
+    predictionCorrect?: boolean;
+    actualOutcome?: boolean;
+    predictedOutcome?: boolean;
+
+    /** Perpetual trade correctness tracking */
+    perpCorrect?: boolean;
+    sentimentAtTrade?: number;
+    priceChange?: number;
+    expectedDirection?: 'up' | 'down';
+
+    /** Sentiment analysis accuracy tracking */
+    sentimentAccuracy?: number;
+    sentimentAtTime?: number;
+    actualSentiment?: number;
+  };
+}
+
+export interface SimulationMetrics {
+  /** Total P&L from all positions */
+  totalPnl: number;
+
+  /** Prediction market metrics */
+  predictionMetrics: {
+    totalPositions: number;
+    correctPredictions: number;
+    incorrectPredictions: number;
+    accuracy: number;
+    avgPnlPerPosition: number;
+  };
+
+  /** Perpetual trading metrics */
+  perpMetrics: {
+    totalTrades: number;
+    profitableTrades: number;
+    winRate: number;
+    avgPnlPerTrade: number;
+    maxDrawdown: number;
+  };
+
+  /** Social metrics */
+  socialMetrics: {
+    postsCreated: number;
+    groupsJoined: number;
+    messagesReceived: number;
+    reputationGained: number;
+  };
+
+  /** Timing metrics */
+  timing: {
+    avgResponseTime: number;
+    maxResponseTime: number;
+    totalDuration: number;
+  };
+
+  /** Compared to optimal actions */
+  optimalityScore: number; // 0-100, how close to optimal
+}