@elizaos/training 2.0.0-alpha.11

package/src/benchmark/BenchmarkValidator.ts

@@ -0,0 +1,206 @@
+/**
+ * Benchmark Data Validator
+ *
+ * Validates benchmark snapshot data to ensure it's properly formatted
+ * and contains all required fields.
+ */
+
+import type { JsonValue } from '../adapter';
+import { logger } from '../utils/logger';
+import type { BenchmarkGameSnapshot } from './BenchmarkDataGenerator';
+
+export interface BenchmarkValidationResult {
+  valid: boolean;
+  errors: string[];
+  warnings: string[];
+}
+
+export class BenchmarkValidator {
+  /**
+   * Validate a benchmark snapshot
+   */
+  static validate(snapshot: unknown): BenchmarkValidationResult {
+    const errors: string[] = [];
+    const warnings: string[] = [];
+
+    // 1. Check required top-level fields
+    if (!snapshot || typeof snapshot !== 'object') {
+      errors.push('Snapshot is null, undefined, or not an object');
+      return { valid: false, errors, warnings };
+    }
+
+    const snap = snapshot as Record<string, JsonValue>;
+
+    if (!snap.id) errors.push('Missing required field: id');
+    if (!snap.version) errors.push('Missing required field: version');
+    if (typeof snap.duration !== 'number')
+      errors.push('Missing or invalid field: duration');
+    if (typeof snap.tickInterval !== 'number')
+      errors.push('Missing or invalid field: tickInterval');
+    if (!snap.initialState) errors.push('Missing required field: initialState');
+    if (!Array.isArray(snap.ticks))
+      errors.push('Missing or invalid field: ticks (must be array)');
+    if (!snap.groundTruth) errors.push('Missing required field: groundTruth');
+
+    // 2. Validate initial state
+    if (snap.initialState && typeof snap.initialState === 'object') {
+      const state = snap.initialState as Record<string, JsonValue>;
+
+      if (typeof state.tick !== 'number')
+        errors.push('initialState.tick must be a number');
+      if (state.tick !== 0) warnings.push('initialState.tick should be 0');
+
+      if (!Array.isArray(state.predictionMarkets)) {
+        errors.push('initialState.predictionMarkets must be an array');
+      }
+
+      if (!Array.isArray(state.perpetualMarkets)) {
+        errors.push('initialState.perpetualMarkets must be an array');
+      }
+
+      if (!Array.isArray(state.agents)) {
+        errors.push('initialState.agents must be an array');
+      }
+    }
+
+    // 3. Validate ticks
+    if (Array.isArray(snap.ticks)) {
+      if (snap.ticks.length === 0) {
+        warnings.push('Ticks array is empty');
+      }
+
+      snap.ticks.forEach((tick: JsonValue, index: number) => {
+        if (!tick || typeof tick !== 'object') {
+          errors.push(`Tick ${index}: invalid tick object`);
+          return;
+        }
+        const tickObj = tick as Record<string, JsonValue>;
+        if (typeof tickObj.number !== 'number') {
+          errors.push(`Tick ${index}: missing or invalid 'number' field`);
+        }
+
+        if (!Array.isArray(tickObj.events)) {
+          errors.push(`Tick ${index}: events must be an array`);
+        }
+
+        if (!tickObj.state) {
+          errors.push(`Tick ${index}: missing state`);
+        }
+      });
+
+      // Check tick numbering is sequential
+      for (let i = 0; i < snap.ticks.length; i++) {
+        const tick = snap.ticks[i] as Record<string, JsonValue> | undefined;
+        if (tick && typeof tick.number === 'number' && tick.number !== i) {
+          warnings.push(`Tick ${i}: number ${tick.number} doesn't match index`);
+        }
+      }
+    }
+
+    // 4. Validate ground truth
+    if (snap.groundTruth && typeof snap.groundTruth === 'object') {
+      const gt = snap.groundTruth as Record<string, JsonValue>;
+
+      if (!gt.marketOutcomes || typeof gt.marketOutcomes !== 'object') {
+        errors.push('groundTruth.marketOutcomes must be an object');
+      }
+
+      if (!gt.priceHistory || typeof gt.priceHistory !== 'object') {
+        errors.push('groundTruth.priceHistory must be an object');
+      }
+
+      if (!Array.isArray(gt.optimalActions)) {
+        errors.push('groundTruth.optimalActions must be an array');
+      }
+
+      if (!Array.isArray(gt.socialOpportunities)) {
+        errors.push('groundTruth.socialOpportunities must be an array');
+      }
+
+      if (!Array.isArray(gt.hiddenFacts)) {
+        errors.push('groundTruth.hiddenFacts must be an array');
+      }
+
+      if (!Array.isArray(gt.hiddenEvents)) {
+        errors.push('groundTruth.hiddenEvents must be an array');
+      }
+
+      if (!gt.trueFacts || typeof gt.trueFacts !== 'object') {
+        errors.push('groundTruth.trueFacts must be an object');
+      }
+    }
+
+    // 5. Cross-validate: markets in initialState should have outcomes in groundTruth
+    if (
+      snap.initialState &&
+      typeof snap.initialState === 'object' &&
+      snap.groundTruth &&
+      typeof snap.groundTruth === 'object'
+    ) {
+      const initialState = snap.initialState as Record<string, JsonValue>;
+      const groundTruth = snap.groundTruth as Record<string, JsonValue>;
+      const markets = (
+        Array.isArray(initialState.predictionMarkets)
+          ? initialState.predictionMarkets
+          : []
+      ) as Array<Record<string, JsonValue>>;
+      const outcomes = (
+        groundTruth.marketOutcomes &&
+        typeof groundTruth.marketOutcomes === 'object'
+          ? groundTruth.marketOutcomes
+          : {}
+      ) as Record<string, JsonValue>;
+
+      markets.forEach((market) => {
+        if (
+          market.id &&
+          typeof market.id === 'string' &&
+          !(market.id in outcomes)
+        ) {
+          warnings.push(
+            `Market ${market.id} in initialState but no outcome in groundTruth`
+          );
+        }
+      });
+    }
+
+    logger.info('Benchmark validation complete', {
+      valid: errors.length === 0,
+      errors: errors.length,
+      warnings: warnings.length,
+    });
+
+    return {
+      valid: errors.length === 0,
+      errors,
+      warnings,
+    };
+  }
+
+  /**
+   * Quick sanity check (fast, minimal validation)
+   */
+  static sanityCheck(snapshot: unknown): snapshot is BenchmarkGameSnapshot {
+    if (!snapshot || typeof snapshot !== 'object') return false;
+    const snap = snapshot as Record<string, JsonValue>;
+    return !!(
+      snap.id &&
+      snap.initialState &&
+      Array.isArray(snap.ticks) &&
+      snap.groundTruth
+    );
+  }
+
+  /**
+   * Validate and throw if invalid
+   */
+  static validateOrThrow(
+    snapshot: unknown
+  ): asserts snapshot is BenchmarkGameSnapshot {
+    const result = this.validate(snapshot);
+
+    if (!result.valid) {
+      throw new Error(`Invalid benchmark data: ${result.errors.join(', ')}`);
+    }
+  }
+}

package/src/benchmark/FastEvalRunner.ts

@@ -0,0 +1,225 @@
+/**
+ * Fast Evaluation Runner
+ *
+ * Provides efficient evaluation of agents on benchmarks with:
+ * - Fast-forward mode (skip waiting)
+ * - Batch processing
+ * - Parallel execution
+ * - Progress tracking
+ */
+
+import { logger } from '../utils/logger';
+import { type BenchmarkRunConfig, BenchmarkRunner } from './BenchmarkRunner';
+import type { SimulationResult } from './SimulationEngine';
+
+export interface FastEvalConfig {
+  /** Benchmark file path */
+  benchmarkPath: string;
+
+  /** Agent runtime to test */
+  agentRuntime: BenchmarkRunConfig['agentRuntime'];
+
+  /** Agent user ID */
+  agentUserId: string;
+
+  /** Number of parallel runs */
+  parallelRuns?: number;
+
+  /** Number of iterations per run */
+  iterations?: number;
+
+  /** Save trajectory data */
+  saveTrajectory?: boolean;
+
+  /** Output directory */
+  outputDir: string;
+
+  /** Fast-forward mode (default: true) */
+  fastForward?: boolean; // Not used directly, passed to BenchmarkRunner
+
+  /** Progress callback */
+  onProgress?: (progress: {
+    completed: number;
+    total: number;
+    currentRun?: string;
+  }) => void;
+}
+
+export interface FastEvalResult {
+  /** All run results */
+  results: SimulationResult[];
+
+  /** Summary statistics */
+  summary: {
+    avgPnl: number;
+    avgAccuracy: number;
+    avgOptimality: number;
+    totalDuration: number;
+    runsCompleted: number;
+  };
+
+  /** Best and worst runs */
+  bestRun: SimulationResult;
+  worstRun: SimulationResult;
+}
+
+export class FastEvalRunner {
+  /**
+   * Run fast evaluation
+   *
+   * Executes efficient batch evaluation of an agent on a benchmark with
+   * parallel runs and progress tracking. Optimized for speed and throughput.
+   *
+   * @param config - Fast evaluation configuration
+   * @returns FastEvalResult with all run results and summary statistics
+   * @throws Error if evaluation fails
+   *
+   * @remarks
+   * - Runs multiple iterations in parallel batches
+   * - Provides progress callbacks for monitoring
+   * - Calculates aggregate statistics across all runs
+   * - Identifies best and worst performing runs
+   *
+   * @example
+   * ```typescript
+   * const result = await FastEvalRunner.run({
+   *   benchmarkPath: './benchmarks/test.json',
+   *   agentRuntime: runtime,
+   *   agentUserId: 'agent-123',
+   *   parallelRuns: 3,
+   *   iterations: 10,
+   *   outputDir: './results'
+   * });
+   * console.log(`Average P&L: ${result.summary.avgPnl}`);
+   * ```
+   */
+  static async run(config: FastEvalConfig): Promise<FastEvalResult> {
+    const startTime = Date.now();
+    const iterations = config.iterations || 1;
+    const parallelRuns = config.parallelRuns || 1;
+
+    logger.info('Starting fast evaluation', {
+      benchmarkPath: config.benchmarkPath,
+      agentUserId: config.agentUserId,
+      iterations,
+      parallelRuns,
+    });
+
+    const results: SimulationResult[] = [];
+    let completed = 0;
+
+    // Run iterations in batches
+    for (
+      let batchStart = 0;
+      batchStart < iterations;
+      batchStart += parallelRuns
+    ) {
+      const batchEnd = Math.min(batchStart + parallelRuns, iterations);
+      const batchSize = batchEnd - batchStart;
+
+      logger.info(
+        `Running batch ${batchStart + 1}-${batchEnd} of ${iterations}`
+      );
+
+      // Run batch in parallel
+      const batchPromises = Array.from({ length: batchSize }, (_, i) => {
+        const runIndex = batchStart + i;
+        const runOutputDir = `${config.outputDir}/run-${runIndex + 1}`;
+
+        return BenchmarkRunner.runSingle({
+          benchmarkPath: config.benchmarkPath,
+          agentRuntime: config.agentRuntime,
+          agentUserId: config.agentUserId,
+          saveTrajectory: config.saveTrajectory ?? false,
+          outputDir: runOutputDir,
+        }).then((result) => {
+          completed++;
+          if (config.onProgress) {
+            config.onProgress({
+              completed,
+              total: iterations,
+              currentRun: `run-${runIndex + 1}`,
+            });
+          }
+          return result;
+        });
+      });
+
+      const batchResults = await Promise.all(batchPromises);
+      results.push(...batchResults);
+    }
+
+    const totalDuration = Date.now() - startTime;
+
+    // Calculate summary
+    const avgPnl =
+      results.reduce((sum, r) => sum + r.metrics.totalPnl, 0) / results.length;
+    const avgAccuracy =
+      results.reduce(
+        (sum, r) => sum + r.metrics.predictionMetrics.accuracy,
+        0
+      ) / results.length;
+    const avgOptimality =
+      results.reduce((sum, r) => sum + r.metrics.optimalityScore, 0) /
+      results.length;
+
+    const bestRun = results.reduce((best, current) =>
+      current.metrics.totalPnl > best.metrics.totalPnl ? current : best
+    );
+
+    const worstRun = results.reduce((worst, current) =>
+      current.metrics.totalPnl < worst.metrics.totalPnl ? current : worst
+    );
+
+    const summary = {
+      avgPnl,
+      avgAccuracy,
+      avgOptimality,
+      totalDuration,
+      runsCompleted: results.length,
+    };
+
+    logger.info('Fast evaluation completed', summary);
+
+    return {
+      results,
+      summary,
+      bestRun,
+      worstRun,
+    };
+  }
+
+  /**
+   * Run evaluation with progress bar
+   */
+  static async runWithProgress(
+    config: FastEvalConfig
+  ): Promise<FastEvalResult> {
+    let lastProgress = 0;
+
+    return this.run({
+      ...config,
+      onProgress: (progress) => {
+        const percent = Math.round((progress.completed / progress.total) * 100);
+        if (percent !== lastProgress) {
+          const barLength = 40;
+          const filled = Math.round(
+            (progress.completed / progress.total) * barLength
+          );
+          const bar = '█'.repeat(filled) + '░'.repeat(barLength - filled);
+          process.stdout.write(
+            `\r[${bar}] ${percent}% (${progress.completed}/${progress.total})`
+          );
+          lastProgress = percent;
+        }
+
+        if (config.onProgress) {
+          config.onProgress(progress);
+        }
+      },
+    }).then((result) => {
+      process.stdout.write('\n');
+      return result;
+    });
+  }
+}

package/src/benchmark/MetricsValidator.ts

@@ -0,0 +1,165 @@
+/**
+ * Metrics Validator
+ *
+ * Validates that benchmark metrics are calculated correctly against ground truth.
+ */
+
+import type { ValidationResult } from '../training/ConfigValidator';
+import { logger } from '../utils/logger';
+import type { GroundTruth } from './BenchmarkDataGenerator';
+import type { AgentAction, SimulationMetrics } from './simulation-types';
+
+export class MetricsValidator {
+  /**
+   * Validate metrics against ground truth
+   */
+  static validate(
+    metrics: SimulationMetrics,
+    actions: AgentAction[],
+    groundTruth: GroundTruth
+  ): ValidationResult {
+    const errors: string[] = [];
+    const warnings: string[] = [];
+
+    // 1. Validate prediction accuracy calculation
+    const predictionValidation = this.validatePredictionMetrics(
+      metrics.predictionMetrics,
+      actions,
+      groundTruth
+    );
+    errors.push(...predictionValidation.errors);
+    warnings.push(...predictionValidation.warnings);
+
+    // 2. Validate optimality score is in valid range
+    if (metrics.optimalityScore < 0 || metrics.optimalityScore > 100) {
+      errors.push(`Optimality score out of range: ${metrics.optimalityScore}`);
+    }
+
+    // 3. Validate timing metrics are reasonable
+    if (metrics.timing.avgResponseTime < 0) {
+      errors.push(
+        `Invalid average response time: ${metrics.timing.avgResponseTime}`
+      );
+    }
+
+    if (metrics.timing.maxResponseTime < metrics.timing.avgResponseTime) {
+      errors.push(
+        `Max response time less than average: ${metrics.timing.maxResponseTime} < ${metrics.timing.avgResponseTime}`
+      );
+    }
+
+    // 4. Validate action counts match
+    const predictionActions = actions.filter(
+      (a) => a.type === 'buy_prediction'
+    );
+    if (predictionActions.length !== metrics.predictionMetrics.totalPositions) {
+      warnings.push(
+        `Prediction action count mismatch: ${predictionActions.length} actions vs ${metrics.predictionMetrics.totalPositions} positions`
+      );
+    }
+
+    // 5. Validate accuracy calculation
+    const { correctPredictions, incorrectPredictions, totalPositions } =
+      metrics.predictionMetrics;
+    const calculatedAccuracy =
+      totalPositions > 0 ? correctPredictions / totalPositions : 0;
+    const accuracyDiff = Math.abs(
+      calculatedAccuracy - metrics.predictionMetrics.accuracy
+    );
+
+    if (accuracyDiff > 0.01) {
+      // Allow 1% tolerance for floating point
+      errors.push(
+        `Accuracy calculation mismatch: reported ${metrics.predictionMetrics.accuracy}, calculated ${calculatedAccuracy}`
+      );
+    }
+
+    // 6. Validate correct + incorrect = total
+    if (correctPredictions + incorrectPredictions !== totalPositions) {
+      errors.push(
+        `Prediction count mismatch: ${correctPredictions} + ${incorrectPredictions} != ${totalPositions}`
+      );
+    }
+
+    // 7. Validate perp win rate calculation
+    if (metrics.perpMetrics.totalTrades > 0) {
+      const calculatedWinRate =
+        metrics.perpMetrics.profitableTrades / metrics.perpMetrics.totalTrades;
+      const winRateDiff = Math.abs(
+        calculatedWinRate - metrics.perpMetrics.winRate
+      );
+
+      if (winRateDiff > 0.01) {
+        errors.push(
+          `Win rate calculation mismatch: reported ${metrics.perpMetrics.winRate}, calculated ${calculatedWinRate}`
+        );
+      }
+    }
+
+    logger.info('Metrics validation complete', {
+      valid: errors.length === 0,
+      errors: errors.length,
+      warnings: warnings.length,
+    });
+
+    return {
+      valid: errors.length === 0,
+      errors,
+      warnings,
+    };
+  }
+
+  /**
+   * Validate prediction metrics against ground truth
+   */
+  private static validatePredictionMetrics(
+    _predictionMetrics: SimulationMetrics['predictionMetrics'],
+    actions: AgentAction[],
+    groundTruth: GroundTruth
+  ): ValidationResult {
+    const errors: string[] = [];
+    const warnings: string[] = [];
+
+    // Get all prediction actions
+    const predictionActions = actions.filter(
+      (a) => a.type === 'buy_prediction'
+    );
+
+    // Validate each action against ground truth
+    for (const action of predictionActions) {
+      const data = action.data as { marketId: string; outcome: string };
+      const marketId = data.marketId;
+
+      // Check if we have ground truth for this market
+      if (!(marketId in groundTruth.marketOutcomes)) {
+        warnings.push(`No ground truth for market ${marketId}`);
+        continue;
+      }
+
+      // Verify the outcome exists in ground truth
+      // (actual verification of correctness happens in SimulationEngine)
+    }
+
+    return { valid: errors.length === 0, errors, warnings };
+  }
+
+  /**
+   * Quick sanity check for metrics
+   */
+  static sanityCheck(metrics: SimulationMetrics): boolean {
+    // Basic sanity checks
+    if (metrics.optimalityScore < 0 || metrics.optimalityScore > 100)
+      return false;
+    if (
+      metrics.predictionMetrics.accuracy < 0 ||
+      metrics.predictionMetrics.accuracy > 1
+    )
+      return false;
+    if (metrics.perpMetrics.winRate < 0 || metrics.perpMetrics.winRate > 1)
+      return false;
+    if (metrics.timing.avgResponseTime < 0) return false;
+    if (metrics.timing.maxResponseTime < 0) return false;
+
+    return true;
+  }
+}