@elizaos/training 2.0.0-alpha.11

package/src/benchmark/BenchmarkDataViewer.ts

@@ -0,0 +1,324 @@
+/**
+ * Benchmark Data Viewer
+ *
+ * Provides utilities to view and inspect benchmark data.
+ * Useful for validation and understanding benchmark structure.
+ */
+
+import type { JsonValue } from '../adapter';
+import { promises as fs } from 'fs';
+import type {
+  BenchmarkGameSnapshot,
+  GameState,
+  GroundTruth,
+  Tick,
+} from './BenchmarkDataGenerator';
+import { BenchmarkValidator } from './BenchmarkValidator';
+
+export interface BenchmarkViewOptions {
+  /** Show detailed information */
+  verbose?: boolean;
+
+  /** Show only summary */
+  summary?: boolean;
+
+  /** Show ground truth data */
+  showGroundTruth?: boolean;
+
+  /** Show hidden facts/events */
+  showHidden?: boolean;
+
+  /** Filter by tick range */
+  tickRange?: { start: number; end: number };
+}
+
+export interface BenchmarkView {
+  /** Basic info */
+  id: string;
+  version: string;
+  createdAt: number;
+  duration: number;
+  tickInterval: number;
+
+  /** State summary */
+  initialState: {
+    predictionMarkets: number;
+    perpetualMarkets: number;
+    agents: number;
+    posts: number;
+    groupChats: number;
+  };
+
+  /** Ticks summary */
+  ticks: {
+    total: number;
+    withEvents: number;
+    eventTypes: Record<string, number>;
+  };
+
+  /** Ground truth summary */
+  groundTruth?: {
+    marketOutcomes: number;
+    priceHistory: Record<string, number>;
+    optimalActions: number;
+    socialOpportunities: number;
+    hiddenFacts: number;
+    hiddenEvents: number;
+    trueFacts: string[];
+  };
+
+  /** Validation results */
+  validation: {
+    valid: boolean;
+    errors: string[];
+    warnings: string[];
+  };
+}
+
+export class BenchmarkDataViewer {
+  /**
+   * Load and view a benchmark file
+   */
+  static async view(
+    filePath: string,
+    options: BenchmarkViewOptions = {}
+  ): Promise<BenchmarkView> {
+    const data = await fs.readFile(filePath, 'utf-8');
+    const snapshot = JSON.parse(data) as BenchmarkGameSnapshot;
+
+    // Validate
+    const validation = BenchmarkValidator.validate(snapshot);
+
+    // Build view
+    const view: BenchmarkView = {
+      id: snapshot.id,
+      version: snapshot.version,
+      createdAt: snapshot.createdAt,
+      duration: snapshot.duration,
+      tickInterval: snapshot.tickInterval,
+
+      initialState: {
+        predictionMarkets: snapshot.initialState.predictionMarkets.length,
+        perpetualMarkets: snapshot.initialState.perpetualMarkets.length,
+        agents: snapshot.initialState.agents.length,
+        posts: snapshot.initialState.posts?.length || 0,
+        groupChats: snapshot.initialState.groupChats?.length || 0,
+      },
+
+      ticks: this.analyzeTicks(snapshot.ticks),
+
+      validation,
+    };
+
+    if (options.showGroundTruth || options.verbose) {
+      view.groundTruth = this.analyzeGroundTruth(snapshot.groundTruth);
+    }
+
+    return view;
+  }
+
+  /**
+   * Analyze ticks
+   */
+  private static analyzeTicks(ticks: Tick[]): BenchmarkView['ticks'] {
+    const eventTypes: Record<string, number> = {};
+    let withEvents = 0;
+
+    for (const tick of ticks) {
+      if (tick.events.length > 0) {
+        withEvents++;
+      }
+
+      for (const event of tick.events) {
+        eventTypes[event.type] = (eventTypes[event.type] || 0) + 1;
+      }
+    }
+
+    return {
+      total: ticks.length,
+      withEvents,
+      eventTypes,
+    };
+  }
+
+  /**
+   * Analyze ground truth
+   */
+  private static analyzeGroundTruth(
+    groundTruth: GroundTruth
+  ): BenchmarkView['groundTruth'] {
+    return {
+      marketOutcomes: Object.keys(groundTruth.marketOutcomes).length,
+      priceHistory: Object.fromEntries(
+        Object.entries(groundTruth.priceHistory).map(([ticker, history]) => [
+          ticker,
+          history.length,
+        ])
+      ),
+      optimalActions: groundTruth.optimalActions.length,
+      socialOpportunities: groundTruth.socialOpportunities.length,
+      hiddenFacts: groundTruth.hiddenFacts?.length || 0,
+      hiddenEvents: groundTruth.hiddenEvents?.length || 0,
+      trueFacts: Object.keys(groundTruth.trueFacts || {}),
+    };
+  }
+
+  /**
+   * Print view to console
+   */
+  static print(view: BenchmarkView, options: BenchmarkViewOptions = {}): void {
+    console.log('\n📊 Benchmark Data View\n');
+    console.log(`ID: ${view.id}`);
+    console.log(`Version: ${view.version}`);
+    console.log(`Created: ${new Date(view.createdAt).toISOString()}`);
+    console.log(`Duration: ${(view.duration / 60).toFixed(1)} minutes`);
+    console.log(`Tick Interval: ${view.tickInterval}s`);
+
+    console.log('\n📈 Initial State:');
+    console.log(`  Prediction Markets: ${view.initialState.predictionMarkets}`);
+    console.log(`  Perpetual Markets: ${view.initialState.perpetualMarkets}`);
+    console.log(`  Agents: ${view.initialState.agents}`);
+    console.log(`  Posts: ${view.initialState.posts}`);
+    console.log(`  Group Chats: ${view.initialState.groupChats}`);
+
+    console.log('\n⏱️  Ticks:');
+    console.log(`  Total: ${view.ticks.total}`);
+    console.log(`  With Events: ${view.ticks.withEvents}`);
+    if (options.verbose) {
+      console.log(`  Event Types:`);
+      for (const [type, count] of Object.entries(view.ticks.eventTypes)) {
+        console.log(`    ${type}: ${count}`);
+      }
+    }
+
+    if (view.groundTruth) {
+      console.log('\n🎯 Ground Truth:');
+      console.log(`  Market Outcomes: ${view.groundTruth.marketOutcomes}`);
+      console.log(`  Price History:`);
+      for (const [ticker, count] of Object.entries(
+        view.groundTruth.priceHistory
+      )) {
+        console.log(`    ${ticker}: ${count} ticks`);
+      }
+      console.log(`  Optimal Actions: ${view.groundTruth.optimalActions}`);
+      console.log(
+        `  Social Opportunities: ${view.groundTruth.socialOpportunities}`
+      );
+      if (options.showHidden) {
+        console.log(`  Hidden Facts: ${view.groundTruth.hiddenFacts}`);
+        console.log(`  Hidden Events: ${view.groundTruth.hiddenEvents}`);
+        console.log(`  True Facts: ${view.groundTruth.trueFacts.join(', ')}`);
+      }
+    }
+
+    console.log('\n✅ Validation:');
+    console.log(`  Valid: ${view.validation.valid ? '✅' : '❌'}`);
+    if (view.validation.errors.length > 0) {
+      console.log(`  Errors: ${view.validation.errors.length}`);
+      if (options.verbose) {
+        for (const error of view.validation.errors) {
+          console.log(`    ❌ ${error}`);
+        }
+      }
+    }
+    if (view.validation.warnings.length > 0) {
+      console.log(`  Warnings: ${view.validation.warnings.length}`);
+      if (options.verbose) {
+        for (const warning of view.validation.warnings) {
+          console.log(`    ⚠️  ${warning}`);
+        }
+      }
+    }
+
+    console.log('');
+  }
+
+  /**
+   * Get tick details
+   */
+  static getTickDetails(
+    snapshot: BenchmarkGameSnapshot,
+    tickNumber: number
+  ): {
+    tick: Tick | null;
+    state: GameState | null;
+    events: Array<{ type: string; data: Record<string, JsonValue> }>;
+  } {
+    const tick = snapshot.ticks[tickNumber] || null;
+
+    if (!tick) {
+      return { tick: null, state: null, events: [] };
+    }
+
+    return {
+      tick,
+      state: tick.state,
+      events: tick.events.map((e) => ({
+        type: e.type,
+        data: e.data,
+      })),
+    };
+  }
+
+  /**
+   * Get ground truth for a specific tick
+   */
+  static getGroundTruthForTick(
+    snapshot: BenchmarkGameSnapshot,
+    tickNumber: number
+  ): {
+    hiddenFacts: Array<{ fact: string; category: string }>;
+    hiddenEvents: Array<{ type: string; description: string }>;
+    marketOutcomes: Record<string, boolean>;
+  } {
+    const gt = snapshot.groundTruth;
+
+    return {
+      hiddenFacts: (gt.hiddenFacts || [])
+        .filter((f) => f.tick === tickNumber)
+        .map((f) => ({ fact: f.fact, category: f.category })),
+      hiddenEvents: (gt.hiddenEvents || [])
+        .filter((e) => e.tick === tickNumber)
+        .map((e) => ({ type: e.type, description: e.description })),
+      marketOutcomes: gt.marketOutcomes,
+    };
+  }
+
+  /**
+   * Check if agent can access hidden facts (should always be false)
+   */
+  static verifyAgentCannotAccessHiddenFacts(snapshot: BenchmarkGameSnapshot): {
+    canAccess: boolean;
+    reason: string;
+  } {
+    // Agents can only access game state via SimulationA2AInterface
+    // Ground truth is stored separately and not exposed
+    // This is a verification check
+
+    const state = snapshot.initialState;
+    const hasGroundTruth = !!snapshot.groundTruth;
+    const hasHiddenFacts = !!snapshot.groundTruth?.hiddenFacts?.length;
+
+    // Check if ground truth is accidentally in state
+    const stateKeys = Object.keys(state);
+    const hasGroundTruthInState =
+      stateKeys.includes('groundTruth') ||
+      stateKeys.includes('hiddenFacts') ||
+      stateKeys.includes('hiddenEvents');
+
+    if (hasGroundTruthInState) {
+      return {
+        canAccess: true,
+        reason: 'Ground truth found in game state (security issue!)',
+      };
+    }
+
+    return {
+      canAccess: false,
+      reason:
+        hasGroundTruth && hasHiddenFacts
+          ? 'Ground truth exists but is properly isolated from game state'
+          : 'No ground truth data found',
+    };
+  }
+}

package/src/benchmark/BenchmarkHistoryService.ts

@@ -0,0 +1,221 @@
+/**
+ * Benchmark History Service
+ *
+ * Persists benchmark results to the database for historical tracking and analysis.
+ */
+
+import {
+  getTrainingDataAdapter,
+  type BenchmarkResultRecord,
+  type JsonValue,
+} from '../adapter';
+import { logger } from '../utils/logger';
+import { generateSnowflakeId } from '../utils/snowflake';
+import type { SimulationMetrics } from './SimulationEngine';
+
+export interface BenchmarkResultInput {
+  modelId: string;
+  benchmarkId: string;
+  benchmarkPath: string;
+  metrics: SimulationMetrics;
+  duration: number;
+  baselineComparison?: {
+    pnlDelta: number;
+    accuracyDelta: number;
+    improved: boolean;
+  };
+}
+
+export interface BenchmarkHistoryQuery {
+  modelId?: string;
+  benchmarkId?: string;
+  startDate?: Date;
+  endDate?: Date;
+  limit?: number;
+}
+
+export interface BenchmarkTrendData {
+  modelId: string;
+  dates: Date[];
+  pnlHistory: number[];
+  accuracyHistory: number[];
+  optimalityHistory: number[];
+}
+
+/**
+ * Service for managing benchmark result history
+ */
+export class BenchmarkHistoryService {
+  /**
+   * Save a benchmark result to the database
+   */
+  static async saveResult(
+    input: BenchmarkResultInput
+  ): Promise<BenchmarkResultRecord> {
+    const id = await generateSnowflakeId();
+    const now = new Date();
+
+    const insertData = {
+      id,
+      modelId: input.modelId,
+      benchmarkId: input.benchmarkId,
+      benchmarkPath: input.benchmarkPath,
+      runAt: now,
+      totalPnl: input.metrics.totalPnl,
+      predictionAccuracy: input.metrics.predictionMetrics.accuracy,
+      perpWinRate: input.metrics.perpMetrics.winRate,
+      optimalityScore: input.metrics.optimalityScore,
+      detailedMetrics: JSON.parse(JSON.stringify(input.metrics)) as JsonValue,
+      baselinePnlDelta: input.baselineComparison?.pnlDelta ?? null,
+      baselineAccuracyDelta: input.baselineComparison?.accuracyDelta ?? null,
+      improved: input.baselineComparison?.improved ?? null,
+      duration: input.duration,
+    };
+
+    await getTrainingDataAdapter().insertBenchmarkResult(insertData);
+
+    logger.info('Saved benchmark result', {
+      id,
+      modelId: input.modelId,
+      benchmarkId: input.benchmarkId,
+      totalPnl: input.metrics.totalPnl,
+    });
+
+    return { ...insertData, createdAt: now };
+  }
+
+  /**
+   * Get benchmark results by query
+   */
+  static async getResults(
+    query: BenchmarkHistoryQuery
+  ): Promise<BenchmarkResultRecord[]> {
+    return getTrainingDataAdapter().queryBenchmarkResults({
+      modelId: query.modelId,
+      benchmarkId: query.benchmarkId,
+      startDate: query.startDate,
+      endDate: query.endDate,
+      limit: query.limit ?? 100,
+    });
+  }
+
+  /**
+   * Get the latest result for a model
+   */
+  static async getLatestResult(
+    modelId: string
+  ): Promise<BenchmarkResultRecord | null> {
+    const results = await getTrainingDataAdapter().queryBenchmarkResults({
+      modelId,
+      limit: 1,
+    });
+    return results[0] ?? null;
+  }
+
+  /**
+   * Get trend data for a model
+   */
+  static async getTrendData(
+    modelId: string,
+    limit = 20
+  ): Promise<BenchmarkTrendData> {
+    const results = await getTrainingDataAdapter().queryBenchmarkResults({
+      modelId,
+      limit,
+    });
+
+    // queryBenchmarkResults returns desc by runAt, reverse for chronological
+    const chronological = results.reverse();
+
+    return {
+      modelId,
+      dates: chronological.map((r) => r.runAt),
+      pnlHistory: chronological.map((r) => r.totalPnl),
+      accuracyHistory: chronological.map((r) => r.predictionAccuracy),
+      optimalityHistory: chronological.map((r) => r.optimalityScore),
+    };
+  }
+
+  /**
+   * Get comparison data for multiple models
+   */
+  static async getModelComparison(
+    modelIds: string[],
+    benchmarkId?: string
+  ): Promise<Map<string, BenchmarkResultRecord[]>> {
+    const adapter = getTrainingDataAdapter();
+    const comparison = new Map<string, BenchmarkResultRecord[]>();
+
+    for (const modelId of modelIds) {
+      const results = await adapter.queryBenchmarkResults({
+        modelId,
+        benchmarkId,
+        limit: 10,
+      });
+      comparison.set(modelId, results);
+    }
+
+    return comparison;
+  }
+
+  /**
+   * Get summary statistics for all models
+   */
+  static async getModelSummary(): Promise<
+    Array<{
+      modelId: string;
+      runCount: number;
+      avgPnl: number;
+      avgAccuracy: number;
+      avgOptimality: number;
+      bestPnl: number;
+      latestRun: Date;
+    }>
+  > {
+    return getTrainingDataAdapter().getBenchmarkModelSummary();
+  }
+
+  /**
+   * Check if a model improved vs baseline
+   */
+  static async checkImprovement(
+    modelId: string,
+    baselineModelId: string,
+    benchmarkId: string
+  ): Promise<{
+    improved: boolean;
+    modelPnl: number;
+    baselinePnl: number;
+    delta: number;
+  } | null> {
+    const adapter = getTrainingDataAdapter();
+
+    const modelResults = await adapter.queryBenchmarkResults({
+      modelId,
+      benchmarkId,
+      limit: 1,
+    });
+
+    const baselineResults = await adapter.queryBenchmarkResults({
+      modelId: baselineModelId,
+      benchmarkId,
+      limit: 1,
+    });
+
+    const modelResult = modelResults[0];
+    const baselineResult = baselineResults[0];
+
+    if (!modelResult || !baselineResult) {
+      return null;
+    }
+
+    const delta = modelResult.totalPnl - baselineResult.totalPnl;
+
+    return {
+      improved: delta > 0,
+      modelPnl: modelResult.totalPnl,
+      baselinePnl: baselineResult.totalPnl,
+      delta,
+    };
+  }
+}