@elizaos/training 2.0.0-alpha.11

package/scripts/real-archetype-benchmark.ts

@@ -0,0 +1,210 @@
+#!/usr/bin/env bun
+/**
+ * Real Archetype Benchmark
+ *
+ * Queries actual agent data from the Babylon database.
+ *
+ * Usage:
+ *   bun run packages/training/scripts/real-archetype-benchmark.ts
+ */
+
+import {
+  agentPerformanceMetrics,
+  and,
+  db,
+  desc,
+  eq,
+  isNull,
+  poolPositions,
+  users,
+} from '@elizaos/db';
+import { ArchetypeConfigService } from '../src/archetypes/ArchetypeConfigService';
+
+// Get the available archetypes from our actual config
+const ARCHETYPES = ArchetypeConfigService.getAvailableArchetypes();
+
+interface RealAgentMetrics {
+  agentId: string;
+  agentName: string;
+  archetype: string;
+  lifetimePnL: number;
+  totalTrades: number;
+  winRate: number;
+  openPositions: number;
+  reputationPoints: number;
+}
+
+async function main() {
+  console.log(
+    '═══════════════════════════════════════════════════════════════'
+  );
+  console.log('  Babylon Real Archetype Benchmark');
+  console.log('  Using ACTUAL data from the game engine');
+  console.log(
+    '═══════════════════════════════════════════════════════════════\n'
+  );
+
+  console.log('Fetching real agent data from database...');
+
+  // Get all agents - use select() without specifying columns
+  const agents = await db
+    .select()
+    .from(users)
+    .where(eq(users.isAgent, true))
+    .orderBy(desc(users.reputationPoints))
+    .limit(100);
+
+  console.log(`Found ${agents.length} agents in database`);
+
+  if (agents.length === 0) {
+    console.log('\n⚠️  No agents found in database.');
+    console.log('   To generate real benchmark data:');
+    console.log('   1. Run the game with agents: bun run dev');
+    console.log('   2. Create agents with archetypes');
+    console.log('   3. Let them trade for a while');
+    console.log('   4. Re-run this benchmark\n');
+    return;
+  }
+
+  const results: RealAgentMetrics[] = [];
+
+  for (const agent of agents) {
+    // Get performance metrics for this agent
+    const performanceMetricsList = await db
+      .select()
+      .from(agentPerformanceMetrics)
+      .where(eq(agentPerformanceMetrics.userId, agent.id))
+      .limit(1);
+
+    const performanceMetrics = performanceMetricsList[0];
+
+    // Get open positions count
+    const openPositionsList = await db
+      .select()
+      .from(poolPositions)
+      .where(
+        and(eq(poolPositions.userId, agent.id), isNull(poolPositions.closedAt))
+      );
+
+    // Infer archetype from username/displayName
+    const agentName = agent.displayName || agent.username || 'Unknown';
+    let archetype = 'default';
+    const lowerName = agentName.toLowerCase();
+    for (const a of ARCHETYPES) {
+      if (
+        lowerName.includes(a.replace('-', '').toLowerCase()) ||
+        lowerName.includes(a.replace('-', ' ').toLowerCase())
+      ) {
+        archetype = a;
+        break;
+      }
+    }
+
+    results.push({
+      agentId: agent.id,
+      agentName,
+      archetype,
+      lifetimePnL: Number(agent.lifetimePnL) || 0,
+      totalTrades: performanceMetrics?.totalTrades || 0,
+      winRate: performanceMetrics?.winRate || 0,
+      openPositions: openPositionsList.length,
+      reputationPoints: agent.reputationPoints || 0,
+    });
+  }
+
+  console.log(`\nProcessed ${results.length} agents\n`);
+
+  // Group by archetype
+  const grouped = new Map<string, RealAgentMetrics[]>();
+  for (const agent of results) {
+    const existing = grouped.get(agent.archetype) || [];
+    existing.push(agent);
+    grouped.set(agent.archetype, existing);
+  }
+
+  // Print summary
+  console.log('Archetype Performance Summary:');
+  console.log('─'.repeat(70));
+  console.log(
+    'Archetype            | Agents | Avg PnL      | Win Rate | Reputation'
+  );
+  console.log('─'.repeat(70));
+
+  const benchmarkResults: Array<{
+    archetype: string;
+    count: number;
+    avgPnL: number;
+    avgWinRate: number;
+    avgReputation: number;
+  }> = [];
+
+  for (const [archetype, agentsList] of grouped) {
+    const count = agentsList.length;
+    const avgPnL =
+      count > 0
+        ? agentsList.reduce((sum, a) => sum + a.lifetimePnL, 0) / count
+        : 0;
+    const avgWinRate =
+      count > 0 ? agentsList.reduce((sum, a) => sum + a.winRate, 0) / count : 0;
+    const avgReputation =
+      count > 0
+        ? agentsList.reduce((sum, a) => sum + a.reputationPoints, 0) / count
+        : 0;
+
+    benchmarkResults.push({
+      archetype,
+      count,
+      avgPnL,
+      avgWinRate,
+      avgReputation,
+    });
+  }
+
+  benchmarkResults.sort((a, b) => b.avgPnL - a.avgPnL);
+
+  for (const r of benchmarkResults) {
+    console.log(
+      `${r.archetype.padEnd(20)} | ` +
+        `${r.count.toString().padStart(6)} | ` +
+        `$${r.avgPnL.toFixed(2).padStart(11)} | ` +
+        `${(r.avgWinRate * 100).toFixed(1).padStart(7)}% | ` +
+        `${r.avgReputation.toFixed(0).padStart(10)}`
+    );
+  }
+
+  console.log('─'.repeat(70));
+
+  // Save report
+  const { mkdirSync, writeFileSync } = await import('fs');
+  const outputDir = './research-output/real-benchmarks';
+  mkdirSync(outputDir, { recursive: true });
+
+  const report = `# Babylon Real Archetype Benchmark
+
+Generated: ${new Date().toISOString()}
+
+## Agents: ${results.length}
+
+| Archetype | Count | Avg PnL | Win Rate | Reputation |
+|-----------|-------|---------|----------|------------|
+${benchmarkResults
+  .map(
+    (r) =>
+      `| ${r.archetype} | ${r.count} | $${r.avgPnL.toFixed(2)} | ${(r.avgWinRate * 100).toFixed(1)}% | ${r.avgReputation.toFixed(0)} |`
+  )
+  .join('\n')}
+`;
+
+  const reportPath = `${outputDir}/benchmark-${Date.now()}.md`;
+  writeFileSync(reportPath, report);
+
+  console.log(`\n✓ Report saved to: ${reportPath}`);
+  console.log(
+    '═══════════════════════════════════════════════════════════════'
+  );
+}
+
+main().catch((err) => {
+  console.error('Benchmark failed:', err);
+  process.exit(1);
+});

package/scripts/run-baseline-comparison.ts

@@ -0,0 +1,116 @@
+#!/usr/bin/env bun
+
+/**
+ * Head-to-Head Benchmark Script
+ *
+ * Runs two parallel simulations on the exact same market conditions ("Fixed Seed").
+ * Compares "Baseline" (Random/Momentum) vs "Challenger" (Smart LLM Agent).
+ *
+ * Usage:
+ *   bun packages/training/scripts/run-baseline-comparison.ts
+ */
+
+// Import initializeJsonMode to enable file-based DB for trajectory recording
+// This prevents "Database not initialized" errors when saveTrajectory is true
+import { initializeJsonMode } from '@elizaos/db';
+import type { IAgentRuntime } from '@elizaos/core';
+import { mkdirSync } from 'fs';
+import * as path from 'path';
+import { type BenchmarkConfig } from '../src/benchmark/BenchmarkDataGenerator';
+import { BenchmarkRunner } from '../src/benchmark/BenchmarkRunner';
+import { MetricsVisualizer } from '../src/benchmark/MetricsVisualizer';
+import { logger } from '../src/utils/logger';
+
+// Mock Agent Runtime for the runner structure
+const mockRuntime = {
+  character: {
+    settings: {
+      model: 'gpt-4-turbo',
+    },
+  },
+} as unknown as IAgentRuntime;
+
+async function main() {
+  console.log(
+    '═══════════════════════════════════════════════════════════════'
+  );
+  console.log('  🥊 HEAD-TO-HEAD BENCHMARK: Random vs LLM Agent');
+  console.log(
+    '═══════════════════════════════════════════════════════════════\n'
+  );
+
+  const outputDir = path.join(
+    process.cwd(),
+    'benchmark-results',
+    `h2h-${Date.now()}`
+  );
+  mkdirSync(outputDir, { recursive: true });
+
+  // 0. Initialize Database in JSON Mode
+  // This ensures TrajectoryRecorder writes to files instead of crashing on missing Postgres
+  const dbPath = path.join(outputDir, 'db_storage');
+  mkdirSync(dbPath, { recursive: true });
+  try {
+    initializeJsonMode(dbPath);
+    logger.info(`Initialized JSON DB at ${dbPath}`);
+  } catch (e) {
+    logger.warn(
+      'Could not initialize JSON DB mode. Trajectory recording might fail if no Postgres connection.',
+      { error: e instanceof Error ? e.message : String(e) }
+    );
+  }
+
+  // 1. Configuration for Fixed Benchmark
+  const generatorConfig: BenchmarkConfig = {
+    durationMinutes: 10,
+    tickInterval: 1,
+    numPredictionMarkets: 5,
+    numPerpetualMarkets: 3,
+    numAgents: 5,
+    seed: 12345, // FIXED SEED for fairness
+  };
+
+  logger.info('Generating fixed benchmark snapshot...');
+
+  // 2. Run Baseline (Random Strategy)
+  // Note: We use the SAME generator config, so the runner will generate the SAME snapshot
+  // because of the fixed seed.
+  logger.info('>>> STARTING RUN A: BASELINE (RANDOM) <<<');
+  const baselineResult = await BenchmarkRunner.runSingle({
+    generatorConfig,
+    agentRuntime: mockRuntime, // Not used for baseline strategy
+    agentUserId: 'baseline-agent',
+    saveTrajectory: false, // Baseline doesn't need trajectory recording
+    outputDir: path.join(outputDir, 'baseline'),
+    forceStrategy: 'random', // Force dumb strategy
+  });
+
+  // 3. Run Challenger (Smart/Momentum Strategy for this demo)
+  // We use 'momentum' here to simulate a "Smart" agent for demonstration.
+  logger.info('>>> STARTING RUN B: CHALLENGER (MOMENTUM/LLM) <<<');
+
+  const challengerResult = await BenchmarkRunner.runSingle({
+    generatorConfig, // Same config -> Same seed -> Same market conditions
+    agentRuntime: mockRuntime,
+    agentUserId: 'challenger-agent',
+    saveTrajectory: true, // Record trajectory for the "Smart" agent to analyze decisions
+    outputDir: path.join(outputDir, 'challenger'),
+    forceStrategy: 'momentum', // Simulating "Smart" behavior
+  });
+
+  // 4. Generate Comparison Report
+  await MetricsVisualizer.generateComparisonReport(
+    baselineResult,
+    challengerResult,
+    outputDir
+  );
+
+  console.log(`\n✅ Benchmark complete. Results saved to: ${outputDir}`);
+
+  process.exit(0);
+}
+
+main().catch((error) => {
+  console.error('Benchmark failed:', error);
+  process.exit(1);
+});

package/scripts/run-full-pipeline.ts

@@ -0,0 +1,272 @@
+#!/usr/bin/env bun
+
+/**
+ * Full Training Pipeline Test
+ *
+ * This script runs the complete training pipeline end-to-end:
+ * 1. Initialize training package
+ * 2. Generate real trajectories (or use existing)
+ * 3. Score trajectories with LLM-as-judge
+ * 4. Export training data
+ * 5. Run archetype matchup benchmark
+ *
+ * Usage:
+ *   bun run packages/training/scripts/run-full-pipeline.ts
+ *
+ * Options:
+ *   --skip-generation    Skip trajectory generation (use existing data)
+ *   --skip-scoring       Skip LLM scoring
+ *   --archetypes         Comma-separated archetypes (default: trader,researcher)
+ *   --agents             Agents per archetype (default: 2)
+ *   --ticks              Ticks per agent (default: 10)
+ */
+
+import { count, db, eq, isNotNull, trajectories } from '@elizaos/db';
+import { parseArgs } from 'util';
+
+// Parse command line arguments
+const { values } = parseArgs({
+  args: process.argv.slice(2),
+  options: {
+    'skip-generation': { type: 'boolean', default: false },
+    'skip-scoring': { type: 'boolean', default: false },
+    archetypes: { type: 'string', default: 'trader,researcher' },
+    agents: { type: 'string', default: '2' },
+    ticks: { type: 'string', default: '10' },
+    help: { type: 'boolean', short: 'h' },
+  },
+});
+
+if (values.help) {
+  console.log(`
+Full Training Pipeline Test
+
+Usage:
+  bun run packages/training/scripts/run-full-pipeline.ts [options]
+
+Options:
+  --skip-generation    Skip trajectory generation (use existing data)
+  --skip-scoring       Skip LLM scoring
+  --archetypes         Comma-separated archetypes (default: trader,researcher)
+  --agents             Agents per archetype (default: 2)
+  --ticks              Ticks per agent (default: 10)
+  -h, --help           Show this help message
+`);
+  process.exit(0);
+}
+
+const config = {
+  skipGeneration: values['skip-generation'] as boolean,
+  skipScoring: values['skip-scoring'] as boolean,
+  archetypes: (values.archetypes as string).split(','),
+  agentsPerArchetype: parseInt(values.agents as string, 10),
+  ticksPerAgent: parseInt(values.ticks as string, 10),
+};
+
+console.log('═══════════════════════════════════════════════════════════════');
+console.log('  Babylon Full Training Pipeline');
+console.log('═══════════════════════════════════════════════════════════════');
+console.log(`  Archetypes: ${config.archetypes.join(', ')}`);
+console.log(`  Agents per archetype: ${config.agentsPerArchetype}`);
+console.log(`  Ticks per agent: ${config.ticksPerAgent}`);
+console.log(`  Skip generation: ${config.skipGeneration}`);
+console.log(`  Skip scoring: ${config.skipScoring}`);
+console.log(
+  '═══════════════════════════════════════════════════════════════\n'
+);
+
+async function runPipeline() {
+  const startTime = Date.now();
+
+  // Step 1: Check database connection
+  console.log('Step 1: Checking database connection...');
+  try {
+    const result = await db.select({ count: count() }).from(trajectories);
+    console.log(
+      `  ✅ Database connected. ${result[0]?.count || 0} existing trajectories.\n`
+    );
+  } catch (error) {
+    console.log(`  ❌ Database connection failed: ${error}`);
+    console.log('  Make sure DATABASE_URL is set correctly.\n');
+    process.exit(1);
+  }
+
+  // Step 2: Initialize training package
+  console.log('Step 2: Initializing training package...');
+  try {
+    const { initializeTrainingPackage } = await import('../src/init-training');
+    await initializeTrainingPackage();
+    console.log('  ✅ Training package initialized.\n');
+  } catch (error) {
+    console.log(`  ⚠️  Training package initialization failed: ${error}`);
+    console.log('  Will continue with limited functionality.\n');
+  }
+
+  // Step 3: Generate trajectories
+  if (!config.skipGeneration) {
+    console.log('Step 3: Generating real trajectories...');
+    try {
+      const { TrajectoryGenerator } = await import(
+        '../src/generation/TrajectoryGenerator'
+      );
+
+      // Get a manager ID (first user in DB or create one)
+      const { users, desc } = await import('@elizaos/db');
+      const managerResult = await db
+        .select({ id: users.id })
+        .from(users)
+        .orderBy(desc(users.createdAt))
+        .limit(1);
+
+      if (managerResult.length === 0) {
+        console.log('  ⚠️  No users found in database. Skipping generation.');
+        console.log('  Create a user first or use --skip-generation.\n');
+      } else {
+        const managerId = managerResult[0].id;
+
+        const generator = new TrajectoryGenerator({
+          archetypes: config.archetypes,
+          agentsPerArchetype: config.agentsPerArchetype,
+          ticksPerAgent: config.ticksPerAgent,
+          parallelAgents: 3,
+          recordTrajectories: true,
+          managerId,
+        });
+
+        const result = await generator.generate();
+        console.log(
+          `  ✅ Generated ${result.trajectoryIds.length} trajectories.`
+        );
+        console.log(`  Agents created: ${result.agentsCreated.length}`);
+        console.log(`  Duration: ${result.duration}ms\n`);
+
+        // Cleanup test agents
+        await generator.cleanup();
+      }
+    } catch (error) {
+      console.log(`  ❌ Generation failed: ${error}`);
+      console.log(
+        '  Make sure the server is running or use --skip-generation.\n'
+      );
+    }
+  } else {
+    console.log(
+      'Step 3: Skipping trajectory generation (--skip-generation).\n'
+    );
+  }
+
+  // Step 4: Score trajectories
+  if (!config.skipScoring) {
+    console.log('Step 4: Scoring trajectories with LLM-as-judge...');
+    try {
+      const { archetypeScoringService } = await import(
+        '../src/scoring/ArchetypeScoringService'
+      );
+
+      // Check for unscored trajectories
+      const unscoredCount = await db
+        .select({ count: count() })
+        .from(trajectories)
+        .where(eq(trajectories.isTrainingData, true));
+
+      const scoredCount = await db
+        .select({ count: count() })
+        .from(trajectories)
+        .where(isNotNull(trajectories.aiJudgeReward));
+
+      console.log(`  Training trajectories: ${unscoredCount[0]?.count || 0}`);
+      console.log(`  Already scored: ${scoredCount[0]?.count || 0}`);
+
+      // Score a batch of unscored trajectories
+      const result = await archetypeScoringService.scoreUnscoredTrajectories(
+        'default',
+        10
+      );
+      console.log(
+        `  ✅ Scored ${result.scored} trajectories (${result.errors} errors).\n`
+      );
+    } catch (error) {
+      console.log(`  ❌ Scoring failed: ${error}`);
+      console.log('  Make sure GROQ_API_KEY is set.\n');
+    }
+  } else {
+    console.log('Step 4: Skipping scoring (--skip-scoring).\n');
+  }
+
+  // Step 5: Run archetype matchup benchmark
+  console.log('Step 5: Running archetype matchup benchmark...');
+  try {
+    const { ArchetypeMatchupBenchmark } = await import(
+      '../src/benchmark/ArchetypeMatchupBenchmark'
+    );
+
+    const benchmark = new ArchetypeMatchupBenchmark({
+      archetypes: config.archetypes,
+      agentsPerArchetype: 2,
+      rounds: 3,
+      ticksPerRound: 50,
+      marketConditions: ['bull', 'bear'],
+      availableVramGb: 16,
+    });
+
+    const results = await benchmark.run();
+
+    console.log(`  ✅ Benchmark complete.`);
+    for (const result of results) {
+      console.log(`  ${result.marketCondition.toUpperCase()} market:`);
+      const top3 = result.archetypeRankings.slice(0, 3);
+      for (const r of top3) {
+        console.log(
+          `    ${r.avgRank.toFixed(1)}. ${r.archetype} (avg PnL: ${r.avgPnl.toFixed(2)})`
+        );
+      }
+    }
+    console.log('');
+  } catch (error) {
+    console.log(`  ❌ Benchmark failed: ${error}\n`);
+  }
+
+  // Step 6: Export training data
+  console.log('Step 6: Checking training data export...');
+  try {
+    const scoredResult = await db
+      .select({ count: count() })
+      .from(trajectories)
+      .where(isNotNull(trajectories.aiJudgeReward));
+
+    const scored = scoredResult[0]?.count || 0;
+    if (scored > 0) {
+      console.log(`  ✅ ${scored} trajectories ready for export.`);
+      console.log('  Run "babylon train export" to export training data.\n');
+    } else {
+      console.log('  ⚠️  No scored trajectories available for export.');
+      console.log('  Generate and score trajectories first.\n');
+    }
+  } catch (error) {
+    console.log(`  ❌ Export check failed: ${error}\n`);
+  }
+
+  // Summary
+  const totalTime = Date.now() - startTime;
+  console.log(
+    '═══════════════════════════════════════════════════════════════'
+  );
+  console.log('  Pipeline Complete');
+  console.log(
+    '═══════════════════════════════════════════════════════════════'
+  );
+  console.log(`  Total time: ${(totalTime / 1000).toFixed(1)}s`);
+  console.log('');
+  console.log('  Next steps:');
+  console.log('  1. Export data: babylon train export');
+  console.log('  2. Train models: python python/scripts/run_full_pipeline.py');
+  console.log('  3. Benchmark: babylon train benchmark');
+  console.log(
+    '═══════════════════════════════════════════════════════════════'
+  );
+}
+
+runPipeline().catch((error) => {
+  console.error('Pipeline failed:', error);
+  process.exit(1);
+});