@elizaos/training 2.0.0-alpha.21 → 2.0.0-alpha.22

package/dist/benchmark/ModelBenchmarkService.js

@@ -0,0 +1,433 @@
+/**
+ * Model Benchmark Service (For HuggingFace Integration)
+ *
+ * Runs benchmark tests on trained RL models for HuggingFace upload decisions.
+ * Compares new models against baselines and previous versions.
+ *
+ * **Purpose:** Evaluate models for HuggingFace upload
+ * **Used by:** HuggingFace integration, weekly CRON, CLI scripts
+ * **Storage:** benchmark_results table (dedicated table)
+ * **Focus:** Public model release, baseline comparison
+ *
+ * **Note:** For training pipeline benchmarking, see BenchmarkService
+ *
+ * @see BenchmarkService - For training pipeline evaluation
+ */
+import { promises as fs } from "node:fs";
+import * as path from "node:path";
+import { ethers } from "ethers";
+import { getTrainingDataAdapter } from "../adapter";
+import { getAgentRuntimeManager } from "../dependencies";
+import { logger } from "../utils/logger";
+import { generateSnowflakeId } from "../utils/snowflake";
+import { BenchmarkRunner } from "./BenchmarkRunner";
+import { parseSimulationMetrics, } from "./parseSimulationMetrics";
+export class ModelBenchmarkService {
+    /**
+     * Benchmark a trained model against standard benchmarks
+     */
+    static async benchmarkModel(options) {
+        logger.info("Starting model benchmark", { modelId: options.modelId });
+        // Load model from database
+        const adapter = getTrainingDataAdapter();
+        const model = await adapter.getModelById(options.modelId);
+        if (!model) {
+            throw new Error(`Model not found: ${options.modelId}`);
+        }
+        // Check if model already benchmarked
+        const existingBenchmarks = await ModelBenchmarkService.getModelBenchmarks(options.modelId);
+        if (existingBenchmarks.length > 0 && !options.saveResults) {
+            logger.info("Model already benchmarked", {
+                modelId: options.modelId,
+                count: existingBenchmarks.length,
+            });
+            return existingBenchmarks;
+        }
+        // Create test agent for benchmarking
+        const testAgentId = await ModelBenchmarkService.getOrCreateTestAgent();
+        const results = [];
+        // Run each benchmark
+        for (const benchmarkPath of options.benchmarkPaths) {
+            logger.info("Running benchmark", {
+                benchmark: benchmarkPath,
+                modelId: options.modelId,
+            });
+            try {
+                // Get agent runtime (will use the RL model if configured)
+                const runtime = await getAgentRuntimeManager().getRuntime(testAgentId);
+                // Run benchmark
+                const simulationResult = await BenchmarkRunner.runSingle({
+                    benchmarkPath,
+                    agentRuntime: runtime,
+                    agentUserId: testAgentId,
+                    saveTrajectory: false,
+                    outputDir: options.outputDir ||
+                        path.join(process.cwd(), "benchmarks", "model-results", model.version),
+                    forceModel: model.storagePath, // Use the RL model
+                });
+                // Create benchmark result
+                const benchmarkResult = {
+                    modelId: options.modelId,
+                    modelVersion: model.version,
+                    benchmarkId: simulationResult.benchmarkId,
+                    benchmarkPath,
+                    runAt: new Date(),
+                    metrics: simulationResult.metrics,
+                };
+                // Compare to baseline if available
+                const baseline = await ModelBenchmarkService.getBaselineBenchmark(benchmarkPath);
+                if (baseline) {
+                    benchmarkResult.comparisonToBaseline = {
+                        pnlDelta: simulationResult.metrics.totalPnl - baseline.totalPnl,
+                        accuracyDelta: simulationResult.metrics.predictionMetrics.accuracy -
+                            baseline.predictionMetrics.accuracy,
+                        optimalityDelta: simulationResult.metrics.optimalityScore -
+                            baseline.optimalityScore,
+                        improved: simulationResult.metrics.totalPnl > baseline.totalPnl,
+                    };
+                }
+                results.push(benchmarkResult);
+                logger.info("Benchmark completed", {
+                    benchmark: benchmarkPath,
+                    pnl: simulationResult.metrics.totalPnl,
+                    accuracy: simulationResult.metrics.predictionMetrics.accuracy,
+                });
+                // Save result if requested (to both database and files)
+                if (options.saveResults) {
+                    await ModelBenchmarkService.saveBenchmarkResultToDatabase(benchmarkResult);
+                    await ModelBenchmarkService.saveBenchmarkResult(benchmarkResult);
+                }
+            }
+            catch (error) {
+                logger.error("Benchmark failed", { benchmark: benchmarkPath, error });
+            }
+        }
+        // Update model with aggregate benchmark score
+        if (results.length > 0) {
+            const avgOptimality = results.reduce((sum, r) => sum + r.metrics.optimalityScore, 0) /
+                results.length;
+            const avgPnl = results.reduce((sum, r) => sum + r.metrics.totalPnl, 0) /
+                results.length;
+            await adapter.updateModelBenchmark(options.modelId, avgOptimality, avgPnl, (model.benchmarkCount || 0) + results.length);
+        }
+        logger.info("Model benchmark complete", {
+            modelId: options.modelId,
+            benchmarksRun: results.length,
+        });
+        return results;
+    }
+    /**
+     * Compare new model against baseline
+     */
+    static async compareToBaseline(modelId) {
+        // Get new model benchmarks
+        const newModelBenchmarks = await ModelBenchmarkService.getModelBenchmarks(modelId);
+        if (newModelBenchmarks.length === 0) {
+            throw new Error(`No benchmarks found for model: ${modelId}`);
+        }
+        // Calculate new model average metrics
+        const newModelMetrics = ModelBenchmarkService.calculateAverageMetrics(newModelBenchmarks.map((b) => b.metrics));
+        // Get baseline benchmarks (use best baseline model)
+        const baselineMetrics = await ModelBenchmarkService.getBaselineAverageMetrics();
+        // Calculate improvement
+        const pnlDelta = newModelMetrics.totalPnl - baselineMetrics.totalPnl;
+        const accuracyDelta = newModelMetrics.accuracy - baselineMetrics.accuracy;
+        const optimalityDelta = newModelMetrics.optimality - baselineMetrics.optimality;
+        // Determine if this is an improvement (weighted score)
+        const improvementScore = (pnlDelta > 0 ? 1 : 0) * 0.4 +
+            (accuracyDelta > 0 ? 1 : 0) * 0.3 +
+            (optimalityDelta > 0 ? 1 : 0) * 0.3;
+        const isImprovement = improvementScore > 0.5;
+        let recommendation;
+        if (isImprovement && pnlDelta > 0) {
+            recommendation = "deploy";
+        }
+        else if (pnlDelta < -100) {
+            recommendation = "baseline_better";
+        }
+        else {
+            recommendation = "keep_training";
+        }
+        return {
+            newModel: {
+                modelId,
+                version: newModelBenchmarks[0]?.modelVersion,
+                avgMetrics: newModelMetrics,
+            },
+            baseline: {
+                modelId: "baseline",
+                avgMetrics: baselineMetrics,
+            },
+            improvement: {
+                pnlDelta,
+                accuracyDelta,
+                optimalityDelta,
+                isImprovement,
+            },
+            recommendation,
+        };
+    }
+    /**
+     * Get all unbenchmarked models
+     */
+    static async getUnbenchmarkedModels() {
+        return getTrainingDataAdapter().getUnbenchmarkedModels();
+    }
+    /**
+     * Get model benchmark results
+     */
+    static async getModelBenchmarks(modelId) {
+        // For now, read from files
+        // In production, you'd store these in a database table
+        const benchmarksDir = path.join(process.cwd(), "benchmarks", "model-results");
+        const results = [];
+        try {
+            const model = await getTrainingDataAdapter().getModelById(modelId);
+            if (!model)
+                return results;
+            const modelDir = path.join(benchmarksDir, model.version);
+            const files = await fs.readdir(modelDir).catch(() => []);
+            for (const file of files) {
+                if (file.endsWith(".json")) {
+                    const filePath = path.join(modelDir, file);
+                    const data = JSON.parse(await fs.readFile(filePath, "utf-8"));
+                    if (data.modelId === modelId) {
+                        results.push(data);
+                    }
+                }
+            }
+        }
+        catch (error) {
+            logger.warn("Could not load benchmark results", { error });
+        }
+        return results;
+    }
+    /**
+     * Save benchmark result to database
+     */
+    static async saveBenchmarkResultToDatabase(result) {
+        await getTrainingDataAdapter().insertBenchmarkResult({
+            id: await generateSnowflakeId(),
+            modelId: result.modelId,
+            benchmarkId: result.benchmarkId,
+            benchmarkPath: result.benchmarkPath,
+            runAt: result.runAt,
+            totalPnl: result.metrics.totalPnl,
+            predictionAccuracy: result.metrics.predictionMetrics.accuracy,
+            perpWinRate: result.metrics.perpMetrics.winRate,
+            optimalityScore: result.metrics.optimalityScore,
+            detailedMetrics: JSON.parse(JSON.stringify(result.metrics)),
+            baselinePnlDelta: result.comparisonToBaseline?.pnlDelta ?? null,
+            baselineAccuracyDelta: result.comparisonToBaseline?.accuracyDelta ?? null,
+            improved: result.comparisonToBaseline?.improved ?? null,
+            duration: result.metrics.timing.totalDuration,
+        });
+        logger.info("Benchmark result saved to database", {
+            modelId: result.modelId,
+            benchmarkId: result.benchmarkId,
+        });
+    }
+    /**
+     * Save benchmark result to file
+     */
+    static async saveBenchmarkResult(result) {
+        const outputDir = path.join(process.cwd(), "benchmarks", "model-results", result.modelVersion);
+        await fs.mkdir(outputDir, { recursive: true });
+        const filename = `benchmark-${result.benchmarkId}-${Date.now()}.json`;
+        const filePath = path.join(outputDir, filename);
+        await fs.writeFile(filePath, JSON.stringify(result, null, 2));
+        logger.info("Benchmark result saved to file", { filePath });
+    }
+    /**
+     * Get benchmark results from database
+     */
+    static async getBenchmarkResultsFromDatabase(modelId) {
+        const results = await getTrainingDataAdapter().getBenchmarkResultsByModel(modelId);
+        return results.map((r) => ({
+            modelId: r.modelId,
+            modelVersion: "", // Not stored in results table
+            benchmarkId: r.benchmarkId,
+            benchmarkPath: r.benchmarkPath,
+            runAt: r.runAt,
+            metrics: parseSimulationMetrics(r.detailedMetrics),
+            comparisonToBaseline: r.baselinePnlDelta !== null
+                ? {
+                    pnlDelta: r.baselinePnlDelta,
+                    accuracyDelta: r.baselineAccuracyDelta ?? 0,
+                    optimalityDelta: 0, // Not stored separately
+                    improved: r.improved || false,
+                }
+                : undefined,
+        }));
+    }
+    /**
+     * Get baseline benchmark for comparison
+     */
+    static async getBaselineBenchmark(benchmarkPath) {
+        try {
+            // Look for baseline result for this benchmark
+            const baselinesDir = path.join(process.cwd(), "benchmarks", "baselines");
+            const files = await fs.readdir(baselinesDir).catch(() => []);
+            for (const file of files) {
+                if (file.endsWith(".json")) {
+                    const filePath = path.join(baselinesDir, file);
+                    const data = JSON.parse(await fs.readFile(filePath, "utf-8"));
+                    if (data.benchmark?.path === benchmarkPath ||
+                        data.benchmark === benchmarkPath) {
+                        return data.metrics;
+                    }
+                }
+            }
+        }
+        catch (error) {
+            logger.warn("Could not load baseline benchmark", { error });
+        }
+        return null;
+    }
+    /**
+     * Calculate average metrics across multiple benchmark results
+     */
+    static calculateAverageMetrics(metricsArray) {
+        if (metricsArray.length === 0) {
+            return {
+                totalPnl: 0,
+                accuracy: 0,
+                winRate: 0,
+                optimality: 0,
+                benchmarkCount: 0,
+            };
+        }
+        const totals = metricsArray.reduce((acc, metrics) => ({
+            pnl: acc.pnl + metrics.totalPnl,
+            accuracy: acc.accuracy + metrics.predictionMetrics.accuracy,
+            winRate: acc.winRate + metrics.perpMetrics.winRate,
+            optimality: acc.optimality + metrics.optimalityScore,
+        }), { pnl: 0, accuracy: 0, winRate: 0, optimality: 0 });
+        const count = metricsArray.length;
+        return {
+            totalPnl: totals.pnl / count,
+            accuracy: totals.accuracy / count,
+            winRate: totals.winRate / count,
+            optimality: totals.optimality / count,
+            benchmarkCount: count,
+        };
+    }
+    /**
+     * Get baseline average metrics
+     */
+    static async getBaselineAverageMetrics() {
+        const baselinesDir = path.join(process.cwd(), "benchmarks", "baselines");
+        const metricsArray = [];
+        try {
+            const files = await fs.readdir(baselinesDir).catch(() => []);
+            for (const file of files) {
+                if (file.endsWith(".json")) {
+                    const filePath = path.join(baselinesDir, file);
+                    const data = JSON.parse(await fs.readFile(filePath, "utf-8"));
+                    if (data.metrics) {
+                        metricsArray.push(data.metrics);
+                    }
+                }
+            }
+        }
+        catch (error) {
+            logger.warn("Could not load baseline metrics", { error });
+        }
+        return ModelBenchmarkService.calculateAverageMetrics(metricsArray);
+    }
+    /**
+     * Get or create test agent for benchmarking
+     */
+    static async getOrCreateTestAgent() {
+        const testAgentUsername = "model-benchmark-agent";
+        const adapter = getTrainingDataAdapter();
+        const existing = await adapter.getUserByUsername(testAgentUsername);
+        if (existing) {
+            return existing.id;
+        }
+        // Create new test agent
+        const agentId = await generateSnowflakeId();
+        const agent = await adapter.createUser({
+            id: agentId,
+            privyId: `did:privy:model-benchmark-${agentId}`,
+            username: testAgentUsername,
+            displayName: "Model Benchmark Agent",
+            walletAddress: ethers.Wallet.createRandom().address,
+            isAgent: true,
+            virtualBalance: "10000",
+            reputationPoints: 1000,
+            isTest: true,
+            updatedAt: new Date(),
+        });
+        // Create agent config in separate table
+        if (agent) {
+            await adapter.createAgentConfig({
+                id: await generateSnowflakeId(),
+                userId: agentId,
+                autonomousTrading: true,
+                autonomousPosting: false,
+                autonomousCommenting: false,
+                systemPrompt: "You are a test agent for benchmarking model performance.",
+                modelTier: "pro",
+                updatedAt: new Date(),
+            });
+        }
+        if (!agent) {
+            throw new Error("Failed to create model benchmark test agent");
+        }
+        logger.info("Created model benchmark test agent", { agentId: agent.id });
+        return agent.id;
+    }
+    /**
+     * Get standard benchmark paths for model evaluation
+     */
+    static async getStandardBenchmarkPaths() {
+        const benchmarksDir = path.join(process.cwd(), "benchmarks");
+        const standardBenchmarks = [];
+        try {
+            // First, look in benchmarks/standard/ directory
+            const standardDir = path.join(benchmarksDir, "standard");
+            if (await fs
+                .access(standardDir)
+                .then(() => true)
+                .catch(() => false)) {
+                const standardFiles = await fs.readdir(standardDir);
+                for (const file of standardFiles) {
+                    if (file.startsWith("standard-") && file.endsWith(".json")) {
+                        standardBenchmarks.push(path.join(standardDir, file));
+                    }
+                }
+            }
+            // If standard benchmarks found, use those
+            if (standardBenchmarks.length > 0) {
+                logger.info(`Using ${standardBenchmarks.length} standard benchmarks from benchmarks/standard/`);
+                return standardBenchmarks;
+            }
+            // Fallback: Look for week-long benchmarks in main directory
+            const files = await fs.readdir(benchmarksDir);
+            for (const file of files) {
+                if (file.startsWith("benchmark-week-") && file.endsWith(".json")) {
+                    standardBenchmarks.push(path.join(benchmarksDir, file));
+                }
+            }
+            // If still nothing, use any benchmark files
+            if (standardBenchmarks.length === 0) {
+                for (const file of files) {
+                    if (file.startsWith("benchmark-") &&
+                        file.endsWith(".json") &&
+                        !file.includes("comparison")) {
+                        const filePath = path.join(benchmarksDir, file);
+                        standardBenchmarks.push(filePath);
+                    }
+                }
+            }
+        }
+        catch (error) {
+            logger.error("Could not load standard benchmarks", { error });
+        }
+        if (standardBenchmarks.length === 0) {
+            logger.warn("No standard benchmarks found. Generate benchmark fixtures before upload.");
+        }
+        return standardBenchmarks;
+    }
+}

package/dist/benchmark/ModelRegistry.js

@@ -0,0 +1,122 @@
+/**
+ * Model Registry
+ *
+ * Centralized configuration for all models available for benchmarking.
+ * Add new models here to make them available for comparison.
+ */
+/**
+ * Registry of all available models for benchmarking
+ */
+export const MODEL_REGISTRY = [
+    {
+        id: "llama-8b",
+        displayName: "LLaMA 3.1 8B",
+        provider: "groq",
+        modelId: "llama-3.1-8b-instant",
+        tier: "lite",
+        parametersBillions: 8,
+        isBaseline: true,
+    },
+    {
+        id: "llama-70b",
+        displayName: "LLaMA 3.1 70B",
+        provider: "groq",
+        modelId: "llama-3.1-70b-versatile",
+        tier: "standard",
+        parametersBillions: 70,
+        isBaseline: false,
+    },
+    {
+        id: "qwen-32b",
+        displayName: "Qwen 3 32B",
+        provider: "groq",
+        modelId: "qwen/qwen3-32b",
+        tier: "standard",
+        parametersBillions: 32,
+        isBaseline: true,
+    },
+    {
+        id: "mixtral-8x7b",
+        displayName: "Mixtral 8x7B",
+        provider: "groq",
+        modelId: "mixtral-8x7b-32768",
+        tier: "standard",
+        parametersBillions: 46,
+        isBaseline: false,
+    },
+    {
+        id: "gpt-4o",
+        displayName: "GPT-4o",
+        provider: "openai",
+        modelId: "gpt-4o",
+        tier: "pro",
+        isBaseline: false,
+    },
+    {
+        id: "gpt-4o-mini",
+        displayName: "GPT-4o Mini",
+        provider: "openai",
+        modelId: "gpt-4o-mini",
+        tier: "lite",
+        isBaseline: false,
+    },
+    {
+        id: "claude-sonnet",
+        displayName: "Claude 3.5 Sonnet",
+        provider: "anthropic",
+        modelId: "claude-3-5-sonnet-20241022",
+        tier: "pro",
+        isBaseline: false,
+    },
+    {
+        id: "claude-haiku",
+        displayName: "Claude 3.5 Haiku",
+        provider: "anthropic",
+        modelId: "claude-3-5-haiku-20241022",
+        tier: "lite",
+        isBaseline: false,
+    },
+];
+/**
+ * Get a model config by ID
+ */
+export function getModelById(id) {
+    return MODEL_REGISTRY.find((m) => m.id === id);
+}
+/**
+ * Get a model config by model ID (API identifier)
+ */
+export function getModelByModelId(modelId) {
+    return MODEL_REGISTRY.find((m) => m.modelId === modelId);
+}
+/**
+ * Get all baseline models
+ */
+export function getBaselineModels() {
+    return MODEL_REGISTRY.filter((m) => m.isBaseline);
+}
+/**
+ * Get models by provider
+ */
+export function getModelsByProvider(provider) {
+    return MODEL_REGISTRY.filter((m) => m.provider === provider);
+}
+/**
+ * Get models by tier
+ */
+export function getModelsByTier(tier) {
+    return MODEL_REGISTRY.filter((m) => m.tier === tier);
+}
+/**
+ * Validate that a model ID exists
+ */
+export function validateModelId(id) {
+    return MODEL_REGISTRY.some((m) => m.id === id || m.modelId === id);
+}
+/**
+ * Get model display name (supports both id and modelId)
+ */
+export function getModelDisplayName(idOrModelId) {
+    const model = getModelById(idOrModelId) ?? getModelByModelId(idOrModelId);
+    return model?.displayName ?? idOrModelId;
+}