@elizaos/training 2.0.0-alpha.21 → 2.0.0-alpha.22

package/dist/benchmark/BenchmarkDataViewer.js

@@ -0,0 +1,197 @@
+/**
+ * Benchmark Data Viewer
+ *
+ * Provides utilities to view and inspect benchmark data.
+ * Useful for validation and understanding benchmark structure.
+ */
+import { promises as fs } from "node:fs";
+import * as BenchmarkValidator from "./BenchmarkValidator";
+function analyzeTicks(ticks) {
+    const eventTypes = {};
+    let withEvents = 0;
+    for (const tick of ticks) {
+        if (tick.events.length > 0) {
+            withEvents++;
+        }
+        for (const event of tick.events) {
+            eventTypes[event.type] = (eventTypes[event.type] || 0) + 1;
+        }
+    }
+    return {
+        total: ticks.length,
+        withEvents,
+        eventTypes,
+    };
+}
+function analyzeGroundTruth(groundTruth) {
+    return {
+        marketOutcomes: Object.keys(groundTruth.marketOutcomes).length,
+        priceHistory: Object.fromEntries(Object.entries(groundTruth.priceHistory).map(([ticker, history]) => [
+            ticker,
+            history.length,
+        ])),
+        optimalActions: groundTruth.optimalActions.length,
+        socialOpportunities: groundTruth.socialOpportunities.length,
+        hiddenFacts: groundTruth.hiddenFacts?.length || 0,
+        hiddenEvents: groundTruth.hiddenEvents?.length || 0,
+        trueFacts: Object.keys(groundTruth.trueFacts || {}),
+    };
+}
+/**
+ * Load and view a benchmark file
+ */
+export async function viewBenchmark(filePath, options = {}) {
+    const data = await fs.readFile(filePath, "utf-8");
+    const snapshot = JSON.parse(data);
+    // Validate
+    const validation = BenchmarkValidator.validate(snapshot);
+    // Build view
+    const view = {
+        id: snapshot.id,
+        version: snapshot.version,
+        createdAt: snapshot.createdAt,
+        duration: snapshot.duration,
+        tickInterval: snapshot.tickInterval,
+        initialState: {
+            predictionMarkets: snapshot.initialState.predictionMarkets.length,
+            perpetualMarkets: snapshot.initialState.perpetualMarkets.length,
+            agents: snapshot.initialState.agents.length,
+            posts: snapshot.initialState.posts?.length || 0,
+            groupChats: snapshot.initialState.groupChats?.length || 0,
+        },
+        ticks: analyzeTicks(snapshot.ticks),
+        validation,
+    };
+    if (options.showGroundTruth || options.verbose) {
+        view.groundTruth = analyzeGroundTruth(snapshot.groundTruth);
+    }
+    return view;
+}
+/**
+ * Print view to console
+ */
+export function printBenchmarkView(view, options = {}) {
+    console.log("\n📊 Benchmark Data View\n");
+    console.log(`ID: ${view.id}`);
+    console.log(`Version: ${view.version}`);
+    console.log(`Created: ${new Date(view.createdAt).toISOString()}`);
+    console.log(`Duration: ${(view.duration / 60).toFixed(1)} minutes`);
+    console.log(`Tick Interval: ${view.tickInterval}s`);
+    console.log("\n📈 Initial State:");
+    console.log(`  Prediction Markets: ${view.initialState.predictionMarkets}`);
+    console.log(`  Perpetual Markets: ${view.initialState.perpetualMarkets}`);
+    console.log(`  Agents: ${view.initialState.agents}`);
+    console.log(`  Posts: ${view.initialState.posts}`);
+    console.log(`  Group Chats: ${view.initialState.groupChats}`);
+    console.log("\n⏱️  Ticks:");
+    console.log(`  Total: ${view.ticks.total}`);
+    console.log(`  With Events: ${view.ticks.withEvents}`);
+    if (options.verbose) {
+        console.log(`  Event Types:`);
+        for (const [type, count] of Object.entries(view.ticks.eventTypes)) {
+            console.log(`    ${type}: ${count}`);
+        }
+    }
+    if (view.groundTruth) {
+        console.log("\n🎯 Ground Truth:");
+        console.log(`  Market Outcomes: ${view.groundTruth.marketOutcomes}`);
+        console.log(`  Price History:`);
+        for (const [ticker, count] of Object.entries(view.groundTruth.priceHistory)) {
+            console.log(`    ${ticker}: ${count} ticks`);
+        }
+        console.log(`  Optimal Actions: ${view.groundTruth.optimalActions}`);
+        console.log(`  Social Opportunities: ${view.groundTruth.socialOpportunities}`);
+        if (options.showHidden) {
+            console.log(`  Hidden Facts: ${view.groundTruth.hiddenFacts}`);
+            console.log(`  Hidden Events: ${view.groundTruth.hiddenEvents}`);
+            console.log(`  True Facts: ${view.groundTruth.trueFacts.join(", ")}`);
+        }
+    }
+    console.log("\n✅ Validation:");
+    console.log(`  Valid: ${view.validation.valid ? "✅" : "❌"}`);
+    if (view.validation.errors.length > 0) {
+        console.log(`  Errors: ${view.validation.errors.length}`);
+        if (options.verbose) {
+            for (const error of view.validation.errors) {
+                console.log(`    ❌ ${error}`);
+            }
+        }
+    }
+    if (view.validation.warnings.length > 0) {
+        console.log(`  Warnings: ${view.validation.warnings.length}`);
+        if (options.verbose) {
+            for (const warning of view.validation.warnings) {
+                console.log(`    ⚠️  ${warning}`);
+            }
+        }
+    }
+    console.log("");
+}
+/**
+ * Get tick details
+ */
+export function getTickDetails(snapshot, tickNumber) {
+    const tick = snapshot.ticks[tickNumber] || null;
+    if (!tick) {
+        return { tick: null, state: null, events: [] };
+    }
+    return {
+        tick,
+        state: tick.state,
+        events: tick.events.map((e) => ({
+            type: e.type,
+            data: e.data,
+        })),
+    };
+}
+/**
+ * Get ground truth for a specific tick
+ */
+export function getGroundTruthForTick(snapshot, tickNumber) {
+    const gt = snapshot.groundTruth;
+    return {
+        hiddenFacts: (gt.hiddenFacts || [])
+            .filter((f) => f.tick === tickNumber)
+            .map((f) => ({ fact: f.fact, category: f.category })),
+        hiddenEvents: (gt.hiddenEvents || [])
+            .filter((e) => e.tick === tickNumber)
+            .map((e) => ({ type: e.type, description: e.description })),
+        marketOutcomes: gt.marketOutcomes,
+    };
+}
+/**
+ * Check if agent can access hidden facts (should always be false)
+ */
+export function verifyAgentCannotAccessHiddenFacts(snapshot) {
+    // Agents can only access game state via SimulationA2AInterface
+    // Ground truth is stored separately and not exposed
+    // This is a verification check
+    const state = snapshot.initialState;
+    const hasGroundTruth = !!snapshot.groundTruth;
+    const hasHiddenFacts = !!snapshot.groundTruth?.hiddenFacts?.length;
+    // Check if ground truth is accidentally in state
+    const stateKeys = Object.keys(state);
+    const hasGroundTruthInState = stateKeys.includes("groundTruth") ||
+        stateKeys.includes("hiddenFacts") ||
+        stateKeys.includes("hiddenEvents");
+    if (hasGroundTruthInState) {
+        return {
+            canAccess: true,
+            reason: "Ground truth found in game state (security issue!)",
+        };
+    }
+    return {
+        canAccess: false,
+        reason: hasGroundTruth && hasHiddenFacts
+            ? "Ground truth exists but is properly isolated from game state"
+            : "No ground truth data found",
+    };
+}
+/** @deprecated Use viewBenchmark, printBenchmarkView, etc. instead */
+export const BenchmarkDataViewer = {
+    view: viewBenchmark,
+    print: printBenchmarkView,
+    getTickDetails,
+    getGroundTruthForTick,
+    verifyAgentCannotAccessHiddenFacts,
+};

package/dist/benchmark/BenchmarkHistoryService.js

@@ -0,0 +1,135 @@
+/**
+ * Benchmark History Service
+ *
+ * Persists benchmark results to the database for historical tracking and analysis.
+ */
+import { getTrainingDataAdapter, } from "../adapter";
+import { logger } from "../utils/logger";
+import { generateSnowflakeId } from "../utils/snowflake";
+/**
+ * Service for managing benchmark result history
+ */
+// biome-ignore lint/complexity/noStaticOnlyClass: Service namespace - methods are logically grouped
+export class BenchmarkHistoryService {
+    /**
+     * Save a benchmark result to the database
+     */
+    static async saveResult(input) {
+        const id = await generateSnowflakeId();
+        const now = new Date();
+        const insertData = {
+            id,
+            modelId: input.modelId,
+            benchmarkId: input.benchmarkId,
+            benchmarkPath: input.benchmarkPath,
+            runAt: now,
+            totalPnl: input.metrics.totalPnl,
+            predictionAccuracy: input.metrics.predictionMetrics.accuracy,
+            perpWinRate: input.metrics.perpMetrics.winRate,
+            optimalityScore: input.metrics.optimalityScore,
+            detailedMetrics: JSON.parse(JSON.stringify(input.metrics)),
+            baselinePnlDelta: input.baselineComparison?.pnlDelta ?? null,
+            baselineAccuracyDelta: input.baselineComparison?.accuracyDelta ?? null,
+            improved: input.baselineComparison?.improved ?? null,
+            duration: input.duration,
+        };
+        await getTrainingDataAdapter().insertBenchmarkResult(insertData);
+        logger.info("Saved benchmark result", {
+            id,
+            modelId: input.modelId,
+            benchmarkId: input.benchmarkId,
+            totalPnl: input.metrics.totalPnl,
+        });
+        return { ...insertData, createdAt: now };
+    }
+    /**
+     * Get benchmark results by query
+     */
+    static async getResults(query) {
+        return getTrainingDataAdapter().queryBenchmarkResults({
+            modelId: query.modelId,
+            benchmarkId: query.benchmarkId,
+            startDate: query.startDate,
+            endDate: query.endDate,
+            limit: query.limit ?? 100,
+        });
+    }
+    /**
+     * Get the latest result for a model
+     */
+    static async getLatestResult(modelId) {
+        const results = await getTrainingDataAdapter().queryBenchmarkResults({
+            modelId,
+            limit: 1,
+        });
+        return results[0] ?? null;
+    }
+    /**
+     * Get trend data for a model
+     */
+    static async getTrendData(modelId, limit = 20) {
+        const results = await getTrainingDataAdapter().queryBenchmarkResults({
+            modelId,
+            limit,
+        });
+        // queryBenchmarkResults returns desc by runAt, reverse for chronological
+        const chronological = results.reverse();
+        return {
+            modelId,
+            dates: chronological.map((r) => r.runAt),
+            pnlHistory: chronological.map((r) => r.totalPnl),
+            accuracyHistory: chronological.map((r) => r.predictionAccuracy),
+            optimalityHistory: chronological.map((r) => r.optimalityScore),
+        };
+    }
+    /**
+     * Get comparison data for multiple models
+     */
+    static async getModelComparison(modelIds, benchmarkId) {
+        const adapter = getTrainingDataAdapter();
+        const comparison = new Map();
+        for (const modelId of modelIds) {
+            const results = await adapter.queryBenchmarkResults({
+                modelId,
+                benchmarkId,
+                limit: 10,
+            });
+            comparison.set(modelId, results);
+        }
+        return comparison;
+    }
+    /**
+     * Get summary statistics for all models
+     */
+    static async getModelSummary() {
+        return getTrainingDataAdapter().getBenchmarkModelSummary();
+    }
+    /**
+     * Check if a model improved vs baseline
+     */
+    static async checkImprovement(modelId, baselineModelId, benchmarkId) {
+        const adapter = getTrainingDataAdapter();
+        const modelResults = await adapter.queryBenchmarkResults({
+            modelId,
+            benchmarkId,
+            limit: 1,
+        });
+        const baselineResults = await adapter.queryBenchmarkResults({
+            modelId: baselineModelId,
+            benchmarkId,
+            limit: 1,
+        });
+        const modelResult = modelResults[0];
+        const baselineResult = baselineResults[0];
+        if (!modelResult || !baselineResult) {
+            return null;
+        }
+        const delta = modelResult.totalPnl - baselineResult.totalPnl;
+        return {
+            improved: delta > 0,
+            modelPnl: modelResult.totalPnl,
+            baselinePnl: baselineResult.totalPnl,
+            delta,
+        };
+    }
+}