@elizaos/training 2.0.0-alpha.77 → 2.0.0-alpha.78

package/dist/benchmark/BenchmarkDataViewer.js

@@ -1,197 +0,0 @@
-/**
- * Benchmark Data Viewer
- *
- * Provides utilities to view and inspect benchmark data.
- * Useful for validation and understanding benchmark structure.
- */
-import { promises as fs } from "node:fs";
-import * as BenchmarkValidator from "./BenchmarkValidator";
-function analyzeTicks(ticks) {
-    const eventTypes = {};
-    let withEvents = 0;
-    for (const tick of ticks) {
-        if (tick.events.length > 0) {
-            withEvents++;
-        }
-        for (const event of tick.events) {
-            eventTypes[event.type] = (eventTypes[event.type] || 0) + 1;
-        }
-    }
-    return {
-        total: ticks.length,
-        withEvents,
-        eventTypes,
-    };
-}
-function analyzeGroundTruth(groundTruth) {
-    return {
-        marketOutcomes: Object.keys(groundTruth.marketOutcomes).length,
-        priceHistory: Object.fromEntries(Object.entries(groundTruth.priceHistory).map(([ticker, history]) => [
-            ticker,
-            history.length,
-        ])),
-        optimalActions: groundTruth.optimalActions.length,
-        socialOpportunities: groundTruth.socialOpportunities.length,
-        hiddenFacts: groundTruth.hiddenFacts?.length || 0,
-        hiddenEvents: groundTruth.hiddenEvents?.length || 0,
-        trueFacts: Object.keys(groundTruth.trueFacts || {}),
-    };
-}
-/**
- * Load and view a benchmark file
- */
-export async function viewBenchmark(filePath, options = {}) {
-    const data = await fs.readFile(filePath, "utf-8");
-    const snapshot = JSON.parse(data);
-    // Validate
-    const validation = BenchmarkValidator.validate(snapshot);
-    // Build view
-    const view = {
-        id: snapshot.id,
-        version: snapshot.version,
-        createdAt: snapshot.createdAt,
-        duration: snapshot.duration,
-        tickInterval: snapshot.tickInterval,
-        initialState: {
-            predictionMarkets: snapshot.initialState.predictionMarkets.length,
-            perpetualMarkets: snapshot.initialState.perpetualMarkets.length,
-            agents: snapshot.initialState.agents.length,
-            posts: snapshot.initialState.posts?.length || 0,
-            groupChats: snapshot.initialState.groupChats?.length || 0,
-        },
-        ticks: analyzeTicks(snapshot.ticks),
-        validation,
-    };
-    if (options.showGroundTruth || options.verbose) {
-        view.groundTruth = analyzeGroundTruth(snapshot.groundTruth);
-    }
-    return view;
-}
-/**
- * Print view to console
- */
-export function printBenchmarkView(view, options = {}) {
-    console.log("\n📊 Benchmark Data View\n");
-    console.log(`ID: ${view.id}`);
-    console.log(`Version: ${view.version}`);
-    console.log(`Created: ${new Date(view.createdAt).toISOString()}`);
-    console.log(`Duration: ${(view.duration / 60).toFixed(1)} minutes`);
-    console.log(`Tick Interval: ${view.tickInterval}s`);
-    console.log("\n📈 Initial State:");
-    console.log(`  Prediction Markets: ${view.initialState.predictionMarkets}`);
-    console.log(`  Perpetual Markets: ${view.initialState.perpetualMarkets}`);
-    console.log(`  Agents: ${view.initialState.agents}`);
-    console.log(`  Posts: ${view.initialState.posts}`);
-    console.log(`  Group Chats: ${view.initialState.groupChats}`);
-    console.log("\n⏱️  Ticks:");
-    console.log(`  Total: ${view.ticks.total}`);
-    console.log(`  With Events: ${view.ticks.withEvents}`);
-    if (options.verbose) {
-        console.log(`  Event Types:`);
-        for (const [type, count] of Object.entries(view.ticks.eventTypes)) {
-            console.log(`    ${type}: ${count}`);
-        }
-    }
-    if (view.groundTruth) {
-        console.log("\n🎯 Ground Truth:");
-        console.log(`  Market Outcomes: ${view.groundTruth.marketOutcomes}`);
-        console.log(`  Price History:`);
-        for (const [ticker, count] of Object.entries(view.groundTruth.priceHistory)) {
-            console.log(`    ${ticker}: ${count} ticks`);
-        }
-        console.log(`  Optimal Actions: ${view.groundTruth.optimalActions}`);
-        console.log(`  Social Opportunities: ${view.groundTruth.socialOpportunities}`);
-        if (options.showHidden) {
-            console.log(`  Hidden Facts: ${view.groundTruth.hiddenFacts}`);
-            console.log(`  Hidden Events: ${view.groundTruth.hiddenEvents}`);
-            console.log(`  True Facts: ${view.groundTruth.trueFacts.join(", ")}`);
-        }
-    }
-    console.log("\n✅ Validation:");
-    console.log(`  Valid: ${view.validation.valid ? "✅" : "❌"}`);
-    if (view.validation.errors.length > 0) {
-        console.log(`  Errors: ${view.validation.errors.length}`);
-        if (options.verbose) {
-            for (const error of view.validation.errors) {
-                console.log(`    ❌ ${error}`);
-            }
-        }
-    }
-    if (view.validation.warnings.length > 0) {
-        console.log(`  Warnings: ${view.validation.warnings.length}`);
-        if (options.verbose) {
-            for (const warning of view.validation.warnings) {
-                console.log(`    ⚠️  ${warning}`);
-            }
-        }
-    }
-    console.log("");
-}
-/**
- * Get tick details
- */
-export function getTickDetails(snapshot, tickNumber) {
-    const tick = snapshot.ticks[tickNumber] || null;
-    if (!tick) {
-        return { tick: null, state: null, events: [] };
-    }
-    return {
-        tick,
-        state: tick.state,
-        events: tick.events.map((e) => ({
-            type: e.type,
-            data: e.data,
-        })),
-    };
-}
-/**
- * Get ground truth for a specific tick
- */
-export function getGroundTruthForTick(snapshot, tickNumber) {
-    const gt = snapshot.groundTruth;
-    return {
-        hiddenFacts: (gt.hiddenFacts || [])
-            .filter((f) => f.tick === tickNumber)
-            .map((f) => ({ fact: f.fact, category: f.category })),
-        hiddenEvents: (gt.hiddenEvents || [])
-            .filter((e) => e.tick === tickNumber)
-            .map((e) => ({ type: e.type, description: e.description })),
-        marketOutcomes: gt.marketOutcomes,
-    };
-}
-/**
- * Check if agent can access hidden facts (should always be false)
- */
-export function verifyAgentCannotAccessHiddenFacts(snapshot) {
-    // Agents can only access game state via SimulationA2AInterface
-    // Ground truth is stored separately and not exposed
-    // This is a verification check
-    const state = snapshot.initialState;
-    const hasGroundTruth = !!snapshot.groundTruth;
-    const hasHiddenFacts = !!snapshot.groundTruth?.hiddenFacts?.length;
-    // Check if ground truth is accidentally in state
-    const stateKeys = Object.keys(state);
-    const hasGroundTruthInState = stateKeys.includes("groundTruth") ||
-        stateKeys.includes("hiddenFacts") ||
-        stateKeys.includes("hiddenEvents");
-    if (hasGroundTruthInState) {
-        return {
-            canAccess: true,
-            reason: "Ground truth found in game state (security issue!)",
-        };
-    }
-    return {
-        canAccess: false,
-        reason: hasGroundTruth && hasHiddenFacts
-            ? "Ground truth exists but is properly isolated from game state"
-            : "No ground truth data found",
-    };
-}
-/** @deprecated Use viewBenchmark, printBenchmarkView, etc. instead */
-export const BenchmarkDataViewer = {
-    view: viewBenchmark,
-    print: printBenchmarkView,
-    getTickDetails,
-    getGroundTruthForTick,
-    verifyAgentCannotAccessHiddenFacts,
-};

package/dist/benchmark/BenchmarkHistoryService.js

@@ -1,135 +0,0 @@
-/**
- * Benchmark History Service
- *
- * Persists benchmark results to the database for historical tracking and analysis.
- */
-import { getTrainingDataAdapter, } from "../adapter";
-import { logger } from "../utils/logger";
-import { generateSnowflakeId } from "../utils/snowflake";
-/**
- * Service for managing benchmark result history
- */
-// biome-ignore lint/complexity/noStaticOnlyClass: Service namespace - methods are logically grouped
-export class BenchmarkHistoryService {
-    /**
-     * Save a benchmark result to the database
-     */
-    static async saveResult(input) {
-        const id = await generateSnowflakeId();
-        const now = new Date();
-        const insertData = {
-            id,
-            modelId: input.modelId,
-            benchmarkId: input.benchmarkId,
-            benchmarkPath: input.benchmarkPath,
-            runAt: now,
-            totalPnl: input.metrics.totalPnl,
-            predictionAccuracy: input.metrics.predictionMetrics.accuracy,
-            perpWinRate: input.metrics.perpMetrics.winRate,
-            optimalityScore: input.metrics.optimalityScore,
-            detailedMetrics: JSON.parse(JSON.stringify(input.metrics)),
-            baselinePnlDelta: input.baselineComparison?.pnlDelta ?? null,
-            baselineAccuracyDelta: input.baselineComparison?.accuracyDelta ?? null,
-            improved: input.baselineComparison?.improved ?? null,
-            duration: input.duration,
-        };
-        await getTrainingDataAdapter().insertBenchmarkResult(insertData);
-        logger.info("Saved benchmark result", {
-            id,
-            modelId: input.modelId,
-            benchmarkId: input.benchmarkId,
-            totalPnl: input.metrics.totalPnl,
-        });
-        return { ...insertData, createdAt: now };
-    }
-    /**
-     * Get benchmark results by query
-     */
-    static async getResults(query) {
-        return getTrainingDataAdapter().queryBenchmarkResults({
-            modelId: query.modelId,
-            benchmarkId: query.benchmarkId,
-            startDate: query.startDate,
-            endDate: query.endDate,
-            limit: query.limit ?? 100,
-        });
-    }
-    /**
-     * Get the latest result for a model
-     */
-    static async getLatestResult(modelId) {
-        const results = await getTrainingDataAdapter().queryBenchmarkResults({
-            modelId,
-            limit: 1,
-        });
-        return results[0] ?? null;
-    }
-    /**
-     * Get trend data for a model
-     */
-    static async getTrendData(modelId, limit = 20) {
-        const results = await getTrainingDataAdapter().queryBenchmarkResults({
-            modelId,
-            limit,
-        });
-        // queryBenchmarkResults returns desc by runAt, reverse for chronological
-        const chronological = results.reverse();
-        return {
-            modelId,
-            dates: chronological.map((r) => r.runAt),
-            pnlHistory: chronological.map((r) => r.totalPnl),
-            accuracyHistory: chronological.map((r) => r.predictionAccuracy),
-            optimalityHistory: chronological.map((r) => r.optimalityScore),
-        };
-    }
-    /**
-     * Get comparison data for multiple models
-     */
-    static async getModelComparison(modelIds, benchmarkId) {
-        const adapter = getTrainingDataAdapter();
-        const comparison = new Map();
-        for (const modelId of modelIds) {
-            const results = await adapter.queryBenchmarkResults({
-                modelId,
-                benchmarkId,
-                limit: 10,
-            });
-            comparison.set(modelId, results);
-        }
-        return comparison;
-    }
-    /**
-     * Get summary statistics for all models
-     */
-    static async getModelSummary() {
-        return getTrainingDataAdapter().getBenchmarkModelSummary();
-    }
-    /**
-     * Check if a model improved vs baseline
-     */
-    static async checkImprovement(modelId, baselineModelId, benchmarkId) {
-        const adapter = getTrainingDataAdapter();
-        const modelResults = await adapter.queryBenchmarkResults({
-            modelId,
-            benchmarkId,
-            limit: 1,
-        });
-        const baselineResults = await adapter.queryBenchmarkResults({
-            modelId: baselineModelId,
-            benchmarkId,
-            limit: 1,
-        });
-        const modelResult = modelResults[0];
-        const baselineResult = baselineResults[0];
-        if (!modelResult || !baselineResult) {
-            return null;
-        }
-        const delta = modelResult.totalPnl - baselineResult.totalPnl;
-        return {
-            improved: delta > 0,
-            modelPnl: modelResult.totalPnl,
-            baselinePnl: baselineResult.totalPnl,
-            delta,
-        };
-    }
-}