@elizaos/training 2.0.0-alpha.77 → 2.0.0-alpha.78

package/dist/training/RewardBackpropagationService.js

@@ -1,117 +0,0 @@
-/**
- * Reward Backpropagation Service
- *
- * Updates trajectory rewards when market outcomes become known.
- * This allows the RL model to learn from actual results, not just immediate actions.
- */
-import { getMarketDataAdapter, getTrainingDataAdapter } from "../adapter";
-import { logger } from "../utils/logger";
-import { MarketOutcomesTracker } from "./MarketOutcomesTracker";
-export class RewardBackpropagationService {
-    outcomesTracker;
-    constructor() {
-        this.outcomesTracker = new MarketOutcomesTracker();
-    }
-    /**
-     * Update rewards for trajectories in a window when outcomes become known
-     */
-    async updateRewardsForWindow(windowId) {
-        logger.info("Updating rewards for window", { windowId });
-        // Get outcomes for this window
-        const outcomes = await this.outcomesTracker.getWindowOutcomes(windowId);
-        if (!outcomes) {
-            logger.info("No outcomes found for window", { windowId });
-            return 0;
-        }
-        // Get all trajectories for this window (filter to training data)
-        const allTrajectories = await getTrainingDataAdapter().getTrajectoriesByWindow(windowId);
-        const trajectoriesResult = allTrajectories.filter((t) => t.isTrainingData);
-        let updated = 0;
-        for (const traj of trajectoriesResult) {
-            if (!traj.stepsJson)
-                continue;
-            const steps = JSON.parse(traj.stepsJson);
-            let totalReward = 0;
-            let hasUpdates = false;
-            // Update rewards for each step based on outcomes
-            for (const step of steps) {
-                const originalReward = step.reward;
-                let updatedReward = originalReward;
-                // Check if this step involved trading
-                if (step.action.actionType.includes("TRADING") ||
-                    step.action.actionType.includes("BUY") ||
-                    step.action.actionType.includes("SELL")) {
-                    // Extract market ID from action parameters
-                    const marketId = step.action.parameters?.marketId;
-                    const ticker = step.action.parameters?.ticker;
-                    if (marketId) {
-                        // Check prediction market outcome
-                        const prediction = outcomes.predictions.find((p) => p.marketId === marketId);
-                        if (prediction) {
-                            // Calculate reward based on whether trade was correct
-                            const side = step.action.parameters?.side;
-                            const isCorrect = (side === "YES" && prediction.outcome === "YES") ||
-                                (side === "NO" && prediction.outcome === "NO");
-                            // Reward: +1 for correct, -1 for incorrect (normalized)
-                            updatedReward = isCorrect ? 1.0 : -1.0;
-                        }
-                    }
-                    else if (ticker) {
-                        // Check perpetual outcome
-                        const stock = outcomes.stocks.find((s) => s.ticker === ticker);
-                        if (stock) {
-                            // Calculate reward based on price movement
-                            const side = step.action.parameters?.side;
-                            const priceChange = stock.changePercent;
-                            // Reward based on whether position direction matched price movement
-                            // Long position: positive reward if price went up
-                            // Short position: positive reward if price went down
-                            if (side === "long") {
-                                updatedReward = Math.max(-1, Math.min(1, priceChange / 10)); // Normalize to -1 to 1
-                            }
-                            else if (side === "short") {
-                                updatedReward = Math.max(-1, Math.min(1, -priceChange / 10)); // Inverted for short
-                            }
-                        }
-                    }
-                }
-                if (updatedReward !== originalReward) {
-                    step.reward = updatedReward;
-                    hasUpdates = true;
-                }
-                totalReward += step.reward;
-            }
-            // Update trajectory if rewards changed
-            if (hasUpdates) {
-                await getTrainingDataAdapter().updateTrajectoryRewards(traj.id, JSON.stringify(steps), totalReward);
-                updated++;
-            }
-        }
-        logger.info("Updated rewards for trajectories", {
-            windowId,
-            updated,
-            total: trajectoriesResult.length,
-        });
-        return updated;
-    }
-    /**
-     * Process all windows that have outcomes but haven't been updated
-     */
-    async processPendingWindows() {
-        // Get all windows with outcomes
-        const marketAdapter = getMarketDataAdapter();
-        if (!marketAdapter) {
-            return 0;
-        }
-        const windowIds = await marketAdapter.getDistinctWindowsWithOutcomes();
-        let processed = 0;
-        for (const windowId of windowIds) {
-            const updated = await this.updateRewardsForWindow(windowId);
-            if (updated > 0) {
-                processed++;
-            }
-        }
-        return processed;
-    }
-}
-export const rewardBackpropagationService = new RewardBackpropagationService();

package/dist/training/RulerScoringService.js

@@ -1,450 +0,0 @@
-/**
- * RULER Scoring Service
- *
- * Implements RULER (Relative Universal LLM-Elicited Rewards) using LLM-as-judge.
- *
- * Key features:
- * - Groups trajectories by scenarioId for relative comparison
- * - Uses LLM judge to score trajectories relative to each other (0-1)
- * - Injects game context (P&L, episode length, actions) into judge prompt
- * - Deduplicates common message prefixes to save tokens
- * - Works with any LiteLLM-compatible provider (Groq, OpenAI, etc.)
- *
- * Based on: https://art.openpipe.ai/fundamentals/ruler
- */
-import { getTrainingDataAdapter } from "../adapter";
-/** Cast string to UUID (replaces @elizaos/core asUUID) */
-function asUUID(id) {
-    return id;
-}
-import { v4 as uuidv4 } from "uuid";
-import { getLLMCaller, getToTrainingMessages, } from "../dependencies";
-import { getRubric, sanitizeArchetype } from "../rubrics";
-import { logger, splitIntoBatches } from "../utils";
-/**
- * Default RULER rubric - works well for most RL tasks
- */
-const DEFAULT_RUBRIC = `
-- A trajectory that achieves its goal should always get a significantly higher score than a trajectory that does not achieve its goal.
-- A trajectory that achieves its goal more efficiently (eg. by avoiding unproductive detours) should get a higher score than a trajectory that achieves its goal less efficiently.
-- If one trajectory is only slightly better than another, the difference in scores should be small. If it is significantly better, the difference in scores should be large.
-- You may give some partial credit for a trajectory that makes progress towards its goal but does not complete it.
-`;
-export class RulerScoringService {
-    minGroupSize = 2; // Minimum trajectories per group for comparison
-    maxGroupSize = 8; // Optimal group size per RULER docs
-    /**
-     * Score trajectories using RULER (LLM-as-judge with relative comparison)
-     *
-     * Groups trajectories by scenarioId and scores them relative to each other.
-     * This is the proper RULER implementation - not simple heuristics!
-     *
-     * @param trajectoryIds - Optional: specific trajectory IDs to score. If not provided, scores all unscored trajectories.
-     * @returns Number of trajectories successfully scored
-     */
-    async scoreTrajectories(trajectoryIds) {
-        const trajectoriesResult = await this.getTrajectoriesToScore(trajectoryIds);
-        if (trajectoriesResult.length === 0) {
-            logger.info("No trajectories to score", {}, "RulerScoring");
-            return 0;
-        }
-        const groups = this.groupByScenario(trajectoriesResult);
-        logger.info("Grouped trajectories for RULER scoring", {
-            totalTrajectories: trajectoriesResult.length,
-            groups: groups.length,
-            avgGroupSize: groups.length > 0 ? trajectoriesResult.length / groups.length : 0,
-        }, "RulerScoring");
-        let totalScored = 0;
-        for (const group of groups) {
-            if (group.trajectories.length < this.minGroupSize) {
-                logger.warn("Skipping group with insufficient trajectories", {
-                    scenarioId: group.scenarioId,
-                    count: group.trajectories.length,
-                    minRequired: this.minGroupSize,
-                }, "RulerScoring");
-                continue;
-            }
-            const batches = splitIntoBatches(group.trajectories, this.maxGroupSize);
-            for (const batch of batches) {
-                const scored = await this.scoreGroup(batch, group.scenarioId);
-                totalScored += scored;
-            }
-        }
-        logger.info("RULER scoring complete", {
-            totalScored,
-            totalTrajectories: trajectoriesResult.length,
-        }, "RulerScoring");
-        return totalScored;
-    }
-    /**
-     * Score a single trajectory (for backward compatibility)
-     *
-     * Note: RULER works best with groups, so this finds other trajectories
-     * in the same scenario and scores them together.
-     */
-    async scoreTrajectory(trajectoryId) {
-        const scored = await this.scoreTrajectories([trajectoryId]);
-        if (scored === 0) {
-            return null;
-        }
-        const updated = await getTrainingDataAdapter().getTrajectoryById(trajectoryId);
-        if (!updated || updated.aiJudgeReward === null) {
-            return null;
-        }
-        return {
-            trajectoryId: updated.trajectoryId,
-            overallScore: updated.aiJudgeReward,
-            reasoning: updated.aiJudgeReasoning || "",
-            scoredAt: updated.judgedAt || new Date(),
-        };
-    }
-    /**
-     * Score a group of trajectories using RULER
-     *
-     * This is the core RULER implementation:
-     * 1. Convert trajectories to message format
-     * 2. Extract common prefix (deduplication)
-     * 3. Build judge prompt with context (P&L, episode length, etc.)
-     * 4. Call LLM judge to score trajectories relative to each other
-     * 5. Save scores to database
-     */
-    async scoreGroup(trajectoriesData, scenarioId) {
-        const richTrajectories = [];
-        for (const dbTraj of trajectoriesData) {
-            if (!dbTraj.stepsJson ||
-                dbTraj.stepsJson === "null" ||
-                dbTraj.stepsJson === "[]") {
-                logger.warn("Skipping trajectory with invalid stepsJson", {
-                    trajectoryId: dbTraj.trajectoryId,
-                }, "RulerScoring");
-                continue;
-            }
-            const steps = JSON.parse(dbTraj.stepsJson);
-            const stepTimestamp = Date.now();
-            const richTraj = {
-                trajectoryId: asUUID(dbTraj.trajectoryId),
-                agentId: asUUID(uuidv4()),
-                startTime: 0,
-                endTime: 0,
-                durationMs: 0,
-                scenarioId: dbTraj.scenarioId || undefined,
-                steps: steps.map((s, idx) => ({
-                    stepId: asUUID(uuidv4()),
-                    stepNumber: idx,
-                    timestamp: s.timestamp || stepTimestamp + idx,
-                    environmentState: {
-                        ...s.environmentState,
-                        timestamp: s.timestamp || stepTimestamp + idx,
-                        agentPoints: s.environmentState.agentPoints ??
-                            0,
-                    },
-                    observation: {},
-                    providerAccesses: (s.providerAccesses || []).map((p) => ({
-                        providerId: uuidv4(),
-                        providerName: p.providerName,
-                        timestamp: s.timestamp || stepTimestamp + idx,
-                        query: p.data,
-                        data: p.data,
-                        purpose: p.purpose,
-                    })),
-                    llmCalls: (s.llmCalls || []).map((l) => ({
-                        callId: uuidv4(),
-                        timestamp: s.timestamp || stepTimestamp + idx,
-                        model: l.model,
-                        modelVersion: l.modelVersion,
-                        systemPrompt: l.systemPrompt,
-                        userPrompt: l.userPrompt,
-                        response: l.response,
-                        reasoning: l.reasoning,
-                        temperature: l.temperature,
-                        maxTokens: l.maxTokens,
-                        latencyMs: l.latencyMs,
-                        purpose: l.purpose,
-                        actionType: l.actionType,
-                    })),
-                    action: {
-                        attemptId: uuidv4(),
-                        timestamp: s.timestamp || stepTimestamp + idx,
-                        actionType: s.action.actionType,
-                        actionName: s.action.actionType,
-                        parameters: s.action.parameters,
-                        reasoning: s.action.reasoning,
-                        success: s.action.success,
-                        result: s.action.result,
-                        error: s.action.error,
-                    },
-                    reward: s.reward,
-                    done: idx === steps.length - 1,
-                    metadata: {},
-                })),
-                totalReward: steps.reduce((sum, s) => sum + s.reward, 0),
-                rewardComponents: {
-                    environmentReward: steps.reduce((sum, s) => sum + s.reward, 0),
-                },
-                metrics: {
-                    episodeLength: dbTraj.episodeLength || steps.length,
-                    finalStatus: "completed",
-                    finalPnL: dbTraj.finalPnL || undefined,
-                },
-                metadata: {
-                    isTrainingData: true,
-                },
-            };
-            const toARTMessages = getToTrainingMessages();
-            const messages = toARTMessages(richTraj);
-            // Sanitize archetype to prevent prompt injection and handle null/empty values
-            const archetype = sanitizeArchetype(dbTraj.archetype);
-            richTrajectories.push({ traj: richTraj, messages, archetype });
-        }
-        if (richTrajectories.length < this.minGroupSize) {
-            logger.warn("Insufficient valid trajectories in group", {
-                scenarioId,
-                validCount: richTrajectories.length,
-            }, "RulerScoring");
-            return 0;
-        }
-        const commonPrefix = this.extractCommonPrefix(richTrajectories.map((rt) => rt.messages));
-        const judgePrompt = this.buildJudgePrompt(richTrajectories, commonPrefix, scenarioId);
-        const judgeResponse = await this.callJudge(judgePrompt);
-        if (!judgeResponse ||
-            judgeResponse.scores.length !== richTrajectories.length) {
-            logger.error("Invalid judge response", {
-                expectedScores: richTrajectories.length,
-                receivedScores: judgeResponse?.scores.length || 0,
-            }, "RulerScoring");
-            return 0;
-        }
-        const scoreMap = new Map();
-        for (const score of judgeResponse.scores) {
-            scoreMap.set(score.trajectory_id, score);
-        }
-        let scored = 0;
-        for (let i = 0; i < richTrajectories.length; i++) {
-            const expectedTrajId = `trajectory-${i + 1}`;
-            const scoreData = scoreMap.get(expectedTrajId);
-            if (!scoreData) {
-                logger.warn("Judge did not return score for trajectory", {
-                    expectedTrajId,
-                    receivedIds: judgeResponse.scores.map((s) => s.trajectory_id),
-                }, "RulerScoring");
-                continue;
-            }
-            const trajectoryId = richTrajectories[i]?.traj.trajectoryId;
-            await getTrainingDataAdapter().updateTrajectoryScore(trajectoryId, Math.max(0, Math.min(1, scoreData.score)), scoreData.explanation);
-            scored++;
-        }
-        logger.info("Scored trajectory group", {
-            scenarioId,
-            scored,
-            groupSize: richTrajectories.length,
-        }, "RulerScoring");
-        return scored;
-    }
-    /**
-     * Build judge prompt with trajectory context
-     *
-     * Injects game knowledge (P&L, episode length, actions) into the prompt
-     * so the judge can make informed relative comparisons.
-     */
-    buildJudgePrompt(richTrajectories, commonPrefix, scenarioId) {
-        // Build context section with game knowledge (injected into prompt)
-        const contextParts = [];
-        contextParts.push(`Scenario: ${scenarioId}`);
-        contextParts.push(`\nTrajectory Performance Context (use this to inform your scoring):`);
-        for (let i = 0; i < richTrajectories.length; i++) {
-            const rt = richTrajectories[i];
-            if (!rt)
-                continue;
-            const trajId = `trajectory-${i + 1}`;
-            contextParts.push(`\n${trajId}:`);
-            contextParts.push(`  - Archetype: ${rt.archetype}`);
-            contextParts.push(`  - Final P&L: $${rt.traj.metrics.finalPnL?.toFixed(2) || "0.00"}`);
-            contextParts.push(`  - Episode Length: ${rt.traj.metrics.episodeLength || 0} steps`);
-            contextParts.push(`  - Total Reward: ${rt.traj.totalReward.toFixed(2)}`);
-            const actionTypes = rt.traj.steps
-                .filter((s) => !!s.action)
-                .map((s) => s.action?.actionType);
-            const uniqueActions = [...new Set(actionTypes)];
-            contextParts.push(`  - Actions Taken: ${uniqueActions.join(", ")} (${actionTypes.length} total)`);
-            // Add success/error info
-            const errors = rt.traj.steps.filter((s) => !!s.action && !s.action.success).length;
-            const successRate = rt.traj.steps.length > 0
-                ? (((rt.traj.steps.length - errors) / rt.traj.steps.length) *
-                    100).toFixed(1)
-                : "0";
-            contextParts.push(`  - Success Rate: ${successRate}%`);
-            if (errors > 0) {
-                contextParts.push(`  - Errors: ${errors}`);
-            }
-        }
-        // Build trajectory messages (with deduplicated prefix)
-        const trajectorySections = [];
-        for (let i = 0; i < richTrajectories.length; i++) {
-            const rt = richTrajectories[i];
-            if (!rt)
-                continue;
-            const trajId = `trajectory-${i + 1}`;
-            // Remove common prefix from messages
-            const uniqueMessages = rt.messages.slice(commonPrefix.length);
-            // Truncate very long messages to save tokens (keep last 20 messages max)
-            const truncatedMessages = uniqueMessages.slice(-20);
-            trajectorySections.push(`<trajectory id="${trajId}">`);
-            trajectorySections.push(JSON.stringify(truncatedMessages, null, 2));
-            trajectorySections.push(`</trajectory>`);
-        }
-        // Build full prompt
-        const userContent = commonPrefix.length > 0
-            ? `<context>\n${JSON.stringify(commonPrefix, null, 2)}\n</context>\n\n`
-            : "";
-        const prompt = `${userContent}${contextParts.join("\n")}\n\nTrajectories:\n\n${trajectorySections.join("\n\n")}`;
-        // Determine archetype-specific rubric
-        // If all trajectories share the same archetype, use that archetype's rubric
-        // Otherwise, fall back to the default rubric
-        const archetypes = [...new Set(richTrajectories.map((rt) => rt.archetype))];
-        const isSingleArchetype = archetypes.length === 1 && archetypes[0] !== "default";
-        const singleArchetype = archetypes[0];
-        const rubric = isSingleArchetype && singleArchetype
-            ? getRubric(singleArchetype)
-            : DEFAULT_RUBRIC;
-        const archetypeContext = isSingleArchetype
-            ? `\n\nYou are evaluating ${archetypes[0]?.toUpperCase()} agents. Score them based on how well they embody that archetype's behavior and goals.`
-            : archetypes.length > 1
-                ? `\n\nNote: This group contains mixed archetypes (${archetypes.join(", ")}). Consider each agent's archetype when scoring.`
-                : "";
-        const systemPrompt = `You are an expert evaluator of AI agent performance. All trajectories below were given the same goal/scenario. Your job is to compare them and assign scores from 0 to 1 based on how well each trajectory achieved its goal.${archetypeContext}
-
-Grading standards:
-${rubric}
-
-Important: Use the performance context provided (P&L, episode length, success rate, archetype) to inform your scoring, but also consider the quality of decision-making, efficiency, and goal achievement shown in the trajectory messages.`;
-        return JSON.stringify({
-            system: systemPrompt,
-            user: prompt,
-        });
-    }
-    /**
-     * Call LLM judge to score trajectories
-     *
-     * Uses structured output format to ensure valid JSON response.
-     */
-    async callJudge(promptJson) {
-        const promptData = JSON.parse(promptJson);
-        const structuredPrompt = `${promptData.user}
-
-Please respond with ONLY a valid JSON object in this exact format:
-{
-  "scores": [
-    {
-      "trajectory_id": "trajectory-1",
-      "explanation": "Brief explanation of score",
-      "score": 0.85
-    },
-    {
-      "trajectory_id": "trajectory-2",
-      "explanation": "Brief explanation of score",
-      "score": 0.65
-    }
-  ]
-}
-
-Return ONLY the JSON, no other text.`;
-        const llmCaller = getLLMCaller();
-        const response = await llmCaller.callGroqDirect({
-            prompt: structuredPrompt,
-            system: promptData.system,
-            modelSize: "large",
-            temperature: 0.3,
-            maxTokens: 2000,
-            actionType: "ruler_score_trajectories",
-        });
-        let jsonText = response.trim();
-        jsonText = jsonText
-            .replace(/```json\n?/g, "")
-            .replace(/```\n?/g, "")
-            .trim();
-        const jsonMatch = jsonText.match(/\{[\s\S]*\}/);
-        if (!jsonMatch) {
-            logger.error("Judge response does not contain JSON", {
-                response: response.substring(0, 500),
-            }, "RulerScoring");
-            return null;
-        }
-        const parsed = JSON.parse(jsonMatch[0]);
-        if (!parsed.scores || !Array.isArray(parsed.scores)) {
-            logger.error("Invalid judge response structure", { parsed }, "RulerScoring");
-            return null;
-        }
-        for (const score of parsed.scores) {
-            if (score.score < 0 || score.score > 1) {
-                score.score = Math.max(0, Math.min(1, score.score));
-            }
-        }
-        return parsed;
-    }
-    /**
-     * Extract common message prefix from trajectories
-     *
-     * RULER deduplicates common prefixes to save tokens.
-     */
-    extractCommonPrefix(messageLists) {
-        if (messageLists.length === 0)
-            return [];
-        const first = messageLists[0];
-        if (!first)
-            return [];
-        const prefix = [];
-        for (let i = 0; i < first.length; i++) {
-            const msg = first[i];
-            if (!msg)
-                continue;
-            const allMatch = messageLists.every((msgs) => msgs[i] &&
-                msgs[i]?.role === msg.role &&
-                msgs[i]?.content === msg.content);
-            if (allMatch) {
-                prefix.push(msg);
-            }
-            else {
-                break;
-            }
-        }
-        return prefix;
-    }
-    /**
-     * Group trajectories by scenarioId
-     */
-    groupByScenario(trajectoriesData) {
-        const groups = new Map();
-        for (const traj of trajectoriesData) {
-            const scenarioId = traj.scenarioId || "default";
-            if (!groups.has(scenarioId)) {
-                groups.set(scenarioId, []);
-            }
-            groups.get(scenarioId)?.push(traj);
-        }
-        return Array.from(groups.entries()).map(([scenarioId, trajs]) => ({
-            scenarioId,
-            trajectories: trajs,
-        }));
-    }
-    /**
-     * Get trajectories to score
-     */
-    async getTrajectoriesToScore(trajectoryIds) {
-        const adapter = getTrainingDataAdapter();
-        return await adapter.getUnscoredTrajectories(trajectoryIds && trajectoryIds.length > 0 ? { trajectoryIds } : undefined);
-    }
-    /**
-     * Score all unscored trajectories in a time window
-     */
-    async scoreWindow(windowId) {
-        const trajectoryIds = await getTrainingDataAdapter().getUnscoredWindowTrajectoryIds(windowId);
-        if (trajectoryIds.length === 0) {
-            return 0;
-        }
-        return await this.scoreTrajectories(trajectoryIds);
-    }
-}
-/**
- * Singleton instance of RulerScoringService
- */
-export const rulerScoringService = new RulerScoringService();

package/dist/training/TrainingMonitor.js

@@ -1,108 +0,0 @@
-/**
- * Training Monitor Service
- *
- * Tracks training job progress and updates database with status.
- * Monitors Python training process and W&B runs.
- */
-import { getTrainingDataAdapter } from "../adapter";
-import { logger } from "../utils/logger";
-export class TrainingMonitor {
-    /**
-     * Start monitoring a training job
-     */
-    async startMonitoring(batchId) {
-        const adapter = getTrainingDataAdapter();
-        await adapter.updateBatchStatus(batchId, "training");
-        logger.info("Started monitoring training job", { batchId }, "TrainingMonitor");
-    }
-    /**
-     * Update training progress
-     */
-    async updateProgress(batchId, progress) {
-        if (progress.status) {
-            const adapter = getTrainingDataAdapter();
-            const errorMsg = progress.status === "failed" ? progress.error : undefined;
-            await adapter.updateBatchStatus(batchId, progress.status, errorMsg);
-        }
-        logger.info("Updated training progress", {
-            batchId,
-            status: progress.status,
-            progress: progress.progress,
-        }, "TrainingMonitor");
-    }
-    /**
-     * Get current progress for a job
-     */
-    async getProgress(batchId) {
-        const adapter = getTrainingDataAdapter();
-        const batch = await adapter.getBatchById(batchId);
-        if (!batch) {
-            return null;
-        }
-        // Calculate progress based on status
-        let progress = 0;
-        switch (batch.status) {
-            case "pending":
-                progress = 0;
-                break;
-            case "preparing":
-                progress = 0.1;
-                break;
-            case "scoring":
-                progress = 0.3;
-                break;
-            case "training":
-                progress = 0.6;
-                break;
-            case "uploading":
-                progress = 0.9;
-                break;
-            case "completed":
-                progress = 1.0;
-                break;
-            case "failed":
-                progress = 0;
-                break;
-        }
-        // Estimate ETA based on average training time
-        let eta;
-        if (batch.status === "training" && batch.startedAt) {
-            const avgTrainingTime = 2 * 60 * 60 * 1000; // 2 hours average
-            const elapsed = Date.now() - batch.startedAt.getTime();
-            eta = Math.max(0, avgTrainingTime - elapsed);
-        }
-        return {
-            batchId,
-            status: batch.status,
-            progress,
-            loss: batch.trainingLoss ?? undefined,
-            eta,
-            error: batch.error ?? undefined,
-        };
-    }
-    /**
-     * Check if training is stuck
-     */
-    async checkForStuckJobs() {
-        const fourHoursMs = 4 * 60 * 60 * 1000;
-        const adapter = getTrainingDataAdapter();
-        const stuckJobs = await adapter.getStuckTrainingBatches(fourHoursMs);
-        if (stuckJobs.length > 0) {
-            logger.warn("Found stuck training jobs", {
-                count: stuckJobs.length,
-                jobs: stuckJobs,
-            }, "TrainingMonitor");
-        }
-        return stuckJobs;
-    }
-    /**
-     * Cancel training job
-     */
-    async cancelJob(batchId, reason) {
-        const adapter = getTrainingDataAdapter();
-        await adapter.updateBatchStatus(batchId, "failed", `Cancelled: ${reason}`);
-        logger.warn("Training job cancelled", { batchId, reason }, "TrainingMonitor");
-    }
-}
-// Singleton
-export const trainingMonitor = new TrainingMonitor();