@elizaos/training 2.0.0-alpha.76 → 2.0.0-alpha.78

package/dist/scoring/ArchetypeScoringService.js

@@ -1,301 +0,0 @@
-/**
- * ArchetypeScoringService
- *
- * Scores trajectories using LLM-as-judge with archetype-specific rubrics.
- * Supports both single trajectory scoring and RULER-style relative comparison.
- *
- * @packageDocumentation
- */
-import { getTrainingDataAdapter } from "../adapter";
-import { getLLMCaller } from "../dependencies";
-import { trajectoryMetricsExtractor } from "../metrics";
-import { hasCustomRubric } from "../rubrics";
-import { logger, splitIntoBatches } from "../utils";
-import { judgePromptBuilder, } from "./JudgePromptBuilder";
-const DEFAULT_OPTIONS = {
-    includeActionDetails: false,
-    saveToDatabase: true,
-};
-/**
- * Service for scoring trajectories with archetype-aware evaluation.
- */
-export class ArchetypeScoringService {
-    minGroupSize = 2;
-    maxGroupSize = 8;
-    /**
-     * Score a single trajectory.
-     * @param trajectoryId - ID of the trajectory to score
-     * @param options - Scoring options
-     * @returns The score or null if trajectory not found
-     */
-    async scoreTrajectory(trajectoryId, options = {}) {
-        const opts = { ...DEFAULT_OPTIONS, ...options };
-        const traj = await getTrainingDataAdapter().getTrajectoryById(trajectoryId);
-        if (!traj) {
-            logger.warn("Trajectory not found", { trajectoryId }, "ArchetypeScoring");
-            return null;
-        }
-        const archetype = traj.archetype || opts.archetype || "default";
-        const steps = JSON.parse(traj.stepsJson);
-        const metrics = trajectoryMetricsExtractor.extractFromRaw({
-            trajectoryId: traj.trajectoryId,
-            agentId: traj.agentId,
-            stepsJson: traj.stepsJson,
-            scenarioId: traj.scenarioId || undefined,
-            finalPnL: traj.finalPnL || undefined,
-        });
-        if (!metrics) {
-            throw new Error(`Failed to extract metrics for trajectory ${trajectoryId}`);
-        }
-        const context = {
-            trajectoryId: traj.trajectoryId,
-            agentId: traj.agentId,
-            archetype,
-            steps,
-            metrics,
-            finalPnL: traj.finalPnL || undefined,
-            episodeLength: traj.episodeLength,
-            totalReward: traj.totalReward,
-        };
-        const { system, user } = judgePromptBuilder.buildSinglePrompt(context, {
-            includeActionDetails: opts.includeActionDetails,
-        });
-        const response = await this.callSingleJudge(system, user);
-        if (!response) {
-            throw new Error(`Judge returned no response for trajectory ${trajectoryId}`);
-        }
-        const score = {
-            trajectoryId: traj.trajectoryId,
-            agentId: traj.agentId,
-            archetype,
-            score: Math.max(0, Math.min(1, response.score)),
-            reasoning: response.reasoning,
-            strengths: response.strengths || [],
-            weaknesses: response.weaknesses || [],
-            metrics,
-            scoredAt: new Date(),
-        };
-        if (opts.saveToDatabase) {
-            await getTrainingDataAdapter().updateTrajectoryScore(trajectoryId, score.score, score.reasoning);
-        }
-        logger.info("Scored trajectory", {
-            trajectoryId,
-            archetype: score.archetype,
-            score: score.score,
-        }, "ArchetypeScoring");
-        return score;
-    }
-    /**
-     * Score multiple trajectories using RULER comparison.
-     * @param trajectoryIds - IDs of trajectories to score
-     * @param options - Scoring options
-     * @returns Array of scores
-     */
-    async scoreTrajectoryGroup(trajectoryIds, options = {}) {
-        const opts = { ...DEFAULT_OPTIONS, ...options };
-        if (trajectoryIds.length < this.minGroupSize) {
-            logger.warn("Group too small for RULER scoring", {
-                size: trajectoryIds.length,
-                minRequired: this.minGroupSize,
-            }, "ArchetypeScoring");
-            return [];
-        }
-        const trajResults = await getTrainingDataAdapter().getTrajectoriesByIds(trajectoryIds);
-        if (trajResults.length < this.minGroupSize) {
-            logger.warn("Not enough valid trajectories", {
-                requested: trajectoryIds.length,
-                found: trajResults.length,
-            }, "ArchetypeScoring");
-            return [];
-        }
-        const contexts = [];
-        const fallbackArchetype = opts.archetype || "default";
-        for (const traj of trajResults) {
-            const steps = JSON.parse(traj.stepsJson);
-            const archetype = traj.archetype || fallbackArchetype;
-            const metrics = trajectoryMetricsExtractor.extractFromRaw({
-                trajectoryId: traj.trajectoryId,
-                agentId: traj.agentId,
-                stepsJson: traj.stepsJson,
-                scenarioId: traj.scenarioId || undefined,
-                finalPnL: traj.finalPnL || undefined,
-            });
-            if (!metrics) {
-                throw new Error(`Failed to extract metrics for trajectory ${traj.trajectoryId}`);
-            }
-            contexts.push({
-                trajectoryId: traj.trajectoryId,
-                agentId: traj.agentId,
-                archetype,
-                steps,
-                metrics,
-                finalPnL: traj.finalPnL || undefined,
-                episodeLength: traj.episodeLength,
-                totalReward: traj.totalReward,
-            });
-        }
-        const batches = splitIntoBatches(contexts, this.maxGroupSize);
-        const scores = [];
-        for (const batch of batches) {
-            const scenarioId = batch[0]?.archetype || "unknown";
-            const { system, user } = judgePromptBuilder.buildComparisonPrompt(batch, scenarioId);
-            const response = await this.callComparisonJudge(system, user);
-            if (!response) {
-                throw new Error("Judge returned no response for batch");
-            }
-            for (let i = 0; i < batch.length; i++) {
-                const ctx = batch[i];
-                if (!ctx)
-                    continue;
-                const expectedId = `trajectory-${i + 1}`;
-                const scoreData = response.scores.find((s) => s.trajectory_id === expectedId);
-                if (!scoreData) {
-                    throw new Error(`Missing score for ${expectedId}`);
-                }
-                const score = {
-                    trajectoryId: ctx.trajectoryId,
-                    agentId: ctx.agentId,
-                    archetype: ctx.archetype || "default",
-                    score: Math.max(0, Math.min(1, scoreData.score)),
-                    reasoning: scoreData.explanation,
-                    strengths: [],
-                    weaknesses: [],
-                    metrics: ctx.metrics,
-                    scoredAt: new Date(),
-                };
-                scores.push(score);
-                if (opts.saveToDatabase) {
-                    await getTrainingDataAdapter().updateTrajectoryScore(ctx.trajectoryId, score.score, score.reasoning);
-                }
-            }
-        }
-        logger.info("Scored trajectory group", {
-            requested: trajectoryIds.length,
-            scored: scores.length,
-        }, "ArchetypeScoring");
-        return scores;
-    }
-    /**
-     * Score trajectories by archetype.
-     * @param archetype - Archetype to use for scoring
-     * @param trajectoryIds - IDs to score
-     * @returns Count of scored and errors
-     */
-    async scoreByArchetype(archetype, trajectoryIds) {
-        if (!hasCustomRubric(archetype)) {
-            logger.warn("No custom rubric for archetype, using default", { archetype }, "ArchetypeScoring");
-        }
-        if (trajectoryIds.length === 0) {
-            return { scored: 0, errors: 0 };
-        }
-        const scores = await this.scoreTrajectoryGroup(trajectoryIds, {
-            archetype,
-            saveToDatabase: true,
-        });
-        return {
-            scored: scores.length,
-            errors: trajectoryIds.length - scores.length,
-        };
-    }
-    /**
-     * Score all unscored trajectories.
-     * @param archetype - Default archetype to use
-     * @param limit - Maximum trajectories to score
-     * @returns Count of scored and errors
-     */
-    async scoreUnscoredTrajectories(archetype = "default", limit = 100) {
-        const unscoredResult = await getTrainingDataAdapter().getUnscoredTrajectories({ limit });
-        if (unscoredResult.length === 0) {
-            logger.info("No unscored trajectories found", {}, "ArchetypeScoring");
-            return { scored: 0, errors: 0 };
-        }
-        const trajectoryIds = unscoredResult.map((r) => r.trajectoryId);
-        return this.scoreByArchetype(archetype, trajectoryIds);
-    }
-    /**
-     * Score trajectories in parallel with rate limiting.
-     * @param trajectoryIds - IDs to score
-     * @param options - Scoring options
-     * @param concurrency - Maximum concurrent calls
-     * @returns Array of scores
-     */
-    async scoreTrajectoriesParallel(trajectoryIds, options = {}, concurrency = 5) {
-        const results = [];
-        const batches = splitIntoBatches(trajectoryIds, concurrency);
-        logger.info("Starting parallel scoring", {
-            total: trajectoryIds.length,
-            batches: batches.length,
-            concurrency,
-        }, "ArchetypeScoring");
-        for (let i = 0; i < batches.length; i++) {
-            const batch = batches[i] ?? [];
-            const batchPromises = batch.map((id) => this.scoreTrajectory(id, options));
-            const batchResults = await Promise.all(batchPromises);
-            for (const result of batchResults) {
-                if (result) {
-                    results.push(result);
-                }
-            }
-            if (i < batches.length - 1) {
-                await new Promise((resolve) => setTimeout(resolve, 100));
-            }
-        }
-        logger.info("Parallel scoring complete", {
-            requested: trajectoryIds.length,
-            scored: results.length,
-        }, "ArchetypeScoring");
-        return results;
-    }
-    /**
-     * Call LLM judge for single trajectory.
-     */
-    async callSingleJudge(system, user) {
-        const llmCaller = getLLMCaller();
-        const prompt = `${user}\n\nReturn ONLY valid JSON, no other text.`;
-        const response = await llmCaller.callGroqDirect({
-            prompt,
-            system,
-            modelSize: "large",
-            temperature: 0.3,
-            maxTokens: 1000,
-            actionType: "archetype_score_trajectory",
-        });
-        return this.parseJudgeResponse(response);
-    }
-    /**
-     * Call LLM judge for trajectory comparison.
-     */
-    async callComparisonJudge(system, user) {
-        const llmCaller = getLLMCaller();
-        const prompt = `${user}\n\nReturn ONLY valid JSON, no other text.`;
-        const response = await llmCaller.callGroqDirect({
-            prompt,
-            system,
-            modelSize: "large",
-            temperature: 0.3,
-            maxTokens: 2000,
-            actionType: "archetype_ruler_score",
-        });
-        return this.parseJudgeResponse(response);
-    }
-    /**
-     * Parse JSON response from judge.
-     */
-    parseJudgeResponse(response) {
-        const jsonText = response
-            .trim()
-            .replace(/```json\n?/g, "")
-            .replace(/```\n?/g, "")
-            .trim();
-        const jsonMatch = jsonText.match(/\{[\s\S]*\}/);
-        if (!jsonMatch) {
-            logger.error("No JSON found in response", {
-                preview: response.substring(0, 200),
-            }, "ArchetypeScoring");
-            return null;
-        }
-        return JSON.parse(jsonMatch[0]);
-    }
-}
-/** Singleton instance */
-export const archetypeScoringService = new ArchetypeScoringService();