@elizaos/training 2.0.0-alpha.13 → 2.0.0-alpha.15

package/src/rubrics/social-butterfly.ts

@@ -63,11 +63,11 @@ Social quality matters too - genuine engagement (meaningful conversations, helpf
 `;
 
 export const SOCIAL_BUTTERFLY_PRIORITY_METRICS = [
-  'social.uniqueUsersInteracted',
-  'social.groupChatsJoined',
-  'social.dmsInitiated',
-  'social.postsCreated',
-  'social.commentsMade',
-  'behavior.socialToTradeRatio',
-  'influence.followersGained',
+  "social.uniqueUsersInteracted",
+  "social.groupChatsJoined",
+  "social.dmsInitiated",
+  "social.postsCreated",
+  "social.commentsMade",
+  "behavior.socialToTradeRatio",
+  "influence.followersGained",
 ];

package/src/rubrics/super-predictor.ts

@@ -88,10 +88,10 @@ Look for evidence of domain expertise:
 `;
 
 export const SUPER_PREDICTOR_PRIORITY_METRICS = [
-  'information.predictionAccuracy',
-  'trading.winRate',
-  'trading.totalPnL',
-  'information.researchActions',
-  'information.predictionsMade',
-  'information.correctPredictions',
+  "information.predictionAccuracy",
+  "trading.winRate",
+  "trading.totalPnL",
+  "information.researchActions",
+  "information.predictionsMade",
+  "information.correctPredictions",
 ];

package/src/rubrics/trader.ts

@@ -59,9 +59,9 @@ If two trajectories have similar P&L, the one with better risk metrics (lower dr
 `;
 
 export const TRADER_PRIORITY_METRICS = [
-  'trading.totalPnL',
-  'trading.sharpeRatio',
-  'trading.winRate',
-  'trading.marketsTraded',
-  'behavior.socialToTradeRatio',
+  "trading.totalPnL",
+  "trading.sharpeRatio",
+  "trading.winRate",
+  "trading.marketsTraded",
+  "behavior.socialToTradeRatio",
 ];

package/src/scoring/ArchetypeScoringService.ts

@@ -7,16 +7,16 @@
  * @packageDocumentation
  */
 
-import { getTrainingDataAdapter } from '../adapter';
-import { getLLMCaller } from '../dependencies';
-import { type BehavioralMetrics, trajectoryMetricsExtractor } from '../metrics';
-import { hasCustomRubric } from '../rubrics';
-import type { TrajectoryStep } from '../training/types';
-import { logger, splitIntoBatches } from '../utils';
+import { getTrainingDataAdapter } from "../adapter";
+import { getLLMCaller } from "../dependencies";
+import { type BehavioralMetrics, trajectoryMetricsExtractor } from "../metrics";
+import { hasCustomRubric } from "../rubrics";
+import type { TrajectoryStep } from "../training/types";
+import { logger, splitIntoBatches } from "../utils";
 import {
   judgePromptBuilder,
   type TrajectoryContext,
-} from './JudgePromptBuilder';
+} from "./JudgePromptBuilder";
 
 /**
  * Score result for a single trajectory.
@@ -86,17 +86,17 @@ export class ArchetypeScoringService {
    */
   async scoreTrajectory(
     trajectoryId: string,
-    options: ScoringOptions = {}
+    options: ScoringOptions = {},
   ): Promise<ArchetypeScore | null> {
     const opts = { ...DEFAULT_OPTIONS, ...options };
 
     const traj = await getTrainingDataAdapter().getTrajectoryById(trajectoryId);
     if (!traj) {
-      logger.warn('Trajectory not found', { trajectoryId }, 'ArchetypeScoring');
+      logger.warn("Trajectory not found", { trajectoryId }, "ArchetypeScoring");
       return null;
     }
 
-    const archetype = traj.archetype || opts.archetype || 'default';
+    const archetype = traj.archetype || opts.archetype || "default";
     const steps = JSON.parse(traj.stepsJson) as TrajectoryStep[];
 
     const metrics = trajectoryMetricsExtractor.extractFromRaw({
@@ -109,7 +109,7 @@ export class ArchetypeScoringService {
 
     if (!metrics) {
       throw new Error(
-        `Failed to extract metrics for trajectory ${trajectoryId}`
+        `Failed to extract metrics for trajectory ${trajectoryId}`,
       );
     }
 
@@ -131,7 +131,7 @@ export class ArchetypeScoringService {
     const response = await this.callSingleJudge(system, user);
     if (!response) {
       throw new Error(
-        `Judge returned no response for trajectory ${trajectoryId}`
+        `Judge returned no response for trajectory ${trajectoryId}`,
       );
     }
 
@@ -151,18 +151,18 @@ export class ArchetypeScoringService {
       await getTrainingDataAdapter().updateTrajectoryScore(
         trajectoryId,
         score.score,
-        score.reasoning
+        score.reasoning,
       );
     }
 
     logger.info(
-      'Scored trajectory',
+      "Scored trajectory",
       {
         trajectoryId,
         archetype: score.archetype,
         score: score.score,
       },
-      'ArchetypeScoring'
+      "ArchetypeScoring",
     );
 
     return score;
@@ -176,38 +176,39 @@ export class ArchetypeScoringService {
    */
   async scoreTrajectoryGroup(
     trajectoryIds: string[],
-    options: ScoringOptions = {}
+    options: ScoringOptions = {},
   ): Promise<ArchetypeScore[]> {
     const opts = { ...DEFAULT_OPTIONS, ...options };
 
     if (trajectoryIds.length < this.minGroupSize) {
       logger.warn(
-        'Group too small for RULER scoring',
+        "Group too small for RULER scoring",
         {
           size: trajectoryIds.length,
           minRequired: this.minGroupSize,
         },
-        'ArchetypeScoring'
+        "ArchetypeScoring",
       );
       return [];
     }
 
-    const trajResults = await getTrainingDataAdapter().getTrajectoriesByIds(trajectoryIds);
+    const trajResults =
+      await getTrainingDataAdapter().getTrajectoriesByIds(trajectoryIds);
 
     if (trajResults.length < this.minGroupSize) {
       logger.warn(
-        'Not enough valid trajectories',
+        "Not enough valid trajectories",
         {
           requested: trajectoryIds.length,
           found: trajResults.length,
         },
-        'ArchetypeScoring'
+        "ArchetypeScoring",
       );
       return [];
     }
 
     const contexts: TrajectoryContext[] = [];
-    const fallbackArchetype = opts.archetype || 'default';
+    const fallbackArchetype = opts.archetype || "default";
 
     for (const traj of trajResults) {
       const steps = JSON.parse(traj.stepsJson) as TrajectoryStep[];
@@ -223,7 +224,7 @@ export class ArchetypeScoringService {
 
       if (!metrics) {
         throw new Error(
-          `Failed to extract metrics for trajectory ${traj.trajectoryId}`
+          `Failed to extract metrics for trajectory ${traj.trajectoryId}`,
         );
       }
 
@@ -243,15 +244,15 @@ export class ArchetypeScoringService {
     const scores: ArchetypeScore[] = [];
 
     for (const batch of batches) {
-      const scenarioId = batch[0]?.archetype || 'unknown';
+      const scenarioId = batch[0]?.archetype || "unknown";
       const { system, user } = judgePromptBuilder.buildComparisonPrompt(
         batch,
-        scenarioId
+        scenarioId,
       );
       const response = await this.callComparisonJudge(system, user);
 
       if (!response) {
-        throw new Error('Judge returned no response for batch');
+        throw new Error("Judge returned no response for batch");
       }
 
       for (let i = 0; i < batch.length; i++) {
@@ -260,7 +261,7 @@ export class ArchetypeScoringService {
 
         const expectedId = `trajectory-${i + 1}`;
         const scoreData = response.scores.find(
-          (s) => s.trajectory_id === expectedId
+          (s) => s.trajectory_id === expectedId,
         );
 
         if (!scoreData) {
@@ -270,7 +271,7 @@ export class ArchetypeScoringService {
         const score: ArchetypeScore = {
           trajectoryId: ctx.trajectoryId,
           agentId: ctx.agentId,
-          archetype: ctx.archetype || 'default',
+          archetype: ctx.archetype || "default",
           score: Math.max(0, Math.min(1, scoreData.score)),
           reasoning: scoreData.explanation,
           strengths: [],
@@ -285,19 +286,19 @@ export class ArchetypeScoringService {
           await getTrainingDataAdapter().updateTrajectoryScore(
             ctx.trajectoryId,
             score.score,
-            score.reasoning
+            score.reasoning,
           );
         }
       }
     }
 
     logger.info(
-      'Scored trajectory group',
+      "Scored trajectory group",
       {
         requested: trajectoryIds.length,
         scored: scores.length,
       },
-      'ArchetypeScoring'
+      "ArchetypeScoring",
     );
 
     return scores;
@@ -311,13 +312,13 @@ export class ArchetypeScoringService {
    */
   async scoreByArchetype(
     archetype: string,
-    trajectoryIds: string[]
+    trajectoryIds: string[],
   ): Promise<{ scored: number; errors: number }> {
     if (!hasCustomRubric(archetype)) {
       logger.warn(
-        'No custom rubric for archetype, using default',
+        "No custom rubric for archetype, using default",
         { archetype },
-        'ArchetypeScoring'
+        "ArchetypeScoring",
       );
     }
 
@@ -343,13 +344,14 @@ export class ArchetypeScoringService {
    * @returns Count of scored and errors
    */
   async scoreUnscoredTrajectories(
-    archetype: string = 'default',
-    limit: number = 100
+    archetype: string = "default",
+    limit: number = 100,
   ): Promise<{ scored: number; errors: number }> {
-    const unscoredResult = await getTrainingDataAdapter().getUnscoredTrajectories({ limit });
+    const unscoredResult =
+      await getTrainingDataAdapter().getUnscoredTrajectories({ limit });
 
     if (unscoredResult.length === 0) {
-      logger.info('No unscored trajectories found', {}, 'ArchetypeScoring');
+      logger.info("No unscored trajectories found", {}, "ArchetypeScoring");
       return { scored: 0, errors: 0 };
     }
 
@@ -367,25 +369,25 @@ export class ArchetypeScoringService {
   async scoreTrajectoriesParallel(
     trajectoryIds: string[],
     options: ScoringOptions = {},
-    concurrency: number = 5
+    concurrency: number = 5,
   ): Promise<ArchetypeScore[]> {
     const results: ArchetypeScore[] = [];
     const batches = splitIntoBatches(trajectoryIds, concurrency);
 
     logger.info(
-      'Starting parallel scoring',
+      "Starting parallel scoring",
       {
         total: trajectoryIds.length,
         batches: batches.length,
         concurrency,
       },
-      'ArchetypeScoring'
+      "ArchetypeScoring",
     );
 
     for (let i = 0; i < batches.length; i++) {
       const batch = batches[i] ?? [];
       const batchPromises = batch.map((id) =>
-        this.scoreTrajectory(id, options)
+        this.scoreTrajectory(id, options),
       );
       const batchResults = await Promise.all(batchPromises);
 
@@ -401,12 +403,12 @@ export class ArchetypeScoringService {
     }
 
     logger.info(
-      'Parallel scoring complete',
+      "Parallel scoring complete",
       {
         requested: trajectoryIds.length,
         scored: results.length,
       },
-      'ArchetypeScoring'
+      "ArchetypeScoring",
     );
 
     return results;
@@ -417,7 +419,7 @@ export class ArchetypeScoringService {
    */
   private async callSingleJudge(
     system: string,
-    user: string
+    user: string,
   ): Promise<TrajectoryScoreResponse | null> {
     const llmCaller = getLLMCaller();
     const prompt = `${user}\n\nReturn ONLY valid JSON, no other text.`;
@@ -425,10 +427,10 @@ export class ArchetypeScoringService {
     const response = await llmCaller.callGroqDirect({
       prompt,
       system,
-      modelSize: 'large',
+      modelSize: "large",
       temperature: 0.3,
       maxTokens: 1000,
-      actionType: 'archetype_score_trajectory',
+      actionType: "archetype_score_trajectory",
     });
 
     return this.parseJudgeResponse<TrajectoryScoreResponse>(response);
@@ -439,7 +441,7 @@ export class ArchetypeScoringService {
    */
   private async callComparisonJudge(
     system: string,
-    user: string
+    user: string,
   ): Promise<RulerScoreResponse | null> {
     const llmCaller = getLLMCaller();
     const prompt = `${user}\n\nReturn ONLY valid JSON, no other text.`;
@@ -447,10 +449,10 @@ export class ArchetypeScoringService {
     const response = await llmCaller.callGroqDirect({
       prompt,
       system,
-      modelSize: 'large',
+      modelSize: "large",
       temperature: 0.3,
       maxTokens: 2000,
-      actionType: 'archetype_ruler_score',
+      actionType: "archetype_ruler_score",
     });
 
     return this.parseJudgeResponse<RulerScoreResponse>(response);
@@ -462,18 +464,18 @@ export class ArchetypeScoringService {
   private parseJudgeResponse<T>(response: string): T | null {
     const jsonText = response
       .trim()
-      .replace(/```json\n?/g, '')
-      .replace(/```\n?/g, '')
+      .replace(/```json\n?/g, "")
+      .replace(/```\n?/g, "")
       .trim();
 
     const jsonMatch = jsonText.match(/\{[\s\S]*\}/);
     if (!jsonMatch) {
       logger.error(
-        'No JSON found in response',
+        "No JSON found in response",
         {
           preview: response.substring(0, 200),
         },
-        'ArchetypeScoring'
+        "ArchetypeScoring",
       );
       return null;
     }