@elizaos/training 2.0.0-alpha.13 → 2.0.0-alpha.15

package/src/huggingface/HuggingFaceIntegrationService.ts

@@ -5,13 +5,13 @@
  * Main entry point for all HuggingFace operations.
  */
 
-import { getTrainingDataAdapter } from '../adapter';
-import { ModelBenchmarkService } from '../benchmark/ModelBenchmarkService';
-import { getExportToHuggingFace } from '../dependencies';
-import { logger } from '../utils';
-import { HuggingFaceDatasetUploader } from './HuggingFaceDatasetUploader';
-import { HuggingFaceModelUploader } from './HuggingFaceModelUploader';
-import { getHuggingFaceToken } from './shared/HuggingFaceUploadUtil';
+import { getTrainingDataAdapter } from "../adapter";
+import { ModelBenchmarkService } from "../benchmark/ModelBenchmarkService";
+import { getExportToHuggingFace } from "../dependencies";
+import { logger } from "../utils";
+import { HuggingFaceDatasetUploader } from "./HuggingFaceDatasetUploader";
+import { HuggingFaceModelUploader } from "./HuggingFaceModelUploader";
+import { getHuggingFaceToken } from "./shared/HuggingFaceUploadUtil";
 
 export interface WeeklyUploadResult {
   success: boolean;
@@ -49,13 +49,13 @@ export class HuggingFaceIntegrationService {
    * Execute complete weekly upload pipeline
    */
   async executeWeeklyUpload(
-    options: DatasetUploadOptions = {}
+    options: DatasetUploadOptions = {},
   ): Promise<WeeklyUploadResult> {
     const startTime = Date.now();
     logger.info(
-      'Starting weekly upload pipeline',
+      "Starting weekly upload pipeline",
       options,
-      'HuggingFaceIntegration'
+      "HuggingFaceIntegration",
     );
 
     const result: WeeklyUploadResult = {
@@ -77,17 +77,16 @@ export class HuggingFaceIntegrationService {
       // Step 1: Upload benchmark dataset
       if (!options.dryRun) {
         logger.info(
-          'Step 1: Uploading benchmark dataset',
+          "Step 1: Uploading benchmark dataset",
           undefined,
-          'HuggingFaceIntegration'
+          "HuggingFaceIntegration",
         );
         const benchmarkResult = await this.datasetUploader.uploadDataset({
           datasetName:
             options.datasetName ||
             process.env.HF_DATASET_NAME ||
-            'elizaos/agent-benchmarks',
-          description:
-            'Weekly benchmark results for autonomous ElizaOS agents',
+            "elizaos/agent-benchmarks",
+          description: "Weekly benchmark results for autonomous ElizaOS agents",
         });
 
         result.datasets.benchmarks = {
@@ -98,14 +97,14 @@ export class HuggingFaceIntegrationService {
 
         if (!benchmarkResult.success) {
           result.errors.push(
-            `Benchmark dataset upload: ${benchmarkResult.error}`
+            `Benchmark dataset upload: ${benchmarkResult.error}`,
           );
         }
       } else {
         logger.info(
-          'DRY RUN: Skipping benchmark dataset upload',
+          "DRY RUN: Skipping benchmark dataset upload",
           undefined,
-          'HuggingFaceIntegration'
+          "HuggingFaceIntegration",
         );
         result.datasets.benchmarks.success = true;
       }
@@ -113,17 +112,17 @@ export class HuggingFaceIntegrationService {
       // Step 2: Upload trajectory dataset
       if (!options.dryRun) {
         logger.info(
-          'Step 2: Uploading trajectory dataset',
+          "Step 2: Uploading trajectory dataset",
           undefined,
-          'HuggingFaceIntegration'
+          "HuggingFaceIntegration",
         );
         const exportToHuggingFace = getExportToHuggingFace();
         const trajectoryResult = await exportToHuggingFace({
           datasetName:
             options.trajectoryDatasetName ||
             process.env.HF_TRAJECTORY_DATASET_NAME ||
-            'elizaos/agent-trajectories',
-          format: 'jsonl',
+            "elizaos/agent-trajectories",
+          format: "jsonl",
         });
 
         result.datasets.trajectories = {
@@ -134,14 +133,14 @@ export class HuggingFaceIntegrationService {
 
         if (!trajectoryResult.success) {
           result.errors.push(
-            `Trajectory dataset upload: ${trajectoryResult.error}`
+            `Trajectory dataset upload: ${trajectoryResult.error}`,
           );
         }
       } else {
         logger.info(
-          'DRY RUN: Skipping trajectory dataset upload',
+          "DRY RUN: Skipping trajectory dataset upload",
           undefined,
-          'HuggingFaceIntegration'
+          "HuggingFaceIntegration",
         );
         result.datasets.trajectories.success = true;
       }
@@ -154,7 +153,7 @@ export class HuggingFaceIntegrationService {
       logger.info(
         `Step 3: Found ${unbenchmarkedModels.length} unbenchmarked models`,
         undefined,
-        'HuggingFaceIntegration'
+        "HuggingFaceIntegration",
       );
 
       if (unbenchmarkedModels.length > 0) {
@@ -162,8 +161,8 @@ export class HuggingFaceIntegrationService {
           await ModelBenchmarkService.getStandardBenchmarkPaths();
 
         if (standardBenchmarks.length === 0) {
-          const error = 'No standard benchmarks available for model evaluation';
-          logger.error(error, undefined, 'HuggingFaceIntegration');
+          const error = "No standard benchmarks available for model evaluation";
+          logger.error(error, undefined, "HuggingFaceIntegration");
           result.errors.push(error);
         } else {
           for (const modelId of unbenchmarkedModels) {
@@ -172,7 +171,7 @@ export class HuggingFaceIntegrationService {
               logger.info(
                 `Benchmarking model: ${modelId}`,
                 undefined,
-                'HuggingFaceIntegration'
+                "HuggingFaceIntegration",
               );
               await ModelBenchmarkService.benchmarkModel({
                 modelId,
@@ -186,14 +185,15 @@ export class HuggingFaceIntegrationService {
                 await ModelBenchmarkService.compareToBaseline(modelId);
 
               // Upload if improved
-              if (comparison.recommendation === 'deploy' && !options.dryRun) {
+              if (comparison.recommendation === "deploy" && !options.dryRun) {
                 logger.info(
                   `Model ${modelId} improved, uploading`,
                   undefined,
-                  'HuggingFaceIntegration'
+                  "HuggingFaceIntegration",
                 );
 
-                const model = await getTrainingDataAdapter().getModelById(modelId);
+                const model =
+                  await getTrainingDataAdapter().getModelById(modelId);
 
                 if (model) {
                   const modelName = options.modelNamePrefix
@@ -205,7 +205,7 @@ export class HuggingFaceIntegrationService {
                   const modelDescription =
                     options.modelDescriptionPrefix ||
                     process.env.HF_MODEL_DESCRIPTION_PREFIX ||
-                    'Autonomous ElizaOS agent';
+                    "Autonomous ElizaOS agent";
 
                   const uploadResult = await this.modelUploader.uploadModel({
                     modelId,
@@ -218,10 +218,13 @@ export class HuggingFaceIntegrationService {
                     result.models.uploaded++;
 
                     // Update model with HuggingFace repo
-                    await getTrainingDataAdapter().updateModelHuggingFaceRepo(modelId, modelName);
+                    await getTrainingDataAdapter().updateModelHuggingFaceRepo(
+                      modelId,
+                      modelName,
+                    );
                   } else {
                     result.errors.push(
-                      `Model upload ${modelId}: ${uploadResult.error}`
+                      `Model upload ${modelId}: ${uploadResult.error}`,
                     );
                   }
                 }
@@ -229,7 +232,7 @@ export class HuggingFaceIntegrationService {
                 logger.info(
                   `Model ${modelId} not ready for deployment: ${comparison.recommendation}`,
                   undefined,
-                  'HuggingFaceIntegration'
+                  "HuggingFaceIntegration",
                 );
               }
             } catch (error) {
@@ -238,7 +241,7 @@ export class HuggingFaceIntegrationService {
               logger.error(
                 `Failed to process model ${modelId}`,
                 { error },
-                'HuggingFaceIntegration'
+                "HuggingFaceIntegration",
               );
               result.errors.push(`Model ${modelId}: ${errorMsg}`);
             }
@@ -250,7 +253,7 @@ export class HuggingFaceIntegrationService {
       result.duration = Date.now() - startTime;
 
       logger.info(
-        'Weekly upload pipeline complete',
+        "Weekly upload pipeline complete",
         {
           success: result.success,
           benchmarkDataset: result.datasets.benchmarks.success,
@@ -261,19 +264,19 @@ export class HuggingFaceIntegrationService {
           errors: result.errors.length,
           duration: result.duration,
         },
-        'HuggingFaceIntegration'
+        "HuggingFaceIntegration",
       );
 
       return result;
     } catch (error) {
       result.duration = Date.now() - startTime;
       result.errors.push(
-        error instanceof Error ? error.message : String(error)
+        error instanceof Error ? error.message : String(error),
       );
       logger.error(
-        'Weekly upload pipeline failed',
+        "Weekly upload pipeline failed",
         { error },
-        'HuggingFaceIntegration'
+        "HuggingFaceIntegration",
       );
       return result;
     }
@@ -295,13 +298,16 @@ export class HuggingFaceIntegrationService {
     const adapter = getTrainingDataAdapter();
 
     // Get last upload time from database
-    const lastUploadTime = (await adapter.getLastDeployedModelDate()) || new Date(0);
+    const lastUploadTime =
+      (await adapter.getLastDeployedModelDate()) || new Date(0);
 
     // Check for new benchmarks since last upload
-    const newBenchmarksCount = await adapter.countBenchmarksSince(lastUploadTime);
+    const newBenchmarksCount =
+      await adapter.countBenchmarksSince(lastUploadTime);
 
     // Check for new trajectories since last upload
-    const newTrajectoriesCount = await adapter.countTrajectoriesSince(lastUploadTime);
+    const newTrajectoriesCount =
+      await adapter.countTrajectoriesSince(lastUploadTime);
 
     // Check for unbenchmarked models
     const unbenchmarkedModels =
@@ -333,7 +339,7 @@ export class HuggingFaceIntegrationService {
     // Check HuggingFace token
     if (!getHuggingFaceToken()) {
       issues.push(
-        'HUGGING_FACE_TOKEN or HF_TOKEN environment variable not set'
+        "HUGGING_FACE_TOKEN or HF_TOKEN environment variable not set",
       );
     }
 
@@ -343,10 +349,10 @@ export class HuggingFaceIntegrationService {
     try {
       const healthy = await adapter.healthCheck();
       if (!healthy) {
-        issues.push('Cannot connect to database');
+        issues.push("Cannot connect to database");
       }
     } catch {
-      issues.push('Cannot connect to database');
+      issues.push("Cannot connect to database");
     }
 
     // Check for standard benchmarks
@@ -354,7 +360,7 @@ export class HuggingFaceIntegrationService {
       await ModelBenchmarkService.getStandardBenchmarkPaths();
     if (standardBenchmarks.length === 0) {
       warnings.push(
-        'No standard benchmarks found. Generate benchmark fixtures before upload.'
+        "No standard benchmarks found. Generate benchmark fixtures before upload.",
       );
     }
 
@@ -364,21 +370,21 @@ export class HuggingFaceIntegrationService {
 
       if (stats.benchmarkCount === 0) {
         warnings.push(
-          'No benchmark results in database. Run some benchmarks first.'
+          "No benchmark results in database. Run some benchmarks first.",
         );
       }
 
       if (stats.trajectoryTraining === 0) {
         warnings.push(
-          'No training trajectories in database. Generate with agents or test data.'
+          "No training trajectories in database. Generate with agents or test data.",
         );
       }
 
       if (stats.modelTotal === 0) {
-        warnings.push('No trained models in database.');
+        warnings.push("No trained models in database.");
       }
     } catch {
-      issues.push('Could not retrieve training statistics');
+      issues.push("Could not retrieve training statistics");
     }
 
     return {

package/src/huggingface/HuggingFaceModelUploader.ts

@@ -4,20 +4,20 @@
  * Uploads trained RL models to HuggingFace Hub with benchmark results and model cards.
  */
 
-import { getTrainingDataAdapter } from '../adapter';
-import { promises as fs } from 'fs';
-import * as path from 'path';
+import { promises as fs } from "node:fs";
+import * as path from "node:path";
+import { getTrainingDataAdapter } from "../adapter";
 import {
   type JsonValue,
   parseSimulationMetrics,
-} from '../benchmark/parseSimulationMetrics';
-import type { SimulationMetrics } from '../benchmark/SimulationEngine';
-import { logger } from '../utils';
+} from "../benchmark/parseSimulationMetrics";
+import type { SimulationMetrics } from "../benchmark/SimulationEngine";
+import { logger } from "../utils";
 import {
   getHuggingFaceToken,
   HuggingFaceUploadUtil,
   requireHuggingFaceToken,
-} from './shared/HuggingFaceUploadUtil';
+} from "./shared/HuggingFaceUploadUtil";
 
 /**
  * Simplified benchmark result for HuggingFace model cards
@@ -76,7 +76,7 @@ export class HuggingFaceModelUploader {
    */
   async uploadModel(options: ModelUploadOptions): Promise<ModelUploadResult> {
     try {
-      logger.info('Starting HuggingFace model upload', {
+      logger.info("Starting HuggingFace model upload", {
         modelId: options.modelId,
       });
 
@@ -93,11 +93,11 @@ export class HuggingFaceModelUploader {
       }
 
       // Step 2: Get benchmark results
-      logger.info('Loading benchmark results', { modelId: options.modelId });
+      logger.info("Loading benchmark results", { modelId: options.modelId });
       const modelBenchmarks = await this.getBenchmarkResults(options.modelId);
 
       if (modelBenchmarks.length === 0) {
-        logger.warn('No benchmark results found for model', {
+        logger.warn("No benchmark results found for model", {
           modelId: options.modelId,
         });
       }
@@ -117,15 +117,15 @@ export class HuggingFaceModelUploader {
       // Step 4: Create output directory
       const outputDir =
         options.outputDir ||
-        path.join(process.cwd(), 'exports', 'models', model.version);
+        path.join(process.cwd(), "exports", "models", model.version);
       await fs.mkdir(outputDir, { recursive: true });
 
       // Step 5: Generate model card
-      logger.info('Generating model card');
+      logger.info("Generating model card");
       await this.generateModelCard(cardData, outputDir);
 
       // Step 6: Save metadata
-      const metadataPath = path.join(outputDir, 'model_metadata.json');
+      const metadataPath = path.join(outputDir, "model_metadata.json");
       await fs.writeFile(
         metadataPath,
         JSON.stringify(
@@ -141,42 +141,42 @@ export class HuggingFaceModelUploader {
             accuracy: model.accuracy,
           },
           null,
-          2
-        )
+          2,
+        ),
       );
 
       // Step 7: Save benchmark results
-      const benchmarksPath = path.join(outputDir, 'benchmark_results.json');
+      const benchmarksPath = path.join(outputDir, "benchmark_results.json");
       await fs.writeFile(
         benchmarksPath,
-        JSON.stringify(modelBenchmarks, null, 2)
+        JSON.stringify(modelBenchmarks, null, 2),
       );
 
       // Step 8: Upload to HuggingFace (if weights available and requested)
       let filesUploaded = 2; // README.md + metadata
 
       if (options.includeWeights && model.storagePath) {
-        logger.info('Uploading model to HuggingFace', {
+        logger.info("Uploading model to HuggingFace", {
           modelName: options.modelName,
         });
         const uploadCount = await this.uploadToHub(
           options.modelName,
           outputDir,
-          options.private ?? false
+          options.private ?? false,
         );
         filesUploaded = uploadCount;
       } else {
         logger.info(
-          'Skipping model weight upload (not requested or no weights available)'
+          "Skipping model weight upload (not requested or no weights available)",
         );
       }
 
       const modelUrl = `https://huggingface.co/${options.modelName}`;
 
-      logger.info('Model uploaded successfully', { modelUrl, filesUploaded });
+      logger.info("Model uploaded successfully", { modelUrl, filesUploaded });
 
       // Update model status in database
-      await adapter.updateModelStatus(options.modelId, 'deployed', {
+      await adapter.updateModelStatus(options.modelId, "deployed", {
         deployedAt: new Date(),
       });
 
@@ -187,12 +187,12 @@ export class HuggingFaceModelUploader {
         filesUploaded,
       };
     } catch (error) {
-      logger.error('Failed to upload model', { error });
+      logger.error("Failed to upload model", { error });
       return {
         success: false,
         modelId: options.modelId,
         filesUploaded: 0,
-        error: error instanceof Error ? error.message : 'Unknown error',
+        error: error instanceof Error ? error.message : "Unknown error",
       };
     }
   }
@@ -201,11 +201,12 @@ export class HuggingFaceModelUploader {
    * Get benchmark results for a model
    */
   private async getBenchmarkResults(
-    modelId: string
+    modelId: string,
   ): Promise<ModelCardBenchmarkResult[]> {
     // Query benchmark results from database
     try {
-      const results = await getTrainingDataAdapter().getBenchmarkResultsByModel(modelId);
+      const results =
+        await getTrainingDataAdapter().getBenchmarkResultsByModel(modelId);
 
       return results.map((r) => ({
         benchmarkId: r.benchmarkId,
@@ -214,7 +215,7 @@ export class HuggingFaceModelUploader {
         metrics: parseSimulationMetrics(r.detailedMetrics as JsonValue),
       }));
     } catch (error) {
-      logger.warn('Could not load benchmark results from database', { error });
+      logger.warn("Could not load benchmark results from database", { error });
 
       // Fallback to files if database fails
       return await this.getBenchmarkResultsFromFiles(modelId);
@@ -225,18 +226,18 @@ export class HuggingFaceModelUploader {
    * Fallback: Get benchmark results from files
    */
   private async getBenchmarkResultsFromFiles(
-    modelId: string
+    modelId: string,
   ): Promise<ModelCardBenchmarkResult[]> {
     const results: ModelCardBenchmarkResult[] = [];
 
     try {
-      const benchmarksDir = path.join(process.cwd(), 'benchmarks');
+      const benchmarksDir = path.join(process.cwd(), "benchmarks");
       const files = await fs.readdir(benchmarksDir);
 
       for (const file of files) {
-        if (file.endsWith('.json') && file.includes(modelId)) {
+        if (file.endsWith(".json") && file.includes(modelId)) {
           const filePath = path.join(benchmarksDir, file);
-          const data = JSON.parse(await fs.readFile(filePath, 'utf-8'));
+          const data = JSON.parse(await fs.readFile(filePath, "utf-8"));
 
           if (data.metrics) {
             results.push({
@@ -248,7 +249,7 @@ export class HuggingFaceModelUploader {
         }
       }
     } catch (error) {
-      logger.warn('Could not load benchmark results from files either', {
+      logger.warn("Could not load benchmark results from files either", {
         error,
       });
     }
@@ -260,7 +261,7 @@ export class HuggingFaceModelUploader {
    * Calculate average metrics across benchmarks
    */
   private calculateAverageMetrics(
-    benchmarkResults: ModelCardBenchmarkResult[]
+    benchmarkResults: ModelCardBenchmarkResult[],
   ): {
     avgPnl: number;
     avgAccuracy: number;
@@ -278,15 +279,15 @@ export class HuggingFaceModelUploader {
 
     const totalPnl = benchmarkResults.reduce(
       (sum, r) => sum + r.metrics.totalPnl,
-      0
+      0,
     );
     const totalAccuracy = benchmarkResults.reduce(
       (sum, r) => sum + r.metrics.predictionMetrics.accuracy,
-      0
+      0,
     );
     const totalOptimality = benchmarkResults.reduce(
       (sum, r) => sum + r.metrics.optimalityScore,
-      0
+      0,
     );
 
     return {
@@ -302,14 +303,14 @@ export class HuggingFaceModelUploader {
    */
   private async generateModelCard(
     data: ModelCardData,
-    outputDir: string
+    outputDir: string,
   ): Promise<void> {
-    const brandName = process.env.TRAINING_BRAND_NAME || 'ElizaOS';
-    const brandOrg = process.env.TRAINING_BRAND_ORG || 'ElizaOS Contributors';
+    const brandName = process.env.TRAINING_BRAND_NAME || "ElizaOS";
+    const brandOrg = process.env.TRAINING_BRAND_ORG || "ElizaOS Contributors";
     const platformName =
-      process.env.TRAINING_PLATFORM_NAME || 'ElizaOS-compatible runtimes';
-    const brandTag = brandName.toLowerCase().replace(/\s+/g, '-');
-    const citationKey = `${brandTag}_agent_${data.version.replace(/\./g, '_')}`;
+      process.env.TRAINING_PLATFORM_NAME || "ElizaOS-compatible runtimes";
+    const brandTag = brandName.toLowerCase().replace(/\s+/g, "-");
+    const citationKey = `${brandTag}_agent_${data.version.replace(/\./g, "_")}`;
 
     const card = `---
 license: mit
@@ -330,9 +331,9 @@ Autonomous agent trained with reinforcement learning for market-style decision m
 
 - **Version:** ${data.version}
 - **Base Model:** ${data.baseModel}
-- **Training Date:** ${data.trainedAt.toISOString().split('T')[0]}
+- **Training Date:** ${data.trainedAt.toISOString().split("T")[0]}
 - **Model ID:** ${data.modelId}
-${data.trainingRunId ? `- **Training Run:** ${data.trainingRunId}` : ''}
+${data.trainingRunId ? `- **Training Run:** ${data.trainingRunId}` : ""}
 
 ## Performance Metrics
 
@@ -351,7 +352,7 @@ ${
 
 ${this.generateBenchmarkTable(data.benchmarkResults)}
 `
-    : 'No benchmark results available yet.'
+    : "No benchmark results available yet."
 }
 
 ## Training Details
@@ -374,7 +375,7 @@ This model was trained using Group Relative Policy Optimization (GRPO) via the A
 
 ### Compute Infrastructure
 
-- **Platform:** ${data.trainingRunId ? 'Atropos GRPO Training' : 'Local training'}
+- **Platform:** ${data.trainingRunId ? "Atropos GRPO Training" : "Local training"}
 - **Training Time:** Continuous learning with hourly updates
 
 ## Intended Use
@@ -471,7 +472,7 @@ This model is part of a research project on autonomous agents in prediction mark
 For questions or issues, please open an issue on the repository.
 `;
 
-    const cardPath = path.join(outputDir, 'README.md');
+    const cardPath = path.join(outputDir, "README.md");
     await fs.writeFile(cardPath, card);
   }
 
@@ -479,14 +480,14 @@ For questions or issues, please open an issue on the repository.
    * Generate benchmark results table
    */
   private generateBenchmarkTable(results: ModelCardBenchmarkResult[]): string {
-    if (results.length === 0) return '';
+    if (results.length === 0) return "";
 
     let table =
-      '| Benchmark | Date | P&L | Accuracy | Win Rate | Optimality |\n';
-    table += '|-----------|------|-----|----------|----------|------------|\n';
+      "| Benchmark | Date | P&L | Accuracy | Win Rate | Optimality |\n";
+    table += "|-----------|------|-----|----------|----------|------------|\n";
 
     results.forEach((result) => {
-      const date = new Date(result.runAt).toISOString().split('T')[0];
+      const date = new Date(result.runAt).toISOString().split("T")[0];
       table += `| ${result.benchmarkId.substring(0, 20)}... | ${date} | ${result.metrics.totalPnl.toFixed(2)} | ${(result.metrics.predictionMetrics.accuracy * 100).toFixed(1)}% | ${(result.metrics.perpMetrics.winRate * 100).toFixed(1)}% | ${result.metrics.optimalityScore.toFixed(1)} |\n`;
     });
 
@@ -500,31 +501,31 @@ For questions or issues, please open an issue on the repository.
   private async uploadToHub(
     modelName: string,
     localDir: string,
-    _isPrivate: boolean
+    _isPrivate: boolean,
   ): Promise<number> {
     if (!this.huggingFaceToken) {
-      throw new Error('HuggingFace token not configured');
+      throw new Error("HuggingFace token not configured");
     }
 
     try {
       // Use shared upload utility
       return await HuggingFaceUploadUtil.uploadDirectory(
         modelName,
-        'model',
+        "model",
         localDir,
-        this.huggingFaceToken
+        this.huggingFaceToken,
       );
     } catch (error) {
-      logger.error('Failed to upload to HuggingFace Hub', { error });
+      logger.error("Failed to upload to HuggingFace Hub", { error });
 
       // Provide helpful manual upload instructions
       const instructions = HuggingFaceUploadUtil.getManualUploadInstructions(
         modelName,
-        'model',
-        localDir
+        "model",
+        localDir,
       );
 
-      logger.info('To upload manually:', { instructions });
+      logger.info("To upload manually:", { instructions });
 
       throw error;
     }