@elizaos/training 2.0.0-alpha.10

package/src/training/pipeline.ts

@@ -0,0 +1,129 @@
+/**
+ * Training Pipeline – public helpers
+ *
+ * IMPORTANT: All heavy modules (AutomationPipeline, ModelDeployer) are loaded
+ * lazily so that importing this file does NOT trigger a database connection.
+ * Consumers that only need types or lightweight utilities can import
+ * "@elizaos/training" without side-effects.
+ */
+
+import type {
+  AutomationStatus,
+  TrainingMonitoringStatus,
+  TrainingReadinessResult,
+  TrainingTriggerOptions,
+  TrainingTriggerResult,
+} from './types';
+import type {
+  DeploymentOptions,
+  DeploymentResult,
+} from './ModelDeployer';
+import type { AutomationPipeline } from './AutomationPipeline';
+
+export type NextTrainingModelSelection = Awaited<
+  ReturnType<AutomationPipeline['getModelSelectionInfo']>
+>;
+
+// ---------------------------------------------------------------------------
+// Lazy singletons – only resolved on first call to avoid DB side-effects at
+// module-load time.
+// ---------------------------------------------------------------------------
+
+let _pipeline: AutomationPipeline | null = null;
+
+async function getPipeline(): Promise<AutomationPipeline> {
+  if (!_pipeline) {
+    const mod = await import('./AutomationPipeline');
+    _pipeline = mod.automationPipeline;
+  }
+  return _pipeline;
+}
+
+async function getDeployer() {
+  const mod = await import('./ModelDeployer');
+  return mod.modelDeployer;
+}
+
+/**
+ * Check whether the current trajectory set is ready for training.
+ */
+export async function checkTrainingReadiness(): Promise<TrainingReadinessResult> {
+  const pipeline = await getPipeline();
+  return pipeline.checkTrainingReadiness();
+}
+
+/**
+ * Trigger a new training job.
+ */
+export async function triggerTraining(
+  options: TrainingTriggerOptions = {}
+): Promise<TrainingTriggerResult> {
+  const pipeline = await getPipeline();
+  return pipeline.triggerTraining(options);
+}
+
+/**
+ * Monitor a training batch by its batch id.
+ */
+export async function monitorTrainingJob(
+  batchId: string
+): Promise<TrainingMonitoringStatus> {
+  const pipeline = await getPipeline();
+  return pipeline.monitorTraining(batchId);
+}
+
+/**
+ * Get summarized status for automation, jobs, and model health.
+ */
+export async function getAutomationPipelineStatus(): Promise<AutomationStatus> {
+  const pipeline = await getPipeline();
+  return pipeline.getStatus();
+}
+
+/**
+ * Get model-selection metadata for the next run.
+ */
+export async function getNextTrainingModelSelection(): Promise<{
+  success: boolean;
+  selection: NextTrainingModelSelection['selection'];
+  summary: NextTrainingModelSelection['summary'];
+}> {
+  const pipeline = await getPipeline();
+  return pipeline.getModelSelectionInfo();
+}
+
+/**
+ * Run benchmark and deploy the model only if it passes thresholds.
+ */
+export async function benchmarkAndMaybeDeployModel(
+  batchId: string,
+  autoDeploy = true
+): Promise<{
+  benchmarked: boolean;
+  deployed: boolean;
+  reason?: string;
+}> {
+  const pipeline = await getPipeline();
+  return pipeline.benchmarkAndDeploy(batchId, autoDeploy);
+}
+
+/**
+ * Deploy a specific model version using the deployment strategy options.
+ */
+export async function deployModelVersion(
+  options: DeploymentOptions
+): Promise<DeploymentResult> {
+  const deployer = await getDeployer();
+  return deployer.deploy(options);
+}
+
+/**
+ * Roll back from one version to another.
+ */
+export async function rollbackModelVersion(
+  currentVersion: string,
+  targetVersion: string
+): Promise<DeploymentResult> {
+  const deployer = await getDeployer();
+  return deployer.rollback(currentVersion, targetVersion);
+}

package/src/training/storage/ModelStorageService.ts

@@ -0,0 +1,279 @@
+/**
+ * Model Storage Service (Vercel Blob)
+ *
+ * Handles model versioning and storage using Vercel Blob.
+ * Stores trained models with metadata for easy deployment.
+ */
+
+import { getTrainingDataAdapter } from '../../adapter';
+import type { JsonValue } from '../../adapter';
+import { del, list, put } from '@vercel/blob';
+import fs from 'fs/promises';
+import path from 'path';
+import { logger } from '../../utils/logger';
+
+export interface ModelMetadata {
+  trainingBatch?: string;
+  accuracy?: number;
+  avgReward?: number;
+  baseModel?: string;
+  modelIdPrefix?: string;
+}
+
+export interface ModelVersion {
+  version: string;
+  baseModel: string;
+  blobUrl: string;
+  size: number;
+  uploadedAt: Date;
+  metadata: ModelMetadata & Record<string, JsonValue | undefined>;
+}
+
+export class ModelStorageService {
+  private readonly blobPrefix = 'models/';
+
+  /**
+   * Upload trained model to Vercel Blob
+   */
+  async uploadModel(options: {
+    version: string;
+    modelPath: string;
+    modelIdPrefix?: string;
+    metadata?: ModelVersion['metadata'];
+  }): Promise<ModelVersion> {
+    logger.info('Uploading model to Vercel Blob', {
+      version: options.version,
+      path: options.modelPath,
+    });
+
+    // Read model file
+    const modelData = await fs.readFile(options.modelPath);
+    const fileName = path.basename(options.modelPath);
+
+    // Upload to Vercel Blob
+    const blob = await put(
+      `${this.blobPrefix}${options.version}/${fileName}`,
+      modelData,
+      {
+        access: 'public', // Models can be publicly downloaded
+        addRandomSuffix: false,
+      }
+    );
+
+    // Upload metadata
+    await put(
+      `${this.blobPrefix}${options.version}/metadata.json`,
+      JSON.stringify(options.metadata || {}, null, 2),
+      {
+        access: 'public',
+        addRandomSuffix: false,
+      }
+    );
+
+    logger.info('Model uploaded to Vercel Blob', {
+      version: options.version,
+      url: blob.url,
+      size: (blob as { size?: number }).size || 0,
+    });
+
+    const metadataModelIdPrefix =
+      typeof options.metadata?.modelIdPrefix === 'string'
+        ? options.metadata.modelIdPrefix
+        : undefined;
+    const modelIdPrefix =
+      options.modelIdPrefix ||
+      metadataModelIdPrefix ||
+      process.env.TRAINING_MODEL_ID_PREFIX ||
+      'eliza-agent';
+
+    // Save to database via adapter
+    const adapter = getTrainingDataAdapter();
+    await adapter.insertModel({
+      id: `model-${Date.now()}`,
+      modelId: `${modelIdPrefix}-${options.version}`,
+      version: options.version,
+      baseModel:
+        (options.metadata?.baseModel as string) || 'unsloth/Qwen3-4B-128K',
+      trainingBatch: (options.metadata?.trainingBatch as string) || null,
+      status: 'ready',
+      deployedAt: null,
+      archivedAt: null,
+      storagePath: blob.url,
+      benchmarkScore: null,
+      accuracy: (options.metadata?.accuracy as number) || null,
+      avgReward: (options.metadata?.avgReward as number) || null,
+      evalMetrics: null,
+      wandbRunId: null,
+      wandbArtifactId: null,
+      huggingFaceRepo: null,
+      agentsUsing: 0,
+      totalInferences: 0,
+      lastBenchmarked: null,
+      benchmarkCount: 0,
+      updatedAt: new Date(),
+    });
+
+    return {
+      version: options.version,
+      baseModel:
+        (options.metadata?.baseModel as string) || 'unsloth/Qwen3-4B-128K',
+      blobUrl: blob.url,
+      size: (blob as { size?: number }).size || 0,
+      uploadedAt: new Date(),
+      metadata: options.metadata || {},
+    };
+  }
+
+  /**
+   * Download model from Vercel Blob
+   */
+  async downloadModel(version: string): Promise<{
+    modelData: Buffer;
+    metadata: ModelVersion['metadata'];
+  }> {
+    const adapter = getTrainingDataAdapter();
+    const model = await adapter.getModelByVersion(version);
+
+    if (!model) {
+      throw new Error(`Model version ${version} not found`);
+    }
+
+    // Download model file
+    const modelResponse = await fetch(model.storagePath);
+    const modelData = Buffer.from(await modelResponse.arrayBuffer());
+
+    // Download metadata
+    const metadataUrl = model.storagePath.replace(/\/[^/]+$/, '/metadata.json');
+    const metadataResponse = await fetch(metadataUrl);
+    const metadata =
+      (await metadataResponse.json()) as ModelVersion['metadata'];
+
+    return {
+      modelData,
+      metadata,
+    };
+  }
+
+  /**
+   * List all model versions
+   */
+  async listModels(): Promise<ModelVersion[]> {
+    const { blobs } = await list({
+      prefix: this.blobPrefix,
+    });
+
+    // Group by version
+    interface BlobInfo {
+      url: string;
+      pathname: string;
+      size: number;
+      uploadedAt: string | Date;
+    }
+
+    interface VersionData {
+      version: string;
+      blobs: BlobInfo[];
+    }
+
+    const versions = new Map<string, VersionData>();
+
+    for (const blob of blobs) {
+      const parts = blob.pathname.split('/');
+      const version = parts[1];
+      if (!version) continue;
+
+      if (!versions.has(version)) {
+        versions.set(version, {
+          version,
+          blobs: [],
+        });
+      }
+      // Convert uploadedAt to string if it's a Date
+      const blobInfo: BlobInfo = {
+        ...blob,
+        uploadedAt:
+          blob.uploadedAt instanceof Date
+            ? blob.uploadedAt.toISOString()
+            : blob.uploadedAt,
+      };
+      versions.get(version)!.blobs.push(blobInfo);
+    }
+
+    // Get metadata for each version
+    const models: ModelVersion[] = [];
+
+    for (const [version, data] of versions) {
+      const modelBlob = data.blobs.find(
+        (b: BlobInfo) =>
+          b.pathname.endsWith('.safetensors') || b.pathname.endsWith('.bin')
+      );
+
+      if (modelBlob) {
+        // Try to get metadata
+        let metadata: ModelVersion['metadata'] = {};
+        try {
+          const metadataBlob = data.blobs.find((b: BlobInfo) =>
+            b.pathname.endsWith('metadata.json')
+          );
+          if (metadataBlob) {
+            const response = await fetch(metadataBlob.url);
+            metadata = (await response.json()) as ModelVersion['metadata'];
+          }
+        } catch {
+          // No metadata, use defaults
+        }
+
+        models.push({
+          version,
+          baseModel: metadata.baseModel || 'unknown',
+          blobUrl: modelBlob.url,
+          size: modelBlob.size,
+          uploadedAt:
+            modelBlob.uploadedAt instanceof Date
+              ? modelBlob.uploadedAt
+              : new Date(modelBlob.uploadedAt),
+          metadata,
+        });
+      }
+    }
+
+    return models.sort(
+      (a, b) => b.uploadedAt.getTime() - a.uploadedAt.getTime()
+    );
+  }
+
+  /**
+   * Delete model version
+   */
+  async deleteModel(version: string): Promise<void> {
+    const { blobs } = await list({
+      prefix: `${this.blobPrefix}${version}/`,
+    });
+
+    for (const blob of blobs) {
+      await del(blob.url);
+    }
+
+    // Update database via adapter
+    const adapter = getTrainingDataAdapter();
+    const model = await adapter.getModelByVersion(version);
+    if (model) {
+      await adapter.updateModelStatus(model.modelId, 'archived', {
+        archivedAt: new Date(),
+      });
+    }
+
+    logger.info('Model deleted from Vercel Blob', { version });
+  }
+
+  /**
+   * Get latest model version
+   */
+  async getLatestVersion(): Promise<ModelVersion | null> {
+    const models = await this.listModels();
+    return models[0] || null;
+  }
+}
+
+// Singleton
+export const modelStorage = new ModelStorageService();

package/src/training/storage/TrainingDataArchiver.ts

@@ -0,0 +1,197 @@
+/**
+ * Training Data Archiver (Vercel Blob)
+ *
+ * Archives training data (exported trajectories, RULER scores) to Vercel Blob
+ * for long-term storage and reproducibility.
+ */
+
+import type { JsonValue } from '../../adapter';
+import { del, list, put } from '@vercel/blob';
+import fs from 'fs/promises';
+import path from 'path';
+import { logger } from '../../utils/logger';
+
+export interface ArchivedWindow {
+  windowId: string;
+  trajectoryCount: number;
+  blobUrls: {
+    trajectories: string;
+    groups?: string;
+    rulerScores?: string;
+    metadata: string;
+  };
+  archivedAt: Date;
+  size: number;
+}
+
+export class TrainingDataArchiver {
+  private readonly blobPrefix = 'training-data/';
+
+  /**
+   * Archive training data for a window
+   */
+  async archiveWindow(options: {
+    windowId: string;
+    trajectoriesPath: string;
+    groupsPath?: string;
+    rulerScoresPath?: string;
+    metadata?: Record<string, unknown>;
+  }): Promise<ArchivedWindow> {
+    logger.info('Archiving training data', { windowId: options.windowId });
+
+    const prefix = `${this.blobPrefix}${options.windowId}/`;
+    interface BlobUrls {
+      trajectories: string;
+      groups?: string;
+      rulerScores?: string;
+      metadata: string;
+    }
+    const urls: BlobUrls = {
+      trajectories: '',
+      metadata: '',
+    };
+    let totalSize = 0;
+
+    // Upload trajectories
+    const trajData = await fs.readFile(options.trajectoriesPath);
+    const trajBlob = await put(`${prefix}trajectories.jsonl`, trajData, {
+      access: 'public',
+      addRandomSuffix: false,
+    });
+    urls.trajectories = trajBlob.url;
+    totalSize += trajData.length;
+
+    // Upload groups if provided
+    if (options.groupsPath) {
+      const groupsData = await fs.readFile(options.groupsPath);
+      const groupsBlob = await put(`${prefix}groups.jsonl`, groupsData, {
+        access: 'public',
+        addRandomSuffix: false,
+      });
+      urls.groups = groupsBlob.url;
+      totalSize += groupsData.length;
+    }
+
+    // Upload RULER scores if provided
+    if (options.rulerScoresPath) {
+      const scoresData = await fs.readFile(options.rulerScoresPath);
+      const scoresBlob = await put(`${prefix}ruler_scores.json`, scoresData, {
+        access: 'public',
+        addRandomSuffix: false,
+      });
+      urls.rulerScores = scoresBlob.url;
+      totalSize += scoresData.length;
+    }
+
+    // Upload metadata
+    const metadataJson = JSON.stringify(options.metadata || {}, null, 2);
+    const metadataBlob = await put(`${prefix}metadata.json`, metadataJson, {
+      access: 'public',
+      addRandomSuffix: false,
+    });
+    urls.metadata = metadataBlob.url;
+    totalSize += Buffer.byteLength(metadataJson, 'utf8');
+
+    logger.info('Training data archived', {
+      windowId: options.windowId,
+      size: totalSize,
+    });
+
+    return {
+      windowId: options.windowId,
+      trajectoryCount: (options.metadata?.trajectoryCount as number) || 0,
+      blobUrls: urls,
+      archivedAt: new Date(),
+      size: totalSize,
+    };
+  }
+
+  /**
+   * Retrieve archived training data
+   */
+  async getWindowData(windowId: string): Promise<{
+    trajectories: string;
+    groups?: string;
+    rulerScores?: Record<string, JsonValue>;
+    metadata: Record<string, JsonValue>;
+  } | null> {
+    const prefix = `${this.blobPrefix}${windowId}/`;
+    const { blobs } = await list({ prefix });
+
+    if (blobs.length === 0) {
+      return null;
+    }
+
+    interface WindowDataResult {
+      trajectories?: string;
+      groups?: string;
+      rulerScores?: Record<string, JsonValue>;
+      metadata?: Record<string, JsonValue>;
+    }
+    const result: WindowDataResult = {};
+
+    for (const blob of blobs) {
+      const response = await fetch(blob.url);
+      const filename = path.basename(blob.pathname);
+
+      if (filename === 'trajectories.jsonl') {
+        result.trajectories = await response.text();
+      } else if (filename === 'groups.jsonl') {
+        result.groups = await response.text();
+      } else if (filename === 'ruler_scores.json') {
+        result.rulerScores = (await response.json()) as Record<
+          string,
+          JsonValue
+        >;
+      } else if (filename === 'metadata.json') {
+        result.metadata = (await response.json()) as Record<string, JsonValue>;
+      }
+    }
+
+    // Ensure required fields are present
+    if (!result.trajectories || !result.metadata) {
+      return null;
+    }
+
+    return {
+      trajectories: result.trajectories,
+      groups: result.groups,
+      rulerScores: result.rulerScores,
+      metadata: result.metadata,
+    };
+  }
+
+  /**
+   * List all archived windows
+   */
+  async listWindows(): Promise<string[]> {
+    const { blobs } = await list({ prefix: this.blobPrefix });
+
+    const windows = new Set<string>();
+    for (const blob of blobs) {
+      const parts = blob.pathname.split('/');
+      if (parts[1]) {
+        windows.add(parts[1]);
+      }
+    }
+
+    return Array.from(windows).sort().reverse();
+  }
+
+  /**
+   * Delete archived window
+   */
+  async deleteWindow(windowId: string): Promise<void> {
+    const prefix = `${this.blobPrefix}${windowId}/`;
+    const { blobs } = await list({ prefix });
+
+    for (const blob of blobs) {
+      await del(blob.url);
+    }
+
+    logger.info('Deleted archived window', { windowId });
+  }
+}
+
+// Singleton
+export const trainingDataArchiver = new TrainingDataArchiver();

package/src/training/storage/index.ts

@@ -0,0 +1,17 @@
+/**
+ * Storage Module
+ *
+ * Services for storing models and training data to Vercel Blob.
+ */
+
+export {
+  ModelStorageService,
+  type ModelVersion,
+  modelStorage,
+} from './ModelStorageService';
+
+export {
+  type ArchivedWindow,
+  TrainingDataArchiver,
+  trainingDataArchiver,
+} from './TrainingDataArchiver';