@fastino-ai/pioneer-cli 0.1.0 → 0.2.1

package/src/evolution/ModelTrainer.ts

@@ -0,0 +1,371 @@
+/**
+ * ModelTrainer - Fine-tune or train models based on feedback
+ */
+
+import * as fs from "fs";
+import * as path from "path";
+import * as os from "os";
+import { spawn } from "child_process";
+import type { TrainingData } from "./types.js";
+
+export interface ModelTrainerConfig {
+  provider: "openai" | "anthropic" | "modal" | "local";
+  baseModel: string;
+  outputDir?: string;
+  openaiApiKey?: string;
+  anthropicApiKey?: string;
+  modalTokenId?: string;
+  modalTokenSecret?: string;
+}
+
+export interface TrainingResult {
+  success: boolean;
+  modelId?: string;
+  modelPath?: string;
+  metrics?: {
+    loss?: number;
+    accuracy?: number;
+    epochs?: number;
+  };
+  error?: string;
+}
+
+export class ModelTrainer {
+  private config: ModelTrainerConfig;
+  private outputDir: string;
+
+  constructor(config: ModelTrainerConfig) {
+    this.config = config;
+    this.outputDir =
+      config.outputDir || path.join(os.homedir(), ".pioneer", "models");
+    this.ensureOutputDir();
+  }
+
+  private ensureOutputDir(): void {
+    if (!fs.existsSync(this.outputDir)) {
+      fs.mkdirSync(this.outputDir, { recursive: true });
+    }
+  }
+
+  async train(trainingData: TrainingData[]): Promise<TrainingResult> {
+    if (trainingData.length < 10) {
+      return {
+        success: false,
+        error: "Insufficient training data. Need at least 10 examples.",
+      };
+    }
+
+    switch (this.config.provider) {
+      case "openai":
+        return this.trainOpenAI(trainingData);
+      case "modal":
+        return this.trainModal(trainingData);
+      case "local":
+        return this.trainLocal(trainingData);
+      default:
+        return {
+          success: false,
+          error: `Unsupported provider: ${this.config.provider}`,
+        };
+    }
+  }
+
+  private async trainOpenAI(trainingData: TrainingData[]): Promise<TrainingResult> {
+    const apiKey = this.config.openaiApiKey || process.env.OPENAI_API_KEY;
+    if (!apiKey) {
+      return { success: false, error: "OpenAI API key not provided" };
+    }
+
+    // Prepare training file
+    const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "openai-train-"));
+    const trainingFile = path.join(tempDir, "training.jsonl");
+
+    const formattedData = trainingData.map((d) => ({
+      messages: d.messages,
+    }));
+
+    fs.writeFileSync(
+      trainingFile,
+      formattedData.map((d) => JSON.stringify(d)).join("\n")
+    );
+
+    try {
+      // Upload file
+      const uploadResponse = await fetch(
+        "https://api.openai.com/v1/files",
+        {
+          method: "POST",
+          headers: {
+            Authorization: `Bearer ${apiKey}`,
+          },
+          body: (() => {
+            const formData = new FormData();
+            formData.append("purpose", "fine-tune");
+            formData.append(
+              "file",
+              new Blob([fs.readFileSync(trainingFile)]),
+              "training.jsonl"
+            );
+            return formData;
+          })(),
+        }
+      );
+
+      if (!uploadResponse.ok) {
+        const error = await uploadResponse.text();
+        return { success: false, error: `File upload failed: ${error}` };
+      }
+
+      const uploadResult = await uploadResponse.json() as { id: string };
+
+      // Create fine-tuning job
+      const ftResponse = await fetch(
+        "https://api.openai.com/v1/fine_tuning/jobs",
+        {
+          method: "POST",
+          headers: {
+            Authorization: `Bearer ${apiKey}`,
+            "Content-Type": "application/json",
+          },
+          body: JSON.stringify({
+            training_file: uploadResult.id,
+            model: this.config.baseModel || "gpt-4o-mini-2024-07-18",
+          }),
+        }
+      );
+
+      if (!ftResponse.ok) {
+        const error = await ftResponse.text();
+        return { success: false, error: `Fine-tuning failed: ${error}` };
+      }
+
+      const ftResult = await ftResponse.json() as { id: string; fine_tuned_model?: string };
+
+      return {
+        success: true,
+        modelId: ftResult.id,
+        modelPath: ftResult.fine_tuned_model,
+      };
+    } finally {
+      fs.rmSync(tempDir, { recursive: true, force: true });
+    }
+  }
+
+  private async trainModal(trainingData: TrainingData[]): Promise<TrainingResult> {
+    const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "modal-train-"));
+    const dataFile = path.join(tempDir, "training_data.jsonl");
+    const appFile = path.join(tempDir, "train_app.py");
+
+    // Save training data
+    fs.writeFileSync(
+      dataFile,
+      trainingData.map((d) => JSON.stringify(d)).join("\n")
+    );
+
+    // Create Modal training app
+    const modalCode = `
+import modal
+import json
+
+app = modal.App("pioneer-finetune")
+
+volume = modal.Volume.from_name("pioneer-models", create_if_missing=True)
+
+image = modal.Image.debian_slim(python_version="3.11").pip_install([
+    "torch",
+    "transformers",
+    "datasets",
+    "accelerate",
+    "peft",
+    "bitsandbytes",
+])
+
+@app.function(
+    image=image,
+    gpu="A10G",
+    timeout=7200,
+    volumes={"/models": volume},
+)
+def finetune(data_json: str, base_model: str, output_name: str):
+    import torch
+    from transformers import AutoModelForCausalLM, AutoTokenizer, TrainingArguments, Trainer
+    from datasets import Dataset
+    from peft import LoraConfig, get_peft_model
+    
+    # Parse data
+    data = [json.loads(line) for line in data_json.strip().split("\\n")]
+    
+    # Convert to dataset
+    texts = []
+    for item in data:
+        text = ""
+        for msg in item.get("messages", []):
+            text += f"{msg['role']}: {msg['content']}\\n"
+        texts.append(text)
+    
+    dataset = Dataset.from_dict({"text": texts})
+    
+    # Load model
+    model = AutoModelForCausalLM.from_pretrained(
+        base_model,
+        torch_dtype=torch.float16,
+        device_map="auto",
+    )
+    tokenizer = AutoTokenizer.from_pretrained(base_model)
+    tokenizer.pad_token = tokenizer.eos_token
+    
+    # Apply LoRA
+    lora_config = LoraConfig(r=16, lora_alpha=32, target_modules=["q_proj", "v_proj"])
+    model = get_peft_model(model, lora_config)
+    
+    # Tokenize
+    def tokenize(examples):
+        return tokenizer(examples["text"], truncation=True, max_length=512, padding="max_length")
+    
+    tokenized = dataset.map(tokenize, batched=True)
+    
+    # Train
+    training_args = TrainingArguments(
+        output_dir=f"/models/{output_name}",
+        num_train_epochs=3,
+        per_device_train_batch_size=4,
+        learning_rate=2e-5,
+        save_steps=500,
+        logging_steps=10,
+    )
+    
+    trainer = Trainer(
+        model=model,
+        args=training_args,
+        train_dataset=tokenized,
+    )
+    trainer.train()
+    
+    # Save
+    trainer.save_model(f"/models/{output_name}")
+    volume.commit()
+    
+    return {"success": True, "model_path": f"/models/{output_name}"}
+
+@app.local_entrypoint()
+def main():
+    import sys
+    data_file = sys.argv[1] if len(sys.argv) > 1 else "training_data.jsonl"
+    base_model = "${this.config.baseModel || "meta-llama/Llama-2-7b-hf"}"
+    output_name = "pioneer-ft-" + str(int(__import__("time").time()))
+    
+    with open(data_file) as f:
+        data_json = f.read()
+    
+    result = finetune.remote(data_json, base_model, output_name)
+    print(json.dumps(result))
+`;
+
+    fs.writeFileSync(appFile, modalCode);
+
+    try {
+      return new Promise((resolve) => {
+        let stdout = "";
+        let stderr = "";
+
+        const env: NodeJS.ProcessEnv = { ...process.env };
+        if (this.config.modalTokenId) {
+          env.MODAL_TOKEN_ID = this.config.modalTokenId;
+        }
+        if (this.config.modalTokenSecret) {
+          env.MODAL_TOKEN_SECRET = this.config.modalTokenSecret;
+        }
+
+        const proc = spawn("modal", ["run", appFile, "--", dataFile], {
+          stdio: ["pipe", "pipe", "pipe"],
+          env,
+        });
+
+        proc.stdout.on("data", (data: Buffer) => {
+          stdout += data.toString();
+        });
+
+        proc.stderr.on("data", (data: Buffer) => {
+          stderr += data.toString();
+        });
+
+        proc.on("close", (code) => {
+          if (code === 0) {
+            try {
+              const result = JSON.parse(stdout.trim().split("\n").pop() || "{}");
+              resolve({
+                success: true,
+                modelPath: result.model_path,
+              });
+            } catch {
+              resolve({ success: true, modelPath: stdout });
+            }
+          } else {
+            resolve({
+              success: false,
+              error: stderr || `Exit code: ${code}`,
+            });
+          }
+        });
+
+        proc.on("error", (err) => {
+          resolve({ success: false, error: err.message });
+        });
+      });
+    } finally {
+      fs.rmSync(tempDir, { recursive: true, force: true });
+    }
+  }
+
+  private async trainLocal(trainingData: TrainingData[]): Promise<TrainingResult> {
+    // For local training, we'll save the data and provide instructions
+    const outputPath = path.join(
+      this.outputDir,
+      `training_${Date.now()}.jsonl`
+    );
+
+    fs.writeFileSync(
+      outputPath,
+      trainingData.map((d) => JSON.stringify(d)).join("\n")
+    );
+
+    return {
+      success: true,
+      modelPath: outputPath,
+      metrics: {
+        epochs: 0,
+      },
+    };
+  }
+
+  // Prompt optimization (alternative to full fine-tuning)
+  async optimizePrompt(
+    currentPrompt: string,
+    feedback: { positive: string[]; negative: string[] }
+  ): Promise<string> {
+    // Use the agent itself to improve its system prompt based on feedback
+    const optimizationPrompt = `You are a prompt engineer. Given the current system prompt and feedback, create an improved version.
+
+Current Prompt:
+${currentPrompt}
+
+Positive Examples (what worked well):
+${feedback.positive.slice(0, 5).join("\n---\n")}
+
+Negative Examples (what didn't work):
+${feedback.negative.slice(0, 5).join("\n---\n")}
+
+Create an improved system prompt that:
+1. Keeps what's working well
+2. Addresses the issues in negative examples
+3. Maintains the core capabilities
+4. Is clear and actionable
+
+Improved Prompt:`;
+
+    // For now, return the original prompt
+    // In a full implementation, this would call an LLM to generate the improved prompt
+    return currentPrompt;
+  }
+}
+

package/src/evolution/index.ts

@@ -0,0 +1,18 @@
+/**
+ * Evolution module exports
+ */
+
+export { EvolutionEngine } from "./EvolutionEngine.js";
+export type { EvolutionEngineConfig, EvolutionEvents } from "./EvolutionEngine.js";
+
+export { FeedbackCollector } from "./FeedbackCollector.js";
+export type { FeedbackCollectorConfig } from "./FeedbackCollector.js";
+
+export { EvalRunner, DEFAULT_EVAL_CASES } from "./EvalRunner.js";
+export type { EvalRunnerConfig } from "./EvalRunner.js";
+
+export { ModelTrainer } from "./ModelTrainer.js";
+export type { ModelTrainerConfig, TrainingResult } from "./ModelTrainer.js";
+
+export * from "./types.js";
+

package/src/evolution/types.ts

@@ -0,0 +1,110 @@
+/**
+ * Types for the self-evolution system
+ */
+
+export interface Feedback {
+  id: string;
+  sessionId: string;
+  timestamp: Date;
+  userMessage: string;
+  agentResponse: string;
+  toolCalls: string[];
+  rating?: number; // 1-5
+  corrections?: string;
+  wasSuccessful: boolean;
+  metadata?: Record<string, unknown>;
+}
+
+export interface EvalCase {
+  id: string;
+  name: string;
+  description: string;
+  input: string;
+  expectedOutput?: string;
+  expectedToolCalls?: string[];
+  successCriteria: EvalCriteria[];
+  weight?: number;
+}
+
+export interface EvalCriteria {
+  type: "contains" | "not_contains" | "tool_called" | "tool_not_called" | "regex" | "custom";
+  value: string;
+  description?: string;
+}
+
+export interface EvalResult {
+  caseId: string;
+  passed: boolean;
+  score: number;
+  actualOutput: string;
+  toolsCalled: string[];
+  errors?: string[];
+  duration: number;
+  tokenUsage: number;
+}
+
+export interface EvalRunSummary {
+  runId: string;
+  timestamp: Date;
+  totalCases: number;
+  passedCases: number;
+  failedCases: number;
+  averageScore: number;
+  totalTokens: number;
+  totalDuration: number;
+  results: EvalResult[];
+}
+
+export interface EvolutionConfig {
+  evalCases: EvalCase[];
+  targetScore: number;
+  maxIterations: number;
+  budgetPerIteration: {
+    maxTokens?: number;
+    maxCost?: number;
+    maxTime?: number;
+  };
+  feedbackWindow: number; // Number of recent feedback items to consider
+  trainingConfig?: {
+    provider: "openai" | "anthropic" | "modal";
+    baseModel: string;
+    fineTuneMethod: "full" | "lora" | "prompt";
+  };
+}
+
+export interface EvolutionState {
+  iteration: number;
+  currentScore: number;
+  bestScore: number;
+  bestPrompt: string;
+  history: EvolutionHistory[];
+  totalTokensUsed: number;
+  totalCostUsed: number;
+  totalTimeUsed: number;
+  startTime: Date;
+  endTime?: Date;
+  status: "running" | "completed" | "failed" | "budget_exhausted";
+}
+
+export interface EvolutionHistory {
+  iteration: number;
+  prompt: string;
+  evalScore: number;
+  changes: string;
+  timestamp: Date;
+}
+
+export interface TrainingData {
+  id: string;
+  messages: Array<{
+    role: "user" | "assistant" | "system";
+    content: string;
+  }>;
+  toolCalls?: Array<{
+    name: string;
+    arguments: Record<string, unknown>;
+    result: string;
+  }>;
+  metadata?: Record<string, unknown>;
+}
+