@fastino-ai/pioneer-cli 0.1.0 → 0.2.0

package/src/evolution/EvolutionEngine.ts

@@ -0,0 +1,319 @@
+/**
+ * EvolutionEngine - Core self-improvement loop
+ * Iteratively improves the agent based on evaluations and feedback
+ */
+
+import * as fs from "fs";
+import * as path from "path";
+import * as os from "os";
+import type { Agent } from "../agent/Agent.js";
+import { FeedbackCollector } from "./FeedbackCollector.js";
+import { EvalRunner, DEFAULT_EVAL_CASES } from "./EvalRunner.js";
+import { ModelTrainer, type TrainingResult } from "./ModelTrainer.js";
+import type {
+  EvolutionConfig,
+  EvolutionState,
+  EvolutionHistory,
+  EvalCase,
+  EvalRunSummary,
+} from "./types.js";
+
+export interface EvolutionEngineConfig {
+  storagePath?: string;
+  evalCases?: EvalCase[];
+  targetScore?: number;
+  maxIterations?: number;
+  budgetPerIteration?: {
+    maxTokens?: number;
+    maxCost?: number;
+    maxTime?: number;
+  };
+  trainingConfig?: {
+    provider: "openai" | "modal" | "local";
+    baseModel: string;
+  };
+}
+
+export interface EvolutionEvents {
+  onIterationStart?: (iteration: number) => void;
+  onIterationEnd?: (iteration: number, score: number) => void;
+  onEvalComplete?: (summary: EvalRunSummary) => void;
+  onTrainingComplete?: (result: TrainingResult) => void;
+  onBudgetWarning?: (message: string) => void;
+  onComplete?: (state: EvolutionState) => void;
+  onError?: (error: Error) => void;
+}
+
+export class EvolutionEngine {
+  private config: EvolutionEngineConfig;
+  private storagePath: string;
+  private feedbackCollector: FeedbackCollector;
+  private evalRunner: EvalRunner;
+  private modelTrainer: ModelTrainer | null = null;
+  private state: EvolutionState;
+  private events: EvolutionEvents;
+
+  constructor(config: EvolutionEngineConfig, events: EvolutionEvents = {}) {
+    this.config = config;
+    this.events = events;
+    this.storagePath =
+      config.storagePath || path.join(os.homedir(), ".pioneer", "evolution");
+    this.ensureStoragePath();
+
+    this.feedbackCollector = new FeedbackCollector({
+      storagePath: path.join(this.storagePath, "feedback"),
+    });
+
+    this.evalRunner = new EvalRunner();
+
+    if (config.trainingConfig) {
+      this.modelTrainer = new ModelTrainer({
+        provider: config.trainingConfig.provider,
+        baseModel: config.trainingConfig.baseModel,
+        outputDir: path.join(this.storagePath, "models"),
+      });
+    }
+
+    this.state = this.loadState() || this.createInitialState();
+  }
+
+  private ensureStoragePath(): void {
+    if (!fs.existsSync(this.storagePath)) {
+      fs.mkdirSync(this.storagePath, { recursive: true });
+    }
+  }
+
+  private createInitialState(): EvolutionState {
+    return {
+      iteration: 0,
+      currentScore: 0,
+      bestScore: 0,
+      bestPrompt: "",
+      history: [],
+      totalTokensUsed: 0,
+      totalCostUsed: 0,
+      totalTimeUsed: 0,
+      startTime: new Date(),
+      status: "running",
+    };
+  }
+
+  private loadState(): EvolutionState | null {
+    const statePath = path.join(this.storagePath, "state.json");
+    try {
+      if (fs.existsSync(statePath)) {
+        const data = fs.readFileSync(statePath, "utf-8");
+        return JSON.parse(data);
+      }
+    } catch {
+      // Ignore errors
+    }
+    return null;
+  }
+
+  private saveState(): void {
+    const statePath = path.join(this.storagePath, "state.json");
+    fs.writeFileSync(statePath, JSON.stringify(this.state, null, 2));
+  }
+
+  async evolve(agent: Agent): Promise<EvolutionState> {
+    const evalCases = this.config.evalCases || DEFAULT_EVAL_CASES;
+    const targetScore = this.config.targetScore || 0.9;
+    const maxIterations = this.config.maxIterations || 10;
+
+    this.state.status = "running";
+    this.state.startTime = new Date();
+
+    while (
+      this.state.iteration < maxIterations &&
+      this.state.status === "running"
+    ) {
+      // Check budget
+      if (!this.checkBudget()) {
+        this.state.status = "budget_exhausted";
+        break;
+      }
+
+      this.state.iteration++;
+      this.events.onIterationStart?.(this.state.iteration);
+
+      try {
+        // Run evaluation
+        const evalSummary = await this.evalRunner.runEvalSuite(agent, evalCases);
+        this.events.onEvalComplete?.(evalSummary);
+
+        // Update state
+        this.state.currentScore = evalSummary.averageScore;
+        this.state.totalTokensUsed += evalSummary.totalTokens;
+        this.state.totalTimeUsed += evalSummary.totalDuration / 1000;
+
+        // Check if target reached
+        if (this.state.currentScore >= targetScore) {
+          this.state.status = "completed";
+          this.events.onIterationEnd?.(this.state.iteration, this.state.currentScore);
+          break;
+        }
+
+        // Update best if improved
+        if (this.state.currentScore > this.state.bestScore) {
+          this.state.bestScore = this.state.currentScore;
+          // Save the current configuration as best
+        }
+
+        // Attempt improvement
+        await this.attemptImprovement(agent, evalSummary);
+
+        // Record history
+        this.state.history.push({
+          iteration: this.state.iteration,
+          prompt: "", // Would store the current prompt
+          evalScore: this.state.currentScore,
+          changes: "Prompt/model adjustment",
+          timestamp: new Date(),
+        });
+
+        this.events.onIterationEnd?.(this.state.iteration, this.state.currentScore);
+        this.saveState();
+      } catch (error) {
+        this.events.onError?.(
+          error instanceof Error ? error : new Error(String(error))
+        );
+        this.state.status = "failed";
+        break;
+      }
+    }
+
+    this.state.endTime = new Date();
+    this.saveState();
+    this.events.onComplete?.(this.state);
+
+    return this.state;
+  }
+
+  private checkBudget(): boolean {
+    const budget = this.config.budgetPerIteration;
+    if (!budget) return true;
+
+    if (budget.maxTokens && this.state.totalTokensUsed >= budget.maxTokens) {
+      this.events.onBudgetWarning?.("Token budget exhausted");
+      return false;
+    }
+
+    if (budget.maxCost && this.state.totalCostUsed >= budget.maxCost) {
+      this.events.onBudgetWarning?.("Cost budget exhausted");
+      return false;
+    }
+
+    if (budget.maxTime && this.state.totalTimeUsed >= budget.maxTime) {
+      this.events.onBudgetWarning?.("Time budget exhausted");
+      return false;
+    }
+
+    return true;
+  }
+
+  private async attemptImprovement(
+    agent: Agent,
+    evalSummary: EvalRunSummary
+  ): Promise<void> {
+    // Get feedback for training
+    const trainingData = this.feedbackCollector.toTrainingData();
+
+    // If we have enough training data, attempt fine-tuning
+    if (trainingData.length >= 50 && this.modelTrainer) {
+      try {
+        const result = await this.modelTrainer.train(trainingData);
+        this.events.onTrainingComplete?.(result);
+
+        if (result.success && result.modelId) {
+          // Would update agent to use the new model
+          console.log(`New model trained: ${result.modelId}`);
+        }
+      } catch (error) {
+        this.events.onError?.(
+          error instanceof Error ? error : new Error(String(error))
+        );
+      }
+    }
+
+    // Analyze failed cases and suggest improvements
+    const failedCases = evalSummary.results.filter((r) => !r.passed);
+    if (failedCases.length > 0) {
+      // Could use the LLM to analyze failures and suggest prompt improvements
+      console.log(
+        `Analyzing ${failedCases.length} failed cases for improvement...`
+      );
+    }
+  }
+
+  // Record interaction feedback
+  recordInteraction(params: {
+    sessionId: string;
+    userMessage: string;
+    agentResponse: string;
+    toolCalls: string[];
+    wasSuccessful: boolean;
+    metadata?: Record<string, unknown>;
+  }): string {
+    return this.feedbackCollector.recordInteraction(params);
+  }
+
+  // Add user rating to feedback
+  rateFeedback(feedbackId: string, rating: number, corrections?: string): void {
+    this.feedbackCollector.addRating(feedbackId, rating, corrections);
+  }
+
+  // Get evolution statistics
+  getStats(): {
+    state: EvolutionState;
+    feedbackStats: ReturnType<FeedbackCollector["getStats"]>;
+  } {
+    return {
+      state: this.state,
+      feedbackStats: this.feedbackCollector.getStats(),
+    };
+  }
+
+  // Reset evolution state
+  reset(): void {
+    this.state = this.createInitialState();
+    this.saveState();
+  }
+
+  // Export feedback for external training
+  exportFeedback(format: "jsonl" | "openai", outputPath: string): void {
+    if (format === "openai") {
+      this.feedbackCollector.exportAsOpenAIFormat(outputPath);
+    } else {
+      this.feedbackCollector.exportAsJsonl(outputPath);
+    }
+  }
+
+  // Run a single evaluation cycle
+  async runEvaluation(agent: Agent): Promise<EvalRunSummary> {
+    const evalCases = this.config.evalCases || DEFAULT_EVAL_CASES;
+    return this.evalRunner.runEvalSuite(agent, evalCases);
+  }
+
+  // Get formatted results
+  formatResults(): string {
+    let output = "\nEvolution Engine Status\n";
+    output += "=".repeat(50) + "\n\n";
+    output += `Status: ${this.state.status}\n`;
+    output += `Iteration: ${this.state.iteration}\n`;
+    output += `Current Score: ${(this.state.currentScore * 100).toFixed(1)}%\n`;
+    output += `Best Score: ${(this.state.bestScore * 100).toFixed(1)}%\n`;
+    output += `Tokens Used: ${this.state.totalTokensUsed.toLocaleString()}\n`;
+    output += `Time Used: ${this.state.totalTimeUsed.toFixed(1)}s\n\n`;
+
+    const feedbackStats = this.feedbackCollector.getStats();
+    output += "Feedback Statistics:\n";
+    output += `  Total: ${feedbackStats.total}\n`;
+    output += `  Rated: ${feedbackStats.rated}\n`;
+    output += `  Avg Rating: ${feedbackStats.avgRating}/5\n`;
+    output += `  Success Rate: ${feedbackStats.successRate}%\n`;
+
+    return output;
+  }
+}
+

package/src/evolution/FeedbackCollector.ts

@@ -0,0 +1,197 @@
+/**
+ * FeedbackCollector - Collects and stores training feedback from interactions
+ */
+
+import * as fs from "fs";
+import * as path from "path";
+import * as os from "os";
+import type { Feedback, TrainingData } from "./types.js";
+
+export interface FeedbackCollectorConfig {
+  storagePath?: string;
+  maxFeedbackItems?: number;
+}
+
+export class FeedbackCollector {
+  private storagePath: string;
+  private maxItems: number;
+  private feedback: Feedback[] = [];
+
+  constructor(config: FeedbackCollectorConfig = {}) {
+    this.storagePath = config.storagePath || path.join(os.homedir(), ".pioneer", "feedback");
+    this.maxItems = config.maxFeedbackItems || 1000;
+    this.ensureStorageDir();
+    this.loadFeedback();
+  }
+
+  private ensureStorageDir(): void {
+    if (!fs.existsSync(this.storagePath)) {
+      fs.mkdirSync(this.storagePath, { recursive: true });
+    }
+  }
+
+  private loadFeedback(): void {
+    const feedbackFile = path.join(this.storagePath, "feedback.json");
+    try {
+      if (fs.existsSync(feedbackFile)) {
+        const data = fs.readFileSync(feedbackFile, "utf-8");
+        this.feedback = JSON.parse(data);
+      }
+    } catch {
+      this.feedback = [];
+    }
+  }
+
+  private saveFeedback(): void {
+    const feedbackFile = path.join(this.storagePath, "feedback.json");
+    fs.writeFileSync(feedbackFile, JSON.stringify(this.feedback, null, 2));
+  }
+
+  recordInteraction(params: {
+    sessionId: string;
+    userMessage: string;
+    agentResponse: string;
+    toolCalls: string[];
+    wasSuccessful: boolean;
+    metadata?: Record<string, unknown>;
+  }): string {
+    const id = `fb_${Date.now()}_${Math.random().toString(36).slice(2, 8)}`;
+    
+    const feedback: Feedback = {
+      id,
+      sessionId: params.sessionId,
+      timestamp: new Date(),
+      userMessage: params.userMessage,
+      agentResponse: params.agentResponse,
+      toolCalls: params.toolCalls,
+      wasSuccessful: params.wasSuccessful,
+      metadata: params.metadata,
+    };
+
+    this.feedback.push(feedback);
+
+    // Trim to max items
+    if (this.feedback.length > this.maxItems) {
+      this.feedback = this.feedback.slice(-this.maxItems);
+    }
+
+    this.saveFeedback();
+    return id;
+  }
+
+  addRating(feedbackId: string, rating: number, corrections?: string): void {
+    const item = this.feedback.find((f) => f.id === feedbackId);
+    if (item) {
+      item.rating = Math.max(1, Math.min(5, rating));
+      if (corrections) {
+        item.corrections = corrections;
+      }
+      this.saveFeedback();
+    }
+  }
+
+  getRecentFeedback(limit = 100): Feedback[] {
+    return this.feedback.slice(-limit);
+  }
+
+  getPositiveFeedback(minRating = 4): Feedback[] {
+    return this.feedback.filter(
+      (f) => f.rating !== undefined && f.rating >= minRating
+    );
+  }
+
+  getNegativeFeedback(maxRating = 2): Feedback[] {
+    return this.feedback.filter(
+      (f) => f.rating !== undefined && f.rating <= maxRating
+    );
+  }
+
+  getSuccessfulInteractions(): Feedback[] {
+    return this.feedback.filter((f) => f.wasSuccessful);
+  }
+
+  getCorrectedInteractions(): Feedback[] {
+    return this.feedback.filter((f) => f.corrections !== undefined);
+  }
+
+  // Convert feedback to training data format
+  toTrainingData(): TrainingData[] {
+    const trainingData: TrainingData[] = [];
+
+    for (const fb of this.feedback) {
+      // Only use successful or highly-rated interactions
+      if (!fb.wasSuccessful && (fb.rating === undefined || fb.rating < 4)) {
+        continue;
+      }
+
+      // Use corrections if available
+      const assistantContent = fb.corrections || fb.agentResponse;
+
+      trainingData.push({
+        id: fb.id,
+        messages: [
+          { role: "user", content: fb.userMessage },
+          { role: "assistant", content: assistantContent },
+        ],
+        toolCalls: fb.toolCalls.map((name) => ({
+          name,
+          arguments: {},
+          result: "",
+        })),
+        metadata: fb.metadata,
+      });
+    }
+
+    return trainingData;
+  }
+
+  // Export training data in various formats
+  exportAsJsonl(outputPath: string): void {
+    const data = this.toTrainingData();
+    const lines = data.map((d) => JSON.stringify(d));
+    fs.writeFileSync(outputPath, lines.join("\n"));
+  }
+
+  exportAsOpenAIFormat(outputPath: string): void {
+    const data = this.toTrainingData();
+    const formatted = data.map((d) => ({
+      messages: d.messages,
+    }));
+    const lines = formatted.map((d) => JSON.stringify(d));
+    fs.writeFileSync(outputPath, lines.join("\n"));
+  }
+
+  // Get statistics
+  getStats(): {
+    total: number;
+    rated: number;
+    avgRating: number;
+    successRate: number;
+    corrected: number;
+  } {
+    const rated = this.feedback.filter((f) => f.rating !== undefined);
+    const avgRating =
+      rated.length > 0
+        ? rated.reduce((sum, f) => sum + (f.rating || 0), 0) / rated.length
+        : 0;
+    const successful = this.feedback.filter((f) => f.wasSuccessful);
+    const corrected = this.feedback.filter((f) => f.corrections !== undefined);
+
+    return {
+      total: this.feedback.length,
+      rated: rated.length,
+      avgRating: Math.round(avgRating * 100) / 100,
+      successRate:
+        this.feedback.length > 0
+          ? Math.round((successful.length / this.feedback.length) * 100)
+          : 0,
+      corrected: corrected.length,
+    };
+  }
+
+  clear(): void {
+    this.feedback = [];
+    this.saveFeedback();
+  }
+}
+