@fastino-ai/pioneer-cli 0.2.1 → 0.2.3

package/src/evolution/EvalRunner.ts

@@ -1,301 +0,0 @@
-/**
- * EvalRunner - Run evaluations to measure agent improvement
- */
-
-import type { Agent } from "../agent/Agent.js";
-import type {
-  EvalCase,
-  EvalResult,
-  EvalRunSummary,
-  EvalCriteria,
-} from "./types.js";
-
-export interface EvalRunnerConfig {
-  timeout?: number;
-  parallelism?: number;
-}
-
-export class EvalRunner {
-  private config: EvalRunnerConfig;
-
-  constructor(config: EvalRunnerConfig = {}) {
-    this.config = {
-      timeout: config.timeout || 60000,
-      parallelism: config.parallelism || 1,
-    };
-  }
-
-  async runEval(agent: Agent, evalCase: EvalCase): Promise<EvalResult> {
-    const startTime = Date.now();
-    const initialUsage = agent.getBudgetStatus().usage.tokensUsed;
-
-    try {
-      // Run the agent with the eval input
-      const response = await agent.chat(evalCase.input, false);
-      const duration = Date.now() - startTime;
-      const tokenUsage = agent.getBudgetStatus().usage.tokensUsed - initialUsage;
-
-      // Get tools that were called (from message history)
-      const messages = agent.getMessages();
-      const toolsCalled: string[] = [];
-      for (const msg of messages) {
-        if (msg.toolCalls) {
-          for (const tc of msg.toolCalls) {
-            toolsCalled.push(tc.name);
-          }
-        }
-      }
-
-      // Evaluate criteria
-      const { passed, score, errors } = this.evaluateCriteria(
-        response,
-        toolsCalled,
-        evalCase
-      );
-
-      return {
-        caseId: evalCase.id,
-        passed,
-        score,
-        actualOutput: response,
-        toolsCalled,
-        errors,
-        duration,
-        tokenUsage,
-      };
-    } catch (error) {
-      return {
-        caseId: evalCase.id,
-        passed: false,
-        score: 0,
-        actualOutput: "",
-        toolsCalled: [],
-        errors: [error instanceof Error ? error.message : String(error)],
-        duration: Date.now() - startTime,
-        tokenUsage: agent.getBudgetStatus().usage.tokensUsed - initialUsage,
-      };
-    }
-  }
-
-  private evaluateCriteria(
-    output: string,
-    toolsCalled: string[],
-    evalCase: EvalCase
-  ): { passed: boolean; score: number; errors: string[] } {
-    const errors: string[] = [];
-    let passedCriteria = 0;
-    const totalCriteria = evalCase.successCriteria.length;
-
-    for (const criteria of evalCase.successCriteria) {
-      const { passed, error } = this.checkCriterion(
-        output,
-        toolsCalled,
-        criteria
-      );
-      if (passed) {
-        passedCriteria++;
-      } else if (error) {
-        errors.push(error);
-      }
-    }
-
-    // Check expected output if provided
-    if (evalCase.expectedOutput) {
-      if (output.toLowerCase().includes(evalCase.expectedOutput.toLowerCase())) {
-        passedCriteria++;
-      } else {
-        errors.push(`Expected output not found: "${evalCase.expectedOutput}"`);
-      }
-    }
-
-    // Check expected tool calls if provided
-    if (evalCase.expectedToolCalls) {
-      const missingTools = evalCase.expectedToolCalls.filter(
-        (t) => !toolsCalled.includes(t)
-      );
-      if (missingTools.length === 0) {
-        passedCriteria++;
-      } else {
-        errors.push(`Expected tools not called: ${missingTools.join(", ")}`);
-      }
-    }
-
-    const score = totalCriteria > 0 ? passedCriteria / totalCriteria : 1;
-    const passed = errors.length === 0 && score >= 0.8;
-
-    return { passed, score, errors };
-  }
-
-  private checkCriterion(
-    output: string,
-    toolsCalled: string[],
-    criteria: EvalCriteria
-  ): { passed: boolean; error?: string } {
-    switch (criteria.type) {
-      case "contains":
-        if (output.toLowerCase().includes(criteria.value.toLowerCase())) {
-          return { passed: true };
-        }
-        return {
-          passed: false,
-          error: `Output should contain: "${criteria.value}"`,
-        };
-
-      case "not_contains":
-        if (!output.toLowerCase().includes(criteria.value.toLowerCase())) {
-          return { passed: true };
-        }
-        return {
-          passed: false,
-          error: `Output should not contain: "${criteria.value}"`,
-        };
-
-      case "tool_called":
-        if (toolsCalled.includes(criteria.value)) {
-          return { passed: true };
-        }
-        return {
-          passed: false,
-          error: `Tool should be called: "${criteria.value}"`,
-        };
-
-      case "tool_not_called":
-        if (!toolsCalled.includes(criteria.value)) {
-          return { passed: true };
-        }
-        return {
-          passed: false,
-          error: `Tool should not be called: "${criteria.value}"`,
-        };
-
-      case "regex":
-        try {
-          const regex = new RegExp(criteria.value, "i");
-          if (regex.test(output)) {
-            return { passed: true };
-          }
-          return {
-            passed: false,
-            error: `Output should match pattern: "${criteria.value}"`,
-          };
-        } catch {
-          return { passed: false, error: `Invalid regex: "${criteria.value}"` };
-        }
-
-      case "custom":
-        // Custom criteria would need to be implemented separately
-        return { passed: true };
-
-      default:
-        return { passed: false, error: `Unknown criteria type` };
-    }
-  }
-
-  async runEvalSuite(agent: Agent, evalCases: EvalCase[]): Promise<EvalRunSummary> {
-    const runId = `eval_${Date.now()}`;
-    const startTime = Date.now();
-    const results: EvalResult[] = [];
-
-    // Run evals sequentially for now (could parallelize)
-    for (const evalCase of evalCases) {
-      // Clear agent history between evals
-      agent.clearHistory();
-
-      const result = await this.runEval(agent, evalCase);
-      results.push(result);
-    }
-
-    const passedCases = results.filter((r) => r.passed).length;
-    const totalTokens = results.reduce((sum, r) => sum + r.tokenUsage, 0);
-    const totalDuration = Date.now() - startTime;
-
-    // Calculate weighted average score
-    let weightedScore = 0;
-    let totalWeight = 0;
-    for (let i = 0; i < results.length; i++) {
-      const weight = evalCases[i].weight || 1;
-      weightedScore += results[i].score * weight;
-      totalWeight += weight;
-    }
-    const averageScore = totalWeight > 0 ? weightedScore / totalWeight : 0;
-
-    return {
-      runId,
-      timestamp: new Date(),
-      totalCases: evalCases.length,
-      passedCases,
-      failedCases: evalCases.length - passedCases,
-      averageScore,
-      totalTokens,
-      totalDuration,
-      results,
-    };
-  }
-
-  // Format results for display
-  formatResults(summary: EvalRunSummary): string {
-    let output = `\nEvaluation Results (${summary.runId})\n`;
-    output += "=".repeat(50) + "\n\n";
-    output += `Total: ${summary.totalCases} | Passed: ${summary.passedCases} | Failed: ${summary.failedCases}\n`;
-    output += `Average Score: ${(summary.averageScore * 100).toFixed(1)}%\n`;
-    output += `Tokens Used: ${summary.totalTokens.toLocaleString()}\n`;
-    output += `Duration: ${(summary.totalDuration / 1000).toFixed(1)}s\n\n`;
-
-    for (const result of summary.results) {
-      const status = result.passed ? "✓" : "✗";
-      const color = result.passed ? "green" : "red";
-      output += `${status} ${result.caseId}: ${(result.score * 100).toFixed(0)}%`;
-      if (result.errors && result.errors.length > 0) {
-        output += `\n   Errors: ${result.errors.join(", ")}`;
-      }
-      output += "\n";
-    }
-
-    return output;
-  }
-}
-
-// Predefined eval cases for common agent capabilities
-export const DEFAULT_EVAL_CASES: EvalCase[] = [
-  {
-    id: "bash_basic",
-    name: "Basic Bash Execution",
-    description: "Test that the agent can execute a simple bash command",
-    input: "What is the current date? Use bash to find out.",
-    successCriteria: [
-      { type: "tool_called", value: "bash" },
-      { type: "regex", value: "\\d{4}" }, // Year should appear
-    ],
-  },
-  {
-    id: "file_read",
-    name: "File Reading",
-    description: "Test that the agent can read a file",
-    input: "Read the contents of package.json and tell me the project name.",
-    successCriteria: [
-      { type: "tool_called", value: "read_file" },
-      { type: "contains", value: "pioneer" },
-    ],
-  },
-  {
-    id: "code_execution",
-    name: "Code Sandbox",
-    description: "Test that the agent can execute code in a sandbox",
-    input: "Calculate the factorial of 5 using Python.",
-    successCriteria: [
-      { type: "tool_called", value: "execute_code" },
-      { type: "contains", value: "120" },
-    ],
-  },
-  {
-    id: "multi_step",
-    name: "Multi-step Task",
-    description: "Test that the agent can complete a multi-step task",
-    input: "List the files in the current directory, then read the README.md file and summarize it.",
-    successCriteria: [
-      { type: "tool_called", value: "list_directory" },
-      { type: "tool_called", value: "read_file" },
-    ],
-  },
-];
-

package/src/evolution/EvolutionEngine.ts

@@ -1,319 +0,0 @@
-/**
- * EvolutionEngine - Core self-improvement loop
- * Iteratively improves the agent based on evaluations and feedback
- */
-
-import * as fs from "fs";
-import * as path from "path";
-import * as os from "os";
-import type { Agent } from "../agent/Agent.js";
-import { FeedbackCollector } from "./FeedbackCollector.js";
-import { EvalRunner, DEFAULT_EVAL_CASES } from "./EvalRunner.js";
-import { ModelTrainer, type TrainingResult } from "./ModelTrainer.js";
-import type {
-  EvolutionConfig,
-  EvolutionState,
-  EvolutionHistory,
-  EvalCase,
-  EvalRunSummary,
-} from "./types.js";
-
-export interface EvolutionEngineConfig {
-  storagePath?: string;
-  evalCases?: EvalCase[];
-  targetScore?: number;
-  maxIterations?: number;
-  budgetPerIteration?: {
-    maxTokens?: number;
-    maxCost?: number;
-    maxTime?: number;
-  };
-  trainingConfig?: {
-    provider: "openai" | "modal" | "local";
-    baseModel: string;
-  };
-}
-
-export interface EvolutionEvents {
-  onIterationStart?: (iteration: number) => void;
-  onIterationEnd?: (iteration: number, score: number) => void;
-  onEvalComplete?: (summary: EvalRunSummary) => void;
-  onTrainingComplete?: (result: TrainingResult) => void;
-  onBudgetWarning?: (message: string) => void;
-  onComplete?: (state: EvolutionState) => void;
-  onError?: (error: Error) => void;
-}
-
-export class EvolutionEngine {
-  private config: EvolutionEngineConfig;
-  private storagePath: string;
-  private feedbackCollector: FeedbackCollector;
-  private evalRunner: EvalRunner;
-  private modelTrainer: ModelTrainer | null = null;
-  private state: EvolutionState;
-  private events: EvolutionEvents;
-
-  constructor(config: EvolutionEngineConfig, events: EvolutionEvents = {}) {
-    this.config = config;
-    this.events = events;
-    this.storagePath =
-      config.storagePath || path.join(os.homedir(), ".pioneer", "evolution");
-    this.ensureStoragePath();
-
-    this.feedbackCollector = new FeedbackCollector({
-      storagePath: path.join(this.storagePath, "feedback"),
-    });
-
-    this.evalRunner = new EvalRunner();
-
-    if (config.trainingConfig) {
-      this.modelTrainer = new ModelTrainer({
-        provider: config.trainingConfig.provider,
-        baseModel: config.trainingConfig.baseModel,
-        outputDir: path.join(this.storagePath, "models"),
-      });
-    }
-
-    this.state = this.loadState() || this.createInitialState();
-  }
-
-  private ensureStoragePath(): void {
-    if (!fs.existsSync(this.storagePath)) {
-      fs.mkdirSync(this.storagePath, { recursive: true });
-    }
-  }
-
-  private createInitialState(): EvolutionState {
-    return {
-      iteration: 0,
-      currentScore: 0,
-      bestScore: 0,
-      bestPrompt: "",
-      history: [],
-      totalTokensUsed: 0,
-      totalCostUsed: 0,
-      totalTimeUsed: 0,
-      startTime: new Date(),
-      status: "running",
-    };
-  }
-
-  private loadState(): EvolutionState | null {
-    const statePath = path.join(this.storagePath, "state.json");
-    try {
-      if (fs.existsSync(statePath)) {
-        const data = fs.readFileSync(statePath, "utf-8");
-        return JSON.parse(data);
-      }
-    } catch {
-      // Ignore errors
-    }
-    return null;
-  }
-
-  private saveState(): void {
-    const statePath = path.join(this.storagePath, "state.json");
-    fs.writeFileSync(statePath, JSON.stringify(this.state, null, 2));
-  }
-
-  async evolve(agent: Agent): Promise<EvolutionState> {
-    const evalCases = this.config.evalCases || DEFAULT_EVAL_CASES;
-    const targetScore = this.config.targetScore || 0.9;
-    const maxIterations = this.config.maxIterations || 10;
-
-    this.state.status = "running";
-    this.state.startTime = new Date();
-
-    while (
-      this.state.iteration < maxIterations &&
-      this.state.status === "running"
-    ) {
-      // Check budget
-      if (!this.checkBudget()) {
-        this.state.status = "budget_exhausted";
-        break;
-      }
-
-      this.state.iteration++;
-      this.events.onIterationStart?.(this.state.iteration);
-
-      try {
-        // Run evaluation
-        const evalSummary = await this.evalRunner.runEvalSuite(agent, evalCases);
-        this.events.onEvalComplete?.(evalSummary);
-
-        // Update state
-        this.state.currentScore = evalSummary.averageScore;
-        this.state.totalTokensUsed += evalSummary.totalTokens;
-        this.state.totalTimeUsed += evalSummary.totalDuration / 1000;
-
-        // Check if target reached
-        if (this.state.currentScore >= targetScore) {
-          this.state.status = "completed";
-          this.events.onIterationEnd?.(this.state.iteration, this.state.currentScore);
-          break;
-        }
-
-        // Update best if improved
-        if (this.state.currentScore > this.state.bestScore) {
-          this.state.bestScore = this.state.currentScore;
-          // Save the current configuration as best
-        }
-
-        // Attempt improvement
-        await this.attemptImprovement(agent, evalSummary);
-
-        // Record history
-        this.state.history.push({
-          iteration: this.state.iteration,
-          prompt: "", // Would store the current prompt
-          evalScore: this.state.currentScore,
-          changes: "Prompt/model adjustment",
-          timestamp: new Date(),
-        });
-
-        this.events.onIterationEnd?.(this.state.iteration, this.state.currentScore);
-        this.saveState();
-      } catch (error) {
-        this.events.onError?.(
-          error instanceof Error ? error : new Error(String(error))
-        );
-        this.state.status = "failed";
-        break;
-      }
-    }
-
-    this.state.endTime = new Date();
-    this.saveState();
-    this.events.onComplete?.(this.state);
-
-    return this.state;
-  }
-
-  private checkBudget(): boolean {
-    const budget = this.config.budgetPerIteration;
-    if (!budget) return true;
-
-    if (budget.maxTokens && this.state.totalTokensUsed >= budget.maxTokens) {
-      this.events.onBudgetWarning?.("Token budget exhausted");
-      return false;
-    }
-
-    if (budget.maxCost && this.state.totalCostUsed >= budget.maxCost) {
-      this.events.onBudgetWarning?.("Cost budget exhausted");
-      return false;
-    }
-
-    if (budget.maxTime && this.state.totalTimeUsed >= budget.maxTime) {
-      this.events.onBudgetWarning?.("Time budget exhausted");
-      return false;
-    }
-
-    return true;
-  }
-
-  private async attemptImprovement(
-    agent: Agent,
-    evalSummary: EvalRunSummary
-  ): Promise<void> {
-    // Get feedback for training
-    const trainingData = this.feedbackCollector.toTrainingData();
-
-    // If we have enough training data, attempt fine-tuning
-    if (trainingData.length >= 50 && this.modelTrainer) {
-      try {
-        const result = await this.modelTrainer.train(trainingData);
-        this.events.onTrainingComplete?.(result);
-
-        if (result.success && result.modelId) {
-          // Would update agent to use the new model
-          console.log(`New model trained: ${result.modelId}`);
-        }
-      } catch (error) {
-        this.events.onError?.(
-          error instanceof Error ? error : new Error(String(error))
-        );
-      }
-    }
-
-    // Analyze failed cases and suggest improvements
-    const failedCases = evalSummary.results.filter((r) => !r.passed);
-    if (failedCases.length > 0) {
-      // Could use the LLM to analyze failures and suggest prompt improvements
-      console.log(
-        `Analyzing ${failedCases.length} failed cases for improvement...`
-      );
-    }
-  }
-
-  // Record interaction feedback
-  recordInteraction(params: {
-    sessionId: string;
-    userMessage: string;
-    agentResponse: string;
-    toolCalls: string[];
-    wasSuccessful: boolean;
-    metadata?: Record<string, unknown>;
-  }): string {
-    return this.feedbackCollector.recordInteraction(params);
-  }
-
-  // Add user rating to feedback
-  rateFeedback(feedbackId: string, rating: number, corrections?: string): void {
-    this.feedbackCollector.addRating(feedbackId, rating, corrections);
-  }
-
-  // Get evolution statistics
-  getStats(): {
-    state: EvolutionState;
-    feedbackStats: ReturnType<FeedbackCollector["getStats"]>;
-  } {
-    return {
-      state: this.state,
-      feedbackStats: this.feedbackCollector.getStats(),
-    };
-  }
-
-  // Reset evolution state
-  reset(): void {
-    this.state = this.createInitialState();
-    this.saveState();
-  }
-
-  // Export feedback for external training
-  exportFeedback(format: "jsonl" | "openai", outputPath: string): void {
-    if (format === "openai") {
-      this.feedbackCollector.exportAsOpenAIFormat(outputPath);
-    } else {
-      this.feedbackCollector.exportAsJsonl(outputPath);
-    }
-  }
-
-  // Run a single evaluation cycle
-  async runEvaluation(agent: Agent): Promise<EvalRunSummary> {
-    const evalCases = this.config.evalCases || DEFAULT_EVAL_CASES;
-    return this.evalRunner.runEvalSuite(agent, evalCases);
-  }
-
-  // Get formatted results
-  formatResults(): string {
-    let output = "\nEvolution Engine Status\n";
-    output += "=".repeat(50) + "\n\n";
-    output += `Status: ${this.state.status}\n`;
-    output += `Iteration: ${this.state.iteration}\n`;
-    output += `Current Score: ${(this.state.currentScore * 100).toFixed(1)}%\n`;
-    output += `Best Score: ${(this.state.bestScore * 100).toFixed(1)}%\n`;
-    output += `Tokens Used: ${this.state.totalTokensUsed.toLocaleString()}\n`;
-    output += `Time Used: ${this.state.totalTimeUsed.toFixed(1)}s\n\n`;
-
-    const feedbackStats = this.feedbackCollector.getStats();
-    output += "Feedback Statistics:\n";
-    output += `  Total: ${feedbackStats.total}\n`;
-    output += `  Rated: ${feedbackStats.rated}\n`;
-    output += `  Avg Rating: ${feedbackStats.avgRating}/5\n`;
-    output += `  Success Rate: ${feedbackStats.successRate}%\n`;
-
-    return output;
-  }
-}
-