@fastino-ai/pioneer-cli 0.1.0 → 0.2.0

package/src/config.ts

@@ -1,15 +1,69 @@
 /**
  * Configuration management for Pioneer CLI.
- * Stores API key and base URL in ~/.pioneer/config.json
+ * Stores API key, base URL, and agent settings in ~/.pioneer/config.json
  */
 
 import fs from "fs";
 import os from "os";
 import path from "path";
 
+export interface AgentProviderConfig {
+  provider: "anthropic" | "openai" | "local";
+  model: string;
+  apiKey?: string;
+  baseUrl?: string;
+}
+
+export interface BudgetConfig {
+  maxTokens?: number;
+  maxCost?: number; // in USD
+  maxTime?: number; // in seconds
+  maxIterations?: number;
+}
+
+export interface SandboxConfig {
+  useDocker?: boolean;
+  dockerImage?: string;
+  timeout?: number;
+  memoryLimit?: string;
+  cpuLimit?: number;
+}
+
+export interface MLConfig {
+  modal?: {
+    tokenId?: string;
+    tokenSecret?: string;
+  };
+  wandb?: {
+    apiKey?: string;
+    entity?: string;
+    project?: string;
+  };
+}
+
+export interface EvolutionConfigOptions {
+  enabled?: boolean;
+  targetScore?: number;
+  maxIterations?: number;
+  budgetPerIteration?: BudgetConfig;
+  trainingProvider?: "openai" | "modal" | "local";
+  trainingBaseModel?: string;
+}
+
 export interface Config {
+  // Existing Pioneer config
   apiKey?: string;
   baseUrl?: string;
+
+  // Agent config
+  agent?: AgentProviderConfig;
+  budget?: BudgetConfig;
+  sandbox?: SandboxConfig;
+  ml?: MLConfig;
+  evolution?: EvolutionConfigOptions;
+
+  // System prompt customization
+  systemPrompt?: string;
 }
 
 const CONFIG_DIR = path.join(os.homedir(), ".pioneer");
@@ -18,6 +72,18 @@ const CONFIG_FILE = path.join(CONFIG_DIR, "config.json");
 export const DEFAULT_BASE_URL =
   process.env.PIONEER_API_URL ?? "http://localhost:5001";
 
+export const DEFAULT_AGENT_CONFIG: AgentProviderConfig = {
+  provider: "anthropic",
+  model: "claude-sonnet-4-5-20250929",
+};
+
+export const DEFAULT_BUDGET: BudgetConfig = {
+  maxTokens: 500000,  // 500k tokens
+  maxCost: 5.0,       // $5 USD
+  maxTime: 7200,      // 2 hours
+  maxIterations: 100,
+};
+
 function ensureConfigDir(): void {
   if (!fs.existsSync(CONFIG_DIR)) {
     fs.mkdirSync(CONFIG_DIR, { recursive: true });
@@ -36,13 +102,37 @@ export function loadConfig(): Config {
   return {};
 }
 
-export function saveConfig(config: Config): void {
+export function saveConfig(config: Partial<Config>): void {
   ensureConfigDir();
   const existing = loadConfig();
-  const merged = { ...existing, ...config };
+  const merged = deepMerge(existing, config);
   fs.writeFileSync(CONFIG_FILE, JSON.stringify(merged, null, 2));
 }
 
+function deepMerge(target: Config, source: Partial<Config>): Config {
+  const result = { ...target };
+  for (const key of Object.keys(source) as Array<keyof Config>) {
+    const sourceValue = source[key];
+    const targetValue = result[key];
+    if (
+      sourceValue !== null &&
+      typeof sourceValue === "object" &&
+      !Array.isArray(sourceValue) &&
+      targetValue !== null &&
+      typeof targetValue === "object" &&
+      !Array.isArray(targetValue)
+    ) {
+      (result as Record<string, unknown>)[key] = {
+        ...targetValue,
+        ...sourceValue,
+      };
+    } else if (sourceValue !== undefined) {
+      (result as Record<string, unknown>)[key] = sourceValue;
+    }
+  }
+  return result;
+}
+
 export function clearApiKey(): void {
   const config = loadConfig();
   delete config.apiKey;
@@ -62,3 +152,95 @@ export function getBaseUrl(): string {
   const config = loadConfig();
   return config.baseUrl ?? DEFAULT_BASE_URL;
 }
+
+// Agent configuration helpers
+export function getAgentConfig(): AgentProviderConfig {
+  const config = loadConfig();
+
+  // Check environment variables for API keys
+  const anthropicKey = process.env.ANTHROPIC_API_KEY;
+  const openaiKey = process.env.OPENAI_API_KEY;
+
+  if (config.agent) {
+    return {
+      ...config.agent,
+      apiKey: config.agent.apiKey ||
+        (config.agent.provider === "anthropic" ? anthropicKey : openaiKey),
+    };
+  }
+
+  // Default to Anthropic if key is available
+  if (anthropicKey) {
+    return {
+      ...DEFAULT_AGENT_CONFIG,
+      apiKey: anthropicKey,
+    };
+  }
+
+  // Fall back to OpenAI
+  if (openaiKey) {
+    return {
+      provider: "openai",
+      model: "gpt-4o",
+      apiKey: openaiKey,
+    };
+  }
+
+  return DEFAULT_AGENT_CONFIG;
+}
+
+export function getBudgetConfig(): BudgetConfig {
+  const config = loadConfig();
+  return {
+    ...DEFAULT_BUDGET,
+    ...config.budget,
+  };
+}
+
+export function getSandboxConfig(): SandboxConfig {
+  const config = loadConfig();
+  return {
+    useDocker: false,
+    dockerImage: "python:3.11-slim",
+    timeout: 30000,
+    memoryLimit: "512m",
+    cpuLimit: 1,
+    ...config.sandbox,
+  };
+}
+
+export function getMLConfig(): MLConfig {
+  const config = loadConfig();
+  return {
+    modal: {
+      tokenId: process.env.MODAL_TOKEN_ID,
+      tokenSecret: process.env.MODAL_TOKEN_SECRET,
+      ...config.ml?.modal,
+    },
+    wandb: {
+      apiKey: process.env.WANDB_API_KEY,
+      ...config.ml?.wandb,
+    },
+  };
+}
+
+export function getEvolutionConfig(): EvolutionConfigOptions {
+  const config = loadConfig();
+  return {
+    enabled: false,
+    targetScore: 0.9,
+    maxIterations: 10,
+    trainingProvider: "local",
+    ...config.evolution,
+  };
+}
+
+export function getSystemPrompt(): string | undefined {
+  return loadConfig().systemPrompt;
+}
+
+// Get config directory path
+export function getConfigDir(): string {
+  ensureConfigDir();
+  return CONFIG_DIR;
+}

package/src/evolution/EvalRunner.ts

@@ -0,0 +1,301 @@
+/**
+ * EvalRunner - Run evaluations to measure agent improvement
+ */
+
+import type { Agent } from "../agent/Agent.js";
+import type {
+  EvalCase,
+  EvalResult,
+  EvalRunSummary,
+  EvalCriteria,
+} from "./types.js";
+
+export interface EvalRunnerConfig {
+  timeout?: number;
+  parallelism?: number;
+}
+
+export class EvalRunner {
+  private config: EvalRunnerConfig;
+
+  constructor(config: EvalRunnerConfig = {}) {
+    this.config = {
+      timeout: config.timeout || 60000,
+      parallelism: config.parallelism || 1,
+    };
+  }
+
+  async runEval(agent: Agent, evalCase: EvalCase): Promise<EvalResult> {
+    const startTime = Date.now();
+    const initialUsage = agent.getBudgetStatus().usage.tokensUsed;
+
+    try {
+      // Run the agent with the eval input
+      const response = await agent.chat(evalCase.input, false);
+      const duration = Date.now() - startTime;
+      const tokenUsage = agent.getBudgetStatus().usage.tokensUsed - initialUsage;
+
+      // Get tools that were called (from message history)
+      const messages = agent.getMessages();
+      const toolsCalled: string[] = [];
+      for (const msg of messages) {
+        if (msg.toolCalls) {
+          for (const tc of msg.toolCalls) {
+            toolsCalled.push(tc.name);
+          }
+        }
+      }
+
+      // Evaluate criteria
+      const { passed, score, errors } = this.evaluateCriteria(
+        response,
+        toolsCalled,
+        evalCase
+      );
+
+      return {
+        caseId: evalCase.id,
+        passed,
+        score,
+        actualOutput: response,
+        toolsCalled,
+        errors,
+        duration,
+        tokenUsage,
+      };
+    } catch (error) {
+      return {
+        caseId: evalCase.id,
+        passed: false,
+        score: 0,
+        actualOutput: "",
+        toolsCalled: [],
+        errors: [error instanceof Error ? error.message : String(error)],
+        duration: Date.now() - startTime,
+        tokenUsage: agent.getBudgetStatus().usage.tokensUsed - initialUsage,
+      };
+    }
+  }
+
+  private evaluateCriteria(
+    output: string,
+    toolsCalled: string[],
+    evalCase: EvalCase
+  ): { passed: boolean; score: number; errors: string[] } {
+    const errors: string[] = [];
+    let passedCriteria = 0;
+    const totalCriteria = evalCase.successCriteria.length;
+
+    for (const criteria of evalCase.successCriteria) {
+      const { passed, error } = this.checkCriterion(
+        output,
+        toolsCalled,
+        criteria
+      );
+      if (passed) {
+        passedCriteria++;
+      } else if (error) {
+        errors.push(error);
+      }
+    }
+
+    // Check expected output if provided
+    if (evalCase.expectedOutput) {
+      if (output.toLowerCase().includes(evalCase.expectedOutput.toLowerCase())) {
+        passedCriteria++;
+      } else {
+        errors.push(`Expected output not found: "${evalCase.expectedOutput}"`);
+      }
+    }
+
+    // Check expected tool calls if provided
+    if (evalCase.expectedToolCalls) {
+      const missingTools = evalCase.expectedToolCalls.filter(
+        (t) => !toolsCalled.includes(t)
+      );
+      if (missingTools.length === 0) {
+        passedCriteria++;
+      } else {
+        errors.push(`Expected tools not called: ${missingTools.join(", ")}`);
+      }
+    }
+
+    const score = totalCriteria > 0 ? passedCriteria / totalCriteria : 1;
+    const passed = errors.length === 0 && score >= 0.8;
+
+    return { passed, score, errors };
+  }
+
+  private checkCriterion(
+    output: string,
+    toolsCalled: string[],
+    criteria: EvalCriteria
+  ): { passed: boolean; error?: string } {
+    switch (criteria.type) {
+      case "contains":
+        if (output.toLowerCase().includes(criteria.value.toLowerCase())) {
+          return { passed: true };
+        }
+        return {
+          passed: false,
+          error: `Output should contain: "${criteria.value}"`,
+        };
+
+      case "not_contains":
+        if (!output.toLowerCase().includes(criteria.value.toLowerCase())) {
+          return { passed: true };
+        }
+        return {
+          passed: false,
+          error: `Output should not contain: "${criteria.value}"`,
+        };
+
+      case "tool_called":
+        if (toolsCalled.includes(criteria.value)) {
+          return { passed: true };
+        }
+        return {
+          passed: false,
+          error: `Tool should be called: "${criteria.value}"`,
+        };
+
+      case "tool_not_called":
+        if (!toolsCalled.includes(criteria.value)) {
+          return { passed: true };
+        }
+        return {
+          passed: false,
+          error: `Tool should not be called: "${criteria.value}"`,
+        };
+
+      case "regex":
+        try {
+          const regex = new RegExp(criteria.value, "i");
+          if (regex.test(output)) {
+            return { passed: true };
+          }
+          return {
+            passed: false,
+            error: `Output should match pattern: "${criteria.value}"`,
+          };
+        } catch {
+          return { passed: false, error: `Invalid regex: "${criteria.value}"` };
+        }
+
+      case "custom":
+        // Custom criteria would need to be implemented separately
+        return { passed: true };
+
+      default:
+        return { passed: false, error: `Unknown criteria type` };
+    }
+  }
+
+  async runEvalSuite(agent: Agent, evalCases: EvalCase[]): Promise<EvalRunSummary> {
+    const runId = `eval_${Date.now()}`;
+    const startTime = Date.now();
+    const results: EvalResult[] = [];
+
+    // Run evals sequentially for now (could parallelize)
+    for (const evalCase of evalCases) {
+      // Clear agent history between evals
+      agent.clearHistory();
+
+      const result = await this.runEval(agent, evalCase);
+      results.push(result);
+    }
+
+    const passedCases = results.filter((r) => r.passed).length;
+    const totalTokens = results.reduce((sum, r) => sum + r.tokenUsage, 0);
+    const totalDuration = Date.now() - startTime;
+
+    // Calculate weighted average score
+    let weightedScore = 0;
+    let totalWeight = 0;
+    for (let i = 0; i < results.length; i++) {
+      const weight = evalCases[i].weight || 1;
+      weightedScore += results[i].score * weight;
+      totalWeight += weight;
+    }
+    const averageScore = totalWeight > 0 ? weightedScore / totalWeight : 0;
+
+    return {
+      runId,
+      timestamp: new Date(),
+      totalCases: evalCases.length,
+      passedCases,
+      failedCases: evalCases.length - passedCases,
+      averageScore,
+      totalTokens,
+      totalDuration,
+      results,
+    };
+  }
+
+  // Format results for display
+  formatResults(summary: EvalRunSummary): string {
+    let output = `\nEvaluation Results (${summary.runId})\n`;
+    output += "=".repeat(50) + "\n\n";
+    output += `Total: ${summary.totalCases} | Passed: ${summary.passedCases} | Failed: ${summary.failedCases}\n`;
+    output += `Average Score: ${(summary.averageScore * 100).toFixed(1)}%\n`;
+    output += `Tokens Used: ${summary.totalTokens.toLocaleString()}\n`;
+    output += `Duration: ${(summary.totalDuration / 1000).toFixed(1)}s\n\n`;
+
+    for (const result of summary.results) {
+      const status = result.passed ? "✓" : "✗";
+      const color = result.passed ? "green" : "red";
+      output += `${status} ${result.caseId}: ${(result.score * 100).toFixed(0)}%`;
+      if (result.errors && result.errors.length > 0) {
+        output += `\n   Errors: ${result.errors.join(", ")}`;
+      }
+      output += "\n";
+    }
+
+    return output;
+  }
+}
+
+// Predefined eval cases for common agent capabilities
+export const DEFAULT_EVAL_CASES: EvalCase[] = [
+  {
+    id: "bash_basic",
+    name: "Basic Bash Execution",
+    description: "Test that the agent can execute a simple bash command",
+    input: "What is the current date? Use bash to find out.",
+    successCriteria: [
+      { type: "tool_called", value: "bash" },
+      { type: "regex", value: "\\d{4}" }, // Year should appear
+    ],
+  },
+  {
+    id: "file_read",
+    name: "File Reading",
+    description: "Test that the agent can read a file",
+    input: "Read the contents of package.json and tell me the project name.",
+    successCriteria: [
+      { type: "tool_called", value: "read_file" },
+      { type: "contains", value: "pioneer" },
+    ],
+  },
+  {
+    id: "code_execution",
+    name: "Code Sandbox",
+    description: "Test that the agent can execute code in a sandbox",
+    input: "Calculate the factorial of 5 using Python.",
+    successCriteria: [
+      { type: "tool_called", value: "execute_code" },
+      { type: "contains", value: "120" },
+    ],
+  },
+  {
+    id: "multi_step",
+    name: "Multi-step Task",
+    description: "Test that the agent can complete a multi-step task",
+    input: "List the files in the current directory, then read the README.md file and summarize it.",
+    successCriteria: [
+      { type: "tool_called", value: "list_directory" },
+      { type: "tool_called", value: "read_file" },
+    ],
+  },
+];
+