@yun520-1/heartflow 0.15.2 → 0.16.0

package/src/core/self-evolution/reflexion-agent.js

@@ -1,782 +0,0 @@
-/**
- * Reflexion Agent - Enhanced Implementation
- * Based on "Reflexion: Language Agents with Verbal Reinforcement Learning" (Shinn et al. 2023)
- * arXiv: 2303.11366
- * 
- * Core Architecture:
- * - Actor (Ma): Generates text/actions based on state observations
- * - Evaluator (Me): Scores trajectory outputs
- * - Self-Reflection (Msr): Generates verbal reinforcement cues
- * 
- * Key Innovation: Verbal reinforcement via semantic gradient signals instead of weight updates
- * Episodic memory buffer stores self-reflections for improved decision-making
- * 
- * @version v0.13.13
- */
-
-'use strict';
-
-const path = require('path');
-const fs = require('fs');
-
-/**
- * Memory buffer for storing self-reflections (episodic long-term memory)
- * Bound by maximum experiences (Ω) to fit within LLM context limits
- */
-class EpisodicMemoryBuffer {
-  constructor(maxSize = 3) {
-    this.maxSize = maxSize;
-    this.memories = [];
-  }
-
-  append(reflection) {
-    this.memories.push(reflection);
-    if (this.memories.length > this.maxSize) {
-      this.memories.shift(); // Remove oldest when exceeding max
-    }
-  }
-
-  getAll() {
-    return [...this.memories];
-  }
-
-  getFormatted() {
-    return this.memories
-      .map((m, i) => `[Experience ${i + 1}]: ${m.content}`)
-      .join('\n');
-  }
-
-  clear() {
-    this.memories = [];
-  }
-
-  size() {
-    return this.memories.length;
-  }
-}
-
-/**
- * Trajectory tracker for short-term memory
- * Records (action, observation) pairs during agent-environment interaction
- */
-class TrajectoryTracker {
-  constructor() {
-    this.trajectory = [];
-  }
-
-  add(action, observation) {
-    this.trajectory.push({ action, observation, timestamp: Date.now() });
-  }
-
-  getTrajectory() {
-    return [...this.trajectory];
-  }
-
-  getFormatted() {
-    return this.trajectory
-      .map((t, i) => `Step ${i + 1}: Action="${t.action}" -> Observation="${t.observation}"`)
-      .join('\n');
-  }
-
-  clear() {
-    this.trajectory = [];
-  }
-
-  length() {
-    return this.trajectory.length;
-  }
-}
-
-/**
- * Self-Reflection Model
- * Generates nuanced verbal feedback from sparse reward signals
- * Converts binary/scalar feedback into actionable insights
- */
-class SelfReflectionModel {
-  constructor() {
-    this.reflectionCount = 0;
-  }
-
-  /**
-   * Generate self-reflection from trajectory and reward
-   * @param {string} trajectory - Formatted trajectory string
-   * @param {number} reward - Scalar reward (0 = fail, 1 = success)
-   * @param {string} task - Task description
-   * @param {string[]} priorMemories - Prior self-reflections for context
-   * @returns {object} Reflection object with content and metadata
-   */
-  reflect(trajectory, reward, task, priorMemories = []) {
-    this.reflectionCount++;
-
-    const isSuccess = reward >= 1;
-    let content;
-    let errorType = null;
-    let suggestedAction = null;
-
-    if (isSuccess) {
-      content = this.generateSuccessReflection(trajectory, task);
-    } else {
-      const analysis = this.analyzeFailure(trajectory, task);
-      content = analysis.content;
-      errorType = analysis.errorType;
-      suggestedAction = analysis.suggestedAction;
-    }
-
-    return {
-      id: `sr-${Date.now()}-${this.reflectionCount}`,
-      content,
-      reward,
-      isSuccess,
-      errorType,
-      suggestedAction,
-      task,
-      trajectoryLength: trajectory.split('\n').length,
-      priorMemoryCount: priorMemories.length,
-      timestamp: Date.now()
-    };
-  }
-
-  generateSuccessReflection(trajectory, task) {
-    const steps = trajectory.split('\n').length;
-    return `Success on "${task}" after ${steps} steps. ` +
-      `The approach of analyzing each step and maintaining context was effective. ` +
-      `Key pattern: Sequential reasoning with intermediate checkpoints ensured progress.`;
-  }
-
-  analyzeFailure(trajectory, task) {
-    const lines = trajectory.split('\n');
-    const hasLongTrajectory = lines.length > 10;
-    const hasRepeatedActions = this.checkRepeatedActions(lines);
-    const hasHallucination = /think|believe|know/i.test(trajectory) && 
-      !/(?:actual|real|verified|correct)/i.test(trajectory);
-
-    let errorType;
-    let suggestedAction;
-    let content;
-
-    if (hasHallucination) {
-      errorType = 'hallucination';
-      suggestedAction = 'Verify each belief against actual observations before proceeding';
-      content = `Failed "${task}" due to hallucination. ` +
-        `The agent made assumptions not grounded in environment feedback. ` +
-        `Recommendation: ${suggestedAction}`;
-    } else if (hasRepeatedActions) {
-      errorType = 'inefficient_planning';
-      suggestedAction = 'Try alternative action sequences when stuck';
-      content = `Failed "${task}" due to inefficient planning. ` +
-        `Agent repeated similar actions without progress. ` +
-        `Recommendation: ${suggestedAction}`;
-    } else if (hasLongTrajectory) {
-      errorType = 'early_mistake';
-      suggestedAction = 'Identify the critical error point in the trajectory';
-      content = `Failed "${task}" with long trajectory. ` +
-        `Early mistake cascaded into subsequent failures. ` +
-        `Need to: ${suggestedAction}`;
-    } else {
-      errorType = 'unknown';
-      suggestedAction = 'Re-analyze problem constraints and environment feedback';
-      content = `Failed "${task}" for unclear reasons. ` +
-        `Consider: ${suggestedAction}`;
-    }
-
-    return { content, errorType, suggestedAction };
-  }
-
-  checkRepeatedActions(lines) {
-    if (lines.length < 3) return false;
-    const lastActions = lines.slice(-3).map(l => l.match(/Action="([^"]+)"/)?.[1]).filter(v => typeof v === 'string');
-    return lastActions.length >= 2 && lastActions.every(a => a === lastActions[0]);
-  }
-}
-
-/**
- * Evaluator Model
- * Assesses trajectory quality and computes reward signals
- */
-class EvaluatorModel {
-  constructor(type = 'binary') {
-    this.type = type; // 'binary', 'heuristic', 'llm'
-    this.evaluationCount = 0;
-  }
-
-  /**
-   * Evaluate a trajectory
-   * @param {string} trajectory - Formatted trajectory
-   * @param {object} context - Evaluation context
-   * @returns {number} Reward score
-   */
-  evaluate(trajectory, context = {}) {
-    this.evaluationCount++;
-
-    switch (this.type) {
-      case 'binary':
-        return this.binaryEvaluate(trajectory, context);
-      case 'heuristic':
-        return this.heuristicEvaluate(trajectory, context);
-      case 'llm':
-        return this.llmEvaluate(trajectory, context);
-      default:
-        return this.binaryEvaluate(trajectory, context);
-    }
-  }
-
-  binaryEvaluate(trajectory, context) {
-    // Simple binary success/failure based on trajectory characteristics
-    const hasError = /(fail|error|wrong|incorrect|not found|cannot)/i.test(trajectory);
-    const hasSuccess = /(success|complete|finish|done|achieved)/i.test(trajectory);
-    
-    if (hasSuccess && !hasError) return 1;
-    if (hasError) return 0;
-    
-    // Default to failure if unclear
-    return 0;
-  }
-
-  heuristicEvaluate(trajectory, context) {
-    let score = 0.5;
-    
-    // Penalize long trajectories (inefficient planning)
-    const steps = trajectory.split('\n').length;
-    if (steps > 20) score -= 0.2;
-    if (steps > 30) score -= 0.2;
-    
-    // Penalize repeated actions
-    if (/(repeated|same action|again)/i.test(trajectory)) score -= 0.15;
-    
-    // Reward clear completion signals
-    if (/task complete|successfully|finished/i.test(trajectory)) score += 0.3;
-    
-    // Penalize halluncination indicators
-    if (/think.*but.*not.*actual/i.test(trajectory)) score -= 0.25;
-    
-    return Math.max(0, Math.min(1, score));
-  }
-
-  llmEvaluate(trajectory, context) {
-    // Simulated LLM evaluation - in production would call actual LLM
-    // Uses heuristic as proxy
-    const hScore = this.heuristicEvaluate(trajectory, context);
-    
-    // Use deterministic evaluation only — variance in LLM evaluation poisons Q-tables
-    return Math.max(0, Math.min(1, hScore));
-  }
-
-  isPassing(score) {
-    return score >= 1 || (this.type !== 'binary' && score >= 0.7);
-  }
-}
-
-/**
- * Actor Model
- * Generates actions based on state and memory context
- * Uses in-context learning from memory buffer
- */
-class ActorModel {
-  constructor() {
-    this.actionCount = 0;
-    this.prompt = '';
-  }
-
-  /**
-   * Generate next action given state and memory
-   * @param {string} state - Current environment state/observation
-   * @param {string} memoryContext - Formatted memory buffer content
-   * @param {object} options - Generation options
-   * @returns {string} Generated action
-   */
-  generateAction(state, memoryContext = '', options = {}) {
-    this.actionCount++;
-    
-    const { strategy = 'react' } = options;
-    
-    // Build prompt with memory context
-    let prompt = `State: ${state}\n`;
-    if (memoryContext) {
-      prompt += `\nPrior experiences:\n${memoryContext}\n`;
-    }
-    prompt += `\nBased on the state and prior experiences, what is the next action?`;
-    
-    this.prompt = prompt;
-    
-    // Simulate action generation based on state analysis
-    const action = this.simulateActionGeneration(state, memoryContext, strategy);
-    
-    return action;
-  }
-
-  simulateActionGeneration(state, memoryContext, strategy) {
-    const stateLower = state.toLowerCase();
-    const memoryLower = memoryContext.toLowerCase();
-    
-    // Check prior memory for similar situations
-    if (memoryLower.includes('retry') || memoryLower.includes('alternative')) {
-      if (/search|find|look/i.test(stateLower)) {
-        return 'Try searching in a different location';
-      }
-      if (/move|go|navigate/i.test(stateLower)) {
-        return 'Try a different path';
-      }
-    }
-    
-    // State-based action selection
-    if (/kitchen|cook|food/i.test(stateLower)) {
-      return 'Examine the kitchen environment';
-    }
-    if (/find|search|look/i.test(stateLower)) {
-      return 'Search the current area thoroughly';
-    }
-    if (/pick|grab|take/i.test(stateLower)) {
-      return 'Pick up the relevant object';
-    }
-    if (/use|interact|apply/i.test(stateLower)) {
-      return 'Use the object on the target';
-    }
-    
-    // Default action
-    return 'Observe and gather more information';
-  }
-
-  getPrompt() {
-    return this.prompt;
-  }
-}
-
-/**
- * Reflexion Agent - Main Class
- * Implements the full Reflexion algorithm:
- * 1. Actor generates trajectory using policy
- * 2. Evaluator scores the trajectory
- * 3. Self-Reflection generates verbal feedback
- * 4. Feedback stored in episodic memory buffer
- * 5. Loop until success or max trials
- */
-class ReflexionAgent {
-  constructor(config = {}) {
-    this.maxTrials = config.maxTrials || 12;
-    this.maxMemorySize = config.maxMemorySize || 3;
-    
-    // Initialize components
-    this.actor = new ActorModel();
-    this.evaluator = new EvaluatorModel(config.evaluatorType || 'binary');
-    this.selfReflection = new SelfReflectionModel();
-    this.memoryBuffer = new EpisodicMemoryBuffer(this.maxMemorySize);
-    this.trajectoryTracker = new TrajectoryTracker();
-    
-    // State
-    this.trial = 0;
-    this.history = [];
-    this.currentTask = null;
-    
-    // Callbacks for LLM integration
-    this.actorFn = config.actorFn || null;      // External LLM actor
-    this.evaluatorFn = config.evaluatorFn || null; // External LLM evaluator
-    this.reflectionFn = config.reflectionFn || null; // External LLM reflection
-    
-    console.log(`[ReflexionAgent] Initialized with maxTrials=${this.maxTrials}, memorySize=${this.maxMemorySize}`);
-  }
-
-  /**
-   * Run Reflexion loop for a given task
-   * @param {string} task - Task description
-   * @param {function} environmentFn - Function to execute action and return observation
-   * @returns {object} Final result with trajectory and reflections
-   */
-  async run(task, environmentFn) {
-    this.currentTask = task;
-    this.trial = 0;
-    this.history = [];
-    this.memoryBuffer.clear();
-    this.trajectoryTracker.clear();
-
-    console.log(`[ReflexionAgent] Starting task: "${task}"`);
-
-    // Generate initial trajectory
-    await this.generateTrajectory(environmentFn);
-
-    // Evaluate initial trajectory
-    let reward = await this.evaluateCurrentTrajectory();
-    this.history.push({
-      trial: this.trial,
-      reward,
-      trajectoryLength: this.trajectoryTracker.length(),
-      memorySize: this.memoryBuffer.size()
-    });
-
-    // Initial self-reflection
-    if (reward < 1) {
-      await this.generateSelfReflection(reward);
-    }
-
-    // Reflexion loop
-    while (this.trial < this.maxTrials && !this.evaluator.isPassing(reward)) {
-      this.trial++;
-      console.log(`[ReflexionAgent] Trial ${this.trial}/${this.maxTrials}`);
-      
-      // Reset environment and generate new trajectory
-      this.trajectoryTracker.clear();
-      await this.generateTrajectory(environmentFn);
-      
-      // Evaluate
-      reward = await this.evaluateCurrentTrajectory();
-      
-      this.history.push({
-        trial: this.trial,
-        reward,
-        trajectoryLength: this.trajectoryTracker.length(),
-        memorySize: this.memoryBuffer.size()
-      });
-
-      // Self-reflect on failure
-      if (!this.evaluator.isPassing(reward)) {
-        await this.generateSelfReflection(reward);
-      }
-    }
-
-    const success = this.evaluator.isPassing(reward);
-    console.log(`[ReflexionAgent] Completed: ${success ? 'SUCCESS' : 'MAX_TRIALS'} (trials: ${this.trial + 1})`);
-
-    return {
-      success,
-      trials: this.trial + 1,
-      finalReward: reward,
-      trajectory: this.trajectoryTracker.getTrajectory(),
-      reflections: this.memoryBuffer.getAll(),
-      history: this.history
-    };
-  }
-
-  /**
-   * Generate trajectory by interacting with environment
-   */
-  async generateTrajectory(environmentFn) {
-    const maxSteps = 30;
-    let currentState = 'initial';
-    let step = 0;
-
-    while (step < maxSteps) {
-      // Get action from actor (or external LLM)
-      let action;
-      if (this.actorFn) {
-        action = await this.actorFn(currentState, this.memoryBuffer.getFormatted(), { 
-          strategy: 'react',
-          trial: this.trial 
-        });
-      } else {
-        action = this.actor.generateAction(currentState, this.memoryBuffer.getFormatted());
-      }
-
-      // Execute action — wrap in try-catch so promise rejection does not crash Reflexion loop
-      let result;
-      try {
-        result = await environmentFn(action, currentState);
-      } catch (envErr) {
-        const errMsg = envErr instanceof Error ? envErr.message : String(envErr);
-        console.warn('[ReflexionAgent] environmentFn failed:', errMsg);
-        result = { done: false, success: false, failure: false, observation: `environment error: ${errMsg}` };
-      }
-      const observation = result.observation || result;
-      
-      // Track trajectory
-      this.trajectoryTracker.add(action, observation);
-      
-      // Check for terminal state
-      if (result.done || result.success || result.failure) {
-        break;
-      }
-      
-      currentState = observation;
-      step++;
-    }
-  }
-
-  /**
-   * Evaluate current trajectory
-   */
-  async evaluateCurrentTrajectory() {
-    const trajectory = this.trajectoryTracker.getFormatted();
-    
-    if (this.evaluatorFn) {
-      return await this.evaluatorFn(trajectory, { task: this.currentTask });
-    }
-    
-    return this.evaluator.evaluate(trajectory, { task: this.currentTask });
-  }
-
-  /**
-   * Generate self-reflection and store in memory buffer
-   */
-  async generateSelfReflection(reward) {
-    const trajectory = this.trajectoryTracker.getFormatted();
-    const priorMemories = this.memoryBuffer.getAll().map(m => m.content);
-    
-    let reflection;
-    if (this.reflectionFn) {
-      reflection = await this.reflectionFn(trajectory, reward, this.currentTask, priorMemories);
-    } else {
-      reflection = this.selfReflection.reflect(trajectory, reward, this.currentTask, priorMemories);
-    }
-    
-    this.memoryBuffer.append(reflection);
-    console.log(`[ReflexionAgent] Self-reflection added: ${reflection.errorType || 'success'}`);
-    
-    return reflection;
-  }
-
-  /**
-   * Get current memory buffer state
-   */
-  getMemoryState() {
-    return {
-      memories: this.memoryBuffer.getAll(),
-      formatted: this.memoryBuffer.getFormatted(),
-      size: this.memoryBuffer.size()
-    };
-  }
-
-  /**
-   * Reset agent state for new task
-   */
-  reset() {
-    this.trial = 0;
-    this.history = [];
-    this.currentTask = null;
-    this.memoryBuffer.clear();
-    this.trajectoryTracker.clear();
-    this.selfReflection = new SelfReflectionModel();
-    console.log('[ReflexionAgent] Reset complete');
-  }
-
-  /**
-   * Get statistics
-   */
-  getStats() {
-    return {
-      totalTrials: this.trial + 1,
-      memorySize: this.memoryBuffer.size(),
-      trajectoryLength: this.trajectoryTracker.length(),
-      actorActions: this.actor.actionCount,
-      evaluations: this.evaluator.evaluationCount,
-      reflections: this.selfReflection.reflectionCount,
-      history: this.history
-    };
-  }
-}
-
-/**
- * Reflection Trigger Mechanism
- * Monitors for conditions that should trigger self-reflection
- * Integrates with HeartFlow's self-evolution module
- */
-class ReflectionTrigger {
-  constructor(config = {}) {
-    this.thresholds = {
-      errorRate: config.errorRateThreshold || 0.3,
-      repeatedFailures: config.repeatedFailuresThreshold || 3,
-      trajectoryLength: config.trajectoryLengthThreshold || 20,
-      noProgress: config.noProgressThreshold || 5
-    };
-    
-    this.state = {
-      consecutiveFailures: 0,
-      lastSuccessTrial: 0,
-      errorPatternCounts: {},
-      reflectionCount: 0
-    };
-  }
-
-  /**
-   * Check if reflection should be triggered
-   * @param {object} context - Current execution context
-   * @returns {boolean} True if reflection should be triggered
-   */
-  shouldReflect(context = {}) {
-    const { error, trajectoryLength, trialNumber, similarErrors } = context;
-    
-    // Trigger on consecutive failures
-    if (error) {
-      this.state.consecutiveFailures++;
-      
-      // Track error patterns
-      const errorKey = error.substring(0, 30);
-      this.state.errorPatternCounts[errorKey] = (this.state.errorPatternCounts[errorKey] || 0) + 1;
-      
-      if (this.state.consecutiveFailures >= this.thresholds.repeatedFailures) {
-        return true;
-      }
-      
-      // Trigger on repeated error patterns
-      if ((this.state.errorPatternCounts[errorKey] || 0) >= 2) {
-        return true;
-      }
-    } else {
-      this.state.consecutiveFailures = 0;
-    }
-    
-    // Trigger on long trajectories without success
-    if (trajectoryLength > this.thresholds.trajectoryLength && !context.success) {
-      return true;
-    }
-    
-    // Trigger on no progress over many trials
-    if (trialNumber - this.state.lastSuccessTrial >= this.thresholds.noProgress) {
-      return true;
-    }
-    
-    return false;
-  }
-
-  /**
-   * Record a successful outcome
-   */
-  recordSuccess(trialNumber) {
-    this.state.lastSuccessTrial = trialNumber;
-    this.state.consecutiveFailures = 0;
-  }
-
-  /**
-   * Get trigger statistics
-   */
-  getStats() {
-    return {
-      ...this.state,
-      thresholds: this.thresholds
-    };
-  }
-
-  /**
-   * Reset trigger state
-   */
-  reset() {
-    this.state = {
-      consecutiveFailures: 0,
-      lastSuccessTrial: 0,
-      errorPatternCounts: {},
-      reflectionCount: 0
-    };
-  }
-}
-
-/**
- * Demo function demonstrating Reflexion Agent
- */
-function demo() {
-  console.log('='.repeat(60));
-  console.log('Reflexion Agent Demo - Verbal Reinforcement Learning');
-  console.log('='.repeat(60));
-  
-  // Create agent
-  const agent = new ReflexionAgent({
-    maxTrials: 5,
-    maxMemorySize: 3,
-    evaluatorType: 'heuristic'
-  });
-  
-  // Simulated environment
-  async function simulatedEnvironment(action, state) {
-    // Simulate some processing
-    await new Promise(resolve => setTimeout(resolve, 10));
-    
-    const actionLower = action.toLowerCase();
-    const stateLower = state.toLowerCase();
-    
-    // Simulate different outcomes based on action
-    if (actionLower.includes('wrong') || actionLower.includes('incorrect')) {
-      return { 
-        observation: 'Action resulted in error - trying again', 
-        failure: false,
-        done: false
-      };
-    }
-    
-    if (actionLower.includes('alternative') || actionLower.includes('different')) {
-      return {
-        observation: 'Task completed successfully',
-        success: true,
-        done: true
-      };
-    }
-    
-    if (stateLower === 'initial' && actionLower.includes('observe')) {
-      return {
-        observation: 'Environment explored, found target location',
-        done: false
-      };
-    }
-    
-    return {
-      observation: 'Action executed, continuing task...',
-      done: false
-    };
-  }
-  
-  // Run agent
-  const task = 'Find and pick up the spatula in the kitchen';
-  
-  console.log(`\nTask: ${task}`);
-  console.log('-'.repeat(60));
-  
-  agent.run(task, simulatedEnvironment).then(result => {
-    console.log('\n' + '='.repeat(60));
-    console.log('RESULTS');
-    console.log('='.repeat(60));
-    console.log(`Success: ${result.success}`);
-    console.log(`Trials: ${result.trials}`);
-    console.log(`Final Reward: ${result.finalReward}`);
-    
-    console.log('\n--- Trajectory ---');
-    result.trajectory.forEach((t, i) => {
-      console.log(`  ${i + 1}. ${t.action} -> ${t.observation}`);
-    });
-    
-    console.log('\n--- Self-Reflections (Episodic Memory) ---');
-    result.reflections.forEach((r, i) => {
-      console.log(`  [${i + 1}] ${r.content}`);
-      if (r.errorType) console.log(`      Error: ${r.errorType}, Suggestion: ${r.suggestedAction}`);
-    });
-    
-    console.log('\n--- Statistics ---');
-    const stats = agent.getStats();
-    console.log(`  Actor actions: ${stats.actorActions}`);
-    console.log(`  Evaluations: ${stats.evaluations}`);
-    console.log(`  Self-reflections generated: ${stats.reflections}`);
-    console.log(`  Final memory size: ${stats.memorySize}`);
-    
-    // Test Reflection Trigger
-    console.log('\n' + '='.repeat(60));
-    console.log('Reflection Trigger Demo');
-    console.log('='.repeat(60));
-    
-    const trigger = new ReflectionTrigger({
-      repeatedFailuresThreshold: 2,
-      trajectoryLengthThreshold: 5
-    });
-    
-    // Simulate failure scenarios
-    const scenarios = [
-      { error: 'Action resulted in error', trajectoryLength: 3, trialNumber: 1 },
-      { error: 'Action resulted in error', trajectoryLength: 4, trialNumber: 2 },
-      { error: 'Action resulted in error', trajectoryLength: 5, trialNumber: 3 },
-      { trajectoryLength: 25, trialNumber: 4 }
-    ];
-    
-    scenarios.forEach((ctx, i) => {
-      const shouldReflect = trigger.shouldReflect(ctx);
-      console.log(`  Scenario ${i + 1}: ${shouldReflect ? 'TRIGGER reflection' : 'Continue normally'}`);
-      if (ctx.error) trigger.state.consecutiveFailures++;
-    });
-    
-    console.log('\nDemo complete!');
-  });
-}
-
-// Export for HeartFlow integration
-module.exports = {
-  ReflexionAgent,
-  EpisodicMemoryBuffer,
-  TrajectoryTracker,
-  SelfReflectionModel,
-  EvaluatorModel,
-  ActorModel,
-  ReflectionTrigger,
-  demo
-};