claude-evolve 1.3.37 → 1.3.38

package/bin/claude-evolve-analyze

@@ -165,7 +165,7 @@ echo "Pending: $pending"
 
 if [[ $count_with_performance -gt 0 ]]; then
   avg_performance=$(echo "scale=4; $total_performance / $count_with_performance" | bc -l 2>/dev/null || echo "0")
-  echo "Average Performance: $avg_performance"
+  echo "Average Performance: $avg_performance"  # Still showing mean for overall stats
 else
   echo "Average Performance: N/A"
 fi
@@ -238,13 +238,24 @@ for gen in $(cut -d' ' -f1 "$gen_stats_file" | sort -u || echo ""); do
   echo -n "$gen: $total_in_gen candidates"
   
   if [[ "$completed_in_gen" -gt 0 ]]; then
-    # Calculate average performance for this generation  
-    sum="0"
+    # Calculate median performance for this generation
+    # AIDEV-NOTE: Changed from mean to median to be more robust to outliers
+    median="0"
     if grep -q "^$gen completed" "$gen_stats_file"; then
-      sum=$(grep "^$gen completed" "$gen_stats_file" | awk '{s+=$3} END {printf "%.6f", s}' 2>/dev/null || echo "0")
+      median=$(grep "^$gen completed" "$gen_stats_file" | awk '{print $3}' | sort -n | awk '{
+        a[NR] = $0
+      }
+      END {
+        if (NR % 2) {
+          # Odd number of elements
+          print a[(NR + 1) / 2]
+        } else {
+          # Even number of elements - average of two middle values
+          printf "%.4f", (a[NR/2] + a[NR/2 + 1]) / 2.0
+        }
+      }' 2>/dev/null || echo "0")
     fi
-    avg=$(echo "scale=4; $sum / $completed_in_gen" | bc -l 2>/dev/null || echo "0")
-    echo " ($completed_in_gen completed, avg: $avg)"
+    echo " ($completed_in_gen completed, median: $median)"
   else
     echo " (0 completed)"
   fi
@@ -290,7 +301,7 @@ if command -v gnuplot >/dev/null 2>&1 && [[ $valid_performance_count -gt 0 ]]; t
   gen_avg_file="/tmp/evolution_gen_avg_$$.dat"
   
   echo "# Row ID Performance Generation" >"$data_file"
-  echo "# Generation AvgPerformance Color" >"$gen_avg_file"
+  echo "# Generation MedianPerformance Color" >"$gen_avg_file"
 
   # Get color by generation number (rotates through 7 colors)
   get_gen_color() {
@@ -417,18 +428,31 @@ print(f'max_id=\"{max_id}\"')
   max_gen_num=0
   for gen in $(cut -d' ' -f1 "$gen_data_temp" | sort -u); do
     if grep -q "^$gen " "$gen_data_temp"; then
-      # Calculate average for this generation
-      sum=$(grep "^$gen " "$gen_data_temp" | awk '{s+=$2} END {printf "%.6f", s}' 2>/dev/null || echo "0")
+      # Calculate median for this generation
+      # AIDEV-NOTE: Changed from mean to median to be more robust to outliers
+      # Extract all performance values for this generation and sort them
+      median=$(grep "^$gen " "$gen_data_temp" | awk '{print $2}' | sort -n | awk '{
+        a[NR] = $0
+      }
+      END {
+        if (NR % 2) {
+          # Odd number of elements
+          print a[(NR + 1) / 2]
+        } else {
+          # Even number of elements - average of two middle values
+          print (a[NR/2] + a[NR/2 + 1]) / 2.0
+        }
+      }' 2>/dev/null || echo "0")
       count=$(grep -c "^$gen " "$gen_data_temp")
       if [[ $count -gt 0 ]]; then
-        avg=$(echo "scale=4; $sum / $count" | bc -l 2>/dev/null || echo "0")
+        avg=$median  # Using median instead of mean
         gen_num=$(echo "$gen" | sed 's/gen0*//')
         # Track max generation number
         if [[ $gen_num =~ ^[0-9]+$ ]] && [[ $gen_num -gt $max_gen_num ]]; then
           max_gen_num=$gen_num
         fi
         color=$(get_gen_color "$gen_num")
-        echo "$gen_index \"Gen$gen_num\" $avg \"$color\"" >>"$gen_avg_file"
+        echo "$gen_index \"Gen$gen_num\" $avg \"$color\"" >>"$gen_avg_file"  # avg is now median
         ((gen_index++))
       fi
     fi
@@ -449,6 +473,19 @@ print(f'max_id=\"{max_id}\"')
     # cat "$data_file"
     # echo "DEBUG: max_gen_num=$max_gen_num"
     
+    # Calculate total data points for dynamic sizing
+    total_data_points=$(awk 'END {print NR-1}' "$data_file")  # Subtract header row
+    
+    # AIDEV-NOTE: Dynamic dot sizing based on data point count
+    # Use significantly larger dots when there are fewer data points for better visibility
+    if [[ $total_data_points -lt 35 ]]; then
+      regular_dot_size="1.8"
+      winner_dot_size="3.0"
+    else
+      regular_dot_size="0.6"
+      winner_dot_size="1.5"
+    fi
+    
     # Plot all algorithms in order of completion, colored by generation
     plot_cmd=""
     gen_plots_added=0
@@ -462,7 +499,7 @@ print(f'max_id=\"{max_id}\"')
         if [[ $gen_plots_added -gt 0 ]]; then
           plot_cmd="$plot_cmd, \\"$'\n'
         fi
-        plot_cmd="${plot_cmd}     \"$data_file\" using (\$4==$gen_num?\$1:1/0):3 with linespoints linewidth 2 linecolor rgb \"$color\" pointsize 1.2 title \"Gen $gen_num\""
+        plot_cmd="${plot_cmd}     \"$data_file\" using (\$4==$gen_num?\$1:1/0):3 with points linecolor rgb \"$color\" pointsize $regular_dot_size title \"Gen $gen_num\""
         ((gen_plots_added++))
       fi
     done
@@ -472,15 +509,15 @@ print(f'max_id=\"{max_id}\"')
       if [[ $gen_plots_added -gt 0 ]]; then
         plot_cmd="$plot_cmd, \\"$'\n'
       fi
-      plot_cmd="${plot_cmd}     \"$winner_file\" using 1:3 with points pointtype 7 pointsize 3 linecolor rgb \"gold\" title \"Best ($max_id)\""
+      plot_cmd="${plot_cmd}     \"$winner_file\" using 1:3 with points pointtype 7 pointsize $winner_dot_size linecolor rgb \"gold\" title \"Best ($max_id)\""
     fi
     
     # Fallback if no generation-specific plots
     if [[ $gen_plots_added -eq 0 ]]; then
-      plot_cmd="\"$data_file\" using 1:3 with linespoints linewidth 2 linecolor rgb \"#1f77b4\" pointsize 1.2 title \"Evolution Progress\""
+      plot_cmd="\"$data_file\" using 1:3 with points linecolor rgb \"#1f77b4\" pointsize $regular_dot_size title \"Evolution Progress\""
       if [[ -n $max_id && -s "$winner_file" ]]; then
         plot_cmd="$plot_cmd, \\"$'\n'
-        plot_cmd="${plot_cmd}     \"$winner_file\" using 1:3 with points pointtype 7 pointsize 3 linecolor rgb \"gold\" title \"Best ($max_id)\""
+        plot_cmd="${plot_cmd}     \"$winner_file\" using 1:3 with points pointtype 7 pointsize $winner_dot_size linecolor rgb \"gold\" title \"Best ($max_id)\""
       fi
     fi
     
@@ -506,21 +543,25 @@ set output "$output_file"
 set multiplot layout 2,1 margins 0.08,0.82,0.15,0.95 spacing 0.1,0.15
 
 #=================== TOP PLOT: Performance Over Time ===================
+# AIDEV-NOTE: Removed x-axis to eliminate tick overlap and formatting issues
 set title "Algorithm Evolution Performance Over Time" font ",14"
-set xlabel "Completion Order (Algorithm #)"
+unset xlabel
 set ylabel "Performance Score"
-set grid
+set grid y  # Only show horizontal grid lines
 set key outside right
-set xtics 1
-set autoscale y
+
+# AIDEV-NOTE: Remove x-axis entirely to avoid tick problems with large datasets
+unset xtics
+set autoscale
+set yrange [*:*]  # Auto-scale y-axis only
 
 # Define colors for generations
 plot $plot_cmd
 
-#=================== BOTTOM PLOT: Generation Averages ===================
-set title "Average Performance by Generation" font ",14"
+#=================== BOTTOM PLOT: Generation Medians ===================
+set title "Median Performance by Generation" font ",14"
 set xlabel "Generation"
-set ylabel "Avg Performance"
+set ylabel "Median Performance"
 set style fill solid 0.8
 set boxwidth 0.6
 unset key
@@ -529,6 +570,10 @@ set grid y
 # Set custom x-axis labels
 set xtics ($xtics_labels)
 
+# Auto-scale for generation plot too
+set autoscale
+set yrange [*:*]
+
 plot "$gen_avg_file" using 1:3 with boxes linecolor rgb "#4CAF50" notitle
 
 unset multiplot

package/bin/claude-evolve-ideate

@@ -14,14 +14,40 @@ else
   load_config
 fi
 
-# Helper function to call AI model (codex o3-pro if available, else Claude)
+# Function to determine which model to use based on generation
+get_model_for_generation() {
+  local generation="$1"
+  local gen_num
+  
+  # Extract numeric part of generation (e.g., "05" from gen05)
+  if [[ $generation =~ ^0*([0-9]+)$ ]]; then
+    gen_num=$((10#${BASH_REMATCH[1]}))
+  else
+    gen_num=1  # Default for malformed input
+  fi
+  
+  # Alternate between models by generation
+  if (( gen_num % 2 == 1 )); then
+    echo "opus"    # Odd generations: use Opus for exploration
+  else
+    echo "o3-pro"  # Even generations: use o3-pro for refinement
+  fi
+}
+
+# Helper function to call AI model (alternating based on generation)
 call_ai_with_limit_check() {
   local prompt="$1"
-  local fallback_model="${2:-opus}"
+  local generation="${2:-01}"  # Default to generation 01 if not provided
+  
+  # Determine which model to use for this generation
+  local preferred_model
+  preferred_model=$(get_model_for_generation "$generation")
+  
+  echo "[INFO] Generation $generation: Using $preferred_model" >&2
   
-  # Check if codex is available
-  if command -v codex >/dev/null 2>&1; then
-    echo "[INFO] Using codex o3-pro for ideation (smartest available model)" >&2
+  # Try preferred model first
+  if [[ "$preferred_model" == "o3-pro" ]] && command -v codex >/dev/null 2>&1; then
+    echo "[INFO] Using codex o3-pro for ideation" >&2
     
     # Call codex with o3-pro model using -q flag and --full-auto
     local ai_output
@@ -32,16 +58,17 @@ call_ai_with_limit_check() {
       echo "$ai_output"
       return 0
     else
-      echo "[WARN] Codex failed, falling back to Claude" >&2
+      echo "[WARN] Codex o3-pro failed, falling back to Claude Opus" >&2
+      preferred_model="opus"
     fi
   fi
   
-  # Fall back to Claude
-  echo "[INFO] Using Claude $fallback_model for ideation" >&2
+  # Use Claude with preferred model (or fallback)
+  echo "[INFO] Using Claude $preferred_model for ideation" >&2
   
   # Call Claude and capture output
   local claude_output
-  claude_output=$(echo "$prompt" | claude --dangerously-skip-permissions --model "$fallback_model" -p 2>&1)
+  claude_output=$(echo "$prompt" | claude --dangerously-skip-permissions --model "$preferred_model" -p 2>&1)
   local claude_exit_code=$?
   
   # Check for usage limit
@@ -309,7 +336,7 @@ Example descriptions:
 Add exactly $count rows to the CSV file now."
 
   echo "[INFO] Generating $count novel exploration ideas..."
-  if ! call_ai_with_limit_check "$prompt" "opus"; then
+  if ! call_ai_with_limit_check "$prompt" "$CURRENT_GENERATION"; then
     echo "[WARN] AI failed to generate novel ideas" >&2
     return 1
   fi
@@ -372,7 +399,7 @@ Example descriptions:
 Add exactly $count parameter tuning rows to the CSV file now."
 
   echo "[INFO] Generating $count hill climbing ideas..."
-  if ! call_ai_with_limit_check "$prompt" "opus"; then
+  if ! call_ai_with_limit_check "$prompt" "$CURRENT_GENERATION"; then
     echo "[WARN] AI failed to generate hill climbing ideas" >&2
     return 1
   fi
@@ -435,7 +462,7 @@ Example descriptions:
 Add exactly $count structural modification rows to the CSV file now."
 
   echo "[INFO] Generating $count structural mutation ideas..."
-  if ! call_ai_with_limit_check "$prompt" "opus"; then
+  if ! call_ai_with_limit_check "$prompt" "$CURRENT_GENERATION"; then
     echo "[WARN] AI failed to generate structural mutation ideas" >&2
     return 1
   fi
@@ -498,7 +525,7 @@ Example descriptions:
 Add exactly $count hybrid combination rows to the CSV file now."
 
   echo "[INFO] Generating $count crossover hybrid ideas..."
-  if ! call_ai_with_limit_check "$prompt" "opus"; then
+  if ! call_ai_with_limit_check "$prompt" "$CURRENT_GENERATION"; then
     echo "[WARN] AI failed to generate crossover ideas" >&2
     return 1
   fi
@@ -567,7 +594,7 @@ CRITICAL CSV FORMAT RULES:
 Add exactly $TOTAL_IDEAS algorithm variation rows to the CSV file now."
 
   echo "[INFO] Generating $TOTAL_IDEAS ideas (legacy mode)..."
-  if ! call_ai_with_limit_check "$prompt" "opus"; then
+  if ! call_ai_with_limit_check "$prompt" "$CURRENT_GENERATION"; then
     echo "[WARN] AI failed to generate ideas" >&2
     return 1
   fi

package/bin/claude-evolve-main

@@ -142,7 +142,7 @@ if [[ $# -eq 0 ]]; then
   case $choice in
   1) exec "$SCRIPT_DIR/claude-evolve-setup" ;;
   2) exec "$SCRIPT_DIR/claude-evolve-ideate" ;;
-  3) exec "$SCRIPT_DIR/claude-evolve-run" ;;
+  3) exec "$SCRIPT_DIR/claude-evolve-run-unified" ;;
   4) exec "$SCRIPT_DIR/claude-evolve-analyze" ;;
   5) exec "$SCRIPT_DIR/claude-evolve-config" ;;
   6) show_help ;;
@@ -174,7 +174,7 @@ ideate)
   ;;
 run)
   shift
-  exec "$SCRIPT_DIR/claude-evolve-run" "$@"
+  exec "$SCRIPT_DIR/claude-evolve-run-unified" "$@"
   ;;
 analyze)
   shift

package/bin/{claude-evolve-run-parallel → claude-evolve-run-parallel.OLD}

@@ -297,12 +297,39 @@ print(count)
     echo "[DISPATCHER] CSV has $((total_rows-1)) total candidates, $complete_count complete"
   fi
   
-  # If no pending work and no active workers, we're done
+  # If no pending work and no active workers, check for auto-ideation
   if [[ $pending_count -eq 0 && $active_workers -eq 0 ]]; then
-    echo "[DISPATCHER] No pending candidates found. Evolution complete."
-    echo "[DISPATCHER] Run 'claude-evolve ideate' to generate more candidates."
-    echo "[DISPATCHER] Exiting main loop: no work remaining" >&2
-    break
+    echo "[DISPATCHER] No pending candidates found."
+    
+    # Check if auto ideation is enabled
+    echo "[DEBUG] AUTO_IDEATE value: '$AUTO_IDEATE'"
+    if [[ "$AUTO_IDEATE" == "true" || "$AUTO_IDEATE" == "1" ]]; then
+      echo "[DISPATCHER] Auto ideation is enabled. Generating new ideas..."
+      
+      # Check if claude-evolve-ideate exists
+      ideate_script="$SCRIPT_DIR/claude-evolve-ideate"
+      if [[ ! -f "$ideate_script" ]]; then
+        echo "[ERROR] claude-evolve-ideate script not found: $ideate_script" >&2
+        echo "[DISPATCHER] Evolution complete - no way to generate more ideas."
+        break
+      fi
+      
+      # Generate new ideas using the multi-strategy approach
+      echo "[DISPATCHER] Calling claude-evolve-ideate to generate new candidates..."
+      if ! "$ideate_script"; then
+        echo "[ERROR] Failed to generate new ideas" >&2
+        echo "[DISPATCHER] Evolution complete - ideation failed."
+        break
+      fi
+      
+      echo "[DISPATCHER] New ideas generated successfully. Continuing evolution..."
+      continue  # Go back to start of loop to find the new candidates
+    else
+      echo "[DISPATCHER] Auto ideation is disabled. Evolution complete."
+      echo "[DISPATCHER] Run 'claude-evolve ideate' to generate more candidates."
+      echo "[DISPATCHER] Exiting main loop: no work remaining" >&2
+      break
+    fi
   fi
   
   # Start workers if we have pending work and capacity

package/bin/claude-evolve-run-unified

@@ -0,0 +1,342 @@
+#!/bin/bash
+
+set -e
+
+# Load configuration  
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+# shellcheck source=../lib/config.sh
+source "$SCRIPT_DIR/../lib/config.sh"
+
+# Use CLAUDE_EVOLVE_CONFIG if set, otherwise default
+if [[ -n ${CLAUDE_EVOLVE_CONFIG:-} ]]; then
+  load_config "$CLAUDE_EVOLVE_CONFIG"
+else
+  load_config
+fi
+
+# Validate configuration
+if ! validate_config; then
+  echo "[ERROR] Configuration validation failed" >&2
+  exit 1
+fi
+
+# AIDEV-NOTE: Unified execution engine that handles both sequential and parallel modes
+# Sequential mode is just parallel mode with max_workers=1
+
+# Default values
+timeout_seconds=""
+force_parallel=""
+force_sequential=""
+use_caffeinate="false"
+
+# Parse command line arguments
+while [[ $# -gt 0 ]]; do
+  case $1 in
+  --timeout)
+    if [[ -z ${2:-} ]] || [[ ! $2 =~ ^[0-9]+$ ]] || [[ $2 -eq 0 ]]; then
+      echo "[ERROR] --timeout requires a positive integer (seconds)" >&2
+      exit 1
+    fi
+    timeout_seconds="$2"
+    shift 2
+    ;;
+  --parallel)
+    force_parallel="true"
+    shift
+    ;;
+  --sequential)
+    force_sequential="true"
+    shift
+    ;;
+  --keep-awake|--caffeinate)
+    use_caffeinate="true"
+    shift
+    ;;
+  --help)
+    cat <<EOF
+claude-evolve run - Execute evolution candidates
+
+USAGE:
+  claude-evolve run [OPTIONS]
+
+OPTIONS:
+  --timeout N       Timeout in seconds for each evaluation
+  --parallel        Force parallel execution mode
+  --sequential      Force sequential execution mode (max_workers=1)
+  --keep-awake      Keep system awake during execution (macOS only)
+  --caffeinate      Alias for --keep-awake
+  --help            Show this help message
+
+DESCRIPTION:
+  Continuously processes evolution candidates by:
+  1. Finding pending candidates in CSV
+  2. Calling Claude to mutate algorithms  
+  3. Updating CSV with performance score and completion status
+  4. Auto-generating new ideas when no pending candidates remain
+
+  Use --timeout to prevent runaway evaluations from blocking progress.
+EOF
+    exit 0
+    ;;
+  *)
+    echo "[ERROR] Unknown option: $1" >&2
+    exit 1
+    ;;
+  esac
+done
+
+# Check if caffeinate should be used
+if [[ "$use_caffeinate" == "true" ]] && command -v caffeinate >/dev/null 2>&1; then
+  echo "[INFO] Using caffeinate to prevent system sleep"
+  # Re-run this script with caffeinate
+  exec caffeinate -dims "$0" "$@"
+fi
+
+# Determine execution mode and worker count
+if [[ "$force_sequential" == "true" ]]; then
+  MAX_WORKERS=1
+  echo "[INFO] Using sequential mode (forced via --sequential, max_workers=1)"
+elif [[ "$force_parallel" == "true" ]]; then
+  echo "[INFO] Using parallel mode (forced via --parallel, max_workers=$MAX_WORKERS)"
+elif [[ "$PARALLEL_ENABLED" == "true" || "$PARALLEL_ENABLED" == "1" ]]; then
+  echo "[INFO] Using parallel mode (enabled in config, max_workers=$MAX_WORKERS)"
+else
+  MAX_WORKERS=1
+  echo "[INFO] Using sequential mode (default, max_workers=1)"
+fi
+
+# Start unified execution engine
+echo "[INFO] Starting evolution run with up to $MAX_WORKERS workers"
+
+# Rest of the logic is the same as claude-evolve-run-parallel but with dynamic MAX_WORKERS
+# and the updated auto-ideation logic we just added
+
+# Worker management
+declare -a worker_pids=()
+
+# Graceful shutdown function
+shutdown_workers() {
+  if [[ ${#worker_pids[@]} -eq 0 ]]; then
+    echo "[DISPATCHER] No workers to shutdown"
+    return 0
+  fi
+  
+  echo "[DISPATCHER] Shutting down workers..."
+  for pid in "${worker_pids[@]}"; do
+    if kill -0 "$pid" 2>/dev/null; then
+      echo "[DISPATCHER] Stopping worker $pid"
+      kill -TERM "$pid" 2>/dev/null || true
+    fi
+  done
+  
+  # Wait for workers to exit
+  local timeout=10
+  while [[ ${#worker_pids[@]} -gt 0 && $timeout -gt 0 ]]; do
+    sleep 1
+    ((timeout--))
+    
+    local new_pids=()
+    for pid in "${worker_pids[@]}"; do
+      if kill -0 "$pid" 2>/dev/null; then
+        new_pids+=($pid)
+      fi
+    done
+    worker_pids=("${new_pids[@]}")
+  done
+  
+  # Force kill remaining workers
+  for pid in "${worker_pids[@]}"; do
+    if kill -0 "$pid" 2>/dev/null; then
+      echo "[DISPATCHER] Force killing worker $pid"
+      kill -KILL "$pid" 2>/dev/null || true
+    fi
+  done
+  
+  echo "[DISPATCHER] Shutdown complete"
+  exit 0
+}
+
+# Signal handling - graceful shutdown with force option
+handle_signal() {
+  local signal="$1"
+  echo "[DISPATCHER] Received signal: $signal" >&2
+  echo "[DISPATCHER] Active workers: ${#worker_pids[@]}" >&2
+  
+  # For expensive workers, give option to force shutdown
+  if [[ ${#worker_pids[@]} -gt 0 ]]; then
+    echo "[DISPATCHER] Warning: ${#worker_pids[@]} expensive workers are still running!" >&2
+    echo "[DISPATCHER] Press Ctrl+C again to force shutdown immediately, or wait for graceful shutdown..." >&2
+    
+    # Give a few seconds for force shutdown option
+    local count=3
+    while [[ $count -gt 0 && "$force_shutdown_requested" != "true" ]]; do
+      sleep 1
+      ((count--))
+    done
+    
+    if [[ "$force_shutdown_requested" == "true" ]]; then
+      echo "[DISPATCHER] Force shutdown in progress..." >&2
+      return  # shutdown_workers already called
+    fi
+    
+    echo "[DISPATCHER] Proceeding with graceful shutdown..." >&2
+  fi
+  
+  shutdown_workers
+}
+
+# Track signals for force shutdown  
+signal_count=0
+force_shutdown_requested=false
+
+# Immediate signal handler for force shutdown detection
+signal_handler() {
+  ((signal_count++))
+  
+  if [[ $signal_count -eq 1 ]]; then
+    # First signal - start graceful shutdown
+    handle_signal "SIGINT"
+  else
+    # Second+ signal - force shutdown immediately
+    echo "[DISPATCHER] Force shutdown requested!" >&2
+    force_shutdown_requested=true
+    shutdown_workers
+  fi
+}
+
+# Set up signal handlers
+trap 'signal_handler' INT
+trap 'handle_signal SIGTERM' TERM
+
+# Function to start a worker
+start_worker() {
+  local worker_script="$SCRIPT_DIR/claude-evolve-worker"
+  if [[ ! -f "$worker_script" ]]; then
+    echo "[ERROR] Worker script not found: $worker_script" >&2
+    exit 1
+  fi
+  
+  local worker_args=()
+  [[ -n $timeout_seconds ]] && worker_args+=(--timeout "$timeout_seconds")
+  
+  echo "[DISPATCHER] Starting worker..."
+  "$worker_script" "${worker_args[@]}" &
+  local worker_pid=$!
+  worker_pids+=($worker_pid)
+  echo "[DISPATCHER] Worker started with PID: $worker_pid"
+}
+
+# Function to clean up finished workers
+cleanup_workers() {
+  local new_pids=()
+  for pid in "${worker_pids[@]}"; do
+    if kill -0 "$pid" 2>/dev/null; then
+      new_pids+=($pid)
+    else
+      # Worker finished
+      if wait "$pid" 2>/dev/null; then
+        echo "[DISPATCHER] Worker $pid completed successfully"
+      else
+        local exit_code=$?
+        if [[ $exit_code -eq 2 ]]; then
+          echo "[DISPATCHER] Worker $pid hit rate limit, will retry later"
+        else
+          echo "[DISPATCHER] Worker $pid failed with exit code $exit_code"
+        fi
+      fi
+    fi
+  done
+  worker_pids=("${new_pids[@]}")
+}
+
+# Function to count pending candidates
+count_pending_candidates() {
+  "$PYTHON_CMD" "$SCRIPT_DIR/../lib/csv_helper.py" find_pending "$FULL_CSV_PATH" >/dev/null 2>&1
+  echo $?  # 0 if found, 1 if not found
+}
+
+# Function to get CSV stats
+get_csv_stats() {
+  if [[ ! -f "$FULL_CSV_PATH" ]]; then
+    echo "0 0 0"
+    return
+  fi
+  
+  local total_rows complete_count pending_count
+  total_rows=$(wc -l < "$FULL_CSV_PATH" | tr -d '[:space:]')
+  complete_count=$(grep -c ',complete' "$FULL_CSV_PATH" || echo "0")
+  
+  # Count pending: empty status or "pending"
+  # Handle potential Windows line endings by stripping carriage returns
+  pending_count=$(awk -F, 'NR>1 {gsub(/\r/, "", $5); if($5=="" || $5=="pending") count++} END {print count+0}' "$FULL_CSV_PATH")
+  
+  echo "$total_rows $complete_count $pending_count"
+}
+
+echo "[DISPATCHER] Starting unified evolution engine"
+echo "[DISPATCHER] Configuration: max_workers=$MAX_WORKERS, timeout=${timeout_seconds:-none}"
+
+# Main dispatch loop
+while true; do
+  # Clean up finished workers
+  cleanup_workers
+  
+  # Get current status
+  read -r total_rows complete_count pending_count <<< "$(get_csv_stats)"
+  active_workers=${#worker_pids[@]}
+  
+  # Status reporting
+  if [[ $total_rows -gt 1 ]]; then
+    echo "[DISPATCHER] Status: $pending_count pending, $active_workers active workers"
+    echo "[DISPATCHER] CSV has $((total_rows-1)) total candidates, $complete_count complete"
+  fi
+  
+  # If no pending work and no active workers, check for auto-ideation
+  if [[ $pending_count -eq 0 && $active_workers -eq 0 ]]; then
+    echo "[DISPATCHER] No pending candidates found."
+    
+    # Check if auto ideation is enabled
+    if [[ "$AUTO_IDEATE" == "true" || "$AUTO_IDEATE" == "1" ]]; then
+      echo "[DISPATCHER] Auto ideation is enabled. Generating new ideas..."
+      
+      # Check if claude-evolve-ideate exists
+      ideate_script="$SCRIPT_DIR/claude-evolve-ideate"
+      if [[ ! -f "$ideate_script" ]]; then
+        echo "[ERROR] claude-evolve-ideate script not found: $ideate_script" >&2
+        echo "[DISPATCHER] Evolution complete - no way to generate more ideas."
+        break
+      fi
+      
+      # Generate new ideas using the multi-strategy approach
+      echo "[DISPATCHER] Calling claude-evolve-ideate to generate new candidates..."
+      if ! "$ideate_script"; then
+        echo "[ERROR] Failed to generate new ideas" >&2
+        echo "[DISPATCHER] Evolution complete - ideation failed."
+        break
+      fi
+      
+      echo "[DISPATCHER] New ideas generated successfully. Continuing evolution..."
+      continue  # Go back to start of loop to find the new candidates
+    else
+      echo "[DISPATCHER] Auto ideation is disabled. Evolution complete."
+      echo "[DISPATCHER] Run 'claude-evolve ideate' to generate more candidates."
+      echo "[DISPATCHER] Exiting main loop: no work remaining" >&2
+      break
+    fi
+  fi
+  
+  # Start workers if we have pending work and capacity
+  while [[ $pending_count -gt 0 && $active_workers -lt $MAX_WORKERS ]]; do
+    start_worker
+    active_workers=${#worker_pids[@]}
+    ((pending_count--))  # Optimistically assume this will be picked up
+  done
+  
+  # Brief pause to avoid busy waiting
+  sleep 2
+done
+
+# Clean shutdown
+shutdown_workers
+echo "[DISPATCHER] Evolution run complete"
+echo "[DISPATCHER] Exiting with code 0"

package/bin/{claude-evolve-run → claude-evolve-run.OLD}

@@ -233,6 +233,7 @@ while true; do
     echo "[INFO] No more pending candidates found."
     
     # Check if auto ideation is enabled
+    echo "[DEBUG] AUTO_IDEATE value: '$AUTO_IDEATE'"
     if [[ "$AUTO_IDEATE" == "true" || "$AUTO_IDEATE" == "1" ]]; then
       echo "[INFO] Auto ideation is enabled. Generating new ideas..."
       
@@ -267,7 +268,7 @@ while true; do
 eval "$("$PYTHON_CMD" "$SCRIPT_DIR/../lib/csv_helper.py" get_row "$FULL_CSV_PATH" "$row_num")"
 
 # Variables are now set: id, basedOnId, description, performance, status
-based_on_id="$basedOnId"  # Convert to expected variable name
+# based_on_id is already set correctly by csv_helper.py
 
 # Check if ID is empty
 if [[ -z $id ]]; then
@@ -282,18 +283,18 @@ echo "[INFO] Based on ID: $based_on_id"
 # Set interrupt handler - just exit without updating CSV status
 trap 'echo "[INFO] Evolution interrupted"; exit 130' INT
 
-# Determine parent algorithm
+# AIDEV-NOTE: Using common evolution processor logic to determine parent/output files
+# and check if processing should be skipped (handles self-parent detection)
+
+# Determine parent algorithm path
 if [[ -z $based_on_id || $based_on_id == "0" || $based_on_id == '""' ]]; then
-  # Empty or zero basedonID means use the base algorithm
   parent_file="$FULL_ALGORITHM_PATH"
   echo "[INFO] Using base algorithm (basedonID is empty or 0)"
 else
   # Handle both old format (numeric) and new format (genXX-XXX)
   if [[ $based_on_id =~ ^[0-9]+$ ]]; then
-    # Old numeric format
     parent_file="$FULL_OUTPUT_DIR/evolution_id${based_on_id}.py"
   else
-    # New generation format
     parent_file="$FULL_OUTPUT_DIR/evolution_${based_on_id}.py"
   fi
   
@@ -310,20 +311,24 @@ fi
 
 echo "[INFO] Using parent algorithm: $parent_file"
 
-# Generate mutation output file
-# Handle both old format (numeric) and new format (genXX-XXX)
+# Generate output file path
 if [[ $id =~ ^[0-9]+$ ]]; then
-  # Old numeric format
   output_file="$FULL_OUTPUT_DIR/evolution_id${id}.py"
 else
-  # New generation format
   output_file="$FULL_OUTPUT_DIR/evolution_${id}.py"
 fi
 echo "[INFO] Generating algorithm mutation..."
 
-# Copy parent algorithm to output file first
-cp "$parent_file" "$output_file"
-echo "[INFO] Copied parent algorithm to: $output_file"
+# Check if processing should be skipped using common logic
+eval "$("$PYTHON_CMD" "$SCRIPT_DIR/../lib/evolution_processor.py" "$id" "$based_on_id" "$FULL_OUTPUT_DIR" "$ROOT_DIR" "$parent_file" "$output_file")"
+
+# Handle copy operation
+if [[ "$skip_copy" == "True" ]]; then
+  echo "[INFO] ⚠️  Skipping copy - $reason"
+else
+  cp "$parent_file" "$output_file"
+  echo "[INFO] Copied parent algorithm to: $output_file"
+fi
 
 # Check for claude CLI
 claude_cmd="${CLAUDE_CMD:-claude}"
@@ -354,9 +359,10 @@ Requirements:
 
 The file currently contains the parent algorithm. Modify it according to the description above while adhering to all guidelines from the CLAUDE.md files."
 
-# Generate mutation (skip for baseline)
-if [[ $id == "000" || $id == "0" ]]; then
-  echo "[INFO] Baseline algorithm - skipping mutation, using original"
+# AIDEV-NOTE: Using common evolution processor logic for Claude processing decisions
+# Handle Claude mutation based on skip flags
+if [[ "$skip_claude" == "True" ]]; then
+  echo "[INFO] ⚠️  Skipping Claude processing - $reason"
 else
   echo "[INFO] Calling Claude $CLAUDE_MODEL to apply mutation..."
   echo "[INFO] Claude will edit: $output_file"

package/bin/claude-evolve-worker

@@ -110,6 +110,7 @@ fi
 echo "[WORKER-$$] Description: $description"
 echo "[WORKER-$$] Based on ID: $based_on_id"
 
+# AIDEV-NOTE: Using common evolution processor logic for consistent handling
 # Determine parent algorithm
 if [[ -z $based_on_id || $based_on_id == "0" || $based_on_id == '""' ]]; then
   parent_file="$FULL_ALGORITHM_PATH"
@@ -136,19 +137,20 @@ else
   output_file="$FULL_OUTPUT_DIR/evolution_${id}.py"
 fi
 
-# Copy parent to output (skip if destination already exists)
-if [[ -f "$output_file" ]]; then
-  echo "[WORKER-$$] ⚠️  Skipping copy - child algorithm already exists: $output_file"
+# Check if processing should be skipped using common logic
+eval "$("$PYTHON_CMD" "$SCRIPT_DIR/../lib/evolution_processor.py" "$id" "$based_on_id" "$FULL_OUTPUT_DIR" "$ROOT_DIR" "$parent_file" "$output_file")"
+
+# Handle copy operation
+if [[ "$skip_copy" == "True" ]]; then
+  echo "[WORKER-$$] ⚠️  Skipping copy - $reason"
 else
   cp "$parent_file" "$output_file"
   echo "[WORKER-$$] Copied parent to: $output_file"
 fi
 
-# Generate mutation (skip for baseline or self-parent)
-if [[ $id == "000" || $id == "0" || $id == "gen00-000" ]]; then
-  echo "[WORKER-$$] Baseline algorithm - skipping mutation"
-elif [[ "$parent_file" == "$output_file" ]]; then
-  echo "[WORKER-$$] ⚠️  Self-parent detected - skipping mutation to preserve existing code"
+# Handle Claude mutation based on skip flags
+if [[ "$skip_claude" == "True" ]]; then
+  echo "[WORKER-$$] ⚠️  Skipping Claude processing - $reason"
 else
   # Check for claude CLI
   claude_cmd="${CLAUDE_CMD:-claude}"

package/lib/config.sh

@@ -40,7 +40,7 @@ DEFAULT_NUM_ELITES=3
 # Default parallel execution values
 DEFAULT_PARALLEL_ENABLED=false
 DEFAULT_MAX_WORKERS=4
-DEFAULT_LOCK_TIMEOUT=30
+DEFAULT_LOCK_TIMEOUT=10
 
 # Default auto ideation value
 DEFAULT_AUTO_IDEATE=true

package/lib/csv-lock.sh

@@ -4,36 +4,95 @@
 # Lock file location
 CSV_LOCKFILE="${EVOLUTION_DIR:-evolution}/.evolution.csv.lock"
 
-# Acquire exclusive lock on CSV file
+# Acquire exclusive lock on CSV file with automatic stale lock cleanup
 # Usage: acquire_csv_lock [timeout_seconds]
 acquire_csv_lock() {
-    local timeout="${1:-30}"
+    local timeout="${1:-${LOCK_TIMEOUT:-10}}"  # Reduced default timeout
     local lockdir="$(dirname "$CSV_LOCKFILE")"
     
     # Ensure lock directory exists
     mkdir -p "$lockdir"
     
-    # Try to acquire lock with timeout
-    if command -v flock >/dev/null 2>&1; then
-        # Use flock if available (Linux)
-        exec 200>"$CSV_LOCKFILE"
-        if ! flock -w "$timeout" -x 200; then
-            echo "ERROR: Failed to acquire CSV lock within $timeout seconds" >&2
-            return 1
-        fi
-    else
-        # Fallback for systems without flock (macOS)
-        local start_time=$(date +%s)
-        while ! (set -C; echo $$ > "$CSV_LOCKFILE") 2>/dev/null; do
-            local current_time=$(date +%s)
-            if [ $((current_time - start_time)) -ge $timeout ]; then
-                echo "ERROR: Failed to acquire CSV lock within $timeout seconds" >&2
-                return 1
+    # AIDEV-NOTE: Robust locking with automatic stale lock detection and cleanup
+    # CSV operations should be fast (<100ms), so long timeouts indicate problems
+    
+    # Clean up stale locks first
+    cleanup_stale_locks
+    
+    # Try to acquire lock with short timeout and fast retry
+    local end_time=$(($(date +%s) + timeout))
+    local sleep_time=0.01  # Start with 10ms sleep
+    
+    while [ $(date +%s) -lt $end_time ]; do
+        if command -v flock >/dev/null 2>&1; then
+            # Use flock if available (Linux) - this should be instant
+            exec 200>"$CSV_LOCKFILE"
+            if flock -n -x 200; then
+                return 0
+            fi
+        else
+            # Fallback for systems without flock (macOS)
+            if (set -C; echo $$ > "$CSV_LOCKFILE") 2>/dev/null; then
+                return 0
+            fi
+            
+            # Check if existing lock is stale and clean it up
+            if [ -f "$CSV_LOCKFILE" ]; then
+                local lock_pid=$(cat "$CSV_LOCKFILE" 2>/dev/null)
+                if [[ "$lock_pid" =~ ^[0-9]+$ ]] && ! kill -0 "$lock_pid" 2>/dev/null; then
+                    echo "[DEBUG] Removing stale lock from dead process $lock_pid" >&2
+                    rm -f "$CSV_LOCKFILE"
+                    continue  # Try again immediately
+                fi
             fi
-            sleep 0.1
-        done
+        fi
+        
+        # Brief sleep with exponential backoff (cap at 100ms)
+        sleep "$sleep_time"
+        sleep_time=$(echo "$sleep_time * 1.5" | bc -l 2>/dev/null | head -c 10)
+        if (( $(echo "$sleep_time > 0.1" | bc -l 2>/dev/null || echo 0) )); then
+            sleep_time=0.1
+        fi
+    done
+    
+    echo "ERROR: Failed to acquire CSV lock within $timeout seconds" >&2
+    echo "ERROR: This indicates a serious problem - CSV operations should be fast" >&2
+    
+    # As a last resort, if lock is very old, break it
+    if [ -f "$CSV_LOCKFILE" ]; then
+        local lock_age=$(($(date +%s) - $(stat -f %m "$CSV_LOCKFILE" 2>/dev/null || stat -c %Y "$CSV_LOCKFILE" 2>/dev/null || echo $(date +%s))))
+        if [ $lock_age -gt 60 ]; then  # Lock older than 1 minute is definitely stale
+            echo "[WARN] Breaking very old lock file (${lock_age}s old)" >&2
+            rm -f "$CSV_LOCKFILE"
+            return 1  # Still return error to trigger retry
+        fi
+    fi
+    
+    return 1
+}
+
+# Clean up stale lock files
+cleanup_stale_locks() {
+    if [ ! -f "$CSV_LOCKFILE" ]; then
+        return 0
+    fi
+    
+    # Check file age - any lock older than 10 seconds is definitely stale
+    local lock_age=$(($(date +%s) - $(stat -f %m "$CSV_LOCKFILE" 2>/dev/null || stat -c %Y "$CSV_LOCKFILE" 2>/dev/null || echo $(date +%s))))
+    if [ $lock_age -gt 10 ]; then
+        echo "[DEBUG] Removing stale lock file (${lock_age}s old)" >&2
+        rm -f "$CSV_LOCKFILE"
+        return 0
+    fi
+    
+    # Check if process is still alive (macOS fallback mode only)
+    if ! command -v flock >/dev/null 2>&1; then
+        local lock_pid=$(cat "$CSV_LOCKFILE" 2>/dev/null)
+        if [[ "$lock_pid" =~ ^[0-9]+$ ]] && ! kill -0 "$lock_pid" 2>/dev/null; then
+            echo "[DEBUG] Removing lock from dead process $lock_pid" >&2
+            rm -f "$CSV_LOCKFILE"
+        fi
     fi
-    return 0
 }
 
 # Release CSV lock

package/lib/evolution_processor.py

@@ -0,0 +1,77 @@
+#!/usr/bin/env python3
+"""
+Evolution processor - common logic for processing evolution candidates.
+Used by both sequential (claude-evolve-run) and parallel (claude-evolve-worker) modes.
+"""
+
+import os
+import sys
+import subprocess
+import re
+from pathlib import Path
+
+def should_skip_processing(id_val, based_on_id, parent_file, output_file):
+    """
+    Determine if evolution processing should be skipped.
+    
+    Simple rule: If file exists, skip everything. This handles all edge cases cleanly.
+    
+    Returns tuple: (skip_copy, skip_claude, reason)
+    """
+    # Baseline algorithm check
+    if id_val in ["000", "0", "gen00-000"]:
+        return True, True, "Baseline algorithm - no processing needed"
+    
+    # File existence check - if file exists, skip both copy and Claude
+    # This automatically handles self-parent cases and re-runs
+    if os.path.exists(output_file):
+        return True, True, "File already exists - skipping all processing"
+    
+    # File doesn't exist - proceed with copy and Claude
+    return False, False, None
+
+def get_parent_file_path(based_on_id, output_dir, root_dir):
+    """Get the parent file path based on based_on_id."""
+    if not based_on_id or based_on_id in ["0", '""']:
+        # Use base algorithm
+        return os.path.join(root_dir, "algorithm.py")
+    
+    # Handle both old format (numeric) and new format (genXX-XXX)
+    if re.match(r'^[0-9]+$', based_on_id):
+        # Old numeric format
+        return os.path.join(output_dir, f"evolution_id{based_on_id}.py")
+    else:
+        # New generation format
+        return os.path.join(output_dir, f"evolution_{based_on_id}.py")
+
+def get_output_file_path(id_val, output_dir):
+    """Get the output file path based on id."""
+    # Handle both old format (numeric) and new format (genXX-XXX)
+    if re.match(r'^[0-9]+$', id_val):
+        # Old numeric format
+        return os.path.join(output_dir, f"evolution_id{id_val}.py")
+    else:
+        # New generation format
+        return os.path.join(output_dir, f"evolution_{id_val}.py")
+
+def main():
+    """Main entry point for standalone testing."""
+    if len(sys.argv) < 7:
+        print("Usage: evolution_processor.py <id> <based_on_id> <output_dir> <root_dir> <parent_file> <output_file>")
+        sys.exit(1)
+    
+    id_val = sys.argv[1]
+    based_on_id = sys.argv[2]
+    output_dir = sys.argv[3]
+    root_dir = sys.argv[4]
+    parent_file = sys.argv[5]
+    output_file = sys.argv[6]
+    
+    skip_copy, skip_claude, reason = should_skip_processing(id_val, based_on_id, parent_file, output_file)
+    
+    print(f"skip_copy={skip_copy}")
+    print(f"skip_claude={skip_claude}")
+    print(f'reason="{reason or ""}"')
+
+if __name__ == "__main__":
+    main()

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "claude-evolve",
-  "version": "1.3.37",
+  "version": "1.3.38",
   "bin": {
     "claude-evolve": "./bin/claude-evolve",
     "claude-evolve-main": "./bin/claude-evolve-main",