claude-evolve 1.5.0 → 1.5.2

package/bin/claude-evolve-ideate

@@ -6,6 +6,10 @@ set -e
 SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 # shellcheck source=../lib/config.sh
 source "$SCRIPT_DIR/../lib/config.sh"
+# shellcheck source=../lib/csv-lock.sh
+source "$SCRIPT_DIR/../lib/csv-lock.sh"
+# shellcheck source=../lib/ai-cli.sh
+source "$SCRIPT_DIR/../lib/ai-cli.sh"
 
 # Use CLAUDE_EVOLVE_CONFIG if set, otherwise default
 if [[ -n ${CLAUDE_EVOLVE_CONFIG:-} ]]; then
@@ -14,197 +18,57 @@ else
   load_config
 fi
 
-# Function to determine which model to use based on generation
-get_model_for_generation() {
-  local generation="$1"
-  local gen_num
-  
-  # Extract numeric part of generation (e.g., "05" from gen05)
-  if [[ $generation =~ ^0*([0-9]+)$ ]]; then
-    gen_num=$((10#${BASH_REMATCH[1]}))
-  else
-    gen_num=1  # Default for malformed input
-  fi
-  
-  # Check which AI tools are available
-  local has_o3=false
-  local has_gemini=false
-  
-  if command -v codex >/dev/null 2>&1; then
-    has_o3=true
-  fi
-  
-  if command -v gemini >/dev/null 2>&1; then
-    has_gemini=true
-  fi
-  
-  # Determine rotation based on what's available
-  if [[ "$has_o3" == "true" && "$has_gemini" == "true" ]]; then
-    # All three available: opus -> o3 -> gemini rotation
-    case $((gen_num % 3)) in
-      1) echo "opus" ;;     # 1, 4, 7, 10...
-      2) echo "o3" ;;       # 2, 5, 8, 11...
-      0) echo "gemini" ;;   # 3, 6, 9, 12...
-    esac
-  elif [[ "$has_o3" == "true" ]]; then
-    # Only opus and o3: alternate between them
-    if (( gen_num % 2 == 1 )); then
-      echo "opus"    # Odd generations
-    else
-      echo "o3"      # Even generations
-    fi
-  elif [[ "$has_gemini" == "true" ]]; then
-    # Only opus and gemini: alternate between them
-    if (( gen_num % 2 == 1 )); then
-      echo "opus"    # Odd generations
-    else
-      echo "gemini" # Even generations
-    fi
-  else
-    # Only opus available
-    echo "opus"
-  fi
-}
-
-# Helper function to call AI model (alternating based on generation)
+# Helper function to call AI with limit check
 call_ai_with_limit_check() {
   local prompt="$1"
   local generation="${2:-01}"  # Default to generation 01 if not provided
   
-  # Determine which model to use for this generation
-  local preferred_model
-  preferred_model=$(get_model_for_generation "$generation")
-  
-  echo "[INFO] Generation $generation: Using $preferred_model" >&2
-  
-  # Try preferred model first
-  if [[ "$preferred_model" == "o3" ]] && command -v codex >/dev/null 2>&1; then
-    echo "[INFO] Using codex o3 for ideation" >&2
-    
-    # Call codex with o3 model using exec subcommand
-    local ai_output
-    ai_output=$(echo "$prompt" | codex exec -m o3 --full-auto 2>&1)
-    local ai_exit_code=$?
-    
-    if [[ $ai_exit_code -eq 0 ]]; then
-      # Clean o3 output - it may be JSON with the response in a field
-      local cleaned_output
-      # Try to extract content from JSON response if present
-      if echo "$ai_output" | grep -q '"content"'; then
-        # Attempt to extract content field from JSON
-        cleaned_output=$(echo "$ai_output" | python3 -c "
-import sys
-import json
-try:
-    data = json.load(sys.stdin)
-    if 'content' in data:
-        print(data['content'])
-    elif 'response' in data:
-        print(data['response'])
-    elif 'text' in data:
-        print(data['text'])
-    else:
-        # If no known field, print the whole thing
-        print(json.dumps(data))
-except:
-    # If not valid JSON, print as-is
-    print(sys.stdin.read())
-" 2>/dev/null || echo "$ai_output")
-      else
-        cleaned_output="$ai_output"
-      fi
-      
-      # Validate the output is not empty and doesn't contain error messages
-      if [[ -n "$cleaned_output" ]] && ! echo "$cleaned_output" | grep -q "error\|failed\|exception"; then
-        echo "$cleaned_output"
-        return 0
-      else
-        echo "[WARN] Codex o3 returned invalid output, falling back to Claude Opus" >&2
-        preferred_model="opus"
-      fi
-    else
-      echo "[WARN] Codex o3 failed with exit code $ai_exit_code, falling back to Claude Opus" >&2
-      preferred_model="opus"
-    fi
-  elif [[ "$preferred_model" == "gemini" ]] && command -v gemini >/dev/null 2>&1; then
-    echo "[INFO] Using gemini 2.5 pro for ideation" >&2
-    
-    # Call gemini with -y and -p flags
-    local ai_output
-    ai_output=$(gemini -y -p "$prompt" 2>&1)
-    local ai_exit_code=$?
-    
-    if [[ $ai_exit_code -eq 0 ]]; then
-      # Check for authentication messages or other non-response content
-      if echo "$ai_output" | grep -q "Attempting to authenticate\|Authenticating\|Loading\|Initializing"; then
-        echo "[WARN] Gemini is still authenticating, falling back to Claude Opus" >&2
-        preferred_model="opus"
-      elif [[ -z "$ai_output" ]] || [[ $(echo "$ai_output" | wc -l) -lt 2 ]]; then
-        echo "[WARN] Gemini returned insufficient output, falling back to Claude Opus" >&2
-        preferred_model="opus"
-      else
-        echo "$ai_output"
-        return 0
-      fi
-    else
-      echo "[WARN] Gemini failed with exit code $ai_exit_code, falling back to Claude Opus" >&2
-      preferred_model="opus"
-    fi
+  # Calculate hash value for round-robin based on generation
+  local gen_num
+  if [[ $generation =~ ^0*([0-9]+)$ ]]; then
+    gen_num=$((10#${BASH_REMATCH[1]}))
+  else
+    gen_num=1
   fi
   
-  # Use Claude with preferred model (or fallback)
-  echo "[INFO] Using Claude $preferred_model for ideation" >&2
-  
-  # Call Claude and capture output
-  local claude_output
-  claude_output=$(echo "$prompt" | claude --dangerously-skip-permissions --model "$preferred_model" -p 2>&1)
-  local claude_exit_code=$?
+  # Use centralized AI library for ideation
+  local ai_output
+  ai_output=$(call_ai_with_round_robin "$prompt" "ideate" "$gen_num")
+  local ai_exit_code=$?
   
-  # Check for usage limit
-  if echo "$claude_output" | grep -q "Claude AI usage limit reached"; then
-    # Extract timestamp if available
-    local limit_timestamp=$(echo "$claude_output" | grep -o "Claude AI usage limit reached|[0-9]*" | cut -d'|' -f2)
-    
-    # Print red error message
-    echo -e "\033[31m[ERROR] CLAUDE AI USAGE LIMIT REACHED!\033[0m" >&2
+  # Handle special exit codes
+  if [[ $ai_exit_code -eq 3 ]]; then
+    # All models hit usage limits
+    echo -e "\033[31m[ERROR] ALL AI MODELS HIT USAGE LIMITS!\033[0m" >&2
     echo -e "\033[31m[ERROR] Ideation halted due to API rate limits.\033[0m" >&2
-    
-    if [[ -n $limit_timestamp ]]; then
-      # Convert timestamp to human-readable format
-      local limit_date=$(date -r "$limit_timestamp" "+%Y-%m-%d %H:%M:%S" 2>/dev/null || echo "Unknown time")
-      echo -e "\033[31m[ERROR] Limit will be released at: $limit_date\033[0m" >&2
-    fi
-    
-    echo -e "\033[33m[INFO] Please wait for the rate limit to reset before continuing.\033[0m" >&2
-    echo -e "\033[33m[INFO] No ideas were generated. Run ideate again when the limit resets.\033[0m" >&2
-    
+    echo -e "\033[33m[INFO] Please wait for the rate limits to reset before continuing.\033[0m" >&2
+    echo -e "\033[33m[INFO] No ideas were generated. Run ideate again when the limits reset.\033[0m" >&2
     exit 1
   fi
   
-  # Validate output doesn't contain shell constructs that could corrupt CSV
-  if echo "$claude_output" | grep -E "EOF.*<.*null|<<.*EOF|<.*dev.*null" >/dev/null 2>&1; then
-    echo "[ERROR] AI output contains invalid shell constructs" >&2
-    echo "[DEBUG] Problematic output: $claude_output" | head -5 >&2
-    return 1
+  if [[ $ai_exit_code -eq 0 ]]; then
+    # For ideation, AI modifies files directly - just return success
+    echo "[INFO] AI succeeded" >&2
+    return 0
+  else
+    return $ai_exit_code
   fi
-  
-  # Output Claude's response
-  echo "$claude_output"
-  
-  return $claude_exit_code
 }
 
+
 # Backward compatibility alias
 call_claude_with_limit_check() {
   call_ai_with_limit_check "$@"
 }
 
 # Robust AI calling with fallbacks across all available models
-call_ai_with_fallbacks() {
+call_ai_for_ideation() {
   local prompt="$1"
   local generation="${2:-01}"
+  local expected_count="${3:-1}"  # Number of ideas expected to be added
+  local temp_csv_file="${4:-temp-csv-$$.csv}"  # Optional temp CSV filename
   
-  # Determine which model to use based on generation (round-robin)
+  # Calculate hash value for round-robin based on generation
   local gen_num
   if [[ $generation =~ ^0*([0-9]+)$ ]]; then
     gen_num=$((10#${BASH_REMATCH[1]}))
@@ -212,100 +76,47 @@ call_ai_with_fallbacks() {
     gen_num=1
   fi
   
-  # Check which AI tools are available
-  local available_models=()
-  if command -v codex >/dev/null 2>&1; then
-    available_models+=("o3")
-  fi
-  if command -v gemini >/dev/null 2>&1; then
-    available_models+=("gemini")
+  # Get the current row count before any modifications
+  local original_csv_count
+  if [[ -f "$temp_csv_file" ]]; then
+    original_csv_count=$(grep -v '^[[:space:]]*$' "$temp_csv_file" | tail -n +2 | wc -l)
+  else
+    original_csv_count=0
   fi
-  available_models+=("opus")  # Claude Opus always available
   
-  # Create ordered list based on round-robin for this generation
-  local num_models=${#available_models[@]}
-  local start_index=$((gen_num % num_models))
-  local models=()
+  # Use centralized AI library
+  local ai_output
+  ai_output=$(call_ai_with_round_robin "$prompt" "ideate" "$gen_num")
+  local ai_exit_code=$?
   
-  # Add models in round-robin order starting from the calculated index
-  for ((i=0; i<num_models; i++)); do
-    local idx=$(((start_index + i) % num_models))
-    models+=("${available_models[$idx]}")
-  done
-  
-  echo "[INFO] Model order for generation $generation (round-robin): ${models[*]}" >&2
+  # Handle special exit codes
+  # No special handling for exit codes anymore
   
-  # Try each model in the ordered sequence
-  for model in "${models[@]}"; do
-    echo "[INFO] Trying $model for ideation" >&2
-    
-    local ai_output
-    local ai_exit_code
-    
-    if [[ "$model" == "o3" ]] && command -v codex >/dev/null 2>&1; then
-      ai_output=$(echo "$prompt" | codex exec -m o3 --full-auto 2>&1)
-      ai_exit_code=$?
+  if [[ $ai_exit_code -eq 0 ]]; then
+    # For ideation, we need to verify the CSV file was actually modified
+    if [[ -f "$temp_csv_file" ]]; then
+      local new_csv_count
+      new_csv_count=$(grep -v '^[[:space:]]*$' "$temp_csv_file" | tail -n +2 | wc -l)
       
-      if [[ $ai_exit_code -eq 0 ]]; then
-        # Clean o3 output like in the original function
-        if echo "$ai_output" | grep -q '"content"'; then
-          ai_output=$(echo "$ai_output" | python3 -c "
-import sys
-import json
-try:
-    data = json.load(sys.stdin)
-    if 'content' in data:
-        print(data['content'])
-    elif 'response' in data:
-        print(data['response'])
-    elif 'text' in data:
-        print(data['text'])
-    else:
-        print(json.dumps(data))
-except:
-    print(sys.stdin.read())
-" 2>/dev/null || echo "$ai_output")
-        fi
-        
-        if [[ -n "$ai_output" ]] && ! echo "$ai_output" | grep -q "error\|failed\|exception"; then
-          echo "$ai_output"
-          return 0
-        fi
-      fi
-      
-    elif [[ "$model" == "gemini" ]] && command -v gemini >/dev/null 2>&1; then
-      ai_output=$(gemini -y -p "$prompt" 2>&1)
-      ai_exit_code=$?
-      
-      if [[ $ai_exit_code -eq 0 ]]; then
-        if ! echo "$ai_output" | grep -q "Attempting to authenticate\|Authenticating\|Loading\|Initializing"; then
-          if [[ -n "$ai_output" ]] && [[ $(echo "$ai_output" | wc -l) -ge 2 ]]; then
-            echo "$ai_output"
-            return 0
-          fi
-        fi
+      if [[ $new_csv_count -gt $original_csv_count ]]; then
+        echo "[INFO] AI completed successfully and modified CSV ($new_csv_count vs $original_csv_count rows)" >&2
+        return 0
+      else
+        echo "[INFO] $model returned exit code 0 but didn't modify CSV file" >&2
+        echo "[DEBUG] Expected file: $temp_csv_file" >&2
+        echo "[DEBUG] Original count: $original_csv_count, New count: $new_csv_count" >&2
+        return 1
       fi
-      
     else
-      # Use Claude with the specified model
-      ai_output=$(echo "$prompt" | claude --dangerously-skip-permissions --model "$model" -p 2>&1)
-      ai_exit_code=$?
-      
-      if [[ $ai_exit_code -eq 0 ]]; then
-        # Check for usage limits
-        if ! echo "$ai_output" | grep -q "Claude AI usage limit reached"; then
-          if ! echo "$ai_output" | grep -E "EOF.*<.*null|<<.*EOF|<.*dev.*null" >/dev/null 2>&1; then
-            echo "$ai_output"
-            return 0
-          fi
-        fi
-      fi
+      echo "[INFO] Exit code 0 but temp CSV file not found: $temp_csv_file" >&2
+      echo "[DEBUG] Current directory: $(pwd)" >&2
+      echo "[DEBUG] Files matching temp-csv-*.csv:" >&2
+      ls -la temp-csv-*.csv 2>&1 >&2
+      return 1
     fi
-    
-    echo "[WARN] $model failed or returned unusable output, trying next model..." >&2
-  done
+  fi
   
-  echo "[ERROR] All AI models failed to generate usable output" >&2
+  echo "[INFO] No AI model successfully modified the CSV file" >&2
   return 1
 }
 
@@ -375,16 +186,15 @@ if [[ $use_strategies == true ]]; then
   fi
 fi
 
-# Get next generation number
+# Get next generation number that doesn't have existing Python files
 get_next_generation() {
-  if [[ ! -f "$FULL_CSV_PATH" ]]; then
-    echo "01"
-    return
-  fi
+  # Start with generation 1 if no CSV exists
+  local start_gen=1
   
-  # Use Python for proper CSV parsing
-  local max_gen
-  max_gen=$("$PYTHON_CMD" -c "
+  if [[ -f "$FULL_CSV_PATH" ]]; then
+    # Use Python for proper CSV parsing to find max generation
+    local max_gen
+    max_gen=$("$PYTHON_CMD" -c "
 import csv
 max_gen = 0
 with open('$FULL_CSV_PATH', 'r') as f:
@@ -402,9 +212,36 @@ with open('$FULL_CSV_PATH', 'r') as f:
                     pass
 print(max_gen)
 ")
+    # Start checking from the next generation after max
+    start_gen=$((max_gen + 1))
+  fi
   
-  # Increment and format with leading zero
-  printf "%02d" $((max_gen + 1))
+  # Keep incrementing until we find a generation with no Python files
+  local candidate_gen=$start_gen
+  while true; do
+    local gen_formatted=$(printf "%02d" $candidate_gen)
+    
+    # Check if any Python files exist for this generation
+    local py_files_exist=false
+    if ls "$FULL_OUTPUT_DIR"/evolution_gen${gen_formatted}-*.py >/dev/null 2>&1; then
+      py_files_exist=true
+    fi
+    
+    if [[ "$py_files_exist" == "false" ]]; then
+      # This generation is safe to use
+      echo "$gen_formatted"
+      return
+    else
+      echo "[WARN] Generation $gen_formatted already has Python files, skipping to next generation" >&2
+      candidate_gen=$((candidate_gen + 1))
+      
+      # Safety check to prevent infinite loop
+      if [[ $candidate_gen -gt 999 ]]; then
+        echo "[ERROR] Could not find a safe generation number (checked up to 999)" >&2
+        exit 1
+      fi
+    fi
+  done
 }
 
 # This function is no longer used with direct CSV modification approach
@@ -439,31 +276,64 @@ validate_direct_csv_modification() {
     return 1
   fi
   
-  # Validate the modified CSV has more entries than original
-  # Count actual data rows (excluding header and empty lines)
-  local original_count
-  original_count=$(grep -v '^[[:space:]]*$' "$FULL_CSV_PATH" | tail -n +2 | wc -l)
+  # Get the count before modification from the temp CSV (which was copied from original before AI ran)
+  # We need to track this before the AI runs by reading from the beginning state
+  # First, get a fresh count from the current main CSV (which reflects any previous operations in this session)
+  local current_original_count
+  current_original_count=$(grep -v '^[[:space:]]*$' "$FULL_CSV_PATH" | tail -n +2 | wc -l)
+  
+  # Count data rows in the modified temp CSV
   local new_count
   new_count=$(grep -v '^[[:space:]]*$' "$temp_csv" | tail -n +2 | wc -l)
   
-  echo "[DEBUG] Original CSV data rows: $original_count" >&2
-  echo "[DEBUG] Modified CSV data rows: $new_count" >&2
+  echo "[DEBUG] Current main CSV data rows: $current_original_count" >&2
+  echo "[DEBUG] Modified temp CSV data rows: $new_count" >&2
   echo "[DEBUG] Expected to add: $expected_count ideas" >&2
   
-  if [[ $new_count -le $original_count ]]; then
-    echo "[ERROR] CSV file wasn't modified - same number of data rows ($new_count <= $original_count)" >&2
+  # Check if AI overwrote the file instead of appending
+  if [[ $new_count -lt $current_original_count ]]; then
+    echo "[ERROR] AI overwrote the CSV file instead of appending ($new_count < $current_original_count)" >&2
     echo "[DEBUG] First 10 lines of CSV after AI attempt:" >&2
     head -10 "$temp_csv" >&2
     return 1
   fi
   
-  local added_count=$((new_count - original_count))
+  # Check if no changes were made
+  if [[ $new_count -eq $current_original_count ]]; then
+    echo "[ERROR] CSV file wasn't modified - same number of data rows ($new_count = $current_original_count)" >&2
+    echo "[DEBUG] First 10 lines of CSV after AI attempt:" >&2
+    head -10 "$temp_csv" >&2
+    return 1
+  fi
+  
+  local added_count=$((new_count - current_original_count))
   if [[ $added_count -ne $expected_count ]]; then
     echo "[WARN] Expected to add $expected_count ideas but added $added_count" >&2
   fi
   
-  # Replace original CSV with modified version
-  mv "$temp_csv" "$FULL_CSV_PATH"
+  # Use proper locking to safely update the CSV
+  echo "[INFO] Acquiring CSV lock to apply changes..."
+  
+  # Set the lockfile path
+  CSV_LOCKFILE="$FULL_EVOLUTION_DIR/.evolution.csv.lock"
+  
+  if ! acquire_csv_lock; then
+    echo "[ERROR] Failed to acquire CSV lock for update" >&2
+    rm -f "$temp_csv"
+    return 1
+  fi
+  
+  # Get just the new entries (skip header and existing entries)
+  local original_line_count=$(wc -l < "$FULL_CSV_PATH")
+  
+  # Append only the new lines from temp CSV to the main CSV
+  tail -n +$((original_line_count + 1)) "$temp_csv" >> "$FULL_CSV_PATH"
+  
+  # Clean up temp file
+  rm -f "$temp_csv"
+  
+  # Release the lock
+  release_csv_lock
   
   echo "[INFO] Successfully added $added_count $idea_type ideas to CSV"
   return 0
@@ -534,8 +404,29 @@ validate_and_apply_csv_modification_old() {
     echo "[WARN] Expected to add $expected_count ideas but added $added_count" >&2
   fi
   
-  # Replace original CSV with modified version
-  mv "$temp_csv" "$FULL_CSV_PATH"
+  # Use proper locking to safely update the CSV
+  echo "[INFO] Acquiring CSV lock to apply changes..."
+  
+  # Set the lockfile path
+  CSV_LOCKFILE="$FULL_EVOLUTION_DIR/.evolution.csv.lock"
+  
+  if ! acquire_csv_lock; then
+    echo "[ERROR] Failed to acquire CSV lock for update" >&2
+    rm -f "$temp_csv"
+    return 1
+  fi
+  
+  # Get just the new entries (skip header and existing entries)
+  local original_line_count=$(wc -l < "$FULL_CSV_PATH")
+  
+  # Append only the new lines from temp CSV to the main CSV
+  tail -n +$((original_line_count + 1)) "$temp_csv" >> "$FULL_CSV_PATH"
+  
+  # Clean up temp file
+  rm -f "$temp_csv"
+  
+  # Release the lock
+  release_csv_lock
   
   echo "[INFO] Successfully added $added_count $idea_type ideas to CSV"
   return 0
@@ -580,8 +471,8 @@ IMPORTANT: Output the complete modified CSV file. Do not add any explanation or
   
   # Get AI to modify the CSV with fallbacks
   local modified_csv
-  local stderr_file="/tmp/claude-evolve-stderr-$$.txt"
-  if ! modified_csv=$(call_ai_with_fallbacks "$csv_prompt" "$CURRENT_GENERATION" 2>"$stderr_file"); then
+  local stderr_file="$FULL_EVOLUTION_DIR/stderr-$$.txt"
+  if ! modified_csv=$(call_ai_for_ideation "$csv_prompt" "$CURRENT_GENERATION" 2>"$stderr_file"); then
     echo "[ERROR] All AI models failed to modify CSV" >&2
     cat "$stderr_file" >&2
     rm -f "$temp_csv" "$stderr_file"
@@ -659,14 +550,72 @@ IMPORTANT: Output the complete modified CSV file. Do not add any explanation or
     echo "[WARN] Expected to add $count ideas but added $added_count" >&2
   fi
   
-  # Replace original CSV with modified version
-  mv "$temp_csv" "$FULL_CSV_PATH"
+  # Use proper locking to safely update the CSV
+  echo "[INFO] Acquiring CSV lock to apply changes..."
+  
+  # Set the lockfile path
+  CSV_LOCKFILE="$FULL_EVOLUTION_DIR/.evolution.csv.lock"
+  
+  if ! acquire_csv_lock; then
+    echo "[ERROR] Failed to acquire CSV lock for update" >&2
+    rm -f "$temp_csv"
+    return 1
+  fi
+  
+  # Get just the new entries (skip header and existing entries)
+  local original_line_count=$(wc -l < "$FULL_CSV_PATH")
+  
+  # Append only the new lines from temp CSV to the main CSV
+  tail -n +$((original_line_count + 1)) "$temp_csv" >> "$FULL_CSV_PATH"
+  
+  # Clean up temp file
+  rm -f "$temp_csv"
+  
+  # Release the lock
+  release_csv_lock
   
   echo "[INFO] Successfully added $added_count $idea_type ideas to CSV"
   
   return 0
 }
 
+# Get list of existing Python files for a generation
+get_existing_py_files_for_generation() {
+  local generation="$1"
+  local py_files=""
+  
+  # List all Python files for this generation
+  for py_file in "$FULL_OUTPUT_DIR"/evolution_gen${generation}-*.py; do
+    if [[ -f "$py_file" ]]; then
+      local basename=$(basename "$py_file" .py)
+      local id="${basename#evolution_}"
+      if [[ -n "$py_files" ]]; then
+        py_files="$py_files, $id"
+      else
+        py_files="$id"
+      fi
+    fi
+  done
+  
+  echo "$py_files"
+}
+
+# Add existing Python files warning to prompt
+add_existing_files_warning() {
+  local prompt="$1"
+  local generation="$2"
+  local existing_py_files=$(get_existing_py_files_for_generation "$generation")
+  
+  if [[ -n "$existing_py_files" ]]; then
+    prompt+="
+
+WARNING: The following IDs already have Python files and MUST NOT be reused: $existing_py_files
+Skip these IDs when assigning new IDs (e.g., if gen$generation-001 and gen$generation-002 exist as Python files, start with gen$generation-003)"
+  fi
+  
+  echo "$prompt"
+}
+
 # Get next available ID for current generation
 get_next_id() {
   local generation="$1"
@@ -796,10 +745,55 @@ ideate_ai_strategies() {
   echo "  Crossover hybrid: $CROSSOVER_HYBRID"
 
   # Generate each type of idea by having Claude directly edit the CSV
-  [[ $NOVEL_EXPLORATION -gt 0 ]] && generate_novel_ideas_direct "$NOVEL_EXPLORATION"
-  [[ $HILL_CLIMBING -gt 0 ]] && generate_hill_climbing_direct "$HILL_CLIMBING" "$top_performers"
-  [[ $STRUCTURAL_MUTATION -gt 0 ]] && generate_structural_mutation_direct "$STRUCTURAL_MUTATION" "$top_performers"
-  [[ $CROSSOVER_HYBRID -gt 0 ]] && generate_crossover_direct "$CROSSOVER_HYBRID" "$top_performers"
+  # Track successes - continue even if some strategies fail
+  local strategies_attempted=0
+  local strategies_succeeded=0
+  
+  if [[ $NOVEL_EXPLORATION -gt 0 ]]; then
+    ((strategies_attempted++))
+    if generate_novel_ideas_direct "$NOVEL_EXPLORATION"; then
+      ((strategies_succeeded++))
+    else
+      echo "[WARN] Novel exploration strategy failed, continuing with other strategies" >&2
+    fi
+  fi
+  
+  if [[ $HILL_CLIMBING -gt 0 ]]; then
+    ((strategies_attempted++))
+    if generate_hill_climbing_direct "$HILL_CLIMBING" "$top_performers"; then
+      ((strategies_succeeded++))
+    else
+      echo "[WARN] Hill climbing strategy failed, continuing with other strategies" >&2
+    fi
+  fi
+  
+  if [[ $STRUCTURAL_MUTATION -gt 0 ]]; then
+    ((strategies_attempted++))
+    if generate_structural_mutation_direct "$STRUCTURAL_MUTATION" "$top_performers"; then
+      ((strategies_succeeded++))
+    else
+      echo "[WARN] Structural mutation strategy failed, continuing with other strategies" >&2
+    fi
+  fi
+  
+  if [[ $CROSSOVER_HYBRID -gt 0 ]]; then
+    ((strategies_attempted++))
+    if generate_crossover_direct "$CROSSOVER_HYBRID" "$top_performers"; then
+      ((strategies_succeeded++))
+    else
+      echo "[WARN] Crossover strategy failed, continuing with other strategies" >&2
+    fi
+  fi
+  
+  echo "[INFO] Strategy results: $strategies_succeeded/$strategies_attempted succeeded" >&2
+  
+  # Success if at least one strategy worked
+  if [[ $strategies_succeeded -gt 0 ]]; then
+    return 0
+  else
+    echo "[ERROR] All ideation strategies failed" >&2
+    return 1
+  fi
 }
 
 # Generate novel exploration ideas using direct CSV modification
@@ -817,34 +811,71 @@ generate_novel_ideas_direct() {
   # Use relative paths and change to evolution directory so AI can access files
   local temp_csv_basename=$(basename "$temp_csv")
   
-  local prompt="I need you to use your file editing capabilities to add exactly $count novel algorithmic ideas to the CSV file: $temp_csv_basename
+  # Get existing Python files for this generation to avoid ID collisions
+  local existing_py_files=$(get_existing_py_files_for_generation "$CURRENT_GENERATION")
+  
+  local prompt="I need you to use your file editing capabilities to APPEND exactly $count novel algorithmic ideas to the CSV file: $temp_csv_basename
 
 Current evolution context:
 - Generation: $CURRENT_GENERATION
 - Algorithm: algorithm.py (base algorithm)
-- Brief: $(head -20 "$FULL_BRIEF_PATH")
+- Brief: $(head -5 "$FULL_BRIEF_PATH" 2>/dev/null | head -c 500 || echo "No brief file found")
 
-Instructions:
+CRITICAL INSTRUCTIONS:
 1. Use the Read tool to examine the current CSV file
-2. Use the Edit or MultiEdit tool to add exactly $count new rows to the CSV file
-3. Use the next available generation numbers (gen$CURRENT_GENERATION-XXX format)
-4. For each idea, create a row with: id,,description,,pending (empty parent_id for novel ideas)
-5. Each description should be one clear sentence describing a novel algorithmic approach
-6. Focus on creative, ambitious ideas that haven't been tried yet
-7. Consider machine learning, new indicators, regime detection, risk management, etc.
-
-CRITICAL: You must use your file editing tools (Edit/MultiEdit) to modify the CSV file. DO NOT return CSV text - use your tools to edit the file directly."
+2. DO NOT DELETE OR REPLACE ANY EXISTING ROWS - YOU MUST PRESERVE ALL EXISTING DATA
+3. Find the highest ID number for generation $CURRENT_GENERATION (e.g., if gen$CURRENT_GENERATION-003 exists, next should be gen$CURRENT_GENERATION-004)
+4. If no gen$CURRENT_GENERATION entries exist yet, start with gen$CURRENT_GENERATION-001"
+  
+  if [[ -n "$existing_py_files" ]]; then
+    prompt+="
+5. IMPORTANT: The following IDs already have Python files and MUST NOT be reused: $existing_py_files
+6. Skip these IDs when assigning new IDs (e.g., if gen$CURRENT_GENERATION-001 and gen$CURRENT_GENERATION-002 exist as Python files, start with gen$CURRENT_GENERATION-003)"
+  else
+    prompt+="
+5. No existing Python files found for generation $CURRENT_GENERATION"
+  fi
+  
+  prompt+="
+6. Use the Edit or MultiEdit tool to APPEND exactly $count new rows AT THE END of the CSV file
+7. For each idea, create a row with: id,,description,,pending (empty parent_id for novel ideas)
+8. Each description should be one clear sentence describing a novel algorithmic approach
+9. Focus on creative, ambitious ideas that haven't been tried yet
+10. Consider machine learning, new indicators, regime detection, risk management, etc.
+
+IMPORTANT: You must APPEND new rows to the existing CSV file. DO NOT replace the file contents. All existing rows must remain unchanged.
+CRITICAL: You must use your file editing tools (Edit/MultiEdit) to modify the CSV file. DO NOT return CSV text - use your tools to edit the file directly.
+CRITICAL: Do NOT use any git commands (git add, git commit, git reset, etc.). Only modify the file directly."
+
+  # Debug prompt size and context
+  echo "[DEBUG] Novel ideas prompt length: ${#prompt} characters" >&2
+  echo "[DEBUG] Working dir: $(pwd)" >&2
+  echo "[DEBUG] Evolution dir: $FULL_EVOLUTION_DIR" >&2
+  echo "[DEBUG] Temp CSV: $temp_csv" >&2
+  if [[ -f "$temp_csv" ]]; then
+    echo "[DEBUG] Temp CSV exists, size: $(wc -l < "$temp_csv") lines" >&2
+  else
+    echo "[DEBUG] Temp CSV does not exist yet" >&2
+  fi
 
   # Change to evolution directory so AI can access files
   local original_pwd=$(pwd)
   cd "$FULL_EVOLUTION_DIR"
   
+  # Debug: Show files in evolution directory
+  echo "[DEBUG] Current directory: $(pwd)" >&2
+  echo "[DEBUG] Temp CSV exists: $(ls -la "$temp_csv_basename" 2>&1)" >&2
+  
   # Get AI to directly edit the CSV file
   local ai_response
   local stderr_file="stderr-$$.txt"
-  if ! ai_response=$(call_ai_with_fallbacks "$prompt" "$CURRENT_GENERATION" 2>"$stderr_file"); then
+  # Temporarily show stderr for debugging
+  if ! ai_response=$(call_ai_for_ideation "$prompt" "$CURRENT_GENERATION" "$count" "$temp_csv_basename"); then
     echo "[ERROR] All AI models failed to generate novel ideas" >&2
+    echo "[DEBUG] Stderr output from AI calls:" >&2
     cat "$stderr_file" >&2
+    echo "[DEBUG] Temp CSV location: $temp_csv" >&2
+    echo "[DEBUG] Working directory: $(pwd)" >&2
     cd "$original_pwd"
     rm -f "$temp_csv" "$stderr_file"
     return 1
@@ -879,6 +910,9 @@ generate_hill_climbing_direct() {
   local data_rows=$(grep -v '^[[:space:]]*$' "$FULL_CSV_PATH" | tail -n +2 | wc -l)
   echo "[DEBUG] Original CSV has $data_rows data rows" >&2
   
+  # Get existing Python files for this generation to avoid ID collisions
+  local existing_py_files=$(get_existing_py_files_for_generation "$CURRENT_GENERATION")
+  
   # Extract just the IDs from top performers for clarity
   local valid_parent_ids
   valid_parent_ids=$(echo "$top_performers" | cut -d',' -f1 | paste -sd ',' -)
@@ -886,7 +920,7 @@ generate_hill_climbing_direct() {
   # Use relative paths and change to evolution directory so AI can access files
   local temp_csv_basename=$(basename "$temp_csv")
   
-  local prompt="I need you to use your file editing capabilities to add exactly $count parameter tuning ideas to the CSV file: $temp_csv_basename
+  local prompt="I need you to use your file editing capabilities to APPEND exactly $count parameter tuning ideas to the CSV file: $temp_csv_basename
 
 IMPORTANT: You MUST use one of these exact parent IDs: $valid_parent_ids
 
@@ -897,16 +931,20 @@ CRITICAL INSTRUCTION: Before generating parameter tuning ideas, you MUST read th
 Algorithm source files are located at: evolution_<PARENT_ID>.py
 For example: evolution_gen01-251.py
 
-Instructions:
+CRITICAL INSTRUCTIONS:
 1. Use the Read tool to examine the current CSV file
-2. Use the Edit or MultiEdit tool to add exactly $count new rows to the CSV file
-3. Use the next available generation numbers (gen$CURRENT_GENERATION-XXX format)
-4. For each idea, create a row with: id,parent_id,description,,pending
-5. Each parent_id MUST be one of: $valid_parent_ids
-6. Each description should focus on adjusting specific parameters that exist in the parent's source code
-7. Include current and new parameter values (e.g., \"Lower rsi_entry from 21 to 18\")
-
-CRITICAL: You must use your file editing tools (Edit/MultiEdit) to modify the CSV file. DO NOT return CSV text - use your tools to edit the file directly."
+2. DO NOT DELETE OR REPLACE ANY EXISTING ROWS - YOU MUST PRESERVE ALL EXISTING DATA
+3. Find the highest ID number for generation $CURRENT_GENERATION (e.g., if gen$CURRENT_GENERATION-003 exists, next should be gen$CURRENT_GENERATION-004)
+4. If no gen$CURRENT_GENERATION entries exist yet, start with gen$CURRENT_GENERATION-001
+5. Use the Edit or MultiEdit tool to APPEND exactly $count new rows AT THE END of the CSV file
+6. For each idea, create a row with: id,parent_id,description,,pending
+7. Each parent_id MUST be one of: $valid_parent_ids
+8. Each description should focus on adjusting specific parameters that exist in the parent's source code
+9. Include current and new parameter values (e.g., \"Lower rsi_entry from 21 to 18\")
+
+IMPORTANT: You must APPEND new rows to the existing CSV file. DO NOT replace the file contents. All existing rows must remain unchanged.
+CRITICAL: You must use your file editing tools (Edit/MultiEdit) to modify the CSV file. DO NOT return CSV text - use your tools to edit the file directly.
+CRITICAL: Do NOT use any git commands (git add, git commit, git reset, etc.). Only modify the file directly."
 
   # Change to evolution directory so AI can access files
   local original_pwd=$(pwd)
@@ -915,7 +953,8 @@ CRITICAL: You must use your file editing tools (Edit/MultiEdit) to modify the CS
   # Get AI to directly edit the CSV file
   local ai_response
   local stderr_file="stderr-$$.txt"
-  if ! ai_response=$(call_ai_with_fallbacks "$prompt" "$CURRENT_GENERATION" 2>"$stderr_file"); then
+  # Temporarily show stderr for debugging
+  if ! ai_response=$(call_ai_for_ideation "$prompt" "$CURRENT_GENERATION" "$count" "$temp_csv_basename"); then
     echo "[ERROR] All AI models failed to generate hill climbing ideas" >&2
     cat "$stderr_file" >&2
     cd "$original_pwd"
@@ -952,6 +991,9 @@ generate_structural_mutation_direct() {
   local data_rows=$(grep -v '^[[:space:]]*$' "$FULL_CSV_PATH" | tail -n +2 | wc -l)
   echo "[DEBUG] Original CSV has $data_rows data rows" >&2
   
+  # Get existing Python files for this generation to avoid ID collisions
+  local existing_py_files=$(get_existing_py_files_for_generation "$CURRENT_GENERATION")
+  
   # Extract just the IDs from top performers for clarity
   local valid_parent_ids
   valid_parent_ids=$(echo "$top_performers" | cut -d',' -f1 | paste -sd ',' -)
@@ -959,7 +1001,7 @@ generate_structural_mutation_direct() {
   # Use relative paths and change to evolution directory so AI can access files
   local temp_csv_basename=$(basename "$temp_csv")
   
-  local prompt="I need you to use your file editing capabilities to add exactly $count structural modification ideas to the CSV file: $temp_csv_basename
+  local prompt="I need you to use your file editing capabilities to APPEND exactly $count structural modification ideas to the CSV file: $temp_csv_basename
 
 IMPORTANT: You MUST use one of these exact parent IDs: $valid_parent_ids
 
@@ -970,16 +1012,20 @@ CRITICAL INSTRUCTION: Before generating structural modification ideas, you MUST
 Algorithm source files are located at: evolution_<PARENT_ID>.py
 For example: evolution_gen01-251.py
 
-Instructions:
+CRITICAL INSTRUCTIONS:
 1. Use the Read tool to examine the current CSV file
-2. Use the Edit or MultiEdit tool to add exactly $count new rows to the CSV file
-3. Use the next available generation numbers (gen$CURRENT_GENERATION-XXX format)
-4. For each idea, create a row with: id,parent_id,description,,pending
-5. Each parent_id MUST be one of: $valid_parent_ids
-6. Each description should focus on architectural/structural changes based on the parent's actual code
-7. Reference actual components/methods found in the source code
-
-CRITICAL: You must use your file editing tools (Edit/MultiEdit) to modify the CSV file. DO NOT return CSV text - use your tools to edit the file directly."
+2. DO NOT DELETE OR REPLACE ANY EXISTING ROWS - YOU MUST PRESERVE ALL EXISTING DATA
+3. Find the highest ID number for generation $CURRENT_GENERATION (e.g., if gen$CURRENT_GENERATION-003 exists, next should be gen$CURRENT_GENERATION-004)
+4. If no gen$CURRENT_GENERATION entries exist yet, start with gen$CURRENT_GENERATION-001
+5. Use the Edit or MultiEdit tool to APPEND exactly $count new rows AT THE END of the CSV file
+6. For each idea, create a row with: id,parent_id,description,,pending
+7. Each parent_id MUST be one of: $valid_parent_ids
+8. Each description should focus on architectural/structural changes based on the parent's actual code
+9. Reference actual components/methods found in the source code
+
+IMPORTANT: You must APPEND new rows to the existing CSV file. DO NOT replace the file contents. All existing rows must remain unchanged.
+CRITICAL: You must use your file editing tools (Edit/MultiEdit) to modify the CSV file. DO NOT return CSV text - use your tools to edit the file directly.
+CRITICAL: Do NOT use any git commands (git add, git commit, git reset, etc.). Only modify the file directly."
 
   # Change to evolution directory so AI can access files
   local original_pwd=$(pwd)
@@ -988,7 +1034,8 @@ CRITICAL: You must use your file editing tools (Edit/MultiEdit) to modify the CS
   # Get AI to directly edit the CSV file
   local ai_response
   local stderr_file="stderr-$$.txt"
-  if ! ai_response=$(call_ai_with_fallbacks "$prompt" "$CURRENT_GENERATION" 2>"$stderr_file"); then
+  # Temporarily show stderr for debugging
+  if ! ai_response=$(call_ai_for_ideation "$prompt" "$CURRENT_GENERATION" "$count" "$temp_csv_basename"); then
     echo "[ERROR] All AI models failed to generate structural mutation ideas" >&2
     cat "$stderr_file" >&2
     cd "$original_pwd"
@@ -1025,6 +1072,9 @@ generate_crossover_direct() {
   local data_rows=$(grep -v '^[[:space:]]*$' "$FULL_CSV_PATH" | tail -n +2 | wc -l)
   echo "[DEBUG] Original CSV has $data_rows data rows" >&2
   
+  # Get existing Python files for this generation to avoid ID collisions
+  local existing_py_files=$(get_existing_py_files_for_generation "$CURRENT_GENERATION")
+  
   # Extract just the IDs from top performers for clarity
   local valid_parent_ids
   valid_parent_ids=$(echo "$top_performers" | cut -d',' -f1 | paste -sd ',' -)
@@ -1032,7 +1082,7 @@ generate_crossover_direct() {
   # Use relative paths and change to evolution directory so AI can access files
   local temp_csv_basename=$(basename "$temp_csv")
   
-  local prompt="I need you to use your file editing capabilities to add exactly $count hybrid combination ideas to the CSV file: $temp_csv_basename
+  local prompt="I need you to use your file editing capabilities to APPEND exactly $count hybrid combination ideas to the CSV file: $temp_csv_basename
 
 IMPORTANT: You MUST use ONLY these exact parent IDs: $valid_parent_ids
 
@@ -1043,16 +1093,20 @@ CRITICAL INSTRUCTION: Before generating hybrid combination ideas, you MUST read
 Algorithm source files are located at: evolution_<PARENT_ID>.py
 For example: evolution_gen01-251.py
 
-Instructions:
+CRITICAL INSTRUCTIONS:
 1. Use the Read tool to examine the current CSV file
-2. Use the Edit or MultiEdit tool to add exactly $count new rows to the CSV file
-3. Use the next available generation numbers (gen$CURRENT_GENERATION-XXX format)
-4. For each idea, create a row with: id,parent_id,description,,pending
-5. Each parent_id MUST be one of: $valid_parent_ids (choose the primary parent)
-6. Each description should combine actual elements from 2+ algorithms based on their source code
-7. Reference specific components/features found in the actual source code
-
-CRITICAL: You must use your file editing tools (Edit/MultiEdit) to modify the CSV file. DO NOT return CSV text - use your tools to edit the file directly."
+2. DO NOT DELETE OR REPLACE ANY EXISTING ROWS - YOU MUST PRESERVE ALL EXISTING DATA
+3. Find the highest ID number for generation $CURRENT_GENERATION (e.g., if gen$CURRENT_GENERATION-003 exists, next should be gen$CURRENT_GENERATION-004)
+4. If no gen$CURRENT_GENERATION entries exist yet, start with gen$CURRENT_GENERATION-001
+5. Use the Edit or MultiEdit tool to APPEND exactly $count new rows AT THE END of the CSV file
+6. For each idea, create a row with: id,parent_id,description,,pending
+7. Each parent_id MUST be one of: $valid_parent_ids (choose the primary parent)
+8. Each description should combine actual elements from 2+ algorithms based on their source code
+9. Reference specific components/features found in the actual source code
+
+IMPORTANT: You must APPEND new rows to the existing CSV file. DO NOT replace the file contents. All existing rows must remain unchanged.
+CRITICAL: You must use your file editing tools (Edit/MultiEdit) to modify the CSV file. DO NOT return CSV text - use your tools to edit the file directly.
+CRITICAL: Do NOT use any git commands (git add, git commit, git reset, etc.). Only modify the file directly."
 
   # Change to evolution directory so AI can access files
   local original_pwd=$(pwd)
@@ -1061,7 +1115,8 @@ CRITICAL: You must use your file editing tools (Edit/MultiEdit) to modify the CS
   # Get AI to directly edit the CSV file
   local ai_response
   local stderr_file="stderr-$$.txt"
-  if ! ai_response=$(call_ai_with_fallbacks "$prompt" "$CURRENT_GENERATION" 2>"$stderr_file"); then
+  # Temporarily show stderr for debugging
+  if ! ai_response=$(call_ai_for_ideation "$prompt" "$CURRENT_GENERATION" "$count" "$temp_csv_basename"); then
     echo "[ERROR] All AI models failed to generate crossover hybrid ideas" >&2
     cat "$stderr_file" >&2
     cd "$original_pwd"
@@ -1133,13 +1188,15 @@ $top_performers"
 
   prompt+="
 
-Instructions:
+CRITICAL INSTRUCTIONS:
 1. Use the Read tool to examine the current CSV file
-2. Use the Edit or MultiEdit tool to add exactly $TOTAL_IDEAS new rows to the CSV file
-3. Use the next available generation numbers (gen$CURRENT_GENERATION-XXX format)
-4. For each idea, create a row with: id,parent_id,description,,pending
-5. Mix both parameter tuning and structural changes
-6. If building on existing algorithms, use their ID as parent_id, otherwise leave parent_id empty
+2. DO NOT DELETE OR REPLACE ANY EXISTING ROWS - YOU MUST PRESERVE ALL EXISTING DATA
+3. Find the highest ID number for generation $CURRENT_GENERATION (e.g., if gen$CURRENT_GENERATION-003 exists, next should be gen$CURRENT_GENERATION-004)
+4. If no gen$CURRENT_GENERATION entries exist yet, start with gen$CURRENT_GENERATION-001
+5. Use the Edit or MultiEdit tool to APPEND exactly $TOTAL_IDEAS new rows AT THE END of the CSV file
+6. For each idea, create a row with: id,parent_id,description,,pending
+7. Mix both parameter tuning and structural changes
+8. If building on existing algorithms, use their ID as parent_id, otherwise leave parent_id empty
 
 ⚠️ AVOID ONLY: Kelly floor/cap adjustments that assume leverage > 1.0 (these get clamped and have no effect)
 
@@ -1153,7 +1210,9 @@ Instructions:
 - Time-Based Patterns: Intraday effects, calendar anomalies, volatility timing
 - Parameter Optimization: Entry thresholds, indicator periods, strategy weights
 
-CRITICAL: You must use your file editing tools (Edit/MultiEdit) to modify the CSV file. DO NOT return CSV text - use your tools to edit the file directly."
+IMPORTANT: You must APPEND new rows to the existing CSV file. DO NOT replace the file contents. All existing rows must remain unchanged.
+CRITICAL: You must use your file editing tools (Edit/MultiEdit) to modify the CSV file. DO NOT return CSV text - use your tools to edit the file directly.
+CRITICAL: Do NOT use any git commands (git add, git commit, git reset, etc.). Only modify the file directly."
 
   # Change to evolution directory so AI can access files
   local original_pwd=$(pwd)
@@ -1162,7 +1221,7 @@ CRITICAL: You must use your file editing tools (Edit/MultiEdit) to modify the CS
   # Get AI to directly edit the CSV file
   local ai_response
   local stderr_file="stderr-$$.txt"
-  if ! ai_response=$(call_ai_with_fallbacks "$prompt" "$CURRENT_GENERATION" 2>"$stderr_file"); then
+  if ! ai_response=$(call_ai_for_ideation "$prompt" "$CURRENT_GENERATION" "$TOTAL_IDEAS" "$temp_csv_basename" 2>"$stderr_file"); then
     echo "[ERROR] All AI models failed to generate ideas" >&2
     cat "$stderr_file" >&2
     cd "$original_pwd"