claude-evolve 1.4.11 → 1.4.13

package/bin/claude-evolve-ideate

@@ -199,6 +199,98 @@ call_claude_with_limit_check() {
   call_ai_with_limit_check "$@"
 }
 
+# Robust AI calling with fallbacks across all available models
+call_ai_with_fallbacks() {
+  local prompt="$1"
+  local generation="${2:-01}"
+  
+  # List of models to try in order
+  local models=()
+  
+  # Check which AI tools are available
+  if command -v codex >/dev/null 2>&1; then
+    models+=("o3")
+  fi
+  if command -v gemini >/dev/null 2>&1; then
+    models+=("gemini")
+  fi
+  # Claude is always available (fallback)
+  models+=("opus" "sonnet" "haiku")
+  
+  # Try each model in sequence
+  for model in "${models[@]}"; do
+    echo "[INFO] Trying $model for ideation (fallback attempt)" >&2
+    
+    local ai_output
+    local ai_exit_code
+    
+    if [[ "$model" == "o3" ]] && command -v codex >/dev/null 2>&1; then
+      ai_output=$(codex -m o3 --full-auto -q "$prompt" 2>&1)
+      ai_exit_code=$?
+      
+      if [[ $ai_exit_code -eq 0 ]]; then
+        # Clean o3 output like in the original function
+        if echo "$ai_output" | grep -q '"content"'; then
+          ai_output=$(echo "$ai_output" | python3 -c "
+import sys
+import json
+try:
+    data = json.load(sys.stdin)
+    if 'content' in data:
+        print(data['content'])
+    elif 'response' in data:
+        print(data['response'])
+    elif 'text' in data:
+        print(data['text'])
+    else:
+        print(json.dumps(data))
+except:
+    print(sys.stdin.read())
+" 2>/dev/null || echo "$ai_output")
+        fi
+        
+        if [[ -n "$ai_output" ]] && ! echo "$ai_output" | grep -q "error\|failed\|exception"; then
+          echo "$ai_output"
+          return 0
+        fi
+      fi
+      
+    elif [[ "$model" == "gemini" ]] && command -v gemini >/dev/null 2>&1; then
+      ai_output=$(gemini -y -p "$prompt" 2>&1)
+      ai_exit_code=$?
+      
+      if [[ $ai_exit_code -eq 0 ]]; then
+        if ! echo "$ai_output" | grep -q "Attempting to authenticate\|Authenticating\|Loading\|Initializing"; then
+          if [[ -n "$ai_output" ]] && [[ $(echo "$ai_output" | wc -l) -ge 2 ]]; then
+            echo "$ai_output"
+            return 0
+          fi
+        fi
+      fi
+      
+    else
+      # Use Claude with the specified model
+      ai_output=$(echo "$prompt" | claude --dangerously-skip-permissions --model "$model" -p 2>&1)
+      ai_exit_code=$?
+      
+      if [[ $ai_exit_code -eq 0 ]]; then
+        # Check for usage limits
+        if ! echo "$ai_output" | grep -q "Claude AI usage limit reached"; then
+          if ! echo "$ai_output" | grep -E "EOF.*<.*null|<<.*EOF|<.*dev.*null" >/dev/null 2>&1; then
+            echo "$ai_output"
+            return 0
+          fi
+        fi
+      fi
+    fi
+    
+    echo "[WARN] $model failed or returned unusable output, trying next model..." >&2
+  done
+  
+  echo "[ERROR] All AI models failed to generate usable output" >&2
+  return 1
+}
+
 # Parse arguments
 use_strategies=true
 
@@ -297,7 +389,8 @@ print(max_gen)
   printf "%02d" $((max_gen + 1))
 }
 
-# Get next available ID number for current generation
+# This function is no longer used with direct CSV modification approach
+# Keeping for backward compatibility but it's not called anywhere
 get_next_id_number() {
   "$PYTHON_CMD" -c "
 import csv
@@ -316,70 +409,242 @@ print(max_id + 1)
 "
 }
 
-# Process AI output and add ideas to CSV
-process_ai_ideas() {
-  local ai_output="$1"
-  local count="$2"
-  local idea_type="$3"  # novel, hill-climbing, structural, crossover
-  local top_performers="${4:-}"  # Optional, for non-novel ideas
+# Validate that AI directly modified the CSV file
+validate_direct_csv_modification() {
+  local temp_csv="$1"
+  local expected_count="$2"
+  local idea_type="$3"
   
-  local next_id_num=$(get_next_id_number)
-  local ideas_added=0
+  # Check if the file was actually modified
+  if [[ ! -f "$temp_csv" ]]; then
+    echo "[ERROR] CSV file was not found after AI modification" >&2
+    return 1
+  fi
   
-  while IFS= read -r line && [[ $ideas_added -lt $count ]]; do
-    # Skip empty lines
-    [[ -z "$line" || "$line" =~ ^[[:space:]]*$ ]] && continue
-    
-    # Skip lines that look like headers or metadata
-    [[ "$line" =~ ^#|^\[|^==|^-- ]] && continue
-    
-    # Clean the line
-    line=$(echo "$line" | sed 's/^[[:space:]]*//;s/[[:space:]]*$//')
-    line=$(echo "$line" | sed 's/^[0-9]\+\.\?[[:space:]]*//')  # Remove numbering
-    line=$(echo "$line" | sed 's/^-[[:space:]]*//')  # Remove bullet points
-    
-    # Extract parent ID and description based on type
-    local parent_id=""
-    local description="$line"
-    
-    if [[ "$idea_type" != "novel" ]]; then
-      # For non-novel ideas, extract parent ID from "From X:" format
-      if [[ "$line" =~ ^From[[:space:]]+([^:]+):[[:space:]]*(.+)$ ]]; then
-        parent_id="${BASH_REMATCH[1]}"
-        description="${BASH_REMATCH[2]}"
-      else
-        # If no parent specified, use the first from top_performers
-        parent_id=$(echo "$top_performers" | head -1 | cut -d',' -f1)
-      fi
-    fi
-    
-    # Skip if description is too short or contains problematic content
-    [[ ${#description} -lt 20 ]] && continue
-    if echo "$description" | grep -qE 'EOF|/dev/null|<<<|>>>'; then
-      continue
-    fi
-    
-    # Generate ID for this idea
-    local idea_id=$(printf "gen%s-%03d" "$CURRENT_GENERATION" $((next_id_num + ideas_added)))
-    
-    # Escape quotes in description for CSV
-    description="${description//\"/\"\"}"
-    
-    # Append to CSV
-    echo "$idea_id,$parent_id,\"$description\",,pending" >> "$FULL_CSV_PATH"
-    
-    ((ideas_added++))
-    if [[ -n "$parent_id" ]]; then
-      echo "[INFO] Added $idea_type idea $idea_id (parent: $parent_id): ${description:0:50}..."
-    else
-      echo "[INFO] Added $idea_type idea $idea_id: ${description:0:70}..."
-    fi
-  done <<< "$ai_output"
+  # Validate the modified CSV has more entries than original
+  local original_count
+  original_count=$(wc -l < "$FULL_CSV_PATH")
+  local new_count
+  new_count=$(wc -l < "$temp_csv")
+  
+  echo "[DEBUG] Original CSV line count: $original_count" >&2
+  echo "[DEBUG] Modified CSV line count: $new_count" >&2
+  echo "[DEBUG] Expected to add: $expected_count ideas" >&2
+  
+  if [[ $new_count -le $original_count ]]; then
+    echo "[ERROR] CSV file wasn't modified - same number of lines ($new_count <= $original_count)" >&2
+    echo "[DEBUG] First 10 lines of CSV after AI attempt:" >&2
+    head -10 "$temp_csv" >&2
+    return 1
+  fi
+  
+  local added_count=$((new_count - original_count))
+  if [[ $added_count -ne $expected_count ]]; then
+    echo "[WARN] Expected to add $expected_count ideas but added $added_count" >&2
+  fi
+  
+  # Replace original CSV with modified version
+  mv "$temp_csv" "$FULL_CSV_PATH"
+  
+  echo "[INFO] Successfully added $added_count $idea_type ideas to CSV"
+  return 0
+}
+
+# DEPRECATED: Old validation function for CSV output approach
+validate_and_apply_csv_modification_old() {
+  local modified_csv="$1"
+  local temp_csv="$2"
+  local expected_count="$3"
+  local idea_type="$4"
+  
+  # Check if the response looks like an error message (but not if it's just CSV data containing these words)
+  if echo "$modified_csv" | head -1 | grep -q "id,basedOnId,description,performance,status"; then
+    # This looks like a CSV file, not an error message
+    echo "[DEBUG] AI returned what appears to be a CSV file" >&2
+  elif echo "$modified_csv" | grep -qi "error\|failed\|limit\|exceeded\|sorry\|cannot\|unable"; then
+    echo "[ERROR] AI failed to modify CSV and returned an error message:" >&2
+    echo "$modified_csv" | head -200 >&2
+    return 1
+  fi
+  
+  # Check if response is too short to be a valid CSV
+  if [[ ${#modified_csv} -lt 50 ]]; then
+    echo "[ERROR] AI response is too short to be a valid CSV (${#modified_csv} chars):" >&2
+    echo "$modified_csv" >&2
+    return 1
+  fi
+  
+  # Extract CSV from AI output (in case there's extra text before it)
+  local csv_start_line
+  csv_start_line=$(echo "$modified_csv" | grep -n "id,basedOnId,description,performance,status" | head -1 | cut -d: -f1)
+  
+  if [[ -n "$csv_start_line" ]]; then
+    # Extract CSV starting from the header line
+    modified_csv=$(echo "$modified_csv" | tail -n +$csv_start_line)
+    echo "[DEBUG] Found CSV header at line $csv_start_line, extracting from there" >&2
+  elif ! echo "$modified_csv" | head -1 | grep -q "id,basedOnId,description,performance,status"; then
+    echo "[ERROR] AI failed to return a valid CSV file. Expected CSV with header, but got:" >&2
+    echo "$modified_csv" | head -c 500 >&2
+    echo "" >&2
+    echo "[DEBUG] The AI was asked to output the complete modified CSV but didn't" >&2
+    return 1
+  fi
+  
+  # Write the modified CSV to temp file
+  echo "$modified_csv" > "$temp_csv"
+  
+  # Validate the modified CSV has more entries than original
+  local original_count
+  original_count=$(wc -l < "$FULL_CSV_PATH")
+  local new_count
+  new_count=$(wc -l < "$temp_csv")
+  
+  echo "[DEBUG] Original CSV line count: $original_count" >&2
+  echo "[DEBUG] Modified CSV line count: $new_count" >&2
+  echo "[DEBUG] Expected to add: $expected_count ideas" >&2
+  
+  if [[ $new_count -le $original_count ]]; then
+    echo "[ERROR] Modified CSV doesn't have more entries ($new_count <= $original_count)" >&2
+    echo "[DEBUG] First 10 lines of modified CSV:" >&2
+    head -10 "$temp_csv" >&2
+    return 1
+  fi
+  
+  local added_count=$((new_count - original_count))
+  if [[ $added_count -ne $expected_count ]]; then
+    echo "[WARN] Expected to add $expected_count ideas but added $added_count" >&2
+  fi
+  
+  # Replace original CSV with modified version
+  mv "$temp_csv" "$FULL_CSV_PATH"
+  
+  echo "[INFO] Successfully added $added_count $idea_type ideas to CSV"
+  return 0
+}
+
+# DEPRECATED: Old two-step process function - kept for reference
+process_ai_ideas_direct_old() {
+  local count="$1"
+  local idea_type="$2"  # novel, hill-climbing, structural, crossover
+  local top_performers="${3:-}"  # Optional, for non-novel ideas
+  local ai_output="$4"  # The AI's response with ideas
+  
+  # Create temporary CSV copy
+  local temp_csv="/tmp/claude-evolve-temp-csv-$$.csv"
+  cp "$FULL_CSV_PATH" "$temp_csv"
+  
+  echo "[DEBUG] Starting CSV modification for $count $idea_type ideas" >&2
+  echo "[DEBUG] Original CSV path: $FULL_CSV_PATH" >&2
+  echo "[DEBUG] Temp CSV path: $temp_csv" >&2
+  echo "[DEBUG] Original CSV size: $(wc -l < "$FULL_CSV_PATH") lines" >&2
+  
+  # Build prompt for AI to directly modify the CSV
+  local csv_prompt="I need you to add exactly $count new $idea_type ideas to this CSV file.
+
+Here are the $count $idea_type ideas to add:
+$ai_output
+
+Current CSV contents:
+$(cat "$temp_csv")
+
+Instructions:
+1. Add exactly $count new rows to the CSV
+2. Use the next available generation numbers (gen$CURRENT_GENERATION-XXX format)
+3. For each idea, create a row with: id,parent_id,description,,pending
+4. For novel ideas: leave parent_id empty
+5. For other idea types: use appropriate parent IDs from these top performers:
+$top_performers
+
+IMPORTANT: Output the complete modified CSV file. Do not add any explanation or other text - just output the CSV."
+  
+  echo "[INFO] Having AI directly modify CSV with $count $idea_type ideas..."
+  
+  # Get AI to modify the CSV with fallbacks
+  local modified_csv
+  local stderr_file="/tmp/claude-evolve-stderr-$$.txt"
+  if ! modified_csv=$(call_ai_with_fallbacks "$csv_prompt" "$CURRENT_GENERATION" 2>"$stderr_file"); then
+    echo "[ERROR] All AI models failed to modify CSV" >&2
+    cat "$stderr_file" >&2
+    rm -f "$temp_csv" "$stderr_file"
+    return 1
+  fi
+  rm -f "$stderr_file"
+  
+  # Check if the response looks like an error message
+  if echo "$modified_csv" | grep -qi "error\|failed\|limit\|exceeded\|sorry\|cannot\|unable"; then
+    echo "[ERROR] AI failed to modify CSV and returned an error message:" >&2
+    echo "$modified_csv" | head -200 >&2
+    rm -f "$temp_csv" 
+    return 1
+  fi
+  
+  # Check if response is too short to be a valid CSV
+  if [[ ${#modified_csv} -lt 50 ]]; then
+    echo "[ERROR] AI response is too short to be a valid CSV (${#modified_csv} chars):" >&2
+    echo "$modified_csv" >&2
+    rm -f "$temp_csv"
+    return 1
+  fi
+  
+  # Extract CSV from AI output (in case there's extra text before it)
+  local csv_start_line
+  csv_start_line=$(echo "$modified_csv" | grep -n "id,basedOnId,description,performance,status" | head -1 | cut -d: -f1)
+  
+  if [[ -n "$csv_start_line" ]]; then
+    # Extract CSV starting from the header line
+    modified_csv=$(echo "$modified_csv" | tail -n +$csv_start_line)
+    echo "[DEBUG] Found CSV header at line $csv_start_line, extracting from there" >&2
+  elif ! echo "$modified_csv" | head -1 | grep -q "id,basedOnId,description,performance,status"; then
+    echo "[ERROR] AI failed to return a valid CSV file. Expected CSV with header, but got:" >&2
+    echo "$modified_csv" | head -c 500 >&2
+    echo "" >&2
+    echo "[DEBUG] The AI was asked to output the complete modified CSV but didn't" >&2
+    rm -f "$temp_csv"
+    return 1
+  fi
+  
+  # Write the modified CSV to temp file
+  echo "$modified_csv" > "$temp_csv"
   
-  if [[ $ideas_added -lt $count ]]; then
-    echo "[WARN] Only generated $ideas_added out of $count requested $idea_type ideas" >&2
+  # Debug: Show the AI's CSV modification attempt
+  echo "[DEBUG] AI response length: ${#modified_csv} characters" >&2
+  echo "[DEBUG] First 300 chars of AI response:" >&2
+  echo "$modified_csv" | head -c 300 >&2
+  echo "" >&2
+  echo "[DEBUG] Last 300 chars of AI response:" >&2
+  echo "$modified_csv" | tail -c 300 >&2
+  echo "" >&2
+  
+  # Validate the modified CSV has more entries than original
+  local original_count
+  original_count=$(wc -l < "$FULL_CSV_PATH")
+  local new_count
+  new_count=$(wc -l < "$temp_csv")
+  
+  echo "[DEBUG] Original CSV line count: $original_count" >&2
+  echo "[DEBUG] Modified CSV line count: $new_count" >&2
+  echo "[DEBUG] Expected to add: $count ideas" >&2
+  
+  if [[ $new_count -le $original_count ]]; then
+    echo "[ERROR] Modified CSV doesn't have more entries ($new_count <= $original_count)" >&2
+    echo "[DEBUG] Temp CSV contents:" >&2
+    cat "$temp_csv" | head -10 >&2
+    echo "[DEBUG] Original CSV contents:" >&2
+    cat "$FULL_CSV_PATH" | head -10 >&2
+    rm -f "$temp_csv"
+    return 1
   fi
   
+  local added_count=$((new_count - original_count))
+  if [[ $added_count -ne $count ]]; then
+    echo "[WARN] Expected to add $count ideas but added $added_count" >&2
+  fi
+  
+  # Replace original CSV with modified version
+  mv "$temp_csv" "$FULL_CSV_PATH"
+  
+  echo "[INFO] Successfully added $added_count $idea_type ideas to CSV"
+  
   return 0
 }
 
@@ -499,10 +764,10 @@ ideate_ai_strategies() {
   top_performers=$(get_top_performers "$NUM_ELITES")
   
   if [[ -z $top_performers ]]; then
-    echo "[INFO] No completed algorithms found, using pure novel exploration"
-    # Generate all ideas as novel exploration
-    generate_novel_ideas_direct "$TOTAL_IDEAS"
-    return 0
+    echo "[INFO] No completed algorithms found, will use baseline algorithm for hill climbing"
+    # For hill climbing and mutations, use the baseline algorithm
+    # Use a special ID that validation script will recognize
+    top_performers="000,Baseline Algorithm (algorithm.py),0.0"
   fi
 
   echo "[INFO] Generating $TOTAL_IDEAS ideas using multi-strategy approach:"
@@ -518,201 +783,239 @@ ideate_ai_strategies() {
   [[ $CROSSOVER_HYBRID -gt 0 ]] && generate_crossover_direct "$CROSSOVER_HYBRID" "$top_performers"
 }
 
-# Generate novel exploration ideas using structured output
+# Generate novel exploration ideas using direct CSV modification
 generate_novel_ideas_direct() {
   local count="$1"
   
-  # Get next available ID for this generation
-  local next_id_num
-  next_id_num=$("$PYTHON_CMD" -c "
-import csv
-import re
-max_id = 0
-pattern = re.compile(r'^gen$CURRENT_GENERATION-(\d+)$')
-with open('$FULL_CSV_PATH', 'r') as f:
-    reader = csv.reader(f)
-    next(reader, None)  # Skip header
-    for row in reader:
-        if row and len(row) > 0:
-            match = pattern.match(row[0].strip())
-            if match:
-                max_id = max(max_id, int(match.group(1)))
-print(max_id + 1)
-")
+  # Create temporary CSV copy
+  local temp_csv="/tmp/claude-evolve-temp-csv-$$.csv"
+  cp "$FULL_CSV_PATH" "$temp_csv"
+  
+  echo "[INFO] Generating $count novel exploration ideas..."
+  echo "[DEBUG] Original CSV has $(wc -l < "$FULL_CSV_PATH") lines" >&2
   
-  local prompt="Generate exactly $count novel algorithmic ideas for trading algorithm evolution.
+  local prompt="Please add exactly $count novel algorithmic ideas directly to the CSV file $temp_csv.
 
 Current evolution context:
 - Generation: $CURRENT_GENERATION
 - Algorithm: $FULL_ALGORITHM_PATH
 - Brief: $(head -20 "$FULL_BRIEF_PATH")
 
-You MUST output EXACTLY $count lines, where each line is a single sentence describing a specific algorithmic change.
-NO PREAMBLE, NO EXPLANATION, NO NUMBERING - just $count lines of algorithm descriptions.
-
-Requirements:
-- Each line should be one clear sentence describing a novel algorithmic approach
-- Focus on creative, ambitious ideas that haven't been tried yet
-- Consider machine learning, new indicators, regime detection, risk management, etc.
-
-Example output format:
-Train LSTM network on 30-day OHLCV sequences to predict next-day direction probability
-Add cross-correlation filter that reduces positions when correlation with market breaks down
-Implement intraday momentum using 30-minute data to adjust daily position sizes
-
-Output exactly $count lines now:"
-
-  echo "[INFO] Generating $count novel exploration ideas..."
-  
-  # Get AI response
-  local ai_output
-  if ! ai_output=$(call_ai_with_limit_check "$prompt" "$CURRENT_GENERATION" 2>&1); then
-    echo "[WARN] AI failed to generate novel ideas" >&2
+Instructions:
+1. Read the current CSV file to see existing entries
+2. Add exactly $count new rows to the CSV file
+3. Use the next available generation numbers (gen$CURRENT_GENERATION-XXX format)
+4. For each idea, create a row with: id,,description,,pending (empty parent_id for novel ideas)
+5. Each description should be one clear sentence describing a novel algorithmic approach
+6. Focus on creative, ambitious ideas that haven't been tried yet
+7. Consider machine learning, new indicators, regime detection, risk management, etc.
+
+IMPORTANT: Edit the CSV file directly. Do not return or print CSV data - just modify the file in place."
+
+  # Get AI to directly edit the CSV file
+  local ai_response
+  local stderr_file="/tmp/claude-evolve-stderr-$$.txt"
+  if ! ai_response=$(call_ai_with_fallbacks "$prompt" "$CURRENT_GENERATION" 2>"$stderr_file"); then
+    echo "[ERROR] All AI models failed to generate novel ideas" >&2
+    cat "$stderr_file" >&2
+    rm -f "$temp_csv" "$stderr_file"
     return 1
   fi
+  rm -f "$stderr_file"
   
-  # Process the output using helper function
-  process_ai_ideas "$ai_output" "$count" "novel"
+  echo "[DEBUG] AI response: $ai_response" >&2
+  
+  # Validate that the CSV file was actually modified
+  if ! validate_direct_csv_modification "$temp_csv" "$count" "novel"; then
+    rm -f "$temp_csv"
+    return 1
+  fi
   
   echo "[INFO] Novel exploration ideas generated successfully"
+  return 0
 }
 
-# Generate hill climbing ideas by getting descriptions from AI
+# Generate hill climbing ideas using direct CSV modification
 generate_hill_climbing_direct() {
   local count="$1"
   local top_performers="$2"
   
-  # Get next available ID for this generation
-  local next_id_num
-  next_id_num=$("$PYTHON_CMD" -c "
-import csv
-import re
-max_id = 0
-pattern = re.compile(r'^gen$CURRENT_GENERATION-(\d+)$')
-with open('$FULL_CSV_PATH', 'r') as f:
-    reader = csv.reader(f)
-    next(reader, None)  # Skip header
-    for row in reader:
-        if row and len(row) > 0:
-            match = pattern.match(row[0].strip())
-            if match:
-                max_id = max(max_id, int(match.group(1)))
-print(max_id + 1)
-")
+  # Create temporary CSV copy
+  local temp_csv="/tmp/claude-evolve-temp-csv-$$.csv"
+  cp "$FULL_CSV_PATH" "$temp_csv"
   
-  local prompt="Generate exactly $count parameter tuning ideas for successful trading algorithms.
-
-Successful algorithms to tune (pick one as parent):
-$top_performers
-
-You MUST output EXACTLY $count lines, where each line describes a parameter tuning idea.
-NO PREAMBLE, NO EXPLANATION, NO NUMBERING - just $count lines of parameter tuning descriptions.
-
-Each line should:
-- Be one clear sentence about adjusting specific parameters
-- Reference the parent algorithm ID at the beginning like \"From gen02-003:\"
-- Focus on hyperparameters, thresholds, periods, weights, etc.
+  echo "[INFO] Generating $count hill climbing ideas..."
+  echo "[DEBUG] Original CSV has $(wc -l < "$FULL_CSV_PATH") lines" >&2
+  
+  # Extract just the IDs from top performers for clarity
+  local valid_parent_ids
+  valid_parent_ids=$(echo "$top_performers" | cut -d',' -f1 | paste -sd ',' -)
+  
+  local prompt="Please add exactly $count parameter tuning ideas directly to the CSV file $temp_csv.
 
-Example output format:
-From gen02-003: Lower IBS_BUY_THRESHOLD from 0.15 to 0.12 to enter deeper oversold conditions
-From gen02-003: Increase TRS_RSI_PERIOD from 2 to 3 for smoother RSI signals
-From gen02-003: Raise WEIGHT_TDD from 0.38 to 0.42 to emphasize best performing strategy
+IMPORTANT: You MUST use one of these exact parent IDs: $valid_parent_ids
 
-Output exactly $count lines now:"
+Successful algorithms to tune:
+$top_performers
 
-  echo "[INFO] Generating $count hill climbing ideas..."
-  
-  # Get AI response
-  local ai_output
-  if ! ai_output=$(call_ai_with_limit_check "$prompt" "$CURRENT_GENERATION" 2>&1); then
-    echo "[WARN] AI failed to generate hill climbing ideas" >&2
+CRITICAL INSTRUCTION: Before generating parameter tuning ideas, you MUST read the source code of the parent algorithms.
+Algorithm source files are located at: $FULL_OUTPUT_DIR/evolution_<PARENT_ID>.py
+For example: $FULL_OUTPUT_DIR/evolution_gen01-251.py
+
+Instructions:
+1. Read the current CSV file to see existing entries
+2. Add exactly $count new rows to the CSV file
+3. Use the next available generation numbers (gen$CURRENT_GENERATION-XXX format)
+4. For each idea, create a row with: id,parent_id,description,,pending
+5. Each parent_id MUST be one of: $valid_parent_ids
+6. Each description should focus on adjusting specific parameters that exist in the parent's source code
+7. Include current and new parameter values (e.g., \"Lower rsi_entry from 21 to 18\")
+
+IMPORTANT: Edit the CSV file directly. Do not return or print CSV data - just modify the file in place."
+
+  # Get AI to directly edit the CSV file
+  local ai_response
+  local stderr_file="/tmp/claude-evolve-stderr-$$.txt"
+  if ! ai_response=$(call_ai_with_fallbacks "$prompt" "$CURRENT_GENERATION" 2>"$stderr_file"); then
+    echo "[ERROR] All AI models failed to generate hill climbing ideas" >&2
+    cat "$stderr_file" >&2
+    rm -f "$temp_csv" "$stderr_file"
     return 1
   fi
+  rm -f "$stderr_file"
   
-  # Process the output using helper function
-  process_ai_ideas "$ai_output" "$count" "hill-climbing" "$top_performers"
+  echo "[DEBUG] AI response: $ai_response" >&2
+  
+  # Validate that the CSV file was actually modified
+  if ! validate_direct_csv_modification "$temp_csv" "$count" "hill-climbing"; then
+    rm -f "$temp_csv"
+    return 1
+  fi
   
   echo "[INFO] Hill climbing ideas generated successfully"
+  return 0
 }
 
-# Generate structural mutation ideas by getting descriptions from AI
+# Generate structural mutation ideas using direct CSV modification
 generate_structural_mutation_direct() {
   local count="$1"
   local top_performers="$2"
   
-  local prompt="Generate exactly $count structural modification ideas for successful trading algorithms.
+  # Create temporary CSV copy
+  local temp_csv="/tmp/claude-evolve-temp-csv-$$.csv"
+  cp "$FULL_CSV_PATH" "$temp_csv"
+  
+  echo "[INFO] Generating $count structural mutation ideas..."
+  echo "[DEBUG] Original CSV has $(wc -l < "$FULL_CSV_PATH") lines" >&2
+  
+  # Extract just the IDs from top performers for clarity
+  local valid_parent_ids
+  valid_parent_ids=$(echo "$top_performers" | cut -d',' -f1 | paste -sd ',' -)
+  
+  local prompt="Please add exactly $count structural modification ideas directly to the CSV file $temp_csv.
+
+IMPORTANT: You MUST use one of these exact parent IDs: $valid_parent_ids
 
 Successful algorithms to modify structurally:
 $top_performers
 
-You MUST output EXACTLY $count lines, where each line describes a structural modification idea.
-NO PREAMBLE, NO EXPLANATION, NO NUMBERING - just $count lines of structural change descriptions.
-
-Each line should:
-- Be one clear sentence about architectural/structural changes
-- Reference the parent algorithm ID at the beginning like \"From gen02-003:\"
-- Focus on replacing components, changing architecture, adding new systems
-
-Example output format:
-From gen02-003: Replace 2-period RSI with LSTM-predicted momentum scores for TRS strategy
-From gen02-003: Add ensemble voting system where sub-strategies vote on market regime
-From gen02-003: Implement hierarchical risk budgeting with correlation-adjusted position sizing
-
-Output exactly $count lines now:"
-
-  echo "[INFO] Generating $count structural mutation ideas..."
-  
-  # Get AI response
-  local ai_output
-  if ! ai_output=$(call_ai_with_limit_check "$prompt" "$CURRENT_GENERATION" 2>&1); then
-    echo "[WARN] AI failed to generate structural mutation ideas" >&2
+CRITICAL INSTRUCTION: Before generating structural modification ideas, you MUST read the source code of the parent algorithms.
+Algorithm source files are located at: $FULL_OUTPUT_DIR/evolution_<PARENT_ID>.py
+For example: $FULL_OUTPUT_DIR/evolution_gen01-251.py
+
+Instructions:
+1. Read the current CSV file to see existing entries
+2. Add exactly $count new rows to the CSV file
+3. Use the next available generation numbers (gen$CURRENT_GENERATION-XXX format)
+4. For each idea, create a row with: id,parent_id,description,,pending
+5. Each parent_id MUST be one of: $valid_parent_ids
+6. Each description should focus on architectural/structural changes based on the parent's actual code
+7. Reference actual components/methods found in the source code
+
+IMPORTANT: Edit the CSV file directly. Do not return or print CSV data - just modify the file in place."
+
+  # Get AI to directly edit the CSV file
+  local ai_response
+  local stderr_file="/tmp/claude-evolve-stderr-$$.txt"
+  if ! ai_response=$(call_ai_with_fallbacks "$prompt" "$CURRENT_GENERATION" 2>"$stderr_file"); then
+    echo "[ERROR] All AI models failed to generate structural mutation ideas" >&2
+    cat "$stderr_file" >&2
+    rm -f "$temp_csv" "$stderr_file"
     return 1
   fi
+  rm -f "$stderr_file"
+  
+  echo "[DEBUG] AI response: $ai_response" >&2
   
-  # Process the output using helper function
-  process_ai_ideas "$ai_output" "$count" "structural" "$top_performers"
+  # Validate that the CSV file was actually modified
+  if ! validate_direct_csv_modification "$temp_csv" "$count" "structural"; then
+    rm -f "$temp_csv"
+    return 1
+  fi
   
   echo "[INFO] Structural mutation ideas generated successfully"
+  return 0
 }
 
-# Generate crossover hybrid ideas by getting descriptions from AI
+# Generate crossover hybrid ideas using direct CSV modification
 generate_crossover_direct() {
   local count="$1"
   local top_performers="$2"
   
-  local prompt="Generate exactly $count hybrid combination ideas from successful trading algorithms.
+  # Create temporary CSV copy
+  local temp_csv="/tmp/claude-evolve-temp-csv-$$.csv"
+  cp "$FULL_CSV_PATH" "$temp_csv"
+  
+  echo "[INFO] Generating $count crossover hybrid ideas..."
+  echo "[DEBUG] Original CSV has $(wc -l < "$FULL_CSV_PATH") lines" >&2
+  
+  # Extract just the IDs from top performers for clarity
+  local valid_parent_ids
+  valid_parent_ids=$(echo "$top_performers" | cut -d',' -f1 | paste -sd ',' -)
+  
+  local prompt="Please add exactly $count hybrid combination ideas directly to the CSV file $temp_csv.
+
+IMPORTANT: You MUST use ONLY these exact parent IDs: $valid_parent_ids
 
 Top performers to combine (reference at least 2 in each idea):
 $top_performers
 
-You MUST output EXACTLY $count lines, where each line describes a hybrid combination idea.
-NO PREAMBLE, NO EXPLANATION, NO NUMBERING - just $count lines of hybrid combination descriptions.
-
-Each line should:
-- Be one clear sentence combining elements from 2+ successful algorithms
-- Reference the base parent algorithm ID at the beginning like \"From gen02-003:\"
-- Explicitly mention which elements to combine from which algorithms
-
-Example output format:
-From gen02-003: Combine VIX regime filter from gen02-003 with LSTM predictions from gen01-005
-From gen02-003: Merge volatility regime detection from gen02-003 with ML momentum from gen01-007
-From gen02-003: Integrate multi-timeframe signals from gen02-003 with correlation sizing from gen01-009
-
-Output exactly $count lines now:"
-
-  echo "[INFO] Generating $count crossover hybrid ideas..."
-  
-  # Get AI response
-  local ai_output
-  if ! ai_output=$(call_ai_with_limit_check "$prompt" "$CURRENT_GENERATION" 2>&1); then
-    echo "[WARN] AI failed to generate crossover ideas" >&2
+CRITICAL INSTRUCTION: Before generating hybrid combination ideas, you MUST read the source code of the parent algorithms.
+Algorithm source files are located at: $FULL_OUTPUT_DIR/evolution_<PARENT_ID>.py
+For example: $FULL_OUTPUT_DIR/evolution_gen01-251.py
+
+Instructions:
+1. Read the current CSV file to see existing entries
+2. Add exactly $count new rows to the CSV file
+3. Use the next available generation numbers (gen$CURRENT_GENERATION-XXX format)
+4. For each idea, create a row with: id,parent_id,description,,pending
+5. Each parent_id MUST be one of: $valid_parent_ids (choose the primary parent)
+6. Each description should combine actual elements from 2+ algorithms based on their source code
+7. Reference specific components/features found in the actual source code
+
+IMPORTANT: Edit the CSV file directly. Do not return or print CSV data - just modify the file in place."
+
+  # Get AI to directly edit the CSV file
+  local ai_response
+  local stderr_file="/tmp/claude-evolve-stderr-$$.txt"
+  if ! ai_response=$(call_ai_with_fallbacks "$prompt" "$CURRENT_GENERATION" 2>"$stderr_file"); then
+    echo "[ERROR] All AI models failed to generate crossover hybrid ideas" >&2
+    cat "$stderr_file" >&2
+    rm -f "$temp_csv" "$stderr_file"
     return 1
   fi
+  rm -f "$stderr_file"
   
-  # Process the output using helper function
-  process_ai_ideas "$ai_output" "$count" "crossover" "$top_performers"
+  echo "[DEBUG] AI response: $ai_response" >&2
+  
+  # Validate that the CSV file was actually modified
+  if ! validate_direct_csv_modification "$temp_csv" "$count" "crossover"; then
+    rm -f "$temp_csv"
+    return 1
+  fi
   
   echo "[INFO] Crossover hybrid ideas generated successfully"
+  return 0
 }
 
 # Legacy AI generation mode (for backward compatibility)
@@ -722,6 +1025,13 @@ ideate_ai_legacy() {
     exit 1
   fi
 
+  # Create temporary CSV copy
+  local temp_csv="/tmp/claude-evolve-temp-csv-$$.csv"
+  cp "$FULL_CSV_PATH" "$temp_csv"
+  
+  echo "[INFO] Generating $TOTAL_IDEAS ideas (legacy mode)..."
+  echo "[DEBUG] Original CSV has $(wc -l < "$FULL_CSV_PATH") lines" >&2
+
   # Get top performers for context
   local top_performers=""
   if [[ -f "$FULL_CSV_PATH" ]]; then
@@ -729,8 +1039,8 @@ ideate_ai_legacy() {
     top_performers=$(awk -F, 'NR > 1 && $4 != "" { print $1 ": " $3 " (score: " $4 ")" }' "$FULL_CSV_PATH" | head -5)
   fi
 
-  # Build prompt for description-only output
-  local prompt="Generate exactly $TOTAL_IDEAS novel algorithmic ideas for trading algorithm evolution.
+  # Build prompt for direct CSV modification
+  local prompt="Please add exactly $TOTAL_IDEAS algorithmic ideas directly to the CSV file $temp_csv.
 
 Algorithm files for context:
 - Base algorithm: $FULL_ALGORITHM_PATH  
@@ -753,13 +1063,13 @@ $top_performers"
 
   prompt+="
 
-You MUST output EXACTLY $TOTAL_IDEAS lines, where each line is a single sentence describing a specific algorithmic change.
-NO PREAMBLE, NO EXPLANATION, NO NUMBERING - just $TOTAL_IDEAS lines of algorithm descriptions.
-
-Requirements:
-- Each line should be one clear sentence describing an algorithmic approach
-- Mix both parameter tuning and structural changes
-- If building on existing algorithms, start with 'From ALGORITHM_ID:'
+Instructions:
+1. Read the current CSV file to see existing entries
+2. Add exactly $TOTAL_IDEAS new rows to the CSV file
+3. Use the next available generation numbers (gen$CURRENT_GENERATION-XXX format)
+4. For each idea, create a row with: id,parent_id,description,,pending
+5. Mix both parameter tuning and structural changes
+6. If building on existing algorithms, use their ID as parent_id, otherwise leave parent_id empty
 
 ⚠️ AVOID ONLY: Kelly floor/cap adjustments that assume leverage > 1.0 (these get clamped and have no effect)
 
@@ -773,22 +1083,29 @@ Requirements:
 - Time-Based Patterns: Intraday effects, calendar anomalies, volatility timing
 - Parameter Optimization: Entry thresholds, indicator periods, strategy weights
 
-Output exactly $TOTAL_IDEAS lines now:"
+IMPORTANT: Edit the CSV file directly. Do not return or print CSV data - just modify the file in place."
 
-  echo "[INFO] Generating $TOTAL_IDEAS ideas (legacy mode)..."
-  
-  # Get AI response
-  local ai_output
-  if ! ai_output=$(call_ai_with_limit_check "$prompt" "$CURRENT_GENERATION" 2>&1); then
-    echo "[WARN] AI failed to generate ideas" >&2
+  # Get AI to directly edit the CSV file
+  local ai_response
+  local stderr_file="/tmp/claude-evolve-stderr-$$.txt"
+  if ! ai_response=$(call_ai_with_fallbacks "$prompt" "$CURRENT_GENERATION" 2>"$stderr_file"); then
+    echo "[ERROR] All AI models failed to generate ideas" >&2
+    cat "$stderr_file" >&2
+    rm -f "$temp_csv" "$stderr_file"
     return 1
   fi
+  rm -f "$stderr_file"
+  
+  echo "[DEBUG] AI response: $ai_response" >&2
   
-  # Process the output using helper function
-  # For legacy mode, we treat all as "novel" since we're not specifying a strategy
-  process_ai_ideas "$ai_output" "$TOTAL_IDEAS" "novel"
+  # Validate that the CSV file was actually modified
+  if ! validate_direct_csv_modification "$temp_csv" "$TOTAL_IDEAS" "mixed"; then
+    rm -f "$temp_csv"
+    return 1
+  fi
   
   echo "[INFO] Legacy ideas generated"
+  return 0
 }
 
 # Determine generation number for this ideation run