claude-evolve 1.3.39 → 1.3.40

package/bin/claude-evolve-edit

@@ -0,0 +1,293 @@
+#!/bin/bash
+
+set -e
+
+# Load configuration
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+# shellcheck source=../lib/config.sh
+source "$SCRIPT_DIR/../lib/config.sh"
+
+# Use CLAUDE_EVOLVE_CONFIG if set, otherwise default
+if [[ -n ${CLAUDE_EVOLVE_CONFIG:-} ]]; then
+  load_config "$CLAUDE_EVOLVE_CONFIG"
+else
+  load_config
+fi
+
+# Function to show help
+show_help() {
+  cat <<EOF
+claude-evolve edit - Manage evolution candidate statuses by generation or status
+
+USAGE:
+  claude-evolve edit <selector> <action>
+
+SELECTORS:
+  gen01, gen02, etc.  Target specific generation
+  all                 Target all generations
+  failed              Target all candidates with failed status
+  complete            Target all candidates with complete status
+  pending             Target all candidates with pending status
+  running             Target all candidates with running status
+
+ACTIONS:
+  failed    Mark candidates as failed (keeps scores)
+  complete  Mark candidates as complete (keeps scores)  
+  pending   Mark candidates as pending (keeps scores)
+  reboot    Reset completely (delete .py files, clear scores, set pending)
+
+EXAMPLES:
+  claude-evolve edit gen03 failed     # Mark all gen03 as failed
+  claude-evolve edit failed pending   # Reset all failed candidates to pending
+  claude-evolve edit complete failed  # Mark all complete as failed for re-run
+  claude-evolve edit all pending      # Mark everything as pending for re-run
+  claude-evolve edit gen02 reboot     # Full reset of gen02 (delete files + clear data)
+
+DESCRIPTION:
+  This command helps manage evolution runs when you need to re-evaluate candidates.
+  Use status selectors (failed, complete, etc.) to bulk-change candidates by status.
+  Use 'reboot' for complete reset including file deletion.
+EOF
+}
+
+# Parse arguments
+if [[ $# -ne 2 ]]; then
+  show_help
+  exit 1
+fi
+
+SELECTOR="$1"
+ACTION="$2"
+
+# Validate configuration
+if ! validate_config; then
+  echo "[ERROR] Configuration validation failed" >&2
+  exit 1
+fi
+
+# Validate selector format
+if [[ "$SELECTOR" != "all" && ! "$SELECTOR" =~ ^gen[0-9]+$ && "$SELECTOR" != "failed" && "$SELECTOR" != "complete" && "$SELECTOR" != "pending" && "$SELECTOR" != "running" ]]; then
+  echo "[ERROR] Selector must be 'all', 'genXX' (e.g., gen01), or status ('failed', 'complete', 'pending', 'running')" >&2
+  exit 1
+fi
+
+# Validate action
+case "$ACTION" in
+  failed|complete|pending|reboot) ;;
+  *)
+    echo "[ERROR] Action must be one of: failed, complete, pending, reboot" >&2
+    exit 1
+    ;;
+esac
+
+# Check if CSV exists
+if [[ ! -f "$FULL_CSV_PATH" ]]; then
+  echo "[ERROR] Evolution CSV not found: $FULL_CSV_PATH" >&2
+  echo "Run 'claude-evolve setup' first or navigate to the correct directory" >&2
+  exit 1
+fi
+
+# Function to update CSV status for specific selector
+update_candidates_status() {
+  local selector="$1"
+  local new_status="$2"
+  local clear_scores="$3"
+  
+  echo "[INFO] Updating candidates matching '$selector' to status: $new_status"
+  
+  # Use Python to safely edit the CSV
+  "$PYTHON_CMD" -c "
+import csv
+import sys
+import os
+
+csv_file = '$FULL_CSV_PATH'
+selector = '$selector'
+new_status = '$new_status'
+clear_scores = '$clear_scores' == 'true'
+
+try:
+    # Read CSV
+    with open(csv_file, 'r') as f:
+        reader = csv.reader(f)
+        rows = list(reader)
+    
+    if not rows:
+        print('[ERROR] CSV is empty')
+        sys.exit(1)
+    
+    header = rows[0]
+    updated_count = 0
+    
+    # Update matching rows
+    for i in range(1, len(rows)):
+        row = rows[i]
+        if len(row) < 1:
+            continue
+        
+        candidate_id = row[0]
+        current_status = row[4] if len(row) > 4 else ''
+        
+        # Check if this row matches selector
+        matches = False
+        if selector == 'all':
+            matches = True
+        elif selector.startswith('gen') and '-' in candidate_id:
+            # Generation selector (e.g., gen01, gen02)
+            matches = candidate_id.startswith(selector + '-')
+        elif selector in ['failed', 'complete', 'pending', 'running']:
+            # Status selector
+            if selector == 'pending':
+                matches = current_status == '' or current_status == 'pending'
+            else:
+                matches = current_status == selector
+        
+        if matches:
+            if clear_scores:
+                # Reboot: clear everything after description (keep id, basedOnId, description)
+                if len(row) >= 3:
+                    rows[i] = [row[0], row[1], row[2], '', '']  # id, basedOnId, description, empty performance, empty status
+                    updated_count += 1
+            else:
+                # Just update status (preserve other fields)
+                # Ensure row has at least 5 fields
+                while len(row) < 5:
+                    row.append('')
+                row[4] = new_status  # Update status field
+                updated_count += 1
+    
+    # Write back to CSV
+    with open(csv_file + '.tmp', 'w', newline='') as f:
+        writer = csv.writer(f)
+        writer.writerows(rows)
+    
+    # Atomic replace
+    os.rename(csv_file + '.tmp', csv_file)
+    
+    print(f'[INFO] Updated {updated_count} candidates')
+    
+except Exception as e:
+    print(f'[ERROR] Failed to update CSV: {e}')
+    sys.exit(1)
+"
+}
+
+# Function to delete evolution files for selector
+delete_evolution_files() {
+  local selector="$1"
+  
+  if [[ ! -d "$FULL_EVOLUTION_DIR" ]]; then
+    echo "[WARN] Evolution directory not found: $FULL_EVOLUTION_DIR"
+    return
+  fi
+  
+  local deleted_count=0
+  
+  if [[ "$selector" == "all" ]]; then
+    echo "[INFO] Deleting all evolution_*.py files..."
+    for file in "$FULL_EVOLUTION_DIR"/evolution_*.py; do
+      if [[ -f "$file" ]]; then
+        rm "$file"
+        ((deleted_count++))
+        echo "[INFO] Deleted: $(basename "$file")"
+      fi
+    done
+  elif [[ "$selector" =~ ^gen[0-9]+$ ]]; then
+    echo "[INFO] Deleting evolution files for $selector..."
+    for file in "$FULL_EVOLUTION_DIR"/evolution_${selector}-*.py; do
+      if [[ -f "$file" ]]; then
+        rm "$file"
+        ((deleted_count++))
+        echo "[INFO] Deleted: $(basename "$file")"
+      fi
+    done
+  else
+    # Status-based selector - need to query CSV for candidate IDs
+    echo "[INFO] Finding files to delete for status '$selector'..."
+    
+    # Use Python to get list of candidate IDs matching the status
+    local candidates_to_delete
+    candidates_to_delete=$("$PYTHON_CMD" -c "
+import csv
+import sys
+
+csv_file = '$FULL_CSV_PATH'
+selector = '$selector'
+
+try:
+    with open(csv_file, 'r') as f:
+        reader = csv.reader(f)
+        next(reader)  # Skip header
+        
+        candidates = []
+        for row in reader:
+            if len(row) < 1:
+                continue
+            
+            candidate_id = row[0]
+            current_status = row[4] if len(row) > 4 else ''
+            
+            # Check if matches status selector
+            matches = False
+            if selector == 'pending':
+                matches = current_status == '' or current_status == 'pending'
+            else:
+                matches = current_status == selector
+            
+            if matches:
+                candidates.append(candidate_id)
+        
+        print(' '.join(candidates))
+        
+except Exception as e:
+    print('', file=sys.stderr)  # Empty output on error
+    sys.exit(1)
+")
+    
+    if [[ -n "$candidates_to_delete" ]]; then
+      for candidate_id in $candidates_to_delete; do
+        # Determine file format
+        if [[ "$candidate_id" =~ ^[0-9]+$ ]]; then
+          file="$FULL_EVOLUTION_DIR/evolution_id${candidate_id}.py"
+        else
+          file="$FULL_EVOLUTION_DIR/evolution_${candidate_id}.py"
+        fi
+        
+        if [[ -f "$file" ]]; then
+          rm "$file"
+          ((deleted_count++))
+          echo "[INFO] Deleted: $(basename "$file")"
+        fi
+      done
+    fi
+  fi
+  
+  echo "[INFO] Deleted $deleted_count evolution files"
+}
+
+# Main execution
+echo "[INFO] Processing '$SELECTOR' with action: $ACTION"
+
+case "$ACTION" in
+  failed)
+    update_candidates_status "$SELECTOR" "failed" "false"
+    ;;
+  complete)
+    update_candidates_status "$SELECTOR" "complete" "false"
+    ;;
+  pending)
+    update_candidates_status "$SELECTOR" "" "false"  # Empty status means pending
+    ;;
+  reboot)
+    echo "[INFO] Performing full reboot of '$SELECTOR'..."
+    delete_evolution_files "$SELECTOR"
+    update_candidates_status "$SELECTOR" "" "true"  # Clear scores and set pending
+    echo "[INFO] Reboot complete: files deleted, scores cleared, status set to pending"
+    ;;
+esac
+
+echo "[INFO] Edit operation complete"
+
+# Call status command to show current state
+echo ""
+"$SCRIPT_DIR/claude-evolve-status" --brief

package/bin/claude-evolve-ideate

@@ -30,7 +30,7 @@ get_model_for_generation() {
   if (( gen_num % 2 == 1 )); then
     echo "opus"    # Odd generations: use Opus for exploration
   else
-    echo "o3-pro"  # Even generations: use o3-pro for refinement
+    echo "o3"      # Even generations: use o3 for refinement
   fi
 }
 
@@ -46,19 +46,19 @@ call_ai_with_limit_check() {
   echo "[INFO] Generation $generation: Using $preferred_model" >&2
   
   # Try preferred model first
-  if [[ "$preferred_model" == "o3-pro" ]] && command -v codex >/dev/null 2>&1; then
-    echo "[INFO] Using codex o3-pro for ideation" >&2
+  if [[ "$preferred_model" == "o3" ]] && command -v codex >/dev/null 2>&1; then
+    echo "[INFO] Using codex o3 for ideation" >&2
     
-    # Call codex with o3-pro model using -q flag and --full-auto
+    # Call codex with o3 model using -q flag and --full-auto
     local ai_output
-    ai_output=$(codex -m o3-pro --full-auto -q "$prompt" 2>&1)
+    ai_output=$(codex -m o3 --full-auto -q "$prompt" 2>&1)
     local ai_exit_code=$?
     
     if [[ $ai_exit_code -eq 0 ]]; then
       echo "$ai_output"
       return 0
     else
-      echo "[WARN] Codex o3-pro failed, falling back to Claude Opus" >&2
+      echo "[WARN] Codex o3 failed, falling back to Claude Opus" >&2
       preferred_model="opus"
     fi
   fi

package/bin/claude-evolve-main

@@ -58,11 +58,15 @@ USAGE:
     claude-evolve [--config=PATH] [COMMAND] [OPTIONS]
 
 COMMANDS:
-    setup       Initialize evolution workspace
-    ideate      Generate new algorithm ideas
-    run         Execute evolution candidates
-    analyze     Analyze evolution results
-    help        Show this help message
+    setup              Initialize evolution workspace
+    ideate             Generate new algorithm ideas
+    run                Execute evolution candidates
+    analyze            Analyze evolution results
+    edit               Manage candidate statuses by generation
+    status             Show evolution progress and current leader
+    cleanup            Clean up unchanged algorithms and descendants
+    cleanup-duplicates Alias for cleanup (deprecated)
+    help               Show this help message
 
 GLOBAL OPTIONS:
     --config=PATH  Use alternate config file (default: evolution/config.yaml)
@@ -91,9 +95,11 @@ show_menu() {
   echo "  2) ideate   - Generate new algorithm ideas"
   echo "  3) run      - Execute evolution candidates"
   echo "  4) analyze  - Analyze evolution results"
-  echo "  5) config   - Manage configuration settings"
-  echo "  6) help     - Show help message"
-  echo "  7) exit     - Exit"
+  echo "  5) edit     - Manage candidate statuses by generation"
+  echo "  6) status   - Show evolution progress and current leader"
+  echo "  7) config   - Manage configuration settings"
+  echo "  8) help     - Show help message"
+  echo "  9) exit     - Exit"
   echo
 
   # Show workspace status
@@ -137,21 +143,23 @@ check_for_updates
 # Main logic
 if [[ $# -eq 0 ]]; then
   show_menu
-  read -r -p "Enter your choice (1-7): " choice
+  read -r -p "Enter your choice (1-9): " choice
 
   case $choice in
   1) exec "$SCRIPT_DIR/claude-evolve-setup" ;;
   2) exec "$SCRIPT_DIR/claude-evolve-ideate" ;;
-  3) exec "$SCRIPT_DIR/claude-evolve-run-unified" ;;
+  3) exec "$SCRIPT_DIR/claude-evolve-run" ;;
   4) exec "$SCRIPT_DIR/claude-evolve-analyze" ;;
-  5) exec "$SCRIPT_DIR/claude-evolve-config" ;;
-  6) show_help ;;
-  7)
+  5) exec "$SCRIPT_DIR/claude-evolve-edit" ;;
+  6) exec "$SCRIPT_DIR/claude-evolve-status" ;;
+  7) exec "$SCRIPT_DIR/claude-evolve-config" ;;
+  8) show_help ;;
+  9)
     echo "Goodbye!"
     exit 0
     ;;
   *)
-    echo -e "${RED}Invalid choice. Please select 1-7.${NC}"
+    echo -e "${RED}Invalid choice. Please select 1-9.${NC}"
     exit 1
     ;;
   esac
@@ -174,12 +182,24 @@ ideate)
   ;;
 run)
   shift
-  exec "$SCRIPT_DIR/claude-evolve-run-unified" "$@"
+  exec "$SCRIPT_DIR/claude-evolve-run" "$@"
   ;;
 analyze)
   shift
   exec "$SCRIPT_DIR/claude-evolve-analyze" "$@"
   ;;
+edit)
+  shift
+  exec "$SCRIPT_DIR/claude-evolve-edit" "$@"
+  ;;
+status)
+  shift
+  exec "$SCRIPT_DIR/claude-evolve-status" "$@"
+  ;;
+cleanup-duplicates|cleanup)
+  shift
+  exec "$SCRIPT_DIR/claude-evolve-cleanup" "$@"
+  ;;
 config)
   shift
   exec "$SCRIPT_DIR/claude-evolve-config" "$@"

package/bin/{claude-evolve-run-unified → claude-evolve-run}

@@ -236,12 +236,32 @@ cleanup_workers() {
       # Worker finished
       if wait "$pid" 2>/dev/null; then
         echo "[DISPATCHER] Worker $pid completed successfully"
+        consecutive_failures=0  # Reset counter on success
       else
         local exit_code=$?
         if [[ $exit_code -eq 2 ]]; then
           echo "[DISPATCHER] Worker $pid hit rate limit, will retry later"
+          # Rate limits don't count as consecutive failures
         else
           echo "[DISPATCHER] Worker $pid failed with exit code $exit_code"
+          ((consecutive_failures++))
+          
+          # Check if we've hit the failure limit
+          if [[ $consecutive_failures -ge $MAX_CONSECUTIVE_FAILURES ]]; then
+            echo "" >&2
+            echo "🚨🚨🚨 EVOLUTION STOPPED: TOO MANY FAILURES 🚨🚨🚨" >&2
+            echo "ERROR: $consecutive_failures consecutive worker failures detected" >&2
+            echo "ERROR: This indicates a systemic problem (Claude API issues, evaluator bugs, etc.)" >&2
+            echo "ERROR: Check logs and fix issues before restarting evolution" >&2
+            echo "🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨" >&2
+            echo "" >&2
+            
+            # Shutdown all workers and exit
+            shutdown_workers
+            exit 1
+          fi
+          
+          echo "[DISPATCHER] Consecutive failures: $consecutive_failures/$MAX_CONSECUTIVE_FAILURES"
         fi
       fi
     fi
@@ -269,9 +289,26 @@ get_csv_stats() {
   total_rows=$(wc -l < "$csv_path" | tr -d '[:space:]')
   complete_count=$(grep ',complete' "$csv_path" 2>/dev/null | wc -l | tr -d '[:space:]')
   
-  # Count pending: empty status or "pending"
-  # Handle potential Windows line endings by stripping carriage returns
-  pending_count=$(awk -F, 'NR>1 {gsub(/\r/, "", $5); if($5=="" || $5=="pending") count++} END {print count+0}' "$csv_path")
+  # Count pending using same logic as find_next_pending_with_lock
+  # This includes rows with <5 fields AND rows with empty/pending status
+  pending_count=$("$PYTHON_CMD" -c "
+import csv
+import sys
+
+pending_count = 0
+with open('$csv_path', 'r') as f:
+    reader = csv.reader(f)
+    rows = list(reader)
+    
+    for i in range(1, len(rows)):
+        # Same logic as find_next_pending_with_lock
+        if len(rows[i]) < 5:
+            pending_count += 1
+        elif len(rows[i]) >= 5 and (rows[i][4] == 'pending' or rows[i][4] == ''):
+            pending_count += 1
+
+print(pending_count)
+")
   
   echo "$total_rows $complete_count $pending_count"
 }
@@ -279,6 +316,100 @@ get_csv_stats() {
 echo "[DISPATCHER] Starting unified evolution engine"
 echo "[DISPATCHER] Configuration: max_workers=$MAX_WORKERS, timeout=${timeout_seconds:-none}"
 
+# Validate CSV and clean up stuck statuses
+if [[ -f "$FULL_CSV_PATH" ]]; then
+  echo "[DISPATCHER] Validating CSV and cleaning up..."
+  if ! "$PYTHON_CMD" -c "
+import csv
+import sys
+
+csv_file = '$FULL_CSV_PATH'
+
+try:
+    # Read CSV - let Python's csv module handle all the complexity
+    with open(csv_file, 'r') as f:
+        reader = csv.reader(f)
+        rows = list(reader)
+    
+    if not rows:
+        print('[ERROR] CSV is empty')
+        sys.exit(1)
+    
+    # Basic sanity checks
+    header = rows[0]
+    num_fields = len(header)
+    
+    if len(rows) == 1:
+        print('[INFO] CSV has no data rows (only header)')
+    
+    # Clean up any stuck 'running' statuses
+    changed = 0
+    for i in range(1, len(rows)):
+        if len(rows[i]) > 4 and rows[i][4] == 'running':
+            rows[i][4] = ''
+            changed += 1
+    
+    if changed > 0:
+        # Write back
+        with open(csv_file + '.tmp', 'w', newline='') as f:
+            writer = csv.writer(f)
+            writer.writerows(rows)
+        import os
+        os.rename(csv_file + '.tmp', csv_file)
+        print(f'[INFO] Reset {changed} stuck running candidates to pending')
+    
+    # Count pending candidates
+    pending = 0
+    for i in range(1, len(rows)):
+        # Row with < 5 fields or empty/pending status in field 5
+        if len(rows[i]) < 5 or (len(rows[i]) >= 5 and rows[i][4] in ['', 'pending']):
+            pending += 1
+    
+    print(f'[INFO] CSV loaded: {len(rows)-1} total candidates, {pending} pending')
+    
+except csv.Error as e:
+    print(f'[ERROR] CSV parsing error: {e}')
+    print('[ERROR] The CSV file appears to be malformed')
+    sys.exit(1)
+except Exception as e:
+    print(f'[ERROR] Failed to read CSV: {e}')
+    sys.exit(1)
+"; then
+    echo "[ERROR] CSV validation failed. Please check the error message above."
+    exit 1
+  fi
+fi
+
+# Automatic cleanup detection - check for unchanged algorithms and warn user
+echo "[DISPATCHER] Checking for duplicate/unchanged algorithms..."
+cleanup_output=$("$SCRIPT_DIR/claude-evolve-cleanup" --dry-run 2>&1)
+
+# Check if cleanup found any issues (look for "UNCHANGED:" in output)
+if echo "$cleanup_output" | grep -q "📋 UNCHANGED:"; then
+  echo "⚠️  WARNING: Issues detected that may need cleanup:"
+  echo "$cleanup_output"
+  echo ""
+  echo "🔧 RECOMMENDATION: Run 'claude-evolve cleanup --force' to fix these issues before continuing"
+  echo "   This will delete unchanged algorithms and reset their descendants to pending status"
+  echo ""
+  echo "⏰ Continuing in 10 seconds (Ctrl+C to abort and run cleanup)..."
+  
+  # Give user time to read and potentially abort
+  for i in {10..1}; do
+    echo -n "  $i..."
+    sleep 1
+  done
+  echo ""
+  echo "🚀 Proceeding with evolution run..."
+  echo ""
+else
+  echo "[DISPATCHER] No cleanup issues detected - proceeding with run"
+fi
+
+# Consecutive failure tracking
+consecutive_failures=0
+MAX_CONSECUTIVE_FAILURES=5
+
 # Main dispatch loop
 while true; do
   # Clean up finished workers
@@ -337,7 +468,7 @@ while true; do
   done
   
   # Brief pause to avoid busy waiting
-  sleep 2
+  sleep 5
 done
 
 # Clean shutdown