claude-evolve 1.3.43 → 1.4.0

package/bin/claude-evolve-run

@@ -251,10 +251,9 @@ cleanup_workers() {
   worker_pids=("${new_pids[@]}")
 }
 
-# Function to count pending candidates
+# Function to count pending candidates - UNIFIED LOGIC
 count_pending_candidates() {
-  "$PYTHON_CMD" "$SCRIPT_DIR/../lib/csv_helper.py" find_pending "$FULL_CSV_PATH" >/dev/null 2>&1
-  echo $?  # 0 if found, 1 if not found
+  "$PYTHON_CMD" "$SCRIPT_DIR/../lib/evolution_csv.py" "$FULL_CSV_PATH" count
 }
 
 # Function to get CSV stats
@@ -271,26 +270,8 @@ get_csv_stats() {
   total_rows=$(wc -l < "$csv_path" | tr -d '[:space:]')
   complete_count=$(grep ',complete' "$csv_path" 2>/dev/null | wc -l | tr -d '[:space:]')
   
-  # Count pending using same logic as find_next_pending_with_lock
-  # This includes rows with <5 fields AND rows with empty/pending status
-  pending_count=$("$PYTHON_CMD" -c "
-import csv
-import sys
-
-pending_count = 0
-with open('$csv_path', 'r') as f:
-    reader = csv.reader(f)
-    rows = list(reader)
-    
-    for i in range(1, len(rows)):
-        # Same logic as find_next_pending_with_lock
-        if len(rows[i]) < 5:
-            pending_count += 1
-        elif len(rows[i]) >= 5 and (rows[i][4] == 'pending' or rows[i][4] == ''):
-            pending_count += 1
-
-print(pending_count)
-")
+  # Count pending using UNIFIED CSV logic
+  pending_count=$("$PYTHON_CMD" "$SCRIPT_DIR/../lib/evolution_csv.py" "$csv_path" count)
   
   echo "$total_rows $complete_count $pending_count"
 }
@@ -298,9 +279,45 @@ print(pending_count)
 echo "[DISPATCHER] Starting unified evolution engine"
 echo "[DISPATCHER] Configuration: max_workers=$MAX_WORKERS, timeout=${timeout_seconds:-none}"
 
-# Validate CSV and clean up stuck statuses
+# Validate CSV and clean up stuck statuses and duplicates
 if [[ -f "$FULL_CSV_PATH" ]]; then
   echo "[DISPATCHER] Validating CSV and cleaning up..."
+  
+  # First check for and clean up duplicates
+  echo "[DISPATCHER] Checking for duplicate entries..."
+  duplicate_check_output=$("$PYTHON_CMD" "$SCRIPT_DIR/claude-evolve-cleanup-duplicates" "$FULL_CSV_PATH" 2>&1)
+  if echo "$duplicate_check_output" | grep -q "Found.*duplicate"; then
+    echo "[DISPATCHER] WARNING: Duplicate entries detected in CSV!"
+    echo "$duplicate_check_output"
+    echo "[DISPATCHER] Automatically cleaning up duplicates..."
+    if "$PYTHON_CMD" "$SCRIPT_DIR/claude-evolve-cleanup-duplicates" "$FULL_CSV_PATH" --fix; then
+      echo "[DISPATCHER] Duplicates cleaned up successfully"
+    else
+      echo "[ERROR] Failed to clean up duplicates" >&2
+      exit 1
+    fi
+  else
+    echo "[DISPATCHER] No duplicates found"
+  fi
+  
+  # Check for and clean up invalid entries
+  echo "[DISPATCHER] Checking for invalid entries..."
+  invalid_check_output=$("$PYTHON_CMD" "$SCRIPT_DIR/claude-evolve-clean-invalid" "$FULL_CSV_PATH" --dry-run 2>&1)
+  if echo "$invalid_check_output" | grep -q "Found.*invalid"; then
+    echo "[DISPATCHER] WARNING: Invalid entries detected in CSV!"
+    echo "$invalid_check_output"
+    echo "[DISPATCHER] Automatically cleaning up invalid entries..."
+    if "$PYTHON_CMD" "$SCRIPT_DIR/claude-evolve-clean-invalid" "$FULL_CSV_PATH"; then
+      echo "[DISPATCHER] Invalid entries cleaned up successfully"
+    else
+      echo "[ERROR] Failed to clean up invalid entries" >&2
+      exit 1
+    fi
+  else
+    echo "[DISPATCHER] No invalid entries found"
+  fi
+  
+  # Then validate and clean stuck statuses
   if ! "$PYTHON_CMD" -c "
 import csv
 import sys
@@ -340,12 +357,13 @@ try:
         os.rename(csv_file + '.tmp', csv_file)
         print(f'[INFO] Reset {changed} stuck running candidates to pending')
     
-    # Count pending candidates
-    pending = 0
-    for i in range(1, len(rows)):
-        # Row with < 5 fields or empty/pending status in field 5
-        if len(rows[i]) < 5 or (len(rows[i]) >= 5 and rows[i][4] in ['', 'pending']):
-            pending += 1
+    # Count pending candidates using UNIFIED logic
+    import sys
+    sys.path.append('$SCRIPT_DIR/../lib')
+    from evolution_csv import EvolutionCSV
+    
+    with EvolutionCSV(csv_file) as csv_ops:
+        pending = csv_ops.count_pending_candidates()
     
     print(f'[INFO] CSV loaded: {len(rows)-1} total candidates, {pending} pending')
     
@@ -388,6 +406,37 @@ else
   echo "[DISPATCHER] No cleanup issues detected - proceeding with run"
 fi
 
+# Ensure baseline algorithm performance is recorded
+ensure_baseline_entry() {
+  # Check if baseline already exists
+  if "$PYTHON_CMD" -c "
+import csv
+with open('$FULL_CSV_PATH', 'r') as f:
+    reader = csv.reader(f)
+    next(reader, None)  # Skip header
+    for row in reader:
+        if len(row) >= 2:
+            candidate_id = row[0]
+            parent_id = row[1] if len(row) > 1 else ''
+            # Check for baseline entry (empty parent and baseline-like ID)
+            if not parent_id and ('baseline' in candidate_id.lower() or candidate_id.startswith('000') or candidate_id == '0'):
+                print('found')
+                exit(0)
+exit(1)
+"; then
+    echo "[DISPATCHER] Baseline performance already recorded"
+  else
+    echo "[DISPATCHER] No baseline found, adding baseline-000 for evaluation..."
+    
+    # Add baseline entry as pending
+    echo "baseline-000,,Original algorithm.py performance,,pending" >> "$FULL_CSV_PATH"
+    echo "[DISPATCHER] Added baseline-000 to evaluation queue"
+  fi
+}
+
+# Check for baseline before starting main loop
+ensure_baseline_entry
+
 # With retry mechanism, we don't need consecutive failure tracking
 # Failures are handled gracefully through the retry system
 

package/bin/claude-evolve-status

@@ -111,6 +111,7 @@ csv_file = '$FULL_CSV_PATH'
 show_brief = '$SHOW_BRIEF' == 'true'
 show_winner_only = '$SHOW_WINNER_ONLY' == 'true'
 evolution_context = '$EVOLUTION_CONTEXT'
+num_novel_to_show = int('${NUM_REVOLUTION:-2}')
 
 def normalize_status(status):
     '''Convert retry statuses to base status for counting.'''
@@ -183,6 +184,7 @@ try:
     if all_candidates:
         winner = max(all_candidates, key=lambda x: x[2])
     
+    
     # Show winner only
     if show_winner_only:
         if winner:
@@ -218,6 +220,27 @@ try:
         print('🏆 CURRENT LEADER: None (no completed candidates)')
         print()
     
+    # Show top novel candidates
+    novel_candidates = []
+    for row in rows[1:]:
+        if len(row) >= 5 and row[3] and row[4] == 'complete' and not row[1]:
+            try:
+                candidate_id = row[0]
+                description = row[2] if len(row) > 2 else ''
+                score = float(row[3])
+                novel_candidates.append((candidate_id, description, score))
+            except ValueError:
+                pass
+    
+    if novel_candidates:
+        novel_candidates.sort(key=lambda x: x[2], reverse=True)
+        print(f'🌟 TOP NOVEL CANDIDATES:')
+        # Use the num_novel_to_show variable set at the top
+        for i, (candidate_id, description, score) in enumerate(novel_candidates[:num_novel_to_show]):
+            print(f'   {i+1}. {candidate_id} (score: {score:.4f})')
+            print(f'      {description}')
+        print()
+    
     # Show per-generation breakdown (unless brief mode)
     if not show_brief and stats_by_gen:
         print('📈 BY GENERATION:')

package/bin/claude-evolve-worker

@@ -31,26 +31,26 @@ handle_failure() {
     
     if [[ $new_retry_num -le $MAX_RETRIES ]]; then
       local new_status="failed-retry${new_retry_num}"
-      update_csv_row_with_lock "$candidate_id" "status" "$new_status"
-      update_csv_row_with_lock "$candidate_id" "performance" "$performance"
+      "$PYTHON_CMD" "$SCRIPT_DIR/../lib/evolution_csv.py" "$FULL_CSV_PATH" update "$candidate_id" "$new_status"
+      "$PYTHON_CMD" "$SCRIPT_DIR/../lib/evolution_csv.py" "$FULL_CSV_PATH" perf "$candidate_id" "$performance"
       echo "[WORKER-$$] ✗ Retry $retry_num failed, marked as $new_status"
       exit 1
     else
-      update_csv_row_with_lock "$candidate_id" "status" "failed"
-      update_csv_row_with_lock "$candidate_id" "performance" "$performance"
+      "$PYTHON_CMD" "$SCRIPT_DIR/../lib/evolution_csv.py" "$FULL_CSV_PATH" update "$candidate_id" "failed"
+      "$PYTHON_CMD" "$SCRIPT_DIR/../lib/evolution_csv.py" "$FULL_CSV_PATH" perf "$candidate_id" "$performance"
       echo "[WORKER-$$] ✗ Max retries ($MAX_RETRIES) exceeded, marking as permanently failed"
       exit 1
     fi
   elif [[ $current_status == "failed" ]]; then
     # Initial failure, convert to retry1 if retries are enabled
     if [[ $MAX_RETRIES -gt 0 ]]; then
-      update_csv_row_with_lock "$candidate_id" "status" "failed-retry1"
-      update_csv_row_with_lock "$candidate_id" "performance" "$performance"
+      "$PYTHON_CMD" "$SCRIPT_DIR/../lib/evolution_csv.py" "$FULL_CSV_PATH" update "$candidate_id" "failed-retry1"
+      "$PYTHON_CMD" "$SCRIPT_DIR/../lib/evolution_csv.py" "$FULL_CSV_PATH" perf "$candidate_id" "$performance"
       echo "[WORKER-$$] ✗ Initial failure, marked as failed-retry1 for retry"
       exit 1
     else
-      update_csv_row_with_lock "$candidate_id" "status" "failed"
-      update_csv_row_with_lock "$candidate_id" "performance" "$performance"
+      "$PYTHON_CMD" "$SCRIPT_DIR/../lib/evolution_csv.py" "$FULL_CSV_PATH" update "$candidate_id" "failed"
+      "$PYTHON_CMD" "$SCRIPT_DIR/../lib/evolution_csv.py" "$FULL_CSV_PATH" perf "$candidate_id" "$performance"
       echo "[WORKER-$$] ✗ Failed (retries disabled)"
       # Use exit code 1 - systemic issue since retries are disabled
       exit 1
@@ -58,13 +58,13 @@ handle_failure() {
   else
     # Not a failure scenario, convert to retry1 if retries enabled
     if [[ $MAX_RETRIES -gt 0 ]]; then
-      update_csv_row_with_lock "$candidate_id" "status" "failed-retry1"
-      update_csv_row_with_lock "$candidate_id" "performance" "$performance"
+      "$PYTHON_CMD" "$SCRIPT_DIR/../lib/evolution_csv.py" "$FULL_CSV_PATH" update "$candidate_id" "failed-retry1"
+      "$PYTHON_CMD" "$SCRIPT_DIR/../lib/evolution_csv.py" "$FULL_CSV_PATH" perf "$candidate_id" "$performance"
       echo "[WORKER-$$] ✗ Evaluation failed, marked as failed-retry1 for retry"
       exit 1
     else
-      update_csv_row_with_lock "$candidate_id" "status" "failed"
-      update_csv_row_with_lock "$candidate_id" "performance" "$performance"
+      "$PYTHON_CMD" "$SCRIPT_DIR/../lib/evolution_csv.py" "$FULL_CSV_PATH" update "$candidate_id" "failed"
+      "$PYTHON_CMD" "$SCRIPT_DIR/../lib/evolution_csv.py" "$FULL_CSV_PATH" perf "$candidate_id" "$performance"
       echo "[WORKER-$$] ✗ Evaluation failed (retries disabled)"
       exit 1
     fi
@@ -110,9 +110,9 @@ while [[ $# -gt 0 ]]; do
   esac
 done
 
-# If no ID provided, find next pending
+# If no ID provided, find next pending using UNIFIED LOGIC
 if [[ -z $candidate_id ]]; then
-  candidate_result=$(find_next_pending_with_lock)
+  candidate_result=$("$PYTHON_CMD" "$SCRIPT_DIR/../lib/evolution_csv.py" "$FULL_CSV_PATH" next)
   if [[ -z $candidate_result ]]; then
     echo "[DEBUG] No pending candidates found" >&2
     exit 0
@@ -128,8 +128,8 @@ if [[ -z $candidate_id ]]; then
     original_candidate_status=""
   fi
 else
-  # Mark specified candidate as running
-  update_csv_row_with_lock "$candidate_id" "status" "running"
+  # Mark specified candidate as running using UNIFIED LOGIC
+  "$PYTHON_CMD" "$SCRIPT_DIR/../lib/evolution_csv.py" "$FULL_CSV_PATH" update "$candidate_id" "running"
   original_candidate_status=""  # Unknown for manually specified candidates
 fi
 
@@ -334,7 +334,7 @@ The file currently contains the parent algorithm. Modify it according to the des
       echo "[WORKER-$$] Cleaned up temp file due to rate limit" >&2
     fi
     # Reset to pending so it can be retried later
-    update_csv_row_with_lock "$candidate_id" "status" "pending"
+    "$PYTHON_CMD" "$SCRIPT_DIR/../lib/evolution_csv.py" "$FULL_CSV_PATH" update "$candidate_id" "pending"
     exit 2  # Special exit code for rate limit
   fi
   
@@ -391,7 +391,7 @@ if [[ -n $timeout_seconds ]]; then
     eval_exit_code=$?
     if [[ $eval_exit_code -eq 124 ]]; then
       echo "[ERROR] Evaluation timed out" >&2
-      update_csv_row_with_lock "$candidate_id" "status" "timeout"
+      "$PYTHON_CMD" "$SCRIPT_DIR/../lib/evolution_csv.py" "$FULL_CSV_PATH" update "$candidate_id" "timeout"
       exit 1
     fi
   fi
@@ -421,8 +421,8 @@ if [[ $eval_exit_code -eq 0 ]]; then
       handle_failure "$candidate_id" "$retry_status" "$score"
       exit 1
     else
-      update_csv_row_with_lock "$candidate_id" "performance" "$score"
-      update_csv_row_with_lock "$candidate_id" "status" "complete"
+      "$PYTHON_CMD" "$SCRIPT_DIR/../lib/evolution_csv.py" "$FULL_CSV_PATH" perf "$candidate_id" "$score"
+      "$PYTHON_CMD" "$SCRIPT_DIR/../lib/evolution_csv.py" "$FULL_CSV_PATH" update "$candidate_id" "complete"
       echo "[WORKER-$$] ✓ Evaluation complete, score: $score"
       exit 0
     fi
@@ -461,8 +461,8 @@ if [[ $eval_exit_code -eq 0 ]]; then
         handle_failure "$candidate_id" "$retry_status" "$score"
         exit 1
       else
-        update_csv_row_with_lock "$candidate_id" "performance" "$score"
-        update_csv_row_with_lock "$candidate_id" "status" "complete"
+        "$PYTHON_CMD" "$SCRIPT_DIR/../lib/evolution_csv.py" "$FULL_CSV_PATH" perf "$candidate_id" "$score"
+        "$PYTHON_CMD" "$SCRIPT_DIR/../lib/evolution_csv.py" "$FULL_CSV_PATH" update "$candidate_id" "complete"
         echo "[WORKER-$$] ✓ Evaluation complete, score: $score"
         exit 0
       fi
@@ -476,8 +476,8 @@ if [[ $eval_exit_code -eq 0 ]]; then
         handle_failure "$candidate_id" "$retry_status" "$score"
         exit 1
       else
-        update_csv_row_with_lock "$candidate_id" "performance" "$score"
-        update_csv_row_with_lock "$candidate_id" "status" "complete"
+        "$PYTHON_CMD" "$SCRIPT_DIR/../lib/evolution_csv.py" "$FULL_CSV_PATH" perf "$candidate_id" "$score"
+        "$PYTHON_CMD" "$SCRIPT_DIR/../lib/evolution_csv.py" "$FULL_CSV_PATH" update "$candidate_id" "complete"
         echo "[WORKER-$$] ✓ Evaluation complete, score: $score"
         exit 0
       fi

package/lib/config.sh

@@ -36,6 +36,7 @@ DEFAULT_HILL_CLIMBING=5
 DEFAULT_STRUCTURAL_MUTATION=3
 DEFAULT_CROSSOVER_HYBRID=4
 DEFAULT_NUM_ELITES=3
+DEFAULT_NUM_REVOLUTION=2  # Number of top novel candidates to include
 
 # Default parallel execution values
 DEFAULT_PARALLEL_ENABLED=false
@@ -70,6 +71,7 @@ load_config() {
   STRUCTURAL_MUTATION="$DEFAULT_STRUCTURAL_MUTATION"
   CROSSOVER_HYBRID="$DEFAULT_CROSSOVER_HYBRID"
   NUM_ELITES="$DEFAULT_NUM_ELITES"
+  NUM_REVOLUTION="$DEFAULT_NUM_REVOLUTION"
   
   # Set parallel execution defaults
   PARALLEL_ENABLED="$DEFAULT_PARALLEL_ENABLED"
@@ -138,6 +140,7 @@ load_config() {
           structural_mutation) STRUCTURAL_MUTATION="$value" ;;
           crossover_hybrid) CROSSOVER_HYBRID="$value" ;;
           num_elites) NUM_ELITES="$value" ;;
+          num_revolution) NUM_REVOLUTION="$value" ;;
         esac
       elif [[ $in_parallel_section == true ]]; then
         # Handle indented keys in parallel section

package/lib/csv_helper_robust.py

@@ -0,0 +1,121 @@
+#!/usr/bin/env python3
+"""Robust CSV helper for evolution system that handles edge cases properly."""
+
+import csv
+import sys
+
+
+def is_valid_candidate_row(row):
+    """Check if a row represents a valid candidate (not empty, has ID)."""
+    if not row:
+        return False
+    if len(row) == 0:
+        return False
+    # First column should have a non-empty ID
+    if not row[0] or row[0].strip() == '':
+        return False
+    return True
+
+
+def is_pending_candidate(row):
+    """Check if a candidate row is pending (needs processing)."""
+    if not is_valid_candidate_row(row):
+        return False
+    
+    # Must have at least 5 columns to check status
+    if len(row) < 5:
+        return True  # Incomplete row is pending
+    
+    # Check status field (5th column, index 4)
+    status = row[4].strip().lower() if row[4] else ''
+    
+    # Blank, missing, "pending", or "running" all mean pending
+    if not status or status in ['pending', 'running']:
+        return True
+    
+    # Check for retry statuses
+    if status.startswith('failed-retry'):
+        return True
+    
+    return False
+
+
+def get_pending_candidates(csv_file):
+    """Get list of pending candidate IDs from CSV."""
+    pending = []
+    
+    try:
+        with open(csv_file, 'r') as f:
+            reader = csv.reader(f)
+            # Skip header
+            next(reader, None)
+            
+            for row in reader:
+                if is_pending_candidate(row):
+                    candidate_id = row[0].strip()
+                    status = row[4].strip() if len(row) > 4 else ''
+                    pending.append((candidate_id, status))
+    
+    except Exception as e:
+        print(f"Error reading CSV: {e}", file=sys.stderr)
+        return []
+    
+    return pending
+
+
+def update_candidate_status(csv_file, candidate_id, new_status):
+    """Update the status of a specific candidate."""
+    rows = []
+    updated = False
+    
+    try:
+        # Read all rows
+        with open(csv_file, 'r') as f:
+            reader = csv.reader(f)
+            rows = list(reader)
+        
+        # Update the specific candidate
+        for i, row in enumerate(rows):
+            if is_valid_candidate_row(row) and row[0].strip() == candidate_id:
+                # Ensure row has at least 5 columns
+                while len(row) < 5:
+                    row.append('')
+                row[4] = new_status
+                updated = True
+                break
+        
+        # Write back if updated
+        if updated:
+            with open(csv_file, 'w', newline='') as f:
+                writer = csv.writer(f)
+                writer.writerows(rows)
+        
+        return updated
+    
+    except Exception as e:
+        print(f"Error updating CSV: {e}", file=sys.stderr)
+        return False
+
+
+if __name__ == '__main__':
+    # Test functionality
+    if len(sys.argv) < 2:
+        print("Usage: csv_helper_robust.py <csv_file> [command]")
+        sys.exit(1)
+    
+    csv_file = sys.argv[1]
+    command = sys.argv[2] if len(sys.argv) > 2 else 'list'
+    
+    if command == 'list':
+        pending = get_pending_candidates(csv_file)
+        for candidate_id, status in pending:
+            print(f"{candidate_id}|{status}")
+    
+    elif command == 'update' and len(sys.argv) >= 5:
+        candidate_id = sys.argv[3]
+        new_status = sys.argv[4]
+        if update_candidate_status(csv_file, candidate_id, new_status):
+            print(f"Updated {candidate_id} to {new_status}")
+        else:
+            print(f"Failed to update {candidate_id}")
+            sys.exit(1)