claude-evolve 1.3.44 → 1.4.1

package/bin/claude-evolve-run

@@ -251,10 +251,9 @@ cleanup_workers() {
   worker_pids=("${new_pids[@]}")
 }
 
-# Function to count pending candidates
+# Function to count pending candidates - UNIFIED LOGIC
 count_pending_candidates() {
-  "$PYTHON_CMD" "$SCRIPT_DIR/../lib/csv_helper.py" find_pending "$FULL_CSV_PATH" >/dev/null 2>&1
-  echo $?  # 0 if found, 1 if not found
+  "$PYTHON_CMD" "$SCRIPT_DIR/../lib/evolution_csv.py" "$FULL_CSV_PATH" count
 }
 
 # Function to get CSV stats
@@ -271,26 +270,8 @@ get_csv_stats() {
   total_rows=$(wc -l < "$csv_path" | tr -d '[:space:]')
   complete_count=$(grep ',complete' "$csv_path" 2>/dev/null | wc -l | tr -d '[:space:]')
   
-  # Count pending using same logic as find_next_pending_with_lock
-  # This includes rows with <5 fields AND rows with empty/pending status
-  pending_count=$("$PYTHON_CMD" -c "
-import csv
-import sys
-
-pending_count = 0
-with open('$csv_path', 'r') as f:
-    reader = csv.reader(f)
-    rows = list(reader)
-    
-    for i in range(1, len(rows)):
-        # Same logic as find_next_pending_with_lock
-        if len(rows[i]) < 5:
-            pending_count += 1
-        elif len(rows[i]) >= 5 and (rows[i][4] == 'pending' or rows[i][4] == ''):
-            pending_count += 1
-
-print(pending_count)
-")
+  # Count pending using UNIFIED CSV logic
+  pending_count=$("$PYTHON_CMD" "$SCRIPT_DIR/../lib/evolution_csv.py" "$csv_path" count)
   
   echo "$total_rows $complete_count $pending_count"
 }
@@ -298,9 +279,45 @@ print(pending_count)
 echo "[DISPATCHER] Starting unified evolution engine"
 echo "[DISPATCHER] Configuration: max_workers=$MAX_WORKERS, timeout=${timeout_seconds:-none}"
 
-# Validate CSV and clean up stuck statuses
+# Validate CSV and clean up stuck statuses and duplicates
 if [[ -f "$FULL_CSV_PATH" ]]; then
   echo "[DISPATCHER] Validating CSV and cleaning up..."
+  
+  # First check for and clean up duplicates
+  echo "[DISPATCHER] Checking for duplicate entries..."
+  duplicate_check_output=$("$PYTHON_CMD" "$SCRIPT_DIR/claude-evolve-cleanup-duplicates" "$FULL_CSV_PATH" 2>&1)
+  if echo "$duplicate_check_output" | grep -q "Found.*duplicate"; then
+    echo "[DISPATCHER] WARNING: Duplicate entries detected in CSV!"
+    echo "$duplicate_check_output"
+    echo "[DISPATCHER] Automatically cleaning up duplicates..."
+    if "$PYTHON_CMD" "$SCRIPT_DIR/claude-evolve-cleanup-duplicates" "$FULL_CSV_PATH" --fix; then
+      echo "[DISPATCHER] Duplicates cleaned up successfully"
+    else
+      echo "[ERROR] Failed to clean up duplicates" >&2
+      exit 1
+    fi
+  else
+    echo "[DISPATCHER] No duplicates found"
+  fi
+  
+  # Check for and clean up invalid entries
+  echo "[DISPATCHER] Checking for invalid entries..."
+  invalid_check_output=$("$PYTHON_CMD" "$SCRIPT_DIR/claude-evolve-clean-invalid" "$FULL_CSV_PATH" --dry-run 2>&1)
+  if echo "$invalid_check_output" | grep -q "Found.*invalid"; then
+    echo "[DISPATCHER] WARNING: Invalid entries detected in CSV!"
+    echo "$invalid_check_output"
+    echo "[DISPATCHER] Automatically cleaning up invalid entries..."
+    if "$PYTHON_CMD" "$SCRIPT_DIR/claude-evolve-clean-invalid" "$FULL_CSV_PATH"; then
+      echo "[DISPATCHER] Invalid entries cleaned up successfully"
+    else
+      echo "[ERROR] Failed to clean up invalid entries" >&2
+      exit 1
+    fi
+  else
+    echo "[DISPATCHER] No invalid entries found"
+  fi
+  
+  # Then validate and clean stuck statuses
   if ! "$PYTHON_CMD" -c "
 import csv
 import sys
@@ -340,12 +357,13 @@ try:
         os.rename(csv_file + '.tmp', csv_file)
         print(f'[INFO] Reset {changed} stuck running candidates to pending')
     
-    # Count pending candidates
-    pending = 0
-    for i in range(1, len(rows)):
-        # Row with < 5 fields or empty/pending status in field 5
-        if len(rows[i]) < 5 or (len(rows[i]) >= 5 and rows[i][4] in ['', 'pending']):
-            pending += 1
+    # Count pending candidates using UNIFIED logic
+    import sys
+    sys.path.append('$SCRIPT_DIR/../lib')
+    from evolution_csv import EvolutionCSV
+    
+    with EvolutionCSV(csv_file) as csv_ops:
+        pending = csv_ops.count_pending_candidates()
     
     print(f'[INFO] CSV loaded: {len(rows)-1} total candidates, {pending} pending')
     
@@ -388,6 +406,37 @@ else
   echo "[DISPATCHER] No cleanup issues detected - proceeding with run"
 fi
 
+# Ensure baseline algorithm performance is recorded
+ensure_baseline_entry() {
+  # Check if baseline already exists
+  if "$PYTHON_CMD" -c "
+import csv
+with open('$FULL_CSV_PATH', 'r') as f:
+    reader = csv.reader(f)
+    next(reader, None)  # Skip header
+    for row in reader:
+        if len(row) >= 2:
+            candidate_id = row[0]
+            parent_id = row[1] if len(row) > 1 else ''
+            # Check for baseline entry (empty parent and baseline-like ID)
+            if not parent_id and ('baseline' in candidate_id.lower() or candidate_id.startswith('000') or candidate_id == '0'):
+                print('found')
+                exit(0)
+exit(1)
+"; then
+    echo "[DISPATCHER] Baseline performance already recorded"
+  else
+    echo "[DISPATCHER] No baseline found, adding baseline-000 for evaluation..."
+    
+    # Add baseline entry as pending
+    echo "baseline-000,,Original algorithm.py performance,,pending" >> "$FULL_CSV_PATH"
+    echo "[DISPATCHER] Added baseline-000 to evaluation queue"
+  fi
+}
+
+# Check for baseline before starting main loop
+ensure_baseline_entry
+
 # With retry mechanism, we don't need consecutive failure tracking
 # Failures are handled gracefully through the retry system
 

package/bin/claude-evolve-status

@@ -111,6 +111,7 @@ csv_file = '$FULL_CSV_PATH'
 show_brief = '$SHOW_BRIEF' == 'true'
 show_winner_only = '$SHOW_WINNER_ONLY' == 'true'
 evolution_context = '$EVOLUTION_CONTEXT'
+num_novel_to_show = int('${NUM_REVOLUTION:-2}')
 
 def normalize_status(status):
     '''Convert retry statuses to base status for counting.'''
@@ -183,6 +184,7 @@ try:
     if all_candidates:
         winner = max(all_candidates, key=lambda x: x[2])
     
+    
     # Show winner only
     if show_winner_only:
         if winner:
@@ -218,6 +220,27 @@ try:
         print('🏆 CURRENT LEADER: None (no completed candidates)')
         print()
     
+    # Show top novel candidates
+    novel_candidates = []
+    for row in rows[1:]:
+        if len(row) >= 5 and row[3] and row[4] == 'complete' and not row[1]:
+            try:
+                candidate_id = row[0]
+                description = row[2] if len(row) > 2 else ''
+                score = float(row[3])
+                novel_candidates.append((candidate_id, description, score))
+            except ValueError:
+                pass
+    
+    if novel_candidates:
+        novel_candidates.sort(key=lambda x: x[2], reverse=True)
+        print(f'🌟 TOP NOVEL CANDIDATES:')
+        # Use the num_novel_to_show variable set at the top
+        for i, (candidate_id, description, score) in enumerate(novel_candidates[:num_novel_to_show]):
+            print(f'   {i+1}. {candidate_id} (score: {score:.4f})')
+            print(f'      {description}')
+        print()
+    
     # Show per-generation breakdown (unless brief mode)
     if not show_brief and stats_by_gen:
         print('📈 BY GENERATION:')
@@ -232,7 +255,12 @@ try:
             status_str = f'{data[\"pending\"]}p {data[\"complete\"]}c {data[\"failed\"]}f {data[\"running\"]}r'
             
             if gen_best:
-                print(f'   {gen}: {total} total ({status_str}) - best: {gen_best[0]} ({gen_best[2]:.4f})')
+                # Check if this generation's best is the overall winner
+                if winner and gen_best[0] == winner[0]:
+                    # Highlight in green if it's the overall winner
+                    print(f'   {gen}: {total} total ({status_str}) - best: \033[32m{gen_best[0]} ({gen_best[2]:.4f})\033[0m')
+                else:
+                    print(f'   {gen}: {total} total ({status_str}) - best: {gen_best[0]} ({gen_best[2]:.4f})')
             else:
                 print(f'   {gen}: {total} total ({status_str}) - best: none')