claude-evolve 1.5.2 → 1.5.4

package/README.md

@@ -30,14 +30,29 @@ Evolution runs indefinitely until you stop it. Perfect for overnight optimizatio
 
 ## Commands
 
+### Core Commands
 ```bash
 claude-evolve           # Interactive menu
 claude-evolve setup     # Initialize workspace
 claude-evolve ideate    # Generate new algorithm ideas
 claude-evolve run       # Start evolution loop (runs forever)
-claude-evolve analyze   # View results and progress
+claude-evolve analyze   # View results and progress with charts
 claude-evolve status    # Quick progress overview
+claude-evolve autostatus # Live auto-updating status display
+```
+
+### Management Commands
+```bash
 claude-evolve edit      # Manage candidate statuses
+claude-evolve config    # View/edit configuration
+claude-evolve cleanup   # Clean up old lock files
+```
+
+### Maintenance Commands
+```bash
+claude-evolve clean-invalid      # Remove invalid candidates
+claude-evolve cleanup-duplicates # Remove duplicate entries
+claude-evolve csv-fix           # Fix CSV formatting issues
 ```
 
 ## Working with Multiple Projects

package/bin/claude-evolve-analyze

@@ -246,7 +246,8 @@ with open('$csv_file', 'r') as f:
 "
 
 # Process generation stats
-for gen in $(cut -d' ' -f1 "$gen_stats_file" | sort -u || echo ""); do
+# Sort generations numerically
+for gen in $(cut -d' ' -f1 "$gen_stats_file" | sort -u | awk '{print substr($0,4), $0}' | sort -n | cut -d' ' -f2 || echo ""); do
   [[ -z "$gen" ]] && continue
   total_in_gen=$(grep -c "^$gen " "$gen_stats_file" 2>/dev/null || echo "0")
   completed_in_gen=$(grep -c "^$gen completed" "$gen_stats_file" 2>/dev/null || echo "0")
@@ -461,7 +462,8 @@ print(f'max_desc=\"{desc_escaped}\"')
   # Create generation averages file and track max generation
   gen_index=1
   max_gen_num=0
-  for gen in $(cut -d' ' -f1 "$gen_data_temp" | sort -u); do
+  # Sort generations numerically for graph  
+  for gen in $(cut -d' ' -f1 "$gen_data_temp" | sort -u | awk '{print substr($0,4), $0}' | sort -n | cut -d' ' -f2); do
     if grep -q "^$gen " "$gen_data_temp"; then
       # Calculate median for this generation
       # AIDEV-NOTE: Changed from mean to median to be more robust to outliers
@@ -511,6 +513,9 @@ print(f'max_desc=\"{desc_escaped}\"')
     # Calculate total data points for dynamic sizing
     total_data_points=$(awk 'END {print NR-1}' "$data_file")  # Subtract header row
     
+    # Count unique generations
+    unique_generations=$(awk '{if(NR>1) print $4}' "$data_file" | sort -nu | wc -l)
+    
     # AIDEV-NOTE: Dynamic dot sizing based on data point count
     # Use significantly larger dots when there are fewer data points for better visibility
     if [[ $total_data_points -lt 35 ]]; then
@@ -528,16 +533,24 @@ print(f'max_desc=\"{desc_escaped}\"')
     # Find all generations that have data
     generations=($(awk '{if(NR>1) print $4}' "$data_file" | sort -n | uniq))
     
-    for gen_num in "${generations[@]}"; do
-      if [[ -n $gen_num ]]; then
-        color=$(get_gen_color "$gen_num")
-        if [[ $gen_plots_added -gt 0 ]]; then
-          plot_cmd="$plot_cmd, \\"$'\n'
+    # If too many generations (>10), use a simplified plot without individual generation legends
+    if [[ $unique_generations -gt 10 ]]; then
+      # Single plot with color gradient based on generation number
+      plot_cmd="\"$data_file\" using 1:3:(\$4) with points palette pointsize $regular_dot_size notitle"
+      gen_plots_added=1
+    else
+      # Original plotting with individual generation legends
+      for gen_num in "${generations[@]}"; do
+        if [[ -n $gen_num ]]; then
+          color=$(get_gen_color "$gen_num")
+          if [[ $gen_plots_added -gt 0 ]]; then
+            plot_cmd="$plot_cmd, \\"$'\n'
+          fi
+          plot_cmd="${plot_cmd}     \"$data_file\" using (\$4==$gen_num?\$1:1/0):3 with points linecolor rgb \"$color\" pointsize $regular_dot_size title \"Gen $gen_num\""
+          ((gen_plots_added++))
         fi
-        plot_cmd="${plot_cmd}     \"$data_file\" using (\$4==$gen_num?\$1:1/0):3 with points linecolor rgb \"$color\" pointsize $regular_dot_size title \"Gen $gen_num\""
-        ((gen_plots_added++))
-      fi
-    done
+      done
+    fi
     
     # Add novel candidates
     if [[ -s "$novel_file" ]] && [[ $(wc -l < "$novel_file") -gt 1 ]]; then
@@ -568,7 +581,8 @@ print(f'max_desc=\"{desc_escaped}\"')
     # Build x-axis labels for generation chart (include all generations from data)
     xtics_labels=""
     label_index=1
-    for gen in $(cut -d' ' -f1 "$gen_data_temp" | sort -u); do
+    # Sort generations numerically for graph  
+  for gen in $(cut -d' ' -f1 "$gen_data_temp" | sort -u | awk '{print substr($0,4), $0}' | sort -n | cut -d' ' -f2); do
       if [[ -n $gen ]]; then
         gen_display=$(echo "$gen" | sed 's/gen0*//')
         if [[ -n $xtics_labels ]]; then
@@ -583,6 +597,9 @@ print(f'max_desc=\"{desc_escaped}\"')
 set terminal png size 1200,800
 set output "$output_file"
 
+# Define unique generations count
+unique_gens = $unique_generations
+
 # Set up multiplot with proper spacing
 set multiplot layout 2,1 margins 0.08,0.82,0.15,0.95 spacing 0.1,0.15
 
@@ -592,7 +609,9 @@ set title "$EVOLUTION_CONTEXT Algorithm Evolution Performance Over Time" font ",
 unset xlabel
 set ylabel "Performance Score"
 set grid y  # Only show horizontal grid lines
-set key outside right
+
+# Show legend only if 10 or fewer generations
+if (unique_gens <= 10) set key outside right; else unset key
 
 # AIDEV-NOTE: Remove x-axis entirely to avoid tick problems with large datasets
 unset xtics
@@ -600,6 +619,9 @@ set autoscale
 set yrange [*:*]  # Auto-scale y-axis only
 
 # Define colors for generations
+# Use palette for many generations
+if (unique_gens > 10) set palette model RGB defined (0 "#1f77b4", 1 "#ff7f0e", 2 "#2ca02c", 3 "#d62728", 4 "#9467bd", 5 "#8c564b", 6 "#e377c2")
+
 plot $plot_cmd
 
 #=================== BOTTOM PLOT: Generation Medians ===================
@@ -611,8 +633,13 @@ set boxwidth 0.6
 unset key
 set grid y
 
-# Set custom x-axis labels
-set xtics ($xtics_labels)
+# Set custom x-axis labels (but hide if too many generations)
+if (unique_gens > 10) {
+    set xtics auto
+    set xtics rotate by -45
+} else {
+    set xtics ($xtics_labels)
+}
 
 # Auto-scale for generation plot too
 set autoscale

package/bin/claude-evolve-autostatus

@@ -246,8 +246,8 @@ class AutoStatus:
             print("-" * min(self.display.cols, len(header_fmt)))
             row += 1
             
-            # Sort generations
-            sorted_gens = sorted(generations.keys())
+            # Sort generations numerically by extracting the number after 'gen'
+            sorted_gens = sorted(generations.keys(), key=lambda g: int(g[3:]) if g.startswith("gen") and g[3:].isdigit() else 0)
             
             # Calculate how many generations we can show
             available_rows = self.display.rows - row - 1  # Leave room at bottom

package/bin/claude-evolve-edit

@@ -20,7 +20,10 @@ show_help() {
 claude-evolve edit - Manage evolution candidate statuses by generation or status
 
 USAGE:
-  claude-evolve edit <selector> <action>
+  claude-evolve edit [--recent-generations=N] <selector> <action>
+
+OPTIONS:
+  --recent-generations=N  Limit operations to the most recent N generations only
 
 SELECTORS:
   gen01, gen02, etc.  Target specific generation
@@ -41,29 +44,56 @@ ACTIONS:
   delete           Delete candidates from CSV and remove .py files (asks confirmation)
 
 EXAMPLES:
-  claude-evolve edit gen03 failed        # Mark all gen03 as failed
-  claude-evolve edit failed pending      # Reset all failed candidates to pending
-  claude-evolve edit failed failed-retry1 # Convert failed to retry status (bug fixing)
-  claude-evolve edit complete failed     # Mark all complete as failed for re-run
-  claude-evolve edit all pending         # Mark everything as pending for re-run
-  claude-evolve edit gen02 reboot        # Full reset of gen02 (delete files + clear data)
-  claude-evolve edit gen02 delete        # Delete gen02 from CSV and remove .py files
+  claude-evolve edit gen03 failed                    # Mark all gen03 as failed
+  claude-evolve edit failed pending                  # Reset all failed candidates to pending
+  claude-evolve edit --recent-generations=15 failed pending  # Reset only recent 15 gen failures
+  claude-evolve edit --recent-generations=5 complete failed  # Re-run recent 5 gen completions
+  claude-evolve edit failed failed-retry1            # Convert failed to retry status (bug fixing)
+  claude-evolve edit complete failed                 # Mark all complete as failed for re-run
+  claude-evolve edit all pending                     # Mark everything as pending for re-run
+  claude-evolve edit gen02 reboot                    # Full reset of gen02 (delete files + clear data)
+  claude-evolve edit gen02 delete                    # Delete gen02 from CSV and remove .py files
 
 DESCRIPTION:
   This command helps manage evolution runs when you need to re-evaluate candidates.
   Use status selectors (failed, complete, etc.) to bulk-change candidates by status.
   Use 'reboot' for complete reset including file deletion.
+  Use --recent-generations to limit operations to recent work only, useful for large systems.
 EOF
 }
 
 # Parse arguments
-if [[ $# -ne 2 ]]; then
+recent_generations=""
+args=()
+
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    --recent-generations=*)
+      recent_generations="${1#*=}"
+      if [[ ! "$recent_generations" =~ ^[1-9][0-9]*$ ]]; then
+        echo "[ERROR] --recent-generations must be a positive integer" >&2
+        exit 1
+      fi
+      shift
+      ;;
+    --help|-h)
+      show_help
+      exit 0
+      ;;
+    *)
+      args+=("$1")
+      shift
+      ;;
+  esac
+done
+
+if [[ ${#args[@]} -ne 2 ]]; then
   show_help
   exit 1
 fi
 
-SELECTOR="$1"
-ACTION="$2"
+SELECTOR="${args[0]}"
+ACTION="${args[1]}"
 
 # Validate configuration
 if ! validate_config; then
@@ -99,7 +129,11 @@ update_candidates_status() {
   local new_status="$2"
   local clear_scores="$3"
   
-  echo "[INFO] Updating candidates matching '$selector' to status: $new_status"
+  local filter_msg=""
+  if [[ -n "$recent_generations" ]]; then
+    filter_msg=" (limited to recent $recent_generations generations)"
+  fi
+  echo "[INFO] Updating candidates matching '$selector' to status: $new_status${filter_msg}"
   
   # Use Python to safely edit the CSV
   "$PYTHON_CMD" -c "
@@ -112,6 +146,7 @@ csv_file = '$FULL_CSV_PATH'
 selector = '$selector'
 new_status = '$new_status'
 clear_scores = '$clear_scores' == 'true'
+recent_generations = '$recent_generations'
 
 
 try:
@@ -127,6 +162,31 @@ try:
     header = rows[0]
     updated_count = 0
     
+    # If recent_generations is specified, determine which generations to include
+    recent_gen_set = set()
+    if recent_generations and recent_generations.isdigit():
+        n_recent = int(recent_generations)
+        
+        # Find all generation numbers from candidate IDs
+        all_generations = set()
+        for i in range(1, len(rows)):
+            row = rows[i]
+            if len(row) < 1:
+                continue
+            candidate_id = row[0]
+            
+            # Extract generation number from candidate_id (e.g., gen01-001 -> 1)
+            match = re.match(r'^gen(\d+)-', candidate_id)
+            if match:
+                gen_num = int(match.group(1))
+                all_generations.add(gen_num)
+        
+        # Get the most recent N generations
+        if all_generations:
+            sorted_generations = sorted(all_generations, reverse=True)
+            recent_gen_set = set(sorted_generations[:n_recent])
+            print(f'[INFO] Filtering to recent generations: {sorted(recent_gen_set)}', file=sys.stderr)
+    
     # Update matching rows
     for i in range(1, len(rows)):
         row = rows[i]
@@ -152,6 +212,18 @@ try:
             else:
                 matches = current_status == selector
         
+        # Apply recent generations filter if specified
+        if matches and recent_gen_set:
+            # Extract generation number from candidate_id
+            gen_match = re.match(r'^gen(\d+)-', candidate_id)
+            if gen_match:
+                candidate_gen = int(gen_match.group(1))
+                if candidate_gen not in recent_gen_set:
+                    matches = False  # Filter out this candidate
+            else:
+                # Non-generation candidates (like baseline) - exclude when filtering by recent generations
+                matches = False
+        
         if matches:
             if clear_scores:
                 # Clear everything after description (keep id, basedOnId, description)
@@ -192,6 +264,12 @@ delete_evolution_files() {
     return
   fi
   
+  local filter_msg=""
+  if [[ -n "$recent_generations" ]]; then
+    filter_msg=" (limited to recent $recent_generations generations)"
+  fi
+  echo "[INFO] Deleting evolution files for '$selector'${filter_msg}..."
+  
   local deleted_count=0
   
   if [[ "$selector" == "all" ]]; then
@@ -225,15 +303,46 @@ import re
 
 csv_file = '$FULL_CSV_PATH'
 selector = '$selector'
+recent_generations = '$recent_generations'
 
 
 try:
     with open(csv_file, 'r') as f:
         reader = csv.reader(f)
-        next(reader)  # Skip header
+        rows = list(reader)
+        
+        if not rows:
+            print('')
+            sys.exit(0)
+        
+        # Skip header if present
+        start_idx = 1 if rows and rows[0] and rows[0][0].lower() == 'id' else 0
+        
+        # Determine recent generations if filtering is requested
+        recent_gen_set = set()
+        if recent_generations and recent_generations.isdigit():
+            n_recent = int(recent_generations)
+            
+            # Find all generation numbers from candidate IDs
+            all_generations = set()
+            for row in rows[start_idx:]:
+                if len(row) < 1:
+                    continue
+                candidate_id = row[0]
+                
+                # Extract generation number from candidate_id (e.g., gen01-001 -> 1)
+                match = re.match(r'^gen(\d+)-', candidate_id)
+                if match:
+                    gen_num = int(match.group(1))
+                    all_generations.add(gen_num)
+            
+            # Get the most recent N generations
+            if all_generations:
+                sorted_generations = sorted(all_generations, reverse=True)
+                recent_gen_set = set(sorted_generations[:n_recent])
         
         candidates = []
-        for row in reader:
+        for row in rows[start_idx:]:
             if len(row) < 1:
                 continue
             
@@ -249,6 +358,18 @@ try:
             else:
                 matches = current_status == selector
             
+            # Apply recent generations filter if specified
+            if matches and recent_gen_set:
+                # Extract generation number from candidate_id
+                gen_match = re.match(r'^gen(\d+)-', candidate_id)
+                if gen_match:
+                    candidate_gen = int(gen_match.group(1))
+                    if candidate_gen not in recent_gen_set:
+                        matches = False  # Filter out this candidate
+                else:
+                    # Non-generation candidates (like baseline) - exclude when filtering by recent generations
+                    matches = False
+            
             if matches:
                 candidates.append(candidate_id)
         
@@ -284,7 +405,11 @@ except Exception as e:
 delete_candidates_from_csv() {
   local selector="$1"
   
-  echo "[INFO] Deleting candidates matching '$selector' from CSV..."
+  local filter_msg=""
+  if [[ -n "$recent_generations" ]]; then
+    filter_msg=" (limited to recent $recent_generations generations)"
+  fi
+  echo "[INFO] Deleting candidates matching '$selector' from CSV${filter_msg}..."
   
   "$PYTHON_CMD" -c "
 import sys
@@ -293,6 +418,7 @@ from lib.evolution_csv import EvolutionCSV
 import re
 
 selector = '$selector'
+recent_generations = '$recent_generations'
 deleted_count = 0
 
 with EvolutionCSV('$FULL_CSV_PATH') as csv:
@@ -306,6 +432,29 @@ with EvolutionCSV('$FULL_CSV_PATH') as csv:
         has_header = rows and rows[0] and rows[0][0].lower() == 'id'
         start_idx = 1 if has_header else 0
         
+        # Determine recent generations if filtering is requested
+        recent_gen_set = set()
+        if recent_generations and recent_generations.isdigit():
+            n_recent = int(recent_generations)
+            
+            # Find all generation numbers from candidate IDs
+            all_generations = set()
+            for row in rows[start_idx:]:
+                if not row or not row[0].strip():
+                    continue
+                candidate_id = row[0].strip()
+                
+                # Extract generation number from candidate_id (e.g., gen01-001 -> 1)
+                match = re.match(r'^gen(\d+)-', candidate_id)
+                if match:
+                    gen_num = int(match.group(1))
+                    all_generations.add(gen_num)
+            
+            # Get the most recent N generations
+            if all_generations:
+                sorted_generations = sorted(all_generations, reverse=True)
+                recent_gen_set = set(sorted_generations[:n_recent])
+        
         for row in rows[start_idx:]:
             if not row or not row[0].strip():
                 continue
@@ -327,6 +476,18 @@ with EvolutionCSV('$FULL_CSV_PATH') as csv:
             else:
                 matches = current_status == selector
             
+            # Apply recent generations filter if specified
+            if matches and recent_gen_set:
+                # Extract generation number from candidate_id
+                gen_match = re.match(r'^gen(\d+)-', candidate_id)
+                if gen_match:
+                    candidate_gen = int(gen_match.group(1))
+                    if candidate_gen not in recent_gen_set:
+                        matches = False  # Filter out this candidate
+                else:
+                    # Non-generation candidates (like baseline) - exclude when filtering by recent generations
+                    matches = False
+            
             if matches:
                 candidates_to_delete.append(candidate_id)
     
@@ -341,7 +502,11 @@ with EvolutionCSV('$FULL_CSV_PATH') as csv:
 }
 
 # Main execution
-echo "[INFO] Processing '$SELECTOR' with action: $ACTION"
+info_msg="Processing '$SELECTOR' with action: $ACTION"
+if [[ -n "$recent_generations" ]]; then
+  info_msg="$info_msg (limited to recent $recent_generations generations)"
+fi
+echo "[INFO] $info_msg"
 
 case "$ACTION" in
   failed)
@@ -387,4 +552,4 @@ echo "[INFO] Edit operation complete"
 
 # Call status command to show current state
 echo ""
-"$SCRIPT_DIR/claude-evolve-status" --brief
+"$SCRIPT_DIR/claude-evolve-status" --brief