claude-evolve 1.3.43 → 1.4.0

package/bin/claude-evolve-analyze

@@ -194,14 +194,9 @@ else
 fi
 
 echo
-echo "=== Top Performer ==="
-if [[ -n $top_id ]]; then
-  echo "ID: $top_id"
-  echo "Performance: $top_score"
-  echo "Description: $top_desc"
-else
-  echo "No completed candidates yet"
-fi
+
+# Call status command to show winners and revolutionary improvers
+"$SCRIPT_DIR/claude-evolve-status" --brief
 
 # Generation analysis
 echo
@@ -321,10 +316,12 @@ if command -v gnuplot >/dev/null 2>&1 && [[ $valid_performance_count -gt 0 ]]; t
   # Create data files for gnuplot
   data_file="/tmp/evolution_data_$$.dat"
   winner_file="/tmp/evolution_winner_$$.dat"
+  novel_file="/tmp/evolution_novel_$$.dat"
   gen_avg_file="/tmp/evolution_gen_avg_$$.dat"
   
   echo "# Row ID Performance Generation" >"$data_file"
   echo "# Generation MedianPerformance Color" >"$gen_avg_file"
+  echo "# Row ID Performance Generation" >"$novel_file"
 
   # Get color by generation number (rotates through 7 colors)
   get_gen_color() {
@@ -350,14 +347,15 @@ if command -v gnuplot >/dev/null 2>&1 && [[ $valid_performance_count -gt 0 ]]; t
   max_row=0
   max_id=""
   
-  # Use Python to generate chart data
+  # Use Python to generate chart data and identify revolutionary improvers
   "$PYTHON_CMD" -c "
 import csv
 import re
 
 with open('$csv_file', 'r') as f:
     reader = csv.reader(f)
-    next(reader)  # Skip header
+    rows = list(reader)
+    header = rows[0]
     
     completed_order = 0  # Track order of completion
     
@@ -367,15 +365,19 @@ with open('$csv_file', 'r') as f:
     with open('$gen_data_temp', 'w') as gen_f:
         pass  # Clear file
     
+    with open('$novel_file', 'w') as novel_f:
+        novel_f.write('# Order ID Performance Generation\\n')
+    
     max_perf = 0
     max_id = ''
     max_order = 0
     
-    for row in reader:
+    
+    for row in rows[1:]:
         if len(row) < 5:
             continue
             
-        id, _, desc, perf, status = row[:5]
+        id, parent_id, desc, perf, status = row[0], row[1] if len(row) > 1 else '', row[2] if len(row) > 2 else '', row[3], row[4]
         
         # Extract generation from ID
         gen = 'gen01'  # default
@@ -396,6 +398,11 @@ with open('$csv_file', 'r') as f:
                     with open('$data_file', 'a') as f:
                         f.write(f'{completed_order} \"{id}\" {perf} {gen_num}\\n')
                     
+                    # Write to novel file if this is a novel candidate
+                    if not parent_id:
+                        with open('$novel_file', 'a') as f:
+                            f.write(f'{completed_order} \"{id}\" {perf} {gen_num}\\n')
+                    
                     # Write to gen temp file
                     with open('$gen_data_temp', 'a') as f:
                         f.write(f'{gen} {perf}\\n')
@@ -532,6 +539,15 @@ print(f'max_desc=\"{desc_escaped}\"')
       fi
     done
     
+    # Add novel candidates
+    if [[ -s "$novel_file" ]] && [[ $(wc -l < "$novel_file") -gt 1 ]]; then
+      if [[ $gen_plots_added -gt 0 ]]; then
+        plot_cmd="$plot_cmd, \\"$'\n'
+      fi
+      plot_cmd="${plot_cmd}     \"$novel_file\" using 1:3 with points pointtype 8 pointsize $winner_dot_size linecolor rgb \"#ff1493\" title \"Novel Candidates\""
+      ((gen_plots_added++))
+    fi
+    
     # Add winner point
     if [[ -n $max_id && -s "$winner_file" ]]; then
       if [[ $gen_plots_added -gt 0 ]]; then
@@ -618,7 +634,7 @@ EOF
     exit 0
   fi
 
-  rm -f "$data_file" "$winner_file" "$gen_avg_file" "$gen_data_temp"
+  rm -f "$data_file" "$winner_file" "$novel_file" "$gen_avg_file" "$gen_data_temp"
   echo "Chart saved to: $output_file"
 
   # Always try to open chart (not just when --open is used)

package/bin/claude-evolve-clean-invalid

@@ -0,0 +1,117 @@
+#!/usr/bin/env python3
+"""
+Cleanup script to remove invalid entries from evolution CSV files.
+This removes entries with IDs that contain shell constructs or other invalid content.
+"""
+
+import csv
+import sys
+import os
+import re
+import argparse
+
+
+def is_invalid_id(candidate_id):
+    """Check if a candidate ID contains invalid content."""
+    if not candidate_id:
+        return True
+    
+    # Check for shell constructs
+    invalid_patterns = [
+        r'EOF.*<.*null',
+        r'<<.*EOF',
+        r'<.*dev.*null',
+        r'^\s*#',  # Comments
+        r'^\s*$',  # Empty
+        r'[<>|&;`$]',  # Shell special characters
+        r'^\s*\[',  # Bash test constructs
+    ]
+    
+    for pattern in invalid_patterns:
+        if re.search(pattern, candidate_id, re.IGNORECASE):
+            return True
+    
+    # Check for excessively long IDs (likely errors)
+    if len(candidate_id) > 50:
+        return True
+    
+    return False
+
+
+def clean_csv(csv_file, dry_run=False):
+    """Remove invalid entries from CSV file."""
+    if not os.path.exists(csv_file):
+        print(f"Error: CSV file {csv_file} not found")
+        return False
+    
+    invalid_entries = []
+    valid_rows = []
+    
+    with open(csv_file, 'r') as f:
+        reader = csv.reader(f)
+        header = next(reader)
+        valid_rows.append(header)
+        
+        for row_num, row in enumerate(reader, start=2):
+            if not row or len(row) == 0:
+                continue
+                
+            candidate_id = row[0].strip() if row[0] else ''
+            
+            if is_invalid_id(candidate_id):
+                invalid_entries.append({
+                    'row_num': row_num,
+                    'id': candidate_id,
+                    'row': row
+                })
+            else:
+                valid_rows.append(row)
+    
+    if not invalid_entries:
+        print("No invalid entries found.")
+        return True
+    
+    print(f"Found {len(invalid_entries)} invalid entries:")
+    for entry in invalid_entries:
+        print(f"  Row {entry['row_num']}: ID='{entry['id']}'")
+        if len(entry['row']) > 4:
+            print(f"    Status: {entry['row'][4]}")
+    
+    if dry_run:
+        print("\nDry run - no changes made.")
+        return True
+    
+    # Write cleaned CSV
+    backup_file = csv_file + '.backup'
+    print(f"\nBacking up to: {backup_file}")
+    os.rename(csv_file, backup_file)
+    
+    try:
+        with open(csv_file, 'w', newline='') as f:
+            writer = csv.writer(f)
+            writer.writerows(valid_rows)
+        
+        print(f"Cleaned CSV written to: {csv_file}")
+        print(f"Removed {len(invalid_entries)} invalid entries")
+        return True
+        
+    except Exception as e:
+        print(f"Error writing cleaned CSV: {e}")
+        print("Restoring backup...")
+        os.rename(backup_file, csv_file)
+        return False
+
+
+def main():
+    parser = argparse.ArgumentParser(description='Remove invalid entries from evolution CSV')
+    parser.add_argument('csv_file', help='Path to evolution CSV file')
+    parser.add_argument('--dry-run', action='store_true', help='Show what would be removed without making changes')
+    
+    args = parser.parse_args()
+    
+    success = clean_csv(args.csv_file, args.dry_run)
+    sys.exit(0 if success else 1)
+
+
+if __name__ == '__main__':
+    main()

package/bin/claude-evolve-cleanup-duplicates

@@ -0,0 +1,131 @@
+#!/usr/bin/env python3
+"""
+Cleanup script to remove duplicate entries from evolution CSV files.
+This fixes the issue where improper CSV parsing caused duplicate IDs.
+"""
+
+import csv
+import sys
+import os
+import argparse
+from collections import defaultdict
+
+
+def find_duplicates(csv_file):
+    """Find duplicate IDs and return information about them."""
+    if not os.path.exists(csv_file):
+        print(f"Error: CSV file {csv_file} not found")
+        return {}
+    
+    id_entries = defaultdict(list)
+    
+    with open(csv_file, 'r') as f:
+        reader = csv.reader(f)
+        header = next(reader)
+        
+        for row_num, row in enumerate(reader, start=2):
+            if row and len(row) > 0:
+                candidate_id = row[0].strip()
+                if candidate_id:
+                    id_entries[candidate_id].append({
+                        'row_num': row_num,
+                        'row': row,
+                        'basedOnId': row[1].strip() if len(row) > 1 else '',
+                        'description': row[2].strip() if len(row) > 2 else '',
+                        'performance': row[3].strip() if len(row) > 3 else '',
+                        'status': row[4].strip() if len(row) > 4 else ''
+                    })
+    
+    # Find duplicates
+    duplicates = {id_val: entries for id_val, entries in id_entries.items() if len(entries) > 1}
+    
+    return duplicates, header
+
+
+def choose_best_entry(entries):
+    """Choose the best entry to keep from duplicates."""
+    # Priority order:
+    # 1. Completed entries with performance score
+    # 2. Entries with empty basedOnId (original entries)
+    # 3. Most complete entry
+    
+    completed_entries = [e for e in entries if e['status'] == 'complete' and e['performance']]
+    
+    if completed_entries:
+        # Keep the one with the highest performance
+        return max(completed_entries, key=lambda x: float(x['performance']) if x['performance'] else 0)
+    
+    # If no completed entries, prefer ones with empty basedOnId (original entries)
+    original_entries = [e for e in entries if not e['basedOnId']]
+    if original_entries:
+        return original_entries[0]
+    
+    # Otherwise, keep the most complete entry
+    return max(entries, key=lambda x: len([f for f in x['row'] if f.strip()]))
+
+
+def cleanup_csv(csv_file, dry_run=True):
+    """Clean up duplicate entries in CSV file."""
+    duplicates, header = find_duplicates(csv_file)
+    
+    if not duplicates:
+        print(f"No duplicates found in {csv_file}")
+        return
+    
+    print(f"Found {len(duplicates)} duplicate IDs in {csv_file}:")
+    
+    # Read all rows
+    with open(csv_file, 'r') as f:
+        reader = csv.reader(f)
+        all_rows = list(reader)
+    
+    rows_to_remove = set()
+    
+    for candidate_id, entries in duplicates.items():
+        print(f"\n  {candidate_id}: {len(entries)} entries")
+        
+        best_entry = choose_best_entry(entries)
+        
+        for entry in entries:
+            if entry == best_entry:
+                print(f"    Row {entry['row_num']}: KEEP - {entry['status']} {entry['performance']}")
+            else:
+                print(f"    Row {entry['row_num']}: REMOVE - {entry['status']} {entry['performance']}")
+                rows_to_remove.add(entry['row_num'] - 1)  # Convert to 0-based index
+    
+    if dry_run:
+        print(f"\nDry run mode: Would remove {len(rows_to_remove)} duplicate rows")
+        print("Run with --fix to actually remove duplicates")
+        return
+    
+    # Remove duplicate rows
+    cleaned_rows = []
+    for i, row in enumerate(all_rows):
+        if i not in rows_to_remove:
+            cleaned_rows.append(row)
+    
+    # Write cleaned CSV
+    backup_file = f"{csv_file}.backup.{os.getpid()}"
+    os.rename(csv_file, backup_file)
+    print(f"\nCreated backup: {backup_file}")
+    
+    with open(csv_file, 'w', newline='') as f:
+        writer = csv.writer(f)
+        writer.writerows(cleaned_rows)
+    
+    print(f"Removed {len(rows_to_remove)} duplicate rows from {csv_file}")
+    print(f"Cleaned CSV has {len(cleaned_rows)} rows (including header)")
+
+
+def main():
+    parser = argparse.ArgumentParser(description='Clean up duplicate entries in evolution CSV files')
+    parser.add_argument('csv_file', help='Path to CSV file to clean')
+    parser.add_argument('--fix', action='store_true', help='Actually fix the file (default is dry run)')
+    
+    args = parser.parse_args()
+    
+    cleanup_csv(args.csv_file, dry_run=not args.fix)
+
+
+if __name__ == '__main__':
+    main()