claude-evolve 1.7.18 → 1.7.21

package/bin/claude-evolve-ideate

@@ -95,66 +95,76 @@ call_ai_for_ideation() {
     gen_num=1
   fi
   
-  # Get the current row count before any modifications
-  local original_csv_count
+  # Make a backup of the pre-populated temp CSV (which includes stub rows from caller)
+  # This preserves the stub rows that the caller added
+  local temp_csv_backup="${temp_csv_file}.backup"
   if [[ -f "$temp_csv_file" ]]; then
-    original_csv_count=$(grep -v '^[[:space:]]*$' "$temp_csv_file" | tail -n +2 | wc -l)
+    cp "$temp_csv_file" "$temp_csv_backup"
   else
-    original_csv_count=0
+    echo "[ERROR] Temp CSV file not found at start: $temp_csv_file" >&2
+    return 1
   fi
-  
-  
+
+  # Get the current row count before any modifications (from the pre-populated file with stubs)
+  local original_csv_count
+  original_csv_count=$(grep -v '^[[:space:]]*$' "$temp_csv_file" | tail -n +2 | wc -l)
+
+  echo "[DEBUG] Pre-populated temp CSV has $original_csv_count rows (includes stub rows with placeholders)" >&2
+
   # Get models for ideation
   local model_list
   model_list=$(get_models_for_command "ideate")
   local models=()
   read -ra models <<< "$model_list"
-  
+
   if [[ ${#models[@]} -eq 0 ]]; then
     echo "[ERROR] No models configured for ideation" >&2
+    rm -f "$temp_csv_backup"
     return 1
   fi
-  
+
   # Calculate starting index for round-robin
   local num_models=${#models[@]}
   local start_index=$((gen_num % num_models))
-  
+
   # Create ordered list based on round-robin
   local ordered_models=()
   for ((i=0; i<num_models; i++)); do
     local idx=$(((start_index + i) % num_models))
     ordered_models+=("${models[$idx]}")
   done
-  
+
   echo "[AI] Model order for ideate (round-robin): ${ordered_models[*]}" >&2
-  
+
   # Try each model until CSV changes
   for model in "${ordered_models[@]}"; do
     echo "[AI] Attempting ideate with $model" >&2
 
-    # Restore temp CSV before each attempt (in case previous model corrupted it)
-    # This ensures each model starts with the original data
-    if [[ -f "$FULL_CSV_PATH" ]]; then
-      cp "$FULL_CSV_PATH" "$temp_csv_file"
-      # Recapture original count in case it changed
-      original_csv_count=$(grep -v '^[[:space:]]*$' "$temp_csv_file" | tail -n +2 | wc -l)
-    fi
+    # Restore temp CSV from backup before each attempt (in case previous model corrupted it)
+    # This preserves the stub rows that the caller pre-populated
+    cp "$temp_csv_backup" "$temp_csv_file"
 
     # Call the model directly
     local ai_output
     ai_output=$(call_ai_model_configured "$model" "$prompt")
     local ai_exit_code=$?
 
-    # Check if the file was modified AND has correct count
+    # Check if the file was modified correctly
     if [[ -f "$temp_csv_file" ]]; then
       local new_csv_count
       new_csv_count=$(grep -v '^[[:space:]]*$' "$temp_csv_file" | tail -n +2 | wc -l)
       local added_count=$((new_csv_count - original_csv_count))
 
-      if [[ $added_count -gt 0 ]]; then
-        # Validate count BEFORE accepting
-        if [[ $added_count -eq $expected_count ]]; then
-          echo "[INFO] CSV modified by $model: added $added_count/$expected_count rows ✓" >&2
+      echo "[DEBUG] After $model: original=$original_csv_count, new=$new_csv_count, added=$added_count" >&2
+
+      # Check if row count is correct (should be same since we're editing stubs, not adding)
+      if [[ $new_csv_count -eq $original_csv_count ]]; then
+        # Count remaining placeholders - there should be none if AI did its job
+        local placeholder_count
+        placeholder_count=$(grep -c "PLACEHOLDER" "$temp_csv_file" 2>/dev/null || echo "0")
+
+        if [[ $placeholder_count -eq 0 ]]; then
+          echo "[INFO] CSV modified by $model: filled $expected_count placeholder rows ✓" >&2
 
           # Post-process to ensure all description fields are quoted
           local fixed_csv_file="${temp_csv_file}.fixed"
@@ -167,15 +177,30 @@ call_ai_for_ideation() {
             echo "[WARN] CSV format validation failed, using original" >&2
           fi
 
+          # Clean up backup file
+          rm -f "$temp_csv_backup"
+
           # Echo the successful model name for caller to capture
           echo "$model"
           return 0
         else
-          echo "[WARN] $model added wrong count: $added_count (expected $expected_count) - trying next model" >&2
-          # Continue to next model - maybe another will get it right
+          echo "[WARN] $model left $placeholder_count placeholders unfilled - trying next model" >&2
+          # Continue to next model
         fi
+      elif [[ $added_count -lt 0 ]]; then
+        echo "[WARN] $model deleted rows ($added_count) - trying next model" >&2
+        # Continue to next model
+      elif [[ $added_count -gt 0 ]]; then
+        echo "[WARN] $model added extra rows ($added_count) instead of editing stubs - trying next model" >&2
+        # Continue to next model
       else
         echo "[INFO] CSV unchanged after $model (exit code: $ai_exit_code)" >&2
+        # Log last few lines of AI output to help debug why it succeeded but didn't change the file
+        if [[ -n "$ai_output" ]]; then
+          echo "[AI] Last 10 lines of $model output:" >&2
+          echo "$ai_output" | tail -n 10 >&2
+          echo "[AI] ---" >&2
+        fi
         # Continue to next model
       fi
     else
@@ -185,6 +210,7 @@ call_ai_for_ideation() {
   done
   
   # All models tried, none changed the file
+  rm -f "$temp_csv_backup"
   echo "[ERROR] All AI models failed to generate ideas" >&2
   return 1
 }
@@ -1102,6 +1128,10 @@ generate_hill_climbing_direct() {
   local temp_csv="$FULL_EVOLUTION_DIR/temp-csv-$$.csv"
   cp "$FULL_CSV_PATH" "$temp_csv"
 
+  # Extract just the IDs from top performers for clarity (needed before pre-populating)
+  local valid_parent_ids
+  valid_parent_ids=$(echo "$top_performers" | cut -d',' -f1 | paste -sd ',' -)
+
   # Pre-populate the CSV with stub rows containing the correct IDs and parent IDs
   echo "[INFO] Pre-populating CSV with stub rows: $required_ids_str"
   # Use first parent as default for stubs (AI will adjust if needed)
@@ -1124,10 +1154,6 @@ generate_hill_climbing_direct() {
   # Get existing Python files for this generation to avoid ID collisions
   local existing_py_files=$(get_existing_py_files_for_generation "$CURRENT_GENERATION")
 
-  # Extract just the IDs from top performers for clarity
-  local valid_parent_ids
-  valid_parent_ids=$(echo "$top_performers" | cut -d',' -f1 | paste -sd ',' -)
-
   # Use relative paths and change to evolution directory so AI can access files
   local temp_csv_basename=$(basename "$temp_csv")
 
@@ -1232,6 +1258,10 @@ generate_structural_mutation_direct() {
   local temp_csv="$FULL_EVOLUTION_DIR/temp-csv-$$.csv"
   cp "$FULL_CSV_PATH" "$temp_csv"
 
+  # Extract just the IDs from top performers for clarity (needed before pre-populating)
+  local valid_parent_ids
+  valid_parent_ids=$(echo "$top_performers" | cut -d',' -f1 | paste -sd ',' -)
+
   # Pre-populate the CSV with stub rows
   echo "[INFO] Pre-populating CSV with stub rows: $required_ids_str"
   local first_parent_id
@@ -1253,10 +1283,6 @@ generate_structural_mutation_direct() {
   # Get existing Python files for this generation to avoid ID collisions
   local existing_py_files=$(get_existing_py_files_for_generation "$CURRENT_GENERATION")
 
-  # Extract just the IDs from top performers for clarity
-  local valid_parent_ids
-  valid_parent_ids=$(echo "$top_performers" | cut -d',' -f1 | paste -sd ',' -)
-
   # Use relative paths and change to evolution directory so AI can access files
   local temp_csv_basename=$(basename "$temp_csv")
 
@@ -1352,6 +1378,10 @@ generate_crossover_direct() {
   local temp_csv="$FULL_EVOLUTION_DIR/temp-csv-$$.csv"
   cp "$FULL_CSV_PATH" "$temp_csv"
 
+  # Extract just the IDs from top performers for clarity (needed before pre-populating)
+  local valid_parent_ids
+  valid_parent_ids=$(echo "$top_performers" | cut -d',' -f1 | paste -sd ',' -)
+
   # Pre-populate the CSV with stub rows
   echo "[INFO] Pre-populating CSV with stub rows: $required_ids_str"
   local first_parent_id
@@ -1373,10 +1403,6 @@ generate_crossover_direct() {
   # Get existing Python files for this generation to avoid ID collisions
   local existing_py_files=$(get_existing_py_files_for_generation "$CURRENT_GENERATION")
 
-  # Extract just the IDs from top performers for clarity
-  local valid_parent_ids
-  valid_parent_ids=$(echo "$top_performers" | cut -d',' -f1 | paste -sd ',' -)
-
   # Use relative paths and change to evolution directory so AI can access files
   local temp_csv_basename=$(basename "$temp_csv")
 

package/bin/claude-evolve-worker

@@ -255,6 +255,14 @@ with EvolutionCSV('$FULL_CSV_PATH') as csv:
     
     if [[ "$current_status" == "complete" ]]; then
       echo "[WORKER-$$] Already evaluated - skipping"
+      # Reset status back to complete since get_next_pending_candidate() set it to running
+      "$PYTHON_CMD" -c "
+import sys
+sys.path.insert(0, '$SCRIPT_DIR/..')
+from lib.evolution_csv import EvolutionCSV
+with EvolutionCSV('$FULL_CSV_PATH') as csv:
+    csv.update_candidate_status('$candidate_id', 'complete')
+" 2>/dev/null || true
       return 0
     fi
     

package/lib/csv_fixer.py

@@ -9,6 +9,31 @@ import csv
 import sys
 import re
 
+def clean_candidate_id(candidate_id):
+    """
+    Clean and normalize a candidate ID.
+    Returns (cleaned_id, was_modified)
+    """
+    if not candidate_id or candidate_id == "id":
+        return candidate_id, False
+
+    original = candidate_id
+    cleaned = candidate_id
+
+    # Strip leading/trailing whitespace
+    cleaned = cleaned.strip()
+
+    # Remove any internal spaces (e.g., "gen01 -001" -> "gen01-001")
+    cleaned = re.sub(r'\s+', '', cleaned)
+
+    # Remove pipe characters and anything before them (line number artifacts)
+    if '|' in cleaned:
+        # Extract the part after the last pipe
+        parts = cleaned.split('|')
+        cleaned = parts[-1].strip()
+
+    return cleaned, (cleaned != original)
+
 def is_valid_candidate_id(candidate_id):
     """
     Check if a candidate ID is valid.
@@ -25,7 +50,7 @@ def is_valid_candidate_id(candidate_id):
     if not candidate_id or candidate_id == "id":
         return True  # Header row
 
-    # Reject IDs containing pipe characters (line number artifacts)
+    # Reject IDs still containing pipe characters after cleaning
     if '|' in candidate_id:
         return False
 
@@ -46,13 +71,14 @@ def fix_csv_format(input_file, output_file):
     """
     Read a CSV file and ensure all fields are properly quoted.
     The csv module handles quoting automatically based on content.
-    Also filters out rows with invalid candidate IDs.
+    Also cleans and validates candidate IDs, filtering out invalid rows.
     """
     with open(input_file, 'r') as infile:
         reader = csv.reader(infile)
         rows = list(reader)
 
     rejected_count = 0
+    cleaned_count = 0
     filtered_rows = []
 
     for i, row in enumerate(rows):
@@ -67,14 +93,27 @@ def fix_csv_format(input_file, output_file):
 
         candidate_id = row[0] if len(row) > 0 else ""
 
-        # Check if candidate ID is valid
-        if not is_valid_candidate_id(candidate_id):
+        # Clean the candidate ID
+        cleaned_id, was_modified = clean_candidate_id(candidate_id)
+
+        if was_modified:
+            cleaned_count += 1
+            print(f"[INFO] Cleaned ID: '{candidate_id}' -> '{cleaned_id}'", file=sys.stderr)
+            row[0] = cleaned_id
+
+        # Check if candidate ID is valid after cleaning
+        if not is_valid_candidate_id(cleaned_id):
             rejected_count += 1
-            print(f"[WARN] Rejecting corrupted record with invalid ID: {candidate_id}", file=sys.stderr)
+            print(f"[WARN] Rejecting corrupted record with invalid ID: {candidate_id} (cleaned: {cleaned_id})", file=sys.stderr)
             continue
 
+        # Trim whitespace from all other fields
+        row = [field.strip() if isinstance(field, str) else field for field in row]
+
         filtered_rows.append(row)
 
+    if cleaned_count > 0:
+        print(f"[INFO] Cleaned {cleaned_count} IDs (removed spaces, pipes, etc.)", file=sys.stderr)
     if rejected_count > 0:
         print(f"[INFO] Filtered out {rejected_count} corrupted records", file=sys.stderr)
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "claude-evolve",
-  "version": "1.7.18",
+  "version": "1.7.21",
   "bin": {
     "claude-evolve": "./bin/claude-evolve",
     "claude-evolve-main": "./bin/claude-evolve-main",