claude-evolve 1.7.20 → 1.7.22

package/bin/claude-evolve-ideate

@@ -95,66 +95,79 @@ call_ai_for_ideation() {
     gen_num=1
   fi
   
-  # Get the current row count before any modifications
-  local original_csv_count
+  # Make a backup of the pre-populated temp CSV (which includes stub rows from caller)
+  # This preserves the stub rows that the caller added
+  local temp_csv_backup="${temp_csv_file}.backup"
   if [[ -f "$temp_csv_file" ]]; then
-    original_csv_count=$(grep -v '^[[:space:]]*$' "$temp_csv_file" | tail -n +2 | wc -l)
+    cp "$temp_csv_file" "$temp_csv_backup"
   else
-    original_csv_count=0
+    echo "[ERROR] Temp CSV file not found at start: $temp_csv_file" >&2
+    return 1
   fi
-  
-  
+
+  # Get the current row count before any modifications (from the pre-populated file with stubs)
+  local original_csv_count
+  original_csv_count=$(grep -v '^[[:space:]]*$' "$temp_csv_file" | tail -n +2 | wc -l | tr -d '[:space:]')
+
+  echo "[DEBUG] Pre-populated temp CSV has $original_csv_count rows (includes stub rows with placeholders)" >&2
+
   # Get models for ideation
   local model_list
   model_list=$(get_models_for_command "ideate")
   local models=()
   read -ra models <<< "$model_list"
-  
+
   if [[ ${#models[@]} -eq 0 ]]; then
     echo "[ERROR] No models configured for ideation" >&2
+    rm -f "$temp_csv_backup"
     return 1
   fi
-  
+
   # Calculate starting index for round-robin
   local num_models=${#models[@]}
   local start_index=$((gen_num % num_models))
-  
+
   # Create ordered list based on round-robin
   local ordered_models=()
   for ((i=0; i<num_models; i++)); do
     local idx=$(((start_index + i) % num_models))
     ordered_models+=("${models[$idx]}")
   done
-  
+
   echo "[AI] Model order for ideate (round-robin): ${ordered_models[*]}" >&2
-  
+
   # Try each model until CSV changes
   for model in "${ordered_models[@]}"; do
     echo "[AI] Attempting ideate with $model" >&2
 
-    # Restore temp CSV before each attempt (in case previous model corrupted it)
-    # This ensures each model starts with the original data
-    if [[ -f "$FULL_CSV_PATH" ]]; then
-      cp "$FULL_CSV_PATH" "$temp_csv_file"
-      # Recapture original count in case it changed
-      original_csv_count=$(grep -v '^[[:space:]]*$' "$temp_csv_file" | tail -n +2 | wc -l)
-    fi
+    # Restore temp CSV from backup before each attempt (in case previous model corrupted it)
+    # This preserves the stub rows that the caller pre-populated
+    cp "$temp_csv_backup" "$temp_csv_file"
 
     # Call the model directly
     local ai_output
     ai_output=$(call_ai_model_configured "$model" "$prompt")
     local ai_exit_code=$?
 
-    # Check if the file was modified AND has correct count
+    # Check if the file was modified correctly
     if [[ -f "$temp_csv_file" ]]; then
       local new_csv_count
-      new_csv_count=$(grep -v '^[[:space:]]*$' "$temp_csv_file" | tail -n +2 | wc -l)
+      new_csv_count=$(grep -v '^[[:space:]]*$' "$temp_csv_file" | tail -n +2 | wc -l | tr -d '[:space:]')
       local added_count=$((new_csv_count - original_csv_count))
 
-      if [[ $added_count -gt 0 ]]; then
-        # Validate count BEFORE accepting
-        if [[ $added_count -eq $expected_count ]]; then
-          echo "[INFO] CSV modified by $model: added $added_count/$expected_count rows ✓" >&2
+      echo "[DEBUG] After $model: original=$original_csv_count, new=$new_csv_count, added=$added_count" >&2
+
+      # Check if row count is correct (should be same since we're editing stubs, not adding)
+      if [[ $new_csv_count -eq $original_csv_count ]]; then
+        # Count remaining placeholders - there should be none if AI did its job
+        local placeholder_count
+        placeholder_count=$(grep -c "PLACEHOLDER" "$temp_csv_file" 2>/dev/null || echo "0")
+        # Strip whitespace and ensure we have a clean integer
+        placeholder_count=$(echo "$placeholder_count" | tr -d '[:space:]')
+        placeholder_count=${placeholder_count:-0}
+
+        if [[ $placeholder_count -eq 0 ]]; then
+          echo "[INFO] CSV modified by $model: filled $expected_count placeholder rows ✓" >&2
 
           # Post-process to ensure all description fields are quoted
           local fixed_csv_file="${temp_csv_file}.fixed"
@@ -167,13 +180,22 @@ call_ai_for_ideation() {
             echo "[WARN] CSV format validation failed, using original" >&2
           fi
 
+          # Clean up backup file
+          rm -f "$temp_csv_backup"
+
           # Echo the successful model name for caller to capture
           echo "$model"
           return 0
         else
-          echo "[WARN] $model added wrong count: $added_count (expected $expected_count) - trying next model" >&2
-          # Continue to next model - maybe another will get it right
+          echo "[WARN] $model left $placeholder_count placeholders unfilled - trying next model" >&2
+          # Continue to next model
         fi
+      elif [[ $added_count -lt 0 ]]; then
+        echo "[WARN] $model deleted rows ($added_count) - trying next model" >&2
+        # Continue to next model
+      elif [[ $added_count -gt 0 ]]; then
+        echo "[WARN] $model added extra rows ($added_count) instead of editing stubs - trying next model" >&2
+        # Continue to next model
       else
         echo "[INFO] CSV unchanged after $model (exit code: $ai_exit_code)" >&2
         # Log last few lines of AI output to help debug why it succeeded but didn't change the file
@@ -191,6 +213,7 @@ call_ai_for_ideation() {
   done
   
   # All models tried, none changed the file
+  rm -f "$temp_csv_backup"
   echo "[ERROR] All AI models failed to generate ideas" >&2
   return 1
 }
@@ -357,11 +380,11 @@ validate_direct_csv_modification() {
   # We need to track this before the AI runs by reading from the beginning state
   # First, get a fresh count from the current main CSV (which reflects any previous operations in this session)
   local current_original_count
-  current_original_count=$(grep -v '^[[:space:]]*$' "$FULL_CSV_PATH" | tail -n +2 | wc -l)
+  current_original_count=$(grep -v '^[[:space:]]*$' "$FULL_CSV_PATH" | tail -n +2 | wc -l | tr -d '[:space:]')
 
   # Count data rows in the modified temp CSV
   local new_count
-  new_count=$(grep -v '^[[:space:]]*$' "$temp_csv" | tail -n +2 | wc -l)
+  new_count=$(grep -v '^[[:space:]]*$' "$temp_csv" | tail -n +2 | wc -l | tr -d '[:space:]')
 
 
   # Check if AI overwrote the file instead of appending
@@ -422,7 +445,7 @@ validate_direct_csv_modification() {
   fi
   
   # Get just the new entries (skip header and existing entries)
-  local original_line_count=$(wc -l < "$FULL_CSV_PATH")
+  local original_line_count=$(wc -l < "$FULL_CSV_PATH" | tr -d '[:space:]')
   
   # Append only the new lines from temp CSV to the main CSV
   tail -n +$((original_line_count + 1)) "$temp_csv" >> "$FULL_CSV_PATH"
@@ -494,9 +517,9 @@ validate_and_apply_csv_modification_old() {
   
   # Validate the modified CSV has more entries than original
   local original_count
-  original_count=$(wc -l < "$FULL_CSV_PATH")
+  original_count=$(wc -l < "$FULL_CSV_PATH" | tr -d '[:space:]')
   local new_count
-  new_count=$(wc -l < "$temp_csv")
+  new_count=$(wc -l < "$temp_csv" | tr -d '[:space:]')
   
   
   if [[ $new_count -le $original_count ]]; then
@@ -523,7 +546,7 @@ validate_and_apply_csv_modification_old() {
   fi
   
   # Get just the new entries (skip header and existing entries)
-  local original_line_count=$(wc -l < "$FULL_CSV_PATH")
+  local original_line_count=$(wc -l < "$FULL_CSV_PATH" | tr -d '[:space:]')
   
   # Append only the new lines from temp CSV to the main CSV
   tail -n +$((original_line_count + 1)) "$temp_csv" >> "$FULL_CSV_PATH"
@@ -645,9 +668,9 @@ IMPORTANT: Output the complete modified CSV file. Do not add any explanation or
   
   # Validate the modified CSV has more entries than original
   local original_count
-  original_count=$(wc -l < "$FULL_CSV_PATH")
+  original_count=$(wc -l < "$FULL_CSV_PATH" | tr -d '[:space:]')
   local new_count
-  new_count=$(wc -l < "$temp_csv")
+  new_count=$(wc -l < "$temp_csv" | tr -d '[:space:]')
   
   
   if [[ $new_count -le $original_count ]]; then
@@ -676,7 +699,7 @@ IMPORTANT: Output the complete modified CSV file. Do not add any explanation or
   fi
   
   # Get just the new entries (skip header and existing entries)
-  local original_line_count=$(wc -l < "$FULL_CSV_PATH")
+  local original_line_count=$(wc -l < "$FULL_CSV_PATH" | tr -d '[:space:]')
   
   # Append only the new lines from temp CSV to the main CSV
   tail -n +$((original_line_count + 1)) "$temp_csv" >> "$FULL_CSV_PATH"
@@ -987,7 +1010,7 @@ generate_novel_ideas_direct() {
 
   # Count total lines in temp CSV (including header)
   local total_lines
-  total_lines=$(wc -l < "$temp_csv")
+  total_lines=$(wc -l < "$temp_csv" | tr -d '[:space:]')
   local read_offset=$((total_lines - 25))
   if [[ $read_offset -lt 1 ]]; then
     read_offset=1
@@ -1108,6 +1131,10 @@ generate_hill_climbing_direct() {
   local temp_csv="$FULL_EVOLUTION_DIR/temp-csv-$$.csv"
   cp "$FULL_CSV_PATH" "$temp_csv"
 
+  # Extract just the IDs from top performers for clarity (needed before pre-populating)
+  local valid_parent_ids
+  valid_parent_ids=$(echo "$top_performers" | cut -d',' -f1 | paste -sd ',' -)
+
   # Pre-populate the CSV with stub rows containing the correct IDs and parent IDs
   echo "[INFO] Pre-populating CSV with stub rows: $required_ids_str"
   # Use first parent as default for stubs (AI will adjust if needed)
@@ -1121,7 +1148,7 @@ generate_hill_climbing_direct() {
 
   # Count total lines in temp CSV (including header)
   local total_lines
-  total_lines=$(wc -l < "$temp_csv")
+  total_lines=$(wc -l < "$temp_csv" | tr -d '[:space:]')
   local read_offset=$((total_lines - 25))
   if [[ $read_offset -lt 1 ]]; then
     read_offset=1
@@ -1130,10 +1157,6 @@ generate_hill_climbing_direct() {
   # Get existing Python files for this generation to avoid ID collisions
   local existing_py_files=$(get_existing_py_files_for_generation "$CURRENT_GENERATION")
 
-  # Extract just the IDs from top performers for clarity
-  local valid_parent_ids
-  valid_parent_ids=$(echo "$top_performers" | cut -d',' -f1 | paste -sd ',' -)
-
   # Use relative paths and change to evolution directory so AI can access files
   local temp_csv_basename=$(basename "$temp_csv")
 
@@ -1238,6 +1261,10 @@ generate_structural_mutation_direct() {
   local temp_csv="$FULL_EVOLUTION_DIR/temp-csv-$$.csv"
   cp "$FULL_CSV_PATH" "$temp_csv"
 
+  # Extract just the IDs from top performers for clarity (needed before pre-populating)
+  local valid_parent_ids
+  valid_parent_ids=$(echo "$top_performers" | cut -d',' -f1 | paste -sd ',' -)
+
   # Pre-populate the CSV with stub rows
   echo "[INFO] Pre-populating CSV with stub rows: $required_ids_str"
   local first_parent_id
@@ -1250,7 +1277,7 @@ generate_structural_mutation_direct() {
 
   # Count total lines in temp CSV (including header)
   local total_lines
-  total_lines=$(wc -l < "$temp_csv")
+  total_lines=$(wc -l < "$temp_csv" | tr -d '[:space:]')
   local read_offset=$((total_lines - 25))
   if [[ $read_offset -lt 1 ]]; then
     read_offset=1
@@ -1259,10 +1286,6 @@ generate_structural_mutation_direct() {
   # Get existing Python files for this generation to avoid ID collisions
   local existing_py_files=$(get_existing_py_files_for_generation "$CURRENT_GENERATION")
 
-  # Extract just the IDs from top performers for clarity
-  local valid_parent_ids
-  valid_parent_ids=$(echo "$top_performers" | cut -d',' -f1 | paste -sd ',' -)
-
   # Use relative paths and change to evolution directory so AI can access files
   local temp_csv_basename=$(basename "$temp_csv")
 
@@ -1358,6 +1381,10 @@ generate_crossover_direct() {
   local temp_csv="$FULL_EVOLUTION_DIR/temp-csv-$$.csv"
   cp "$FULL_CSV_PATH" "$temp_csv"
 
+  # Extract just the IDs from top performers for clarity (needed before pre-populating)
+  local valid_parent_ids
+  valid_parent_ids=$(echo "$top_performers" | cut -d',' -f1 | paste -sd ',' -)
+
   # Pre-populate the CSV with stub rows
   echo "[INFO] Pre-populating CSV with stub rows: $required_ids_str"
   local first_parent_id
@@ -1370,7 +1397,7 @@ generate_crossover_direct() {
 
   # Count total lines in temp CSV (including header)
   local total_lines
-  total_lines=$(wc -l < "$temp_csv")
+  total_lines=$(wc -l < "$temp_csv" | tr -d '[:space:]')
   local read_offset=$((total_lines - 25))
   if [[ $read_offset -lt 1 ]]; then
     read_offset=1
@@ -1379,10 +1406,6 @@ generate_crossover_direct() {
   # Get existing Python files for this generation to avoid ID collisions
   local existing_py_files=$(get_existing_py_files_for_generation "$CURRENT_GENERATION")
 
-  # Extract just the IDs from top performers for clarity
-  local valid_parent_ids
-  valid_parent_ids=$(echo "$top_performers" | cut -d',' -f1 | paste -sd ',' -)
-
   # Use relative paths and change to evolution directory so AI can access files
   local temp_csv_basename=$(basename "$temp_csv")
 

package/lib/csv_fixer.py

@@ -9,6 +9,31 @@ import csv
 import sys
 import re
 
+def clean_candidate_id(candidate_id):
+    """
+    Clean and normalize a candidate ID.
+    Returns (cleaned_id, was_modified)
+    """
+    if not candidate_id or candidate_id == "id":
+        return candidate_id, False
+
+    original = candidate_id
+    cleaned = candidate_id
+
+    # Strip leading/trailing whitespace
+    cleaned = cleaned.strip()
+
+    # Remove any internal spaces (e.g., "gen01 -001" -> "gen01-001")
+    cleaned = re.sub(r'\s+', '', cleaned)
+
+    # Remove pipe characters and anything before them (line number artifacts)
+    if '|' in cleaned:
+        # Extract the part after the last pipe
+        parts = cleaned.split('|')
+        cleaned = parts[-1].strip()
+
+    return cleaned, (cleaned != original)
+
 def is_valid_candidate_id(candidate_id):
     """
     Check if a candidate ID is valid.
@@ -25,7 +50,7 @@ def is_valid_candidate_id(candidate_id):
     if not candidate_id or candidate_id == "id":
         return True  # Header row
 
-    # Reject IDs containing pipe characters (line number artifacts)
+    # Reject IDs still containing pipe characters after cleaning
     if '|' in candidate_id:
         return False
 
@@ -46,13 +71,14 @@ def fix_csv_format(input_file, output_file):
     """
     Read a CSV file and ensure all fields are properly quoted.
     The csv module handles quoting automatically based on content.
-    Also filters out rows with invalid candidate IDs.
+    Also cleans and validates candidate IDs, filtering out invalid rows.
     """
     with open(input_file, 'r') as infile:
         reader = csv.reader(infile)
         rows = list(reader)
 
     rejected_count = 0
+    cleaned_count = 0
     filtered_rows = []
 
     for i, row in enumerate(rows):
@@ -67,14 +93,27 @@ def fix_csv_format(input_file, output_file):
 
         candidate_id = row[0] if len(row) > 0 else ""
 
-        # Check if candidate ID is valid
-        if not is_valid_candidate_id(candidate_id):
+        # Clean the candidate ID
+        cleaned_id, was_modified = clean_candidate_id(candidate_id)
+
+        if was_modified:
+            cleaned_count += 1
+            print(f"[INFO] Cleaned ID: '{candidate_id}' -> '{cleaned_id}'", file=sys.stderr)
+            row[0] = cleaned_id
+
+        # Check if candidate ID is valid after cleaning
+        if not is_valid_candidate_id(cleaned_id):
             rejected_count += 1
-            print(f"[WARN] Rejecting corrupted record with invalid ID: {candidate_id}", file=sys.stderr)
+            print(f"[WARN] Rejecting corrupted record with invalid ID: {candidate_id} (cleaned: {cleaned_id})", file=sys.stderr)
             continue
 
+        # Trim whitespace from all other fields
+        row = [field.strip() if isinstance(field, str) else field for field in row]
+
         filtered_rows.append(row)
 
+    if cleaned_count > 0:
+        print(f"[INFO] Cleaned {cleaned_count} IDs (removed spaces, pipes, etc.)", file=sys.stderr)
     if rejected_count > 0:
         print(f"[INFO] Filtered out {rejected_count} corrupted records", file=sys.stderr)
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "claude-evolve",
-  "version": "1.7.20",
+  "version": "1.7.22",
   "bin": {
     "claude-evolve": "./bin/claude-evolve",
     "claude-evolve-main": "./bin/claude-evolve-main",