claude-evolve 1.7.8 → 1.7.12

package/bin/claude-evolve-ideate

@@ -145,28 +145,35 @@ call_ai_for_ideation() {
     ai_output=$(call_ai_model_configured "$model" "$prompt")
     local ai_exit_code=$?
 
-    # Check if the file was modified - this is ALL that matters
+    # Check if the file was modified AND has correct count
     if [[ -f "$temp_csv_file" ]]; then
       local new_csv_count
       new_csv_count=$(grep -v '^[[:space:]]*$' "$temp_csv_file" | tail -n +2 | wc -l)
-
-      if [[ $new_csv_count -gt $original_csv_count ]]; then
-        echo "[INFO] CSV was modified by $model ($new_csv_count vs $original_csv_count rows) - validating format..." >&2
-        
-        # Post-process to ensure all description fields are quoted
-        local fixed_csv_file="${temp_csv_file}.fixed"
-        
-        # Use the CSV fixer script
-        if "$PYTHON_CMD" "$SCRIPT_DIR/../lib/csv_fixer.py" "$temp_csv_file" "$fixed_csv_file"; then
-          mv "$fixed_csv_file" "$temp_csv_file"
-          echo "[INFO] CSV format validated and fixed if needed" >&2
+      local added_count=$((new_csv_count - original_csv_count))
+
+      if [[ $added_count -gt 0 ]]; then
+        # Validate count BEFORE accepting
+        if [[ $added_count -eq $expected_count ]]; then
+          echo "[INFO] CSV modified by $model: added $added_count/$expected_count rows ✓" >&2
+
+          # Post-process to ensure all description fields are quoted
+          local fixed_csv_file="${temp_csv_file}.fixed"
+
+          # Use the CSV fixer script
+          if "$PYTHON_CMD" "$SCRIPT_DIR/../lib/csv_fixer.py" "$temp_csv_file" "$fixed_csv_file"; then
+            mv "$fixed_csv_file" "$temp_csv_file"
+            echo "[INFO] CSV format validated and fixed if needed" >&2
+          else
+            echo "[WARN] CSV format validation failed, using original" >&2
+          fi
+
+          # Echo the successful model name for caller to capture
+          echo "$model"
+          return 0
         else
-          echo "[WARN] CSV format validation failed, using original" >&2
+          echo "[WARN] $model added wrong count: $added_count (expected $expected_count) - trying next model" >&2
+          # Continue to next model - maybe another will get it right
         fi
-        
-        # Echo the successful model name for caller to capture
-        echo "$model"
-        return 0
       else
         echo "[INFO] CSV unchanged after $model (exit code: $ai_exit_code)" >&2
         # Continue to next model
@@ -366,9 +373,12 @@ validate_direct_csv_modification() {
   
   local added_count=$((new_count - current_original_count))
   if [[ $added_count -ne $expected_count ]]; then
-    echo "[WARN] Expected to add $expected_count ideas but added $added_count" >&2
+    echo "[ERROR] Expected to add $expected_count ideas but only added $added_count" >&2
+    echo "[ERROR] Ideation failed - rejecting partial results to prevent endless loops" >&2
+    rm -f "$temp_csv"
+    return 1
   fi
-  
+
   # Use proper locking to safely update the CSV
   echo "[INFO] Acquiring CSV lock to apply changes..."
   
@@ -859,12 +869,14 @@ ideate_ai_strategies() {
   fi
   
   echo "[INFO] Strategy results: $strategies_succeeded/$strategies_attempted succeeded" >&2
-  
-  # Success if at least one strategy worked
-  if [[ $strategies_succeeded -gt 0 ]]; then
+
+  # REQUIRE ALL strategies to succeed - no partial results
+  # Accepting partial results leads to endless loops with 1 idea per generation
+  if [[ $strategies_succeeded -eq $strategies_attempted ]]; then
     return 0
   else
-    echo "[ERROR] All ideation strategies failed" >&2
+    echo "[ERROR] Not all ideation strategies succeeded ($strategies_succeeded/$strategies_attempted)" >&2
+    echo "[ERROR] Rejecting partial results - will retry with exponential backoff" >&2
     return 1
   fi
 }

package/bin/claude-evolve-run

@@ -553,17 +553,70 @@ except Exception as e:
   return $?
 }
 
+# Track loop iterations for periodic cleanup
+loop_iteration=0
+last_stuck_check=0
+
 # Main dispatch loop
 while true; do
   # Clean up finished workers
   cleanup_workers
-  
+
   # Check if API limit was reached
   if [[ "$api_limit_reached" == "true" ]]; then
     echo "[DISPATCHER] Stopping evolution run due to API usage limits" >&2
     break
   fi
-  
+
+  # Periodic cleanup of stuck candidates (every 5 iterations, ~25 seconds)
+  ((loop_iteration++))
+  if [[ $((loop_iteration - last_stuck_check)) -ge 5 ]]; then
+    last_stuck_check=$loop_iteration
+
+    # Reset stuck 'running' candidates (no active worker) and unknown statuses
+    if [[ -f "$FULL_CSV_PATH" && ${#worker_pids[@]} -eq 0 ]]; then
+      echo "[DISPATCHER] Checking for stuck candidates (no active workers)..."
+      "$PYTHON_CMD" -c "
+import csv
+import sys
+from pathlib import Path
+
+csv_file = '$FULL_CSV_PATH'
+rows = []
+
+with open(csv_file, 'r') as f:
+    rows = list(csv.reader(f))
+
+reset_count = 0
+has_header = rows and rows[0] and rows[0][0].lower() == 'id'
+start_idx = 1 if has_header else 0
+
+for i in range(start_idx, len(rows)):
+    if len(rows[i]) > 4:
+        status = rows[i][4].strip() if rows[i][4] else ''
+        candidate_id = rows[i][0] if rows[i] else ''
+
+        # Reset 'running' when no workers are active
+        if status == 'running':
+            print(f'[INFO] Resetting stuck running candidate: {candidate_id}', file=sys.stderr)
+            rows[i][4] = 'pending'
+            reset_count += 1
+        # Reset unknown statuses like 'ready'
+        elif status not in ['', 'pending', 'complete', 'failed', 'failed-ai-retry',
+                           'failed-retry1', 'failed-retry2', 'failed-retry3', 'skipped']:
+            print(f'[WARN] Resetting unknown status \"{status}\" to pending: {candidate_id}', file=sys.stderr)
+            rows[i][4] = 'pending'
+            reset_count += 1
+
+if reset_count > 0:
+    with open(csv_file + '.tmp', 'w', newline='') as f:
+        csv.writer(f).writerows(rows)
+    Path(csv_file + '.tmp').rename(csv_file)
+    print(f'[INFO] Reset {reset_count} stuck/unknown candidates to pending', file=sys.stderr)
+" || true
+    fi
+  fi
+
   # Get current status
   csv_stats=$(get_csv_stats "$FULL_CSV_PATH")
   read -r total_rows complete_count pending_count <<< "$csv_stats"
@@ -578,18 +631,71 @@ while true; do
   # If no pending work and no active workers, check for auto-ideation
   if [[ $pending_count -eq 0 && $active_workers -eq 0 ]]; then
     echo "[DISPATCHER] No pending candidates found."
-    
+
+    # Before blocking, do final check for stuck work (immediate, not periodic)
+    echo "[DISPATCHER] Performing final verification for stuck candidates..."
+    stuck_work_count=$("$PYTHON_CMD" -c "
+import csv
+csv_file = '$FULL_CSV_PATH'
+stuck = 0
+with open(csv_file, 'r') as f:
+    reader = csv.reader(f)
+    next(reader, None)  # Skip header
+    for row in reader:
+        if len(row) > 4:
+            status = row[4].strip() if row[4] else ''
+            # Count running, unknown, and retry statuses
+            if status in ['running'] or \
+               (status and status not in ['', 'pending', 'complete', 'failed', 'skipped',
+                                          'failed-ai-retry', 'failed-retry1', 'failed-retry2', 'failed-retry3']):
+                stuck += 1
+print(stuck)
+" 2>/dev/null || echo "0")
+
+    if [[ $stuck_work_count -gt 0 ]]; then
+      echo "[DISPATCHER] Found $stuck_work_count stuck candidates - running cleanup cycle..."
+      # Force immediate cleanup
+      loop_iteration=$((last_stuck_check + 5))
+      continue  # Go back to top of loop to trigger cleanup
+    fi
+
     # Check if auto ideation is enabled
     if [[ "$AUTO_IDEATE" == "true" || "$AUTO_IDEATE" == "1" ]]; then
       echo "[DISPATCHER] Auto ideation is enabled. Checking prerequisites..."
-      
-      # Check if previous generation has at least one completed item
-      if ! check_previous_generation_has_completed "$FULL_CSV_PATH"; then
-        echo "[DISPATCHER] Evolution complete - previous generation has no completed items."
-        echo "[DISPATCHER] This prevents endless ideation loops when API limits are hit."
-        echo "[DISPATCHER] Wait for current generation to complete, then run 'claude-evolve ideate' manually."
+
+      # Check if we have enough completed work to generate meaningful ideas
+      # Note: With bottom-up processing, highest gen might be processed first and fail
+      # So we check overall completion rate, not just the highest generation
+      total_completed=$("$PYTHON_CMD" -c "
+import csv
+completed = 0
+with open('$FULL_CSV_PATH', 'r') as f:
+    reader = csv.reader(f)
+    next(reader, None)
+    for row in reader:
+        if len(row) > 4 and row[4].strip() == 'complete':
+            completed += 1
+print(completed)
+" 2>/dev/null || echo "0")
+
+      if [[ $total_completed -lt 3 ]]; then
+        echo "[DISPATCHER] Evolution blocked - insufficient completed candidates ($total_completed < 3)."
+        echo "[DISPATCHER] This prevents ideation when there's no successful work to learn from."
+        echo "[DISPATCHER] Possible causes:"
+        echo "[DISPATCHER]   - Early generations failing (common with bottom-up processing)"
+        echo "[DISPATCHER]   - Algorithm bugs preventing any completions"
+        echo "[DISPATCHER]   - API rate limits or infrastructure issues"
+        echo "[DISPATCHER] Recommendation: Fix evaluation issues and run 'claude-evolve run' again"
         break
       fi
+
+      # Also check highest generation (catches endless loop case)
+      if ! check_previous_generation_has_completed "$FULL_CSV_PATH"; then
+        echo "[DISPATCHER] WARNING: Highest generation has no completed items."
+        echo "[DISPATCHER] This can happen with bottom-up processing if early gens fail."
+        echo "[DISPATCHER] Continuing since we have $total_completed total completions..."
+        # Don't break - allow ideation if we have enough overall completions
+      fi
       
       echo "[DISPATCHER] Prerequisites met. Generating new ideas..."
       

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "claude-evolve",
-  "version": "1.7.8",
+  "version": "1.7.12",
   "bin": {
     "claude-evolve": "./bin/claude-evolve",
     "claude-evolve-main": "./bin/claude-evolve-main",