claude-evolve 1.3.38 → 1.3.40

package/bin/claude-evolve-run-parallel.OLD

@@ -1,389 +0,0 @@
-#!/bin/bash
-# Parallel evolution dispatcher - manages worker pool
-
-set -e
-
-# Load configuration
-SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
-# shellcheck source=../lib/config.sh
-source "$SCRIPT_DIR/../lib/config.sh"
-# shellcheck source=../lib/csv-lock.sh
-source "$SCRIPT_DIR/../lib/csv-lock.sh"
-
-# Use CLAUDE_EVOLVE_CONFIG if set, otherwise default
-if [[ -n ${CLAUDE_EVOLVE_CONFIG:-} ]]; then
-  load_config "$CLAUDE_EVOLVE_CONFIG"
-else
-  load_config
-fi
-
-# Parse arguments
-timeout_seconds=""
-
-while [[ $# -gt 0 ]]; do
-  case $1 in
-  --timeout)
-    timeout_seconds="$2"
-    shift 2
-    ;;
-  *)
-    echo "[ERROR] Unknown option: $1" >&2
-    exit 1
-    ;;
-  esac
-done
-
-echo "[DISPATCHER] Starting parallel evolution with $MAX_WORKERS max workers"
-[[ -n $timeout_seconds ]] && echo "[DISPATCHER] Using timeout: ${timeout_seconds} seconds per worker"
-
-# Validate workspace
-if [[ ! -d "$FULL_EVOLUTION_DIR" ]]; then
-  echo "[ERROR] Evolution directory not found: $FULL_EVOLUTION_DIR" >&2
-  exit 1
-fi
-
-if [[ ! -f "$FULL_CSV_PATH" ]]; then
-  echo "[ERROR] CSV file not found: $FULL_CSV_PATH" >&2
-  exit 1
-fi
-
-# Prepare logging
-mkdir -p logs
-
-# Track active workers (using regular array for compatibility)
-worker_pids=()
-consecutive_failures=0
-MAX_FAILURES=10
-rate_limit_hit=false
-
-# Count pending candidates
-count_pending() {
-  if ! read_csv_with_lock csv_content; then
-    echo "0"
-    return
-  fi
-  
-  # Use Python for proper CSV parsing with quoted fields
-  echo "$csv_content" | "$PYTHON_CMD" -c "
-import csv
-import sys
-reader = csv.reader(sys.stdin)
-next(reader)  # Skip header
-count = 0
-for row in reader:
-    # If row has fewer than 5 fields, treat as pending
-    if len(row) < 5:
-        count += 1
-    elif len(row) >= 5 and (row[4] == 'pending' or row[4] == ''):
-        count += 1
-print(count)
-"
-}
-
-# Start a worker
-start_worker() {
-  local worker_cmd="$SCRIPT_DIR/claude-evolve-worker"
-  [[ -n $timeout_seconds ]] && worker_cmd="$worker_cmd --timeout $timeout_seconds"
-  
-  echo "[DISPATCHER] Starting worker..."
-  $worker_cmd &
-  local pid=$!
-  
-  # Verify worker started successfully
-  sleep 0.1
-  if kill -0 "$pid" 2>/dev/null; then
-    worker_pids+=($pid)
-    echo "[DISPATCHER] Worker $pid started"
-  else
-    echo "[ERROR] Worker failed to start" >&2
-    ((consecutive_failures++))
-  fi
-}
-
-# Check worker exit status and handle accordingly
-handle_worker_exit() {
-  local pid="$1"
-  local exit_code="$2"
-  
-  case $exit_code in
-    0)
-      echo "[DISPATCHER] Worker $pid completed successfully"
-      consecutive_failures=0
-      ;;
-    1)
-      echo "[DISPATCHER] Worker $pid failed"
-      ((consecutive_failures++))
-      ;;
-    2)
-      echo "[DISPATCHER] Worker $pid hit rate limit"
-      rate_limit_hit=true
-      ;;
-    130)
-      echo "[DISPATCHER] Worker $pid interrupted"
-      ;;
-    *)
-      echo "[DISPATCHER] Worker $pid exited with code $exit_code"
-      ((consecutive_failures++))
-      ;;
-  esac
-  
-  # Remove from tracking
-  local new_pids=()
-  for p in "${worker_pids[@]}"; do
-    [[ $p != "$pid" ]] && new_pids+=($p)
-  done
-  worker_pids=("${new_pids[@]}")
-}
-
-# Signal handler for graceful shutdown
-shutdown_workers() {
-  echo "[DISPATCHER] Shutting down workers..."
-  for pid in "${worker_pids[@]}"; do
-    if kill -0 "$pid" 2>/dev/null; then
-      echo "[DISPATCHER] Stopping worker $pid"
-      kill -TERM "$pid" 2>/dev/null || true
-    fi
-  done
-  
-  # Wait for workers to exit
-  local timeout=10
-  while [[ ${#worker_pids[@]} -gt 0 && $timeout -gt 0 ]]; do
-    sleep 1
-    ((timeout--))
-    
-    local new_pids=()
-    for pid in "${worker_pids[@]}"; do
-      if kill -0 "$pid" 2>/dev/null; then
-        new_pids+=($pid)
-      fi
-    done
-    worker_pids=("${new_pids[@]}")
-  done
-  
-  # Force kill remaining workers
-  for pid in "${worker_pids[@]}"; do
-    if kill -0 "$pid" 2>/dev/null; then
-      echo "[DISPATCHER] Force killing worker $pid"
-      kill -KILL "$pid" 2>/dev/null || true
-    fi
-  done
-  
-  echo "[DISPATCHER] Shutdown complete"
-  exit 0
-}
-
-# Better signal handling with logging
-handle_signal() {
-  local signal="$1"
-  echo "[DISPATCHER] Received signal: $signal" >&2
-  echo "[DISPATCHER] Active workers: ${#worker_pids[@]}" >&2
-  
-  # For expensive workers, give option to continue
-  if [[ ${#worker_pids[@]} -gt 0 ]]; then
-    echo "[DISPATCHER] Warning: ${#worker_pids[@]} expensive workers are still running!" >&2
-    echo "[DISPATCHER] Press Ctrl+C again within 5 seconds to force shutdown, or wait..." >&2
-    
-    # Give 5 seconds to reconsider
-    local count=5
-    while [[ $count -gt 0 ]]; do
-      sleep 1
-      ((count--))
-      # Check if we got another signal
-      if [[ -f /tmp/evolve-force-shutdown-$$ ]]; then
-        echo "[DISPATCHER] Force shutdown requested" >&2
-        rm -f /tmp/evolve-force-shutdown-$$
-        shutdown_workers
-        exit 1
-      fi
-    done
-    
-    echo "[DISPATCHER] Continuing with active workers..." >&2
-    return
-  fi
-  
-  shutdown_workers
-}
-
-# Set up signal handlers
-trap 'handle_signal INT' INT
-trap 'handle_signal TERM' TERM
-trap 'echo "[DISPATCHER] Exiting with code $?" >&2' EXIT
-
-# Check for stuck "running" candidates from previous runs
-check_stuck_candidates() {
-  if read_csv_with_lock csv_content; then
-    local stuck_count=$(echo "$csv_content" | "$PYTHON_CMD" -c "
-import csv
-import sys
-reader = csv.reader(sys.stdin)
-next(reader)  # Skip header
-count = 0
-for row in reader:
-    if len(row) >= 5 and row[4] == 'running':
-        count += 1
-print(count)
-")
-    if [[ $stuck_count -gt 0 ]]; then
-      echo "[DISPATCHER] Found $stuck_count candidates stuck in 'running' status"
-      echo "[DISPATCHER] Resetting them to 'pending' for retry..."
-      
-      # Reset stuck candidates
-      if acquire_csv_lock; then
-        "$PYTHON_CMD" -c "
-import csv
-import sys
-
-# Read CSV
-with open('$FULL_CSV_PATH', 'r') as f:
-    reader = csv.reader(f)
-    rows = list(reader)
-
-# Reset running to pending
-for i in range(1, len(rows)):
-    if len(rows[i]) >= 5 and rows[i][4] == 'running':
-        rows[i][4] = 'pending'
-
-# Write back
-with open('${FULL_CSV_PATH}.tmp', 'w', newline='') as f:
-    writer = csv.writer(f)
-    writer.writerows(rows)
-" && mv -f "${FULL_CSV_PATH}.tmp" "$FULL_CSV_PATH"
-        release_csv_lock
-      fi
-    fi
-  fi
-}
-
-# Main dispatcher loop
-echo "[DISPATCHER] Starting main dispatch loop"
-
-# Check for stuck candidates from previous runs
-check_stuck_candidates
-
-# Set error handling
-set +e  # Don't exit on error in the main loop
-
-while true; do
-  # In parallel mode, let individual algorithm failures happen
-  # The generation is finite, so worst case it just completes with many failures
-  
-  # Check if rate limit was hit
-  if [[ $rate_limit_hit == true ]]; then
-    echo "[DISPATCHER] Rate limit detected. Waiting 60 seconds before retrying..."
-    sleep 60
-    rate_limit_hit=false
-  fi
-  
-  # Count pending work
-  pending_count=$(count_pending || echo "0")
-  active_workers=${#worker_pids[@]}
-  
-  echo "[DISPATCHER] Status: $pending_count pending, $active_workers active workers"
-  
-  # Debug: Show CSV status if no pending
-  if [[ $pending_count -eq 0 ]]; then
-    total_rows=$(read_csv_with_lock csv_content && echo "$csv_content" | wc -l | xargs)
-    complete_count=$(read_csv_with_lock csv_content && echo "$csv_content" | "$PYTHON_CMD" -c "
-import csv
-import sys
-reader = csv.reader(sys.stdin)
-next(reader)  # Skip header
-count = 0
-for row in reader:
-    if len(row) >= 5 and row[4] == 'complete':
-        count += 1
-print(count)
-")
-    echo "[DISPATCHER] CSV has $((total_rows-1)) total candidates, $complete_count complete"
-  fi
-  
-  # If no pending work and no active workers, check for auto-ideation
-  if [[ $pending_count -eq 0 && $active_workers -eq 0 ]]; then
-    echo "[DISPATCHER] No pending candidates found."
-    
-    # Check if auto ideation is enabled
-    echo "[DEBUG] AUTO_IDEATE value: '$AUTO_IDEATE'"
-    if [[ "$AUTO_IDEATE" == "true" || "$AUTO_IDEATE" == "1" ]]; then
-      echo "[DISPATCHER] Auto ideation is enabled. Generating new ideas..."
-      
-      # Check if claude-evolve-ideate exists
-      ideate_script="$SCRIPT_DIR/claude-evolve-ideate"
-      if [[ ! -f "$ideate_script" ]]; then
-        echo "[ERROR] claude-evolve-ideate script not found: $ideate_script" >&2
-        echo "[DISPATCHER] Evolution complete - no way to generate more ideas."
-        break
-      fi
-      
-      # Generate new ideas using the multi-strategy approach
-      echo "[DISPATCHER] Calling claude-evolve-ideate to generate new candidates..."
-      if ! "$ideate_script"; then
-        echo "[ERROR] Failed to generate new ideas" >&2
-        echo "[DISPATCHER] Evolution complete - ideation failed."
-        break
-      fi
-      
-      echo "[DISPATCHER] New ideas generated successfully. Continuing evolution..."
-      continue  # Go back to start of loop to find the new candidates
-    else
-      echo "[DISPATCHER] Auto ideation is disabled. Evolution complete."
-      echo "[DISPATCHER] Run 'claude-evolve ideate' to generate more candidates."
-      echo "[DISPATCHER] Exiting main loop: no work remaining" >&2
-      break
-    fi
-  fi
-  
-  # Start workers if we have pending work and capacity
-  while [[ $pending_count -gt 0 && $active_workers -lt $MAX_WORKERS ]]; do
-    start_worker
-    active_workers=${#worker_pids[@]}
-    ((pending_count--))  # Optimistically assume this will be picked up
-  done
-  
-  # If no active workers and no pending work, we're done
-  if [[ $active_workers -eq 0 && $pending_count -eq 0 ]]; then
-    echo "[DISPATCHER] No active workers and no pending work. Evolution complete."
-    break
-  fi
-  
-  # Wait for any worker to finish
-  if [[ $active_workers -gt 0 ]]; then
-    # Poll for finished workers (macOS compatible)
-    sleep 5  # Check every 5 seconds
-    
-    # Check which workers have finished
-    for pid in "${worker_pids[@]}"; do
-      if ! kill -0 "$pid" 2>/dev/null; then
-        # Get exit status
-        wait "$pid" 2>/dev/null
-        exit_code=$?
-        handle_worker_exit "$pid" "$exit_code"
-      fi
-    done
-    
-    # Safety check - if we have workers but the array is corrupted
-    if [[ ${#worker_pids[@]} -eq 0 ]] && jobs -r | grep -q .; then
-      echo "[DISPATCHER] Warning: Lost track of workers but jobs still running!" >&2
-      echo "[DISPATCHER] Attempting to recover..." >&2
-      
-      # Try to recover PIDs from jobs
-      while read -r job_info; do
-        if [[ $job_info =~ \[([0-9]+)\][[:space:]]+([0-9]+) ]]; then
-          local recovered_pid="${BASH_REMATCH[2]}"
-          echo "[DISPATCHER] Recovered worker PID: $recovered_pid" >&2
-          worker_pids+=($recovered_pid)
-        fi
-      done < <(jobs -l)
-    fi
-  else
-    # No active workers but we might be waiting for ideation or have pending work
-    sleep 1
-  fi
-done
-
-echo "[DISPATCHER] Evolution run complete"
-
-# Final cleanup check
-if [[ ${#worker_pids[@]} -gt 0 ]]; then
-  echo "[DISPATCHER] Warning: ${#worker_pids[@]} workers still active at exit"
-  shutdown_workers
-fi