claude-evolve 1.8.51 → 1.9.1

package/lib/evolve_run.py

@@ -0,0 +1,402 @@
+#!/usr/bin/env python3
+"""
+Main orchestrator for claude-evolve.
+Manages worker processes and coordinates ideation.
+
+AIDEV-NOTE: This is the Python port of bin/claude-evolve-run.
+Exit codes:
+  0 - Success (evolution complete)
+  1 - Error
+  2 - Rate limit (workers should retry later)
+  3 - API exhausted
+"""
+
+import argparse
+import os
+import signal
+import subprocess
+import sys
+import time
+from dataclasses import dataclass
+from pathlib import Path
+from typing import List, Optional, Set
+
+# Add lib to path
+SCRIPT_DIR = Path(__file__).parent
+sys.path.insert(0, str(SCRIPT_DIR.parent))
+
+from lib.evolution_csv import EvolutionCSV
+
+
+@dataclass
+class RunConfig:
+    """Configuration for the run orchestrator."""
+    csv_path: str
+    evolution_dir: str
+    max_workers: int = 4
+    auto_ideate: bool = True
+    worker_timeout: int = 600
+    poll_interval: int = 5
+    min_completed_for_ideation: int = 3
+    config_path: Optional[str] = None
+
+
+class WorkerPool:
+    """Manages worker subprocess pool."""
+
+    def __init__(self, max_workers: int, worker_script: Path, config_path: Optional[str], timeout: int):
+        self.max_workers = max_workers
+        self.worker_script = worker_script
+        self.config_path = config_path
+        self.timeout = timeout
+        self.workers: dict[int, subprocess.Popen] = {}  # pid -> process
+
+    def spawn_worker(self) -> Optional[int]:
+        """Spawn a new worker. Returns pid or None if at capacity."""
+        if len(self.workers) >= self.max_workers:
+            return None
+
+        cmd = [sys.executable, str(self.worker_script)]
+        if self.config_path:
+            cmd.extend(['--config', self.config_path])
+        if self.timeout:
+            cmd.extend(['--timeout', str(self.timeout)])
+
+        try:
+            proc = subprocess.Popen(
+                cmd,
+                stdout=subprocess.PIPE,
+                stderr=subprocess.STDOUT,
+                text=True
+            )
+            self.workers[proc.pid] = proc
+            print(f"[RUN] Spawned worker {proc.pid}", file=sys.stderr)
+            return proc.pid
+        except Exception as e:
+            print(f"[RUN] Failed to spawn worker: {e}", file=sys.stderr)
+            return None
+
+    def cleanup_finished(self) -> List[int]:
+        """Clean up finished workers. Returns list of exit codes."""
+        exit_codes = []
+        finished_pids = []
+
+        for pid, proc in list(self.workers.items()):
+            ret = proc.poll()
+            if ret is not None:
+                finished_pids.append(pid)
+                exit_codes.append(ret)
+
+                # Log output
+                if proc.stdout:
+                    output = proc.stdout.read()
+                    if output:
+                        for line in output.strip().split('\n'):
+                            print(f"[WORKER-{pid}] {line}", file=sys.stderr)
+
+                print(f"[RUN] Worker {pid} exited with code {ret}", file=sys.stderr)
+
+        for pid in finished_pids:
+            del self.workers[pid]
+
+        return exit_codes
+
+    def shutdown(self, timeout: int = 10):
+        """Shutdown all workers gracefully."""
+        if not self.workers:
+            return
+
+        print(f"[RUN] Shutting down {len(self.workers)} workers...", file=sys.stderr)
+
+        # Send SIGTERM
+        for pid, proc in self.workers.items():
+            try:
+                proc.terminate()
+            except Exception:
+                pass
+
+        # Wait for graceful shutdown
+        deadline = time.time() + timeout
+        while self.workers and time.time() < deadline:
+            self.cleanup_finished()
+            if self.workers:
+                time.sleep(0.5)
+
+        # Force kill remaining
+        for pid, proc in list(self.workers.items()):
+            try:
+                proc.kill()
+                print(f"[RUN] Force killed worker {pid}", file=sys.stderr)
+            except Exception:
+                pass
+
+        self.workers.clear()
+
+    @property
+    def active_count(self) -> int:
+        return len(self.workers)
+
+
+class EvolutionRunner:
+    """Main evolution orchestrator."""
+
+    def __init__(self, config: RunConfig):
+        self.config = config
+        self.worker_script = SCRIPT_DIR / "evolve_worker.py"
+        self.ideate_script = SCRIPT_DIR / "evolve_ideate.py"
+        self.pool = WorkerPool(
+            max_workers=config.max_workers,
+            worker_script=self.worker_script,
+            config_path=config.config_path,
+            timeout=config.worker_timeout
+        )
+        self.api_limit_reached = False
+        self.shutdown_requested = False
+        self._setup_signal_handlers()
+
+    def _setup_signal_handlers(self):
+        """Setup signal handlers for graceful shutdown."""
+        signal.signal(signal.SIGTERM, self._handle_signal)
+        signal.signal(signal.SIGINT, self._handle_signal)
+
+    def _handle_signal(self, signum, frame):
+        """Handle termination signal."""
+        sig_name = signal.Signals(signum).name
+        print(f"\n[RUN] Received {sig_name}, shutting down...", file=sys.stderr)
+        self.shutdown_requested = True
+        self.pool.shutdown()
+        sys.exit(128 + signum)
+
+    def cleanup_csv(self):
+        """Clean up CSV at startup."""
+        print("[RUN] Cleaning up CSV...", file=sys.stderr)
+        with EvolutionCSV(self.config.csv_path) as csv:
+            # Remove duplicates
+            removed = csv.remove_duplicate_candidates()
+            if removed:
+                print(f"[RUN] Removed {removed} duplicate candidates", file=sys.stderr)
+
+            # Reset stuck candidates
+            reset = csv.reset_stuck_candidates()
+            if reset:
+                print(f"[RUN] Reset {reset} stuck candidates", file=sys.stderr)
+
+            # Clean corrupted status fields
+            fixed = csv.cleanup_corrupted_status_fields()
+            if fixed:
+                print(f"[RUN] Fixed {fixed} corrupted status fields", file=sys.stderr)
+
+    def ensure_baseline(self):
+        """Ensure baseline entry exists in CSV."""
+        with EvolutionCSV(self.config.csv_path) as csv:
+            info = csv.get_candidate_info('baseline-000')
+            if not info:
+                print("[RUN] Adding baseline-000 entry", file=sys.stderr)
+                csv.append_candidates([{
+                    'id': 'baseline-000',
+                    'basedOnId': '',
+                    'description': 'Original algorithm.py performance',
+                    'status': 'pending'
+                }])
+
+    def get_stats(self) -> dict:
+        """Get CSV statistics."""
+        with EvolutionCSV(self.config.csv_path) as csv:
+            return csv.get_csv_stats()
+
+    def should_ideate(self, stats: dict) -> bool:
+        """Check if we should run ideation."""
+        if not self.config.auto_ideate:
+            return False
+
+        # Need minimum completed algorithms to learn from
+        if stats['complete'] < self.config.min_completed_for_ideation:
+            print(f"[RUN] Not enough completed ({stats['complete']} < {self.config.min_completed_for_ideation})", file=sys.stderr)
+            return False
+
+        return True
+
+    def run_ideation(self) -> bool:
+        """Run ideation. Returns True on success."""
+        print("[RUN] Running ideation...", file=sys.stderr)
+
+        cmd = [sys.executable, str(self.ideate_script)]
+        if self.config.config_path:
+            cmd.extend(['--config', self.config.config_path])
+
+        try:
+            result = subprocess.run(
+                cmd,
+                capture_output=True,
+                text=True,
+                cwd=self.config.evolution_dir
+            )
+
+            # Log output
+            if result.stdout:
+                for line in result.stdout.strip().split('\n'):
+                    print(f"[IDEATE] {line}", file=sys.stderr)
+            if result.stderr:
+                for line in result.stderr.strip().split('\n'):
+                    print(f"[IDEATE] {line}", file=sys.stderr)
+
+            return result.returncode == 0
+
+        except Exception as e:
+            print(f"[RUN] Ideation failed: {e}", file=sys.stderr)
+            return False
+
+    def run(self) -> int:
+        """
+        Main orchestration loop.
+
+        Returns:
+            Exit code
+        """
+        print("[RUN] Starting evolution run", file=sys.stderr)
+        print(f"[RUN] Max workers: {self.config.max_workers}", file=sys.stderr)
+        print(f"[RUN] Auto ideate: {self.config.auto_ideate}", file=sys.stderr)
+
+        # Startup cleanup
+        self.cleanup_csv()
+        self.ensure_baseline()
+
+        iteration = 0
+
+        while not self.shutdown_requested:
+            iteration += 1
+
+            # Clean up finished workers
+            exit_codes = self.pool.cleanup_finished()
+
+            # Check for API limit
+            if 2 in exit_codes or 3 in exit_codes:
+                print("[RUN] API limit reached, waiting 5 minutes...", file=sys.stderr)
+                self.api_limit_reached = True
+                time.sleep(300)  # 5 minute wait
+                self.api_limit_reached = False
+                self.cleanup_csv()  # Reset stuck candidates
+                continue
+
+            # Periodic cleanup (every 5 iterations)
+            if iteration % 5 == 0 and self.pool.active_count == 0:
+                with EvolutionCSV(self.config.csv_path) as csv:
+                    csv.reset_stuck_candidates()
+
+            # Get stats
+            stats = self.get_stats()
+            print(f"[RUN] Stats: {stats['pending']} pending, {stats['complete']} complete, {stats['running']} running", file=sys.stderr)
+
+            # Check if we need ideation
+            if stats['pending'] == 0 and self.pool.active_count == 0:
+                # First reset any stuck candidates
+                with EvolutionCSV(self.config.csv_path) as csv:
+                    csv.reset_stuck_candidates()
+
+                # Re-check stats after reset
+                stats = self.get_stats()
+
+                if stats['pending'] == 0:
+                    if self.should_ideate(stats):
+                        if self.run_ideation():
+                            continue  # Loop back to check for new work
+                        else:
+                            print("[RUN] Ideation failed, waiting...", file=sys.stderr)
+                            time.sleep(30)
+                            continue
+                    else:
+                        print("[RUN] Evolution complete!", file=sys.stderr)
+                        break
+
+            # Spawn workers for pending work
+            while stats['pending'] > 0 and self.pool.active_count < self.config.max_workers:
+                pid = self.pool.spawn_worker()
+                if pid is None:
+                    break
+                stats['pending'] -= 1  # Optimistic decrement
+
+            # Sleep before next iteration
+            time.sleep(self.config.poll_interval)
+
+        # Cleanup
+        self.pool.shutdown()
+        print("[RUN] Exiting", file=sys.stderr)
+        return 0
+
+
+def load_config(config_path: Optional[str] = None) -> RunConfig:
+    """Load configuration from YAML."""
+    import yaml
+
+    # Find config
+    if config_path:
+        yaml_path = Path(config_path)
+    elif os.environ.get('CLAUDE_EVOLVE_CONFIG'):
+        yaml_path = Path(os.environ['CLAUDE_EVOLVE_CONFIG'])
+    else:
+        yaml_path = Path('evolution/config.yaml')
+        if not yaml_path.exists():
+            yaml_path = Path('config.yaml')
+
+    if not yaml_path.exists():
+        raise FileNotFoundError(f"Config not found: {yaml_path}")
+
+    with open(yaml_path) as f:
+        data = yaml.safe_load(f) or {}
+
+    base_dir = yaml_path.parent
+
+    def resolve(path: str) -> str:
+        p = Path(path)
+        if not p.is_absolute():
+            p = base_dir / p
+        return str(p.resolve())
+
+    parallel = data.get('parallel', {})
+
+    return RunConfig(
+        csv_path=resolve(data.get('csv_file', 'evolution.csv')),
+        evolution_dir=str(base_dir.resolve()),
+        max_workers=parallel.get('max_workers', 4),
+        auto_ideate=data.get('auto_ideate', True),
+        worker_timeout=data.get('timeout_seconds', 600),
+        poll_interval=parallel.get('poll_interval', 5),
+        min_completed_for_ideation=data.get('min_completed_for_ideation', 3),
+        config_path=str(yaml_path.resolve())
+    )
+
+
+def main():
+    parser = argparse.ArgumentParser(description='Claude Evolve Runner')
+    parser.add_argument('--config', help='Path to config.yaml')
+    parser.add_argument('--parallel', type=int, help='Max parallel workers')
+    parser.add_argument('--sequential', action='store_true', help='Run sequentially (1 worker)')
+    parser.add_argument('--timeout', type=int, help='Worker timeout in seconds')
+    args = parser.parse_args()
+
+    try:
+        config = load_config(args.config)
+
+        if args.sequential:
+            config.max_workers = 1
+        elif args.parallel:
+            config.max_workers = args.parallel
+
+        if args.timeout:
+            config.worker_timeout = args.timeout
+
+        runner = EvolutionRunner(config)
+        sys.exit(runner.run())
+
+    except FileNotFoundError as e:
+        print(f"Error: {e}", file=sys.stderr)
+        sys.exit(1)
+    except Exception as e:
+        print(f"Error: {e}", file=sys.stderr)
+        import traceback
+        traceback.print_exc()
+        sys.exit(1)
+
+
+if __name__ == '__main__':
+    main()