claude-evolve 1.9.5 → 1.9.7

package/lib/ai_cli.py

@@ -5,11 +5,13 @@ AIDEV-NOTE: This keeps ai-cli.sh as the source of truth for model configs and ti
 """
 
 import os
+import random
 import subprocess
 import sys
 import tempfile
+import time
 from pathlib import Path
-from typing import Optional, Tuple
+from typing import Optional, Tuple, List
 
 # Path to ai-cli.sh relative to this file
 SCRIPT_DIR = Path(__file__).parent
@@ -155,6 +157,178 @@ def call_ai(
         raise AIError(f"Failed to call AI: {e}")
 
 
+def get_models_for_command(command: str) -> List[str]:
+    """
+    Get the list of available models for a command.
+
+    Args:
+        command: Either "run" or "ideate"
+
+    Returns:
+        List of model names
+    """
+    bash_script = f'''
+        source "{SCRIPT_DIR}/config.sh"
+        load_config
+        case "$1" in
+            run) echo "$LLM_RUN" ;;
+            ideate) echo "$LLM_IDEATE" ;;
+        esac
+    '''
+
+    result = subprocess.run(
+        ["bash", "-c", bash_script, "bash", command],
+        capture_output=True,
+        text=True
+    )
+
+    if result.returncode != 0:
+        return []
+
+    model_list = result.stdout.strip()
+    if not model_list:
+        return []
+
+    return model_list.split()
+
+
+def call_ai_model(
+    prompt: str,
+    model_name: str,
+    working_dir: Optional[str] = None,
+    env_vars: Optional[dict] = None
+) -> Tuple[str, str]:
+    """
+    Call a specific AI model.
+
+    Args:
+        prompt: The prompt to send to the AI
+        model_name: The specific model to use
+        working_dir: Directory to run the command in
+        env_vars: Additional environment variables
+
+    Returns:
+        Tuple of (output, model_name)
+
+    Raises:
+        TimeoutError, RateLimitError, APIExhaustedError, AIError
+    """
+    bash_script = f'''
+        source "{SCRIPT_DIR}/config.sh"
+        load_config
+        source "{AI_CLI_PATH}"
+        call_ai_model_configured "$1" "$2"
+    '''
+
+    env = os.environ.copy()
+    if working_dir:
+        env['CLAUDE_EVOLVE_WORKING_DIR'] = working_dir
+    if env_vars:
+        env.update(env_vars)
+
+    try:
+        result = subprocess.run(
+            ["bash", "-c", bash_script, "bash", model_name, prompt],
+            capture_output=True,
+            text=True,
+            cwd=working_dir,
+            env=env
+        )
+
+        output = result.stdout
+        stderr = result.stderr
+        exit_code = result.returncode
+
+        # Print stderr (contains debug info)
+        if stderr:
+            for line in stderr.strip().split('\n'):
+                if line:
+                    print(f"  {line}", file=sys.stderr)
+
+        # Handle exit codes
+        if exit_code == 124:
+            raise TimeoutError(f"AI call timed out (model: {model_name})")
+        elif exit_code == 2:
+            raise RateLimitError(f"Rate limit hit (model: {model_name})")
+        elif exit_code == 3:
+            raise APIExhaustedError(f"API quota exhausted (model: {model_name})")
+        elif exit_code != 0:
+            raise AIError(f"AI call failed with exit code {exit_code}: {stderr}")
+
+        return output, model_name
+
+    except subprocess.SubprocessError as e:
+        raise AIError(f"Failed to call AI: {e}")
+
+
+def call_ai_with_backoff(
+    prompt: str,
+    command: str = "ideate",
+    working_dir: Optional[str] = None,
+    env_vars: Optional[dict] = None,
+    max_rounds: int = 10,
+    initial_wait: int = 60,
+    max_wait: int = 600
+) -> Tuple[str, str]:
+    """
+    Call AI with round-based retries and exponential backoff.
+
+    AIDEV-NOTE: This is the robust retry mechanism for handling rate limits.
+    - Tries each model in the pool (shuffled order)
+    - If all models fail in a round, waits with exponential backoff
+    - Keeps going until success or max_rounds exhausted
+
+    Args:
+        prompt: The prompt to send
+        command: "run" or "ideate" - determines model pool
+        working_dir: Directory for file operations
+        env_vars: Additional environment variables
+        max_rounds: Maximum number of full rounds to attempt
+        initial_wait: Initial wait time in seconds after first failed round
+        max_wait: Maximum wait time in seconds between rounds
+
+    Returns:
+        Tuple of (output, model_name)
+
+    Raises:
+        AIError: If all rounds exhausted without success
+    """
+    models = get_models_for_command(command)
+    if not models:
+        raise AIError(f"No models configured for command: {command}")
+
+    wait_time = initial_wait
+    last_errors = {}
+
+    for round_num in range(max_rounds):
+        # Shuffle models each round for fairness
+        shuffled_models = models.copy()
+        random.shuffle(shuffled_models)
+
+        print(f"[AI] Round {round_num + 1}/{max_rounds}: trying {len(shuffled_models)} models", file=sys.stderr)
+
+        for model in shuffled_models:
+            try:
+                output, model_name = call_ai_model(prompt, model, working_dir, env_vars)
+                if round_num > 0:
+                    print(f"[AI] Succeeded on round {round_num + 1} with {model}", file=sys.stderr)
+                return output, model_name
+            except AIError as e:
+                last_errors[model] = str(e)
+                # Continue to next model
+
+        # All models failed in this round
+        if round_num < max_rounds - 1:
+            print(f"[AI] All models failed in round {round_num + 1}, waiting {wait_time}s before retry...", file=sys.stderr)
+            time.sleep(wait_time)
+            # Exponential backoff: 60 -> 120 -> 240 -> 480 (capped at max_wait)
+            wait_time = min(wait_time * 2, max_wait)
+
+    # All rounds exhausted
+    error_summary = "; ".join(f"{m}: {e[:50]}" for m, e in list(last_errors.items())[:3])
+    raise AIError(f"All {max_rounds} rounds exhausted. Last errors: {error_summary}")
+
+
 def call_ai_for_file_edit(
     prompt: str,
     file_path: str,

package/lib/evolve_ideate.py

@@ -23,7 +23,7 @@ SCRIPT_DIR = Path(__file__).parent
 sys.path.insert(0, str(SCRIPT_DIR.parent))
 
 from lib.evolution_csv import EvolutionCSV
-from lib.ai_cli import call_ai, get_git_protection_warning, AIError
+from lib.ai_cli import call_ai_with_backoff, get_git_protection_warning, AIError
 from lib.embedding import check_novelty as check_embedding_novelty, get_embedding, set_cache_file, save_cache
 
 
@@ -47,6 +47,13 @@ class IdeationConfig:
     novelty_enabled: bool = True
     novelty_threshold: float = 0.92
 
+    # Retry configuration with exponential backoff
+    # AIDEV-NOTE: This implements round-based retries like the shell version.
+    # Each round tries ALL models. If all fail, wait and retry.
+    max_rounds: int = 10          # Max full rounds of all models
+    initial_wait: int = 60        # Seconds to wait after first failed round
+    max_wait: int = 600           # Max wait between rounds (10 minutes)
+
 
 @dataclass
 class Idea:
@@ -85,8 +92,13 @@ class IdeationStrategy(ABC):
         """Build the AI prompt."""
         pass
 
-    def generate(self, context: IdeationContext, count: int) -> List[Idea]:
-        """Generate ideas using this strategy."""
+    def generate(self, context: IdeationContext, count: int,
+                 max_rounds: int = 10, initial_wait: int = 60, max_wait: int = 600) -> List[Idea]:
+        """Generate ideas using this strategy with round-based retry and backoff.
+
+        AIDEV-NOTE: Uses call_ai_with_backoff for robust retry handling.
+        Each round tries ALL models. If all fail, waits with exponential backoff.
+        """
         if count <= 0:
             return []
 
@@ -110,8 +122,15 @@ class IdeationStrategy(ABC):
             # Build prompt
             prompt = self.build_prompt(context, ids, temp_csv.name)
 
-            # Call AI
-            output, model = call_ai(prompt, command="ideate", working_dir=self.config.evolution_dir)
+            # Call AI with round-based retry and backoff
+            output, model = call_ai_with_backoff(
+                prompt,
+                command="ideate",
+                working_dir=self.config.evolution_dir,
+                max_rounds=max_rounds,
+                initial_wait=initial_wait,
+                max_wait=max_wait
+            )
 
             # Parse results from modified CSV
             ideas = self._parse_results(temp_csv, ids)
@@ -120,12 +139,15 @@ class IdeationStrategy(ABC):
                 # Record model used
                 for idea in ideas:
                     idea.strategy = f"{self.name} ({model})"
-
-            return ideas
+                return ideas
+            else:
+                print(f"[IDEATE] AI completed but no ideas parsed from output", file=sys.stderr)
+                return []
 
         except AIError as e:
-            print(f"[IDEATE] AI error in {self.name}: {e}", file=sys.stderr)
+            print(f"[IDEATE] All retries exhausted in {self.name}: {e}", file=sys.stderr)
             return []
+
         finally:
             temp_csv.unlink(missing_ok=True)
 
@@ -382,7 +404,12 @@ class Ideator:
             if count <= 0:
                 continue
 
-            ideas = strategy.generate(context, count)
+            ideas = strategy.generate(
+                context, count,
+                max_rounds=self.config.max_rounds,
+                initial_wait=self.config.initial_wait,
+                max_wait=self.config.max_wait
+            )
 
             if ideas:
                 strategies_succeeded += 1
@@ -472,7 +499,10 @@ def load_config(config_path: Optional[str] = None) -> IdeationConfig:
         crossover_hybrid=ideation.get('crossover_hybrid', 4),
         num_elites=ideation.get('num_elites', 3),
         novelty_enabled=novelty.get('enabled', True),
-        novelty_threshold=novelty.get('threshold', 0.92)
+        novelty_threshold=novelty.get('threshold', 0.92),
+        max_rounds=ideation.get('max_rounds', 10),
+        initial_wait=ideation.get('initial_wait', 60),
+        max_wait=ideation.get('max_wait', 600)
     )
 
 

package/lib/evolve_run.py

@@ -63,12 +63,9 @@ class WorkerPool:
             cmd.extend(['--timeout', str(self.timeout)])
 
         try:
-            proc = subprocess.Popen(
-                cmd,
-                stdout=subprocess.PIPE,
-                stderr=subprocess.STDOUT,
-                text=True
-            )
+            # Don't capture output - let it stream directly to terminal
+            # This provides real-time visibility into which models are being used
+            proc = subprocess.Popen(cmd)
             self.workers[proc.pid] = proc
             print(f"[RUN] Spawned worker {proc.pid}", file=sys.stderr)
             return proc.pid
@@ -86,14 +83,6 @@ class WorkerPool:
             if ret is not None:
                 finished_pids.append(pid)
                 exit_codes.append(ret)
-
-                # Log output
-                if proc.stdout:
-                    output = proc.stdout.read()
-                    if output:
-                        for line in output.strip().split('\n'):
-                            print(f"[WORKER-{pid}] {line}", file=sys.stderr)
-
                 print(f"[RUN] Worker {pid} exited with code {ret}", file=sys.stderr)
 
         for pid in finished_pids:

package/lib/evolve_worker.py

@@ -31,7 +31,7 @@ SCRIPT_DIR = Path(__file__).parent
 sys.path.insert(0, str(SCRIPT_DIR.parent))
 
 from lib.evolution_csv import EvolutionCSV
-from lib.ai_cli import call_ai, get_git_protection_warning, AIError, RateLimitError, APIExhaustedError, TimeoutError
+from lib.ai_cli import call_ai_with_backoff, get_git_protection_warning, AIError
 
 
 @dataclass
@@ -46,8 +46,11 @@ class Config:
     python_cmd: str = "python3"
     memory_limit_mb: int = 0
     timeout_seconds: int = 600
-    max_ai_retries: int = 3
     max_candidates: int = 5
+    # Retry configuration with exponential backoff
+    max_rounds: int = 10
+    initial_wait: int = 60
+    max_wait: int = 600
 
 
 @dataclass
@@ -139,52 +142,42 @@ This is especially important for models with smaller context windows (like GLM).
 
 CRITICAL: If you do not know how to implement what was asked for, or if the requested change is unclear or not feasible, you MUST refuse to make any changes. DO NOT modify the code if you are uncertain about the implementation. Simply respond that you cannot implement the requested change and explain why. It is better to refuse than to make incorrect or random changes."""
 
-    def _call_ai_with_retries(self, prompt: str, target_file: Path, source_file: Path) -> Tuple[bool, str]:
+    def _call_ai_with_backoff(self, prompt: str, target_file: Path) -> Tuple[bool, str]:
         """
-        Call AI with retries.
+        Call AI with round-based retry and exponential backoff.
+
+        AIDEV-NOTE: Uses call_ai_with_backoff which tries all models in the pool,
+        then waits with exponential backoff if all fail, and repeats.
 
         Returns:
             Tuple of (success, model_name)
         """
-        for attempt in range(1, self.config.max_ai_retries + 1):
-            print(f"[WORKER-{os.getpid()}] AI attempt {attempt}/{self.config.max_ai_retries}", file=sys.stderr)
+        # Get file hash before AI call
+        hash_before = self._file_hash(target_file) if target_file.exists() else None
 
-            # Re-copy source if this is a retry
-            if attempt > 1:
-                print(f"[WORKER-{os.getpid()}] Re-copying source file for retry", file=sys.stderr)
-                shutil.copy(source_file, target_file)
+        try:
+            output, model = call_ai_with_backoff(
+                prompt,
+                command="run",
+                working_dir=self.config.evolution_dir,
+                max_rounds=self.config.max_rounds,
+                initial_wait=self.config.initial_wait,
+                max_wait=self.config.max_wait
+            )
 
-            # Get file hash before AI call
-            hash_before = self._file_hash(target_file) if target_file.exists() else None
+            # Check if file was modified
+            hash_after = self._file_hash(target_file) if target_file.exists() else None
 
-            try:
-                output, model = call_ai(prompt, command="run", working_dir=self.config.evolution_dir)
-
-                # Check if file was modified
-                hash_after = self._file_hash(target_file) if target_file.exists() else None
-
-                if hash_before != hash_after and hash_after is not None:
-                    print(f"[WORKER-{os.getpid()}] AI successfully modified file (model: {model})", file=sys.stderr)
-                    return True, model
-                else:
-                    print(f"[WORKER-{os.getpid()}] AI did not modify file", file=sys.stderr)
-
-            except RateLimitError as e:
-                print(f"[WORKER-{os.getpid()}] Rate limit: {e}", file=sys.stderr)
-                raise  # Propagate to caller
-            except APIExhaustedError as e:
-                print(f"[WORKER-{os.getpid()}] API exhausted: {e}", file=sys.stderr)
-                raise  # Propagate to caller
-            except TimeoutError as e:
-                print(f"[WORKER-{os.getpid()}] Timeout: {e}", file=sys.stderr)
-            except AIError as e:
-                print(f"[WORKER-{os.getpid()}] AI error: {e}", file=sys.stderr)
-
-            if attempt < self.config.max_ai_retries:
-                print(f"[WORKER-{os.getpid()}] Will retry with different model...", file=sys.stderr)
-                time.sleep(2)
-
-        return False, ""
+            if hash_before != hash_after and hash_after is not None:
+                print(f"[WORKER-{os.getpid()}] AI successfully modified file (model: {model})", file=sys.stderr)
+                return True, model
+            else:
+                print(f"[WORKER-{os.getpid()}] AI completed but did not modify file", file=sys.stderr)
+                return False, model
+
+        except AIError as e:
+            print(f"[WORKER-{os.getpid()}] All AI retries exhausted: {e}", file=sys.stderr)
+            return False, ""
 
     def _file_hash(self, path: Path) -> Optional[str]:
         """Get file hash."""
@@ -329,33 +322,19 @@ CRITICAL: If you do not know how to implement what was asked for, or if the requ
             print(f"[WORKER-{os.getpid()}] Copying {source_file} to {target_file}", file=sys.stderr)
             shutil.copy(source_file, target_file)
 
-            # Call AI to modify
+            # Call AI to modify (uses round-based retry with backoff)
             prompt = self._build_prompt(candidate, target_file.name)
+            success, model = self._call_ai_with_backoff(prompt, target_file)
 
-            try:
-                success, model = self._call_ai_with_retries(prompt, target_file, source_file)
-
-                if not success:
-                    print(f"[WORKER-{os.getpid()}] AI failed after all retries", file=sys.stderr)
-                    target_file.unlink(missing_ok=True)
-                    return 77  # AI generation failed
-
-                # Record model used
-                if model:
-                    with EvolutionCSV(self.config.csv_path) as csv:
-                        csv.update_candidate_field(candidate.id, 'run-LLM', model)
-
-            except RateLimitError:
+            if not success:
+                print(f"[WORKER-{os.getpid()}] AI failed after all retries", file=sys.stderr)
                 target_file.unlink(missing_ok=True)
-                with EvolutionCSV(self.config.csv_path) as csv:
-                    csv.update_candidate_status(candidate.id, 'pending')
-                return 2  # Rate limit
+                return 77  # AI generation failed
 
-            except APIExhaustedError:
-                target_file.unlink(missing_ok=True)
+            # Record model used
+            if model:
                 with EvolutionCSV(self.config.csv_path) as csv:
-                    csv.update_candidate_status(candidate.id, 'pending')
-                return 3  # API exhausted
+                    csv.update_candidate_field(candidate.id, 'run-LLM', model)
 
             # Check syntax
             if not self._check_syntax(target_file):
@@ -475,6 +454,8 @@ def load_config_from_yaml(config_path: Optional[str] = None) -> Config:
             p = base_dir / p
         return str(p.resolve())
 
+    ideation = data.get('ideation', {})
+
     return Config(
         csv_path=resolve(data.get('csv_file', 'evolution.csv')),
         evolution_dir=str(base_dir.resolve()),
@@ -485,8 +466,10 @@ def load_config_from_yaml(config_path: Optional[str] = None) -> Config:
         python_cmd=data.get('python_cmd', 'python3'),
         memory_limit_mb=data.get('memory_limit_mb', 0),
         timeout_seconds=data.get('timeout_seconds', 600),
-        max_ai_retries=data.get('max_retries', 3),
-        max_candidates=data.get('worker_max_candidates', 5)
+        max_candidates=data.get('worker_max_candidates', 5),
+        max_rounds=ideation.get('max_rounds', 10),
+        initial_wait=ideation.get('initial_wait', 60),
+        max_wait=ideation.get('max_wait', 600)
     )
 
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "claude-evolve",
-  "version": "1.9.5",
+  "version": "1.9.7",
   "bin": {
     "claude-evolve": "bin/claude-evolve",
     "claude-evolve-main": "bin/claude-evolve-main",