claude-evolve 1.9.5 → 1.9.6

package/lib/ai_cli.py

@@ -5,11 +5,13 @@ AIDEV-NOTE: This keeps ai-cli.sh as the source of truth for model configs and ti
 """
 
 import os
+import random
 import subprocess
 import sys
 import tempfile
+import time
 from pathlib import Path
-from typing import Optional, Tuple
+from typing import Optional, Tuple, List
 
 # Path to ai-cli.sh relative to this file
 SCRIPT_DIR = Path(__file__).parent
@@ -155,6 +157,178 @@ def call_ai(
         raise AIError(f"Failed to call AI: {e}")
 
 
+def get_models_for_command(command: str) -> List[str]:
+    """
+    Get the list of available models for a command.
+
+    Args:
+        command: Either "run" or "ideate"
+
+    Returns:
+        List of model names
+    """
+    bash_script = f'''
+        source "{SCRIPT_DIR}/config.sh"
+        load_config
+        case "$1" in
+            run) echo "$LLM_RUN" ;;
+            ideate) echo "$LLM_IDEATE" ;;
+        esac
+    '''
+
+    result = subprocess.run(
+        ["bash", "-c", bash_script, "bash", command],
+        capture_output=True,
+        text=True
+    )
+
+    if result.returncode != 0:
+        return []
+
+    model_list = result.stdout.strip()
+    if not model_list:
+        return []
+
+    return model_list.split()
+
+
+def call_ai_model(
+    prompt: str,
+    model_name: str,
+    working_dir: Optional[str] = None,
+    env_vars: Optional[dict] = None
+) -> Tuple[str, str]:
+    """
+    Call a specific AI model.
+
+    Args:
+        prompt: The prompt to send to the AI
+        model_name: The specific model to use
+        working_dir: Directory to run the command in
+        env_vars: Additional environment variables
+
+    Returns:
+        Tuple of (output, model_name)
+
+    Raises:
+        TimeoutError, RateLimitError, APIExhaustedError, AIError
+    """
+    bash_script = f'''
+        source "{SCRIPT_DIR}/config.sh"
+        load_config
+        source "{AI_CLI_PATH}"
+        call_ai_model_configured "$1" "$2"
+    '''
+
+    env = os.environ.copy()
+    if working_dir:
+        env['CLAUDE_EVOLVE_WORKING_DIR'] = working_dir
+    if env_vars:
+        env.update(env_vars)
+
+    try:
+        result = subprocess.run(
+            ["bash", "-c", bash_script, "bash", model_name, prompt],
+            capture_output=True,
+            text=True,
+            cwd=working_dir,
+            env=env
+        )
+
+        output = result.stdout
+        stderr = result.stderr
+        exit_code = result.returncode
+
+        # Print stderr (contains debug info)
+        if stderr:
+            for line in stderr.strip().split('\n'):
+                if line:
+                    print(f"  {line}", file=sys.stderr)
+
+        # Handle exit codes
+        if exit_code == 124:
+            raise TimeoutError(f"AI call timed out (model: {model_name})")
+        elif exit_code == 2:
+            raise RateLimitError(f"Rate limit hit (model: {model_name})")
+        elif exit_code == 3:
+            raise APIExhaustedError(f"API quota exhausted (model: {model_name})")
+        elif exit_code != 0:
+            raise AIError(f"AI call failed with exit code {exit_code}: {stderr}")
+
+        return output, model_name
+
+    except subprocess.SubprocessError as e:
+        raise AIError(f"Failed to call AI: {e}")
+
+
+def call_ai_with_backoff(
+    prompt: str,
+    command: str = "ideate",
+    working_dir: Optional[str] = None,
+    env_vars: Optional[dict] = None,
+    max_rounds: int = 10,
+    initial_wait: int = 60,
+    max_wait: int = 600
+) -> Tuple[str, str]:
+    """
+    Call AI with round-based retries and exponential backoff.
+
+    AIDEV-NOTE: This is the robust retry mechanism for handling rate limits.
+    - Tries each model in the pool (shuffled order)
+    - If all models fail in a round, waits with exponential backoff
+    - Keeps going until success or max_rounds exhausted
+
+    Args:
+        prompt: The prompt to send
+        command: "run" or "ideate" - determines model pool
+        working_dir: Directory for file operations
+        env_vars: Additional environment variables
+        max_rounds: Maximum number of full rounds to attempt
+        initial_wait: Initial wait time in seconds after first failed round
+        max_wait: Maximum wait time in seconds between rounds
+
+    Returns:
+        Tuple of (output, model_name)
+
+    Raises:
+        AIError: If all rounds exhausted without success
+    """
+    models = get_models_for_command(command)
+    if not models:
+        raise AIError(f"No models configured for command: {command}")
+
+    wait_time = initial_wait
+    last_errors = {}
+
+    for round_num in range(max_rounds):
+        # Shuffle models each round for fairness
+        shuffled_models = models.copy()
+        random.shuffle(shuffled_models)
+
+        print(f"[AI] Round {round_num + 1}/{max_rounds}: trying {len(shuffled_models)} models", file=sys.stderr)
+
+        for model in shuffled_models:
+            try:
+                output, model_name = call_ai_model(prompt, model, working_dir, env_vars)
+                if round_num > 0:
+                    print(f"[AI] Succeeded on round {round_num + 1} with {model}", file=sys.stderr)
+                return output, model_name
+            except AIError as e:
+                last_errors[model] = str(e)
+                # Continue to next model
+
+        # All models failed in this round
+        if round_num < max_rounds - 1:
+            print(f"[AI] All models failed in round {round_num + 1}, waiting {wait_time}s before retry...", file=sys.stderr)
+            time.sleep(wait_time)
+            # Exponential backoff: 60 -> 120 -> 240 -> 480 (capped at max_wait)
+            wait_time = min(wait_time * 2, max_wait)
+
+    # All rounds exhausted
+    error_summary = "; ".join(f"{m}: {e[:50]}" for m, e in list(last_errors.items())[:3])
+    raise AIError(f"All {max_rounds} rounds exhausted. Last errors: {error_summary}")
+
+
 def call_ai_for_file_edit(
     prompt: str,
     file_path: str,

package/lib/evolve_ideate.py

@@ -23,7 +23,7 @@ SCRIPT_DIR = Path(__file__).parent
 sys.path.insert(0, str(SCRIPT_DIR.parent))
 
 from lib.evolution_csv import EvolutionCSV
-from lib.ai_cli import call_ai, get_git_protection_warning, AIError
+from lib.ai_cli import call_ai_with_backoff, get_git_protection_warning, AIError
 from lib.embedding import check_novelty as check_embedding_novelty, get_embedding, set_cache_file, save_cache
 
 
@@ -47,6 +47,13 @@ class IdeationConfig:
     novelty_enabled: bool = True
     novelty_threshold: float = 0.92
 
+    # Retry configuration with exponential backoff
+    # AIDEV-NOTE: This implements round-based retries like the shell version.
+    # Each round tries ALL models. If all fail, wait and retry.
+    max_rounds: int = 10          # Max full rounds of all models
+    initial_wait: int = 60        # Seconds to wait after first failed round
+    max_wait: int = 600           # Max wait between rounds (10 minutes)
+
 
 @dataclass
 class Idea:
@@ -85,8 +92,13 @@ class IdeationStrategy(ABC):
         """Build the AI prompt."""
         pass
 
-    def generate(self, context: IdeationContext, count: int) -> List[Idea]:
-        """Generate ideas using this strategy."""
+    def generate(self, context: IdeationContext, count: int,
+                 max_rounds: int = 10, initial_wait: int = 60, max_wait: int = 600) -> List[Idea]:
+        """Generate ideas using this strategy with round-based retry and backoff.
+
+        AIDEV-NOTE: Uses call_ai_with_backoff for robust retry handling.
+        Each round tries ALL models. If all fail, waits with exponential backoff.
+        """
         if count <= 0:
             return []
 
@@ -110,8 +122,15 @@ class IdeationStrategy(ABC):
             # Build prompt
             prompt = self.build_prompt(context, ids, temp_csv.name)
 
-            # Call AI
-            output, model = call_ai(prompt, command="ideate", working_dir=self.config.evolution_dir)
+            # Call AI with round-based retry and backoff
+            output, model = call_ai_with_backoff(
+                prompt,
+                command="ideate",
+                working_dir=self.config.evolution_dir,
+                max_rounds=max_rounds,
+                initial_wait=initial_wait,
+                max_wait=max_wait
+            )
 
             # Parse results from modified CSV
             ideas = self._parse_results(temp_csv, ids)
@@ -120,12 +139,15 @@ class IdeationStrategy(ABC):
                 # Record model used
                 for idea in ideas:
                     idea.strategy = f"{self.name} ({model})"
-
-            return ideas
+                return ideas
+            else:
+                print(f"[IDEATE] AI completed but no ideas parsed from output", file=sys.stderr)
+                return []
 
         except AIError as e:
-            print(f"[IDEATE] AI error in {self.name}: {e}", file=sys.stderr)
+            print(f"[IDEATE] All retries exhausted in {self.name}: {e}", file=sys.stderr)
             return []
+
         finally:
             temp_csv.unlink(missing_ok=True)
 
@@ -382,7 +404,12 @@ class Ideator:
             if count <= 0:
                 continue
 
-            ideas = strategy.generate(context, count)
+            ideas = strategy.generate(
+                context, count,
+                max_rounds=self.config.max_rounds,
+                initial_wait=self.config.initial_wait,
+                max_wait=self.config.max_wait
+            )
 
             if ideas:
                 strategies_succeeded += 1
@@ -472,7 +499,10 @@ def load_config(config_path: Optional[str] = None) -> IdeationConfig:
         crossover_hybrid=ideation.get('crossover_hybrid', 4),
         num_elites=ideation.get('num_elites', 3),
         novelty_enabled=novelty.get('enabled', True),
-        novelty_threshold=novelty.get('threshold', 0.92)
+        novelty_threshold=novelty.get('threshold', 0.92),
+        max_rounds=ideation.get('max_rounds', 10),
+        initial_wait=ideation.get('initial_wait', 60),
+        max_wait=ideation.get('max_wait', 600)
     )
 
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "claude-evolve",
-  "version": "1.9.5",
+  "version": "1.9.6",
   "bin": {
     "claude-evolve": "bin/claude-evolve",
     "claude-evolve-main": "bin/claude-evolve-main",