claude-evolve 1.8.51 → 1.9.1

package/bin/claude-evolve-ideate-py

@@ -0,0 +1,15 @@
+#!/usr/bin/env python3
+"""
+Python implementation of claude-evolve-ideate.
+Test this alongside the shell version before switching.
+"""
+import sys
+import os
+
+# Add lib to path
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', 'lib'))
+
+from evolve_ideate import main
+
+if __name__ == '__main__':
+    sys.exit(main())

package/bin/claude-evolve-main

@@ -196,10 +196,18 @@ ideate)
   shift
   exec "$SCRIPT_DIR/claude-evolve-ideate" "$@"
   ;;
+ideate-py)
+  shift
+  exec "$SCRIPT_DIR/claude-evolve-ideate-py" "$@"
+  ;;
 run)
   shift
   exec "$SCRIPT_DIR/claude-evolve-run" "$@"
   ;;
+run-py)
+  shift
+  exec "$SCRIPT_DIR/claude-evolve-run-py" "$@"
+  ;;
 analyze)
   shift
   exec "$SCRIPT_DIR/claude-evolve-analyze" "$@"
@@ -232,6 +240,10 @@ killall)
   shift
   exec "$SCRIPT_DIR/claude-evolve-killall" "$@"
   ;;
+worker-py)
+  shift
+  exec "$SCRIPT_DIR/claude-evolve-worker-py" "$@"
+  ;;
 *)
   echo "Unknown command: ${1:-}"
   echo

package/bin/claude-evolve-run-py

@@ -0,0 +1,15 @@
+#!/usr/bin/env python3
+"""
+Python implementation of claude-evolve-run.
+Test this alongside the shell version before switching.
+"""
+import sys
+import os
+
+# Add lib to path
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', 'lib'))
+
+from evolve_run import main
+
+if __name__ == '__main__':
+    sys.exit(main())

package/bin/claude-evolve-worker-py

@@ -0,0 +1,15 @@
+#!/usr/bin/env python3
+"""
+Python implementation of claude-evolve-worker.
+Test this alongside the shell version before switching.
+"""
+import sys
+import os
+
+# Add lib to path
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', 'lib'))
+
+from evolve_worker import main
+
+if __name__ == '__main__':
+    sys.exit(main())

package/lib/ai_cli.py

@@ -0,0 +1,196 @@
+#!/usr/bin/env python3
+"""
+Python wrapper around ai-cli.sh for AI model invocation.
+AIDEV-NOTE: This keeps ai-cli.sh as the source of truth for model configs and timeouts.
+"""
+
+import os
+import subprocess
+import tempfile
+from pathlib import Path
+from typing import Optional, Tuple
+
+# Path to ai-cli.sh relative to this file
+SCRIPT_DIR = Path(__file__).parent
+AI_CLI_PATH = SCRIPT_DIR / "ai-cli.sh"
+
+
+class AIError(Exception):
+    """Base exception for AI errors."""
+    pass
+
+
+class RateLimitError(AIError):
+    """Rate limit hit - should retry later."""
+    pass
+
+
+class APIExhaustedError(AIError):
+    """API quota exhausted - stop processing."""
+    pass
+
+
+class TimeoutError(AIError):
+    """AI call timed out."""
+    pass
+
+
+def get_git_protection_warning() -> str:
+    """Get the git protection warning that must prefix all AI prompts."""
+    return '''!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+!!!
+!!! ⛔ ABSOLUTE PROHIBITION - READ THIS FIRST ⛔
+!!!
+!!! YOU ARE STRICTLY FORBIDDEN FROM USING ANY GIT COMMANDS WHATSOEVER
+!!!
+!!! ❌ FORBIDDEN: git commit, git add, git reset, git checkout, git revert,
+!!!              git branch, git merge, git stash, git clean, git push, git pull
+!!!              OR ANY OTHER COMMAND STARTING WITH 'git'
+!!!
+!!! ⚠️  WHY: This runs in production. Git operations have caused DATA LOSS.
+!!!          Multiple times AIs have corrupted evolution runs with git commands.
+!!!          Version control is ONLY managed by the human operator.
+!!!
+!!! ✅ WHAT YOU CAN DO: Edit files directly using file editing tools ONLY.
+!!!                     Never touch version control. Ever.
+!!!
+!!! 💀 IF YOU USE GIT: You will corrupt the entire evolution run and lose data.
+!!!                    This is an automated system. No git operations allowed.
+!!!
+!!! 🚨 CONSEQUENCES: If you execute ANY git command, the human operator will be
+!!!                  forced to SHUT DOWN ALL AI-BASED EVOLUTION WORK and switch
+!!!                  to manual-only mode. You will cause the termination of this
+!!!                  entire automated evolution system. DO NOT BE THAT AI.
+!!!
+!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+'''
+
+
+def call_ai(
+    prompt: str,
+    command: str = "run",
+    working_dir: Optional[str] = None,
+    env_vars: Optional[dict] = None
+) -> Tuple[str, str]:
+    """
+    Call AI using the configured models via ai-cli.sh.
+
+    Args:
+        prompt: The prompt to send to the AI
+        command: Either "run" or "ideate" - determines which model pool to use
+        working_dir: Directory to run the command in (for file editing)
+        env_vars: Additional environment variables to pass
+
+    Returns:
+        Tuple of (output, model_name)
+
+    Raises:
+        TimeoutError: If the AI call times out
+        RateLimitError: If rate limited
+        APIExhaustedError: If API quota exhausted
+        AIError: For other AI errors
+    """
+    # Create temp file for model name (ai-cli.sh writes to /tmp/.claude-evolve-model-$$)
+    pid = os.getpid()
+    model_file = f"/tmp/.claude-evolve-model-{pid}"
+
+    # Build the bash command that sources config and calls the AI
+    # We need to source config.sh first to get LLM_RUN/LLM_IDEATE variables
+    bash_script = f'''
+        source "{SCRIPT_DIR}/config.sh"
+        source "{AI_CLI_PATH}"
+        call_ai_random "$1" "$2"
+    '''
+
+    # Setup environment
+    env = os.environ.copy()
+    if env_vars:
+        env.update(env_vars)
+
+    try:
+        result = subprocess.run(
+            ["bash", "-c", bash_script, "bash", prompt, command],
+            capture_output=True,
+            text=True,
+            cwd=working_dir,
+            env=env
+        )
+
+        output = result.stdout
+        stderr = result.stderr
+        exit_code = result.returncode
+
+        # Read model name from temp file
+        model_name = "unknown"
+        if os.path.exists(model_file):
+            with open(model_file) as f:
+                model_name = f.read().strip()
+            os.remove(model_file)
+
+        # Handle exit codes
+        if exit_code == 124:
+            raise TimeoutError(f"AI call timed out (model: {model_name})")
+        elif exit_code == 2:
+            raise RateLimitError(f"Rate limit hit (model: {model_name})")
+        elif exit_code == 3:
+            raise APIExhaustedError(f"API quota exhausted (model: {model_name})")
+        elif exit_code != 0:
+            raise AIError(f"AI call failed with exit code {exit_code}: {stderr}")
+
+        return output, model_name
+
+    except subprocess.SubprocessError as e:
+        raise AIError(f"Failed to call AI: {e}")
+
+
+def call_ai_for_file_edit(
+    prompt: str,
+    file_path: str,
+    command: str = "run",
+    working_dir: Optional[str] = None
+) -> Tuple[bool, str]:
+    """
+    Call AI to edit a specific file.
+
+    This is used when the AI needs to modify files directly (like CSV editing
+    during ideation). The file path is passed in the prompt context.
+
+    Args:
+        prompt: The prompt including file editing instructions
+        file_path: Path to the file being edited (for verification)
+        command: Either "run" or "ideate"
+        working_dir: Directory to run in
+
+    Returns:
+        Tuple of (success: bool, model_name: str)
+    """
+    # Get file mtime before
+    before_mtime = None
+    if os.path.exists(file_path):
+        before_mtime = os.path.getmtime(file_path)
+
+    try:
+        output, model_name = call_ai(prompt, command, working_dir)
+
+        # Verify file was modified
+        if os.path.exists(file_path):
+            after_mtime = os.path.getmtime(file_path)
+            if before_mtime is not None and after_mtime > before_mtime:
+                return True, model_name
+
+        # File not modified - might be an error
+        return False, model_name
+
+    except AIError:
+        raise
+
+
+if __name__ == "__main__":
+    # Quick test
+    print("Testing AI CLI wrapper...")
+    print(f"AI CLI path: {AI_CLI_PATH}")
+    print(f"AI CLI exists: {AI_CLI_PATH.exists()}")
+    print("\nGit protection warning:")
+    print(get_git_protection_warning()[:200] + "...")

package/lib/embedding.py

@@ -0,0 +1,200 @@
+#!/usr/bin/env python3
+"""
+Embedding helper using Ollama's nomic-embed-text model.
+AIDEV-NOTE: Requires ollama with nomic-embed-text model pulled.
+AIDEV-NOTE: Embeddings are cached to disk for efficiency.
+"""
+
+import hashlib
+import json
+import math
+import os
+import urllib.request
+from pathlib import Path
+from typing import List, Optional, Tuple, Dict
+
+EMBEDDING_MODEL = os.environ.get("EMBEDDING_MODEL", "nomic-embed-text")
+OLLAMA_URL = os.environ.get("OLLAMA_URL", "http://localhost:11434")
+
+# Global cache - maps text hash to embedding
+_embedding_cache: Dict[str, List[float]] = {}
+_cache_file: Optional[Path] = None
+_cache_dirty = False
+
+
+def _text_hash(text: str) -> str:
+    """Create a short hash of text for cache key."""
+    return hashlib.sha256(text.encode('utf-8')).hexdigest()[:16]
+
+
+def set_cache_file(path: str) -> None:
+    """Set the file path for persistent embedding cache."""
+    global _cache_file, _embedding_cache
+    _cache_file = Path(path)
+    _load_cache()
+
+
+def _load_cache() -> None:
+    """Load cache from disk."""
+    global _embedding_cache
+    if _cache_file and _cache_file.exists():
+        try:
+            with open(_cache_file, 'r') as f:
+                _embedding_cache = json.load(f)
+            print(f"[EMBED] Loaded {len(_embedding_cache)} cached embeddings", file=__import__('sys').stderr)
+        except Exception as e:
+            print(f"[EMBED] Cache load error: {e}", file=__import__('sys').stderr)
+            _embedding_cache = {}
+
+
+def save_cache() -> None:
+    """Save cache to disk."""
+    global _cache_dirty
+    if _cache_file and _cache_dirty:
+        try:
+            _cache_file.parent.mkdir(parents=True, exist_ok=True)
+            with open(_cache_file, 'w') as f:
+                json.dump(_embedding_cache, f)
+            _cache_dirty = False
+        except Exception as e:
+            print(f"[EMBED] Cache save error: {e}", file=__import__('sys').stderr)
+
+
+def get_embedding(text: str, use_cache: bool = True) -> Optional[List[float]]:
+    """Get embedding vector for text using Ollama. Uses cache if available."""
+    global _cache_dirty
+
+    # Check cache first
+    if use_cache:
+        key = _text_hash(text)
+        if key in _embedding_cache:
+            return _embedding_cache[key]
+
+    try:
+        req_data = json.dumps({"model": EMBEDDING_MODEL, "input": text}).encode('utf-8')
+        req = urllib.request.Request(
+            f"{OLLAMA_URL}/api/embed",
+            data=req_data,
+            headers={"Content-Type": "application/json"}
+        )
+        with urllib.request.urlopen(req, timeout=30) as response:
+            data = json.loads(response.read().decode('utf-8'))
+            embedding = data.get("embeddings", [[]])[0]
+
+            # Store in cache
+            if use_cache and embedding:
+                key = _text_hash(text)
+                _embedding_cache[key] = embedding
+                _cache_dirty = True
+
+            return embedding
+    except Exception as e:
+        print(f"Embedding error: {e}")
+        return None
+
+
+def get_file_embedding(file_path: str) -> Optional[List[float]]:
+    """Get embedding for a file's contents."""
+    try:
+        with open(file_path, 'r') as f:
+            return get_embedding(f.read())
+    except Exception as e:
+        print(f"File read error: {e}")
+        return None
+
+
+def cosine_similarity(a: List[float], b: List[float]) -> float:
+    """Compute cosine similarity between two embedding vectors."""
+    if not a or not b or len(a) != len(b):
+        return 0.0
+
+    dot = sum(x * y for x, y in zip(a, b))
+    norm_a = math.sqrt(sum(x * x for x in a))
+    norm_b = math.sqrt(sum(x * x for x in b))
+
+    if norm_a == 0 or norm_b == 0:
+        return 0.0
+
+    return dot / (norm_a * norm_b)
+
+
+def is_similar(text1: str, text2: str, threshold: float = 0.9) -> bool:
+    """Check if two texts are semantically similar."""
+    emb1 = get_embedding(text1)
+    emb2 = get_embedding(text2)
+
+    if not emb1 or not emb2:
+        return False
+
+    return cosine_similarity(emb1, emb2) >= threshold
+
+
+def find_most_similar(
+    query: str,
+    candidates: List[str],
+    top_k: int = 5
+) -> List[Tuple[int, float, str]]:
+    """
+    Find most similar texts from candidates.
+    Returns list of (index, similarity, text) tuples.
+    """
+    query_emb = get_embedding(query)
+    if not query_emb:
+        return []
+
+    results = []
+    for i, candidate in enumerate(candidates):
+        cand_emb = get_embedding(candidate)
+        if cand_emb:
+            sim = cosine_similarity(query_emb, cand_emb)
+            results.append((i, sim, candidate))
+
+    results.sort(key=lambda x: x[1], reverse=True)
+    return results[:top_k]
+
+
+def check_novelty(
+    new_code: str,
+    existing_codes: List[str],
+    threshold: float = 0.95
+) -> Tuple[bool, float]:
+    """
+    Check if new code is novel enough compared to existing code.
+    Returns (is_novel, max_similarity).
+    Uses cache for efficiency - subsequent calls with same texts are instant.
+    """
+    new_emb = get_embedding(new_code)
+    if not new_emb:
+        return True, 0.0  # Can't check, assume novel
+
+    max_sim = 0.0
+    for existing in existing_codes:
+        existing_emb = get_embedding(existing)
+        if existing_emb:
+            sim = cosine_similarity(new_emb, existing_emb)
+            max_sim = max(max_sim, sim)
+
+    # Save cache after checking (batched save)
+    save_cache()
+
+    return max_sim < threshold, max_sim
+
+
+if __name__ == "__main__":
+    # Test
+    print("Testing embedding...")
+    emb = get_embedding("def hello(): print('hello world')")
+    if emb:
+        print(f"Embedding (first 5 dims): {emb[:5]}")
+        print(f"Full dimensions: {len(emb)}")
+
+        # Test similarity
+        code1 = "def add(a, b): return a + b"
+        code2 = "def sum(x, y): return x + y"
+        code3 = "def multiply(a, b): return a * b"
+
+        sim12 = cosine_similarity(get_embedding(code1), get_embedding(code2))
+        sim13 = cosine_similarity(get_embedding(code1), get_embedding(code3))
+
+        print(f"\nSimilarity (add vs sum): {sim12:.4f}")
+        print(f"Similarity (add vs multiply): {sim13:.4f}")