claude-evolve 1.8.12 → 1.8.13

package/bin/claude-evolve-worker

@@ -407,19 +407,41 @@ with EvolutionCSV('$FULL_CSV_PATH') as csv:
     eval_arg=""
   fi
   local eval_cmd=("$PYTHON_CMD" "$FULL_EVALUATOR_PATH" "$eval_arg")
-  
+
   # Add memory limiting if configured
+  # CRITICAL: Use multiple layers of protection (ulimit + Python wrapper + monitoring)
+  local memory_protection=""
   if [[ -n "$MEMORY_LIMIT_MB" ]] && [[ "$MEMORY_LIMIT_MB" -gt 0 ]]; then
+    # Layer 1: ulimit for hard memory limit (kernel-enforced)
+    # IMPORTANT: Use -m (RSS) not -v (virtual memory) because:
+    # - Neural networks use mmap() which bypasses RLIMIT_AS (-v)
+    # - RSS limit is more reliable for actual memory consumption
+    # Convert MB to KB for ulimit
+    local memory_limit_kb=$((MEMORY_LIMIT_MB * 1024))
+
+    # Try -m first (RSS limit), fall back to -v if not supported
+    if ulimit -m $memory_limit_kb 2>/dev/null; then
+      memory_protection="ulimit -m $memory_limit_kb 2>/dev/null; "
+      echo "[MEMORY] Layer 1: ulimit -m ${memory_limit_kb}KB (RSS limit - catches neural networks)" >&2
+    else
+      memory_protection="ulimit -v $memory_limit_kb 2>/dev/null; "
+      echo "[MEMORY] Layer 1: ulimit -v ${memory_limit_kb}KB (fallback - may not catch neural networks)" >&2
+    fi
+
+    # Layer 2: Python wrapper with PROCESS TREE monitoring (backup protection)
     eval_cmd=("$PYTHON_CMD" "$SCRIPT_DIR/../lib/memory_limit_wrapper.py" "$MEMORY_LIMIT_MB" "${eval_cmd[@]}")
+
+    echo "[MEMORY] Layer 2: Python process tree monitoring (kills entire subprocess tree)" >&2
   fi
-  
+
   # Add timeout if configured
   [[ -n "$timeout_seconds" ]] && eval_cmd=(timeout "$timeout_seconds" "${eval_cmd[@]}")
-  
-  # Run evaluation with tee to both display and capture output
+
+  # Run evaluation with memory protection, tee to both display and capture output
   # Use stdbuf to disable buffering for real-time output
   # IMPORTANT: Use PIPESTATUS to get the exit code of the evaluation command, not tee
-  stdbuf -o0 -e0 "${eval_cmd[@]}" 2>&1 | tee "$eval_output_file" >&2
+  # The subshell ensures ulimit is applied before the command runs
+  stdbuf -o0 -e0 bash -c "${memory_protection}$(printf '%q ' "${eval_cmd[@]}")" 2>&1 | tee "$eval_output_file" >&2
   local eval_exit_code=${PIPESTATUS[0]}  # Get exit code of first command in pipe
   
   if [[ $eval_exit_code -eq 0 ]]; then

package/lib/memory_limit_wrapper.py

@@ -4,6 +4,16 @@ Memory-limited execution wrapper for claude-evolve evaluations.
 
 This script runs a command with memory limits to prevent runaway algorithms
 from consuming all system memory and crashing the machine.
+
+CRITICAL: Multi-layer protection approach (both must work together):
+1. ulimit -m (RSS limit) set by calling shell script - kernel-enforced, catches neural networks
+2. This Python wrapper monitors ENTIRE PROCESS TREE every 0.1s and kills if limit exceeded
+
+AIDEV-NOTE: Previous bugs fixed:
+- ulimit -v (virtual memory) doesn't catch neural networks that use mmap()
+- Was only monitoring direct child, not entire process tree (missed grandchildren)
+- Monitoring interval was 0.5s - too slow for fast memory allocations
+- Resource limit failures were silently ignored instead of failing fast
 """
 import sys
 import os
@@ -11,119 +21,197 @@ import subprocess
 import signal
 import time
 import resource
-from typing import Optional
+from typing import Optional, Tuple
 
-def set_memory_limit(limit_mb: int) -> None:
-    """Set memory limit in MB using resource module."""
+def verify_memory_limit_set(limit_mb: int) -> Tuple[bool, str]:
+    """Verify that memory limits are actually enforced."""
+    try:
+        limit_bytes = limit_mb * 1024 * 1024
+
+        # Check RLIMIT_AS (virtual memory)
+        soft_as, hard_as = resource.getrlimit(resource.RLIMIT_AS)
+        if soft_as != resource.RLIM_INFINITY and soft_as <= limit_bytes * 1.1:
+            return True, f"RLIMIT_AS set to {soft_as / (1024*1024):.0f}MB"
+
+        # Check RLIMIT_DATA (data segment)
+        try:
+            soft_data, hard_data = resource.getrlimit(resource.RLIMIT_DATA)
+            if soft_data != resource.RLIM_INFINITY and soft_data <= limit_bytes * 1.1:
+                return True, f"RLIMIT_DATA set to {soft_data / (1024*1024):.0f}MB"
+        except (OSError, ValueError):
+            pass
+
+        return False, "No hard memory limits detected"
+    except Exception as e:
+        return False, f"Error checking limits: {e}"
+
+def set_memory_limit(limit_mb: int) -> bool:
+    """
+    Set memory limit in MB using resource module.
+    Returns True if successful, False otherwise.
+    """
     try:
         # Convert MB to bytes
         limit_bytes = limit_mb * 1024 * 1024
-        
+
         # Set virtual memory limit (address space)
         # On macOS this is the most reliable way to limit memory
         resource.setrlimit(resource.RLIMIT_AS, (limit_bytes, limit_bytes))
-        
+
         # Also try to set data segment limit if available
         try:
             resource.setrlimit(resource.RLIMIT_DATA, (limit_bytes, limit_bytes))
         except (OSError, ValueError):
             # Not available on all systems
             pass
-            
-        print(f"[MEMORY] Set memory limit to {limit_mb}MB", file=sys.stderr)
-        
+
+        # Verify it was actually set
+        is_set, msg = verify_memory_limit_set(limit_mb)
+        if is_set:
+            print(f"[MEMORY] ✓ Hard limit enforced: {msg}", file=sys.stderr)
+            return True
+        else:
+            print(f"[MEMORY] ✗ Hard limit NOT enforced: {msg}", file=sys.stderr)
+            return False
+
     except (OSError, ValueError) as e:
-        print(f"[MEMORY] Warning: Could not set memory limit: {e}", file=sys.stderr)
+        print(f"[MEMORY] ✗ Could not set memory limit: {e}", file=sys.stderr)
+        return False
+
+def get_process_tree_memory_native(pid: int) -> float:
+    """Get total memory usage of entire process group using native ps command."""
+    try:
+        # Get the process group ID
+        pgid = os.getpgid(pid)
+
+        # Get all processes in the process group
+        ps_result = subprocess.run(
+            ["ps", "-o", "rss=", "-g", str(pgid)],
+            capture_output=True,
+            text=True,
+            timeout=1
+        )
+
+        if ps_result.returncode != 0:
+            return 0.0
+
+        # Sum all RSS values from the process group
+        total_rss_kb = 0
+        for line in ps_result.stdout.strip().split('\n'):
+            line = line.strip()
+            if line:
+                try:
+                    total_rss_kb += int(line)
+                except ValueError:
+                    continue
+
+        # Convert KB to MB
+        return total_rss_kb / 1024.0
+    except Exception:
+        return 0.0
 
 def monitor_memory_usage_native(process: subprocess.Popen, limit_mb: int) -> Optional[str]:
-    """Monitor process memory usage using native tools and kill if it exceeds limits."""
-    # print(f"[MEMORY] Starting native monitoring for PID {process.pid} with limit {limit_mb}MB", file=sys.stderr)
-    
+    """Monitor ENTIRE PROCESS TREE memory usage using native tools and kill if it exceeds limits."""
+    print(f"[MEMORY] Monitoring process tree from root PID {process.pid} (limit: {limit_mb}MB)", file=sys.stderr)
+
     while process.poll() is None:
         try:
-            # Use ps command to get memory usage
-            ps_result = subprocess.run(
-                ["ps", "-o", "rss=", "-p", str(process.pid)], 
-                capture_output=True, 
-                text=True, 
-                timeout=1
-            )
-            
-            if ps_result.returncode == 0 and ps_result.stdout.strip():
-                # ps returns RSS in KB, convert to MB
-                memory_kb = int(ps_result.stdout.strip())
-                memory_mb = memory_kb / 1024
-                
-                # print(f"[MEMORY] PID {process.pid} using {memory_mb:.1f}MB (limit: {limit_mb}MB)", file=sys.stderr)
-                
-                if memory_mb > limit_mb:
-                    print(f"[MEMORY] Process exceeded {limit_mb}MB limit (using {memory_mb:.1f}MB), terminating", file=sys.stderr)
-                    # Kill the entire process group - fix race condition
-                    try:
-                        pgid = os.getpgid(process.pid)
-                        os.killpg(pgid, signal.SIGTERM)
-                    except ProcessLookupError:
-                        return f"Memory limit exceeded: {memory_mb:.1f}MB > {limit_mb}MB"
-                    
-                    time.sleep(2)  # Give it time to cleanup
-                    
-                    try:
-                        if process.poll() is None:
-                            pgid = os.getpgid(process.pid) 
-                            os.killpg(pgid, signal.SIGKILL)
-                    except ProcessLookupError:
-                        pass
+            # Get total memory for entire process tree
+            memory_mb = get_process_tree_memory_native(process.pid)
+
+            if memory_mb > limit_mb:
+                print(f"[MEMORY] Process tree exceeded {limit_mb}MB limit (using {memory_mb:.1f}MB), terminating entire tree", file=sys.stderr)
+                # Kill the entire process group
+                try:
+                    pgid = os.getpgid(process.pid)
+                    os.killpg(pgid, signal.SIGTERM)
+                except ProcessLookupError:
                     return f"Memory limit exceeded: {memory_mb:.1f}MB > {limit_mb}MB"
-            
-            time.sleep(0.5)  # Check every 500ms
-            
+
+                time.sleep(2)  # Give it time to cleanup
+
+                try:
+                    if process.poll() is None:
+                        pgid = os.getpgid(process.pid)
+                        os.killpg(pgid, signal.SIGKILL)
+                except ProcessLookupError:
+                    pass
+                return f"Memory limit exceeded: {memory_mb:.1f}MB > {limit_mb}MB"
+
+            time.sleep(0.1)  # Check every 100ms for faster response
+
         except (subprocess.TimeoutExpired, ValueError, ProcessLookupError):
             # Process might have terminated or ps command failed
-            time.sleep(0.5)
+            time.sleep(0.1)
             continue
-    
-    # print(f"[MEMORY] Monitoring stopped for PID {process.pid}", file=sys.stderr)
+
     return None
 
+def get_process_tree_memory_psutil(ps_process) -> float:
+    """Get total memory usage of entire process tree using psutil."""
+    try:
+        import psutil
+        total_mb = 0.0
+
+        # Get memory of root process
+        try:
+            total_mb += ps_process.memory_info().rss / (1024 * 1024)
+        except (psutil.NoSuchProcess, psutil.AccessDenied):
+            return 0.0
+
+        # Get memory of all children (recursive)
+        try:
+            for child in ps_process.children(recursive=True):
+                try:
+                    total_mb += child.memory_info().rss / (1024 * 1024)
+                except (psutil.NoSuchProcess, psutil.AccessDenied):
+                    continue
+        except (psutil.NoSuchProcess, psutil.AccessDenied):
+            pass
+
+        return total_mb
+    except ImportError:
+        return 0.0
+
 def monitor_memory_usage(process: subprocess.Popen, limit_mb: int) -> Optional[str]:
-    """Monitor process memory usage and kill if it exceeds limits."""
+    """Monitor ENTIRE PROCESS TREE memory usage and kill if it exceeds limits."""
     try:
         import psutil
         ps_process = psutil.Process(process.pid)
-        
+        print(f"[MEMORY] Monitoring process tree from root PID {process.pid} (limit: {limit_mb}MB, using psutil)", file=sys.stderr)
+
         while process.poll() is None:
             try:
-                # Get memory usage in MB
-                memory_info = ps_process.memory_info()
-                memory_mb = memory_info.rss / (1024 * 1024)
-                
+                # Get total memory for entire process tree
+                memory_mb = get_process_tree_memory_psutil(ps_process)
+
                 if memory_mb > limit_mb:
-                    print(f"[MEMORY] Process exceeded {limit_mb}MB limit (using {memory_mb:.1f}MB), terminating", file=sys.stderr)
-                    # Kill the entire process group - fix race condition
+                    print(f"[MEMORY] Process tree exceeded {limit_mb}MB limit (using {memory_mb:.1f}MB), terminating entire tree", file=sys.stderr)
+                    # Kill the entire process group
                     try:
                         pgid = os.getpgid(process.pid)
                         os.killpg(pgid, signal.SIGTERM)
                     except ProcessLookupError:
                         return f"Memory limit exceeded: {memory_mb:.1f}MB > {limit_mb}MB"
-                    
+
                     time.sleep(2)  # Give it time to cleanup
-                    
+
                     try:
                         if process.poll() is None:
-                            pgid = os.getpgid(process.pid) 
+                            pgid = os.getpgid(process.pid)
                             os.killpg(pgid, signal.SIGKILL)
                     except ProcessLookupError:
                         pass
                     return f"Memory limit exceeded: {memory_mb:.1f}MB > {limit_mb}MB"
-                
-                time.sleep(0.5)  # Check every 500ms
+
+                time.sleep(0.1)  # Check every 100ms for faster response
             except (psutil.NoSuchProcess, psutil.AccessDenied):
                 # Process already terminated
                 break
     except ImportError:
         # psutil not available, use native monitoring
         return monitor_memory_usage_native(process, limit_mb)
-    
+
     return None
 
 def validate_memory_limit(limit_mb: int) -> bool:

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "claude-evolve",
-  "version": "1.8.12",
+  "version": "1.8.13",
   "bin": {
     "claude-evolve": "./bin/claude-evolve",
     "claude-evolve-main": "./bin/claude-evolve-main",