gitarsenal-cli 1.7.1 → 1.7.3

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "gitarsenal-cli",
-  "version": "1.7.1",
+  "version": "1.7.3",
   "description": "CLI tool for creating Modal sandboxes with GitHub repositories",
   "main": "index.js",
   "bin": {

package/python/requirements.txt

@@ -3,4 +3,5 @@ requests>=2.31.0
 pathlib>=1.0.1
 python-dotenv>=1.0.0
 flask>=2.0.0
-flask-cors>=3.0.0 
+flask-cors>=3.0.0
+pexpect>=4.8.0 

package/python/test_modalSandboxScript.py

@@ -10,6 +10,9 @@ import requests
 import secrets
 import string
 import argparse
+import threading
+import uuid
+import signal
 from pathlib import Path
 
 # Parse command-line arguments
@@ -33,6 +36,382 @@ if args.proxy_api_key:
     os.environ["MODAL_PROXY_API_KEY"] = args.proxy_api_key
     # print(f"✅ Set MODAL_PROXY_API_KEY from command line")
 
+class PersistentShell:
+    """A persistent bash shell using subprocess.Popen for executing commands with state persistence."""
+    
+    def __init__(self, working_dir="/root", timeout=60):
+        self.working_dir = working_dir
+        self.timeout = timeout
+        self.process = None
+        self.stdout_lines = []  # Use list instead of queue
+        self.stderr_lines = []  # Use list instead of queue
+        self.stdout_lock = threading.Lock()
+        self.stderr_lock = threading.Lock()
+        self.stdout_thread = None
+        self.stderr_thread = None
+        self.command_counter = 0
+        self.is_running = False
+        self.virtual_env_path = None  # Track activated virtual environment
+        
+    def start(self):
+        """Start the persistent bash shell."""
+        if self.is_running:
+            return
+            
+        print(f"🐚 Starting persistent bash shell in {self.working_dir}")
+        
+        # Start bash with unbuffered output
+        self.process = subprocess.Popen(
+            ['bash', '-i'],  # Interactive bash
+            stdin=subprocess.PIPE,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+            text=True,
+            bufsize=0,  # Unbuffered
+            cwd=self.working_dir,
+            preexec_fn=os.setsid  # Create new process group
+        )
+        
+        # Start threads to read stdout and stderr
+        self.stdout_thread = threading.Thread(target=self._read_stdout, daemon=True)
+        self.stderr_thread = threading.Thread(target=self._read_stderr, daemon=True)
+        
+        self.stdout_thread.start()
+        self.stderr_thread.start()
+        
+        self.is_running = True
+        
+        # Initial setup commands
+        self._send_command_raw("set +h")  # Disable hash table for commands
+        self._send_command_raw("export PS1='$ '")  # Simpler prompt
+        self._send_command_raw("cd " + self.working_dir)  # Change to working directory
+        time.sleep(0.5)  # Let initial commands settle
+        
+        print("✅ Persistent shell started successfully")
+        
+    def _read_stdout(self):
+        """Read stdout in a separate thread."""
+        while self.process and self.process.poll() is None:
+            try:
+                line = self.process.stdout.readline()
+                if line:
+                    with self.stdout_lock:
+                        self.stdout_lines.append(line.rstrip('\n'))
+                else:
+                    time.sleep(0.01)
+            except Exception as e:
+                print(f"Error reading stdout: {e}")
+                break
+                
+    def _read_stderr(self):
+        """Read stderr in a separate thread."""
+        while self.process and self.process.poll() is None:
+            try:
+                line = self.process.stderr.readline()
+                if line:
+                    with self.stderr_lock:
+                        self.stderr_lines.append(line.rstrip('\n'))
+                else:
+                    time.sleep(0.01)
+            except Exception as e:
+                print(f"Error reading stderr: {e}")
+                break
+    
+    def _send_command_raw(self, command):
+        """Send a raw command to the shell without waiting for completion."""
+        if not self.is_running or not self.process:
+            raise RuntimeError("Shell is not running")
+            
+        try:
+            self.process.stdin.write(command + '\n')
+            self.process.stdin.flush()
+        except Exception as e:
+            print(f"Error sending command: {e}")
+            raise
+    
+    def _preprocess_command(self, command):
+        """Preprocess commands to handle special cases like virtual environment activation."""
+        # Handle virtual environment creation and activation
+        if "uv venv" in command and "&&" in command and "source" in command:
+            # Split the compound command into separate parts
+            parts = [part.strip() for part in command.split("&&")]
+            return parts
+        elif command.strip().startswith("source ") and "/bin/activate" in command:
+            # Handle standalone source command
+            venv_path = command.replace("source ", "").replace("/bin/activate", "").strip()
+            self.virtual_env_path = venv_path
+            return [command]
+        elif "source" in command and "activate" in command:
+            # Handle any other source activation pattern
+            return [command]
+        elif "uv pip install" in command and self.is_in_venv():
+            # If we're in a virtual environment, ensure we use the right pip
+            return [command]
+        else:
+            return [command]
+    
+    def execute(self, command, timeout=None):
+        """Execute a command and return (success, stdout, stderr)."""
+        if not self.is_running:
+            self.start()
+            
+        if timeout is None:
+            timeout = self.timeout
+        
+        # Preprocess the command to handle special cases
+        command_parts = self._preprocess_command(command)
+        
+        # If we have multiple parts, execute them sequentially
+        if len(command_parts) > 1:
+            print(f"🔧 Executing compound command in {len(command_parts)} parts")
+            all_stdout = []
+            all_stderr = []
+            
+            for i, part in enumerate(command_parts):
+                print(f"   Part {i+1}/{len(command_parts)}: {part}")
+                success, stdout, stderr = self._execute_single(part, timeout)
+                
+                if stdout:
+                    all_stdout.append(stdout)
+                if stderr:
+                    all_stderr.append(stderr)
+                
+                if not success:
+                    # If any part fails, return the failure
+                    return False, '\n'.join(all_stdout), '\n'.join(all_stderr)
+                
+                # Small delay between parts to let environment changes take effect
+                time.sleep(0.1)
+            
+            return True, '\n'.join(all_stdout), '\n'.join(all_stderr)
+        else:
+            return self._execute_single(command_parts[0], timeout)
+    
+    def _execute_single(self, command, timeout):
+        """Execute a single command and return (success, stdout, stderr)."""
+        self.command_counter += 1
+        marker = f"CMD_DONE_{self.command_counter}_{uuid.uuid4().hex[:8]}"
+        
+        print(f"🔧 Executing: {command}")
+        
+        # Clear any existing output
+        self._clear_lines()
+        
+        # Wait for shell to be ready (prompt should be visible)
+        if not self.wait_for_prompt(timeout=2):
+            print("⚠️ Shell not ready, waiting...")
+            time.sleep(0.5)
+        
+        # For source commands, we need special handling
+        if command.strip().startswith("source "):
+            # Send the source command in a way that preserves the environment
+            try:
+                # Extract the virtual environment path
+                venv_path = command.replace("source ", "").replace("/bin/activate", "").strip()
+                
+                # Use a more robust approach that actually activates the environment
+                activation_script = f"""
+                if [ -f "{venv_path}/bin/activate" ]; then
+                    source "{venv_path}/bin/activate"
+                    echo "VIRTUAL_ENV=$VIRTUAL_ENV"
+                    echo "PATH=$PATH"
+                    echo 'SOURCE_SUCCESS'
+                else
+                    echo 'SOURCE_FAILED - activation script not found'
+                fi
+                """
+                
+                self._send_command_raw(activation_script)
+                time.sleep(0.3)  # Give more time for environment changes
+                self._send_command_raw(f'echo "EXIT_CODE:$?"')
+                self._send_command_raw(f'echo "{marker}"')
+            except Exception as e:
+                return False, "", f"Failed to send source command: {e}"
+        else:
+            # Send the command followed by markers
+            try:
+                self._send_command_raw(command)
+                # Wait a moment for the command to start
+                time.sleep(0.1)
+                self._send_command_raw(f'echo "EXIT_CODE:$?"')
+                self._send_command_raw(f'echo "{marker}"')
+            except Exception as e:
+                return False, "", f"Failed to send command: {e}"
+        
+        # Collect output until we see the marker
+        command_stdout = []
+        command_stderr = []
+        start_time = time.time()
+        found_marker = False
+        exit_code = None
+        last_stdout_index = 0
+        last_stderr_index = 0
+        source_success = None
+        
+        while time.time() - start_time < timeout:
+            # Check for new stdout lines
+            with self.stdout_lock:
+                current_stdout = self.stdout_lines[last_stdout_index:]
+                last_stdout_index = len(self.stdout_lines)
+            
+            for line in current_stdout:
+                if line == marker:
+                    found_marker = True
+                    break
+                elif line.startswith("EXIT_CODE:"):
+                    try:
+                        exit_code = int(line.split(":", 1)[1])
+                    except (ValueError, IndexError):
+                        exit_code = 1
+                elif line == "SOURCE_SUCCESS":
+                    source_success = True
+                elif line.startswith("SOURCE_FAILED"):
+                    source_success = False
+                    command_stderr.append(line)
+                elif line.startswith("VIRTUAL_ENV="):
+                    # Extract and store the virtual environment path
+                    venv_path = line.split("=", 1)[1]
+                    self.virtual_env_path = venv_path
+                    command_stdout.append(line)
+                elif line.startswith("PATH="):
+                    # Store the updated PATH
+                    command_stdout.append(line)
+                elif line.strip() and not line.startswith("$"):  # Skip empty lines and prompt lines
+                    command_stdout.append(line)
+            
+            if found_marker:
+                break
+            
+            # Check for new stderr lines
+            with self.stderr_lock:
+                current_stderr = self.stderr_lines[last_stderr_index:]
+                last_stderr_index = len(self.stderr_lines)
+            
+            for line in current_stderr:
+                if line.strip():  # Skip empty lines
+                    command_stderr.append(line)
+                
+            time.sleep(0.1)
+        
+        if not found_marker:
+            print(f"⚠️ Command timed out after {timeout} seconds")
+            return False, '\n'.join(command_stdout), f"Command timed out after {timeout} seconds"
+        
+        stdout_text = '\n'.join(command_stdout)
+        stderr_text = '\n'.join(command_stderr)
+        
+        # Determine success based on multiple factors
+        if source_success is not None:
+            success = source_success
+        else:
+            success = exit_code == 0 if exit_code is not None else len(command_stderr) == 0
+        
+        if success:
+            if stdout_text:
+                print(f"✅ Output: {stdout_text}")
+            # Track virtual environment activation
+            if command.strip().startswith("source ") and "/bin/activate" in command:
+                venv_path = command.replace("source ", "").replace("/bin/activate", "").strip()
+                self.virtual_env_path = venv_path
+                print(f"✅ Virtual environment activated: {venv_path}")
+        else:
+            print(f"❌ Command failed with exit code: {exit_code}")
+            if stderr_text:
+                print(f"❌ Error: {stderr_text}")
+        
+        # Wait a moment for the shell to be ready for the next command
+        time.sleep(0.2)
+        
+        return success, stdout_text, stderr_text
+    
+    def _clear_lines(self):
+        """Clear both output line lists."""
+        with self.stdout_lock:
+            self.stdout_lines.clear()
+        with self.stderr_lock:
+            self.stderr_lines.clear()
+    
+    def get_cwd(self):
+        """Get current working directory."""
+        success, output, _ = self._execute_single("pwd", 10)
+        if success:
+            return output.strip()
+        return self.working_dir
+    
+    def get_virtual_env(self):
+        """Get the currently activated virtual environment path."""
+        return self.virtual_env_path
+    
+    def is_in_venv(self):
+        """Check if we're currently in a virtual environment."""
+        return self.virtual_env_path is not None and self.virtual_env_path != ""
+    
+    def get_venv_name(self):
+        """Get the name of the current virtual environment if active."""
+        if self.is_in_venv():
+            return os.path.basename(self.virtual_env_path)
+        return None
+    
+    def exec(self, *args, **kwargs):
+        """Compatibility method to make PersistentShell work with call_openai_for_debug."""
+        # Convert exec call to execute method
+        if len(args) >= 2 and args[0] == "bash" and args[1] == "-c":
+            command = args[2]
+            success, stdout, stderr = self.execute(command)
+            
+            # Create a mock result object that mimics the expected interface
+            class MockResult:
+                def __init__(self, stdout, stderr, returncode):
+                    self.stdout = [stdout] if stdout else []
+                    self.stderr = [stderr] if stderr else []
+                    self.returncode = 0 if returncode else 1
+                
+                def wait(self):
+                    pass
+            
+            return MockResult(stdout, stderr, success)
+        else:
+            raise NotImplementedError("exec method only supports bash -c commands")
+    
+    def wait_for_prompt(self, timeout=5):
+        """Wait for the shell prompt to appear, indicating readiness for next command."""
+        start_time = time.time()
+        while time.time() - start_time < timeout:
+            with self.stdout_lock:
+                if self.stdout_lines and self.stdout_lines[-1].strip().endswith('$'):
+                    return True
+            time.sleep(0.1)
+        return False
+    
+    def cleanup(self):
+        """Clean up the shell process."""
+        print("🧹 Cleaning up persistent shell...")
+        self.is_running = False
+        
+        if self.process:
+            try:
+                # Send exit command
+                self._send_command_raw("exit")
+                
+                # Wait for process to terminate
+                try:
+                    self.process.wait(timeout=5)
+                except subprocess.TimeoutExpired:
+                    # Force kill if it doesn't exit gracefully
+                    os.killpg(os.getpgid(self.process.pid), signal.SIGTERM)
+                    try:
+                        self.process.wait(timeout=2)
+                    except subprocess.TimeoutExpired:
+                        os.killpg(os.getpgid(self.process.pid), signal.SIGKILL)
+                        
+            except Exception as e:
+                print(f"Error during cleanup: {e}")
+            finally:
+                self.process = None
+        
+        print("✅ Shell cleanup completed")
+
+
 # First, try to fetch tokens from the proxy server
 try:
     # Import the fetch_modal_tokens module
@@ -501,7 +880,7 @@ def call_openai_for_debug(command, error_output, api_key=None, current_dir=None,
         print("💡 To enable LLM debugging, set the OPENAI_API_KEY environment variable")
         return None
     
-    print(f"✅ OpenAI API key available (length: {len(api_key)})")
+    # print(f"✅ OpenAI API key available (length: {len(api_key)})")
     
     # Gather additional context to help with debugging
     directory_context = ""
@@ -1015,7 +1394,7 @@ def generate_random_password(length=16):
     return password
 
 
-# Now modify the create_modal_ssh_container function to use the standalone ssh_container_function
+# Now modify the create_modal_ssh_container function to use the PersistentShell
 def create_modal_ssh_container(gpu_type, repo_url=None, repo_name=None, setup_commands=None, 
                                volume_name=None, timeout_minutes=60, ssh_password=None, interactive=False):
     """Create a Modal SSH container with GPU support and tunneling"""
@@ -1239,7 +1618,7 @@ def create_modal_ssh_container(gpu_type, repo_url=None, repo_name=None, setup_co
                 "python3", "python3-pip", "build-essential", "tmux", "screen", "nano",
                 "gpg", "ca-certificates", "software-properties-common"
             )
-            .pip_install("uv", "modal", "requests", "openai")  # Remove problematic CUDA packages
+            .uv_pip_install("uv", "modal", "requests", "openai")  # Remove problematic CUDA packages
             .run_commands(
                 # Create SSH directory
                 "mkdir -p /var/run/sshd",
@@ -1309,6 +1688,7 @@ def create_modal_ssh_container(gpu_type, repo_url=None, repo_name=None, setup_co
         subprocess.run(["service", "ssh", "start"], check=True)
         
         # Clone repository if provided
+        repo_dir = "/root"
         if repo_url:
             repo_name_from_url = repo_name or repo_url.split('/')[-1].replace('.git', '')
             print(f"📥 Cloning repository: {repo_url}")
@@ -1320,119 +1700,77 @@ def create_modal_ssh_container(gpu_type, repo_url=None, repo_name=None, setup_co
                 # Change to repository directory
                 repo_dir = f"/root/{repo_name_from_url}"
                 if os.path.exists(repo_dir):
-                    os.chdir(repo_dir)
-                    print(f"📂 Changed to repository directory: {repo_dir}")
+                    print(f"📂 Will run setup commands in repository directory: {repo_dir}")
                 
             except subprocess.CalledProcessError as e:
                 print(f"❌ Failed to clone repository: {e}")
         
-        # Run setup commands if provided
+        # Run setup commands if provided using PersistentShell
         if setup_commands:
-            print(f"⚙️ Running {len(setup_commands)} setup commands...")
+            print(f"⚙️ Running {len(setup_commands)} setup commands with persistent shell...")
             
-            # Define a helper function for running commands with LLM debugging
-            def run_command_with_basic_error_handling(cmd, show_output=True, retry_count=0, max_retries=2):
-                """Execute a command with LLM debugging enabled"""
-                print(f"🔧 Executing: {cmd}")
-                try:
-                    # Handle special case for source command which doesn't work with subprocess.run
-                    if cmd.strip().startswith("source ") or " source " in cmd:
-                        print("⚠️ Detected 'source' command which doesn't work with subprocess.run")
-                        print("🔄 Converting to bash -c with dot (.) instead of source")
-                        # Replace source with . (dot) which is the same as source but works in sh
-                        modified_cmd = cmd.replace("source ", ". ")
-                        # Wrap in bash -c to ensure it runs in bash
-                        bash_cmd = f"bash -c '{modified_cmd}'"
-                        print(f"🔄 Modified command: {bash_cmd}")
-                        result = subprocess.run(bash_cmd, shell=True, check=True, 
-                                              capture_output=True, text=True)
-                    else:
-                        result = subprocess.run(cmd, shell=True, check=True, 
-                                              capture_output=True, text=True)
+            # Create persistent shell instance
+            shell = PersistentShell(working_dir=repo_dir, timeout=120)
+            
+            try:
+                # Start the persistent shell
+                shell.start()
+                
+                # Execute each setup command
+                for i, cmd in enumerate(setup_commands, 1):
+                    print(f"📋 Executing command {i}/{len(setup_commands)}: {cmd}")
                     
-                    if result.stdout and show_output:
-                        print(f"✅ Output: {result.stdout}")
-                    return True, result.stdout, ""
-                except subprocess.CalledProcessError as e:
-                    error_output = e.stderr if e.stderr else str(e)
-                    print(f"❌ Command failed: {e}")
-                    print(f"❌ Error: {error_output}")
+                    success, stdout, stderr = shell.execute(cmd, timeout=120)
                     
-                    # Call OpenAI for debugging
-                    print("🔍 Attempting to debug the failed command with OpenAI...")
-                    try:
-                        # Get the current directory for context
-                        current_dir = os.getcwd()
+                    if not success:
+                        print(f"⚠️ Command {i} failed, attempting LLM debugging...")
                         
                         # Call OpenAI for debugging
-                        print(f"🔍 DEBUG: About to call call_openai_for_debug...")
-                        print(f"🔍 DEBUG: Command: {cmd}")
-                        print(f"🔍 DEBUG: Error output length: {len(error_output)}")
-                        print(f"🔍 DEBUG: Current directory: {current_dir}")
-                        
-                        # Get the API key from environment or use the one that was fetched earlier
-                        api_key = os.environ.get("OPENAI_API_KEY")
-                        fix_command = call_openai_for_debug(cmd, error_output, api_key=api_key, current_dir=current_dir)
-                        
-                        print(f"🔍 DEBUG: call_openai_for_debug returned: {fix_command}")
-                        
-                        if fix_command:
-                            print(f"🔧 OpenAI suggested fix command: {fix_command}")
+                        try:
+                            current_dir = shell.get_cwd()
+                            api_key = os.environ.get("OPENAI_API_KEY")
                             
-                            # Run the fix command
-                            print(f"🔄 Running suggested fix command: {fix_command}")
-                            try:
-                                fix_result = subprocess.run(fix_command, shell=True, check=True, 
-                                                          capture_output=True, text=True)
-                                if fix_result.stdout:
-                                    print(f"✅ Fix command output: {fix_result.stdout}")
-                                
-                                # Retry the original command
-                                print(f"🔄 Retrying original command: {cmd}")
-                                return run_command_with_basic_error_handling(cmd, show_output, retry_count + 1, max_retries)
-                            except subprocess.CalledProcessError as fix_e:
-                                print(f"❌ Fix command also failed: {fix_e}")
-                                return False, "", error_output
-                        else:
-                            print("❌ No fix suggested by OpenAI")
-                            return False, "", error_output
+                            # Use your existing call_openai_for_debug function
+                            fix_command = call_openai_for_debug(cmd, stderr, api_key=api_key, current_dir=current_dir, sandbox=shell)
                             
-                    except Exception as debug_e:
-                        print(f"❌ LLM debugging failed: {debug_e}")
-                        return False, "", error_output
-            
-            # Run each setup command
-            for i, cmd in enumerate(setup_commands, 1):
-                print(f"📋 Executing command {i}/{len(setup_commands)}: {cmd}")
-                
-                # Check if this is a cd command and if the directory exists
-                if cmd.strip().startswith("cd "):
-                    cd_parts = cmd.split(None, 1)
-                    if len(cd_parts) >= 2:
-                        target_dir = cd_parts[1].strip('"\'')
-                        print(f"🔍 Checking if directory exists: {target_dir}")
-                        try:
-                            check_result = subprocess.run(f"test -d '{target_dir}'", shell=True, 
-                                                       capture_output=True, text=True)
-                            if check_result.returncode != 0:
-                                print(f"⚠️ Directory does not exist: {target_dir}")
-                                print(f"🔍 Current directory contents:")
-                                subprocess.run("pwd && ls -la", shell=True, check=False)
+                            if fix_command:
+                                print(f"🔧 OpenAI suggested fix command: {fix_command}")
+                                
+                                # Run the fix command in the persistent shell
+                                print(f"🔄 Running suggested fix command: {fix_command}")
+                                fix_success, fix_stdout, fix_stderr = shell.execute(fix_command, timeout=120)
                                 
-                                # Try to find similar directories
-                                print(f"🔍 Looking for similar directories...")
-                                subprocess.run("find . -type d -name '*llama*' -o -name '*nano*' 2>/dev/null | head -10", shell=True, check=False)
-                        except Exception as e:
-                            print(f"⚠️ Could not check directory: {e}")
+                                if fix_success:
+                                    print(f"✅ Fix command succeeded")
+                                    
+                                    # Retry the original command
+                                    print(f"🔄 Retrying original command: {cmd}")
+                                    retry_success, retry_stdout, retry_stderr = shell.execute(cmd, timeout=120)
+                                    
+                                    if retry_success:
+                                        print(f"✅ Original command succeeded after fix!")
+                                    else:
+                                        print(f"⚠️ Original command still failed after fix, continuing...")
+                                else:
+                                    print(f"❌ Fix command failed: {fix_stderr}")
+                                    print(f"⚠️ Continuing with remaining commands...")
+                            else:
+                                print("❌ No fix suggested by OpenAI")
+                                print(f"⚠️ Continuing with remaining commands...")
+                                
+                        except Exception as debug_e:
+                            print(f"❌ LLM debugging failed: {debug_e}")
+                            print(f"⚠️ Continuing with remaining commands...")
+                    else:
+                        print(f"✅ Command {i} completed successfully")
                 
-                success, stdout, stderr = run_command_with_basic_error_handling(cmd, show_output=True)
-                if not success:
-                    print(f"⚠️ Command {i} failed, but continuing with remaining commands...")
-                    
-                    # If this was a cd command that failed, try to understand the directory structure
-                    if cmd.strip().startswith("cd ") and "No such file or directory" in stderr:
-                        print(f"🔍 Analyzing directory structure after failed cd command...")
-                        subprocess.run("pwd && ls -la && echo '--- Parent directory ---' && ls -la ..", shell=True, check=False)
+                print("✅ All setup commands processed")
+                
+            except Exception as e:
+                print(f"❌ Error during setup command execution: {e}")
+            finally:
+                # Clean up the shell
+                shell.cleanup()
         
         # Create SSH tunnel
         with modal.forward(22, unencrypted=True) as tunnel:

package/test_modalSandboxScript.py

@@ -10,6 +10,9 @@ import requests
 import secrets
 import string
 import argparse
+import threading
+import uuid
+import signal
 from pathlib import Path
 
 # Parse command-line arguments
@@ -33,6 +36,382 @@ if args.proxy_api_key:
     os.environ["MODAL_PROXY_API_KEY"] = args.proxy_api_key
     # print(f"✅ Set MODAL_PROXY_API_KEY from command line")
 
+class PersistentShell:
+    """A persistent bash shell using subprocess.Popen for executing commands with state persistence."""
+    
+    def __init__(self, working_dir="/root", timeout=60):
+        self.working_dir = working_dir
+        self.timeout = timeout
+        self.process = None
+        self.stdout_lines = []  # Use list instead of queue
+        self.stderr_lines = []  # Use list instead of queue
+        self.stdout_lock = threading.Lock()
+        self.stderr_lock = threading.Lock()
+        self.stdout_thread = None
+        self.stderr_thread = None
+        self.command_counter = 0
+        self.is_running = False
+        self.virtual_env_path = None  # Track activated virtual environment
+        
+    def start(self):
+        """Start the persistent bash shell."""
+        if self.is_running:
+            return
+            
+        print(f"🐚 Starting persistent bash shell in {self.working_dir}")
+        
+        # Start bash with unbuffered output
+        self.process = subprocess.Popen(
+            ['bash', '-i'],  # Interactive bash
+            stdin=subprocess.PIPE,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+            text=True,
+            bufsize=0,  # Unbuffered
+            cwd=self.working_dir,
+            preexec_fn=os.setsid  # Create new process group
+        )
+        
+        # Start threads to read stdout and stderr
+        self.stdout_thread = threading.Thread(target=self._read_stdout, daemon=True)
+        self.stderr_thread = threading.Thread(target=self._read_stderr, daemon=True)
+        
+        self.stdout_thread.start()
+        self.stderr_thread.start()
+        
+        self.is_running = True
+        
+        # Initial setup commands
+        self._send_command_raw("set +h")  # Disable hash table for commands
+        self._send_command_raw("export PS1='$ '")  # Simpler prompt
+        self._send_command_raw("cd " + self.working_dir)  # Change to working directory
+        time.sleep(0.5)  # Let initial commands settle
+        
+        print("✅ Persistent shell started successfully")
+        
+    def _read_stdout(self):
+        """Read stdout in a separate thread."""
+        while self.process and self.process.poll() is None:
+            try:
+                line = self.process.stdout.readline()
+                if line:
+                    with self.stdout_lock:
+                        self.stdout_lines.append(line.rstrip('\n'))
+                else:
+                    time.sleep(0.01)
+            except Exception as e:
+                print(f"Error reading stdout: {e}")
+                break
+                
+    def _read_stderr(self):
+        """Read stderr in a separate thread."""
+        while self.process and self.process.poll() is None:
+            try:
+                line = self.process.stderr.readline()
+                if line:
+                    with self.stderr_lock:
+                        self.stderr_lines.append(line.rstrip('\n'))
+                else:
+                    time.sleep(0.01)
+            except Exception as e:
+                print(f"Error reading stderr: {e}")
+                break
+    
+    def _send_command_raw(self, command):
+        """Send a raw command to the shell without waiting for completion."""
+        if not self.is_running or not self.process:
+            raise RuntimeError("Shell is not running")
+            
+        try:
+            self.process.stdin.write(command + '\n')
+            self.process.stdin.flush()
+        except Exception as e:
+            print(f"Error sending command: {e}")
+            raise
+    
+    def _preprocess_command(self, command):
+        """Preprocess commands to handle special cases like virtual environment activation."""
+        # Handle virtual environment creation and activation
+        if "uv venv" in command and "&&" in command and "source" in command:
+            # Split the compound command into separate parts
+            parts = [part.strip() for part in command.split("&&")]
+            return parts
+        elif command.strip().startswith("source ") and "/bin/activate" in command:
+            # Handle standalone source command
+            venv_path = command.replace("source ", "").replace("/bin/activate", "").strip()
+            self.virtual_env_path = venv_path
+            return [command]
+        elif "source" in command and "activate" in command:
+            # Handle any other source activation pattern
+            return [command]
+        elif "uv pip install" in command and self.is_in_venv():
+            # If we're in a virtual environment, ensure we use the right pip
+            return [command]
+        else:
+            return [command]
+    
+    def execute(self, command, timeout=None):
+        """Execute a command and return (success, stdout, stderr)."""
+        if not self.is_running:
+            self.start()
+            
+        if timeout is None:
+            timeout = self.timeout
+        
+        # Preprocess the command to handle special cases
+        command_parts = self._preprocess_command(command)
+        
+        # If we have multiple parts, execute them sequentially
+        if len(command_parts) > 1:
+            print(f"🔧 Executing compound command in {len(command_parts)} parts")
+            all_stdout = []
+            all_stderr = []
+            
+            for i, part in enumerate(command_parts):
+                print(f"   Part {i+1}/{len(command_parts)}: {part}")
+                success, stdout, stderr = self._execute_single(part, timeout)
+                
+                if stdout:
+                    all_stdout.append(stdout)
+                if stderr:
+                    all_stderr.append(stderr)
+                
+                if not success:
+                    # If any part fails, return the failure
+                    return False, '\n'.join(all_stdout), '\n'.join(all_stderr)
+                
+                # Small delay between parts to let environment changes take effect
+                time.sleep(0.1)
+            
+            return True, '\n'.join(all_stdout), '\n'.join(all_stderr)
+        else:
+            return self._execute_single(command_parts[0], timeout)
+    
+    def _execute_single(self, command, timeout):
+        """Execute a single command and return (success, stdout, stderr)."""
+        self.command_counter += 1
+        marker = f"CMD_DONE_{self.command_counter}_{uuid.uuid4().hex[:8]}"
+        
+        print(f"🔧 Executing: {command}")
+        
+        # Clear any existing output
+        self._clear_lines()
+        
+        # Wait for shell to be ready (prompt should be visible)
+        if not self.wait_for_prompt(timeout=2):
+            print("⚠️ Shell not ready, waiting...")
+            time.sleep(0.5)
+        
+        # For source commands, we need special handling
+        if command.strip().startswith("source "):
+            # Send the source command in a way that preserves the environment
+            try:
+                # Extract the virtual environment path
+                venv_path = command.replace("source ", "").replace("/bin/activate", "").strip()
+                
+                # Use a more robust approach that actually activates the environment
+                activation_script = f"""
+                if [ -f "{venv_path}/bin/activate" ]; then
+                    source "{venv_path}/bin/activate"
+                    echo "VIRTUAL_ENV=$VIRTUAL_ENV"
+                    echo "PATH=$PATH"
+                    echo 'SOURCE_SUCCESS'
+                else
+                    echo 'SOURCE_FAILED - activation script not found'
+                fi
+                """
+                
+                self._send_command_raw(activation_script)
+                time.sleep(0.3)  # Give more time for environment changes
+                self._send_command_raw(f'echo "EXIT_CODE:$?"')
+                self._send_command_raw(f'echo "{marker}"')
+            except Exception as e:
+                return False, "", f"Failed to send source command: {e}"
+        else:
+            # Send the command followed by markers
+            try:
+                self._send_command_raw(command)
+                # Wait a moment for the command to start
+                time.sleep(0.1)
+                self._send_command_raw(f'echo "EXIT_CODE:$?"')
+                self._send_command_raw(f'echo "{marker}"')
+            except Exception as e:
+                return False, "", f"Failed to send command: {e}"
+        
+        # Collect output until we see the marker
+        command_stdout = []
+        command_stderr = []
+        start_time = time.time()
+        found_marker = False
+        exit_code = None
+        last_stdout_index = 0
+        last_stderr_index = 0
+        source_success = None
+        
+        while time.time() - start_time < timeout:
+            # Check for new stdout lines
+            with self.stdout_lock:
+                current_stdout = self.stdout_lines[last_stdout_index:]
+                last_stdout_index = len(self.stdout_lines)
+            
+            for line in current_stdout:
+                if line == marker:
+                    found_marker = True
+                    break
+                elif line.startswith("EXIT_CODE:"):
+                    try:
+                        exit_code = int(line.split(":", 1)[1])
+                    except (ValueError, IndexError):
+                        exit_code = 1
+                elif line == "SOURCE_SUCCESS":
+                    source_success = True
+                elif line.startswith("SOURCE_FAILED"):
+                    source_success = False
+                    command_stderr.append(line)
+                elif line.startswith("VIRTUAL_ENV="):
+                    # Extract and store the virtual environment path
+                    venv_path = line.split("=", 1)[1]
+                    self.virtual_env_path = venv_path
+                    command_stdout.append(line)
+                elif line.startswith("PATH="):
+                    # Store the updated PATH
+                    command_stdout.append(line)
+                elif line.strip() and not line.startswith("$"):  # Skip empty lines and prompt lines
+                    command_stdout.append(line)
+            
+            if found_marker:
+                break
+            
+            # Check for new stderr lines
+            with self.stderr_lock:
+                current_stderr = self.stderr_lines[last_stderr_index:]
+                last_stderr_index = len(self.stderr_lines)
+            
+            for line in current_stderr:
+                if line.strip():  # Skip empty lines
+                    command_stderr.append(line)
+                
+            time.sleep(0.1)
+        
+        if not found_marker:
+            print(f"⚠️ Command timed out after {timeout} seconds")
+            return False, '\n'.join(command_stdout), f"Command timed out after {timeout} seconds"
+        
+        stdout_text = '\n'.join(command_stdout)
+        stderr_text = '\n'.join(command_stderr)
+        
+        # Determine success based on multiple factors
+        if source_success is not None:
+            success = source_success
+        else:
+            success = exit_code == 0 if exit_code is not None else len(command_stderr) == 0
+        
+        if success:
+            if stdout_text:
+                print(f"✅ Output: {stdout_text}")
+            # Track virtual environment activation
+            if command.strip().startswith("source ") and "/bin/activate" in command:
+                venv_path = command.replace("source ", "").replace("/bin/activate", "").strip()
+                self.virtual_env_path = venv_path
+                print(f"✅ Virtual environment activated: {venv_path}")
+        else:
+            print(f"❌ Command failed with exit code: {exit_code}")
+            if stderr_text:
+                print(f"❌ Error: {stderr_text}")
+        
+        # Wait a moment for the shell to be ready for the next command
+        time.sleep(0.2)
+        
+        return success, stdout_text, stderr_text
+    
+    def _clear_lines(self):
+        """Clear both output line lists."""
+        with self.stdout_lock:
+            self.stdout_lines.clear()
+        with self.stderr_lock:
+            self.stderr_lines.clear()
+    
+    def get_cwd(self):
+        """Get current working directory."""
+        success, output, _ = self._execute_single("pwd", 10)
+        if success:
+            return output.strip()
+        return self.working_dir
+    
+    def get_virtual_env(self):
+        """Get the currently activated virtual environment path."""
+        return self.virtual_env_path
+    
+    def is_in_venv(self):
+        """Check if we're currently in a virtual environment."""
+        return self.virtual_env_path is not None and self.virtual_env_path != ""
+    
+    def get_venv_name(self):
+        """Get the name of the current virtual environment if active."""
+        if self.is_in_venv():
+            return os.path.basename(self.virtual_env_path)
+        return None
+    
+    def exec(self, *args, **kwargs):
+        """Compatibility method to make PersistentShell work with call_openai_for_debug."""
+        # Convert exec call to execute method
+        if len(args) >= 2 and args[0] == "bash" and args[1] == "-c":
+            command = args[2]
+            success, stdout, stderr = self.execute(command)
+            
+            # Create a mock result object that mimics the expected interface
+            class MockResult:
+                def __init__(self, stdout, stderr, returncode):
+                    self.stdout = [stdout] if stdout else []
+                    self.stderr = [stderr] if stderr else []
+                    self.returncode = 0 if returncode else 1
+                
+                def wait(self):
+                    pass
+            
+            return MockResult(stdout, stderr, success)
+        else:
+            raise NotImplementedError("exec method only supports bash -c commands")
+    
+    def wait_for_prompt(self, timeout=5):
+        """Wait for the shell prompt to appear, indicating readiness for next command."""
+        start_time = time.time()
+        while time.time() - start_time < timeout:
+            with self.stdout_lock:
+                if self.stdout_lines and self.stdout_lines[-1].strip().endswith('$'):
+                    return True
+            time.sleep(0.1)
+        return False
+    
+    def cleanup(self):
+        """Clean up the shell process."""
+        print("🧹 Cleaning up persistent shell...")
+        self.is_running = False
+        
+        if self.process:
+            try:
+                # Send exit command
+                self._send_command_raw("exit")
+                
+                # Wait for process to terminate
+                try:
+                    self.process.wait(timeout=5)
+                except subprocess.TimeoutExpired:
+                    # Force kill if it doesn't exit gracefully
+                    os.killpg(os.getpgid(self.process.pid), signal.SIGTERM)
+                    try:
+                        self.process.wait(timeout=2)
+                    except subprocess.TimeoutExpired:
+                        os.killpg(os.getpgid(self.process.pid), signal.SIGKILL)
+                        
+            except Exception as e:
+                print(f"Error during cleanup: {e}")
+            finally:
+                self.process = None
+        
+        print("✅ Shell cleanup completed")
+
+
 # First, try to fetch tokens from the proxy server
 try:
     # Import the fetch_modal_tokens module
@@ -501,7 +880,7 @@ def call_openai_for_debug(command, error_output, api_key=None, current_dir=None,
         print("💡 To enable LLM debugging, set the OPENAI_API_KEY environment variable")
         return None
     
-    print(f"✅ OpenAI API key available (length: {len(api_key)})")
+    # print(f"✅ OpenAI API key available (length: {len(api_key)})")
     
     # Gather additional context to help with debugging
     directory_context = ""
@@ -1015,7 +1394,7 @@ def generate_random_password(length=16):
     return password
 
 
-# Now modify the create_modal_ssh_container function to use the standalone ssh_container_function
+# Now modify the create_modal_ssh_container function to use the PersistentShell
 def create_modal_ssh_container(gpu_type, repo_url=None, repo_name=None, setup_commands=None, 
                                volume_name=None, timeout_minutes=60, ssh_password=None, interactive=False):
     """Create a Modal SSH container with GPU support and tunneling"""
@@ -1239,7 +1618,7 @@ def create_modal_ssh_container(gpu_type, repo_url=None, repo_name=None, setup_co
                 "python3", "python3-pip", "build-essential", "tmux", "screen", "nano",
                 "gpg", "ca-certificates", "software-properties-common"
             )
-            .pip_install("uv", "modal", "requests", "openai")  # Remove problematic CUDA packages
+            .uv_pip_install("uv", "modal", "requests", "openai")  # Remove problematic CUDA packages
             .run_commands(
                 # Create SSH directory
                 "mkdir -p /var/run/sshd",
@@ -1309,6 +1688,7 @@ def create_modal_ssh_container(gpu_type, repo_url=None, repo_name=None, setup_co
         subprocess.run(["service", "ssh", "start"], check=True)
         
         # Clone repository if provided
+        repo_dir = "/root"
         if repo_url:
             repo_name_from_url = repo_name or repo_url.split('/')[-1].replace('.git', '')
             print(f"📥 Cloning repository: {repo_url}")
@@ -1320,119 +1700,77 @@ def create_modal_ssh_container(gpu_type, repo_url=None, repo_name=None, setup_co
                 # Change to repository directory
                 repo_dir = f"/root/{repo_name_from_url}"
                 if os.path.exists(repo_dir):
-                    os.chdir(repo_dir)
-                    print(f"📂 Changed to repository directory: {repo_dir}")
+                    print(f"📂 Will run setup commands in repository directory: {repo_dir}")
                 
             except subprocess.CalledProcessError as e:
                 print(f"❌ Failed to clone repository: {e}")
         
-        # Run setup commands if provided
+        # Run setup commands if provided using PersistentShell
         if setup_commands:
-            print(f"⚙️ Running {len(setup_commands)} setup commands...")
+            print(f"⚙️ Running {len(setup_commands)} setup commands with persistent shell...")
             
-            # Define a helper function for running commands with LLM debugging
-            def run_command_with_basic_error_handling(cmd, show_output=True, retry_count=0, max_retries=2):
-                """Execute a command with LLM debugging enabled"""
-                print(f"🔧 Executing: {cmd}")
-                try:
-                    # Handle special case for source command which doesn't work with subprocess.run
-                    if cmd.strip().startswith("source ") or " source " in cmd:
-                        print("⚠️ Detected 'source' command which doesn't work with subprocess.run")
-                        print("🔄 Converting to bash -c with dot (.) instead of source")
-                        # Replace source with . (dot) which is the same as source but works in sh
-                        modified_cmd = cmd.replace("source ", ". ")
-                        # Wrap in bash -c to ensure it runs in bash
-                        bash_cmd = f"bash -c '{modified_cmd}'"
-                        print(f"🔄 Modified command: {bash_cmd}")
-                        result = subprocess.run(bash_cmd, shell=True, check=True, 
-                                              capture_output=True, text=True)
-                    else:
-                        result = subprocess.run(cmd, shell=True, check=True, 
-                                              capture_output=True, text=True)
+            # Create persistent shell instance
+            shell = PersistentShell(working_dir=repo_dir, timeout=120)
+            
+            try:
+                # Start the persistent shell
+                shell.start()
+                
+                # Execute each setup command
+                for i, cmd in enumerate(setup_commands, 1):
+                    print(f"📋 Executing command {i}/{len(setup_commands)}: {cmd}")
                     
-                    if result.stdout and show_output:
-                        print(f"✅ Output: {result.stdout}")
-                    return True, result.stdout, ""
-                except subprocess.CalledProcessError as e:
-                    error_output = e.stderr if e.stderr else str(e)
-                    print(f"❌ Command failed: {e}")
-                    print(f"❌ Error: {error_output}")
+                    success, stdout, stderr = shell.execute(cmd, timeout=120)
                     
-                    # Call OpenAI for debugging
-                    print("🔍 Attempting to debug the failed command with OpenAI...")
-                    try:
-                        # Get the current directory for context
-                        current_dir = os.getcwd()
+                    if not success:
+                        print(f"⚠️ Command {i} failed, attempting LLM debugging...")
                         
                         # Call OpenAI for debugging
-                        print(f"🔍 DEBUG: About to call call_openai_for_debug...")
-                        print(f"🔍 DEBUG: Command: {cmd}")
-                        print(f"🔍 DEBUG: Error output length: {len(error_output)}")
-                        print(f"🔍 DEBUG: Current directory: {current_dir}")
-                        
-                        # Get the API key from environment or use the one that was fetched earlier
-                        api_key = os.environ.get("OPENAI_API_KEY")
-                        fix_command = call_openai_for_debug(cmd, error_output, api_key=api_key, current_dir=current_dir)
-                        
-                        print(f"🔍 DEBUG: call_openai_for_debug returned: {fix_command}")
-                        
-                        if fix_command:
-                            print(f"🔧 OpenAI suggested fix command: {fix_command}")
+                        try:
+                            current_dir = shell.get_cwd()
+                            api_key = os.environ.get("OPENAI_API_KEY")
                             
-                            # Run the fix command
-                            print(f"🔄 Running suggested fix command: {fix_command}")
-                            try:
-                                fix_result = subprocess.run(fix_command, shell=True, check=True, 
-                                                          capture_output=True, text=True)
-                                if fix_result.stdout:
-                                    print(f"✅ Fix command output: {fix_result.stdout}")
-                                
-                                # Retry the original command
-                                print(f"🔄 Retrying original command: {cmd}")
-                                return run_command_with_basic_error_handling(cmd, show_output, retry_count + 1, max_retries)
-                            except subprocess.CalledProcessError as fix_e:
-                                print(f"❌ Fix command also failed: {fix_e}")
-                                return False, "", error_output
-                        else:
-                            print("❌ No fix suggested by OpenAI")
-                            return False, "", error_output
+                            # Use your existing call_openai_for_debug function
+                            fix_command = call_openai_for_debug(cmd, stderr, api_key=api_key, current_dir=current_dir, sandbox=shell)
                             
-                    except Exception as debug_e:
-                        print(f"❌ LLM debugging failed: {debug_e}")
-                        return False, "", error_output
-            
-            # Run each setup command
-            for i, cmd in enumerate(setup_commands, 1):
-                print(f"📋 Executing command {i}/{len(setup_commands)}: {cmd}")
-                
-                # Check if this is a cd command and if the directory exists
-                if cmd.strip().startswith("cd "):
-                    cd_parts = cmd.split(None, 1)
-                    if len(cd_parts) >= 2:
-                        target_dir = cd_parts[1].strip('"\'')
-                        print(f"🔍 Checking if directory exists: {target_dir}")
-                        try:
-                            check_result = subprocess.run(f"test -d '{target_dir}'", shell=True, 
-                                                       capture_output=True, text=True)
-                            if check_result.returncode != 0:
-                                print(f"⚠️ Directory does not exist: {target_dir}")
-                                print(f"🔍 Current directory contents:")
-                                subprocess.run("pwd && ls -la", shell=True, check=False)
+                            if fix_command:
+                                print(f"🔧 OpenAI suggested fix command: {fix_command}")
+                                
+                                # Run the fix command in the persistent shell
+                                print(f"🔄 Running suggested fix command: {fix_command}")
+                                fix_success, fix_stdout, fix_stderr = shell.execute(fix_command, timeout=120)
                                 
-                                # Try to find similar directories
-                                print(f"🔍 Looking for similar directories...")
-                                subprocess.run("find . -type d -name '*llama*' -o -name '*nano*' 2>/dev/null | head -10", shell=True, check=False)
-                        except Exception as e:
-                            print(f"⚠️ Could not check directory: {e}")
+                                if fix_success:
+                                    print(f"✅ Fix command succeeded")
+                                    
+                                    # Retry the original command
+                                    print(f"🔄 Retrying original command: {cmd}")
+                                    retry_success, retry_stdout, retry_stderr = shell.execute(cmd, timeout=120)
+                                    
+                                    if retry_success:
+                                        print(f"✅ Original command succeeded after fix!")
+                                    else:
+                                        print(f"⚠️ Original command still failed after fix, continuing...")
+                                else:
+                                    print(f"❌ Fix command failed: {fix_stderr}")
+                                    print(f"⚠️ Continuing with remaining commands...")
+                            else:
+                                print("❌ No fix suggested by OpenAI")
+                                print(f"⚠️ Continuing with remaining commands...")
+                                
+                        except Exception as debug_e:
+                            print(f"❌ LLM debugging failed: {debug_e}")
+                            print(f"⚠️ Continuing with remaining commands...")
+                    else:
+                        print(f"✅ Command {i} completed successfully")
                 
-                success, stdout, stderr = run_command_with_basic_error_handling(cmd, show_output=True)
-                if not success:
-                    print(f"⚠️ Command {i} failed, but continuing with remaining commands...")
-                    
-                    # If this was a cd command that failed, try to understand the directory structure
-                    if cmd.strip().startswith("cd ") and "No such file or directory" in stderr:
-                        print(f"🔍 Analyzing directory structure after failed cd command...")
-                        subprocess.run("pwd && ls -la && echo '--- Parent directory ---' && ls -la ..", shell=True, check=False)
+                print("✅ All setup commands processed")
+                
+            except Exception as e:
+                print(f"❌ Error during setup command execution: {e}")
+            finally:
+                # Clean up the shell
+                shell.cleanup()
         
         # Create SSH tunnel
         with modal.forward(22, unencrypted=True) as tunnel: