gitarsenal-cli 1.9.21 → 1.9.23

package/python/setup_modal_token.py

@@ -56,45 +56,6 @@ def setup_modal_token():
     
     return True
     
-def get_token_from_gitarsenal():
-    """Get Modal token from GitArsenal credentials file (fallback method)"""
-    try:
-        from credentials_manager import CredentialsManager
-        credentials_manager = CredentialsManager()
-        token = credentials_manager.get_modal_token()
-        if token:
-            logger.info("Found Modal token in GitArsenal credentials")
-            return token
-    except ImportError:
-        logger.warning("Could not import CredentialsManager")
-    except Exception as e:
-        logger.warning(f"Error getting token from GitArsenal credentials: {e}")
-    
-    return None
-    
-def get_token_from_modal_file():
-    """Get Modal token from Modal token file (fallback method)"""
-    try:
-        import json
-        token_file = Path.home() / ".modal" / "token.json"
-        
-        if not token_file.exists():
-            logger.warning(f"Modal token file not found at {token_file}")
-            return None
-            
-        with open(token_file, 'r') as f:
-            token_data = json.load(f)
-            
-        # The token file contains both token_id and token
-        token_id = token_data.get("token_id")
-        if token_id:
-            logger.info("Found token_id in Modal token file")
-            return token_id
-    except Exception as e:
-        logger.warning(f"Error reading Modal token file: {e}")
-    
-    return None
-    
 if __name__ == "__main__":
     if setup_modal_token():
         print("✅ Modal token set up successfully")

package/python/shell.py

@@ -0,0 +1,610 @@
+import threading
+import subprocess
+import os
+import time
+import uuid
+import re
+
+class PersistentShell:
+    """A persistent bash shell using subprocess.Popen for executing commands with state persistence."""
+    
+    def __init__(self, working_dir="/root", timeout=60):
+        self.working_dir = working_dir
+        self.timeout = timeout
+        self.process = None
+        self.stdout_lines = []  # Use list instead of queue
+        self.stderr_lines = []  # Use list instead of queue
+        self.stdout_lock = threading.Lock()
+        self.stderr_lock = threading.Lock()
+        self.stdout_thread = None
+        self.stderr_thread = None
+        self.command_counter = 0
+        self.is_running = False
+        self.virtual_env_path = None  # Track activated virtual environment
+        self.suggested_alternative = None  # Store suggested alternative commands
+        self.should_remove_command = False  # Flag to indicate if a command should be removed
+        self.removal_reason = None  # Reason for removing a command
+        
+    def start(self):
+        """Start the persistent bash shell."""
+        if self.is_running:
+            return
+            
+        print(f"🐚 Starting persistent bash shell in {self.working_dir}")
+        
+        # Start bash with unbuffered output
+        self.process = subprocess.Popen(
+            ['bash', '-i'],  # Interactive bash
+            stdin=subprocess.PIPE,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+            text=True,
+            bufsize=0,  # Unbuffered
+            cwd=self.working_dir,
+            preexec_fn=os.setsid  # Create new process group
+        )
+        
+        # Start threads to read stdout and stderr
+        self.stdout_thread = threading.Thread(target=self._read_stdout, daemon=True)
+        self.stderr_thread = threading.Thread(target=self._read_stderr, daemon=True)
+        
+        self.stdout_thread.start()
+        self.stderr_thread.start()
+        
+        self.is_running = True
+        
+        # Initial setup commands
+        self._send_command_raw("set +h")  # Disable hash table for commands
+        self._send_command_raw("export PS1='$ '")  # Simpler prompt
+        self._send_command_raw("cd " + self.working_dir)  # Change to working directory
+        time.sleep(0.5)  # Let initial commands settle
+        
+        
+    def _read_stdout(self):
+        """Read stdout in a separate thread."""
+        while self.process and self.process.poll() is None:
+            try:
+                line = self.process.stdout.readline()
+                if line:
+                    with self.stdout_lock:
+                        self.stdout_lines.append(line.rstrip('\n'))
+                else:
+                    time.sleep(0.01)
+            except Exception as e:
+                print(f"Error reading stdout: {e}")
+                break
+                
+    def _read_stderr(self):
+        """Read stderr in a separate thread."""
+        while self.process and self.process.poll() is None:
+            try:
+                line = self.process.stderr.readline()
+                if line:
+                    with self.stderr_lock:
+                        self.stderr_lines.append(line.rstrip('\n'))
+                else:
+                    time.sleep(0.01)
+            except Exception as e:
+                print(f"Error reading stderr: {e}")
+                break
+    
+    def _send_command_raw(self, command):
+        """Send a raw command to the shell without waiting for completion."""
+        if not self.is_running or not self.process:
+            raise RuntimeError("Shell is not running")
+            
+        try:
+            self.process.stdin.write(command + '\n')
+            self.process.stdin.flush()
+        except Exception as e:
+            print(f"Error sending command: {e}")
+            raise
+    
+    def _preprocess_command(self, command):
+        """Preprocess commands to handle special cases like virtual environment activation."""
+        # Handle virtual environment creation and activation
+        if "uv venv" in command and "&&" in command and "source" in command:
+            # Split the compound command into separate parts
+            parts = [part.strip() for part in command.split("&&")]
+            return parts
+        elif command.strip().startswith("source ") and "/bin/activate" in command:
+            # Handle standalone source command
+            venv_path = command.replace("source ", "").replace("/bin/activate", "").strip()
+            self.virtual_env_path = venv_path
+            return [command]
+        elif "source" in command and "activate" in command:
+            # Handle any other source activation pattern
+            return [command]
+        elif "uv pip install" in command and self.is_in_venv():
+            # If we're in a virtual environment, ensure we use the right pip
+            return [command]
+        else:
+            return [command]
+    
+    def execute(self, command, timeout=None):
+        """Execute a command and return (success, stdout, stderr)."""
+        if not self.is_running:
+            self.start()
+            
+        if timeout is None:
+            timeout = self.timeout
+        
+        # Preprocess the command to handle special cases
+        command_parts = self._preprocess_command(command)
+        
+        # If we have multiple parts, execute them sequentially
+        if len(command_parts) > 1:
+            print(f"🔧 Executing compound command in {len(command_parts)} parts")
+            all_stdout = []
+            all_stderr = []
+            
+            for i, part in enumerate(command_parts):
+                print(f"   Part {i+1}/{len(command_parts)}: {part}")
+                success, stdout, stderr = self._execute_single(part, timeout)
+                
+                if stdout:
+                    all_stdout.append(stdout)
+                if stderr:
+                    all_stderr.append(stderr)
+                
+                if not success:
+                    # If any part fails, return the failure
+                    return False, '\n'.join(all_stdout), '\n'.join(all_stderr)
+                
+                # Small delay between parts to let environment changes take effect
+                time.sleep(0.1)
+            
+            return True, '\n'.join(all_stdout), '\n'.join(all_stderr)
+        else:
+            return self._execute_single(command_parts[0], timeout)
+    
+    def _execute_single(self, command, timeout):
+        """Execute a single command and return (success, stdout, stderr)."""
+        self.command_counter += 1
+        marker = f"CMD_DONE_{self.command_counter}_{uuid.uuid4().hex[:8]}"
+        
+        print(f"🔧 Executing: {command}")
+        
+        # Clear any existing output
+        self._clear_lines()
+        
+        # Wait for shell to be ready (prompt should be visible)
+        if not self.wait_for_prompt(timeout=2):
+            # print("⚠️ Shell not ready, waiting...")
+            time.sleep(0.5)
+        
+        # For source commands, we need special handling
+        if command.strip().startswith("source "):
+            # Send the source command in a way that preserves the environment
+            try:
+                # Extract the virtual environment path
+                venv_path = command.replace("source ", "").replace("/bin/activate", "").strip()
+                
+                # Use a more robust approach that actually activates the environment
+                activation_script = f"""
+                if [ -f "{venv_path}/bin/activate" ]; then
+                    source "{venv_path}/bin/activate"
+                    echo "VIRTUAL_ENV=$VIRTUAL_ENV"
+                    echo "PATH=$PATH"
+                    echo 'SOURCE_SUCCESS'
+                else
+                    echo 'SOURCE_FAILED - activation script not found'
+                fi
+                """
+                
+                self._send_command_raw(activation_script)
+                time.sleep(0.3)  # Give more time for environment changes
+                self._send_command_raw(f'echo "EXIT_CODE:$?"')
+                self._send_command_raw(f'echo "{marker}"')
+            except Exception as e:
+                return False, "", f"Failed to send source command: {e}"
+        else:
+            # Send the command followed by markers
+            try:
+                self._send_command_raw(command)
+                # Wait a moment for the command to start
+                time.sleep(0.1)
+                self._send_command_raw(f'echo "EXIT_CODE:$?"')
+                self._send_command_raw(f'echo "{marker}"')
+            except Exception as e:
+                return False, "", f"Failed to send command: {e}"
+        
+        # Collect output until we see the marker
+        command_stdout = []
+        command_stderr = []
+        start_time = time.time()
+        found_marker = False
+        exit_code = None
+        last_stdout_index = 0
+        last_stderr_index = 0
+        source_success = None
+        
+        while time.time() - start_time < timeout:
+            # Check for new stdout lines
+            with self.stdout_lock:
+                current_stdout = self.stdout_lines[last_stdout_index:]
+                last_stdout_index = len(self.stdout_lines)
+            
+            for line in current_stdout:
+                if line == marker:
+                    found_marker = True
+                    break
+                elif line.startswith("EXIT_CODE:"):
+                    try:
+                        exit_code = int(line.split(":", 1)[1])
+                    except (ValueError, IndexError):
+                        exit_code = 1
+                elif line == "SOURCE_SUCCESS":
+                    source_success = True
+                elif line.startswith("SOURCE_FAILED"):
+                    source_success = False
+                    command_stderr.append(line)
+                elif line.startswith("VIRTUAL_ENV="):
+                    # Extract and store the virtual environment path
+                    venv_path = line.split("=", 1)[1]
+                    self.virtual_env_path = venv_path
+                    command_stdout.append(line)
+                elif line.startswith("PATH="):
+                    # Store the updated PATH
+                    command_stdout.append(line)
+                elif line.strip() and not line.startswith("$"):  # Skip empty lines and prompt lines
+                    command_stdout.append(line)
+            
+            if found_marker:
+                break
+            
+            # Check for new stderr lines
+            with self.stderr_lock:
+                current_stderr = self.stderr_lines[last_stderr_index:]
+                last_stderr_index = len(self.stderr_lines)
+            
+            for line in current_stderr:
+                if line.strip():  # Skip empty lines
+                    command_stderr.append(line)
+            
+            # Check if command is waiting for user input
+            if not found_marker and time.time() - start_time > 5:  # Wait at least 5 seconds before checking
+                if self._is_waiting_for_input(command_stdout, command_stderr):
+                    print("⚠️ Command appears to be waiting for user input")
+                    # Try to handle the input requirement
+                    input_handled = self._handle_input_requirement(command, command_stdout, command_stderr)
+                    
+                    if input_handled is True and self.should_remove_command:
+                        # If LLM suggested to remove the command
+                        self._send_command_raw("\x03")  # Send Ctrl+C
+                        time.sleep(0.5)
+                        return False, '\n'.join(command_stdout), f"Command removed - {self.removal_reason}"
+                    elif not input_handled:
+                        # If we couldn't handle the input, abort the command
+                        self._send_command_raw("\x03")  # Send Ctrl+C
+                        time.sleep(0.5)
+                        return False, '\n'.join(command_stdout), "Command aborted - requires user input"
+            
+            time.sleep(0.1)
+        
+        if not found_marker:
+            print(f"⚠️ Command timed out after {timeout} seconds")
+            return False, '\n'.join(command_stdout), f"Command timed out after {timeout} seconds"
+        
+        stdout_text = '\n'.join(command_stdout)
+        stderr_text = '\n'.join(command_stderr)
+        
+        # Determine success based on multiple factors
+        if source_success is not None:
+            success = source_success
+        else:
+            success = exit_code == 0 if exit_code is not None else len(command_stderr) == 0
+        
+        if success:
+            if stdout_text:
+                print(f"✅ Output: {stdout_text}")
+            # Track virtual environment activation
+            if command.strip().startswith("source ") and "/bin/activate" in command:
+                venv_path = command.replace("source ", "").replace("/bin/activate", "").strip()
+                self.virtual_env_path = venv_path
+                print(f"✅ Virtual environment activated: {venv_path}")
+        else:
+            print(f"❌ Command failed with exit code: {exit_code}")
+            if stderr_text:
+                print(f"❌ Error: {stderr_text}")
+        
+        # Wait a moment for the shell to be ready for the next command
+        time.sleep(0.2)
+        
+        return success, stdout_text, stderr_text
+    
+    def _is_waiting_for_input(self, stdout_lines, stderr_lines):
+        """Detect if a command is waiting for user input."""
+        # Common patterns that indicate waiting for user input
+        input_patterns = [
+            r'(?i)(y/n|yes/no)\??\s*$',  # Yes/No prompts
+            r'(?i)password:?\s*$',        # Password prompts
+            r'(?i)continue\??\s*$',       # Continue prompts
+            r'(?i)proceed\??\s*$',        # Proceed prompts
+            r'\[\s*[Yy]/[Nn]\s*\]\s*$',   # [Y/n] style prompts
+            r'(?i)username:?\s*$',        # Username prompts
+            r'(?i)token:?\s*$',           # Token prompts
+            r'(?i)api key:?\s*$',         # API key prompts
+            r'(?i)press enter to continue', # Press enter prompts
+            r'(?i)select an option:?\s*$', # Selection prompts
+            r'(?i)choose an option:?\s*$', # Choice prompts
+        ]
+        
+        # Check the last few lines of stdout and stderr for input patterns
+        last_lines = []
+        if stdout_lines:
+            last_lines.extend(stdout_lines[-3:])  # Check last 3 lines of stdout
+        if stderr_lines:
+            last_lines.extend(stderr_lines[-3:])  # Check last 3 lines of stderr
+        
+        for line in last_lines:
+            for pattern in input_patterns:
+                if re.search(pattern, line):
+                    print(f"🔍 Detected input prompt: {line}")
+                    return True
+                
+        # Check if there's no output for a while but the command is still running
+        if len(stdout_lines) == 0 and len(stderr_lines) == 0:
+            # This might be a command waiting for input without a prompt
+            # We'll be cautious and only return True if we're sure
+            return False
+            
+        return False
+        
+    def _handle_input_requirement(self, command, stdout_lines, stderr_lines):
+        """Attempt to handle commands that require input."""
+        # Extract the last few lines to analyze what kind of input is needed
+        last_lines = []
+        if stdout_lines:
+            last_lines.extend(stdout_lines[-3:])
+        if stderr_lines:
+            last_lines.extend(stderr_lines[-3:])
+        
+        last_line = last_lines[-1] if last_lines else ""
+        
+        # Try to determine what kind of input is needed
+        if re.search(r'(?i)(y/n|yes/no|\[y/n\])', last_line):
+            # For yes/no prompts, usually 'yes' is safer
+            print("🔧 Auto-responding with 'y' to yes/no prompt")
+            self._send_command_raw("y")
+            return True
+            
+        elif re.search(r'(?i)password', last_line):
+            # For password prompts, check if we have stored credentials
+            stored_creds = get_stored_credentials()
+            if stored_creds and 'ssh_password' in stored_creds:
+                print("🔧 Auto-responding with stored SSH password")
+                self._send_command_raw(stored_creds['ssh_password'])
+                return True
+            else:
+                print("⚠️ Password prompt detected but no stored password available")
+                return False
+                
+        elif re.search(r'(?i)token|api.key', last_line):
+            # For token/API key prompts
+            stored_creds = get_stored_credentials()
+            if stored_creds:
+                if 'openai_api_key' in stored_creds and re.search(r'(?i)openai|api.key', last_line):
+                    print("🔧 Auto-responding with stored OpenAI API key")
+                    self._send_command_raw(stored_creds['openai_api_key'])
+                    return True
+                elif 'hf_token' in stored_creds and re.search(r'(?i)hugg|hf|token', last_line):
+                    print("🔧 Auto-responding with stored Hugging Face token")
+                    self._send_command_raw(stored_creds['hf_token'])
+                    return True
+            
+            print("⚠️ Token/API key prompt detected but no matching stored credentials")
+            return False
+            
+        elif re.search(r'(?i)press enter|continue|proceed', last_line):
+            # For "press enter to continue" prompts
+            print("🔧 Auto-responding with Enter to continue")
+            self._send_command_raw("")  # Empty string sends just Enter
+            return True
+            
+        # If we can't determine the type of input needed
+        print("⚠️ Couldn't determine the type of input needed")
+        
+        # Try to use LLM to suggest an alternative command
+        try:
+            # Get current working directory for context
+            cwd = self.get_cwd()
+            
+            # Reset command removal flags
+            self.should_remove_command = False
+            self.removal_reason = None
+            
+            # Call LLM to suggest an alternative
+            alternative = self._suggest_alternative_command(command, stdout_lines, stderr_lines, cwd)
+            
+            # Check if LLM suggested to remove the command
+            if self.should_remove_command:
+                print(f"🚫 Command will be removed: {self.removal_reason}")
+                return True  # Return True to indicate the command has been handled (by removing it)
+            
+            if alternative:
+                print(f"🔧 LLM suggested alternative command: {alternative}")
+                # We don't execute the alternative here, but return False so the calling code
+                # can handle it (e.g., by adding it to the command list)
+                
+                # Store the suggested alternative for later use
+                self.suggested_alternative = alternative
+                return False
+        except Exception as e:
+            print(f"⚠️ Error getting LLM suggestion: {e}")
+        
+        return False
+        
+    def _suggest_alternative_command(self, command, stdout_lines, stderr_lines, current_dir):
+        """Use LLM to suggest an alternative command that doesn't require user input."""
+        try:
+            # Get API key
+            api_key = os.environ.get("OPENAI_API_KEY")
+            if not api_key:
+                # Try to load from saved file
+                key_file = os.path.expanduser("~/.gitarsenal/openai_key")
+                if os.path.exists(key_file):
+                    with open(key_file, "r") as f:
+                        api_key = f.read().strip()
+            
+            if not api_key:
+                print("⚠️ No OpenAI API key available for suggesting alternative command")
+                return None
+                
+            # Prepare the prompt
+            stdout_text = '\n'.join(stdout_lines[-10:]) if stdout_lines else ""
+            stderr_text = '\n'.join(stderr_lines[-10:]) if stderr_lines else ""
+            
+            prompt = f"""
+            The command '{command}' appears to be waiting for user input.
+            
+            Current directory: {current_dir}
+            
+            Last stdout output:
+            {stdout_text}
+            
+            Last stderr output:
+            {stderr_text}
+            
+            Please analyze this command and determine if it's useful to continue with it.
+            If it's useful, suggest an alternative command that achieves the same goal but doesn't require user input.
+            For example, add flags like -y, --yes, --no-input, etc., or provide the required input in the command.
+            
+            If the command is not useful or cannot be executed non-interactively, respond with "REMOVE_COMMAND" and explain why.
+            
+            Format your response as:
+            ALTERNATIVE: <alternative command>
+            or
+            REMOVE_COMMAND: <reason>
+            """
+            
+            # Call OpenAI API
+            import openai
+            client = openai.OpenAI(api_key=api_key)
+            
+            response = client.chat.completions.create(
+                model="gpt-4o-mini",
+                messages=[
+                    {"role": "system", "content": "You are a helpful assistant that suggests alternative commands that don't require user input."},
+                    {"role": "user", "content": prompt}
+                ],
+                max_tokens=150,
+                temperature=0.7
+            )
+            
+            response_text = response.choices[0].message.content.strip()
+            
+            # Check if the response suggests removing the command
+            if response_text.startswith("REMOVE_COMMAND:"):
+                reason = response_text.replace("REMOVE_COMMAND:", "").strip()
+                print(f"🚫 LLM suggests removing command: {reason}")
+                self.should_remove_command = True
+                self.removal_reason = reason
+                return None
+            
+            # Extract the alternative command
+            if response_text.startswith("ALTERNATIVE:"):
+                alternative_command = response_text.replace("ALTERNATIVE:", "").strip()
+            else:
+                # Try to extract the command from a free-form response
+                lines = response_text.split('\n')
+                for line in lines:
+                    line = line.strip()
+                    if line and not line.startswith(('Here', 'I', 'You', 'The', 'This', 'Use', 'Try')):
+                        alternative_command = line
+                        break
+                else:
+                    alternative_command = lines[0].strip()
+            
+            return alternative_command
+            
+        except Exception as e:
+            print(f"⚠️ Error suggesting alternative command: {e}")
+            return None
+    
+    def _clear_lines(self):
+        """Clear both output line lists."""
+        with self.stdout_lock:
+            self.stdout_lines.clear()
+        with self.stderr_lock:
+            self.stderr_lines.clear()
+    
+    def get_cwd(self):
+        """Get current working directory."""
+        success, output, _ = self._execute_single("pwd", 10)
+        if success:
+            return output.strip()
+        return self.working_dir
+    
+    def get_virtual_env(self):
+        """Get the currently activated virtual environment path."""
+        return self.virtual_env_path
+    
+    def is_in_venv(self):
+        """Check if we're currently in a virtual environment."""
+        return self.virtual_env_path is not None and self.virtual_env_path != ""
+    
+    def get_venv_name(self):
+        """Get the name of the current virtual environment if active."""
+        if self.is_in_venv():
+            return os.path.basename(self.virtual_env_path)
+        return None
+    
+    def exec(self, *args, **kwargs):
+        """Compatibility method to make PersistentShell work with call_openai_for_debug."""
+        # Convert exec call to execute method
+        if len(args) >= 2 and args[0] == "bash" and args[1] == "-c":
+            command = args[2]
+            success, stdout, stderr = self.execute(command)
+            
+            # Create a mock result object that mimics the expected interface
+            class MockResult:
+                def __init__(self, stdout, stderr, returncode):
+                    self.stdout = [stdout] if stdout else []
+                    self.stderr = [stderr] if stderr else []
+                    self.returncode = 0 if returncode else 1
+                
+                def wait(self):
+                    pass
+            
+            return MockResult(stdout, stderr, success)
+        else:
+            raise NotImplementedError("exec method only supports bash -c commands")
+    
+    def wait_for_prompt(self, timeout=5):
+        """Wait for the shell prompt to appear, indicating readiness for next command."""
+        start_time = time.time()
+        while time.time() - start_time < timeout:
+            with self.stdout_lock:
+                if self.stdout_lines and self.stdout_lines[-1].strip().endswith('$'):
+                    return True
+            time.sleep(0.1)
+        return False
+    
+    def cleanup(self):
+        """Clean up the shell process."""
+        print("🧹 Cleaning up persistent shell...")
+        self.is_running = False
+        
+        if self.process:
+            try:
+                # Send exit command
+                self._send_command_raw("exit")
+                
+                # Wait for process to terminate
+                try:
+                    self.process.wait(timeout=5)
+                except subprocess.TimeoutExpired:
+                    # Force kill if it doesn't exit gracefully
+                    os.killpg(os.getpgid(self.process.pid), signal.SIGTERM)
+                    try:
+                        self.process.wait(timeout=2)
+                    except subprocess.TimeoutExpired:
+                        os.killpg(os.getpgid(self.process.pid), signal.SIGKILL)
+                        
+            except Exception as e:
+                print(f"Error during cleanup: {e}")
+            finally:
+                self.process = None
+        
+        print("✅ Shell cleanup completed")

package/python/test_modalSandboxScript.py

@@ -2411,7 +2411,7 @@ def switch_to_openai_models():
 
 def get_current_debug_model():
     """Get the currently configured debugging model preference"""
-    return os.environ.get("GITARSENAL_DEBUG_MODEL", "anthropic")
+    return os.environ.get("GITARSENAL_DEBUG_MODEL", "openai")
 
 def call_llm_for_debug(command, error_output, api_key=None, current_dir=None, sandbox=None):
     """Unified function to call LLM for debugging - routes to OpenAI or Anthropic based on configuration"""
@@ -5001,4 +5001,4 @@ if __name__ == "__main__":
         # print(f"\n❌ Error: {e}")
         # print("🧹 Cleaning up resources...")
         cleanup_modal_token()
-        sys.exit(1)
+