bone-agent 1.3.0

package/src/tools/helpers/loader.py

@@ -0,0 +1,231 @@
+"""Tool auto-discovery and loading mechanism.
+
+This module provides automatic discovery and loading of tools from
+multiple directories. Tools are imported to trigger @tool decorator
+registration.
+"""
+
+import importlib.util
+import logging
+import sys
+from pathlib import Path
+from typing import List, Optional
+
+from .base import ToolRegistry
+
+logger = logging.getLogger(__name__)
+
+
+def _is_python_file(path: Path) -> bool:
+    """Check if a file is a Python module.
+
+    Args:
+        path: File path to check
+
+    Returns:
+        True if file is a .py file (not __pycache__ or test file)
+    """
+    return (
+        path.suffix == ".py"
+        and path.name != "__init__.py"
+        and not path.name.startswith("test_")
+        and not path.name.startswith("_")
+    )
+
+
+def _load_module_from_path(module_name: str, file_path: Path) -> Optional[object]:
+    """Load a Python module from a file path.
+
+    Args:
+        module_name: Name to give the module
+        file_path: Path to the Python file
+
+    Returns:
+        Loaded module or None if loading failed
+    """
+    try:
+        spec = importlib.util.spec_from_file_location(module_name, file_path)
+        if spec is None or spec.loader is None:
+            logger.warning(f"Could not load spec for {file_path}")
+            return None
+
+        module = importlib.util.module_from_spec(spec)
+
+        # For user tools (not in src/utils/tools/), set package to None
+        # to force absolute imports instead of relative imports
+        from tools import __file__ as tools_init_file
+        tools_dir = Path(tools_init_file).parent
+
+        # User tools are those not in the main tools directory
+        # (helper modules are in src/tools/helpers/)
+        if file_path.parent != tools_dir and file_path.parent != tools_dir / "helpers":
+            # User tool - set to None to force absolute imports
+            module.__package__ = None
+
+        sys.modules[module_name] = module
+        spec.loader.exec_module(module)
+
+        logger.debug(f"Successfully loaded module: {module_name}")
+        return module
+
+    except Exception as e:
+        logger.warning(f"Failed to load module {module_name} from {file_path}: {e}")
+        return None
+
+
+def discover_tools(directories: List[str]) -> int:
+    """Discover and load tools from specified directories.
+
+    This scans directories for Python files and imports them,
+    which triggers @tool decorator registration.
+
+    Args:
+        directories: List of directory paths to scan
+
+    Returns:
+        Number of tools successfully loaded
+
+    Note:
+        - Only .py files are considered (excluding __pycache__, tests)
+        - Import errors are logged but don't stop discovery
+        - User tools can override built-in tools (with warning)
+    """
+    initial_count = ToolRegistry.tool_count()
+    loaded_count = 0
+
+    for directory in directories:
+        dir_path = Path(directory)
+
+        if not dir_path.exists():
+            logger.debug(f"Tool directory does not exist: {directory}")
+            continue
+
+        if not dir_path.is_dir():
+            logger.warning(f"Tool path is not a directory: {directory}")
+            continue
+
+        logger.info(f"Discovering tools in: {directory}")
+
+        # Find all Python files
+        python_files = [f for f in dir_path.iterdir() if _is_python_file(f)]
+
+        for py_file in python_files:
+            # Create unique module name
+            module_name = f"tools_{py_file.stem}_{hash(str(py_file)) & 0xFFFFFFFF}"
+
+            # Skip modules already loaded (e.g. cron re-calling load_all_tools)
+            if module_name in sys.modules:
+                logger.debug(f"Module already loaded, skipping: {module_name}")
+                continue
+
+            module = _load_module_from_path(module_name, py_file)
+            if module:
+                loaded_count += 1
+
+    final_count = ToolRegistry.tool_count()
+    new_tools = final_count - initial_count
+
+    logger.info(
+        f"Tool discovery complete: Loaded {loaded_count} modules, "
+        f"registered {new_tools} new tools (total: {final_count})"
+    )
+
+    return new_tools
+
+
+def load_builtin_tools() -> int:
+    """Load built-in tools from src/utils/tools/.
+
+    Returns:
+        Number of tools loaded
+
+    Note:
+        Built-in tools are already imported in __init__.py, which triggers
+        @tool decorator registration. This function returns the count of
+        currently registered built-in tools.
+    """
+    # Built-in tools are already imported in utils/tools/__init__.py
+    # which triggers @tool decorator registration.
+    # Just return the count of tools currently in registry.
+    return ToolRegistry.tool_count()
+
+
+def load_plugin_tools() -> int:
+    """Load plugin tools from tool_plugins/ directory.
+
+    Plugin modules are imported, which triggers @tool(tier="plugin") decorator
+    registration into the PluginManifest (NOT ToolRegistry). This keeps plugin
+    schemas out of the LLM context window until activated via search_plugins.
+
+    Returns:
+        Number of plugin modules loaded
+
+    Note:
+        - tool_plugins/ directory at repository root
+        - Plugin tools registered in PluginManifest, not ToolRegistry
+    """
+    from .plugin_manifest import plugin_manifest
+
+    # Get repository root (assumes we're in src/tools/helpers/)
+    current_dir = Path(__file__).parent
+    repo_root = current_dir.parent.parent.parent
+
+    # Define plugin tool directories
+    plugin_directories = [
+        str(repo_root / "tool_plugins"),
+    ]
+
+    # Discover tools in plugin directories
+    modules_loaded = discover_tools(plugin_directories)
+
+    logger.info(
+        f"Plugin manifest: {plugin_manifest.plugin_count()} plugins available "
+        f"(categories: {plugin_manifest.get_categories()})"
+    )
+
+    return modules_loaded
+
+
+def load_all_tools() -> int:
+    """Load all tools (built-in and plugin tools).
+
+    Returns:
+        Total number of tools loaded
+
+    Discovery order:
+        1. Built-in tools (src/tools/*.py)
+        2. Plugin tools (tool_plugins/*.py)
+
+    Note:
+        Plugin tools loaded later can override built-in tools.
+    """
+    logger.info("Starting tool loading...")
+
+    # Load built-in tools first
+    builtin_count = load_builtin_tools()
+
+    # Then load plugin tools (can override built-ins)
+    plugin_count = load_plugin_tools()
+
+    total_count = builtin_count + plugin_count
+    total_registered = ToolRegistry.tool_count()
+
+    logger.info(
+        f"Tool loading complete: {builtin_count} built-in + "
+        f"{plugin_count} plugin modules = {total_count} modules, "
+        f"{total_registered} tools registered"
+    )
+
+    return total_count
+
+
+def list_registered_tools() -> List[str]:
+    """List names of all registered tools.
+
+    Returns:
+        List of tool names
+    """
+    return [tool.name for tool in ToolRegistry.get_all()]
+
+
+

package/src/tools/helpers/parallel_executor.py

@@ -0,0 +1,231 @@
+"""Concurrent tool execution engine.
+
+This module provides parallel execution of multiple tool calls using
+ThreadPoolExecutor for I/O-bound operations like file reads and web searches.
+"""
+
+import concurrent.futures
+from typing import List, Tuple
+from dataclasses import dataclass
+
+from .base import ToolRegistry, build_context
+
+
+@dataclass
+class ToolCall:
+    """Represents a single tool call.
+
+    Attributes:
+        tool_id: Unique identifier for this tool call
+        function_name: Name of the tool function to execute
+        arguments: Dictionary of arguments to pass to the tool handler
+        call_index: Index in original tool_calls array (for order preservation)
+    """
+    tool_id: str
+    function_name: str
+    arguments: dict
+    call_index: int
+
+
+@dataclass
+class ToolResult:
+    """Result of a tool execution.
+
+    Attributes:
+        tool_id: Unique identifier for the tool call
+        call_index: Index in original tool_calls array (for order preservation)
+        success: Whether the tool executed successfully
+        result: String result from tool execution (if successful)
+        error: Error message (if failed)
+        should_exit: Whether the tool requested the orchestration loop to exit
+        requires_approval: Whether this tool requires user approval (for orchestrator)
+    """
+    tool_id: str
+    call_index: int
+    success: bool
+    result: str
+    error: str = None
+    should_exit: bool = False
+    requires_approval: bool = False
+
+
+class ParallelToolExecutor:
+    """Executes multiple tool calls concurrently with proper error handling.
+
+    This class provides thread-safe concurrent execution of tool calls using
+    ThreadPoolExecutor. Key features:
+    - Executes independent tools concurrently for performance
+    - Preserves result order using call_index tracking
+    - Isolates errors (one failure doesn't stop others)
+    - Fast-path optimization for single tool calls (no threading overhead)
+    """
+
+    def __init__(self, max_workers: int = 5):
+        """Initialize executor.
+
+        Args:
+            max_workers: Maximum number of concurrent tool executions
+        """
+        self.max_workers = max_workers
+
+    def execute_tools(
+        self,
+        tool_calls: List[ToolCall],
+        context: dict
+    ) -> Tuple[List[ToolResult], bool]:
+        """Execute multiple tools concurrently.
+
+        Args:
+            tool_calls: List of ToolCall objects
+            context: Dictionary containing repo_root, console, chat_manager, etc.
+
+        Returns:
+            Tuple of (results in call_index order, had_any_errors)
+        """
+        if len(tool_calls) == 1:
+            # Fast path for single tool (no threading overhead)
+            return self._execute_single(tool_calls[0], context)
+
+        # Parallel execution for multiple tools
+        results = []
+
+        with concurrent.futures.ThreadPoolExecutor(
+            max_workers=min(self.max_workers, len(tool_calls))
+        ) as executor:
+            # Submit all tool executions
+            future_to_call = {
+                executor.submit(
+                    self._execute_single_tool,
+                    tool_call,
+                    context
+                ): tool_call
+                for tool_call in tool_calls
+            }
+
+            # Collect results as they complete
+            for future in concurrent.futures.as_completed(future_to_call):
+                tool_call = future_to_call[future]
+                try:
+                    result = future.result()
+                    results.append(result)
+                except Exception as e:
+                    results.append(ToolResult(
+                        tool_id=tool_call.tool_id,
+                        call_index=tool_call.call_index,
+                        success=False,
+                        result="",
+                        error=str(e)
+                    ))
+
+        # Sort by call_index to maintain order
+        results.sort(key=lambda r: r.call_index)
+
+        # Check for errors
+        had_errors = any(not r.success for r in results)
+
+        return results, had_errors
+
+    def _execute_single(
+        self,
+        tool_call: ToolCall,
+        context: dict
+    ) -> Tuple[List[ToolResult], bool]:
+        """Execute single tool (fast path, no threading overhead).
+
+        Args:
+            tool_call: Single ToolCall to execute
+            context: Execution context dict
+
+        Returns:
+            Tuple of (single-element result list, had_errors)
+        """
+        result = self._execute_single_tool(tool_call, context)
+        return [result], not result.success
+
+    def _execute_single_tool(
+        self,
+        tool_call: ToolCall,
+        context: dict
+    ) -> ToolResult:
+        """Execute a single tool call with error handling.
+
+        Args:
+            tool_call: ToolCall to execute
+            context: Execution context dict
+
+        Returns:
+            ToolResult with execution outcome
+        """
+        tool = ToolRegistry.get(tool_call.function_name)
+        if tool:
+            try:
+                # For tools requiring approval, return preview without executing
+                # The orchestrator will handle the approval workflow
+                if tool.requires_approval:
+                    # Build context from context dict
+                    cm = context.get('chat_manager')
+
+                    tool_context = build_context(
+                        repo_root=context.get('repo_root'),
+                        console=context.get('console'),
+                        gitignore_spec=context.get('gitignore_spec'),
+                        debug_mode=context.get('debug_mode', False),
+                        chat_manager=cm,
+                        rg_exe_path=context.get('rg_exe_path'),
+                        panel_updater=context.get('panel_updater'),
+                        vault_root=context.get('vault_root')
+                    )
+
+                    # For edit_file: return preview (orchestrator handles approval)
+                    if tool_call.function_name == "edit_file":
+                        tool_result = tool.execute(tool_call.arguments, tool_context)
+                        return ToolResult(
+                            tool_id=tool_call.tool_id,
+                            call_index=tool_call.call_index,
+                            success=True,
+                            result=tool_result,
+                            should_exit=False,
+                            requires_approval=True  # Flag for orchestrator to handle
+                        )
+                    # Other approval-required tools would go here in the future
+
+                # Normal execution for tools without approval
+                # Build context from context dict
+                cm = context.get('chat_manager')
+
+                tool_context = build_context(
+                    repo_root=context.get('repo_root'),
+                    console=context.get('console'),
+                    gitignore_spec=context.get('gitignore_spec'),
+                    debug_mode=context.get('debug_mode', False),
+                    chat_manager=cm,
+                    rg_exe_path=context.get('rg_exe_path'),
+                    panel_updater=context.get('panel_updater'),
+                    vault_root=context.get('vault_root')
+                )
+
+                tool_result = tool.execute(tool_call.arguments, tool_context)
+                return ToolResult(
+                    tool_id=tool_call.tool_id,
+                    call_index=tool_call.call_index,
+                    success=True,
+                    result=tool_result,
+                    should_exit=False
+                )
+            except Exception as e:
+                return ToolResult(
+                    tool_id=tool_call.tool_id,
+                    call_index=tool_call.call_index,
+                    success=False,
+                    result="",
+                    error=f"Error executing tool '{tool_call.function_name}': {str(e)}"
+                )
+
+        # Tool not found in registry
+        return ToolResult(
+            tool_id=tool_call.tool_id,
+            call_index=tool_call.call_index,
+            success=False,
+            result="",
+            error=f"Unknown tool '{tool_call.function_name}'"
+        )

package/src/tools/helpers/path_resolver.py

@@ -0,0 +1,226 @@
+"""Path resolution and validation utilities.
+
+This module provides centralized path validation and resolution logic,
+ensuring consistent behavior across all tools that work with file paths.
+"""
+
+import os
+import time
+from pathlib import Path
+from typing import Optional, Tuple
+
+from .file_helpers import _is_reserved_windows_name
+
+# Performance metrics tracking
+_path_resolution_times = []
+_path_validation_errors = {}
+
+
+class PathResolver:
+    """Centralized path resolution and validation.
+
+    This class provides a single source of truth for path validation logic,
+    including Windows-specific checks, path resolution, and gitignore filtering.
+
+    Example:
+        resolver = PathResolver(repo_root=Path("/project"), gitignore_spec=spec)
+        resolved_path, error = resolver.resolve_and_validate("src/file.py")
+        if error:
+            return f"Error: {error}"
+        # Use resolved_path...
+    """
+
+    def __init__(
+        self,
+        repo_root: Path,
+        gitignore_spec = None,
+        vault_path: Path = None
+    ):
+        """Initialize the path resolver.
+
+        Args:
+            repo_root: Repository root directory for resolving relative paths
+            gitignore_spec: Optional PathSpec for .gitignore filtering
+            vault_path: Optional Obsidian vault root (allowed as second base path)
+        """
+        self.repo_root = repo_root
+        self.vault_path = vault_path
+        self.gitignore_spec = gitignore_spec
+
+    def resolve_and_validate(
+        self,
+        path_str: str,
+        check_gitignore: bool = True,
+        must_exist: bool = True,
+        must_be_file: bool = False,
+        must_be_dir: bool = False,
+        enforce_boundary: bool = False,
+    ) -> Tuple[Optional[Path], Optional[str]]:
+        """Validate and resolve a path string.
+
+        Performs comprehensive validation including:
+        - Windows filename validation (invalid chars, reserved names)
+        - Path resolution (absolute vs relative)
+        - Gitignore filtering (optional, within repo only)
+        - Path existence check (optional)
+        - Type validation (file/directory, optional)
+
+        Args:
+            path_str: Path string to validate
+            check_gitignore: Whether to apply gitignore filtering
+            must_exist: Whether the path must exist on disk
+            must_be_file: Whether the path must be a file (requires must_exist=True)
+            must_be_dir: Whether the path must be a directory (requires must_exist=True)
+            enforce_boundary: Whether to restrict paths to repo_root or vault_path (default: False)
+
+        Returns:
+            Tuple of (resolved_path, error_message)
+            - resolved_path: Path object if valid, None if invalid
+            - error_message: None if valid, error description if invalid
+        """
+        start_time = time.time()
+
+        try:
+            # Step 1: Validate filename for Windows-specific issues
+            if os.name == 'nt':  # Windows-specific validation
+                # Check for invalid characters
+                invalid_chars = '<>:"|?*[]{}"\n\r\t'
+                if any(char in path_str for char in invalid_chars):
+                    elapsed = time.time() - start_time
+                    _track_validation_error("invalid_chars")
+                    _path_resolution_times.append(elapsed)
+                    return None, f"Filename contains invalid characters: {invalid_chars}"
+
+                # Check for reserved device names
+                filename = Path(path_str).name
+                if _is_reserved_windows_name(filename):
+                    elapsed = time.time() - start_time
+                    _track_validation_error("reserved_name")
+                    _path_resolution_times.append(elapsed)
+                    return None, f"Filename is a reserved Windows device name: {filename}"
+
+            # Step 2: Resolve the path
+            path = Path(path_str)
+            if not path.is_absolute():
+                path = self.repo_root / path
+
+            # Resolve to absolute path (handles .. and symlinks)
+            path = path.resolve()
+
+            # Step 2b: Security boundary — path must be within repo_root or vault_path
+            if enforce_boundary:
+                try:
+                    path.relative_to(self.repo_root)
+                except ValueError:
+                    if self.vault_path is not None:
+                        try:
+                            path.relative_to(self.vault_path)
+                        except ValueError:
+                            elapsed = time.time() - start_time
+                            _track_validation_error("outside_allowed_roots")
+                            _path_resolution_times.append(elapsed)
+                            return None, f"Path is outside allowed directories: {path_str}"
+                    else:
+                        elapsed = time.time() - start_time
+                        _track_validation_error("outside_repo")
+                        _path_resolution_times.append(elapsed)
+                        return None, f"Path is outside repository: {path_str}"
+
+            # Step 3: Check existence if required
+            if must_exist:
+                if not path.exists():
+                    elapsed = time.time() - start_time
+                    _track_validation_error("not_found")
+                    _path_resolution_times.append(elapsed)
+                    return None, f"Path not found: {path_str}"
+
+                # Step 4: Validate type if required
+                if must_be_file and not path.is_file():
+                    elapsed = time.time() - start_time
+                    _track_validation_error("not_a_file")
+                    _path_resolution_times.append(elapsed)
+                    return None, f"Path is not a file: {path_str}"
+
+                if must_be_dir and not path.is_dir():
+                    elapsed = time.time() - start_time
+                    _track_validation_error("not_a_dir")
+                    _path_resolution_times.append(elapsed)
+                    return None, f"Path is not a directory: {path_str}"
+
+            # Step 5: Check gitignore if requested and within repo
+            if check_gitignore and self.gitignore_spec is not None:
+                # Only check gitignore for paths within the repo
+                try:
+                    path.relative_to(self.repo_root)
+                    from .file_helpers import _is_ignored_cached, _register_gitignore_spec
+
+                    # Full gitignore check
+                    spec_key = _register_gitignore_spec(self.gitignore_spec)
+                    if _is_ignored_cached(str(path), str(self.repo_root), spec_key):
+                        elapsed = time.time() - start_time
+                        _track_validation_error("gitignore_filtered")
+                        _path_resolution_times.append(elapsed)
+                        return None, f"Path is excluded by .gitignore: {path_str}"
+                except ValueError:
+                    # Path is outside repo, skip gitignore check
+                    pass
+
+            # Success - track timing
+            elapsed = time.time() - start_time
+            _path_resolution_times.append(elapsed)
+            return path, None
+
+        except OSError as e:
+            elapsed = time.time() - start_time
+            _track_validation_error("os_error")
+            _path_resolution_times.append(elapsed)
+            return None, f"Error accessing path '{path_str}': {e}"
+        except Exception as e:
+            elapsed = time.time() - start_time
+            _track_validation_error("unexpected_error")
+            _path_resolution_times.append(elapsed)
+            return None, f"Unexpected error resolving path '{path_str}': {e}"
+
+
+def _track_validation_error(error_type: str):
+    """Track validation errors for metrics.
+
+    Args:
+        error_type: Type of validation error
+    """
+    _path_validation_errors[error_type] = _path_validation_errors.get(error_type, 0) + 1
+
+
+def get_path_resolver_metrics() -> dict:
+    """Get performance metrics for path resolution operations.
+
+    Returns:
+        Dictionary with metrics:
+        - total_resolutions: Total number of path resolutions
+        - avg_resolution_time: Average resolution time in seconds
+        - max_resolution_time: Maximum resolution time
+        - min_resolution_time: Minimum resolution time
+        - validation_errors: Dict of error types and counts
+    """
+    if not _path_resolution_times:
+        return {
+            "total_resolutions": 0,
+            "avg_resolution_time": 0,
+            "max_resolution_time": 0,
+            "min_resolution_time": 0,
+            "validation_errors": _path_validation_errors.copy()
+        }
+
+    return {
+        "total_resolutions": len(_path_resolution_times),
+        "avg_resolution_time": sum(_path_resolution_times) / len(_path_resolution_times),
+        "max_resolution_time": max(_path_resolution_times),
+        "min_resolution_time": min(_path_resolution_times),
+        "validation_errors": _path_validation_errors.copy()
+    }
+
+
+def clear_path_resolver_metrics():
+    """Clear all accumulated metrics for testing or monitoring reset."""
+    _path_resolution_times.clear()
+    _path_validation_errors.clear()