claude-code-workflow 6.2.5 → 6.2.6

package/codex-lens/src/codexlens/semantic/code_extractor.py

@@ -0,0 +1,274 @@
+"""Smart code extraction for complete code blocks."""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import List, Optional, Tuple
+
+from codexlens.entities import SearchResult, Symbol
+
+
+def extract_complete_code_block(
+    result: SearchResult,
+    source_file_path: Optional[str] = None,
+    context_lines: int = 0,
+) -> str:
+    """Extract complete code block from a search result.
+    
+    Args:
+        result: SearchResult from semantic search.
+        source_file_path: Optional path to source file for re-reading.
+        context_lines: Additional lines of context to include above/below.
+    
+    Returns:
+        Complete code block as string.
+    """
+    # If we have full content stored, use it
+    if result.content:
+        if context_lines == 0:
+            return result.content
+        # Need to add context, read from file
+        
+    # Try to read from source file
+    file_path = source_file_path or result.path
+    if not file_path or not Path(file_path).exists():
+        # Fall back to excerpt
+        return result.excerpt or ""
+    
+    try:
+        content = Path(file_path).read_text(encoding="utf-8", errors="ignore")
+        lines = content.splitlines()
+        
+        # Get line range
+        start_line = result.start_line or 1
+        end_line = result.end_line or len(lines)
+        
+        # Add context
+        start_idx = max(0, start_line - 1 - context_lines)
+        end_idx = min(len(lines), end_line + context_lines)
+        
+        return "\n".join(lines[start_idx:end_idx])
+    except Exception:
+        return result.excerpt or result.content or ""
+
+
+def extract_symbol_with_context(
+    file_path: str,
+    symbol: Symbol,
+    include_docstring: bool = True,
+    include_decorators: bool = True,
+) -> str:
+    """Extract a symbol (function/class) with its docstring and decorators.
+    
+    Args:
+        file_path: Path to source file.
+        symbol: Symbol to extract.
+        include_docstring: Include docstring if present.
+        include_decorators: Include decorators/annotations above symbol.
+    
+    Returns:
+        Complete symbol code with context.
+    """
+    try:
+        content = Path(file_path).read_text(encoding="utf-8", errors="ignore")
+        lines = content.splitlines()
+        
+        start_line, end_line = symbol.range
+        start_idx = start_line - 1
+        end_idx = end_line
+        
+        # Look for decorators above the symbol
+        if include_decorators and start_idx > 0:
+            decorator_start = start_idx
+            # Search backwards for decorators
+            i = start_idx - 1
+            while i >= 0 and i >= start_idx - 20:  # Look up to 20 lines back
+                line = lines[i].strip()
+                if line.startswith("@"):
+                    decorator_start = i
+                    i -= 1
+                elif line == "" or line.startswith("#"):
+                    # Skip empty lines and comments, continue looking
+                    i -= 1
+                elif line.startswith("//") or line.startswith("/*") or line.startswith("*"):
+                    # JavaScript/Java style comments
+                    decorator_start = i
+                    i -= 1
+                else:
+                    # Found non-decorator, non-comment line, stop
+                    break
+            start_idx = decorator_start
+        
+        return "\n".join(lines[start_idx:end_idx])
+    except Exception:
+        return ""
+
+
+def format_search_result_code(
+    result: SearchResult,
+    max_lines: Optional[int] = None,
+    show_line_numbers: bool = True,
+    highlight_match: bool = False,
+) -> str:
+    """Format search result code for display.
+    
+    Args:
+        result: SearchResult to format.
+        max_lines: Maximum lines to show (None for all).
+        show_line_numbers: Include line numbers in output.
+        highlight_match: Add markers for matched region.
+    
+    Returns:
+        Formatted code string.
+    """
+    content = result.content or result.excerpt or ""
+    if not content:
+        return ""
+    
+    lines = content.splitlines()
+    
+    # Truncate if needed
+    truncated = False
+    if max_lines and len(lines) > max_lines:
+        lines = lines[:max_lines]
+        truncated = True
+    
+    # Format with line numbers
+    if show_line_numbers:
+        start = result.start_line or 1
+        formatted_lines = []
+        for i, line in enumerate(lines):
+            line_num = start + i
+            formatted_lines.append(f"{line_num:4d} | {line}")
+        output = "\n".join(formatted_lines)
+    else:
+        output = "\n".join(lines)
+    
+    if truncated:
+        output += "\n... (truncated)"
+    
+    return output
+
+
+def get_code_block_summary(result: SearchResult) -> str:
+    """Get a concise summary of a code block.
+    
+    Args:
+        result: SearchResult to summarize.
+    
+    Returns:
+        Summary string like "function hello_world (lines 10-25)"
+    """
+    parts = []
+    
+    if result.symbol_kind:
+        parts.append(result.symbol_kind)
+    
+    if result.symbol_name:
+        parts.append(f"`{result.symbol_name}`")
+    elif result.excerpt:
+        # Extract first meaningful identifier
+        first_line = result.excerpt.split("\n")[0][:50]
+        parts.append(f'"{first_line}..."')
+    
+    if result.start_line and result.end_line:
+        if result.start_line == result.end_line:
+            parts.append(f"(line {result.start_line})")
+        else:
+            parts.append(f"(lines {result.start_line}-{result.end_line})")
+    
+    if result.path:
+        file_name = Path(result.path).name
+        parts.append(f"in {file_name}")
+    
+    return " ".join(parts) if parts else "unknown code block"
+
+
+class CodeBlockResult:
+    """Enhanced search result with complete code block."""
+    
+    def __init__(self, result: SearchResult, source_path: Optional[str] = None):
+        self.result = result
+        self.source_path = source_path or result.path
+        self._full_code: Optional[str] = None
+    
+    @property
+    def score(self) -> float:
+        return self.result.score
+    
+    @property
+    def path(self) -> str:
+        return self.result.path
+    
+    @property
+    def file_name(self) -> str:
+        return Path(self.result.path).name
+    
+    @property
+    def symbol_name(self) -> Optional[str]:
+        return self.result.symbol_name
+    
+    @property
+    def symbol_kind(self) -> Optional[str]:
+        return self.result.symbol_kind
+    
+    @property
+    def line_range(self) -> Tuple[int, int]:
+        return (
+            self.result.start_line or 1,
+            self.result.end_line or 1
+        )
+    
+    @property
+    def full_code(self) -> str:
+        """Get full code block content."""
+        if self._full_code is None:
+            self._full_code = extract_complete_code_block(self.result, self.source_path)
+        return self._full_code
+    
+    @property
+    def excerpt(self) -> str:
+        """Get short excerpt."""
+        return self.result.excerpt or ""
+    
+    @property
+    def summary(self) -> str:
+        """Get code block summary."""
+        return get_code_block_summary(self.result)
+    
+    def format(
+        self,
+        max_lines: Optional[int] = None,
+        show_line_numbers: bool = True,
+    ) -> str:
+        """Format code for display."""
+        # Use full code if available
+        display_result = SearchResult(
+            path=self.result.path,
+            score=self.result.score,
+            content=self.full_code,
+            start_line=self.result.start_line,
+            end_line=self.result.end_line,
+        )
+        return format_search_result_code(
+            display_result,
+            max_lines=max_lines,
+            show_line_numbers=show_line_numbers
+        )
+    
+    def __repr__(self) -> str:
+        return f"<CodeBlockResult {self.summary} score={self.score:.3f}>"
+
+
+def enhance_search_results(
+    results: List[SearchResult],
+) -> List[CodeBlockResult]:
+    """Enhance search results with complete code block access.
+    
+    Args:
+        results: List of SearchResult from semantic search.
+    
+    Returns:
+        List of CodeBlockResult with full code access.
+    """
+    return [CodeBlockResult(r) for r in results]

package/codex-lens/src/codexlens/semantic/embedder.py

@@ -0,0 +1,244 @@
+"""Embedder for semantic code search using fastembed.
+
+Supports GPU acceleration via ONNX execution providers (CUDA, TensorRT, DirectML, ROCm, CoreML).
+GPU acceleration is automatic when available, with transparent CPU fallback.
+"""
+
+from __future__ import annotations
+
+import gc
+import logging
+import threading
+from typing import Dict, Iterable, List, Optional
+
+import numpy as np
+
+from . import SEMANTIC_AVAILABLE
+from .gpu_support import get_optimal_providers, is_gpu_available, get_gpu_summary
+
+logger = logging.getLogger(__name__)
+
+# Global embedder cache for singleton pattern
+_embedder_cache: Dict[str, "Embedder"] = {}
+_cache_lock = threading.Lock()
+
+
+def get_embedder(profile: str = "code", use_gpu: bool = True) -> "Embedder":
+    """Get or create a cached Embedder instance (thread-safe singleton).
+
+    This function provides significant performance improvement by reusing
+    Embedder instances across multiple searches, avoiding repeated model
+    loading overhead (~0.8s per load).
+
+    Args:
+        profile: Model profile ("fast", "code", "multilingual", "balanced")
+        use_gpu: If True, use GPU acceleration when available (default: True)
+
+    Returns:
+        Cached Embedder instance for the given profile
+    """
+    global _embedder_cache
+
+    # Cache key includes GPU preference to support mixed configurations
+    cache_key = f"{profile}:{'gpu' if use_gpu else 'cpu'}"
+
+    # Fast path: check cache without lock
+    if cache_key in _embedder_cache:
+        return _embedder_cache[cache_key]
+
+    # Slow path: acquire lock for initialization
+    with _cache_lock:
+        # Double-check after acquiring lock
+        if cache_key in _embedder_cache:
+            return _embedder_cache[cache_key]
+
+        # Create new embedder and cache it
+        embedder = Embedder(profile=profile, use_gpu=use_gpu)
+        # Pre-load model to ensure it's ready
+        embedder._load_model()
+        _embedder_cache[cache_key] = embedder
+
+        # Log GPU status on first embedder creation
+        if use_gpu and is_gpu_available():
+            logger.info(f"Embedder initialized with GPU: {get_gpu_summary()}")
+        elif use_gpu:
+            logger.debug("GPU not available, using CPU for embeddings")
+
+        return embedder
+
+
+def clear_embedder_cache() -> None:
+    """Clear the embedder cache and release ONNX resources.
+
+    This method ensures proper cleanup of ONNX model resources to prevent
+    memory leaks when embedders are no longer needed.
+    """
+    global _embedder_cache
+    with _cache_lock:
+        # Release ONNX resources before clearing cache
+        for embedder in _embedder_cache.values():
+            if embedder._model is not None:
+                del embedder._model
+                embedder._model = None
+        _embedder_cache.clear()
+        gc.collect()
+
+
+class Embedder:
+    """Generate embeddings for code chunks using fastembed (ONNX-based).
+
+    Supported Model Profiles:
+    - fast: BAAI/bge-small-en-v1.5 (384 dim) - Fast, lightweight, English-optimized
+    - code: jinaai/jina-embeddings-v2-base-code (768 dim) - Code-optimized, best for programming languages
+    - multilingual: intfloat/multilingual-e5-large (1024 dim) - Multilingual + code support
+    - balanced: mixedbread-ai/mxbai-embed-large-v1 (1024 dim) - High accuracy, general purpose
+    """
+
+    # Model profiles for different use cases
+    MODELS = {
+        "fast": "BAAI/bge-small-en-v1.5",           # 384 dim - Fast, lightweight
+        "code": "jinaai/jina-embeddings-v2-base-code",  # 768 dim - Code-optimized
+        "multilingual": "intfloat/multilingual-e5-large",  # 1024 dim - Multilingual
+        "balanced": "mixedbread-ai/mxbai-embed-large-v1",  # 1024 dim - High accuracy
+    }
+
+    # Dimension mapping for each model
+    MODEL_DIMS = {
+        "BAAI/bge-small-en-v1.5": 384,
+        "jinaai/jina-embeddings-v2-base-code": 768,
+        "intfloat/multilingual-e5-large": 1024,
+        "mixedbread-ai/mxbai-embed-large-v1": 1024,
+    }
+
+    # Default model (fast profile)
+    DEFAULT_MODEL = "BAAI/bge-small-en-v1.5"
+    DEFAULT_PROFILE = "fast"
+
+    def __init__(
+        self,
+        model_name: str | None = None,
+        profile: str | None = None,
+        use_gpu: bool = True,
+        providers: List[str] | None = None,
+    ) -> None:
+        """Initialize embedder with model or profile.
+
+        Args:
+            model_name: Explicit model name (e.g., "jinaai/jina-embeddings-v2-base-code")
+            profile: Model profile shortcut ("fast", "code", "multilingual", "balanced")
+                    If both provided, model_name takes precedence.
+            use_gpu: If True, use GPU acceleration when available (default: True)
+            providers: Explicit ONNX providers list (overrides use_gpu if provided)
+        """
+        if not SEMANTIC_AVAILABLE:
+            raise ImportError(
+                "Semantic search dependencies not available. "
+                "Install with: pip install codexlens[semantic]"
+            )
+
+        # Resolve model name from profile or use explicit name
+        if model_name:
+            self.model_name = model_name
+        elif profile and profile in self.MODELS:
+            self.model_name = self.MODELS[profile]
+        else:
+            self.model_name = self.DEFAULT_MODEL
+
+        # Configure ONNX execution providers
+        if providers is not None:
+            self._providers = providers
+        else:
+            self._providers = get_optimal_providers(use_gpu=use_gpu)
+
+        self._use_gpu = use_gpu
+        self._model = None
+
+    @property
+    def embedding_dim(self) -> int:
+        """Get embedding dimension for current model."""
+        return self.MODEL_DIMS.get(self.model_name, 768)  # Default to 768 if unknown
+
+    @property
+    def providers(self) -> List[str]:
+        """Get configured ONNX execution providers."""
+        return self._providers
+
+    @property
+    def is_gpu_enabled(self) -> bool:
+        """Check if GPU acceleration is enabled for this embedder."""
+        gpu_providers = {"CUDAExecutionProvider", "TensorrtExecutionProvider",
+                        "DmlExecutionProvider", "ROCMExecutionProvider", "CoreMLExecutionProvider"}
+        return any(p in gpu_providers for p in self._providers)
+
+    def _load_model(self) -> None:
+        """Lazy load the embedding model with configured providers."""
+        if self._model is not None:
+            return
+
+        from fastembed import TextEmbedding
+
+        # fastembed supports 'providers' parameter for ONNX execution providers
+        try:
+            self._model = TextEmbedding(
+                model_name=self.model_name,
+                providers=self._providers,
+            )
+            logger.debug(f"Model loaded with providers: {self._providers}")
+        except TypeError:
+            # Fallback for older fastembed versions without providers parameter
+            logger.warning(
+                "fastembed version doesn't support 'providers' parameter. "
+                "Upgrade fastembed for GPU acceleration: pip install --upgrade fastembed"
+            )
+            self._model = TextEmbedding(model_name=self.model_name)
+
+    def embed(self, texts: str | Iterable[str]) -> List[List[float]]:
+        """Generate embeddings for one or more texts.
+
+        Args:
+            texts: Single text or iterable of texts to embed.
+
+        Returns:
+            List of embedding vectors (each is a list of floats).
+
+        Note:
+            This method converts numpy arrays to Python lists for backward compatibility.
+            For memory-efficient processing, use embed_to_numpy() instead.
+        """
+        self._load_model()
+
+        if isinstance(texts, str):
+            texts = [texts]
+        else:
+            texts = list(texts)
+
+        embeddings = list(self._model.embed(texts))
+        return [emb.tolist() for emb in embeddings]
+
+    def embed_to_numpy(self, texts: str | Iterable[str]) -> np.ndarray:
+        """Generate embeddings for one or more texts (returns numpy arrays).
+
+        This method is more memory-efficient than embed() as it avoids converting
+        numpy arrays to Python lists, which can significantly reduce memory usage
+        during batch processing.
+
+        Args:
+            texts: Single text or iterable of texts to embed.
+
+        Returns:
+            numpy.ndarray of shape (n_texts, embedding_dim) containing embeddings.
+        """
+        self._load_model()
+
+        if isinstance(texts, str):
+            texts = [texts]
+        else:
+            texts = list(texts)
+
+        # Return embeddings as numpy array directly (no .tolist() conversion)
+        embeddings = list(self._model.embed(texts))
+        return np.array(embeddings)
+
+    def embed_single(self, text: str) -> List[float]:
+        """Generate embedding for a single text."""
+        return self.embed(text)[0]

package/codex-lens/src/codexlens/semantic/gpu_support.py

@@ -0,0 +1,192 @@
+"""GPU acceleration support for semantic embeddings.
+
+This module provides GPU detection, initialization, and fallback handling
+for ONNX-based embedding generation.
+"""
+
+from __future__ import annotations
+
+import logging
+from dataclasses import dataclass
+from typing import List, Optional
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass
+class GPUInfo:
+    """GPU availability and configuration info."""
+
+    gpu_available: bool = False
+    cuda_available: bool = False
+    gpu_count: int = 0
+    gpu_name: Optional[str] = None
+    onnx_providers: List[str] = None
+
+    def __post_init__(self):
+        if self.onnx_providers is None:
+            self.onnx_providers = ["CPUExecutionProvider"]
+
+
+_gpu_info_cache: Optional[GPUInfo] = None
+
+
+def detect_gpu(force_refresh: bool = False) -> GPUInfo:
+    """Detect available GPU resources for embedding acceleration.
+
+    Args:
+        force_refresh: If True, re-detect GPU even if cached.
+
+    Returns:
+        GPUInfo with detection results.
+    """
+    global _gpu_info_cache
+
+    if _gpu_info_cache is not None and not force_refresh:
+        return _gpu_info_cache
+
+    info = GPUInfo()
+
+    # Check PyTorch CUDA availability (most reliable detection)
+    try:
+        import torch
+        if torch.cuda.is_available():
+            info.cuda_available = True
+            info.gpu_available = True
+            info.gpu_count = torch.cuda.device_count()
+            if info.gpu_count > 0:
+                info.gpu_name = torch.cuda.get_device_name(0)
+            logger.debug(f"PyTorch CUDA detected: {info.gpu_count} GPU(s)")
+    except ImportError:
+        logger.debug("PyTorch not available for GPU detection")
+
+    # Check ONNX Runtime providers with validation
+    try:
+        import onnxruntime as ort
+        available_providers = ort.get_available_providers()
+
+        # Build provider list with priority order
+        providers = []
+
+        # Test each provider to ensure it actually works
+        def test_provider(provider_name: str) -> bool:
+            """Test if a provider actually works by creating a dummy session."""
+            try:
+                # Create a minimal ONNX model to test provider
+                import numpy as np
+                # Simple test: just check if provider can be instantiated
+                sess_options = ort.SessionOptions()
+                sess_options.log_severity_level = 4  # Suppress warnings
+                return True
+            except Exception:
+                return False
+
+        # CUDA provider (NVIDIA GPU) - check if CUDA runtime is available
+        if "CUDAExecutionProvider" in available_providers:
+            # Verify CUDA is actually usable by checking for cuBLAS
+            cuda_works = False
+            try:
+                import ctypes
+                # Try to load cuBLAS to verify CUDA installation
+                try:
+                    ctypes.CDLL("cublas64_12.dll")
+                    cuda_works = True
+                except OSError:
+                    try:
+                        ctypes.CDLL("cublas64_11.dll")
+                        cuda_works = True
+                    except OSError:
+                        pass
+            except Exception:
+                pass
+
+            if cuda_works:
+                providers.append("CUDAExecutionProvider")
+                info.gpu_available = True
+                logger.debug("ONNX CUDAExecutionProvider available and working")
+            else:
+                logger.debug("ONNX CUDAExecutionProvider listed but CUDA runtime not found")
+
+        # TensorRT provider (optimized NVIDIA inference)
+        if "TensorrtExecutionProvider" in available_providers:
+            # TensorRT requires additional libraries, skip for now
+            logger.debug("ONNX TensorrtExecutionProvider available (requires TensorRT SDK)")
+
+        # DirectML provider (Windows GPU - AMD/Intel/NVIDIA)
+        if "DmlExecutionProvider" in available_providers:
+            providers.append("DmlExecutionProvider")
+            info.gpu_available = True
+            logger.debug("ONNX DmlExecutionProvider available (DirectML)")
+
+        # ROCm provider (AMD GPU on Linux)
+        if "ROCMExecutionProvider" in available_providers:
+            providers.append("ROCMExecutionProvider")
+            info.gpu_available = True
+            logger.debug("ONNX ROCMExecutionProvider available (AMD)")
+
+        # CoreML provider (Apple Silicon)
+        if "CoreMLExecutionProvider" in available_providers:
+            providers.append("CoreMLExecutionProvider")
+            info.gpu_available = True
+            logger.debug("ONNX CoreMLExecutionProvider available (Apple)")
+
+        # Always include CPU as fallback
+        providers.append("CPUExecutionProvider")
+
+        info.onnx_providers = providers
+
+    except ImportError:
+        logger.debug("ONNX Runtime not available")
+        info.onnx_providers = ["CPUExecutionProvider"]
+
+    _gpu_info_cache = info
+    return info
+
+
+def get_optimal_providers(use_gpu: bool = True) -> List[str]:
+    """Get optimal ONNX execution providers based on availability.
+
+    Args:
+        use_gpu: If True, include GPU providers when available.
+                 If False, force CPU-only execution.
+
+    Returns:
+        List of provider names in priority order.
+    """
+    if not use_gpu:
+        return ["CPUExecutionProvider"]
+
+    gpu_info = detect_gpu()
+    return gpu_info.onnx_providers
+
+
+def is_gpu_available() -> bool:
+    """Check if any GPU acceleration is available."""
+    return detect_gpu().gpu_available
+
+
+def get_gpu_summary() -> str:
+    """Get human-readable GPU status summary."""
+    info = detect_gpu()
+
+    if not info.gpu_available:
+        return "GPU: Not available (using CPU)"
+
+    parts = []
+    if info.gpu_name:
+        parts.append(f"GPU: {info.gpu_name}")
+    if info.gpu_count > 1:
+        parts.append(f"({info.gpu_count} devices)")
+
+    # Show active providers (excluding CPU fallback)
+    gpu_providers = [p for p in info.onnx_providers if p != "CPUExecutionProvider"]
+    if gpu_providers:
+        parts.append(f"Providers: {', '.join(gpu_providers)}")
+
+    return " | ".join(parts) if parts else "GPU: Available"
+
+
+def clear_gpu_cache() -> None:
+    """Clear cached GPU detection info."""
+    global _gpu_info_cache
+    _gpu_info_cache = None