claude-code-workflow 6.2.7 → 6.3.0

package/codex-lens/src/codexlens/cli/model_manager.py

@@ -79,36 +79,38 @@ def get_cache_dir() -> Path:
     """Get fastembed cache directory.
 
     Returns:
-        Path to cache directory (usually ~/.cache/fastembed or %LOCALAPPDATA%\\Temp\\fastembed_cache)
+        Path to cache directory (~/.cache/huggingface or custom path)
     """
     # Check HF_HOME environment variable first
     if "HF_HOME" in os.environ:
         return Path(os.environ["HF_HOME"])
 
-    # Default cache locations
-    if os.name == "nt":  # Windows
-        cache_dir = Path(os.environ.get("LOCALAPPDATA", Path.home() / "AppData" / "Local")) / "Temp" / "fastembed_cache"
-    else:  # Unix-like
-        cache_dir = Path.home() / ".cache" / "fastembed"
-
-    return cache_dir
+    # fastembed 0.7.4+ uses HuggingFace cache when cache_dir is specified
+    # Models are stored directly under the cache directory
+    return Path.home() / ".cache" / "huggingface"
 
 
 def _get_model_cache_path(cache_dir: Path, info: Dict) -> Path:
     """Get the actual cache path for a model.
 
-    fastembed uses ONNX versions of models with different names than the original.
-    This function returns the correct path based on the cache_name field.
+    fastembed 0.7.4+ uses HuggingFace Hub's naming convention:
+    - Prefix: 'models--'
+    - Replace '/' with '--' in model name
+    Example: jinaai/jina-embeddings-v2-base-code
+             -> models--jinaai--jina-embeddings-v2-base-code
 
     Args:
-        cache_dir: The fastembed cache directory
+        cache_dir: The fastembed cache directory (HuggingFace hub path)
         info: Model profile info dictionary
 
     Returns:
         Path to the model cache directory
     """
-    cache_name = info.get("cache_name", info["model_name"])
-    return cache_dir / f"models--{cache_name.replace('/', '--')}"
+    # HuggingFace Hub naming: models--{org}--{model}
+    # Use cache_name if available (for mapped ONNX models), else model_name
+    target_name = info.get("cache_name", info["model_name"])
+    sanitized_name = f"models--{target_name.replace('/', '--')}"
+    return cache_dir / sanitized_name
 
 
 def list_models() -> Dict[str, any]:
@@ -194,18 +196,29 @@ def download_model(profile: str, progress_callback: Optional[callable] = None) -
     model_name = info["model_name"]
 
     try:
-        # Download model by instantiating TextEmbedding
-        # This will automatically download to cache if not present
+        # Get cache directory
+        cache_dir = get_cache_dir()
+
+        # Download model by instantiating TextEmbedding with explicit cache_dir
+        # This ensures fastembed uses the correct HuggingFace Hub cache location
         if progress_callback:
             progress_callback(f"Downloading {model_name}...")
 
-        embedder = TextEmbedding(model_name=model_name)
+        # CRITICAL: Must specify cache_dir to use HuggingFace cache
+        # and call embed() to trigger actual download
+        embedder = TextEmbedding(model_name=model_name, cache_dir=str(cache_dir))
+
+        # Trigger actual download by calling embed
+        # TextEmbedding.__init__ alone doesn't download files
+        if progress_callback:
+            progress_callback(f"Initializing {model_name}...")
+
+        list(embedder.embed(["test"]))  # Trigger download
 
         if progress_callback:
             progress_callback(f"Model {model_name} downloaded successfully")
 
-        # Get cache info using correct cache_name
-        cache_dir = get_cache_dir()
+        # Get cache info using correct HuggingFace Hub path
         model_cache_path = _get_model_cache_path(cache_dir, info)
 
         cache_size = 0

package/codex-lens/src/codexlens/cli/output.py

@@ -35,12 +35,23 @@ def _to_jsonable(value: Any) -> Any:
     return value
 
 
-def print_json(*, success: bool, result: Any = None, error: str | None = None) -> None:
+def print_json(*, success: bool, result: Any = None, error: str | None = None, **kwargs: Any) -> None:
+    """Print JSON output with optional additional fields.
+
+    Args:
+        success: Whether the operation succeeded
+        result: Result data (used when success=True)
+        error: Error message (used when success=False)
+        **kwargs: Additional fields to include in the payload (e.g., code, details)
+    """
     payload: dict[str, Any] = {"success": success}
     if success:
         payload["result"] = _to_jsonable(result)
     else:
         payload["error"] = error or "Unknown error"
+        # Include additional error details if provided
+        for key, value in kwargs.items():
+            payload[key] = _to_jsonable(value)
     console.print_json(json.dumps(payload, ensure_ascii=False))
 
 

package/codex-lens/src/codexlens/config.py

@@ -2,6 +2,7 @@
 
 from __future__ import annotations
 
+import json
 import os
 from dataclasses import dataclass, field
 from functools import cached_property
@@ -14,6 +15,9 @@ from .errors import ConfigError
 # Workspace-local directory name
 WORKSPACE_DIR_NAME = ".codexlens"
 
+# Settings file name
+SETTINGS_FILE_NAME = "settings.json"
+
 
 def _default_global_dir() -> Path:
     """Get global CodexLens data directory."""
@@ -89,6 +93,19 @@ class Config:
     # Hybrid chunker configuration
     hybrid_max_chunk_size: int = 2000  # Max characters per chunk before LLM refinement
     hybrid_llm_refinement: bool = False  # Enable LLM-based semantic boundary refinement
+
+    # Embedding configuration
+    embedding_backend: str = "fastembed"  # "fastembed" (local) or "litellm" (API)
+    embedding_model: str = "code"  # For fastembed: profile (fast/code/multilingual/balanced)
+                                   # For litellm: model name from config (e.g., "qwen3-embedding")
+    embedding_use_gpu: bool = True  # For fastembed: whether to use GPU acceleration
+
+    # Multi-endpoint configuration for litellm backend
+    embedding_endpoints: List[Dict[str, Any]] = field(default_factory=list)
+    # List of endpoint configs: [{"model": "...", "api_key": "...", "api_base": "...", "weight": 1.0}]
+    embedding_strategy: str = "latency_aware"  # round_robin, latency_aware, weighted_random
+    embedding_cooldown: float = 60.0  # Default cooldown seconds for rate-limited endpoints
+
     def __post_init__(self) -> None:
         try:
             self.data_dir = self.data_dir.expanduser().resolve()
@@ -133,6 +150,82 @@ class Config:
         """Get parsing rules for a specific language, falling back to defaults."""
         return {**self.parsing_rules.get("default", {}), **self.parsing_rules.get(language_id, {})}
 
+    @cached_property
+    def settings_path(self) -> Path:
+        """Path to the settings file."""
+        return self.data_dir / SETTINGS_FILE_NAME
+
+    def save_settings(self) -> None:
+        """Save embedding and other settings to file."""
+        embedding_config = {
+            "backend": self.embedding_backend,
+            "model": self.embedding_model,
+            "use_gpu": self.embedding_use_gpu,
+        }
+        # Include multi-endpoint config if present
+        if self.embedding_endpoints:
+            embedding_config["endpoints"] = self.embedding_endpoints
+            embedding_config["strategy"] = self.embedding_strategy
+            embedding_config["cooldown"] = self.embedding_cooldown
+
+        settings = {
+            "embedding": embedding_config,
+            "llm": {
+                "enabled": self.llm_enabled,
+                "tool": self.llm_tool,
+                "timeout_ms": self.llm_timeout_ms,
+                "batch_size": self.llm_batch_size,
+            },
+        }
+        with open(self.settings_path, "w", encoding="utf-8") as f:
+            json.dump(settings, f, indent=2)
+
+    def load_settings(self) -> None:
+        """Load settings from file if exists."""
+        if not self.settings_path.exists():
+            return
+
+        try:
+            with open(self.settings_path, "r", encoding="utf-8") as f:
+                settings = json.load(f)
+
+            # Load embedding settings
+            embedding = settings.get("embedding", {})
+            if "backend" in embedding:
+                self.embedding_backend = embedding["backend"]
+            if "model" in embedding:
+                self.embedding_model = embedding["model"]
+            if "use_gpu" in embedding:
+                self.embedding_use_gpu = embedding["use_gpu"]
+
+            # Load multi-endpoint configuration
+            if "endpoints" in embedding:
+                self.embedding_endpoints = embedding["endpoints"]
+            if "strategy" in embedding:
+                self.embedding_strategy = embedding["strategy"]
+            if "cooldown" in embedding:
+                self.embedding_cooldown = embedding["cooldown"]
+
+            # Load LLM settings
+            llm = settings.get("llm", {})
+            if "enabled" in llm:
+                self.llm_enabled = llm["enabled"]
+            if "tool" in llm:
+                self.llm_tool = llm["tool"]
+            if "timeout_ms" in llm:
+                self.llm_timeout_ms = llm["timeout_ms"]
+            if "batch_size" in llm:
+                self.llm_batch_size = llm["batch_size"]
+        except Exception:
+            pass  # Silently ignore errors
+
+    @classmethod
+    def load(cls) -> "Config":
+        """Load config with settings from file."""
+        config = cls()
+        config.load_settings()
+        return config
+
 
 @dataclass
 class WorkspaceConfig:

package/codex-lens/src/codexlens/search/chain_search.py

@@ -494,9 +494,13 @@ class ChainSearchEngine:
                     else:
                         # Use fuzzy FTS if enable_fuzzy=True (mode="fuzzy"), otherwise exact FTS
                         if enable_fuzzy:
-                            fts_results = store.search_fts_fuzzy(query, limit=limit)
+                            fts_results = store.search_fts_fuzzy(
+                                query, limit=limit, return_full_content=True
+                            )
                         else:
-                            fts_results = store.search_fts(query, limit=limit)
+                            fts_results = store.search_fts_exact(
+                                query, limit=limit, return_full_content=True
+                            )
 
                     # Optionally add semantic keyword results
                     if include_semantic:

package/codex-lens/src/codexlens/search/hybrid_search.py

@@ -27,11 +27,11 @@ class HybridSearchEngine:
         default_weights: Default RRF weights for each source
     """
 
-    # Default RRF weights (exact: 40%, fuzzy: 30%, vector: 30%)
+    # Default RRF weights (vector: 60%, exact: 30%, fuzzy: 10%)
     DEFAULT_WEIGHTS = {
-        "exact": 0.4,
-        "fuzzy": 0.3,
-        "vector": 0.3,
+        "exact": 0.3,
+        "fuzzy": 0.1,
+        "vector": 0.6,
     }
 
     def __init__(self, weights: Optional[Dict[str, float]] = None):
@@ -200,7 +200,9 @@ class HybridSearchEngine:
         """
         try:
             with DirIndexStore(index_path) as store:
-                return store.search_fts_exact(query, limit=limit)
+                return store.search_fts_exact(
+                    query, limit=limit, return_full_content=True
+                )
         except Exception as exc:
             self.logger.debug("Exact search error: %s", exc)
             return []
@@ -220,7 +222,9 @@ class HybridSearchEngine:
         """
         try:
             with DirIndexStore(index_path) as store:
-                return store.search_fts_fuzzy(query, limit=limit)
+                return store.search_fts_fuzzy(
+                    query, limit=limit, return_full_content=True
+                )
         except Exception as exc:
             self.logger.debug("Fuzzy search error: %s", exc)
             return []
@@ -260,7 +264,7 @@ class HybridSearchEngine:
                 return []
 
             # Initialize embedder and vector store
-            from codexlens.semantic.embedder import get_embedder
+            from codexlens.semantic.factory import get_embedder
             from codexlens.semantic.vector_store import VectorStore
 
             vector_store = VectorStore(index_path)
@@ -277,32 +281,51 @@ class HybridSearchEngine:
             # Get stored model configuration (preferred) or auto-detect from dimension
             model_config = vector_store.get_model_config()
             if model_config:
-                profile = model_config["model_profile"]
+                backend = model_config.get("backend", "fastembed")
+                model_name = model_config["model_name"]
+                model_profile = model_config["model_profile"]
                 self.logger.debug(
-                    "Using stored model config: %s (%s, %dd)",
-                    profile, model_config["model_name"], model_config["embedding_dim"]
+                    "Using stored model config: %s backend, %s (%s, %dd)",
+                    backend, model_profile, model_name, model_config["embedding_dim"]
                 )
+                
+                # Get embedder based on backend
+                if backend == "litellm":
+                    embedder = get_embedder(backend="litellm", model=model_name)
+                else:
+                    embedder = get_embedder(backend="fastembed", profile=model_profile)
             else:
                 # Fallback: auto-detect from embedding dimension
                 detected_dim = vector_store.dimension
                 if detected_dim is None:
                     self.logger.info("Vector store dimension unknown, using default profile")
-                    profile = "code"  # Default fallback
+                    embedder = get_embedder(backend="fastembed", profile="code")
                 elif detected_dim == 384:
-                    profile = "fast"
+                    embedder = get_embedder(backend="fastembed", profile="fast")
                 elif detected_dim == 768:
-                    profile = "code"
+                    embedder = get_embedder(backend="fastembed", profile="code")
                 elif detected_dim == 1024:
-                    profile = "multilingual"  # or balanced, both are 1024
+                    embedder = get_embedder(backend="fastembed", profile="multilingual")
+                elif detected_dim == 1536:
+                    # Likely OpenAI text-embedding-3-small or ada-002
+                    self.logger.info(
+                        "Detected 1536-dim embeddings (likely OpenAI), using litellm backend with text-embedding-3-small"
+                    )
+                    embedder = get_embedder(backend="litellm", model="text-embedding-3-small")
+                elif detected_dim == 3072:
+                    # Likely OpenAI text-embedding-3-large
+                    self.logger.info(
+                        "Detected 3072-dim embeddings (likely OpenAI), using litellm backend with text-embedding-3-large"
+                    )
+                    embedder = get_embedder(backend="litellm", model="text-embedding-3-large")
                 else:
-                    profile = "code"  # Default fallback
-                self.logger.debug(
-                    "No stored model config, auto-detected profile '%s' from dimension %s",
-                    profile, detected_dim
-                )
+                    self.logger.debug(
+                        "Unknown dimension %s, using default fastembed profile 'code'",
+                        detected_dim
+                    )
+                    embedder = get_embedder(backend="fastembed", profile="code")
+
 
-            # Use cached embedder (singleton) for performance
-            embedder = get_embedder(profile=profile)
 
             # Generate query embedding
             query_embedding = embedder.embed_single(query)

package/codex-lens/src/codexlens/search/ranking.py

@@ -25,7 +25,7 @@ def reciprocal_rank_fusion(
         results_map: Dictionary mapping source name to list of SearchResult objects
                      Sources: 'exact', 'fuzzy', 'vector'
         weights: Dictionary mapping source name to weight (default: equal weights)
-                 Example: {'exact': 0.4, 'fuzzy': 0.3, 'vector': 0.3}
+                 Example: {'exact': 0.3, 'fuzzy': 0.1, 'vector': 0.6}
         k: Constant to avoid division by zero and control rank influence (default 60)
 
     Returns:

package/codex-lens/src/codexlens/semantic/__init__.py

@@ -14,6 +14,7 @@ from __future__ import annotations
 SEMANTIC_AVAILABLE = False
 SEMANTIC_BACKEND: str | None = None
 GPU_AVAILABLE = False
+LITELLM_AVAILABLE = False
 _import_error: str | None = None
 
 
@@ -67,10 +68,51 @@ def check_gpu_available() -> tuple[bool, str]:
         return False, "GPU support module not available"
 
 
+# Export embedder components
+# BaseEmbedder is always available (abstract base class)
+from .base import BaseEmbedder
+
+# Factory function for creating embedders
+from .factory import get_embedder as get_embedder_factory
+
+# Optional: LiteLLMEmbedderWrapper (only if ccw-litellm is installed)
+try:
+    import ccw_litellm  # noqa: F401
+    from .litellm_embedder import LiteLLMEmbedderWrapper
+    LITELLM_AVAILABLE = True
+except ImportError:
+    LiteLLMEmbedderWrapper = None
+    LITELLM_AVAILABLE = False
+
+
+def is_embedding_backend_available(backend: str) -> tuple[bool, str | None]:
+    """Check whether a specific embedding backend can be used.
+
+    Notes:
+    - "fastembed" requires the optional semantic deps (pip install codexlens[semantic]).
+    - "litellm" requires ccw-litellm to be installed in the same environment.
+    """
+    backend = (backend or "").strip().lower()
+    if backend == "fastembed":
+        if SEMANTIC_AVAILABLE:
+            return True, None
+        return False, _import_error or "fastembed not available. Install with: pip install codexlens[semantic]"
+    if backend == "litellm":
+        if LITELLM_AVAILABLE:
+            return True, None
+        return False, "ccw-litellm not available. Install with: pip install ccw-litellm"
+    return False, f"Invalid embedding backend: {backend}. Must be 'fastembed' or 'litellm'."
+
+
 __all__ = [
     "SEMANTIC_AVAILABLE",
     "SEMANTIC_BACKEND",
     "GPU_AVAILABLE",
+    "LITELLM_AVAILABLE",
     "check_semantic_available",
+    "is_embedding_backend_available",
     "check_gpu_available",
+    "BaseEmbedder",
+    "get_embedder_factory",
+    "LiteLLMEmbedderWrapper",
 ]

package/codex-lens/src/codexlens/semantic/base.py

@@ -0,0 +1,61 @@
+"""Base class for embedders.
+
+Defines the interface that all embedders must implement.
+"""
+
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+from typing import Iterable
+
+import numpy as np
+
+
+class BaseEmbedder(ABC):
+    """Base class for all embedders.
+
+    All embedder implementations must inherit from this class and implement
+    the abstract methods to ensure a consistent interface.
+    """
+
+    @property
+    @abstractmethod
+    def embedding_dim(self) -> int:
+        """Return embedding dimensions.
+
+        Returns:
+            int: Dimension of the embedding vectors.
+        """
+        ...
+
+    @property
+    @abstractmethod
+    def model_name(self) -> str:
+        """Return model name.
+
+        Returns:
+            str: Name or identifier of the underlying model.
+        """
+        ...
+
+    @property
+    def max_tokens(self) -> int:
+        """Return maximum token limit for embeddings.
+
+        Returns:
+            int: Maximum number of tokens that can be embedded at once.
+                Default is 8192 if not overridden by implementation.
+        """
+        return 8192
+
+    @abstractmethod
+    def embed_to_numpy(self, texts: str | Iterable[str]) -> np.ndarray:
+        """Embed texts to numpy array.
+
+        Args:
+            texts: Single text or iterable of texts to embed.
+
+        Returns:
+            numpy.ndarray: Array of shape (n_texts, embedding_dim) containing embeddings.
+        """
+        ...

package/codex-lens/src/codexlens/semantic/chunker.py

@@ -39,7 +39,7 @@ from codexlens.parsers.tokenizer import get_default_tokenizer
 class ChunkConfig:
     """Configuration for chunking strategies."""
     max_chunk_size: int = 1000  # Max characters per chunk
-    overlap: int = 100  # Overlap for sliding window
+    overlap: int = 200  # Overlap for sliding window (increased from 100 for better context)
     strategy: str = "auto"  # Chunking strategy: auto, symbol, sliding_window, hybrid
     min_chunk_size: int = 50  # Minimum chunk size
     skip_token_count: bool = False  # Skip expensive token counting (use char/4 estimate)
@@ -80,6 +80,7 @@ class Chunker:
         """Chunk code by extracted symbols (functions, classes).
 
         Each symbol becomes one chunk with its full content.
+        Large symbols exceeding max_chunk_size are recursively split using sliding window.
 
         Args:
             content: Source code content
@@ -101,27 +102,49 @@ class Chunker:
             if len(chunk_content.strip()) < self.config.min_chunk_size:
                 continue
 
-            # Calculate token count if not provided
-            token_count = None
-            if symbol_token_counts and symbol.name in symbol_token_counts:
-                token_count = symbol_token_counts[symbol.name]
+            # Check if symbol content exceeds max_chunk_size
+            if len(chunk_content) > self.config.max_chunk_size:
+                # Create line mapping for correct line number tracking
+                line_mapping = list(range(start_line, end_line + 1))
+
+                # Use sliding window to split large symbol
+                sub_chunks = self.chunk_sliding_window(
+                    chunk_content,
+                    file_path=file_path,
+                    language=language,
+                    line_mapping=line_mapping
+                )
+
+                # Update sub_chunks with parent symbol metadata
+                for sub_chunk in sub_chunks:
+                    sub_chunk.metadata["symbol_name"] = symbol.name
+                    sub_chunk.metadata["symbol_kind"] = symbol.kind
+                    sub_chunk.metadata["strategy"] = "symbol_split"
+                    sub_chunk.metadata["parent_symbol_range"] = (start_line, end_line)
+
+                chunks.extend(sub_chunks)
             else:
-                token_count = self._estimate_token_count(chunk_content)
+                # Calculate token count if not provided
+                token_count = None
+                if symbol_token_counts and symbol.name in symbol_token_counts:
+                    token_count = symbol_token_counts[symbol.name]
+                else:
+                    token_count = self._estimate_token_count(chunk_content)
 
-            chunks.append(SemanticChunk(
-                content=chunk_content,
-                embedding=None,
-                metadata={
-                    "file": str(file_path),
-                    "language": language,
-                    "symbol_name": symbol.name,
-                    "symbol_kind": symbol.kind,
-                    "start_line": start_line,
-                    "end_line": end_line,
-                    "strategy": "symbol",
-                    "token_count": token_count,
-                }
-            ))
+                chunks.append(SemanticChunk(
+                    content=chunk_content,
+                    embedding=None,
+                    metadata={
+                        "file": str(file_path),
+                        "language": language,
+                        "symbol_name": symbol.name,
+                        "symbol_kind": symbol.kind,
+                        "start_line": start_line,
+                        "end_line": end_line,
+                        "strategy": "symbol",
+                        "token_count": token_count,
+                    }
+                ))
 
         return chunks