claude-code-workflow 6.3.2 → 6.3.5

package/codex-lens/src/codexlens/search/hybrid_search.py

@@ -7,12 +7,38 @@ results via Reciprocal Rank Fusion (RRF) algorithm.
 from __future__ import annotations
 
 import logging
+import time
 from concurrent.futures import ThreadPoolExecutor, as_completed
+from contextlib import contextmanager
 from pathlib import Path
-from typing import Dict, List, Optional
+from typing import Any, Dict, List, Optional
 
+
+@contextmanager
+def timer(name: str, logger: logging.Logger, level: int = logging.DEBUG):
+    """Context manager for timing code blocks.
+
+    Args:
+        name: Name of the operation being timed
+        logger: Logger instance to use
+        level: Logging level (default DEBUG)
+    """
+    start = time.perf_counter()
+    try:
+        yield
+    finally:
+        elapsed_ms = (time.perf_counter() - start) * 1000
+        logger.log(level, "[TIMING] %s: %.2fms", name, elapsed_ms)
+
+from codexlens.config import Config
 from codexlens.entities import SearchResult
-from codexlens.search.ranking import reciprocal_rank_fusion, tag_search_source
+from codexlens.search.ranking import (
+    apply_symbol_boost,
+    get_rrf_weights,
+    reciprocal_rank_fusion,
+    rerank_results,
+    tag_search_source,
+)
 from codexlens.storage.dir_index import DirIndexStore
 
 
@@ -34,14 +60,23 @@ class HybridSearchEngine:
         "vector": 0.6,
     }
 
-    def __init__(self, weights: Optional[Dict[str, float]] = None):
+    def __init__(
+        self,
+        weights: Optional[Dict[str, float]] = None,
+        config: Optional[Config] = None,
+        embedder: Any = None,
+    ):
         """Initialize hybrid search engine.
 
         Args:
             weights: Optional custom RRF weights (default: DEFAULT_WEIGHTS)
+            config: Optional runtime config (enables optional reranking features)
+            embedder: Optional embedder instance for embedding-based reranking
         """
         self.logger = logging.getLogger(__name__)
         self.weights = weights or self.DEFAULT_WEIGHTS.copy()
+        self._config = config
+        self.embedder = embedder
 
     def search(
         self,
@@ -101,7 +136,8 @@ class HybridSearchEngine:
                 backends["vector"] = True
 
         # Execute parallel searches
-        results_map = self._search_parallel(index_path, query, backends, limit)
+        with timer("parallel_search_total", self.logger):
+            results_map = self._search_parallel(index_path, query, backends, limit)
 
         # Provide helpful message if pure-vector mode returns no results
         if pure_vector and enable_vector and len(results_map.get("vector", [])) == 0:
@@ -120,11 +156,72 @@ class HybridSearchEngine:
             if source in results_map
         }
 
-        fused_results = reciprocal_rank_fusion(results_map, active_weights)
+        with timer("rrf_fusion", self.logger):
+            adaptive_weights = get_rrf_weights(query, active_weights)
+            fused_results = reciprocal_rank_fusion(results_map, adaptive_weights)
+
+        # Optional: boost results that include explicit symbol matches
+        boost_factor = (
+            self._config.symbol_boost_factor
+            if self._config is not None
+            else 1.5
+        )
+        with timer("symbol_boost", self.logger):
+            fused_results = apply_symbol_boost(
+                fused_results, boost_factor=boost_factor
+            )
+
+        # Optional: embedding-based reranking on top results
+        if self._config is not None and self._config.enable_reranking:
+            with timer("reranking", self.logger):
+                if self.embedder is None:
+                    self.embedder = self._get_reranking_embedder()
+                fused_results = rerank_results(
+                    query,
+                    fused_results[:100],
+                    self.embedder,
+                    top_k=self._config.reranking_top_k,
+                )
 
         # Apply final limit
         return fused_results[:limit]
 
+    def _get_reranking_embedder(self) -> Any:
+        """Create an embedder for reranking based on Config embedding settings."""
+        if self._config is None:
+            return None
+
+        try:
+            from codexlens.semantic.factory import get_embedder
+        except Exception as exc:
+            self.logger.debug("Reranking embedder unavailable: %s", exc)
+            return None
+
+        try:
+            if self._config.embedding_backend == "fastembed":
+                return get_embedder(
+                    backend="fastembed",
+                    profile=self._config.embedding_model,
+                    use_gpu=self._config.embedding_use_gpu,
+                )
+            if self._config.embedding_backend == "litellm":
+                return get_embedder(
+                    backend="litellm",
+                    model=self._config.embedding_model,
+                    endpoints=self._config.embedding_endpoints,
+                    strategy=self._config.embedding_strategy,
+                    cooldown=self._config.embedding_cooldown,
+                )
+        except Exception as exc:
+            self.logger.debug("Failed to initialize reranking embedder: %s", exc)
+            return None
+
+        self.logger.debug(
+            "Unknown embedding backend for reranking: %s",
+            self._config.embedding_backend,
+        )
+        return None
+
     def _search_parallel(
         self,
         index_path: Path,
@@ -144,25 +241,30 @@ class HybridSearchEngine:
             Dictionary mapping source name to results list
         """
         results_map: Dict[str, List[SearchResult]] = {}
+        timing_data: Dict[str, float] = {}
 
         # Use ThreadPoolExecutor for parallel I/O-bound searches
         with ThreadPoolExecutor(max_workers=len(backends)) as executor:
-            # Submit search tasks
+            # Submit search tasks with timing
             future_to_source = {}
+            submit_times = {}
 
             if backends.get("exact"):
+                submit_times["exact"] = time.perf_counter()
                 future = executor.submit(
                     self._search_exact, index_path, query, limit
                 )
                 future_to_source[future] = "exact"
 
             if backends.get("fuzzy"):
+                submit_times["fuzzy"] = time.perf_counter()
                 future = executor.submit(
                     self._search_fuzzy, index_path, query, limit
                 )
                 future_to_source[future] = "fuzzy"
 
             if backends.get("vector"):
+                submit_times["vector"] = time.perf_counter()
                 future = executor.submit(
                     self._search_vector, index_path, query, limit
                 )
@@ -171,18 +273,26 @@ class HybridSearchEngine:
             # Collect results as they complete
             for future in as_completed(future_to_source):
                 source = future_to_source[future]
+                elapsed_ms = (time.perf_counter() - submit_times[source]) * 1000
+                timing_data[source] = elapsed_ms
                 try:
                     results = future.result()
                     # Tag results with source for debugging
                     tagged_results = tag_search_source(results, source)
                     results_map[source] = tagged_results
                     self.logger.debug(
-                        "Got %d results from %s search", len(results), source
+                        "[TIMING] %s_search: %.2fms (%d results)",
+                        source, elapsed_ms, len(results)
                     )
                 except Exception as exc:
                     self.logger.error("Search failed for %s: %s", source, exc)
                     results_map[source] = []
 
+        # Log timing summary
+        if timing_data:
+            timing_str = ", ".join(f"{k}={v:.1f}ms" for k, v in timing_data.items())
+            self.logger.debug("[TIMING] search_backends: {%s}", timing_str)
+
         return results_map
 
     def _search_exact(
@@ -245,6 +355,8 @@ class HybridSearchEngine:
         try:
             # Check if semantic chunks table exists
             import sqlite3
+
+            start_check = time.perf_counter()
             try:
                 with sqlite3.connect(index_path) as conn:
                     cursor = conn.execute(
@@ -254,6 +366,10 @@ class HybridSearchEngine:
             except sqlite3.Error as e:
                 self.logger.error("Database check failed in vector search: %s", e)
                 return []
+            self.logger.debug(
+                "[TIMING] vector_table_check: %.2fms",
+                (time.perf_counter() - start_check) * 1000
+            )
 
             if not has_semantic_table:
                 self.logger.info(
@@ -267,7 +383,12 @@ class HybridSearchEngine:
             from codexlens.semantic.factory import get_embedder
             from codexlens.semantic.vector_store import VectorStore
 
+            start_init = time.perf_counter()
             vector_store = VectorStore(index_path)
+            self.logger.debug(
+                "[TIMING] vector_store_init: %.2fms",
+                (time.perf_counter() - start_init) * 1000
+            )
 
             # Check if vector store has data
             if vector_store.count_chunks() == 0:
@@ -279,6 +400,7 @@ class HybridSearchEngine:
                 return []
 
             # Get stored model configuration (preferred) or auto-detect from dimension
+            start_embedder = time.perf_counter()
             model_config = vector_store.get_model_config()
             if model_config:
                 backend = model_config.get("backend", "fastembed")
@@ -288,7 +410,7 @@ class HybridSearchEngine:
                     "Using stored model config: %s backend, %s (%s, %dd)",
                     backend, model_profile, model_name, model_config["embedding_dim"]
                 )
-                
+
                 # Get embedder based on backend
                 if backend == "litellm":
                     embedder = get_embedder(backend="litellm", model=model_name)
@@ -324,21 +446,32 @@ class HybridSearchEngine:
                         detected_dim
                     )
                     embedder = get_embedder(backend="fastembed", profile="code")
-
-
+            self.logger.debug(
+                "[TIMING] embedder_init: %.2fms",
+                (time.perf_counter() - start_embedder) * 1000
+            )
 
             # Generate query embedding
+            start_embed = time.perf_counter()
             query_embedding = embedder.embed_single(query)
+            self.logger.debug(
+                "[TIMING] query_embedding: %.2fms",
+                (time.perf_counter() - start_embed) * 1000
+            )
 
             # Search for similar chunks
+            start_search = time.perf_counter()
             results = vector_store.search_similar(
                 query_embedding=query_embedding,
                 top_k=limit,
                 min_score=0.0,  # Return all results, let RRF handle filtering
                 return_full_content=True,
             )
+            self.logger.debug(
+                "[TIMING] vector_similarity_search: %.2fms (%d results)",
+                (time.perf_counter() - start_search) * 1000, len(results)
+            )
 
-            self.logger.debug("Vector search found %d results", len(results))
             return results
 
         except ImportError as exc: