claude-code-workflow 6.2.5 → 6.2.7

package/codex-lens/src/codexlens/search/enrichment.py

@@ -0,0 +1,150 @@
+# codex-lens/src/codexlens/search/enrichment.py
+"""Relationship enrichment for search results."""
+import sqlite3
+from pathlib import Path
+from typing import List, Dict, Any, Optional
+
+
+class RelationshipEnricher:
+    """Enriches search results with code graph relationships."""
+
+    def __init__(self, index_path: Path):
+        """Initialize with path to index database.
+
+        Args:
+            index_path: Path to _index.db SQLite database
+        """
+        self.index_path = index_path
+        self.db_conn: Optional[sqlite3.Connection] = None
+        self._connect()
+
+    def _connect(self) -> None:
+        """Establish read-only database connection."""
+        if self.index_path.exists():
+            self.db_conn = sqlite3.connect(
+                f"file:{self.index_path}?mode=ro",
+                uri=True,
+                check_same_thread=False
+            )
+            self.db_conn.row_factory = sqlite3.Row
+
+    def enrich(self, results: List[Dict[str, Any]], limit: int = 10) -> List[Dict[str, Any]]:
+        """Add relationship data to search results.
+
+        Args:
+            results: List of search result dictionaries
+            limit: Maximum number of results to enrich
+
+        Returns:
+            Results with relationships field added
+        """
+        if not self.db_conn:
+            return results
+
+        for result in results[:limit]:
+            file_path = result.get('file') or result.get('path')
+            symbol_name = result.get('symbol')
+            result['relationships'] = self._find_relationships(file_path, symbol_name)
+        return results
+
+    def _find_relationships(self, file_path: Optional[str], symbol_name: Optional[str]) -> List[Dict[str, Any]]:
+        """Query relationships for a symbol.
+
+        Args:
+            file_path: Path to file containing the symbol
+            symbol_name: Name of the symbol
+
+        Returns:
+            List of relationship dictionaries with type, direction, target/source, file, line
+        """
+        if not self.db_conn or not symbol_name:
+            return []
+
+        relationships = []
+        cursor = self.db_conn.cursor()
+
+        try:
+            # Find symbol ID(s) by name and optionally file
+            if file_path:
+                cursor.execute(
+                    'SELECT id FROM symbols WHERE name = ? AND file_path = ?',
+                    (symbol_name, file_path)
+                )
+            else:
+                cursor.execute('SELECT id FROM symbols WHERE name = ?', (symbol_name,))
+
+            symbol_ids = [row[0] for row in cursor.fetchall()]
+
+            if not symbol_ids:
+                return []
+
+            # Query outgoing relationships (symbol is source)
+            placeholders = ','.join('?' * len(symbol_ids))
+            cursor.execute(f'''
+                SELECT sr.relationship_type, sr.target_symbol_fqn, sr.file_path, sr.line
+                FROM symbol_relationships sr
+                WHERE sr.source_symbol_id IN ({placeholders})
+            ''', symbol_ids)
+
+            for row in cursor.fetchall():
+                relationships.append({
+                    'type': row[0],
+                    'direction': 'outgoing',
+                    'target': row[1],
+                    'file': row[2],
+                    'line': row[3],
+                })
+
+            # Query incoming relationships (symbol is target)
+            # Match against symbol name or qualified name patterns
+            cursor.execute('''
+                SELECT sr.relationship_type, s.name AS source_name, sr.file_path, sr.line
+                FROM symbol_relationships sr
+                JOIN symbols s ON sr.source_symbol_id = s.id
+                WHERE sr.target_symbol_fqn = ? OR sr.target_symbol_fqn LIKE ?
+            ''', (symbol_name, f'%.{symbol_name}'))
+
+            for row in cursor.fetchall():
+                rel_type = row[0]
+                # Convert to incoming type
+                incoming_type = self._to_incoming_type(rel_type)
+                relationships.append({
+                    'type': incoming_type,
+                    'direction': 'incoming',
+                    'source': row[1],
+                    'file': row[2],
+                    'line': row[3],
+                })
+
+        except sqlite3.Error:
+            return []
+
+        return relationships
+
+    def _to_incoming_type(self, outgoing_type: str) -> str:
+        """Convert outgoing relationship type to incoming type.
+
+        Args:
+            outgoing_type: The outgoing relationship type (e.g., 'calls', 'imports')
+
+        Returns:
+            Corresponding incoming type (e.g., 'called_by', 'imported_by')
+        """
+        type_map = {
+            'calls': 'called_by',
+            'imports': 'imported_by',
+            'extends': 'extended_by',
+        }
+        return type_map.get(outgoing_type, f'{outgoing_type}_by')
+
+    def close(self) -> None:
+        """Close database connection."""
+        if self.db_conn:
+            self.db_conn.close()
+            self.db_conn = None
+
+    def __enter__(self) -> 'RelationshipEnricher':
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb) -> None:
+        self.close()

package/codex-lens/src/codexlens/search/hybrid_search.py

@@ -0,0 +1,326 @@
+"""Hybrid search engine orchestrating parallel exact/fuzzy/vector searches with RRF fusion.
+
+Coordinates multiple search backends in parallel using ThreadPoolExecutor and combines
+results via Reciprocal Rank Fusion (RRF) algorithm.
+"""
+
+from __future__ import annotations
+
+import logging
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from pathlib import Path
+from typing import Dict, List, Optional
+
+from codexlens.entities import SearchResult
+from codexlens.search.ranking import reciprocal_rank_fusion, tag_search_source
+from codexlens.storage.dir_index import DirIndexStore
+
+
+class HybridSearchEngine:
+    """Hybrid search engine with parallel execution and RRF fusion.
+
+    Orchestrates searches across exact FTS, fuzzy FTS, and optional vector backends,
+    executing them in parallel and fusing results via Reciprocal Rank Fusion.
+
+    Attributes:
+        logger: Python logger instance
+        default_weights: Default RRF weights for each source
+    """
+
+    # Default RRF weights (exact: 40%, fuzzy: 30%, vector: 30%)
+    DEFAULT_WEIGHTS = {
+        "exact": 0.4,
+        "fuzzy": 0.3,
+        "vector": 0.3,
+    }
+
+    def __init__(self, weights: Optional[Dict[str, float]] = None):
+        """Initialize hybrid search engine.
+
+        Args:
+            weights: Optional custom RRF weights (default: DEFAULT_WEIGHTS)
+        """
+        self.logger = logging.getLogger(__name__)
+        self.weights = weights or self.DEFAULT_WEIGHTS.copy()
+
+    def search(
+        self,
+        index_path: Path,
+        query: str,
+        limit: int = 20,
+        enable_fuzzy: bool = True,
+        enable_vector: bool = False,
+        pure_vector: bool = False,
+    ) -> List[SearchResult]:
+        """Execute hybrid search with parallel retrieval and RRF fusion.
+
+        Args:
+            index_path: Path to _index.db file
+            query: FTS5 query string (for FTS) or natural language query (for vector)
+            limit: Maximum results to return after fusion
+            enable_fuzzy: Enable fuzzy FTS search (default True)
+            enable_vector: Enable vector search (default False)
+            pure_vector: If True, only use vector search without FTS fallback (default False)
+
+        Returns:
+            List of SearchResult objects sorted by fusion score
+
+        Examples:
+            >>> engine = HybridSearchEngine()
+            >>> # Hybrid search (exact + fuzzy + vector)
+            >>> results = engine.search(Path("project/_index.db"), "authentication",
+            ...                         enable_vector=True)
+            >>> # Pure vector search (semantic only)
+            >>> results = engine.search(Path("project/_index.db"),
+            ...                         "how to authenticate users",
+            ...                         enable_vector=True, pure_vector=True)
+            >>> for r in results[:5]:
+            ...     print(f"{r.path}: {r.score:.3f}")
+        """
+        # Determine which backends to use
+        backends = {}
+
+        if pure_vector:
+            # Pure vector mode: only use vector search, no FTS fallback
+            if enable_vector:
+                backends["vector"] = True
+            else:
+                # Invalid configuration: pure_vector=True but enable_vector=False
+                self.logger.warning(
+                    "pure_vector=True requires enable_vector=True. "
+                    "Falling back to exact search. "
+                    "To use pure vector search, enable vector search mode."
+                )
+                backends["exact"] = True
+        else:
+            # Hybrid mode: always include exact search as baseline
+            backends["exact"] = True
+            if enable_fuzzy:
+                backends["fuzzy"] = True
+            if enable_vector:
+                backends["vector"] = True
+
+        # Execute parallel searches
+        results_map = self._search_parallel(index_path, query, backends, limit)
+
+        # Provide helpful message if pure-vector mode returns no results
+        if pure_vector and enable_vector and len(results_map.get("vector", [])) == 0:
+            self.logger.warning(
+                "Pure vector search returned no results. "
+                "This usually means embeddings haven't been generated. "
+                "Run: codexlens embeddings-generate %s",
+                index_path.parent if index_path.name == "_index.db" else index_path
+            )
+
+        # Apply RRF fusion
+        # Filter weights to only active backends
+        active_weights = {
+            source: weight
+            for source, weight in self.weights.items()
+            if source in results_map
+        }
+
+        fused_results = reciprocal_rank_fusion(results_map, active_weights)
+
+        # Apply final limit
+        return fused_results[:limit]
+
+    def _search_parallel(
+        self,
+        index_path: Path,
+        query: str,
+        backends: Dict[str, bool],
+        limit: int,
+    ) -> Dict[str, List[SearchResult]]:
+        """Execute parallel searches across enabled backends.
+
+        Args:
+            index_path: Path to _index.db file
+            query: FTS5 query string
+            backends: Dictionary of backend name to enabled flag
+            limit: Results limit per backend
+
+        Returns:
+            Dictionary mapping source name to results list
+        """
+        results_map: Dict[str, List[SearchResult]] = {}
+
+        # Use ThreadPoolExecutor for parallel I/O-bound searches
+        with ThreadPoolExecutor(max_workers=len(backends)) as executor:
+            # Submit search tasks
+            future_to_source = {}
+
+            if backends.get("exact"):
+                future = executor.submit(
+                    self._search_exact, index_path, query, limit
+                )
+                future_to_source[future] = "exact"
+
+            if backends.get("fuzzy"):
+                future = executor.submit(
+                    self._search_fuzzy, index_path, query, limit
+                )
+                future_to_source[future] = "fuzzy"
+
+            if backends.get("vector"):
+                future = executor.submit(
+                    self._search_vector, index_path, query, limit
+                )
+                future_to_source[future] = "vector"
+
+            # Collect results as they complete
+            for future in as_completed(future_to_source):
+                source = future_to_source[future]
+                try:
+                    results = future.result()
+                    # Tag results with source for debugging
+                    tagged_results = tag_search_source(results, source)
+                    results_map[source] = tagged_results
+                    self.logger.debug(
+                        "Got %d results from %s search", len(results), source
+                    )
+                except Exception as exc:
+                    self.logger.error("Search failed for %s: %s", source, exc)
+                    results_map[source] = []
+
+        return results_map
+
+    def _search_exact(
+        self, index_path: Path, query: str, limit: int
+    ) -> List[SearchResult]:
+        """Execute exact FTS search using unicode61 tokenizer.
+
+        Args:
+            index_path: Path to _index.db file
+            query: FTS5 query string
+            limit: Maximum results
+
+        Returns:
+            List of SearchResult objects
+        """
+        try:
+            with DirIndexStore(index_path) as store:
+                return store.search_fts_exact(query, limit=limit)
+        except Exception as exc:
+            self.logger.debug("Exact search error: %s", exc)
+            return []
+
+    def _search_fuzzy(
+        self, index_path: Path, query: str, limit: int
+    ) -> List[SearchResult]:
+        """Execute fuzzy FTS search using trigram/extended unicode61 tokenizer.
+
+        Args:
+            index_path: Path to _index.db file
+            query: FTS5 query string
+            limit: Maximum results
+
+        Returns:
+            List of SearchResult objects
+        """
+        try:
+            with DirIndexStore(index_path) as store:
+                return store.search_fts_fuzzy(query, limit=limit)
+        except Exception as exc:
+            self.logger.debug("Fuzzy search error: %s", exc)
+            return []
+
+    def _search_vector(
+        self, index_path: Path, query: str, limit: int
+    ) -> List[SearchResult]:
+        """Execute vector similarity search using semantic embeddings.
+
+        Args:
+            index_path: Path to _index.db file
+            query: Natural language query string
+            limit: Maximum results
+
+        Returns:
+            List of SearchResult objects ordered by semantic similarity
+        """
+        try:
+            # Check if semantic chunks table exists
+            import sqlite3
+            try:
+                with sqlite3.connect(index_path) as conn:
+                    cursor = conn.execute(
+                        "SELECT name FROM sqlite_master WHERE type='table' AND name='semantic_chunks'"
+                    )
+                    has_semantic_table = cursor.fetchone() is not None
+            except sqlite3.Error as e:
+                self.logger.error("Database check failed in vector search: %s", e)
+                return []
+
+            if not has_semantic_table:
+                self.logger.info(
+                    "No embeddings found in index. "
+                    "Generate embeddings with: codexlens embeddings-generate %s",
+                    index_path.parent if index_path.name == "_index.db" else index_path
+                )
+                return []
+
+            # Initialize embedder and vector store
+            from codexlens.semantic.embedder import get_embedder
+            from codexlens.semantic.vector_store import VectorStore
+
+            vector_store = VectorStore(index_path)
+
+            # Check if vector store has data
+            if vector_store.count_chunks() == 0:
+                self.logger.info(
+                    "Vector store is empty (0 chunks). "
+                    "Generate embeddings with: codexlens embeddings-generate %s",
+                    index_path.parent if index_path.name == "_index.db" else index_path
+                )
+                return []
+
+            # Get stored model configuration (preferred) or auto-detect from dimension
+            model_config = vector_store.get_model_config()
+            if model_config:
+                profile = model_config["model_profile"]
+                self.logger.debug(
+                    "Using stored model config: %s (%s, %dd)",
+                    profile, model_config["model_name"], model_config["embedding_dim"]
+                )
+            else:
+                # Fallback: auto-detect from embedding dimension
+                detected_dim = vector_store.dimension
+                if detected_dim is None:
+                    self.logger.info("Vector store dimension unknown, using default profile")
+                    profile = "code"  # Default fallback
+                elif detected_dim == 384:
+                    profile = "fast"
+                elif detected_dim == 768:
+                    profile = "code"
+                elif detected_dim == 1024:
+                    profile = "multilingual"  # or balanced, both are 1024
+                else:
+                    profile = "code"  # Default fallback
+                self.logger.debug(
+                    "No stored model config, auto-detected profile '%s' from dimension %s",
+                    profile, detected_dim
+                )
+
+            # Use cached embedder (singleton) for performance
+            embedder = get_embedder(profile=profile)
+
+            # Generate query embedding
+            query_embedding = embedder.embed_single(query)
+
+            # Search for similar chunks
+            results = vector_store.search_similar(
+                query_embedding=query_embedding,
+                top_k=limit,
+                min_score=0.0,  # Return all results, let RRF handle filtering
+                return_full_content=True,
+            )
+
+            self.logger.debug("Vector search found %d results", len(results))
+            return results
+
+        except ImportError as exc:
+            self.logger.debug("Semantic dependencies not available: %s", exc)
+            return []
+        except Exception as exc:
+            self.logger.error("Vector search error: %s", exc)
+            return []