claude-code-workflow 6.2.1 → 6.2.3

package/codex-lens/src/codexlens/search/enrichment.py

@@ -0,0 +1,150 @@
+# codex-lens/src/codexlens/search/enrichment.py
+"""Relationship enrichment for search results."""
+import sqlite3
+from pathlib import Path
+from typing import List, Dict, Any, Optional
+
+
+class RelationshipEnricher:
+    """Enriches search results with code graph relationships."""
+
+    def __init__(self, index_path: Path):
+        """Initialize with path to index database.
+
+        Args:
+            index_path: Path to _index.db SQLite database
+        """
+        self.index_path = index_path
+        self.db_conn: Optional[sqlite3.Connection] = None
+        self._connect()
+
+    def _connect(self) -> None:
+        """Establish read-only database connection."""
+        if self.index_path.exists():
+            self.db_conn = sqlite3.connect(
+                f"file:{self.index_path}?mode=ro",
+                uri=True,
+                check_same_thread=False
+            )
+            self.db_conn.row_factory = sqlite3.Row
+
+    def enrich(self, results: List[Dict[str, Any]], limit: int = 10) -> List[Dict[str, Any]]:
+        """Add relationship data to search results.
+
+        Args:
+            results: List of search result dictionaries
+            limit: Maximum number of results to enrich
+
+        Returns:
+            Results with relationships field added
+        """
+        if not self.db_conn:
+            return results
+
+        for result in results[:limit]:
+            file_path = result.get('file') or result.get('path')
+            symbol_name = result.get('symbol')
+            result['relationships'] = self._find_relationships(file_path, symbol_name)
+        return results
+
+    def _find_relationships(self, file_path: Optional[str], symbol_name: Optional[str]) -> List[Dict[str, Any]]:
+        """Query relationships for a symbol.
+
+        Args:
+            file_path: Path to file containing the symbol
+            symbol_name: Name of the symbol
+
+        Returns:
+            List of relationship dictionaries with type, direction, target/source, file, line
+        """
+        if not self.db_conn or not symbol_name:
+            return []
+
+        relationships = []
+        cursor = self.db_conn.cursor()
+
+        try:
+            # Find symbol ID(s) by name and optionally file
+            if file_path:
+                cursor.execute(
+                    'SELECT id FROM symbols WHERE name = ? AND file_path = ?',
+                    (symbol_name, file_path)
+                )
+            else:
+                cursor.execute('SELECT id FROM symbols WHERE name = ?', (symbol_name,))
+
+            symbol_ids = [row[0] for row in cursor.fetchall()]
+
+            if not symbol_ids:
+                return []
+
+            # Query outgoing relationships (symbol is source)
+            placeholders = ','.join('?' * len(symbol_ids))
+            cursor.execute(f'''
+                SELECT sr.relationship_type, sr.target_symbol_fqn, sr.file_path, sr.line
+                FROM symbol_relationships sr
+                WHERE sr.source_symbol_id IN ({placeholders})
+            ''', symbol_ids)
+
+            for row in cursor.fetchall():
+                relationships.append({
+                    'type': row[0],
+                    'direction': 'outgoing',
+                    'target': row[1],
+                    'file': row[2],
+                    'line': row[3],
+                })
+
+            # Query incoming relationships (symbol is target)
+            # Match against symbol name or qualified name patterns
+            cursor.execute('''
+                SELECT sr.relationship_type, s.name AS source_name, sr.file_path, sr.line
+                FROM symbol_relationships sr
+                JOIN symbols s ON sr.source_symbol_id = s.id
+                WHERE sr.target_symbol_fqn = ? OR sr.target_symbol_fqn LIKE ?
+            ''', (symbol_name, f'%.{symbol_name}'))
+
+            for row in cursor.fetchall():
+                rel_type = row[0]
+                # Convert to incoming type
+                incoming_type = self._to_incoming_type(rel_type)
+                relationships.append({
+                    'type': incoming_type,
+                    'direction': 'incoming',
+                    'source': row[1],
+                    'file': row[2],
+                    'line': row[3],
+                })
+
+        except sqlite3.Error:
+            return []
+
+        return relationships
+
+    def _to_incoming_type(self, outgoing_type: str) -> str:
+        """Convert outgoing relationship type to incoming type.
+
+        Args:
+            outgoing_type: The outgoing relationship type (e.g., 'calls', 'imports')
+
+        Returns:
+            Corresponding incoming type (e.g., 'called_by', 'imported_by')
+        """
+        type_map = {
+            'calls': 'called_by',
+            'imports': 'imported_by',
+            'extends': 'extended_by',
+        }
+        return type_map.get(outgoing_type, f'{outgoing_type}_by')
+
+    def close(self) -> None:
+        """Close database connection."""
+        if self.db_conn:
+            self.db_conn.close()
+            self.db_conn = None
+
+    def __enter__(self) -> 'RelationshipEnricher':
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb) -> None:
+        self.close()

package/codex-lens/src/codexlens/search/hybrid_search.py

@@ -0,0 +1,313 @@
+"""Hybrid search engine orchestrating parallel exact/fuzzy/vector searches with RRF fusion.
+
+Coordinates multiple search backends in parallel using ThreadPoolExecutor and combines
+results via Reciprocal Rank Fusion (RRF) algorithm.
+"""
+
+from __future__ import annotations
+
+import logging
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from pathlib import Path
+from typing import Dict, List, Optional
+
+from codexlens.entities import SearchResult
+from codexlens.search.ranking import reciprocal_rank_fusion, tag_search_source
+from codexlens.storage.dir_index import DirIndexStore
+
+
+class HybridSearchEngine:
+    """Hybrid search engine with parallel execution and RRF fusion.
+
+    Orchestrates searches across exact FTS, fuzzy FTS, and optional vector backends,
+    executing them in parallel and fusing results via Reciprocal Rank Fusion.
+
+    Attributes:
+        logger: Python logger instance
+        default_weights: Default RRF weights for each source
+    """
+
+    # Default RRF weights (exact: 40%, fuzzy: 30%, vector: 30%)
+    DEFAULT_WEIGHTS = {
+        "exact": 0.4,
+        "fuzzy": 0.3,
+        "vector": 0.3,
+    }
+
+    def __init__(self, weights: Optional[Dict[str, float]] = None):
+        """Initialize hybrid search engine.
+
+        Args:
+            weights: Optional custom RRF weights (default: DEFAULT_WEIGHTS)
+        """
+        self.logger = logging.getLogger(__name__)
+        self.weights = weights or self.DEFAULT_WEIGHTS.copy()
+
+    def search(
+        self,
+        index_path: Path,
+        query: str,
+        limit: int = 20,
+        enable_fuzzy: bool = True,
+        enable_vector: bool = False,
+        pure_vector: bool = False,
+    ) -> List[SearchResult]:
+        """Execute hybrid search with parallel retrieval and RRF fusion.
+
+        Args:
+            index_path: Path to _index.db file
+            query: FTS5 query string (for FTS) or natural language query (for vector)
+            limit: Maximum results to return after fusion
+            enable_fuzzy: Enable fuzzy FTS search (default True)
+            enable_vector: Enable vector search (default False)
+            pure_vector: If True, only use vector search without FTS fallback (default False)
+
+        Returns:
+            List of SearchResult objects sorted by fusion score
+
+        Examples:
+            >>> engine = HybridSearchEngine()
+            >>> # Hybrid search (exact + fuzzy + vector)
+            >>> results = engine.search(Path("project/_index.db"), "authentication",
+            ...                         enable_vector=True)
+            >>> # Pure vector search (semantic only)
+            >>> results = engine.search(Path("project/_index.db"),
+            ...                         "how to authenticate users",
+            ...                         enable_vector=True, pure_vector=True)
+            >>> for r in results[:5]:
+            ...     print(f"{r.path}: {r.score:.3f}")
+        """
+        # Determine which backends to use
+        backends = {}
+
+        if pure_vector:
+            # Pure vector mode: only use vector search, no FTS fallback
+            if enable_vector:
+                backends["vector"] = True
+            else:
+                # Invalid configuration: pure_vector=True but enable_vector=False
+                self.logger.warning(
+                    "pure_vector=True requires enable_vector=True. "
+                    "Falling back to exact search. "
+                    "To use pure vector search, enable vector search mode."
+                )
+                backends["exact"] = True
+        else:
+            # Hybrid mode: always include exact search as baseline
+            backends["exact"] = True
+            if enable_fuzzy:
+                backends["fuzzy"] = True
+            if enable_vector:
+                backends["vector"] = True
+
+        # Execute parallel searches
+        results_map = self._search_parallel(index_path, query, backends, limit)
+
+        # Provide helpful message if pure-vector mode returns no results
+        if pure_vector and enable_vector and len(results_map.get("vector", [])) == 0:
+            self.logger.warning(
+                "Pure vector search returned no results. "
+                "This usually means embeddings haven't been generated. "
+                "Run: codexlens embeddings-generate %s",
+                index_path.parent if index_path.name == "_index.db" else index_path
+            )
+
+        # Apply RRF fusion
+        # Filter weights to only active backends
+        active_weights = {
+            source: weight
+            for source, weight in self.weights.items()
+            if source in results_map
+        }
+
+        fused_results = reciprocal_rank_fusion(results_map, active_weights)
+
+        # Apply final limit
+        return fused_results[:limit]
+
+    def _search_parallel(
+        self,
+        index_path: Path,
+        query: str,
+        backends: Dict[str, bool],
+        limit: int,
+    ) -> Dict[str, List[SearchResult]]:
+        """Execute parallel searches across enabled backends.
+
+        Args:
+            index_path: Path to _index.db file
+            query: FTS5 query string
+            backends: Dictionary of backend name to enabled flag
+            limit: Results limit per backend
+
+        Returns:
+            Dictionary mapping source name to results list
+        """
+        results_map: Dict[str, List[SearchResult]] = {}
+
+        # Use ThreadPoolExecutor for parallel I/O-bound searches
+        with ThreadPoolExecutor(max_workers=len(backends)) as executor:
+            # Submit search tasks
+            future_to_source = {}
+
+            if backends.get("exact"):
+                future = executor.submit(
+                    self._search_exact, index_path, query, limit
+                )
+                future_to_source[future] = "exact"
+
+            if backends.get("fuzzy"):
+                future = executor.submit(
+                    self._search_fuzzy, index_path, query, limit
+                )
+                future_to_source[future] = "fuzzy"
+
+            if backends.get("vector"):
+                future = executor.submit(
+                    self._search_vector, index_path, query, limit
+                )
+                future_to_source[future] = "vector"
+
+            # Collect results as they complete
+            for future in as_completed(future_to_source):
+                source = future_to_source[future]
+                try:
+                    results = future.result()
+                    # Tag results with source for debugging
+                    tagged_results = tag_search_source(results, source)
+                    results_map[source] = tagged_results
+                    self.logger.debug(
+                        "Got %d results from %s search", len(results), source
+                    )
+                except Exception as exc:
+                    self.logger.error("Search failed for %s: %s", source, exc)
+                    results_map[source] = []
+
+        return results_map
+
+    def _search_exact(
+        self, index_path: Path, query: str, limit: int
+    ) -> List[SearchResult]:
+        """Execute exact FTS search using unicode61 tokenizer.
+
+        Args:
+            index_path: Path to _index.db file
+            query: FTS5 query string
+            limit: Maximum results
+
+        Returns:
+            List of SearchResult objects
+        """
+        try:
+            with DirIndexStore(index_path) as store:
+                return store.search_fts_exact(query, limit=limit)
+        except Exception as exc:
+            self.logger.debug("Exact search error: %s", exc)
+            return []
+
+    def _search_fuzzy(
+        self, index_path: Path, query: str, limit: int
+    ) -> List[SearchResult]:
+        """Execute fuzzy FTS search using trigram/extended unicode61 tokenizer.
+
+        Args:
+            index_path: Path to _index.db file
+            query: FTS5 query string
+            limit: Maximum results
+
+        Returns:
+            List of SearchResult objects
+        """
+        try:
+            with DirIndexStore(index_path) as store:
+                return store.search_fts_fuzzy(query, limit=limit)
+        except Exception as exc:
+            self.logger.debug("Fuzzy search error: %s", exc)
+            return []
+
+    def _search_vector(
+        self, index_path: Path, query: str, limit: int
+    ) -> List[SearchResult]:
+        """Execute vector similarity search using semantic embeddings.
+
+        Args:
+            index_path: Path to _index.db file
+            query: Natural language query string
+            limit: Maximum results
+
+        Returns:
+            List of SearchResult objects ordered by semantic similarity
+        """
+        try:
+            # Check if semantic chunks table exists
+            import sqlite3
+            try:
+                with sqlite3.connect(index_path) as conn:
+                    cursor = conn.execute(
+                        "SELECT name FROM sqlite_master WHERE type='table' AND name='semantic_chunks'"
+                    )
+                    has_semantic_table = cursor.fetchone() is not None
+            except sqlite3.Error as e:
+                self.logger.error("Database check failed in vector search: %s", e)
+                return []
+
+            if not has_semantic_table:
+                self.logger.info(
+                    "No embeddings found in index. "
+                    "Generate embeddings with: codexlens embeddings-generate %s",
+                    index_path.parent if index_path.name == "_index.db" else index_path
+                )
+                return []
+
+            # Initialize embedder and vector store
+            from codexlens.semantic.embedder import get_embedder
+            from codexlens.semantic.vector_store import VectorStore
+
+            vector_store = VectorStore(index_path)
+
+            # Check if vector store has data
+            if vector_store.count_chunks() == 0:
+                self.logger.info(
+                    "Vector store is empty (0 chunks). "
+                    "Generate embeddings with: codexlens embeddings-generate %s",
+                    index_path.parent if index_path.name == "_index.db" else index_path
+                )
+                return []
+
+            # Auto-detect embedding dimension and select appropriate profile
+            detected_dim = vector_store.dimension
+            if detected_dim is None:
+                self.logger.info("Vector store dimension unknown, using default profile")
+                profile = "code"  # Default fallback
+            elif detected_dim == 384:
+                profile = "fast"
+            elif detected_dim == 768:
+                profile = "code"
+            elif detected_dim == 1024:
+                profile = "multilingual"  # or balanced, both are 1024
+            else:
+                profile = "code"  # Default fallback
+
+            # Use cached embedder (singleton) for performance
+            embedder = get_embedder(profile=profile)
+
+            # Generate query embedding
+            query_embedding = embedder.embed_single(query)
+
+            # Search for similar chunks
+            results = vector_store.search_similar(
+                query_embedding=query_embedding,
+                top_k=limit,
+                min_score=0.0,  # Return all results, let RRF handle filtering
+                return_full_content=True,
+            )
+
+            self.logger.debug("Vector search found %d results", len(results))
+            return results
+
+        except ImportError as exc:
+            self.logger.debug("Semantic dependencies not available: %s", exc)
+            return []
+        except Exception as exc:
+            self.logger.error("Vector search error: %s", exc)
+            return []

package/codex-lens/src/codexlens/search/query_parser.py

@@ -0,0 +1,242 @@
+"""Query preprocessing for CodexLens search.
+
+Provides query expansion for better identifier matching:
+- CamelCase splitting: UserAuth → User OR Auth
+- snake_case splitting: user_auth → user OR auth
+- Preserves original query for exact matching
+"""
+
+from __future__ import annotations
+
+import logging
+import re
+from typing import Set, List
+
+log = logging.getLogger(__name__)
+
+
+class QueryParser:
+    """Parser for preprocessing search queries before FTS5 execution.
+
+    Expands identifier-style queries (CamelCase, snake_case) into OR queries
+    to improve recall when searching for code symbols.
+
+    Example transformations:
+        - 'UserAuth' → 'UserAuth OR User OR Auth'
+        - 'user_auth' → 'user_auth OR user OR auth'
+        - 'getUserData' → 'getUserData OR get OR User OR Data'
+    """
+
+    # Patterns for identifier splitting
+    CAMEL_CASE_PATTERN = re.compile(r'([a-z])([A-Z])')
+    SNAKE_CASE_PATTERN = re.compile(r'_+')
+    KEBAB_CASE_PATTERN = re.compile(r'-+')
+
+    # Minimum token length to include in expansion (avoid noise from single chars)
+    MIN_TOKEN_LENGTH = 2
+
+    # All-caps acronyms pattern (e.g., HTTP, SQL, API)
+    ALL_CAPS_PATTERN = re.compile(r'^[A-Z]{2,}$')
+
+    def __init__(self, enable: bool = True, min_token_length: int = 2):
+        """Initialize query parser.
+
+        Args:
+            enable: Whether to enable query preprocessing
+            min_token_length: Minimum token length to include in expansion
+        """
+        self.enable = enable
+        self.min_token_length = min_token_length
+
+    def preprocess_query(self, query: str) -> str:
+        """Preprocess query with identifier expansion.
+
+        Args:
+            query: Original search query
+
+        Returns:
+            Expanded query with OR operator connecting original and split tokens
+
+        Example:
+            >>> parser = QueryParser()
+            >>> parser.preprocess_query('UserAuth')
+            'UserAuth OR User OR Auth'
+            >>> parser.preprocess_query('get_user_data')
+            'get_user_data OR get OR user OR data'
+        """
+        if not self.enable:
+            return query
+
+        query = query.strip()
+        if not query:
+            return query
+
+        # Extract tokens from query (handle multiple words/terms)
+        # For simple queries, just process the whole thing
+        # For complex FTS5 queries with operators, preserve structure
+        if self._is_simple_query(query):
+            return self._expand_simple_query(query)
+        else:
+            # Complex query with FTS5 operators, don't expand
+            log.debug(f"Skipping expansion for complex FTS5 query: {query}")
+            return query
+
+    def _is_simple_query(self, query: str) -> bool:
+        """Check if query is simple (no FTS5 operators).
+
+        Args:
+            query: Search query
+
+        Returns:
+            True if query is simple (safe to expand), False otherwise
+        """
+        # Check for FTS5 operators that indicate complex query
+        fts5_operators = ['OR', 'AND', 'NOT', 'NEAR', '*', '^', '"']
+        return not any(op in query for op in fts5_operators)
+
+    def _expand_simple_query(self, query: str) -> str:
+        """Expand a simple query with identifier splitting.
+
+        Args:
+            query: Simple search query
+
+        Returns:
+            Expanded query with OR operators
+        """
+        tokens: Set[str] = set()
+
+        # Always include original query
+        tokens.add(query)
+
+        # Split on whitespace first
+        words = query.split()
+
+        for word in words:
+            # Extract tokens from this word
+            word_tokens = self._extract_tokens(word)
+            tokens.update(word_tokens)
+
+        # Filter out short tokens and duplicates
+        filtered_tokens = [
+            t for t in tokens
+            if len(t) >= self.min_token_length
+        ]
+
+        # Remove duplicates while preserving original query first
+        unique_tokens: List[str] = []
+        seen: Set[str] = set()
+
+        # Always put original query first
+        if query not in seen and len(query) >= self.min_token_length:
+            unique_tokens.append(query)
+            seen.add(query)
+
+        # Add other tokens
+        for token in filtered_tokens:
+            if token not in seen:
+                unique_tokens.append(token)
+                seen.add(token)
+
+        # Join with OR operator (only if we have multiple tokens)
+        if len(unique_tokens) > 1:
+            expanded = ' OR '.join(unique_tokens)
+            log.debug(f"Expanded query: '{query}' → '{expanded}'")
+            return expanded
+        else:
+            return query
+
+    def _extract_tokens(self, word: str) -> Set[str]:
+        """Extract tokens from a single word using various splitting strategies.
+
+        Args:
+            word: Single word/identifier to split
+
+        Returns:
+            Set of extracted tokens
+        """
+        tokens: Set[str] = set()
+
+        # Add original word
+        tokens.add(word)
+
+        # Handle all-caps acronyms (don't split)
+        if self.ALL_CAPS_PATTERN.match(word):
+            return tokens
+
+        # CamelCase splitting
+        camel_tokens = self._split_camel_case(word)
+        tokens.update(camel_tokens)
+
+        # snake_case splitting
+        snake_tokens = self._split_snake_case(word)
+        tokens.update(snake_tokens)
+
+        # kebab-case splitting
+        kebab_tokens = self._split_kebab_case(word)
+        tokens.update(kebab_tokens)
+
+        return tokens
+
+    def _split_camel_case(self, word: str) -> List[str]:
+        """Split CamelCase identifier into tokens.
+
+        Args:
+            word: CamelCase identifier (e.g., 'getUserData')
+
+        Returns:
+            List of tokens (e.g., ['get', 'User', 'Data'])
+        """
+        # Insert space before uppercase letters preceded by lowercase
+        spaced = self.CAMEL_CASE_PATTERN.sub(r'\1 \2', word)
+        # Split on spaces and filter empty
+        return [t for t in spaced.split() if t]
+
+    def _split_snake_case(self, word: str) -> List[str]:
+        """Split snake_case identifier into tokens.
+
+        Args:
+            word: snake_case identifier (e.g., 'get_user_data')
+
+        Returns:
+            List of tokens (e.g., ['get', 'user', 'data'])
+        """
+        # Split on underscores
+        return [t for t in self.SNAKE_CASE_PATTERN.split(word) if t]
+
+    def _split_kebab_case(self, word: str) -> List[str]:
+        """Split kebab-case identifier into tokens.
+
+        Args:
+            word: kebab-case identifier (e.g., 'get-user-data')
+
+        Returns:
+            List of tokens (e.g., ['get', 'user', 'data'])
+        """
+        # Split on hyphens
+        return [t for t in self.KEBAB_CASE_PATTERN.split(word) if t]
+
+
+# Global default parser instance
+_default_parser = QueryParser(enable=True)
+
+
+def preprocess_query(query: str, enable: bool = True) -> str:
+    """Convenience function for query preprocessing.
+
+    Args:
+        query: Original search query
+        enable: Whether to enable preprocessing
+
+    Returns:
+        Preprocessed query with identifier expansion
+    """
+    if not enable:
+        return query
+
+    return _default_parser.preprocess_query(query)
+
+
+__all__ = [
+    "QueryParser",
+    "preprocess_query",
+]