claude-code-workflow 6.3.53 → 7.0.0

package/codex-lens/src/codexlens/parsers/treesitter_parser.py

@@ -11,7 +11,7 @@ return `None`; callers should use a regex-based fallback such as
 from __future__ import annotations
 
 from pathlib import Path
-from typing import Dict, List, Optional
+from typing import Dict, List, Optional, TYPE_CHECKING
 
 try:
     from tree_sitter import Language as TreeSitterLanguage
@@ -27,26 +27,46 @@ except ImportError:
 from codexlens.entities import CodeRelationship, IndexedFile, RelationshipType, Symbol
 from codexlens.parsers.tokenizer import get_default_tokenizer
 
+if TYPE_CHECKING:
+    from codexlens.config import Config
+
 
 class TreeSitterSymbolParser:
-    """Parser using tree-sitter for AST-level symbol extraction."""
+    """Parser using tree-sitter for AST-level symbol extraction.
+
+    Supports optional ast-grep integration for relationship extraction
+    (Python/JavaScript/TypeScript) when config.use_astgrep is True and
+    ast-grep-py is available.
+    """
 
-    def __init__(self, language_id: str, path: Optional[Path] = None) -> None:
+    def __init__(
+        self,
+        language_id: str,
+        path: Optional[Path] = None,
+        config: Optional["Config"] = None,
+    ) -> None:
         """Initialize tree-sitter parser for a language.
 
         Args:
             language_id: Language identifier (python, javascript, typescript, etc.)
             path: Optional file path for language variant detection (e.g., .tsx)
+            config: Optional Config instance for parser feature toggles
         """
         self.language_id = language_id
         self.path = path
+        self._config = config
         self._parser: Optional[object] = None
         self._language: Optional[TreeSitterLanguage] = None
         self._tokenizer = get_default_tokenizer()
+        self._astgrep_processor = None
 
         if TREE_SITTER_AVAILABLE:
             self._initialize_parser()
 
+        # Initialize ast-grep processor for Python if config enables it
+        if self._should_use_astgrep():
+            self._initialize_astgrep_processor()
+
     def _initialize_parser(self) -> None:
         """Initialize tree-sitter parser and language."""
         if TreeSitterParser is None or TreeSitterLanguage is None:
@@ -82,6 +102,40 @@ class TreeSitterSymbolParser:
             self._parser = None
             self._language = None
 
+    def _should_use_astgrep(self) -> bool:
+        """Check if ast-grep should be used for relationship extraction.
+
+        Returns:
+            True if config.use_astgrep is True and language is supported
+        """
+        if self._config is None:
+            return False
+        if not getattr(self._config, "use_astgrep", False):
+            return False
+        return self.language_id in {"python", "javascript", "typescript"}
+
+    def _initialize_astgrep_processor(self) -> None:
+        """Initialize ast-grep processor for relationship extraction."""
+        try:
+            from codexlens.parsers.astgrep_processor import (
+                AstGrepPythonProcessor,
+                is_astgrep_processor_available,
+            )
+            from codexlens.parsers.astgrep_js_ts_processor import (
+                AstGrepJavaScriptProcessor,
+                AstGrepTypeScriptProcessor,
+            )
+
+            if is_astgrep_processor_available():
+                if self.language_id == "python":
+                    self._astgrep_processor = AstGrepPythonProcessor(self.path)
+                elif self.language_id == "javascript":
+                    self._astgrep_processor = AstGrepJavaScriptProcessor(self.path)
+                elif self.language_id == "typescript":
+                    self._astgrep_processor = AstGrepTypeScriptProcessor(self.path)
+        except ImportError:
+            self._astgrep_processor = None
+
     def is_available(self) -> bool:
         """Check if tree-sitter parser is available.
 
@@ -138,7 +192,10 @@ class TreeSitterSymbolParser:
         source_bytes, root = parsed
         try:
             symbols = self._extract_symbols(source_bytes, root)
-            relationships = self._extract_relationships(source_bytes, root, path)
+            # Pass source_code for ast-grep integration
+            relationships = self._extract_relationships(
+                source_bytes, root, path, source_code=text
+            )
 
             return IndexedFile(
                 path=str(path.resolve()),
@@ -173,13 +230,67 @@ class TreeSitterSymbolParser:
         source_bytes: bytes,
         root: TreeSitterNode,
         path: Path,
+        source_code: Optional[str] = None,
     ) -> List[CodeRelationship]:
+        """Extract relationships, optionally using ast-grep.
+
+        When config.use_astgrep is True and an ast-grep processor is available,
+        uses ast-grep for relationship extraction. Otherwise, uses tree-sitter.
+
+        Args:
+            source_bytes: Source code as bytes
+            root: Root AST node from tree-sitter
+            path: File path
+            source_code: Optional source code string (required for ast-grep)
+
+        Returns:
+            List of extracted relationships
+        """
+        # Try ast-grep first if configured and available for this language.
+        if self._astgrep_processor is not None and source_code is not None:
+            try:
+                astgrep_rels = self._extract_relationships_astgrep(source_code, path)
+                if astgrep_rels is not None:
+                    return astgrep_rels
+            except Exception:
+                # Fall back to tree-sitter on ast-grep failure
+                pass
+
         if self.language_id == "python":
             return self._extract_python_relationships(source_bytes, root, path)
         if self.language_id in {"javascript", "typescript"}:
             return self._extract_js_ts_relationships(source_bytes, root, path)
         return []
 
+    def _extract_relationships_astgrep(
+        self,
+        source_code: str,
+        path: Path,
+    ) -> Optional[List[CodeRelationship]]:
+        """Extract relationships using ast-grep processor.
+
+        Args:
+            source_code: Source code text
+            path: File path
+
+        Returns:
+            List of relationships, or None if ast-grep unavailable
+        """
+        if self._astgrep_processor is None:
+            return None
+
+        if not self._astgrep_processor.is_available():
+            return None
+
+        try:
+            indexed = self._astgrep_processor.parse(source_code, path)
+            if indexed is not None:
+                return indexed.relationships
+        except Exception:
+            pass
+
+        return None
+
     def _extract_python_relationships(
         self,
         source_bytes: bytes,
@@ -189,7 +300,9 @@ class TreeSitterSymbolParser:
         source_file = str(path.resolve())
         relationships: List[CodeRelationship] = []
 
-        scope_stack: List[str] = []
+        # Use a synthetic module scope so module-level imports/calls can be recorded
+        # (useful for static global graph persistence).
+        scope_stack: List[str] = ["<module>"]
         alias_stack: List[Dict[str, str]] = [{}]
 
         def record_import(target_symbol: str, source_line: int) -> None:
@@ -296,7 +409,9 @@ class TreeSitterSymbolParser:
         source_file = str(path.resolve())
         relationships: List[CodeRelationship] = []
 
-        scope_stack: List[str] = []
+        # Use a synthetic module scope so module-level imports/calls can be recorded
+        # (useful for static global graph persistence).
+        scope_stack: List[str] = ["<module>"]
         alias_stack: List[Dict[str, str]] = [{}]
 
         def record_import(target_symbol: str, source_line: int) -> None:
@@ -489,7 +604,7 @@ class TreeSitterSymbolParser:
         targets: List[str] = []
 
         if node.type == "import_statement":
-            for child in node.children:
+            for i, child in enumerate(node.children):
                 if child.type == "aliased_import":
                     name_node = child.child_by_field_name("name")
                     alias_node = child.child_by_field_name("alias")
@@ -526,7 +641,7 @@ class TreeSitterSymbolParser:
             if module_node is not None:
                 module_name = self._node_text(source_bytes, module_node).strip()
 
-            for child in node.children:
+            for i, child in enumerate(node.children):
                 if child.type == "aliased_import":
                     name_node = child.child_by_field_name("name")
                     alias_node = child.child_by_field_name("alias")
@@ -544,7 +659,18 @@ class TreeSitterSymbolParser:
                     if bound_name:
                         aliases[bound_name] = target
                     targets.append(target)
-                elif child.type == "identifier":
+                elif child.type == "dotted_name" and node.field_name_for_child(i) == "name":
+                    # tree-sitter-python represents `from X import A, B, C` as
+                    # multiple dotted_name nodes (field: "name").
+                    imported_name = self._node_text(source_bytes, child).strip()
+                    if not imported_name:
+                        continue
+                    target = (
+                        f"{module_name}.{imported_name}" if module_name else imported_name
+                    )
+                    aliases[imported_name] = target
+                    targets.append(target)
+                elif child.type == "identifier" and node.field_name_for_child(i) == "name":
                     imported_name = self._node_text(source_bytes, child).strip()
                     if not imported_name or imported_name in {"from", "import", "*"}:
                         continue

package/codex-lens/src/codexlens/search/__init__.py

@@ -5,6 +5,7 @@ from .chain_search import (
     ChainSearchResult,
     quick_search,
 )
+from .global_graph_expander import GlobalGraphExpander
 
 # Clustering availability flag (lazy import pattern)
 CLUSTERING_AVAILABLE = False
@@ -46,6 +47,7 @@ __all__ = [
     "SearchStats",
     "ChainSearchResult",
     "quick_search",
+    "GlobalGraphExpander",
     # Clustering
     "CLUSTERING_AVAILABLE",
     "check_clustering_available",

package/codex-lens/src/codexlens/search/binary_searcher.py

@@ -42,7 +42,7 @@ class BinarySearcher:
     - Database (fallback): Loads all vectors into memory from SQLite
     """
 
-    def __init__(self, index_root_or_meta_path: Path) -> None:
+    def __init__(self, index_root_or_meta_path: Path) -> None:
         """Initialize BinarySearcher.
 
         Args:
@@ -62,12 +62,16 @@ class BinarySearcher:
             self.index_root = path
             self.meta_store_path = path / "_vectors_meta.db"
 
-        self._chunk_ids: Optional[np.ndarray] = None
-        self._binary_matrix: Optional[np.ndarray] = None
-        self._is_memmap = False
-        self._loaded = False
-
-    def load(self) -> bool:
+        self._chunk_ids: Optional[np.ndarray] = None
+        self._binary_matrix: Optional[np.ndarray] = None
+        self._is_memmap = False
+        self._loaded = False
+        self._embedding_dim: Optional[int] = None
+        self._backend: Optional[str] = None
+        self._model: Optional[str] = None
+        self._model_profile: Optional[str] = None
+
+    def load(self) -> bool:
         """Load binary vectors using memory-mapped file or database fallback.
 
         Tries to load from memory-mapped file first (preferred for large indexes),
@@ -83,17 +87,21 @@ class BinarySearcher:
         mmap_path = self.index_root / "_binary_vectors.mmap"
         meta_path = mmap_path.with_suffix('.meta.json')
 
-        if mmap_path.exists() and meta_path.exists():
-            try:
-                with open(meta_path, 'r') as f:
-                    meta = json.load(f)
-
-                shape = tuple(meta['shape'])
-                self._chunk_ids = np.array(meta['chunk_ids'], dtype=np.int64)
-
-                # Memory-map the binary matrix (read-only)
-                self._binary_matrix = np.memmap(
-                    str(mmap_path),
+        if mmap_path.exists() and meta_path.exists():
+            try:
+                with open(meta_path, 'r') as f:
+                    meta = json.load(f)
+
+                shape = tuple(meta['shape'])
+                self._chunk_ids = np.array(meta['chunk_ids'], dtype=np.int64)
+                self._embedding_dim = meta.get("embedding_dim")
+                self._backend = meta.get("backend")
+                self._model = meta.get("model") or meta.get("model_name")
+                self._model_profile = meta.get("model_profile")
+
+                # Memory-map the binary matrix (read-only)
+                self._binary_matrix = np.memmap(
+                    str(mmap_path),
                     dtype=np.uint8,
                     mode='r',
                     shape=shape
@@ -101,11 +109,11 @@ class BinarySearcher:
                 self._is_memmap = True
                 self._loaded = True
 
-                logger.info(
-                    "Memory-mapped %d binary vectors (%d bytes each)",
-                    len(self._chunk_ids), shape[1]
-                )
-                return True
+                logger.info(
+                    "Memory-mapped %d binary vectors (%d bytes each)",
+                    len(self._chunk_ids), shape[1]
+                )
+                return True
 
             except Exception as e:
                 logger.warning("Failed to load mmap binary vectors, falling back to DB: %s", e)
@@ -138,9 +146,13 @@ class BinarySearcher:
                 arr = np.frombuffer(vec_bytes, dtype=np.uint8)
                 binary_arrays.append(arr)
 
-            self._binary_matrix = np.vstack(binary_arrays)
-            self._is_memmap = False
-            self._loaded = True
+            self._binary_matrix = np.vstack(binary_arrays)
+            self._is_memmap = False
+            self._loaded = True
+            self._embedding_dim = None
+            self._backend = None
+            self._model = None
+            self._model_profile = None
 
             logger.info(
                 "Loaded %d binary vectors from DB (%d bytes each)",
@@ -256,15 +268,35 @@ class BinarySearcher:
 
         return results
 
-    @property
-    def vector_count(self) -> int:
-        """Get number of loaded binary vectors."""
-        return len(self._chunk_ids) if self._chunk_ids is not None else 0
-
-    @property
-    def is_memmap(self) -> bool:
-        """Check if using memory-mapped file (vs in-memory array)."""
-        return self._is_memmap
+    @property
+    def vector_count(self) -> int:
+        """Get number of loaded binary vectors."""
+        return len(self._chunk_ids) if self._chunk_ids is not None else 0
+
+    @property
+    def embedding_dim(self) -> Optional[int]:
+        """Embedding dimension used to build these binary vectors (if known)."""
+        return int(self._embedding_dim) if self._embedding_dim is not None else None
+
+    @property
+    def backend(self) -> Optional[str]:
+        """Embedding backend used to build these vectors (if known)."""
+        return self._backend
+
+    @property
+    def model(self) -> Optional[str]:
+        """Embedding model name used to build these vectors (if known)."""
+        return self._model
+
+    @property
+    def model_profile(self) -> Optional[str]:
+        """Embedding profile name (fastembed) used to build these vectors (if known)."""
+        return self._model_profile
+
+    @property
+    def is_memmap(self) -> bool:
+        """Check if using memory-mapped file (vs in-memory array)."""
+        return self._is_memmap
 
     def clear(self) -> None:
         """Clear loaded vectors from memory."""