claude-code-workflow 6.1.4 → 6.2.2

package/codex-lens/src/codexlens/entities.py

@@ -0,0 +1,121 @@
+"""Pydantic entity models for CodexLens."""
+
+from __future__ import annotations
+
+from enum import Enum
+from typing import Any, Dict, List, Optional, Tuple
+
+from pydantic import BaseModel, Field, field_validator
+
+
+class Symbol(BaseModel):
+    """A code symbol discovered in a file."""
+
+    name: str = Field(..., min_length=1)
+    kind: str = Field(..., min_length=1)
+    range: Tuple[int, int] = Field(..., description="(start_line, end_line), 1-based inclusive")
+    file: Optional[str] = Field(default=None, description="Full path to the file containing this symbol")
+
+    @field_validator("range")
+    @classmethod
+    def validate_range(cls, value: Tuple[int, int]) -> Tuple[int, int]:
+        if len(value) != 2:
+            raise ValueError("range must be a (start_line, end_line) tuple")
+        start_line, end_line = value
+        if start_line < 1 or end_line < 1:
+            raise ValueError("range lines must be >= 1")
+        if end_line < start_line:
+            raise ValueError("end_line must be >= start_line")
+        return value
+
+
+class SemanticChunk(BaseModel):
+    """A semantically meaningful chunk of content, optionally embedded."""
+
+    content: str = Field(..., min_length=1)
+    embedding: Optional[List[float]] = Field(default=None, description="Vector embedding for semantic search")
+    metadata: Dict[str, Any] = Field(default_factory=dict)
+
+    @field_validator("embedding")
+    @classmethod
+    def validate_embedding(cls, value: Optional[List[float]]) -> Optional[List[float]]:
+        if value is None:
+            return value
+        if not value:
+            raise ValueError("embedding cannot be empty when provided")
+        return value
+
+
+class IndexedFile(BaseModel):
+    """An indexed source file with symbols and optional semantic chunks."""
+
+    path: str = Field(..., min_length=1)
+    language: str = Field(..., min_length=1)
+    symbols: List[Symbol] = Field(default_factory=list)
+    chunks: List[SemanticChunk] = Field(default_factory=list)
+
+    @field_validator("path", "language")
+    @classmethod
+    def strip_and_validate_nonempty(cls, value: str) -> str:
+        cleaned = value.strip()
+        if not cleaned:
+            raise ValueError("value cannot be blank")
+        return cleaned
+
+
+class RelationshipType(str, Enum):
+    """Types of code relationships."""
+    CALL = "call"
+    INHERITS = "inherits"
+    IMPORTS = "imports"
+
+
+class CodeRelationship(BaseModel):
+    """A relationship between code symbols (e.g., function calls, inheritance)."""
+
+    source_symbol: str = Field(..., min_length=1, description="Name of source symbol")
+    target_symbol: str = Field(..., min_length=1, description="Name of target symbol")
+    relationship_type: RelationshipType = Field(..., description="Type of relationship (call, inherits, etc.)")
+    source_file: str = Field(..., min_length=1, description="File path containing source symbol")
+    target_file: Optional[str] = Field(default=None, description="File path containing target (None if same file)")
+    source_line: int = Field(..., ge=1, description="Line number where relationship occurs (1-based)")
+
+
+class AdditionalLocation(BaseModel):
+    """A pointer to another location where a similar result was found.
+
+    Used for grouping search results with similar scores and content,
+    where the primary result is stored in SearchResult and secondary
+    locations are stored in this model.
+    """
+
+    path: str = Field(..., min_length=1)
+    score: float = Field(..., ge=0.0)
+    start_line: Optional[int] = Field(default=None, description="Start line of the result (1-based)")
+    end_line: Optional[int] = Field(default=None, description="End line of the result (1-based)")
+    symbol_name: Optional[str] = Field(default=None, description="Name of matched symbol")
+
+
+class SearchResult(BaseModel):
+    """A unified search result for lexical or semantic search."""
+
+    path: str = Field(..., min_length=1)
+    score: float = Field(..., ge=0.0)
+    excerpt: Optional[str] = None
+    content: Optional[str] = Field(default=None, description="Full content of matched code block")
+    symbol: Optional[Symbol] = None
+    chunk: Optional[SemanticChunk] = None
+    metadata: Dict[str, Any] = Field(default_factory=dict)
+
+    # Additional context for complete code blocks
+    start_line: Optional[int] = Field(default=None, description="Start line of code block (1-based)")
+    end_line: Optional[int] = Field(default=None, description="End line of code block (1-based)")
+    symbol_name: Optional[str] = Field(default=None, description="Name of matched symbol/function/class")
+    symbol_kind: Optional[str] = Field(default=None, description="Kind of symbol (function/class/method)")
+
+    # Field for grouping similar results
+    additional_locations: List["AdditionalLocation"] = Field(
+        default_factory=list,
+        description="Other locations for grouped results with similar scores and content."
+    )
+

package/codex-lens/src/codexlens/errors.py

@@ -0,0 +1,55 @@
+"""CodexLens exception hierarchy."""
+
+from __future__ import annotations
+
+
+class CodexLensError(Exception):
+    """Base class for all CodexLens errors."""
+
+
+class ConfigError(CodexLensError):
+    """Raised when configuration is invalid or cannot be loaded."""
+
+
+class ParseError(CodexLensError):
+    """Raised when parsing or indexing a file fails."""
+
+
+class StorageError(CodexLensError):
+    """Raised when reading/writing index storage fails.
+
+    Attributes:
+        message: Human-readable error description
+        db_path: Path to the database file (if applicable)
+        operation: The operation that failed (e.g., 'query', 'initialize', 'migrate')
+        details: Additional context for debugging
+    """
+
+    def __init__(
+        self,
+        message: str,
+        db_path: str | None = None,
+        operation: str | None = None,
+        details: dict | None = None
+    ) -> None:
+        super().__init__(message)
+        self.message = message
+        self.db_path = db_path
+        self.operation = operation
+        self.details = details or {}
+
+    def __str__(self) -> str:
+        parts = [self.message]
+        if self.db_path:
+            parts.append(f"[db: {self.db_path}]")
+        if self.operation:
+            parts.append(f"[op: {self.operation}]")
+        if self.details:
+            detail_str = ", ".join(f"{k}={v}" for k, v in self.details.items())
+            parts.append(f"[{detail_str}]")
+        return " ".join(parts)
+
+
+class SearchError(CodexLensError):
+    """Raised when a search operation fails."""
+

package/codex-lens/src/codexlens/indexing/README.md

@@ -0,0 +1,77 @@
+# Symbol Extraction and Indexing
+
+This module provides symbol extraction and relationship tracking for code graph enrichment.
+
+## Overview
+
+The `SymbolExtractor` class extracts code symbols (functions, classes) and their relationships (calls, imports) from source files using regex-based pattern matching.
+
+## Supported Languages
+
+- Python (.py)
+- TypeScript (.ts, .tsx)
+- JavaScript (.js, .jsx)
+
+## Database Schema
+
+### Symbols Table
+Stores code symbols with their location information:
+- `id`: Primary key
+- `qualified_name`: Fully qualified name (e.g., "module.ClassName")
+- `name`: Symbol name
+- `kind`: Symbol type (function, class)
+- `file_path`: Path to source file
+- `start_line`: Starting line number
+- `end_line`: Ending line number
+
+### Symbol Relationships Table
+Stores relationships between symbols:
+- `id`: Primary key
+- `source_symbol_id`: Foreign key to symbols table
+- `target_symbol_fqn`: Fully qualified name of target symbol
+- `relationship_type`: Type of relationship (calls, imports)
+- `file_path`: Path to source file
+- `line`: Line number where relationship occurs
+
+## Usage Example
+
+```python
+from pathlib import Path
+from codexlens.indexing.symbol_extractor import SymbolExtractor
+
+# Initialize extractor
+db_path = Path("./code_index.db")
+extractor = SymbolExtractor(db_path)
+extractor.connect()
+
+# Extract from file
+file_path = Path("src/my_module.py")
+with open(file_path) as f:
+    content = f.read()
+
+symbols, relationships = extractor.extract_from_file(file_path, content)
+
+# Save to database
+name_to_id = extractor.save_symbols(symbols)
+extractor.save_relationships(relationships, name_to_id)
+
+# Clean up
+extractor.close()
+```
+
+## Pattern Matching
+
+The extractor uses regex patterns to identify:
+
+- **Functions**: Function definitions (including async, export keywords)
+- **Classes**: Class definitions (including export keyword)
+- **Imports**: Import/require statements
+- **Calls**: Function/method invocations
+
+## Future Enhancements
+
+- Tree-sitter integration for more accurate parsing
+- Support for additional languages
+- Method and variable extraction
+- Enhanced scope tracking
+- Relationship type expansion (inherits, implements, etc.)

package/codex-lens/src/codexlens/indexing/__init__.py

@@ -0,0 +1,4 @@
+"""Code indexing and symbol extraction."""
+from codexlens.indexing.symbol_extractor import SymbolExtractor
+
+__all__ = ["SymbolExtractor"]

package/codex-lens/src/codexlens/indexing/symbol_extractor.py

@@ -0,0 +1,243 @@
+"""Symbol and relationship extraction from source code."""
+import re
+import sqlite3
+from pathlib import Path
+from typing import Any, Dict, List, Optional, Tuple
+
+
+class SymbolExtractor:
+    """Extract symbols and relationships from source code using regex patterns."""
+
+    # Pattern definitions for different languages
+    PATTERNS = {
+        'python': {
+            'function': r'^(?:async\s+)?def\s+(\w+)\s*\(',
+            'class': r'^class\s+(\w+)\s*[:\(]',
+            'import': r'^(?:from\s+([\w.]+)\s+)?import\s+([\w.,\s]+)',
+            'call': r'(?<![.\w])(\w+)\s*\(',
+        },
+        'typescript': {
+            'function': r'(?:export\s+)?(?:async\s+)?function\s+(\w+)\s*[<\(]',
+            'class': r'(?:export\s+)?class\s+(\w+)',
+            'import': r"import\s+.*\s+from\s+['\"]([^'\"]+)['\"]",
+            'call': r'(?<![.\w])(\w+)\s*[<\(]',
+        },
+        'javascript': {
+            'function': r'(?:export\s+)?(?:async\s+)?function\s+(\w+)\s*\(',
+            'class': r'(?:export\s+)?class\s+(\w+)',
+            'import': r"(?:import|require)\s*\(?['\"]([^'\"]+)['\"]",
+            'call': r'(?<![.\w])(\w+)\s*\(',
+        }
+    }
+
+    LANGUAGE_MAP = {
+        '.py': 'python',
+        '.ts': 'typescript',
+        '.tsx': 'typescript',
+        '.js': 'javascript',
+        '.jsx': 'javascript',
+    }
+
+    def __init__(self, db_path: Path):
+        self.db_path = db_path
+        self.db_conn: Optional[sqlite3.Connection] = None
+
+    def connect(self) -> None:
+        """Connect to database and ensure schema exists."""
+        self.db_conn = sqlite3.connect(str(self.db_path))
+        self._ensure_tables()
+
+    def __enter__(self) -> "SymbolExtractor":
+        """Context manager entry: connect to database."""
+        self.connect()
+        return self
+
+    def __exit__(self, exc_type: Any, exc_val: Any, exc_tb: Any) -> None:
+        """Context manager exit: close database connection."""
+        self.close()
+
+    def _ensure_tables(self) -> None:
+        """Create symbols and relationships tables if they don't exist."""
+        if not self.db_conn:
+            return
+        cursor = self.db_conn.cursor()
+
+        # Create symbols table with qualified_name
+        cursor.execute('''
+            CREATE TABLE IF NOT EXISTS symbols (
+                id INTEGER PRIMARY KEY AUTOINCREMENT,
+                qualified_name TEXT NOT NULL,
+                name TEXT NOT NULL,
+                kind TEXT NOT NULL,
+                file_path TEXT NOT NULL,
+                start_line INTEGER NOT NULL,
+                end_line INTEGER NOT NULL,
+                UNIQUE(file_path, name, start_line)
+            )
+        ''')
+
+        # Create relationships table with target_symbol_fqn
+        cursor.execute('''
+            CREATE TABLE IF NOT EXISTS symbol_relationships (
+                id INTEGER PRIMARY KEY AUTOINCREMENT,
+                source_symbol_id INTEGER NOT NULL,
+                target_symbol_fqn TEXT NOT NULL,
+                relationship_type TEXT NOT NULL,
+                file_path TEXT NOT NULL,
+                line INTEGER,
+                FOREIGN KEY (source_symbol_id) REFERENCES symbols(id) ON DELETE CASCADE
+            )
+        ''')
+
+        # Create performance indexes
+        cursor.execute('CREATE INDEX IF NOT EXISTS idx_symbols_name ON symbols(name)')
+        cursor.execute('CREATE INDEX IF NOT EXISTS idx_symbols_file ON symbols(file_path)')
+        cursor.execute('CREATE INDEX IF NOT EXISTS idx_rel_source ON symbol_relationships(source_symbol_id)')
+        cursor.execute('CREATE INDEX IF NOT EXISTS idx_rel_target ON symbol_relationships(target_symbol_fqn)')
+        cursor.execute('CREATE INDEX IF NOT EXISTS idx_rel_type ON symbol_relationships(relationship_type)')
+
+        self.db_conn.commit()
+
+    def extract_from_file(self, file_path: Path, content: str) -> Tuple[List[Dict], List[Dict]]:
+        """Extract symbols and relationships from file content.
+
+        Args:
+            file_path: Path to the source file
+            content: File content as string
+
+        Returns:
+            Tuple of (symbols, relationships) where:
+            - symbols: List of symbol dicts with qualified_name, name, kind, file_path, start_line, end_line
+            - relationships: List of relationship dicts with source_scope, target, type, file_path, line
+        """
+        ext = file_path.suffix.lower()
+        lang = self.LANGUAGE_MAP.get(ext)
+
+        if not lang or lang not in self.PATTERNS:
+            return [], []
+
+        patterns = self.PATTERNS[lang]
+        symbols = []
+        relationships = []
+        lines = content.split('\n')
+
+        current_scope = None
+
+        for line_num, line in enumerate(lines, 1):
+            # Extract function/class definitions
+            for kind in ['function', 'class']:
+                if kind in patterns:
+                    match = re.search(patterns[kind], line)
+                    if match:
+                        name = match.group(1)
+                        qualified_name = f"{file_path.stem}.{name}"
+                        symbols.append({
+                            'qualified_name': qualified_name,
+                            'name': name,
+                            'kind': kind,
+                            'file_path': str(file_path),
+                            'start_line': line_num,
+                            'end_line': line_num,  # Simplified - would need proper parsing for actual end
+                        })
+                        current_scope = name
+
+            # Extract imports
+            if 'import' in patterns:
+                match = re.search(patterns['import'], line)
+                if match:
+                    import_target = match.group(1) or match.group(2) if match.lastindex >= 2 else match.group(1)
+                    if import_target and current_scope:
+                        relationships.append({
+                            'source_scope': current_scope,
+                            'target': import_target.strip(),
+                            'type': 'imports',
+                            'file_path': str(file_path),
+                            'line': line_num,
+                        })
+
+            # Extract function calls (simplified)
+            if 'call' in patterns and current_scope:
+                for match in re.finditer(patterns['call'], line):
+                    call_name = match.group(1)
+                    # Skip common keywords and the current function
+                    if call_name not in ['if', 'for', 'while', 'return', 'print', 'len', 'str', 'int', 'float', 'list', 'dict', 'set', 'tuple', current_scope]:
+                        relationships.append({
+                            'source_scope': current_scope,
+                            'target': call_name,
+                            'type': 'calls',
+                            'file_path': str(file_path),
+                            'line': line_num,
+                        })
+
+        return symbols, relationships
+
+    def save_symbols(self, symbols: List[Dict]) -> Dict[str, int]:
+        """Save symbols to database and return name->id mapping.
+
+        Args:
+            symbols: List of symbol dicts with qualified_name, name, kind, file_path, start_line, end_line
+
+        Returns:
+            Dictionary mapping symbol name to database id
+        """
+        if not self.db_conn or not symbols:
+            return {}
+
+        cursor = self.db_conn.cursor()
+        name_to_id = {}
+
+        for sym in symbols:
+            try:
+                cursor.execute('''
+                    INSERT OR IGNORE INTO symbols
+                    (qualified_name, name, kind, file_path, start_line, end_line)
+                    VALUES (?, ?, ?, ?, ?, ?)
+                ''', (sym['qualified_name'], sym['name'], sym['kind'],
+                      sym['file_path'], sym['start_line'], sym['end_line']))
+
+                # Get the id
+                cursor.execute('''
+                    SELECT id FROM symbols
+                    WHERE file_path = ? AND name = ? AND start_line = ?
+                ''', (sym['file_path'], sym['name'], sym['start_line']))
+
+                row = cursor.fetchone()
+                if row:
+                    name_to_id[sym['name']] = row[0]
+            except sqlite3.Error:
+                continue
+
+        self.db_conn.commit()
+        return name_to_id
+
+    def save_relationships(self, relationships: List[Dict], name_to_id: Dict[str, int]) -> None:
+        """Save relationships to database.
+
+        Args:
+            relationships: List of relationship dicts with source_scope, target, type, file_path, line
+            name_to_id: Dictionary mapping symbol names to database ids
+        """
+        if not self.db_conn or not relationships:
+            return
+
+        cursor = self.db_conn.cursor()
+
+        for rel in relationships:
+            source_id = name_to_id.get(rel['source_scope'])
+            if source_id:
+                try:
+                    cursor.execute('''
+                        INSERT INTO symbol_relationships
+                        (source_symbol_id, target_symbol_fqn, relationship_type, file_path, line)
+                        VALUES (?, ?, ?, ?, ?)
+                    ''', (source_id, rel['target'], rel['type'], rel['file_path'], rel['line']))
+                except sqlite3.Error:
+                    continue
+
+        self.db_conn.commit()
+
+    def close(self) -> None:
+        """Close database connection."""
+        if self.db_conn:
+            self.db_conn.close()
+            self.db_conn = None

package/codex-lens/src/codexlens/parsers/__init__.py

@@ -0,0 +1,8 @@
+"""Parsers for CodexLens."""
+
+from __future__ import annotations
+
+from .factory import ParserFactory
+
+__all__ = ["ParserFactory"]
+