claude-code-workflow 6.2.4 → 6.2.5

package/codex-lens/src/codexlens/parsers/encoding.py

@@ -1,202 +0,0 @@
-"""Optional encoding detection module for CodexLens.
-
-Provides automatic encoding detection with graceful fallback to UTF-8.
-Install with: pip install codexlens[encoding]
-"""
-
-from __future__ import annotations
-
-import logging
-from pathlib import Path
-from typing import Tuple, Optional
-
-log = logging.getLogger(__name__)
-
-# Feature flag for encoding detection availability
-ENCODING_DETECTION_AVAILABLE = False
-_import_error: Optional[str] = None
-
-
-def _detect_chardet_backend() -> Tuple[bool, Optional[str]]:
-    """Detect if chardet or charset-normalizer is available."""
-    try:
-        import chardet
-        return True, None
-    except ImportError:
-        pass
-
-    try:
-        from charset_normalizer import from_bytes
-        return True, None
-    except ImportError:
-        pass
-
-    return False, "chardet not available. Install with: pip install codexlens[encoding]"
-
-
-# Initialize on module load
-ENCODING_DETECTION_AVAILABLE, _import_error = _detect_chardet_backend()
-
-
-def check_encoding_available() -> Tuple[bool, Optional[str]]:
-    """Check if encoding detection dependencies are available.
-
-    Returns:
-        Tuple of (available, error_message)
-    """
-    return ENCODING_DETECTION_AVAILABLE, _import_error
-
-
-def detect_encoding(content_bytes: bytes, confidence_threshold: float = 0.7) -> str:
-    """Detect encoding from file content bytes.
-
-    Uses chardet or charset-normalizer with configurable confidence threshold.
-    Falls back to UTF-8 if confidence is too low or detection unavailable.
-
-    Args:
-        content_bytes: Raw file content as bytes
-        confidence_threshold: Minimum confidence (0.0-1.0) to accept detection
-
-    Returns:
-        Detected encoding name (e.g., 'utf-8', 'iso-8859-1', 'gbk')
-        Returns 'utf-8' as fallback if detection fails or confidence too low
-    """
-    if not ENCODING_DETECTION_AVAILABLE:
-        log.debug("Encoding detection not available, using UTF-8 fallback")
-        return "utf-8"
-
-    if not content_bytes:
-        return "utf-8"
-
-    try:
-        # Try chardet first
-        try:
-            import chardet
-            result = chardet.detect(content_bytes)
-            encoding = result.get("encoding")
-            confidence = result.get("confidence", 0.0)
-
-            if encoding and confidence >= confidence_threshold:
-                log.debug(f"Detected encoding: {encoding} (confidence: {confidence:.2f})")
-                # Normalize encoding name: replace underscores with hyphens
-                return encoding.lower().replace('_', '-')
-            else:
-                log.debug(
-                    f"Low confidence encoding detection: {encoding} "
-                    f"(confidence: {confidence:.2f}), using UTF-8 fallback"
-                )
-                return "utf-8"
-        except ImportError:
-            pass
-
-        # Fallback to charset-normalizer
-        try:
-            from charset_normalizer import from_bytes
-            results = from_bytes(content_bytes)
-            if results:
-                best = results.best()
-                if best and best.encoding:
-                    log.debug(f"Detected encoding via charset-normalizer: {best.encoding}")
-                    # Normalize encoding name: replace underscores with hyphens
-                    return best.encoding.lower().replace('_', '-')
-        except ImportError:
-            pass
-
-    except Exception as e:
-        log.warning(f"Encoding detection failed: {e}, using UTF-8 fallback")
-
-    return "utf-8"
-
-
-def read_file_safe(
-    path: Path | str,
-    confidence_threshold: float = 0.7,
-    max_detection_bytes: int = 100_000
-) -> Tuple[str, str]:
-    """Read file with automatic encoding detection and safe decoding.
-
-    Reads file bytes, detects encoding, and decodes with error replacement
-    to preserve file structure even with encoding issues.
-
-    Args:
-        path: Path to file to read
-        confidence_threshold: Minimum confidence for encoding detection
-        max_detection_bytes: Maximum bytes to use for encoding detection (default 100KB)
-
-    Returns:
-        Tuple of (content, detected_encoding)
-        - content: Decoded file content (with � for unmappable bytes)
-        - detected_encoding: Detected encoding name
-
-    Raises:
-        OSError: If file cannot be read
-        IsADirectoryError: If path is a directory
-    """
-    file_path = Path(path) if isinstance(path, str) else path
-
-    # Read file bytes
-    try:
-        content_bytes = file_path.read_bytes()
-    except Exception as e:
-        log.error(f"Failed to read file {file_path}: {e}")
-        raise
-
-    # Detect encoding from first N bytes for performance
-    detection_sample = content_bytes[:max_detection_bytes] if len(content_bytes) > max_detection_bytes else content_bytes
-    encoding = detect_encoding(detection_sample, confidence_threshold)
-
-    # Decode with error replacement to preserve structure
-    try:
-        content = content_bytes.decode(encoding, errors='replace')
-        log.debug(f"Successfully decoded {file_path} using {encoding}")
-        return content, encoding
-    except Exception as e:
-        # Final fallback to UTF-8 with replacement
-        log.warning(f"Failed to decode {file_path} with {encoding}, using UTF-8: {e}")
-        content = content_bytes.decode('utf-8', errors='replace')
-        return content, 'utf-8'
-
-
-def is_binary_file(path: Path | str, sample_size: int = 8192) -> bool:
-    """Check if file is likely binary by sampling first bytes.
-
-    Uses heuristic: if >30% of sample bytes are null or non-text, consider binary.
-
-    Args:
-        path: Path to file to check
-        sample_size: Number of bytes to sample (default 8KB)
-
-    Returns:
-        True if file appears to be binary, False otherwise
-    """
-    file_path = Path(path) if isinstance(path, str) else path
-
-    try:
-        with file_path.open('rb') as f:
-            sample = f.read(sample_size)
-
-        if not sample:
-            return False
-
-        # Count null bytes and non-printable characters
-        null_count = sample.count(b'\x00')
-        non_text_count = sum(1 for byte in sample if byte < 0x20 and byte not in (0x09, 0x0a, 0x0d))
-
-        # If >30% null bytes or >50% non-text, consider binary
-        null_ratio = null_count / len(sample)
-        non_text_ratio = non_text_count / len(sample)
-
-        return null_ratio > 0.3 or non_text_ratio > 0.5
-
-    except Exception as e:
-        log.debug(f"Binary check failed for {file_path}: {e}, assuming text")
-        return False
-
-
-__all__ = [
-    "ENCODING_DETECTION_AVAILABLE",
-    "check_encoding_available",
-    "detect_encoding",
-    "read_file_safe",
-    "is_binary_file",
-]

package/codex-lens/src/codexlens/parsers/factory.py

@@ -1,256 +0,0 @@
-"""Parser factory for CodexLens.
-
-Python and JavaScript/TypeScript parsing use Tree-Sitter grammars when
-available. Regex fallbacks are retained to preserve the existing parser
-interface and behavior in minimal environments.
-"""
-
-from __future__ import annotations
-
-import re
-from dataclasses import dataclass
-from pathlib import Path
-from typing import Dict, List, Optional, Protocol
-
-from codexlens.config import Config
-from codexlens.entities import IndexedFile, Symbol
-from codexlens.parsers.treesitter_parser import TreeSitterSymbolParser
-
-
-class Parser(Protocol):
-    def parse(self, text: str, path: Path) -> IndexedFile: ...
-
-
-@dataclass
-class SimpleRegexParser:
-    language_id: str
-
-    def parse(self, text: str, path: Path) -> IndexedFile:
-        # Try tree-sitter first for supported languages
-        if self.language_id in {"python", "javascript", "typescript"}:
-            ts_parser = TreeSitterSymbolParser(self.language_id, path)
-            if ts_parser.is_available():
-                symbols = ts_parser.parse_symbols(text)
-                if symbols is not None:
-                    return IndexedFile(
-                        path=str(path.resolve()),
-                        language=self.language_id,
-                        symbols=symbols,
-                        chunks=[],
-                    )
-
-        # Fallback to regex parsing
-        if self.language_id == "python":
-            symbols = _parse_python_symbols_regex(text)
-        elif self.language_id in {"javascript", "typescript"}:
-            symbols = _parse_js_ts_symbols_regex(text)
-        elif self.language_id == "java":
-            symbols = _parse_java_symbols(text)
-        elif self.language_id == "go":
-            symbols = _parse_go_symbols(text)
-        elif self.language_id == "markdown":
-            symbols = _parse_markdown_symbols(text)
-        elif self.language_id == "text":
-            symbols = _parse_text_symbols(text)
-        else:
-            symbols = _parse_generic_symbols(text)
-
-        return IndexedFile(
-            path=str(path.resolve()),
-            language=self.language_id,
-            symbols=symbols,
-            chunks=[],
-        )
-
-
-class ParserFactory:
-    def __init__(self, config: Config) -> None:
-        self.config = config
-        self._parsers: Dict[str, Parser] = {}
-
-    def get_parser(self, language_id: str) -> Parser:
-        if language_id not in self._parsers:
-            self._parsers[language_id] = SimpleRegexParser(language_id)
-        return self._parsers[language_id]
-
-
-# Regex-based fallback parsers
-_PY_CLASS_RE = re.compile(r"^\s*class\s+([A-Za-z_]\w*)\b")
-_PY_DEF_RE = re.compile(r"^\s*(?:async\s+)?def\s+([A-Za-z_]\w*)\s*\(")
-
-
-
-
-def _parse_python_symbols(text: str) -> List[Symbol]:
-    """Parse Python symbols, using tree-sitter if available, regex fallback."""
-    ts_parser = TreeSitterSymbolParser("python")
-    if ts_parser.is_available():
-        symbols = ts_parser.parse_symbols(text)
-        if symbols is not None:
-            return symbols
-    return _parse_python_symbols_regex(text)
-
-
-def _parse_js_ts_symbols(
-    text: str,
-    language_id: str = "javascript",
-    path: Optional[Path] = None,
-) -> List[Symbol]:
-    """Parse JS/TS symbols, using tree-sitter if available, regex fallback."""
-    ts_parser = TreeSitterSymbolParser(language_id, path)
-    if ts_parser.is_available():
-        symbols = ts_parser.parse_symbols(text)
-        if symbols is not None:
-            return symbols
-    return _parse_js_ts_symbols_regex(text)
-
-
-def _parse_python_symbols_regex(text: str) -> List[Symbol]:
-    symbols: List[Symbol] = []
-    current_class_indent: Optional[int] = None
-    for i, line in enumerate(text.splitlines(), start=1):
-        class_match = _PY_CLASS_RE.match(line)
-        if class_match:
-            current_class_indent = len(line) - len(line.lstrip(" "))
-            symbols.append(Symbol(name=class_match.group(1), kind="class", range=(i, i)))
-            continue
-        def_match = _PY_DEF_RE.match(line)
-        if def_match:
-            indent = len(line) - len(line.lstrip(" "))
-            kind = "method" if current_class_indent is not None and indent > current_class_indent else "function"
-            symbols.append(Symbol(name=def_match.group(1), kind=kind, range=(i, i)))
-            continue
-        if current_class_indent is not None:
-            indent = len(line) - len(line.lstrip(" "))
-            if line.strip() and indent <= current_class_indent:
-                current_class_indent = None
-    return symbols
-
-
-_JS_FUNC_RE = re.compile(r"^\s*(?:export\s+)?(?:async\s+)?function\s+([A-Za-z_$][\w$]*)\s*\(")
-_JS_CLASS_RE = re.compile(r"^\s*(?:export\s+)?class\s+([A-Za-z_$][\w$]*)\b")
-_JS_ARROW_RE = re.compile(
-    r"^\s*(?:export\s+)?(?:const|let|var)\s+([A-Za-z_$][\w$]*)\s*=\s*(?:async\s*)?\(?[^)]*\)?\s*=>"
-)
-_JS_METHOD_RE = re.compile(r"^\s+(?:async\s+)?([A-Za-z_$][\w$]*)\s*\([^)]*\)\s*\{")
-
-
-def _parse_js_ts_symbols_regex(text: str) -> List[Symbol]:
-    symbols: List[Symbol] = []
-    in_class = False
-    class_brace_depth = 0
-    brace_depth = 0
-
-    for i, line in enumerate(text.splitlines(), start=1):
-        brace_depth += line.count("{") - line.count("}")
-
-        class_match = _JS_CLASS_RE.match(line)
-        if class_match:
-            symbols.append(Symbol(name=class_match.group(1), kind="class", range=(i, i)))
-            in_class = True
-            class_brace_depth = brace_depth
-            continue
-
-        if in_class and brace_depth < class_brace_depth:
-            in_class = False
-
-        func_match = _JS_FUNC_RE.match(line)
-        if func_match:
-            symbols.append(Symbol(name=func_match.group(1), kind="function", range=(i, i)))
-            continue
-
-        arrow_match = _JS_ARROW_RE.match(line)
-        if arrow_match:
-            symbols.append(Symbol(name=arrow_match.group(1), kind="function", range=(i, i)))
-            continue
-
-        if in_class:
-            method_match = _JS_METHOD_RE.match(line)
-            if method_match:
-                name = method_match.group(1)
-                if name != "constructor":
-                    symbols.append(Symbol(name=name, kind="method", range=(i, i)))
-
-    return symbols
-
-
-_JAVA_CLASS_RE = re.compile(r"^\s*(?:public\s+)?class\s+([A-Za-z_]\w*)\b")
-_JAVA_METHOD_RE = re.compile(
-    r"^\s*(?:public|private|protected|static|\s)+[\w<>\[\]]+\s+([A-Za-z_]\w*)\s*\("
-)
-
-
-def _parse_java_symbols(text: str) -> List[Symbol]:
-    symbols: List[Symbol] = []
-    for i, line in enumerate(text.splitlines(), start=1):
-        class_match = _JAVA_CLASS_RE.match(line)
-        if class_match:
-            symbols.append(Symbol(name=class_match.group(1), kind="class", range=(i, i)))
-            continue
-        method_match = _JAVA_METHOD_RE.match(line)
-        if method_match:
-            symbols.append(Symbol(name=method_match.group(1), kind="method", range=(i, i)))
-    return symbols
-
-
-_GO_FUNC_RE = re.compile(r"^\s*func\s+(?:\([^)]+\)\s+)?([A-Za-z_]\w*)\s*\(")
-_GO_TYPE_RE = re.compile(r"^\s*type\s+([A-Za-z_]\w*)\s+(?:struct|interface)\b")
-
-
-def _parse_go_symbols(text: str) -> List[Symbol]:
-    symbols: List[Symbol] = []
-    for i, line in enumerate(text.splitlines(), start=1):
-        type_match = _GO_TYPE_RE.match(line)
-        if type_match:
-            symbols.append(Symbol(name=type_match.group(1), kind="class", range=(i, i)))
-            continue
-        func_match = _GO_FUNC_RE.match(line)
-        if func_match:
-            symbols.append(Symbol(name=func_match.group(1), kind="function", range=(i, i)))
-    return symbols
-
-
-_GENERIC_DEF_RE = re.compile(r"^\s*(?:def|function|func)\s+([A-Za-z_]\w*)\b")
-_GENERIC_CLASS_RE = re.compile(r"^\s*(?:class|struct|interface)\s+([A-Za-z_]\w*)\b")
-
-
-def _parse_generic_symbols(text: str) -> List[Symbol]:
-    symbols: List[Symbol] = []
-    for i, line in enumerate(text.splitlines(), start=1):
-        class_match = _GENERIC_CLASS_RE.match(line)
-        if class_match:
-            symbols.append(Symbol(name=class_match.group(1), kind="class", range=(i, i)))
-            continue
-        def_match = _GENERIC_DEF_RE.match(line)
-        if def_match:
-            symbols.append(Symbol(name=def_match.group(1), kind="function", range=(i, i)))
-    return symbols
-
-
-# Markdown heading regex: # Heading, ## Heading, etc.
-_MD_HEADING_RE = re.compile(r"^(#{1,6})\s+(.+)$")
-
-
-def _parse_markdown_symbols(text: str) -> List[Symbol]:
-    """Parse Markdown headings as symbols.
-    
-    Extracts # headings as 'section' symbols with heading level as kind suffix.
-    """
-    symbols: List[Symbol] = []
-    for i, line in enumerate(text.splitlines(), start=1):
-        heading_match = _MD_HEADING_RE.match(line)
-        if heading_match:
-            level = len(heading_match.group(1))
-            title = heading_match.group(2).strip()
-            # Use 'section' kind with level indicator
-            kind = f"h{level}"
-            symbols.append(Symbol(name=title, kind=kind, range=(i, i)))
-    return symbols
-
-
-def _parse_text_symbols(text: str) -> List[Symbol]:
-    """Parse plain text files - no symbols, just index content."""
-    # Text files don't have structured symbols, return empty list
-    # The file content will still be indexed for FTS search
-    return []
-

package/codex-lens/src/codexlens/parsers/tokenizer.py

@@ -1,98 +0,0 @@
-"""Token counting utilities for CodexLens.
-
-Provides accurate token counting using tiktoken with character count fallback.
-"""
-
-from __future__ import annotations
-
-from typing import Optional
-
-try:
-    import tiktoken
-    TIKTOKEN_AVAILABLE = True
-except ImportError:
-    TIKTOKEN_AVAILABLE = False
-
-
-class Tokenizer:
-    """Token counter with tiktoken primary and character count fallback."""
-
-    def __init__(self, encoding_name: str = "cl100k_base") -> None:
-        """Initialize tokenizer.
-
-        Args:
-            encoding_name: Tiktoken encoding name (default: cl100k_base for GPT-4)
-        """
-        self._encoding: Optional[object] = None
-        self._encoding_name = encoding_name
-
-        if TIKTOKEN_AVAILABLE:
-            try:
-                self._encoding = tiktoken.get_encoding(encoding_name)
-            except Exception:
-                # Fallback to character counting if encoding fails
-                self._encoding = None
-
-    def count_tokens(self, text: str) -> int:
-        """Count tokens in text.
-
-        Uses tiktoken if available, otherwise falls back to character count / 4.
-
-        Args:
-            text: Text to count tokens for
-
-        Returns:
-            Estimated token count
-        """
-        if not text:
-            return 0
-
-        if self._encoding is not None:
-            try:
-                return len(self._encoding.encode(text))  # type: ignore[attr-defined]
-            except Exception:
-                # Fall through to character count fallback
-                pass
-
-        # Fallback: rough estimate using character count
-        # Average of ~4 characters per token for English text
-        return max(1, len(text) // 4)
-
-    def is_using_tiktoken(self) -> bool:
-        """Check if tiktoken is being used.
-
-        Returns:
-            True if tiktoken is available and initialized
-        """
-        return self._encoding is not None
-
-
-# Global default tokenizer instance
-_default_tokenizer: Optional[Tokenizer] = None
-
-
-def get_default_tokenizer() -> Tokenizer:
-    """Get the global default tokenizer instance.
-
-    Returns:
-        Shared Tokenizer instance
-    """
-    global _default_tokenizer
-    if _default_tokenizer is None:
-        _default_tokenizer = Tokenizer()
-    return _default_tokenizer
-
-
-def count_tokens(text: str, tokenizer: Optional[Tokenizer] = None) -> int:
-    """Count tokens in text using default or provided tokenizer.
-
-    Args:
-        text: Text to count tokens for
-        tokenizer: Optional tokenizer instance (uses default if None)
-
-    Returns:
-        Estimated token count
-    """
-    if tokenizer is None:
-        tokenizer = get_default_tokenizer()
-    return tokenizer.count_tokens(text)