claude-code-workflow 6.1.4 → 6.2.2

package/codex-lens/src/codexlens/cli/model_manager.py

@@ -0,0 +1,289 @@
+"""Model Manager - Manage fastembed models for semantic search."""
+
+import json
+import os
+import shutil
+from pathlib import Path
+from typing import Dict, List, Optional
+
+try:
+    from fastembed import TextEmbedding
+    FASTEMBED_AVAILABLE = True
+except ImportError:
+    FASTEMBED_AVAILABLE = False
+
+
+# Model profiles with metadata
+MODEL_PROFILES = {
+    "fast": {
+        "model_name": "BAAI/bge-small-en-v1.5",
+        "dimensions": 384,
+        "size_mb": 80,
+        "description": "Fast, lightweight, English-optimized",
+        "use_case": "Quick prototyping, resource-constrained environments",
+    },
+    "code": {
+        "model_name": "jinaai/jina-embeddings-v2-base-code",
+        "dimensions": 768,
+        "size_mb": 150,
+        "description": "Code-optimized, best for programming languages",
+        "use_case": "Open source projects, code semantic search",
+    },
+    "multilingual": {
+        "model_name": "intfloat/multilingual-e5-large",
+        "dimensions": 1024,
+        "size_mb": 1000,
+        "description": "Multilingual + code support",
+        "use_case": "Enterprise multilingual projects",
+    },
+    "balanced": {
+        "model_name": "mixedbread-ai/mxbai-embed-large-v1",
+        "dimensions": 1024,
+        "size_mb": 600,
+        "description": "High accuracy, general purpose",
+        "use_case": "High-quality semantic search, balanced performance",
+    },
+}
+
+
+def get_cache_dir() -> Path:
+    """Get fastembed cache directory.
+
+    Returns:
+        Path to cache directory (usually ~/.cache/fastembed or %LOCALAPPDATA%\\Temp\\fastembed_cache)
+    """
+    # Check HF_HOME environment variable first
+    if "HF_HOME" in os.environ:
+        return Path(os.environ["HF_HOME"])
+
+    # Default cache locations
+    if os.name == "nt":  # Windows
+        cache_dir = Path(os.environ.get("LOCALAPPDATA", Path.home() / "AppData" / "Local")) / "Temp" / "fastembed_cache"
+    else:  # Unix-like
+        cache_dir = Path.home() / ".cache" / "fastembed"
+
+    return cache_dir
+
+
+def list_models() -> Dict[str, any]:
+    """List available model profiles and their installation status.
+
+    Returns:
+        Dictionary with model profiles, installed status, and cache info
+    """
+    if not FASTEMBED_AVAILABLE:
+        return {
+            "success": False,
+            "error": "fastembed not installed. Install with: pip install codexlens[semantic]",
+        }
+
+    cache_dir = get_cache_dir()
+    cache_exists = cache_dir.exists()
+
+    models = []
+    for profile, info in MODEL_PROFILES.items():
+        model_name = info["model_name"]
+
+        # Check if model is cached
+        installed = False
+        cache_size_mb = 0
+
+        if cache_exists:
+            # Check for model directory in cache
+            model_cache_path = cache_dir / f"models--{model_name.replace('/', '--')}"
+            if model_cache_path.exists():
+                installed = True
+                # Calculate cache size
+                total_size = sum(
+                    f.stat().st_size
+                    for f in model_cache_path.rglob("*")
+                    if f.is_file()
+                )
+                cache_size_mb = round(total_size / (1024 * 1024), 1)
+
+        models.append({
+            "profile": profile,
+            "model_name": model_name,
+            "dimensions": info["dimensions"],
+            "estimated_size_mb": info["size_mb"],
+            "actual_size_mb": cache_size_mb if installed else None,
+            "description": info["description"],
+            "use_case": info["use_case"],
+            "installed": installed,
+        })
+
+    return {
+        "success": True,
+        "result": {
+            "models": models,
+            "cache_dir": str(cache_dir),
+            "cache_exists": cache_exists,
+        },
+    }
+
+
+def download_model(profile: str, progress_callback: Optional[callable] = None) -> Dict[str, any]:
+    """Download a model by profile name.
+
+    Args:
+        profile: Model profile name (fast, code, multilingual, balanced)
+        progress_callback: Optional callback function to report progress
+
+    Returns:
+        Result dictionary with success status
+    """
+    if not FASTEMBED_AVAILABLE:
+        return {
+            "success": False,
+            "error": "fastembed not installed. Install with: pip install codexlens[semantic]",
+        }
+
+    if profile not in MODEL_PROFILES:
+        return {
+            "success": False,
+            "error": f"Unknown profile: {profile}. Available: {', '.join(MODEL_PROFILES.keys())}",
+        }
+
+    model_name = MODEL_PROFILES[profile]["model_name"]
+
+    try:
+        # Download model by instantiating TextEmbedding
+        # This will automatically download to cache if not present
+        if progress_callback:
+            progress_callback(f"Downloading {model_name}...")
+
+        embedder = TextEmbedding(model_name=model_name)
+
+        if progress_callback:
+            progress_callback(f"Model {model_name} downloaded successfully")
+
+        # Get cache info
+        cache_dir = get_cache_dir()
+        model_cache_path = cache_dir / f"models--{model_name.replace('/', '--')}"
+
+        cache_size = 0
+        if model_cache_path.exists():
+            total_size = sum(
+                f.stat().st_size
+                for f in model_cache_path.rglob("*")
+                if f.is_file()
+            )
+            cache_size = round(total_size / (1024 * 1024), 1)
+
+        return {
+            "success": True,
+            "result": {
+                "profile": profile,
+                "model_name": model_name,
+                "cache_size_mb": cache_size,
+                "cache_path": str(model_cache_path),
+            },
+        }
+
+    except Exception as e:
+        return {
+            "success": False,
+            "error": f"Failed to download model: {str(e)}",
+        }
+
+
+def delete_model(profile: str) -> Dict[str, any]:
+    """Delete a downloaded model from cache.
+
+    Args:
+        profile: Model profile name to delete
+
+    Returns:
+        Result dictionary with success status
+    """
+    if profile not in MODEL_PROFILES:
+        return {
+            "success": False,
+            "error": f"Unknown profile: {profile}. Available: {', '.join(MODEL_PROFILES.keys())}",
+        }
+
+    model_name = MODEL_PROFILES[profile]["model_name"]
+    cache_dir = get_cache_dir()
+    model_cache_path = cache_dir / f"models--{model_name.replace('/', '--')}"
+
+    if not model_cache_path.exists():
+        return {
+            "success": False,
+            "error": f"Model {profile} ({model_name}) is not installed",
+        }
+
+    try:
+        # Calculate size before deletion
+        total_size = sum(
+            f.stat().st_size
+            for f in model_cache_path.rglob("*")
+            if f.is_file()
+        )
+        size_mb = round(total_size / (1024 * 1024), 1)
+
+        # Delete model directory
+        shutil.rmtree(model_cache_path)
+
+        return {
+            "success": True,
+            "result": {
+                "profile": profile,
+                "model_name": model_name,
+                "deleted_size_mb": size_mb,
+                "cache_path": str(model_cache_path),
+            },
+        }
+
+    except Exception as e:
+        return {
+            "success": False,
+            "error": f"Failed to delete model: {str(e)}",
+        }
+
+
+def get_model_info(profile: str) -> Dict[str, any]:
+    """Get detailed information about a model profile.
+
+    Args:
+        profile: Model profile name
+
+    Returns:
+        Result dictionary with model information
+    """
+    if profile not in MODEL_PROFILES:
+        return {
+            "success": False,
+            "error": f"Unknown profile: {profile}. Available: {', '.join(MODEL_PROFILES.keys())}",
+        }
+
+    info = MODEL_PROFILES[profile]
+    model_name = info["model_name"]
+
+    # Check installation status
+    cache_dir = get_cache_dir()
+    model_cache_path = cache_dir / f"models--{model_name.replace('/', '--')}"
+    installed = model_cache_path.exists()
+
+    cache_size_mb = None
+    if installed:
+        total_size = sum(
+            f.stat().st_size
+            for f in model_cache_path.rglob("*")
+            if f.is_file()
+        )
+        cache_size_mb = round(total_size / (1024 * 1024), 1)
+
+    return {
+        "success": True,
+        "result": {
+            "profile": profile,
+            "model_name": model_name,
+            "dimensions": info["dimensions"],
+            "estimated_size_mb": info["size_mb"],
+            "actual_size_mb": cache_size_mb,
+            "description": info["description"],
+            "use_case": info["use_case"],
+            "installed": installed,
+            "cache_path": str(model_cache_path) if installed else None,
+        },
+    }

package/codex-lens/src/codexlens/cli/output.py

@@ -0,0 +1,124 @@
+"""Rich and JSON output helpers for CodexLens CLI."""
+
+from __future__ import annotations
+
+import json
+import sys
+from dataclasses import asdict, is_dataclass
+from pathlib import Path
+from typing import Any, Iterable, Mapping, Sequence
+
+from rich.console import Console
+from rich.table import Table
+from rich.text import Text
+
+from codexlens.entities import SearchResult, Symbol
+
+# Force UTF-8 encoding for Windows console to properly display Chinese text
+# Use force_terminal=True and legacy_windows=False to avoid GBK encoding issues
+console = Console(force_terminal=True, legacy_windows=False)
+
+
+def _to_jsonable(value: Any) -> Any:
+    if value is None:
+        return None
+    if hasattr(value, "model_dump"):
+        return value.model_dump()
+    if is_dataclass(value):
+        return asdict(value)
+    if isinstance(value, Path):
+        return str(value)
+    if isinstance(value, Mapping):
+        return {k: _to_jsonable(v) for k, v in value.items()}
+    if isinstance(value, (list, tuple, set)):
+        return [_to_jsonable(v) for v in value]
+    return value
+
+
+def print_json(*, success: bool, result: Any = None, error: str | None = None) -> None:
+    payload: dict[str, Any] = {"success": success}
+    if success:
+        payload["result"] = _to_jsonable(result)
+    else:
+        payload["error"] = error or "Unknown error"
+    console.print_json(json.dumps(payload, ensure_ascii=False))
+
+
+def render_search_results(
+    results: Sequence[SearchResult], *, title: str = "Search Results", verbose: bool = False
+) -> None:
+    """Render search results with optional source tags in verbose mode.
+
+    Args:
+        results: Search results to display
+        title: Table title
+        verbose: If True, show search source tags ([E], [F], [V]) and fusion scores
+    """
+    table = Table(title=title, show_lines=False)
+
+    if verbose:
+        # Verbose mode: show source tags
+        table.add_column("Source", style="dim", width=6, justify="center")
+
+    table.add_column("Path", style="cyan", no_wrap=True)
+    table.add_column("Score", style="magenta", justify="right")
+    table.add_column("Excerpt", style="white")
+
+    for res in results:
+        excerpt = res.excerpt or ""
+        score_str = f"{res.score:.3f}"
+
+        if verbose:
+            # Extract search source tag if available
+            source = getattr(res, "search_source", None)
+            source_tag = ""
+            if source == "exact":
+                source_tag = "[E]"
+            elif source == "fuzzy":
+                source_tag = "[F]"
+            elif source == "vector":
+                source_tag = "[V]"
+            elif source == "fusion":
+                source_tag = "[RRF]"
+            table.add_row(source_tag, res.path, score_str, excerpt)
+        else:
+            table.add_row(res.path, score_str, excerpt)
+
+    console.print(table)
+
+
+def render_symbols(symbols: Sequence[Symbol], *, title: str = "Symbols") -> None:
+    table = Table(title=title)
+    table.add_column("Name", style="green")
+    table.add_column("Kind", style="yellow")
+    table.add_column("Range", style="white", justify="right")
+
+    for sym in symbols:
+        start, end = sym.range
+        table.add_row(sym.name, sym.kind, f"{start}-{end}")
+
+    console.print(table)
+
+
+def render_status(stats: Mapping[str, Any]) -> None:
+    table = Table(title="Index Status")
+    table.add_column("Metric", style="cyan")
+    table.add_column("Value", style="white")
+
+    for key, value in stats.items():
+        if isinstance(value, Mapping):
+            value_text = ", ".join(f"{k}:{v}" for k, v in value.items())
+        elif isinstance(value, (list, tuple)):
+            value_text = ", ".join(str(v) for v in value)
+        else:
+            value_text = str(value)
+        table.add_row(str(key), value_text)
+
+    console.print(table)
+
+
+def render_file_inspect(path: str, language: str, symbols: Iterable[Symbol]) -> None:
+    header = Text.assemble(("File: ", "bold"), (path, "cyan"), ("  Language: ", "bold"), (language, "green"))
+    console.print(header)
+    render_symbols(list(symbols), title="Discovered Symbols")
+

package/codex-lens/src/codexlens/config.py

@@ -0,0 +1,201 @@
+"""Configuration system for CodexLens."""
+
+from __future__ import annotations
+
+import os
+from dataclasses import dataclass, field
+from functools import cached_property
+from pathlib import Path
+from typing import Any, Dict, List, Optional
+
+from .errors import ConfigError
+
+
+# Workspace-local directory name
+WORKSPACE_DIR_NAME = ".codexlens"
+
+
+def _default_global_dir() -> Path:
+    """Get global CodexLens data directory."""
+    env_override = os.getenv("CODEXLENS_DATA_DIR")
+    if env_override:
+        return Path(env_override).expanduser().resolve()
+    return (Path.home() / ".codexlens").resolve()
+
+
+def find_workspace_root(start_path: Path) -> Optional[Path]:
+    """Find the workspace root by looking for .codexlens directory.
+
+    Searches from start_path upward to find an existing .codexlens directory.
+    Returns None if not found.
+    """
+    current = start_path.resolve()
+
+    # Search up to filesystem root
+    while current != current.parent:
+        workspace_dir = current / WORKSPACE_DIR_NAME
+        if workspace_dir.is_dir():
+            return current
+        current = current.parent
+
+    # Check root as well
+    workspace_dir = current / WORKSPACE_DIR_NAME
+    if workspace_dir.is_dir():
+        return current
+
+    return None
+
+
+@dataclass
+class Config:
+    """Runtime configuration for CodexLens.
+
+    - data_dir: Base directory for all persistent CodexLens data.
+    - venv_path: Optional virtualenv used for language tooling.
+    - supported_languages: Language IDs and their associated file extensions.
+    - parsing_rules: Per-language parsing and chunking hints.
+    """
+
+    data_dir: Path = field(default_factory=_default_global_dir)
+    venv_path: Path = field(default_factory=lambda: _default_global_dir() / "venv")
+    supported_languages: Dict[str, Dict[str, Any]] = field(
+        default_factory=lambda: {
+            "python": {"extensions": [".py"], "tree_sitter_language": "python"},
+            "javascript": {"extensions": [".js", ".jsx"], "tree_sitter_language": "javascript"},
+            "typescript": {"extensions": [".ts", ".tsx"], "tree_sitter_language": "typescript"},
+            "java": {"extensions": [".java"], "tree_sitter_language": "java"},
+            "go": {"extensions": [".go"], "tree_sitter_language": "go"},
+            "zig": {"extensions": [".zig"], "tree_sitter_language": "zig"},
+            "objective-c": {"extensions": [".m", ".mm"], "tree_sitter_language": "objc"},
+            "markdown": {"extensions": [".md", ".mdx"], "tree_sitter_language": None},
+            "text": {"extensions": [".txt"], "tree_sitter_language": None},
+        }
+    )
+    parsing_rules: Dict[str, Dict[str, Any]] = field(
+        default_factory=lambda: {
+            "default": {
+                "max_chunk_chars": 4000,
+                "max_chunk_lines": 200,
+                "overlap_lines": 20,
+            }
+        }
+    )
+
+    llm_enabled: bool = False
+    llm_tool: str = "gemini"
+    llm_timeout_ms: int = 300000
+    llm_batch_size: int = 5
+
+    # Hybrid chunker configuration
+    hybrid_max_chunk_size: int = 2000  # Max characters per chunk before LLM refinement
+    hybrid_llm_refinement: bool = False  # Enable LLM-based semantic boundary refinement
+    def __post_init__(self) -> None:
+        try:
+            self.data_dir = self.data_dir.expanduser().resolve()
+            self.venv_path = self.venv_path.expanduser().resolve()
+            self.data_dir.mkdir(parents=True, exist_ok=True)
+        except Exception as exc:
+            raise ConfigError(f"Failed to initialize data_dir at {self.data_dir}: {exc}") from exc
+
+    @cached_property
+    def cache_dir(self) -> Path:
+        """Directory for transient caches."""
+        return self.data_dir / "cache"
+
+    @cached_property
+    def index_dir(self) -> Path:
+        """Directory where index artifacts are stored."""
+        return self.data_dir / "index"
+
+    @cached_property
+    def db_path(self) -> Path:
+        """Default SQLite index path."""
+        return self.index_dir / "codexlens.db"
+
+    def ensure_runtime_dirs(self) -> None:
+        """Create standard runtime directories if missing."""
+        for directory in (self.cache_dir, self.index_dir):
+            try:
+                directory.mkdir(parents=True, exist_ok=True)
+            except Exception as exc:
+                raise ConfigError(f"Failed to create directory {directory}: {exc}") from exc
+
+    def language_for_path(self, path: str | Path) -> str | None:
+        """Infer a supported language ID from a file path."""
+        extension = Path(path).suffix.lower()
+        for language_id, spec in self.supported_languages.items():
+            extensions: List[str] = spec.get("extensions", [])
+            if extension in extensions:
+                return language_id
+        return None
+
+    def rules_for_language(self, language_id: str) -> Dict[str, Any]:
+        """Get parsing rules for a specific language, falling back to defaults."""
+        return {**self.parsing_rules.get("default", {}), **self.parsing_rules.get(language_id, {})}
+
+
+@dataclass
+class WorkspaceConfig:
+    """Workspace-local configuration for CodexLens.
+
+    Stores index data in project/.codexlens/ directory.
+    """
+
+    workspace_root: Path
+
+    def __post_init__(self) -> None:
+        self.workspace_root = Path(self.workspace_root).resolve()
+
+    @property
+    def codexlens_dir(self) -> Path:
+        """The .codexlens directory in workspace root."""
+        return self.workspace_root / WORKSPACE_DIR_NAME
+
+    @property
+    def db_path(self) -> Path:
+        """SQLite index path for this workspace."""
+        return self.codexlens_dir / "index.db"
+
+    @property
+    def cache_dir(self) -> Path:
+        """Cache directory for this workspace."""
+        return self.codexlens_dir / "cache"
+
+    def initialize(self) -> None:
+        """Create the .codexlens directory structure."""
+        try:
+            self.codexlens_dir.mkdir(parents=True, exist_ok=True)
+            self.cache_dir.mkdir(parents=True, exist_ok=True)
+
+            # Create .gitignore to exclude cache but keep index
+            gitignore_path = self.codexlens_dir / ".gitignore"
+            if not gitignore_path.exists():
+                gitignore_path.write_text(
+                    "# CodexLens workspace data\n"
+                    "cache/\n"
+                    "*.log\n"
+                )
+        except Exception as exc:
+            raise ConfigError(f"Failed to initialize workspace at {self.codexlens_dir}: {exc}") from exc
+
+    def exists(self) -> bool:
+        """Check if workspace is already initialized."""
+        return self.codexlens_dir.is_dir() and self.db_path.exists()
+
+    @classmethod
+    def from_path(cls, path: Path) -> Optional["WorkspaceConfig"]:
+        """Create WorkspaceConfig from a path by finding workspace root.
+
+        Returns None if no workspace found.
+        """
+        root = find_workspace_root(path)
+        if root is None:
+            return None
+        return cls(workspace_root=root)
+
+    @classmethod
+    def create_at(cls, path: Path) -> "WorkspaceConfig":
+        """Create a new workspace at the given path."""
+        config = cls(workspace_root=path)
+        config.initialize()
+        return config