claude-code-workflow 6.2.7 → 6.3.0

package/codex-lens/src/codexlens/cli/commands.py

@@ -62,10 +62,27 @@ def _parse_languages(raw: Optional[List[str]]) -> Optional[List[str]]:
 
 
 def _get_index_root() -> Path:
-    """Get the index root directory from config or default."""
+    """Get the index root directory from config or default.
+
+    Priority order:
+    1. CODEXLENS_INDEX_DIR environment variable
+    2. index_dir from ~/.codexlens/config.json
+    3. Default: ~/.codexlens/indexes
+    """
     env_override = os.getenv("CODEXLENS_INDEX_DIR")
     if env_override:
         return Path(env_override).expanduser().resolve()
+
+    # Read from config.json
+    config_file = Path.home() / ".codexlens" / "config.json"
+    if config_file.exists():
+        try:
+            cfg = json.loads(config_file.read_text(encoding="utf-8"))
+            if "index_dir" in cfg:
+                return Path(cfg["index_dir"]).expanduser().resolve()
+        except (json.JSONDecodeError, OSError):
+            pass  # Fall through to default
+
     return Path.home() / ".codexlens" / "indexes"
 
 
@@ -86,10 +103,12 @@ def init(
         "-l",
         help="Limit indexing to specific languages (repeat or comma-separated).",
     ),
-    workers: Optional[int] = typer.Option(None, "--workers", "-w", min=1, max=16, help="Parallel worker processes (default: auto-detect based on CPU count, max 16)."),
+    workers: Optional[int] = typer.Option(None, "--workers", "-w", min=1, help="Parallel worker processes (default: auto-detect based on CPU count)."),
     force: bool = typer.Option(False, "--force", "-f", help="Force full reindex (skip incremental mode)."),
     no_embeddings: bool = typer.Option(False, "--no-embeddings", help="Skip automatic embedding generation (if semantic deps installed)."),
-    embedding_model: str = typer.Option("code", "--embedding-model", help="Embedding model profile: fast, code, multilingual, balanced."),
+    embedding_backend: str = typer.Option("fastembed", "--embedding-backend", help="Embedding backend: fastembed (local) or litellm (remote API)."),
+    embedding_model: str = typer.Option("code", "--embedding-model", help="Embedding model: profile name for fastembed (fast/code/multilingual/balanced) or model name for litellm (e.g. text-embedding-3-small)."),
+    max_workers: int = typer.Option(1, "--max-workers", min=1, help="Max concurrent API calls for embedding generation. Recommended: 4-8 for litellm backend."),
     json_mode: bool = typer.Option(False, "--json", help="Output JSON response."),
     verbose: bool = typer.Option(False, "--verbose", "-v", help="Enable debug logging."),
 ) -> None:
@@ -103,6 +122,14 @@ def init(
 
     If semantic search dependencies are installed, automatically generates embeddings
     after indexing completes. Use --no-embeddings to skip this step.
+
+    Embedding Backend Options:
+      - fastembed: Local ONNX-based embeddings (default, no API calls)
+      - litellm: Remote API embeddings via ccw-litellm (requires API keys)
+
+    Embedding Model Options:
+      - For fastembed backend: Use profile names (fast, code, multilingual, balanced)
+      - For litellm backend: Use model names (e.g., text-embedding-3-small, text-embedding-ada-002)
     """
     _configure_logging(verbose, json_mode)
     config = Config()
@@ -139,26 +166,37 @@ def init(
             "errors": len(build_result.errors),
         }
 
-        if json_mode:
-            print_json(success=True, result=result)
-        else:
+        if not json_mode:
             console.print(f"[green]OK[/green] Indexed [bold]{build_result.total_files}[/bold] files in [bold]{build_result.total_dirs}[/bold] directories")
             console.print(f"  Index root: {build_result.index_root}")
             if build_result.errors:
                 console.print(f"  [yellow]Warnings:[/yellow] {len(build_result.errors)} errors")
 
-        # Auto-generate embeddings if semantic search is available
+        # Auto-generate embeddings if the requested backend is available
         if not no_embeddings:
             try:
-                from codexlens.semantic import SEMANTIC_AVAILABLE
+                from codexlens.semantic import is_embedding_backend_available
                 from codexlens.cli.embedding_manager import generate_embeddings_recursive, get_embeddings_status
 
-                if SEMANTIC_AVAILABLE:
+                # Validate embedding backend
+                valid_backends = ["fastembed", "litellm"]
+                if embedding_backend not in valid_backends:
+                    error_msg = f"Invalid embedding backend: {embedding_backend}. Must be one of: {', '.join(valid_backends)}"
+                    if json_mode:
+                        print_json(success=False, error=error_msg)
+                    else:
+                        console.print(f"[red]Error:[/red] {error_msg}")
+                    raise typer.Exit(code=1)
+
+                backend_available, backend_error = is_embedding_backend_available(embedding_backend)
+
+                if backend_available:
                     # Use the index root directory (not the _index.db file)
                     index_root = Path(build_result.index_root)
 
                     if not json_mode:
                         console.print("\n[bold]Generating embeddings...[/bold]")
+                        console.print(f"Backend: [cyan]{embedding_backend}[/cyan]")
                         console.print(f"Model: [cyan]{embedding_model}[/cyan]")
                     else:
                         # Output progress message for JSON mode (parsed by Node.js)
@@ -179,10 +217,12 @@ def init(
 
                     embed_result = generate_embeddings_recursive(
                         index_root,
+                        embedding_backend=embedding_backend,
                         model_profile=embedding_model,
                         force=False,  # Don't force regenerate during init
                         chunk_size=2000,
                         progress_callback=progress_update,  # Always use callback
+                        max_workers=max_workers,
                     )
 
                     if embed_result["success"]:
@@ -224,10 +264,10 @@ def init(
                         }
                 else:
                     if not json_mode and verbose:
-                        console.print("[dim]Semantic search not available. Skipping embeddings.[/dim]")
+                        console.print(f"[dim]Embedding backend '{embedding_backend}' not available. Skipping embeddings.[/dim]")
                     result["embeddings"] = {
                         "generated": False,
-                        "error": "Semantic dependencies not installed",
+                        "error": backend_error or "Embedding backend not available",
                     }
             except Exception as e:
                 if not json_mode and verbose:
@@ -242,6 +282,10 @@ def init(
                 "error": "Skipped (--no-embeddings)",
             }
 
+        # Output final JSON result with embeddings status
+        if json_mode:
+            print_json(success=True, result=result)
+
     except StorageError as exc:
         if json_mode:
             print_json(success=False, error=f"Storage error: {exc}")
@@ -307,7 +351,7 @@ def search(
       Use 'codexlens embeddings-generate' to create embeddings first.
 
     Hybrid Mode:
-      Default weights: exact=0.4, fuzzy=0.3, vector=0.3
+      Default weights: exact=0.3, fuzzy=0.1, vector=0.6
       Use --weights to customize (e.g., --weights 0.5,0.3,0.2)
 
     Examples:
@@ -434,6 +478,7 @@ def search(
                     "path": r.path,
                     "score": r.score,
                     "excerpt": r.excerpt,
+                    "content": r.content,  # Full function/class body
                     "source": getattr(r, "search_source", None),
                     "symbol": getattr(r, "symbol", None),
                 }
@@ -715,6 +760,16 @@ def status(
                 console.print(f"  Coverage: {embeddings_info['coverage_percent']:.1f}%")
                 console.print(f"  Total Chunks: {embeddings_info['total_chunks']}")
 
+                # Display model information if available
+                model_info = embeddings_info.get('model_info')
+                if model_info:
+                    console.print("\n[bold]Embedding Model:[/bold]")
+                    console.print(f"  Backend: [cyan]{model_info.get('backend', 'unknown')}[/cyan]")
+                    console.print(f"  Model: [cyan]{model_info.get('model_profile', 'unknown')}[/cyan] ({model_info.get('model_name', '')})")
+                    console.print(f"  Dimensions: {model_info.get('embedding_dim', 'unknown')}")
+                    if model_info.get('updated_at'):
+                        console.print(f"  Last Updated: {model_info['updated_at']}")
+
     except StorageError as exc:
         if json_mode:
             print_json(success=False, error=f"Storage error: {exc}")
@@ -1764,11 +1819,17 @@ def embeddings_generate(
         exists=True,
         help="Path to _index.db file or project directory.",
     ),
+    backend: str = typer.Option(
+        "fastembed",
+        "--backend",
+        "-b",
+        help="Embedding backend: fastembed (local) or litellm (remote API).",
+    ),
     model: str = typer.Option(
         "code",
         "--model",
         "-m",
-        help="Model profile: fast, code, multilingual, balanced.",
+        help="Model: profile name for fastembed (fast/code/multilingual/balanced) or model name for litellm (e.g. text-embedding-3-small).",
     ),
     force: bool = typer.Option(
         False,
@@ -1787,6 +1848,13 @@ def embeddings_generate(
         "-r",
         help="Recursively process all _index.db files in directory tree.",
     ),
+    max_workers: int = typer.Option(
+        1,
+        "--max-workers",
+        "-w",
+        min=1,
+        help="Max concurrent API calls. Recommended: 4-8 for litellm backend. Default: 1 (sequential).",
+    ),
     json_mode: bool = typer.Option(False, "--json", help="Output JSON response."),
     verbose: bool = typer.Option(False, "--verbose", "-v", help="Enable verbose output."),
 ) -> None:
@@ -1796,20 +1864,48 @@ def embeddings_generate(
     semantic search capabilities. Embeddings are stored in the same
     database as the FTS index.
 
-    Model Profiles:
-      - fast: BAAI/bge-small-en-v1.5 (384 dims, ~80MB)
-      - code: jinaai/jina-embeddings-v2-base-code (768 dims, ~150MB) [recommended]
-      - multilingual: intfloat/multilingual-e5-large (1024 dims, ~1GB)
-      - balanced: mixedbread-ai/mxbai-embed-large-v1 (1024 dims, ~600MB)
+    Embedding Backend Options:
+      - fastembed: Local ONNX-based embeddings (default, no API calls)
+      - litellm: Remote API embeddings via ccw-litellm (requires API keys)
+
+    Model Options:
+      For fastembed backend (profiles):
+        - fast: BAAI/bge-small-en-v1.5 (384 dims, ~80MB)
+        - code: jinaai/jina-embeddings-v2-base-code (768 dims, ~150MB) [recommended]
+        - multilingual: intfloat/multilingual-e5-large (1024 dims, ~1GB)
+        - balanced: mixedbread-ai/mxbai-embed-large-v1 (1024 dims, ~600MB)
+
+      For litellm backend (model names):
+        - text-embedding-3-small, text-embedding-3-large (OpenAI)
+        - text-embedding-ada-002 (OpenAI legacy)
+        - Any model supported by ccw-litellm
 
     Examples:
-        codexlens embeddings-generate ~/projects/my-app              # Auto-find index for project
+        codexlens embeddings-generate ~/projects/my-app              # Auto-find index (fastembed, code profile)
         codexlens embeddings-generate ~/.codexlens/indexes/project/_index.db  # Specific index
-        codexlens embeddings-generate ~/projects/my-app --model fast --force  # Regenerate with fast model
+        codexlens embeddings-generate ~/projects/my-app --backend litellm --model text-embedding-3-small  # Use LiteLLM
+        codexlens embeddings-generate ~/projects/my-app --model fast --force  # Regenerate with fast profile
     """
     _configure_logging(verbose, json_mode)
 
-    from codexlens.cli.embedding_manager import generate_embeddings, generate_embeddings_recursive
+    from codexlens.cli.embedding_manager import (
+        generate_embeddings,
+        generate_embeddings_recursive,
+        scan_for_model_conflicts,
+        check_global_model_lock,
+        set_locked_model_config,
+    )
+
+    # Validate backend
+    valid_backends = ["fastembed", "litellm"]
+    if backend not in valid_backends:
+        error_msg = f"Invalid backend: {backend}. Must be one of: {', '.join(valid_backends)}"
+        if json_mode:
+            print_json(success=False, error=error_msg)
+        else:
+            console.print(f"[red]Error:[/red] {error_msg}")
+            console.print(f"[dim]Valid backends: {', '.join(valid_backends)}[/dim]")
+        raise typer.Exit(code=1)
 
     # Resolve path
     target_path = path.expanduser().resolve()
@@ -1860,23 +1956,100 @@ def embeddings_generate(
         console.print(f"Mode: [yellow]Recursive[/yellow]")
     else:
         console.print(f"Index: [dim]{index_path}[/dim]")
-    console.print(f"Model: [cyan]{model}[/cyan]\n")
+    console.print(f"Backend: [cyan]{backend}[/cyan]")
+    console.print(f"Model: [cyan]{model}[/cyan]")
+    if max_workers > 1:
+        console.print(f"Concurrency: [cyan]{max_workers} workers[/cyan]")
+    console.print()
+
+    # Check global model lock (prevents mixing different models)
+    if not force:
+        lock_result = check_global_model_lock(backend, model)
+        if lock_result["has_conflict"]:
+            locked = lock_result["locked_config"]
+            if json_mode:
+                print_json(
+                    success=False,
+                    error="Global model lock conflict",
+                    code="MODEL_LOCKED",
+                    locked_config=locked,
+                    target_config=lock_result["target_config"],
+                    hint="Use --force to override the lock and switch to a different model (will regenerate all embeddings)",
+                )
+                raise typer.Exit(code=1)
+            else:
+                console.print("[red]⛔ Global Model Lock Active[/red]")
+                console.print(f"  Locked model: [cyan]{locked['backend']}/{locked['model']}[/cyan]")
+                console.print(f"  Requested: [yellow]{backend}/{model}[/yellow]")
+                console.print(f"  Locked at: {locked.get('locked_at', 'unknown')}")
+                console.print()
+                console.print("[dim]All indexes must use the same embedding model.[/dim]")
+                console.print("[dim]Use --force to switch models (will regenerate all embeddings).[/dim]")
+                raise typer.Exit(code=1)
+
+    # Pre-check for model conflicts (only if not forcing)
+    if not force:
+        # Determine the index root for conflict scanning
+        scan_root = index_root if use_recursive else (index_path.parent if index_path else None)
+
+        if scan_root:
+            conflict_result = scan_for_model_conflicts(scan_root, backend, model)
+
+            if conflict_result["has_conflict"]:
+                existing = conflict_result["existing_config"]
+                conflict_count = len(conflict_result["conflicts"])
+
+                if json_mode:
+                    # JSON mode: return structured error for UI handling
+                    print_json(
+                        success=False,
+                        error="Model conflict detected",
+                        code="MODEL_CONFLICT",
+                        existing_config=existing,
+                        target_config=conflict_result["target_config"],
+                        conflict_count=conflict_count,
+                        conflicts=conflict_result["conflicts"][:5],  # Show first 5 conflicts
+                        hint="Use --force to overwrite existing embeddings with the new model",
+                    )
+                    raise typer.Exit(code=1)
+                else:
+                    # Interactive mode: show warning and ask for confirmation
+                    console.print("[yellow]⚠ Model Conflict Detected[/yellow]")
+                    console.print(f"  Existing: [red]{existing['backend']}/{existing['model']}[/red] ({existing.get('embedding_dim', '?')} dim)")
+                    console.print(f"  Requested: [green]{backend}/{model}[/green]")
+                    console.print(f"  Affected indexes: [yellow]{conflict_count}[/yellow]")
+                    console.print()
+                    console.print("[dim]Mixing different embedding models in the same index is not supported.[/dim]")
+                    console.print("[dim]Overwriting will delete all existing embeddings and regenerate with the new model.[/dim]")
+                    console.print()
+
+                    # Ask for confirmation
+                    if typer.confirm("Overwrite existing embeddings with the new model?", default=False):
+                        force = True
+                        console.print("[green]Confirmed.[/green] Proceeding with overwrite...\n")
+                    else:
+                        console.print("[yellow]Cancelled.[/yellow] Use --force to skip this prompt.")
+                        raise typer.Exit(code=0)
 
     if use_recursive:
         result = generate_embeddings_recursive(
             index_root,
+            embedding_backend=backend,
             model_profile=model,
             force=force,
             chunk_size=chunk_size,
             progress_callback=progress_update,
+            max_workers=max_workers,
         )
     else:
         result = generate_embeddings(
             index_path,
+            embedding_backend=backend,
             model_profile=model,
             force=force,
             chunk_size=chunk_size,
             progress_callback=progress_update,
+            max_workers=max_workers,
         )
 
     if json_mode:
@@ -1889,14 +2062,21 @@ def embeddings_generate(
             # Provide helpful hints
             if "already has" in error_msg:
                 console.print("\n[dim]Use --force to regenerate existing embeddings[/dim]")
-            elif "Semantic search not available" in error_msg:
+            elif "fastembed not available" in error_msg or "Semantic search not available" in error_msg:
                 console.print("\n[dim]Install semantic dependencies:[/dim]")
                 console.print("  [cyan]pip install codexlens[semantic][/cyan]")
+            elif "ccw-litellm not available" in error_msg:
+                console.print("\n[dim]Install LiteLLM backend dependencies:[/dim]")
+                console.print("  [cyan]pip install ccw-litellm[/cyan]")
 
             raise typer.Exit(code=1)
 
         data = result["result"]
 
+        # Set global model lock after successful generation
+        # This prevents using different models for future indexes
+        set_locked_model_config(backend, model)
+
         if use_recursive:
             # Recursive mode output
             console.print(f"[green]✓[/green] Recursive embeddings generation complete!")
@@ -1938,3 +2118,178 @@ def embeddings_generate(
 
         console.print("\n[dim]Use vector search with:[/dim]")
         console.print("  [cyan]codexlens search 'your query' --mode pure-vector[/cyan]")
+
+
+# ==================== GPU Management Commands ====================
+
+@app.command(name="gpu-list")
+def gpu_list(
+    json_mode: bool = typer.Option(False, "--json", help="Output JSON response."),
+) -> None:
+    """List available GPU devices for embedding acceleration.
+
+    Shows all detected GPU devices with their capabilities and selection status.
+    Discrete GPUs (NVIDIA, AMD) are automatically preferred over integrated GPUs.
+
+    Examples:
+        codexlens gpu-list                    # List all GPUs
+        codexlens gpu-list --json             # JSON output for scripting
+    """
+    from codexlens.semantic.gpu_support import get_gpu_devices, detect_gpu, get_selected_device_id
+
+    gpu_info = detect_gpu()
+    devices = get_gpu_devices()
+    selected_id = get_selected_device_id()
+
+    if json_mode:
+        print_json(
+            success=True,
+            result={
+                "devices": devices,
+                "selected_device_id": selected_id,
+                "gpu_available": gpu_info.gpu_available,
+                "providers": gpu_info.onnx_providers,
+            }
+        )
+    else:
+        if not devices:
+            console.print("[yellow]No GPU devices detected[/yellow]")
+            console.print(f"ONNX Providers: [dim]{', '.join(gpu_info.onnx_providers)}[/dim]")
+            return
+
+        console.print("[bold]Available GPU Devices[/bold]\n")
+
+        table = Table(show_header=True, header_style="bold")
+        table.add_column("ID", justify="center")
+        table.add_column("Name")
+        table.add_column("Vendor", justify="center")
+        table.add_column("Type", justify="center")
+        table.add_column("Status", justify="center")
+
+        for dev in devices:
+            type_str = "[green]Discrete[/green]" if dev["is_discrete"] else "[dim]Integrated[/dim]"
+            vendor_color = {
+                "nvidia": "green",
+                "amd": "red",
+                "intel": "blue"
+            }.get(dev["vendor"], "white")
+            vendor_str = f"[{vendor_color}]{dev['vendor'].upper()}[/{vendor_color}]"
+
+            status_parts = []
+            if dev["is_preferred"]:
+                status_parts.append("[cyan]Auto[/cyan]")
+            if dev["is_selected"]:
+                status_parts.append("[green]✓ Selected[/green]")
+
+            status_str = " ".join(status_parts) if status_parts else "[dim]—[/dim]"
+
+            table.add_row(
+                str(dev["device_id"]),
+                dev["name"],
+                vendor_str,
+                type_str,
+                status_str,
+            )
+
+        console.print(table)
+        console.print(f"\nONNX Providers: [dim]{', '.join(gpu_info.onnx_providers)}[/dim]")
+        console.print("\n[dim]Select GPU with:[/dim]")
+        console.print("  [cyan]codexlens gpu-select <device_id>[/cyan]")
+
+
+@app.command(name="gpu-select")
+def gpu_select(
+    device_id: int = typer.Argument(
+        ...,
+        help="GPU device ID to use for embeddings. Use 'codexlens gpu-list' to see available IDs.",
+    ),
+    json_mode: bool = typer.Option(False, "--json", help="Output JSON response."),
+) -> None:
+    """Select a specific GPU device for embedding generation.
+
+    By default, CodexLens automatically selects the most powerful GPU (discrete over integrated).
+    Use this command to override the selection.
+
+    Examples:
+        codexlens gpu-select 1                # Use GPU device 1
+        codexlens gpu-select 0 --json         # Select GPU 0 with JSON output
+    """
+    from codexlens.semantic.gpu_support import set_selected_device_id, get_gpu_devices
+    from codexlens.semantic.embedder import clear_embedder_cache
+
+    devices = get_gpu_devices()
+    valid_ids = [dev["device_id"] for dev in devices]
+
+    if device_id not in valid_ids:
+        if json_mode:
+            print_json(success=False, error=f"Invalid device_id {device_id}. Valid IDs: {valid_ids}")
+        else:
+            console.print(f"[red]Error:[/red] Invalid device_id {device_id}")
+            console.print(f"Valid IDs: {valid_ids}")
+            console.print("\n[dim]Use 'codexlens gpu-list' to see available devices[/dim]")
+        raise typer.Exit(code=1)
+
+    success = set_selected_device_id(device_id)
+
+    if success:
+        # Clear embedder cache to force reload with new GPU
+        clear_embedder_cache()
+
+        device_name = next((dev["name"] for dev in devices if dev["device_id"] == device_id), "Unknown")
+
+        if json_mode:
+            print_json(
+                success=True,
+                result={
+                    "device_id": device_id,
+                    "device_name": device_name,
+                    "message": f"GPU selection set to device {device_id}: {device_name}",
+                }
+            )
+        else:
+            console.print(f"[green]✓[/green] GPU selection updated")
+            console.print(f"  Device ID: {device_id}")
+            console.print(f"  Device: [cyan]{device_name}[/cyan]")
+            console.print("\n[dim]New embeddings will use this GPU[/dim]")
+    else:
+        if json_mode:
+            print_json(success=False, error="Failed to set GPU selection")
+        else:
+            console.print("[red]Error:[/red] Failed to set GPU selection")
+        raise typer.Exit(code=1)
+
+
+@app.command(name="gpu-reset")
+def gpu_reset(
+    json_mode: bool = typer.Option(False, "--json", help="Output JSON response."),
+) -> None:
+    """Reset GPU selection to automatic detection.
+
+    Clears any manual GPU selection and returns to automatic selection
+    (discrete GPU preferred over integrated).
+
+    Examples:
+        codexlens gpu-reset                   # Reset to auto-detection
+    """
+    from codexlens.semantic.gpu_support import set_selected_device_id, detect_gpu
+    from codexlens.semantic.embedder import clear_embedder_cache
+
+    set_selected_device_id(None)
+    clear_embedder_cache()
+
+    gpu_info = detect_gpu(force_refresh=True)
+
+    if json_mode:
+        print_json(
+            success=True,
+            result={
+                "message": "GPU selection reset to auto-detection",
+                "preferred_device_id": gpu_info.preferred_device_id,
+                "preferred_device_name": gpu_info.gpu_name,
+            }
+        )
+    else:
+        console.print("[green]✓[/green] GPU selection reset to auto-detection")
+        if gpu_info.preferred_device_id is not None:
+            console.print(f"  Auto-selected device: {gpu_info.preferred_device_id}")
+            console.print(f"  Device: [cyan]{gpu_info.gpu_name}[/cyan]")