claude-code-workflow 6.3.54 → 7.0.0

package/codex-lens/src/codexlens/cli/embedding_manager.py

@@ -1,4 +1,42 @@
-"""Embedding Manager - Manage semantic embeddings for code indexes."""
+"""Embedding Manager - Manage semantic embeddings for code indexes.
+
+This module provides functions for generating and managing semantic embeddings
+for code indexes, supporting both fastembed and litellm backends.
+
+Example Usage:
+    Generate embeddings for a single index:
+
+    >>> from pathlib import Path
+    >>> from codexlens.cli.embedding_manager import generate_embeddings
+    >>> result = generate_embeddings(
+    ...     index_path=Path("path/to/_index.db"),
+    ...     force=True
+    ... )
+    >>> if result["success"]:
+    ...     print(f"Generated {result['total_chunks_created']} embeddings")
+
+    Generate embeddings for an entire project with centralized index:
+
+    >>> from codexlens.cli.embedding_manager import generate_dense_embeddings_centralized
+    >>> result = generate_dense_embeddings_centralized(
+    ...     index_root=Path("path/to/project"),
+    ...     force=True,
+    ...     progress_callback=lambda msg: print(msg)
+    ... )
+
+    Check if embeddings exist:
+
+    >>> from codexlens.cli.embedding_manager import check_index_embeddings
+    >>> status = check_index_embeddings(Path("path/to/_index.db"))
+    >>> print(status["result"]["has_embeddings"])
+
+Backward Compatibility:
+    The deprecated `discover_all_index_dbs()` function is maintained for compatibility.
+    `generate_embeddings_recursive()` is deprecated but functional; use
+    `generate_dense_embeddings_centralized()` instead.
+    The `EMBEDDING_BATCH_SIZE` constant is kept as a reference but actual batch size
+    is calculated dynamically via `calculate_dynamic_batch_size()`.
+"""
 
 import gc
 import json
@@ -53,11 +91,11 @@ def calculate_dynamic_batch_size(config, embedder) -> int:
     - Utilization factor (default 80% to leave headroom)
 
     Args:
-        config: Config object with api_batch_size_* settings
-        embedder: Embedding model object with max_tokens property
+        config: Config object with api_batch_size_* settings.
+        embedder: Embedding model object with max_tokens property.
 
     Returns:
-        Calculated batch size, clamped to [1, api_batch_size_max]
+        int: Calculated batch size, clamped to [1, api_batch_size_max].
     """
     # If dynamic calculation is disabled, return static value
     if not getattr(config, 'api_batch_size_dynamic', False):
@@ -147,17 +185,12 @@ def _cleanup_fastembed_resources() -> None:
     try:
         from codexlens.semantic.embedder import clear_embedder_cache
         clear_embedder_cache()
-    except Exception:
-        pass
-
-
-def _cleanup_splade_resources() -> None:
-    """Release SPLADE encoder ONNX resources."""
-    try:
-        from codexlens.semantic.splade_encoder import clear_splade_cache
-        clear_splade_cache()
-    except Exception:
+    except (ImportError, AttributeError):
+        # Expected when semantic module unavailable or cache function doesn't exist
         pass
+    except Exception as exc:
+        # Log unexpected errors but don't fail cleanup
+        logger.debug(f"Unexpected error during fastembed cleanup: {exc}")
 
 
 def _generate_chunks_from_cursor(
@@ -210,9 +243,18 @@ def _generate_chunks_from_cursor(
                     total_files += 1
                     for chunk in chunks:
                         yield (chunk, file_path)
+            except (OSError, UnicodeDecodeError) as e:
+                # File access or encoding errors
+                logger.error(f"Failed to read file {file_path}: {e}")
+                failed_files.append((file_path, f"File read error: {e}"))
+            except ValueError as e:
+                # Chunking configuration errors
+                logger.error(f"Chunking config error for {file_path}: {e}")
+                failed_files.append((file_path, f"Chunking error: {e}"))
             except Exception as e:
-                logger.error(f"Failed to chunk {file_path}: {e}")
-                failed_files.append((file_path, str(e)))
+                # Other unexpected errors
+                logger.error(f"Unexpected error processing {file_path}: {e}")
+                failed_files.append((file_path, f"Unexpected error: {e}"))
 
 
 def _create_token_aware_batches(
@@ -380,10 +422,155 @@ def _get_embedding_defaults() -> tuple[str, str, bool, List, str, float]:
             config.embedding_strategy,
             config.embedding_cooldown,
         )
-    except Exception:
+    except (ImportError, AttributeError, OSError, ValueError) as exc:
+        # Config not available or malformed - use defaults
+        logger.debug(f"Using default embedding config (config load failed): {exc}")
+        return "fastembed", "code", True, [], "latency_aware", 60.0
+    except Exception as exc:
+        # Unexpected error - still use defaults but log
+        logger.warning(f"Unexpected error loading embedding config: {exc}")
         return "fastembed", "code", True, [], "latency_aware", 60.0
 
 
+def _apply_embedding_config_defaults(
+    embedding_backend: Optional[str],
+    model_profile: Optional[str],
+    use_gpu: Optional[bool],
+    endpoints: Optional[List],
+    strategy: Optional[str],
+    cooldown: Optional[float],
+) -> tuple[str, str, bool, List, str, float]:
+    """Apply config defaults to embedding parameters.
+
+    This helper function reduces code duplication across embedding generation
+    functions by centralizing the default value application logic.
+
+    Args:
+        embedding_backend: Embedding backend (fastembed/litellm) or None for default
+        model_profile: Model profile/name or None for default
+        use_gpu: GPU flag or None for default
+        endpoints: API endpoints list or None for default
+        strategy: Selection strategy or None for default
+        cooldown: Cooldown seconds or None for default
+
+    Returns:
+        Tuple of (backend, model, use_gpu, endpoints, strategy, cooldown) with
+        defaults applied where None was passed.
+    """
+    (default_backend, default_model, default_gpu,
+     default_endpoints, default_strategy, default_cooldown) = _get_embedding_defaults()
+
+    backend = embedding_backend if embedding_backend is not None else default_backend
+    model = model_profile if model_profile is not None else default_model
+    gpu = use_gpu if use_gpu is not None else default_gpu
+    eps = endpoints if endpoints is not None else default_endpoints
+    strat = strategy if strategy is not None else default_strategy
+    cool = cooldown if cooldown is not None else default_cooldown
+
+    return backend, model, gpu, eps, strat, cool
+
+
+def _calculate_max_workers(
+    embedding_backend: str,
+    endpoints: Optional[List],
+    max_workers: Optional[int],
+) -> int:
+    """Calculate optimal max_workers based on backend and endpoint count.
+
+    Args:
+        embedding_backend: The embedding backend being used
+        endpoints: List of API endpoints (for litellm multi-endpoint mode)
+        max_workers: Explicitly specified max_workers or None for auto-calculation
+
+    Returns:
+        Calculated or specified max_workers value
+    """
+    if max_workers is not None:
+        return max_workers
+
+    endpoint_count = len(endpoints) if endpoints else 1
+
+    # Set dynamic max_workers default based on backend type and endpoint count
+    # - FastEmbed: CPU-bound, sequential is optimal (1 worker)
+    # - LiteLLM single endpoint: 4 workers default
+    # - LiteLLM multi-endpoint: workers = endpoint_count * 2 (to saturate all APIs)
+    if embedding_backend == "litellm":
+        if endpoint_count > 1:
+            return endpoint_count * 2  # No cap, scale with endpoints
+        else:
+            return 4
+    else:
+        return 1
+
+
+def _initialize_embedder_and_chunker(
+    embedding_backend: str,
+    model_profile: str,
+    use_gpu: bool,
+    endpoints: Optional[List],
+    strategy: str,
+    cooldown: float,
+    chunk_size: int,
+    overlap: int,
+) -> tuple:
+    """Initialize embedder and chunker for embedding generation.
+
+    This helper function reduces code duplication by centralizing embedder
+    and chunker initialization logic.
+
+    Args:
+        embedding_backend: The embedding backend (fastembed/litellm)
+        model_profile: Model profile or name
+        use_gpu: Whether to use GPU acceleration
+        endpoints: Optional API endpoints for load balancing
+        strategy: Selection strategy for multi-endpoint mode
+        cooldown: Cooldown seconds for rate-limited endpoints
+        chunk_size: Maximum chunk size in characters
+        overlap: Overlap size in characters
+
+    Returns:
+        Tuple of (embedder, chunker, endpoint_count)
+
+    Raises:
+        ValueError: If embedding_backend is invalid
+    """
+    from codexlens.semantic.factory import get_embedder as get_embedder_factory
+    from codexlens.semantic.chunker import Chunker, ChunkConfig
+    from codexlens.config import Config
+
+    # Initialize embedder using factory (supports fastembed, litellm, and rotational)
+    # For fastembed: model_profile is a profile name (fast/code/multilingual/balanced)
+    # For litellm: model_profile is a model name (e.g., qwen3-embedding)
+    # For multi-endpoint: endpoints list enables load balancing
+    if embedding_backend == "fastembed":
+        embedder = get_embedder_factory(backend="fastembed", profile=model_profile, use_gpu=use_gpu)
+    elif embedding_backend == "litellm":
+        embedder = get_embedder_factory(
+            backend="litellm",
+            model=model_profile,
+            endpoints=endpoints if endpoints else None,
+            strategy=strategy,
+            cooldown=cooldown,
+        )
+    else:
+        raise ValueError(f"Invalid embedding backend: {embedding_backend}. Must be 'fastembed' or 'litellm'.")
+
+    # skip_token_count=True: Use fast estimation (len/4) instead of expensive tiktoken
+    # This significantly reduces CPU usage with minimal impact on metadata accuracy
+    # Load chunk stripping config from settings
+    chunk_cfg = Config.load()
+    chunker = Chunker(config=ChunkConfig(
+        max_chunk_size=chunk_size,
+        overlap=overlap,
+        skip_token_count=True,
+        strip_comments=getattr(chunk_cfg, 'chunk_strip_comments', True),
+        strip_docstrings=getattr(chunk_cfg, 'chunk_strip_docstrings', True),
+    ))
+
+    endpoint_count = len(endpoints) if endpoints else 1
+    return embedder, chunker, endpoint_count
+
+
 def generate_embeddings(
     index_path: Path,
     embedding_backend: Optional[str] = None,
@@ -398,7 +585,6 @@ def generate_embeddings(
     endpoints: Optional[List] = None,
     strategy: Optional[str] = None,
     cooldown: Optional[float] = None,
-    splade_db_path: Optional[Path] = None,
 ) -> Dict[str, any]:
     """Generate embeddings for an index using memory-efficient batch processing.
 
@@ -407,16 +593,16 @@ def generate_embeddings(
     LiteLLM backend to improve throughput.
 
     Args:
-        index_path: Path to _index.db file
+        index_path: Path to _index.db file.
         embedding_backend: Embedding backend to use (fastembed or litellm).
                           Defaults to config setting.
         model_profile: Model profile for fastembed (fast, code, multilingual, balanced)
                       or model name for litellm (e.g., qwen3-embedding).
                       Defaults to config setting.
-        force: If True, regenerate even if embeddings exist
-        chunk_size: Maximum chunk size in characters
-        overlap: Overlap size in characters for sliding window chunking (default: 200)
-        progress_callback: Optional callback for progress updates
+        force: If True, regenerate even if embeddings exist.
+        chunk_size: Maximum chunk size in characters.
+        overlap: Overlap size in characters for sliding window chunking (default: 200).
+        progress_callback: Optional callback for progress updates.
         use_gpu: Whether to use GPU acceleration (fastembed only).
                 Defaults to config setting.
         max_tokens_per_batch: Maximum tokens per batch for token-aware batching.
@@ -428,45 +614,24 @@ def generate_embeddings(
                   Each dict has keys: model, api_key, api_base, weight.
         strategy: Selection strategy for multi-endpoint mode (round_robin, latency_aware).
         cooldown: Default cooldown seconds for rate-limited endpoints.
-        splade_db_path: Optional path to centralized SPLADE database. If None, SPLADE
-                       is written to index_path (legacy behavior). Use index_root / SPLADE_DB_NAME
-                       for centralized storage.
 
     Returns:
-        Result dictionary with generation statistics
-    """
-    # Get defaults from config if not specified
-    (default_backend, default_model, default_gpu,
-     default_endpoints, default_strategy, default_cooldown) = _get_embedding_defaults()
+        Dict[str, any]: Result dictionary with generation statistics.
+            Contains keys: success, error (if failed), files_processed,
+            total_chunks_created, execution_time, etc.
 
-    if embedding_backend is None:
-        embedding_backend = default_backend
-    if model_profile is None:
-        model_profile = default_model
-    if use_gpu is None:
-        use_gpu = default_gpu
-    if endpoints is None:
-        endpoints = default_endpoints
-    if strategy is None:
-        strategy = default_strategy
-    if cooldown is None:
-        cooldown = default_cooldown
-
-    # Calculate endpoint count for worker scaling
-    endpoint_count = len(endpoints) if endpoints else 1
+    Raises:
+        ValueError: If embedding_backend is invalid.
+        ImportError: If semantic module is not available.
+    """
+    # Apply config defaults
+    embedding_backend, model_profile, use_gpu, endpoints, strategy, cooldown = \
+        _apply_embedding_config_defaults(
+            embedding_backend, model_profile, use_gpu, endpoints, strategy, cooldown
+        )
 
-    # Set dynamic max_workers default based on backend type and endpoint count
-    # - FastEmbed: CPU-bound, sequential is optimal (1 worker)
-    # - LiteLLM single endpoint: 4 workers default
-    # - LiteLLM multi-endpoint: workers = endpoint_count * 2 (to saturate all APIs)
-    if max_workers is None:
-        if embedding_backend == "litellm":
-            if endpoint_count > 1:
-                max_workers = endpoint_count * 2  # No cap, scale with endpoints
-            else:
-                max_workers = 4
-        else:
-            max_workers = 1
+    # Calculate max_workers
+    max_workers = _calculate_max_workers(embedding_backend, endpoints, max_workers)
 
     backend_available, backend_error = is_embedding_backend_available(embedding_backend)
     if not backend_available:
@@ -500,51 +665,23 @@ def generate_embeddings(
             with sqlite3.connect(index_path) as conn:
                 conn.execute("DELETE FROM semantic_chunks")
                 conn.commit()
-        except Exception as e:
+        except sqlite3.DatabaseError as e:
             return {
                 "success": False,
-                "error": f"Failed to clear existing chunks: {str(e)}",
+                "error": f"Database error clearing chunks: {str(e)}",
             }
-
-    # Initialize components
-    try:
-        # Import factory function to support both backends
-        from codexlens.semantic.factory import get_embedder as get_embedder_factory
-        from codexlens.semantic.vector_store import VectorStore
-        from codexlens.semantic.chunker import Chunker, ChunkConfig
-
-        # Initialize embedder using factory (supports fastembed, litellm, and rotational)
-        # For fastembed: model_profile is a profile name (fast/code/multilingual/balanced)
-        # For litellm: model_profile is a model name (e.g., qwen3-embedding)
-        # For multi-endpoint: endpoints list enables load balancing
-        if embedding_backend == "fastembed":
-            embedder = get_embedder_factory(backend="fastembed", profile=model_profile, use_gpu=use_gpu)
-        elif embedding_backend == "litellm":
-            embedder = get_embedder_factory(
-                backend="litellm",
-                model=model_profile,
-                endpoints=endpoints if endpoints else None,
-                strategy=strategy,
-                cooldown=cooldown,
-            )
-        else:
+        except Exception as e:
             return {
                 "success": False,
-                "error": f"Invalid embedding backend: {embedding_backend}. Must be 'fastembed' or 'litellm'.",
+                "error": f"Failed to clear existing chunks: {str(e)}",
             }
 
-        # skip_token_count=True: Use fast estimation (len/4) instead of expensive tiktoken
-        # This significantly reduces CPU usage with minimal impact on metadata accuracy
-        # Load chunk stripping config from settings
-        from codexlens.config import Config
-        chunk_cfg = Config.load()
-        chunker = Chunker(config=ChunkConfig(
-            max_chunk_size=chunk_size,
-            overlap=overlap,
-            skip_token_count=True,
-            strip_comments=getattr(chunk_cfg, 'chunk_strip_comments', True),
-            strip_docstrings=getattr(chunk_cfg, 'chunk_strip_docstrings', True),
-        ))
+    # Initialize embedder and chunker using helper
+    try:
+        embedder, chunker, endpoint_count = _initialize_embedder_and_chunker(
+            embedding_backend, model_profile, use_gpu, endpoints, strategy, cooldown,
+            chunk_size, overlap
+        )
 
         # Log embedder info with endpoint count for multi-endpoint mode
         if progress_callback:
@@ -560,10 +697,17 @@ def generate_embeddings(
         if progress_callback and batch_config.api_batch_size_dynamic:
             progress_callback(f"Dynamic batch size: {effective_batch_size} (model max_tokens={getattr(embedder, 'max_tokens', 8192)})")
 
+    except (ImportError, ValueError) as e:
+        # Missing dependency or invalid configuration
+        return {
+            "success": False,
+            "error": f"Failed to initialize embedding components: {str(e)}",
+        }
     except Exception as e:
+        # Other unexpected errors
         return {
             "success": False,
-            "error": f"Failed to initialize components: {str(e)}",
+            "error": f"Unexpected error initializing components: {str(e)}",
         }
 
     # --- STREAMING PROCESSING ---
@@ -822,100 +966,13 @@ def generate_embeddings(
                 if progress_callback:
                     progress_callback(f"Finalizing index... Building ANN index for {total_chunks_created} chunks")
 
-            # --- SPLADE SPARSE ENCODING (after dense embeddings) ---
-            # Add SPLADE encoding if enabled in config
-            splade_success = False
-            splade_error = None
-
-            try:
-                from codexlens.config import Config, SPLADE_DB_NAME
-                config = Config.load()
-
-                if config.enable_splade:
-                    from codexlens.semantic.splade_encoder import check_splade_available, get_splade_encoder
-                    from codexlens.storage.splade_index import SpladeIndex
-
-                    ok, err = check_splade_available()
-                    if ok:
-                        if progress_callback:
-                            progress_callback(f"Generating SPLADE sparse vectors for {total_chunks_created} chunks...")
-
-                        # Initialize SPLADE encoder and index
-                        splade_encoder = get_splade_encoder(use_gpu=use_gpu)
-                        # Use centralized SPLADE database if provided, otherwise fallback to index_path
-                        effective_splade_path = splade_db_path if splade_db_path else index_path
-                        splade_index = SpladeIndex(effective_splade_path)
-                        splade_index.create_tables()
-
-                        # Retrieve all chunks from database for SPLADE encoding
-                        with sqlite3.connect(index_path) as conn:
-                            conn.row_factory = sqlite3.Row
-                            cursor = conn.execute("SELECT id, content FROM semantic_chunks ORDER BY id")
-
-                            # Batch encode for efficiency
-                            SPLADE_BATCH_SIZE = 32
-                            batch_postings = []
-                            chunk_batch = []
-                            chunk_ids = []
-
-                            for row in cursor:
-                                chunk_id = row["id"]
-                                content = row["content"]
-
-                                chunk_ids.append(chunk_id)
-                                chunk_batch.append(content)
-
-                                # Process batch when full
-                                if len(chunk_batch) >= SPLADE_BATCH_SIZE:
-                                    sparse_vecs = splade_encoder.encode_batch(chunk_batch, batch_size=SPLADE_BATCH_SIZE)
-                                    for cid, sparse_vec in zip(chunk_ids, sparse_vecs):
-                                        batch_postings.append((cid, sparse_vec))
-
-                                    chunk_batch = []
-                                    chunk_ids = []
-
-                            # Process remaining chunks
-                            if chunk_batch:
-                                sparse_vecs = splade_encoder.encode_batch(chunk_batch, batch_size=SPLADE_BATCH_SIZE)
-                                for cid, sparse_vec in zip(chunk_ids, sparse_vecs):
-                                    batch_postings.append((cid, sparse_vec))
-
-                            # Batch insert all postings
-                            if batch_postings:
-                                splade_index.add_postings_batch(batch_postings)
-
-                                # Set metadata
-                                splade_index.set_metadata(
-                                    model_name=splade_encoder.model_name,
-                                    vocab_size=splade_encoder.vocab_size
-                                )
-
-                                splade_success = True
-                                if progress_callback:
-                                    stats = splade_index.get_stats()
-                                    progress_callback(
-                                        f"SPLADE index created: {stats['total_postings']} postings, "
-                                        f"{stats['unique_tokens']} unique tokens"
-                                    )
-                    else:
-                        logger.debug("SPLADE not available: %s", err)
-                        splade_error = f"SPLADE not available: {err}"
-            except Exception as e:
-                splade_error = str(e)
-                logger.warning("SPLADE encoding failed: %s", e)
-
-            # Report SPLADE status after processing
-            if progress_callback and not splade_success and splade_error:
-                progress_callback(f"SPLADE index: FAILED - {splade_error}")
-
     except Exception as e:
         # Cleanup on error to prevent process hanging
         try:
             _cleanup_fastembed_resources()
-            _cleanup_splade_resources()
             gc.collect()
-        except Exception:
-            pass
+        except Exception as cleanup_exc:
+            logger.debug(f"Cleanup error during exception handling: {cleanup_exc}")
         return {"success": False, "error": f"Failed to read or process files: {str(e)}"}
 
     elapsed_time = time.time() - start_time
@@ -924,10 +981,9 @@ def generate_embeddings(
     # This is critical - without it, ONNX Runtime threads prevent Python from exiting
     try:
         _cleanup_fastembed_resources()
-        _cleanup_splade_resources()
         gc.collect()
-    except Exception:
-        pass
+    except Exception as cleanup_exc:
+        logger.debug(f"Cleanup error during finalization: {cleanup_exc}")
 
     return {
         "success": True,
@@ -961,6 +1017,296 @@ def _discover_index_dbs_internal(index_root: Path) -> List[Path]:
     return sorted(index_root.rglob("_index.db"))
 
 
+def build_centralized_binary_vectors_from_existing(
+    index_root: Path,
+    *,
+    force: bool = False,
+    embedding_dim: Optional[int] = None,
+    progress_callback: Optional[callable] = None,
+) -> Dict[str, Any]:
+    """Build centralized binary vectors + metadata from existing semantic_chunks embeddings.
+
+    This is a fast-path for enabling the staged binary coarse search without
+    regenerating embeddings (and without triggering global model locks).
+
+    It scans all distributed `_index.db` files under `index_root`, reads
+    existing `semantic_chunks.embedding` blobs, assigns new global chunk_ids,
+    and writes:
+      - `<index_root>/_binary_vectors.mmap` (+ `.meta.json`)
+      - `<index_root>/_vectors_meta.db` (chunk_metadata + binary_vectors)
+    """
+    from codexlens.config import BINARY_VECTORS_MMAP_NAME, VECTORS_META_DB_NAME
+    from codexlens.storage.vector_meta_store import VectorMetadataStore
+
+    index_root = Path(index_root).resolve()
+    vectors_meta_path = index_root / VECTORS_META_DB_NAME
+    mmap_path = index_root / BINARY_VECTORS_MMAP_NAME
+    meta_path = mmap_path.with_suffix(".meta.json")
+
+    index_files = _discover_index_dbs_internal(index_root)
+    if not index_files:
+        return {"success": False, "error": f"No _index.db files found under {index_root}"}
+
+    if progress_callback:
+        progress_callback(f"Scanning {len(index_files)} index databases for existing embeddings...")
+
+    # First pass: detect embedding dims present.
+    dims_seen: Dict[int, int] = {}
+    selected_config: Optional[Dict[str, Any]] = None
+
+    for index_path in index_files:
+        try:
+            with sqlite3.connect(index_path) as conn:
+                conn.row_factory = sqlite3.Row
+                has_table = conn.execute(
+                    "SELECT 1 FROM sqlite_master WHERE type='table' AND name='semantic_chunks'"
+                ).fetchone()
+                if not has_table:
+                    continue
+
+                dim_row = conn.execute(
+                    "SELECT backend, model_profile, model_name, embedding_dim FROM embeddings_config WHERE id=1"
+                ).fetchone()
+                if dim_row and dim_row[3]:
+                    dim_val = int(dim_row[3])
+                    dims_seen[dim_val] = dims_seen.get(dim_val, 0) + 1
+                    if selected_config is None:
+                        selected_config = {
+                            "backend": dim_row[0],
+                            "model_profile": dim_row[1],
+                            "model_name": dim_row[2],
+                            "embedding_dim": dim_val,
+                        }
+
+                # We count per-dim later after selecting a target dim.
+        except (sqlite3.DatabaseError, ValueError, TypeError):
+            # Skip corrupted or malformed indexes
+            continue
+
+    if not dims_seen:
+        return {"success": False, "error": "No embeddings_config found under index_root"}
+
+    if embedding_dim is None:
+        # Default: pick the most common embedding dim across indexes.
+        embedding_dim = max(dims_seen.items(), key=lambda kv: kv[1])[0]
+
+    embedding_dim = int(embedding_dim)
+
+    if progress_callback and len(dims_seen) > 1:
+        progress_callback(f"Mixed embedding dims detected, selecting dim={embedding_dim} (seen={dims_seen})")
+
+    # Re-detect the selected model config for this dim (do not reuse an arbitrary first-seen config).
+    selected_config = None
+
+    # Second pass: count only chunks matching selected dim.
+    total_chunks = 0
+    for index_path in index_files:
+        try:
+            with sqlite3.connect(index_path) as conn:
+                conn.row_factory = sqlite3.Row
+                has_table = conn.execute(
+                    "SELECT 1 FROM sqlite_master WHERE type='table' AND name='semantic_chunks'"
+                ).fetchone()
+                if not has_table:
+                    continue
+
+                dim_row = conn.execute(
+                    "SELECT backend, model_profile, model_name, embedding_dim FROM embeddings_config WHERE id=1"
+                ).fetchone()
+                dim_val = int(dim_row[3]) if dim_row and dim_row[3] else None
+                if dim_val != embedding_dim:
+                    continue
+
+                if selected_config is None:
+                    selected_config = {
+                        "backend": dim_row[0],
+                        "model_profile": dim_row[1],
+                        "model_name": dim_row[2],
+                        "embedding_dim": dim_val,
+                    }
+
+                row = conn.execute(
+                    "SELECT COUNT(*) FROM semantic_chunks WHERE embedding IS NOT NULL AND length(embedding) > 0"
+                ).fetchone()
+                total_chunks += int(row[0] if row else 0)
+        except (sqlite3.DatabaseError, ValueError, TypeError):
+            # Skip corrupted or malformed indexes
+            continue
+
+    if not total_chunks:
+        return {
+            "success": False,
+            "error": f"No existing embeddings found for embedding_dim={embedding_dim}",
+            "dims_seen": dims_seen,
+        }
+
+    if progress_callback:
+        progress_callback(f"Found {total_chunks} embedded chunks (dim={embedding_dim}). Building binary vectors...")
+
+    # Prepare output files / DB.
+    try:
+        import numpy as np
+    except ImportError as exc:
+        return {"success": False, "error": f"numpy required to build binary vectors: {exc}"}
+
+    store = VectorMetadataStore(vectors_meta_path)
+    store._ensure_schema()
+
+    if force:
+        try:
+            store.clear()
+        except Exception:
+            pass
+        try:
+            store.clear_binary_vectors()
+        except Exception:
+            pass
+        try:
+            if mmap_path.exists():
+                mmap_path.unlink()
+        except Exception:
+            pass
+        try:
+            if meta_path.exists():
+                meta_path.unlink()
+        except Exception:
+            pass
+
+    bytes_per_vec = (int(embedding_dim) + 7) // 8
+    mmap = np.memmap(
+        str(mmap_path),
+        dtype=np.uint8,
+        mode="w+",
+        shape=(int(total_chunks), int(bytes_per_vec)),
+    )
+
+    chunk_ids: List[int] = []
+    chunks_batch: List[Dict[str, Any]] = []
+    bin_ids_batch: List[int] = []
+    bin_vecs_batch: List[bytes] = []
+    batch_limit = 500
+
+    global_id = 1
+    write_idx = 0
+
+    skipped_indexes: Dict[str, int] = {}
+    for index_path in index_files:
+        try:
+            with sqlite3.connect(index_path) as conn:
+                conn.row_factory = sqlite3.Row
+                has_table = conn.execute(
+                    "SELECT 1 FROM sqlite_master WHERE type='table' AND name='semantic_chunks'"
+                ).fetchone()
+                if not has_table:
+                    continue
+
+                dim_row = conn.execute(
+                    "SELECT embedding_dim FROM embeddings_config WHERE id=1"
+                ).fetchone()
+                dim_val = int(dim_row[0]) if dim_row and dim_row[0] else None
+                if dim_val != embedding_dim:
+                    skipped_indexes[str(index_path)] = dim_val or -1
+                    continue
+
+                rows = conn.execute(
+                    "SELECT file_path, content, embedding, metadata, category FROM semantic_chunks "
+                    "WHERE embedding IS NOT NULL AND length(embedding) > 0"
+                ).fetchall()
+
+                for row in rows:
+                    emb = np.frombuffer(row["embedding"], dtype=np.float32)
+                    if emb.size != int(embedding_dim):
+                        continue
+
+                    packed = np.packbits((emb > 0).astype(np.uint8))
+                    if packed.size != bytes_per_vec:
+                        continue
+
+                    mmap[write_idx] = packed
+                    write_idx += 1
+
+                    cid = global_id
+                    global_id += 1
+                    chunk_ids.append(cid)
+
+                    meta_raw = row["metadata"]
+                    meta_dict: Dict[str, Any] = {}
+                    if meta_raw:
+                        try:
+                            meta_dict = json.loads(meta_raw) if isinstance(meta_raw, str) else dict(meta_raw)
+                        except Exception:
+                            meta_dict = {}
+
+                    chunks_batch.append(
+                        {
+                            "chunk_id": cid,
+                            "file_path": row["file_path"],
+                            "content": row["content"],
+                            "start_line": meta_dict.get("start_line"),
+                            "end_line": meta_dict.get("end_line"),
+                            "category": row["category"],
+                            "metadata": meta_dict,
+                            "source_index_db": str(index_path),
+                        }
+                    )
+
+                    bin_ids_batch.append(cid)
+                    bin_vecs_batch.append(packed.tobytes())
+
+                    if len(chunks_batch) >= batch_limit:
+                        store.add_chunks(chunks_batch)
+                        store.add_binary_vectors(bin_ids_batch, bin_vecs_batch)
+                        chunks_batch = []
+                        bin_ids_batch = []
+                        bin_vecs_batch = []
+
+        except Exception:
+            continue
+
+    if chunks_batch:
+        store.add_chunks(chunks_batch)
+        store.add_binary_vectors(bin_ids_batch, bin_vecs_batch)
+
+    mmap.flush()
+    del mmap
+
+    # If we skipped inconsistent vectors, truncate metadata to actual write count.
+    chunk_ids = chunk_ids[:write_idx]
+
+    # Write sidecar metadata.
+    with open(meta_path, "w", encoding="utf-8") as f:
+        json.dump(
+            {
+                "shape": [int(write_idx), int(bytes_per_vec)],
+                "chunk_ids": chunk_ids,
+                "embedding_dim": int(embedding_dim),
+                "backend": (selected_config or {}).get("backend"),
+                "model_profile": (selected_config or {}).get("model_profile"),
+                "model_name": (selected_config or {}).get("model_name"),
+            },
+            f,
+        )
+
+    if progress_callback:
+        progress_callback(f"Binary vectors ready: {mmap_path} (rows={write_idx})")
+
+    return {
+        "success": True,
+        "result": {
+            "index_root": str(index_root),
+            "index_files_scanned": len(index_files),
+            "chunks_total": int(total_chunks),
+            "chunks_written": int(write_idx),
+            "embedding_dim": int(embedding_dim),
+            "bytes_per_vector": int(bytes_per_vec),
+            "skipped_indexes": len(skipped_indexes),
+            "vectors_meta_db": str(vectors_meta_path),
+            "binary_mmap": str(mmap_path),
+            "binary_meta_json": str(meta_path),
+        },
+    }
+
+
 def discover_all_index_dbs(index_root: Path) -> List[Path]:
     """Recursively find all _index.db files in an index tree.
 
@@ -1056,35 +1402,14 @@ def generate_embeddings_recursive(
         stacklevel=2
     )
 
-    # Get defaults from config if not specified
-    (default_backend, default_model, default_gpu,
-     default_endpoints, default_strategy, default_cooldown) = _get_embedding_defaults()
-
-    if embedding_backend is None:
-        embedding_backend = default_backend
-    if model_profile is None:
-        model_profile = default_model
-    if use_gpu is None:
-        use_gpu = default_gpu
-    if endpoints is None:
-        endpoints = default_endpoints
-    if strategy is None:
-        strategy = default_strategy
-    if cooldown is None:
-        cooldown = default_cooldown
-
-    # Calculate endpoint count for worker scaling
-    endpoint_count = len(endpoints) if endpoints else 1
+    # Apply config defaults
+    embedding_backend, model_profile, use_gpu, endpoints, strategy, cooldown = \
+        _apply_embedding_config_defaults(
+            embedding_backend, model_profile, use_gpu, endpoints, strategy, cooldown
+        )
 
-    # Set dynamic max_workers default based on backend type and endpoint count
-    if max_workers is None:
-        if embedding_backend == "litellm":
-            if endpoint_count > 1:
-                max_workers = endpoint_count * 2  # No cap, scale with endpoints
-            else:
-                max_workers = 4
-        else:
-            max_workers = 1
+    # Calculate max_workers
+    max_workers = _calculate_max_workers(embedding_backend, endpoints, max_workers)
 
     # Discover all _index.db files (using internal helper to avoid double deprecation warning)
     index_files = _discover_index_dbs_internal(index_root)
@@ -1098,10 +1423,6 @@ def generate_embeddings_recursive(
     if progress_callback:
         progress_callback(f"Found {len(index_files)} index databases to process")
 
-    # Calculate centralized SPLADE database path
-    from codexlens.config import SPLADE_DB_NAME
-    splade_db_path = index_root / SPLADE_DB_NAME
-
     # Process each index database
     all_results = []
     total_chunks = 0
@@ -1131,7 +1452,6 @@ def generate_embeddings_recursive(
             endpoints=endpoints,
             strategy=strategy,
             cooldown=cooldown,
-            splade_db_path=splade_db_path,  # Use centralized SPLADE storage
         )
 
         all_results.append({
@@ -1153,7 +1473,6 @@ def generate_embeddings_recursive(
     # Each generate_embeddings() call does its own cleanup, but do a final one to be safe
     try:
         _cleanup_fastembed_resources()
-        _cleanup_splade_resources()
         gc.collect()
     except Exception:
         pass
@@ -1197,7 +1516,6 @@ def generate_dense_embeddings_centralized(
     Target architecture:
         <index_root>/
         |-- _vectors.hnsw         # Centralized dense vector ANN index
-        |-- _splade.db            # Centralized sparse vector index
         |-- src/
             |-- _index.db         # No longer contains .hnsw file
 
@@ -1219,36 +1537,16 @@ def generate_dense_embeddings_centralized(
     Returns:
         Result dictionary with generation statistics
     """
-    from codexlens.config import VECTORS_HNSW_NAME, SPLADE_DB_NAME
+    from codexlens.config import VECTORS_HNSW_NAME
 
-    # Get defaults from config if not specified
-    (default_backend, default_model, default_gpu,
-     default_endpoints, default_strategy, default_cooldown) = _get_embedding_defaults()
+    # Apply config defaults
+    embedding_backend, model_profile, use_gpu, endpoints, strategy, cooldown = \
+        _apply_embedding_config_defaults(
+            embedding_backend, model_profile, use_gpu, endpoints, strategy, cooldown
+        )
 
-    if embedding_backend is None:
-        embedding_backend = default_backend
-    if model_profile is None:
-        model_profile = default_model
-    if use_gpu is None:
-        use_gpu = default_gpu
-    if endpoints is None:
-        endpoints = default_endpoints
-    if strategy is None:
-        strategy = default_strategy
-    if cooldown is None:
-        cooldown = default_cooldown
-
-    # Calculate endpoint count for worker scaling
-    endpoint_count = len(endpoints) if endpoints else 1
-
-    if max_workers is None:
-        if embedding_backend == "litellm":
-            if endpoint_count > 1:
-                max_workers = endpoint_count * 2
-            else:
-                max_workers = 4
-        else:
-            max_workers = 1
+    # Calculate max_workers
+    max_workers = _calculate_max_workers(embedding_backend, endpoints, max_workers)
 
     backend_available, backend_error = is_embedding_backend_available(embedding_backend)
     if not backend_available:
@@ -1290,38 +1588,18 @@ def generate_dense_embeddings_centralized(
             "error": f"Centralized vector index already exists at {central_hnsw_path}. Use --force to regenerate.",
         }
 
-    # Initialize embedder
+    # Initialize embedder and chunker using helper
     try:
-        from codexlens.semantic.factory import get_embedder as get_embedder_factory
-        from codexlens.semantic.chunker import Chunker, ChunkConfig
         from codexlens.semantic.ann_index import ANNIndex
 
-        if embedding_backend == "fastembed":
-            embedder = get_embedder_factory(backend="fastembed", profile=model_profile, use_gpu=use_gpu)
-        elif embedding_backend == "litellm":
-            embedder = get_embedder_factory(
-                backend="litellm",
-                model=model_profile,
-                endpoints=endpoints if endpoints else None,
-                strategy=strategy,
-                cooldown=cooldown,
-            )
-        else:
-            return {
-                "success": False,
-                "error": f"Invalid embedding backend: {embedding_backend}",
-            }
+        embedder, chunker, endpoint_count = _initialize_embedder_and_chunker(
+            embedding_backend, model_profile, use_gpu, endpoints, strategy, cooldown,
+            chunk_size, overlap
+        )
 
-        # Load chunk stripping config from settings
+        # Load chunk stripping config for batch size calculation
         from codexlens.config import Config
-        chunk_cfg = Config.load()
-        chunker = Chunker(config=ChunkConfig(
-            max_chunk_size=chunk_size,
-            overlap=overlap,
-            skip_token_count=True,
-            strip_comments=getattr(chunk_cfg, 'chunk_strip_comments', True),
-            strip_docstrings=getattr(chunk_cfg, 'chunk_strip_docstrings', True),
-        ))
+        batch_config = Config.load()
 
         if progress_callback:
             if endpoint_count > 1:
@@ -1329,7 +1607,6 @@ def generate_dense_embeddings_centralized(
             progress_callback(f"Using model: {embedder.model_name} ({embedder.embedding_dim} dimensions)")
 
         # Calculate dynamic batch size based on model capacity
-        batch_config = chunk_cfg  # Reuse already loaded config
         effective_batch_size = calculate_dynamic_batch_size(batch_config, embedder)
 
         if progress_callback and batch_config.api_batch_size_dynamic:
@@ -1543,90 +1820,6 @@ def generate_dense_embeddings_centralized(
         logger.warning("Binary vector generation failed: %s", e)
         # Non-fatal: continue without binary vectors
 
-    # --- SPLADE Sparse Index Generation (Centralized) ---
-    splade_success = False
-    splade_chunks_count = 0
-    try:
-        from codexlens.config import Config
-        config = Config.load()
-
-        if config.enable_splade and chunk_id_to_info:
-            from codexlens.semantic.splade_encoder import check_splade_available, get_splade_encoder
-            from codexlens.storage.splade_index import SpladeIndex
-            import json
-
-            ok, err = check_splade_available()
-            if ok:
-                if progress_callback:
-                    progress_callback(f"Generating SPLADE sparse vectors for {len(chunk_id_to_info)} chunks...")
-
-                # Initialize SPLADE encoder and index
-                splade_encoder = get_splade_encoder(use_gpu=use_gpu)
-                splade_db_path = index_root / SPLADE_DB_NAME
-                splade_index = SpladeIndex(splade_db_path)
-                splade_index.create_tables()
-
-                # Batch encode for efficiency
-                SPLADE_BATCH_SIZE = 32
-                all_postings = []
-                all_chunk_metadata = []
-
-                # Create batches from chunk_id_to_info
-                chunk_items = list(chunk_id_to_info.items())
-
-                for i in range(0, len(chunk_items), SPLADE_BATCH_SIZE):
-                    batch_items = chunk_items[i:i + SPLADE_BATCH_SIZE]
-                    chunk_ids = [item[0] for item in batch_items]
-                    chunk_contents = [item[1]["content"] for item in batch_items]
-
-                    # Generate sparse vectors
-                    sparse_vecs = splade_encoder.encode_batch(chunk_contents, batch_size=SPLADE_BATCH_SIZE)
-                    for cid, sparse_vec in zip(chunk_ids, sparse_vecs):
-                        all_postings.append((cid, sparse_vec))
-
-                    if progress_callback and (i + SPLADE_BATCH_SIZE) % 100 == 0:
-                        progress_callback(f"SPLADE encoding: {min(i + SPLADE_BATCH_SIZE, len(chunk_items))}/{len(chunk_items)}")
-
-                # Batch insert all postings
-                if all_postings:
-                    splade_index.add_postings_batch(all_postings)
-
-                # CRITICAL FIX: Populate splade_chunks table
-                for cid, info in chunk_id_to_info.items():
-                    metadata_str = json.dumps(info.get("metadata", {})) if info.get("metadata") else None
-                    all_chunk_metadata.append((
-                        cid,
-                        info["file_path"],
-                        info["content"],
-                        metadata_str,
-                        info.get("source_index_db")
-                    ))
-
-                if all_chunk_metadata:
-                    splade_index.add_chunks_metadata_batch(all_chunk_metadata)
-                    splade_chunks_count = len(all_chunk_metadata)
-
-                # Set metadata
-                splade_index.set_metadata(
-                    model_name=splade_encoder.model_name,
-                    vocab_size=splade_encoder.vocab_size
-                )
-
-                splade_index.close()
-                splade_success = True
-
-                if progress_callback:
-                    progress_callback(f"SPLADE index created: {len(all_postings)} postings, {splade_chunks_count} chunks")
-
-            else:
-                if progress_callback:
-                    progress_callback(f"SPLADE not available, skipping sparse index: {err}")
-
-    except Exception as e:
-        logger.warning("SPLADE encoding failed: %s", e)
-        if progress_callback:
-            progress_callback(f"SPLADE encoding failed: {e}")
-
     elapsed_time = time.time() - start_time
 
     # Cleanup
@@ -1647,8 +1840,6 @@ def generate_dense_embeddings_centralized(
             "model_name": embedder.model_name,
             "central_index_path": str(central_hnsw_path),
             "failed_files": failed_files[:5],
-            "splade_success": splade_success,
-            "splade_chunks": splade_chunks_count,
             "binary_success": binary_success,
             "binary_count": binary_count,
         },
@@ -1998,4 +2189,4 @@ def check_global_model_lock(
         "has_conflict": has_conflict,
         "locked_config": locked_config,
         "target_config": {"backend": target_backend, "model": target_model},
-    }
+    }