claude-code-workflow 6.3.18 → 6.3.20

package/codex-lens/src/codexlens/cli/embedding_manager.py

@@ -17,6 +17,25 @@ except ImportError:
     def is_embedding_backend_available(_backend: str):  # type: ignore[no-redef]
         return False, "codexlens.semantic not available"
 
+try:
+    from codexlens.config import VECTORS_META_DB_NAME
+except ImportError:
+    VECTORS_META_DB_NAME = "_vectors_meta.db"
+
+try:
+    from codexlens.search.ranking import get_file_category
+except ImportError:
+    def get_file_category(path: str):  # type: ignore[no-redef]
+        """Fallback: map common extensions to category."""
+        ext = Path(path).suffix.lower()
+        code_exts = {".py", ".js", ".jsx", ".ts", ".tsx", ".java", ".go", ".c", ".cpp", ".rs"}
+        doc_exts = {".md", ".mdx", ".txt", ".rst"}
+        if ext in code_exts:
+            return "code"
+        elif ext in doc_exts:
+            return "doc"
+        return None
+
 logger = logging.getLogger(__name__)
 
 # Embedding batch size - larger values improve throughput on modern hardware
@@ -24,6 +43,105 @@ logger = logging.getLogger(__name__)
 EMBEDDING_BATCH_SIZE = 256
 
 
+def calculate_dynamic_batch_size(config, embedder) -> int:
+    """Calculate batch size dynamically based on model token capacity.
+
+    This function computes an optimal batch size by considering:
+    - Maximum chunk character size from parsing rules
+    - Estimated tokens per chunk (chars / chars_per_token_estimate)
+    - Model's maximum token capacity
+    - Utilization factor (default 80% to leave headroom)
+
+    Args:
+        config: Config object with api_batch_size_* settings
+        embedder: Embedding model object with max_tokens property
+
+    Returns:
+        Calculated batch size, clamped to [1, api_batch_size_max]
+    """
+    # If dynamic calculation is disabled, return static value
+    if not getattr(config, 'api_batch_size_dynamic', False):
+        return getattr(config, 'api_batch_size', 8)
+
+    # Get maximum chunk character size from ALL parsing rules (not just default)
+    # This ensures we use the worst-case chunk size across all languages
+    parsing_rules = getattr(config, 'parsing_rules', {})
+    all_max_chunk_chars = [
+        rule.get('max_chunk_chars', 0)
+        for rule in parsing_rules.values()
+        if isinstance(rule, dict)
+    ]
+    max_chunk_chars = max(all_max_chunk_chars) if all_max_chunk_chars else 4000
+    if max_chunk_chars <= 0:
+        max_chunk_chars = 4000  # Final fallback
+
+    # Get characters per token estimate
+    chars_per_token = getattr(config, 'chars_per_token_estimate', 4)
+    if chars_per_token <= 0:
+        chars_per_token = 4  # Safe default
+
+    # Estimate tokens per chunk
+    estimated_tokens_per_chunk = max_chunk_chars / chars_per_token
+
+    # Prevent division by zero
+    if estimated_tokens_per_chunk <= 0:
+        return getattr(config, 'api_batch_size', 8)
+
+    # Get model's maximum token capacity
+    model_max_tokens = getattr(embedder, 'max_tokens', 8192)
+
+    # Get utilization factor (default 80%, max 95% to leave safety margin)
+    utilization_factor = getattr(config, 'api_batch_size_utilization_factor', 0.8)
+    if utilization_factor <= 0 or utilization_factor > 0.95:
+        if utilization_factor > 0.95:
+            logger.warning(
+                "Utilization factor %.2f exceeds safe limit 0.95. "
+                "Token estimation is approximate, high values risk API errors. "
+                "Clamping to 0.95.",
+                utilization_factor
+            )
+            utilization_factor = 0.95
+        else:
+            utilization_factor = 0.8
+
+    # Calculate safe token limit
+    safe_token_limit = model_max_tokens * utilization_factor
+
+    # Calculate dynamic batch size
+    dynamic_batch_size = int(safe_token_limit / estimated_tokens_per_chunk)
+
+    # Get maximum batch size limit
+    batch_size_max = getattr(config, 'api_batch_size_max', 2048)
+
+    # Clamp to [1, batch_size_max]
+    result = max(1, min(dynamic_batch_size, batch_size_max))
+
+    logger.debug(
+        "Dynamic batch size calculated: %d (max_chunk_chars=%d, chars_per_token=%d, "
+        "model_max_tokens=%d, utilization=%.1f%%, limit=%d)",
+        result, max_chunk_chars, chars_per_token, model_max_tokens,
+        utilization_factor * 100, batch_size_max
+    )
+
+    return result
+
+
+def _build_categories_from_batch(chunk_batch: List[Tuple[Any, str]]) -> List[str]:
+    """Build categories list from chunk batch for index-level category filtering.
+
+    Args:
+        chunk_batch: List of (chunk, file_path) tuples
+
+    Returns:
+        List of category strings ('code' or 'doc'), defaulting to 'code' for unknown
+    """
+    categories = []
+    for _, file_path in chunk_batch:
+        cat = get_file_category(file_path)
+        categories.append(cat if cat else "code")  # Default to 'code' for unknown extensions
+    return categories
+
+
 def _cleanup_fastembed_resources() -> None:
     """Best-effort cleanup for fastembed/ONNX resources (no-op for other backends)."""
     try:
@@ -33,6 +151,15 @@ def _cleanup_fastembed_resources() -> None:
         pass
 
 
+def _cleanup_splade_resources() -> None:
+    """Release SPLADE encoder ONNX resources."""
+    try:
+        from codexlens.semantic.splade_encoder import clear_splade_cache
+        clear_splade_cache()
+    except Exception:
+        pass
+
+
 def _generate_chunks_from_cursor(
     cursor,
     chunker,
@@ -271,6 +398,7 @@ def generate_embeddings(
     endpoints: Optional[List] = None,
     strategy: Optional[str] = None,
     cooldown: Optional[float] = None,
+    splade_db_path: Optional[Path] = None,
 ) -> Dict[str, any]:
     """Generate embeddings for an index using memory-efficient batch processing.
 
@@ -300,6 +428,9 @@ def generate_embeddings(
                   Each dict has keys: model, api_key, api_base, weight.
         strategy: Selection strategy for multi-endpoint mode (round_robin, latency_aware).
         cooldown: Default cooldown seconds for rate-limited endpoints.
+        splade_db_path: Optional path to centralized SPLADE database. If None, SPLADE
+                       is written to index_path (legacy behavior). Use index_root / SPLADE_DB_NAME
+                       for centralized storage.
 
     Returns:
         Result dictionary with generation statistics
@@ -404,10 +535,15 @@ def generate_embeddings(
 
         # skip_token_count=True: Use fast estimation (len/4) instead of expensive tiktoken
         # This significantly reduces CPU usage with minimal impact on metadata accuracy
+        # Load chunk stripping config from settings
+        from codexlens.config import Config
+        chunk_cfg = Config.load()
         chunker = Chunker(config=ChunkConfig(
             max_chunk_size=chunk_size,
             overlap=overlap,
-            skip_token_count=True
+            skip_token_count=True,
+            strip_comments=getattr(chunk_cfg, 'chunk_strip_comments', True),
+            strip_docstrings=getattr(chunk_cfg, 'chunk_strip_docstrings', True),
         ))
 
         # Log embedder info with endpoint count for multi-endpoint mode
@@ -416,6 +552,14 @@ def generate_embeddings(
                 progress_callback(f"Using {endpoint_count} API endpoints with {strategy} strategy")
             progress_callback(f"Using model: {embedder.model_name} ({embedder.embedding_dim} dimensions)")
 
+        # Calculate dynamic batch size based on model capacity
+        from codexlens.config import Config
+        batch_config = Config.load()
+        effective_batch_size = calculate_dynamic_batch_size(batch_config, embedder)
+
+        if progress_callback and batch_config.api_batch_size_dynamic:
+            progress_callback(f"Dynamic batch size: {effective_batch_size} (model max_tokens={getattr(embedder, 'max_tokens', 8192)})")
+
     except Exception as e:
         return {
             "success": False,
@@ -429,7 +573,7 @@ def generate_embeddings(
     total_chunks_created = 0
     total_files_processed = 0
     FILE_BATCH_SIZE = 100  # Process 100 files at a time
-    # EMBEDDING_BATCH_SIZE is defined at module level (default: 256)
+    # effective_batch_size is calculated above (dynamic or EMBEDDING_BATCH_SIZE fallback)
 
     try:
         with VectorStore(index_path) as vector_store:
@@ -487,7 +631,7 @@ def generate_embeddings(
                         # Fallback to fixed-size batching for backward compatibility
                         def fixed_size_batches():
                             while True:
-                                batch = list(islice(chunk_generator, EMBEDDING_BATCH_SIZE))
+                                batch = list(islice(chunk_generator, effective_batch_size))
                                 if not batch:
                                     break
                                 yield batch
@@ -518,7 +662,7 @@ def generate_embeddings(
                         for attempt in range(max_retries + 1):
                             try:
                                 batch_contents = [chunk.content for chunk, _ in chunk_batch]
-                                embeddings_numpy = embedder.embed_to_numpy(batch_contents, batch_size=EMBEDDING_BATCH_SIZE)
+                                embeddings_numpy = embedder.embed_to_numpy(batch_contents, batch_size=effective_batch_size)
                                 return batch_num, chunk_batch, embeddings_numpy, batch_files, None
 
                             except Exception as e:
@@ -566,10 +710,11 @@ def generate_embeddings(
                                 try:
                                     # Generate embeddings
                                     batch_contents = [chunk.content for chunk, _ in chunk_batch]
-                                    embeddings_numpy = embedder.embed_to_numpy(batch_contents, batch_size=EMBEDDING_BATCH_SIZE)
+                                    embeddings_numpy = embedder.embed_to_numpy(batch_contents, batch_size=effective_batch_size)
 
-                                    # Store embeddings
-                                    vector_store.add_chunks_batch_numpy(chunk_batch, embeddings_numpy)
+                                    # Store embeddings with category
+                                    categories = _build_categories_from_batch(chunk_batch)
+                                    vector_store.add_chunks_batch_numpy(chunk_batch, embeddings_numpy, categories=categories)
 
                                     files_seen.update(batch_files)
                                     total_chunks_created += len(chunk_batch)
@@ -621,7 +766,8 @@ def generate_embeddings(
                                         batch_num, chunk_batch, embeddings_numpy, batch_files, error = f.result()
                                         if embeddings_numpy is not None and error is None:
                                             # Write to DB in main thread (no contention)
-                                            vector_store.add_chunks_batch_numpy(chunk_batch, embeddings_numpy)
+                                            categories = _build_categories_from_batch(chunk_batch)
+                                            vector_store.add_chunks_batch_numpy(chunk_batch, embeddings_numpy, categories=categories)
                                             total_chunks_created += len(chunk_batch)
                                         files_seen.update(batch_files)
                                         total_files_processed = len(files_seen)
@@ -658,7 +804,8 @@ def generate_embeddings(
                                 try:
                                     batch_num, chunk_batch, embeddings_numpy, batch_files, error = future.result()
                                     if embeddings_numpy is not None and error is None:
-                                        vector_store.add_chunks_batch_numpy(chunk_batch, embeddings_numpy)
+                                        categories = _build_categories_from_batch(chunk_batch)
+                                        vector_store.add_chunks_batch_numpy(chunk_batch, embeddings_numpy, categories=categories)
                                         total_chunks_created += len(chunk_batch)
                                     files_seen.update(batch_files)
                                     total_files_processed = len(files_seen)
@@ -675,10 +822,97 @@ def generate_embeddings(
                 if progress_callback:
                     progress_callback(f"Finalizing index... Building ANN index for {total_chunks_created} chunks")
 
+            # --- SPLADE SPARSE ENCODING (after dense embeddings) ---
+            # Add SPLADE encoding if enabled in config
+            splade_success = False
+            splade_error = None
+
+            try:
+                from codexlens.config import Config, SPLADE_DB_NAME
+                config = Config.load()
+
+                if config.enable_splade:
+                    from codexlens.semantic.splade_encoder import check_splade_available, get_splade_encoder
+                    from codexlens.storage.splade_index import SpladeIndex
+
+                    ok, err = check_splade_available()
+                    if ok:
+                        if progress_callback:
+                            progress_callback(f"Generating SPLADE sparse vectors for {total_chunks_created} chunks...")
+
+                        # Initialize SPLADE encoder and index
+                        splade_encoder = get_splade_encoder(use_gpu=use_gpu)
+                        # Use centralized SPLADE database if provided, otherwise fallback to index_path
+                        effective_splade_path = splade_db_path if splade_db_path else index_path
+                        splade_index = SpladeIndex(effective_splade_path)
+                        splade_index.create_tables()
+
+                        # Retrieve all chunks from database for SPLADE encoding
+                        with sqlite3.connect(index_path) as conn:
+                            conn.row_factory = sqlite3.Row
+                            cursor = conn.execute("SELECT id, content FROM semantic_chunks ORDER BY id")
+
+                            # Batch encode for efficiency
+                            SPLADE_BATCH_SIZE = 32
+                            batch_postings = []
+                            chunk_batch = []
+                            chunk_ids = []
+
+                            for row in cursor:
+                                chunk_id = row["id"]
+                                content = row["content"]
+
+                                chunk_ids.append(chunk_id)
+                                chunk_batch.append(content)
+
+                                # Process batch when full
+                                if len(chunk_batch) >= SPLADE_BATCH_SIZE:
+                                    sparse_vecs = splade_encoder.encode_batch(chunk_batch, batch_size=SPLADE_BATCH_SIZE)
+                                    for cid, sparse_vec in zip(chunk_ids, sparse_vecs):
+                                        batch_postings.append((cid, sparse_vec))
+
+                                    chunk_batch = []
+                                    chunk_ids = []
+
+                            # Process remaining chunks
+                            if chunk_batch:
+                                sparse_vecs = splade_encoder.encode_batch(chunk_batch, batch_size=SPLADE_BATCH_SIZE)
+                                for cid, sparse_vec in zip(chunk_ids, sparse_vecs):
+                                    batch_postings.append((cid, sparse_vec))
+
+                            # Batch insert all postings
+                            if batch_postings:
+                                splade_index.add_postings_batch(batch_postings)
+
+                                # Set metadata
+                                splade_index.set_metadata(
+                                    model_name=splade_encoder.model_name,
+                                    vocab_size=splade_encoder.vocab_size
+                                )
+
+                                splade_success = True
+                                if progress_callback:
+                                    stats = splade_index.get_stats()
+                                    progress_callback(
+                                        f"SPLADE index created: {stats['total_postings']} postings, "
+                                        f"{stats['unique_tokens']} unique tokens"
+                                    )
+                    else:
+                        logger.debug("SPLADE not available: %s", err)
+                        splade_error = f"SPLADE not available: {err}"
+            except Exception as e:
+                splade_error = str(e)
+                logger.warning("SPLADE encoding failed: %s", e)
+
+            # Report SPLADE status after processing
+            if progress_callback and not splade_success and splade_error:
+                progress_callback(f"SPLADE index: FAILED - {splade_error}")
+
     except Exception as e:
         # Cleanup on error to prevent process hanging
         try:
             _cleanup_fastembed_resources()
+            _cleanup_splade_resources()
             gc.collect()
         except Exception:
             pass
@@ -690,6 +924,7 @@ def generate_embeddings(
     # This is critical - without it, ONNX Runtime threads prevent Python from exiting
     try:
         _cleanup_fastembed_resources()
+        _cleanup_splade_resources()
         gc.collect()
     except Exception:
         pass
@@ -709,8 +944,10 @@ def generate_embeddings(
     }
 
 
-def discover_all_index_dbs(index_root: Path) -> List[Path]:
-    """Recursively find all _index.db files in an index tree.
+def _discover_index_dbs_internal(index_root: Path) -> List[Path]:
+    """Internal helper to find all _index.db files (no deprecation warning).
+
+    Used internally by generate_dense_embeddings_centralized.
 
     Args:
         index_root: Root directory to scan for _index.db files
@@ -724,6 +961,30 @@ def discover_all_index_dbs(index_root: Path) -> List[Path]:
     return sorted(index_root.rglob("_index.db"))
 
 
+def discover_all_index_dbs(index_root: Path) -> List[Path]:
+    """Recursively find all _index.db files in an index tree.
+
+    .. deprecated::
+        This function is deprecated. Use centralized indexing with
+        ``generate_dense_embeddings_centralized`` instead, which handles
+        index discovery internally.
+
+    Args:
+        index_root: Root directory to scan for _index.db files
+
+    Returns:
+        Sorted list of paths to _index.db files
+    """
+    import warnings
+    warnings.warn(
+        "discover_all_index_dbs is deprecated. Use centralized indexing with "
+        "generate_dense_embeddings_centralized instead.",
+        DeprecationWarning,
+        stacklevel=2
+    )
+    return _discover_index_dbs_internal(index_root)
+
+
 def find_all_indexes(scan_dir: Path) -> List[Path]:
     """Find all _index.db files in directory tree.
 
@@ -757,6 +1018,11 @@ def generate_embeddings_recursive(
 ) -> Dict[str, any]:
     """Generate embeddings for all index databases in a project recursively.
 
+    .. deprecated::
+        This function is deprecated. Use ``generate_dense_embeddings_centralized``
+        instead, which creates a single centralized vector index for the entire project
+        rather than per-directory indexes.
+
     Args:
         index_root: Root index directory containing _index.db files
         embedding_backend: Embedding backend to use (fastembed or litellm).
@@ -782,6 +1048,14 @@ def generate_embeddings_recursive(
     Returns:
         Aggregated result dictionary with generation statistics
     """
+    import warnings
+    warnings.warn(
+        "generate_embeddings_recursive is deprecated. Use "
+        "generate_dense_embeddings_centralized instead for centralized indexing.",
+        DeprecationWarning,
+        stacklevel=2
+    )
+
     # Get defaults from config if not specified
     (default_backend, default_model, default_gpu,
      default_endpoints, default_strategy, default_cooldown) = _get_embedding_defaults()
@@ -812,8 +1086,8 @@ def generate_embeddings_recursive(
         else:
             max_workers = 1
 
-    # Discover all _index.db files
-    index_files = discover_all_index_dbs(index_root)
+    # Discover all _index.db files (using internal helper to avoid double deprecation warning)
+    index_files = _discover_index_dbs_internal(index_root)
 
     if not index_files:
         return {
@@ -824,6 +1098,10 @@ def generate_embeddings_recursive(
     if progress_callback:
         progress_callback(f"Found {len(index_files)} index databases to process")
 
+    # Calculate centralized SPLADE database path
+    from codexlens.config import SPLADE_DB_NAME
+    splade_db_path = index_root / SPLADE_DB_NAME
+
     # Process each index database
     all_results = []
     total_chunks = 0
@@ -853,6 +1131,7 @@ def generate_embeddings_recursive(
             endpoints=endpoints,
             strategy=strategy,
             cooldown=cooldown,
+            splade_db_path=splade_db_path,  # Use centralized SPLADE storage
         )
 
         all_results.append({
@@ -874,6 +1153,7 @@ def generate_embeddings_recursive(
     # Each generate_embeddings() call does its own cleanup, but do a final one to be safe
     try:
         _cleanup_fastembed_resources()
+        _cleanup_splade_resources()
         gc.collect()
     except Exception:
         pass
@@ -893,6 +1173,488 @@ def generate_embeddings_recursive(
     }
 
 
+def generate_dense_embeddings_centralized(
+    index_root: Path,
+    embedding_backend: Optional[str] = None,
+    model_profile: Optional[str] = None,
+    force: bool = False,
+    chunk_size: int = 2000,
+    overlap: int = 200,
+    progress_callback: Optional[callable] = None,
+    use_gpu: Optional[bool] = None,
+    max_tokens_per_batch: Optional[int] = None,
+    max_workers: Optional[int] = None,
+    endpoints: Optional[List] = None,
+    strategy: Optional[str] = None,
+    cooldown: Optional[float] = None,
+) -> Dict[str, any]:
+    """Generate dense embeddings with centralized vector storage.
+
+    This function creates a single HNSW index at the project root instead of
+    per-directory indexes. All chunks from all _index.db files are combined
+    into one central _vectors.hnsw file.
+
+    Target architecture:
+        <index_root>/
+        |-- _vectors.hnsw         # Centralized dense vector ANN index
+        |-- _splade.db            # Centralized sparse vector index
+        |-- src/
+            |-- _index.db         # No longer contains .hnsw file
+
+    Args:
+        index_root: Root index directory containing _index.db files
+        embedding_backend: Embedding backend (fastembed or litellm)
+        model_profile: Model profile or name
+        force: If True, regenerate even if embeddings exist
+        chunk_size: Maximum chunk size in characters
+        overlap: Overlap size in characters
+        progress_callback: Optional callback for progress updates
+        use_gpu: Whether to use GPU acceleration
+        max_tokens_per_batch: Maximum tokens per batch
+        max_workers: Maximum concurrent workers
+        endpoints: Multi-endpoint configurations
+        strategy: Endpoint selection strategy
+        cooldown: Rate-limit cooldown seconds
+
+    Returns:
+        Result dictionary with generation statistics
+    """
+    from codexlens.config import VECTORS_HNSW_NAME, SPLADE_DB_NAME
+
+    # Get defaults from config if not specified
+    (default_backend, default_model, default_gpu,
+     default_endpoints, default_strategy, default_cooldown) = _get_embedding_defaults()
+
+    if embedding_backend is None:
+        embedding_backend = default_backend
+    if model_profile is None:
+        model_profile = default_model
+    if use_gpu is None:
+        use_gpu = default_gpu
+    if endpoints is None:
+        endpoints = default_endpoints
+    if strategy is None:
+        strategy = default_strategy
+    if cooldown is None:
+        cooldown = default_cooldown
+
+    # Calculate endpoint count for worker scaling
+    endpoint_count = len(endpoints) if endpoints else 1
+
+    if max_workers is None:
+        if embedding_backend == "litellm":
+            if endpoint_count > 1:
+                max_workers = endpoint_count * 2
+            else:
+                max_workers = 4
+        else:
+            max_workers = 1
+
+    backend_available, backend_error = is_embedding_backend_available(embedding_backend)
+    if not backend_available:
+        return {"success": False, "error": backend_error or "Embedding backend not available"}
+
+    # Discover all _index.db files
+    index_files = _discover_index_dbs_internal(index_root)
+
+    if not index_files:
+        return {
+            "success": False,
+            "error": f"No index databases found in {index_root}",
+        }
+
+    if progress_callback:
+        progress_callback(f"Found {len(index_files)} index databases for centralized embedding")
+
+    # Pre-calculate estimated chunk count for HNSW capacity
+    # This avoids expensive resize operations during indexing
+    estimated_total_files = 0
+    for index_path in index_files:
+        try:
+            with sqlite3.connect(index_path) as conn:
+                cursor = conn.execute("SELECT COUNT(*) FROM files")
+                estimated_total_files += cursor.fetchone()[0]
+        except Exception:
+            pass
+    # Heuristic: ~15 chunks per file on average
+    estimated_chunks = max(100000, estimated_total_files * 15)
+
+    if progress_callback:
+        progress_callback(f"Estimated {estimated_total_files} files, ~{estimated_chunks} chunks")
+
+    # Check for existing centralized index
+    central_hnsw_path = index_root / VECTORS_HNSW_NAME
+    if central_hnsw_path.exists() and not force:
+        return {
+            "success": False,
+            "error": f"Centralized vector index already exists at {central_hnsw_path}. Use --force to regenerate.",
+        }
+
+    # Initialize embedder
+    try:
+        from codexlens.semantic.factory import get_embedder as get_embedder_factory
+        from codexlens.semantic.chunker import Chunker, ChunkConfig
+        from codexlens.semantic.ann_index import ANNIndex
+
+        if embedding_backend == "fastembed":
+            embedder = get_embedder_factory(backend="fastembed", profile=model_profile, use_gpu=use_gpu)
+        elif embedding_backend == "litellm":
+            embedder = get_embedder_factory(
+                backend="litellm",
+                model=model_profile,
+                endpoints=endpoints if endpoints else None,
+                strategy=strategy,
+                cooldown=cooldown,
+            )
+        else:
+            return {
+                "success": False,
+                "error": f"Invalid embedding backend: {embedding_backend}",
+            }
+
+        # Load chunk stripping config from settings
+        from codexlens.config import Config
+        chunk_cfg = Config.load()
+        chunker = Chunker(config=ChunkConfig(
+            max_chunk_size=chunk_size,
+            overlap=overlap,
+            skip_token_count=True,
+            strip_comments=getattr(chunk_cfg, 'chunk_strip_comments', True),
+            strip_docstrings=getattr(chunk_cfg, 'chunk_strip_docstrings', True),
+        ))
+
+        if progress_callback:
+            if endpoint_count > 1:
+                progress_callback(f"Using {endpoint_count} API endpoints with {strategy} strategy")
+            progress_callback(f"Using model: {embedder.model_name} ({embedder.embedding_dim} dimensions)")
+
+        # Calculate dynamic batch size based on model capacity
+        batch_config = chunk_cfg  # Reuse already loaded config
+        effective_batch_size = calculate_dynamic_batch_size(batch_config, embedder)
+
+        if progress_callback and batch_config.api_batch_size_dynamic:
+            progress_callback(f"Dynamic batch size: {effective_batch_size} (model max_tokens={getattr(embedder, 'max_tokens', 8192)})")
+
+    except Exception as e:
+        return {
+            "success": False,
+            "error": f"Failed to initialize components: {str(e)}",
+        }
+
+    # Create centralized ANN index with pre-calculated capacity
+    # Using estimated_chunks avoids expensive resize operations during indexing
+    central_ann_index = ANNIndex.create_central(
+        index_root=index_root,
+        dim=embedder.embedding_dim,
+        initial_capacity=estimated_chunks,
+        auto_save=False,
+    )
+
+    # Process all index databases
+    start_time = time.time()
+    failed_files = []
+    total_chunks_created = 0
+    total_files_processed = 0
+    all_chunk_ids = []
+    all_embeddings = []
+
+    # Track chunk ID to file_path mapping for metadata
+    chunk_id_to_info: Dict[int, Dict[str, Any]] = {}
+    next_chunk_id = 1
+    # Track current index_path for source_index_db field
+    current_index_path: Optional[str] = None
+
+    for idx, index_path in enumerate(index_files, 1):
+        if progress_callback:
+            try:
+                rel_path = index_path.relative_to(index_root)
+            except ValueError:
+                rel_path = index_path
+            progress_callback(f"Processing {idx}/{len(index_files)}: {rel_path}")
+
+        # Track current index_path for source_index_db
+        current_index_path = str(index_path)
+
+        try:
+            with sqlite3.connect(index_path) as conn:
+                conn.row_factory = sqlite3.Row
+                path_column = _get_path_column(conn)
+
+                # Get files from this index
+                cursor = conn.execute(f"SELECT {path_column}, content, language FROM files")
+                file_rows = cursor.fetchall()
+
+                for file_row in file_rows:
+                    file_path = file_row[path_column]
+                    content = file_row["content"]
+                    language = file_row["language"] or "python"
+
+                    try:
+                        chunks = chunker.chunk_sliding_window(
+                            content,
+                            file_path=file_path,
+                            language=language
+                        )
+
+                        if not chunks:
+                            continue
+
+                        total_files_processed += 1
+
+                        # Generate embeddings for this file's chunks
+                        batch_contents = [chunk.content for chunk in chunks]
+                        embeddings_numpy = embedder.embed_to_numpy(batch_contents, batch_size=effective_batch_size)
+
+                        # Assign chunk IDs and store embeddings
+                        for i, chunk in enumerate(chunks):
+                            chunk_id = next_chunk_id
+                            next_chunk_id += 1
+
+                            all_chunk_ids.append(chunk_id)
+                            all_embeddings.append(embeddings_numpy[i])
+
+                            # Store metadata for later retrieval
+                            chunk_id_to_info[chunk_id] = {
+                                "file_path": file_path,
+                                "content": chunk.content,
+                                "metadata": chunk.metadata,
+                                "category": get_file_category(file_path) or "code",
+                                "source_index_db": current_index_path,
+                            }
+                            total_chunks_created += 1
+
+                    except Exception as e:
+                        logger.error(f"Failed to process {file_path}: {e}")
+                        failed_files.append((file_path, str(e)))
+
+        except Exception as e:
+            logger.error(f"Failed to read index {index_path}: {e}")
+            failed_files.append((str(index_path), str(e)))
+
+    # Add all embeddings to centralized ANN index
+    if all_embeddings:
+        if progress_callback:
+            progress_callback(f"Building centralized ANN index with {len(all_embeddings)} vectors...")
+
+        try:
+            import numpy as np
+            embeddings_matrix = np.vstack(all_embeddings)
+            central_ann_index.add_vectors(all_chunk_ids, embeddings_matrix)
+            central_ann_index.save()
+
+            if progress_callback:
+                progress_callback(f"Saved centralized index to {central_hnsw_path}")
+
+        except Exception as e:
+            return {
+                "success": False,
+                "error": f"Failed to build centralized ANN index: {str(e)}",
+            }
+
+    # Store chunk metadata in a centralized metadata database
+    vectors_meta_path = index_root / VECTORS_META_DB_NAME
+    if chunk_id_to_info:
+        if progress_callback:
+            progress_callback(f"Storing {len(chunk_id_to_info)} chunk metadata records...")
+
+        try:
+            from codexlens.storage.vector_meta_store import VectorMetadataStore
+
+            with VectorMetadataStore(vectors_meta_path) as meta_store:
+                # Convert chunk_id_to_info dict to list of dicts for batch insert
+                chunks_to_store = []
+                for cid, info in chunk_id_to_info.items():
+                    metadata = info.get("metadata", {})
+                    chunks_to_store.append({
+                        "chunk_id": cid,
+                        "file_path": info["file_path"],
+                        "content": info["content"],
+                        "start_line": metadata.get("start_line"),
+                        "end_line": metadata.get("end_line"),
+                        "category": info.get("category"),
+                        "metadata": metadata,
+                        "source_index_db": info.get("source_index_db"),
+                    })
+
+                meta_store.add_chunks(chunks_to_store)
+
+            if progress_callback:
+                progress_callback(f"Saved metadata to {vectors_meta_path}")
+
+        except Exception as e:
+            logger.warning("Failed to store vector metadata: %s", e)
+            # Non-fatal: continue without centralized metadata
+
+    # --- Binary Vector Generation for Cascade Search (Memory-Mapped) ---
+    binary_success = False
+    binary_count = 0
+    try:
+        from codexlens.config import Config, BINARY_VECTORS_MMAP_NAME
+        config = Config.load()
+
+        if getattr(config, 'enable_binary_cascade', True) and all_embeddings:
+            import numpy as np
+
+            if progress_callback:
+                progress_callback(f"Generating binary vectors for {len(all_embeddings)} chunks...")
+
+            # Binarize dense vectors: sign(x) -> 1 if x > 0, 0 otherwise
+            # Pack into bytes for efficient storage and Hamming distance computation
+            embeddings_matrix = np.vstack(all_embeddings)
+            binary_matrix = (embeddings_matrix > 0).astype(np.uint8)
+
+            # Pack bits into bytes (8 bits per byte) - vectorized for all rows
+            packed_matrix = np.packbits(binary_matrix, axis=1)
+            binary_count = len(packed_matrix)
+
+            # Save as memory-mapped file for efficient loading
+            binary_mmap_path = index_root / BINARY_VECTORS_MMAP_NAME
+            mmap_array = np.memmap(
+                str(binary_mmap_path),
+                dtype=np.uint8,
+                mode='w+',
+                shape=packed_matrix.shape
+            )
+            mmap_array[:] = packed_matrix
+            mmap_array.flush()
+            del mmap_array  # Close the memmap
+
+            # Save metadata (shape and chunk_ids) to sidecar JSON
+            import json
+            meta_path = binary_mmap_path.with_suffix('.meta.json')
+            with open(meta_path, 'w') as f:
+                json.dump({
+                    'shape': list(packed_matrix.shape),
+                    'chunk_ids': all_chunk_ids,
+                    'embedding_dim': embeddings_matrix.shape[1],
+                }, f)
+
+            # Also store in DB for backward compatibility
+            from codexlens.storage.vector_meta_store import VectorMetadataStore
+            binary_packed_bytes = [row.tobytes() for row in packed_matrix]
+            with VectorMetadataStore(vectors_meta_path) as meta_store:
+                meta_store.add_binary_vectors(all_chunk_ids, binary_packed_bytes)
+
+            binary_success = True
+            if progress_callback:
+                progress_callback(f"Generated {binary_count} binary vectors ({embeddings_matrix.shape[1]} dims -> {packed_matrix.shape[1]} bytes, mmap: {binary_mmap_path.name})")
+
+    except Exception as e:
+        logger.warning("Binary vector generation failed: %s", e)
+        # Non-fatal: continue without binary vectors
+
+    # --- SPLADE Sparse Index Generation (Centralized) ---
+    splade_success = False
+    splade_chunks_count = 0
+    try:
+        from codexlens.config import Config
+        config = Config.load()
+
+        if config.enable_splade and chunk_id_to_info:
+            from codexlens.semantic.splade_encoder import check_splade_available, get_splade_encoder
+            from codexlens.storage.splade_index import SpladeIndex
+            import json
+
+            ok, err = check_splade_available()
+            if ok:
+                if progress_callback:
+                    progress_callback(f"Generating SPLADE sparse vectors for {len(chunk_id_to_info)} chunks...")
+
+                # Initialize SPLADE encoder and index
+                splade_encoder = get_splade_encoder(use_gpu=use_gpu)
+                splade_db_path = index_root / SPLADE_DB_NAME
+                splade_index = SpladeIndex(splade_db_path)
+                splade_index.create_tables()
+
+                # Batch encode for efficiency
+                SPLADE_BATCH_SIZE = 32
+                all_postings = []
+                all_chunk_metadata = []
+
+                # Create batches from chunk_id_to_info
+                chunk_items = list(chunk_id_to_info.items())
+
+                for i in range(0, len(chunk_items), SPLADE_BATCH_SIZE):
+                    batch_items = chunk_items[i:i + SPLADE_BATCH_SIZE]
+                    chunk_ids = [item[0] for item in batch_items]
+                    chunk_contents = [item[1]["content"] for item in batch_items]
+
+                    # Generate sparse vectors
+                    sparse_vecs = splade_encoder.encode_batch(chunk_contents, batch_size=SPLADE_BATCH_SIZE)
+                    for cid, sparse_vec in zip(chunk_ids, sparse_vecs):
+                        all_postings.append((cid, sparse_vec))
+
+                    if progress_callback and (i + SPLADE_BATCH_SIZE) % 100 == 0:
+                        progress_callback(f"SPLADE encoding: {min(i + SPLADE_BATCH_SIZE, len(chunk_items))}/{len(chunk_items)}")
+
+                # Batch insert all postings
+                if all_postings:
+                    splade_index.add_postings_batch(all_postings)
+
+                # CRITICAL FIX: Populate splade_chunks table
+                for cid, info in chunk_id_to_info.items():
+                    metadata_str = json.dumps(info.get("metadata", {})) if info.get("metadata") else None
+                    all_chunk_metadata.append((
+                        cid,
+                        info["file_path"],
+                        info["content"],
+                        metadata_str,
+                        info.get("source_index_db")
+                    ))
+
+                if all_chunk_metadata:
+                    splade_index.add_chunks_metadata_batch(all_chunk_metadata)
+                    splade_chunks_count = len(all_chunk_metadata)
+
+                # Set metadata
+                splade_index.set_metadata(
+                    model_name=splade_encoder.model_name,
+                    vocab_size=splade_encoder.vocab_size
+                )
+
+                splade_index.close()
+                splade_success = True
+
+                if progress_callback:
+                    progress_callback(f"SPLADE index created: {len(all_postings)} postings, {splade_chunks_count} chunks")
+
+            else:
+                if progress_callback:
+                    progress_callback(f"SPLADE not available, skipping sparse index: {err}")
+
+    except Exception as e:
+        logger.warning("SPLADE encoding failed: %s", e)
+        if progress_callback:
+            progress_callback(f"SPLADE encoding failed: {e}")
+
+    elapsed_time = time.time() - start_time
+
+    # Cleanup
+    try:
+        _cleanup_fastembed_resources()
+        gc.collect()
+    except Exception:
+        pass
+
+    return {
+        "success": True,
+        "result": {
+            "chunks_created": total_chunks_created,
+            "files_processed": total_files_processed,
+            "files_failed": len(failed_files),
+            "elapsed_time": elapsed_time,
+            "model_profile": model_profile,
+            "model_name": embedder.model_name,
+            "central_index_path": str(central_hnsw_path),
+            "failed_files": failed_files[:5],
+            "splade_success": splade_success,
+            "splade_chunks": splade_chunks_count,
+            "binary_success": binary_success,
+            "binary_count": binary_count,
+        },
+    }
+
+
 def get_embeddings_status(index_root: Path) -> Dict[str, any]:
     """Get comprehensive embeddings coverage status for all indexes.
 
@@ -902,7 +1664,7 @@ def get_embeddings_status(index_root: Path) -> Dict[str, any]:
     Returns:
         Aggregated status with coverage statistics, model info, and timestamps
     """
-    index_files = discover_all_index_dbs(index_root)
+    index_files = _discover_index_dbs_internal(index_root)
 
     if not index_files:
         return {
@@ -1071,7 +1833,7 @@ def scan_for_model_conflicts(
         - conflicts: List of conflicting index paths with their configs
         - indexes_with_embeddings: Count of indexes that have embeddings
     """
-    index_files = discover_all_index_dbs(index_root)
+    index_files = _discover_index_dbs_internal(index_root)
 
     if not index_files:
         return {