@mc1global/opencode-jarvis 0.13.1 → 0.14.0

package/src/infrastructure/rag/document_loader_repository.py

@@ -1,1495 +0,0 @@
-"""Document Loader Repository - Abstraction for document ingestion."""
-
-import logging
-import os
-import yaml
-from pathlib import Path
-from abc import ABC, abstractmethod
-from typing import Any, Dict, List, Optional, Type
-
-from langchain_community.document_loaders import (
-    DirectoryLoader,
-    TextLoader,
-    PyPDFLoader,
-)
-from langchain_text_splitters import (
-    Language,
-    PythonCodeTextSplitter,
-    MarkdownTextSplitter,
-    MarkdownHeaderTextSplitter,
-    RecursiveJsonSplitter,
-    HTMLSemanticPreservingSplitter,
-    LatexTextSplitter,
-    RecursiveCharacterTextSplitter,
-    TextSplitter,
-)
-
-from .workspace_utils import normalize_workspace_name
-from .domain_map_loader import DomainMapLoader
-
-logger = logging.getLogger(__name__)
-
-
-# ===== METADATA EXTRACTORS =====
-
-
-class MetadataExtractor(ABC):
-    """Base class for metadata extractors."""
-
-    @abstractmethod
-    def extract(self, content: str, file_path: Optional[str] = None) -> Dict[str, Any]:
-        """Extract metadata from content."""
-        pass
-
-
-class PythonMetadataExtractor(MetadataExtractor):
-    """Extract metadata from Python files."""
-
-    def extract(self, content: str, file_path: Optional[str] = None) -> Dict[str, Any]:
-        """Extract imports, classes, functions, docstrings, complexity from Python code."""
-        import ast
-        import re
-
-        metadata = {
-            "imports": [],
-            "classes": [],
-            "functions": [],
-            "docstrings": [],
-            "complexity": 0,
-        }
-
-        try:
-            tree = ast.parse(content)
-
-            for node in ast.walk(tree):
-                if isinstance(node, ast.Import):
-                    for alias in node.names:
-                        metadata["imports"].append(alias.name)
-                elif isinstance(node, ast.ImportFrom):
-                    module = node.module or ""
-                    for alias in node.names:
-                        metadata["imports"].append(f"{module}.{alias.name}")
-                elif isinstance(node, ast.ClassDef):
-                    metadata["classes"].append(node.name)
-                    if node.docstring:
-                        metadata["docstrings"].append(node.docstring.strip())
-                elif isinstance(node, ast.FunctionDef):
-                    metadata["functions"].append(node.name)
-                    if node.docstring:
-                        metadata["docstrings"].append(node.docstring.strip())
-
-            # Simple complexity metric
-            metadata["complexity"] = len(
-                re.findall(r"\b(if|for|while|try|def|class)\b", content)
-            )
-
-        except SyntaxError:
-            logger.warning(f"Syntax error in Python file: {file_path}")
-
-        return metadata
-
-
-class JavaScriptMetadataExtractor(MetadataExtractor):
-    """Extract metadata from JavaScript/TypeScript files."""
-
-    def extract(self, content: str, file_path: Optional[str] = None) -> Dict[str, Any]:
-        """Extract imports, exports, classes, functions, components."""
-        import re
-
-        metadata = {
-            "imports": [],
-            "exports": [],
-            "classes": [],
-            "functions": [],
-            "components": [],
-        }
-
-        # Extract imports
-        import_matches = re.findall(
-            r'import\s+.*?from\s+[\'"]([^\'"]+)[\'"]', content, re.MULTILINE
-        )
-        metadata["imports"].extend(import_matches)
-
-        # Extract exports
-        export_matches = re.findall(
-            r"export\s+(?:const|let|var|function|class)\s+(\w+)", content
-        )
-        metadata["exports"].extend(export_matches)
-
-        # Extract classes and functions
-        class_matches = re.findall(r"class\s+(\w+)", content)
-        metadata["classes"].extend(class_matches)
-
-        func_matches = re.findall(
-            r"(?:function\s+|const\s+|let\s+|var\s+)(\w+)\s*=", content
-        )
-        metadata["functions"].extend(func_matches)
-
-        # Extract React components (basic heuristic)
-        component_matches = re.findall(
-            r"(?:function|const)\s+(\w+)\s*\([^)]*\)\s*{[^}]*return\s*\(", content
-        )
-        metadata["components"].extend(component_matches)
-
-        return metadata
-
-
-class MarkdownMetadataExtractor(MetadataExtractor):
-    """Extract metadata from Markdown files."""
-
-    def extract(self, content: str, file_path: Optional[str] = None) -> Dict[str, Any]:
-        """Extract headers, sections, links, code blocks."""
-        import re
-
-        metadata = {"headers": [], "sections": [], "links": [], "code_blocks": []}
-
-        # Extract headers
-        header_matches = re.findall(r"^(#{1,6})\s+(.+)$", content, re.MULTILINE)
-        metadata["headers"] = [f"{level} {text}" for level, text in header_matches]
-
-        # Extract sections (H1, H2, H3)
-        section_matches = re.findall(r"^#{1,3}\s+(.+)$", content, re.MULTILINE)
-        metadata["sections"] = section_matches
-
-        # Extract links
-        link_matches = re.findall(r"\[([^\]]+)\]\(([^)]+)\)", content)
-        metadata["links"] = [text for text, url in link_matches]
-
-        # Extract code blocks
-        code_matches = re.findall(r"```(\w+)?\n(.*?)\n```", content, re.DOTALL)
-        metadata["code_blocks"] = [lang for lang, code in code_matches if lang]
-
-        return metadata
-
-
-class JavaMetadataExtractor(MetadataExtractor):
-    """Extract metadata from Java files."""
-
-    def extract(self, content: str, file_path: Optional[str] = None) -> Dict[str, Any]:
-        """Extract package, imports, classes, interfaces, methods, annotations."""
-        import re
-
-        metadata = {
-            "package": None,
-            "imports": [],
-            "classes": [],
-            "interfaces": [],
-            "methods": [],
-            "annotations": [],
-        }
-
-        # Extract package
-        package_match = re.search(r"package\s+([^\s;]+)", content)
-        if package_match:
-            metadata["package"] = package_match.group(1)
-
-        # Extract imports
-        import_matches = re.findall(r"import\s+([^\s;]+)", content)
-        metadata["imports"] = import_matches
-
-        # Extract classes
-        class_matches = re.findall(r"class\s+(\w+)", content)
-        metadata["classes"] = class_matches
-
-        # Extract interfaces
-        interface_matches = re.findall(r"interface\s+(\w+)", content)
-        metadata["interfaces"] = interface_matches
-
-        # Extract methods
-        method_matches = re.findall(
-            r"(?:public|private|protected)?\s*\w+\s+(\w+)\s*\([^)]*\)", content
-        )
-        metadata["methods"] = method_matches
-
-        # Extract annotations
-        annotation_matches = re.findall(r"@\w+", content)
-        metadata["annotations"] = list(set(annotation_matches))  # Remove duplicates
-
-        return metadata
-
-
-class RustMetadataExtractor(MetadataExtractor):
-    """Extract metadata from Rust files."""
-
-    def extract(self, content: str, file_path: Optional[str] = None) -> Dict[str, Any]:
-        """Extract modules, structs, enums, functions, traits, macros, impl blocks."""
-        import re
-
-        metadata = {
-            "modules": [],
-            "structs": [],
-            "enums": [],
-            "functions": [],
-            "traits": [],
-            "macros": [],
-            "impl_blocks": [],
-        }
-
-        # Extract modules
-        module_matches = re.findall(r"(?:pub\s+)?mod\s+(\w+)", content)
-        metadata["modules"] = module_matches
-
-        # Extract structs
-        struct_matches = re.findall(r"(?:pub\s+)?struct\s+(\w+)", content)
-        metadata["structs"] = struct_matches
-
-        # Extract enums
-        enum_matches = re.findall(r"(?:pub\s+)?enum\s+(\w+)", content)
-        metadata["enums"] = enum_matches
-
-        # Extract functions
-        func_matches = re.findall(r"(?:pub\s+)?fn\s+(\w+)", content)
-        metadata["functions"] = func_matches
-
-        # Extract traits
-        trait_matches = re.findall(r"(?:pub\s+)?trait\s+(\w+)", content)
-        metadata["traits"] = trait_matches
-
-        # Extract macros
-        macro_matches = re.findall(r"macro_rules!\s*(\w+)", content)
-        metadata["macros"] = macro_matches
-
-        # Extract impl blocks
-        impl_matches = re.findall(r"impl(?:<[^>]+>)?\s+(\w+)", content)
-        metadata["impl_blocks"] = impl_matches
-
-        return metadata
-
-
-class YAMLMetadataExtractor(MetadataExtractor):
-    """Extract metadata from YAML files."""
-
-    def extract(self, content: str, file_path: Optional[str] = None) -> Dict[str, Any]:
-        """Extract keys, structure, anchors."""
-        metadata = {"keys": [], "structure": {}, "anchors": []}
-
-        try:
-            import yaml
-
-            data = yaml.safe_load(content)
-
-            if isinstance(data, dict):
-                metadata["keys"] = list(data.keys())
-                metadata["structure"] = self._analyze_structure(data)
-            elif isinstance(data, list):
-                metadata["keys"] = [f"item_{i}" for i in range(len(data))]
-                metadata["structure"] = {"type": "list", "length": len(data)}
-
-            # Extract anchors (basic detection)
-            anchor_matches = re.findall(r"&(\w+)", content)
-            metadata["anchors"] = anchor_matches
-
-        except Exception as e:
-            logger.warning(f"Error parsing YAML metadata: {e}")
-
-        return metadata
-
-    def _analyze_structure(self, data: Any, depth: int = 0) -> Dict[str, Any]:
-        """Analyze the structure of YAML data."""
-        if depth > 3:  # Limit depth
-            return {"type": "complex"}
-
-        if isinstance(data, dict):
-            return {
-                "type": "dict",
-                "keys": len(data),
-                "nested": {
-                    k: self._analyze_structure(v, depth + 1)
-                    for k, v in list(data.items())[:3]
-                },
-            }
-        elif isinstance(data, list):
-            return {
-                "type": "list",
-                "length": len(data),
-                "item_types": list(set(type(item).__name__ for item in data[:5])),
-            }
-        else:
-            return {"type": type(data).__name__}
-
-
-class JSONMetadataExtractor(MetadataExtractor):
-    """Extract metadata from JSON files."""
-
-    def extract(self, content: str, file_path: Optional[str] = None) -> Dict[str, Any]:
-        """Extract schema, keys, structure."""
-        import json
-
-        metadata = {"schema": {}, "keys": [], "structure": {}}
-
-        try:
-            data = json.loads(content)
-
-            if isinstance(data, dict):
-                metadata["keys"] = list(data.keys())
-                metadata["schema"] = self._infer_schema(data)
-                metadata["structure"] = {"type": "object", "properties": len(data)}
-            elif isinstance(data, list):
-                if data:
-                    metadata["schema"] = self._infer_schema(data[0])
-                metadata["structure"] = {"type": "array", "length": len(data)}
-
-        except Exception as e:
-            logger.warning(f"Error parsing JSON metadata: {e}")
-
-        return metadata
-
-    def _infer_schema(self, data: Any, depth: int = 0) -> Dict[str, Any]:
-        """Infer basic schema from data."""
-        if depth > 3:
-            return {"type": "complex"}
-
-        if isinstance(data, dict):
-            return {
-                "type": "object",
-                "properties": {
-                    k: self._infer_schema(v, depth + 1)
-                    for k, v in list(data.items())[:5]
-                },
-            }
-        elif isinstance(data, list):
-            if data:
-                return {
-                    "type": "array",
-                    "items": self._infer_schema(data[0], depth + 1),
-                }
-            return {"type": "array", "items": {}}
-        else:
-            return {"type": type(data).__name__}
-
-
-class CSVMetadataExtractor(MetadataExtractor):
-    """Extract metadata from CSV files."""
-
-    def extract(self, content: str, file_path: Optional[str] = None) -> Dict[str, Any]:
-        """Extract headers, row count, columns."""
-        import csv
-        import io
-
-        metadata = {"headers": [], "row_count": 0, "columns": 0}
-
-        try:
-            # Try to detect delimiter
-            sample = content[:1024]
-            sniffer = csv.Sniffer()
-            delimiter = sniffer.sniff(sample).delimiter
-
-            # Parse CSV
-            reader = csv.reader(io.StringIO(content), delimiter=delimiter)
-            rows = list(reader)
-
-            if rows:
-                metadata["headers"] = rows[0]
-                metadata["row_count"] = len(rows) - 1  # Excluding header
-                metadata["columns"] = len(rows[0])
-
-        except Exception as e:
-            logger.warning(f"Error parsing CSV metadata: {e}")
-
-        return metadata
-
-
-class SwiftMetadataExtractor(MetadataExtractor):
-    """Extract metadata from Swift files."""
-
-    def extract(self, content: str, file_path: Optional[str] = None) -> Dict[str, Any]:
-        """Extract modules, classes, structs, protocols, functions, extensions."""
-        import re
-
-        metadata = {
-            "modules": [],
-            "classes": [],
-            "structs": [],
-            "protocols": [],
-            "functions": [],
-            "extensions": [],
-        }
-
-        # Extract modules
-        module_matches = re.findall(r"module\s+(\w+)", content)
-        metadata["modules"] = module_matches
-
-        # Extract classes
-        class_matches = re.findall(r"class\s+(\w+)", content)
-        metadata["classes"] = class_matches
-
-        # Extract structs
-        struct_matches = re.findall(r"struct\s+(\w+)", content)
-        metadata["structs"] = struct_matches
-
-        # Extract protocols
-        protocol_matches = re.findall(r"protocol\s+(\w+)", content)
-        metadata["protocols"] = protocol_matches
-
-        # Extract functions
-        func_matches = re.findall(
-            r"(?:func\s+|public\s+func\s+|private\s+func\s+|internal\s+func\s+)(\w+)",
-            content,
-        )
-        metadata["functions"] = func_matches
-
-        # Extract extensions
-        extension_matches = re.findall(r"extension\s+(\w+)", content)
-        metadata["extensions"] = extension_matches
-
-        return metadata
-
-
-class CSharpMetadataExtractor(MetadataExtractor):
-    """Extract metadata from C# files."""
-
-    def extract(self, content: str, file_path: Optional[str] = None) -> Dict[str, Any]:
-        """Extract namespace, using statements, classes, interfaces, methods, properties, attributes."""
-        import re
-
-        metadata = {
-            "namespace": None,
-            "using_statements": [],
-            "classes": [],
-            "interfaces": [],
-            "methods": [],
-            "properties": [],
-            "attributes": [],
-        }
-
-        # Extract namespace
-        namespace_match = re.search(r"namespace\s+([^\s;{]+)", content)
-        if namespace_match:
-            metadata["namespace"] = namespace_match.group(1)
-
-        # Extract using statements
-        using_matches = re.findall(r"using\s+([^\s;]+)", content)
-        metadata["using_statements"] = using_matches
-
-        # Extract classes
-        class_matches = re.findall(
-            r"(?:public\s+|internal\s+|private\s+|protected\s+)?class\s+(\w+)", content
-        )
-        metadata["classes"] = class_matches
-
-        # Extract interfaces
-        interface_matches = re.findall(
-            r"(?:public\s+|internal\s+|private\s+|protected\s+)?interface\s+(\w+)",
-            content,
-        )
-        metadata["interfaces"] = interface_matches
-
-        # Extract methods
-        method_matches = re.findall(
-            r"(?:public\s+|private\s+|protected\s+|internal\s+)?(?:async\s+|static\s+|virtual\s+|override\s+|abstract\s+)*(\w+(?:<[^>]+>)?)\s+(\w+)\s*\([^)]*\)",
-            content,
-        )
-        metadata["methods"] = [
-            method
-            for return_type, method in method_matches
-            if not return_type in ["class", "interface", "struct", "enum"]
-        ]
-
-        # Extract properties
-        property_matches = re.findall(
-            r"(?:public\s+|private\s+|protected\s+|internal\s+)?(?:static\s+|virtual\s+|override\s+|abstract\s+)*\w+(?:<[^>]+>)?\s+(\w+)\s*{\s*get;\s*(?:set;)?\s*}",
-            content,
-        )
-        metadata["properties"] = property_matches
-
-        # Extract attributes
-        attribute_matches = re.findall(r"\[([^\]]+)\]", content)
-        metadata["attributes"] = list(set(attribute_matches))  # Remove duplicates
-
-        return metadata
-
-
-# Registry of metadata extractors
-METADATA_EXTRACTORS: Dict[str, Type[MetadataExtractor]] = {
-    "python": PythonMetadataExtractor,
-    "javascript": JavaScriptMetadataExtractor,
-    "typescript": JavaScriptMetadataExtractor,  # Same as JS
-    "markdown": MarkdownMetadataExtractor,
-    "java": JavaMetadataExtractor,
-    "rust": RustMetadataExtractor,
-    "csharp": CSharpMetadataExtractor,
-    "swift": SwiftMetadataExtractor,
-    "kotlin": JavaScriptMetadataExtractor,  # Reuse JS extractor for now (similar syntax)
-    "yaml": YAMLMetadataExtractor,
-    "yml": YAMLMetadataExtractor,
-    "json": JSONMetadataExtractor,
-    "csv": CSVMetadataExtractor,
-}
-
-
-class DocumentLoaderRepositoryInterface(ABC):
-    """Interface for document loading and chunking."""
-
-    @abstractmethod
-    def load_and_split(
-        self,
-        data_path: str,
-        workspace_name: Optional[str] = None,
-        domain_name: Optional[str] = None,
-    ) -> List[Any]:
-        """Load documents and split into chunks."""
-        pass
-
-
-class DocumentLoaderRepository(DocumentLoaderRepositoryInterface):
-    """
-    Document loader with DDD metadata injection and adaptive text splitting.
-
-    Handles:
-    - Multiple file type support (py, md, json, yaml, pdf, etc.)
-    - Language-aware chunking for code files (swift, cs, kt, java, ts, js, go, etc.)
-    - Adaptive chunking strategy per file type
-    - Metadata injection (workspace, domain, file_type)
-    - Advanced metadata extraction per file type
-    - Domain inference from directory structure
-    - Detection and handling of minified files
-    """
-
-    def _extract_metadata(
-        self, content: str, file_type: str, file_path: str
-    ) -> Dict[str, Any]:
-        """Extract metadata based on file type configuration."""
-        metadata = {}
-
-        # Get file type configuration
-        config = self._config.get("rag", {}).get("file_types", {}).get(file_type, {})
-        if not config.get("extract_metadata", False):
-            return metadata
-
-        metadata_fields = config.get("metadata_fields", [])
-        if not metadata_fields:
-            return metadata
-
-        # Extract metadata using appropriate extractor
-        extractor_class = METADATA_EXTRACTORS.get(file_type)
-        if extractor_class:
-            try:
-                extractor = extractor_class()
-                extracted = extractor.extract(content, file_path)
-
-                # Filter to only requested fields
-                for field in metadata_fields:
-                    if field in extracted:
-                        metadata[field] = extracted[field]
-
-                logger.debug(
-                    f"Extracted metadata for {file_path}: {list(metadata.keys())}"
-                )
-
-            except Exception as e:
-                logger.warning(f"Failed to extract metadata from {file_path}: {e}")
-
-        return metadata
-
-    # Supported file loaders
-    FILE_LOADERS: Dict[str, Type[Any]] = {
-        ".pdf": PyPDFLoader,
-        ".txt": TextLoader,
-        ".md": TextLoader,
-        ".csv": TextLoader,
-        ".json": TextLoader,
-        ".yaml": TextLoader,
-        ".yml": TextLoader,
-        ".py": TextLoader,
-        ".ts": TextLoader,
-        ".js": TextLoader,
-        ".jsx": TextLoader,
-        ".tsx": TextLoader,
-        ".swift": TextLoader,
-        ".cs": TextLoader,
-        ".kt": TextLoader,
-        ".go": TextLoader,
-        ".java": TextLoader,
-        ".cpp": TextLoader,
-        ".c": TextLoader,
-        ".h": TextLoader,
-        ".html": TextLoader,
-        ".css": TextLoader,
-        ".sh": TextLoader,
-        ".bash": TextLoader,
-        ".zsh": TextLoader,
-        ".fish": TextLoader,
-        ".rs": TextLoader,
-        ".rb": TextLoader,
-        ".php": TextLoader,
-    }
-
-    # Mapping of file extensions to Language enum for language-aware splitting
-    EXT_TO_LANGUAGE: Dict[str, Language] = {
-        ".swift": Language.SWIFT,
-        ".cs": Language.CSHARP,
-        ".kt": Language.KOTLIN,
-        ".java": Language.JAVA,
-        ".ts": Language.TS,
-        ".js": Language.JS,
-        ".go": Language.GO,
-        ".cpp": Language.CPP,
-        ".c": Language.C,
-        ".rs": Language.RUST,
-        ".rb": Language.RUBY,
-        ".php": Language.PHP,
-    }
-
-    def __init__(self, config_path: str = "config/rag.yaml") -> None:
-        """
-        Initialize document loader with dynamic chunk sizing per language.
-
-        Args:
-            config_path: Path to RAG configuration file
-        """
-        # Load and store configuration
-        self._config = self._load_rag_config(config_path)
-
-        # Load included content configuration
-        self.included_config = self._config.get("rag", {}).get("included_content", {})
-
-        # Domain map loaders cache
-        self._domain_map_loaders: Dict[str, DomainMapLoader] = {}
-
-        # Track logged warnings to prevent repetition (CD-161 P2 fix)
-        self._logged_warnings: set = set()
-
-        logger.info(
-            "Document loader initialized with dynamic chunk sizing per language"
-        )
-
-    def _load_rag_config(self, config_path: str) -> Dict[str, Any]:
-        """
-        Load complete RAG configuration from YAML file.
-
-        Args:
-            config_path: Path to the RAG configuration file
-
-        Returns:
-            Dictionary with complete RAG configuration
-        """
-        try:
-            config_path = Path(config_path)
-            if not config_path.exists():
-                logger.warning(f"Config file not found: {config_path}")
-                return {}
-
-            with open(config_path, "r") as f:
-                config = yaml.safe_load(f)
-
-            logger.info(f"Loaded RAG configuration from {config_path}")
-
-            # Log included content configuration if present
-            included = config.get("rag", {}).get("included_content", {})
-            if included:
-                logger.debug(
-                    f"Documentation dirs: {included.get('documentation_dirs', [])}"
-                )
-                logger.debug(f"Root files: {included.get('root_files', [])}")
-                logger.debug(f"Code dirs: {included.get('code_dirs', [])}")
-
-            return config
-        except Exception as e:
-            logger.error(f"Failed to load config from {config_path}: {e}")
-            return {}
-
-    def _load_language_config(self, ext: str) -> Dict[str, int]:
-        """
-        Load language-specific chunk configuration dynamically.
-
-        Args:
-            ext: File extension (e.g., '.py', '.cs', '.sh')
-
-        Returns:
-            Dict with chunk_size and chunk_overlap
-        """
-        chunking_config = self._config.get("rag", {}).get("chunking", {})
-
-        # Map extension to language key in config
-        lang_map = {
-            # Python
-            ".py": "python",
-            ".pyi": "python",
-            # Web/JavaScript
-            ".ts": "typescript",
-            ".tsx": "typescript",
-            ".js": "javascript",
-            ".jsx": "javascript",
-            ".mjs": "javascript",
-            # Mobile/Native
-            ".swift": "swift",
-            ".kt": "kotlin",
-            ".kts": "kotlin",
-            # Systems Programming
-            ".go": "go",
-            ".rs": "rust",
-            # Enterprise Languages
-            ".cs": "csharp",
-            ".java": "java",
-            # C/C++ Family
-            ".c": "cpp",
-            ".cpp": "cpp",
-            ".cc": "cpp",
-            ".cxx": "cpp",
-            ".h": "cpp",
-            ".hpp": "cpp",
-            ".hxx": "cpp",
-            # Web Backend
-            ".php": "php",
-            ".rb": "ruby",
-            # Shell Scripts
-            ".sh": "shell",
-            ".bash": "shell",
-            ".zsh": "shell",
-            ".fish": "shell",
-            # Data/Configuration
-            ".json": "json",
-            ".yaml": "yaml",
-            ".yml": "yaml",
-            ".csv": "csv",
-            # Documentation
-            ".md": "markdown",
-            ".txt": "text",
-            # Web Assets
-            ".html": "html",
-            ".htm": "html",
-            ".css": "css",
-            # Other
-            ".tex": "latex",
-        }
-
-        lang_key = lang_map.get(ext, "default")
-        lang_config = chunking_config.get(lang_key, chunking_config.get("default", {}))
-
-        return {
-            "chunk_size": lang_config.get("chunk_size", 500),
-            "chunk_overlap": lang_config.get("chunk_overlap", 50),
-        }
-
-    def _is_minified(self, content: str, ext: str) -> bool:
-        """
-        Detect if content is minified (no line breaks or very long lines).
-
-        Args:
-            content: File content to check
-            ext: File extension
-
-        Returns:
-            True if content appears to be minified
-        """
-        # Extensions that can be minified
-        minifiable = {".js", ".jsx", ".ts", ".tsx", ".json", ".css", ".html"}
-
-        if ext not in minifiable:
-            return False
-
-        # Quick check: if content is all on one line and reasonably long
-        lines = content.split("\n")
-
-        # Remove empty lines for accurate counting
-        non_empty_lines = [line for line in lines if line.strip()]
-
-        if not non_empty_lines:
-            return False
-
-        # If only 1-2 non-empty lines and content is long, likely minified
-        if len(non_empty_lines) <= 2 and len(content) > 200:
-            return True
-
-        # Check average line length
-        avg_line_length = sum(len(line) for line in non_empty_lines) / len(
-            non_empty_lines
-        )
-
-        # Lower threshold for better detection
-        if avg_line_length > 150:  # Reduced from 300
-            return True
-
-        # Check for typical minification patterns
-        if ext in {".js", ".jsx", ".ts", ".tsx"}:
-            # Check if first line has multiple statements (semicolons)
-            first_line = non_empty_lines[0] if non_empty_lines else ""
-            # Reduced threshold from 20 to 5 for better detection
-            if first_line.count(";") > 5:
-                return True
-            # Check for compressed function syntax
-            if "){" in first_line and "}" in first_line and first_line.count("){") > 2:
-                return True
-
-        if ext == ".json":
-            # Minified JSON has no spaces after colons/commas
-            first_line = non_empty_lines[0] if non_empty_lines else ""
-            # Check for lack of formatting
-            if '": ' not in first_line and '":' in first_line and len(first_line) > 100:
-                return True
-            if ", " not in first_line and "," in first_line and len(first_line) > 100:
-                return True
-
-        return False
-
-    def _get_splitter_for_extension(
-        self, ext: str, content: Optional[str] = None
-    ) -> TextSplitter:
-        """
-        Get adaptive text splitter with DYNAMIC language-specific configuration.
-
-        Loads chunk size and overlap dynamically per file type from config.
-
-        Args:
-            ext: File extension (e.g., '.py', '.md', '.json')
-            content: Optional file content for minification detection
-
-        Returns:
-            Configured TextSplitter instance
-        """
-        # Load dynamic config for this extension
-        config = self._load_language_config(ext)
-        chunk_size = config["chunk_size"]
-        chunk_overlap = config["chunk_overlap"]
-
-        logger.debug(f"Splitter for {ext}: size={chunk_size}, overlap={chunk_overlap}")
-
-        # Python: Always use PythonCodeTextSplitter
-        if ext == ".py":
-            return PythonCodeTextSplitter(
-                chunk_size=chunk_size,
-                chunk_overlap=chunk_overlap,
-            )
-
-        # Markdown: Simple character-based splitting (temporarily disabled header hierarchy)
-        if ext == ".md":
-            return RecursiveCharacterTextSplitter(
-                chunk_size=chunk_size,
-                chunk_overlap=chunk_overlap,
-            )
-
-        # Shell scripts: Line-based with special separators
-        if ext in {".sh", ".bash", ".zsh", ".fish"}:
-            return RecursiveCharacterTextSplitter(
-                chunk_size=chunk_size,
-                chunk_overlap=chunk_overlap,
-                separators=["\n\n", "\n", ";", "&&", "||", "|", " ", ""],
-            )
-
-        # YAML: Line-based with structure awareness
-        if ext in {".yaml", ".yml"}:
-            return RecursiveCharacterTextSplitter(
-                chunk_size=chunk_size,
-                chunk_overlap=chunk_overlap,
-                separators=["\n\n", "\n", ": ", "- ", " ", ""],
-            )
-
-        # Code files with language support (Swift, Kotlin, C#, Go, C++, etc.)
-        if ext in self.EXT_TO_LANGUAGE:
-            lang = self.EXT_TO_LANGUAGE[ext]
-
-            # Check if JS/TS files are minified
-            if ext in {".js", ".jsx", ".ts", ".tsx"} and content:
-                if self._is_minified(content, ext):
-                    logger.debug(
-                        f"Minified {ext} detected, using character-based splitter"
-                    )
-                    return RecursiveCharacterTextSplitter(
-                        chunk_size=chunk_size,
-                        chunk_overlap=chunk_overlap,
-                        separators=[";", "}", "{", ",", " ", ""],
-                    )
-
-            # Language-specific splitters for better chunking
-            if ext == ".java":
-                # Java: Split on class/interface boundaries
-                return RecursiveCharacterTextSplitter(
-                    chunk_size=chunk_size,
-                    chunk_overlap=chunk_overlap,
-                    separators=[
-                        "\n\npublic class",
-                        "\n\npublic interface",
-                        "\n\nclass",
-                        "\n\ninterface",
-                        "\n\npublic",
-                        "\n\nprivate",
-                        "\n\nprotected",
-                        "\n\n",
-                        "\n",
-                        " ",
-                        "",
-                    ],
-                )
-            elif ext == ".rs":
-                # Rust: Split on module/impl boundaries
-                return RecursiveCharacterTextSplitter(
-                    chunk_size=chunk_size,
-                    chunk_overlap=chunk_overlap,
-                    separators=[
-                        "\n\npub mod",
-                        "\n\nmod",
-                        "\n\nimpl",
-                        "\n\npub struct",
-                        "\n\npub enum",
-                        "\n\nfn",
-                        "\n\n",
-                        "\n",
-                        " ",
-                        "",
-                    ],
-                )
-            elif ext == ".php":
-                # PHP: Split on class/function boundaries
-                return RecursiveCharacterTextSplitter(
-                    chunk_size=chunk_size,
-                    chunk_overlap=chunk_overlap,
-                    separators=[
-                        "\n\nclass",
-                        "\n\ninterface",
-                        "\n\ntrait",
-                        "\n\nfunction",
-                        "\n\npublic function",
-                        "\n\nprivate function",
-                        "\n\nprotected function",
-                        "\n\n",
-                        "\n",
-                        " ",
-                        "",
-                    ],
-                )
-            elif ext == ".rb":
-                # Ruby: Split on class/module boundaries
-                return RecursiveCharacterTextSplitter(
-                    chunk_size=chunk_size,
-                    chunk_overlap=chunk_overlap,
-                    separators=[
-                        "\n\nclass",
-                        "\n\nmodule",
-                        "\n\npublic",
-                        "\n\nprivate",
-                        "\n\ndef",
-                        "\n\n",
-                        "\n",
-                        " ",
-                        "",
-                    ],
-                )
-            elif ext in {".cs"}:
-                # C#: Split on class/namespace boundaries
-                return RecursiveCharacterTextSplitter(
-                    chunk_size=chunk_size,
-                    chunk_overlap=chunk_overlap,
-                    separators=[
-                        "\n\npublic class",
-                        "\n\ninternal class",
-                        "\n\nclass",
-                        "\n\npublic interface",
-                        "\n\ninterface",
-                        "\n\nnamespace",
-                        "\n\npublic",
-                        "\n\nprivate",
-                        "\n\n",
-                        "\n",
-                        " ",
-                        "",
-                    ],
-                )
-            else:
-                # Use language-aware splitter with DYNAMIC sizes for other languages
-                logger.debug(f"Language-aware splitter for {ext} ({lang.value})")
-                return RecursiveCharacterTextSplitter.from_language(
-                    language=lang,
-                    chunk_size=chunk_size,
-                    chunk_overlap=chunk_overlap,
-                )
-
-        # JSON: Check if minified
-        if ext == ".json":
-            if content and self._is_minified(content, ext):
-                logger.debug(f"Minified JSON detected, using character splitter")
-                return RecursiveCharacterTextSplitter(
-                    chunk_size=chunk_size,
-                    chunk_overlap=chunk_overlap,
-                    separators=["},", "],", ",", " ", ""],
-                )
-
-        # CSV: Line-based splitting
-        if ext == ".csv":
-            return RecursiveCharacterTextSplitter(
-                chunk_size=chunk_size,
-                chunk_overlap=chunk_overlap,
-                separators=["\n", ","],
-            )
-
-        # CSS: Rule-based splitting
-        if ext == ".css":
-            return RecursiveCharacterTextSplitter(
-                chunk_size=chunk_size,
-                chunk_overlap=chunk_overlap,
-                separators=["}\n", "}\n\n", "\n", " ", ""],
-            )
-            return RecursiveCharacterTextSplitter(
-                chunk_size=chunk_size,
-                chunk_overlap=chunk_overlap,
-                separators=["\n", ",", " ", ""],
-            )
-
-        # HTML: Semantic preservation
-        if ext == ".html":
-            if content and self._is_minified(content, ext):
-                logger.debug(f"Minified HTML detected")
-                return RecursiveCharacterTextSplitter(
-                    chunk_size=chunk_size,
-                    chunk_overlap=chunk_overlap,
-                    separators=[">", "</", "/>", " ", ""],
-                )
-            else:
-                logger.debug(f"HTML with semantic preservation")
-                return RecursiveCharacterTextSplitter(
-                    chunk_size=chunk_size,
-                    chunk_overlap=chunk_overlap,
-                    separators=["</div>", "</p>", "</h", ">", "\n\n", "\n", " ", ""],
-                )
-
-        # CSS: Check if minified
-        if ext == ".css":
-            if content and self._is_minified(content, ext):
-                logger.debug(f"Minified CSS detected")
-                return RecursiveCharacterTextSplitter(
-                    chunk_size=chunk_size,
-                    chunk_overlap=chunk_overlap,
-                    separators=["}", ";", "{", " ", ""],
-                )
-            else:
-                logger.debug(f"CSS with line-based splitting")
-                return RecursiveCharacterTextSplitter(
-                    chunk_size=chunk_size,
-                    chunk_overlap=chunk_overlap,
-                    separators=["\n", "}", ";", " ", ""],
-                )
-
-        # LaTeX: Special handling
-        if ext == ".tex":
-            logger.debug(f"LaTeX-specific splitter for {ext}")
-            return LatexTextSplitter(
-                chunk_size=chunk_size,
-                chunk_overlap=chunk_overlap,
-            )
-
-        # Fallback: Adaptive based on common patterns
-        logger.debug(f"Fallback adaptive splitter for {ext}")
-        return RecursiveCharacterTextSplitter(
-            chunk_size=chunk_size,
-            chunk_overlap=chunk_overlap,
-            separators=["\n", "\n\n", ".", "!", "?", ";", ",", " ", ""],
-        )
-
-    def load_and_split(
-        self,
-        data_path: str,
-        workspace_name: Optional[str] = None,
-        domain_name: Optional[str] = None,
-    ) -> List[Any]:
-        """
-        Load documents with DDD metadata and split into chunks.
-        Uses included_content configuration from rag.yaml if available.
-
-        Args:
-            data_path: Root directory containing source files
-            workspace_name: Top-level workspace (auto-inferred if None)
-            domain_name: DDD domain for filtering
-
-        Returns:
-            List of chunked documents with metadata
-
-        Raises:
-            ValueError: If no supported documents found
-        """
-        logger.info(f"Loading documents from {data_path}")
-
-        all_docs: List[Any] = []
-        data_path = Path(data_path).resolve()
-
-        # Infer workspace name from directory if not provided
-        if not workspace_name:
-            workspace_name = os.path.basename(os.path.abspath(data_path))
-            logger.info(f"Inferred workspace: {workspace_name}")
-
-        # Normalize workspace name for case-insensitive lookups
-        workspace_name = normalize_workspace_name(workspace_name)
-        logger.info(f"Normalized workspace name: {workspace_name}")
-
-        # Check if we're indexing the root directory (not a subdirectory)
-        # Use config only for root-level indexing (current dir or '.')
-        should_use_config = self.included_config and (
-            str(data_path) == "."
-            or str(data_path) == os.getcwd()
-            or data_path == Path(".").resolve()
-        )
-
-        if should_use_config:
-            # Use configured directories for root-level indexing
-            # 1. Load documentation directories
-            doc_dirs = self.included_config.get("documentation_dirs", [])
-            for doc_dir in doc_dirs:
-                dir_path = data_path / doc_dir
-                if dir_path.exists() and dir_path.is_dir():
-                    logger.info(f"Loading documentation directory: {doc_dir}")
-                    self._load_directory(
-                        dir_path, all_docs, workspace_name, domain_name
-                    )
-
-            # 2. Load specific root files
-            root_files = self.included_config.get("root_files", [])
-            for root_file in root_files:
-                file_path = data_path / root_file
-                if file_path.exists() and file_path.is_file():
-                    logger.info(f"Loading root file: {root_file}")
-                    self._load_single_file(
-                        file_path, all_docs, workspace_name, domain_name
-                    )
-
-            # 3. Load code directories (src, scripts)
-            code_dirs = self.included_config.get("code_dirs", [])
-            for code_dir in code_dirs:
-                dir_path = data_path / code_dir
-                if dir_path.exists() and dir_path.is_dir():
-                    logger.info(f"Loading code directory: {code_dir}")
-                    self._load_directory(
-                        dir_path, all_docs, workspace_name, domain_name
-                    )
-        else:
-            # Load the specific directory passed (e.g., for skills)
-            logger.info(f"Loading specific directory: {data_path}")
-            self._load_directory(data_path, all_docs, workspace_name, domain_name)
-
-        if not all_docs:
-            raise ValueError(
-                f"No supported documents found in {data_path}. "
-                f"Supported: {', '.join(self.FILE_LOADERS.keys())}"
-            )
-
-        logger.info(f"Total chunks created: {len(all_docs)}")
-        return all_docs
-
-    def _load_directory(
-        self,
-        dir_path: Path,
-        all_docs: List[Any],
-        workspace_name: str,
-        domain_name: Optional[str],
-    ) -> None:
-        """
-        Load all supported files from a directory.
-
-        Args:
-            dir_path: Directory path to load from
-            all_docs: List to append documents to
-            workspace_name: Workspace identifier
-            domain_name: Optional domain filter
-        """
-        for ext, loader_cls in self.FILE_LOADERS.items():
-            glob_pattern = f"**/*{ext}"
-            try:
-                dir_loader = DirectoryLoader(
-                    str(dir_path),
-                    glob=glob_pattern,
-                    loader_cls=loader_cls,
-                    silent_errors=True,
-                )
-                docs_for_ext = list(dir_loader.lazy_load())
-
-                # Split documents using adaptive splitter FIRST
-                if docs_for_ext:
-                    # Try to get content for minification detection
-                    content = None
-                    if (
-                        ext in {".js", ".jsx", ".ts", ".tsx", ".json", ".css", ".html"}
-                        and docs_for_ext
-                    ):
-                        content = (
-                            docs_for_ext[0].page_content
-                            if hasattr(docs_for_ext[0], "page_content")
-                            else None
-                        )
-
-                    splitter_result = self._get_splitter_for_extension(ext, content)
-
-                    # Handle two-stage splitting for markdown (returns tuple)
-                    if isinstance(splitter_result, tuple):
-                        header_splitter, char_splitter = splitter_result
-                        # Stage 1: Split by headers to extract hierarchy
-                        from langchain_core.documents import Document
-
-                        doc_chunks = []
-                        for doc in docs_for_ext:
-                            # Split single document by headers
-                            header_splits = header_splitter.split_text(doc.page_content)
-                            # header_splits is a list of Document objects, NOT tuples
-                            for header_doc in header_splits:
-                                # Extract hierarchy metadata from Document
-                                header_metadata = (
-                                    header_doc.metadata
-                                    if hasattr(header_doc, "metadata")
-                                    else {}
-                                )
-                                chunk_content = (
-                                    header_doc.page_content
-                                    if hasattr(header_doc, "page_content")
-                                    else str(header_doc)
-                                )
-                                # Create Document with hierarchy metadata
-                                chunk_doc = Document(
-                                    page_content=chunk_content,
-                                    metadata=header_metadata.copy(),
-                                )
-                                # Preserve original document metadata (source, etc)
-                                if hasattr(doc, "metadata"):
-                                    chunk_doc.metadata.update(doc.metadata)
-                                    # But keep the hierarchy metadata we extracted
-                                    chunk_doc.metadata.update(header_metadata)
-                                doc_chunks.append(chunk_doc)
-                        # Stage 2: Further split large sections by character
-                        chunks_for_ext = char_splitter.split_documents(doc_chunks)
-                    else:
-                        # Single-stage splitting for other file types
-                        chunks_for_ext = splitter_result.split_documents(docs_for_ext)
-
-                    # FIX: Inject metadata AFTER splitting to ensure it's preserved
-                    for chunk in chunks_for_ext:
-                        chunk.metadata["file_type"] = ext.lstrip(".")
-                        chunk.metadata["workspace_name"] = workspace_name
-
-                        # Infer domain from path if not provided
-                        if domain_name:
-                            chunk.metadata["domain_name"] = domain_name
-                        else:
-                            chunk.metadata["domain_name"] = self._infer_domain(
-                                chunk.metadata.get("source", "")
-                            )
-
-                        # Enrich with domain map metadata (domain_description, etc)
-                        chunk.metadata = self._enrich_with_domain_map(
-                            chunk.metadata, workspace_name
-                        )
-
-                        # Extract section type and build section path (for markdown with hierarchy)
-                        if ext == ".md" and chunk.metadata.get("h3"):
-                            h3 = chunk.metadata.get("h3", "")
-                            # Extract section type from H3 prefix
-                            if h3.startswith("COMO"):
-                                chunk.metadata["section_type"] = "COMO"
-                            elif h3.startswith("DIRETIVA"):
-                                chunk.metadata["section_type"] = "DIRETIVA"
-                            elif h3.startswith("O QUE EVITAR"):
-                                chunk.metadata["section_type"] = "EVITAR"
-                            elif "O QUE" in chunk.metadata.get("h2", ""):
-                                chunk.metadata["section_type"] = "O_QUE"
-                            else:
-                                chunk.metadata["section_type"] = "OTHER"
-
-                            # Build section path for hierarchical navigation
-                            parts = [
-                                chunk.metadata.get("h1", ""),
-                                chunk.metadata.get("h2", ""),
-                                chunk.metadata.get("h3", ""),
-                            ]
-                            chunk.metadata["section_path"] = "/".join(
-                                [p for p in parts if p]
-                            )
-
-                    all_docs.extend(chunks_for_ext)
-                    logger.info(
-                        f"Loaded {len(docs_for_ext)} {ext} files from {dir_path.name}, "
-                        f"split into {len(chunks_for_ext)} chunks"
-                    )
-
-            except Exception as e:
-                logger.warning(f"Failed to load {ext} files from {dir_path}: {e}")
-
-    def _load_single_file(
-        self,
-        file_path: Path,
-        all_docs: List[Any],
-        workspace_name: str,
-        domain_name: Optional[str],
-    ) -> None:
-        """
-        Load a single file.
-
-        Args:
-            file_path: File path to load
-            all_docs: List to append documents to
-            workspace_name: Workspace identifier
-            domain_name: Optional domain filter
-        """
-        ext = file_path.suffix
-        if ext not in self.FILE_LOADERS:
-            logger.warning(f"Unsupported file type: {ext}")
-            return
-
-        try:
-            loader_cls = self.FILE_LOADERS[ext]
-            loader = loader_cls(str(file_path))
-            docs = list(loader.lazy_load())
-
-            # Split documents FIRST
-            if docs:
-                # Try to get content for minification detection
-                content = None
-                if (
-                    ext in {".js", ".jsx", ".ts", ".tsx", ".json", ".css", ".html"}
-                    and docs
-                ):
-                    content = (
-                        docs[0].page_content
-                        if hasattr(docs[0], "page_content")
-                        else None
-                    )
-
-                text_splitter = self._get_splitter_for_extension(ext, content)
-                chunks = text_splitter.split_documents(docs)
-
-                # FIX: Inject metadata AFTER splitting to ensure it's preserved
-                for chunk in chunks:
-                    chunk.metadata["file_type"] = ext.lstrip(".")
-                    chunk.metadata["workspace_name"] = workspace_name
-                    chunk.metadata["domain_name"] = domain_name or self._infer_domain(
-                        file_path, workspace_name
-                    )
-
-                    # Extract additional metadata based on file type
-                    file_type = ext.lstrip(".")
-                    if hasattr(chunk, "page_content") and chunk.page_content:
-                        additional_metadata = self._extract_metadata(
-                            chunk.page_content, file_type, str(file_path)
-                        )
-                        chunk.metadata.update(additional_metadata)
-
-                    # Enrich with domain map metadata (domain_description, etc)
-                    chunk.metadata = self._enrich_with_domain_map(
-                        chunk.metadata, workspace_name
-                    )
-
-                all_docs.extend(chunks)
-                logger.info(
-                    f"Loaded file {file_path.name}, split into {len(chunks)} chunks"
-                )
-
-        except Exception as e:
-            logger.warning(f"Failed to load file {file_path}: {e}")
-
-    @staticmethod
-    def _infer_domain(source_path: str) -> str:
-        """
-        Infer DDD domain from file path.
-
-        Delegates to HierarchicalDomainInference which supports:
-        - force_domain_mapping from config/rag.yaml (highest priority)
-        - DDD structure detection (src/domain|application|infrastructure)
-        - Workspace hierarchy inference
-
-        Args:
-            source_path: File path
-
-        Returns:
-            Inferred domain name
-        """
-        from src.infrastructure.rag.hierarchical_domain_inference import (
-            HierarchicalDomainInference,
-        )
-
-        result = HierarchicalDomainInference.parse_path_hierarchy(source_path)
-        return result["domain_name"]
-
-    def load_from_path(
-        self,
-        path: str,
-        workspace_name: Optional[str] = None,
-        domain_name: Optional[str] = None,
-    ) -> List[Any]:
-        """
-        Load documents from file path (compatibility method).
-
-        Args:
-            path: Directory or file path
-            workspace_name: Optional workspace identifier
-            domain_name: Optional domain filter
-
-        Returns:
-            List of Document entities
-        """
-        # This delegates to the existing load_and_split but without splitting
-        # We'll return the raw documents for chunking later
-        return self.load_and_split(path, workspace_name, domain_name)
-
-    def chunk_document(
-        self, document: Any, chunk_size: int, chunk_overlap: int
-    ) -> List[Any]:
-        """
-        Split document into chunks (compatibility method).
-
-        Args:
-            document: Document to chunk
-            chunk_size: Maximum characters per chunk
-            chunk_overlap: Overlapping characters
-
-        Returns:
-            List of chunks
-        """
-        # Simple fallback implementation
-        from langchain_text_splitters import RecursiveCharacterTextSplitter
-
-        splitter = RecursiveCharacterTextSplitter(
-            chunk_size=chunk_size,
-            chunk_overlap=chunk_overlap,
-            separators=["\n\n", "\n", " ", ""],
-        )
-
-        if hasattr(document, "page_content"):
-            # It's already a Document object
-            return splitter.split_documents([document])
-        else:
-            # It's raw text
-            return splitter.split_text(str(document))
-
-    def _enrich_with_domain_map(
-        self, base_metadata: Dict[str, Any], workspace_name: str
-    ) -> Dict[str, Any]:
-        """
-        Enrich chunk metadata with domain map information.
-
-        Args:
-            base_metadata: Base metadata dict with domain_name
-            workspace_name: Workspace name
-
-        Returns:
-            Enriched metadata with domain_description, dependencies, classes
-        """
-        if not hasattr(self, "_domain_map_loaders"):
-            logger.debug(
-                f"Initializing _domain_map_loaders in {self.__class__.__name__}"
-            )
-            self._domain_map_loaders = {}
-
-        try:
-            # Get or create domain map loader for workspace
-            if workspace_name not in self._domain_map_loaders:
-                loader = DomainMapLoader(workspace_name)
-                if loader.load_domain_map():
-                    self._domain_map_loaders[workspace_name] = loader
-                    logger.info(f"Domain map loaded for workspace: {workspace_name}")
-                else:
-                    # No domain map available - log warning once per workspace (CD-161 P2 fix)
-                    warning_key = f"no_domain_map:{workspace_name}"
-                    if not hasattr(self, '_logged_warnings'):
-                        self._logged_warnings = set()
-                    if warning_key not in self._logged_warnings:
-                        logger.warning(f"No domain map found for workspace: {workspace_name}")
-                        self._logged_warnings.add(warning_key)
-                    # Cache negative result to avoid repeated lookups
-                    self._domain_map_loaders[workspace_name] = None
-                    return base_metadata
-
-            loader = self._domain_map_loaders.get(workspace_name)
-            if loader is None:
-                return base_metadata
-
-            # Enrich metadata using domain map
-            enriched = loader.enrich_metadata(base_metadata)
-
-            return enriched
-
-        except Exception as e:
-            logger.error(f"Error enriching metadata: {e}", exc_info=True)
-            return base_metadata