@mc1global/opencode-jarvis 0.12.0 → 0.13.1

package/src/infrastructure/rag/document_loader_repository.py

@@ -0,0 +1,1495 @@
+"""Document Loader Repository - Abstraction for document ingestion."""
+
+import logging
+import os
+import yaml
+from pathlib import Path
+from abc import ABC, abstractmethod
+from typing import Any, Dict, List, Optional, Type
+
+from langchain_community.document_loaders import (
+    DirectoryLoader,
+    TextLoader,
+    PyPDFLoader,
+)
+from langchain_text_splitters import (
+    Language,
+    PythonCodeTextSplitter,
+    MarkdownTextSplitter,
+    MarkdownHeaderTextSplitter,
+    RecursiveJsonSplitter,
+    HTMLSemanticPreservingSplitter,
+    LatexTextSplitter,
+    RecursiveCharacterTextSplitter,
+    TextSplitter,
+)
+
+from .workspace_utils import normalize_workspace_name
+from .domain_map_loader import DomainMapLoader
+
+logger = logging.getLogger(__name__)
+
+
+# ===== METADATA EXTRACTORS =====
+
+
+class MetadataExtractor(ABC):
+    """Base class for metadata extractors."""
+
+    @abstractmethod
+    def extract(self, content: str, file_path: Optional[str] = None) -> Dict[str, Any]:
+        """Extract metadata from content."""
+        pass
+
+
+class PythonMetadataExtractor(MetadataExtractor):
+    """Extract metadata from Python files."""
+
+    def extract(self, content: str, file_path: Optional[str] = None) -> Dict[str, Any]:
+        """Extract imports, classes, functions, docstrings, complexity from Python code."""
+        import ast
+        import re
+
+        metadata = {
+            "imports": [],
+            "classes": [],
+            "functions": [],
+            "docstrings": [],
+            "complexity": 0,
+        }
+
+        try:
+            tree = ast.parse(content)
+
+            for node in ast.walk(tree):
+                if isinstance(node, ast.Import):
+                    for alias in node.names:
+                        metadata["imports"].append(alias.name)
+                elif isinstance(node, ast.ImportFrom):
+                    module = node.module or ""
+                    for alias in node.names:
+                        metadata["imports"].append(f"{module}.{alias.name}")
+                elif isinstance(node, ast.ClassDef):
+                    metadata["classes"].append(node.name)
+                    if node.docstring:
+                        metadata["docstrings"].append(node.docstring.strip())
+                elif isinstance(node, ast.FunctionDef):
+                    metadata["functions"].append(node.name)
+                    if node.docstring:
+                        metadata["docstrings"].append(node.docstring.strip())
+
+            # Simple complexity metric
+            metadata["complexity"] = len(
+                re.findall(r"\b(if|for|while|try|def|class)\b", content)
+            )
+
+        except SyntaxError:
+            logger.warning(f"Syntax error in Python file: {file_path}")
+
+        return metadata
+
+
+class JavaScriptMetadataExtractor(MetadataExtractor):
+    """Extract metadata from JavaScript/TypeScript files."""
+
+    def extract(self, content: str, file_path: Optional[str] = None) -> Dict[str, Any]:
+        """Extract imports, exports, classes, functions, components."""
+        import re
+
+        metadata = {
+            "imports": [],
+            "exports": [],
+            "classes": [],
+            "functions": [],
+            "components": [],
+        }
+
+        # Extract imports
+        import_matches = re.findall(
+            r'import\s+.*?from\s+[\'"]([^\'"]+)[\'"]', content, re.MULTILINE
+        )
+        metadata["imports"].extend(import_matches)
+
+        # Extract exports
+        export_matches = re.findall(
+            r"export\s+(?:const|let|var|function|class)\s+(\w+)", content
+        )
+        metadata["exports"].extend(export_matches)
+
+        # Extract classes and functions
+        class_matches = re.findall(r"class\s+(\w+)", content)
+        metadata["classes"].extend(class_matches)
+
+        func_matches = re.findall(
+            r"(?:function\s+|const\s+|let\s+|var\s+)(\w+)\s*=", content
+        )
+        metadata["functions"].extend(func_matches)
+
+        # Extract React components (basic heuristic)
+        component_matches = re.findall(
+            r"(?:function|const)\s+(\w+)\s*\([^)]*\)\s*{[^}]*return\s*\(", content
+        )
+        metadata["components"].extend(component_matches)
+
+        return metadata
+
+
+class MarkdownMetadataExtractor(MetadataExtractor):
+    """Extract metadata from Markdown files."""
+
+    def extract(self, content: str, file_path: Optional[str] = None) -> Dict[str, Any]:
+        """Extract headers, sections, links, code blocks."""
+        import re
+
+        metadata = {"headers": [], "sections": [], "links": [], "code_blocks": []}
+
+        # Extract headers
+        header_matches = re.findall(r"^(#{1,6})\s+(.+)$", content, re.MULTILINE)
+        metadata["headers"] = [f"{level} {text}" for level, text in header_matches]
+
+        # Extract sections (H1, H2, H3)
+        section_matches = re.findall(r"^#{1,3}\s+(.+)$", content, re.MULTILINE)
+        metadata["sections"] = section_matches
+
+        # Extract links
+        link_matches = re.findall(r"\[([^\]]+)\]\(([^)]+)\)", content)
+        metadata["links"] = [text for text, url in link_matches]
+
+        # Extract code blocks
+        code_matches = re.findall(r"```(\w+)?\n(.*?)\n```", content, re.DOTALL)
+        metadata["code_blocks"] = [lang for lang, code in code_matches if lang]
+
+        return metadata
+
+
+class JavaMetadataExtractor(MetadataExtractor):
+    """Extract metadata from Java files."""
+
+    def extract(self, content: str, file_path: Optional[str] = None) -> Dict[str, Any]:
+        """Extract package, imports, classes, interfaces, methods, annotations."""
+        import re
+
+        metadata = {
+            "package": None,
+            "imports": [],
+            "classes": [],
+            "interfaces": [],
+            "methods": [],
+            "annotations": [],
+        }
+
+        # Extract package
+        package_match = re.search(r"package\s+([^\s;]+)", content)
+        if package_match:
+            metadata["package"] = package_match.group(1)
+
+        # Extract imports
+        import_matches = re.findall(r"import\s+([^\s;]+)", content)
+        metadata["imports"] = import_matches
+
+        # Extract classes
+        class_matches = re.findall(r"class\s+(\w+)", content)
+        metadata["classes"] = class_matches
+
+        # Extract interfaces
+        interface_matches = re.findall(r"interface\s+(\w+)", content)
+        metadata["interfaces"] = interface_matches
+
+        # Extract methods
+        method_matches = re.findall(
+            r"(?:public|private|protected)?\s*\w+\s+(\w+)\s*\([^)]*\)", content
+        )
+        metadata["methods"] = method_matches
+
+        # Extract annotations
+        annotation_matches = re.findall(r"@\w+", content)
+        metadata["annotations"] = list(set(annotation_matches))  # Remove duplicates
+
+        return metadata
+
+
+class RustMetadataExtractor(MetadataExtractor):
+    """Extract metadata from Rust files."""
+
+    def extract(self, content: str, file_path: Optional[str] = None) -> Dict[str, Any]:
+        """Extract modules, structs, enums, functions, traits, macros, impl blocks."""
+        import re
+
+        metadata = {
+            "modules": [],
+            "structs": [],
+            "enums": [],
+            "functions": [],
+            "traits": [],
+            "macros": [],
+            "impl_blocks": [],
+        }
+
+        # Extract modules
+        module_matches = re.findall(r"(?:pub\s+)?mod\s+(\w+)", content)
+        metadata["modules"] = module_matches
+
+        # Extract structs
+        struct_matches = re.findall(r"(?:pub\s+)?struct\s+(\w+)", content)
+        metadata["structs"] = struct_matches
+
+        # Extract enums
+        enum_matches = re.findall(r"(?:pub\s+)?enum\s+(\w+)", content)
+        metadata["enums"] = enum_matches
+
+        # Extract functions
+        func_matches = re.findall(r"(?:pub\s+)?fn\s+(\w+)", content)
+        metadata["functions"] = func_matches
+
+        # Extract traits
+        trait_matches = re.findall(r"(?:pub\s+)?trait\s+(\w+)", content)
+        metadata["traits"] = trait_matches
+
+        # Extract macros
+        macro_matches = re.findall(r"macro_rules!\s*(\w+)", content)
+        metadata["macros"] = macro_matches
+
+        # Extract impl blocks
+        impl_matches = re.findall(r"impl(?:<[^>]+>)?\s+(\w+)", content)
+        metadata["impl_blocks"] = impl_matches
+
+        return metadata
+
+
+class YAMLMetadataExtractor(MetadataExtractor):
+    """Extract metadata from YAML files."""
+
+    def extract(self, content: str, file_path: Optional[str] = None) -> Dict[str, Any]:
+        """Extract keys, structure, anchors."""
+        metadata = {"keys": [], "structure": {}, "anchors": []}
+
+        try:
+            import yaml
+
+            data = yaml.safe_load(content)
+
+            if isinstance(data, dict):
+                metadata["keys"] = list(data.keys())
+                metadata["structure"] = self._analyze_structure(data)
+            elif isinstance(data, list):
+                metadata["keys"] = [f"item_{i}" for i in range(len(data))]
+                metadata["structure"] = {"type": "list", "length": len(data)}
+
+            # Extract anchors (basic detection)
+            anchor_matches = re.findall(r"&(\w+)", content)
+            metadata["anchors"] = anchor_matches
+
+        except Exception as e:
+            logger.warning(f"Error parsing YAML metadata: {e}")
+
+        return metadata
+
+    def _analyze_structure(self, data: Any, depth: int = 0) -> Dict[str, Any]:
+        """Analyze the structure of YAML data."""
+        if depth > 3:  # Limit depth
+            return {"type": "complex"}
+
+        if isinstance(data, dict):
+            return {
+                "type": "dict",
+                "keys": len(data),
+                "nested": {
+                    k: self._analyze_structure(v, depth + 1)
+                    for k, v in list(data.items())[:3]
+                },
+            }
+        elif isinstance(data, list):
+            return {
+                "type": "list",
+                "length": len(data),
+                "item_types": list(set(type(item).__name__ for item in data[:5])),
+            }
+        else:
+            return {"type": type(data).__name__}
+
+
+class JSONMetadataExtractor(MetadataExtractor):
+    """Extract metadata from JSON files."""
+
+    def extract(self, content: str, file_path: Optional[str] = None) -> Dict[str, Any]:
+        """Extract schema, keys, structure."""
+        import json
+
+        metadata = {"schema": {}, "keys": [], "structure": {}}
+
+        try:
+            data = json.loads(content)
+
+            if isinstance(data, dict):
+                metadata["keys"] = list(data.keys())
+                metadata["schema"] = self._infer_schema(data)
+                metadata["structure"] = {"type": "object", "properties": len(data)}
+            elif isinstance(data, list):
+                if data:
+                    metadata["schema"] = self._infer_schema(data[0])
+                metadata["structure"] = {"type": "array", "length": len(data)}
+
+        except Exception as e:
+            logger.warning(f"Error parsing JSON metadata: {e}")
+
+        return metadata
+
+    def _infer_schema(self, data: Any, depth: int = 0) -> Dict[str, Any]:
+        """Infer basic schema from data."""
+        if depth > 3:
+            return {"type": "complex"}
+
+        if isinstance(data, dict):
+            return {
+                "type": "object",
+                "properties": {
+                    k: self._infer_schema(v, depth + 1)
+                    for k, v in list(data.items())[:5]
+                },
+            }
+        elif isinstance(data, list):
+            if data:
+                return {
+                    "type": "array",
+                    "items": self._infer_schema(data[0], depth + 1),
+                }
+            return {"type": "array", "items": {}}
+        else:
+            return {"type": type(data).__name__}
+
+
+class CSVMetadataExtractor(MetadataExtractor):
+    """Extract metadata from CSV files."""
+
+    def extract(self, content: str, file_path: Optional[str] = None) -> Dict[str, Any]:
+        """Extract headers, row count, columns."""
+        import csv
+        import io
+
+        metadata = {"headers": [], "row_count": 0, "columns": 0}
+
+        try:
+            # Try to detect delimiter
+            sample = content[:1024]
+            sniffer = csv.Sniffer()
+            delimiter = sniffer.sniff(sample).delimiter
+
+            # Parse CSV
+            reader = csv.reader(io.StringIO(content), delimiter=delimiter)
+            rows = list(reader)
+
+            if rows:
+                metadata["headers"] = rows[0]
+                metadata["row_count"] = len(rows) - 1  # Excluding header
+                metadata["columns"] = len(rows[0])
+
+        except Exception as e:
+            logger.warning(f"Error parsing CSV metadata: {e}")
+
+        return metadata
+
+
+class SwiftMetadataExtractor(MetadataExtractor):
+    """Extract metadata from Swift files."""
+
+    def extract(self, content: str, file_path: Optional[str] = None) -> Dict[str, Any]:
+        """Extract modules, classes, structs, protocols, functions, extensions."""
+        import re
+
+        metadata = {
+            "modules": [],
+            "classes": [],
+            "structs": [],
+            "protocols": [],
+            "functions": [],
+            "extensions": [],
+        }
+
+        # Extract modules
+        module_matches = re.findall(r"module\s+(\w+)", content)
+        metadata["modules"] = module_matches
+
+        # Extract classes
+        class_matches = re.findall(r"class\s+(\w+)", content)
+        metadata["classes"] = class_matches
+
+        # Extract structs
+        struct_matches = re.findall(r"struct\s+(\w+)", content)
+        metadata["structs"] = struct_matches
+
+        # Extract protocols
+        protocol_matches = re.findall(r"protocol\s+(\w+)", content)
+        metadata["protocols"] = protocol_matches
+
+        # Extract functions
+        func_matches = re.findall(
+            r"(?:func\s+|public\s+func\s+|private\s+func\s+|internal\s+func\s+)(\w+)",
+            content,
+        )
+        metadata["functions"] = func_matches
+
+        # Extract extensions
+        extension_matches = re.findall(r"extension\s+(\w+)", content)
+        metadata["extensions"] = extension_matches
+
+        return metadata
+
+
+class CSharpMetadataExtractor(MetadataExtractor):
+    """Extract metadata from C# files."""
+
+    def extract(self, content: str, file_path: Optional[str] = None) -> Dict[str, Any]:
+        """Extract namespace, using statements, classes, interfaces, methods, properties, attributes."""
+        import re
+
+        metadata = {
+            "namespace": None,
+            "using_statements": [],
+            "classes": [],
+            "interfaces": [],
+            "methods": [],
+            "properties": [],
+            "attributes": [],
+        }
+
+        # Extract namespace
+        namespace_match = re.search(r"namespace\s+([^\s;{]+)", content)
+        if namespace_match:
+            metadata["namespace"] = namespace_match.group(1)
+
+        # Extract using statements
+        using_matches = re.findall(r"using\s+([^\s;]+)", content)
+        metadata["using_statements"] = using_matches
+
+        # Extract classes
+        class_matches = re.findall(
+            r"(?:public\s+|internal\s+|private\s+|protected\s+)?class\s+(\w+)", content
+        )
+        metadata["classes"] = class_matches
+
+        # Extract interfaces
+        interface_matches = re.findall(
+            r"(?:public\s+|internal\s+|private\s+|protected\s+)?interface\s+(\w+)",
+            content,
+        )
+        metadata["interfaces"] = interface_matches
+
+        # Extract methods
+        method_matches = re.findall(
+            r"(?:public\s+|private\s+|protected\s+|internal\s+)?(?:async\s+|static\s+|virtual\s+|override\s+|abstract\s+)*(\w+(?:<[^>]+>)?)\s+(\w+)\s*\([^)]*\)",
+            content,
+        )
+        metadata["methods"] = [
+            method
+            for return_type, method in method_matches
+            if not return_type in ["class", "interface", "struct", "enum"]
+        ]
+
+        # Extract properties
+        property_matches = re.findall(
+            r"(?:public\s+|private\s+|protected\s+|internal\s+)?(?:static\s+|virtual\s+|override\s+|abstract\s+)*\w+(?:<[^>]+>)?\s+(\w+)\s*{\s*get;\s*(?:set;)?\s*}",
+            content,
+        )
+        metadata["properties"] = property_matches
+
+        # Extract attributes
+        attribute_matches = re.findall(r"\[([^\]]+)\]", content)
+        metadata["attributes"] = list(set(attribute_matches))  # Remove duplicates
+
+        return metadata
+
+
+# Registry of metadata extractors
+METADATA_EXTRACTORS: Dict[str, Type[MetadataExtractor]] = {
+    "python": PythonMetadataExtractor,
+    "javascript": JavaScriptMetadataExtractor,
+    "typescript": JavaScriptMetadataExtractor,  # Same as JS
+    "markdown": MarkdownMetadataExtractor,
+    "java": JavaMetadataExtractor,
+    "rust": RustMetadataExtractor,
+    "csharp": CSharpMetadataExtractor,
+    "swift": SwiftMetadataExtractor,
+    "kotlin": JavaScriptMetadataExtractor,  # Reuse JS extractor for now (similar syntax)
+    "yaml": YAMLMetadataExtractor,
+    "yml": YAMLMetadataExtractor,
+    "json": JSONMetadataExtractor,
+    "csv": CSVMetadataExtractor,
+}
+
+
+class DocumentLoaderRepositoryInterface(ABC):
+    """Interface for document loading and chunking."""
+
+    @abstractmethod
+    def load_and_split(
+        self,
+        data_path: str,
+        workspace_name: Optional[str] = None,
+        domain_name: Optional[str] = None,
+    ) -> List[Any]:
+        """Load documents and split into chunks."""
+        pass
+
+
+class DocumentLoaderRepository(DocumentLoaderRepositoryInterface):
+    """
+    Document loader with DDD metadata injection and adaptive text splitting.
+
+    Handles:
+    - Multiple file type support (py, md, json, yaml, pdf, etc.)
+    - Language-aware chunking for code files (swift, cs, kt, java, ts, js, go, etc.)
+    - Adaptive chunking strategy per file type
+    - Metadata injection (workspace, domain, file_type)
+    - Advanced metadata extraction per file type
+    - Domain inference from directory structure
+    - Detection and handling of minified files
+    """
+
+    def _extract_metadata(
+        self, content: str, file_type: str, file_path: str
+    ) -> Dict[str, Any]:
+        """Extract metadata based on file type configuration."""
+        metadata = {}
+
+        # Get file type configuration
+        config = self._config.get("rag", {}).get("file_types", {}).get(file_type, {})
+        if not config.get("extract_metadata", False):
+            return metadata
+
+        metadata_fields = config.get("metadata_fields", [])
+        if not metadata_fields:
+            return metadata
+
+        # Extract metadata using appropriate extractor
+        extractor_class = METADATA_EXTRACTORS.get(file_type)
+        if extractor_class:
+            try:
+                extractor = extractor_class()
+                extracted = extractor.extract(content, file_path)
+
+                # Filter to only requested fields
+                for field in metadata_fields:
+                    if field in extracted:
+                        metadata[field] = extracted[field]
+
+                logger.debug(
+                    f"Extracted metadata for {file_path}: {list(metadata.keys())}"
+                )
+
+            except Exception as e:
+                logger.warning(f"Failed to extract metadata from {file_path}: {e}")
+
+        return metadata
+
+    # Supported file loaders
+    FILE_LOADERS: Dict[str, Type[Any]] = {
+        ".pdf": PyPDFLoader,
+        ".txt": TextLoader,
+        ".md": TextLoader,
+        ".csv": TextLoader,
+        ".json": TextLoader,
+        ".yaml": TextLoader,
+        ".yml": TextLoader,
+        ".py": TextLoader,
+        ".ts": TextLoader,
+        ".js": TextLoader,
+        ".jsx": TextLoader,
+        ".tsx": TextLoader,
+        ".swift": TextLoader,
+        ".cs": TextLoader,
+        ".kt": TextLoader,
+        ".go": TextLoader,
+        ".java": TextLoader,
+        ".cpp": TextLoader,
+        ".c": TextLoader,
+        ".h": TextLoader,
+        ".html": TextLoader,
+        ".css": TextLoader,
+        ".sh": TextLoader,
+        ".bash": TextLoader,
+        ".zsh": TextLoader,
+        ".fish": TextLoader,
+        ".rs": TextLoader,
+        ".rb": TextLoader,
+        ".php": TextLoader,
+    }
+
+    # Mapping of file extensions to Language enum for language-aware splitting
+    EXT_TO_LANGUAGE: Dict[str, Language] = {
+        ".swift": Language.SWIFT,
+        ".cs": Language.CSHARP,
+        ".kt": Language.KOTLIN,
+        ".java": Language.JAVA,
+        ".ts": Language.TS,
+        ".js": Language.JS,
+        ".go": Language.GO,
+        ".cpp": Language.CPP,
+        ".c": Language.C,
+        ".rs": Language.RUST,
+        ".rb": Language.RUBY,
+        ".php": Language.PHP,
+    }
+
+    def __init__(self, config_path: str = "config/rag.yaml") -> None:
+        """
+        Initialize document loader with dynamic chunk sizing per language.
+
+        Args:
+            config_path: Path to RAG configuration file
+        """
+        # Load and store configuration
+        self._config = self._load_rag_config(config_path)
+
+        # Load included content configuration
+        self.included_config = self._config.get("rag", {}).get("included_content", {})
+
+        # Domain map loaders cache
+        self._domain_map_loaders: Dict[str, DomainMapLoader] = {}
+
+        # Track logged warnings to prevent repetition (CD-161 P2 fix)
+        self._logged_warnings: set = set()
+
+        logger.info(
+            "Document loader initialized with dynamic chunk sizing per language"
+        )
+
+    def _load_rag_config(self, config_path: str) -> Dict[str, Any]:
+        """
+        Load complete RAG configuration from YAML file.
+
+        Args:
+            config_path: Path to the RAG configuration file
+
+        Returns:
+            Dictionary with complete RAG configuration
+        """
+        try:
+            config_path = Path(config_path)
+            if not config_path.exists():
+                logger.warning(f"Config file not found: {config_path}")
+                return {}
+
+            with open(config_path, "r") as f:
+                config = yaml.safe_load(f)
+
+            logger.info(f"Loaded RAG configuration from {config_path}")
+
+            # Log included content configuration if present
+            included = config.get("rag", {}).get("included_content", {})
+            if included:
+                logger.debug(
+                    f"Documentation dirs: {included.get('documentation_dirs', [])}"
+                )
+                logger.debug(f"Root files: {included.get('root_files', [])}")
+                logger.debug(f"Code dirs: {included.get('code_dirs', [])}")
+
+            return config
+        except Exception as e:
+            logger.error(f"Failed to load config from {config_path}: {e}")
+            return {}
+
+    def _load_language_config(self, ext: str) -> Dict[str, int]:
+        """
+        Load language-specific chunk configuration dynamically.
+
+        Args:
+            ext: File extension (e.g., '.py', '.cs', '.sh')
+
+        Returns:
+            Dict with chunk_size and chunk_overlap
+        """
+        chunking_config = self._config.get("rag", {}).get("chunking", {})
+
+        # Map extension to language key in config
+        lang_map = {
+            # Python
+            ".py": "python",
+            ".pyi": "python",
+            # Web/JavaScript
+            ".ts": "typescript",
+            ".tsx": "typescript",
+            ".js": "javascript",
+            ".jsx": "javascript",
+            ".mjs": "javascript",
+            # Mobile/Native
+            ".swift": "swift",
+            ".kt": "kotlin",
+            ".kts": "kotlin",
+            # Systems Programming
+            ".go": "go",
+            ".rs": "rust",
+            # Enterprise Languages
+            ".cs": "csharp",
+            ".java": "java",
+            # C/C++ Family
+            ".c": "cpp",
+            ".cpp": "cpp",
+            ".cc": "cpp",
+            ".cxx": "cpp",
+            ".h": "cpp",
+            ".hpp": "cpp",
+            ".hxx": "cpp",
+            # Web Backend
+            ".php": "php",
+            ".rb": "ruby",
+            # Shell Scripts
+            ".sh": "shell",
+            ".bash": "shell",
+            ".zsh": "shell",
+            ".fish": "shell",
+            # Data/Configuration
+            ".json": "json",
+            ".yaml": "yaml",
+            ".yml": "yaml",
+            ".csv": "csv",
+            # Documentation
+            ".md": "markdown",
+            ".txt": "text",
+            # Web Assets
+            ".html": "html",
+            ".htm": "html",
+            ".css": "css",
+            # Other
+            ".tex": "latex",
+        }
+
+        lang_key = lang_map.get(ext, "default")
+        lang_config = chunking_config.get(lang_key, chunking_config.get("default", {}))
+
+        return {
+            "chunk_size": lang_config.get("chunk_size", 500),
+            "chunk_overlap": lang_config.get("chunk_overlap", 50),
+        }
+
+    def _is_minified(self, content: str, ext: str) -> bool:
+        """
+        Detect if content is minified (no line breaks or very long lines).
+
+        Args:
+            content: File content to check
+            ext: File extension
+
+        Returns:
+            True if content appears to be minified
+        """
+        # Extensions that can be minified
+        minifiable = {".js", ".jsx", ".ts", ".tsx", ".json", ".css", ".html"}
+
+        if ext not in minifiable:
+            return False
+
+        # Quick check: if content is all on one line and reasonably long
+        lines = content.split("\n")
+
+        # Remove empty lines for accurate counting
+        non_empty_lines = [line for line in lines if line.strip()]
+
+        if not non_empty_lines:
+            return False
+
+        # If only 1-2 non-empty lines and content is long, likely minified
+        if len(non_empty_lines) <= 2 and len(content) > 200:
+            return True
+
+        # Check average line length
+        avg_line_length = sum(len(line) for line in non_empty_lines) / len(
+            non_empty_lines
+        )
+
+        # Lower threshold for better detection
+        if avg_line_length > 150:  # Reduced from 300
+            return True
+
+        # Check for typical minification patterns
+        if ext in {".js", ".jsx", ".ts", ".tsx"}:
+            # Check if first line has multiple statements (semicolons)
+            first_line = non_empty_lines[0] if non_empty_lines else ""
+            # Reduced threshold from 20 to 5 for better detection
+            if first_line.count(";") > 5:
+                return True
+            # Check for compressed function syntax
+            if "){" in first_line and "}" in first_line and first_line.count("){") > 2:
+                return True
+
+        if ext == ".json":
+            # Minified JSON has no spaces after colons/commas
+            first_line = non_empty_lines[0] if non_empty_lines else ""
+            # Check for lack of formatting
+            if '": ' not in first_line and '":' in first_line and len(first_line) > 100:
+                return True
+            if ", " not in first_line and "," in first_line and len(first_line) > 100:
+                return True
+
+        return False
+
+    def _get_splitter_for_extension(
+        self, ext: str, content: Optional[str] = None
+    ) -> TextSplitter:
+        """
+        Get adaptive text splitter with DYNAMIC language-specific configuration.
+
+        Loads chunk size and overlap dynamically per file type from config.
+
+        Args:
+            ext: File extension (e.g., '.py', '.md', '.json')
+            content: Optional file content for minification detection
+
+        Returns:
+            Configured TextSplitter instance
+        """
+        # Load dynamic config for this extension
+        config = self._load_language_config(ext)
+        chunk_size = config["chunk_size"]
+        chunk_overlap = config["chunk_overlap"]
+
+        logger.debug(f"Splitter for {ext}: size={chunk_size}, overlap={chunk_overlap}")
+
+        # Python: Always use PythonCodeTextSplitter
+        if ext == ".py":
+            return PythonCodeTextSplitter(
+                chunk_size=chunk_size,
+                chunk_overlap=chunk_overlap,
+            )
+
+        # Markdown: Simple character-based splitting (temporarily disabled header hierarchy)
+        if ext == ".md":
+            return RecursiveCharacterTextSplitter(
+                chunk_size=chunk_size,
+                chunk_overlap=chunk_overlap,
+            )
+
+        # Shell scripts: Line-based with special separators
+        if ext in {".sh", ".bash", ".zsh", ".fish"}:
+            return RecursiveCharacterTextSplitter(
+                chunk_size=chunk_size,
+                chunk_overlap=chunk_overlap,
+                separators=["\n\n", "\n", ";", "&&", "||", "|", " ", ""],
+            )
+
+        # YAML: Line-based with structure awareness
+        if ext in {".yaml", ".yml"}:
+            return RecursiveCharacterTextSplitter(
+                chunk_size=chunk_size,
+                chunk_overlap=chunk_overlap,
+                separators=["\n\n", "\n", ": ", "- ", " ", ""],
+            )
+
+        # Code files with language support (Swift, Kotlin, C#, Go, C++, etc.)
+        if ext in self.EXT_TO_LANGUAGE:
+            lang = self.EXT_TO_LANGUAGE[ext]
+
+            # Check if JS/TS files are minified
+            if ext in {".js", ".jsx", ".ts", ".tsx"} and content:
+                if self._is_minified(content, ext):
+                    logger.debug(
+                        f"Minified {ext} detected, using character-based splitter"
+                    )
+                    return RecursiveCharacterTextSplitter(
+                        chunk_size=chunk_size,
+                        chunk_overlap=chunk_overlap,
+                        separators=[";", "}", "{", ",", " ", ""],
+                    )
+
+            # Language-specific splitters for better chunking
+            if ext == ".java":
+                # Java: Split on class/interface boundaries
+                return RecursiveCharacterTextSplitter(
+                    chunk_size=chunk_size,
+                    chunk_overlap=chunk_overlap,
+                    separators=[
+                        "\n\npublic class",
+                        "\n\npublic interface",
+                        "\n\nclass",
+                        "\n\ninterface",
+                        "\n\npublic",
+                        "\n\nprivate",
+                        "\n\nprotected",
+                        "\n\n",
+                        "\n",
+                        " ",
+                        "",
+                    ],
+                )
+            elif ext == ".rs":
+                # Rust: Split on module/impl boundaries
+                return RecursiveCharacterTextSplitter(
+                    chunk_size=chunk_size,
+                    chunk_overlap=chunk_overlap,
+                    separators=[
+                        "\n\npub mod",
+                        "\n\nmod",
+                        "\n\nimpl",
+                        "\n\npub struct",
+                        "\n\npub enum",
+                        "\n\nfn",
+                        "\n\n",
+                        "\n",
+                        " ",
+                        "",
+                    ],
+                )
+            elif ext == ".php":
+                # PHP: Split on class/function boundaries
+                return RecursiveCharacterTextSplitter(
+                    chunk_size=chunk_size,
+                    chunk_overlap=chunk_overlap,
+                    separators=[
+                        "\n\nclass",
+                        "\n\ninterface",
+                        "\n\ntrait",
+                        "\n\nfunction",
+                        "\n\npublic function",
+                        "\n\nprivate function",
+                        "\n\nprotected function",
+                        "\n\n",
+                        "\n",
+                        " ",
+                        "",
+                    ],
+                )
+            elif ext == ".rb":
+                # Ruby: Split on class/module boundaries
+                return RecursiveCharacterTextSplitter(
+                    chunk_size=chunk_size,
+                    chunk_overlap=chunk_overlap,
+                    separators=[
+                        "\n\nclass",
+                        "\n\nmodule",
+                        "\n\npublic",
+                        "\n\nprivate",
+                        "\n\ndef",
+                        "\n\n",
+                        "\n",
+                        " ",
+                        "",
+                    ],
+                )
+            elif ext in {".cs"}:
+                # C#: Split on class/namespace boundaries
+                return RecursiveCharacterTextSplitter(
+                    chunk_size=chunk_size,
+                    chunk_overlap=chunk_overlap,
+                    separators=[
+                        "\n\npublic class",
+                        "\n\ninternal class",
+                        "\n\nclass",
+                        "\n\npublic interface",
+                        "\n\ninterface",
+                        "\n\nnamespace",
+                        "\n\npublic",
+                        "\n\nprivate",
+                        "\n\n",
+                        "\n",
+                        " ",
+                        "",
+                    ],
+                )
+            else:
+                # Use language-aware splitter with DYNAMIC sizes for other languages
+                logger.debug(f"Language-aware splitter for {ext} ({lang.value})")
+                return RecursiveCharacterTextSplitter.from_language(
+                    language=lang,
+                    chunk_size=chunk_size,
+                    chunk_overlap=chunk_overlap,
+                )
+
+        # JSON: Check if minified
+        if ext == ".json":
+            if content and self._is_minified(content, ext):
+                logger.debug(f"Minified JSON detected, using character splitter")
+                return RecursiveCharacterTextSplitter(
+                    chunk_size=chunk_size,
+                    chunk_overlap=chunk_overlap,
+                    separators=["},", "],", ",", " ", ""],
+                )
+
+        # CSV: Line-based splitting
+        if ext == ".csv":
+            return RecursiveCharacterTextSplitter(
+                chunk_size=chunk_size,
+                chunk_overlap=chunk_overlap,
+                separators=["\n", ","],
+            )
+
+        # CSS: Rule-based splitting
+        if ext == ".css":
+            return RecursiveCharacterTextSplitter(
+                chunk_size=chunk_size,
+                chunk_overlap=chunk_overlap,
+                separators=["}\n", "}\n\n", "\n", " ", ""],
+            )
+            return RecursiveCharacterTextSplitter(
+                chunk_size=chunk_size,
+                chunk_overlap=chunk_overlap,
+                separators=["\n", ",", " ", ""],
+            )
+
+        # HTML: Semantic preservation
+        if ext == ".html":
+            if content and self._is_minified(content, ext):
+                logger.debug(f"Minified HTML detected")
+                return RecursiveCharacterTextSplitter(
+                    chunk_size=chunk_size,
+                    chunk_overlap=chunk_overlap,
+                    separators=[">", "</", "/>", " ", ""],
+                )
+            else:
+                logger.debug(f"HTML with semantic preservation")
+                return RecursiveCharacterTextSplitter(
+                    chunk_size=chunk_size,
+                    chunk_overlap=chunk_overlap,
+                    separators=["</div>", "</p>", "</h", ">", "\n\n", "\n", " ", ""],
+                )
+
+        # CSS: Check if minified
+        if ext == ".css":
+            if content and self._is_minified(content, ext):
+                logger.debug(f"Minified CSS detected")
+                return RecursiveCharacterTextSplitter(
+                    chunk_size=chunk_size,
+                    chunk_overlap=chunk_overlap,
+                    separators=["}", ";", "{", " ", ""],
+                )
+            else:
+                logger.debug(f"CSS with line-based splitting")
+                return RecursiveCharacterTextSplitter(
+                    chunk_size=chunk_size,
+                    chunk_overlap=chunk_overlap,
+                    separators=["\n", "}", ";", " ", ""],
+                )
+
+        # LaTeX: Special handling
+        if ext == ".tex":
+            logger.debug(f"LaTeX-specific splitter for {ext}")
+            return LatexTextSplitter(
+                chunk_size=chunk_size,
+                chunk_overlap=chunk_overlap,
+            )
+
+        # Fallback: Adaptive based on common patterns
+        logger.debug(f"Fallback adaptive splitter for {ext}")
+        return RecursiveCharacterTextSplitter(
+            chunk_size=chunk_size,
+            chunk_overlap=chunk_overlap,
+            separators=["\n", "\n\n", ".", "!", "?", ";", ",", " ", ""],
+        )
+
+    def load_and_split(
+        self,
+        data_path: str,
+        workspace_name: Optional[str] = None,
+        domain_name: Optional[str] = None,
+    ) -> List[Any]:
+        """
+        Load documents with DDD metadata and split into chunks.
+        Uses included_content configuration from rag.yaml if available.
+
+        Args:
+            data_path: Root directory containing source files
+            workspace_name: Top-level workspace (auto-inferred if None)
+            domain_name: DDD domain for filtering
+
+        Returns:
+            List of chunked documents with metadata
+
+        Raises:
+            ValueError: If no supported documents found
+        """
+        logger.info(f"Loading documents from {data_path}")
+
+        all_docs: List[Any] = []
+        data_path = Path(data_path).resolve()
+
+        # Infer workspace name from directory if not provided
+        if not workspace_name:
+            workspace_name = os.path.basename(os.path.abspath(data_path))
+            logger.info(f"Inferred workspace: {workspace_name}")
+
+        # Normalize workspace name for case-insensitive lookups
+        workspace_name = normalize_workspace_name(workspace_name)
+        logger.info(f"Normalized workspace name: {workspace_name}")
+
+        # Check if we're indexing the root directory (not a subdirectory)
+        # Use config only for root-level indexing (current dir or '.')
+        should_use_config = self.included_config and (
+            str(data_path) == "."
+            or str(data_path) == os.getcwd()
+            or data_path == Path(".").resolve()
+        )
+
+        if should_use_config:
+            # Use configured directories for root-level indexing
+            # 1. Load documentation directories
+            doc_dirs = self.included_config.get("documentation_dirs", [])
+            for doc_dir in doc_dirs:
+                dir_path = data_path / doc_dir
+                if dir_path.exists() and dir_path.is_dir():
+                    logger.info(f"Loading documentation directory: {doc_dir}")
+                    self._load_directory(
+                        dir_path, all_docs, workspace_name, domain_name
+                    )
+
+            # 2. Load specific root files
+            root_files = self.included_config.get("root_files", [])
+            for root_file in root_files:
+                file_path = data_path / root_file
+                if file_path.exists() and file_path.is_file():
+                    logger.info(f"Loading root file: {root_file}")
+                    self._load_single_file(
+                        file_path, all_docs, workspace_name, domain_name
+                    )
+
+            # 3. Load code directories (src, scripts)
+            code_dirs = self.included_config.get("code_dirs", [])
+            for code_dir in code_dirs:
+                dir_path = data_path / code_dir
+                if dir_path.exists() and dir_path.is_dir():
+                    logger.info(f"Loading code directory: {code_dir}")
+                    self._load_directory(
+                        dir_path, all_docs, workspace_name, domain_name
+                    )
+        else:
+            # Load the specific directory passed (e.g., for skills)
+            logger.info(f"Loading specific directory: {data_path}")
+            self._load_directory(data_path, all_docs, workspace_name, domain_name)
+
+        if not all_docs:
+            raise ValueError(
+                f"No supported documents found in {data_path}. "
+                f"Supported: {', '.join(self.FILE_LOADERS.keys())}"
+            )
+
+        logger.info(f"Total chunks created: {len(all_docs)}")
+        return all_docs
+
+    def _load_directory(
+        self,
+        dir_path: Path,
+        all_docs: List[Any],
+        workspace_name: str,
+        domain_name: Optional[str],
+    ) -> None:
+        """
+        Load all supported files from a directory.
+
+        Args:
+            dir_path: Directory path to load from
+            all_docs: List to append documents to
+            workspace_name: Workspace identifier
+            domain_name: Optional domain filter
+        """
+        for ext, loader_cls in self.FILE_LOADERS.items():
+            glob_pattern = f"**/*{ext}"
+            try:
+                dir_loader = DirectoryLoader(
+                    str(dir_path),
+                    glob=glob_pattern,
+                    loader_cls=loader_cls,
+                    silent_errors=True,
+                )
+                docs_for_ext = list(dir_loader.lazy_load())
+
+                # Split documents using adaptive splitter FIRST
+                if docs_for_ext:
+                    # Try to get content for minification detection
+                    content = None
+                    if (
+                        ext in {".js", ".jsx", ".ts", ".tsx", ".json", ".css", ".html"}
+                        and docs_for_ext
+                    ):
+                        content = (
+                            docs_for_ext[0].page_content
+                            if hasattr(docs_for_ext[0], "page_content")
+                            else None
+                        )
+
+                    splitter_result = self._get_splitter_for_extension(ext, content)
+
+                    # Handle two-stage splitting for markdown (returns tuple)
+                    if isinstance(splitter_result, tuple):
+                        header_splitter, char_splitter = splitter_result
+                        # Stage 1: Split by headers to extract hierarchy
+                        from langchain_core.documents import Document
+
+                        doc_chunks = []
+                        for doc in docs_for_ext:
+                            # Split single document by headers
+                            header_splits = header_splitter.split_text(doc.page_content)
+                            # header_splits is a list of Document objects, NOT tuples
+                            for header_doc in header_splits:
+                                # Extract hierarchy metadata from Document
+                                header_metadata = (
+                                    header_doc.metadata
+                                    if hasattr(header_doc, "metadata")
+                                    else {}
+                                )
+                                chunk_content = (
+                                    header_doc.page_content
+                                    if hasattr(header_doc, "page_content")
+                                    else str(header_doc)
+                                )
+                                # Create Document with hierarchy metadata
+                                chunk_doc = Document(
+                                    page_content=chunk_content,
+                                    metadata=header_metadata.copy(),
+                                )
+                                # Preserve original document metadata (source, etc)
+                                if hasattr(doc, "metadata"):
+                                    chunk_doc.metadata.update(doc.metadata)
+                                    # But keep the hierarchy metadata we extracted
+                                    chunk_doc.metadata.update(header_metadata)
+                                doc_chunks.append(chunk_doc)
+                        # Stage 2: Further split large sections by character
+                        chunks_for_ext = char_splitter.split_documents(doc_chunks)
+                    else:
+                        # Single-stage splitting for other file types
+                        chunks_for_ext = splitter_result.split_documents(docs_for_ext)
+
+                    # FIX: Inject metadata AFTER splitting to ensure it's preserved
+                    for chunk in chunks_for_ext:
+                        chunk.metadata["file_type"] = ext.lstrip(".")
+                        chunk.metadata["workspace_name"] = workspace_name
+
+                        # Infer domain from path if not provided
+                        if domain_name:
+                            chunk.metadata["domain_name"] = domain_name
+                        else:
+                            chunk.metadata["domain_name"] = self._infer_domain(
+                                chunk.metadata.get("source", "")
+                            )
+
+                        # Enrich with domain map metadata (domain_description, etc)
+                        chunk.metadata = self._enrich_with_domain_map(
+                            chunk.metadata, workspace_name
+                        )
+
+                        # Extract section type and build section path (for markdown with hierarchy)
+                        if ext == ".md" and chunk.metadata.get("h3"):
+                            h3 = chunk.metadata.get("h3", "")
+                            # Extract section type from H3 prefix
+                            if h3.startswith("COMO"):
+                                chunk.metadata["section_type"] = "COMO"
+                            elif h3.startswith("DIRETIVA"):
+                                chunk.metadata["section_type"] = "DIRETIVA"
+                            elif h3.startswith("O QUE EVITAR"):
+                                chunk.metadata["section_type"] = "EVITAR"
+                            elif "O QUE" in chunk.metadata.get("h2", ""):
+                                chunk.metadata["section_type"] = "O_QUE"
+                            else:
+                                chunk.metadata["section_type"] = "OTHER"
+
+                            # Build section path for hierarchical navigation
+                            parts = [
+                                chunk.metadata.get("h1", ""),
+                                chunk.metadata.get("h2", ""),
+                                chunk.metadata.get("h3", ""),
+                            ]
+                            chunk.metadata["section_path"] = "/".join(
+                                [p for p in parts if p]
+                            )
+
+                    all_docs.extend(chunks_for_ext)
+                    logger.info(
+                        f"Loaded {len(docs_for_ext)} {ext} files from {dir_path.name}, "
+                        f"split into {len(chunks_for_ext)} chunks"
+                    )
+
+            except Exception as e:
+                logger.warning(f"Failed to load {ext} files from {dir_path}: {e}")
+
+    def _load_single_file(
+        self,
+        file_path: Path,
+        all_docs: List[Any],
+        workspace_name: str,
+        domain_name: Optional[str],
+    ) -> None:
+        """
+        Load a single file.
+
+        Args:
+            file_path: File path to load
+            all_docs: List to append documents to
+            workspace_name: Workspace identifier
+            domain_name: Optional domain filter
+        """
+        ext = file_path.suffix
+        if ext not in self.FILE_LOADERS:
+            logger.warning(f"Unsupported file type: {ext}")
+            return
+
+        try:
+            loader_cls = self.FILE_LOADERS[ext]
+            loader = loader_cls(str(file_path))
+            docs = list(loader.lazy_load())
+
+            # Split documents FIRST
+            if docs:
+                # Try to get content for minification detection
+                content = None
+                if (
+                    ext in {".js", ".jsx", ".ts", ".tsx", ".json", ".css", ".html"}
+                    and docs
+                ):
+                    content = (
+                        docs[0].page_content
+                        if hasattr(docs[0], "page_content")
+                        else None
+                    )
+
+                text_splitter = self._get_splitter_for_extension(ext, content)
+                chunks = text_splitter.split_documents(docs)
+
+                # FIX: Inject metadata AFTER splitting to ensure it's preserved
+                for chunk in chunks:
+                    chunk.metadata["file_type"] = ext.lstrip(".")
+                    chunk.metadata["workspace_name"] = workspace_name
+                    chunk.metadata["domain_name"] = domain_name or self._infer_domain(
+                        file_path, workspace_name
+                    )
+
+                    # Extract additional metadata based on file type
+                    file_type = ext.lstrip(".")
+                    if hasattr(chunk, "page_content") and chunk.page_content:
+                        additional_metadata = self._extract_metadata(
+                            chunk.page_content, file_type, str(file_path)
+                        )
+                        chunk.metadata.update(additional_metadata)
+
+                    # Enrich with domain map metadata (domain_description, etc)
+                    chunk.metadata = self._enrich_with_domain_map(
+                        chunk.metadata, workspace_name
+                    )
+
+                all_docs.extend(chunks)
+                logger.info(
+                    f"Loaded file {file_path.name}, split into {len(chunks)} chunks"
+                )
+
+        except Exception as e:
+            logger.warning(f"Failed to load file {file_path}: {e}")
+
+    @staticmethod
+    def _infer_domain(source_path: str) -> str:
+        """
+        Infer DDD domain from file path.
+
+        Delegates to HierarchicalDomainInference which supports:
+        - force_domain_mapping from config/rag.yaml (highest priority)
+        - DDD structure detection (src/domain|application|infrastructure)
+        - Workspace hierarchy inference
+
+        Args:
+            source_path: File path
+
+        Returns:
+            Inferred domain name
+        """
+        from src.infrastructure.rag.hierarchical_domain_inference import (
+            HierarchicalDomainInference,
+        )
+
+        result = HierarchicalDomainInference.parse_path_hierarchy(source_path)
+        return result["domain_name"]
+
+    def load_from_path(
+        self,
+        path: str,
+        workspace_name: Optional[str] = None,
+        domain_name: Optional[str] = None,
+    ) -> List[Any]:
+        """
+        Load documents from file path (compatibility method).
+
+        Args:
+            path: Directory or file path
+            workspace_name: Optional workspace identifier
+            domain_name: Optional domain filter
+
+        Returns:
+            List of Document entities
+        """
+        # This delegates to the existing load_and_split but without splitting
+        # We'll return the raw documents for chunking later
+        return self.load_and_split(path, workspace_name, domain_name)
+
+    def chunk_document(
+        self, document: Any, chunk_size: int, chunk_overlap: int
+    ) -> List[Any]:
+        """
+        Split document into chunks (compatibility method).
+
+        Args:
+            document: Document to chunk
+            chunk_size: Maximum characters per chunk
+            chunk_overlap: Overlapping characters
+
+        Returns:
+            List of chunks
+        """
+        # Simple fallback implementation
+        from langchain_text_splitters import RecursiveCharacterTextSplitter
+
+        splitter = RecursiveCharacterTextSplitter(
+            chunk_size=chunk_size,
+            chunk_overlap=chunk_overlap,
+            separators=["\n\n", "\n", " ", ""],
+        )
+
+        if hasattr(document, "page_content"):
+            # It's already a Document object
+            return splitter.split_documents([document])
+        else:
+            # It's raw text
+            return splitter.split_text(str(document))
+
+    def _enrich_with_domain_map(
+        self, base_metadata: Dict[str, Any], workspace_name: str
+    ) -> Dict[str, Any]:
+        """
+        Enrich chunk metadata with domain map information.
+
+        Args:
+            base_metadata: Base metadata dict with domain_name
+            workspace_name: Workspace name
+
+        Returns:
+            Enriched metadata with domain_description, dependencies, classes
+        """
+        if not hasattr(self, "_domain_map_loaders"):
+            logger.debug(
+                f"Initializing _domain_map_loaders in {self.__class__.__name__}"
+            )
+            self._domain_map_loaders = {}
+
+        try:
+            # Get or create domain map loader for workspace
+            if workspace_name not in self._domain_map_loaders:
+                loader = DomainMapLoader(workspace_name)
+                if loader.load_domain_map():
+                    self._domain_map_loaders[workspace_name] = loader
+                    logger.info(f"Domain map loaded for workspace: {workspace_name}")
+                else:
+                    # No domain map available - log warning once per workspace (CD-161 P2 fix)
+                    warning_key = f"no_domain_map:{workspace_name}"
+                    if not hasattr(self, '_logged_warnings'):
+                        self._logged_warnings = set()
+                    if warning_key not in self._logged_warnings:
+                        logger.warning(f"No domain map found for workspace: {workspace_name}")
+                        self._logged_warnings.add(warning_key)
+                    # Cache negative result to avoid repeated lookups
+                    self._domain_map_loaders[workspace_name] = None
+                    return base_metadata
+
+            loader = self._domain_map_loaders.get(workspace_name)
+            if loader is None:
+                return base_metadata
+
+            # Enrich metadata using domain map
+            enriched = loader.enrich_metadata(base_metadata)
+
+            return enriched
+
+        except Exception as e:
+            logger.error(f"Error enriching metadata: {e}", exc_info=True)
+            return base_metadata