@musashishao/agent-kit 1.2.2 → 1.4.0

package/.agent/skills/rag-engineering/scripts/chunk_code.py

@@ -0,0 +1,916 @@
+#!/usr/bin/env python3
+"""
+Code Chunker - Intelligent code chunking by logical boundaries.
+
+Splits code files into meaningful chunks (functions, classes, modules)
+instead of arbitrary character/token limits.
+
+Usage:
+    python chunk_code.py --src ./src --output chunks.json --lang auto
+"""
+
+import os
+import re
+import json
+import hashlib
+import argparse
+from pathlib import Path
+from typing import List, Dict, Optional, Tuple
+from dataclasses import dataclass, asdict
+from datetime import datetime
+
+
+@dataclass
+class Chunk:
+    """Represents a code chunk."""
+    id: str
+    content: str
+    metadata: Dict
+
+
+def generate_chunk_id(file_path: str, content: str) -> str:
+    """Generate unique chunk ID."""
+    hash_input = f"{file_path}:{content[:100]}"
+    return hashlib.sha256(hash_input.encode()).hexdigest()[:16]
+
+
+class TypeScriptChunker:
+    """Chunk TypeScript/JavaScript files by logical boundaries."""
+    
+    EXTENSIONS = {'.ts', '.tsx', '.js', '.jsx', '.mjs', '.cjs'}
+    
+    def __init__(self, max_chunk_size: int = 1500, overlap: int = 100):
+        self.max_chunk_size = max_chunk_size  # in characters
+        self.overlap = overlap
+    
+    def chunk_file(self, file_path: Path, base_path: Path) -> List[Chunk]:
+        """Chunk a single file."""
+        try:
+            content = file_path.read_text(encoding='utf-8')
+        except (UnicodeDecodeError, PermissionError):
+            return []
+        
+        relative_path = str(file_path.relative_to(base_path))
+        chunks = []
+        
+        # Try to extract functions and classes
+        code_units = self._extract_code_units(content)
+        
+        if code_units:
+            # Chunk by code units
+            for unit in code_units:
+                chunk_content = unit['code']
+                
+                # If unit is too large, split it
+                if len(chunk_content) > self.max_chunk_size:
+                    sub_chunks = self._split_large_chunk(chunk_content)
+                    for i, sub in enumerate(sub_chunks):
+                        chunks.append(self._create_chunk(
+                            content=sub,
+                            file_path=relative_path,
+                            unit_type=unit['type'],
+                            unit_name=f"{unit['name']}_part{i+1}",
+                            start_line=unit['start_line'],
+                            end_line=unit['end_line']
+                        ))
+                else:
+                    chunks.append(self._create_chunk(
+                        content=chunk_content,
+                        file_path=relative_path,
+                        unit_type=unit['type'],
+                        unit_name=unit['name'],
+                        start_line=unit['start_line'],
+                        end_line=unit['end_line']
+                    ))
+        else:
+            # Fallback: chunk entire file
+            if len(content) <= self.max_chunk_size:
+                chunks.append(self._create_chunk(
+                    content=content,
+                    file_path=relative_path,
+                    unit_type='module',
+                    unit_name=file_path.stem,
+                    start_line=1,
+                    end_line=content.count('\n') + 1
+                ))
+            else:
+                # Split by size with overlap
+                sub_chunks = self._split_large_chunk(content)
+                for i, sub in enumerate(sub_chunks):
+                    chunks.append(self._create_chunk(
+                        content=sub,
+                        file_path=relative_path,
+                        unit_type='module_part',
+                        unit_name=f"{file_path.stem}_part{i+1}",
+                        start_line=1,
+                        end_line=content.count('\n') + 1
+                    ))
+        
+        return chunks
+    
+    def _extract_code_units(self, content: str) -> List[Dict]:
+        """Extract functions and classes from code."""
+        units = []
+        lines = content.split('\n')
+        
+        # Patterns for detecting code blocks
+        patterns = [
+            # Exported function
+            (r'^export\s+(?:async\s+)?function\s+(\w+)', 'function'),
+            # Regular function
+            (r'^(?:async\s+)?function\s+(\w+)', 'function'),
+            # Arrow function (const/let/var)
+            (r'^(?:export\s+)?(?:const|let|var)\s+(\w+)\s*=\s*(?:async\s+)?\([^)]*\)\s*=>', 'function'),
+            # Class
+            (r'^(?:export\s+)?class\s+(\w+)', 'class'),
+            # Interface/Type (TypeScript)
+            (r'^(?:export\s+)?(?:interface|type)\s+(\w+)', 'type'),
+        ]
+        
+        i = 0
+        while i < len(lines):
+            line = lines[i]
+            
+            for pattern, unit_type in patterns:
+                match = re.match(pattern, line.strip())
+                if match:
+                    name = match.group(1)
+                    start_line = i + 1
+                    
+                    # Find the end of this block (matching braces)
+                    end_line = self._find_block_end(lines, i)
+                    
+                    code = '\n'.join(lines[i:end_line])
+                    units.append({
+                        'type': unit_type,
+                        'name': name,
+                        'code': code,
+                        'start_line': start_line,
+                        'end_line': end_line
+                    })
+                    
+                    i = end_line
+                    break
+            else:
+                i += 1
+        
+        return units
+    
+    def _find_block_end(self, lines: List[str], start: int) -> int:
+        """Find the end of a code block by matching braces."""
+        brace_count = 0
+        found_open = False
+        
+        for i in range(start, len(lines)):
+            line = lines[i]
+            for char in line:
+                if char == '{':
+                    brace_count += 1
+                    found_open = True
+                elif char == '}':
+                    brace_count -= 1
+            
+            if found_open and brace_count == 0:
+                return i + 1
+        
+        return len(lines)
+    
+    def _split_large_chunk(self, content: str) -> List[str]:
+        """Split large content into smaller chunks with overlap."""
+        chunks = []
+        lines = content.split('\n')
+        
+        current_chunk = []
+        current_size = 0
+        
+        for line in lines:
+            line_size = len(line) + 1  # +1 for newline
+            
+            if current_size + line_size > self.max_chunk_size and current_chunk:
+                chunks.append('\n'.join(current_chunk))
+                # Keep overlap lines
+                overlap_lines = current_chunk[-3:] if len(current_chunk) > 3 else current_chunk
+                current_chunk = overlap_lines.copy()
+                current_size = sum(len(l) + 1 for l in current_chunk)
+            
+            current_chunk.append(line)
+            current_size += line_size
+        
+        if current_chunk:
+            chunks.append('\n'.join(current_chunk))
+        
+        return chunks
+    
+    def _create_chunk(
+        self,
+        content: str,
+        file_path: str,
+        unit_type: str,
+        unit_name: str,
+        start_line: int,
+        end_line: int
+    ) -> Chunk:
+        """Create a Chunk object with metadata."""
+        return Chunk(
+            id=generate_chunk_id(file_path, content),
+            content=content,
+            metadata={
+                'file_path': file_path,
+                'file_type': 'typescript',
+                'chunk_type': unit_type,
+                'name': unit_name,
+                'start_line': start_line,
+                'end_line': end_line,
+                'char_count': len(content),
+                'line_count': content.count('\n') + 1
+            }
+        )
+
+
+class PythonChunker:
+    """Chunk Python files by logical boundaries."""
+    
+    def __init__(self, max_chunk_size: int = 1500, overlap: int = 100):
+        self.max_chunk_size = max_chunk_size
+        self.overlap = overlap
+    
+    def chunk_file(self, file_path: Path, base_path: Path) -> List[Chunk]:
+        """Chunk a single Python file."""
+        try:
+            content = file_path.read_text(encoding='utf-8')
+        except (UnicodeDecodeError, PermissionError):
+            return []
+        
+        relative_path = str(file_path.relative_to(base_path))
+        chunks = []
+        
+        # Try to use AST for extraction
+        try:
+            import ast
+            tree = ast.parse(content)
+            chunks = self._chunk_with_ast(content, tree, relative_path)
+        except SyntaxError:
+            # Fallback to regex-based extraction
+            chunks = self._chunk_with_regex(content, relative_path)
+        
+        return chunks
+    
+    def _chunk_with_ast(self, content: str, tree, file_path: str) -> List[Chunk]:
+        """Extract chunks using Python AST."""
+        import ast
+        
+        chunks = []
+        lines = content.split('\n')
+        
+        for node in ast.iter_child_nodes(tree):
+            if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)):
+                chunk_type = 'function'
+                name = node.name
+            elif isinstance(node, ast.ClassDef):
+                chunk_type = 'class'
+                name = node.name
+            else:
+                continue
+            
+            # Get source segment
+            start_line = node.lineno
+            end_line = node.end_lineno or start_line
+            code = '\n'.join(lines[start_line - 1:end_line])
+            
+            # Handle large chunks
+            if len(code) > self.max_chunk_size:
+                # For classes, try to split by methods
+                if isinstance(node, ast.ClassDef):
+                    method_chunks = self._split_class_by_methods(
+                        code, node, lines, file_path, name
+                    )
+                    chunks.extend(method_chunks)
+                else:
+                    # Split function (rare case)
+                    sub_chunks = self._split_large_content(code)
+                    for i, sub in enumerate(sub_chunks):
+                        chunks.append(self._create_chunk(
+                            content=sub,
+                            file_path=file_path,
+                            chunk_type=f'{chunk_type}_part',
+                            name=f'{name}_part{i+1}',
+                            start_line=start_line,
+                            end_line=end_line
+                        ))
+            else:
+                chunks.append(self._create_chunk(
+                    content=code,
+                    file_path=file_path,
+                    chunk_type=chunk_type,
+                    name=name,
+                    start_line=start_line,
+                    end_line=end_line
+                ))
+        
+        # If no chunks extracted, treat as module
+        if not chunks:
+            if len(content) <= self.max_chunk_size:
+                chunks.append(self._create_chunk(
+                    content=content,
+                    file_path=file_path,
+                    chunk_type='module',
+                    name=Path(file_path).stem,
+                    start_line=1,
+                    end_line=len(lines)
+                ))
+            else:
+                sub_chunks = self._split_large_content(content)
+                for i, sub in enumerate(sub_chunks):
+                    chunks.append(self._create_chunk(
+                        content=sub,
+                        file_path=file_path,
+                        chunk_type='module_part',
+                        name=f'{Path(file_path).stem}_part{i+1}',
+                        start_line=1,
+                        end_line=len(lines)
+                    ))
+        
+        return chunks
+    
+    def _split_class_by_methods(
+        self,
+        code: str,
+        class_node,
+        lines: List[str],
+        file_path: str,
+        class_name: str
+    ) -> List[Chunk]:
+        """Split a large class into method-level chunks."""
+        import ast
+        
+        chunks = []
+        
+        # First chunk: class definition + docstring + class variables
+        class_start = class_node.lineno - 1
+        first_method_line = None
+        
+        for item in class_node.body:
+            if isinstance(item, (ast.FunctionDef, ast.AsyncFunctionDef)):
+                first_method_line = item.lineno - 1
+                break
+        
+        if first_method_line:
+            class_header = '\n'.join(lines[class_start:first_method_line])
+            chunks.append(self._create_chunk(
+                content=class_header,
+                file_path=file_path,
+                chunk_type='class_header',
+                name=f'{class_name}_header',
+                start_line=class_start + 1,
+                end_line=first_method_line
+            ))
+        
+        # Each method as separate chunk
+        for item in class_node.body:
+            if isinstance(item, (ast.FunctionDef, ast.AsyncFunctionDef)):
+                start = item.lineno - 1
+                end = item.end_lineno or start + 1
+                method_code = '\n'.join(lines[start:end])
+                
+                chunks.append(self._create_chunk(
+                    content=method_code,
+                    file_path=file_path,
+                    chunk_type='method',
+                    name=f'{class_name}.{item.name}',
+                    start_line=start + 1,
+                    end_line=end
+                ))
+        
+        return chunks
+    
+    def _chunk_with_regex(self, content: str, file_path: str) -> List[Chunk]:
+        """Fallback regex-based chunking."""
+        chunks = []
+        lines = content.split('\n')
+        
+        # Simple pattern for function/class definitions
+        def_pattern = re.compile(r'^(class|def|async\s+def)\s+(\w+)')
+        
+        current_block = []
+        current_type = None
+        current_name = None
+        block_start = 0
+        base_indent = None
+        
+        for i, line in enumerate(lines):
+            match = def_pattern.match(line)
+            
+            if match:
+                # Save previous block
+                if current_block and current_name:
+                    chunks.append(self._create_chunk(
+                        content='\n'.join(current_block),
+                        file_path=file_path,
+                        chunk_type=current_type or 'code',
+                        name=current_name,
+                        start_line=block_start + 1,
+                        end_line=i
+                    ))
+                
+                # Start new block
+                current_type = 'class' if match.group(1) == 'class' else 'function'
+                current_name = match.group(2)
+                current_block = [line]
+                block_start = i
+                base_indent = len(line) - len(line.lstrip())
+            elif current_block:
+                # Check if still in block (indent-based)
+                stripped = line.lstrip()
+                if stripped:  # Non-empty line
+                    current_indent = len(line) - len(stripped)
+                    if current_indent <= base_indent and not line.strip().startswith('#'):
+                        # End of block
+                        chunks.append(self._create_chunk(
+                            content='\n'.join(current_block),
+                            file_path=file_path,
+                            chunk_type=current_type or 'code',
+                            name=current_name,
+                            start_line=block_start + 1,
+                            end_line=i
+                        ))
+                        current_block = []
+                        current_name = None
+                        current_type = None
+                    else:
+                        current_block.append(line)
+                else:
+                    current_block.append(line)
+        
+        # Don't forget last block
+        if current_block and current_name:
+            chunks.append(self._create_chunk(
+                content='\n'.join(current_block),
+                file_path=file_path,
+                chunk_type=current_type or 'code',
+                name=current_name,
+                start_line=block_start + 1,
+                end_line=len(lines)
+            ))
+        
+        return chunks
+    
+    def _split_large_content(self, content: str) -> List[str]:
+        """Split large content with overlap."""
+        chunks = []
+        lines = content.split('\n')
+        
+        current_chunk = []
+        current_size = 0
+        
+        for line in lines:
+            line_size = len(line) + 1
+            
+            if current_size + line_size > self.max_chunk_size and current_chunk:
+                chunks.append('\n'.join(current_chunk))
+                overlap_lines = current_chunk[-3:] if len(current_chunk) > 3 else current_chunk
+                current_chunk = overlap_lines.copy()
+                current_size = sum(len(l) + 1 for l in current_chunk)
+            
+            current_chunk.append(line)
+            current_size += line_size
+        
+        if current_chunk:
+            chunks.append('\n'.join(current_chunk))
+        
+        return chunks
+    
+    def _create_chunk(
+        self,
+        content: str,
+        file_path: str,
+        chunk_type: str,
+        name: str,
+        start_line: int,
+        end_line: int
+    ) -> Chunk:
+        """Create a Chunk object."""
+        return Chunk(
+            id=generate_chunk_id(file_path, content),
+            content=content,
+            metadata={
+                'file_path': file_path,
+                'file_type': 'python',
+                'chunk_type': chunk_type,
+                'name': name,
+                'start_line': start_line,
+                'end_line': end_line,
+                'char_count': len(content),
+                'line_count': content.count('\n') + 1
+            }
+        )
+
+
+class MarkdownChunker:
+    """Chunk Markdown files by heading sections."""
+    
+    EXTENSIONS = {'.md', '.mdx', '.markdown'}
+    
+    def __init__(self, max_chunk_size: int = 2000, overlap: int = 100):
+        self.max_chunk_size = max_chunk_size
+        self.overlap = overlap
+    
+    def chunk_file(self, file_path: Path, base_path: Path) -> List[Chunk]:
+        """Chunk a Markdown file by headings."""
+        try:
+            content = file_path.read_text(encoding='utf-8')
+        except (UnicodeDecodeError, PermissionError):
+            return []
+        
+        relative_path = str(file_path.relative_to(base_path))
+        chunks = []
+        
+        # Extract sections by heading
+        sections = self._extract_sections(content)
+        
+        if sections:
+            for section in sections:
+                chunk_content = section['content']
+                
+                # If section is too large, split it
+                if len(chunk_content) > self.max_chunk_size:
+                    sub_chunks = self._split_by_paragraphs(chunk_content)
+                    for i, sub in enumerate(sub_chunks):
+                        chunks.append(self._create_chunk(
+                            content=sub,
+                            file_path=relative_path,
+                            section_title=f"{section['title']}_part{i+1}",
+                            heading_level=section['level'],
+                            parent_headings=section['parents'],
+                            start_line=section['start_line'],
+                            end_line=section['end_line']
+                        ))
+                else:
+                    chunks.append(self._create_chunk(
+                        content=chunk_content,
+                        file_path=relative_path,
+                        section_title=section['title'],
+                        heading_level=section['level'],
+                        parent_headings=section['parents'],
+                        start_line=section['start_line'],
+                        end_line=section['end_line']
+                    ))
+        else:
+            # No headings found, treat entire file as one chunk or split by paragraphs
+            if len(content) <= self.max_chunk_size:
+                chunks.append(self._create_chunk(
+                    content=content,
+                    file_path=relative_path,
+                    section_title=file_path.stem,
+                    heading_level=0,
+                    parent_headings=[],
+                    start_line=1,
+                    end_line=content.count('\n') + 1
+                ))
+            else:
+                sub_chunks = self._split_by_paragraphs(content)
+                for i, sub in enumerate(sub_chunks):
+                    chunks.append(self._create_chunk(
+                        content=sub,
+                        file_path=relative_path,
+                        section_title=f"{file_path.stem}_part{i+1}",
+                        heading_level=0,
+                        parent_headings=[],
+                        start_line=1,
+                        end_line=content.count('\n') + 1
+                    ))
+        
+        return chunks
+    
+    def _extract_sections(self, content: str) -> List[Dict]:
+        """Extract sections based on Markdown headings."""
+        sections = []
+        lines = content.split('\n')
+        
+        heading_pattern = re.compile(r'^(#{1,6})\s+(.+)$')
+        
+        current_section = None
+        current_lines = []
+        parent_stack = []  # Track parent headings for context
+        
+        for i, line in enumerate(lines):
+            match = heading_pattern.match(line)
+            
+            if match:
+                # Save previous section
+                if current_section:
+                    current_section['content'] = '\n'.join(current_lines).strip()
+                    current_section['end_line'] = i
+                    if current_section['content']:
+                        sections.append(current_section)
+                
+                # Update parent stack
+                level = len(match.group(1))
+                title = match.group(2).strip()
+                
+                # Pop parents that are same or deeper level
+                while parent_stack and parent_stack[-1]['level'] >= level:
+                    parent_stack.pop()
+                
+                parents = [p['title'] for p in parent_stack]
+                
+                # Start new section
+                current_section = {
+                    'title': title,
+                    'level': level,
+                    'parents': parents.copy(),
+                    'start_line': i + 1,
+                    'end_line': i + 1,
+                    'content': ''
+                }
+                current_lines = [line]
+                
+                # Add this heading to parent stack
+                parent_stack.append({'level': level, 'title': title})
+            elif current_section:
+                current_lines.append(line)
+        
+        # Don't forget last section
+        if current_section:
+            current_section['content'] = '\n'.join(current_lines).strip()
+            current_section['end_line'] = len(lines)
+            if current_section['content']:
+                sections.append(current_section)
+        
+        return sections
+    
+    def _split_by_paragraphs(self, content: str) -> List[str]:
+        """Split content by paragraphs when too large."""
+        chunks = []
+        paragraphs = re.split(r'\n\s*\n', content)
+        
+        current_chunk = []
+        current_size = 0
+        
+        for para in paragraphs:
+            para_size = len(para) + 2  # +2 for paragraph break
+            
+            if current_size + para_size > self.max_chunk_size and current_chunk:
+                chunks.append('\n\n'.join(current_chunk))
+                current_chunk = []
+                current_size = 0
+            
+            current_chunk.append(para)
+            current_size += para_size
+        
+        if current_chunk:
+            chunks.append('\n\n'.join(current_chunk))
+        
+        return chunks
+    
+    def _create_chunk(
+        self,
+        content: str,
+        file_path: str,
+        section_title: str,
+        heading_level: int,
+        parent_headings: List[str],
+        start_line: int,
+        end_line: int
+    ) -> Chunk:
+        """Create a Chunk object with rich context."""
+        # Build context string for better retrieval
+        context_path = ' > '.join(parent_headings + [section_title]) if parent_headings else section_title
+        
+        return Chunk(
+            id=generate_chunk_id(file_path, content),
+            content=content,
+            metadata={
+                'file_path': file_path,
+                'file_type': 'markdown',
+                'chunk_type': f'heading_{heading_level}' if heading_level > 0 else 'paragraph',
+                'name': section_title,
+                'context_path': context_path,
+                'heading_level': heading_level,
+                'parent_headings': parent_headings,
+                'start_line': start_line,
+                'end_line': end_line,
+                'char_count': len(content),
+                'line_count': content.count('\n') + 1
+            }
+        )
+
+
+class TextChunker:
+    """Chunk plain text files by paragraphs."""
+    
+    EXTENSIONS = {'.txt', '.text', '.log'}
+    
+    def __init__(self, max_chunk_size: int = 1500, overlap: int = 100):
+        self.max_chunk_size = max_chunk_size
+        self.overlap = overlap
+    
+    def chunk_file(self, file_path: Path, base_path: Path) -> List[Chunk]:
+        """Chunk a text file by paragraphs."""
+        try:
+            content = file_path.read_text(encoding='utf-8')
+        except (UnicodeDecodeError, PermissionError):
+            return []
+        
+        relative_path = str(file_path.relative_to(base_path))
+        chunks = []
+        
+        if len(content) <= self.max_chunk_size:
+            chunks.append(self._create_chunk(
+                content=content,
+                file_path=relative_path,
+                name=file_path.stem,
+                start_line=1,
+                end_line=content.count('\n') + 1
+            ))
+        else:
+            # Split by paragraphs
+            paragraphs = re.split(r'\n\s*\n', content)
+            current_chunk = []
+            current_size = 0
+            chunk_index = 0
+            
+            for para in paragraphs:
+                para_size = len(para) + 2
+                
+                if current_size + para_size > self.max_chunk_size and current_chunk:
+                    chunk_index += 1
+                    chunks.append(self._create_chunk(
+                        content='\n\n'.join(current_chunk),
+                        file_path=relative_path,
+                        name=f"{file_path.stem}_part{chunk_index}",
+                        start_line=1,
+                        end_line=content.count('\n') + 1
+                    ))
+                    current_chunk = []
+                    current_size = 0
+                
+                current_chunk.append(para)
+                current_size += para_size
+            
+            if current_chunk:
+                chunk_index += 1
+                chunks.append(self._create_chunk(
+                    content='\n\n'.join(current_chunk),
+                    file_path=relative_path,
+                    name=f"{file_path.stem}_part{chunk_index}",
+                    start_line=1,
+                    end_line=content.count('\n') + 1
+                ))
+        
+        return chunks
+    
+    def _create_chunk(
+        self,
+        content: str,
+        file_path: str,
+        name: str,
+        start_line: int,
+        end_line: int
+    ) -> Chunk:
+        """Create a Chunk object."""
+        return Chunk(
+            id=generate_chunk_id(file_path, content),
+            content=content,
+            metadata={
+                'file_path': file_path,
+                'file_type': 'text',
+                'chunk_type': 'paragraph',
+                'name': name,
+                'start_line': start_line,
+                'end_line': end_line,
+                'char_count': len(content),
+                'line_count': content.count('\n') + 1
+            }
+        )
+
+
+def chunk_directory(
+    src_path: Path,
+    lang: str,
+    exclude_patterns: List[str],
+    max_chunk_size: int = 1500
+) -> List[Chunk]:
+    """Chunk all files in a directory - Universal support for Code, Docs, and Text."""
+    all_chunks = []
+    
+    # Initialize all chunkers
+    ts_chunker = TypeScriptChunker(max_chunk_size=max_chunk_size)
+    py_chunker = PythonChunker(max_chunk_size=max_chunk_size)
+    md_chunker = MarkdownChunker(max_chunk_size=max_chunk_size)
+    txt_chunker = TextChunker(max_chunk_size=max_chunk_size)
+    
+    # Map extensions to chunkers
+    extension_map = {}
+    for ext in TypeScriptChunker.EXTENSIONS:
+        extension_map[ext] = ts_chunker
+    extension_map['.py'] = py_chunker
+    for ext in MarkdownChunker.EXTENSIONS:
+        extension_map[ext] = md_chunker
+    for ext in TextChunker.EXTENSIONS:
+        extension_map[ext] = txt_chunker
+    
+    # All supported extensions
+    all_extensions = set(extension_map.keys())
+    
+    # Process files
+    for file_path in src_path.rglob('*'):
+        if not file_path.is_file():
+            continue
+        
+        # Check if extension is supported
+        if file_path.suffix not in all_extensions:
+            continue
+        
+        # Check exclusions
+        path_str = str(file_path)
+        if any(pattern in path_str for pattern in exclude_patterns):
+            continue
+        
+        # Select appropriate chunker
+        chunker = extension_map.get(file_path.suffix)
+        if chunker:
+            chunks = chunker.chunk_file(file_path, src_path)
+            all_chunks.extend(chunks)
+    
+    return all_chunks
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description='Universal Chunker - Code, Markdown, and Text files'
+    )
+    parser.add_argument('--src', default='./src', help='Source directory')
+    parser.add_argument('--output', default='.agent/rag/chunks.json', help='Output file')
+    parser.add_argument('--lang', choices=['typescript', 'python', 'auto', 'universal'], 
+                       default='universal', help='Language mode (universal = all file types)')
+    parser.add_argument('--max-size', type=int, default=1500, help='Max chunk size in chars')
+    parser.add_argument('--exclude', default='node_modules,__pycache__,.git,dist,build,.agent',
+                       help='Patterns to exclude')
+    
+    args = parser.parse_args()
+    
+    src_path = Path(args.src).resolve()
+    output_path = Path(args.output)
+    exclude_patterns = args.exclude.split(',')
+    
+    if not src_path.exists():
+        print(f"Error: Source directory '{src_path}' does not exist")
+        return 1
+    
+    # Mode selection
+    lang = args.lang
+    if lang == 'auto':
+        ts_files = list(src_path.rglob('*.ts')) + list(src_path.rglob('*.tsx'))
+        py_files = list(src_path.rglob('*.py'))
+        lang = 'typescript' if len(ts_files) >= len(py_files) else 'python'
+        print(f"Auto-detected language: {lang}")
+    elif lang == 'universal':
+        print("Universal mode: Processing Code, Markdown, and Text files")
+    
+    print(f"Chunking {src_path}...")
+    chunks = chunk_directory(src_path, lang, exclude_patterns, args.max_size)
+    print(f"Created {len(chunks)} chunks")
+    
+    if len(chunks) == 0:
+        print("Warning: No chunks created. Check if source directory has supported files.")
+        print("Supported: .ts, .tsx, .js, .jsx, .py, .md, .mdx, .txt")
+    
+    # Save output
+    output_path.parent.mkdir(parents=True, exist_ok=True)
+    
+    # Categorize chunks by file type
+    file_types = {}
+    for c in chunks:
+        ft = c.metadata.get('file_type', 'unknown')
+        file_types[ft] = file_types.get(ft, 0) + 1
+    
+    output_data = {
+        'metadata': {
+            'generated_at': datetime.now().isoformat(),
+            'source_path': str(src_path),
+            'mode': lang,
+            'total_chunks': len(chunks),
+            'max_chunk_size': args.max_size,
+            'file_types': file_types
+        },
+        'chunks': [asdict(c) for c in chunks]
+    }
+    
+    output_path.write_text(json.dumps(output_data, indent=2), encoding='utf-8')
+    print(f"Saved to: {output_path}")
+    
+    # Print summary
+    chunk_types = {}
+    for c in chunks:
+        t = c.metadata['chunk_type']
+        chunk_types[t] = chunk_types.get(t, 0) + 1
+    
+    print("\nChunk summary:")
+    for t, count in sorted(chunk_types.items()):
+        print(f"  {t}: {count}")
+    
+    return 0
+
+
+if __name__ == '__main__':
+    exit(main())