@voodocs/cli 0.1.0

package/lib/darkarts/annotations/parser.py

@@ -0,0 +1,618 @@
+"""
+DarkArts Annotation Parser
+
+Extracts DarkArts annotations from source code in multiple languages.
+"""
+
+import re
+import ast
+import json
+import subprocess
+from typing import List, Optional, Dict, Any
+from pathlib import Path
+
+from .types import (
+    ParsedAnnotations,
+    ModuleAnnotation,
+    ClassAnnotation,
+    FunctionAnnotation,
+    ComplexityAnnotation,
+    ErrorCase,
+    StateTransition,
+    Language,
+)
+
+try:
+    from darkarts.exceptions import ParserNotBuiltError
+    from darkarts.parser_checker import check_parser_for_language
+except ImportError:
+    # Fallback if exceptions module not available
+    class ParserNotBuiltError(Exception):
+        pass
+    def check_parser_for_language(lang):
+        return True
+
+
+class AnnotationParser:
+    """Parser for DarkArts annotations."""
+    
+    # Regex patterns for different languages
+    PATTERNS = {
+        Language.PYTHON: r'"""@voodocs\s*(.*?)\s*"""',
+        Language.TYPESCRIPT: r'/\*@voodocs\s*(.*?)\s*\*/',
+        Language.JAVASCRIPT: r'/\*@voodocs\s*(.*?)\s*\*/',
+        Language.JAVA: r'/\*@voodocs\s*(.*?)\s*\*/',
+        Language.CPP: r'/\*@voodocs\s*(.*?)\s*\*/',
+        Language.CSHARP: r'/\*@voodocs\s*(.*?)\s*\*/',
+        Language.GO: r'/\*@voodocs\s*(.*?)\s*\*/',
+        Language.RUST: r'/\*@voodocs\s*(.*?)\s*\*/',
+    }
+    
+    def __init__(self):
+        pass
+    
+    def detect_language(self, source_file: str) -> Language:
+        """Detect programming language from file extension."""
+        ext = Path(source_file).suffix.lower()
+        
+        mapping = {
+            '.py': Language.PYTHON,
+            '.ts': Language.TYPESCRIPT,
+            '.tsx': Language.TYPESCRIPT,
+            '.js': Language.JAVASCRIPT,
+            '.jsx': Language.JAVASCRIPT,
+            '.java': Language.JAVA,
+            '.cpp': Language.CPP,
+            '.cc': Language.CPP,
+            '.cxx': Language.CPP,
+            '.h': Language.CPP,
+            '.hpp': Language.CPP,
+            '.cs': Language.CSHARP,
+            '.go': Language.GO,
+            '.rs': Language.RUST,
+        }
+        
+        return mapping.get(ext, Language.PYTHON)
+    
+    def parse_file(self, source_file: str) -> ParsedAnnotations:
+        """Parse annotations from a source file."""
+        with open(source_file, 'r', encoding='utf-8') as f:
+            source_code = f.read()
+        
+        language = self.detect_language(source_file)
+        return self.parse_source(source_code, source_file, language)
+    
+    def parse_source(
+        self, 
+        source_code: str, 
+        source_file: str = "<string>",
+        language: Optional[Language] = None
+    ) -> ParsedAnnotations:
+        """Parse annotations from source code."""
+        if language is None:
+            language = self.detect_language(source_file)
+        
+        # Extract all annotation blocks
+        pattern = self.PATTERNS[language]
+        matches = re.finditer(pattern, source_code, re.DOTALL)
+        
+        annotations = []
+        for match in matches:
+            annotation_text = match.group(1)
+            line_number = source_code[:match.start()].count('\n') + 1
+            annotations.append({
+                'text': annotation_text,
+                'line': line_number,
+                'start': match.start(),
+                'end': match.end(),
+            })
+        
+        # Parse based on language
+        if language == Language.PYTHON:
+            return self._parse_python(source_code, source_file, annotations)
+        elif language in (Language.TYPESCRIPT, Language.JAVASCRIPT):
+            return self._parse_typescript(source_file)
+        else:
+            return self._parse_generic(source_code, source_file, language, annotations)
+    
+    def _parse_python(
+        self, 
+        source_code: str, 
+        source_file: str,
+        annotations: List[Dict]
+    ) -> ParsedAnnotations:
+        """Parse Python source code with AST."""
+        try:
+            tree = ast.parse(source_code)
+        except SyntaxError:
+            # Fallback to generic parsing
+            return self._parse_generic(source_code, source_file, Language.PYTHON, annotations)
+        
+        module = ModuleAnnotation(
+            name=Path(source_file).stem,
+            source_file=source_file,
+            language=Language.PYTHON,
+        )
+        
+        # Map annotations to AST nodes
+        annotation_map = self._build_annotation_map(source_code, annotations)
+        
+        # Parse module-level annotations (first few lines)
+        for line in range(1, 20):
+            if line in annotation_map:
+                parsed = self._parse_annotation_text(annotation_map[line])
+                if 'module_purpose' in parsed or 'dependencies' in parsed or 'assumptions' in parsed:
+                    self._parse_module_annotation(annotation_map[line], module)
+                    break
+        
+        # Parse classes and functions
+        for node in ast.walk(tree):
+            if isinstance(node, ast.ClassDef):
+                cls_annotation = self._parse_class_node(node, annotation_map, source_file)
+                if cls_annotation:
+                    module.classes.append(cls_annotation)
+            
+            elif isinstance(node, ast.FunctionDef):
+                # Only top-level functions (not methods)
+                if isinstance(node, ast.FunctionDef) and node.col_offset == 0:
+                    func_annotation = self._parse_function_node(node, annotation_map, source_file)
+                    if func_annotation:
+                        module.functions.append(func_annotation)
+        
+        return ParsedAnnotations(
+            module=module,
+            language=Language.PYTHON,
+            source_file=source_file,
+        )
+    
+    def _build_annotation_map(self, source_code: str, annotations: List[Dict]) -> Dict[int, str]:
+        """Build a map from line numbers to annotation text."""
+        annotation_map = {}
+        for ann in annotations:
+            annotation_map[ann['line']] = ann['text']
+        return annotation_map
+    
+    def _parse_class_node(
+        self, 
+        node: ast.ClassDef, 
+        annotation_map: Dict[int, str],
+        source_file: str
+    ) -> Optional[ClassAnnotation]:
+        """Parse a class node."""
+        # Look for annotation in docstring (after class definition)
+        class_line = node.lineno
+        annotation_text = None
+        
+        # Check a few lines after the class definition (docstring)
+        for offset in range(1, 10):
+            if class_line + offset in annotation_map:
+                annotation_text = annotation_map[class_line + offset]
+                break
+        
+        if not annotation_text:
+            return None
+        
+        cls_annotation = ClassAnnotation(
+            name=node.name,
+            line_number=class_line,
+            source_file=source_file,
+            raw_annotation=annotation_text,
+        )
+        
+        # Parse class annotation
+        parsed = self._parse_annotation_text(annotation_text)
+        
+        if 'class_invariants' in parsed:
+            cls_annotation.class_invariants = parsed['class_invariants']
+        
+        if 'state_transitions' in parsed:
+            cls_annotation.state_transitions = [
+                self._parse_state_transition(st) 
+                for st in parsed['state_transitions']
+            ]
+        
+        # Parse methods
+        for item in node.body:
+            if isinstance(item, ast.FunctionDef):
+                method_annotation = self._parse_function_node(item, annotation_map, source_file)
+                if method_annotation:
+                    cls_annotation.methods.append(method_annotation)
+        
+        return cls_annotation
+    
+    def _parse_function_node(
+        self, 
+        node: ast.FunctionDef, 
+        annotation_map: Dict[int, str],
+        source_file: str
+    ) -> Optional[FunctionAnnotation]:
+        """Parse a function node."""
+        # Look for annotation in docstring (after function definition)
+        func_line = node.lineno
+        annotation_text = None
+        
+        # Check a few lines after the function definition (docstring)
+        for offset in range(1, 10):
+            if func_line + offset in annotation_map:
+                annotation_text = annotation_map[func_line + offset]
+                break
+        
+        if not annotation_text:
+            return None
+        
+        func_annotation = FunctionAnnotation(
+            name=node.name,
+            line_number=func_line,
+            source_file=source_file,
+            raw_annotation=annotation_text,
+        )
+        
+        # Parse function annotation
+        parsed = self._parse_annotation_text(annotation_text)
+        
+        if 'solve' in parsed:
+            func_annotation.solve = parsed['solve']
+        
+        if 'preconditions' in parsed:
+            func_annotation.preconditions = parsed['preconditions']
+        
+        if 'postconditions' in parsed:
+            func_annotation.postconditions = parsed['postconditions']
+        
+        if 'invariants' in parsed:
+            func_annotation.invariants = parsed['invariants']
+        
+        if 'optimize' in parsed:
+            func_annotation.optimize = parsed['optimize']
+        
+        if 'complexity' in parsed:
+            func_annotation.complexity = self._parse_complexity(parsed['complexity'])
+        
+        if 'error_cases' in parsed:
+            func_annotation.error_cases = [
+                self._parse_error_case(ec) 
+                for ec in parsed['error_cases']
+            ]
+        
+        return func_annotation
+    
+    def _parse_module_annotation(self, annotation_text: str, module: ModuleAnnotation):
+        """Parse module-level annotation."""
+        parsed = self._parse_annotation_text(annotation_text)
+        
+        if 'module_purpose' in parsed:
+            module.module_purpose = parsed['module_purpose']
+        
+        if 'dependencies' in parsed:
+            module.dependencies = parsed['dependencies']
+        
+        if 'assumptions' in parsed:
+            module.assumptions = parsed['assumptions']
+    
+    def _parse_annotation_text(self, text: str) -> Dict[str, Any]:
+        """Parse annotation text into structured data."""
+        result = {}
+        
+        # Simple key-value parsing
+        lines = text.strip().split('\n')
+        current_key = None
+        current_value = []
+        
+        for line in lines:
+            line = line.strip()
+            if not line:
+                continue
+            
+            # Check if this is a key line
+            if ':' in line and not line.startswith('"') and not line.startswith('['):
+                # Save previous key-value
+                if current_key:
+                    result[current_key] = self._parse_value(current_key, current_value)
+                
+                # Start new key
+                key, value = line.split(':', 1)
+                current_key = key.strip()
+                current_value = [value.strip()]
+            else:
+                # Continuation of current value
+                if current_key:
+                    current_value.append(line)
+        
+        # Save last key-value
+        if current_key:
+            result[current_key] = self._parse_value(current_key, current_value)
+        
+        return result
+    
+    def _parse_value(self, key: str, value_lines: List[str]) -> Any:
+        """Parse a value based on its key."""
+        text = ' '.join(value_lines).strip()
+        
+        # Remove quotes
+        if text.startswith('"') and text.endswith('"'):
+            return text[1:-1]
+        
+        # Parse lists
+        if text.startswith('['):
+            return self._parse_list(text)
+        
+        # Parse objects
+        if text.startswith('{'):
+            return self._parse_object(text)
+        
+        return text
+    
+    def _parse_list(self, text: str) -> List[str]:
+        """Parse a list value."""
+        # Remove brackets
+        text = text.strip('[]').strip()
+        
+        # Split by commas (simple approach)
+        items = []
+        current = []
+        depth = 0
+        in_string = False
+        
+        for char in text:
+            if char == '"' and (not current or current[-1] != '\\'):
+                in_string = not in_string
+            elif char in '{[' and not in_string:
+                depth += 1
+            elif char in '}]' and not in_string:
+                depth -= 1
+            elif char == ',' and depth == 0 and not in_string:
+                items.append(''.join(current).strip().strip('"'))
+                current = []
+                continue
+            
+            current.append(char)
+        
+        if current:
+            items.append(''.join(current).strip().strip('"'))
+        
+        return items
+    
+    def _parse_object(self, text: str) -> Dict[str, Any]:
+        """Parse an object value."""
+        # Simple object parsing
+        result = {}
+        text = text.strip('{}').strip()
+        
+        # Split by commas
+        pairs = text.split(',')
+        for pair in pairs:
+            if ':' in pair:
+                key, value = pair.split(':', 1)
+                result[key.strip().strip('"')] = value.strip().strip('"')
+        
+        return result
+    
+    def _parse_complexity(self, data: Any) -> ComplexityAnnotation:
+        """Parse complexity annotation."""
+        if isinstance(data, dict):
+            return ComplexityAnnotation(
+                time=data.get('time', 'O(1)'),
+                space=data.get('space', 'O(1)'),
+                best_case=data.get('best_case'),
+                worst_case=data.get('worst_case'),
+                average_case=data.get('average_case'),
+            )
+        return ComplexityAnnotation()
+    
+    def _parse_error_case(self, text: str) -> ErrorCase:
+        """Parse error case annotation."""
+        # Format: "condition → ErrorType"
+        if '→' in text:
+            condition, error_type = text.split('→', 1)
+            return ErrorCase(
+                condition=condition.strip(),
+                error_type=error_type.strip(),
+            )
+        return ErrorCase(condition=text, error_type="Error")
+    
+    def _parse_state_transition(self, text: str) -> StateTransition:
+        """Parse state transition annotation."""
+        # Format: "FROM → TO: condition" or "FROM → TO"
+        if ':' in text:
+            transition, condition = text.split(':', 1)
+            condition = condition.strip()
+        else:
+            transition = text
+            condition = None
+        
+        if '→' in transition:
+            from_state, to_state = transition.split('→', 1)
+            return StateTransition(
+                from_state=from_state.strip(),
+                to_state=to_state.strip(),
+                condition=condition,
+            )
+        
+        return StateTransition(from_state="UNKNOWN", to_state="UNKNOWN")
+    
+    def _parse_generic(
+        self, 
+        source_code: str, 
+        source_file: str,
+        language: Language,
+        annotations: List[Dict]
+    ) -> ParsedAnnotations:
+        """Generic parsing for non-Python languages."""
+        module = ModuleAnnotation(
+            name=Path(source_file).stem,
+            source_file=source_file,
+            language=language,
+        )
+        
+        # Parse each annotation
+        for ann in annotations:
+            parsed = self._parse_annotation_text(ann['text'])
+            
+            # Determine annotation type based on content
+            if 'module_purpose' in parsed or 'dependencies' in parsed:
+                self._parse_module_annotation(ann['text'], module)
+            elif 'class_invariants' in parsed:
+                # Class annotation
+                cls = ClassAnnotation(
+                    name="UnknownClass",
+                    line_number=ann['line'],
+                    source_file=source_file,
+                    raw_annotation=ann['text'],
+                )
+                if 'class_invariants' in parsed:
+                    cls.class_invariants = parsed['class_invariants']
+                if 'state_transitions' in parsed:
+                    cls.state_transitions = [
+                        self._parse_state_transition(st) 
+                        for st in parsed['state_transitions']
+                    ]
+                module.classes.append(cls)
+            else:
+                # Function annotation
+                func = FunctionAnnotation(
+                    name="UnknownFunction",
+                    line_number=ann['line'],
+                    source_file=source_file,
+                    raw_annotation=ann['text'],
+                )
+                
+                if 'solve' in parsed:
+                    func.solve = parsed['solve']
+                if 'preconditions' in parsed:
+                    func.preconditions = parsed['preconditions']
+                if 'postconditions' in parsed:
+                    func.postconditions = parsed['postconditions']
+                if 'invariants' in parsed:
+                    func.invariants = parsed['invariants']
+                if 'optimize' in parsed:
+                    func.optimize = parsed['optimize']
+                if 'complexity' in parsed:
+                    func.complexity = self._parse_complexity(parsed['complexity'])
+                if 'error_cases' in parsed:
+                    func.error_cases = [
+                        self._parse_error_case(ec) 
+                        for ec in parsed['error_cases']
+                    ]
+                
+                module.functions.append(func)
+        
+        return ParsedAnnotations(
+            module=module,
+            language=language,
+            source_file=source_file,
+        )
+
+    def _parse_typescript(self, source_file: str) -> ParsedAnnotations:
+        """Parse TypeScript/JavaScript source code using the TypeScript parser."""
+        # Check if TypeScript parser is built
+        try:
+            check_parser_for_language('typescript')
+        except ParserNotBuiltError:
+            raise
+        
+        # Get the path to the TypeScript parser CLI
+        parser_dir = Path(__file__).parent.parent / 'parsers' / 'typescript'
+        parser_cli = parser_dir / 'dist' / 'cli.js'
+        
+        if not parser_cli.exists():
+            raise ParserNotBuiltError("TypeScript")
+        
+        # Run the TypeScript parser
+        try:
+            result = subprocess.run(
+                ['node', str(parser_cli), source_file],
+                capture_output=True,
+                text=True,
+                check=True
+            )
+            data = json.loads(result.stdout)
+        except subprocess.CalledProcessError as e:
+            raise RuntimeError(f"TypeScript parser failed: {e.stderr}")
+        except json.JSONDecodeError as e:
+            raise RuntimeError(f"Failed to parse TypeScript parser output: {e}")
+        
+        # Convert the TypeScript parser output to our internal format
+        language = Language.TYPESCRIPT if data['language'] == 'typescript' else Language.JAVASCRIPT
+        
+        module = ModuleAnnotation(
+            name=Path(source_file).stem,
+            source_file=source_file,
+            language=language,
+        )
+        
+        # Parse module annotations
+        if data.get('module_annotations'):
+            self._apply_module_annotations(module, data['module_annotations'])
+        
+        # Parse classes
+        for cls_data in data.get('classes', []):
+            cls = ClassAnnotation(
+                name=cls_data['name'],
+                line_number=cls_data['line'],
+            )
+            
+            if cls_data.get('annotations'):
+                self._apply_class_annotations(cls, cls_data['annotations'])
+            
+            # Parse methods
+            for method_data in cls_data.get('methods', []):
+                method = FunctionAnnotation(
+                    name=method_data['name'],
+                    line_number=method_data['line'],
+                )
+                
+                if method_data.get('annotations'):
+                    self._apply_function_annotations(method, method_data['annotations'])
+                
+                cls.methods.append(method)
+            
+            module.classes.append(cls)
+        
+        # Parse functions
+        for func_data in data.get('functions', []):
+            func = FunctionAnnotation(
+                name=func_data['name'],
+                line_number=func_data['line'],
+            )
+            
+            if func_data.get('annotations'):
+                self._apply_function_annotations(func, func_data['annotations'])
+            
+            module.functions.append(func)
+        
+        return ParsedAnnotations(
+            module=module,
+            language=language,
+            source_file=source_file,
+        )
+    
+    def _apply_module_annotations(self, module: ModuleAnnotation, annotations: Dict[str, Any]):
+        """Apply annotations to a module."""
+        if 'module_purpose' in annotations:
+            module.purpose = annotations['module_purpose']
+        if 'dependencies' in annotations:
+            module.dependencies = annotations['dependencies'] if isinstance(annotations['dependencies'], list) else [annotations['dependencies']]
+        if 'assumptions' in annotations:
+            module.assumptions = annotations['assumptions'] if isinstance(annotations['assumptions'], list) else [annotations['assumptions']]
+    
+    def _apply_class_annotations(self, cls: ClassAnnotation, annotations: Dict[str, Any]):
+        """Apply annotations to a class."""
+        if 'class_invariants' in annotations:
+            cls.invariants = annotations['class_invariants'] if isinstance(annotations['class_invariants'], list) else [annotations['class_invariants']]
+        if 'state_transitions' in annotations:
+            transitions = annotations['state_transitions']
+            if isinstance(transitions, list):
+                cls.state_transitions = [self._parse_state_transition(t) for t in transitions]
+    
+    def _apply_function_annotations(self, func: FunctionAnnotation, annotations: Dict[str, Any]):
+        """Apply annotations to a function."""
+        if 'preconditions' in annotations:
+            func.preconditions = annotations['preconditions'] if isinstance(annotations['preconditions'], list) else [annotations['preconditions']]
+        if 'postconditions' in annotations:
+            func.postconditions = annotations['postconditions'] if isinstance(annotations['postconditions'], list) else [annotations['postconditions']]
+        if 'invariants' in annotations:
+            func.invariants = annotations['invariants'] if isinstance(annotations['invariants'], list) else [annotations['invariants']]
+        if 'complexity' in annotations:
+            func.complexity = self._parse_complexity(annotations['complexity'])
+        if 'side_effects' in annotations:
+            func.side_effects = annotations['side_effects'] if isinstance(annotations['side_effects'], list) else [annotations['side_effects']]