@voodocs/cli 0.4.1 → 1.0.0

package/lib/darkarts/validation/benchmark.py

@@ -0,0 +1,426 @@
+#!/usr/bin/env python3
+"""
+DarkArts Benchmark Suite
+
+Comprehensive benchmarking framework for validating complexity claims
+with real performance data.
+
+Usage:
+    python3 benchmark_suite.py --module darkarts.executor.symbolic
+    python3 benchmark_suite.py --all
+    python3 benchmark_suite.py --critical-paths
+"""
+
+import time
+import sys
+import json
+import statistics
+from pathlib import Path
+from typing import Dict, List, Tuple, Any, Callable
+from dataclasses import dataclass, asdict
+import importlib
+import inspect
+
+
+@dataclass
+class BenchmarkResult:
+    """Result of a single benchmark run."""
+    module: str
+    function: str
+    input_size: int
+    execution_time: float  # seconds
+    memory_usage: int  # bytes
+    iterations: int
+    claimed_complexity: str
+    actual_complexity: str
+    status: str  # 'pass', 'fail', 'warning'
+
+
+class ComplexityAnalyzer:
+    """Analyzes actual runtime complexity from benchmark data."""
+    
+    @staticmethod
+    def fit_complexity(sizes: List[int], times: List[float]) -> str:
+        """
+        Fit benchmark data to complexity classes.
+        
+        Returns best-fit complexity (O(1), O(log n), O(n), O(n log n), O(n²), O(2^n))
+        """
+        if len(sizes) < 3:
+            return "Unknown (insufficient data)"
+        
+        # Normalize times
+        base_time = times[0]
+        if base_time == 0:
+            return "O(1)"
+        
+        ratios = [t / base_time for t in times]
+        size_ratios = [s / sizes[0] for s in sizes]
+        
+        # Check O(1) - constant time
+        if max(ratios) / min(ratios) < 1.5:
+            return "O(1)"
+        
+        # Check O(log n)
+        log_ratios = [size_ratios[i] ** 0.5 for i in range(len(size_ratios))]
+        if ComplexityAnalyzer._fits(ratios, log_ratios, tolerance=0.3):
+            return "O(log n)"
+        
+        # Check O(n)
+        if ComplexityAnalyzer._fits(ratios, size_ratios, tolerance=0.3):
+            return "O(n)"
+        
+        # Check O(n log n)
+        nlogn_ratios = [sr * (sr ** 0.5) for sr in size_ratios]
+        if ComplexityAnalyzer._fits(ratios, nlogn_ratios, tolerance=0.4):
+            return "O(n log n)"
+        
+        # Check O(n²)
+        n2_ratios = [sr ** 2 for sr in size_ratios]
+        if ComplexityAnalyzer._fits(ratios, n2_ratios, tolerance=0.5):
+            return "O(n²)"
+        
+        # Check O(n³)
+        n3_ratios = [sr ** 3 for sr in size_ratios]
+        if ComplexityAnalyzer._fits(ratios, n3_ratios, tolerance=0.6):
+            return "O(n³)"
+        
+        # Check O(2^n) - exponential
+        exp_ratios = [2 ** (i * 2) for i in range(len(size_ratios))]
+        if ComplexityAnalyzer._fits(ratios, exp_ratios, tolerance=0.7):
+            return "O(2^n)"
+        
+        return "Unknown (doesn't fit standard complexity)"
+    
+    @staticmethod
+    def _fits(actual: List[float], expected: List[float], tolerance: float) -> bool:
+        """Check if actual ratios fit expected ratios within tolerance."""
+        if len(actual) != len(expected):
+            return False
+        
+        errors = [abs(a - e) / e if e != 0 else 0 for a, e in zip(actual, expected)]
+        avg_error = statistics.mean(errors)
+        return avg_error < tolerance
+
+
+class BenchmarkSuite:
+    """Main benchmarking suite."""
+    
+    def __init__(self):
+        self.results: List[BenchmarkResult] = []
+        self.critical_paths = [
+            ('darkarts.executor.symbolic', 'solve'),
+            ('darkarts.executor.numerical', 'solve'),
+            ('darkarts.executor.z3_solver', 'solve'),
+            ('darkarts.parser.unified', 'parse'),
+            ('darkarts.parser.latex', 'parse'),
+            ('darkarts.annotations.parser', 'parse_darkarts'),
+            ('darkarts.cli_darkarts', 'main'),
+            ('darkarts.context.diagram', 'generate_dependency_graph'),
+        ]
+    
+    def benchmark_function(
+        self,
+        func: Callable,
+        input_generator: Callable[[int], Any],
+        sizes: List[int],
+        iterations: int = 10
+    ) -> List[Tuple[int, float]]:
+        """
+        Benchmark a function with different input sizes.
+        
+        Args:
+            func: Function to benchmark
+            input_generator: Function that generates input of given size
+            sizes: List of input sizes to test
+            iterations: Number of iterations per size
+        
+        Returns:
+            List of (size, avg_time) tuples
+        """
+        results = []
+        
+        for size in sizes:
+            times = []
+            
+            for _ in range(iterations):
+                input_data = input_generator(size)
+                
+                start = time.perf_counter()
+                try:
+                    func(input_data)
+                except Exception as e:
+                    print(f"  ⚠️  Error with size {size}: {e}")
+                    continue
+                end = time.perf_counter()
+                
+                times.append(end - start)
+            
+            if times:
+                avg_time = statistics.mean(times)
+                results.append((size, avg_time))
+                print(f"  Size {size:6d}: {avg_time:.6f}s (avg of {len(times)} runs)")
+        
+        return results
+    
+    def benchmark_module(self, module_path: str) -> List[BenchmarkResult]:
+        """Benchmark all functions in a module."""
+        print(f"\n{'='*60}")
+        print(f"Benchmarking: {module_path}")
+        print(f"{'='*60}\n")
+        
+        try:
+            module = importlib.import_module(module_path)
+        except ImportError as e:
+            print(f"❌ Failed to import {module_path}: {e}")
+            return []
+        
+        # Get claimed complexity from module docstring
+        claimed_complexity = self._extract_complexity_claim(module)
+        
+        results = []
+        
+        # Find benchmarkable functions
+        for name, obj in inspect.getmembers(module):
+            if inspect.isfunction(obj) and not name.startswith('_'):
+                print(f"\nFunction: {name}")
+                
+                # Generate appropriate test sizes
+                sizes = [10, 50, 100, 500, 1000]
+                
+                # Create input generator (customize per function type)
+                input_gen = self._create_input_generator(name, obj)
+                
+                if input_gen is None:
+                    print(f"  ⚠️  No input generator available")
+                    continue
+                
+                # Run benchmark
+                bench_results = self.benchmark_function(obj, input_gen, sizes, iterations=5)
+                
+                if not bench_results:
+                    continue
+                
+                # Analyze complexity
+                sizes_tested, times = zip(*bench_results)
+                actual_complexity = ComplexityAnalyzer.fit_complexity(list(sizes_tested), list(times))
+                
+                # Compare with claim
+                status = self._compare_complexity(claimed_complexity, actual_complexity)
+                
+                result = BenchmarkResult(
+                    module=module_path,
+                    function=name,
+                    input_size=max(sizes_tested),
+                    execution_time=times[-1],
+                    memory_usage=0,  # TODO: Add memory profiling
+                    iterations=5,
+                    claimed_complexity=claimed_complexity,
+                    actual_complexity=actual_complexity,
+                    status=status
+                )
+                
+                results.append(result)
+                self.results.append(result)
+                
+                print(f"  Claimed: {claimed_complexity}")
+                print(f"  Actual:  {actual_complexity}")
+                print(f"  Status:  {status}")
+        
+        return results
+    
+    def benchmark_critical_paths(self) -> List[BenchmarkResult]:
+        """Benchmark all critical paths."""
+        print("\n" + "="*60)
+        print("BENCHMARKING CRITICAL PATHS")
+        print("="*60)
+        
+        all_results = []
+        
+        for module_path, func_name in self.critical_paths:
+            print(f"\n{'='*60}")
+            print(f"Critical Path: {module_path}.{func_name}")
+            print(f"{'='*60}\n")
+            
+            try:
+                module = importlib.import_module(module_path)
+                func = getattr(module, func_name, None)
+                
+                if func is None:
+                    print(f"❌ Function {func_name} not found in {module_path}")
+                    continue
+                
+                # Get claimed complexity
+                claimed_complexity = self._extract_complexity_claim(module)
+                
+                # Create input generator
+                input_gen = self._create_input_generator(func_name, func)
+                
+                if input_gen is None:
+                    print(f"  ⚠️  No input generator available")
+                    continue
+                
+                # Run benchmark with larger sizes for critical paths
+                sizes = [10, 50, 100, 500, 1000, 5000]
+                bench_results = self.benchmark_function(func, input_gen, sizes, iterations=10)
+                
+                if not bench_results:
+                    continue
+                
+                # Analyze
+                sizes_tested, times = zip(*bench_results)
+                actual_complexity = ComplexityAnalyzer.fit_complexity(list(sizes_tested), list(times))
+                
+                status = self._compare_complexity(claimed_complexity, actual_complexity)
+                
+                result = BenchmarkResult(
+                    module=module_path,
+                    function=func_name,
+                    input_size=max(sizes_tested),
+                    execution_time=times[-1],
+                    memory_usage=0,
+                    iterations=10,
+                    claimed_complexity=claimed_complexity,
+                    actual_complexity=actual_complexity,
+                    status=status
+                )
+                
+                all_results.append(result)
+                self.results.append(result)
+                
+                print(f"\n  Claimed: {claimed_complexity}")
+                print(f"  Actual:  {actual_complexity}")
+                print(f"  Status:  {status}")
+                
+            except Exception as e:
+                print(f"❌ Error benchmarking {module_path}.{func_name}: {e}")
+        
+        return all_results
+    
+    def _extract_complexity_claim(self, module) -> str:
+        """Extract ⚡{} complexity claim from module docstring."""
+        docstring = inspect.getdoc(module)
+        if not docstring:
+            return "Unknown"
+        
+        # Look for ⚡{...}
+        import re
+        match = re.search(r'⚡\{([^}]+)\}', docstring)
+        if match:
+            return match.group(1)
+        
+        return "Unknown"
+    
+    def _create_input_generator(self, func_name: str, func: Callable) -> Callable[[int], Any]:
+        """Create appropriate input generator for function."""
+        # Inspect function signature
+        sig = inspect.signature(func)
+        params = list(sig.parameters.values())
+        
+        if not params:
+            return lambda n: None
+        
+        first_param = params[0]
+        
+        # Generate based on function name and parameter type
+        if 'parse' in func_name.lower():
+            # Parser functions - generate strings
+            return lambda n: "x + y = " + " + ".join([f"term{i}" for i in range(n)])
+        
+        elif 'solve' in func_name.lower():
+            # Solver functions - generate equations
+            return lambda n: f"x^{n} + {n}*x + {n*2} = 0"
+        
+        elif 'generate' in func_name.lower():
+            # Generator functions - generate lists
+            return lambda n: list(range(n))
+        
+        elif first_param.annotation == str or 'str' in str(first_param.annotation):
+            # String input
+            return lambda n: "a" * n
+        
+        elif first_param.annotation == list or 'list' in str(first_param.annotation):
+            # List input
+            return lambda n: list(range(n))
+        
+        elif first_param.annotation == int or 'int' in str(first_param.annotation):
+            # Integer input
+            return lambda n: n
+        
+        else:
+            # Default: list of integers
+            return lambda n: list(range(n))
+    
+    def _compare_complexity(self, claimed: str, actual: str) -> str:
+        """Compare claimed vs actual complexity."""
+        # Normalize
+        claimed_norm = claimed.split(':')[0].split('|')[0].strip()
+        actual_norm = actual.strip()
+        
+        # Extract complexity class
+        complexity_order = {
+            'O(1)': 0,
+            'O(log n)': 1,
+            'O(n)': 2,
+            'O(n log n)': 3,
+            'O(n²)': 4,
+            'O(n³)': 5,
+            'O(2^n)': 6,
+        }
+        
+        claimed_order = complexity_order.get(claimed_norm, -1)
+        actual_order = complexity_order.get(actual_norm, -1)
+        
+        if claimed_order == -1 or actual_order == -1:
+            return 'warning'
+        
+        if claimed_order == actual_order:
+            return 'pass'
+        elif claimed_order > actual_order:
+            return 'pass'  # Conservative claim (safe)
+        else:
+            return 'fail'  # Underestimated (dangerous)
+    
+    def generate_report(self, output_path: str = None):
+        """Generate benchmark report."""
+        print("\n" + "="*60)
+        print("BENCHMARK REPORT")
+        print("="*60 + "\n")
+        
+        if not self.results:
+            print("No benchmark results available")
+            return
+        
+        # Summary statistics
+        total = len(self.results)
+        passed = sum(1 for r in self.results if r.status == 'pass')
+        failed = sum(1 for r in self.results if r.status == 'fail')
+        warnings = sum(1 for r in self.results if r.status == 'warning')
+        
+        print(f"Total Benchmarks: {total}")
+        print(f"Passed:  {passed} ({passed/total*100:.1f}%)")
+        print(f"Failed:  {failed} ({failed/total*100:.1f}%)")
+        print(f"Warnings: {warnings} ({warnings/total*100:.1f}%)")
+        print()
+        
+        # Detailed results
+        print("Detailed Results:")
+        print("-" * 60)
+        
+        for result in self.results:
+            status_icon = {'pass': '✅', 'fail': '❌', 'warning': '⚠️'}[result.status]
+            print(f"{status_icon} {result.module}.{result.function}")
+            print(f"   Claimed: {result.claimed_complexity}")
+            print(f"   Actual:  {result.actual_complexity}")
+            print(f"   Time:    {result.execution_time:.6f}s (n={result.input_size})")
+            print()
+        
+        # Save to file
+        if output_path:
+            with open(output_path, 'w') as f:
+                json.dump([asdict(r) for r in self.results], f, indent=2)
+            print(f"Report saved to: {output_path}")
+
+
+
+# CLI interface removed - use darkarts.validation API or voodocs CLI instead

package/lib/darkarts/validation/benchmark_wrapper.py

@@ -0,0 +1,22 @@
+"""@darkarts
+⊢validation:benchmark.wrapper
+∂{pathlib,typing}
+⚠{python≥3.7}
+⊨{∀method→delegates,¬state}
+🔒{read-only}
+⚡{O(1):creation,O(n*m):benchmarks|n=files,m=iterations}
+"""
+
+from pathlib import Path
+from typing import List, Dict, Any
+
+from . import benchmark
+from .types import BenchmarkResult
+
+
+class BenchmarkSuite:
+    """Benchmark suite for @darkarts annotations."""
+    
+    def run_benchmarks(self, config: Dict[str, Any]) -> List[BenchmarkResult]:
+        """Run benchmarks based on configuration."""
+        return benchmark.run_benchmarks(config)

package/lib/darkarts/validation/config.py

@@ -0,0 +1,257 @@
+"""@darkarts
+⊢config:validation.configuration
+∂{pathlib,typing,dataclasses,json,re}
+⚠{python≥3.7,config-file:valid-json}
+⊨{∀load→Config,∀validate→rules-applied,∀merge→precedence-respected,serializable:to-json}
+🔒{read:config-files,¬write-except-defaults,¬network,¬exec}
+⚡{O(n):load-and-parse|n=config-entries,O(n):rule-application}
+
+Configuration File Support for Validation
+
+Customizable validation rules and settings with:
+- JSON configuration files (.darkarts.json)
+- Per-project settings (project root)
+- Per-directory overrides (nested configs)
+- Rule customization (ignore patterns, custom stdlib)
+- Validation thresholds (error vs warning)
+- Auto-fix preferences (backup settings, dry-run defaults)
+"""
+
+import json
+from pathlib import Path
+from typing import Optional, Set, List, Dict, Any
+from dataclasses import dataclass, field, asdict
+
+
+@dataclass
+class ValidationConfig:
+    """Configuration for validation behavior."""
+    
+    # Files and directories to ignore
+    ignore_patterns: List[str] = field(default_factory=lambda: [
+        "*/test_*.py",
+        "*/__pycache__/*",
+        "*/.*",
+        "*/venv/*",
+        "*/env/*",
+        "*/.venv/*"
+    ])
+    
+    # Additional stdlib modules to keep in validation
+    stdlib_whitelist: Set[str] = field(default_factory=lambda: {
+        'typing', 'dataclasses', 'enum', 'abc', 'pathlib',
+        'datetime', 'json', 'yaml', 're', 'os', 'sys',
+        'time', 'uuid', 'collections', 'itertools', 'functools'
+    })
+    
+    # Modules to always ignore (even if imported)
+    ignore_modules: Set[str] = field(default_factory=set)
+    
+    # Whether to treat extra dependencies as errors or warnings
+    extra_deps_as_error: bool = True
+    
+    # Whether to treat missing dependencies as errors or warnings
+    missing_deps_as_error: bool = True
+    
+    # Minimum number of dependencies before validation
+    min_deps_threshold: int = 0
+    
+    # Whether to validate __init__.py files
+    validate_init_files: bool = True
+    
+    # Whether to validate test files
+    validate_test_files: bool = True
+
+
+@dataclass
+class AutoFixConfig:
+    """Configuration for auto-fix behavior."""
+    
+    # Whether to create backups before fixing
+    create_backups: bool = True
+    
+    # Backup directory
+    backup_dir: str = ".backups"
+    
+    # Whether to run in dry-run mode by default
+    dry_run_default: bool = False
+    
+    # Whether to auto-fix on watch mode
+    auto_fix_on_watch: bool = False
+    
+    # Maximum number of files to fix in one batch
+    max_batch_size: int = 100
+
+
+@dataclass
+class WatchConfig:
+    """Configuration for watch mode."""
+    
+    # Check interval in seconds
+    interval: float = 1.0
+    
+    # Debounce time in seconds
+    debounce: float = 0.5
+    
+    # Whether to show verbose output
+    verbose: bool = False
+    
+    # Whether to beep on errors
+    beep_on_error: bool = False
+
+
+@dataclass
+class Config:
+    """Complete configuration for darkarts validation."""
+    
+    validation: ValidationConfig = field(default_factory=ValidationConfig)
+    autofix: AutoFixConfig = field(default_factory=AutoFixConfig)
+    watch: WatchConfig = field(default_factory=WatchConfig)
+    
+    # Project-specific settings
+    project_name: Optional[str] = None
+    project_root: Optional[str] = None
+    
+    @classmethod
+    def from_dict(cls, data: Dict[str, Any]) -> 'Config':
+        """Create Config from dictionary."""
+        validation_data = data.get('validation', {})
+        autofix_data = data.get('autofix', {})
+        watch_data = data.get('watch', {})
+        
+        # Convert lists to sets where needed
+        if 'stdlib_whitelist' in validation_data:
+            validation_data['stdlib_whitelist'] = set(validation_data['stdlib_whitelist'])
+        if 'ignore_modules' in validation_data:
+            validation_data['ignore_modules'] = set(validation_data['ignore_modules'])
+        
+        return cls(
+            validation=ValidationConfig(**validation_data),
+            autofix=AutoFixConfig(**autofix_data),
+            watch=WatchConfig(**watch_data),
+            project_name=data.get('project_name'),
+            project_root=data.get('project_root')
+        )
+    
+    def to_dict(self) -> Dict[str, Any]:
+        """Convert Config to dictionary."""
+        result = asdict(self)
+        
+        # Convert sets to lists for JSON serialization
+        if 'stdlib_whitelist' in result['validation']:
+            result['validation']['stdlib_whitelist'] = sorted(result['validation']['stdlib_whitelist'])
+        if 'ignore_modules' in result['validation']:
+            result['validation']['ignore_modules'] = sorted(result['validation']['ignore_modules'])
+        
+        return result
+    
+    def save(self, path: Path):
+        """Save configuration to a JSON file."""
+        with open(path, 'w') as f:
+            json.dump(self.to_dict(), f, indent=2)
+    
+    @classmethod
+    def load(cls, path: Path) -> 'Config':
+        """Load configuration from a JSON file."""
+        with open(path, 'r') as f:
+            data = json.load(f)
+        return cls.from_dict(data)
+    
+    @classmethod
+    def find_config(cls, start_path: Path) -> Optional['Config']:
+        """
+        Find and load configuration file starting from a path.
+        
+        Searches upward from start_path for .darkarts.json
+        
+        Args:
+            start_path: Path to start searching from
+        
+        Returns:
+            Config if found, None otherwise
+        """
+        current = start_path if start_path.is_dir() else start_path.parent
+        
+        while current != current.parent:
+            config_file = current / ".darkarts.json"
+            if config_file.exists():
+                return cls.load(config_file)
+            current = current.parent
+        
+        return None
+    
+    def merge(self, other: 'Config') -> 'Config':
+        """
+        Merge another config into this one.
+        
+        Other config takes precedence for non-default values.
+        
+        Args:
+            other: Config to merge
+        
+        Returns:
+            New merged Config
+        """
+        # This is a simple merge - you could make it more sophisticated
+        merged_dict = self.to_dict()
+        other_dict = other.to_dict()
+        
+        # Deep merge
+        for key, value in other_dict.items():
+            if isinstance(value, dict) and key in merged_dict:
+                merged_dict[key].update(value)
+            else:
+                merged_dict[key] = value
+        
+        return Config.from_dict(merged_dict)
+
+
+def create_default_config(path: Path):
+    """Create a default configuration file."""
+    config = Config()
+    config.save(path)
+    print(f"✅ Created default configuration: {path}")
+
+
+def validate_config(path: Path) -> bool:
+    """
+    Validate a configuration file.
+    
+    Args:
+        path: Path to the config file
+    
+    Returns:
+        True if valid, False otherwise
+    """
+    try:
+        Config.load(path)
+        print(f"✅ Configuration valid: {path}")
+        return True
+    except json.JSONDecodeError as e:
+        print(f"❌ Invalid JSON in {path}: {e}")
+        return False
+    except Exception as e:
+        print(f"❌ Error loading {path}: {e}")
+        return False
+
+
+def show_config(path: Optional[Path] = None):
+    """Display current configuration."""
+    if path:
+        config = Config.load(path)
+    else:
+        config = Config.find_config(Path.cwd())
+        if config is None:
+            print("No configuration file found. Using defaults.")
+            config = Config()
+    
+    print("\n" + "="*60)
+    print("Current Configuration")
+    print("="*60 + "\n")
+    
+    print(json.dumps(config.to_dict(), indent=2))
+    print()
+
+
+
+# CLI interface removed - use darkarts.validation API or voodocs CLI instead