anvil-dev-framework 0.1.6

package/global/lib/verification_runner.py

@@ -0,0 +1,557 @@
+#!/usr/bin/env python3
+"""
+verification_runner.py - Verification suite runner for /verify command (ANV-149)
+
+Executes test, lint, and type check commands with output capture and error parsing.
+Supports project-specific configuration via .claude/settings.yaml.
+
+Usage:
+    from verification_runner import VerificationRunner, VerificationResult
+
+    runner = VerificationRunner()
+    result = runner.run_all()
+
+    if result.passed:
+        print("All checks passed!")
+    else:
+        for error in result.errors:
+            print(f"{error.file}:{error.line} - {error.message}")
+"""
+
+import json
+import re
+import subprocess
+import time
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any, Dict, List, Literal, Optional, Tuple
+
+try:
+    import yaml
+    HAS_YAML = True
+except ImportError:
+    HAS_YAML = False
+
+
+# =============================================================================
+# Data Models
+# =============================================================================
+
+@dataclass
+class VerificationError:
+    """A single verification error with location information."""
+    file: str
+    line: Optional[int] = None
+    column: Optional[int] = None
+    message: str = ""
+    severity: Literal["error", "warning"] = "error"
+    source: Literal["test", "lint", "types"] = "test"
+    raw_output: str = ""
+
+    def to_dict(self) -> Dict[str, Any]:
+        return {
+            "file": self.file,
+            "line": self.line,
+            "column": self.column,
+            "message": self.message,
+            "severity": self.severity,
+            "source": self.source,
+        }
+
+
+@dataclass
+class CheckResult:
+    """Result of a single verification check (test/lint/types)."""
+    check_type: Literal["test", "lint", "types"]
+    passed: bool
+    exit_code: int
+    duration_ms: int
+    output: str
+    errors: List[VerificationError] = field(default_factory=list)
+    count: Optional[int] = None  # Number of tests passed, etc.
+
+    def to_dict(self) -> Dict[str, Any]:
+        return {
+            "check_type": self.check_type,
+            "passed": self.passed,
+            "exit_code": self.exit_code,
+            "duration_ms": self.duration_ms,
+            "errors": [e.to_dict() for e in self.errors],
+            "count": self.count,
+        }
+
+
+@dataclass
+class VerificationResult:
+    """Complete verification suite result."""
+    passed: bool
+    tests: Optional[CheckResult] = None
+    lint: Optional[CheckResult] = None
+    types: Optional[CheckResult] = None
+    total_duration_ms: int = 0
+    errors: List[VerificationError] = field(default_factory=list)
+
+    def to_dict(self) -> Dict[str, Any]:
+        return {
+            "passed": self.passed,
+            "tests": self.tests.to_dict() if self.tests else None,
+            "lint": self.lint.to_dict() if self.lint else None,
+            "types": self.types.to_dict() if self.types else None,
+            "total_duration_ms": self.total_duration_ms,
+            "error_count": len(self.errors),
+        }
+
+    def to_json(self) -> str:
+        return json.dumps(self.to_dict(), indent=2)
+
+
+@dataclass
+class VerificationConfig:
+    """Configuration for verification commands."""
+    test_command: Optional[str] = None
+    lint_command: Optional[str] = None
+    types_command: Optional[str] = None
+    max_iterations: int = 3
+    required_for_completion: bool = True
+    timeout_seconds: int = 300
+
+    @classmethod
+    def from_dict(cls, data: Dict[str, Any]) -> "VerificationConfig":
+        commands = data.get("commands", {})
+        return cls(
+            test_command=commands.get("test"),
+            lint_command=commands.get("lint"),
+            types_command=commands.get("types"),
+            max_iterations=data.get("max_iterations", 3),
+            required_for_completion=data.get("required_for_completion", True),
+            timeout_seconds=data.get("timeout_seconds", 300),
+        )
+
+
+# =============================================================================
+# Project Type Detection
+# =============================================================================
+
+class ProjectDetector:
+    """Detect project type and default verification commands."""
+
+    NODEJS_DEFAULTS = {
+        "test": "npm test",
+        "lint": "npm run lint",
+        "types": "npm run typecheck",
+    }
+
+    PYTHON_DEFAULTS = {
+        "test": "pytest",
+        "lint": "ruff check .",
+        "types": "mypy .",
+    }
+
+    RUST_DEFAULTS = {
+        "test": "cargo test",
+        "lint": "cargo clippy",
+        "types": "cargo check",
+    }
+
+    GO_DEFAULTS = {
+        "test": "go test ./...",
+        "lint": "golangci-lint run",
+        "types": None,  # Go is statically typed at compile
+    }
+
+    @classmethod
+    def detect(cls, project_root: Path) -> Tuple[str, Dict[str, Optional[str]]]:
+        """Detect project type and return default commands."""
+        if (project_root / "package.json").exists():
+            return "nodejs", cls.NODEJS_DEFAULTS
+
+        if (project_root / "pyproject.toml").exists() or \
+           (project_root / "setup.py").exists() or \
+           (project_root / "requirements.txt").exists():
+            return "python", cls.PYTHON_DEFAULTS
+
+        if (project_root / "Cargo.toml").exists():
+            return "rust", cls.RUST_DEFAULTS
+
+        if (project_root / "go.mod").exists():
+            return "go", cls.GO_DEFAULTS
+
+        # Default to Node.js style
+        return "unknown", cls.NODEJS_DEFAULTS
+
+
+# =============================================================================
+# Error Parsers
+# =============================================================================
+
+class ErrorParser:
+    """Parse errors from command output."""
+
+    # Common patterns for extracting file:line:column
+    FILE_LINE_PATTERNS = [
+        # TypeScript/JavaScript: src/file.ts(10,5): error TS2345
+        re.compile(r"^(.+?)\((\d+),(\d+)\):\s*(.+)$", re.MULTILINE),
+        # ESLint/Ruff: src/file.ts:10:5: error message
+        re.compile(r"^(.+?):(\d+):(\d+):\s*(.+)$", re.MULTILINE),
+        # Python traceback: File "src/file.py", line 10
+        re.compile(r'File "(.+?)", line (\d+)', re.MULTILINE),
+        # Jest/Vitest: at Object.<anonymous> (src/file.test.ts:10:5)
+        re.compile(r"at .+? \((.+?):(\d+):(\d+)\)", re.MULTILINE),
+        # Generic: src/file.ts:10
+        re.compile(r"^(.+?):(\d+)(?::\d+)?(?:\s|$)", re.MULTILINE),
+    ]
+
+    # Test failure patterns
+    TEST_FAILURE_PATTERNS = [
+        # Jest/Vitest: FAIL src/file.test.ts
+        re.compile(r"FAIL\s+(.+\.(?:test|spec)\.[jt]sx?)"),
+        # Pytest: FAILED test_file.py::test_name
+        re.compile(r"FAILED\s+(.+?)::", re.MULTILINE),
+    ]
+
+    @classmethod
+    def parse_errors(
+        cls,
+        output: str,
+        source: Literal["test", "lint", "types"]
+    ) -> List[VerificationError]:
+        """Parse errors from command output."""
+        errors = []
+        seen = set()
+
+        for pattern in cls.FILE_LINE_PATTERNS:
+            for match in pattern.finditer(output):
+                groups = match.groups()
+                file_path = groups[0] if groups else None
+
+                if not file_path or file_path in seen:
+                    continue
+
+                # Skip common false positives
+                if any(skip in file_path for skip in [
+                    "node_modules", ".git", "__pycache__", "site-packages"
+                ]):
+                    continue
+
+                line = int(groups[1]) if len(groups) > 1 and groups[1] else None
+                column = int(groups[2]) if len(groups) > 2 and groups[2] and groups[2].isdigit() else None
+                message = groups[3] if len(groups) > 3 else ""
+
+                # Clean up message
+                message = message.strip()[:200]
+
+                errors.append(VerificationError(
+                    file=file_path,
+                    line=line,
+                    column=column,
+                    message=message,
+                    source=source,
+                ))
+                seen.add(file_path)
+
+        # Also check for test-specific failure patterns
+        if source == "test":
+            for pattern in cls.TEST_FAILURE_PATTERNS:
+                for match in pattern.finditer(output):
+                    file_path = match.group(1)
+                    if file_path and file_path not in seen:
+                        errors.append(VerificationError(
+                            file=file_path,
+                            line=None,
+                            message="Test failed",
+                            source=source,
+                        ))
+                        seen.add(file_path)
+
+        return errors
+
+    @classmethod
+    def extract_test_count(cls, output: str) -> Optional[int]:
+        """Extract number of passed tests from output."""
+        patterns = [
+            # Jest/Vitest: Tests: 47 passed
+            re.compile(r"Tests?:\s*(\d+)\s*passed"),
+            # Pytest: 47 passed
+            re.compile(r"(\d+)\s*passed"),
+            # Cargo: test result: ok. 47 passed
+            re.compile(r"(\d+)\s*passed"),
+        ]
+        for pattern in patterns:
+            match = pattern.search(output)
+            if match:
+                return int(match.group(1))
+        return None
+
+
+# =============================================================================
+# Verification Runner
+# =============================================================================
+
+class VerificationRunner:
+    """Run verification suite with output capture and error parsing."""
+
+    def __init__(
+        self,
+        project_root: Optional[Path] = None,
+        config: Optional[VerificationConfig] = None,
+    ):
+        self.project_root = project_root or Path.cwd()
+        self.config = config or self._load_config()
+        self.project_type, self.default_commands = ProjectDetector.detect(self.project_root)
+
+    def _load_config(self) -> VerificationConfig:
+        """Load configuration from .claude/settings.yaml."""
+        config_path = self.project_root / ".claude" / "settings.yaml"
+
+        if not config_path.exists() or not HAS_YAML:
+            return VerificationConfig()
+
+        try:
+            with open(config_path) as f:
+                data = yaml.safe_load(f) or {}
+            verification_data = data.get("verification", {})
+            return VerificationConfig.from_dict(verification_data)
+        except Exception:
+            return VerificationConfig()
+
+    def _get_command(self, check_type: Literal["test", "lint", "types"]) -> Optional[str]:
+        """Get command for a check type, preferring config over defaults."""
+        if check_type == "test":
+            return self.config.test_command or self.default_commands.get("test")
+        elif check_type == "lint":
+            return self.config.lint_command or self.default_commands.get("lint")
+        elif check_type == "types":
+            return self.config.types_command or self.default_commands.get("types")
+        return None
+
+    def _run_command(
+        self,
+        command: str,
+        check_type: Literal["test", "lint", "types"],
+    ) -> CheckResult:
+        """Run a single verification command."""
+        start_time = time.time()
+
+        try:
+            result = subprocess.run(
+                command,
+                shell=True,
+                capture_output=True,
+                text=True,
+                timeout=self.config.timeout_seconds,
+                cwd=str(self.project_root),
+            )
+            output = result.stdout + "\n" + result.stderr
+            exit_code = result.returncode
+            passed = exit_code == 0
+
+        except subprocess.TimeoutExpired:
+            output = f"Command timed out after {self.config.timeout_seconds}s"
+            exit_code = -1
+            passed = False
+
+        except Exception as e:
+            output = f"Command failed: {str(e)}"
+            exit_code = -1
+            passed = False
+
+        duration_ms = int((time.time() - start_time) * 1000)
+
+        # Parse errors from output
+        errors = []
+        if not passed:
+            errors = ErrorParser.parse_errors(output, check_type)
+
+        # Extract test count if applicable
+        count = None
+        if check_type == "test" and passed:
+            count = ErrorParser.extract_test_count(output)
+
+        return CheckResult(
+            check_type=check_type,
+            passed=passed,
+            exit_code=exit_code,
+            duration_ms=duration_ms,
+            output=output,
+            errors=errors,
+            count=count,
+        )
+
+    def run_tests(self) -> Optional[CheckResult]:
+        """Run test suite."""
+        command = self._get_command("test")
+        if not command:
+            return None
+        return self._run_command(command, "test")
+
+    def run_lint(self) -> Optional[CheckResult]:
+        """Run linter."""
+        command = self._get_command("lint")
+        if not command:
+            return None
+        return self._run_command(command, "lint")
+
+    def run_types(self) -> Optional[CheckResult]:
+        """Run type checker."""
+        command = self._get_command("types")
+        if not command:
+            return None
+        return self._run_command(command, "types")
+
+    def run_all(
+        self,
+        skip_tests: bool = False,
+        skip_lint: bool = False,
+        skip_types: bool = False,
+    ) -> VerificationResult:
+        """Run all verification checks."""
+        start_time = time.time()
+
+        tests = None if skip_tests else self.run_tests()
+        lint = None if skip_lint else self.run_lint()
+        types = None if skip_types else self.run_types()
+
+        # Collect all errors
+        all_errors = []
+        if tests and not tests.passed:
+            all_errors.extend(tests.errors)
+        if lint and not lint.passed:
+            all_errors.extend(lint.errors)
+        if types and not types.passed:
+            all_errors.extend(types.errors)
+
+        # Determine overall pass/fail
+        passed = True
+        if tests and not tests.passed:
+            passed = False
+        if lint and not lint.passed:
+            passed = False
+        if types and not types.passed:
+            passed = False
+
+        total_duration_ms = int((time.time() - start_time) * 1000)
+
+        return VerificationResult(
+            passed=passed,
+            tests=tests,
+            lint=lint,
+            types=types,
+            total_duration_ms=total_duration_ms,
+            errors=all_errors,
+        )
+
+    def run_single(
+        self,
+        check_type: Literal["test", "lint", "types"]
+    ) -> Optional[CheckResult]:
+        """Run a single verification check."""
+        if check_type == "test":
+            return self.run_tests()
+        elif check_type == "lint":
+            return self.run_lint()
+        elif check_type == "types":
+            return self.run_types()
+        return None
+
+
+# =============================================================================
+# State Management
+# =============================================================================
+
+class VerificationState:
+    """Manage verification state persistence."""
+
+    STATE_FILE = ".claude/verification-state.json"
+
+    def __init__(self, project_root: Optional[Path] = None):
+        self.project_root = project_root or Path.cwd()
+        self.state_path = self.project_root / self.STATE_FILE
+
+    def save(self, result: VerificationResult, iteration: int = 0) -> None:
+        """Save verification state to file."""
+        from datetime import datetime
+
+        state = {
+            "last_run": datetime.now().isoformat(),
+            "status": "passed" if result.passed else "failed",
+            "iteration": iteration,
+            "results": result.to_dict(),
+        }
+
+        self.state_path.parent.mkdir(parents=True, exist_ok=True)
+        with open(self.state_path, "w") as f:
+            json.dump(state, f, indent=2)
+
+    def load(self) -> Optional[Dict[str, Any]]:
+        """Load verification state from file."""
+        if not self.state_path.exists():
+            return None
+
+        try:
+            with open(self.state_path) as f:
+                return json.load(f)
+        except Exception:
+            return None
+
+    def clear(self) -> None:
+        """Clear verification state."""
+        if self.state_path.exists():
+            self.state_path.unlink()
+
+
+# =============================================================================
+# CLI Interface
+# =============================================================================
+
+def main():
+    """CLI entry point for verification runner."""
+    import argparse
+
+    parser = argparse.ArgumentParser(description="Run verification suite")
+    parser.add_argument("--test-only", action="store_true", help="Run tests only")
+    parser.add_argument("--lint-only", action="store_true", help="Run lint only")
+    parser.add_argument("--types-only", action="store_true", help="Run types only")
+    parser.add_argument("--quick", action="store_true", help="Skip type checking")
+    parser.add_argument("--json", action="store_true", help="Output JSON")
+    parser.add_argument("--project", type=str, help="Project root path")
+
+    args = parser.parse_args()
+
+    project_root = Path(args.project) if args.project else Path.cwd()
+    runner = VerificationRunner(project_root=project_root)
+
+    if args.test_only:
+        result = runner.run_tests()
+        if result:
+            print(result.to_dict() if args.json else f"Tests: {'PASS' if result.passed else 'FAIL'}")
+    elif args.lint_only:
+        result = runner.run_lint()
+        if result:
+            print(result.to_dict() if args.json else f"Lint: {'PASS' if result.passed else 'FAIL'}")
+    elif args.types_only:
+        result = runner.run_types()
+        if result:
+            print(result.to_dict() if args.json else f"Types: {'PASS' if result.passed else 'FAIL'}")
+    else:
+        result = runner.run_all(skip_types=args.quick)
+        if args.json:
+            print(result.to_json())
+        else:
+            status = "PASS" if result.passed else "FAIL"
+            print(f"Verification: {status}")
+            if result.tests:
+                count_str = f" ({result.tests.count} passed)" if result.tests.count else ""
+                print(f"  Tests: {'PASS' if result.tests.passed else 'FAIL'}{count_str}")
+            if result.lint:
+                print(f"  Lint: {'PASS' if result.lint.passed else 'FAIL'}")
+            if result.types:
+                print(f"  Types: {'PASS' if result.types.passed else 'FAIL'}")
+            if result.errors:
+                print(f"\nErrors ({len(result.errors)}):")
+                for err in result.errors[:10]:
+                    loc = f"{err.file}:{err.line}" if err.line else err.file
+                    print(f"  - {loc}: {err.message[:80]}")
+
+
+if __name__ == "__main__":
+    main()