@pennyfarthing/core 11.3.2 → 11.3.4

package/pennyfarthing_scripts/hotspots/analyze.py

@@ -1,613 +0,0 @@
-"""
-Core hotspot analysis engine.
-
-Uses PyDriller to mine git history, computes per-file metrics, and produces
-scored hotspot results. Falls back to raw git log parsing if PyDriller is unavailable.
-"""
-
-from __future__ import annotations
-
-import asyncio
-import fnmatch
-import logging
-import re
-from collections import defaultdict
-from datetime import UTC, datetime, timedelta
-from pathlib import Path
-
-from pennyfarthing_scripts.hotspots.models import (
-    DirectoryHotspot,
-    FileHotspot,
-    HotspotResult,
-    MultiRepoHotspotResult,
-)
-
-logger = logging.getLogger("hotspots")
-
-# Scoring weights (must sum to 1.0)
-WEIGHT_BUG_FIXES = 0.35
-WEIGHT_CHANGES = 0.30
-WEIGHT_AUTHORS = 0.20
-WEIGHT_CHURN = 0.10
-WEIGHT_RECENCY = 0.05
-
-# Default file patterns to exclude from analysis
-DEFAULT_EXCLUDES = [
-    "node_modules/*",
-    "dist/*",
-    "build/*",
-    "*.lock",
-    "*.min.js",
-    "*.min.css",
-    "*.map",
-    "package-lock.json",
-    "pnpm-lock.yaml",
-    # Dotfiles
-    ".*",
-    # Images
-    "*.png",
-    "*.jpg",
-    "*.jpeg",
-    "*.gif",
-    "*.svg",
-    "*.ico",
-    # Fonts
-    "*.woff",
-    "*.woff2",
-    "*.ttf",
-    "*.eot",
-    # Generated files
-    "*.d.ts",
-    "*.snap",
-    "*.d.ts.map",
-    # CI config
-    ".github/*",
-    # Sprint/session operational files (not code quality signals)
-    "sprint/*",
-    ".session/*",
-    # Config/manifest files — high churn but not code quality signals
-    "package.json",
-    "*/package.json",
-    "tsconfig.json",
-    "*/tsconfig.json",
-    "tsconfig.*.json",
-    "*/tsconfig.*.json",
-    # Documentation — frequently edited but not code hotspots
-    "*.md",
-    "CLAUDE.md",
-    "*/CLAUDE.md",
-    "CLAUDE-*.md",
-    "*/CLAUDE-*.md",
-    "README.md",
-    "*/README.md",
-    "CHANGELOG.md",
-    "*/CHANGELOG.md",
-    "docs/*",
-    # YAML config (sprint, workflow, etc.)
-    "*.yaml",
-    "*.yml",
-]
-
-# Regex for identifying bug-fix commits
-BUG_FIX_PATTERN = re.compile(
-    r"\b(fix|bug|patch|hotfix|regression|resolve[ds]?)\b", re.IGNORECASE
-)
-
-
-async def _run_git_command(args: list[str], cwd: Path) -> tuple[str, str, int]:
-    """Run a git command asynchronously.
-
-    Args:
-        args: Git command arguments (without 'git')
-        cwd: Working directory
-
-    Returns:
-        (stdout, stderr, return_code)
-    """
-    proc = await asyncio.create_subprocess_exec(
-        "git",
-        *args,
-        cwd=cwd,
-        stdout=asyncio.subprocess.PIPE,
-        stderr=asyncio.subprocess.PIPE,
-    )
-    stdout, stderr = await proc.communicate()
-    return (
-        stdout.decode("utf-8", errors="replace").strip(),
-        stderr.decode("utf-8", errors="replace").strip(),
-        proc.returncode or 0,
-    )
-
-
-async def _run_git_log(
-    repo_path: Path, since_days: int, branch: str = "--all"
-) -> tuple[str, str, int]:
-    """Run git log with numstat for hotspot analysis."""
-    args = [
-        "log",
-        f"--since={since_days} days ago",
-        branch,
-        "--numstat",
-        "--pretty=format:COMMIT:%H|%an|%aI|%s",
-    ]
-    return await _run_git_command(args, repo_path)
-
-
-def is_bug_fix_commit(message: str) -> bool:
-    """Check if a commit message indicates a bug fix."""
-    return bool(BUG_FIX_PATTERN.search(message))
-
-
-def _parse_git_log(output: str) -> list[dict]:
-    """Parse git log --numstat output into structured commit dicts.
-
-    Each dict has: hash, author, date, message, files (list of {path, added, deleted}).
-
-    Handles binary files (numstat shows '-' for added/deleted).
-    """
-    if not output.strip():
-        return []
-
-    commits = []
-    current_commit: dict | None = None
-
-    for line in output.split("\n"):
-        line = line.rstrip()
-
-        if line.startswith("COMMIT:"):
-            # Save previous commit
-            if current_commit is not None:
-                commits.append(current_commit)
-
-            parts = line[7:].split("|", 3)
-            if len(parts) >= 4:
-                current_commit = {
-                    "hash": parts[0],
-                    "author": parts[1],
-                    "date": parts[2],
-                    "message": parts[3],
-                    "files": [],
-                }
-            else:
-                current_commit = None
-            continue
-
-        if current_commit is None:
-            continue
-
-        # Numstat lines: "added\tdeleted\tpath" or "-\t-\tpath" for binary
-        if "\t" in line:
-            parts = line.split("\t", 2)
-            if len(parts) == 3:
-                added_str, deleted_str, path = parts
-                # Binary files show "-" for numstat
-                added = int(added_str) if added_str != "-" else 0
-                deleted = int(deleted_str) if deleted_str != "-" else 0
-                current_commit["files"].append(
-                    {"path": path, "added": added, "deleted": deleted}
-                )
-
-    # Don't forget the last commit
-    if current_commit is not None:
-        commits.append(current_commit)
-
-    return commits
-
-
-def calculate_hotspot_score(
-    change_count: int,
-    bug_fix_count: int,
-    author_count: int,
-    churn: int,
-    age_days: float,
-    *,
-    max_changes: int = 1,
-    max_bugs: int = 1,
-    max_authors: int = 1,
-    max_churn: int = 1,
-    window_days: int = 90,
-) -> float:
-    """Compute weighted composite hotspot score (0–100).
-
-    Each dimension is normalized against the max value seen in the dataset,
-    then weighted and summed.
-
-    Args:
-        change_count: Number of commits touching this file
-        bug_fix_count: Number of bug-fix commits touching this file
-        author_count: Distinct authors who changed this file
-        churn: Total lines added + deleted
-        age_days: Days since last change
-        max_changes: Max change_count in dataset (for normalization)
-        max_bugs: Max bug_fix_count in dataset
-        max_authors: Max author_count in dataset
-        max_churn: Max churn in dataset
-        window_days: Analysis time window in days
-
-    Returns:
-        Score between 0.0 and 100.0
-    """
-    norm_changes = change_count / max_changes if max_changes > 0 else 0
-    norm_bugs = bug_fix_count / max_bugs if max_bugs > 0 else 0
-    norm_authors = author_count / max_authors if max_authors > 0 else 0
-    norm_churn = churn / max_churn if max_churn > 0 else 0
-    # Recency: recently changed files score higher
-    recency = max(0.0, 1.0 - (age_days / window_days)) if window_days > 0 else 0
-
-    raw = (
-        WEIGHT_BUG_FIXES * norm_bugs
-        + WEIGHT_CHANGES * norm_changes
-        + WEIGHT_AUTHORS * norm_authors
-        + WEIGHT_CHURN * norm_churn
-        + WEIGHT_RECENCY * recency
-    )
-
-    return round(min(raw * 100, 100.0), 1)
-
-
-def _should_exclude(path: str, patterns: list[str]) -> bool:
-    """Check if a file path matches any exclusion pattern."""
-    for pattern in patterns:
-        if fnmatch.fnmatch(path, pattern):
-            return True
-        # Also check the basename for patterns like "*.lock"
-        if fnmatch.fnmatch(path.split("/")[-1], pattern):
-            return True
-    return False
-
-
-def _aggregate_by_directory(file_hotspots: list[FileHotspot]) -> list[DirectoryHotspot]:
-    """Roll up file-level hotspots to parent directories."""
-    dir_data: dict[str, dict] = defaultdict(
-        lambda: {
-            "file_count": 0,
-            "total_changes": 0,
-            "total_bug_fixes": 0,
-            "author_counts": [],
-            "scores": [],
-        }
-    )
-
-    for fh in file_hotspots:
-        parent = str(Path(fh.path).parent)
-        if parent == ".":
-            parent = "/"
-        entry = dir_data[parent]
-        entry["file_count"] += 1
-        entry["total_changes"] += fh.change_count
-        entry["total_bug_fixes"] += fh.bug_fix_count
-        entry["author_counts"].append(fh.author_count)
-        entry["scores"].append(fh.hotspot_score)
-
-    result = []
-    for path, data in dir_data.items():
-        avg_authors = (
-            sum(data["author_counts"]) / len(data["author_counts"])
-            if data["author_counts"]
-            else 0
-        )
-        avg_score = (
-            sum(data["scores"]) / len(data["scores"]) if data["scores"] else 0
-        )
-        result.append(
-            DirectoryHotspot(
-                path=path,
-                file_count=data["file_count"],
-                total_changes=data["total_changes"],
-                total_bug_fixes=data["total_bug_fixes"],
-                avg_author_count=round(avg_authors, 1),
-                hotspot_score=round(avg_score, 1),
-            )
-        )
-
-    result.sort(key=lambda d: d.hotspot_score, reverse=True)
-    return result
-
-
-async def analyze_repo(
-    name: str,
-    path: Path,
-    days: int = 90,
-    excludes: list[str] | None = None,
-    branch: str = "--all",
-) -> HotspotResult:
-    """Analyze a single repository for code hotspots.
-
-    Uses PyDriller for git mining when available, falls back to raw git log parsing.
-
-    Args:
-        name: Display name for the repository
-        path: Path to the git repository
-        days: Time window in days
-        excludes: Additional file patterns to exclude
-        branch: Branch spec (default --all)
-
-    Returns:
-        HotspotResult with file and directory hotspots
-    """
-    all_excludes = DEFAULT_EXCLUDES + (excludes or [])
-    resolved = Path(path).resolve()
-
-    if not resolved.exists():
-        return HotspotResult(
-            success=False,
-            repo_name=name,
-            repo_path=str(resolved),
-            time_window_days=days,
-            error=f"Path not found: {resolved}",
-        )
-
-    try:
-        return await _analyze_repo_pydriller(name, resolved, days, all_excludes)
-    except ImportError:
-        logger.info("[hotspots] PyDriller not available, using git log fallback")
-        return await _analyze_repo_gitlog(name, resolved, days, all_excludes, branch)
-
-
-async def _analyze_repo_pydriller(
-    name: str,
-    resolved: Path,
-    days: int,
-    all_excludes: list[str],
-) -> HotspotResult:
-    """PyDriller-backed hotspot analysis."""
-    from pydriller import Repository
-
-    since = datetime.now(UTC) - timedelta(days=days)
-    now = datetime.now(UTC)
-
-    file_metrics: dict[str, dict] = defaultdict(
-        lambda: {
-            "change_count": 0,
-            "bug_fix_count": 0,
-            "authors": set(),
-            "lines_added": 0,
-            "lines_deleted": 0,
-            "last_changed": "",
-        }
-    )
-
-    def _collect() -> int:
-        commit_count = 0
-        repo = Repository(str(resolved), since=since)
-        for commit in repo.traverse_commits():
-            commit_count += 1
-            is_fix = is_bug_fix_commit(commit.msg)
-            commit_date = commit.committer_date.isoformat()
-
-            for mod in commit.modified_files:
-                fpath = mod.new_path or mod.old_path
-                if not fpath:
-                    continue
-                if _should_exclude(fpath, all_excludes):
-                    continue
-
-                m = file_metrics[fpath]
-                m["change_count"] += 1
-                if is_fix:
-                    m["bug_fix_count"] += 1
-                m["authors"].add(commit.author.name)
-                m["lines_added"] += mod.added_lines
-                m["lines_deleted"] += mod.deleted_lines
-                if not m["last_changed"] or commit_date > m["last_changed"]:
-                    m["last_changed"] = commit_date
-        return commit_count
-
-    loop = asyncio.get_event_loop()
-    commit_count = await loop.run_in_executor(None, _collect)
-    logger.info("[hotspots] PyDriller: %d commits, %d files after filtering",
-                commit_count, len(file_metrics))
-
-    if not file_metrics:
-        return HotspotResult(
-            success=True,
-            repo_name=name,
-            repo_path=str(resolved),
-            time_window_days=days,
-            commit_count=commit_count,
-        )
-
-    return _build_hotspot_result(name, resolved, days, commit_count, file_metrics, now)
-
-
-async def _analyze_repo_gitlog(
-    name: str,
-    resolved: Path,
-    days: int,
-    all_excludes: list[str],
-    branch: str,
-) -> HotspotResult:
-    """Fallback: raw git log parsing."""
-    stdout, stderr, rc = await _run_git_log(resolved, days, branch)
-
-    if rc != 0:
-        return HotspotResult(
-            success=False,
-            repo_name=name,
-            repo_path=str(resolved),
-            time_window_days=days,
-            error=f"git log failed: {stderr}",
-        )
-
-    commits = _parse_git_log(stdout)
-
-    if not commits:
-        return HotspotResult(
-            success=True,
-            repo_name=name,
-            repo_path=str(resolved),
-            time_window_days=days,
-            commit_count=0,
-        )
-
-    file_metrics: dict[str, dict] = defaultdict(
-        lambda: {
-            "change_count": 0,
-            "bug_fix_count": 0,
-            "authors": set(),
-            "lines_added": 0,
-            "lines_deleted": 0,
-            "last_changed": "",
-        }
-    )
-
-    now = datetime.now(UTC)
-
-    for commit in commits:
-        is_fix = is_bug_fix_commit(commit["message"])
-        for file_info in commit["files"]:
-            fpath = file_info["path"]
-            if _should_exclude(fpath, all_excludes):
-                continue
-
-            m = file_metrics[fpath]
-            m["change_count"] += 1
-            if is_fix:
-                m["bug_fix_count"] += 1
-            m["authors"].add(commit["author"])
-            m["lines_added"] += file_info["added"]
-            m["lines_deleted"] += file_info["deleted"]
-            if not m["last_changed"] or commit["date"] > m["last_changed"]:
-                m["last_changed"] = commit["date"]
-
-    if not file_metrics:
-        return HotspotResult(
-            success=True,
-            repo_name=name,
-            repo_path=str(resolved),
-            time_window_days=days,
-            commit_count=len(commits),
-        )
-
-    return _build_hotspot_result(name, resolved, days, len(commits), file_metrics, now)
-
-
-def _build_hotspot_result(
-    name: str,
-    resolved: Path,
-    days: int,
-    commit_count: int,
-    file_metrics: dict[str, dict],
-    now: datetime,
-) -> HotspotResult:
-    """Build scored HotspotResult from aggregated file metrics."""
-    max_changes = max(m["change_count"] for m in file_metrics.values())
-    max_bugs = max(m["bug_fix_count"] for m in file_metrics.values()) or 1
-    max_authors = max(len(m["authors"]) for m in file_metrics.values())
-    max_churn = max(
-        m["lines_added"] + m["lines_deleted"] for m in file_metrics.values()
-    ) or 1
-
-    file_hotspots = []
-    for fpath, m in file_metrics.items():
-        churn = m["lines_added"] + m["lines_deleted"]
-
-        age_days = days
-        if m["last_changed"]:
-            try:
-                last_dt = datetime.fromisoformat(m["last_changed"])
-                age_days = max(0, (now - last_dt).total_seconds() / 86400)
-            except (ValueError, TypeError):
-                pass
-
-        score = calculate_hotspot_score(
-            change_count=m["change_count"],
-            bug_fix_count=m["bug_fix_count"],
-            author_count=len(m["authors"]),
-            churn=churn,
-            age_days=age_days,
-            max_changes=max_changes,
-            max_bugs=max_bugs,
-            max_authors=max_authors,
-            max_churn=max_churn,
-            window_days=days,
-        )
-
-        file_hotspots.append(
-            FileHotspot(
-                path=fpath,
-                change_count=m["change_count"],
-                bug_fix_count=m["bug_fix_count"],
-                author_count=len(m["authors"]),
-                lines_added=m["lines_added"],
-                lines_deleted=m["lines_deleted"],
-                churn=churn,
-                last_changed=m["last_changed"],
-                hotspot_score=score,
-            )
-        )
-
-    file_hotspots.sort(key=lambda h: h.hotspot_score, reverse=True)
-    directory_hotspots = _aggregate_by_directory(file_hotspots)
-
-    return HotspotResult(
-        success=True,
-        repo_name=name,
-        repo_path=str(resolved),
-        time_window_days=days,
-        commit_count=commit_count,
-        file_hotspots=file_hotspots,
-        directory_hotspots=directory_hotspots,
-    )
-
-
-async def analyze_all_repos(
-    project_root: Path,
-    days: int = 90,
-    excludes: list[str] | None = None,
-    branch: str = "--all",
-    skip_types: list[str] | None = None,
-) -> MultiRepoHotspotResult:
-    """Analyze all repos found under project root in parallel.
-
-    Discovers repos via repos.yaml if available, otherwise analyzes project_root itself.
-
-    Args:
-        project_root: Root directory of the project
-        days: Time window in days
-        excludes: Additional file patterns to exclude
-        branch: Branch spec
-        skip_types: Repo types to exclude (e.g. ["orchestrator"])
-
-    Returns:
-        MultiRepoHotspotResult with per-repo results
-    """
-    from pennyfarthing_scripts.common.config import load_yaml_config
-
-    repos_yaml = load_yaml_config(project_root / ".pennyfarthing" / "repos.yaml")
-
-    repos: list[tuple[str, Path]] = []
-
-    if repos_yaml and isinstance(repos_yaml, dict):
-        # Extract repos from repos.yaml
-        for repo_name, repo_config in repos_yaml.items():
-            if isinstance(repo_config, dict):
-                # Filter by type if skip_types is provided
-                if skip_types:
-                    repo_type = repo_config.get("type")
-                    if repo_type and repo_type in skip_types:
-                        continue
-                repo_path = repo_config.get("path", repo_name)
-            else:
-                repo_path = str(repo_config)
-            full_path = project_root / repo_path
-            if full_path.exists() and (full_path / ".git").exists():
-                repos.append((repo_name, full_path))
-    else:
-        # No repos.yaml — analyze the project root itself if it's a git repo
-        if (project_root / ".git").exists():
-            repos.append((project_root.name, project_root))
-
-    if not repos:
-        return MultiRepoHotspotResult(
-            success=False,
-            error="No git repositories found to analyze",
-        )
-
-    tasks = [analyze_repo(name, path, days, excludes, branch) for name, path in repos]
-    results = await asyncio.gather(*tasks, return_exceptions=False)
-
-    return MultiRepoHotspotResult(
-        success=True,
-        repo_results=list(results),
-    )