@pennyfarthing/core 10.0.5 → 10.2.0

package/pennyfarthing_scripts/codemarkers/analyze.py

@@ -0,0 +1,501 @@
+"""
+Core code marker analysis engine.
+
+Greps source files for TODO/FIXME/HACK/XXX markers, runs git blame for
+author and age data, and computes staleness.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import fnmatch
+import re
+import time
+from collections import defaultdict
+from datetime import UTC, datetime
+from pathlib import Path
+
+from pennyfarthing_scripts.codemarkers.models import (
+    CodeMarker,
+    CodeMarkersResult,
+    DeprecationMarker,
+    MarkerSummary,
+)
+
+# Marker types to detect (case-sensitive, uppercase only)
+MARKER_PATTERN = re.compile(r"\b(TODO|FIXME|HACK|XXX)\b")
+
+# Default file patterns to exclude from analysis
+DEFAULT_EXCLUDES = [
+    "node_modules/*",
+    "dist/*",
+    "build/*",
+    "*.lock",
+    "*.min.js",
+    "*.min.css",
+    "*.map",
+    "package-lock.json",
+    "pnpm-lock.yaml",
+]
+
+# Binary file extensions to skip
+_BINARY_EXTENSIONS = frozenset({
+    ".png", ".jpg", ".jpeg", ".gif", ".bmp", ".ico", ".svg",
+    ".woff", ".woff2", ".ttf", ".eot", ".otf",
+    ".zip", ".gz", ".tar", ".bz2", ".7z", ".rar",
+    ".pdf", ".doc", ".docx", ".xls", ".xlsx",
+    ".exe", ".dll", ".so", ".dylib", ".o", ".a",
+    ".pyc", ".pyo", ".class", ".jar",
+    ".mp3", ".mp4", ".wav", ".avi", ".mov",
+    ".sqlite", ".db",
+})
+
+
+def _should_exclude(path: str, patterns: list[str]) -> bool:
+    """Check if a file path matches any exclusion pattern."""
+    for pattern in patterns:
+        if fnmatch.fnmatch(path, pattern):
+            return True
+        # Also check the basename for patterns like "*.lock"
+        if fnmatch.fnmatch(path.split("/")[-1], pattern):
+            return True
+    return False
+
+
+def _grep_markers(root: Path, excludes: list[str]) -> list[dict]:
+    """Scan files under root for TODO/FIXME/HACK/XXX markers.
+
+    Args:
+        root: Directory to scan recursively
+        excludes: Glob patterns for files/dirs to skip
+
+    Returns:
+        List of dicts: {path, line, marker_type, text}
+    """
+    results: list[dict] = []
+
+    for file_path in sorted(root.rglob("*")):
+        if not file_path.is_file():
+            continue
+
+        # Skip binary files by extension
+        if file_path.suffix.lower() in _BINARY_EXTENSIONS:
+            continue
+
+        # Get relative path for exclusion matching
+        rel_path = str(file_path.relative_to(root))
+
+        if _should_exclude(rel_path, excludes):
+            continue
+
+        try:
+            content = file_path.read_text(encoding="utf-8", errors="strict")
+        except (UnicodeDecodeError, OSError):
+            continue
+
+        for line_num, line_text in enumerate(content.split("\n"), start=1):
+            match = MARKER_PATTERN.search(line_text)
+            if match:
+                results.append({
+                    "path": rel_path,
+                    "line": line_num,
+                    "marker_type": match.group(1),
+                    "text": line_text.strip(),
+                })
+
+    return results
+
+
+def _parse_blame_porcelain(output: str, line: int) -> dict:
+    """Parse git blame --porcelain output for a specific line.
+
+    Extracts author name and author-time (Unix timestamp).
+
+    Args:
+        output: Raw porcelain output from git blame
+        line: Line number to extract (used for context)
+
+    Returns:
+        Dict with 'author' and 'author_time' keys, or empty dict
+    """
+    if not output.strip():
+        return {}
+
+    author = ""
+    author_time = 0
+
+    for raw_line in output.split("\n"):
+        if raw_line.startswith("author "):
+            author = raw_line[7:]
+        elif raw_line.startswith("author-time "):
+            try:
+                author_time = int(raw_line[12:])
+            except ValueError:
+                pass
+
+    if not author:
+        return {}
+
+    return {"author": author, "author_time": author_time}
+
+
+async def _run_git_command(args: list[str], cwd: Path) -> tuple[str, str, int]:
+    """Run a git command asynchronously.
+
+    Args:
+        args: Git command arguments (without 'git')
+        cwd: Working directory
+
+    Returns:
+        (stdout, stderr, return_code)
+    """
+    proc = await asyncio.create_subprocess_exec(
+        "git",
+        *args,
+        cwd=cwd,
+        stdout=asyncio.subprocess.PIPE,
+        stderr=asyncio.subprocess.PIPE,
+    )
+    stdout, stderr = await proc.communicate()
+    return (
+        stdout.decode("utf-8", errors="replace").strip(),
+        stderr.decode("utf-8", errors="replace").strip(),
+        proc.returncode or 0,
+    )
+
+
+async def _batch_blame_file(
+    repo_path: Path, file_path: str, lines: list[int]
+) -> dict[int, dict]:
+    """Blame an entire file once and extract data for requested lines.
+
+    Args:
+        repo_path: Root of the git repository
+        file_path: Relative path to file within repo
+        lines: Line numbers to extract blame data for
+
+    Returns:
+        Dict mapping line number -> {author, author_time}
+    """
+    stdout, stderr, rc = await _run_git_command(
+        ["blame", "--porcelain", file_path], repo_path
+    )
+
+    if rc != 0:
+        return {}
+
+    # Parse porcelain: each block starts with "<hash> <orig_line> <final_line> <count>"
+    results: dict[int, dict] = {}
+    current_line = 0
+    current_author = ""
+    current_time = 0
+    lines_set = set(lines)
+
+    for raw_line in stdout.split("\n"):
+        # Header line: hash orig_line final_line [count]
+        # Detect by checking that parts[1] is a digit (orig_line number)
+        parts = raw_line.split(" ")
+        if len(parts) >= 3 and len(parts[0]) >= 6 and parts[1].isdigit():
+            try:
+                current_line = int(parts[2])
+            except (ValueError, IndexError):
+                pass
+            current_author = ""
+            current_time = 0
+        elif raw_line.startswith("author "):
+            current_author = raw_line[7:]
+        elif raw_line.startswith("author-time "):
+            try:
+                current_time = int(raw_line[12:])
+            except ValueError:
+                pass
+        elif raw_line.startswith("\t"):
+            # Content line — end of this block
+            if current_line in lines_set:
+                results[current_line] = {
+                    "author": current_author,
+                    "author_time": current_time,
+                }
+
+    return results
+
+
+async def analyze_repo(
+    name: str,
+    path: Path,
+    days: int = 90,
+    excludes: list[str] | None = None,
+) -> CodeMarkersResult:
+    """Analyze a repository for code markers.
+
+    Args:
+        name: Display name for the repository
+        path: Path to the git repository
+        days: Stale threshold in days
+        excludes: Additional file patterns to exclude
+
+    Returns:
+        CodeMarkersResult with markers and summary
+    """
+    resolved = Path(path).resolve()
+
+    if not resolved.exists():
+        return CodeMarkersResult(
+            success=False,
+            repo_name=name,
+            repo_path=str(resolved),
+            stale_threshold_days=days,
+            error=f"Path not found: {resolved}",
+        )
+
+    all_excludes = DEFAULT_EXCLUDES + (excludes or [])
+
+    # Grep for markers
+    raw_markers = _grep_markers(resolved, all_excludes)
+
+    if not raw_markers:
+        return CodeMarkersResult(
+            success=True,
+            repo_name=name,
+            repo_path=str(resolved),
+            stale_threshold_days=days,
+            markers=[],
+            summary=MarkerSummary(),
+        )
+
+    # Group markers by file for batch blame
+    by_file: dict[str, list[dict]] = defaultdict(list)
+    for m in raw_markers:
+        by_file[m["path"]].append(m)
+
+    # Blame each file once
+    now = time.time()
+    markers: list[CodeMarker] = []
+    stale_count = 0
+    type_counts: dict[str, int] = defaultdict(int)
+
+    for file_path, file_markers in by_file.items():
+        line_numbers = [m["line"] for m in file_markers]
+        blame_data = await _batch_blame_file(resolved, file_path, line_numbers)
+
+        for m in file_markers:
+            blame = blame_data.get(m["line"], {})
+            author = blame.get("author", "")
+            author_time = blame.get("author_time", 0)
+
+            # Compute age
+            if author_time > 0:
+                age_days = (now - author_time) / 86400
+                date_str = datetime.fromtimestamp(
+                    author_time, tz=UTC
+                ).isoformat()
+            else:
+                age_days = 0.0
+                date_str = ""
+
+            is_stale = age_days > days
+
+            marker = CodeMarker(
+                path=m["path"],
+                line=m["line"],
+                marker_type=m["marker_type"],
+                text=m["text"],
+                author=author,
+                date=date_str,
+                age_days=round(age_days, 1),
+                is_stale=is_stale,
+            )
+            markers.append(marker)
+
+            type_counts[m["marker_type"]] += 1
+            if is_stale:
+                stale_count += 1
+
+    summary = MarkerSummary(
+        total_markers=len(markers),
+        stale_markers=stale_count,
+        by_type=dict(type_counts),
+    )
+
+    return CodeMarkersResult(
+        success=True,
+        repo_name=name,
+        repo_path=str(resolved),
+        stale_threshold_days=days,
+        markers=markers,
+        summary=summary,
+    )
+
+
+# =============================================================================
+# @deprecated detection (Story 80-2)
+# =============================================================================
+
+# File extensions to scan for @deprecated JSDoc tags
+_DEPRECATION_EXTENSIONS = frozenset({".ts", ".tsx", ".js"})
+
+# Pattern to find @deprecated in JSDoc comments
+_DEPRECATED_PATTERN = re.compile(r"@deprecated\b(.*)")
+
+# Pattern to extract symbol name from export declarations
+_SYMBOL_PATTERN = re.compile(
+    r"export\s+(?:default\s+)?(?:function|class|const|let|var|interface|type|enum)\s+(\w+)"
+)
+
+
+def _grep_deprecations(root: Path, excludes: list[str]) -> list[dict]:
+    """Scan TypeScript/JS files for @deprecated JSDoc tags.
+
+    For each @deprecated tag found, extracts the symbol name from the
+    next export declaration line following the JSDoc block.
+
+    Args:
+        root: Directory to scan recursively
+        excludes: Glob patterns for files/dirs to skip
+
+    Returns:
+        List of dicts: {path, line, symbol, text}
+    """
+    results: list[dict] = []
+
+    for file_path in sorted(root.rglob("*")):
+        if not file_path.is_file():
+            continue
+
+        if file_path.suffix.lower() not in _DEPRECATION_EXTENSIONS:
+            continue
+
+        rel_path = str(file_path.relative_to(root))
+
+        if _should_exclude(rel_path, excludes):
+            continue
+
+        try:
+            content = file_path.read_text(encoding="utf-8", errors="strict")
+        except (UnicodeDecodeError, OSError):
+            continue
+
+        lines = content.split("\n")
+        for line_num, line_text in enumerate(lines, start=1):
+            match = _DEPRECATED_PATTERN.search(line_text)
+            if not match:
+                continue
+
+            # Extract the @deprecated text
+            deprecated_text = line_text.strip()
+
+            # Look ahead for the symbol declaration
+            symbol = ""
+            for ahead in lines[line_num:]:  # line_num is already 1-indexed, so lines[line_num:] starts after current
+                sym_match = _SYMBOL_PATTERN.search(ahead)
+                if sym_match:
+                    symbol = sym_match.group(1)
+                    break
+                # Stop looking if we hit another JSDoc or a blank line after the block closes
+                stripped = ahead.strip()
+                if stripped and not stripped.startswith("*") and not stripped.startswith("/") and not stripped == "":
+                    break
+
+            if symbol:
+                results.append({
+                    "path": rel_path,
+                    "line": line_num,
+                    "symbol": symbol,
+                    "text": deprecated_text,
+                })
+
+    return results
+
+
+def _count_callers(
+    symbol: str, root: Path, defining_file: str
+) -> tuple[int, list[str]]:
+    """Count files that import/reference a deprecated symbol.
+
+    Greps all TypeScript/JS files for the symbol name, excluding
+    the file that defines it.
+
+    Args:
+        symbol: The deprecated symbol name to search for
+        root: Directory to scan
+        defining_file: Relative path of the file defining the symbol (excluded)
+
+    Returns:
+        (caller_count, list_of_caller_paths)
+    """
+    callers: list[str] = []
+
+    for file_path in sorted(root.rglob("*")):
+        if not file_path.is_file():
+            continue
+
+        if file_path.suffix.lower() not in _DEPRECATION_EXTENSIONS:
+            continue
+
+        rel_path = str(file_path.relative_to(root))
+
+        if rel_path == defining_file:
+            continue
+
+        try:
+            content = file_path.read_text(encoding="utf-8", errors="strict")
+        except (UnicodeDecodeError, OSError):
+            continue
+
+        if symbol in content:
+            callers.append(rel_path)
+
+    return len(callers), callers
+
+
+async def analyze_deprecations(
+    path: Path,
+    excludes: list[str] | None = None,
+) -> dict:
+    """Analyze a directory for @deprecated symbols and their callers.
+
+    Args:
+        path: Directory to scan
+        excludes: Additional file patterns to exclude
+
+    Returns:
+        Dict with success, deprecations (list of DeprecationMarker),
+        summary (total_deprecations, deprecations_with_callers), and
+        optionally error.
+    """
+    resolved = Path(path).resolve()
+
+    if not resolved.exists():
+        return {
+            "success": False,
+            "error": f"Path not found: {resolved}",
+        }
+
+    all_excludes = DEFAULT_EXCLUDES + (excludes or [])
+
+    raw = _grep_deprecations(resolved, all_excludes)
+
+    markers: list[DeprecationMarker] = []
+    for item in raw:
+        count, caller_list = _count_callers(
+            item["symbol"], resolved, item["path"]
+        )
+        markers.append(DeprecationMarker(
+            path=item["path"],
+            line=item["line"],
+            symbol=item["symbol"],
+            text=item["text"],
+            caller_count=count,
+            callers=caller_list,
+        ))
+
+    with_callers = sum(1 for m in markers if m.caller_count > 0)
+
+    return {
+        "success": True,
+        "deprecations": markers,
+        "summary": {
+            "total_deprecations": len(markers),
+            "deprecations_with_callers": with_callers,
+        },
+    }

package/pennyfarthing_scripts/codemarkers/cli.py

@@ -0,0 +1,179 @@
+"""
+CLI commands for code marker analysis.
+
+Usage:
+    pf codemarkers analyze [OPTIONS]
+    pf codemarkers stale [OPTIONS]
+    pf codemarkers summary [OPTIONS]
+"""
+
+from __future__ import annotations
+
+import asyncio
+from pathlib import Path
+
+import click
+
+
+@click.group()
+def codemarkers():
+    """Code marker detection (TODO, FIXME, HACK, XXX).
+
+    \b
+    Commands:
+      analyze  - Full marker analysis with blame data
+      stale    - Show only stale markers (older than threshold)
+      summary  - Summary counts by marker type
+    """
+    pass
+
+
+def _common_options(fn):
+    """Shared options for all codemarkers commands."""
+    fn = click.option("--repo", help="Analyze a single named repo from repos.yaml")(fn)
+    fn = click.option("--path", "repo_path", type=click.Path(), help="Analyze a standalone repo path")(fn)
+    fn = click.option("--days", default=90, show_default=True, help="Stale threshold in days")(fn)
+    fn = click.option("--top", default=20, show_default=True, help="Number of top results to show")(fn)
+    fn = click.option("--format", "fmt", type=click.Choice(["table", "json", "csv"]), default="table", show_default=True)(fn)
+    fn = click.option("--output", "output_file", type=click.Path(), help="Write output to file")(fn)
+    fn = click.option("--exclude", multiple=True, help="Additional exclude patterns (repeatable)")(fn)
+    return fn
+
+
+def _run_analysis(repo: str | None, repo_path: str | None, days: int, exclude: tuple):
+    """Run analysis and return result."""
+    from pennyfarthing_scripts.codemarkers.analyze import analyze_repo
+    from pennyfarthing_scripts.common.config import get_project_root
+
+    excludes = list(exclude) if exclude else None
+
+    if repo_path:
+        p = Path(repo_path).resolve()
+        return asyncio.run(analyze_repo(p.name, p, days, excludes))
+    elif repo:
+        project_root = get_project_root()
+        from pennyfarthing_scripts.common.config import load_yaml_config
+        repos_yaml = load_yaml_config(project_root / ".pennyfarthing" / "repos.yaml")
+        if repos_yaml and repo in repos_yaml:
+            cfg = repos_yaml[repo]
+            rpath = cfg.get("path", repo) if isinstance(cfg, dict) else str(cfg)
+            return asyncio.run(
+                analyze_repo(repo, project_root / rpath, days, excludes)
+            )
+        else:
+            candidate = project_root / repo
+            if candidate.exists():
+                return asyncio.run(analyze_repo(repo, candidate, days, excludes))
+            raise click.ClickException(f"Repo not found: {repo}")
+    else:
+        project_root = get_project_root()
+        return asyncio.run(
+            analyze_repo(project_root.name, project_root, days, excludes)
+        )
+
+
+def _output_result(result, fmt: str, output_file: str | None, top: int, mode: str):
+    """Format and output the analysis result."""
+    from pennyfarthing_scripts.codemarkers.formatters import (
+        export_csv,
+        export_json,
+        format_marker_table,
+        format_summary,
+    )
+
+    if fmt == "json":
+        text = export_json(result)
+    elif fmt == "csv":
+        text = export_csv(result.markers[:top])
+    else:
+        format_summary(result)
+        if mode == "stale":
+            stale = [m for m in result.markers if m.is_stale]
+            text = format_marker_table(stale, top)
+        else:
+            text = format_marker_table(result.markers, top)
+
+    if output_file:
+        Path(output_file).write_text(text)
+        click.echo(f"Output written to {output_file}", err=True)
+    else:
+        click.echo(text)
+
+
+@codemarkers.command()
+@_common_options
+def analyze(repo, repo_path, days, top, fmt, output_file, exclude):
+    """Full marker analysis with blame data."""
+    result = _run_analysis(repo, repo_path, days, exclude)
+    _output_result(result, fmt, output_file, top, "analyze")
+
+
+@codemarkers.command()
+@_common_options
+def stale(repo, repo_path, days, top, fmt, output_file, exclude):
+    """Show only stale markers (older than threshold)."""
+    result = _run_analysis(repo, repo_path, days, exclude)
+    _output_result(result, fmt, output_file, top, "stale")
+
+
+@codemarkers.command()
+@_common_options
+def summary(repo, repo_path, days, top, fmt, output_file, exclude):
+    """Summary counts by marker type."""
+    result = _run_analysis(repo, repo_path, days, exclude)
+    if fmt == "json":
+        from pennyfarthing_scripts.codemarkers.formatters import export_json
+        click.echo(export_json(result))
+    else:
+        from pennyfarthing_scripts.codemarkers.formatters import format_summary
+        format_summary(result, file=click.get_text_stream("stdout"))
+
+
+def _run_deprecation_analysis(repo_path, exclude):
+    """Run deprecation analysis and return result dict."""
+    from pennyfarthing_scripts.codemarkers.analyze import analyze_deprecations
+
+    excludes = list(exclude) if exclude else None
+    if repo_path:
+        p = Path(repo_path).resolve()
+    else:
+        from pennyfarthing_scripts.common.config import get_project_root
+        p = get_project_root()
+
+    return asyncio.run(analyze_deprecations(p, excludes))
+
+
+@codemarkers.command()
+@_common_options
+def deprecations(repo, repo_path, days, top, fmt, output_file, exclude):
+    """Scan for @deprecated symbols and cross-reference callers."""
+    result = _run_deprecation_analysis(repo_path, exclude)
+
+    if fmt == "json":
+        import json
+        from dataclasses import asdict
+
+        output = {
+            "success": result["success"],
+            "deprecations": [asdict(m) for m in result.get("deprecations", [])],
+            "summary": result.get("summary", {}),
+        }
+        text = json.dumps(output, indent=2)
+    else:
+        lines = []
+        deps = result.get("deprecations", [])
+        summary = result.get("summary", {})
+        lines.append(f"Total deprecations: {summary.get('total_deprecations', 0)}")
+        lines.append(f"With active callers: {summary.get('deprecations_with_callers', 0)}")
+        lines.append("")
+        for m in deps[:top]:
+            callers_str = f" ({m.caller_count} callers)" if m.caller_count else ""
+            lines.append(f"  {m.symbol} @ {m.path}:{m.line}{callers_str}")
+            lines.append(f"    {m.text}")
+        text = "\n".join(lines)
+
+    if output_file:
+        Path(output_file).write_text(text)
+        click.echo(f"Output written to {output_file}", err=True)
+    else:
+        click.echo(text)