clckernel 1.2.19 → 2.0.0

package/tools/verify_tdd.py

@@ -1,209 +1,451 @@
 #!/usr/bin/env python3
 """
-TDD Flow Validator
-Validates that test files exist for changed code, checks for test patterns,
-and reports on test coverage presence (pytest, unittest, manage.py test).
-Self-contained: uses only Python stdlib (pathlib, subprocess, json).
+Red -> Green TDD Verifier (pytest)
+
+Proves that a change to Python production code ships with tests that
+FAIL without the change (red) and PASS with it (green).
+
+Two trees are compared:
+  - something staged:  BASE = HEAD,   CANDIDATE = the index (staged content)
+  - nothing staged:    BASE = HEAD~1, CANDIDATE = HEAD
+Both trees are materialized from git into temporary directories, so the
+working copy is never evaluated and the caller's index/HEAD/worktree are
+never touched.
+
+Each changed file belongs to its project root: the nearest ancestor (inside
+the repository) holding a Python project marker, else the repository root.
+Files are grouped by root and each group is verified on its own, with
+cwd and PYTHONPATH set to that root in the temporary tree; a group whose code
+changed without a changed test in the same group fails. With a single root
+the result is the same as a whole-repository run.
+
+The project config (.clckernel.yaml) widens or narrows what counts as
+production code: a file matching `tdd.code` is code whatever its language,
+a file matching `tdd.ignore` never is. A .py file matching `tdd.code` is
+handed to verify_tdd.js when it is installed next to this file (it is the
+code of the JS suite, e.g. Python tools tested by JS tests).
+
+Self-contained: Python 3.9+ standard library only.
 """
 from __future__ import annotations
 
+import importlib.util
 import json
 import os
 import pathlib
+import shutil
 import subprocess
 import sys
+import tarfile
+import tempfile
 
+# Sibling helper shipped next to the guards; works both as a script and when
+# loaded by audit.py through importlib.
+_TOOLS_DIR = os.path.dirname(os.path.abspath(__file__))
+if _TOOLS_DIR not in sys.path:
+    sys.path.insert(0, _TOOLS_DIR)
+import _changes  # noqa: E402
 
-def _get_changed_files(target_dir: str) -> list[str] | None:
-    """Return list of changed files from git diff, or None on failure."""
-    for ref in ("HEAD~1", "--cached"):
-        try:
-            args = ["git", "diff", "--name-only"]
-            if ref == "--cached":
-                args.append("--cached")
-            else:
-                args.append(ref)
-            out = subprocess.check_output(
-                args, cwd=target_dir,
-                stderr=subprocess.DEVNULL, text=True,
-            )
-            files = [f.strip() for f in out.splitlines() if f.strip()]
-            if files:
-                return files
-        except (subprocess.CalledProcessError, FileNotFoundError):
-            continue
-    return None
+NAME = "verify_tdd"
+DEFAULT_TIMEOUT = 300
+OUTPUT_TAIL_LINES = 40
+TEST_DIRS = {"tests", "test"}
+# Collected test-support files: overlaid on BASE, never passed to pytest.
+SUPPORT_NAMES = {"conftest.py", "__init__.py"}
+# Files that make their directory a Python project root.
+PROJECT_MARKERS = ("pyproject.toml", "requirements.txt", "setup.cfg", "setup.py", "manage.py")
 
 
-def _detect_test_framework(target_dir: str) -> str:
-    """Detect which test framework is in use."""
-    root = pathlib.Path(target_dir)
+class GitError(Exception):
+    """Raised when a git command fails."""
 
-    # Check for pytest.ini or pytest config
-    for name in ("pytest.ini", "setup.cfg", "pyproject.toml"):
-        fpath = root / name
-        if fpath.exists():
-            try:
-                text = fpath.read_text(encoding="utf-8", errors="replace").lower()
-            except OSError:
-                continue
-            if "pytest" in text:
-                return "pytest"
 
-    # Check for Django manage.py test
-    if (root / "manage.py").exists():
-        return "django"
+def _never(_rel: str) -> bool:
+    return False
 
-    # Check for unittest patterns
-    for py_file in root.rglob("test_*.py"):
-        try:
-            content = py_file.read_text(encoding="utf-8", errors="replace")
-            if "import unittest" in content or "unittest.TestCase" in content:
-                return "unittest"
-        except OSError:
-            continue
 
-    return "unknown"
+NO_RULES = {"extra": _never, "ignore": _never, "handed": _never}
+# The other-language verifier: when it is installed next to this one, a .py
+# file listed in tdd.code is its code (e.g. Python tools tested by JS tests).
+SIBLING_VERIFIER = "verify_tdd.js"
 
 
-def _find_test_files(target_dir: str) -> list[str]:
-    """Find all test files in the project."""
-    root = pathlib.Path(target_dir)
-    test_files: list[str] = []
+def _code_rules(repo: str) -> dict:
+    """
+    What counts as code beyond .py (tdd.code), what never does (tdd.ignore),
+    and which .py files are handed to the sibling verifier: those matching
+    tdd.code when verify_tdd.js is installed next to this file. Listing a .py
+    file in tdd.code has no other purpose, and verify_tdd.js already counts it
+    as code, so each file has exactly one owner.
+    """
+    config = _changes.project_config(repo)
+    extra = _changes.glob_matcher(config["tdd"]["code"] if config else [])
+    sibling = os.path.isfile(os.path.join(_TOOLS_DIR, SIBLING_VERIFIER))
+    return {
+        "extra": extra,
+        "ignore": _changes.glob_matcher(config["tdd"]["ignore"] if config else []),
+        "handed": (lambda rel: rel.endswith(".py") and extra(rel)) if sibling else _never,
+    }
 
-    for py_file in root.rglob("test_*.py"):
-        rel = str(py_file.relative_to(root))
-        if "__pycache__" not in rel:
-            test_files.append(rel)
 
-    for py_file in root.rglob("*_test.py"):
-        rel = str(py_file.relative_to(root))
-        if "__pycache__" not in rel and rel not in test_files:
-            test_files.append(rel)
+def _in_scope(rel: str, rules: dict) -> bool:
+    """True for a file the verifier looks at: .py, or listed in tdd.code (minus handed files)."""
+    if rules["handed"](rel):
+        return False
+    return rel.endswith(".py") or rules["extra"](rel)
 
-    # Check tests/ directories
-    for tests_dir in root.rglob("tests"):
-        if tests_dir.is_dir():
-            for py_file in tests_dir.glob("*.py"):
-                rel = str(py_file.relative_to(root))
-                if "__pycache__" not in rel and py_file.name != "__init__.py" and rel not in test_files:
-                    test_files.append(rel)
 
-    return sorted(test_files)
+def _git(repo: str, *args: str) -> str:
+    try:
+        return subprocess.run(
+            ["git", *args], cwd=repo, check=True,
+            stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True,
+        ).stdout
+    except (subprocess.CalledProcessError, FileNotFoundError) as exc:
+        raise GitError(str(exc)) from exc
 
 
-def _count_test_methods(target_dir: str, filepath: str) -> int:
-    """Count test methods in a test file."""
-    full_path = pathlib.Path(target_dir) / filepath
+def _rev_exists(repo: str, rev: str) -> bool:
     try:
-        content = full_path.read_text(encoding="utf-8", errors="replace")
-    except OSError:
-        return 0
+        _git(repo, "rev-parse", "--verify", "--quiet", f"{rev}^{{commit}}")
+        return True
+    except GitError:
+        return False
 
-    import re
-    # Count test_ method definitions and TestCase subclasses
-    test_funcs = len(re.findall(r"def (test_\w+)", content))
-    return test_funcs
 
-
-def check(target_dir: str | None = None) -> dict:
+def _resolve_trees(repo: str, rules: dict = NO_RULES) -> dict | None:
     """
-    Validate TDD flow: tests exist, test framework detected, coverage signals.
-
-    Returns:
-        dict with keys: exit_code, skipped, name, data
-        data.framework, data.test_files, data.changed_code_files,
-        data.test_count, data.has_test_coverage
+    Pick BASE/CANDIDATE and list the changed files (deletions ignored). Pure
+    renames (identical content moved to a new path) are not changes to verify:
+    they are listed apart in "renamed" and count neither as code nor as tests.
     """
-    dir_path = target_dir or os.getcwd()
+    all_changes = _changes.changed_files(repo)
+    if all_changes["mode"] is None:
+        raise GitError("could not list the changed files")
+    pure = _changes.pure_renames(repo, all_changes["mode"])
+    changes = {"mode": all_changes["mode"], "files": [f for f in all_changes["files"] if f not in pure]}
+    renamed = [f for f in all_changes["files"] if f in pure and _in_scope(f, rules)]
+    has_head = _rev_exists(repo, "HEAD")
+
+    if changes["mode"] == "staged":
+        if not has_head and not changes["files"]:
+            return None  # no commits and nothing staged
+        return {"mode": "staged", "base": "HEAD" if has_head else None,
+                "candidate": None, "changed": changes["files"], "renamed": renamed,
+                "snapshot": changes["mode"]}
+
+    base = "HEAD~1" if _rev_exists(repo, "HEAD~1") else None
+    return {"mode": "manual", "base": base, "candidate": "HEAD", "changed": changes["files"],
+            "renamed": renamed, "snapshot": changes["mode"]}
+
+
+def _project_root(rel: str, files: set) -> str:
+    """Nearest ancestor dir of rel holding a project marker; "." for the repository root."""
+    parts = pathlib.PurePosixPath(rel).parts[:-1]
+    for depth in range(len(parts), 0, -1):
+        root = "/".join(parts[:depth])
+        if any(f"{root}/{marker}" in files for marker in PROJECT_MARKERS):
+            return root
+    return "."
+
+
+def _by_root(repo: str, trees: dict, rules: dict = NO_RULES) -> dict:
+    """Changed .py (and tdd.code) files grouped by project root (insertion order: first seen)."""
+    files = set(_changes.list_files(repo, trees["snapshot"]))
+    roots: dict = {}
+    for rel in trees["changed"]:
+        if _in_scope(rel, rules):
+            roots.setdefault(_project_root(rel, files), []).append(rel)
+    return roots
+
+
+def _relative(rel: str, root: str) -> str:
+    return rel if root == "." else rel[len(root) + 1:]
+
+
+def _is_test_path(rel: str) -> bool:
+    parts = pathlib.PurePosixPath(rel).parts
+    name = parts[-1]
+    return (
+        name.startswith("test_")
+        or name.endswith("_test.py")
+        or any(part in TEST_DIRS for part in parts[:-1])
+    )
+
+
+def _classify(changed: list[str], rules: dict = NO_RULES) -> dict:
+    """Split into tests, support and code; tdd.code files are code, tdd.ignore files never are."""
+    tests: list[str] = []
+    support: list[str] = []
+    code: list[str] = []
+    for rel in changed:
+        if not _in_scope(rel, rules):
+            continue
+        if rules["extra"](rel):
+            if not rules["ignore"](rel):
+                code.append(rel)
+            continue
+        name = pathlib.PurePosixPath(rel).name
+        if name == "conftest.py":
+            support.append(rel)
+        elif _is_test_path(rel):
+            (support if name in SUPPORT_NAMES else tests).append(rel)
+        elif not rules["ignore"](rel):
+            code.append(rel)
+    return {"tests": tests, "support": support, "code": code}
+
+
+def _materialize(repo: str, rev: str | None, dest: str) -> None:
+    """Write a tree into dest: the index when rev is None, else a commit."""
+    if rev is None:
+        _git(repo, "checkout-index", "-a", f"--prefix={dest.rstrip(os.sep)}{os.sep}")
+        return
+    proc = subprocess.Popen(
+        ["git", "archive", "--format=tar", rev], cwd=repo,
+        stdout=subprocess.PIPE, stderr=subprocess.PIPE,
+    )
+    root = os.path.realpath(dest)
+    try:
+        with tarfile.open(fileobj=proc.stdout, mode="r|") as tar:
+            for member in tar:
+                target = os.path.realpath(os.path.join(root, member.name))
+                if target != root and not target.startswith(root + os.sep):
+                    raise GitError(f"unsafe path in archive: {member.name}")
+                if hasattr(tarfile, "data_filter"):
+                    tar.extract(member, root, filter="data")
+                else:
+                    tar.extract(member, root)
+    finally:
+        if proc.stdout:
+            proc.stdout.close()
+        stderr = proc.stderr.read().decode("utf-8", "replace") if proc.stderr else ""
+        if proc.wait() != 0:
+            raise GitError(f"git archive {rev} failed: {stderr.strip()}")
+
+
+def _overlay(src_root: str, dest_root: str, files: list[str]) -> None:
+    for rel in files:
+        dest = os.path.join(dest_root, rel)
+        os.makedirs(os.path.dirname(dest), exist_ok=True)
+        shutil.copy2(os.path.join(src_root, rel), dest)
+
+
+def _timeout() -> float:
+    try:
+        value = float(os.environ.get("CLC_TDD_TIMEOUT", DEFAULT_TIMEOUT))
+        return value if value > 0 else DEFAULT_TIMEOUT
+    except ValueError:
+        return DEFAULT_TIMEOUT
+
+
+def _tail(text: str) -> str:
+    return "\n".join(text.splitlines()[-OUTPUT_TAIL_LINES:])
+
 
-    changed = _get_changed_files(dir_path)
-    if changed is None:
-        return {
-            "exit_code": 0, "skipped": True, "name": "verify_tdd",
-            "data": {"message": "No git diff available"},
-        }
-
-    if not changed:
-        return {
-            "exit_code": 0, "skipped": False, "name": "verify_tdd",
-            "data": {
-                "framework": "unknown", "test_files": [],
-                "changed_code_files": [], "test_count": 0,
-                "has_test_coverage": False,
-                "message": "No changed files",
-            },
-        }
-
-    framework = _detect_test_framework(dir_path)
-    test_files = _find_test_files(dir_path)
-
-    # Separate test files from code files in the diff
-    changed_tests = [f for f in changed if f.endswith(".py") and _is_test_file(f)]
-    changed_code = [f for f in changed if f.endswith(".py") and not _is_test_file(f)]
-
-    # Count total test methods
-    total_tests = sum(_count_test_methods(dir_path, tf) for tf in test_files)
-
-    # Determine if there's meaningful test coverage
-    has_tests = len(test_files) > 0
-    has_test_in_diff = len(changed_tests) > 0
-    has_code_in_diff = len(changed_code) > 0
-
-    # TDD indicator: if code changed but no test changed, flag it
-    if has_code_in_diff and not has_test_in_diff and has_tests:
-        msg = (
-            f"Code files changed without corresponding test changes "
-            f"({len(changed_code)} code files, {len(changed_tests)} test files). "
-            f"Framework: {framework}. Total test methods: {total_tests}"
+def _run_pytest(tree: str, tests: list[str]) -> dict:
+    env = dict(os.environ)
+    env["PYTHONDONTWRITEBYTECODE"] = "1"
+    # Put the materialized tree ahead of anything installed (e.g. an editable
+    # install pointing at the working copy).
+    roots = [tree] + ([os.path.join(tree, "src")] if os.path.isdir(os.path.join(tree, "src")) else [])
+    env["PYTHONPATH"] = os.pathsep.join(roots + ([env["PYTHONPATH"]] if env.get("PYTHONPATH") else []))
+    cmd = [sys.executable, "-m", "pytest", *tests, "-q", "-p", "no:cacheprovider"]
+    try:
+        proc = subprocess.run(
+            cmd, cwd=tree, env=env, timeout=_timeout(),
+            stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True,
         )
-        exit_code = 1
-    elif has_code_in_diff and not has_test_in_diff and not has_tests:
-        msg = (
-            f"No test files found in project. "
-            f"{len(changed_code)} code file(s) changed without tests. "
-            f"Framework: {framework}"
+        return {"exit_code": proc.returncode, "timed_out": False, "output": _tail(proc.stdout)}
+    except subprocess.TimeoutExpired as exc:
+        out = exc.stdout.decode("utf-8", "replace") if isinstance(exc.stdout, bytes) else (exc.stdout or "")
+        return {"exit_code": None, "timed_out": True, "output": _tail(out)}
+
+
+def _result(exit_code: int, data: dict, skipped: bool = False) -> dict:
+    return {"exit_code": exit_code, "skipped": skipped, "name": NAME, "data": data}
+
+
+def _pytest_missing(data: dict) -> dict | None:
+    if importlib.util.find_spec("pytest") is not None:
+        return None
+    data["message"] = (
+        f"pytest is not importable by {sys.executable}: red->green cannot be verified. "
+        "Install it with `pip install pytest`."
+    )
+    return _result(1, data)
+
+
+def _materialize_trees(repo: str, trees: dict, tmp: str) -> tuple:
+    candidate_dir = os.path.join(tmp, "candidate")
+    base_dir = os.path.join(tmp, "base")
+    os.makedirs(candidate_dir)
+    os.makedirs(base_dir)
+    _materialize(repo, trees["candidate"], candidate_dir)
+    if trees["base"] is not None:
+        _materialize(repo, trees["base"], base_dir)
+    return candidate_dir, base_dir
+
+
+def _red_green(candidate_root: str, base_root: str, root: str, groups: dict, data: dict) -> dict:
+    """Green on CANDIDATE, then red on BASE + the changed tests, inside one project root."""
+    run_files = [_relative(f, root) for f in groups["tests"]]
+    overlay_files = [_relative(f, root) for f in groups["tests"] + groups["support"]]
+    candidate_dir = candidate_root if root == "." else os.path.join(candidate_root, root)
+    base_dir = base_root if root == "." else os.path.join(base_root, root)
+    os.makedirs(base_dir, exist_ok=True)
+
+    green = _run_pytest(candidate_dir, run_files)
+    data["green"] = green
+    if green["exit_code"] != 0:
+        reason = "timed out" if green["timed_out"] else f"pytest exit {green['exit_code']}"
+        data["message"] = f"Changed tests do not pass with the change ({reason})."
+        return _result(1, data)
+
+    _overlay(candidate_dir, base_dir, overlay_files)
+    red = _run_pytest(base_dir, run_files)
+    data["red"] = red
+    if red["exit_code"] in (1, 2):
+        data["message"] = (
+            f"TDD red->green verified: {len(run_files)} test file(s) fail without the change "
+            f"and pass with it."
         )
-        exit_code = 1
-    elif has_test_in_diff:
-        msg = (
-            f"TDD flow detected: {len(changed_tests)} test file(s) changed "
-            f"alongside {len(changed_code)} code file(s). "
-            f"Framework: {framework}. Total test methods: {total_tests}"
+        return _result(0, data)
+    if red["exit_code"] == 0:
+        data["message"] = (
+            "Changed tests pass without the change: they do not prove the new behavior."
         )
-        exit_code = 0
-    else:
-        msg = (
-            f"No Python code changes detected. "
-            f"Framework: {framework}. Total test methods: {total_tests}"
+        return _result(1, data)
+    reason = "timed out" if red["timed_out"] else f"pytest exit {red['exit_code']}"
+    data["message"] = f"Red run on BASE did not fail cleanly ({reason}); see red output."
+    return _result(1, data)
+
+
+def _verify(repo: str, trees: dict, groups: dict, data: dict, root: str = ".") -> dict:
+    missing = _pytest_missing(data)
+    if missing:
+        return missing
+    with tempfile.TemporaryDirectory(prefix="clc-verify-tdd-") as tmp:
+        candidate_dir, base_dir = _materialize_trees(repo, trees, tmp)
+        return _red_green(candidate_dir, base_dir, root, groups, data)
+
+
+def _precheck(groups: dict, data: dict, lang_label: str = "Python") -> dict | None:
+    """Outcome decided without running tests, or None when red/green must run."""
+    if not groups["code"]:
+        if groups["tests"]:
+            data["message"] = "Only tests changed: characterization tests allowed."
+            data["warning"] = "No production code changed, so red->green was not verified."
+        elif data.get("pure_renames"):
+            data["message"] = "Only pure renames: nothing to verify."
+        else:
+            data["message"] = f"No {lang_label} production code changed: nothing to verify."
+        return _result(0, data)
+    if not groups["tests"]:
+        data["message"] = (
+            f"No test evidence: {len(groups['code'])} production code file(s) changed "
+            "without any changed test file."
         )
-        exit_code = 0
+        return _result(1, data)
+    return None
 
+
+def _group_data(root: str, groups: dict) -> dict:
     return {
-        "exit_code": exit_code,
-        "skipped": False,
-        "name": "verify_tdd",
-        "data": {
-            "framework": framework,
-            "test_files": test_files[:20],  # Cap at 20 for readability
-            "changed_code_files": changed_code,
-            "changed_test_files": changed_tests,
-            "test_count": total_tests,
-            "has_test_coverage": has_tests,
-            "message": msg,
-        },
+        "root": root,
+        "changed_tests": groups["tests"],
+        "changed_support": groups["support"],
+        "changed_code": groups["code"],
     }
 
 
-def _is_test_file(filepath: str) -> bool:
-    """Check if a filepath is a test file."""
-    basename = pathlib.Path(filepath).name
-    return (
-        basename.startswith("test_")
-        or basename.endswith("_test.py")
-        or "/tests/" in filepath
-        or "/test/" in filepath
-    )
+def _verify_roots(repo: str, trees: dict, roots: dict, data: dict, rules: dict = NO_RULES) -> dict:
+    """Several project roots: each group passes on its own, or the change fails."""
+    results = []
+    pending = []
+    for root, files in roots.items():
+        groups = _classify(files, rules)
+        gdata = _group_data(root, groups)
+        early = _precheck(groups, gdata)
+        if early is not None and early["exit_code"] == 1:
+            gdata["message"] = (
+                f"No test evidence: {len(groups['code'])} production code file(s) changed in {root} "
+                f"without any changed test file in {root}."
+            )
+        results.append((root, groups, gdata, early))
+        if early is None:
+            pending.append(len(results) - 1)
+
+    if pending:
+        missing = _pytest_missing({})
+        try:
+            with tempfile.TemporaryDirectory(prefix="clc-verify-tdd-") as tmp:
+                candidate_dir, base_dir = (None, None) if missing else _materialize_trees(repo, trees, tmp)
+                for idx in pending:
+                    root, groups, gdata, _ = results[idx]
+                    outcome = _pytest_missing(gdata) or _red_green(candidate_dir, base_dir, root, groups, gdata)
+                    results[idx] = (root, groups, gdata, outcome)
+        except (GitError, OSError) as exc:
+            data["message"] = f"Could not materialize the trees to verify: {exc}"
+            return _result(1, data)
+
+    data["groups"] = [dict(gdata, exit_code=outcome["exit_code"]) for _, _, gdata, outcome in results]
+    failed = [g for g in data["groups"] if g["exit_code"] != 0]
+    shown = failed or data["groups"]
+    data["message"] = " ".join(f"[{g['root']}] {g['message']}" for g in shown)
+    return _result(1 if failed else 0, data)
+
+
+def check(target_dir: str | None = None) -> dict:
+    """
+    Verify red->green for the current change.
+
+    Returns:
+        dict with keys exit_code, skipped, name, data. data includes mode,
+        base, candidate, changed_tests, changed_support, changed_code,
+        message and, when tests ran, green/red pytest results.
+    """
+    dir_path = target_dir or os.getcwd()
+    try:
+        repo = _git(dir_path, "rev-parse", "--show-toplevel").strip()
+    except GitError:
+        return _result(0, {"message": "Not a git repository: nothing to verify"}, skipped=True)
+    # An invalid config raises here: never verify against a config we cannot read.
+    rules = _code_rules(repo)
+    try:
+        trees = _resolve_trees(repo, rules)
+    except GitError:
+        return _result(0, {"message": "Not a git repository: nothing to verify"}, skipped=True)
+    if trees is None:
+        return _result(0, {"message": "No commits and nothing staged: nothing to verify"}, skipped=True)
+
+    groups = _classify(trees["changed"], rules)
+    data = {
+        "mode": trees["mode"],
+        "base": trees["base"] or "(empty tree)",
+        "candidate": trees["candidate"] or "(index)",
+        "changed_tests": groups["tests"],
+        "changed_support": groups["support"],
+        "changed_code": groups["code"],
+    }
+    if trees["renamed"]:
+        data["pure_renames"] = trees["renamed"]
+
+    roots = _by_root(repo, trees, rules)
+    if len(roots) > 1:
+        return _verify_roots(repo, trees, roots, data, rules)
+    root = next(iter(roots), ".")
+    if root != ".":
+        data["root"] = root
+
+    early = _precheck(groups, data)
+    if early is not None:
+        return early
+
+    try:
+        return _verify(repo, trees, groups, data, root)
+    except (GitError, OSError) as exc:
+        data["message"] = f"Could not materialize the trees to verify: {exc}"
+        return _result(1, data)
 
 
 if __name__ == "__main__":