devcouncil 0.3.1 → 0.4.0

package/src/devcouncil/verification/command_malformation.py

@@ -0,0 +1,147 @@
+"""Detect when a verification command failed to run vs. a real test failure."""
+
+from __future__ import annotations
+
+import re
+from pathlib import Path
+from typing import Optional, Tuple
+
+from devcouncil.domain.evidence import CommandResult
+
+# Signatures that mean the verification command itself could not run (or had
+# nothing to run), so its non-zero exit says nothing about whether the
+# implementation is correct — a tooling/plan defect, not a code defect.
+MALFORMED_COMMAND_SIGNATURES = (
+    "syntaxerror",
+    "invalid syntax",
+    "indentationerror",
+    "no module named",
+    "can't open file",
+    "no such file or directory",
+    "file or directory not found",
+    "no tests ran",
+    "no tests collected",
+    "error: not found",
+    "is not recognized as an internal or external command",
+    "command not found",
+    "executable file not found",
+    "failed to run command",
+    "importerror",
+    "modulenotfounderror",
+)
+
+UNCONDITIONAL_UNRUNNABLE_SIGNATURES = (
+    "syntaxerror",
+    "invalid syntax",
+    "indentationerror",
+    "can't open file",
+    "is not recognized as an internal or external command",
+    "command not found",
+    "executable file not found",
+    "failed to run command",
+    "no tests ran",
+    "no tests collected",
+    "error: not found",
+)
+
+PYTEST_NONRUN_EXIT_CODES = {4, 5}
+
+TRACEBACK_FRAME_RE = re.compile(r'File "(?P<file>[^"]+)", line (?P<line>\d+)')
+
+
+def is_traceback_frame(line: str) -> bool:
+    """True for a Python traceback frame line: ``  File "...", line N``."""
+    stripped = line.strip()
+    return stripped.startswith('File "') and ", line " in stripped
+
+
+def malformed_signature_precedes_traceback(text: str) -> bool:
+    """Decide whether an unrunnable-launcher signature is authoritative."""
+    if not text:
+        return False
+    low_all = text.lower()
+    if any(sig in low_all for sig in UNCONDITIONAL_UNRUNNABLE_SIGNATURES):
+        return True
+    lines = text.splitlines()
+    lowered_lines = [ln.lower() for ln in lines]
+    first_frame_idx: Optional[int] = None
+    for idx, line in enumerate(lines):
+        if is_traceback_frame(line):
+            first_frame_idx = idx
+            break
+    for idx, low in enumerate(lowered_lines):
+        if any(sig in low for sig in MALFORMED_COMMAND_SIGNATURES):
+            if first_frame_idx is None or idx < first_frame_idx:
+                return True
+            return False
+    return False
+
+
+def launcher_text(result: CommandResult) -> str:
+    """Captured output for launcher-vs-test analysis, ordered stderr then stdout."""
+    parts: list[str] = []
+    for path in (result.stderr_path, result.stdout_path):
+        if not path:
+            continue
+        try:
+            content = Path(path).read_text(encoding="utf-8", errors="replace")
+            if content.strip():
+                parts.append(content)
+        except Exception:
+            pass
+    if parts:
+        return "\n".join(parts)
+    return result.summary or ""
+
+
+def relativize(project_root: Path, raw_path: str) -> str:
+    """Normalize a traceback file path to a repo-relative posix path when possible."""
+    normalized = raw_path.replace("\\", "/")
+    try:
+        candidate = Path(raw_path)
+        if candidate.is_absolute():
+            rel = candidate.resolve().relative_to(project_root.resolve())
+            return rel.as_posix()
+    except Exception:
+        pass
+    return normalized
+
+
+def failure_location(project_root: Path, result: CommandResult) -> Tuple[Optional[str], Optional[int]]:
+    """Best-effort (file, line) of a failing command's deepest traceback frame."""
+    sources: list[str] = []
+    for path in (result.stdout_path, result.stderr_path):
+        if path:
+            try:
+                content = Path(path).read_text(encoding="utf-8", errors="replace")
+                if content.strip():
+                    sources.append(content)
+            except Exception:
+                pass
+    sources.append(result.summary or "")
+    best_file: Optional[str] = None
+    best_line: Optional[int] = None
+    for text in sources:
+        for match in TRACEBACK_FRAME_RE.finditer(text):
+            raw_file = match.group("file")
+            if not raw_file or raw_file.startswith("<"):
+                continue
+            best_file = relativize(project_root, raw_file)
+            try:
+                best_line = int(match.group("line"))
+            except ValueError:
+                best_line = None
+        if best_file is not None:
+            return best_file, best_line
+    return best_file, best_line
+
+
+def command_is_malformed(result: CommandResult) -> bool:
+    """True when a non-zero exit reflects a broken/unrunnable command."""
+    if result.timed_out:
+        return False
+    is_pytest = "pytest" in (result.command or "")
+    if is_pytest and result.exit_code in PYTEST_NONRUN_EXIT_CODES:
+        return True
+    text = launcher_text(result)
+    return malformed_signature_precedes_traceback(text)

package/src/devcouncil/verification/command_runner.py

@@ -0,0 +1,164 @@
+"""Run verification evidence commands with sanitized env and logging.
+
+Extracted from ``verifier.py`` so orchestration stays focused on gate logic.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import logging
+import shlex
+import shutil
+import subprocess
+import sys
+from pathlib import Path
+from typing import List
+
+from devcouncil.domain.evidence import CommandResult
+from devcouncil.utils.redaction import redact_string
+from devcouncil.utils.subprocess_env import clean_subprocess_env
+
+logger = logging.getLogger(__name__)
+
+
+def _timeout_stream(content: str | bytes | None) -> str:
+    """Normalize partial output attached to ``TimeoutExpired``."""
+    if isinstance(content, bytes):
+        return content.decode("utf-8", errors="replace")
+    return content or ""
+
+
+def split_command(command: str) -> List[str]:
+    # Use POSIX splitting so quotes are interpreted, not preserved. With
+    # posix=False, `python -c "assert x"` keeps the surrounding quotes, so the
+    # interpreter receives the literal string `"assert x"` and treats it as a
+    # no-op string expression that exits 0 — every quoted-argument evidence
+    # command would then silently "pass" without running, producing false
+    # verification. posix=True strips the quotes correctly; planner-generated
+    # commands use forward-slash paths, which the interpreter accepts on Windows.
+    return shlex.split(command, posix=True)
+
+
+def summarize_stream(content: str, budget: int = 360) -> str:
+    """Condense a command's stdout/stderr for the evidence summary."""
+    if not content or not content.strip():
+        return "(empty)"
+    lines = [ln.rstrip() for ln in content.splitlines() if ln.strip()]
+    markers = ("error", "exception", "assert", "traceback", "failed", "not found", "no module named")
+    salient = ""
+    for ln in reversed(lines):
+        low = ln.lower()
+        if any(m in low for m in markers):
+            salient = ln.strip()
+            break
+    if not salient:
+        salient = lines[-1].strip()
+    salient = salient[:240]
+    tail = content.strip()[-budget:]
+    summary = f"{salient} | {tail}" if salient not in tail[: len(salient) + 5] else tail
+    return summary[: budget + len(salient) + 8]
+
+
+def save_command_log(
+    project_root: Path,
+    label: str,
+    command: str,
+    stream: str,
+    content: str,
+) -> str:
+    """Save command output to a log file and return the path."""
+    log_dir = project_root / ".devcouncil" / "logs"
+    log_dir.mkdir(parents=True, exist_ok=True)
+    cmd_hash = hashlib.sha256(command.encode()).hexdigest()[:8]
+    filename = f"{label}-{cmd_hash}-{stream}.log"
+    log_path = log_dir / filename
+    log_path.write_text(redact_string(content), encoding="utf-8")
+    return str(log_path)
+
+
+def run_verification_command(
+    project_root: Path,
+    command: str,
+    *,
+    task_id: str = "verify",
+    timeout: int = 900,
+) -> CommandResult:
+    """Execute one evidence command in the target repo with a cleaned environment."""
+    env = clean_subprocess_env()
+    try:
+        argv = split_command(command)
+    except ValueError as e:
+        return CommandResult(
+            command=command,
+            exit_code=-1,
+            stdout_path="",
+            stderr_path="",
+            summary=f"Failed to run command: unparseable shell syntax ({e})",
+        )
+    if argv and argv[0] in {"python", "python3"}:
+        # Planner-generated checks use portable launcher names, but macOS may have
+        # no ``python`` and its system ``python3`` does not contain project deps.
+        # Verification must run under the interpreter hosting DevCouncil.
+        argv = [sys.executable, *argv[1:]]
+    elif argv:
+        resolved = shutil.which(argv[0], path=env.get("PATH"))
+        if resolved:
+            argv = [resolved, *argv[1:]]
+
+    try:
+        result = subprocess.run(
+            argv,
+            shell=False,
+            capture_output=True,
+            text=True,
+            encoding="utf-8",
+            errors="replace",
+            cwd=project_root,
+            timeout=timeout,
+            env=env,
+        )
+        stdout = result.stdout or ""
+        stderr = result.stderr or ""
+        stdout_path = save_command_log(project_root, task_id, command, "stdout", stdout)
+        stderr_path = save_command_log(project_root, task_id, command, "stderr", stderr)
+        stdout_summary = redact_string(summarize_stream(stdout))
+        stderr_summary = redact_string(summarize_stream(stderr))
+        return CommandResult(
+            command=command,
+            exit_code=result.returncode,
+            stdout_path=stdout_path,
+            stderr_path=stderr_path,
+            summary=(
+                f"Exit code {result.returncode}. "
+                f"stderr: {stderr_summary}. "
+                f"stdout: {stdout_summary}"
+            ),
+        )
+    except subprocess.TimeoutExpired as exc:
+        stdout = _timeout_stream(exc.stdout)
+        stderr = _timeout_stream(exc.stderr)
+        stdout_path = save_command_log(project_root, task_id, command, "stdout", stdout)
+        stderr_path = save_command_log(project_root, task_id, command, "stderr", stderr)
+        stdout_summary = redact_string(summarize_stream(stdout))
+        stderr_summary = redact_string(summarize_stream(stderr))
+        return CommandResult(
+            command=command,
+            exit_code=124,
+            stdout_path=stdout_path,
+            stderr_path=stderr_path,
+            summary=(
+                f"Verification command timed out after {timeout} seconds. "
+                f"stderr: {stderr_summary}. "
+                f"stdout: {stdout_summary}"
+            ),
+            timed_out=True,
+        )
+    except Exception as e:
+        logger.debug("Command run failed for %r: %s", command, e)
+        return CommandResult(
+            command=command,
+            exit_code=-1,
+            stdout_path="",
+            stderr_path="",
+            summary=f"Failed to run command: {e}",
+        )

package/src/devcouncil/verification/coverage_measurement.py

@@ -0,0 +1,292 @@
+"""Diff-coverage instrumentation extracted from Verifier."""
+
+from __future__ import annotations
+
+import json
+import logging
+import shutil
+import subprocess
+import sys
+from pathlib import Path
+from typing import Callable, Dict, List, Optional
+
+from devcouncil.app.config import load_config
+from devcouncil.domain.task import Task
+from devcouncil.utils.subprocess_env import clean_subprocess_env
+from devcouncil.verification import command_runner as cmd_runner
+from devcouncil.verification import diff_coverage as dc
+
+logger = logging.getLogger(__name__)
+
+SplitCommand = Callable[[str], List[str]]
+VerificationEnv = Callable[[], Dict[str, str]]
+LoadCommands = Callable[[], Dict[str, List[str]]]
+CommandCanProve = Callable[[str, str], bool]
+
+
+class DiffCoverageMeasurer:
+    """Run task test commands under coverage and intersect with a diff."""
+
+    def __init__(
+        self,
+        project_root: Path,
+        *,
+        get_coverage_python: Callable[[], Optional[str]],
+        split_command: SplitCommand,
+        verification_env: VerificationEnv,
+        load_commands: LoadCommands,
+        command_can_prove_acceptance: CommandCanProve,
+    ) -> None:
+        self.project_root = project_root
+        self._get_coverage_python = get_coverage_python
+        self._split_command = split_command
+        self._verification_env = verification_env
+        self._load_commands = load_commands
+        self._command_can_prove_acceptance = command_can_prove_acceptance
+
+    @classmethod
+    def from_verifier(cls, verifier: object) -> DiffCoverageMeasurer:
+        """Build a measurer bound to a Verifier instance."""
+        return cls(
+            verifier.project_root,  # type: ignore[attr-defined]
+            get_coverage_python=lambda: verifier._coverage_python,  # type: ignore[attr-defined]
+            split_command=cmd_runner.split_command,
+            verification_env=clean_subprocess_env,
+            load_commands=verifier._load_commands,  # type: ignore[attr-defined]
+            command_can_prove_acceptance=verifier._command_can_prove_acceptance,  # type: ignore[attr-defined]
+        )
+
+    def coverage_target_commands(self, task: Task) -> List[str]:
+        """The test command(s) to instrument — the ones that purport to prove the ACs."""
+        if task.expected_tests:
+            return list(task.expected_tests)
+        test_like = [
+            c for c in task.allowed_commands
+            if self._command_can_prove_acceptance("allowed", c)
+        ]
+        if test_like:
+            return test_like
+        return list(self._load_commands().get("test", []))
+
+    def measure(self, task: Task, diff_content: str) -> dc.DiffCoverageResult:
+        """Run instrumentation for Python, JS/TS, and Go changed paths."""
+        parsed = dc.parse_changed_lines(diff_content)
+        py_changed = dc.measurable_python_changes(parsed)
+        js_changed = dc.measurable_js_changes(parsed)
+        go_changed = dc.measurable_go_changes(parsed)
+        if not py_changed and not js_changed and not go_changed:
+            return dc.DiffCoverageResult(measured=False, reason="no measurable source changes in diff")
+
+        commands = self.coverage_target_commands(task)
+        if not commands:
+            return dc.DiffCoverageResult(measured=False, reason="no test command to instrument")
+
+        try:
+            timeout = load_config(self.project_root).execution.command_timeout
+        except Exception:
+            timeout = 300
+
+        env = self._verification_env()
+        tmp_dir = self.project_root / ".devcouncil" / "tmp"
+        tmp_dir.mkdir(parents=True, exist_ok=True)
+        results: List[dc.DiffCoverageResult] = []
+
+        if py_changed:
+            results.append(self._measure_python(task, commands, py_changed, tmp_dir, env, timeout))
+        if js_changed:
+            results.append(self._measure_js(task, commands, js_changed, tmp_dir, env, timeout))
+        if go_changed:
+            results.append(self._measure_go(task, commands, go_changed, tmp_dir, env, timeout))
+
+        return dc.merge_diff_coverage_results(results)
+
+    def _resolve_coverage_python(self, env: Dict[str, str]) -> str:
+        override = self._get_coverage_python()
+        if override:
+            return override
+        for name in ("python", "python3", "py"):
+            found = shutil.which(name, path=env.get("PATH"))
+            if found:
+                return found
+        return sys.executable
+
+    def _coverage_available(self, python: str, env: Dict[str, str]) -> bool:
+        try:
+            result = subprocess.run(
+                [python, "-m", "coverage", "--version"],
+                cwd=self.project_root,
+                capture_output=True,
+                text=True,
+                encoding="utf-8",
+                errors="replace",
+                timeout=30,
+                env=env,
+            )
+            return result.returncode == 0
+        except Exception:
+            return False
+
+    def _measure_python(
+        self,
+        task: Task,
+        commands: List[str],
+        changed: Dict[str, Dict[int, str]],
+        tmp_dir: Path,
+        env: dict,
+        timeout: int,
+    ) -> dc.DiffCoverageResult:
+        python = self._resolve_coverage_python(env)
+        if not self._coverage_available(python, env):
+            return dc.DiffCoverageResult(measured=False, reason="coverage.py not available")
+        data_file = tmp_dir / f"diffcov-{task.id}.coverage"
+        json_file = tmp_dir / f"diffcov-{task.id}.json"
+        for stale in (data_file, json_file):
+            try:
+                stale.unlink()
+            except FileNotFoundError:
+                pass
+        ran_any = False
+        append = False
+        inline_scripts: List[Path] = []
+        try:
+            for idx, cmd in enumerate(commands):
+                try:
+                    argv = self._split_command(cmd)
+                except ValueError as exc:
+                    logger.warning("Diff-coverage skipping unparseable command %r: %s", cmd, exc)
+                    continue
+                inline = dc.inline_python_code(argv)
+                if inline is not None:
+                    script = tmp_dir / f"diffcov-inline-{task.id}-{idx}.py"
+                    try:
+                        script.write_text(dc.inline_script_content(inline, self.project_root), encoding="utf-8")
+                    except Exception as exc:
+                        logger.warning("Diff-coverage inline script write failed for %s: %s", task.id, exc)
+                        continue
+                    inline_scripts.append(script)
+                    cov_argv: Optional[List[str]] = dc.coverage_run_script_argv(
+                        str(script), python, append=append, data_file=str(data_file)
+                    )
+                else:
+                    cov_argv = dc.coverage_run_argv(argv, python, append=append, data_file=str(data_file))
+                if cov_argv is None:
+                    continue
+                try:
+                    subprocess.run(
+                        cov_argv, cwd=self.project_root, capture_output=True,
+                        text=True, encoding="utf-8", errors="replace", timeout=timeout, env=env,
+                    )
+                except Exception as exc:
+                    logger.warning("Diff-coverage run failed for %s: %s", task.id, exc)
+                    continue
+                ran_any = True
+                append = True
+            if not ran_any or not data_file.exists():
+                return dc.DiffCoverageResult(measured=False, reason="no instrumentable Python test command")
+            subprocess.run(
+                [python, "-m", "coverage", "json", f"--data-file={data_file}", "-o", str(json_file)],
+                cwd=self.project_root, capture_output=True, text=True,
+                encoding="utf-8", errors="replace", timeout=120, env=env,
+            )
+            data = json.loads(json_file.read_text(encoding="utf-8"))
+            coverage = dc.parse_coverage_json(data, self.project_root)
+            return dc.intersect(changed, coverage, tool="coverage.py")
+        except Exception as exc:
+            return dc.DiffCoverageResult(measured=False, reason=f"coverage.py failed: {exc}")
+        finally:
+            for path in [data_file, json_file, *inline_scripts]:
+                try:
+                    path.unlink()
+                except OSError:
+                    pass
+
+    def _measure_js(
+        self,
+        task: Task,
+        commands: List[str],
+        changed: Dict[str, Dict[int, str]],
+        tmp_dir: Path,
+        env: dict,
+        timeout: int,
+    ) -> dc.DiffCoverageResult:
+        reports_dir = tmp_dir / f"c8-{task.id}"
+        reports_dir.mkdir(parents=True, exist_ok=True)
+        try:
+            for stale in reports_dir.glob("coverage-final.json"):
+                stale.unlink()
+        except OSError:
+            pass
+        ran_any = False
+        for cmd in commands:
+            try:
+                argv = self._split_command(cmd)
+            except ValueError as exc:
+                logger.warning("Diff-coverage skipping unparseable command %r: %s", cmd, exc)
+                continue
+            cov_argv = dc.c8_run_argv(argv, reports_dir=str(reports_dir))
+            if cov_argv is None:
+                continue
+            try:
+                subprocess.run(
+                    cov_argv, cwd=self.project_root, capture_output=True,
+                    text=True, encoding="utf-8", errors="replace", timeout=timeout, env=env,
+                )
+            except Exception as exc:
+                logger.warning("c8 diff-coverage run failed for %s: %s", task.id, exc)
+                continue
+            ran_any = True
+        candidates = list(reports_dir.glob("**/coverage-final.json"))
+        if not ran_any or not candidates:
+            return dc.DiffCoverageResult(measured=False, reason="c8 not available or no JS test command")
+        try:
+            data = json.loads(candidates[0].read_text(encoding="utf-8"))
+            coverage = dc.parse_istanbul_json(data, self.project_root)
+            return dc.intersect(changed, coverage, tool="c8")
+        except Exception as exc:
+            return dc.DiffCoverageResult(measured=False, reason=f"c8 report unreadable: {exc}")
+
+    def _measure_go(
+        self,
+        task: Task,
+        commands: List[str],
+        changed: Dict[str, Dict[int, str]],
+        tmp_dir: Path,
+        env: dict,
+        timeout: int,
+    ) -> dc.DiffCoverageResult:
+        profile = tmp_dir / f"diffcov-{task.id}.out"
+        try:
+            profile.unlink()
+        except FileNotFoundError:
+            pass
+        ran_any = False
+        for cmd in commands:
+            try:
+                argv = self._split_command(cmd)
+            except ValueError as exc:
+                logger.warning("Diff-coverage skipping unparseable command %r: %s", cmd, exc)
+                continue
+            cov_argv = dc.go_cover_run_argv(argv, str(profile))
+            if cov_argv is None:
+                continue
+            try:
+                subprocess.run(
+                    cov_argv, cwd=self.project_root, capture_output=True,
+                    text=True, encoding="utf-8", errors="replace", timeout=timeout, env=env,
+                )
+            except Exception as exc:
+                logger.warning("go cover run failed for %s: %s", task.id, exc)
+                continue
+            ran_any = True
+        if not ran_any or not profile.exists():
+            return dc.DiffCoverageResult(measured=False, reason="go test -coverprofile not available")
+        try:
+            coverage = dc.parse_go_coverprofile(profile.read_text(encoding="utf-8"), self.project_root)
+            return dc.intersect(changed, coverage, tool="go-cover")
+        except Exception as exc:
+            return dc.DiffCoverageResult(measured=False, reason=f"go cover profile unreadable: {exc}")
+        finally:
+            try:
+                profile.unlink()
+            except OSError:
+                pass