devcouncil 0.1.1 → 0.2.0

package/src/devcouncil/verification/acceptance_compiler.py

@@ -0,0 +1,125 @@
+"""Compile natural-language acceptance criteria into self-contained executable
+checks that DevCouncil owns and runs.
+
+This is the difference between trusting the planner/agent's word and gathering
+real evidence: instead of running planner-authored ``expected_tests`` (which the
+benchmark showed often reference tools or test files that do not exist), DevCouncil
+derives one runnable check per acceptance criterion directly from the criterion
+text and the code under review, then maps each check 1:1 to its criterion.
+"""
+
+from __future__ import annotations
+
+import json
+from typing import Dict, List
+
+from pydantic import BaseModel
+
+from devcouncil.domain.requirement import Requirement
+from devcouncil.domain.task import Task
+from devcouncil.llm.router import ModelRouter
+
+
+class CompiledCheck(BaseModel):
+    acceptance_criterion_id: str
+    command: str  # a single shell command that exits 0 iff the criterion holds
+
+
+class CompiledChecks(BaseModel):
+    checks: List[CompiledCheck]
+
+
+class AcceptanceTestCompiler:
+    def __init__(self, router: ModelRouter, role: str = "implementation_reviewer"):
+        self.router = router
+        self.role = role
+
+    async def compile(
+        self,
+        task: Task,
+        requirements: List[Requirement],
+        code_context: str,
+    ) -> Dict[str, List[str]]:
+        """Return {acceptance_criterion_id: [self-contained check command(s)]}.
+
+        Best-effort: returns {} if the model cannot produce usable checks, so the
+        caller can fall back to the task's declared expected_tests.
+        """
+        ac_by_id = {ac.id: ac for req in requirements for ac in req.acceptance_criteria}
+        target = [ac_by_id[i] for i in task.acceptance_criterion_ids if i in ac_by_id]
+        if not target:
+            return {}
+
+        acs_json = json.dumps(
+            [{"id": ac.id, "description": ac.description, "method": ac.verification_method} for ac in target],
+            indent=2,
+        )
+        prompt = f"""
+You are DevCouncil's acceptance-test compiler. Convert each acceptance criterion
+below into exactly ONE shell command that EXITS 0 if and only if the BEHAVIOR
+described by the criterion holds for the code shown.
+
+Acceptance criteria:
+{acs_json}
+
+Code under review (the diff / current files):
+{code_context}
+
+What a check must verify — BEHAVIOR ONLY:
+- A check exists to confirm the code DOES what the criterion describes when its
+  public API is exercised: import the module/symbol and call its function(s), or
+  run its CLI/entrypoint, and assert on the observable result (return value,
+  raised exception, stdout, exit code).
+- DevCouncil already enforces scope, file ownership, and append-only/orphan-diff
+  constraints with its OWN gates. Acceptance checks must therefore NEVER re-assert
+  repository or filesystem STATE — that is not their job and it produces false
+  BLOCKED results because `dev` itself adds workspace files (AGENTS.md, CLAUDE.md,
+  .gitignore, .devcouncil/config.yaml, etc.).
+
+Rules — the commands are executed verbatim by the verifier:
+- One command per acceptance_criterion_id (reference the id exactly).
+- Each command MUST be a single, SELF-CONTAINED, immediately-runnable command:
+  import the real module/symbol from the code and assert the behavior directly.
+  Do NOT depend on test files, fixtures, or any external setup.
+- Prefer: python -c "import <module>; assert <expr>". For an expected exception,
+  use a one-line guard, e.g.
+  python -c "import m; \\ntry: m.f([])\\nexcept ValueError: pass\\nelse: raise SystemExit(1)"
+  (real newlines are fine; never put try/if/for after a ';').
+- Use the actual module name implied by the code (e.g. file 'stats.py' -> import stats).
+
+HARD PROHIBITIONS — a command that does any of these is INVALID; omit the
+criterion instead of emitting such a command:
+- NEVER assert exact git or filesystem state. Forbidden: `git status`,
+  `git status --porcelain`, `git diff`, `git diff --name-only`, `git show`,
+  `git ls-files`, `ls`/`find`/`os.listdir` equality checks, asserting a precise
+  set or count of changed/created files, or asserting a file does/does not exist
+  as the criterion's pass condition.
+- NEVER do append-only or byte-level file/content comparisons (e.g.
+  `git show HEAD:file`, diffing bytes, asserting only N bytes/lines were added).
+  Assert the resulting BEHAVIOR instead, not how the file changed.
+- NEVER invoke linters, type checkers, formatters, or build/package tools that
+  may be absent: flake8, mypy, ruff, pylint, black, isort, eslint, tsc, prettier,
+  npm, npx, yarn, pnpm, cargo, go vet, etc. Only use such a tool if the code
+  context clearly shows it is configured for this repo (e.g. a matching config
+  section/file is present in the context) AND it is essential to the criterion.
+- If a criterion cannot be checked by a behavioral command (e.g. pure 'manual'
+  review, or it only describes repo/tooling state), OMIT it rather than inventing
+  a state-based or bogus command.
+"""
+        try:
+            result = await self.router.complete_structured(
+                role=self.role,
+                messages=[{"role": "user", "content": prompt}],
+                schema=CompiledChecks,
+                fallback=CompiledChecks(checks=[]),
+            )
+        except Exception:
+            return {}
+
+        out: Dict[str, List[str]] = {}
+        valid_ids = {ac.id for ac in target}
+        for check in result.checks:
+            cmd = (check.command or "").strip()
+            if check.acceptance_criterion_id in valid_ids and cmd:
+                out.setdefault(check.acceptance_criterion_id, []).append(cmd)
+        return out

package/src/devcouncil/verification/ad_hoc_check.py

@@ -0,0 +1,129 @@
+"""Verify an ad-hoc working-tree diff against an inline requirement — no planning, no keys.
+
+This powers ``dev check``'s evidence-gate mode (the lite entry point): wrap whatever is
+in the working tree as a synthetic Requirement→Task, run the *same* deterministic
+:class:`~devcouncil.verification.verifier.Verifier` the full workflow uses — orphan-diff,
+secret scan, acceptance evidence, and the diff↔coverage gate — and return the verdict
+plus the typed next-actions contract. ``router=None`` keeps it provider-key-free so a
+newcomer can taste the evidence gate before committing to the full council flow.
+
+The logic lives here (not in the CLI command) so it is unit-testable without Typer and
+resilient to churn in the command module.
+"""
+
+from __future__ import annotations
+
+import asyncio
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import List, Optional
+
+from devcouncil.domain.evidence import DiffCoverageEvidence
+from devcouncil.domain.gap import Gap
+from devcouncil.domain.requirement import AcceptanceCriterion, Requirement
+from devcouncil.domain.task import PlannedFile, Task
+from devcouncil.llm.router import ModelRouter
+from devcouncil.verification.next_actions import NextAction, build_next_actions
+from devcouncil.verification.verifier import Verifier
+
+_REQ_ID = "REQ-CHECK"
+_AC_ID = "AC-CHECK"
+_TASK_ID = "CHECK"
+
+_DEFAULT_CRITERION = "The working-tree changes are correct and exercised by tests."
+
+
+@dataclass
+class AdHocCheckResult:
+    requirement: str
+    changed_files: List[str] = field(default_factory=list)
+    gaps: List[Gap] = field(default_factory=list)
+    next_actions: List[NextAction] = field(default_factory=list)
+    diff_coverage: Optional[DiffCoverageEvidence] = None
+    passed: bool = True
+    reason: str = ""
+
+    def to_dict(self) -> dict:
+        return {
+            "ok": True,
+            "verified": self.passed,
+            "requirement": self.requirement,
+            "changed_files": self.changed_files,
+            "reason": self.reason,
+            "gap_count": len(self.gaps),
+            "blocking_gap_count": len([g for g in self.gaps if g.blocking]),
+            "gaps": [g.model_dump() for g in self.gaps],
+            "next_actions": [a.model_dump() for a in self.next_actions],
+            "diff_coverage": self.diff_coverage.model_dump() if self.diff_coverage else None,
+        }
+
+
+def run_working_tree_check(
+    project_root: Path,
+    requirement: Optional[str] = None,
+    *,
+    test_commands: Optional[List[str]] = None,
+    enforce_coverage: bool = False,
+    min_ratio: float = 0.0,
+    router: Optional[ModelRouter] = None,
+    verifier: Optional[Verifier] = None,
+) -> AdHocCheckResult:
+    """Verify the current working-tree diff against a one-line requirement.
+
+    Builds a synthetic task whose planned files are exactly the changed files (so the
+    result is about evidence, not scope noise) and whose expected tests are
+    ``test_commands``. Diff coverage is always measured; pass ``enforce_coverage`` (or a
+    positive ``min_ratio``) to make an unexercised diff blocking.
+    """
+    verifier = verifier or Verifier(project_root, router=router)
+
+    diff = verifier.get_diff()
+    changed_files = verifier.get_changed_files()
+    if not diff or not changed_files:
+        return AdHocCheckResult(requirement="", passed=True, reason="no_changes")
+
+    criterion = requirement or _DEFAULT_CRITERION
+    req = Requirement(
+        id=_REQ_ID,
+        title=(requirement or "Working-tree change")[:80],
+        description=criterion,
+        priority="high",
+        source="user",
+        acceptance_criteria=[
+            AcceptanceCriterion(id=_AC_ID, description=criterion, verification_method="unit_test"),
+        ],
+    )
+    untracked = set(verifier._get_untracked_files())
+    task = Task(
+        id=_TASK_ID,
+        title="Ad-hoc working-tree check",
+        description=criterion,
+        requirement_ids=[_REQ_ID],
+        acceptance_criterion_ids=[_AC_ID],
+        planned_files=[
+            PlannedFile(
+                path=path,
+                reason="working-tree change",
+                allowed_change="create" if path in untracked else "modify",
+            )
+            for path in changed_files
+        ],
+        expected_tests=list(test_commands or []),
+    )
+
+    # Always measure diff coverage in lite mode; block on it only when asked. A positive
+    # --min-coverage implies enforcement so the flag is never silently inert.
+    enforce = enforce_coverage or min_ratio > 0
+    verifier._diff_coverage_override = (True, enforce, float(min_ratio))
+
+    gaps, evidence = asyncio.run(verifier.verify_task(task, [req]))
+    coverage = next((ev for ev in evidence if isinstance(ev, DiffCoverageEvidence)), None)
+    blocking = [g for g in gaps if g.blocking]
+    return AdHocCheckResult(
+        requirement=criterion,
+        changed_files=changed_files,
+        gaps=gaps,
+        next_actions=build_next_actions(gaps),
+        diff_coverage=coverage,
+        passed=not blocking,
+    )

package/src/devcouncil/verification/diff_coverage.py

@@ -0,0 +1,353 @@
+"""Diff↔coverage intersection — proof that the *changed lines* were exercised.
+
+DevCouncil's headline promise is that a passing test must prove the **new logic
+was exercised**, not merely that *some* suite exited 0. An agent can make a green
+suite pass while the changed code is never imported, never called, or shadowed by
+an unrelated passing test. This module closes that gap: it runs a task's test
+command under coverage instrumentation, then intersects the lines the tests
+actually executed with the lines the diff changed.
+
+Two failure shapes are caught:
+
+1. **Touched-but-not-exercised** — the changed file *is* in the coverage report,
+   but the changed executable lines were never executed (e.g. a passing test that
+   exercises a different branch).
+2. **Never-imported** — the changed source file is *absent* from the coverage
+   report entirely, meaning the tests never loaded it.
+
+False-positive discipline (mirrors :class:`~devcouncil.verification.verifier.Verifier`):
+this analysis only ever produces a *signal* when it has reliable data — a
+parseable diff with real hunks, a detected coverage tool, and changed *executable*
+lines to measure. When any of those is missing it returns
+``DiffCoverageResult(measured=False, ...)`` and the verifier degrades to its prior
+behaviour rather than blocking correct work. Coverage is read from the **target
+repository's** own tooling (coverage.py for Python); DevCouncil never forces its
+own coverage dependency into the project under verification.
+"""
+
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Dict, List, Optional, Set
+
+# A unified-diff hunk header: ``@@ -<old> +<newStart>[,<newLen>] @@``. We only need
+# the new-file starting line to walk added/context lines into new-file numbers.
+_HUNK_RE = re.compile(r"^@@ -\d+(?:,\d+)? \+(\d+)(?:,\d+)? @@")
+
+# Path fragments that mark a file as test code. Test files are excluded from the
+# "must be exercised" denominator: a test exercising itself is not the new logic
+# whose behaviour an acceptance criterion is about.
+_TEST_MARKERS = (
+    "/tests/",
+    "tests/",
+    "/test_",
+    "test_",
+    "_test.",
+    ".test.",
+    ".spec.",
+    "_spec.",
+    "/spec/",
+)
+
+
+def _strip_diff_prefix(path: str) -> str:
+    """Strip a leading ``a/`` or ``b/`` and normalise to forward slashes."""
+    path = path.strip().strip('"')
+    if path.startswith(("a/", "b/")):
+        path = path[2:]
+    return path.replace("\\", "/")
+
+
+def is_test_path(path: str) -> bool:
+    lowered = path.replace("\\", "/").lower()
+    return any(marker in lowered for marker in _TEST_MARKERS)
+
+
+def is_code_like(line: str) -> bool:
+    """Conservative heuristic: a non-blank line that is not a pure comment.
+
+    Used only for changed files that are *absent* from the coverage report, where
+    no authoritative executable-line set exists. Imports, ``def``/``class``,
+    decorators and statements all count; blank lines and ``#`` comments do not.
+    Kept deliberately conservative so it never inflates the denominator.
+    """
+    stripped = line.strip()
+    return bool(stripped) and not stripped.startswith("#")
+
+
+def parse_changed_lines(diff: str) -> Dict[str, Dict[int, str]]:
+    """Parse a unified diff into ``{file_path: {new_line_number: added_text}}``.
+
+    Only *added* lines (``+`` in the new file) are recorded, keyed by their line
+    number in the post-change file so they can be intersected with coverage data.
+    Deletions and context lines advance the counter but are not themselves
+    "changed lines" we require coverage for. Paths are normalised relative to the
+    repo (``a/``/``b/`` prefixes stripped, forward slashes).
+    """
+    changed: Dict[str, Dict[int, str]] = {}
+    current_file: Optional[str] = None
+    new_line = 0
+    in_hunk = False
+
+    for raw in diff.splitlines():
+        if raw.startswith("diff --git") or raw.startswith("--- "):
+            in_hunk = False
+            continue
+        if raw.startswith("+++ "):
+            target = raw[4:].strip()
+            if target == "/dev/null":
+                current_file = None
+            else:
+                current_file = _strip_diff_prefix(target)
+                changed.setdefault(current_file, {})
+            in_hunk = False
+            continue
+        if raw.startswith("@@"):
+            match = _HUNK_RE.match(raw)
+            if match:
+                new_line = int(match.group(1))
+                in_hunk = True
+            else:
+                # A header we can't number (e.g. a combined-merge ``@@@``). Stay out
+                # of hunk mode rather than mis-attribute added lines to line 0.
+                in_hunk = False
+            continue
+        if not in_hunk or current_file is None:
+            continue
+        if raw.startswith("\\"):  # ""
+            continue
+        if raw.startswith("+"):
+            changed[current_file][new_line] = raw[1:]
+            new_line += 1
+        elif raw.startswith("-"):
+            continue  # old-file only; does not advance the new-file counter
+        else:
+            new_line += 1  # context line
+
+    return {path: lines for path, lines in changed.items() if lines}
+
+
+def parse_coverage_json(data: dict, root: Path) -> "CoverageData":
+    """Extract executed and executable lines per file from ``coverage json`` output.
+
+    ``coverage.py`` reports ``executed_lines`` and ``missing_lines`` per file; their
+    union is the set of statements coverage knows are executable. Paths are
+    normalised to repo-relative POSIX so they intersect with diff paths.
+    """
+    executed: Dict[str, Set[int]] = {}
+    executable: Dict[str, Set[int]] = {}
+    files = data.get("files", {}) if isinstance(data, dict) else {}
+    for raw_path, payload in files.items():
+        if not isinstance(payload, dict):
+            continue
+        rel = _relativize(raw_path, root)
+        if rel is None:
+            continue
+        run = {int(n) for n in payload.get("executed_lines", []) or []}
+        miss = {int(n) for n in payload.get("missing_lines", []) or []}
+        executed[rel] = run
+        executable[rel] = run | miss
+    return CoverageData(executed=executed, executable=executable)
+
+
+def _relativize(raw_path: str, root: Path) -> Optional[str]:
+    candidate = Path(raw_path)
+    try:
+        if candidate.is_absolute():
+            rel = candidate.resolve().relative_to(root.resolve())
+        else:
+            rel = candidate
+    except ValueError:
+        # Outside the repo (site-packages, stdlib) — not a changed-file candidate.
+        return None
+    return rel.as_posix()
+
+
+@dataclass
+class CoverageData:
+    executed: Dict[str, Set[int]] = field(default_factory=dict)
+    executable: Dict[str, Set[int]] = field(default_factory=dict)
+
+
+@dataclass
+class DiffCoverageResult:
+    """Outcome of intersecting changed lines with executed lines.
+
+    ``measured`` is True only when there was a meaningful signal to compute — at
+    least one changed *executable* line. When False, callers must NOT treat the
+    result as evidence of a problem (false-positive discipline).
+    """
+
+    measured: bool
+    tool: str = ""
+    reason: str = ""
+    changed_executable_lines: int = 0
+    covered_changed_lines: int = 0
+    uncovered_by_file: Dict[str, List[int]] = field(default_factory=dict)
+    absent_files: List[str] = field(default_factory=list)
+
+    @property
+    def ratio(self) -> float:
+        if self.changed_executable_lines == 0:
+            return 1.0
+        return self.covered_changed_lines / self.changed_executable_lines
+
+    def summary(self) -> str:
+        if not self.measured:
+            return f"diff coverage not measured ({self.reason})" if self.reason else "diff coverage not measured"
+        pct = round(self.ratio * 100)
+        base = (
+            f"{self.covered_changed_lines}/{self.changed_executable_lines} changed lines exercised "
+            f"({pct}%) via {self.tool or 'coverage'}"
+        )
+        if self.absent_files:
+            base += f"; not imported by tests: {', '.join(self.absent_files)}"
+        return base
+
+
+def intersect(
+    changed: Dict[str, Dict[int, str]],
+    coverage: CoverageData,
+    *,
+    tool: str = "coverage.py",
+) -> DiffCoverageResult:
+    """Intersect changed lines with executed lines to measure diff coverage.
+
+    ``changed`` should already be filtered to measurable source files (e.g. ``.py``
+    non-test files). For each file present in the coverage report we use coverage's
+    authoritative executable-line set; for changed source files *absent* from the
+    report (never imported) we fall back to the conservative ``is_code_like``
+    heuristic and count those added lines as executable-but-uncovered.
+    """
+    total_executable = 0
+    total_covered = 0
+    uncovered_by_file: Dict[str, List[int]] = {}
+    absent_files: List[str] = []
+
+    for path, line_map in changed.items():
+        changed_nums = set(line_map.keys())
+        if path in coverage.executable:
+            file_executable = changed_nums & coverage.executable[path]
+            file_covered = changed_nums & coverage.executed.get(path, set())
+            total_executable += len(file_executable)
+            total_covered += len(file_covered)
+            missing = sorted(file_executable - file_covered)
+            if missing:
+                uncovered_by_file[path] = missing
+        else:
+            # Absent from the coverage report -> the tests never loaded this file.
+            code_like = sorted(num for num, text in line_map.items() if is_code_like(text))
+            if code_like:
+                total_executable += len(code_like)
+                uncovered_by_file[path] = code_like
+                absent_files.append(path)
+
+    if total_executable == 0:
+        return DiffCoverageResult(
+            measured=False,
+            tool=tool,
+            reason="no changed executable lines to measure",
+        )
+
+    return DiffCoverageResult(
+        measured=True,
+        tool=tool,
+        changed_executable_lines=total_executable,
+        covered_changed_lines=total_covered,
+        uncovered_by_file=uncovered_by_file,
+        absent_files=absent_files,
+    )
+
+
+def measurable_python_changes(changed: Dict[str, Dict[int, str]]) -> Dict[str, Dict[int, str]]:
+    """Filter parsed diff lines to Python source files (non-test) coverage.py can measure."""
+    return {
+        path: lines
+        for path, lines in changed.items()
+        if path.endswith(".py") and not is_test_path(path)
+    }
+
+
+def coverage_run_argv(
+    command_argv: List[str],
+    python: str,
+    *,
+    append: bool,
+    data_file: str,
+    source: str = ".",
+) -> Optional[List[str]]:
+    """Transform a test command's argv into a ``coverage run`` invocation.
+
+    Supports the common Python entry points. Returns ``None`` for commands that
+    cannot be instrumented (the caller then leaves diff coverage unmeasured rather
+    than guessing). ``python -c "<code>"`` is handled separately by the caller
+    because it must materialise a temp script first.
+    """
+    if not command_argv:
+        return None
+
+    prefix = [python, "-m", "coverage", "run", f"--source={source}", f"--data-file={data_file}"]
+    if append:
+        prefix.append("-a")
+
+    head = Path(command_argv[0]).name.lower()
+    if head.endswith(".exe"):  # Windows: python.exe / pytest.exe
+        head = head[:-4]
+    rest = command_argv[1:]
+
+    # python -m pytest / python -m unittest -> reuse the same module entry point.
+    if head in {"python", "python3", "py"} and len(rest) >= 2 and rest[0] == "-m":
+        module = rest[1]
+        if module in {"pytest", "unittest"}:
+            return [*prefix, "-m", module, *rest[2:]]
+        return None
+    # bare pytest -> run via the module entry point under coverage.
+    if head in {"pytest", "py.test"}:
+        return [*prefix, "-m", "pytest", *rest]
+    return None
+
+
+def inline_python_code(command_argv: List[str]) -> Optional[str]:
+    """Return the ``CODE`` of a ``python -c "CODE"`` command, else None.
+
+    DevCouncil's acceptance compiler and many planner ``expected_tests`` are inline
+    assertions (``python -c "import m; assert m.f()==1"``). These are exactly the
+    checks whose diff coverage matters, so they are instrumented via a temp script
+    (see :func:`coverage_run_script_argv`) rather than left unmeasured.
+    """
+    if len(command_argv) < 3:
+        return None
+    head = Path(command_argv[0]).name.lower()
+    if head.endswith(".exe"):
+        head = head[:-4]
+    if head in {"python", "python3", "py"} and command_argv[1] == "-c":
+        return command_argv[2]
+    return None
+
+
+def inline_script_content(code: str, root: Path) -> str:
+    """Wrap inline ``-c`` code as a script that imports like ``python -c`` would.
+
+    ``python -c`` puts the current working directory on ``sys.path``; a plain script
+    instead puts the script's own directory there. Since the temp script lives under
+    ``.devcouncil/tmp`` we re-insert the repo root so ``import <module>`` resolves the
+    same way the original inline check did.
+    """
+    return f"import sys\nsys.path.insert(0, {str(Path(root))!r})\n{code}\n"
+
+
+def coverage_run_script_argv(
+    script_path: str,
+    python: str,
+    *,
+    append: bool,
+    data_file: str,
+    source: str = ".",
+) -> List[str]:
+    """A ``coverage run`` invocation for a materialised script (used for inline checks)."""
+    prefix = [python, "-m", "coverage", "run", f"--source={source}", f"--data-file={data_file}"]
+    if append:
+        prefix.append("-a")
+    return [*prefix, script_path]