devcouncil 0.1.1 → 0.3.0

package/src/devcouncil/verification/acceptance_compiler.py

@@ -0,0 +1,253 @@
+"""Compile natural-language acceptance criteria into self-contained executable
+checks that DevCouncil owns and runs.
+
+This is the difference between trusting the planner/agent's word and gathering
+real evidence: instead of running planner-authored ``expected_tests`` (which the
+benchmark showed often reference tools or test files that do not exist), DevCouncil
+derives one runnable check per acceptance criterion directly from the criterion
+text and the code under review, then maps each check 1:1 to its criterion.
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+from typing import Dict, List
+
+from pydantic import BaseModel
+
+from devcouncil.domain.requirement import Requirement
+from devcouncil.domain.task import Task
+from devcouncil.llm.router import ModelRouter
+
+logger = logging.getLogger(__name__)
+
+
+class CompiledCheck(BaseModel):
+    acceptance_criterion_id: str
+    command: str  # a single shell command that exits 0 iff the criterion holds
+
+
+class CompiledChecks(BaseModel):
+    checks: List[CompiledCheck]
+
+
+# A worked example pinned to the prompt. Weak/local models follow a concrete
+# AC->command pairing far more reliably than rules alone — it anchors the import
+# style, the exception-guard idiom, and "assert behavior, not state".
+_WORKED_EXAMPLE = """Worked example (follow this shape exactly):
+  Acceptance criterion: "median([]) raises ValueError"
+  Code under review: a new file `stats.py` defining `def median(values): ...`
+  CORRECT command:
+    python -c "import stats\\ntry: stats.median([])\\nexcept ValueError: pass\\nelse: raise SystemExit(1)"
+  Why: it imports the REAL module name implied by the file (stats.py -> import stats),
+  exercises the public function, and asserts the OBSERVABLE behavior (a raised
+  exception) — not any file/git state."""
+
+
+class AcceptanceTestCompiler:
+    def __init__(self, router: ModelRouter, role: str = "implementation_reviewer"):
+        self.router = router
+        self.role = role
+
+    async def compile(
+        self,
+        task: Task,
+        requirements: List[Requirement],
+        code_context: str,
+    ) -> Dict[str, List[str]]:
+        """Return {acceptance_criterion_id: [self-contained check command(s)]}.
+
+        Best-effort: returns {} if the model cannot produce usable checks, so the
+        caller can fall back to the task's declared expected_tests. Single-shot
+        wrapper over :meth:`compile_candidates` (samples=1) for back-compat.
+        """
+        return await self.compile_candidates(task, requirements, code_context, samples=1)
+
+    async def compile_candidates(
+        self,
+        task: Task,
+        requirements: List[Requirement],
+        code_context: str,
+        samples: int = 1,
+        per_criterion: bool = False,
+    ) -> Dict[str, List[str]]:
+        """Return {acceptance_criterion_id: [up to ``samples`` INDEPENDENT check commands]}.
+
+        Each criterion gets several independently-generated behavioral checks so the
+        caller can decide by majority vote — outvoting a single mis-generated check
+        without auto-passing a real defect. ``samples=1`` yields one command per
+        criterion (identical to the original single-shot ``compile``). Local sampling
+        is cost-free, so a weak model benefits most from a higher count.
+
+        ``per_criterion`` compiles ONE criterion per model call instead of batching them
+        all into a single prompt. A weak model batching N criteria into one JSON response
+        routinely omits or mis-attributes some — producing false ``incomplete`` verdicts —
+        whereas a focused single-criterion prompt is far more reliable. It costs N× the
+        calls (cheap on a local monitor), so it is opt-in.
+        """
+        ac_by_id = {ac.id: ac for req in requirements for ac in req.acceptance_criteria}
+        target = [ac_by_id[i] for i in task.acceptance_criterion_ids if i in ac_by_id]
+        if not target:
+            return {}
+
+        if per_criterion and len(target) > 1:
+            out: Dict[str, List[str]] = {}
+            for ac in target:
+                out.update(await self._sample_checks([ac], code_context, samples))
+            return out
+        return await self._sample_checks(target, code_context, samples)
+
+    async def _sample_checks(
+        self, target: List, code_context: str, samples: int
+    ) -> Dict[str, List[str]]:
+        """Generate up to ``samples`` independent check commands for ``target`` criteria."""
+        valid_ids = {ac.id for ac in target}
+        out: Dict[str, List[str]] = {}
+        # Independent attempts. Each varies temperature + an attempt marker so the router
+        # cache keys differ (otherwise an identical prompt+temp returns the cached answer
+        # and every "sample" is the same command). Attempt 0 stays deterministic (temp 0).
+        for attempt in range(max(1, samples)):
+            temperature = 0.0 if attempt == 0 else min(0.8, 0.3 + 0.2 * attempt)
+            try:
+                result = await self.router.complete_structured(
+                    role=self.role,
+                    messages=[{"role": "user", "content": self._compile_prompt(target, code_context, attempt)}],
+                    schema=CompiledChecks,
+                    temperature=temperature,
+                    fallback=CompiledChecks(checks=[]),
+                )
+            except Exception as e:
+                logger.warning("Acceptance-check compile attempt %d failed: %s", attempt, e)
+                continue
+            for check in result.checks:
+                cmd = (check.command or "").strip()
+                if check.acceptance_criterion_id in valid_ids and cmd:
+                    bucket = out.setdefault(check.acceptance_criterion_id, [])
+                    if cmd not in bucket:  # dedup identical candidates across attempts
+                        bucket.append(cmd)
+        logger.info(
+            "Compiled acceptance checks for %d/%d criteria (%d samples)",
+            len(out), len(valid_ids), max(1, samples),
+        )
+        return out
+
+    async def repair(
+        self,
+        ac_id: str,
+        ac_description: str,
+        failing_command: str,
+        error_summary: str,
+        code_context: str,
+    ) -> str | None:
+        """Regenerate a compiled check that FAILED TO RUN (malformed/unrunnable).
+
+        Given the broken command and the launcher error, ask the model to fix the
+        COMMAND so it runs against the code — never to change what it asserts. Returns
+        the repaired command, or None if the model cannot produce a usable one. Safe by
+        construction: a check that did not run proves nothing, so regenerating it cannot
+        weaken the gate (a repaired check still has to genuinely pass to count)."""
+        prompt = f"""The following DevCouncil acceptance check FAILED TO RUN — it is malformed or
+its tooling/import is wrong, so it proves nothing about the code. Fix the COMMAND so it
+RUNS and correctly tests the SAME criterion. Do NOT weaken or change what it asserts;
+only fix what stops it from running (wrong module/import name, broken Python one-liner
+syntax, unavailable tool).
+
+Acceptance criterion ({ac_id}): {ac_description}
+
+Failing command:
+{failing_command}
+
+Launcher error / output:
+{error_summary}
+
+Code under review:
+{code_context}
+
+{_WORKED_EXAMPLE}
+
+Return JSON: one CompiledCheck with acceptance_criterion_id={ac_id!r} and the corrected
+single self-contained command. If you cannot produce a runnable behavioral command,
+return an empty checks list."""
+        try:
+            result = await self.router.complete_structured(
+                role=self.role,
+                messages=[{"role": "user", "content": prompt}],
+                schema=CompiledChecks,
+                fallback=CompiledChecks(checks=[]),
+            )
+        except Exception:
+            return None
+        for check in result.checks:
+            cmd = (check.command or "").strip()
+            if check.acceptance_criterion_id == ac_id and cmd and cmd != failing_command.strip():
+                return cmd
+        return None
+
+    def _compile_prompt(self, target, code_context: str, attempt: int = 0) -> str:
+        acs_json = json.dumps(
+            [{"id": ac.id, "description": ac.description, "method": ac.verification_method} for ac in target],
+            indent=2,
+        )
+        # Independent-attempt marker: nudges diversity across samples AND differentiates
+        # the router cache key so a second sample is actually regenerated, not replayed.
+        variant = "" if attempt == 0 else (
+            f"\nIndependent attempt #{attempt}: derive each check FROM SCRATCH; do not assume "
+            "a previous attempt's wording. Prefer a different but equivalent way to exercise "
+            "the same behavior.\n"
+        )
+        prompt = f"""
+You are DevCouncil's acceptance-test compiler. Convert each acceptance criterion
+below into exactly ONE shell command that EXITS 0 if and only if the BEHAVIOR
+described by the criterion holds for the code shown.
+{variant}
+Acceptance criteria:
+{acs_json}
+
+Code under review (the diff / current files):
+{code_context}
+
+What a check must verify — BEHAVIOR ONLY:
+- A check exists to confirm the code DOES what the criterion describes when its
+  public API is exercised: import the module/symbol and call its function(s), or
+  run its CLI/entrypoint, and assert on the observable result (return value,
+  raised exception, stdout, exit code).
+- DevCouncil already enforces scope, file ownership, and append-only/orphan-diff
+  constraints with its OWN gates. Acceptance checks must therefore NEVER re-assert
+  repository or filesystem STATE — that is not their job and it produces false
+  BLOCKED results because `dev` itself adds workspace files (AGENTS.md, CLAUDE.md,
+  .gitignore, .devcouncil/config.yaml, etc.).
+
+Rules — the commands are executed verbatim by the verifier:
+- One command per acceptance_criterion_id (reference the id exactly).
+- Each command MUST be a single, SELF-CONTAINED, immediately-runnable command:
+  import the real module/symbol from the code and assert the behavior directly.
+  Do NOT depend on test files, fixtures, or any external setup.
+- Prefer: python -c "import <module>; assert <expr>". For an expected exception,
+  use a one-line guard, e.g.
+  python -c "import m; \\ntry: m.f([])\\nexcept ValueError: pass\\nelse: raise SystemExit(1)"
+  (real newlines are fine; never put try/if/for after a ';').
+- Use the actual module name implied by the code (e.g. file 'stats.py' -> import stats).
+
+HARD PROHIBITIONS — a command that does any of these is INVALID; omit the
+criterion instead of emitting such a command:
+- NEVER assert exact git or filesystem state. Forbidden: `git status`,
+  `git status --porcelain`, `git diff`, `git diff --name-only`, `git show`,
+  `git ls-files`, `ls`/`find`/`os.listdir` equality checks, asserting a precise
+  set or count of changed/created files, or asserting a file does/does not exist
+  as the criterion's pass condition.
+- NEVER do append-only or byte-level file/content comparisons (e.g.
+  `git show HEAD:file`, diffing bytes, asserting only N bytes/lines were added).
+  Assert the resulting BEHAVIOR instead, not how the file changed.
+- NEVER invoke linters, type checkers, formatters, or build/package tools that
+  may be absent: flake8, mypy, ruff, pylint, black, isort, eslint, tsc, prettier,
+  npm, npx, yarn, pnpm, cargo, go vet, etc. Only use such a tool if the code
+  context clearly shows it is configured for this repo (e.g. a matching config
+  section/file is present in the context) AND it is essential to the criterion.
+- If a criterion cannot be checked by a behavioral command (e.g. pure 'manual'
+  review, or it only describes repo/tooling state), OMIT it rather than inventing
+  a state-based or bogus command.
+
+{_WORKED_EXAMPLE}
+"""
+        return prompt

package/src/devcouncil/verification/ad_hoc_check.py

@@ -0,0 +1,135 @@
+"""Verify an ad-hoc working-tree diff against an inline requirement — no planning, no keys.
+
+This powers ``dev check``'s evidence-gate mode (the lite entry point): wrap whatever is
+in the working tree as a synthetic Requirement→Task, run the *same* deterministic
+:class:`~devcouncil.verification.verifier.Verifier` the full workflow uses — orphan-diff,
+secret scan, acceptance evidence, and the diff↔coverage gate — and return the verdict
+plus the typed next-actions contract. ``router=None`` keeps it provider-key-free so a
+newcomer can taste the evidence gate before committing to the full council flow.
+
+The logic lives here (not in the CLI command) so it is unit-testable without Typer and
+resilient to churn in the command module.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import logging
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import List, Optional
+
+from devcouncil.domain.evidence import DiffCoverageEvidence
+from devcouncil.domain.gap import Gap
+from devcouncil.domain.requirement import AcceptanceCriterion, Requirement
+from devcouncil.domain.task import PlannedFile, Task
+from devcouncil.llm.router import ModelRouter
+from devcouncil.verification.next_actions import NextAction, build_next_actions
+from devcouncil.verification.verifier import Verifier
+
+_REQ_ID = "REQ-CHECK"
+_AC_ID = "AC-CHECK"
+_TASK_ID = "CHECK"
+
+_DEFAULT_CRITERION = "The working-tree changes are correct and exercised by tests."
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass
+class AdHocCheckResult:
+    requirement: str
+    changed_files: List[str] = field(default_factory=list)
+    gaps: List[Gap] = field(default_factory=list)
+    next_actions: List[NextAction] = field(default_factory=list)
+    diff_coverage: Optional[DiffCoverageEvidence] = None
+    passed: bool = True
+    reason: str = ""
+
+    def to_dict(self) -> dict:
+        return {
+            "ok": True,
+            "verified": self.passed,
+            "requirement": self.requirement,
+            "changed_files": self.changed_files,
+            "reason": self.reason,
+            "gap_count": len(self.gaps),
+            "blocking_gap_count": len([g for g in self.gaps if g.blocking]),
+            "gaps": [g.model_dump() for g in self.gaps],
+            "next_actions": [a.model_dump() for a in self.next_actions],
+            "diff_coverage": self.diff_coverage.model_dump() if self.diff_coverage else None,
+        }
+
+
+def run_working_tree_check(
+    project_root: Path,
+    requirement: Optional[str] = None,
+    *,
+    test_commands: Optional[List[str]] = None,
+    enforce_coverage: bool = False,
+    min_ratio: float = 0.0,
+    router: Optional[ModelRouter] = None,
+    verifier: Optional[Verifier] = None,
+) -> AdHocCheckResult:
+    """Verify the current working-tree diff against a one-line requirement.
+
+    Builds a synthetic task whose planned files are exactly the changed files (so the
+    result is about evidence, not scope noise) and whose expected tests are
+    ``test_commands``. Diff coverage is always measured; pass ``enforce_coverage`` (or a
+    positive ``min_ratio``) to make an unexercised diff blocking.
+    """
+    verifier = verifier or Verifier(project_root, router=router)
+
+    diff = verifier.get_diff()
+    changed_files = verifier.get_changed_files()
+    if not diff or not changed_files:
+        logger.info("Ad-hoc check: no working-tree changes; passing trivially")
+        return AdHocCheckResult(requirement="", passed=True, reason="no_changes")
+    logger.info("Ad-hoc check: %d changed file(s), enforce_coverage=%s", len(changed_files), enforce_coverage or min_ratio > 0)
+
+    criterion = requirement or _DEFAULT_CRITERION
+    req = Requirement(
+        id=_REQ_ID,
+        title=(requirement or "Working-tree change")[:80],
+        description=criterion,
+        priority="high",
+        source="user",
+        acceptance_criteria=[
+            AcceptanceCriterion(id=_AC_ID, description=criterion, verification_method="unit_test"),
+        ],
+    )
+    untracked = set(verifier._get_untracked_files())
+    task = Task(
+        id=_TASK_ID,
+        title="Ad-hoc working-tree check",
+        description=criterion,
+        requirement_ids=[_REQ_ID],
+        acceptance_criterion_ids=[_AC_ID],
+        planned_files=[
+            PlannedFile(
+                path=path,
+                reason="working-tree change",
+                allowed_change="create" if path in untracked else "modify",
+            )
+            for path in changed_files
+        ],
+        expected_tests=list(test_commands or []),
+    )
+
+    # Always measure diff coverage in lite mode; block on it only when asked. A positive
+    # --min-coverage implies enforcement so the flag is never silently inert.
+    enforce = enforce_coverage or min_ratio > 0
+    verifier._diff_coverage_override = (True, enforce, float(min_ratio))
+
+    gaps, evidence = asyncio.run(verifier.verify_task(task, [req]))
+    coverage = next((ev for ev in evidence if isinstance(ev, DiffCoverageEvidence)), None)
+    blocking = [g for g in gaps if g.blocking]
+    logger.info("Ad-hoc check result: passed=%s (%d gap(s), %d blocking)", not blocking, len(gaps), len(blocking))
+    return AdHocCheckResult(
+        requirement=criterion,
+        changed_files=changed_files,
+        gaps=gaps,
+        next_actions=build_next_actions(gaps),
+        diff_coverage=coverage,
+        passed=not blocking,
+    )