devcouncil 0.2.0 → 0.3.1

package/src/devcouncil/verification/acceptance_compiler.py

@@ -11,6 +11,7 @@ text and the code under review, then maps each check 1:1 to its criterion.
 from __future__ import annotations
 
 import json
+import logging
 from typing import Dict, List
 
 from pydantic import BaseModel
@@ -19,6 +20,8 @@ from devcouncil.domain.requirement import Requirement
 from devcouncil.domain.task import Task
 from devcouncil.llm.router import ModelRouter
 
+logger = logging.getLogger(__name__)
+
 
 class CompiledCheck(BaseModel):
     acceptance_criterion_id: str
@@ -29,6 +32,19 @@ class CompiledChecks(BaseModel):
     checks: List[CompiledCheck]
 
 
+# A worked example pinned to the prompt. Weak/local models follow a concrete
+# AC->command pairing far more reliably than rules alone — it anchors the import
+# style, the exception-guard idiom, and "assert behavior, not state".
+_WORKED_EXAMPLE = """Worked example (follow this shape exactly):
+  Acceptance criterion: "median([]) raises ValueError"
+  Code under review: a new file `stats.py` defining `def median(values): ...`
+  CORRECT command:
+    python -c "import stats\\ntry: stats.median([])\\nexcept ValueError: pass\\nelse: raise SystemExit(1)"
+  Why: it imports the REAL module name implied by the file (stats.py -> import stats),
+  exercises the public function, and asserts the OBSERVABLE behavior (a raised
+  exception) — not any file/git state."""
+
+
 class AcceptanceTestCompiler:
     def __init__(self, router: ModelRouter, role: str = "implementation_reviewer"):
         self.router = router
@@ -43,22 +59,148 @@ class AcceptanceTestCompiler:
         """Return {acceptance_criterion_id: [self-contained check command(s)]}.
 
         Best-effort: returns {} if the model cannot produce usable checks, so the
-        caller can fall back to the task's declared expected_tests.
+        caller can fall back to the task's declared expected_tests. Single-shot
+        wrapper over :meth:`compile_candidates` (samples=1) for back-compat.
+        """
+        return await self.compile_candidates(task, requirements, code_context, samples=1)
+
+    async def compile_candidates(
+        self,
+        task: Task,
+        requirements: List[Requirement],
+        code_context: str,
+        samples: int = 1,
+        per_criterion: bool = False,
+    ) -> Dict[str, List[str]]:
+        """Return {acceptance_criterion_id: [up to ``samples`` INDEPENDENT check commands]}.
+
+        Each criterion gets several independently-generated behavioral checks so the
+        caller can decide by majority vote — outvoting a single mis-generated check
+        without auto-passing a real defect. ``samples=1`` yields one command per
+        criterion (identical to the original single-shot ``compile``). Local sampling
+        is cost-free, so a weak model benefits most from a higher count.
+
+        ``per_criterion`` compiles ONE criterion per model call instead of batching them
+        all into a single prompt. A weak model batching N criteria into one JSON response
+        routinely omits or mis-attributes some — producing false ``incomplete`` verdicts —
+        whereas a focused single-criterion prompt is far more reliable. It costs N× the
+        calls (cheap on a local monitor), so it is opt-in.
         """
         ac_by_id = {ac.id: ac for req in requirements for ac in req.acceptance_criteria}
         target = [ac_by_id[i] for i in task.acceptance_criterion_ids if i in ac_by_id]
         if not target:
             return {}
 
+        if per_criterion and len(target) > 1:
+            out: Dict[str, List[str]] = {}
+            for ac in target:
+                out.update(await self._sample_checks([ac], code_context, samples))
+            return out
+        return await self._sample_checks(target, code_context, samples)
+
+    async def _sample_checks(
+        self, target: List, code_context: str, samples: int
+    ) -> Dict[str, List[str]]:
+        """Generate up to ``samples`` independent check commands for ``target`` criteria."""
+        valid_ids = {ac.id for ac in target}
+        out: Dict[str, List[str]] = {}
+        # Independent attempts. Each varies temperature + an attempt marker so the router
+        # cache keys differ (otherwise an identical prompt+temp returns the cached answer
+        # and every "sample" is the same command). Attempt 0 stays deterministic (temp 0).
+        for attempt in range(max(1, samples)):
+            temperature = 0.0 if attempt == 0 else min(0.8, 0.3 + 0.2 * attempt)
+            try:
+                result = await self.router.complete_structured(
+                    role=self.role,
+                    messages=[{"role": "user", "content": self._compile_prompt(target, code_context, attempt)}],
+                    schema=CompiledChecks,
+                    temperature=temperature,
+                    fallback=CompiledChecks(checks=[]),
+                )
+            except Exception as e:
+                logger.warning("Acceptance-check compile attempt %d failed: %s", attempt, e)
+                continue
+            for check in result.checks:
+                cmd = (check.command or "").strip()
+                if check.acceptance_criterion_id in valid_ids and cmd:
+                    bucket = out.setdefault(check.acceptance_criterion_id, [])
+                    if cmd not in bucket:  # dedup identical candidates across attempts
+                        bucket.append(cmd)
+        logger.info(
+            "Compiled acceptance checks for %d/%d criteria (%d samples)",
+            len(out), len(valid_ids), max(1, samples),
+        )
+        return out
+
+    async def repair(
+        self,
+        ac_id: str,
+        ac_description: str,
+        failing_command: str,
+        error_summary: str,
+        code_context: str,
+    ) -> str | None:
+        """Regenerate a compiled check that FAILED TO RUN (malformed/unrunnable).
+
+        Given the broken command and the launcher error, ask the model to fix the
+        COMMAND so it runs against the code — never to change what it asserts. Returns
+        the repaired command, or None if the model cannot produce a usable one. Safe by
+        construction: a check that did not run proves nothing, so regenerating it cannot
+        weaken the gate (a repaired check still has to genuinely pass to count)."""
+        prompt = f"""The following DevCouncil acceptance check FAILED TO RUN — it is malformed or
+its tooling/import is wrong, so it proves nothing about the code. Fix the COMMAND so it
+RUNS and correctly tests the SAME criterion. Do NOT weaken or change what it asserts;
+only fix what stops it from running (wrong module/import name, broken Python one-liner
+syntax, unavailable tool).
+
+Acceptance criterion ({ac_id}): {ac_description}
+
+Failing command:
+{failing_command}
+
+Launcher error / output:
+{error_summary}
+
+Code under review:
+{code_context}
+
+{_WORKED_EXAMPLE}
+
+Return JSON: one CompiledCheck with acceptance_criterion_id={ac_id!r} and the corrected
+single self-contained command. If you cannot produce a runnable behavioral command,
+return an empty checks list."""
+        try:
+            result = await self.router.complete_structured(
+                role=self.role,
+                messages=[{"role": "user", "content": prompt}],
+                schema=CompiledChecks,
+                fallback=CompiledChecks(checks=[]),
+            )
+        except Exception:
+            return None
+        for check in result.checks:
+            cmd = (check.command or "").strip()
+            if check.acceptance_criterion_id == ac_id and cmd and cmd != failing_command.strip():
+                return cmd
+        return None
+
+    def _compile_prompt(self, target, code_context: str, attempt: int = 0) -> str:
         acs_json = json.dumps(
             [{"id": ac.id, "description": ac.description, "method": ac.verification_method} for ac in target],
             indent=2,
         )
+        # Independent-attempt marker: nudges diversity across samples AND differentiates
+        # the router cache key so a second sample is actually regenerated, not replayed.
+        variant = "" if attempt == 0 else (
+            f"\nIndependent attempt #{attempt}: derive each check FROM SCRATCH; do not assume "
+            "a previous attempt's wording. Prefer a different but equivalent way to exercise "
+            "the same behavior.\n"
+        )
         prompt = f"""
 You are DevCouncil's acceptance-test compiler. Convert each acceptance criterion
 below into exactly ONE shell command that EXITS 0 if and only if the BEHAVIOR
 described by the criterion holds for the code shown.
-
+{variant}
 Acceptance criteria:
 {acs_json}
 
@@ -105,21 +247,7 @@ criterion instead of emitting such a command:
 - If a criterion cannot be checked by a behavioral command (e.g. pure 'manual'
   review, or it only describes repo/tooling state), OMIT it rather than inventing
   a state-based or bogus command.
-"""
-        try:
-            result = await self.router.complete_structured(
-                role=self.role,
-                messages=[{"role": "user", "content": prompt}],
-                schema=CompiledChecks,
-                fallback=CompiledChecks(checks=[]),
-            )
-        except Exception:
-            return {}
 
-        out: Dict[str, List[str]] = {}
-        valid_ids = {ac.id for ac in target}
-        for check in result.checks:
-            cmd = (check.command or "").strip()
-            if check.acceptance_criterion_id in valid_ids and cmd:
-                out.setdefault(check.acceptance_criterion_id, []).append(cmd)
-        return out
+{_WORKED_EXAMPLE}
+"""
+        return prompt

package/src/devcouncil/verification/ad_hoc_check.py

@@ -14,6 +14,7 @@ resilient to churn in the command module.
 from __future__ import annotations
 
 import asyncio
+import logging
 from dataclasses import dataclass, field
 from pathlib import Path
 from typing import List, Optional
@@ -32,6 +33,8 @@ _TASK_ID = "CHECK"
 
 _DEFAULT_CRITERION = "The working-tree changes are correct and exercised by tests."
 
+logger = logging.getLogger(__name__)
+
 
 @dataclass
 class AdHocCheckResult:
@@ -80,7 +83,9 @@ def run_working_tree_check(
     diff = verifier.get_diff()
     changed_files = verifier.get_changed_files()
     if not diff or not changed_files:
+        logger.info("Ad-hoc check: no working-tree changes; passing trivially")
         return AdHocCheckResult(requirement="", passed=True, reason="no_changes")
+    logger.info("Ad-hoc check: %d changed file(s), enforce_coverage=%s", len(changed_files), enforce_coverage or min_ratio > 0)
 
     criterion = requirement or _DEFAULT_CRITERION
     req = Requirement(
@@ -119,6 +124,7 @@ def run_working_tree_check(
     gaps, evidence = asyncio.run(verifier.verify_task(task, [req]))
     coverage = next((ev for ev in evidence if isinstance(ev, DiffCoverageEvidence)), None)
     blocking = [g for g in gaps if g.blocking]
+    logger.info("Ad-hoc check result: passed=%s (%d gap(s), %d blocking)", not blocking, len(gaps), len(blocking))
     return AdHocCheckResult(
         requirement=criterion,
         changed_files=changed_files,

package/src/devcouncil/verification/implementation_reviewer.py

@@ -1,4 +1,5 @@
 import json
+import logging
 from typing import List
 from pydantic import BaseModel
 from devcouncil.domain.task import Task
@@ -7,6 +8,8 @@ from devcouncil.domain.gap import Gap
 from devcouncil.llm.router import ModelRouter
 from devcouncil.utils.redaction import redact_string
 
+logger = logging.getLogger(__name__)
+
 class ReviewOutput(BaseModel):
     is_satisfactory: bool
     findings: List[Gap]
@@ -47,9 +50,15 @@ Your task is to identify if the implementation is complete, correct, and follows
 Return a JSON object with 'is_satisfactory' and a list of 'findings' (as Gap objects).
 """
         messages = [{"role": "user", "content": prompt}]
-        
-        return await self.router.complete_structured(
+
+        logger.info("Implementation review: task=%s diff_bytes=%d", task.id, len(diff))
+        result = await self.router.complete_structured(
             role="implementation_reviewer",
             messages=messages,
             schema=ReviewOutput
         )
+        logger.info(
+            "Implementation review for %s: satisfactory=%s findings=%d",
+            task.id, result.is_satisfactory, len(result.findings),
+        )
+        return result

package/src/devcouncil/verification/sandbox.py

@@ -2,8 +2,11 @@
 
 from __future__ import annotations
 
+import asyncio
 import hashlib
+import logging
 import platform
+import shutil
 import subprocess
 import sys
 from pathlib import Path
@@ -17,6 +20,8 @@ from devcouncil.storage.db import get_db
 from devcouncil.storage.native import VerificationRunRepository
 from devcouncil.verification.verifier import Verifier
 
+logger = logging.getLogger(__name__)
+
 
 class SandboxResult(BaseModel):
     sandbox: str
@@ -38,6 +43,8 @@ def _save_run(
     commands: list[dict],
     status: Literal["passed", "failed", "unsupported"],
 ) -> None:
+    log = logger.info if status == "passed" else logger.warning
+    log("Sandbox %s for %s: %s (%d command(s))", sandbox, task.id, status, len(commands))
     db = get_db(project_root)
     if not db:
         return
@@ -56,8 +63,6 @@ class LocalSandbox(VerificationSandbox):
         self.project_root = project_root
 
     def run(self, task: Task, commands: list[str], requirements: list) -> SandboxResult:
-        import asyncio
-
         gaps, _ = asyncio.run(Verifier(self.project_root).verify_task(task, requirements))
         status: Literal["passed", "failed", "unsupported"] = (
             "failed" if any(g.blocking for g in gaps) else "passed"
@@ -148,8 +153,6 @@ class NixSandbox(VerificationSandbox):
 
 
 def shutil_which(name: str) -> str | None:
-    import shutil
-
     return shutil.which(name)