devcouncil 0.1.1 → 0.2.0

package/src/devcouncil/verification/next_actions.py

@@ -0,0 +1,189 @@
+"""Typed, machine-actionable next actions derived from verification gaps.
+
+A human-readable "here is what's wrong" report is fine for a person, but an agent
+in a closed loop needs a structured contract it can route on without parsing
+prose. ``build_next_actions`` turns the gaps from a verification run into a list of
+:class:`NextAction` records — ``{category, action, file, line, missing_evidence,
+suggested_command}`` — so a coding agent (over MCP) can self-repair and re-verify
+without a human pasting anything.
+
+The mapping is deterministic. Where a gap was created with explicit hints
+(``gap.file``/``gap.line``/``gap.suggested_command``) those are used directly; when
+a gap is reloaded from the database (which does not persist those hint columns) the
+fields are reconstructed best-effort from the gap's evidence and description.
+"""
+
+from __future__ import annotations
+
+from typing import List, Optional
+
+from pydantic import BaseModel, Field
+
+from devcouncil.domain.gap import Gap
+
+# Machine-routable buckets an agent can branch on. Kept small and stable so the
+# contract is predictable across releases.
+Category = str
+
+_CATEGORY_BY_GAP_TYPE = {
+    "orphan_diff": "scope",
+    "planned_file_not_changed": "scope",
+    "dependency_risk": "scope",
+    "test_failed": "fix_code",
+    "invalid_verification_command": "fix_verification",
+    "acceptance_criteria_unproven": "add_test",
+    "diff_not_exercised": "add_test",
+    "missing_test": "add_test",
+    "security_risk": "security",
+    "architecture_drift": "review",
+    "assumption_violated": "review",
+    "migration_gap": "fix_code",
+    "requirement_not_planned": "plan",
+    "task_not_implemented": "plan",
+}
+
+
+class NextAction(BaseModel):
+    """One concrete, routable step the agent can take to clear a gap."""
+
+    gap_id: str
+    gap_type: str
+    category: Category
+    severity: str
+    blocking: bool
+    action: str
+    file: Optional[str] = None
+    line: Optional[int] = None
+    acceptance_criterion_id: Optional[str] = None
+    expected_verification_method: Optional[str] = None
+    missing_evidence: Optional[str] = None
+    suggested_command: Optional[str] = None
+    evidence: List[str] = Field(default_factory=list)
+    # Paths to the captured stdout/stderr logs for the failing command behind this
+    # gap, so the agent can open the full failure output without re-running. Present
+    # only on a fresh verify run (the gap store does not persist them).
+    stdout_path: Optional[str] = None
+    stderr_path: Optional[str] = None
+
+
+def _looks_like_path(value: str) -> bool:
+    value = value.strip()
+    if not value or " " in value or "\n" in value:
+        return False
+    return "/" in value or "." in value
+
+
+def _derive_file(gap: Gap) -> Optional[str]:
+    if gap.file:
+        return gap.file
+    for item in gap.evidence:
+        if isinstance(item, str) and _looks_like_path(item):
+            return item
+    return None
+
+
+def _action_text(gap: Gap, file: Optional[str]) -> str:
+    target = file or "the affected file"
+    if gap.gap_type == "orphan_diff":
+        return f"Revert changes to {target} or add it to the task's planned files."
+    if gap.gap_type == "planned_file_not_changed":
+        return f"Modify {target} as planned, or remove it from the task's planned files."
+    if gap.gap_type == "dependency_risk":
+        return f"Justify or revert the unplanned dependency/config change in {target}."
+    if gap.gap_type == "test_failed":
+        cmd = gap.suggested_command
+        return f"Fix the failing check, then re-run: {cmd}" if cmd else "Fix the failing verification check, then re-verify."
+    if gap.gap_type == "invalid_verification_command":
+        return "Replace the unrunnable verification command with a single runnable command, then re-verify."
+    if gap.gap_type == "diff_not_exercised":
+        loc = f" ({target}{':' + str(gap.line) if gap.line else ''})" if file else ""
+        return f"Add or extend a test that executes the changed lines{loc}, then re-verify."
+    if gap.gap_type in {"acceptance_criteria_unproven", "missing_test"}:
+        return "Provide a passing verification command that proves this acceptance criterion."
+    if gap.gap_type == "security_risk":
+        return "Remove the detected secret/finding from the diff and rotate any exposed credential."
+    if gap.gap_type == "architecture_drift":
+        return "Address the flagged change or resolve the open critique card, then re-verify."
+    # Fall back to the gap's own recommended fix for anything unmapped.
+    return gap.recommended_fix
+
+
+def _missing_evidence(gap: Gap) -> Optional[str]:
+    """A concrete description of WHAT evidence is missing — not a restatement of the
+    description.
+
+    For an unproven acceptance criterion we name the criterion and the expected
+    verification method (and, where the verifier knew one, the expected check command
+    via ``gap.suggested_command``) so the agent can author the right proof rather than
+    re-reading prose. Falls back to the gap description for the other add-test gap
+    types (diff_not_exercised, missing_test) which already carry concrete locations."""
+    if gap.gap_type == "acceptance_criteria_unproven":
+        parts: List[str] = []
+        ac = gap.acceptance_criterion_id
+        method = gap.expected_verification_method
+        if ac:
+            parts.append(f"No passing evidence for acceptance criterion {ac}")
+        else:
+            parts.append("No passing acceptance evidence")
+        if method:
+            parts.append(f"expected verification method: {method}")
+        if gap.suggested_command:
+            parts.append(f"run/repair check: {gap.suggested_command}")
+        elif gap.file:
+            loc = f"{gap.file}:{gap.line}" if gap.line else gap.file
+            parts.append(f"uncovered: {loc}")
+        return "; ".join(parts)
+    if gap.gap_type in {"diff_not_exercised", "missing_test"}:
+        return gap.description
+    return None
+
+
+def next_action_for(gap: Gap) -> NextAction:
+    file = _derive_file(gap)
+    return NextAction(
+        gap_id=gap.id,
+        gap_type=gap.gap_type,
+        category=_CATEGORY_BY_GAP_TYPE.get(gap.gap_type, "review"),
+        severity=gap.severity,
+        blocking=gap.blocking,
+        action=_action_text(gap, file),
+        file=file,
+        line=gap.line,
+        acceptance_criterion_id=gap.acceptance_criterion_id,
+        expected_verification_method=gap.expected_verification_method,
+        missing_evidence=_missing_evidence(gap),
+        suggested_command=gap.suggested_command,
+        evidence=list(gap.evidence),
+        stdout_path=gap.stdout_path,
+        stderr_path=gap.stderr_path,
+    )
+
+
+def build_next_actions(gaps: List[Gap], *, blocking_only: bool = True) -> List[NextAction]:
+    """Build the next-actions contract from verification gaps.
+
+    By default only *blocking* gaps become next actions — those are what the agent
+    must clear to pass verification. Pass ``blocking_only=False`` to include
+    advisory signals (non-blocking gaps) as well. Blocking actions are ordered
+    first, then by severity.
+    """
+    selected = [g for g in gaps if g.blocking] if blocking_only else list(gaps)
+    severity_rank = {"critical": 0, "high": 1, "medium": 2, "low": 3}
+    selected.sort(key=lambda g: (not g.blocking, severity_rank.get(g.severity, 4)))
+    return [next_action_for(gap) for gap in selected]
+
+
+def split_next_actions(gaps: List[Gap]) -> tuple[List[NextAction], List[NextAction]]:
+    """Return ``(blocking_actions, advisory_actions)``.
+
+    Blocking actions are what the agent MUST clear to pass the gate. Advisory
+    actions are non-blocking signals worth acting on — most importantly the
+    diff↔coverage ``diff_not_exercised`` finding ("tests passed but the new code
+    was never run") and security/add-test hints, which ``build_next_actions``
+    filters out by default. Surfacing them as a distinct array lets an autonomous
+    agent improve quality without confusing them with the pass/fail gate.
+    """
+    all_actions = build_next_actions(gaps, blocking_only=False)
+    blocking = [a for a in all_actions if a.blocking]
+    advisory = [a for a in all_actions if not a.blocking]
+    return blocking, advisory

package/src/devcouncil/verification/sandbox.py

@@ -0,0 +1,178 @@
+"""Verification sandbox abstraction."""
+
+from __future__ import annotations
+
+import hashlib
+import platform
+import subprocess
+import sys
+from pathlib import Path
+from typing import Literal
+
+from pydantic import BaseModel
+
+from devcouncil.app.config import DevCouncilConfig, load_config
+from devcouncil.domain.task import Task
+from devcouncil.storage.db import get_db
+from devcouncil.storage.native import VerificationRunRepository
+from devcouncil.verification.verifier import Verifier
+
+
+class SandboxResult(BaseModel):
+    sandbox: str
+    status: Literal["passed", "failed", "unsupported"]
+    environment: dict[str, str]
+    commands: list[dict]
+
+
+class VerificationSandbox:
+    def run(self, task: Task, commands: list[str], requirements: list) -> SandboxResult:
+        raise NotImplementedError
+
+
+def _save_run(
+    project_root: Path,
+    task: Task,
+    sandbox: str,
+    environment: dict[str, str],
+    commands: list[dict],
+    status: Literal["passed", "failed", "unsupported"],
+) -> None:
+    db = get_db(project_root)
+    if not db:
+        return
+    with db.get_session() as session:
+        VerificationRunRepository(session).save(
+            task.id,
+            sandbox,
+            environment,
+            commands,
+            status,
+        )
+
+
+class LocalSandbox(VerificationSandbox):
+    def __init__(self, project_root: Path):
+        self.project_root = project_root
+
+    def run(self, task: Task, commands: list[str], requirements: list) -> SandboxResult:
+        import asyncio
+
+        gaps, _ = asyncio.run(Verifier(self.project_root).verify_task(task, requirements))
+        status: Literal["passed", "failed", "unsupported"] = (
+            "failed" if any(g.blocking for g in gaps) else "passed"
+        )
+        env = _environment_metadata(self.project_root)
+        command_results = [{"command": cmd, "status": status} for cmd in commands]
+        _save_run(self.project_root, task, "local", env, command_results, status)
+        return SandboxResult(sandbox="local", status=status, environment=env, commands=command_results)
+
+
+class DockerSandbox(VerificationSandbox):
+    def __init__(self, project_root: Path, config: DevCouncilConfig):
+        self.project_root = project_root
+        self.config = config
+
+    def run(self, task: Task, commands: list[str], requirements: list) -> SandboxResult:
+        if not shutil_which("docker"):
+            result = SandboxResult(
+                sandbox="docker",
+                status="unsupported",
+                environment={},
+                commands=[{"reason": "docker not available"}],
+            )
+            _save_run(self.project_root, task, "docker", result.environment, result.commands, result.status)
+            return result
+        image = self.config.verification.sandbox.docker_image
+        setup = self.config.verification.sandbox.docker_setup_commands
+        results: list[dict] = []
+        for setup_cmd in setup:
+            proc = subprocess.run(
+                ["docker", "run", "--rm", "-v", f"{self.project_root}:/work", "-w", "/work", image, "sh", "-c", setup_cmd],
+                capture_output=True,
+                text=True,
+            )
+            results.append({"command": setup_cmd, "exit_code": proc.returncode})
+            if proc.returncode != 0:
+                result = SandboxResult(sandbox="docker", status="failed", environment={"image": image}, commands=results)
+                _save_run(self.project_root, task, "docker", result.environment, result.commands, result.status)
+                return result
+        for cmd in commands:
+            proc = subprocess.run(
+                ["docker", "run", "--rm", "-v", f"{self.project_root}:/work", "-w", "/work", image, "sh", "-c", cmd],
+                capture_output=True,
+                text=True,
+            )
+            results.append({"command": cmd, "exit_code": proc.returncode})
+            if proc.returncode != 0:
+                result = SandboxResult(sandbox="docker", status="failed", environment={"image": image}, commands=results)
+                _save_run(self.project_root, task, "docker", result.environment, result.commands, result.status)
+                return result
+        result = SandboxResult(sandbox="docker", status="passed", environment={"image": image}, commands=results)
+        _save_run(self.project_root, task, "docker", result.environment, result.commands, result.status)
+        return result
+
+
+class NixSandbox(VerificationSandbox):
+    def __init__(self, project_root: Path, config: DevCouncilConfig):
+        self.project_root = project_root
+        self.config = config
+
+    def run(self, task: Task, commands: list[str], requirements: list) -> SandboxResult:
+        if not (self.project_root / "flake.nix").exists() or not shutil_which("nix"):
+            result = SandboxResult(
+                sandbox="nix",
+                status="unsupported",
+                environment={},
+                commands=[{"reason": "nix or flake.nix unavailable"}],
+            )
+            _save_run(self.project_root, task, "nix", result.environment, result.commands, result.status)
+            return result
+        attr = self.config.verification.sandbox.nix_flake_attr or "devShells.default"
+        results: list[dict] = []
+        for cmd in commands:
+            proc = subprocess.run(
+                ["nix", "develop", f".#{attr}", "-c", "sh", "-c", cmd],
+                cwd=self.project_root,
+                capture_output=True,
+                text=True,
+            )
+            results.append({"command": cmd, "exit_code": proc.returncode})
+            if proc.returncode != 0:
+                result = SandboxResult(sandbox="nix", status="failed", environment={"attr": attr}, commands=results)
+                _save_run(self.project_root, task, "nix", result.environment, result.commands, result.status)
+                return result
+        result = SandboxResult(sandbox="nix", status="passed", environment={"attr": attr}, commands=results)
+        _save_run(self.project_root, task, "nix", result.environment, result.commands, result.status)
+        return result
+
+
+def shutil_which(name: str) -> str | None:
+    import shutil
+
+    return shutil.which(name)
+
+
+def _environment_metadata(project_root: Path) -> dict[str, str]:
+    env = {
+        "python": sys.version.split()[0],
+        "platform": platform.platform(),
+    }
+    try:
+        uv = subprocess.check_output(["uv", "--version"], text=True).strip()
+        env["uv"] = uv
+    except (FileNotFoundError, subprocess.CalledProcessError, OSError):
+        pass
+    lock = project_root / "uv.lock"
+    if lock.exists():
+        env["uv_lock_hash"] = hashlib.sha256(lock.read_bytes()).hexdigest()[:16]
+    return env
+
+
+def get_sandbox(name: str, project_root: Path) -> VerificationSandbox:
+    config = load_config(project_root)
+    if name == "docker":
+        return DockerSandbox(project_root, config)
+    if name == "nix":
+        return NixSandbox(project_root, config)
+    return LocalSandbox(project_root)

package/src/devcouncil/verification/test_resolver.py

@@ -0,0 +1,91 @@
+"""Evidence suggestion from changed files."""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Literal
+
+from pydantic import BaseModel
+
+from devcouncil.domain.task import Task
+
+
+class EvidenceSuggestion(BaseModel):
+    command: str
+    confidence: Literal["high", "medium", "low"]
+    reason: str
+    paths: list[str] = []
+
+
+class TestResolver:
+    __test__ = False
+
+    def __init__(self, project_root: Path):
+        self.project_root = project_root.resolve()
+
+    def suggest_for_task(
+        self,
+        task: Task,
+        changed_files: list[str] | None = None,
+    ) -> list[EvidenceSuggestion]:
+        files = changed_files or [pf.path for pf in task.planned_files]
+        suggestions: list[EvidenceSuggestion] = []
+        seen: set[str] = set()
+
+        for path in files:
+            normalized = path.replace("\\", "/")
+            for candidate in self._candidates_for_path(normalized):
+                if candidate in seen:
+                    continue
+                seen.add(candidate)
+                confidence, reason = self._confidence_for(normalized, candidate)
+                suggestions.append(
+                    EvidenceSuggestion(
+                        command=candidate,
+                        confidence=confidence,
+                        reason=reason,
+                        paths=[normalized],
+                    )
+                )
+        return suggestions
+
+    def _candidates_for_path(self, path: str) -> list[str]:
+        candidates: list[str] = []
+        if path.startswith("src/") and path.endswith(".py"):
+            module = path.removeprefix("src/").removesuffix(".py")
+            parts = module.split("/")
+            if len(parts) >= 2:
+                nested = self.project_root / "tests" / parts[0] / f"test_{parts[-1]}.py"
+                if nested.exists():
+                    candidates.append(f"pytest {nested.relative_to(self.project_root).as_posix()}")
+            flat = self.project_root / "tests" / f"test_{parts[-1]}.py"
+            if flat.exists():
+                candidates.append(f"pytest {flat.relative_to(self.project_root).as_posix()}")
+            if "cli/commands/" in path:
+                unit = self.project_root / "tests/unit/test_cli_commands.py"
+                if unit.exists():
+                    candidates.append(f"pytest {unit.relative_to(self.project_root).as_posix()}")
+            if path.endswith("policy_engine.py"):
+                unit = self.project_root / "tests/unit/test_task_policy_engine.py"
+                if unit.exists():
+                    candidates.append(f"pytest {unit.relative_to(self.project_root).as_posix()}")
+        if path == "src/auth.py":
+            auth_test = self.project_root / "tests/test_auth.py"
+            if auth_test.exists():
+                candidates.append("pytest tests/test_auth.py")
+        if not candidates:
+            candidates.append("pytest tests/unit")
+        return candidates
+
+    def _confidence_for(self, path: str, command: str) -> tuple[Literal["high", "medium", "low"], str]:
+        if "test_auth.py" in command and path.endswith("auth.py"):
+            return "high", "Direct auth module mapping"
+        if path.endswith("policy_engine.py") and "test_task_policy_engine.py" in command:
+            return "high", "Policy engine unit test exists"
+        if "cli/commands" in path and "test_cli_commands.py" in command:
+            return "high", "CLI command module maps to cli command tests"
+        if command.startswith("pytest tests/") and command != "pytest tests/unit":
+            return "high", "Nested module test path exists"
+        if "test_" in command:
+            return "medium", "Related test file match"
+        return "low", "Package-level fallback"