devcouncil 0.3.0 → 0.4.0

package/src/devcouncil/verification/git_diff_fallback.py

@@ -0,0 +1,272 @@
+"""Git diff fallbacks when no batched snapshot is available.
+
+Extracted from ``verifier.py`` so the verifier orchestrator stays focused on gate
+logic while pre-head / untracked / walk-the-tree paths live in one place.
+"""
+
+from __future__ import annotations
+
+import fnmatch
+import logging
+import subprocess
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import TYPE_CHECKING, List, Optional
+
+from devcouncil.domain.checkpoint_refs import task_before_ref
+from devcouncil.utils.proc import GIT_TIMEOUT
+
+if TYPE_CHECKING:
+    from devcouncil.utils.git_snapshot import GitWorktreeSnapshot
+
+logger = logging.getLogger(__name__)
+
+IGNORED_CHANGE_PATTERNS = (
+    "__pycache__/*",
+    "*/__pycache__/*",
+    "*.pyc",
+    "*.pyo",
+    ".pytest_cache/*",
+    ".mypy_cache/*",
+    ".ruff_cache/*",
+    ".devcouncil/*",
+    "AGENTS.md",
+    "CLAUDE.md",
+    ".gitignore",
+)
+
+MAX_UNTRACKED_DIFF_BYTES = 256_000
+
+
+@dataclass
+class GitDiffFallback:
+    """Pre-snapshot git diff helpers used by :class:`~devcouncil.verification.verifier.Verifier`."""
+
+    project_root: Path
+    git_snapshot: Optional["GitWorktreeSnapshot"] = None
+    untracked_cache: Optional[List[str]] = field(default=None, repr=False)
+    # When set, diff/changed-files use ``git diff <ref>`` (PR/base scope) instead of HEAD/worktree.
+    diff_base: Optional[str] = None
+
+    def has_head(self) -> bool:
+        if self.git_snapshot is not None:
+            return self.git_snapshot.has_head
+        try:
+            return subprocess.run(
+                ["git", "rev-parse", "--verify", "HEAD"],
+                cwd=self.project_root,
+                stdout=subprocess.DEVNULL,
+                stderr=subprocess.DEVNULL,
+                timeout=GIT_TIMEOUT,
+            ).returncode == 0
+        except (subprocess.TimeoutExpired, OSError) as exc:
+            logger.warning("git rev-parse HEAD failed: %s", exc)
+            return False
+
+    def get_initial_repo_diff(self) -> str:
+        parts: List[str] = []
+        for cmd in (["git", "diff", "--cached"], ["git", "diff"]):
+            result = subprocess.run(
+                cmd,
+                cwd=self.project_root,
+                capture_output=True,
+                text=True,
+                encoding="utf-8",
+                errors="replace",
+                timeout=GIT_TIMEOUT,
+            )
+            if result.returncode == 0 and result.stdout:
+                parts.append(result.stdout)
+        untracked_diff = self.get_untracked_files_diff()
+        if untracked_diff:
+            parts.append(untracked_diff)
+        return "\n".join(parts)
+
+    def get_status_files(self) -> List[str]:
+        files: set[str] = set()
+        commands = (
+            ["git", "diff", "--cached", "--name-only"],
+            ["git", "diff", "--name-only"],
+            ["git", "ls-files", "--others", "--exclude-standard"],
+        )
+        for cmd in commands:
+            try:
+                output = subprocess.check_output(
+                    cmd,
+                    cwd=self.project_root,
+                    stderr=subprocess.DEVNULL,
+                    timeout=GIT_TIMEOUT,
+                ).decode("utf-8", errors="replace").splitlines()
+                files.update(path.replace("\\", "/") for path in output if path.strip())
+            except (subprocess.CalledProcessError, subprocess.TimeoutExpired):
+                continue
+        if not files:
+            files.update(self.walk_project_files())
+        return self.filter_change_paths(sorted(files))
+
+    def get_untracked_files(self) -> List[str]:
+        if self.git_snapshot is not None:
+            return self.filter_change_paths(self.git_snapshot.untracked_files)
+        if self.untracked_cache is not None:
+            return self.untracked_cache
+        try:
+            output = subprocess.check_output(
+                ["git", "ls-files", "--others", "--exclude-standard"],
+                cwd=self.project_root,
+                stderr=subprocess.DEVNULL,
+                timeout=GIT_TIMEOUT,
+            ).decode("utf-8", errors="replace").splitlines()
+            return self.filter_change_paths(output)
+        except Exception as exc:
+            logger.debug("Failed to list untracked files: %s", exc)
+            return []
+
+    def get_untracked_files_diff(self) -> str:
+        parts: List[str] = []
+        for rel_path in self.get_untracked_files():
+            full_path = self.project_root / rel_path
+            if not full_path.is_file():
+                continue
+            parts.append(self.format_new_file_diff(rel_path, full_path))
+        return "\n".join(part for part in parts if part)
+
+    def format_new_file_diff(self, rel_path: str, full_path: Path) -> str:
+        try:
+            raw = full_path.read_bytes()
+        except Exception as exc:
+            logger.debug("Failed to read untracked file %s: %s", rel_path, exc)
+            return ""
+
+        header = [
+            f"diff --git a/{rel_path} b/{rel_path}",
+            "new file mode 100644",
+            "--- /dev/null",
+            f"+++ b/{rel_path}",
+        ]
+        if b"\0" in raw[:8192]:
+            return "\n".join([*header, f"Binary files /dev/null and b/{rel_path} differ"])
+
+        truncated = len(raw) > MAX_UNTRACKED_DIFF_BYTES
+        if truncated:
+            raw = raw[:MAX_UNTRACKED_DIFF_BYTES]
+        text = raw.decode("utf-8", errors="replace")
+        lines = text.splitlines()
+        if text.endswith(("\n", "\r")):
+            line_count = len(lines)
+        else:
+            line_count = max(len(lines), 1 if text else 0)
+
+        diff_lines = [*header, f"@@ -0,0 +1,{line_count} @@"]
+        if not text:
+            return "\n".join(header) + "\n"
+
+        diff_lines.extend(f"+{line}" for line in lines)
+        if truncated:
+            diff_lines.append("+[devcouncil: untracked file diff truncated]")
+        return "\n".join(diff_lines)
+
+    def walk_project_files(self) -> List[str]:
+        files: List[str] = []
+        for path in self.project_root.rglob("*"):
+            if not path.is_file():
+                continue
+            rel = path.relative_to(self.project_root).as_posix()
+            if rel.startswith(".git/"):
+                continue
+            files.append(rel)
+        return files
+
+    def filter_change_paths(self, paths: List[str]) -> List[str]:
+        return [
+            path
+            for path in (p.strip().replace("\\", "/") for p in paths)
+            if path and not self.is_ignored_change(path)
+        ]
+
+    def is_ignored_change(self, path: str) -> bool:
+        return any(fnmatch.fnmatch(path, pattern) for pattern in IGNORED_CHANGE_PATTERNS)
+
+    def get_diff(self) -> str:
+        if self.diff_base:
+            try:
+                return subprocess.check_output(
+                    ["git", "diff", self.diff_base],
+                    cwd=self.project_root,
+                    timeout=GIT_TIMEOUT,
+                ).decode("utf-8", errors="replace")
+            except Exception as e:
+                logger.warning("Failed to get git diff against %s: %s", self.diff_base, e)
+                return ""
+        snap = self.git_snapshot
+        try:
+            if snap is not None:
+                if not snap.has_head:
+                    parts = [snap.cached_diff, snap.working_diff]
+                    untracked_diff = self.get_untracked_files_diff()
+                    return "\n".join(part for part in [*parts, untracked_diff] if part)
+                untracked_diff = self.get_untracked_files_diff()
+                return "\n".join(part for part in [snap.tracked_diff, untracked_diff] if part)
+            if not self.has_head():
+                return self.get_initial_repo_diff()
+            tracked_diff = subprocess.check_output(
+                ["git", "diff", "HEAD"], cwd=self.project_root, timeout=GIT_TIMEOUT
+            ).decode("utf-8", errors="replace")
+            untracked_diff = self.get_untracked_files_diff()
+            return "\n".join(part for part in [tracked_diff, untracked_diff] if part)
+        except Exception as e:
+            logger.warning("Failed to get git diff: %s", e)
+            return ""
+
+    def get_changed_files(self) -> List[str]:
+        if self.diff_base:
+            try:
+                output = subprocess.check_output(
+                    ["git", "diff", self.diff_base, "--name-only"],
+                    cwd=self.project_root,
+                    timeout=GIT_TIMEOUT,
+                ).decode("utf-8", errors="replace").splitlines()
+                return self.filter_change_paths(output)
+            except Exception as e:
+                logger.warning("Failed to get changed files against %s: %s", self.diff_base, e)
+                return []
+        snap = self.git_snapshot
+        try:
+            if snap is not None:
+                if not snap.has_head:
+                    return self.filter_change_paths(snap.status_paths or snap.untracked_files)
+                files = set(snap.tracked_changed_files)
+                files.update(snap.untracked_files)
+                return self.filter_change_paths(sorted(files))
+            if not self.has_head():
+                return self.get_status_files()
+            output = subprocess.check_output(
+                ["git", "diff", "HEAD", "--name-only"], cwd=self.project_root, timeout=GIT_TIMEOUT
+            ).decode("utf-8", errors="replace").splitlines()
+            files = set(output)
+            files.update(self.get_untracked_files())
+            return self.filter_change_paths(sorted(files))
+        except Exception as e:
+            logger.warning("Failed to get changed files: %s", e)
+            return []
+
+    def committed_task_diff(self, task_id: str) -> str:
+        """Diff of work committed since the task's ``before`` checkpoint, or ""."""
+        before_ref = task_before_ref(task_id)
+        try:
+            has_ref = subprocess.run(
+                ["git", "rev-parse", "--verify", before_ref],
+                cwd=self.project_root,
+                stdout=subprocess.DEVNULL,
+                stderr=subprocess.DEVNULL,
+                timeout=GIT_TIMEOUT,
+            ).returncode == 0
+            if has_ref:
+                return subprocess.check_output(
+                    ["git", "diff", before_ref],
+                    cwd=self.project_root,
+                    stderr=subprocess.DEVNULL,
+                    timeout=GIT_TIMEOUT,
+                ).decode("utf-8", errors="replace")
+        except Exception as e:
+            logger.debug("Failed to read committed task diff for %s: %s", task_id, e)
+        return ""

package/src/devcouncil/verification/implementation_reviewer.py

@@ -47,6 +47,19 @@ Your task is to identify if the implementation is complete, correct, and follows
 - Identify architectural drift.
 - Identify security risks not caught by static scans.
 
+Review rules — follow them exactly:
+- The diff above IS the complete, authoritative record of the change. Review the CODE in it.
+  Do not ask for proof that the change was made, and do not report "no evidence of
+  implementation" — you are looking at the implementation.
+- Every finding must cite the specific file/hunk in the diff that demonstrates the problem
+  (in its 'evidence'). A concern you cannot tie to a concrete line in the diff is not a finding.
+- Do not review prose, commit-message style, response formatting, or your estimate of effort.
+- Severity rubric: 'critical' is reserved for a demonstrable correctness or security defect
+  visible in the diff (wrong behavior, data loss, injection, broken requirement). Missing
+  edge cases are 'high' at most; style and structure are 'low'.
+- If the diff implements the requirements correctly, return is_satisfactory=true and an
+  empty findings list. An empty review of correct code is a correct review.
+
 Return a JSON object with 'is_satisfactory' and a list of 'findings' (as Gap objects).
 """
         messages = [{"role": "user", "content": prompt}]

package/src/devcouncil/verification/incremental_check.py

@@ -0,0 +1,241 @@
+"""Incremental gate runner: the sub-second sidecar behind ``dev check --watch``.
+
+Ties :mod:`devcouncil.verification.gate_selector` (path → relevant/narrowed gates) to
+:mod:`devcouncil.verification.gate_cache` (content-hash skip of previously-green gates).
+On an iterative edit only the gates whose inputs actually changed are executed; the rest
+are served from cache. This is deliberately separate from the full ``verify_task`` path,
+which release / ``dev go`` still run in full and which never consults this cache.
+
+Given the working-tree changed files and the project's configured commands, it:
+1. selects the gates relevant to the changed stacks (and narrows linters to the touched
+   files),
+2. skips any gate that passed last time with byte-identical inputs,
+3. runs the remainder and records their pass/fail in the cache,
+4. returns a compact, render-friendly result.
+
+Best-effort and side-effect-light: it shells out only to run the gates themselves and
+persists a single small JSON cache file.
+"""
+
+from __future__ import annotations
+
+import logging
+import time
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Callable, List, Mapping, Optional, Sequence
+
+from devcouncil.utils.json_persist import read_json
+from devcouncil.verification.gate_cache import GateResultCache
+from devcouncil.verification.gate_selector import GateSpec, select_gates
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass
+class GateOutcome:
+    name: str
+    kind: str
+    command: str
+    passed: bool
+    cached: bool
+    summary: str = ""
+    duration_s: float = 0.0
+    narrowed: bool = False
+
+
+@dataclass
+class IncrementalResult:
+    changed_files: List[str] = field(default_factory=list)
+    outcomes: List[GateOutcome] = field(default_factory=list)
+    skipped: List[tuple[str, str]] = field(default_factory=list)
+    duration_s: float = 0.0
+    no_changes: bool = False
+    no_gates: bool = False
+    narrowed: bool = False
+
+    @property
+    def ran(self) -> List[GateOutcome]:
+        return [o for o in self.outcomes if not o.cached]
+
+    @property
+    def cached(self) -> List[GateOutcome]:
+        return [o for o in self.outcomes if o.cached]
+
+    @property
+    def passed(self) -> bool:
+        return all(o.passed for o in self.outcomes)
+
+    def to_dict(self) -> dict:
+        return {
+            "passed": self.passed,
+            "no_changes": self.no_changes,
+            "no_gates": self.no_gates,
+            "narrowed": self.narrowed,
+            "changed_files": self.changed_files,
+            "duration_s": round(self.duration_s, 4),
+            "gates": [
+                {
+                    "name": o.name,
+                    "kind": o.kind,
+                    "command": o.command,
+                    "passed": o.passed,
+                    "cached": o.cached,
+                    "narrowed": o.narrowed,
+                    "duration_s": round(o.duration_s, 4),
+                }
+                for o in self.outcomes
+            ],
+            "ran": len(self.ran),
+            "cached_hits": len(self.cached),
+            "skipped": [{"command": c, "reason": r} for c, r in self.skipped],
+        }
+
+
+def _default_commands(project_root: Path) -> dict[str, list[str]]:
+    """Configured test/lint/typecheck commands, or ``{}`` on any config failure."""
+    try:
+        from devcouncil.app.config import load_config
+
+        cfg = load_config(project_root)
+        return {
+            "test": list(cfg.commands.test),
+            "lint": list(cfg.commands.lint),
+            "typecheck": list(cfg.commands.typecheck),
+        }
+    except Exception as exc:
+        logger.debug("incremental gates: config commands unavailable: %s", exc)
+        return {}
+
+
+def _load_repo_map(project_root: Path) -> dict | None:
+    """Best-effort load of ``.devcouncil/repo_map.json`` for dependent expansion."""
+    map_path = project_root / ".devcouncil" / "repo_map.json"
+    if not map_path.is_file():
+        return None
+    try:
+        data = read_json(map_path)
+        return data if isinstance(data, dict) else None
+    except Exception as exc:
+        logger.debug("incremental gates: repo map unavailable: %s", exc)
+        return None
+
+
+def _default_changed_files(project_root: Path) -> list[str]:
+    from devcouncil.verification.verifier import Verifier
+
+    return Verifier(project_root).get_changed_files()
+
+
+def _default_runner(project_root: Path, timeout: int) -> Callable[[str], tuple[bool, str]]:
+    from devcouncil.verification.command_runner import run_verification_command
+
+    def _run(command: str) -> tuple[bool, str]:
+        result = run_verification_command(
+            project_root, command, task_id="check-watch", timeout=timeout
+        )
+        return result.exit_code == 0, result.summary
+
+    return _run
+
+
+def run_incremental_gates(
+    project_root: Path,
+    *,
+    commands: Optional[Mapping[str, Sequence[str]]] = None,
+    extra_test_commands: Optional[Sequence[str]] = None,
+    changed_files: Optional[Sequence[str]] = None,
+    cache: Optional[GateResultCache] = None,
+    use_cache: bool = True,
+    narrow: bool = True,
+    runner: Optional[Callable[[str], tuple[bool, str]]] = None,
+    timeout: int = 300,
+) -> IncrementalResult:
+    """Run only the gates affected by ``changed_files``, skipping cached-green ones.
+
+    ``commands`` defaults to the project's configured test/lint/typecheck commands;
+    ``extra_test_commands`` (e.g. ``dev check --test``) are appended to the test kind.
+    ``runner`` is a ``command -> (passed, summary)`` callable (defaults to the real
+    verification command runner) — the seam that keeps this unit-testable without
+    subprocesses.
+    """
+    started = time.perf_counter()
+    root = Path(project_root)
+
+    if changed_files is None:
+        changed_files = _default_changed_files(root)
+    changed = [p for p in changed_files if p and p.strip()]
+
+    resolved_commands: dict[str, list[str]] = {
+        k: list(v) for k, v in (commands or _default_commands(root)).items()
+    }
+    if extra_test_commands:
+        resolved_commands.setdefault("test", [])
+        resolved_commands["test"] = list(resolved_commands["test"]) + [
+            c for c in extra_test_commands if c and c.strip()
+        ]
+
+    if not changed:
+        return IncrementalResult(no_changes=True, duration_s=time.perf_counter() - started)
+
+    repo_map = _load_repo_map(root)
+    selection = select_gates(changed, resolved_commands, narrow=narrow, repo_map=repo_map)
+    any_narrowed = any(g.narrowed for g in selection.gates)
+    if not selection.gates:
+        return IncrementalResult(
+            changed_files=changed,
+            skipped=selection.skipped,
+            no_gates=True,
+            narrowed=any_narrowed,
+            duration_s=time.perf_counter() - started,
+        )
+
+    if cache is None:
+        cache = GateResultCache(root)
+    cache.load()
+    run = runner or _default_runner(root, timeout)
+
+    outcomes: List[GateOutcome] = []
+    for gate in selection.gates:
+        if use_cache and cache.is_green(gate):
+            outcomes.append(GateOutcome(
+                name=gate.name, kind=gate.kind, command=gate.command,
+                passed=True, cached=True,
+                summary=cache.cached_summary(gate) or "", narrowed=gate.narrowed,
+            ))
+            continue
+        gate_started = time.perf_counter()
+        try:
+            passed, summary = run(gate.command)
+        except Exception as exc:  # a gate crash must not kill the sidecar loop
+            logger.debug("incremental gate %s crashed: %s", gate.name, exc)
+            passed, summary = False, f"gate crashed: {exc}"
+        gate_dt = time.perf_counter() - gate_started
+        cache.record(gate, passed=passed, summary=summary)
+        outcomes.append(GateOutcome(
+            name=gate.name, kind=gate.kind, command=gate.command,
+            passed=passed, cached=False, summary=summary,
+            duration_s=gate_dt, narrowed=gate.narrowed,
+        ))
+
+    cache.save()
+    return IncrementalResult(
+        changed_files=changed,
+        outcomes=outcomes,
+        skipped=selection.skipped,
+        narrowed=any_narrowed,
+        duration_s=time.perf_counter() - started,
+    )
+
+
+def selected_gate_specs(
+    project_root: Path,
+    changed_files: Sequence[str],
+    *,
+    commands: Optional[Mapping[str, Sequence[str]]] = None,
+    narrow: bool = True,
+) -> list[GateSpec]:
+    """Convenience: the gates that *would* run for these changed files (no execution)."""
+    resolved = commands or _default_commands(Path(project_root))
+    repo_map = _load_repo_map(Path(project_root))
+    return select_gates(changed_files, resolved, narrow=narrow, repo_map=repo_map).gates

package/src/devcouncil/verification/next_actions.py

@@ -40,6 +40,14 @@ _CATEGORY_BY_GAP_TYPE = {
     "migration_gap": "fix_code",
     "requirement_not_planned": "plan",
     "task_not_implemented": "plan",
+    "stub_detected": "fix_code",
+    "stub_declared": "review",
+    "suspicious_effort": "review",
+    "coarse_acceptance_proof": "add_test",
+    "unwired_file": "fix_code",
+    "dead_symbol": "fix_code",
+    "stranded_code": "fix_code",
+    "stale_map": "refresh_map",
 }
 
 
@@ -85,7 +93,10 @@ def _derive_file(gap: Gap) -> Optional[str]:
 def _action_text(gap: Gap, file: Optional[str]) -> str:
     target = file or "the affected file"
     if gap.gap_type == "orphan_diff":
-        return f"Revert changes to {target} or add it to the task's planned files."
+        return (
+            f"Revert changes to {target} or append it with "
+            f"`dev scope update <task_id> --lease-token <token> --planned-file {target}`."
+        )
     if gap.gap_type == "planned_file_not_changed":
         return f"Modify {target} as planned, or remove it from the task's planned files."
     if gap.gap_type == "dependency_risk":
@@ -104,6 +115,54 @@ def _action_text(gap: Gap, file: Optional[str]) -> str:
         return "Remove the detected secret/finding from the diff and rotate any exposed credential."
     if gap.gap_type == "architecture_drift":
         return "Address the flagged change or resolve the open critique card, then re-verify."
+    if gap.gap_type == "stub_detected":
+        loc = f"{target}{':' + str(gap.line) if gap.line else ''}"
+        return (
+            f"Replace the stub/placeholder at {loc} with a real implementation, then re-verify. "
+            "Do not mark work complete while placeholders remain."
+        )
+    if gap.gap_type == "stub_declared":
+        loc = f"{target}{':' + str(gap.line) if gap.line else ''}"
+        return (
+            f"Review the intentional stub declared at {loc}; replace it before marking done."
+        )
+    if gap.gap_type == "suspicious_effort":
+        return (
+            "The diff looks too small or superficial for the planned scope. Complete the "
+            "planned work (or restore removed tests), then re-verify."
+        )
+    if gap.gap_type == "coarse_acceptance_proof":
+        return (
+            "Acceptance criteria were proven only by a coarse passing command, not a "
+            "per-criterion check. Add a verification command or test that exercises each "
+            "listed criterion specifically, then re-verify."
+        )
+    if gap.gap_type == "unwired_file":
+        return (
+            f"Import or register `{target}` from its intended non-test caller "
+            f"(append the caller with `dev scope update <task_id> --lease-token <token> "
+            f"--planned-file <caller>` if needed), or delete the unused file."
+        )
+    if gap.gap_type == "dead_symbol":
+        loc = f"{target}{':' + str(gap.line) if gap.line else ''}"
+        name = ""
+        for item in gap.evidence:
+            if isinstance(item, str) and item.startswith("symbol:"):
+                name = item.split(":", 1)[1].strip()
+                break
+        symbol = name or "the new symbol"
+        return (
+            f"Call or register `{symbol}` at {loc} from the code that needs it "
+            f"(use `dev scope update ... --planned-file <caller>` if the caller is "
+            f"out of scope), or remove it."
+        )
+    if gap.gap_type == "stranded_code":
+        return (
+            f"Restore the import/call that kept `{target}` live, or delete the "
+            "stranded module if it is intentionally unused."
+        )
+    if gap.gap_type == "stale_map":
+        return "Run `dev map` to regenerate the repository map, then re-verify."
     # Fall back to the gap's own recommended fix for anything unmapped.
     return gap.recommended_fix