devcouncil 0.3.1 → 0.4.0

package/src/devcouncil/verification/checks/diff_coverage_gate.py

@@ -0,0 +1,101 @@
+"""Diff↔coverage gate extracted from Verifier.verify_task."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Callable, List, Optional
+
+from devcouncil.domain.evidence import DiffCoverageEvidence
+from devcouncil.domain.gap import Gap
+from devcouncil.domain.task import Task
+from devcouncil.verification import diff_coverage as dc
+
+
+@dataclass
+class DiffCoverageGateResult:
+    gaps: List[Gap]
+    evidence: List[DiffCoverageEvidence]
+    coverage_measured: bool
+    coverage_skipped_reason: Optional[str]
+
+
+def run_diff_coverage_gate(
+    *,
+    task: Task,
+    diff_content: str,
+    any_passing: bool,
+    enforce_cov: bool,
+    measure_cov: bool,
+    min_ratio: float,
+    measure_diff_coverage: Callable[[Task, str], dc.DiffCoverageResult],
+    coverage_target_commands: Callable[[Task], List[str]],
+    next_gap_id: Callable[[str, str], str],
+) -> DiffCoverageGateResult:
+    """Run the diff↔coverage gate and return gaps + evidence."""
+    gaps: List[Gap] = []
+    evidence: List[DiffCoverageEvidence] = []
+    coverage_measured = False
+    coverage_skipped_reason: Optional[str] = None
+
+    if not measure_cov:
+        coverage_skipped_reason = "diff coverage disabled in config"
+    elif not diff_content:
+        coverage_skipped_reason = "no diff to measure"
+    elif not task.acceptance_criterion_ids:
+        coverage_skipped_reason = "task has no acceptance criteria"
+    elif not any_passing:
+        coverage_skipped_reason = "no passing verification command to instrument"
+
+    if measure_cov and diff_content and task.acceptance_criterion_ids and any_passing:
+        cov = measure_diff_coverage(task, diff_content)
+        if not cov.measured:
+            coverage_skipped_reason = cov.reason or "diff coverage could not be measured"
+        if cov.measured:
+            coverage_measured = True
+            coverage_skipped_reason = None
+            evidence.append(DiffCoverageEvidence(
+                task_id=task.id,
+                tool=cov.tool,
+                measured=True,
+                changed_lines=cov.changed_executable_lines,
+                covered_lines=cov.covered_changed_lines,
+                coverage_ratio=cov.ratio,
+                uncovered_by_file=cov.uncovered_by_file,
+                absent_files=cov.absent_files,
+                summary=cov.summary(),
+            ))
+            failing = cov.covered_changed_lines == 0 if min_ratio <= 0 else cov.ratio < min_ratio
+            if failing:
+                first_file = next(iter(cov.uncovered_by_file), None)
+                first_lines = cov.uncovered_by_file.get(first_file or "", [])
+                target_cmds = coverage_target_commands(task)
+                gaps.append(Gap(
+                    id=next_gap_id(task.id, "DIFFCOV"),
+                    severity="high" if enforce_cov else "medium",
+                    gap_type="diff_not_exercised",
+                    task_id=task.id,
+                    description=(
+                        f"Verification commands passed but exercised "
+                        f"{cov.covered_changed_lines}/{cov.changed_executable_lines} changed line(s): "
+                        f"{cov.summary()}. The acceptance criteria are not proven because the new "
+                        "logic was never executed by the tests."
+                    ),
+                    evidence=[cov.summary()] + [
+                        f"{path}: lines {lines}" for path, lines in list(cov.uncovered_by_file.items())[:5]
+                    ],
+                    recommended_fix=(
+                        "Add or extend a test that executes the changed lines, then re-verify. "
+                        "A passing suite that does not run the new code is not acceptance evidence."
+                    ),
+                    blocking=enforce_cov,
+                    file=first_file,
+                    line=first_lines[0] if first_lines else None,
+                    suggested_command=target_cmds[0] if target_cmds else None,
+                ))
+
+    return DiffCoverageGateResult(
+        gaps=gaps,
+        evidence=evidence,
+        coverage_measured=coverage_measured,
+        coverage_skipped_reason=coverage_skipped_reason,
+    )

package/src/devcouncil/verification/checks/doc_code_ref.py

@@ -0,0 +1,79 @@
+"""Heuristic doc→code reference gate for changed documentation."""
+
+from __future__ import annotations
+
+import logging
+import re
+from pathlib import Path
+from typing import Callable, List
+
+from devcouncil.domain.gap import Gap
+from devcouncil.domain.task import Task
+
+logger = logging.getLogger(__name__)
+
+_CODE_REF_RE = re.compile(
+    r"(?<![`\w])(?:src|tests|docs)/[\w./-]+\.(?:py|md|ts|tsx|js|jsx|json|yaml|yml|toml)"
+)
+
+
+def _norm(path: str) -> str:
+    return path.replace("\\", "/").lstrip("./")
+
+
+def _broken_refs_in_text(project_root: Path, text: str) -> List[str]:
+    broken: List[str] = []
+    for match in _CODE_REF_RE.finditer(text):
+        ref = match.group(0)
+        if not (project_root / ref).is_file():
+            broken.append(ref)
+    return broken
+
+
+def detect_doc_code_ref_gaps(
+    *,
+    task: Task,
+    project_root: Path,
+    changed_files: List[str],
+    diff_content: str,
+    next_gap_id: Callable[[str, str], str],
+    doc_code_ref_enabled: bool = True,
+    doc_code_ref_blocking: bool = False,
+) -> List[Gap]:
+    if not doc_code_ref_enabled:
+        return []
+    gaps: List[Gap] = []
+    try:
+        doc_paths = sorted(
+            p for p in changed_files
+            if _norm(p).endswith(".md") or _norm(p).startswith("docs/")
+        )
+        for rel in doc_paths:
+            path = project_root / rel
+            if not path.is_file():
+                continue
+            try:
+                text = path.read_text(encoding="utf-8", errors="replace")
+            except OSError:
+                continue
+            broken = _broken_refs_in_text(project_root, text)
+            if not broken:
+                continue
+            gaps.append(Gap(
+                id=next_gap_id(task.id, f"DOCREF-{rel}"),
+                severity="high" if doc_code_ref_blocking else "medium",
+                gap_type="doc_code_ref",
+                task_id=task.id,
+                description=(
+                    f"Changed doc `{rel}` references missing code paths: "
+                    f"{', '.join(broken[:5])}"
+                    + (" …" if len(broken) > 5 else "")
+                ),
+                evidence=[rel, *broken[:8]],
+                recommended_fix="Fix or remove broken code references in the changed documentation.",
+                blocking=doc_code_ref_blocking,
+                file=rel,
+            ))
+    except Exception:
+        logger.debug("detect_doc_code_ref_gaps failed", exc_info=True)
+    return gaps

package/src/devcouncil/verification/checks/liveness_ratchet.py

@@ -0,0 +1,336 @@
+"""Liveness ratchet: flag existing code newly stranded by this task.
+
+Diff-scoped gates catch *new* unwired files / dead symbols. This check catches
+the complementary failure: a task removes the last importer/caller of *existing*
+code, leaving it stranded relative to the checkout-time baseline.
+
+Skip when no complete baseline exists (pre-feature tasks). Never raises.
+"""
+
+from __future__ import annotations
+
+import logging
+from pathlib import Path
+from typing import Any, Callable, List, Mapping, Optional, Set
+
+from devcouncil.domain.gap import Gap
+from devcouncil.domain.task import Task
+
+logger = logging.getLogger(__name__)
+
+
+def _norm(path: str) -> str:
+    s = str(path).replace("\\", "/")
+    while s.startswith("./"):
+        s = s[2:]
+    return s
+
+
+def _as_str_set(values: Any) -> Set[str]:
+    if not isinstance(values, list):
+        return set()
+    return {_norm(str(v)) for v in values if v is not None and str(v).strip()}
+
+
+def _as_list(value: object) -> list[Any]:
+    return list(value) if isinstance(value, list) else []
+
+
+def _symbol_key(entry: str) -> str:
+    """Normalize ``path:line name`` / ``path:line:name`` to a stable identity key.
+
+    Prefer ``path::name`` so line-number drift does not create false regressions.
+    """
+    text = str(entry).strip()
+    if not text:
+        return ""
+    # Common map format: "path/to.py:12 symbol_name"
+    if " " in text:
+        loc, _, name = text.partition(" ")
+        path = loc.rsplit(":", 1)[0] if ":" in loc else loc
+        return f"{_norm(path)}::{name.strip()}"
+    # Fallback: path:line:name or path:name
+    parts = text.split(":")
+    if len(parts) >= 3 and parts[-2].isdigit():
+        return f"{_norm(parts[0])}::{parts[-1].strip()}"
+    if len(parts) == 2:
+        return f"{_norm(parts[0])}::{parts[1].strip()}"
+    return _norm(text)
+
+
+def _symbol_display(entry: str) -> tuple[str, Optional[int], str]:
+    """Return (path, line|None, name) for gap fields."""
+    text = str(entry).strip()
+    if " " in text:
+        loc, _, name = text.partition(" ")
+        if ":" in loc:
+            path, _, line_s = loc.rpartition(":")
+            line = int(line_s) if line_s.isdigit() else None
+            return _norm(path), line, name.strip()
+        return _norm(loc), None, name.strip()
+    parts = text.split(":")
+    if len(parts) >= 3 and parts[-2].isdigit():
+        return _norm(parts[0]), int(parts[-2]), parts[-1].strip()
+    if len(parts) == 2:
+        return _norm(parts[0]), None, parts[1].strip()
+    return _norm(text), None, ""
+
+
+def baseline_is_complete(baseline: Mapping[str, Any] | None) -> bool:
+    """True when baseline exists and carries the write-completed marker."""
+    return bool(baseline and isinstance(baseline, Mapping) and baseline.get("complete") is True)
+
+
+def _liveness_roots_unreliable(snap: Mapping[str, Any] | None) -> bool:
+    """True when unreachable lists must not be trusted (empty roots / flag)."""
+    if not snap or not isinstance(snap, Mapping):
+        return True
+    if snap.get("liveness_unreachable_unreliable") is True:
+        return True
+    roots = snap.get("entry_roots")
+    if not isinstance(roots, list) or not roots:
+        return True
+    return False
+
+
+def detect_liveness_regressions(
+    baseline: Mapping[str, Any] | None,
+    current: Mapping[str, Any] | None,
+    task_added_files: Set[str] | None = None,
+    *,
+    task: Task | None = None,
+    next_gap_id: Callable[[str, str], str] | None = None,
+    blocking: bool = False,
+    diff_added_lines: Mapping[str, Set[int]] | None = None,
+) -> List[Gap]:
+    """Diff baseline vs current liveness; flag newly stranded pre-existing code.
+
+    Files/symbols added by this task are excluded (covered by ``unwired_file`` /
+    ``dead_symbol``). Dead symbols must have existed at baseline (``symbol_index``)
+    — a brand-new unused def in an existing file is not ``stranded_code``.
+
+    Returns an empty list when ``baseline`` is missing or incomplete.
+    When either side has empty entry roots / unreliable unreachable, skip
+    unreachable diffs (avoids empty-root stranded floods).
+    """
+    gaps: List[Gap] = []
+    if not baseline_is_complete(baseline):
+        return gaps
+    if not current or not isinstance(current, Mapping):
+        return gaps
+    assert baseline is not None
+    try:
+        added = {_norm(p) for p in (task_added_files or set())}
+        task_id = task.id if task is not None else "TASK"
+        gap_id = next_gap_id or (lambda tid, kind: f"{tid}-{kind}-1")
+        added_lines = {
+            _norm(p): set(lines)
+            for p, lines in (diff_added_lines or {}).items()
+        }
+
+        base_unwired = _as_str_set(baseline.get("unwired_candidates"))
+        base_unreachable = _as_str_set(baseline.get("unreachable_files"))
+        cur_unwired = _as_str_set(current.get("unwired_candidates"))
+        cur_unreachable = _as_str_set(current.get("unreachable_files"))
+
+        newly_unwired = (cur_unwired - base_unwired) - added
+        skip_unreachable = (
+            _liveness_roots_unreliable(baseline) or _liveness_roots_unreliable(current)
+        )
+        newly_unreachable = (
+            set()
+            if skip_unreachable
+            else (cur_unreachable - base_unreachable) - added
+        )
+        stranded_files = sorted(newly_unwired | newly_unreachable)
+
+        for path in stranded_files:
+            reasons = []
+            if path in newly_unwired:
+                reasons.append("unwired")
+            if path in newly_unreachable:
+                reasons.append("unreachable")
+            reason = "/".join(reasons) or "stranded"
+            gaps.append(Gap(
+                id=gap_id(task_id, "STRAND"),
+                severity="high" if blocking else "medium",
+                gap_type="stranded_code",
+                task_id=task_id,
+                description=(
+                    f"Pre-existing file `{path}` became {reason} during this task "
+                    "(lost its last production importer/reachability path)."
+                ),
+                evidence=[path, f"regression:{reason}"],
+                recommended_fix=(
+                    "Restore the import/call that kept this module live, or delete "
+                    "the stranded module if it is intentionally unused."
+                ),
+                blocking=blocking,
+                file=path,
+            ))
+
+        # Symbol-level diff only when baseline was recorded with the same scan
+        # algorithm — stale/missing scan_version would otherwise flag every newly
+        # detected dead symbol as stranded_code after a detector hardening.
+        from devcouncil.indexing.wiring import LIVENESS_SCAN_VERSION
+
+        if baseline.get("scan_version") != LIVENESS_SCAN_VERSION:
+            return gaps
+
+        base_syms = {
+            _symbol_key(s): str(s)
+            for s in (baseline.get("dead_symbol_candidates") or [])
+            if isinstance(s, str) and _symbol_key(s)
+        }
+        cur_syms = {
+            _symbol_key(s): str(s)
+            for s in (current.get("dead_symbol_candidates") or [])
+            if isinstance(s, str) and _symbol_key(s)
+        }
+        # Symbols known at checkout. Without this index (legacy baselines), refuse
+        # to flag new dead symbols — avoids double-firing on brand-new defs.
+        base_index_raw = baseline.get("symbol_index") or []
+        base_index = {
+            _norm(str(k)) if "::" in str(k) else _symbol_key(str(k))
+            for k in base_index_raw
+            if k
+        }
+        # Normalize path::name keys
+        base_index_norm: Set[str] = set()
+        for k in base_index:
+            if "::" in k:
+                path_part, _, name = k.partition("::")
+                base_index_norm.add(f"{_norm(path_part)}::{name}")
+            else:
+                base_index_norm.add(k)
+
+        for key in sorted(set(cur_syms) - set(base_syms)):
+            entry = cur_syms[key]
+            path, line, name = _symbol_display(entry)
+            if path in added:
+                continue
+            # Require the symbol existed at baseline (was live then, dead now).
+            if not base_index_norm or key not in base_index_norm:
+                continue
+            # Defining line added in this task's diff → new symbol, not stranded.
+            if line is not None and line in added_lines.get(path, ()):
+                continue
+            label = name or entry
+            gaps.append(Gap(
+                id=gap_id(task_id, "STRANDSYM"),
+                severity="high" if blocking else "medium",
+                gap_type="stranded_code",
+                task_id=task_id,
+                description=(
+                    f"Pre-existing symbol `{label}` at {path}"
+                    f"{':' + str(line) if line else ''} became unreferenced "
+                    "during this task (lost its last caller)."
+                ),
+                evidence=[entry, f"symbol:{label}", "regression:dead_symbol"],
+                recommended_fix=(
+                    "Restore the import/call that referenced this symbol, or delete "
+                    "the stranded module/symbol if it is intentionally unused."
+                ),
+                blocking=blocking,
+                file=path,
+                line=line,
+            ))
+    except Exception:
+        logger.debug("detect_liveness_regressions failed; degrading to zero gaps", exc_info=True)
+        return []
+    return gaps
+
+
+def load_liveness_baseline(project_root: Path, task_id: str) -> Optional[dict]:
+    """Load checkout-time liveness baseline; None when absent, unreadable, or incomplete."""
+    path = project_root / ".devcouncil" / "liveness_baseline" / f"{task_id}.json"
+    if not path.is_file():
+        return None
+    try:
+        from devcouncil.utils.json_persist import read_json
+
+        data = read_json(path)
+        if not isinstance(data, dict):
+            return None
+        if data.get("complete") is not True:
+            logger.warning(
+                "liveness baseline for %s is incomplete; treating as missing",
+                task_id,
+            )
+            return None
+        return data
+    except Exception:
+        logger.debug("failed to load liveness baseline for %s", task_id, exc_info=True)
+        return None
+
+
+def delete_liveness_baseline(project_root: Path, task_id: str) -> bool:
+    """Remove the checkout baseline for ``task_id``. Returns True when deleted."""
+    path = project_root / ".devcouncil" / "liveness_baseline" / f"{task_id}.json"
+    try:
+        if path.is_file():
+            path.unlink()
+            return True
+    except OSError:
+        logger.debug("failed to delete liveness baseline for %s", task_id, exc_info=True)
+    return False
+
+
+def snapshot_liveness_baseline(
+    project_root: Path,
+    task_id: str,
+    *,
+    reset: bool = False,
+) -> Optional[Path]:
+    """Write-once uncapped liveness snapshot for ``task_id``.
+
+    Skips when a complete baseline already exists unless ``reset=True``. Writes
+    only after a full successful scan and marks ``complete: true`` so partial
+    failures never become ratchet inputs. Returns the path, or None on failure.
+    Never raises.
+    """
+    try:
+        from devcouncil.indexing.repo_mapper import RepoMapper
+        from devcouncil.utils.json_persist import write_json
+
+        out_dir = project_root / ".devcouncil" / "liveness_baseline"
+        out_dir.mkdir(parents=True, exist_ok=True)
+        out_path = out_dir / f"{task_id}.json"
+
+        if out_path.is_file() and not reset:
+            try:
+                from devcouncil.utils.json_persist import read_json
+
+                existing = read_json(out_path)
+                if isinstance(existing, dict) and existing.get("complete") is True:
+                    return out_path
+            except Exception:
+                pass
+            # Incomplete/corrupt on disk — fall through and rewrite.
+
+        mapper = RepoMapper(project_root)
+        snap = mapper.liveness_snapshot()
+        from devcouncil.indexing.wiring import LIVENESS_SCAN_VERSION
+
+        unreliable = bool(snap.get("liveness_unreachable_unreliable")) or not _as_list(
+            snap.get("entry_roots")
+        )
+        # Empty-root scans must not become ratchet baselines (would flood or
+        # falsely look "clean" after fail-soft unreachable=[]).
+        payload: dict[str, Any] = {
+            "unwired_candidates": _as_list(snap.get("unwired_candidates")),
+            "unreachable_files": _as_list(snap.get("unreachable_files")),
+            "dead_symbol_candidates": _as_list(snap.get("dead_symbol_candidates")),
+            "entry_roots": _as_list(snap.get("entry_roots")),
+            "symbol_index": _as_list(snap.get("symbol_index")),
+            "liveness_unreachable_unreliable": unreliable,
+            "generated_head": mapper._git_head(),
+            "source": "fresh_scan",
+            "scan_version": LIVENESS_SCAN_VERSION,
+            "complete": not unreliable,
+        }
+        write_json(out_path, payload)
+        return out_path if not unreliable else None
+    except Exception:
+        logger.debug("snapshot_liveness_baseline failed for %s", task_id, exc_info=True)
+        return None

package/src/devcouncil/verification/checks/orphan_diff.py

@@ -0,0 +1,104 @@
+"""Orphan-diff detection helpers extracted from Verifier."""
+
+from __future__ import annotations
+
+import logging
+import subprocess
+from pathlib import Path
+from typing import Callable, List, Tuple
+
+from devcouncil.domain.gap import Gap
+from devcouncil.domain.task import Task
+from devcouncil.indexing.wiring import is_test_path
+from devcouncil.verification.stub_detector import detect_stubs
+
+logger = logging.getLogger(__name__)
+
+
+def classify_change_paths(
+    project_root: Path,
+    changed_files: List[str],
+    get_untracked_files: Callable[[], List[str]],
+) -> Tuple[List[str], List[str]]:
+    """Return (added, deleted) paths intersecting changed_files."""
+    changed_set = set(changed_files)
+    added = set(get_untracked_files())
+    deleted: set[str] = set()
+    try:
+        from devcouncil.utils.proc import GIT_TIMEOUT
+
+        output = subprocess.check_output(
+            ["git", "diff", "HEAD", "--name-status"],
+            cwd=project_root,
+            stderr=subprocess.DEVNULL,
+            timeout=GIT_TIMEOUT,
+        ).decode("utf-8", errors="replace").splitlines()
+        for line in output:
+            parts = line.split("\t")
+            if len(parts) < 2:
+                continue
+            status = parts[0]
+            path = parts[-1].replace("\\", "/")
+            if status.startswith("A"):
+                added.add(path)
+            elif status.startswith("D"):
+                deleted.add(path)
+    except Exception as e:
+        logger.debug("Failed to classify changed files: %s", e)
+    return sorted(added & changed_set), sorted(deleted & changed_set)
+
+
+def detect_orphan_diff_gaps(
+    *,
+    task: Task,
+    changed_files: List[str],
+    planned_paths: set[str],
+    diff_content: str,
+    project_root: Path,
+    get_untracked_files: Callable[[], List[str]],
+    next_gap_id: Callable[[str, str], str],
+    classify_fn: Callable[[List[str]], Tuple[List[str], List[str]]] | None = None,
+) -> List[Gap]:
+    """Detect orphan-diff gaps for files changed outside the task plan."""
+    gaps: List[Gap] = []
+    if classify_fn is not None:
+        orphan_added, _orphan_deleted = classify_fn(changed_files)
+    else:
+        orphan_added, _orphan_deleted = classify_change_paths(
+            project_root, changed_files, get_untracked_files,
+        )
+    assert_free_test_files: set[str] = set()
+    if diff_content:
+        for finding in detect_stubs(project_root, diff_content, honor_allow_stub=False):
+            if "no assertions" in finding.reason:
+                assert_free_test_files.add(finding.file)
+    for cf in changed_files:
+        if cf not in planned_paths:
+            new_test_file = (
+                cf in orphan_added
+                and is_test_path(cf)
+                and cf not in assert_free_test_files
+            )
+            gaps.append(Gap(
+                id=next_gap_id(task.id, f"ORPHAN-{cf}"),
+                severity="medium" if new_test_file else "high",
+                gap_type="orphan_diff",
+                task_id=task.id,
+                description=(
+                    f"New test file {cf} was added but not planned for this task "
+                    "(advisory: added tests cannot change shipped behavior)."
+                    if new_test_file else
+                    f"File {cf} was modified but not planned for this task."
+                ),
+                evidence=[cf],
+                recommended_fix=(
+                    f"Append {cf} with `dev scope update <task_id> --lease-token <token> "
+                    f"--planned-file {cf}` (or fold the tests into a planned test file)."
+                    if new_test_file else
+                    f"Revert changes to {cf} or append it with "
+                    f"`dev scope update <task_id> --lease-token <token> --planned-file {cf}`."
+                ),
+                blocking=not new_test_file,
+                file=cf,
+            ))
+    return gaps