devcouncil 0.3.1 → 0.4.0

package/src/devcouncil/verification/checks/compiled_acceptance.py

@@ -0,0 +1,179 @@
+"""Compiled per-criterion acceptance check orchestration extracted from Verifier."""
+
+from __future__ import annotations
+
+import logging
+from dataclasses import dataclass, field
+from typing import Any, Awaitable, Callable, Dict, List, Tuple
+
+from devcouncil.domain.evidence import CommandResult
+from devcouncil.domain.gap import Gap
+from devcouncil.domain.requirement import Requirement
+from devcouncil.domain.task import Task
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass
+class CompiledAcceptanceResult:
+    gaps: List[Gap] = field(default_factory=list)
+    compiled_pass: Dict[str, bool] = field(default_factory=dict)
+    compiled_cmds_by_ac: Dict[str, List[str]] = field(default_factory=dict)
+    failing_results_by_ac: Dict[str, List[CommandResult]] = field(default_factory=dict)
+    compiled_vote: Dict[str, Tuple[int, int, bool]] = field(default_factory=dict)
+    inconclusive_acs: set[str] = field(default_factory=set)
+    command_results: List[CommandResult] = field(default_factory=list)
+    evidence: List[Any] = field(default_factory=list)
+    genuine_failure: bool = False
+    had_unrunnable: bool = False
+
+
+async def run_compiled_acceptance_checks(
+    *,
+    task: Task,
+    requirements: List[Requirement],
+    compile_future: Any | None,
+    diff_content: str,
+    ac_repair_attempts: int,
+    acceptance_compiler: Any | None,
+    command_applicable: Callable[[str], Tuple[bool, str]],
+    run_command: Callable[[str], CommandResult],
+    command_is_malformed: Callable[[CommandResult], bool],
+    failure_location: Callable[[CommandResult], Tuple[str | None, int | None]],
+    next_gap_id: Callable[[str, str], str],
+    repair_command: Callable[
+        [str, str, str, str, str],
+        Awaitable[str | None],
+    ] | None = None,
+    had_unrunnable: bool = False,
+    genuine_failure: bool = False,
+) -> CompiledAcceptanceResult:
+    """Run DevCouncil-owned per-criterion checks from the acceptance compiler."""
+    result = CompiledAcceptanceResult(
+        had_unrunnable=had_unrunnable,
+        genuine_failure=genuine_failure,
+    )
+    if compile_future is None:
+        return result
+
+    try:
+        compiled = await compile_future
+    except Exception as exc:  # pragma: no cover - best effort
+        logger.warning("Acceptance compiler failed for %s: %s", task.id, exc)
+        compiled = {}
+
+    ac_meta = {ac.id: ac for req in requirements for ac in req.acceptance_criteria}
+    for ac_id, raw_cmds in compiled.items():
+        candidates = [c for c in raw_cmds if command_applicable(c)[0]]
+        result.compiled_cmds_by_ac[ac_id] = list(candidates)
+        if not candidates:
+            result.had_unrunnable = True
+            continue
+
+        passes = 0
+        genuine_fails = 0
+        repaired = False
+        fail_results: List[Tuple[str, CommandResult]] = []
+        for cmd in candidates:
+            cmd_result = run_command(cmd)
+            result.command_results.append(cmd_result)
+            result.evidence.append(cmd_result)
+            attempts = 0
+            while (
+                cmd_result.exit_code != 0
+                and command_is_malformed(cmd_result)
+                and attempts < ac_repair_attempts
+                and repair_command is not None
+            ):
+                attempts += 1
+                ac_desc = ac_meta[ac_id].description if ac_id in ac_meta else ac_id
+                try:
+                    fixed = await repair_command(
+                        ac_id, ac_desc, cmd, cmd_result.summary[:800], diff_content
+                    )
+                except Exception:
+                    fixed = None
+                if not fixed or not command_applicable(fixed)[0]:
+                    break
+                cmd = fixed
+                result.compiled_cmds_by_ac[ac_id].append(cmd)
+                cmd_result = run_command(cmd)
+                result.command_results.append(cmd_result)
+                result.evidence.append(cmd_result)
+            if cmd_result.exit_code == 0:
+                passes += 1
+                if attempts > 0:
+                    repaired = True
+            elif command_is_malformed(cmd_result):
+                result.had_unrunnable = True
+                result.failing_results_by_ac.setdefault(ac_id, []).append(cmd_result)
+            else:
+                genuine_fails += 1
+                fail_results.append((cmd, cmd_result))
+                result.failing_results_by_ac.setdefault(ac_id, []).append(cmd_result)
+
+        decisive = passes + genuine_fails
+        if decisive == 0:
+            continue
+
+        ac_proven = passes > genuine_fails
+        result.compiled_pass[ac_id] = ac_proven
+        if ac_proven:
+            result.compiled_vote[ac_id] = (passes, decisive, repaired)
+            continue
+        if passes == 0 and genuine_fails > 0:
+            result.genuine_failure = True
+            cmd, cmd_result = fail_results[0]
+            fail_file, fail_line = failure_location(cmd_result)
+            agree = (
+                f" {genuine_fails}/{decisive} independent checks agreed it fails."
+                if decisive > 1 else ""
+            )
+            result.gaps.append(Gap(
+                id=next_gap_id(task.id, f"ACCHK-{ac_id}"),
+                severity="high",
+                gap_type="test_failed",
+                task_id=task.id,
+                description=(
+                    f"Acceptance check for {ac_id} failed: '{cmd}' "
+                    f"(exit {cmd_result.exit_code}).{agree}"
+                ),
+                evidence=[cmd_result.summary[:500]],
+                recommended_fix=f"Fix the implementation so acceptance criterion {ac_id} holds.",
+                blocking=True,
+                acceptance_criterion_id=ac_id,
+                suggested_command=cmd,
+                file=fail_file,
+                line=fail_line,
+                stdout_path=cmd_result.stdout_path or None,
+                stderr_path=cmd_result.stderr_path or None,
+            ))
+        elif passes > 0 and genuine_fails > 0:
+            result.inconclusive_acs.add(ac_id)
+
+    return result
+
+
+def promote_demoted_failures_when_compiler_incomplete(
+    *,
+    task: Task,
+    compiler_active: bool,
+    compiled_cmds_by_ac: Dict[str, List[str]],
+    demoted_failures: List[Gap],
+    genuine_failure: bool,
+) -> bool:
+    """Re-promote demoted planner test failures when compile coverage is partial."""
+    compiler_covered_all = bool(task.acceptance_criterion_ids) and all(
+        compiled_cmds_by_ac.get(ac_id) for ac_id in task.acceptance_criterion_ids
+    )
+    if compiler_active and not compiler_covered_all and demoted_failures:
+        for gap in demoted_failures:
+            gap.blocking = True
+            gap.severity = "high"
+            genuine_failure = True
+            logger.info(
+                "Re-promoted demoted test failure %s to blocking: acceptance compiler "
+                "did not produce a check for every criterion of task %s.",
+                gap.id, task.id,
+            )
+    return genuine_failure

package/src/devcouncil/verification/checks/corpus_stale.py

@@ -0,0 +1,124 @@
+"""Corpus freshness gate for tasks touching docs/corpus paths."""
+
+from __future__ import annotations
+
+import logging
+from pathlib import Path
+from typing import Callable, List, Optional
+
+from devcouncil.domain.gap import Gap
+from devcouncil.domain.task import Task
+
+logger = logging.getLogger(__name__)
+
+
+def _norm(path: str) -> str:
+    return path.replace("\\", "/").lstrip("./")
+
+
+def _task_touches_corpus(
+    *,
+    task: Task,
+    changed_files: List[str],
+    corpus_paths: List[str],
+) -> bool:
+    corpus_norm = {_norm(p) for p in corpus_paths}
+    for pf in task.planned_files:
+        rel = _norm(pf.path)
+        if rel in corpus_norm or any(rel.startswith(c + "/") for c in corpus_norm):
+            return True
+        if rel.endswith(".md") or rel.startswith("docs/"):
+            return True
+    for path in changed_files:
+        rel = _norm(path)
+        if rel in corpus_norm or any(rel.startswith(c + "/") for c in corpus_norm):
+            return True
+        if rel.endswith(".md") or rel.startswith("docs/"):
+            return True
+    return False
+
+
+def _corpus_is_stale(project_root: Path) -> tuple[bool, str]:
+    try:
+        from devcouncil.indexing.wiring import corpus_graph_path, load_corpus_settings
+        from devcouncil.indexing.repo_mapper import RepoMapper
+
+        settings = load_corpus_settings(project_root)
+        if not settings.enabled:
+            return False, "corpus disabled"
+        graph_path = corpus_graph_path(project_root)
+        if not graph_path.is_file():
+            return True, "corpus graph missing"
+        mapper = RepoMapper(project_root)
+        map_path = project_root / ".devcouncil" / "repo_map.json"
+        if map_path.is_file():
+            from devcouncil.utils.json_persist import read_json
+
+            loaded = read_json(map_path)
+            if isinstance(loaded, dict) and mapper.map_is_stale(loaded):
+                return True, "repo map stale vs corpus fingerprint"
+        return False, "fresh"
+    except Exception as exc:
+        logger.debug("corpus stale check failed: %s", exc)
+        return True, "corpus check error"
+
+
+def detect_corpus_stale_gaps(
+    *,
+    task: Task,
+    project_root: Path,
+    changed_files: List[str],
+    next_gap_id: Callable[[str, str], str],
+    corpus_stale_enabled: bool = True,
+    corpus_stale_blocking: bool = False,
+    corpus_paths: Optional[List[str]] = None,
+) -> List[Gap]:
+    if not corpus_stale_enabled:
+        return []
+    try:
+        from devcouncil.indexing.wiring import load_corpus_settings
+
+        settings = load_corpus_settings(project_root)
+        paths = corpus_paths if corpus_paths is not None else list(settings.paths)
+        if not _task_touches_corpus(task=task, changed_files=changed_files, corpus_paths=paths):
+            return []
+        stale, reason = _corpus_is_stale(project_root)
+        if not stale:
+            return []
+        return [Gap(
+            id=next_gap_id(task.id, "CORPUSSTALE"),
+            severity="high" if corpus_stale_blocking else "medium",
+            gap_type="corpus_stale",
+            task_id=task.id,
+            description=(
+                f"Corpus index is stale or missing ({reason}). "
+                "Doc/corpus navigation and doc-code-ref gates may be wrong."
+            ),
+            evidence=[".devcouncil/corpus/graph.json", reason],
+            recommended_fix="Run `dev map` (corpus rebuild) or enable auto_refresh_on_verify.",
+            blocking=corpus_stale_blocking,
+            suggested_command="dev map",
+        )]
+    except Exception:
+        logger.debug("detect_corpus_stale_gaps failed", exc_info=True)
+        return []
+
+
+def refresh_corpus_on_verify_if_needed(project_root: Path) -> bool:
+    """Best-effort corpus rebuild when enabled and stale."""
+    try:
+        from devcouncil.indexing.wiring import load_corpus_settings
+
+        settings = load_corpus_settings(project_root)
+        if not settings.enabled or not settings.auto_refresh_on_verify:
+            return False
+        stale, _ = _corpus_is_stale(project_root)
+        if not stale:
+            return False
+        from devcouncil.indexing.wiring import build_corpus
+
+        build_corpus(project_root)
+        return True
+    except Exception:
+        logger.debug("corpus verify refresh failed", exc_info=True)
+        return False

package/src/devcouncil/verification/checks/corpus_verification.py

@@ -0,0 +1,9 @@
+"""Corpus verification gate facade (re-exports split gate modules)."""
+
+from __future__ import annotations
+
+from devcouncil.verification.checks.acceptance_corpus import detect_acceptance_corpus_gaps
+from devcouncil.verification.checks.corpus_stale import detect_corpus_stale_gaps
+from devcouncil.verification.checks.doc_code_ref import detect_doc_code_ref_gaps
+
+__all__ = ["detect_acceptance_corpus_gaps", "detect_corpus_stale_gaps", "detect_doc_code_ref_gaps"]

package/src/devcouncil/verification/checks/dead_symbols.py

@@ -0,0 +1,360 @@
+"""Dead-symbol gate: flag newly added public top-level symbols nothing references.
+
+Diff-scoped via ``added_lines_by_file`` + AST/span intersection. Test references
+and intent-text naming clear a symbol; files already flagged ``unwired_file`` are
+skipped. Never raises.
+"""
+
+from __future__ import annotations
+
+import ast
+import logging
+import re
+from pathlib import Path
+from typing import Callable, List, Optional, Set, Tuple
+
+from devcouncil.domain.gap import Gap
+from devcouncil.domain.requirement import Requirement
+from devcouncil.domain.task import Task
+from devcouncil.indexing.wiring import (
+    ALLOW_UNWIRED,
+    decorator_names,
+    is_liveness_code_file,
+    is_private_symbol,
+    is_test_path,
+    is_vendored_path,
+    is_wiring_decorated,
+    iter_js_export_symbols,
+    parse_python_all_exports,
+    parse_python_reexport_names,
+    strip_js_comments,
+    strip_py_comments,
+    strip_string_literals,
+)
+from devcouncil.verification.checks.semantic_diff import task_intent_text
+from devcouncil.verification.stub_detector import added_lines_by_file, task_allows_scaffolding
+
+logger = logging.getLogger(__name__)
+
+_IDENT_RE = re.compile(r"\b([A-Za-z_][A-Za-z0-9_]*)\b")
+_CODE_SUFFIXES = {".py", ".ts", ".tsx", ".js", ".jsx", ".mjs", ".cjs"}
+
+# (path, start_line, end_line, name)
+_SymbolCand = Tuple[str, int, int, str]
+
+
+def _norm(path: str) -> str:
+    s = path.replace("\\", "/")
+    while s.startswith("./"):
+        s = s[2:]
+    return s
+
+
+def _python_candidates(
+    project_root: Path,
+    path: str,
+    added_lines: Set[int],
+) -> List[_SymbolCand]:
+    """Return (path, start, end, name) for public top-level defs intersecting added lines."""
+    try:
+        source = (project_root / path).read_text(encoding="utf-8", errors="replace")
+        tree = ast.parse(source)
+    except Exception:
+        return []
+    all_exports = parse_python_all_exports(source)
+    reexports = parse_python_reexport_names(path, source)
+    protected = all_exports | reexports
+    out: List[_SymbolCand] = []
+    for node in tree.body:
+        if not isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)):
+            continue
+        name = node.name
+        if is_private_symbol(name):
+            continue
+        # ``__all__`` / barrel re-exports = public API surface (parity with graph).
+        if name in protected:
+            continue
+        start = getattr(node, "lineno", None)
+        end = getattr(node, "end_lineno", start) or start
+        if start is None:
+            continue
+        span = set(range(start, (end or start) + 1))
+        if not span & added_lines:
+            continue
+        if is_wiring_decorated(decorator_names(node)):
+            continue
+        out.append((path, start, end or start, name))
+    return out
+
+
+def _js_definition_span(source: str, name: str) -> Optional[Tuple[int, int]]:
+    """Best-effort start/end lines for a JS/TS binding named ``name``."""
+    patterns = (
+        rf"(?m)^\s*(?:export\s+)?(?:async\s+)?function\s+{re.escape(name)}\b",
+        rf"(?m)^\s*(?:export\s+)?class\s+{re.escape(name)}\b",
+        rf"(?m)^\s*(?:export\s+)?(?:const|let|var)\s+{re.escape(name)}\b",
+    )
+    for pat in patterns:
+        m = re.search(pat, source)
+        if m:
+            line = source[: m.start()].count("\n") + 1
+            # Single-line approx is enough for token outside-span checks.
+            return line, line
+    return None
+
+
+def _js_candidates(
+    project_root: Path,
+    path: str,
+    added_lines: Set[int],
+) -> List[_SymbolCand]:
+    try:
+        source = (project_root / path).read_text(encoding="utf-8", errors="replace")
+    except OSError:
+        return []
+    out: List[_SymbolCand] = []
+    seen: Set[str] = set()
+    export_hits = iter_js_export_symbols(source)
+    for line, name in export_hits:
+        if name in seen:
+            continue
+        span = _js_definition_span(source, name)
+        if span is not None:
+            start, end = span
+        else:
+            start = end = line
+        # Fold export-list lines into the defining span so `export { name }` is
+        # not treated as an external reference that clears the symbol.
+        for eline, ename in export_hits:
+            if ename == name:
+                start = min(start, eline)
+                end = max(end, eline)
+        if line not in added_lines and not (set(range(start, end + 1)) & added_lines):
+            continue
+        lines = source.splitlines()
+        if 0 < line <= len(lines):
+            prev_lines = lines[: line - 1]
+            if prev_lines:
+                prev = prev_lines[-1].strip()
+                if prev.startswith("@"):
+                    continue
+        seen.add(name)
+        out.append((path, start, end, name))
+    return out
+
+
+def _build_token_index(
+    project_root: Path,
+    files: List[str],
+    *,
+    exclude: Set[str],
+) -> tuple[dict[str, Set[str]], dict[str, dict[str, Set[int]]]]:
+    """Map identifier token -> files, and token -> file -> line numbers."""
+    index: dict[str, Set[str]] = {}
+    lines_index: dict[str, dict[str, Set[int]]] = {}
+    try:
+        for rel in files:
+            norm = _norm(rel)
+            if norm in exclude:
+                continue
+            if is_vendored_path(norm):
+                continue
+            if Path(norm).suffix.lower() not in _CODE_SUFFIXES:
+                continue
+            path = project_root / norm
+            if not path.is_file():
+                continue
+            try:
+                text = path.read_text(encoding="utf-8", errors="replace")
+            except OSError:
+                continue
+            if path.suffix.lower() == ".py":
+                cleaned = strip_string_literals(strip_py_comments(text))
+            else:
+                cleaned = strip_string_literals(strip_js_comments(text))
+            for lineno, line in enumerate(cleaned.splitlines(), 1):
+                for tok in _IDENT_RE.findall(line):
+                    if len(tok) < 2:
+                        continue
+                    index.setdefault(tok, set()).add(norm)
+                    lines_index.setdefault(tok, {}).setdefault(norm, set()).add(lineno)
+    except Exception:
+        logger.debug("token index build failed", exc_info=True)
+    return index, lines_index
+
+
+def _symbol_is_referenced(
+    name: str,
+    path: str,
+    start: int,
+    end: int,
+    token_index: dict[str, Set[str]],
+    lines_index: dict[str, dict[str, Set[int]]],
+) -> bool:
+    """True when ``name`` is used outside its defining span (other file or same-file)."""
+    refs = token_index.get(name, set())
+    if refs - {path}:
+        return True
+    same_lines = lines_index.get(name, {}).get(path, set())
+    return any(ln < start or ln > end for ln in same_lines)
+
+
+def detect_dead_symbol_gaps(
+    *,
+    task: Task,
+    project_root: Path,
+    diff_content: str,
+    next_gap_id: Callable[[str, str], str],
+    dead_symbol_enabled: bool = True,
+    dead_symbol_blocking: bool = False,
+    requirements: Optional[List[Requirement]] = None,
+    unwired_files: Optional[Set[str]] = None,
+    git_files: Optional[List[str]] = None,
+    lsp_refs: Optional[bool] = None,
+) -> List[Gap]:
+    """Flag diff-added public top-level symbols with zero external references.
+
+    When ``lsp_refs`` is True (or config ``indexing.lsp_refs`` when ``None``),
+    token-scan dead candidates are confirmed via the optional live LSP client
+    before becoming gaps — external references clear false positives.
+    """
+    gaps: List[Gap] = []
+    if not dead_symbol_enabled or not diff_content:
+        return gaps
+    try:
+        by_file = added_lines_by_file(diff_content)
+        if not by_file:
+            return gaps
+
+        scaffolding_ok = task_allows_scaffolding(task)
+        intent = task_intent_text(task, requirements) if requirements is not None else (
+            f"{task.title} {task.description}"
+        )
+        unwired = {_norm(p) for p in (unwired_files or set())}
+
+        candidates: List[_SymbolCand] = []
+        for path, added in by_file.items():
+            norm = _norm(path)
+            if not is_liveness_code_file(norm) or is_test_path(norm):
+                continue
+            if norm in unwired:
+                continue
+            line_nums = {ln for ln, _ in added}
+            if Path(norm).suffix.lower() == ".py":
+                candidates.extend(_python_candidates(project_root, norm, line_nums))
+            else:
+                candidates.extend(_js_candidates(project_root, norm, line_nums))
+
+        if not candidates:
+            return gaps
+
+        candidates.sort(key=lambda item: (item[0], item[1], item[3]))
+
+        if git_files is None:
+            try:
+                from devcouncil.indexing.repo_mapper import RepoMapper
+
+                tracked = RepoMapper(project_root).get_git_files()
+            except Exception:
+                tracked = []
+        else:
+            tracked = list(git_files)
+        token_index, lines_index = _build_token_index(project_root, tracked, exclude=set())
+
+        use_lsp = lsp_refs
+        if use_lsp is None:
+            try:
+                from devcouncil.indexing.lsp_client import lsp_refs_enabled
+
+                use_lsp = lsp_refs_enabled(project_root)
+            except Exception:
+                use_lsp = False
+        lsp_pool = None
+        if use_lsp:
+            try:
+                from devcouncil.indexing.lsp_client import LspSessionPool
+
+                lsp_pool = LspSessionPool(project_root)
+            except Exception:
+                lsp_pool = None
+
+        try:
+            for path, start, end, name in candidates:
+                # Intent-text naming = deliberate API addition.
+                if re.search(rf"\b{re.escape(name)}\b", intent or ""):
+                    continue
+
+                # allow-unwired on the defining line/file with scaffolding parity.
+                try:
+                    source = (project_root / path).read_text(encoding="utf-8", errors="replace")
+                    src_lines = source.splitlines()
+                    line_text = src_lines[start - 1] if 0 < start <= len(src_lines) else ""
+                except OSError:
+                    line_text = ""
+                    source = ""
+                if ALLOW_UNWIRED in line_text or ALLOW_UNWIRED in source:
+                    gaps.append(Gap(
+                        id=next_gap_id(task.id, f"DEADDECL-{path}:{start}:{name}"),
+                        severity="medium",
+                        gap_type="dead_symbol",
+                        task_id=task.id,
+                        description=(
+                            f"Intentional unused symbol `{name}` declared at {path}:{start}."
+                        ),
+                        evidence=[f"{path}:{start}", f"symbol:{name}", ALLOW_UNWIRED],
+                        recommended_fix=(
+                            f"Wire `{name}` into its caller when scaffolding is complete, "
+                            "or remove it."
+                        ),
+                        blocking=False,
+                        file=path,
+                        line=start,
+                    ))
+                    if scaffolding_ok:
+                        continue
+
+                if _symbol_is_referenced(name, path, start, end, token_index, lines_index):
+                    continue
+
+                try:
+                    from devcouncil.indexing.graph.query import symbol_has_non_test_inbound
+
+                    if symbol_has_non_test_inbound(project_root, path, name):
+                        continue
+                except Exception:
+                    logger.debug("graph inbound check failed for %s", name, exc_info=True)
+
+                if lsp_pool is not None:
+                    try:
+                        confirmed = lsp_pool.confirm_unreferenced(path, start, name)
+                        if confirmed is False:
+                            continue
+                    except Exception:
+                        logger.debug("LSP dead-symbol confirm failed for %s", name, exc_info=True)
+
+                gaps.append(Gap(
+                    id=next_gap_id(task.id, f"DEAD-{path}:{start}:{name}"),
+                    severity="high" if dead_symbol_blocking else "medium",
+                    gap_type="dead_symbol",
+                    task_id=task.id,
+                    description=(
+                        f"New public symbol `{name}` at {path}:{start} is never referenced "
+                        "outside its own definition."
+                    ),
+                    evidence=[f"{path}:{start}", f"symbol:{name}"],
+                    recommended_fix=(
+                        f"Call or register `{name}` from the code that needs it "
+                        f"(use `dev scope update {task.id} --lease-token <token> "
+                        f"--planned-file <caller>` if the caller is out of scope), or remove it."
+                    ),
+                    blocking=dead_symbol_blocking,
+                    file=path,
+                    line=start,
+                ))
+        finally:
+            if lsp_pool is not None:
+                lsp_pool.close()
+    except Exception:
+        logger.debug("detect_dead_symbol_gaps failed; degrading to zero gaps", exc_info=True)
+        return []
+    return gaps