devcouncil 0.3.1 → 0.4.1

package/src/devcouncil/verification/checks/acceptance_corpus.py

@@ -0,0 +1,194 @@
+"""Acceptance-criteria ↔ corpus alignment gate (soft by default)."""
+
+from __future__ import annotations
+
+import logging
+import re
+from pathlib import Path
+from typing import Callable, List, Optional, Set
+
+from devcouncil.domain.gap import Gap
+from devcouncil.domain.requirement import Requirement
+from devcouncil.domain.task import Task
+
+logger = logging.getLogger(__name__)
+
+_DOC_PATH_RE = re.compile(
+    r"(?<![`\w])(?:docs/[\w./-]+\.(?:md|rst|txt)|README(?:\.md)?)(?![`\w])",
+    re.IGNORECASE,
+)
+_DOC_CONCEPT_RE = re.compile(
+    r"\b(?:documentation|corpus|wiki|architecture doc(?:umentation)?|user guide|dev guide)\b",
+    re.IGNORECASE,
+)
+_EVIDENCE_PATH_RE = re.compile(
+    r"(?:evidence|see|per|from)\s*:?\s*[`']?((?:docs/|README)[\w./-]*(?:\.md)?)[`']?",
+    re.IGNORECASE,
+)
+_BACKTICK_TERM_RE = re.compile(r"`([^`]{3,80})`")
+
+
+def _norm(path: str) -> str:
+    return path.replace("\\", "/").lstrip("./")
+
+
+def _ac_by_id(requirements: List[Requirement], ac_id: str):
+    for req in requirements:
+        for ac in req.acceptance_criteria:
+            if ac.id == ac_id:
+                return ac, req.id
+    return None, None
+
+
+def _doc_paths_in_text(text: str) -> List[str]:
+    paths: List[str] = []
+    for match in _DOC_PATH_RE.finditer(text):
+        paths.append(_norm(match.group(0)))
+    for match in _EVIDENCE_PATH_RE.finditer(text):
+        paths.append(_norm(match.group(1)))
+    return list(dict.fromkeys(paths))
+
+
+def _cites_doc_concepts(description: str) -> bool:
+    if _DOC_PATH_RE.search(description):
+        return True
+    if _DOC_CONCEPT_RE.search(description):
+        return True
+    for match in _BACKTICK_TERM_RE.finditer(description):
+        term = match.group(1)
+        if term.startswith("docs/") or term.endswith(".md") or "documentation" in term.lower():
+            return True
+    return False
+
+
+def _corpus_terms(description: str, doc_paths: List[str]) -> List[str]:
+    terms: List[str] = []
+    for path in doc_paths:
+        stem = Path(path).stem.replace("-", " ").replace("_", " ")
+        if stem and stem.lower() != "readme":
+            terms.append(stem)
+        terms.append(path)
+    for match in _BACKTICK_TERM_RE.finditer(description):
+        term = match.group(1).strip()
+        if len(term) >= 3 and not term.startswith("src/"):
+            terms.append(term)
+    if _DOC_CONCEPT_RE.search(description):
+        for word in re.findall(r"[A-Za-z]{4,}", description):
+            if word.lower() not in {"documentation", "acceptance", "criterion", "criteria", "verify", "behavior", "matches", "section"}:
+                terms.append(word)
+    # Residual phrase tokens from the description (e.g. "verification gates").
+    for chunk in re.findall(r"[A-Za-z][A-Za-z0-9 -]{2,40}", description):
+        stripped = chunk.strip()
+        if " " in stripped and len(stripped) >= 8:
+            terms.append(stripped)
+    return list(dict.fromkeys(t for t in terms if t.strip()))
+
+
+def _corpus_has_hit(project_root: Path, terms: List[str], doc_paths: List[str]) -> bool:
+    if not terms and not doc_paths:
+        return False
+    try:
+        from devcouncil.indexing.wiring import load_corpus_graph, query_corpus
+
+        graph = load_corpus_graph(project_root)
+        if graph is not None:
+            doc_norm = {_norm(p) for p in doc_paths}
+            for node in graph.nodes:
+                node_path = _norm(node.path or "")
+                if node_path in doc_norm or any(
+                    node_path.endswith(p) for p in doc_norm if p
+                ):
+                    return True
+        for term in terms[:8]:
+            result = query_corpus(project_root, term, limit=3)
+            if result.get("matches"):
+                return True
+    except Exception:
+        logger.debug("corpus query failed during acceptance_corpus check", exc_info=True)
+    return False
+
+
+def _explicit_evidence_path(
+    *,
+    project_root: Path,
+    doc_paths: List[str],
+    changed_files: List[str],
+    planned_paths: Set[str],
+    expected_tests: List[str],
+    diff_content: str,
+) -> Optional[str]:
+    changed_norm = {_norm(p) for p in changed_files}
+    tests_blob = "\n".join(expected_tests)
+    for path in doc_paths:
+        rel = _norm(path)
+        full = project_root / rel
+        if not full.is_file():
+            continue
+        if rel in changed_norm or rel in planned_paths:
+            return rel
+        if rel in diff_content or rel in tests_blob:
+            return rel
+    return None
+
+
+def detect_acceptance_corpus_gaps(
+    *,
+    task: Task,
+    requirements: List[Requirement],
+    project_root: Path,
+    changed_files: List[str],
+    diff_content: str,
+    next_gap_id: Callable[[str, str], str],
+    acceptance_corpus_enabled: bool = True,
+    acceptance_corpus_blocking: bool = False,
+) -> List[Gap]:
+    if not acceptance_corpus_enabled or not task.acceptance_criterion_ids:
+        return []
+    gaps: List[Gap] = []
+    planned_paths = {_norm(pf.path) for pf in task.planned_files}
+    try:
+        for ac_id in task.acceptance_criterion_ids:
+            ac, req_id = _ac_by_id(requirements, ac_id)
+            if ac is None:
+                continue
+            description = ac.description or ""
+            if not _cites_doc_concepts(description):
+                continue
+            doc_paths = _doc_paths_in_text(description)
+            terms = _corpus_terms(description, doc_paths)
+            if _corpus_has_hit(project_root, terms, doc_paths):
+                continue
+            evidence_path = _explicit_evidence_path(
+                project_root=project_root,
+                doc_paths=doc_paths,
+                changed_files=changed_files,
+                planned_paths=planned_paths,
+                expected_tests=list(task.expected_tests or []),
+                diff_content=diff_content or "",
+            )
+            if evidence_path:
+                continue
+            gaps.append(Gap(
+                id=next_gap_id(task.id, f"ACCORPUS-{ac_id}"),
+                severity="high" if acceptance_corpus_blocking else "medium",
+                gap_type="acceptance_corpus",
+                task_id=task.id,
+                requirement_id=req_id,
+                acceptance_criterion_id=ac_id,
+                description=(
+                    f"Acceptance criterion `{ac_id}` cites documentation concepts but "
+                    "has no corpus match and no explicit evidence path in the task diff, "
+                    "planned files, or expected tests."
+                ),
+                evidence=[ac_id, description[:200], *(doc_paths[:3])],
+                recommended_fix=(
+                    "Add a corpus hit (run `dev corpus build` and align docs), cite an "
+                    "existing doc path in planned/changed files, or add an explicit "
+                    "`evidence: docs/...` path with supporting changes."
+                ),
+                blocking=acceptance_corpus_blocking,
+                suggested_command="dev corpus build",
+            ))
+    except Exception:
+        logger.debug("detect_acceptance_corpus_gaps failed", exc_info=True)
+    return gaps

package/src/devcouncil/verification/checks/acceptance_evidence.py

@@ -0,0 +1,239 @@
+"""Acceptance-criteria evidence mapping extracted from Verifier.verify_task."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Callable, Dict, List, Literal, Optional, Set, Tuple
+
+from devcouncil.domain.evidence import CommandResult, TestEvidence
+from devcouncil.domain.gap import Gap
+from devcouncil.domain.requirement import Requirement
+from devcouncil.domain.task import Task
+from devcouncil.verification.command_evidence import command_is_trivial_evidence
+from devcouncil.verification.difficulty import RigorPolicy
+
+_AD_HOC_TASK_ID = "CHECK"
+
+
+@dataclass
+class AcceptanceEvidenceResult:
+    gaps: List[Gap] = field(default_factory=list)
+    evidence: List[TestEvidence] = field(default_factory=list)
+
+
+def requirement_id_for_ac(requirements: List[Requirement], ac_id: str) -> Optional[str]:
+    for req in requirements:
+        if any(ac.id == ac_id for ac in req.acceptance_criteria):
+            return req.id
+    return None
+
+
+def _declared_test_proves_ac(
+    task: Task,
+    ac_id: str,
+    successful_commands: List[CommandResult],
+) -> bool:
+    """True when a user/planner declared test command proves this criterion.
+
+    For ad-hoc ``CHECK`` runs, ``--test`` commands are criterion-specific by
+    construction (one inline AC). They must not fall through to the coarse
+    "any passing command" bucket.
+    """
+    declared = list(task.expected_tests or [])
+    if not declared:
+        return False
+    passing_declared = [r.command for r in successful_commands if r.command in declared]
+    if not passing_declared:
+        return False
+    # Only ad-hoc CHECK treats inline --test commands as per-criterion proof.
+    # Workflow tasks with a single expected_test still use the coarse bucket so
+    # hard-rigor coarse_acceptance_proof blocking remains meaningful.
+    return task.id == _AD_HOC_TASK_ID
+
+
+def map_acceptance_criteria_evidence(
+    *,
+    task: Task,
+    requirements: List[Requirement],
+    evidence_results: List[CommandResult],
+    compiled_pass: Dict[str, bool],
+    compiled_cmds_by_ac: Dict[str, List[str]],
+    failing_results_by_ac: Dict[str, List[CommandResult]],
+    compiled_vote: Dict[str, Tuple[int, int, bool]],
+    inconclusive_acs: Set[str],
+    work_present: bool,
+    genuine_failure: bool,
+    had_unrunnable: bool,
+    rigor: RigorPolicy,
+    is_quality_only_command: Callable[[str], bool],
+    next_gap_id: Callable[[str, str], str],
+) -> AcceptanceEvidenceResult:
+    """Map compiled + coarse verification evidence to gaps and TestEvidence records."""
+    result = AcceptanceEvidenceResult()
+    agent_appended = set(task.agent_appended_expected_tests or []) | set(
+        task.agent_appended_allowed_commands or []
+    )
+    successful_commands = [
+        item for item in evidence_results
+        if item.exit_code == 0
+        and not is_quality_only_command(item.command)
+        and item.command not in agent_appended
+        and not command_is_trivial_evidence(item.command)
+    ]
+    coarse_proof_available = work_present and bool(successful_commands)
+
+    if task.acceptance_criterion_ids:
+        req_by_ac = {ac.id: req.id for req in requirements for ac in req.acceptance_criteria}
+        unproven_acs: List[str] = []
+        coarse_proven_acs: List[str] = []
+        for ac_id in task.acceptance_criterion_ids:
+            proven: Optional[bool] = compiled_pass.get(ac_id)
+            coarse = False
+            if proven is None:
+                if _declared_test_proves_ac(task, ac_id, successful_commands):
+                    proven = True
+                else:
+                    proven = coarse_proof_available
+                    coarse = proven
+            if proven:
+                if coarse:
+                    coarse_proven_acs.append(ac_id)
+                if not (coarse and genuine_failure):
+                    proof_mode: Literal["compiled", "vote", "coarse", ""]
+                    if coarse:
+                        proof_summary = (
+                            "Acceptance criterion proven only by a COARSE signal (a passing "
+                            "acceptance-capable command, not a per-criterion check); behavior "
+                            "not precisely verified."
+                        )
+                        proof_mode = "coarse"
+                    elif task.id == _AD_HOC_TASK_ID and task.expected_tests:
+                        proof_mode = "compiled"
+                        proof_summary = (
+                            "Acceptance criterion proven by user-supplied verification "
+                            "command(s) for this check."
+                        )
+                    else:
+                        passes_n, decisive_n, was_repaired = compiled_vote.get(ac_id, (1, 1, False))
+                        proof_mode = "vote" if decisive_n > 1 else "compiled"
+                        how = (
+                            f"a majority vote of independent compiled checks ({passes_n}/{decisive_n} passed)"
+                            if decisive_n > 1 else
+                            "a per-criterion compiled check"
+                        )
+                        repaired_note = (
+                            " (one check was regenerated from its launcher error to run)"
+                            if was_repaired else ""
+                        )
+                        proof_summary = f"Acceptance criterion proven by {how}.{repaired_note}"
+                    result.evidence.append(TestEvidence(
+                        requirement_id=req_by_ac.get(
+                            ac_id, task.requirement_ids[0] if task.requirement_ids else ""
+                        ),
+                        acceptance_criterion_id=ac_id,
+                        command="(devcouncil acceptance check)",
+                        status="passed",
+                        evidence_summary=proof_summary,
+                        mode=proof_mode,
+                    ))
+            else:
+                unproven_acs.append(ac_id)
+
+        if coarse_proven_acs and rigor.coarse_acceptance_enabled:
+            result.gaps.append(Gap(
+                id=next_gap_id(task.id, "COARSE"),
+                severity="high" if rigor.coarse_acceptance_blocking else "low",
+                gap_type="coarse_acceptance_proof",
+                task_id=task.id,
+                description=(
+                    "Verification mode = COARSE for "
+                    f"{', '.join(coarse_proven_acs)}: proven by a passing acceptance-capable "
+                    "command, not a per-criterion check. Behavior is not precisely verified."
+                ),
+                evidence=[f"coarse-proven: {', '.join(coarse_proven_acs)}"],
+                recommended_fix=(
+                    "Add a verification command (or test) that exercises each listed criterion "
+                    "specifically, so DevCouncil can compile a per-criterion check instead of "
+                    "relying on the coarse fallback."
+                ),
+                blocking=rigor.coarse_acceptance_blocking,
+            ))
+
+        if unproven_acs:
+            couldnt_verify = had_unrunnable and not genuine_failure and work_present
+            ac_by_id = {ac.id: ac for req in requirements for ac in req.acceptance_criteria}
+            automatable_methods = {"unit_test", "integration_test"}
+            for ac_id in unproven_acs:
+                ac = ac_by_id.get(ac_id)
+                method = ac.verification_method if ac else "unit_test"
+                is_automatable = (ac.required if ac else True) and method in automatable_methods
+                if not is_automatable:
+                    blocks = False
+                    optional = "" if (ac is None or ac.required) else " optional"
+                    fix = (
+                        f"This{optional} criterion's verification method is '{method}'; it cannot be "
+                        "proven by running code. Review it manually (it does not block the gate)."
+                    )
+                    suffix = f" (non-blocking: {method})"
+                elif ac_id in inconclusive_acs:
+                    blocks = False
+                    fix = (
+                        "Auto-generated acceptance checks disagreed on this criterion (some "
+                        "passed, some failed). Add a precise verification command that "
+                        "unambiguously proves it so the result is decisive."
+                    )
+                    suffix = " (auto-checks inconclusive)"
+                elif couldnt_verify:
+                    blocks = False
+                    fix = (
+                        "Could not verify this criterion: the verification commands did not run "
+                        "(missing tooling or tests). Regenerate them with 'dev repair' to confirm the work."
+                    )
+                    suffix = " (verification commands could not run)"
+                else:
+                    blocks = True
+                    fix = "Add or fix a verification command that proves this acceptance criterion."
+                    suffix = ""
+                ac_compiled = compiled_cmds_by_ac.get(ac_id, [])
+                ac_failures = failing_results_by_ac.get(ac_id, [])
+                ac_evidence: List[str] = []
+                suggested_cmd: Optional[str] = None
+                if ac_compiled:
+                    suggested_cmd = ac_compiled[0]
+                    ac_evidence.extend(f"compiled check: {c}" for c in ac_compiled)
+                    ac_evidence.extend(r.summary[:500] for r in ac_failures)
+                else:
+                    ac_evidence.append(
+                        f"no DevCouncil check compiled for {ac_id} "
+                        f"(expected verification method: {method})"
+                    )
+                result.gaps.append(Gap(
+                    id=next_gap_id(task.id, f"AC-{ac_id}"),
+                    severity="high" if blocks else "medium",
+                    gap_type="acceptance_criteria_unproven",
+                    requirement_id=requirement_id_for_ac(requirements, ac_id),
+                    task_id=task.id,
+                    description=(
+                        f"Acceptance criterion {ac_id} has no passing verification evidence "
+                        f"for task {task.id}.{suffix}"
+                    ),
+                    evidence=ac_evidence,
+                    recommended_fix=fix,
+                    blocking=blocks,
+                    acceptance_criterion_id=ac_id,
+                    expected_verification_method=method,
+                    suggested_command=suggested_cmd,
+                ))
+    elif task.requirement_ids:
+        result.gaps.append(Gap(
+            id=next_gap_id(task.id, "NOAC"),
+            severity="high",
+            gap_type="acceptance_criteria_unproven",
+            requirement_id=task.requirement_ids[0],
+            task_id=task.id,
+            description=f"Task {task.id} is linked to requirements but no acceptance criteria.",
+            recommended_fix="Link the task to specific acceptance_criterion_ids before verification.",
+            blocking=True,
+        ))
+
+    return result

package/src/devcouncil/verification/checks/command_evidence.py

@@ -0,0 +1,148 @@
+"""Expected verification command execution extracted from Verifier."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Callable, Dict, List, Tuple
+
+from devcouncil.domain.evidence import CommandResult
+from devcouncil.domain.gap import Gap
+from devcouncil.domain.task import Task
+
+
+@dataclass
+class CommandEvidenceRunResult:
+    gaps: List[Gap] = field(default_factory=list)
+    command_results: List[CommandResult] = field(default_factory=list)
+    evidence_results: List[CommandResult] = field(default_factory=list)
+    genuine_failure: bool = False
+    had_unrunnable: bool = False
+    demoted_failures: List[Gap] = field(default_factory=list)
+
+
+def run_verification_commands(
+    *,
+    task: Task,
+    commands_for_task: Dict[str, List[str]],
+    compiler_active: bool,
+    command_applicable: Callable[[str], Tuple[bool, str]],
+    run_command: Callable[[str], CommandResult],
+    command_can_prove_acceptance: Callable[[str, str], bool],
+    command_is_malformed: Callable[[CommandResult], bool],
+    failure_location: Callable[[CommandResult], Tuple[str | None, int | None]],
+    is_quality_only_command: Callable[[str], bool],
+    next_gap_id: Callable[[str, str], str],
+    retry_flaky: bool = True,
+) -> CommandEvidenceRunResult:
+    """Run planner/config verification commands and collect gaps + evidence.
+
+    With ``retry_flaky`` (config ``verification.retry_flaky``, default on), a genuinely
+    failing acceptance-capable command gets ONE immediate re-run; a pass on the re-run
+    counts as passed with its summary tagged ``[flaky: passed on retry 2/2]`` so
+    next-actions/reports can distinguish a flaky pass from a stable one.
+    """
+    result = CommandEvidenceRunResult()
+    for cmd_type, cmds in commands_for_task.items():
+        for cmd in cmds:
+            applicable, skip_reason = command_applicable(cmd)
+            if not applicable:
+                result.gaps.append(Gap(
+                    id=next_gap_id(task.id, f"SKIP-{cmd}"),
+                    severity="low",
+                    gap_type="skipped_verification_command",
+                    task_id=task.id,
+                    description=f"Skipped verification command '{cmd}': {skip_reason}.",
+                    evidence=[skip_reason],
+                    recommended_fix=(
+                        "Replace it with a command for this repo's stack, or remove it "
+                        "from .devcouncil/config.yaml / the task's expected_tests."
+                    ),
+                    blocking=False,
+                    suggested_command=cmd,
+                ))
+                continue
+            cmd_result = run_command(cmd)
+            can_prove = command_can_prove_acceptance(cmd_type, cmd)
+            if (
+                retry_flaky
+                and can_prove
+                and cmd_result.exit_code > 0
+                and not cmd_result.timed_out
+                and not command_is_malformed(cmd_result)
+            ):
+                # Flaky-evidence retry: one immediate re-run of a genuinely failing
+                # acceptance-capable command. Only exit codes > 0 qualify — the
+                # synthetic -1 (unlaunchable / timed out) and malformed commands
+                # prove nothing and would fail identically or burn a second full
+                # timeout window. On a failed re-run the ORIGINAL result is kept.
+                retry_result = run_command(cmd)
+                if retry_result.exit_code == 0:
+                    cmd_result = retry_result.model_copy(update={
+                        "summary": f"[flaky: passed on retry 2/2] {retry_result.summary}",
+                    })
+            result.command_results.append(cmd_result)
+            if can_prove:
+                result.evidence_results.append(cmd_result)
+            if cmd_result.exit_code != 0:
+                if command_is_malformed(cmd_result):
+                    result.had_unrunnable = True
+                    result.gaps.append(Gap(
+                        id=next_gap_id(task.id, f"BADCMD-{cmd}"),
+                        severity="medium",
+                        gap_type="invalid_verification_command",
+                        task_id=task.id,
+                        description=(
+                            f"Verification command could not run (not a code failure): '{cmd}'. "
+                            "It appears malformed or its tooling is unavailable, so this command "
+                            "proves nothing either way."
+                        ),
+                        evidence=[cmd_result.summary[:500]],
+                        recommended_fix=(
+                            "Regenerate the task's verification commands with 'dev repair', or edit "
+                            "them to be a single runnable command (e.g. 'python -m pytest <file>')."
+                        ),
+                        blocking=False,
+                        suggested_command=cmd,
+                        stdout_path=cmd_result.stdout_path or None,
+                        stderr_path=cmd_result.stderr_path or None,
+                    ))
+                else:
+                    is_quality_gate = cmd_type in {"lint", "typecheck"} or is_quality_only_command(cmd)
+                    blocking = cmd_result.timed_out or (
+                        (not compiler_active) and not is_quality_gate
+                    )
+                    if blocking:
+                        result.genuine_failure = True
+                    fail_file, fail_line = failure_location(cmd_result)
+                    failure_label = (
+                        "timed out after running for the configured limit"
+                        if cmd_result.timed_out
+                        else f"failed with exit code {cmd_result.exit_code}"
+                    )
+                    gap = Gap(
+                        id=next_gap_id(task.id, f"{cmd_type.upper()}-{cmd}"),
+                        severity="high" if blocking else "medium",
+                        gap_type="quality_gate_failed" if is_quality_gate else "test_failed",
+                        task_id=task.id,
+                        description=(
+                            f"{'Quality gate' if is_quality_gate else 'Command'} '{cmd}' "
+                            f"{failure_label}"
+                            + (
+                                " (advisory: style/type, not a correctness gate)."
+                                if is_quality_gate and not cmd_result.timed_out
+                                else "."
+                            )
+                        ),
+                        evidence=[cmd_result.summary[:500]],
+                        recommended_fix=f"Fix the issues reported by '{cmd}'.",
+                        blocking=blocking,
+                        suggested_command=cmd,
+                        file=fail_file,
+                        line=fail_line,
+                        stdout_path=cmd_result.stdout_path or None,
+                        stderr_path=cmd_result.stderr_path or None,
+                    )
+                    result.gaps.append(gap)
+                    if compiler_active and not is_quality_gate and not blocking:
+                        result.demoted_failures.append(gap)
+    return result