devcouncil 0.3.1 → 0.4.1

package/src/devcouncil/verification/claims/mapper.py

@@ -0,0 +1,168 @@
+"""Claim-to-check mapper: free-text completion claims → structured assertions.
+
+Port of Oversight semantics: sentence-scoped negation/hedging; specific
+assertions suppress GENERIC_DONE; capped to MAX_ASSERTIONS.
+"""
+
+from __future__ import annotations
+
+import re
+
+from devcouncil.verification.claims.models import Assertion, Kind
+
+MAX_ASSERTIONS = 5
+
+_SENTENCE_SPLIT = re.compile(r"(?<=[.!?])\s+|\n+")
+
+_BENIGN_NEGATIVE = re.compile(
+    r"\b(?:0|no|zero|without)\s+(?:\w+\s+)?(?:failures?|errors?|warnings?|regressions?|issues?)\b"
+    r"|\bnone\s+failed\b",
+    re.I,
+)
+
+_HEDGE = re.compile(
+    r"\b(?:not|never|no|fail(?:s|ed|ing|ure)?|should|would|could|might|"
+    r"will|going\s+to|plan(?:s|ning)?\s+to|need(?:s)?\s+to|todo|"
+    r"haven'?t|hasn'?t|didn'?t|don'?t|doesn'?t|won'?t|can'?t|isn'?t|aren'?t|"
+    r"still|yet|broken|remaining\s+work)\b",
+    re.I,
+)
+
+_TESTS_PASS = re.compile(
+    r"\btests?\s+(?:are\s+|all\s+|now\s+)*pass(?:es|ing)?\b"
+    r"|\btest\s+suite\s+passes\b"
+    r"|\b(?:pytest|jest|vitest|unittest)\b.*\b(?:green|passed|passing)\b"
+    r"|\b\d+\s+(?:tests?\s+)?(?:passed|passing)\b"
+    r"|\b(?:test\s+)?suite\s+is\s+green\b"
+    r"|\b(?:all\s+)?checks\s+(?:are\s+)?green\b"
+    r"|\b(?:unit\s+)?tests\s+succeed(?:ed)?\b"
+    r"|\bci\s+is\s+(?:happy|green|passing)\b"
+    r"|\beverything\s+passed\b",
+    re.I,
+)
+_BUILD = re.compile(
+    r"\bbuild\s+(?:succeeds|succeeded|passes|passed|completed\s+successfully|is\s+(?:now\s+)?(?:passing|clean|green))\b"
+    r"|\bcompiles?(?:\s+cleanly)?\b"
+    r"|\bbuilds\s+cleanly\b"
+    r"|\bbuild\s+completed\b",
+    re.I,
+)
+_LINT = re.compile(
+    r"\blint(?:ing|er)?\s+(?:passes|passed|is\s+clean|clean)\b"
+    r"|\bno\s+(?:type|lint(?:ing)?)\s+errors\b"
+    r"|\btype-?check(?:ing)?\s+passes\b"
+    r"|\bno\s+lint\s+(?:errors|warnings)\b"
+    r"|\b(?:mypy|ruff|eslint|flake8)\b.*\b(?:passes|passed|clean)\b",
+    re.I,
+)
+_GENERIC_DONE = re.compile(
+    r"\b(?:done|completed?|finished|everything\s+works|all\s+set|task\s+is\s+complete|implementation\s+is\s+complete)\b",
+    re.I,
+)
+_CREATE_VERB = re.compile(r"\b(?:created|added|wrote|generated|new\s+file)\b", re.I)
+_MODIFY_VERB = re.compile(r"\b(?:updated|edited|modified|changed|fixed|refactored|patched|rewrote)\b", re.I)
+_RAN_COMMAND = re.compile(r"\b(?:ran|executed|invoked)\s+[`\"']([^`\"'\n]+)[`\"'].*?\bsuccess", re.I)
+
+_QUOTED_SPAN = re.compile(r"[`\"']([^`\"'\n]+)[`\"']")
+_BARE_PATH = re.compile(r"(?<![\w./\\-])(?:[A-Za-z]:[/\\])?[\w.-]+(?:[/\\][\w.-]+)+(?![\w/\\])")
+_KNOWN_EXTENSIONS = {
+    "py", "js", "ts", "tsx", "jsx", "json", "md", "txt", "toml", "yaml", "yml",
+    "html", "css", "scss", "sql", "sh", "ps1", "psm1", "bat", "cs", "java",
+    "go", "rs", "c", "cpp", "h", "hpp", "rb", "php", "xml", "csv", "ini",
+    "cfg", "lock", "env",
+}
+
+
+def _has_known_extension(token: str) -> bool:
+    name = token.replace("\\", "/").rsplit("/", 1)[-1]
+    if "." not in name:
+        return False
+    return name.rsplit(".", 1)[-1].lower() in _KNOWN_EXTENSIONS
+
+
+def _looks_like_path(token: str, bare: bool = False) -> bool:
+    token = token.strip()
+    if not token or " " in token:
+        return False
+    if bare:
+        return _has_known_extension(token)
+    if "/" in token or "\\" in token:
+        return True
+    return _has_known_extension(token)
+
+
+def _extract_paths(sentence: str) -> list[str]:
+    paths: list[str] = []
+    seen: set[str] = set()
+    for match in _QUOTED_SPAN.finditer(sentence):
+        token = match.group(1).strip().rstrip(".,;:!?")
+        if _looks_like_path(token) and token not in seen:
+            seen.add(token)
+            paths.append(token)
+    without_quotes = _QUOTED_SPAN.sub(" ", sentence)
+    for match in _BARE_PATH.finditer(without_quotes):
+        token = match.group(0).rstrip(".,;:!?")
+        if _looks_like_path(token, bare=True) and token not in seen:
+            seen.add(token)
+            paths.append(token)
+    return paths
+
+
+def map_claims(text: str) -> list[Assertion]:
+    """Parse a completion message into a deduplicated, capped assertion list."""
+    if not text or not text.strip():
+        return []
+
+    specific: list[Assertion] = []
+    generic: list[Assertion] = []
+    seen: set[tuple[Kind, str | None]] = set()
+
+    def add(bucket: list[Assertion], kind: Kind, target: str | None, source: str) -> None:
+        key = (kind, target)
+        if key in seen:
+            return
+        seen.add(key)
+        bucket.append(Assertion(kind=kind, target=target, source_text=source.strip()))
+
+    for sentence in _SENTENCE_SPLIT.split(text):
+        sentence = sentence.strip()
+        if not sentence:
+            continue
+        sanitized = _BENIGN_NEGATIVE.sub(" ", sentence)
+        if _HEDGE.search(sanitized):
+            continue
+
+        if _TESTS_PASS.search(sentence):
+            add(specific, Kind.TESTS_PASS, None, sentence)
+        if _BUILD.search(sentence):
+            add(specific, Kind.BUILD_SUCCEEDS, None, sentence)
+        if _LINT.search(sentence):
+            add(specific, Kind.LINT_CLEAN, None, sentence)
+
+        ran = _RAN_COMMAND.search(sentence)
+        if ran:
+            add(specific, Kind.COMMAND_SUCCEEDED, ran.group(1).strip(), sentence)
+
+        paths = _extract_paths(sentence)
+        if paths:
+            if _CREATE_VERB.search(sentence):
+                for path in paths:
+                    add(specific, Kind.FILE_CREATED, path, sentence)
+            elif _MODIFY_VERB.search(sentence):
+                for path in paths:
+                    add(specific, Kind.FILE_UPDATED, path, sentence)
+
+        if _GENERIC_DONE.search(sentence):
+            add(generic, Kind.GENERIC_DONE, None, sentence)
+
+    priority = {
+        Kind.TESTS_PASS: 0,
+        Kind.BUILD_SUCCEEDS: 1,
+        Kind.LINT_CLEAN: 2,
+        Kind.COMMAND_SUCCEEDED: 3,
+        Kind.FILE_CREATED: 4,
+        Kind.FILE_UPDATED: 5,
+    }
+    specific.sort(key=lambda a: priority.get(a.kind, 9))
+    result = specific if specific else generic
+    return result[:MAX_ASSERTIONS]

package/src/devcouncil/verification/claims/models.py

@@ -0,0 +1,39 @@
+"""Core data types for claim verification."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from enum import Enum
+
+
+class Kind(str, Enum):
+    TESTS_PASS = "tests_pass"
+    BUILD_SUCCEEDS = "build_succeeds"
+    LINT_CLEAN = "lint_clean"
+    FILE_CREATED = "file_created"
+    FILE_UPDATED = "file_updated"
+    COMMAND_SUCCEEDED = "command_succeeded"
+    GENERIC_DONE = "generic_done"
+
+
+@dataclass(frozen=True)
+class Assertion:
+    """A checkable claim extracted from the agent's completion message."""
+
+    kind: Kind
+    target: str | None = None
+    source_text: str = ""
+
+
+class Status(str, Enum):
+    PASS = "pass"
+    FAIL = "fail"
+    UNVERIFIABLE = "unverifiable"
+    SKIPPED = "skipped"
+
+
+@dataclass
+class CheckResult:
+    assertion: Assertion
+    status: Status
+    detail: str = ""

package/src/devcouncil/verification/claims/transcript.py

@@ -0,0 +1,92 @@
+"""Extract the final assistant message from a Claude transcript JSONL."""
+
+from __future__ import annotations
+
+import json
+import re
+from pathlib import Path
+
+_SENTENCE_SPLIT = re.compile(r"(?<=[.!?])\s+")
+
+
+def _text_of(entry: dict) -> str | None:
+    message = entry.get("message")
+    if not isinstance(message, dict):
+        return None
+    content = message.get("content")
+    if isinstance(content, str):
+        return content.strip() or None
+    if isinstance(content, list):
+        texts = [
+            block.get("text", "")
+            for block in content
+            if isinstance(block, dict) and block.get("type") == "text"
+        ]
+        joined = "\n".join(t for t in texts if t.strip())
+        return joined.strip() or None
+    return None
+
+
+def last_assistant_text(path: Path) -> str | None:
+    """Last assistant message that contains text (tool-use-only turns skipped)."""
+    try:
+        lines = path.read_text(encoding="utf-8-sig", errors="replace").splitlines()
+    except OSError:
+        return None
+    for line in reversed(lines):
+        line = line.strip()
+        if not line:
+            continue
+        try:
+            entry = json.loads(line)
+        except json.JSONDecodeError:
+            continue
+        if not isinstance(entry, dict) or entry.get("type") != "assistant":
+            continue
+        text = _text_of(entry)
+        if text:
+            return text
+    return None
+
+
+def last_assistant_sentence(path: Path) -> str | None:
+    """Last sentence of the last assistant text, or None."""
+    text = last_assistant_text(path)
+    if not text:
+        return None
+    parts = [p.strip() for p in _SENTENCE_SPLIT.split(text) if p.strip()]
+    return parts[-1] if parts else text.strip()
+
+
+def ends_on_open_question(path: Path) -> bool:
+    """Heuristic: last assistant turn ends with '?' and no later user turn."""
+    try:
+        lines = path.read_text(encoding="utf-8-sig", errors="replace").splitlines()
+    except OSError:
+        return False
+    last_assistant_idx = -1
+    last_user_idx = -1
+    last_assistant_msg: str | None = None
+    for i, line in enumerate(lines):
+        line = line.strip()
+        if not line:
+            continue
+        try:
+            entry = json.loads(line)
+        except json.JSONDecodeError:
+            continue
+        if not isinstance(entry, dict):
+            continue
+        etype = entry.get("type")
+        if etype == "assistant":
+            text = _text_of(entry)
+            if text:
+                last_assistant_idx = i
+                last_assistant_msg = text
+        elif etype == "user":
+            last_user_idx = i
+    if last_assistant_idx < 0 or not last_assistant_msg:
+        return False
+    if last_user_idx > last_assistant_idx:
+        return False
+    return last_assistant_msg.rstrip().endswith("?")

package/src/devcouncil/verification/claims/verdict.py

@@ -0,0 +1,88 @@
+"""Merge claim check results into a corrective message / report."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+from devcouncil.verification.claims.models import CheckResult, Status
+
+
+@dataclass
+class ClaimVerdict:
+    block: bool
+    reason: str
+    report: str
+
+
+def render_report(results: list[CheckResult]) -> str:
+    if not results:
+        return ""
+    lines = [
+        "# DevCouncil stop-gate claim report",
+        "",
+        "| Claim | Check | Status |",
+        "|---|---|---|",
+    ]
+    for r in results:
+        claim = (r.assertion.source_text or r.assertion.kind.value).replace("|", "\\|")
+        target = f" `{r.assertion.target}`" if r.assertion.target else ""
+        lines.append(f"| {claim} | {r.assertion.kind.value}{target} | {r.status.name} |")
+    lines.append("")
+    for r in results:
+        if r.detail:
+            lines.append(f"## {r.assertion.kind.value} — {r.status.name}")
+            lines.append(r.detail)
+            lines.append("")
+    return "\n".join(lines)
+
+
+_STATUS_ICONS = {
+    Status.PASS: "✓",
+    Status.FAIL: "✗",
+    Status.UNVERIFIABLE: "?",
+    Status.SKIPPED: "–",
+}
+
+
+def summary_line(results: list[CheckResult], *, prefix: str = "🛡 devcouncil") -> str:
+    """One-line verdict summary for Claude systemMessage."""
+    if not results:
+        return f"{prefix}: (no claims)"
+    parts = []
+    for r in results:
+        icon = _STATUS_ICONS.get(r.status, "?")
+        target = f" {r.assertion.target}" if r.assertion.target else ""
+        parts.append(f"{r.assertion.kind.value}{target} {icon}")
+    return f"{prefix}: " + " | ".join(parts)
+
+
+def _corrective_message(failures: list[CheckResult]) -> str:
+    bullets = []
+    for r in failures:
+        claim = r.assertion.source_text or r.assertion.kind.value
+        bullets.append(f'- You claimed: "{claim}" — but verification found: {r.detail}')
+    joined = "\n".join(bullets)
+    return (
+        "CLAIM VERIFICATION FAILED. DevCouncil re-checked your completion claims "
+        "against the real environment and found discrepancies:\n\n"
+        f"{joined}\n\n"
+        "Do not repeat the claim. Investigate the filesystem, git, and command exit "
+        "codes, fix the discrepancy, and only finish when these verifications would succeed."
+    )
+
+
+def decide_claims(results: list[CheckResult], blocks_so_far: int, max_blocks: int) -> ClaimVerdict:
+    failures = [r for r in results if r.status is Status.FAIL]
+    report = render_report(results)
+
+    if not failures:
+        return ClaimVerdict(block=False, reason="", report=report)
+
+    if blocks_so_far >= max_blocks:
+        report += (
+            f"\n> Block cap reached ({blocks_so_far}/{max_blocks}): failures above were "
+            "NOT re-injected into the agent. Review them yourself.\n"
+        )
+        return ClaimVerdict(block=False, reason="", report=report)
+
+    return ClaimVerdict(block=True, reason=_corrective_message(failures), report=report)

package/src/devcouncil/verification/command_evidence.py

@@ -0,0 +1,170 @@
+"""Shared helpers for deciding whether a command qualifies as acceptance evidence."""
+
+from __future__ import annotations
+
+import re
+import shlex
+
+# Executables (basename of argv[0]) that ARE test/check runners.
+_RUNNER_EXECUTABLES = frozenset({
+    "pytest", "py.test", "vitest", "jest", "mocha", "ava", "tap",
+    "unittest", "tox", "nox", "behave", "rspec", "minitest", "phpunit",
+    "mypy", "pyright", "tsc",
+})
+
+# (argv0 basename, first subcommand) pairs that are runners.
+_RUNNER_SUBCOMMANDS = frozenset({
+    ("cargo", "test"), ("go", "test"), ("mvn", "test"), ("gradle", "test"),
+    ("gradlew", "test"), ("make", "test"), ("make", "check"), ("ruff", "check"),
+    ("swift", "test"), ("dotnet", "test"), ("mix", "test"), ("stack", "test"),
+})
+
+# Package managers whose ``test`` / ``run <script>`` invocations count.
+_PKG_MANAGERS = frozenset({"npm", "yarn", "pnpm", "bun"})
+_SCRIPT_KEYWORDS = ("test", "typecheck", "type-check", "vitest", "jest", "check")
+
+# Interpreters whose inline payloads (-c / -e / --eval) can carry assertions.
+_INLINE_INTERPRETERS = frozenset({"python", "python3", "python2", "py", "node", "deno", "bun"})
+_INLINE_FLAGS = frozenset({"-c", "-e", "--eval"})
+# Assertion-ish content inside an inline payload. ``print(...)`` alone is not
+# evidence; an assert / raise / expect() is.
+_INLINE_EVIDENCE_RE = re.compile(
+    r"(\bassert\b|\braise\s+\w+|\bexpect\s*\(|\.to(?:Be|Equal|Throw)\b|\bthrow\s)",
+    re.IGNORECASE,
+)
+
+# A single argument token that references a test file/directory. Path-like — a
+# quoted sentence containing the word "test" has spaces and cannot match.
+_TEST_PATH_RE = re.compile(
+    r"(^|/)(tests?(/|$)|test_[^/\s]*$|[^/\s]*(?:_tests?|\.test|\.spec)\.[a-z]+$)",
+    re.IGNORECASE,
+)
+
+
+def _split(command: str) -> list[str]:
+    text = (command or "").strip()
+    if not text:
+        return []
+    try:
+        return shlex.split(text)
+    except ValueError:
+        return text.split()
+
+
+def command_has_acceptance_evidence(command: str) -> bool:
+    """True when ``command``'s actual INVOCATION can produce behavioral evidence.
+
+    STRICT, structure-aware allowlist — use for UNTRUSTED input (an agent appending
+    expected_tests via ``update_task_scope``). The runner must be the executable, a
+    subcommand, a package-manager script, an inline interpreter assertion, or a
+    test-file path argument — never just a keyword substring anywhere in the line
+    (``echo "tests assert ok"`` must not qualify). For trusted planner/config
+    commands at verify time use :func:`command_is_trivial_evidence` instead, whose
+    deny-list does not punish legitimate behavioral commands that lack a keyword
+    (``make check``, ``./run_smoke.sh``, ``python scripts/validate.py``)."""
+    tokens = _split(command)
+    if not tokens:
+        return False
+    base = tokens[0].rsplit("/", 1)[-1].lower()
+    if base in _TRIVIAL_EXECUTABLES:
+        return False
+    if base in _RUNNER_EXECUTABLES:
+        return True
+
+    args = tokens[1:]
+    lowered_args = [a.lower() for a in args]
+    first_arg = lowered_args[0] if lowered_args else ""
+
+    # Subcommand runners: `cargo test`, `go test ./...`, `ruff check .`, ...
+    if (base, first_arg) in _RUNNER_SUBCOMMANDS:
+        return True
+
+    # Package-manager scripts: `npm test`, `npm run typecheck`, `yarn vitest`.
+    if base in _PKG_MANAGERS and lowered_args:
+        script = lowered_args[1] if first_arg == "run" and len(lowered_args) > 1 else first_arg
+        if any(keyword in script for keyword in _SCRIPT_KEYWORDS):
+            return True
+
+    # Interpreters: `python -m pytest`, `python -c "assert ..."`, `node -e "expect(...)"`.
+    if base in _INLINE_INTERPRETERS:
+        for i, arg in enumerate(lowered_args):
+            if arg == "-m" and i + 1 < len(lowered_args):
+                module = lowered_args[i + 1]
+                if module in _RUNNER_EXECUTABLES or module == "unittest":
+                    return True
+            if arg in _INLINE_FLAGS and i + 1 < len(args) and _INLINE_EVIDENCE_RE.search(args[i + 1]):
+                return True
+
+    # A test-file/dir reference as the executable or an argument: `./run_tests.sh`,
+    # `python tests/test_x.py`, `bash tests/run.sh`.
+    for token in tokens:
+        if " " not in token and _TEST_PATH_RE.search(token):
+            return True
+
+    return False
+
+
+# Executables whose success proves nothing about the code under review: pure
+# printing, filesystem/system queries, and no-ops. First-token match (after
+# common wrappers) so e.g. ``echo done`` or ``date`` can never coarse-prove an
+# acceptance criterion, while ``make check`` / ``./run_smoke.sh`` still can.
+_TRIVIAL_EXECUTABLES = frozenset({
+    "echo", "printf", "true", "false", ":", "exit", "sleep", "ls", "dir", "cat",
+    "head", "tail", "wc", "pwd", "cd", "which", "whereis", "whoami", "date",
+    "env", "printenv", "type", "touch", "mkdir", "cp", "mv", "hostname", "uname",
+})
+
+# Wrapper tokens to skip before resolving the effective executable.
+_WRAPPER_TOKENS = frozenset({"sudo", "time", "nice", "nohup", "uv", "poetry", "pdm", "hatch", "rye", "npx"})
+
+# Version/help style flags: a command that only reports its own identity.
+_IDENTITY_FLAGS = frozenset({"--version", "-v", "-version", "--help", "-h", "-help", "version", "--about"})
+
+
+def command_is_trivial_evidence(command: str) -> bool:
+    """True when ``command`` cannot possibly prove behavior — deny-list, not allowlist.
+
+    Use at VERIFY time to filter which passing planner/config commands may
+    coarse-prove an acceptance criterion. Deliberately permissive: a behavioral
+    command without a test keyword (``make check``, ``python scripts/validate.py``,
+    ``./run_smoke.sh``) is NOT trivial and keeps its evidential value; only commands
+    that provably exercise nothing are rejected:
+
+    * empty commands;
+    * pure printing / system queries (``echo``, ``ls``, ``date``, ...);
+    * identity checks (``python --version``, ``node --help``);
+    * ``git`` state queries (repo state is not behavior);
+    * interpreter one-liners that neither import nor assert nor raise
+      (``python -c "print('ok')"``).
+    """
+    lowered = (command or "").strip().lower()
+    if not lowered:
+        return True
+    if command_has_acceptance_evidence(lowered):
+        return False
+
+    tokens = lowered.split()
+    index = 0
+    while index < len(tokens) and tokens[index] in _WRAPPER_TOKENS:
+        index += 1
+        if index < len(tokens) and tokens[index] == "run":
+            index += 1
+    if index >= len(tokens):
+        return True
+    executable = tokens[index].rsplit("/", 1)[-1]
+
+    if executable in _TRIVIAL_EXECUTABLES:
+        return True
+    # `git status` / `git diff` / `git log` etc.: repository state, not behavior.
+    if executable == "git":
+        return True
+    # Bare identity check: every remaining argument is a version/help flag.
+    rest = tokens[index + 1:]
+    if rest and all(tok in _IDENTITY_FLAGS for tok in rest):
+        return True
+    # Interpreter one-liner that neither imports the code, asserts, raises, nor
+    # calls anything that could fail meaningfully: `python -c "print('ok')"`.
+    if executable.startswith(("python", "node", "ruby", "perl")) and (" -c " in f" {lowered} " or " -e " in f" {lowered} "):
+        if not any(marker in lowered for marker in ("import", "require", "assert", "raise", "exit(", "sys.exit")):
+            return True
+    return False