devcouncil 0.2.0 → 0.2.1

package/src/devcouncil/utils/redaction.py

@@ -1,147 +0,0 @@
-"""Proactive secret and PII redaction for LLM-bound prompts.
-
-Two entry points:
-  - redact_string(text)  -- regex-based redaction of known secret patterns
-  - redact_text(text, extra_patterns)  -- enhanced version with custom patterns and typed labels
-  - redact_env_vars(text)  -- redact values in environment variable assignments
-  - redact_dict(data)  -- recursively redact all string values in a dict
-"""
-
-import re
-from typing import Any, Dict, List, Optional, Pattern
-
-# Common patterns for sensitive data — each with a human-readable label
-SECRET_PATTERNS: Dict[str, Pattern] = {
-    "aws_access_key": re.compile(r"(?i)\b(AKIA[0-9A-Z]{16})\b"),
-    "aws_secret_key": re.compile(r"(?i)(?:aws_secret_access_key|aws_secret|secret_key)\s*[=:]\s*([0-9a-zA-Z/+]{40})"),
-    "github_token": re.compile(r"(?i)\b(gh[pusr]_[A-Za-z0-9_]{36})\b"),
-    # Anthropic keys are matched before the generic OpenAI-style sk- pattern so the
-    # more specific label wins.
-    "anthropic_key": re.compile(r"\b(sk-ant-[A-Za-z0-9_-]{20,})\b"),
-    "openai_key": re.compile(r"\b(sk-[A-Za-z0-9_-]{20,})\b"),
-    "google_api_key": re.compile(r"\b(AIza[0-9A-Za-z_-]{35})\b"),
-    "stripe_live_key": re.compile(r"\b((?:sk|rk)_live_[0-9A-Za-z]{16,})\b"),
-    "slack_token": re.compile(r"(?i)\b(xox[baprs]-[0-9]{12}-[0-9]{12}-[0-9]{12}-[a-z0-9]{32})\b"),
-    "jwt": re.compile(r"(?i)\b(ey[a-zA-Z0-9_-]{10,}\.ey[a-zA-Z0-9_-]{10,}\.[a-zA-Z0-9_-]{10,})\b"),
-    "generic_api_key": re.compile(r"(?i)(api[_-]?key|secret|token|password)[\"'\s]*[:=][\"'\s]*([a-zA-Z0-9_\-\.]{16,})"),
-    "private_key": re.compile(r"(?s)-----BEGIN [A-Z]+ PRIVATE KEY-----.*?-----END [A-Z]+ PRIVATE KEY-----"),
-    "bearer": re.compile(r"(?i)\b(Bearer\s+)([a-zA-Z0-9_\-\.]{16,})\b"),
-    "database_url": re.compile(r"(?i)((?:postgresql|mysql|mongodb|redis)://[^:]+:)([^@]+)(@.+)"),
-}
-
-# For backward compatibility
-def redact_string(text: str) -> str:
-    """Redact known sensitive patterns from a string (legacy API)."""
-    return redact_text(text)
-
-
-def redact_text(text: str, extra_patterns: Optional[List[str]] = None) -> str:
-    """Redact known sensitive patterns from a string.
-    
-    Args:
-        text: The input text to redact.
-        extra_patterns: Optional list of regex patterns to additionally redact,
-                       labeled as [REDACTED:custom_N].
-    
-    Returns:
-        Text with all sensitive patterns replaced with [REDACTED:type] labels.
-    """
-    if not isinstance(text, str):
-        return text
-        
-    redacted_text = text
-    
-    for key_type, pattern in SECRET_PATTERNS.items():
-        if key_type == "generic_api_key":
-            # For the generic pattern, replace the value part (group 2)
-            def _make_generic_replacer(kt: str):
-                def replacer(match):
-                    prefix = match.group(1)
-                    separator = match.group(0)[len(match.group(1)):-len(match.group(2))]
-                    return f"{prefix}{separator}[REDACTED:{kt}]"
-                return replacer
-            redacted_text = pattern.sub(_make_generic_replacer(key_type), redacted_text)
-        elif key_type == "bearer":
-            # Preserve "Bearer " prefix, redact the token
-            def _bearer_replacer(match):
-                return f"{match.group(1)}[REDACTED:bearer]"
-            redacted_text = pattern.sub(_bearer_replacer, redacted_text)
-        elif key_type == "database_url":
-            # Preserve protocol and host, redact password
-            def _db_url_replacer(match):
-                return f"{match.group(1)}[REDACTED:database_url]{match.group(3)}"
-            redacted_text = pattern.sub(_db_url_replacer, redacted_text)
-        else:
-            redacted_text = pattern.sub(f"[REDACTED:{key_type}]", redacted_text)
-    
-    # Apply custom extra patterns
-    if extra_patterns:
-        for i, pat_str in enumerate(extra_patterns):
-            try:
-                pat = re.compile(pat_str)
-                redacted_text = pat.sub(f"[REDACTED:custom_{i}]", redacted_text)
-            except re.error:
-                pass  # Skip invalid patterns
-            
-    return redacted_text
-
-
-def redact_env_vars(text: str) -> str:
-    """Redact values in environment variable assignments.
-    
-    Handles patterns like:
-      export KEY=value
-      KEY='value'
-      KEY="value"
-    """
-    if not isinstance(text, str):
-        return text
-
-    # Match: optional export, VAR_NAME = value (with optional quotes)
-    env_pattern = re.compile(
-        r"(?m)^(\s*(?:export\s+)?)"               # optional export
-        r"([A-Z_][A-Z0-9_]*)"                      # variable name
-        r"(\s*=\s*)"                                # equals sign
-        r"(?:'([^']*)'|\"([^\"]*)\"|(\S+))"         # value (quoted or unquoted)
-    )
-
-    sensitive_keys = {
-        "api_key", "secret", "token", "password", "passwd", "credential",
-        "private_key", "access_key", "secret_key", "auth",
-    }
-
-    def _env_replacer(match):
-        prefix = match.group(1)
-        var_name = match.group(2)
-        eq_sign = match.group(3)
-        
-        # Check if the variable name contains a sensitive keyword
-        var_lower = var_name.lower()
-        is_sensitive = any(kw in var_lower for kw in sensitive_keys)
-        
-        if is_sensitive:
-            return f"{prefix}{var_name}{eq_sign}[REDACTED]"
-        
-        return match.group(0)
-
-    return env_pattern.sub(_env_replacer, text)
-
-
-def redact_dict(data: dict[str, Any]) -> dict[str, Any]:
-    """Recursively redact sensitive patterns from a dictionary (e.g. JSON response)."""
-    result: dict[str, Any] = {}
-    for k, v in data.items():
-        if isinstance(v, str):
-            result[k] = redact_text(v)
-        elif isinstance(v, dict):
-            result[k] = redact_dict(v)
-        elif isinstance(v, list):
-            result[k] = [
-                redact_dict(item) if isinstance(item, dict)
-                else redact_text(item) if isinstance(item, str)
-                else item
-                for item in v
-            ]
-        else:
-            result[k] = v
-    return result

package/src/devcouncil/utils/subprocess_env.py

@@ -1,69 +0,0 @@
-"""A sanitized environment for spawning *external* executables.
-
-When DevCouncil runs from a virtualenv (a project ``.venv`` or a ``uv tool
-install``), its interpreter exports markers — ``VIRTUAL_ENV``, ``PYTHONHOME``,
-``PYTHONPATH``, ``UV_INTERNAL__PYTHONHOME`` — that any child process inherits.
-Those markers forcibly re-point a freshly-spawned, *differently-built* Python
-(e.g. the globally-installed ``devcouncil`` CLI, or a project's own
-interpreter) at DevCouncil's stdlib/site-packages. The classic symptoms are
-``AssertionError: SRE module mismatch`` (the child loads its own ``_sre`` C
-extension against our ``re`` stdlib) and spurious ``No module named pytest``.
-
-This mirrors :meth:`Verifier._verification_env` but is dependency-free so the
-integration probes (``dev integrate check``) and any other call-site that
-shells out to an *external* program can share one correct implementation.
-"""
-
-from __future__ import annotations
-
-import os
-import sys
-from pathlib import Path
-from typing import Dict
-
-
-def clean_subprocess_env() -> Dict[str, str]:
-    """Return a copy of ``os.environ`` with DevCouncil's own virtualenv stripped.
-
-    No-op (returns a plain copy) when DevCouncil is not running inside a venv,
-    so behaviour is unchanged for system / pipx-style installs.
-    """
-    env = dict(os.environ)
-    venv_prefix = Path(sys.prefix).resolve()
-    base_prefix = Path(getattr(sys, "base_prefix", sys.prefix)).resolve()
-    if venv_prefix == base_prefix:
-        return env  # Not inside a venv; nothing to strip.
-
-    venv_dirs = {
-        str(venv_prefix).lower(),
-        str((venv_prefix / "Scripts").resolve()).lower(),
-        str((venv_prefix / "bin").resolve()).lower(),
-    }
-    path = env.get("PATH", "")
-    kept = []
-    for entry in path.split(os.pathsep):
-        if not entry:
-            continue
-        try:
-            normalized = str(Path(entry).resolve()).lower()
-        except Exception:
-            normalized = entry.lower()
-        if normalized in venv_dirs:
-            continue
-        kept.append(entry)
-    env["PATH"] = os.pathsep.join(kept)
-
-    own_prefixes = {str(venv_prefix), str(base_prefix)}
-    for marker in ("VIRTUAL_ENV", "PYTHONHOME"):
-        value = env.get(marker)
-        if not value:
-            continue
-        try:
-            resolved = str(Path(value).resolve())
-        except Exception:
-            resolved = value
-        if resolved in own_prefixes:
-            env.pop(marker, None)
-    # uv stashes the interpreter home here and re-applies it to child pythons.
-    env.pop("UV_INTERNAL__PYTHONHOME", None)
-    return env

package/src/devcouncil/verification/__init__.py

@@ -1 +0,0 @@
-

package/src/devcouncil/verification/acceptance_compiler.py

@@ -1,125 +0,0 @@
-"""Compile natural-language acceptance criteria into self-contained executable
-checks that DevCouncil owns and runs.
-
-This is the difference between trusting the planner/agent's word and gathering
-real evidence: instead of running planner-authored ``expected_tests`` (which the
-benchmark showed often reference tools or test files that do not exist), DevCouncil
-derives one runnable check per acceptance criterion directly from the criterion
-text and the code under review, then maps each check 1:1 to its criterion.
-"""
-
-from __future__ import annotations
-
-import json
-from typing import Dict, List
-
-from pydantic import BaseModel
-
-from devcouncil.domain.requirement import Requirement
-from devcouncil.domain.task import Task
-from devcouncil.llm.router import ModelRouter
-
-
-class CompiledCheck(BaseModel):
-    acceptance_criterion_id: str
-    command: str  # a single shell command that exits 0 iff the criterion holds
-
-
-class CompiledChecks(BaseModel):
-    checks: List[CompiledCheck]
-
-
-class AcceptanceTestCompiler:
-    def __init__(self, router: ModelRouter, role: str = "implementation_reviewer"):
-        self.router = router
-        self.role = role
-
-    async def compile(
-        self,
-        task: Task,
-        requirements: List[Requirement],
-        code_context: str,
-    ) -> Dict[str, List[str]]:
-        """Return {acceptance_criterion_id: [self-contained check command(s)]}.
-
-        Best-effort: returns {} if the model cannot produce usable checks, so the
-        caller can fall back to the task's declared expected_tests.
-        """
-        ac_by_id = {ac.id: ac for req in requirements for ac in req.acceptance_criteria}
-        target = [ac_by_id[i] for i in task.acceptance_criterion_ids if i in ac_by_id]
-        if not target:
-            return {}
-
-        acs_json = json.dumps(
-            [{"id": ac.id, "description": ac.description, "method": ac.verification_method} for ac in target],
-            indent=2,
-        )
-        prompt = f"""
-You are DevCouncil's acceptance-test compiler. Convert each acceptance criterion
-below into exactly ONE shell command that EXITS 0 if and only if the BEHAVIOR
-described by the criterion holds for the code shown.
-
-Acceptance criteria:
-{acs_json}
-
-Code under review (the diff / current files):
-{code_context}
-
-What a check must verify — BEHAVIOR ONLY:
-- A check exists to confirm the code DOES what the criterion describes when its
-  public API is exercised: import the module/symbol and call its function(s), or
-  run its CLI/entrypoint, and assert on the observable result (return value,
-  raised exception, stdout, exit code).
-- DevCouncil already enforces scope, file ownership, and append-only/orphan-diff
-  constraints with its OWN gates. Acceptance checks must therefore NEVER re-assert
-  repository or filesystem STATE — that is not their job and it produces false
-  BLOCKED results because `dev` itself adds workspace files (AGENTS.md, CLAUDE.md,
-  .gitignore, .devcouncil/config.yaml, etc.).
-
-Rules — the commands are executed verbatim by the verifier:
-- One command per acceptance_criterion_id (reference the id exactly).
-- Each command MUST be a single, SELF-CONTAINED, immediately-runnable command:
-  import the real module/symbol from the code and assert the behavior directly.
-  Do NOT depend on test files, fixtures, or any external setup.
-- Prefer: python -c "import <module>; assert <expr>". For an expected exception,
-  use a one-line guard, e.g.
-  python -c "import m; \\ntry: m.f([])\\nexcept ValueError: pass\\nelse: raise SystemExit(1)"
-  (real newlines are fine; never put try/if/for after a ';').
-- Use the actual module name implied by the code (e.g. file 'stats.py' -> import stats).
-
-HARD PROHIBITIONS — a command that does any of these is INVALID; omit the
-criterion instead of emitting such a command:
-- NEVER assert exact git or filesystem state. Forbidden: `git status`,
-  `git status --porcelain`, `git diff`, `git diff --name-only`, `git show`,
-  `git ls-files`, `ls`/`find`/`os.listdir` equality checks, asserting a precise
-  set or count of changed/created files, or asserting a file does/does not exist
-  as the criterion's pass condition.
-- NEVER do append-only or byte-level file/content comparisons (e.g.
-  `git show HEAD:file`, diffing bytes, asserting only N bytes/lines were added).
-  Assert the resulting BEHAVIOR instead, not how the file changed.
-- NEVER invoke linters, type checkers, formatters, or build/package tools that
-  may be absent: flake8, mypy, ruff, pylint, black, isort, eslint, tsc, prettier,
-  npm, npx, yarn, pnpm, cargo, go vet, etc. Only use such a tool if the code
-  context clearly shows it is configured for this repo (e.g. a matching config
-  section/file is present in the context) AND it is essential to the criterion.
-- If a criterion cannot be checked by a behavioral command (e.g. pure 'manual'
-  review, or it only describes repo/tooling state), OMIT it rather than inventing
-  a state-based or bogus command.
-"""
-        try:
-            result = await self.router.complete_structured(
-                role=self.role,
-                messages=[{"role": "user", "content": prompt}],
-                schema=CompiledChecks,
-                fallback=CompiledChecks(checks=[]),
-            )
-        except Exception:
-            return {}
-
-        out: Dict[str, List[str]] = {}
-        valid_ids = {ac.id for ac in target}
-        for check in result.checks:
-            cmd = (check.command or "").strip()
-            if check.acceptance_criterion_id in valid_ids and cmd:
-                out.setdefault(check.acceptance_criterion_id, []).append(cmd)
-        return out

package/src/devcouncil/verification/ad_hoc_check.py

@@ -1,129 +0,0 @@
-"""Verify an ad-hoc working-tree diff against an inline requirement — no planning, no keys.
-
-This powers ``dev check``'s evidence-gate mode (the lite entry point): wrap whatever is
-in the working tree as a synthetic Requirement→Task, run the *same* deterministic
-:class:`~devcouncil.verification.verifier.Verifier` the full workflow uses — orphan-diff,
-secret scan, acceptance evidence, and the diff↔coverage gate — and return the verdict
-plus the typed next-actions contract. ``router=None`` keeps it provider-key-free so a
-newcomer can taste the evidence gate before committing to the full council flow.
-
-The logic lives here (not in the CLI command) so it is unit-testable without Typer and
-resilient to churn in the command module.
-"""
-
-from __future__ import annotations
-
-import asyncio
-from dataclasses import dataclass, field
-from pathlib import Path
-from typing import List, Optional
-
-from devcouncil.domain.evidence import DiffCoverageEvidence
-from devcouncil.domain.gap import Gap
-from devcouncil.domain.requirement import AcceptanceCriterion, Requirement
-from devcouncil.domain.task import PlannedFile, Task
-from devcouncil.llm.router import ModelRouter
-from devcouncil.verification.next_actions import NextAction, build_next_actions
-from devcouncil.verification.verifier import Verifier
-
-_REQ_ID = "REQ-CHECK"
-_AC_ID = "AC-CHECK"
-_TASK_ID = "CHECK"
-
-_DEFAULT_CRITERION = "The working-tree changes are correct and exercised by tests."
-
-
-@dataclass
-class AdHocCheckResult:
-    requirement: str
-    changed_files: List[str] = field(default_factory=list)
-    gaps: List[Gap] = field(default_factory=list)
-    next_actions: List[NextAction] = field(default_factory=list)
-    diff_coverage: Optional[DiffCoverageEvidence] = None
-    passed: bool = True
-    reason: str = ""
-
-    def to_dict(self) -> dict:
-        return {
-            "ok": True,
-            "verified": self.passed,
-            "requirement": self.requirement,
-            "changed_files": self.changed_files,
-            "reason": self.reason,
-            "gap_count": len(self.gaps),
-            "blocking_gap_count": len([g for g in self.gaps if g.blocking]),
-            "gaps": [g.model_dump() for g in self.gaps],
-            "next_actions": [a.model_dump() for a in self.next_actions],
-            "diff_coverage": self.diff_coverage.model_dump() if self.diff_coverage else None,
-        }
-
-
-def run_working_tree_check(
-    project_root: Path,
-    requirement: Optional[str] = None,
-    *,
-    test_commands: Optional[List[str]] = None,
-    enforce_coverage: bool = False,
-    min_ratio: float = 0.0,
-    router: Optional[ModelRouter] = None,
-    verifier: Optional[Verifier] = None,
-) -> AdHocCheckResult:
-    """Verify the current working-tree diff against a one-line requirement.
-
-    Builds a synthetic task whose planned files are exactly the changed files (so the
-    result is about evidence, not scope noise) and whose expected tests are
-    ``test_commands``. Diff coverage is always measured; pass ``enforce_coverage`` (or a
-    positive ``min_ratio``) to make an unexercised diff blocking.
-    """
-    verifier = verifier or Verifier(project_root, router=router)
-
-    diff = verifier.get_diff()
-    changed_files = verifier.get_changed_files()
-    if not diff or not changed_files:
-        return AdHocCheckResult(requirement="", passed=True, reason="no_changes")
-
-    criterion = requirement or _DEFAULT_CRITERION
-    req = Requirement(
-        id=_REQ_ID,
-        title=(requirement or "Working-tree change")[:80],
-        description=criterion,
-        priority="high",
-        source="user",
-        acceptance_criteria=[
-            AcceptanceCriterion(id=_AC_ID, description=criterion, verification_method="unit_test"),
-        ],
-    )
-    untracked = set(verifier._get_untracked_files())
-    task = Task(
-        id=_TASK_ID,
-        title="Ad-hoc working-tree check",
-        description=criterion,
-        requirement_ids=[_REQ_ID],
-        acceptance_criterion_ids=[_AC_ID],
-        planned_files=[
-            PlannedFile(
-                path=path,
-                reason="working-tree change",
-                allowed_change="create" if path in untracked else "modify",
-            )
-            for path in changed_files
-        ],
-        expected_tests=list(test_commands or []),
-    )
-
-    # Always measure diff coverage in lite mode; block on it only when asked. A positive
-    # --min-coverage implies enforcement so the flag is never silently inert.
-    enforce = enforce_coverage or min_ratio > 0
-    verifier._diff_coverage_override = (True, enforce, float(min_ratio))
-
-    gaps, evidence = asyncio.run(verifier.verify_task(task, [req]))
-    coverage = next((ev for ev in evidence if isinstance(ev, DiffCoverageEvidence)), None)
-    blocking = [g for g in gaps if g.blocking]
-    return AdHocCheckResult(
-        requirement=criterion,
-        changed_files=changed_files,
-        gaps=gaps,
-        next_actions=build_next_actions(gaps),
-        diff_coverage=coverage,
-        passed=not blocking,
-    )