devcouncil 0.1.0 → 0.2.0

package/src/devcouncil/execution/fs_watcher.py

@@ -0,0 +1,180 @@
+"""Filesystem change attribution with polling fallback."""
+
+from __future__ import annotations
+
+import time
+import uuid
+from pathlib import Path
+from typing import Callable
+
+from devcouncil.domain.gap import Gap
+from devcouncil.domain.task import Task
+from devcouncil.execution.policy_engine import TaskPolicyEngine
+from devcouncil.storage.db import get_db
+from devcouncil.storage.native import FileChangeRepository, TaskLeaseRepository
+from devcouncil.storage.repositories import GapRepository, TaskRepository
+from devcouncil.verification.verifier import Verifier
+
+_IGNORED_PREFIXES = (
+    ".git/",
+    ".devcouncil/cache/",
+    ".devcouncil/logs/",
+    "__pycache__/",
+    ".pytest_cache/",
+    ".mypy_cache/",
+    ".ruff_cache/",
+    "dist/",
+    "build/",
+    "target/",
+    "node_modules/",
+)
+
+# Event mode must additionally ignore everything DevCouncil writes while
+# recording events (state DB, traces, run artifacts) — otherwise recording a
+# file-change event triggers another filesystem event, feeding back forever.
+_EVENT_IGNORED_PREFIXES = _IGNORED_PREFIXES + (".devcouncil/", ".gitignore")
+
+_TASK_CACHE_TTL_SECONDS = 10.0
+_EVENT_DEBOUNCE_SECONDS = 0.5
+
+
+class FilesystemWatcher:
+    def __init__(
+        self,
+        project_root: Path,
+        task_id: str,
+        *,
+        poll_interval: float = 1.0,
+        on_event: Callable[[dict], None] | None = None,
+    ):
+        self.project_root = project_root.resolve()
+        self.task_id = task_id
+        self.poll_interval = poll_interval
+        self.on_event = on_event
+        self.policy = TaskPolicyEngine(self.project_root)
+        self._seen: dict[str, float] = {}
+        self._task_cache: tuple[float, Task | None] | None = None
+
+    def should_ignore(self, path: str) -> bool:
+        normalized = path.replace("\\", "/")
+        return any(normalized.startswith(prefix) for prefix in _IGNORED_PREFIXES)
+
+    def scan_once(self) -> list[dict]:
+        task = self._load_task()
+        changed = Verifier(self.project_root).get_changed_files()
+        events: list[dict] = []
+        for path in changed:
+            if self.should_ignore(path):
+                continue
+            events.append(self._record_path(path, task, operation="modify"))
+        return events
+
+    def watch(self) -> None:
+        observer = self._start_event_observer()
+        if observer is None:
+            # Polling fallback when watchdog is unavailable.
+            while True:
+                for event in self.scan_once():
+                    self._notify(event)
+                time.sleep(self.poll_interval)
+        try:
+            while True:
+                time.sleep(self.poll_interval)
+        finally:
+            observer.stop()
+            observer.join(timeout=5)
+
+    def _start_event_observer(self):
+        try:
+            from watchdog.events import FileSystemEventHandler
+            from watchdog.observers import Observer
+        except ImportError:
+            return None
+
+        watcher = self
+
+        class _Handler(FileSystemEventHandler):
+            def on_any_event(self, event):
+                if getattr(event, "is_directory", False):
+                    return
+                operation = {"created": "create", "deleted": "delete"}.get(event.event_type, "modify")
+                # For moves, attribute the destination path.
+                path = getattr(event, "dest_path", "") or event.src_path
+                watcher.handle_event(str(path), operation=operation)
+
+        observer = Observer()
+        observer.schedule(_Handler(), str(self.project_root), recursive=True)
+        observer.start()
+        return observer
+
+    def handle_event(self, path: str, *, operation: str = "modify") -> dict | None:
+        """Attribute one raw filesystem event (event-driven mode)."""
+        try:
+            rel = Path(path).resolve().relative_to(self.project_root).as_posix()
+        except (ValueError, OSError):
+            return None
+        if any(rel.startswith(prefix) for prefix in _EVENT_IGNORED_PREFIXES):
+            return None
+        if self._debounced(rel):
+            return None
+        event = self._record_path(rel, self._task_cached(), operation=operation)
+        self._notify(event)
+        return event
+
+    def _notify(self, event: dict) -> None:
+        if self.on_event is not None:
+            self.on_event(event)
+
+    def _debounced(self, rel: str, *, window: float = _EVENT_DEBOUNCE_SECONDS) -> bool:
+        now = time.monotonic()
+        last = self._seen.get(rel)
+        self._seen[rel] = now
+        return last is not None and (now - last) < window
+
+    def _task_cached(self) -> Task | None:
+        now = time.monotonic()
+        if self._task_cache is not None and now - self._task_cache[0] < _TASK_CACHE_TTL_SECONDS:
+            return self._task_cache[1]
+        task = self._load_task()
+        self._task_cache = (now, task)
+        return task
+
+    def _load_task(self) -> Task | None:
+        db = get_db(self.project_root)
+        if not db:
+            return None
+        with db.get_session() as session:
+            return TaskRepository(session).get_by_id(self.task_id)
+
+    def _record_path(self, path: str, task: Task | None, *, operation: str) -> dict:
+        decision = self.policy.evaluate_file_change(path, task, operation=operation)  # type: ignore[arg-type]
+        allowed = decision.action in {"allow", "warn"}
+        db = get_db(self.project_root)
+        if db:
+            # Resolve the lease and record the event in one session so the
+            # recorded lease_id cannot go stale between lookups.
+            with db.get_session() as session:
+                active = TaskLeaseRepository(session).active_for_task(self.task_id)
+                lease_id = active.id if active else None
+                FileChangeRepository(session).record(
+                    path,
+                    operation,
+                    allowed,
+                    task_id=self.task_id,
+                    lease_id=lease_id,
+                    reason=decision.reason,
+                )
+                if not allowed:
+                    gap_repo = GapRepository(session)
+                    gap_repo.save(
+                        Gap(
+                            id=f"GAP-{self.task_id}-ORPHAN-{uuid.uuid4().hex[:12]}",
+                            severity="high",
+                            gap_type="orphan_diff",
+                            task_id=self.task_id,
+                            description=f"Unplanned file change: {path}",
+                            recommended_fix="Revert change or update planned_files.",
+                            blocking=True,
+                        )
+                    )
+        return {"path": path, "operation": operation, "allowed": allowed, "reason": decision.reason}

package/src/devcouncil/execution/handoff.py

@@ -0,0 +1,102 @@
+"""Native agent handoff manifest builder."""
+
+from __future__ import annotations
+
+import uuid
+from datetime import datetime, timezone
+from pathlib import Path
+
+from pydantic import BaseModel, Field
+
+from devcouncil.execution.checkpoints import CheckpointService
+from devcouncil.storage.db import get_db
+from devcouncil.storage.native import AgentHandoffRepository, SemanticDiffRepository, TaskLeaseRepository
+from devcouncil.storage.repositories import EvidenceRepository, GapRepository, RequirementRepository, TaskRepository
+from devcouncil.verification.verifier import Verifier
+
+
+class HandoffManifest(BaseModel):
+    task: dict
+    requirements: list[dict] = Field(default_factory=list)
+    planned_files: list[dict] = Field(default_factory=list)
+    changed_files: list[str] = Field(default_factory=list)
+    semantic_diff: dict | None = None
+    checkpoint_refs: dict[str, str] = Field(default_factory=dict)
+    command_evidence: list[dict] = Field(default_factory=list)
+    open_gaps: list[dict] = Field(default_factory=list)
+    from_agent: str
+    to_agent: str
+    instruction: str = ""
+    created_at: str
+
+
+class HandoffService:
+    def __init__(self, project_root: Path):
+        self.project_root = project_root.resolve()
+
+    def create(
+        self,
+        task_id: str,
+        from_agent: str,
+        to_agent: str,
+        *,
+        instruction: str = "",
+    ) -> tuple[HandoffManifest, Path, str]:
+        db = get_db(self.project_root)
+        if not db:
+            raise RuntimeError("DevCouncil not initialized.")
+        run_id = str(uuid.uuid4())
+        run_dir = self.project_root / ".devcouncil" / "runs" / run_id
+        run_dir.mkdir(parents=True, exist_ok=True)
+        manifest_path = run_dir / "handoff.json"
+
+        with db.get_session() as session:
+            task = TaskRepository(session).get_by_id(task_id)
+            if not task:
+                raise ValueError(f"Task {task_id} not found")
+            reqs = RequirementRepository(session).get_all()
+            gaps = [g for g in GapRepository(session).get_all() if g.task_id == task_id and g.blocking]
+            evidence = EvidenceRepository(session).get_command_results_for_task(task_id)
+            semantic = SemanticDiffRepository(session).latest_for_task(task_id)
+
+        changed = Verifier(self.project_root).get_task_changed_files(task_id)
+        refs = {
+            "before": CheckpointService.REF_BEFORE.format(task_id=task_id),
+            "after": CheckpointService.REF_AFTER.format(task_id=task_id),
+        }
+        manifest = HandoffManifest(
+            task=task.model_dump(),
+            requirements=[r.model_dump() for r in reqs if r.id in task.requirement_ids],
+            planned_files=[pf.model_dump() for pf in task.planned_files],
+            changed_files=changed,
+            semantic_diff={"classifications": semantic.classifications, "summary": semantic.summary} if semantic else None,
+            checkpoint_refs=refs,
+            command_evidence=[ev.model_dump() for ev in evidence],
+            open_gaps=[g.model_dump() for g in gaps],
+            from_agent=from_agent,
+            to_agent=to_agent,
+            instruction=instruction,
+            created_at=datetime.now(timezone.utc).isoformat(),
+        )
+        manifest_path.write_text(manifest.model_dump_json(indent=2), encoding="utf-8")
+
+        status = "manifest_only"
+        with db.get_session() as session:
+            active = TaskLeaseRepository(session).active_for_task(task_id)
+            if active and active.agent == from_agent:
+                TaskLeaseRepository(session).release(task_id, active.lease_token)
+                TaskLeaseRepository(session).acquire(
+                    task_id,
+                    owner=f"handoff:{to_agent}",
+                    agent=to_agent,
+                )
+                status = "lease_transferred"
+            AgentHandoffRepository(session).save(
+                task_id,
+                from_agent,
+                to_agent,
+                run_id,
+                str(manifest_path),
+                status,
+            )
+        return manifest, manifest_path, run_id

package/src/devcouncil/execution/hook_policy.py

@@ -2,10 +2,29 @@ import fnmatch
 import re
 from dataclasses import dataclass
 from pathlib import Path
-from pathlib import PurePosixPath
 from typing import Any, Optional
 
 from devcouncil.domain.task import Task
+from devcouncil.execution.policy_engine import (
+    PROTECTED_WRITE_PATTERNS,
+    SECRET_PATH_PATTERNS,
+    TaskPolicyEngine,
+    normalize_repo_path,
+)
+from devcouncil.utils.redaction import SECRET_PATTERNS
+
+# Splits a shell command into the segments a shell would execute independently, on
+# the chaining/pipe/sequence operators. We deliberately do NOT try to parse quoting
+# perfectly — any operator we miss only makes us evaluate a *larger* segment as a
+# single command (which then fails the allowlist), so this errs toward DENY.
+_SEGMENT_SPLIT_RE = re.compile(r"\s*(?:\|\||&&|\||;|\n)\s*")
+
+# Strips a `bash -c "..."` / `sh -c '...'` wrapper so the inner command is what gets
+# checked against the allowlist, closing the trivial obfuscation where a denied
+# command is smuggled inside a shell wrapper.
+_SHELL_WRAPPER_RE = re.compile(
+    r"""^\s*(?:[A-Za-z0-9_./\\-]*?(?:bash|sh|zsh|dash|ksh))(?:\.exe)?\s+-[A-Za-z]*c\s+(?P<quote>['"])(?P<inner>.*)(?P=quote)\s*$"""
+)
 
 
 @dataclass(frozen=True)
@@ -20,35 +39,60 @@ class HookDecision:
 
 
 class HookPolicy:
-    """Policy-backed hook checks for Claude-style pre-tool-use events."""
+    """Policy-backed hook checks for coding CLI tool-use events."""
 
     def __init__(self, project_root: Path | None = None):
-        self.project_root = project_root.resolve() if project_root else None
-
-    secret_path_patterns = (
-        ".env",
-        ".env.*",
-        "**/.env",
-        "**/.env.*",
-        "**/credentials/**",
-        "**/secrets/**",
-        "**/*.pem",
-        "**/*.key",
-    )
-    protected_path_patterns = (
-        "package.json",
-        "pyproject.toml",
-        "uv.lock",
-        "Dockerfile",
-        "docker-compose.yml",
-        ".github/workflows/*.yml",
-        ".github/workflows/*.yaml",
-        "schema.prisma",
-        "wrangler.toml",
-        "index.html",
-    )
-    write_tools = {"write_file", "edit_file", "replace", "Write", "Edit", "MultiEdit"}
-    shell_tools = {"bash", "shell", "run_command", "Bash"}
+        self.project_root = (project_root or Path(".")).resolve()
+        self.policy_engine = TaskPolicyEngine(
+            self.project_root,
+            global_allowed_commands=self._load_global_allowed_commands(),
+        )
+
+    def _load_global_allowed_commands(self) -> list[str]:
+        """Best-effort load of repo-wide allowed commands from config.
+
+        Never raises — a missing/invalid config must not disable the gate, and the
+        gate stays fail-closed (empty allowlist) when config can't be read."""
+        try:
+            from devcouncil.app.config import load_config
+
+            execution = load_config(self.project_root).execution
+            configured = getattr(execution, "global_allowed_commands", None)
+            if isinstance(configured, (list, tuple)):
+                return [str(item) for item in configured]
+        except Exception:
+            pass
+        return []
+
+    secret_path_patterns = SECRET_PATH_PATTERNS
+    protected_path_patterns = PROTECTED_WRITE_PATTERNS
+    write_tools = {
+        "apply_patch",
+        "edit",
+        "edit_file",
+        "replace",
+        "write",
+        "write_file",
+        "Edit",
+        "MultiEdit",
+        "Write",
+        "create_file",
+        "str_replace",
+        "search_replace",
+    }
+    shell_tools = {
+        "bash",
+        "exec",
+        "exec_command",
+        "local_shell",
+        "run_command",
+        "run_shell_command",
+        "run_terminal_cmd",
+        "shell",
+        "shell_command",
+        "Bash",
+        "Shell",
+    }
 
     def evaluate(self, call_data: dict[str, Any], active_task: Optional[Task]) -> HookDecision:
         tool_name = str(call_data.get("name") or call_data.get("tool_name") or call_data.get("tool") or "")
@@ -58,57 +102,131 @@ class HookPolicy:
 
         if tool_name in self.shell_tools:
             command = self._extract_command(arguments)
-            return self.evaluate_command(command)
+            return self.evaluate_command(command, active_task)
 
         if tool_name in self.write_tools:
             target = self._extract_path(arguments)
-            return self.evaluate_file_write(target, active_task)
+            content = self._extract_content(arguments)
+            return self.evaluate_file_write(target, active_task, content=content)
 
         return HookDecision("allow", "Tool is outside DevCouncil hook policy.")
 
-    def evaluate_command(self, command: str) -> HookDecision:
-        normalized = " ".join(command.split())
-        lowered = normalized.lower()
+    def evaluate_command(self, command: str, active_task: Optional[Task] = None) -> HookDecision:
+        """Evaluate a shell command before execution.
+
+        Two gates, deny wins:
+          1. Git-safety regexes (force push, --no-verify, protected-branch resets).
+          2. The task command allowlist: every segment of a chained command must be
+             authorized by the active task (or the global allowlist). With no active
+             task only the read-only no-task allowlist applies, so a Bash-routed
+             write/destructive command can no longer escape the planned-files gate.
+        This fails closed: anything we cannot positively authorize is denied."""
+        if self.policy_engine is None:
+            return HookDecision("deny", "No project root configured.", command)
+
+        # 1) Git-safety check first — a hard deny here wins regardless of allowlist.
+        git_decision = self.policy_engine.evaluate_hook_command(command)
+        if git_decision.action == "deny":
+            return HookDecision(git_decision.action, git_decision.reason, git_decision.target)
+
+        # 2) Allowlist enforcement over every executed segment.
+        segments = self._split_command_segments(command)
+        if not segments:
+            return HookDecision("deny", "Empty command is not allowed.", command)
+
+        warn: HookDecision | None = None
+        for segment in segments:
+            decision = self.policy_engine.evaluate_command(segment, active_task)
+            if decision.action == "deny":
+                return HookDecision("deny", decision.reason, decision.target)
+            if decision.action == "warn" and warn is None:
+                warn = HookDecision("warn", decision.reason, decision.target)
+
+        # A git-safety warn (e.g. direct push to a protected branch) should surface even
+        # when every segment is otherwise allowed.
+        if git_decision.action == "warn":
+            return HookDecision(git_decision.action, git_decision.reason, git_decision.target)
+        if warn is not None:
+            return warn
+        return HookDecision("allow", "Command authorized by task allowlist.", command)
+
+    def _split_command_segments(self, command: str) -> list[str]:
+        """Split a shell command on ; && || | and newlines, unwrapping bash -c wrappers.
+
+        Each returned segment is itself unwrapped/re-split so a denied command nested
+        inside a `bash -c "..."` wrapper is still checked. Errs toward DENY: anything
+        ambiguous collapses into a larger segment that the allowlist will reject."""
+        normalized = command.strip()
         if not normalized:
-            return HookDecision("allow", "No command detected.")
-
-        if "--no-verify" in lowered or "--no-gpg-sign" in lowered:
-            return HookDecision("deny", "Verification bypass flags are not allowed.", normalized)
-
-        if re.search(r"\bgit\s+reset\s+--hard\s+(origin/)?(main|master)\b", lowered):
-            return HookDecision("deny", "Protected branch hard resets are not allowed.", normalized)
-
-        if re.search(r"\bgit\s+push\b.*(\s--force(?:-with-lease)?\b|\s-f\b)", lowered):
-            return HookDecision("deny", "Force pushes are not allowed.", normalized)
-
-        if re.search(r"\bgit\s+push\s+\S+\s+((head:)?(main|master)|(main|master):\S+)\b", lowered):
-            return HookDecision("warn", "Direct pushes to protected branches should go through verification gates.", normalized)
-
-        return HookDecision("allow", "Command is allowed.", normalized)
-
-    def evaluate_file_write(self, raw_path: Optional[str], active_task: Optional[Task]) -> HookDecision:
+            return []
+
+        wrapper = _SHELL_WRAPPER_RE.match(normalized)
+        if wrapper is not None:
+            # Recurse into the wrapped command so its own chaining is evaluated.
+            return self._split_command_segments(wrapper.group("inner"))
+
+        segments: list[str] = []
+        for raw in _SEGMENT_SPLIT_RE.split(normalized):
+            piece = raw.strip()
+            if not piece:
+                continue
+            nested = _SHELL_WRAPPER_RE.match(piece)
+            if nested is not None:
+                segments.extend(self._split_command_segments(nested.group("inner")))
+            else:
+                segments.append(piece)
+        return segments
+
+    def evaluate_file_write(
+        self,
+        raw_path: Optional[str],
+        active_task: Optional[Task],
+        *,
+        content: Optional[str] = None,
+    ) -> HookDecision:
         if not raw_path:
             return HookDecision("allow", "No file path detected.")
-
-        path = self._normalize_path(raw_path)
-        if self._matches_any(path, self.secret_path_patterns):
-            return HookDecision("deny", "Secret and credential paths are never writable through hooks.", path)
-
-        if active_task is None:
-            return HookDecision("deny", "No running DevCouncil task authorizes this file write.", path)
-
-        if active_task and not self._is_planned_file(path, active_task):
-            return HookDecision("deny", f"Task {active_task.id} does not authorize changes to {path}.", path)
-
-        if self._matches_any(path, self.protected_path_patterns):
-            return HookDecision("warn", f"{path} is a protected high-impact file; verification gates must approve it.", path)
-
-        return HookDecision("allow", "File write is allowed.", path)
+        if self.policy_engine is None:
+            return HookDecision("deny", "No project root configured.", raw_path)
+
+        # Pre-action secret scan: refuse to write content that contains a secret,
+        # before it ever lands on disk. Reuses the shared secret regexes.
+        if content is not None:
+            secret = self._scan_content_for_secret(content)
+            if secret is not None:
+                return HookDecision(
+                    "deny",
+                    f"Refusing to write content containing a potential {secret}.",
+                    raw_path,
+                )
+
+        # Delegate to the engine, which normalizes via the shared normalize_repo_path and
+        # denies out-of-root targets — so the path that is checked is the path enforced.
+        decision = self.policy_engine.evaluate_file_change(raw_path, active_task)
+        return HookDecision(decision.action, decision.reason, decision.target)
+
+    def _scan_content_for_secret(self, content: str) -> Optional[str]:
+        if not isinstance(content, str) or not content:
+            return None
+        for key_type, pattern in SECRET_PATTERNS.items():
+            if pattern.search(content):
+                return key_type
+        return None
 
     def _extract_command(self, arguments: dict[str, Any]) -> str:
         value = arguments.get("command") or arguments.get("cmd") or arguments.get("script") or ""
         return str(value)
 
+    def _extract_content(self, arguments: dict[str, Any]) -> Optional[str]:
+        """Pull the to-be-written content from a write tool's arguments.
+
+        Covers the common shapes across coding CLIs (content/new_str/new_string/text)."""
+        for key in ("content", "new_str", "new_string", "text", "file_text", "contents"):
+            value = arguments.get(key)
+            if isinstance(value, str) and value:
+                return value
+        return None
+
     def _extract_path(self, arguments: dict[str, Any]) -> Optional[str]:
         value = (
             arguments.get("path")
@@ -116,22 +234,13 @@ class HookPolicy:
             or arguments.get("filepath")
             or arguments.get("filePath")
             or arguments.get("target")
+            or arguments.get("target_file")
         )
         return str(value) if value else None
 
     def _normalize_path(self, raw_path: str) -> str:
-        path = raw_path.strip().strip('"').replace("\\", "/")
-        if self.project_root:
-            try:
-                candidate = Path(path)
-                resolved = candidate.resolve() if candidate.is_absolute() else (self.project_root / path).resolve()
-                return resolved.relative_to(self.project_root).as_posix()
-            except (OSError, ValueError):
-                pass
-        if re.match(r"^[A-Za-z]:/", path):
-            parts = PurePosixPath(path).parts
-            path = "/".join(parts[1:])
-        return path[2:] if path.startswith("./") else path
+        # Shared normalizer — single source of truth with TaskPolicyEngine.
+        return normalize_repo_path(self.project_root, raw_path)[0]
 
     def _is_planned_file(self, path: str, task: Task) -> bool:
         for planned in task.planned_files: