devcouncil 0.1.1 → 0.2.0

package/src/devcouncil/execution/checkpoints.py

@@ -0,0 +1,246 @@
+"""Git-native checkpoint service with legacy patch compatibility."""
+
+from __future__ import annotations
+
+import json
+import logging
+import os
+import subprocess
+import tempfile
+import uuid
+from pathlib import Path
+
+from pydantic import BaseModel
+
+from devcouncil.verification.verifier import Verifier
+
+logger = logging.getLogger(__name__)
+
+
+class CheckpointResult(BaseModel):
+    task_id: str
+    ref: str | None = None
+    patch_path: str | None = None
+    json_path: str | None = None
+    git_ref_created: bool = False
+    message: str = ""
+
+
+class CheckpointService:
+    REF_BEFORE = "refs/devcouncil/tasks/{task_id}/before"
+    REF_AFTER = "refs/devcouncil/tasks/{task_id}/after"
+    REF_ATTEMPT = "refs/devcouncil/tasks/{task_id}/attempts/{attempt}"
+
+    def __init__(self, project_root: Path):
+        self.project_root = project_root.resolve()
+        self.checkpoint_dir = self.project_root / ".devcouncil" / "checkpoints"
+        self.checkpoint_dir.mkdir(parents=True, exist_ok=True)
+
+    def create_before(self, task_id: str) -> CheckpointResult:
+        return self._create(task_id, stage="before")
+
+    def create_after(self, task_id: str) -> CheckpointResult:
+        return self._create(task_id, stage="after")
+
+    def create_attempt(self, task_id: str, attempt: int) -> CheckpointResult:
+        ref_template = self.REF_ATTEMPT.format(task_id=task_id, attempt=attempt)
+        return self._create(task_id, stage="attempt", ref_name=ref_template)
+
+    def rollback(self, task_id: str) -> CheckpointResult:
+        before_ref = self.REF_BEFORE.format(task_id=task_id)
+        after_ref = self.REF_AFTER.format(task_id=task_id)
+        after_patch = self.checkpoint_dir / f"{task_id}-after.patch"
+
+        if self._ref_exists(after_ref) and self._ref_exists(before_ref):
+            try:
+                diff = subprocess.check_output(
+                    ["git", "diff", before_ref, after_ref],
+                    cwd=self.project_root,
+                    text=True,
+                    encoding="utf-8",
+                    errors="replace",
+                )
+                if diff.strip():
+                    subprocess.run(
+                        ["git", "apply", "-R", "--whitespace=nowarn"],
+                        cwd=self.project_root,
+                        input=diff,
+                        text=True,
+                        check=True,
+                    )
+                    return CheckpointResult(
+                        task_id=task_id,
+                        ref=after_ref,
+                        git_ref_created=True,
+                        message="Rolled back using git refs.",
+                    )
+            except subprocess.CalledProcessError as exc:
+                return CheckpointResult(
+                    task_id=task_id,
+                    message=f"Git ref rollback failed: {exc}",
+                )
+
+        if after_patch.exists():
+            try:
+                subprocess.check_call(
+                    ["git", "apply", "-R", str(after_patch)],
+                    cwd=self.project_root,
+                )
+                return CheckpointResult(
+                    task_id=task_id,
+                    patch_path=str(after_patch),
+                    message="Rolled back using after patch.",
+                )
+            except subprocess.CalledProcessError as exc:
+                return CheckpointResult(
+                    task_id=task_id,
+                    patch_path=str(after_patch),
+                    message=f"Patch rollback failed: {exc}",
+                )
+
+        return CheckpointResult(
+            task_id=task_id,
+            message="No checkpoint refs or after patch found.",
+        )
+
+    def import_legacy_patch(self, task_id: str) -> CheckpointResult:
+        before_patch = self.checkpoint_dir / f"{task_id}-before.patch"
+        if not before_patch.exists():
+            return CheckpointResult(
+                task_id=task_id,
+                message="No legacy before patch to import.",
+            )
+        ref = self.REF_BEFORE.format(task_id=task_id)
+        created = self._update_ref(ref)
+        return CheckpointResult(
+            task_id=task_id,
+            ref=ref if created else None,
+            patch_path=str(before_patch),
+            git_ref_created=created,
+            message="Imported legacy before patch ref when possible.",
+        )
+
+    def _create(
+        self,
+        task_id: str,
+        *,
+        stage: str,
+        ref_name: str | None = None,
+    ) -> CheckpointResult:
+        ref = ref_name or (
+            self.REF_BEFORE.format(task_id=task_id)
+            if stage == "before"
+            else self.REF_AFTER.format(task_id=task_id)
+        )
+        patch_path = self.checkpoint_dir / f"{task_id}-{stage}.patch"
+        json_path: str | None = None
+
+        # Point the ref at a snapshot of the *working tree* (the task's actual state),
+        # not bare HEAD — otherwise before/after refs both resolve to the same commit and
+        # the git-ref rollback path can never fire. Falls back to HEAD if snapshotting is
+        # impossible (unborn HEAD / not a git repo), preserving the patch-based rollback.
+        git_ref_created = self._update_ref(ref, self._snapshot_commit())
+        try:
+            diff = Verifier(self.project_root).get_diff()
+            if diff:
+                patch_path.write_text(diff, encoding="utf-8")
+        except Exception as exc:
+            # Without a patch (and if the ref also failed) rollback is impossible —
+            # never let this fail silently.
+            logger.warning("Failed to capture %s checkpoint patch for %s: %s", stage, task_id, exc)
+
+        if stage == "before":
+            snapshot = {
+                "task_id": task_id,
+                "changed_files": Verifier(self.project_root).get_changed_files(),
+            }
+            snapshot_path = self.checkpoint_dir / f"{task_id}-before.json"
+            snapshot_path.write_text(json.dumps(snapshot, indent=2), encoding="utf-8")
+            json_path = str(snapshot_path)
+
+        return CheckpointResult(
+            task_id=task_id,
+            ref=ref if git_ref_created else None,
+            patch_path=str(patch_path) if patch_path.exists() else None,
+            json_path=json_path,
+            git_ref_created=git_ref_created,
+            message=f"Checkpoint {stage} created.",
+        )
+
+    def _update_ref(self, ref: str, commit: str | None = None) -> bool:
+        """Point ``ref`` at ``commit`` (a working-tree snapshot) or, if None, at HEAD."""
+        try:
+            target = commit
+            if target is None:
+                target = subprocess.check_output(
+                    ["git", "rev-parse", "HEAD"],
+                    cwd=self.project_root,
+                    text=True,
+                    encoding="utf-8",
+                    errors="replace",
+                ).strip()
+            if not target:
+                return False
+            subprocess.check_call(
+                ["git", "update-ref", ref, target],
+                cwd=self.project_root,
+            )
+            return True
+        except (subprocess.CalledProcessError, FileNotFoundError):
+            return False
+
+    def _snapshot_commit(self) -> str | None:
+        """Create a commit object capturing the FULL working tree (tracked + untracked,
+        honoring .gitignore) WITHOUT touching the user's index or working tree.
+
+        Uses a throwaway ``GIT_INDEX_FILE`` so staging happens in isolation. Returns the
+        commit sha, or None if it can't be built (unborn HEAD, not a git repo) so the
+        caller falls back to a HEAD ref + the patch-based rollback."""
+        # The temp index lives OUTSIDE the repo: if it sat under the working tree (even in
+        # the gitignored .devcouncil/), `git add -A` could stage the index file itself into
+        # the snapshot. ``.devcouncil`` is excluded from staging for the same reason —
+        # DevCouncil's own run state must never be snapshotted or rolled back.
+        index_path = Path(tempfile.gettempdir()) / f"devcouncil-snapshot-index-{uuid.uuid4().hex}"
+        env = {**os.environ, "GIT_INDEX_FILE": str(index_path)}
+        try:
+            # Seed the temp index from HEAD, stage every change (incl. new files), then
+            # write a tree + commit from that isolated index.
+            subprocess.run(
+                ["git", "read-tree", "HEAD"],
+                cwd=self.project_root, env=env, check=True, capture_output=True, text=True,
+            )
+            subprocess.run(
+                ["git", "add", "-A", "--", ".", ":(exclude).devcouncil"],
+                cwd=self.project_root, env=env, check=True, capture_output=True, text=True,
+            )
+            tree = subprocess.check_output(
+                ["git", "write-tree"],
+                cwd=self.project_root, env=env, text=True, encoding="utf-8", errors="replace",
+            ).strip()
+            if not tree:
+                return None
+            commit = subprocess.check_output(
+                ["git", "-c", "user.name=DevCouncil", "-c", "user.email=devcouncil@local",
+                 "commit-tree", tree, "-p", "HEAD", "-m", "devcouncil checkpoint"],
+                cwd=self.project_root, text=True, encoding="utf-8", errors="replace",
+            ).strip()
+            return commit or None
+        except (subprocess.CalledProcessError, FileNotFoundError):
+            return None
+        finally:
+            try:
+                index_path.unlink()
+            except OSError:
+                pass
+
+    def _ref_exists(self, ref: str) -> bool:
+        try:
+            subprocess.check_output(
+                ["git", "rev-parse", "--verify", ref],
+                cwd=self.project_root,
+                stderr=subprocess.DEVNULL,
+                text=True,
+            )
+            return True
+        except (subprocess.CalledProcessError, FileNotFoundError):
+            return False

package/src/devcouncil/execution/context_builder.py

@@ -41,7 +41,7 @@ class ContextBuilder:
         
         return json.dumps(context, indent=2)
 
-    def get_structure_summary(self, task: Task = None) -> List[str]:
+    def get_structure_summary(self, task: Task | None = None) -> List[str]:
         """Simple list of files in the project for context."""
         try:
             import subprocess

package/src/devcouncil/execution/fs_watcher.py

@@ -0,0 +1,180 @@
+"""Filesystem change attribution with polling fallback."""
+
+from __future__ import annotations
+
+import time
+import uuid
+from pathlib import Path
+from typing import Callable
+
+from devcouncil.domain.gap import Gap
+from devcouncil.domain.task import Task
+from devcouncil.execution.policy_engine import TaskPolicyEngine
+from devcouncil.storage.db import get_db
+from devcouncil.storage.native import FileChangeRepository, TaskLeaseRepository
+from devcouncil.storage.repositories import GapRepository, TaskRepository
+from devcouncil.verification.verifier import Verifier
+
+_IGNORED_PREFIXES = (
+    ".git/",
+    ".devcouncil/cache/",
+    ".devcouncil/logs/",
+    "__pycache__/",
+    ".pytest_cache/",
+    ".mypy_cache/",
+    ".ruff_cache/",
+    "dist/",
+    "build/",
+    "target/",
+    "node_modules/",
+)
+
+# Event mode must additionally ignore everything DevCouncil writes while
+# recording events (state DB, traces, run artifacts) — otherwise recording a
+# file-change event triggers another filesystem event, feeding back forever.
+_EVENT_IGNORED_PREFIXES = _IGNORED_PREFIXES + (".devcouncil/", ".gitignore")
+
+_TASK_CACHE_TTL_SECONDS = 10.0
+_EVENT_DEBOUNCE_SECONDS = 0.5
+
+
+class FilesystemWatcher:
+    def __init__(
+        self,
+        project_root: Path,
+        task_id: str,
+        *,
+        poll_interval: float = 1.0,
+        on_event: Callable[[dict], None] | None = None,
+    ):
+        self.project_root = project_root.resolve()
+        self.task_id = task_id
+        self.poll_interval = poll_interval
+        self.on_event = on_event
+        self.policy = TaskPolicyEngine(self.project_root)
+        self._seen: dict[str, float] = {}
+        self._task_cache: tuple[float, Task | None] | None = None
+
+    def should_ignore(self, path: str) -> bool:
+        normalized = path.replace("\\", "/")
+        return any(normalized.startswith(prefix) for prefix in _IGNORED_PREFIXES)
+
+    def scan_once(self) -> list[dict]:
+        task = self._load_task()
+        changed = Verifier(self.project_root).get_changed_files()
+        events: list[dict] = []
+        for path in changed:
+            if self.should_ignore(path):
+                continue
+            events.append(self._record_path(path, task, operation="modify"))
+        return events
+
+    def watch(self) -> None:
+        observer = self._start_event_observer()
+        if observer is None:
+            # Polling fallback when watchdog is unavailable.
+            while True:
+                for event in self.scan_once():
+                    self._notify(event)
+                time.sleep(self.poll_interval)
+        try:
+            while True:
+                time.sleep(self.poll_interval)
+        finally:
+            observer.stop()
+            observer.join(timeout=5)
+
+    def _start_event_observer(self):
+        try:
+            from watchdog.events import FileSystemEventHandler
+            from watchdog.observers import Observer
+        except ImportError:
+            return None
+
+        watcher = self
+
+        class _Handler(FileSystemEventHandler):
+            def on_any_event(self, event):
+                if getattr(event, "is_directory", False):
+                    return
+                operation = {"created": "create", "deleted": "delete"}.get(event.event_type, "modify")
+                # For moves, attribute the destination path.
+                path = getattr(event, "dest_path", "") or event.src_path
+                watcher.handle_event(str(path), operation=operation)
+
+        observer = Observer()
+        observer.schedule(_Handler(), str(self.project_root), recursive=True)
+        observer.start()
+        return observer
+
+    def handle_event(self, path: str, *, operation: str = "modify") -> dict | None:
+        """Attribute one raw filesystem event (event-driven mode)."""
+        try:
+            rel = Path(path).resolve().relative_to(self.project_root).as_posix()
+        except (ValueError, OSError):
+            return None
+        if any(rel.startswith(prefix) for prefix in _EVENT_IGNORED_PREFIXES):
+            return None
+        if self._debounced(rel):
+            return None
+        event = self._record_path(rel, self._task_cached(), operation=operation)
+        self._notify(event)
+        return event
+
+    def _notify(self, event: dict) -> None:
+        if self.on_event is not None:
+            self.on_event(event)
+
+    def _debounced(self, rel: str, *, window: float = _EVENT_DEBOUNCE_SECONDS) -> bool:
+        now = time.monotonic()
+        last = self._seen.get(rel)
+        self._seen[rel] = now
+        return last is not None and (now - last) < window
+
+    def _task_cached(self) -> Task | None:
+        now = time.monotonic()
+        if self._task_cache is not None and now - self._task_cache[0] < _TASK_CACHE_TTL_SECONDS:
+            return self._task_cache[1]
+        task = self._load_task()
+        self._task_cache = (now, task)
+        return task
+
+    def _load_task(self) -> Task | None:
+        db = get_db(self.project_root)
+        if not db:
+            return None
+        with db.get_session() as session:
+            return TaskRepository(session).get_by_id(self.task_id)
+
+    def _record_path(self, path: str, task: Task | None, *, operation: str) -> dict:
+        decision = self.policy.evaluate_file_change(path, task, operation=operation)  # type: ignore[arg-type]
+        allowed = decision.action in {"allow", "warn"}
+        db = get_db(self.project_root)
+        if db:
+            # Resolve the lease and record the event in one session so the
+            # recorded lease_id cannot go stale between lookups.
+            with db.get_session() as session:
+                active = TaskLeaseRepository(session).active_for_task(self.task_id)
+                lease_id = active.id if active else None
+                FileChangeRepository(session).record(
+                    path,
+                    operation,
+                    allowed,
+                    task_id=self.task_id,
+                    lease_id=lease_id,
+                    reason=decision.reason,
+                )
+                if not allowed:
+                    gap_repo = GapRepository(session)
+                    gap_repo.save(
+                        Gap(
+                            id=f"GAP-{self.task_id}-ORPHAN-{uuid.uuid4().hex[:12]}",
+                            severity="high",
+                            gap_type="orphan_diff",
+                            task_id=self.task_id,
+                            description=f"Unplanned file change: {path}",
+                            recommended_fix="Revert change or update planned_files.",
+                            blocking=True,
+                        )
+                    )
+        return {"path": path, "operation": operation, "allowed": allowed, "reason": decision.reason}

package/src/devcouncil/execution/handoff.py

@@ -0,0 +1,102 @@
+"""Native agent handoff manifest builder."""
+
+from __future__ import annotations
+
+import uuid
+from datetime import datetime, timezone
+from pathlib import Path
+
+from pydantic import BaseModel, Field
+
+from devcouncil.execution.checkpoints import CheckpointService
+from devcouncil.storage.db import get_db
+from devcouncil.storage.native import AgentHandoffRepository, SemanticDiffRepository, TaskLeaseRepository
+from devcouncil.storage.repositories import EvidenceRepository, GapRepository, RequirementRepository, TaskRepository
+from devcouncil.verification.verifier import Verifier
+
+
+class HandoffManifest(BaseModel):
+    task: dict
+    requirements: list[dict] = Field(default_factory=list)
+    planned_files: list[dict] = Field(default_factory=list)
+    changed_files: list[str] = Field(default_factory=list)
+    semantic_diff: dict | None = None
+    checkpoint_refs: dict[str, str] = Field(default_factory=dict)
+    command_evidence: list[dict] = Field(default_factory=list)
+    open_gaps: list[dict] = Field(default_factory=list)
+    from_agent: str
+    to_agent: str
+    instruction: str = ""
+    created_at: str
+
+
+class HandoffService:
+    def __init__(self, project_root: Path):
+        self.project_root = project_root.resolve()
+
+    def create(
+        self,
+        task_id: str,
+        from_agent: str,
+        to_agent: str,
+        *,
+        instruction: str = "",
+    ) -> tuple[HandoffManifest, Path, str]:
+        db = get_db(self.project_root)
+        if not db:
+            raise RuntimeError("DevCouncil not initialized.")
+        run_id = str(uuid.uuid4())
+        run_dir = self.project_root / ".devcouncil" / "runs" / run_id
+        run_dir.mkdir(parents=True, exist_ok=True)
+        manifest_path = run_dir / "handoff.json"
+
+        with db.get_session() as session:
+            task = TaskRepository(session).get_by_id(task_id)
+            if not task:
+                raise ValueError(f"Task {task_id} not found")
+            reqs = RequirementRepository(session).get_all()
+            gaps = [g for g in GapRepository(session).get_all() if g.task_id == task_id and g.blocking]
+            evidence = EvidenceRepository(session).get_command_results_for_task(task_id)
+            semantic = SemanticDiffRepository(session).latest_for_task(task_id)
+
+        changed = Verifier(self.project_root).get_task_changed_files(task_id)
+        refs = {
+            "before": CheckpointService.REF_BEFORE.format(task_id=task_id),
+            "after": CheckpointService.REF_AFTER.format(task_id=task_id),
+        }
+        manifest = HandoffManifest(
+            task=task.model_dump(),
+            requirements=[r.model_dump() for r in reqs if r.id in task.requirement_ids],
+            planned_files=[pf.model_dump() for pf in task.planned_files],
+            changed_files=changed,
+            semantic_diff={"classifications": semantic.classifications, "summary": semantic.summary} if semantic else None,
+            checkpoint_refs=refs,
+            command_evidence=[ev.model_dump() for ev in evidence],
+            open_gaps=[g.model_dump() for g in gaps],
+            from_agent=from_agent,
+            to_agent=to_agent,
+            instruction=instruction,
+            created_at=datetime.now(timezone.utc).isoformat(),
+        )
+        manifest_path.write_text(manifest.model_dump_json(indent=2), encoding="utf-8")
+
+        status = "manifest_only"
+        with db.get_session() as session:
+            active = TaskLeaseRepository(session).active_for_task(task_id)
+            if active and active.agent == from_agent:
+                TaskLeaseRepository(session).release(task_id, active.lease_token)
+                TaskLeaseRepository(session).acquire(
+                    task_id,
+                    owner=f"handoff:{to_agent}",
+                    agent=to_agent,
+                )
+                status = "lease_transferred"
+            AgentHandoffRepository(session).save(
+                task_id,
+                from_agent,
+                to_agent,
+                run_id,
+                str(manifest_path),
+                status,
+            )
+        return manifest, manifest_path, run_id