devcouncil 0.3.1 → 0.4.0

package/src/devcouncil/domain/checkpoint_refs.py

@@ -0,0 +1,17 @@
+"""Git ref names for per-task checkpoints.
+
+Owned by the domain layer so the writer (execution.checkpoints.CheckpointService)
+and the reader (verification.verifier.Verifier) share one definition of the ref
+layout without importing each other — checkpoints.py imports Verifier, so the
+verifier must not import checkpoints back. This module must stay dependency-free
+(no imports from execution/ or verification/).
+"""
+
+TASK_REF_PREFIX = "refs/devcouncil/tasks"
+
+REF_BEFORE_TEMPLATE = TASK_REF_PREFIX + "/{task_id}/before"
+
+
+def task_before_ref(task_id: str) -> str:
+    """Ref pointing at the snapshot taken before ``task_id`` started."""
+    return REF_BEFORE_TEMPLATE.format(task_id=task_id)

package/src/devcouncil/domain/evidence.py

@@ -7,6 +7,7 @@ class CommandResult(BaseModel):
     stdout_path: str
     stderr_path: str
     summary: str
+    timed_out: bool = False
 
 class DiffEvidence(BaseModel):
     task_id: str

package/src/devcouncil/domain/gap.py

@@ -22,6 +22,16 @@ class Gap(BaseModel):
         "quality_gate_failed",
         "skipped_verification_command",
         "coarse_acceptance_proof",
+        "stub_detected",
+        "stub_declared",
+        "suspicious_effort",
+        "unwired_file",
+        "dead_symbol",
+        "stranded_code",
+        "stale_map",
+        "corpus_stale",
+        "doc_code_ref",
+        "acceptance_corpus",
     ]
     requirement_id: Optional[str] = None
     task_id: Optional[str] = None

package/src/devcouncil/domain/requirement.py

@@ -18,5 +18,9 @@ class Requirement(BaseModel):
     title: str
     description: str
     priority: Literal["low", "medium", "high", "critical"]
-    source: Literal["user", "planner", "critic", "arbiter"]
+    # Defaulted: requirements come back through LLM round-trips (arbiter/critic
+    # rewrites) where weaker models sometimes drop provenance metadata; a missing
+    # source shouldn't invalidate an otherwise-sound requirement. "planner" is the
+    # most common origin for machine-generated requirements.
+    source: Literal["user", "planner", "critic", "arbiter"] = "planner"
     acceptance_criteria: List[AcceptanceCriterion] = Field(default_factory=list)

package/src/devcouncil/domain/task.py

@@ -1,5 +1,5 @@
 from pydantic import BaseModel, Field
-from typing import Literal, List
+from typing import Literal, List, Optional
 
 class PlannedFile(BaseModel):
     path: str
@@ -27,6 +27,14 @@ class Task(BaseModel):
             "repo is already configured for them."
         ),
     )
+    agent_appended_expected_tests: List[str] = Field(
+        default_factory=list,
+        description=(
+            "Subset of expected_tests appended at runtime by a leased agent via "
+            "devcouncil_update_task_scope. These may run during verification but "
+            "cannot coarse-prove acceptance criteria."
+        ),
+    )
     allowed_commands: List[str] = Field(
         default_factory=list,
         description=(
@@ -36,7 +44,39 @@ class Task(BaseModel):
             "only when the task needs no commands beyond those in expected_tests."
         ),
     )
+    agent_appended_allowed_commands: List[str] = Field(
+        default_factory=list,
+        description=(
+            "Subset of allowed_commands appended at runtime by a leased agent via "
+            "devcouncil_update_task_scope. These may run during execution/verification "
+            "but cannot coarse-prove acceptance criteria (self-certification guard, "
+            "mirroring agent_appended_expected_tests)."
+        ),
+    )
+    agent_appended_planned_files: List[str] = Field(
+        default_factory=list,
+        description=(
+            "Paths appended to planned_files at runtime by a leased agent via "
+            "devcouncil_update_task_scope / `dev scope update --planned-file`. "
+            "Always added as modify-op only; secret/restricted paths are rejected. "
+            "Lets an agent wire an unwired file into its intended caller without "
+            "hitting orphan_diff / write-policy deadlock."
+        ),
+    )
     forbidden_changes: List[str] = Field(default_factory=list)
+    difficulty: Optional[Literal["easy", "normal", "hard"]] = Field(
+        default=None,
+        description=(
+            "Manual difficulty override. When set (by a planner or a human) it wins over "
+            "the deterministic estimator in devcouncil.verification.difficulty; hard tasks "
+            "get stricter verification (stub/effort gates block, coverage enforced) per "
+            "the verification.rigor config."
+        ),
+    )
+    priority: Optional[Literal["high", "medium", "low"]] = Field(
+        default=None,
+        description="Optional human/planner priority hint (high/medium/low). Default None.",
+    )
     depends_on: List[str] = Field(
         default_factory=list,
         description=(
@@ -51,5 +91,6 @@ class Task(BaseModel):
         "running",
         "blocked",
         "verified",
-        "done"
+        "done",
+        "cancelled",
     ] = "planned"

package/src/devcouncil/execution/checkpoints.py

@@ -2,7 +2,6 @@
 
 from __future__ import annotations
 
-import json
 import logging
 import os
 import subprocess
@@ -12,6 +11,10 @@ from pathlib import Path
 
 from pydantic import BaseModel
 
+from devcouncil.domain.checkpoint_refs import REF_BEFORE_TEMPLATE
+from devcouncil.utils.fsio import atomic_write_text
+from devcouncil.utils.proc import GIT_TIMEOUT
+from devcouncil.utils.json_persist import write_json
 from devcouncil.verification.verifier import Verifier
 
 logger = logging.getLogger(__name__)
@@ -27,9 +30,10 @@ class CheckpointResult(BaseModel):
 
 
 class CheckpointService:
-    REF_BEFORE = "refs/devcouncil/tasks/{task_id}/before"
+    # The before-ref layout lives in devcouncil.domain.checkpoint_refs because the
+    # Verifier reads it too; kept as a class attr for compat with .format() callers.
+    REF_BEFORE = REF_BEFORE_TEMPLATE
     REF_AFTER = "refs/devcouncil/tasks/{task_id}/after"
-    REF_ATTEMPT = "refs/devcouncil/tasks/{task_id}/attempts/{attempt}"
 
     def __init__(self, project_root: Path):
         self.project_root = project_root.resolve()
@@ -42,10 +46,6 @@ class CheckpointService:
     def create_after(self, task_id: str) -> CheckpointResult:
         return self._create(task_id, stage="after")
 
-    def create_attempt(self, task_id: str, attempt: int) -> CheckpointResult:
-        ref_template = self.REF_ATTEMPT.format(task_id=task_id, attempt=attempt)
-        return self._create(task_id, stage="attempt", ref_name=ref_template)
-
     def rollback(self, task_id: str) -> CheckpointResult:
         logger.info("Rollback requested for %s", task_id)
         before_ref = self.REF_BEFORE.format(task_id=task_id)
@@ -60,6 +60,7 @@ class CheckpointService:
                     text=True,
                     encoding="utf-8",
                     errors="replace",
+                    timeout=GIT_TIMEOUT,
                 )
                 if diff.strip():
                     subprocess.run(
@@ -68,6 +69,7 @@ class CheckpointService:
                         input=diff,
                         text=True,
                         check=True,
+                        timeout=GIT_TIMEOUT,
                     )
                     return CheckpointResult(
                         task_id=task_id,
@@ -75,7 +77,7 @@ class CheckpointService:
                         git_ref_created=True,
                         message="Rolled back using git refs.",
                     )
-            except subprocess.CalledProcessError as exc:
+            except (subprocess.CalledProcessError, subprocess.TimeoutExpired) as exc:
                 return CheckpointResult(
                     task_id=task_id,
                     message=f"Git ref rollback failed: {exc}",
@@ -86,13 +88,14 @@ class CheckpointService:
                 subprocess.check_call(
                     ["git", "apply", "-R", str(after_patch)],
                     cwd=self.project_root,
+                    timeout=GIT_TIMEOUT,
                 )
                 return CheckpointResult(
                     task_id=task_id,
                     patch_path=str(after_patch),
                     message="Rolled back using after patch.",
                 )
-            except subprocess.CalledProcessError as exc:
+            except (subprocess.CalledProcessError, subprocess.TimeoutExpired) as exc:
                 return CheckpointResult(
                     task_id=task_id,
                     patch_path=str(after_patch),
@@ -104,23 +107,6 @@ class CheckpointService:
             message="No checkpoint refs or after patch found.",
         )
 
-    def import_legacy_patch(self, task_id: str) -> CheckpointResult:
-        before_patch = self.checkpoint_dir / f"{task_id}-before.patch"
-        if not before_patch.exists():
-            return CheckpointResult(
-                task_id=task_id,
-                message="No legacy before patch to import.",
-            )
-        ref = self.REF_BEFORE.format(task_id=task_id)
-        created = self._update_ref(ref)
-        return CheckpointResult(
-            task_id=task_id,
-            ref=ref if created else None,
-            patch_path=str(before_patch),
-            git_ref_created=created,
-            message="Imported legacy before patch ref when possible.",
-        )
-
     def _create(
         self,
         task_id: str,
@@ -145,7 +131,7 @@ class CheckpointService:
         try:
             diff = verifier.get_diff()
             if diff:
-                patch_path.write_text(diff, encoding="utf-8")
+                atomic_write_text(patch_path, diff)
         except Exception as exc:
             # Without a patch (and if the ref also failed) rollback is impossible —
             # never let this fail silently.
@@ -157,7 +143,7 @@ class CheckpointService:
                 "changed_files": verifier.get_changed_files(),
             }
             snapshot_path = self.checkpoint_dir / f"{task_id}-before.json"
-            snapshot_path.write_text(json.dumps(snapshot, indent=2), encoding="utf-8")
+            write_json(snapshot_path, snapshot)
             json_path = str(snapshot_path)
 
         # Routine bookkeeping (fires before+after every task and repair attempt) — DEBUG
@@ -188,15 +174,17 @@ class CheckpointService:
                     text=True,
                     encoding="utf-8",
                     errors="replace",
+                    timeout=GIT_TIMEOUT,
                 ).strip()
             if not target:
                 return False
             subprocess.check_call(
                 ["git", "update-ref", ref, target],
                 cwd=self.project_root,
+                timeout=GIT_TIMEOUT,
             )
             return True
-        except (subprocess.CalledProcessError, FileNotFoundError):
+        except (subprocess.CalledProcessError, subprocess.TimeoutExpired, FileNotFoundError):
             return False
 
     def _snapshot_commit(self) -> str | None:
@@ -218,14 +206,18 @@ class CheckpointService:
             subprocess.run(
                 ["git", "read-tree", "HEAD"],
                 cwd=self.project_root, env=env, check=True, capture_output=True, text=True,
+                timeout=GIT_TIMEOUT,
             )
+            # Staging the full tree scales with repo size — give it extra room.
             subprocess.run(
                 ["git", "add", "-A", "--", ".", ":(exclude).devcouncil"],
                 cwd=self.project_root, env=env, check=True, capture_output=True, text=True,
+                timeout=GIT_TIMEOUT * 2,
             )
             tree = subprocess.check_output(
                 ["git", "write-tree"],
                 cwd=self.project_root, env=env, text=True, encoding="utf-8", errors="replace",
+                timeout=GIT_TIMEOUT,
             ).strip()
             if not tree:
                 return None
@@ -233,9 +225,10 @@ class CheckpointService:
                 ["git", "-c", "user.name=DevCouncil", "-c", "user.email=devcouncil@local",
                  "commit-tree", tree, "-p", "HEAD", "-m", "devcouncil checkpoint"],
                 cwd=self.project_root, text=True, encoding="utf-8", errors="replace",
+                timeout=GIT_TIMEOUT,
             ).strip()
             return commit or None
-        except (subprocess.CalledProcessError, FileNotFoundError):
+        except (subprocess.CalledProcessError, subprocess.TimeoutExpired, FileNotFoundError):
             return None
         finally:
             try:
@@ -250,7 +243,8 @@ class CheckpointService:
                 cwd=self.project_root,
                 stderr=subprocess.DEVNULL,
                 text=True,
+                timeout=GIT_TIMEOUT,
             )
             return True
-        except (subprocess.CalledProcessError, FileNotFoundError):
+        except (subprocess.CalledProcessError, subprocess.TimeoutExpired, FileNotFoundError):
             return False

package/src/devcouncil/execution/context_builder.py

@@ -1,60 +1,31 @@
 from pathlib import Path
 from typing import List
+
 from devcouncil.domain.task import Task
-from devcouncil.domain.requirement import Requirement
-from devcouncil.utils.redaction import redact_string
-import json
+
 
 class ContextBuilder:
-    """Gathers repo-level and task-level context for agent prompts."""
-    
+    """Gathers repo-level context for agent prompts."""
+
     def __init__(self, project_root: Path):
         self.project_root = project_root
 
-    def build_task_context(self, task: Task, requirements: List[Requirement]) -> str:
-        """Collects all relevant information for implementing a task."""
-        
-        # 1. Map requirements relevant to this task
-        req_map = {r.id: r for r in requirements}
-        task_reqs = [req_map[rid] for rid in task.requirement_ids if rid in req_map]
-        
-        # 2. Gather content of planned files (if they exist)
-        file_contents = {}
-        for pf in task.planned_files:
-            file_path = self.project_root / pf.path
-            if file_path.exists() and file_path.is_file():
-                try:
-                    raw = file_path.read_text(encoding="utf-8")
-                    file_contents[pf.path] = redact_string(raw)
-                except Exception:
-                    file_contents[pf.path] = "[Error reading file]"
-            else:
-                file_contents[pf.path] = "[New file - does not exist yet]"
-
-        # 3. Assemble the context string
-        context = {
-            "task": task.model_dump(),
-            "relevant_requirements": [r.model_dump() for r in task_reqs],
-            "file_contents": file_contents,
-            "project_structure": self.get_structure_summary(task)
-        }
-        
-        return json.dumps(context, indent=2)
-
     def get_structure_summary(self, task: Task | None = None) -> List[str]:
         """Simple list of files in the project for context."""
         try:
-            import subprocess
-            output = subprocess.check_output(
-                ["git", "ls-files", "--cached", "--others", "--exclude-standard"],
+            from devcouncil.utils.proc import git_output
+
+            output = git_output(
+                ["ls-files", "--cached", "--others", "--exclude-standard"],
                 cwd=self.project_root,
-                stderr=subprocess.DEVNULL
-            ).decode().splitlines()
-            
+            ).splitlines()
+
             if task and task.planned_files:
                 planned_paths = {pf.path for pf in task.planned_files}
-                output = [p for p in output if p in planned_paths] + [p for p in output if p not in planned_paths]
-                
-            return output[:100] # Limit to avoid context overflow
+                output = [p for p in output if p in planned_paths] + [
+                    p for p in output if p not in planned_paths
+                ]
+
+            return output[:100]  # Limit to avoid context overflow
         except Exception:
             return []

package/src/devcouncil/execution/fs_watcher.py

@@ -14,6 +14,7 @@ from devcouncil.execution.policy_engine import TaskPolicyEngine
 from devcouncil.storage.db import get_db
 from devcouncil.storage.native import FileChangeRepository, TaskLeaseRepository
 from devcouncil.storage.repositories import GapRepository, TaskRepository
+from devcouncil.verification.stub_detector import detect_stubs, task_allows_scaffolding
 from devcouncil.verification.verifier import Verifier
 
 _IGNORED_PREFIXES = (
@@ -37,6 +38,7 @@ _EVENT_IGNORED_PREFIXES = _IGNORED_PREFIXES + (".devcouncil/", ".gitignore")
 
 _TASK_CACHE_TTL_SECONDS = 10.0
 _EVENT_DEBOUNCE_SECONDS = 0.5
+_STUB_SCAN_INTERVAL_SECONDS = 3.0
 
 logger = logging.getLogger(__name__)
 
@@ -58,6 +60,8 @@ class FilesystemWatcher:
         self._seen: dict[str, float] = {}
         self._seen_last_cleanup: float = 0.0
         self._task_cache: tuple[float, Task | None] | None = None
+        self._last_stub_scan: float = 0.0
+        self._reported_stub_keys: set[tuple[str, int, str]] = set()
         # Reuse one Verifier across polls (avoids rebuilding its scanners each tick) but
         # still run get_changed_files() fresh every poll — caching the git result would
         # make the watcher miss changes for the cache TTL, defeating the poll interval.
@@ -75,6 +79,7 @@ class FilesystemWatcher:
             if self.should_ignore(path):
                 continue
             events.append(self._record_path(path, task, operation="modify"))
+        self._scan_stubs_live(task)
         return events
 
     def watch(self) -> None:
@@ -129,6 +134,7 @@ class FilesystemWatcher:
             return None
         event = self._record_path(rel, self._task_cached(), operation=operation)
         self._notify(event)
+        self._scan_stubs_live(self._task_cached())
         return event
 
     def _notify(self, event: dict) -> None:
@@ -203,3 +209,61 @@ class FilesystemWatcher:
                         )
                     )
         return {"path": path, "operation": operation, "allowed": allowed, "reason": decision.reason}
+
+    def _scan_stubs_live(self, task: Task | None) -> None:
+        """Run stub detection on the current working-tree diff during execution.
+
+        Surfaces non-blocking ``stub_detected`` gaps mid-session — cheaper to fix
+        than waiting for ``dev verify``. Throttled so rapid saves do not spam gaps.
+        """
+        now = time.monotonic()
+        if now - self._last_stub_scan < _STUB_SCAN_INTERVAL_SECONDS:
+            return
+        self._last_stub_scan = now
+        if task is None:
+            return
+        if self._verifier is None:
+            self._verifier = Verifier(self.project_root)
+        try:
+            diff = self._verifier.get_diff()
+        except Exception:
+            logger.debug("Live stub scan skipped: could not load diff", exc_info=True)
+            return
+        if not diff.strip():
+            return
+        scaffolding_ok = task_allows_scaffolding(task)
+        findings = detect_stubs(self.project_root, diff, honor_allow_stub=scaffolding_ok)
+        if not findings:
+            return
+        db = get_db(self.project_root)
+        if not db:
+            return
+        with db.get_session() as session:
+            gap_repo = GapRepository(session)
+            for finding in findings:
+                key = (finding.file, finding.line, finding.reason)
+                if key in self._reported_stub_keys:
+                    continue
+                self._reported_stub_keys.add(key)
+                gap_repo.save(
+                    Gap(
+                        id=f"GAP-{self.task_id}-LIVESTUB-{uuid.uuid4().hex[:10]}",
+                        severity="medium",
+                        gap_type="stub_detected",
+                        task_id=self.task_id,
+                        description=(
+                            f"Live stub/placeholder at {finding.file}:{finding.line}: {finding.reason}."
+                        ),
+                        evidence=[f"{finding.file}:{finding.line}", finding.snippet],
+                        recommended_fix=(
+                            f"Replace the placeholder at {finding.file}:{finding.line} before continuing."
+                        ),
+                        blocking=False,
+                        file=finding.file,
+                        line=finding.line,
+                    )
+                )
+                logger.info(
+                    "Live stub detected for %s at %s:%s (%s)",
+                    self.task_id, finding.file, finding.line, finding.reason,
+                )