devcouncil 0.3.1 → 0.4.0

package/src/devcouncil/cli/commands/rollback.py

@@ -1,11 +1,14 @@
+import logging
 import typer
 from pathlib import Path
 from rich.console import Console
 
 from devcouncil.execution.checkpoints import CheckpointService
+from devcouncil.telemetry.stages import log_stage, log_step
 
 app = typer.Typer()
 console = Console()
+logger = logging.getLogger(__name__)
 
 @app.callback(invoke_without_command=True)
 def rollback(
@@ -20,40 +23,48 @@ def rollback(
         return
 
     root = project_root.expanduser().resolve()
-    checkpoint_dir = root / ".devcouncil" / "checkpoints"
-    checkpoint_file = checkpoint_dir / f"{task_id}-before.patch"
-    after_patch = checkpoint_dir / f"{task_id}-after.patch"
-    service = CheckpointService(root)
-
-    if not checkpoint_file.exists() and not after_patch.exists():
-        before_ref = CheckpointService.REF_BEFORE.format(task_id=task_id)
-        after_ref = CheckpointService.REF_AFTER.format(task_id=task_id)
-        if not service._ref_exists(before_ref) and not service._ref_exists(after_ref):
-            console.print(
-                f"[red]No checkpoint found for task {task_id}. Expected {after_patch} "
-                f"or {checkpoint_file}.[/red]"
-            )
+    from devcouncil.telemetry.logging_setup import set_log_dir
+    set_log_dir(root)
+    logger.info("dev rollback: task=%s", task_id)
+
+    with log_stage("rollback", project_root=root, task_id=task_id):
+        log_step("rollback/1: locating checkpoint", project_root=root, task_id=task_id, trace=True)
+        checkpoint_dir = root / ".devcouncil" / "checkpoints"
+        checkpoint_file = checkpoint_dir / f"{task_id}-before.patch"
+        after_patch = checkpoint_dir / f"{task_id}-after.patch"
+        service = CheckpointService(root)
+
+        if not checkpoint_file.exists() and not after_patch.exists():
+            before_ref = CheckpointService.REF_BEFORE.format(task_id=task_id)
+            after_ref = CheckpointService.REF_AFTER.format(task_id=task_id)
+            if not service._ref_exists(before_ref) and not service._ref_exists(after_ref):
+                console.print(
+                    f"[red]No checkpoint found for task {task_id}. Expected {after_patch} "
+                    f"or {checkpoint_file}.[/red]"
+                )
+                raise typer.Exit(code=1)
+
+        console.print(f"Rolling back task [bold]{task_id}[/bold]...")
+        log_step("rollback/2: reverting checkpoint", project_root=root, task_id=task_id)
+        result = service.rollback(task_id)
+        if "failed" in result.message.lower() or result.message.startswith("No checkpoint"):
+            console.print(f"[yellow]{result.message}[/yellow]")
+            if checkpoint_file.exists():
+                console.print(
+                    f"The before-patch at {checkpoint_file} captured the state before the task ran.\n"
+                    f"To manually reset:\n"
+                    f"  1. [bold]git stash[/bold] (if you want to keep current changes)\n"
+                    f"  2. [bold]git checkout -- .[/bold] (discard working tree changes)\n"
+                    f"  3. [bold]git apply {checkpoint_file}[/bold] (restore pre-task state)"
+                )
+            elif after_patch.exists():
+                console.print(
+                    f"Only the after-patch at {after_patch} exists (it captured the task's changes).\n"
+                    f"To manually revert those changes from the working tree:\n"
+                    f"  1. [bold]git apply --stat {after_patch}[/bold] (inspect what the task changed)\n"
+                    f"  2. [bold]git apply -R {after_patch}[/bold] (reverse-apply the task's changes)"
+                )
             raise typer.Exit(code=1)
 
-    console.print(f"Rolling back task [bold]{task_id}[/bold]...")
-    result = service.rollback(task_id)
-    if "failed" in result.message.lower() or result.message.startswith("No checkpoint"):
-        console.print(f"[yellow]{result.message}[/yellow]")
-        if checkpoint_file.exists():
-            console.print(
-                f"The before-patch at {checkpoint_file} captured the state before the task ran.\n"
-                f"To manually reset:\n"
-                f"  1. [bold]git stash[/bold] (if you want to keep current changes)\n"
-                f"  2. [bold]git checkout -- .[/bold] (discard working tree changes)\n"
-                f"  3. [bold]git apply {checkpoint_file}[/bold] (restore pre-task state)"
-            )
-        elif after_patch.exists():
-            console.print(
-                f"Only the after-patch at {after_patch} exists (it captured the task's changes).\n"
-                f"To manually revert those changes from the working tree:\n"
-                f"  1. [bold]git apply --stat {after_patch}[/bold] (inspect what the task changed)\n"
-                f"  2. [bold]git apply -R {after_patch}[/bold] (reverse-apply the task's changes)"
-            )
-        raise typer.Exit(code=1)
-
-    console.print(f"[green]Successfully rolled back task {task_id}.[/green] {result.message}")
+        console.print(f"[green]Successfully rolled back task {task_id}.[/green] {result.message}")
+        log_step("rollback/complete", project_root=root, task_id=task_id, trace=True)

package/src/devcouncil/cli/commands/run.py

@@ -10,6 +10,7 @@ from devcouncil.executors.coding_cli import CodingCliExecutor
 from devcouncil.executors.agent_registry import (
     AGENT_ALIASES,
     BUILTIN_CODING_EXECUTOR_NAMES,
+    list_run_executor_names,
     load_cli_agent_specs,
 )
 from devcouncil.executors.native.agent import NativeAgent
@@ -17,11 +18,14 @@ from devcouncil.llm.provider import create_provider, validate_model_provider
 from devcouncil.llm.router import ModelRouter
 from devcouncil.app.config import load_config, get_api_key
 from devcouncil.domain.evidence import CommandResult, DiffEvidence, TestEvidence
+from devcouncil.domain.task import Task
+from devcouncil.domain.requirement import Requirement
 from devcouncil.storage.repositories import GapRepository, EvidenceRepository, StateRepository
 from devcouncil.verification.verifier import Verifier
 from devcouncil.app.state_machine import ProjectPhase
 from devcouncil.cli.commands.init import initialize_project
 from devcouncil.telemetry.traces import TraceLogger
+from devcouncil.telemetry.stages import log_stage, log_step
 
 console = Console()
 logger = logging.getLogger(__name__)
@@ -39,14 +43,59 @@ def _current_changed_files(project_root: Path = Path(".")) -> list[str]:
 
     return Verifier(project_root).get_changed_files()
 
+
+def _verify_executor_output_if_present(
+    session,
+    task,
+    reqs,
+    *,
+    root: Path,
+    executor_label: str,
+    cli_client: str | None = None,
+    cli_executor=None,
+) -> bool:
+    """Run verification when the executor failed but left implementable changes."""
+    if not _current_changed_files(root):
+        return False
+    console.print(
+        f"[yellow]{executor_label} failed, but the workspace has changes — "
+        f"verifying {task.id} against them anyway.[/yellow]"
+    )
+    verified = _verify_after_execution(
+        session, task, reqs, router=_build_verification_router(root), project_root=root
+    )
+    _record_project_phase(
+        session,
+        ProjectPhase.TASK_VERIFIED if verified else ProjectPhase.TASK_BLOCKED,
+    )
+    TaskRepository(session).save(task)
+    if cli_client:
+        _record_agent_verification(
+            root,
+            task.id,
+            cli_client,
+            getattr(cli_executor, "last_run_id", None) if cli_executor else None,
+            verified,
+        )
+        _run_live_review_after_execution(root, cli_client, task.id)
+    _log_exec_outcome(executor_label, task.id, verified=verified)
+    if verified:
+        console.print(f"\n[green]{executor_label} left verifiable work; {task.id} verified.[/green]")
+    else:
+        console.print(
+            f"\n[yellow]{executor_label} left changes, but {task.id} is blocked by verification gaps.[/yellow]"
+        )
+    return True
+
 def _capture_after_patch(task_id: str, project_root: Path = Path(".")):
     """Capture the diff after task execution for use by rollback."""
     try:
         from devcouncil.execution.checkpoints import CheckpointService
 
         CheckpointService(project_root).create_after(task_id)
-    except Exception:
-        pass  # Non-critical — don't block execution
+    except Exception as e:
+        # Non-critical — don't block execution, but a missing after checkpoint breaks rollback.
+        logger.warning("Failed to capture after checkpoint for %s: %s", task_id, e)
 
 def _capture_before_snapshot(task_id: str, project_root: Path = Path(".")):
     from devcouncil.execution.checkpoints import CheckpointService
@@ -56,11 +105,22 @@ def _capture_before_snapshot(task_id: str, project_root: Path = Path(".")):
 def _record_project_phase(session, phase: ProjectPhase):
     StateRepository(session).record_phase(phase.value)
 
-def _verify_after_execution(session, task, reqs, router=None, project_root: Path = Path(".")) -> bool:
+def _verify_after_execution(
+    session,
+    task: Task,
+    reqs: list[Requirement],
+    router=None,
+    project_root: Path = Path("."),
+) -> bool:
     """Run deterministic verification after an automated executor finishes."""
     import asyncio
 
     logger.info("Verifying task %s (router=%s)", task.id, "yes" if router else "no")
+    log_step(
+        "run/verify: verifying executor output",
+        project_root=project_root,
+        task_id=task.id,
+    )
     verifier = Verifier(project_root, router=router)
     gaps, evidence = asyncio.run(verifier.verify_task(task, reqs))
     blocking = [g for g in gaps if g.blocking]
@@ -148,13 +208,14 @@ def _run_live_review_after_execution(project_root: Path, client: str, task_id: s
         )
         from devcouncil.live.transcripts import latest_assistant_turn
 
-        transcript = _resolve_transcript(project_root, client, latest=True)
+        transcript = _resolve_transcript(project_root, client, latest=True, task_id=task_id)
         if transcript is None or not transcript.exists():
             return
         turn = latest_assistant_turn(transcript, client=client)
         if turn is None:
             return
         card = asyncio.run(_review_turn(turn, project_root, client, use_llm=True, task_id=task_id))
+        card = card.model_copy(update={"task_id": task_id, "blocks_gate": False})
         saved_path, duplicate = _save_card_once(project_root, card, persist=True, force=False)
         if saved_path:
             _log_card_reviewed(project_root, card, saved_path, duplicate=duplicate, source="e2e")
@@ -188,8 +249,8 @@ def run(
         "--executor",
         "-e",
         help=(
-            "Executor to use (manual, mini, openhands, native-preview, "
-            "codex, gemini, claude, opencode, antigravity, warp, cursor, aider, "
+            "Executor to use (manual, mini, openhands, native-preview, claude-sdk, "
+            "codex, gemini, claude, opencode, antigravity, warp, cursor, grok, aider, "
             "copilot, goose, amp, qwen, crush, or a configured agent)"
         ),
     ),
@@ -214,6 +275,11 @@ def run(
         console.print("[red]DevCouncil state is unavailable in this directory.[/red]")
         return
 
+    with log_stage("run", project_root=root, task_id=task_id, executor=executor):
+        _run_task_body(root, task_id, executor, profile, stream, db)
+
+
+def _run_task_body(root, task_id, executor, profile, stream, db):
     with db.get_session() as session:
         task_repo = TaskRepository(session)
         task = task_repo.get_by_id(task_id)
@@ -237,6 +303,7 @@ def run(
 
         console.print(f"Running task [bold]{task_id}[/bold] using [bold]{executor}[/bold] executor...")
 
+        log_step("run/1: creating git checkpoint", project_root=root, task_id=task_id)
         # 1. Create Git checkpoint
         try:
             from devcouncil.execution.checkpoints import CheckpointService
@@ -250,6 +317,38 @@ def run(
             console.print(f"[yellow]Warning: Failed to create git checkpoint: {e}[/yellow]")
 
         executor = executor.strip().lower().replace("_", "-")
+        if executor == "claude-sdk":
+            # Preflight the optional SDK BEFORE mutating any state: previously a
+            # missing claude-agent-sdk was only discovered inside run_task, after
+            # planning had already spent its budget — every such run failed at the
+            # finish line with "SDK is not installed" (observed repeatedly in logs).
+            # Degrade to the Claude Code CLI executor when it is available (same
+            # agent, hook-based gating instead of in-process gating); otherwise
+            # fail fast with the actionable install hint.
+            import importlib.util
+            import shutil as _shutil
+
+            if importlib.util.find_spec("claude_agent_sdk") is None:
+                if _shutil.which("claude"):
+                    logger.warning(
+                        "claude-agent-sdk is not installed; falling back to the "
+                        "'claude' CLI executor for %s (install claude-agent-sdk "
+                        "to restore in-process tool gating).", task_id,
+                    )
+                    console.print(
+                        "[yellow]claude-agent-sdk is not installed — falling back to the "
+                        "'claude' CLI executor. Install it with "
+                        "`pip install claude-agent-sdk` to use --executor claude-sdk.[/yellow]"
+                    )
+                    executor = "claude"
+                else:
+                    logger.error("claude-sdk executor unavailable for %s: claude-agent-sdk not installed", task_id)
+                    console.print(
+                        "[red]The Claude Agent SDK is not installed and no `claude` CLI was found. "
+                        "Install the SDK with `pip install claude-agent-sdk`, or use another "
+                        "executor (e.g. --executor claude).[/red]"
+                    )
+                    return
         custom_agents = _custom_cli_agents(root)
         if executor not in CODING_EXECUTORS and executor not in custom_agents:
             ignored = [flag for flag, value in (("--profile", profile), ("--stream", stream)) if value]
@@ -266,6 +365,12 @@ def run(
             console.print("Use 'dev prompt TASK-ID' to get the prompt for this task.")
             console.print("When finished, use 'dev verify TASK-ID' to check the results.")
         elif executor in CODING_EXECUTORS or executor in custom_agents:
+            log_step(
+                f"run/2: executing with {executor}",
+                project_root=root,
+                task_id=task_id,
+                executor=executor,
+            )
             _record_project_phase(session, ProjectPhase.TASK_EXECUTING)
             req_repo = RequirementRepository(session)
             reqs = req_repo.get_all()
@@ -304,7 +409,18 @@ def run(
             else:
                 logger.error("%s failed to start or execute for %s: %s", executor, task_id, exec_result.message)
                 console.print(f"\n[red]{executor.upper()} failed to start or execute: {exec_result.message}[/red]")
+                if not _verify_executor_output_if_present(
+                    session,
+                    task,
+                    reqs,
+                    root=root,
+                    executor_label=executor.upper(),
+                    cli_client=cli_client,
+                    cli_executor=cli_executor,
+                ):
+                    task_repo.save(task)
         elif executor == "mini":
+            log_step("run/2: executing with mini-SWE-agent", project_root=root, task_id=task_id)
             _record_project_phase(session, ProjectPhase.TASK_EXECUTING)
             req_repo = RequirementRepository(session)
             reqs = req_repo.get_all()
@@ -330,6 +446,7 @@ def run(
                 logger.error("mini-SWE-agent failed to start or execute for %s", task_id)
                 console.print("\n[red]mini-SWE-agent failed to start or execute.[/red]")
         elif executor == "openhands":
+            log_step("run/2: executing with OpenHands", project_root=root, task_id=task_id)
             _record_project_phase(session, ProjectPhase.TASK_EXECUTING)
             req_repo = RequirementRepository(session)
             reqs = req_repo.get_all()
@@ -354,7 +471,52 @@ def run(
             else:
                 logger.error("OpenHands failed to start or execute for %s", task_id)
                 console.print("\n[red]OpenHands failed to start or execute.[/red]")
+        elif executor == "claude-sdk":
+            log_step("run/2: executing with claude-sdk", project_root=root, task_id=task_id)
+            # In-process Claude Agent SDK executor: every tool call is gated live against
+            # this task's scope, so out-of-scope writes/commands are denied before they land.
+            from devcouncil.executors.claude_sdk import ClaudeSdkExecutor
+            from devcouncil.executors.agent_registry import load_agent_profiles
+
+            _record_project_phase(session, ProjectPhase.TASK_EXECUTING)
+            req_repo = RequirementRepository(session)
+            reqs = req_repo.get_all()
+            # Honor --profile for model and env overrides (e.g. ANTHROPIC_BASE_URL to
+            # target an alternative Anthropic-compatible endpoint). permission_mode is
+            # deliberately NOT taken from the profile: the SDK executor's containment
+            # relies on "default" routing every tool call through can_use_tool; a
+            # profile's acceptEdits/auto would silently bypass the live scope gate.
+            sdk_profile = load_agent_profiles(root).get(profile or "default")
+            sdk_executor = ClaudeSdkExecutor(
+                root,
+                active_task=task,
+                model=(sdk_profile.model if sdk_profile else None),
+                advisor_model=(sdk_profile.advisor_model if sdk_profile else None),
+                env=(dict(sdk_profile.env) if sdk_profile and sdk_profile.env else None),
+            )
+            exec_result = sdk_executor.run_task(task, reqs)
+            _capture_after_patch(task_id, root)
+            if exec_result.success:
+                _record_project_phase(session, ProjectPhase.TASK_VERIFYING)
+                verified = _verify_after_execution(
+                    session, task, reqs, router=_build_verification_router(root), project_root=root
+                )
+                _record_project_phase(
+                    session,
+                    ProjectPhase.TASK_VERIFIED if verified else ProjectPhase.TASK_BLOCKED,
+                )
+                task_repo.save(task)
+                _run_live_review_after_execution(root, "claude", task.id)
+                _log_exec_outcome("claude-sdk", task_id, verified=verified)
+                if verified:
+                    console.print(f"\n[green]claude-sdk finished and task {task_id} verified.[/green]")
+                else:
+                    console.print(f"\n[yellow]claude-sdk finished, but task {task_id} is blocked by verification gaps.[/yellow]")
+            else:
+                logger.error("claude-sdk failed to start or execute for %s: %s", task_id, exec_result.message)
+                console.print(f"\n[red]claude-sdk failed to start or execute: {exec_result.message}[/red]")
         elif executor in {"native", "native-preview"}:
+            log_step("run/2: executing with native agent", project_root=root, task_id=task_id)
             # Load config for model routing and permissions
             try:
                 config = load_config(root)
@@ -406,5 +568,8 @@ def run(
                 logger.error("Native agent failed during execution for %s", task_id)
                 console.print("\n[red]Native agent failed during execution.[/red]")
         else:
-            logger.error("Executor %r not implemented", executor)
-            console.print(f"[red]Executor {executor} not yet implemented.[/red]")
+            available = ", ".join(list_run_executor_names(root))
+            logger.error("Executor %r not recognized; available: %s", executor, available)
+            console.print(f"[red]Unknown executor {executor!r}.[/red]")
+            console.print(f"[dim]Available executors: {available}[/dim]")
+        log_step("run/complete", project_root=root, task_id=task_id, executor=executor, trace=True)