devcouncil 0.3.1 → 0.4.1

package/src/devcouncil/cli/main.py

@@ -32,11 +32,13 @@ from devcouncil.cli.commands import (  # noqa: E402 - imports follow stdio recon
     artifacts,
     agents,
     baseline,
+    boot,
     check,
     config,
     cost,
     ast,
     dashboard,
+    debug_cmd,
     design,
     doctor,
     go,
@@ -46,6 +48,7 @@ from devcouncil.cli.commands import (  # noqa: E402 - imports follow stdio recon
     logs,
     lsp,
     map,
+    graph_cmd,
     mcp_server,
     okf,
     plan,
@@ -57,19 +60,28 @@ from devcouncil.cli.commands import (  # noqa: E402 - imports follow stdio recon
     run,
     runs,
     setup,
+    campaign,
     show,
     status,
+    gaps,
+    requirements,
+    export,
     tasks,
     trace,
     verify,
     version,
     watch,
+    wiki,
     shell,
     semantic,
     evidence,
     handoff,
     skills,
     scaffold,
+    provenance,
+    lease,
+    gated_write,
+    task_gate,
 )
 from devcouncil.cli.commands.watch_fs import watch_fs  # noqa: E402 - imports follow stdio reconfiguration
 
@@ -101,19 +113,29 @@ app.add_typer(setup.app, name="setup")
 app.add_typer(lsp.app, name="lsp")
 app.add_typer(ast.app, name="ast")
 app.add_typer(dashboard.app, name="dashboard")
+app.add_typer(debug_cmd.app, name="debug")
 app.add_typer(watch.app, name="watch")
 app.add_typer(semantic.app, name="semantic")
 app.add_typer(evidence.app, name="evidence")
 app.add_typer(skills.app, name="skills")
 app.add_typer(okf.app, name="okf")
 app.add_typer(design.app, name="design")
+app.add_typer(wiki.app, name="wiki")
+app.add_typer(campaign.app, name="campaign")
+app.add_typer(provenance.resource_app, name="resource")
+app.add_typer(lease.lease_app, name="lease")
+app.add_typer(task_gate.scope_app, name="scope")
 watch.app.command("fs")(watch_fs)
 
 # Direct command registrations (those defined as def cmd())
 app.command(name="baseline")(baseline.baseline)
+app.command(name="boot")(boot.boot)
 app.command(name="e2e")(go.go)
 app.command(name="go")(go.go)
 app.command(name="map")(map.map_repo)
+app.add_typer(graph_cmd.app, name="graph")
+app.add_typer(graph_cmd.corpus_app, name="corpus")
+app.command(name="graph-context")(map.graph_context_cmd)
 app.command(name="scaffold-ci")(scaffold.scaffold_ci_command)
 app.command(name="plan")(plan.plan)
 app.command(name="approve")(plan.approve)
@@ -130,6 +152,23 @@ app.command(name="verify")(verify.verify)
 app.command(name="check")(check.check)
 app.command(name="repair")(repair.repair)
 app.command(name="status")(status.status)
+app.command(name="gaps")(gaps.gaps)
+app.command(name="provenance")(provenance.provenance)
+app.command(name="checkout")(lease.checkout)
+app.command(name="release")(lease.release)
+app.command(name="write")(gated_write.write)
+app.command(name="apply-patch")(gated_write.apply_patch)
+app.command(name="next-task")(task_gate.next_task)
+app.command(name="policy-check")(task_gate.policy_check)
+app.command(name="record-command")(task_gate.record_command)
+app.command(name="run-cmd")(task_gate.run_cmd)
+app.command(name="attach-committed-range")(task_gate.attach_committed_range)
+app.command(name="verify-leased")(task_gate.verify_leased)
+app.command(name="evidence-append")(task_gate.evidence_append)
+app.command(name="evidence-list")(task_gate.evidence_list)
+app.command(name="handoff-leased")(task_gate.handoff_leased)
+app.command(name="requirements")(requirements.requirements)
+app.command(name="export")(export.export_state)
 app.command(name="optimize")(agents.optimize_agent)
 
 @app.callback()
@@ -168,5 +207,23 @@ def main(
     configure_logging(verbosity=verbose, quiet=quiet, log_level=log_level)
     return
 
+def run_cli() -> None:
+    """Console-script entry: run the CLI with EPIPE-safe stdout.
+
+    ``dev map | head`` (or any consumer closing the pipe early) must exit with
+    the conventional SIGPIPE status instead of a BrokenPipeError traceback.
+    """
+    import os
+
+    try:
+        app()
+    except BrokenPipeError:
+        try:
+            os.dup2(os.open(os.devnull, os.O_WRONLY), sys.stdout.fileno())
+        except OSError:
+            pass
+        sys.exit(128 + 13)
+
+
 if __name__ == "__main__":
-    app()
+    run_cli()

package/src/devcouncil/codeintel/__init__.py

@@ -0,0 +1,16 @@
+"""Transactional, project-scoped code intelligence services.
+
+The legacy :mod:`devcouncil.indexing.graph` package remains the public
+compatibility surface.  New code should obtain a :class:`CodeIntelService` for a
+canonical project root and query its committed SQLite generation.
+"""
+
+from devcouncil.codeintel.service import CodeIntelService, get_codeintel_service
+from devcouncil.codeintel.store import CodeIntelStore, StoreStatus
+
+__all__ = [
+    "CodeIntelService",
+    "CodeIntelStore",
+    "StoreStatus",
+    "get_codeintel_service",
+]

package/src/devcouncil/codeintel/build_control.py

@@ -0,0 +1,429 @@
+"""Single-writer graph build sessions and supervised full-build isolation."""
+
+from __future__ import annotations
+
+import json
+import os
+import queue
+import signal
+import subprocess
+import sys
+import threading
+import time
+import uuid
+from collections import deque
+from contextlib import contextmanager
+from dataclasses import asdict, dataclass
+from pathlib import Path
+from typing import TYPE_CHECKING, Iterator
+
+from devcouncil.codeintel.service import get_codeintel_service
+from devcouncil.indexing.graph.schema import CodeGraph
+from devcouncil.utils.json_persist import read_json, write_json
+
+# WriterLease is imported lazily inside graph_build_session to avoid
+# sync.__init__ → incremental → build_control circular import at module load.
+if TYPE_CHECKING:
+    from devcouncil.codeintel.sync.lease import WriterLease
+
+STATUS_REL = Path(".devcouncil") / "codeintel" / "build_status.json"
+
+
+class GraphBuildBusy(RuntimeError):
+    """Another thread or process owns the graph writer session."""
+
+
+class GraphBuildTimeout(TimeoutError):
+    """A supervised graph build exceeded its progress or total deadline."""
+
+
+class GraphBuildFailed(RuntimeError):
+    """A supervised graph worker exited without committing a graph."""
+
+
+@dataclass
+class BuildStatus:
+    build_id: str = ""
+    state: str = "idle"
+    mode: str = ""
+    pid: int | None = None
+    phase: str = ""
+    completed: int = 0
+    total: int = 0
+    started_at: float | None = None
+    last_progress_at: float | None = None
+    generation_before: int | None = None
+    generation_after: int | None = None
+    degraded_reason: str = ""
+    compatibility_export: str = "unknown"
+    stall_timeout_seconds: float = 90.0
+    total_timeout_seconds: float = 900.0
+
+
+@dataclass
+class IsolatedBuildResult:
+    graph: CodeGraph
+    status: BuildStatus
+
+
+_LOCKS: dict[Path, threading.RLock] = {}
+_LOCKS_GUARD = threading.Lock()
+_LOCAL = threading.local()
+
+
+def status_path(root: Path) -> Path:
+    return root.expanduser().resolve() / STATUS_REL
+
+
+def read_build_status(root: Path) -> BuildStatus:
+    try:
+        raw = read_json(status_path(root))
+        status = BuildStatus(**{
+            key: raw[key] for key in BuildStatus.__dataclass_fields__ if key in raw
+        })
+        if status.state == "building":
+            if status.pid is not None and not _pid_alive(status.pid):
+                status.state = "stale"
+                status.degraded_reason = "recorded graph worker is no longer running"
+            elif (
+                status.last_progress_at is not None
+                and time.time() - status.last_progress_at > status.stall_timeout_seconds
+            ):
+                status.state = "stalled"
+                status.degraded_reason = (
+                    f"no recorded graph progress for {status.stall_timeout_seconds:.1f}s"
+                )
+        return status
+    except Exception:
+        return BuildStatus()
+
+
+def _pid_alive(pid: int) -> bool:
+    try:
+        os.kill(pid, 0)
+    except ProcessLookupError:
+        return False
+    except (PermissionError, OSError):
+        return True
+    return True
+
+
+def _write_status(root: Path, status: BuildStatus) -> None:
+    path = status_path(root)
+    path.parent.mkdir(parents=True, exist_ok=True)
+    write_json(path, asdict(status))
+
+
+def record_inline_build_status(
+    root: Path,
+    *,
+    state: str,
+    mode: str,
+    phase: str,
+    generation_before: int | None,
+    generation_after: int | None,
+    reason: str = "",
+    compatibility_export: str = "healthy",
+) -> BuildStatus:
+    now = time.time()
+    status = BuildStatus(
+        build_id=uuid.uuid4().hex,
+        state=state,
+        mode=mode,
+        pid=os.getpid(),
+        phase=phase,
+        completed=1,
+        total=1,
+        started_at=now,
+        last_progress_at=now,
+        generation_before=generation_before,
+        generation_after=generation_after,
+        degraded_reason=reason,
+        compatibility_export=compatibility_export,
+        stall_timeout_seconds=90.0,
+        total_timeout_seconds=900.0,
+    )
+    _write_status(root.expanduser().resolve(), status)
+    return status
+
+
+def _root_lock(root: Path) -> threading.RLock:
+    with _LOCKS_GUARD:
+        return _LOCKS.setdefault(root, threading.RLock())
+
+
+def _lease_timeouts(root: Path) -> tuple[float, float]:
+    """Return ``(build_timeout, sync_timeout)`` seconds for writer-lease waits."""
+    build_timeout = 30.0
+    sync_timeout = 5.0
+    try:
+        from devcouncil.app.config import load_config
+
+        cfg = load_config(root).code_intelligence
+        build_timeout = float(
+            getattr(cfg, "writer_lease_timeout_seconds", build_timeout) or build_timeout
+        )
+        sync_timeout = float(
+            getattr(cfg, "writer_lease_sync_timeout_seconds", sync_timeout) or sync_timeout
+        )
+    except Exception:
+        pass
+    return max(0.1, build_timeout), max(0.1, sync_timeout)
+
+
+@contextmanager
+def graph_build_session(
+    root: Path,
+    *,
+    lease: WriterLease | None = None,
+    timeout: float | None = None,
+) -> Iterator[None]:
+    """Hold the process-local and cross-process writer locks at outermost depth.
+
+    ``timeout`` bounds how long a contending writer waits for ``writer.lock``
+    (exponential backoff). Defaults to ``code_intelligence.writer_lease_timeout_seconds``.
+    """
+    from devcouncil.codeintel.sync.lease import WriterLease
+
+    root = root.expanduser().resolve()
+    lock = _root_lock(root)
+    with lock:
+        depths = getattr(_LOCAL, "depths", None)
+        if depths is None:
+            depths = {}
+            _LOCAL.depths = depths
+        leases = getattr(_LOCAL, "leases", None)
+        if leases is None:
+            leases = {}
+            _LOCAL.leases = leases
+        depth = int(depths.get(root, 0))
+        owned_lease = None
+        if depth == 0:
+            owned_lease = lease or WriterLease(
+                root / ".devcouncil" / "codeintel" / "writer.lock"
+            )
+            wait = timeout if timeout is not None else _lease_timeouts(root)[0]
+            if not owned_lease.acquire_with_retry(timeout=wait):
+                raise GraphBuildBusy("another process owns the code-intelligence writer lease")
+            leases[root] = owned_lease
+        depths[root] = depth + 1
+        try:
+            yield
+        finally:
+            remaining = int(depths.get(root, 1)) - 1
+            if remaining:
+                depths[root] = remaining
+            else:
+                depths.pop(root, None)
+                held = leases.pop(root, owned_lease)
+                if held is not None:
+                    held.release()
+
+
+@contextmanager
+def yield_writer_lease_for_child(root: Path) -> Iterator[None]:
+    """Temporarily release this process's writer lease so a supervised child can own it.
+
+    Isolated graph workers must hold ``writer.lock`` themselves. When the parent already
+    owns the lease (nested ``graph_build_session``), release it for the child lifetime and
+    re-acquire afterward with bounded backoff so a concurrent watcher cannot permanently
+    starve the parent. If the parent does not own the lease, this is a no-op.
+    """
+    root = root.expanduser().resolve()
+    leases = getattr(_LOCAL, "leases", {}) or {}
+    held = leases.get(root)
+    if held is None:
+        yield
+        return
+    held.release()
+    try:
+        yield
+    finally:
+        # Another watcher may have claimed the lock in the yield window; wait out a
+        # full build-lease budget before failing the otherwise-successful child commit.
+        wait = _lease_timeouts(root)[0]
+        if not held.acquire_with_retry(timeout=wait):
+            raise GraphBuildBusy(
+                "could not re-acquire the code-intelligence writer lease after isolated build"
+            )
+
+
+def _terminate_worker(process: subprocess.Popen[str]) -> None:
+    if process.poll() is not None:
+        return
+    try:
+        if os.name == "nt":
+            process.terminate()
+        else:
+            os.killpg(process.pid, signal.SIGTERM)
+        process.wait(timeout=5.0)
+    except (OSError, subprocess.TimeoutExpired):
+        try:
+            if os.name == "nt":
+                process.kill()
+            else:
+                os.killpg(process.pid, signal.SIGKILL)
+        except OSError:
+            pass
+
+
+def run_isolated_full_build(
+    root: Path,
+    *,
+    changed_paths: set[str] | None = None,
+    liveness: bool = True,
+) -> IsolatedBuildResult:
+    """Build and persist a full graph in a heartbeat-supervised child process."""
+    from devcouncil.app.config import IndexingConfig, load_config
+
+    root = root.expanduser().resolve()
+    try:
+        config = load_config(root).indexing
+    except FileNotFoundError:
+        config = IndexingConfig()
+    before = get_codeintel_service(root).store.current_generation()
+    build_id = uuid.uuid4().hex
+    command = [
+        sys.executable,
+        "-m",
+        "devcouncil.codeintel.build_worker",
+        "--root",
+        str(root),
+        "--build-id",
+        build_id,
+    ]
+    if not liveness:
+        command.append("--no-liveness")
+    for path in sorted(changed_paths or set()):
+        command.extend(("--changed-path", path))
+    creationflags = 0
+    if os.name == "nt":
+        creationflags = getattr(subprocess, "CREATE_NEW_PROCESS_GROUP", 0)
+    process = subprocess.Popen(
+        command,
+        cwd=root,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.PIPE,
+        text=True,
+        bufsize=1,
+        start_new_session=os.name != "nt",
+        creationflags=creationflags,
+    )
+    now = time.time()
+    status = BuildStatus(
+        build_id=build_id,
+        state="building",
+        mode="full",
+        pid=process.pid,
+        phase="starting",
+        started_at=now,
+        last_progress_at=now,
+        generation_before=before,
+        stall_timeout_seconds=float(config.build_stall_timeout_seconds),
+        total_timeout_seconds=float(config.build_total_timeout_seconds),
+    )
+    _write_status(root, status)
+    messages: queue.Queue[str] = queue.Queue()
+    stderr_lines: deque[str] = deque(maxlen=200)
+
+    def _read_stdout() -> None:
+        assert process.stdout is not None
+        for line in process.stdout:
+            messages.put(line)
+
+    reader = threading.Thread(target=_read_stdout, name="devcouncil-graph-worker-output", daemon=True)
+    reader.start()
+
+    def _read_stderr() -> None:
+        assert process.stderr is not None
+        for line in process.stderr:
+            stderr_lines.append(line)
+
+    error_reader = threading.Thread(
+        target=_read_stderr,
+        name="devcouncil-graph-worker-errors",
+        daemon=True,
+    )
+    error_reader.start()
+    started = time.monotonic()
+    last_progress = started
+    stall = float(config.build_stall_timeout_seconds)
+    total = float(config.build_total_timeout_seconds)
+    final_payload: dict[str, object] = {}
+    after: int | None = None
+    # Child must own writer.lock; release any parent-held lease for the supervision window.
+    with yield_writer_lease_for_child(root):
+        try:
+            while True:
+                try:
+                    line = messages.get(timeout=0.2)
+                except queue.Empty:
+                    line = ""
+                if line:
+                    try:
+                        payload = json.loads(line)
+                    except json.JSONDecodeError:
+                        payload = {}
+                    if payload:
+                        last_progress = time.monotonic()
+                        status.phase = str(payload.get("phase") or status.phase)
+                        status.completed = int(payload.get("completed") or 0)
+                        status.total = int(payload.get("total") or 0)
+                        status.last_progress_at = time.time()
+                        status.compatibility_export = str(
+                            payload.get("compatibility_export") or status.compatibility_export
+                        )
+                        if payload.get("state") in {"complete", "degraded"}:
+                            final_payload = payload
+                        _write_status(root, status)
+                elapsed = time.monotonic() - started
+                idle = time.monotonic() - last_progress
+                if elapsed > total or idle > stall:
+                    reason = (
+                        f"graph build exceeded {total:.1f}s total timeout"
+                        if elapsed > total
+                        else f"graph build made no progress for {stall:.1f}s"
+                    )
+                    _terminate_worker(process)
+                    status.state = "timed_out"
+                    status.degraded_reason = reason
+                    status.generation_after = get_codeintel_service(root).store.current_generation()
+                    _write_status(root, status)
+                    raise GraphBuildTimeout(reason)
+                if process.poll() is not None:
+                    reader.join(timeout=0.2)
+                    if not reader.is_alive() and messages.empty():
+                        break
+            reader.join(timeout=1.0)
+            error_reader.join(timeout=1.0)
+            after = get_codeintel_service(root).store.current_generation()
+            if process.returncode != 0 or after is None or after == before:
+                stderr = "".join(stderr_lines).strip()
+                reason = stderr[-4000:] or f"graph worker exited with code {process.returncode}"
+                status.state = "failed"
+                status.degraded_reason = reason
+                status.generation_after = after
+                _write_status(root, status)
+                raise GraphBuildFailed(reason)
+            status.state = str(final_payload.get("state") or "complete")
+            status.phase = "complete"
+            status.completed = status.total or 1
+            status.total = status.total or 1
+            status.generation_after = after
+            status.compatibility_export = str(
+                final_payload.get("compatibility_export") or "healthy"
+            )
+            status.degraded_reason = str(final_payload.get("reason") or "")
+            _write_status(root, status)
+        finally:
+            if process.poll() is None:
+                _terminate_worker(process)
+    # Reload under the re-acquired lease so a concurrent watcher that wrote during
+    # the yield window cannot leave the parent holding a stale in-memory graph.
+    after = get_codeintel_service(root).store.current_generation()
+    status.generation_after = after
+    graph = get_codeintel_service(root).load()
+    if graph is None:
+        raise GraphBuildFailed("graph worker committed a generation that could not be loaded")
+    _write_status(root, status)
+    return IsolatedBuildResult(graph=graph, status=status)

package/src/devcouncil/codeintel/build_worker.py

@@ -0,0 +1,78 @@
+"""Internal subprocess entry point for supervised full graph builds.
+
+devcouncil: allow-unwired — launched with ``python -m`` by build_control.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+from pathlib import Path
+
+
+def _emit(**payload: object) -> None:
+    print(json.dumps(payload, separators=(",", ":")), flush=True)
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(add_help=False)
+    parser.add_argument("--root", required=True)
+    parser.add_argument("--build-id", required=True)
+    parser.add_argument("--changed-path", action="append", default=[])
+    parser.add_argument("--no-liveness", action="store_true")
+    args = parser.parse_args()
+    root = Path(args.root).expanduser().resolve()
+
+    def progress(phase: str, completed: int, total: int) -> None:
+        _emit(
+            build_id=args.build_id,
+            state="building",
+            phase=phase,
+            completed=completed,
+            total=total,
+        )
+
+    from devcouncil.codeintel.build_control import graph_build_session
+    from devcouncil.indexing.graph.build import (
+        CompatibilityGraphTooLarge,
+        build_code_graph,
+        write_code_graph,
+    )
+
+    # Acquire the cross-process writer lease in the child so an orphaned worker
+    # (parent crash) still serializes against watchers/MCP writers.
+    with graph_build_session(root):
+        graph = build_code_graph(
+            root,
+            changed_paths=set(args.changed_path),
+            liveness=not args.no_liveness,
+            progress=progress,
+        )
+        graph.meta.update({
+            "incremental": False,
+            "changed_paths": sorted(set(args.changed_path)),
+            "affected_paths": sorted(set(args.changed_path)),
+            "affected_fraction": 1.0,
+            "resolution_scope": "full",
+        })
+        compatibility_export = "healthy"
+        reason = ""
+        try:
+            write_code_graph(root, graph, _lease_held=True)
+        except CompatibilityGraphTooLarge as exc:
+            compatibility_export = "degraded"
+            reason = str(exc)
+        _emit(
+            build_id=args.build_id,
+            state="degraded" if reason else "complete",
+            phase="complete",
+            completed=1,
+            total=1,
+            compatibility_export=compatibility_export,
+            reason=reason,
+        )
+        return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

package/src/devcouncil/codeintel/debug/__init__.py

@@ -0,0 +1,17 @@
+"""Debugger control and fingerprint-scoped runtime evidence."""
+
+from devcouncil.codeintel.debug.consent import require_debug_consent
+from devcouncil.codeintel.debug.discovery import AdapterInfo, discover_adapters
+from devcouncil.codeintel.debug.protocol import DAPClient, DAPError
+from devcouncil.codeintel.debug.session import DebugSession, DebugSessionManager, get_debug_manager
+
+__all__ = [
+    "AdapterInfo",
+    "DAPClient",
+    "DAPError",
+    "DebugSession",
+    "DebugSessionManager",
+    "discover_adapters",
+    "get_debug_manager",
+    "require_debug_consent",
+]