devcouncil 0.3.0 → 0.4.0

package/src/devcouncil/codeintel/debug/python_trace_runner.py

@@ -0,0 +1,81 @@
+"""Child-process entrypoint for exact Python call/return tracing."""
+
+from __future__ import annotations
+
+import argparse
+import json
+import runpy
+import sys
+import time
+from collections import Counter
+from pathlib import Path
+from types import FrameType
+from typing import Any
+
+
+def _frame_id(frame: FrameType, root: Path) -> str | None:
+    path = Path(frame.f_code.co_filename).resolve()
+    try:
+        rel = path.relative_to(root).as_posix()
+    except ValueError:
+        return None
+    return f"{rel}::{frame.f_code.co_qualname}"
+
+
+def run_trace(root: Path, output: Path, script: Path, argv: list[str]) -> int:
+    root = root.resolve()
+    calls: Counter[tuple[str, str]] = Counter()
+    first_seen: dict[tuple[str, str], float] = {}
+    last_seen: dict[tuple[str, str], float] = {}
+
+    def profile(frame: FrameType, event: str, _arg: Any):
+        if event != "call" or frame.f_back is None:
+            return profile
+        caller = _frame_id(frame.f_back, root)
+        callee = _frame_id(frame, root)
+        if caller and callee:
+            key = (caller, callee)
+            now = time.time()
+            calls[key] += 1
+            first_seen.setdefault(key, now)
+            last_seen[key] = now
+        return profile
+
+    previous_argv = sys.argv
+    sys.argv = [str(script), *argv]
+    sys.setprofile(profile)
+    exit_code = 0
+    try:
+        runpy.run_path(str(script), run_name="__main__")
+    except SystemExit as exc:
+        exit_code = int(exc.code) if isinstance(exc.code, int) else 1
+    finally:
+        sys.setprofile(None)
+        sys.argv = previous_argv
+    output.parent.mkdir(parents=True, exist_ok=True)
+    with output.open("w", encoding="utf-8") as handle:
+        for (source, target), count in sorted(calls.items()):
+            handle.write(json.dumps({
+                "source": source,
+                "target": target,
+                "kind": "observed_calls",
+                "count": count,
+                "first_seen": first_seen[(source, target)],
+                "last_seen": last_seen[(source, target)],
+                "evidence": {"provider": "python-sys-setprofile"},
+            }, sort_keys=True) + "\n")
+    return exit_code
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--root", type=Path, required=True)
+    parser.add_argument("--output", type=Path, required=True)
+    parser.add_argument("script", type=Path)
+    parser.add_argument("args", nargs=argparse.REMAINDER)
+    ns = parser.parse_args()
+    return run_trace(ns.root, ns.output, ns.script, list(ns.args))
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

package/src/devcouncil/codeintel/debug/session.py

@@ -0,0 +1,238 @@
+"""Persistent in-process debug-session manager used by MCP and embedded CLI."""
+
+from __future__ import annotations
+
+import re
+import threading
+import uuid
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any, Sequence, cast
+
+from devcouncil.codeintel.debug.discovery import adapter_by_command
+from devcouncil.codeintel.debug.fingerprint import build_fingerprint, executable_hash, source_fingerprint
+from devcouncil.codeintel.debug.protocol import DAPClient
+from devcouncil.codeintel.service import get_codeintel_service
+
+_SECRET = re.compile(r"(?i)(token|secret|password|api[_-]?key|authorization)\s*[:=]\s*([^,;\s]+)")
+
+
+def redact_value(value: str, *, limit: int = 4096) -> str:
+    redacted = _SECRET.sub(lambda match: f"{match.group(1)}=[REDACTED]", value)
+    return redacted if len(redacted) <= limit else redacted[:limit] + "…[truncated]"
+
+
+@dataclass
+class DebugSession:
+    id: str
+    root: Path
+    client: DAPClient
+    adapter_command: tuple[str, ...]
+    adapter_id: str
+    adapter_version: str
+    adapter_requests: tuple[str, ...]
+    request: str
+    capabilities: dict[str, Any]
+    source_fingerprint: str
+    build_fingerprint: str
+    executable_hash: str
+    breakpoints: dict[str, list[int]] = field(default_factory=dict)
+
+    def as_dict(self) -> dict[str, Any]:
+        return {
+            "id": self.id,
+            "project_root": str(self.root),
+            "adapter_command": list(self.adapter_command),
+            "adapter_id": self.adapter_id,
+            "adapter_version": self.adapter_version,
+            "adapter_requests": list(self.adapter_requests),
+            "request": self.request,
+            "capabilities": self.capabilities,
+            "source_fingerprint": self.source_fingerprint,
+            "build_fingerprint": self.build_fingerprint,
+            "executable_hash": self.executable_hash,
+            "breakpoints": self.breakpoints,
+        }
+
+
+class DebugSessionManager:
+    def __init__(self):
+        self._sessions: dict[str, DebugSession] = {}
+        self._lock = threading.Lock()
+
+    def start(
+        self,
+        root: Path,
+        adapter_command: Sequence[str],
+        *,
+        request: str,
+        arguments: dict[str, Any],
+        initial_breakpoints: dict[str, Sequence[int]] | None = None,
+        timeout: float = 30.0,
+    ) -> DebugSession:
+        if request not in {"launch", "attach"}:
+            raise ValueError("request must be launch or attach")
+        root = root.expanduser().resolve()
+        adapter = adapter_by_command(list(adapter_command))
+        client = DAPClient.start_stdio(adapter_command, cwd=str(root))
+        try:
+            capabilities = client.initialize(adapter_id=adapter.id if adapter else "devcouncil")
+            pending = client.begin_request(request, arguments)
+            client.wait_event("initialized", timeout=timeout, fail_on=pending)
+            normalized_breakpoints: dict[str, list[int]] = {}
+            for source, lines in (initial_breakpoints or {}).items():
+                normalized = str(Path(source).expanduser().resolve())
+                normalized_breakpoints[normalized] = [int(line) for line in lines]
+                client.request("setBreakpoints", {
+                    "source": {"path": normalized},
+                    "breakpoints": [{"line": line} for line in normalized_breakpoints[normalized]],
+                    "sourceModified": False,
+                }, timeout=timeout)
+            client.request("configurationDone", timeout=timeout)
+            client.wait_response(pending, timeout=timeout)
+        except Exception:
+            client.close()
+            raise
+        executable = adapter_command[0]
+        session = DebugSession(
+            id=uuid.uuid4().hex,
+            root=root,
+            client=client,
+            adapter_command=tuple(adapter_command),
+            adapter_id=adapter.id if adapter else "custom",
+            adapter_version=adapter.version if adapter else "",
+            adapter_requests=adapter.requests if adapter else (request,),
+            request=request,
+            capabilities=capabilities,
+            source_fingerprint=source_fingerprint(root),
+            build_fingerprint=build_fingerprint(root, executable),
+            executable_hash=executable_hash(executable),
+            breakpoints=normalized_breakpoints,
+        )
+        with self._lock:
+            self._sessions[session.id] = session
+        return session
+
+    def get(self, session_id: str) -> DebugSession:
+        with self._lock:
+            session = self._sessions.get(session_id)
+        if session is None:
+            raise KeyError(f"unknown debug session: {session_id}")
+        return session
+
+    def list(self) -> list[dict[str, Any]]:
+        with self._lock:
+            return [session.as_dict() for session in self._sessions.values()]
+
+    def set_breakpoints(self, session_id: str, source: str, lines: Sequence[int]) -> dict[str, Any]:
+        session = self.get(session_id)
+        normalized = str(Path(source).expanduser().resolve())
+        body = session.client.request("setBreakpoints", {
+            "source": {"path": normalized},
+            "breakpoints": [{"line": int(line)} for line in lines],
+            "sourceModified": False,
+        })
+        session.breakpoints[normalized] = [int(line) for line in lines]
+        return body
+
+    def control(self, session_id: str, action: str, *, thread_id: int | None = None) -> dict[str, Any]:
+        session = self.get(session_id)
+        commands = {
+            "continue": "continue",
+            "pause": "pause",
+            "next": "next",
+            "stepIn": "stepIn",
+            "stepOut": "stepOut",
+        }
+        command = commands.get(action)
+        if command is None:
+            raise ValueError(f"unsupported debug action: {action}")
+        arguments = {"threadId": thread_id} if thread_id is not None else {}
+        return session.client.request(command, arguments)
+
+    def inspect(self, session_id: str, operation: str, arguments: dict[str, Any]) -> dict[str, Any]:
+        session = self.get(session_id)
+        allowed = {"threads", "stackTrace", "scopes", "variables", "source", "disassemble"}
+        if operation not in allowed:
+            raise ValueError(f"unsupported inspect operation: {operation}")
+        body = session.client.request(operation, arguments)
+        return cast(dict[str, Any], self._redact(body))
+
+    def evaluate(
+        self,
+        session_id: str,
+        expression: str,
+        *,
+        frame_id: int | None,
+        allow_side_effects: bool,
+    ) -> dict[str, Any]:
+        session = self.get(session_id)
+        return cast(dict[str, Any], self._redact(session.client.evaluate(
+            expression,
+            frame_id=frame_id,
+            allow_side_effects=allow_side_effects,
+        )))
+
+    def capture_stack(self, session_id: str, *, thread_id: int) -> dict[str, Any]:
+        session = self.get(session_id)
+        body = session.client.request("stackTrace", {"threadId": thread_id})
+        frames = list(body.get("stackFrames") or [])
+        observations = []
+        for caller, callee in zip(reversed(frames[1:]), reversed(frames[:-1])):
+            observations.append({
+                "source": self._frame_id(caller),
+                "target": self._frame_id(callee),
+                "kind": "sampled_stack",
+                "evidence": {"thread_id": thread_id, "provider": "dap-stack"},
+            })
+        store = get_codeintel_service(session.root).store
+        runtime_id = store.start_runtime_session(
+            provider="dap-stack",
+            source_fingerprint=session.source_fingerprint,
+            build_fingerprint=session.build_fingerprint,
+            executable_hash=session.executable_hash,
+            metadata={
+                "debug_session": session.id,
+                "thread_id": thread_id,
+                "adapter_id": session.adapter_id,
+                "adapter_version": session.adapter_version,
+                "adapter_requests": list(session.adapter_requests),
+                "adapter_capabilities": session.capabilities,
+            },
+        )
+        store.add_runtime_observations(runtime_id, observations)
+        store.end_runtime_session(runtime_id)
+        return {"session_id": runtime_id, "observations": observations, "frames": self._redact(frames)}
+
+    def stop(self, session_id: str, *, terminate_debuggee: bool = True) -> None:
+        session = self.get(session_id)
+        try:
+            session.client.request("disconnect", {"terminateDebuggee": terminate_debuggee}, timeout=5.0)
+        except Exception:
+            pass
+        session.client.close()
+        with self._lock:
+            self._sessions.pop(session_id, None)
+
+    @classmethod
+    def _redact(cls, value: Any) -> Any:
+        if isinstance(value, str):
+            return redact_value(value)
+        if isinstance(value, list):
+            return [cls._redact(item) for item in value]
+        if isinstance(value, dict):
+            return {key: cls._redact(item) for key, item in value.items()}
+        return value
+
+    @staticmethod
+    def _frame_id(frame: dict[str, Any]) -> str:
+        source = frame.get("source") or {}
+        path = source.get("path") or source.get("name") or "<unknown>"
+        return f"{path}:{frame.get('name', '<anonymous>')}:{frame.get('line', 0)}"
+
+
+_MANAGER = DebugSessionManager()
+
+
+def get_debug_manager() -> DebugSessionManager:
+    return _MANAGER

package/src/devcouncil/codeintel/debug/tracing.py

@@ -0,0 +1,201 @@
+"""Runtime trace providers independent from debugger control."""
+
+from __future__ import annotations
+
+import json
+import shutil
+import subprocess
+import sys
+import uuid
+from pathlib import Path
+from typing import Any, Sequence
+
+from devcouncil.codeintel.debug.fingerprint import build_fingerprint, executable_hash, source_fingerprint
+from devcouncil.codeintel.service import get_codeintel_service
+
+
+class PythonTraceProvider:
+    def __init__(self, root: Path):
+        self.root = root.expanduser().resolve()
+
+    def run(self, script: Path, args: Sequence[str] = (), *, timeout: float = 300.0) -> dict[str, Any]:
+        script = script if script.is_absolute() else self.root / script
+        trace_path = self.root / ".devcouncil" / "codeintel" / "traces" / f"python-{uuid.uuid4().hex}.jsonl"
+        source = source_fingerprint(self.root)
+        build = build_fingerprint(self.root, sys.executable)
+        store = get_codeintel_service(self.root).store
+        session_id = store.start_runtime_session(
+            provider="python-sys-setprofile",
+            source_fingerprint=source,
+            build_fingerprint=build,
+            executable_hash=executable_hash(sys.executable),
+            metadata={"script": str(script), "args": list(args), "trace_path": str(trace_path)},
+        )
+        try:
+            proc = subprocess.run(
+                [
+                    sys.executable,
+                    "-m",
+                    "devcouncil.codeintel.debug.python_trace_runner",
+                    "--root",
+                    str(self.root),
+                    "--output",
+                    str(trace_path),
+                    str(script),
+                    *args,
+                ],
+                cwd=self.root,
+                check=False,
+                timeout=timeout,
+            )
+            observations = load_jsonl_observations(trace_path)
+            store.add_runtime_observations(session_id, observations)
+        finally:
+            store.end_runtime_session(session_id)
+        return {
+            "session_id": session_id,
+            "provider": "python-sys-setprofile",
+            "exit_code": proc.returncode,
+            "observation_count": len(observations),
+            "trace_path": str(trace_path),
+            "source_fingerprint": source,
+            "build_fingerprint": build,
+        }
+
+
+class NodeCpuProfileProvider:
+    def __init__(self, root: Path, executable: str | Path | None = None):
+        self.root = root.expanduser().resolve()
+        resolved = str(executable) if executable else shutil.which("node")
+        if not resolved:
+            raise FileNotFoundError("Node.js executable was not found")
+        self.executable = str(Path(resolved).expanduser().resolve())
+
+    def run(self, script: Path, args: Sequence[str] = (), *, timeout: float = 300.0) -> dict[str, Any]:
+        script = script if script.is_absolute() else self.root / script
+        trace_dir = self.root / ".devcouncil" / "codeintel" / "traces"
+        trace_dir.mkdir(parents=True, exist_ok=True)
+        trace_path = trace_dir / f"node-{uuid.uuid4().hex}.cpuprofile"
+        source = source_fingerprint(self.root)
+        build = build_fingerprint(self.root, self.executable)
+        executable_fingerprint = executable_hash(self.executable)
+        store = get_codeintel_service(self.root).store
+        session_id = store.start_runtime_session(
+            provider="node-cpu-profile",
+            source_fingerprint=source,
+            build_fingerprint=build,
+            executable_hash=executable_fingerprint,
+            metadata={
+                "script": str(script),
+                "args": list(args),
+                "trace_path": str(trace_path),
+                "runtime_version": _runtime_version(self.executable),
+            },
+        )
+        try:
+            proc = subprocess.run(
+                [
+                    self.executable,
+                    "--cpu-prof",
+                    f"--cpu-prof-dir={trace_dir}",
+                    f"--cpu-prof-name={trace_path.name}",
+                    str(script),
+                    *args,
+                ],
+                cwd=self.root,
+                check=False,
+                timeout=timeout,
+            )
+            observations = load_node_cpu_profile(trace_path)
+            store.add_runtime_observations(session_id, observations)
+        finally:
+            store.end_runtime_session(session_id)
+        return {
+            "session_id": session_id,
+            "provider": "node-cpu-profile",
+            "exit_code": proc.returncode,
+            "observation_count": len(observations),
+            "trace_path": str(trace_path),
+            "source_fingerprint": source,
+            "build_fingerprint": build,
+            "executable_hash": executable_fingerprint,
+        }
+
+
+def load_jsonl_observations(path: Path) -> list[dict[str, Any]]:
+    rows: list[dict[str, Any]] = []
+    for line in path.read_text(encoding="utf-8").splitlines() if path.is_file() else []:
+        try:
+            value = json.loads(line)
+        except json.JSONDecodeError:
+            continue
+        if isinstance(value, dict) and value.get("source") and value.get("target"):
+            rows.append(value)
+    return rows
+
+
+def load_node_cpu_profile(path: Path) -> list[dict[str, Any]]:
+    """Convert a Chrome/Node CPU profile's parent/child nodes to sampled edges."""
+
+    data = json.loads(path.read_text(encoding="utf-8"))
+    nodes = {int(node["id"]): node for node in data.get("nodes") or [] if isinstance(node, dict) and "id" in node}
+    samples = [int(value) for value in data.get("samples") or []]
+    counts: dict[int, int] = {}
+    for sample in samples:
+        counts[sample] = counts.get(sample, 0) + 1
+    rows: list[dict[str, Any]] = []
+    for parent_id, parent in nodes.items():
+        parent_name = _profile_frame(parent)
+        for child_id in parent.get("children") or []:
+            child = nodes.get(int(child_id))
+            if child is None:
+                continue
+            rows.append({
+                "source": parent_name,
+                "target": _profile_frame(child),
+                "kind": "sampled_calls",
+                "count": max(1, counts.get(int(child_id), 0)),
+                "evidence": {"provider": "node-cpu-profile", "parent_id": parent_id, "child_id": child_id},
+            })
+    return rows
+
+
+def import_runtime_trace(root: Path, path: Path, *, provider: str = "jsonl") -> dict[str, Any]:
+    root = root.expanduser().resolve()
+    path = path.expanduser().resolve()
+    observations = load_node_cpu_profile(path) if path.suffix == ".cpuprofile" else load_jsonl_observations(path)
+    store = get_codeintel_service(root).store
+    source = source_fingerprint(root)
+    session_id = store.start_runtime_session(
+        provider=provider,
+        source_fingerprint=source,
+        build_fingerprint=build_fingerprint(root),
+        metadata={"imported_path": str(path)},
+    )
+    store.add_runtime_observations(session_id, observations)
+    store.end_runtime_session(session_id)
+    return {"session_id": session_id, "provider": provider, "observation_count": len(observations)}
+
+
+def _profile_frame(node: dict[str, Any]) -> str:
+    frame = node.get("callFrame") or {}
+    url = frame.get("url") or "<unknown>"
+    function = frame.get("functionName") or "<anonymous>"
+    line = int(frame.get("lineNumber", -1)) + 1
+    return f"{url}:{function}:{line}"
+
+
+def _runtime_version(executable: str) -> str:
+    try:
+        completed = subprocess.run(
+            [executable, "--version"],
+            stdin=subprocess.DEVNULL,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.STDOUT,
+            check=False,
+            timeout=2.0,
+            text=True,
+        )
+    except (OSError, subprocess.TimeoutExpired):
+        return ""
+    return completed.stdout.strip().splitlines()[0][:128] if completed.stdout.strip() else ""

package/src/devcouncil/codeintel/languages/__init__.py

@@ -0,0 +1,17 @@
+"""Language manifests and Tree-sitter grammar diagnostics."""
+
+from devcouncil.codeintel.languages.registry import (
+    LANGUAGE_SPECS,
+    LanguageSpec,
+    detect_language,
+    grammar_status,
+    supported_languages,
+)
+
+__all__ = [
+    "LANGUAGE_SPECS",
+    "LanguageSpec",
+    "detect_language",
+    "grammar_status",
+    "supported_languages",
+]