devcouncil 0.3.1 → 0.4.1

package/src/devcouncil/indexing/graph/pdg/schema.py

@@ -0,0 +1,253 @@
+"""Serializable PDG layer types."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Any, cast, Dict, List, Literal, Optional
+
+PDG_VERSION = 1
+
+TaintCategory = Literal[
+    "command-injection",
+    "path-traversal",
+    "sql-injection",
+    "code-injection",
+    "ssrf",
+    "deserialization",
+    "other",
+]
+
+
+@dataclass
+class BasicBlock:
+    id: str
+    start_line: int
+    end_line: int
+    text: str = ""
+    lines: List[int] = field(default_factory=list)
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "id": self.id,
+            "start_line": self.start_line,
+            "end_line": self.end_line,
+            "text": self.text,
+            "lines": self.lines,
+        }
+
+    @classmethod
+    def from_dict(cls, raw: dict[str, Any]) -> "BasicBlock":
+        return cls(
+            id=str(raw["id"]),
+            start_line=int(raw["start_line"]),
+            end_line=int(raw["end_line"]),
+            text=str(raw.get("text") or ""),
+            lines=[int(x) for x in raw.get("lines") or []],
+        )
+
+
+@dataclass
+class CFGEdge:
+    source: str
+    target: str
+    kind: str  # fallthrough | true | false | loop | exception | return
+
+    def to_dict(self) -> dict[str, Any]:
+        return {"source": self.source, "target": self.target, "kind": self.kind}
+
+    @classmethod
+    def from_dict(cls, raw: dict[str, Any]) -> "CFGEdge":
+        return cls(
+            source=str(raw["source"]),
+            target=str(raw["target"]),
+            kind=str(raw.get("kind") or "fallthrough"),
+        )
+
+
+@dataclass
+class DataDepEdge:
+    variable: str
+    def_line: int
+    use_line: int
+    def_block: str
+    use_block: str
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "variable": self.variable,
+            "def_line": self.def_line,
+            "use_line": self.use_line,
+            "def_block": self.def_block,
+            "use_block": self.use_block,
+        }
+
+    @classmethod
+    def from_dict(cls, raw: dict[str, Any]) -> "DataDepEdge":
+        return cls(
+            variable=str(raw["variable"]),
+            def_line=int(raw["def_line"]),
+            use_line=int(raw["use_line"]),
+            def_block=str(raw["def_block"]),
+            use_block=str(raw["use_block"]),
+        )
+
+
+@dataclass
+class CDGEdge:
+    controller: str
+    dependent: str
+    branch: str  # T | F | *
+    guard: bool = False
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "controller": self.controller,
+            "dependent": self.dependent,
+            "branch": self.branch,
+            "guard": self.guard,
+        }
+
+    @classmethod
+    def from_dict(cls, raw: dict[str, Any]) -> "CDGEdge":
+        return cls(
+            controller=str(raw["controller"]),
+            dependent=str(raw["dependent"]),
+            branch=str(raw.get("branch") or "*"),
+            guard=bool(raw.get("guard")),
+        )
+
+
+@dataclass
+class TaintFinding:
+    path: str
+    function: str
+    category: TaintCategory
+    source_line: int
+    sink_line: int
+    variable: str
+    source_expr: str
+    sink_expr: str
+    confidence: str = "extracted"
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "path": self.path,
+            "function": self.function,
+            "category": self.category,
+            "source_line": self.source_line,
+            "sink_line": self.sink_line,
+            "variable": self.variable,
+            "source_expr": self.source_expr,
+            "sink_expr": self.sink_expr,
+            "confidence": self.confidence,
+        }
+
+    @classmethod
+    def from_dict(cls, raw: dict[str, Any]) -> "TaintFinding":
+        return cls(
+            path=str(raw["path"]),
+            function=str(raw["function"]),
+            category=cast(TaintCategory, raw.get("category") or "other"),
+            source_line=int(raw["source_line"]),
+            sink_line=int(raw["sink_line"]),
+            variable=str(raw.get("variable") or ""),
+            source_expr=str(raw.get("source_expr") or ""),
+            sink_expr=str(raw.get("sink_expr") or ""),
+            confidence=str(raw.get("confidence") or "extracted"),
+        )
+
+
+@dataclass
+class FunctionPDG:
+    path: str
+    qualname: str
+    start_line: int
+    end_line: int
+    blocks: List[BasicBlock] = field(default_factory=list)
+    cfg_edges: List[CFGEdge] = field(default_factory=list)
+    reaching_def: List[DataDepEdge] = field(default_factory=list)
+    cdg: List[CDGEdge] = field(default_factory=list)
+    taint: List[TaintFinding] = field(default_factory=list)
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "path": self.path,
+            "qualname": self.qualname,
+            "start_line": self.start_line,
+            "end_line": self.end_line,
+            "blocks": [b.to_dict() for b in self.blocks],
+            "cfg_edges": [e.to_dict() for e in self.cfg_edges],
+            "reaching_def": [e.to_dict() for e in self.reaching_def],
+            "cdg": [e.to_dict() for e in self.cdg],
+            "taint": [t.to_dict() for t in self.taint],
+        }
+
+    @classmethod
+    def from_dict(cls, raw: dict[str, Any]) -> "FunctionPDG":
+        return cls(
+            path=str(raw["path"]),
+            qualname=str(raw["qualname"]),
+            start_line=int(raw["start_line"]),
+            end_line=int(raw["end_line"]),
+            blocks=[BasicBlock.from_dict(b) for b in raw.get("blocks") or []],
+            cfg_edges=[CFGEdge.from_dict(e) for e in raw.get("cfg_edges") or []],
+            reaching_def=[DataDepEdge.from_dict(e) for e in raw.get("reaching_def") or []],
+            cdg=[CDGEdge.from_dict(e) for e in raw.get("cdg") or []],
+            taint=[TaintFinding.from_dict(t) for t in raw.get("taint") or []],
+        )
+
+
+@dataclass
+class FilePDG:
+    path: str
+    language: str
+    functions: List[FunctionPDG] = field(default_factory=list)
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "path": self.path,
+            "language": self.language,
+            "functions": [f.to_dict() for f in self.functions],
+        }
+
+    @classmethod
+    def from_dict(cls, raw: dict[str, Any]) -> "FilePDG":
+        return cls(
+            path=str(raw["path"]),
+            language=str(raw.get("language") or ""),
+            functions=[FunctionPDG.from_dict(f) for f in raw.get("functions") or []],
+        )
+
+
+@dataclass
+class PDGLayer:
+    version: int = PDG_VERSION
+    files: Dict[str, FilePDG] = field(default_factory=dict)
+    taint_findings: List[TaintFinding] = field(default_factory=list)
+
+    def to_meta(self) -> dict[str, Any]:
+        findings = [f.to_dict() for f in self.taint_findings]
+        stats = {
+            "file_count": len(self.files),
+            "function_count": sum(len(f.functions) for f in self.files.values()),
+            "cfg_edge_count": sum(
+                len(fn.cfg_edges) for f in self.files.values() for fn in f.functions
+            ),
+            "reaching_def_count": sum(
+                len(fn.reaching_def) for f in self.files.values() for fn in f.functions
+            ),
+            "cdg_edge_count": sum(len(fn.cdg) for f in self.files.values() for fn in f.functions),
+            "taint_count": len(findings),
+        }
+        return {
+            "version": self.version,
+            "stats": stats,
+            "files": sorted(self.files.keys()),
+            "taint_findings": findings[:500],
+        }
+
+    def shard_payload(self, path: str) -> Optional[dict[str, Any]]:
+        file_pdg = self.files.get(path)
+        if file_pdg is None:
+            return None
+        return {"pdg": file_pdg.to_dict()}

package/src/devcouncil/indexing/graph/pdg/taint.py

@@ -0,0 +1,154 @@
+"""Heuristic intra-procedural taint analysis."""
+
+from __future__ import annotations
+
+import ast
+from dataclasses import dataclass
+from typing import Dict, Iterable, List, Set, Tuple
+
+from devcouncil.indexing.graph.pdg.schema import DataDepEdge, TaintCategory, TaintFinding
+
+
+@dataclass(frozen=True)
+class _Site:
+    line: int
+    variable: str
+    expr: str
+    category: TaintCategory
+
+
+_SOURCE_PATTERNS: Dict[TaintCategory, Tuple[str, ...]] = {
+    "command-injection": ("input", "sys.argv", "os.environ.get", "request.args", "request.form"),
+    "path-traversal": ("input", "sys.argv", "os.environ.get"),
+    "sql-injection": ("input", "sys.argv", "request.args"),
+    "code-injection": ("input", "sys.argv"),
+    "ssrf": ("input", "sys.argv", "request.url"),
+    "deserialization": ("json.loads", "yaml.load", "pickle.loads"),
+    "other": (),
+}
+
+_SINK_PATTERNS: Dict[TaintCategory, Tuple[str, ...]] = {
+    "command-injection": ("os.system", "subprocess.call", "subprocess.run", "subprocess.Popen", "os.popen"),
+    "path-traversal": ("open", "Path", "shutil.copy", "shutil.move"),
+    "sql-injection": (".execute", ".raw", "executemany"),
+    "code-injection": ("eval", "exec", "compile"),
+    "ssrf": ("urlopen", "requests.get", "requests.post", "httpx.get", "httpx.post"),
+    "deserialization": (),
+    "other": (),
+}
+
+
+def _call_name(node: ast.Call) -> str:
+    func = node.func
+    if isinstance(func, ast.Name):
+        return func.id
+    if isinstance(func, ast.Attribute):
+        parts: List[str] = []
+        cur: ast.AST = func
+        while isinstance(cur, ast.Attribute):
+            parts.append(cur.attr)
+            cur = cur.value
+        if isinstance(cur, ast.Name):
+            parts.append(cur.id)
+        return ".".join(reversed(parts))
+    return ""
+
+
+class _SiteVisitor(ast.NodeVisitor):
+    def __init__(self) -> None:
+        self.sites: List[_Site] = []
+
+    def visit_Call(self, node: ast.Call) -> None:
+        name = _call_name(node)
+        line = int(node.lineno)
+        for category, patterns in _SINK_PATTERNS.items():
+            if any(p in name for p in patterns):
+                var = ""
+                if node.args and isinstance(node.args[0], ast.Name):
+                    var = node.args[0].id
+                self.sites.append(_Site(line=line, variable=var, expr=name, category=category))
+        for category, patterns in _SOURCE_PATTERNS.items():
+            if any(name.endswith(p) or name == p for p in patterns):
+                var = ""
+                if node.args and isinstance(node.args[0], ast.Name):
+                    var = node.args[0].id
+                self.sites.append(_Site(line=line, variable=var or name, expr=name, category=category))
+        self.generic_visit(node)
+
+    def visit_Name(self, node: ast.Name) -> None:
+        if isinstance(node.ctx, ast.Load) and node.id == "input":
+            self.sites.append(_Site(line=int(node.lineno), variable="input", expr="input", category="command-injection"))
+        self.generic_visit(node)
+
+
+def _param_sources(func_node: ast.AST) -> List[_Site]:
+    sites: List[_Site] = []
+    if not isinstance(func_node, (ast.FunctionDef, ast.AsyncFunctionDef)):
+        return sites
+    for arg in func_node.args.args:
+        sites.append(_Site(line=int(arg.lineno), variable=arg.arg, expr=arg.arg, category="other"))
+    return sites
+
+
+def _reaching_map(edges: Iterable[DataDepEdge]) -> Dict[Tuple[int, str], Set[Tuple[int, str]]]:
+    m: Dict[Tuple[int, str], Set[Tuple[int, str]]] = {}
+    for edge in edges:
+        m.setdefault((edge.use_line, edge.variable), set()).add((edge.def_line, edge.variable))
+    return m
+
+
+def analyze_taint(
+    path: str,
+    qualname: str,
+    func_node: ast.AST,
+    reaching: List[DataDepEdge],
+) -> List[TaintFinding]:
+    visitor = _SiteVisitor()
+    visitor.visit(func_node)
+    sites = _param_sources(func_node) + visitor.sites
+    sources = [s for s in sites if s.category != "other" or s.expr in {p for pats in _SOURCE_PATTERNS.values() for p in pats}]
+    sinks = [s for s in sites if any(s.expr.startswith(p) or p in s.expr for pats in _SINK_PATTERNS.values() for p in pats if pats)]
+    reach = _reaching_map(reaching)
+    findings: List[TaintFinding] = []
+    seen: Set[tuple[int, int, str]] = set()
+    for src in sources:
+        for sink in sinks:
+            if sink.line < src.line:
+                continue
+            if src.category != "other" and sink.category != "other" and src.category != sink.category:
+                continue
+            tainted = {src.variable}
+            if sink.variable and sink.variable not in tainted:
+                # walk reaching defs from sink arg
+                queue = [(sink.line, sink.variable)]
+                ok = False
+                visited: Set[Tuple[int, str]] = set()
+                while queue:
+                    use_line, var = queue.pop()
+                    if (use_line, var) in visited:
+                        continue
+                    visited.add((use_line, var))
+                    if var == src.variable or (src.line, src.variable) in reach.get((use_line, var), set()):
+                        ok = True
+                        break
+                    for def_line, def_var in reach.get((use_line, var), set()):
+                        queue.append((def_line, def_var))
+                if not ok and sink.variable != src.variable:
+                    continue
+            key = (src.line, sink.line, sink.category)
+            if key in seen:
+                continue
+            seen.add(key)
+            findings.append(
+                TaintFinding(
+                    path=path,
+                    function=qualname,
+                    category=sink.category if sink.category != "other" else src.category,
+                    source_line=src.line,
+                    sink_line=sink.line,
+                    variable=sink.variable or src.variable,
+                    source_expr=src.expr,
+                    sink_expr=sink.expr,
+                )
+            )
+    return findings

package/src/devcouncil/indexing/graph/query.py

@@ -0,0 +1,302 @@
+"""Query helpers over a persisted CodeGraph.
+
+devcouncil: allow-unwired — package-private; reached via package ``__init__`` / CLI.
+"""
+
+from __future__ import annotations
+
+from collections import defaultdict, deque
+from pathlib import Path
+from typing import Any, Dict, List, Optional, Set
+
+from devcouncil.indexing.graph.build import load_code_graph
+from devcouncil.indexing.graph.schema import CodeGraph, GraphNode
+
+
+def _load(root: Path, graph: Optional[CodeGraph] = None) -> Optional[CodeGraph]:
+    return graph if graph is not None else load_code_graph(root)
+
+
+def _match_nodes(graph: CodeGraph, name_or_path: str) -> List[GraphNode]:
+    q = name_or_path.replace("\\", "/")
+    hits: List[GraphNode] = []
+    for n in graph.nodes:
+        if n.id == q or n.path == q or n.name == q:
+            hits.append(n)
+            continue
+        if q in n.id or n.path.endswith(q) or n.id.endswith(f"::{q}"):
+            hits.append(n)
+    return hits
+
+
+def query_symbol(
+    root: Path,
+    name_or_path: str,
+    *,
+    graph: Optional[CodeGraph] = None,
+) -> Dict[str, Any]:
+    """360° view: definition, callers, callees, importers."""
+    g = _load(root, graph)
+    if g is None:
+        return {"error": "no code graph; run `dev map` first", "query": name_or_path}
+    nodes = _match_nodes(g, name_or_path)
+    if not nodes:
+        return {"query": name_or_path, "matches": [], "definitions": []}
+
+    callers: Dict[str, List[str]] = defaultdict(list)
+    callees: Dict[str, List[str]] = defaultdict(list)
+    importers: Dict[str, List[str]] = defaultdict(list)
+    imports: Dict[str, List[str]] = defaultdict(list)
+    for e in g.edges:
+        if e.kind == "calls":
+            callers[e.target].append(e.source)
+            callees[e.source].append(e.target)
+        elif e.kind == "imports":
+            importers[e.target].append(e.source)
+            imports[e.source].append(e.target)
+
+    results = []
+    for n in nodes[:20]:
+        results.append(
+            {
+                "id": n.id,
+                "kind": n.kind.value if hasattr(n.kind, "value") else n.kind,
+                "path": n.path,
+                "name": n.name,
+                "line": n.line,
+                "area": n.area,
+                "callers": sorted(set(callers.get(n.id, [])))[:50],
+                "callees": sorted(set(callees.get(n.id, [])))[:50],
+                "importers": sorted(set(importers.get(n.id, [])))[:50],
+                "imports": sorted(set(imports.get(n.id, [])))[:50],
+            }
+        )
+    return {"query": name_or_path, "matches": len(nodes), "definitions": results}
+
+
+def symbol_has_non_test_inbound(
+    root: Path,
+    path: str,
+    name: str,
+    *,
+    graph: Optional[CodeGraph] = None,
+) -> bool:
+    """Return whether a symbol has a non-structural inbound production edge.
+
+    Structural ownership edges (``contains``/``defines``) exist for every
+    declaration and therefore do not prove wiring. Test-only callers also do not
+    clear a production dead-symbol finding.
+    """
+    from devcouncil.indexing.wiring import is_test_path
+
+    loaded = _load(root, graph)
+    if loaded is None:
+        return False
+    normalized = path.replace("\\", "/")
+    targets = {
+        node.id
+        for node in loaded.nodes
+        if node.path.replace("\\", "/") == normalized and node.name == name
+    }
+    if not targets:
+        return False
+    by_id = {node.id: node for node in loaded.nodes}
+    structural = {"contains", "defines", "documents"}
+    for edge in loaded.edges:
+        if edge.target not in targets or edge.kind in structural or edge.source in targets:
+            continue
+        source = by_id.get(edge.source)
+        source_path = source.path if source is not None else edge.source.split("::", 1)[0]
+        if source_path and not is_test_path(source_path):
+            return True
+    return False
+
+
+def trace_path(
+    root: Path,
+    start: str,
+    end: str,
+    *,
+    graph: Optional[CodeGraph] = None,
+    max_depth: int = 32,
+) -> Dict[str, Any]:
+    """Shortest path between two nodes (by id/name/path) over imports+calls+contains."""
+    g = _load(root, graph)
+    if g is None:
+        return {"error": "no code graph; run `dev map` first", "from": start, "to": end}
+
+    starts = _match_nodes(g, start)
+    ends = _match_nodes(g, end)
+    if not starts or not ends:
+        return {
+            "from": start,
+            "to": end,
+            "path": [],
+            "found": False,
+            "reason": "endpoint not found",
+        }
+
+    start_ids = {n.id for n in starts}
+    end_ids = {n.id for n in ends}
+    adj: Dict[str, Set[str]] = defaultdict(set)
+    for e in g.edges:
+        if e.kind in {"imports", "calls", "contains", "defines", "inherits"}:
+            adj[e.source].add(e.target)
+            adj[e.target].add(e.source)  # undirected for reachability UX
+
+    # BFS from all starts
+    queue: deque = deque()
+    parent: Dict[str, Optional[str]] = {}
+    for s in start_ids:
+        queue.append(s)
+        parent[s] = None
+    found: Optional[str] = None
+    depth = 0
+    while queue and depth < max_depth:
+        for _ in range(len(queue)):
+            cur = queue.popleft()
+            if cur in end_ids:
+                found = cur
+                break
+            for nxt in adj.get(cur, ()):
+                if nxt not in parent:
+                    parent[nxt] = cur
+                    queue.append(nxt)
+        if found:
+            break
+        depth += 1
+
+    if not found:
+        return {"from": start, "to": end, "path": [], "found": False}
+
+    path: List[str] = []
+    walk: Optional[str] = found
+    while walk is not None:
+        path.append(walk)
+        walk = parent.get(walk)
+    path.reverse()
+    # Prefer a path that starts at a start_id
+    return {"from": start, "to": end, "path": path, "found": True, "length": len(path) - 1}
+
+
+# --- Opt-in PDG query helpers ---
+
+
+def _load_file_pdg_from_store(root: Path, path: str):
+    from devcouncil.indexing.graph.pdg.schema import FilePDG
+
+    path = path.replace("\\", "/")
+    try:
+        from devcouncil.codeintel import get_codeintel_service
+
+        shards = get_codeintel_service(root).store.analysis_shards()
+        raw = (shards.get(path) or {}).get("pdg")
+        if isinstance(raw, dict):
+            return FilePDG.from_dict(raw)
+    except Exception:
+        pass
+    return None
+
+
+def _match_pdg_functions(root: Path, graph: CodeGraph, target: str):
+    q = target.replace("\\", "/")
+    paths: set[str] = set()
+    if q.endswith(".py"):
+        paths.add(q)
+    else:
+        for n in graph.nodes:
+            if n.name == q or n.id.endswith(f"::{q}") or q in n.id:
+                if n.path:
+                    paths.add(n.path.replace("\\", "/"))
+    hits = []
+    for path in paths:
+        file_pdg = _load_file_pdg_from_store(root, path)
+        if file_pdg is None:
+            continue
+        for fn in file_pdg.functions:
+            if q.endswith(".py") or fn.qualname == q or fn.qualname.endswith(f".{q}") or q in fn.qualname:
+                hits.append(fn)
+    return hits
+
+
+def explain_pdg_taint(
+    root: Path,
+    *,
+    graph: Optional[CodeGraph] = None,
+    path: Optional[str] = None,
+    category: Optional[str] = None,
+) -> Dict[str, Any]:
+    from devcouncil.indexing.graph.build import load_pdg_layer
+    from devcouncil.indexing.graph.pdg.schema import TaintFinding
+
+    g = _load(root, graph)
+    if g is None:
+        return {"ok": False, "error": "no code graph; run `dev map --pdg` or `dev graph pdg build` first"}
+    layer = load_pdg_layer(g)
+    findings: List[TaintFinding] = list(layer.taint_findings) if layer else []
+    if not findings and isinstance(g.meta.get("pdg"), dict):
+        for item in g.meta["pdg"].get("taint_findings") or []:
+            if isinstance(item, dict):
+                findings.append(TaintFinding.from_dict(item))
+    if path:
+        path = path.replace("\\", "/")
+        findings = [f for f in findings if f.path == path]
+    if category:
+        findings = [f for f in findings if f.category == category]
+    return {"ok": True, "count": len(findings), "findings": [f.to_dict() for f in findings]}
+
+
+def query_pdg_controls(
+    root: Path,
+    target: str,
+    *,
+    graph: Optional[CodeGraph] = None,
+) -> Dict[str, Any]:
+    g = _load(root, graph)
+    if g is None:
+        return {"ok": False, "error": "no code graph"}
+    functions = _match_pdg_functions(root, g, target)
+    if not functions:
+        return {"ok": False, "error": f"no PDG for target {target!r}; run `dev graph pdg build`"}
+    return {
+        "ok": True,
+        "target": target,
+        "functions": [
+            {"path": fn.path, "qualname": fn.qualname, "cdg": [e.to_dict() for e in fn.cdg]}
+            for fn in functions
+        ],
+    }
+
+
+def query_pdg_flows(
+    root: Path,
+    target: str,
+    *,
+    variable: Optional[str] = None,
+    graph: Optional[CodeGraph] = None,
+) -> Dict[str, Any]:
+    g = _load(root, graph)
+    if g is None:
+        return {"ok": False, "error": "no code graph"}
+    functions = _match_pdg_functions(root, g, target)
+    if not functions:
+        return {"ok": False, "error": f"no PDG for target {target!r}; run `dev graph pdg build`"}
+    out: List[dict[str, Any]] = []
+    for fn in functions:
+        flows = fn.reaching_def
+        if variable:
+            flows = [e for e in flows if e.variable == variable]
+        out.append(
+            {
+                "path": fn.path,
+                "qualname": fn.qualname,
+                "reaching_def": [e.to_dict() for e in flows],
+            }
+        )
+    return {"ok": True, "target": target, "variable": variable, "functions": out}
+
+
+# Aliases matching pdg.query public API
+explain_taint = explain_pdg_taint
+query_controls = query_pdg_controls
+query_flows = query_pdg_flows