adl-toolkit 0.0.0-stage → 0.1.1

package/adl_drift/lang/python/taint.py

@@ -0,0 +1,365 @@
+"""Python intra-function taint mechanics (P2.19, ported unchanged onto the
+P4.6 seam).
+
+`PythonTaintStrategy` is the exact pre-P4.6 `detectors.taint._analyze_one`
+CST walk, now a `detectors.taint.TaintStrategy` subclass reading its
+source/sink tables from `self.config` (a `CFGConfig`) instead of hardcoded
+module constants. No behavioral change for Python — `default_python_taint_
+strategy()` below is what `lang/python/cfg.py::default_python_cfg_config()`
+packages `TAINT_SINKS`/`SINK_METHODS` into, so every existing call site sees
+byte-identical output.
+"""
+
+from __future__ import annotations
+
+from typing import Dict, List, Optional, Set, Tuple
+
+from tree_sitter import Node
+
+from adl_drift.detectors.taint import TaintPath, TaintStrategy, _OutCall
+from adl_drift.lang.base import CFGConfig
+from adl_drift.utils.types import SourcePosition
+
+__all__ = [
+    "TAINT_SINKS",
+    "SINK_METHODS",
+    "PythonTaintStrategy",
+    "default_python_taint_strategy",
+]
+
+# ─── sinks ────────────────────────────────────────────────────────────────────
+
+# Exact dotted/identifier callable text → vulnerability category.
+TAINT_SINKS: Dict[str, str] = {
+    "eval": "code_injection",
+    "exec": "code_injection",
+    "os.system": "command_injection",
+    "os.popen": "command_injection",
+    "subprocess.run": "command_injection",
+    "subprocess.Popen": "command_injection",
+    "subprocess.call": "command_injection",
+    "subprocess.check_call": "command_injection",
+    "subprocess.check_output": "command_injection",
+    "pickle.load": "unsafe_deserialization",
+    "pickle.loads": "unsafe_deserialization",
+    "yaml.load": "unsafe_deserialization",
+}
+
+# Final attribute name (any receiver) → category. Matches `db.execute(q)`,
+# `conn.cursor().execute(q)`, `self.session.execute(q)`, etc.
+SINK_METHODS: Dict[str, str] = {
+    "execute": "sql_injection",
+    "executemany": "sql_injection",
+    "executescript": "sql_injection",
+    "raw": "sql_injection",
+}
+
+
+# ─── CST helpers (self-contained; mirror lang/python/cfg.py conventions) ─────
+
+def _text(node: Node, source: bytes) -> str:
+    try:
+        return source[node.start_byte:node.end_byte].decode("utf-8", errors="replace")
+    except Exception:
+        return ""
+
+
+def _pos(node: Node) -> SourcePosition:
+    return SourcePosition(
+        line=node.start_point[0] + 1,
+        column=node.start_point[1],
+        end_line=node.end_point[0] + 1,
+        end_column=node.end_point[1],
+    )
+
+
+def _ids_in(node: Optional[Node], source: bytes) -> Set[str]:
+    """Identifier *reads* under *node* (attribute receivers only, not field names)."""
+    out: Set[str] = set()
+    if node is None:
+        return out
+    stack = [node]
+    while stack:
+        n = stack.pop()
+        t = n.type
+        if t == "attribute":
+            obj = n.child_by_field_name("object")
+            if obj is not None:
+                stack.append(obj)
+            continue
+        if t == "keyword_argument":
+            val = n.child_by_field_name("value")
+            if val is not None:
+                stack.append(val)
+            continue
+        if t in ("function_definition", "class_definition", "lambda"):
+            continue
+        if t == "identifier":
+            txt = _text(n, source)
+            if txt:
+                out.add(txt)
+            continue
+        stack.extend(n.children)
+    return out
+
+
+def _lhs_names(node: Node, source: bytes) -> List[str]:
+    if node.type == "identifier":
+        return [_text(node, source)]
+    names: List[str] = []
+    for child in node.children:
+        if child.type not in (",", "(", ")", "[", "]", "*", "."):
+            names.extend(_lhs_names(child, source))
+    return names
+
+
+def _param_names(func_node: Node, source: bytes) -> List[str]:
+    """Ordered parameter identifiers (includes ``self``/``cls``)."""
+    params = func_node.child_by_field_name("parameters")
+    if params is None:
+        return []
+    names: List[str] = []
+    for child in params.children:
+        ct = child.type
+        if ct == "identifier":
+            names.append(_text(child, source))
+        elif ct in ("typed_parameter", "default_parameter", "typed_default_parameter",
+                    "list_splat_pattern", "dictionary_splat_pattern"):
+            inner = child.child_by_field_name("name") or (
+                child.children[0] if child.children else None
+            )
+            if inner is not None and inner.type == "identifier":
+                names.append(_text(inner, source))
+    return names
+
+
+def _collect_calls(body: Node) -> List[Node]:
+    """Every ``call`` node under *body*, not descending into nested functions."""
+    out: List[Node] = []
+    stack = [body]
+    while stack:
+        node = stack.pop()
+        if node.type == "call":
+            out.append(node)
+        if node.type == "function_definition" and node is not body:
+            continue
+        stack.extend(node.children)
+    return out
+
+
+def _positional_args(call: Node) -> List[Node]:
+    args = call.child_by_field_name("arguments")
+    if args is None:
+        return []
+    return [c for c in args.children
+            if c.type not in ("(", ")", ",", "comment", "keyword_argument",
+                              "list_splat", "dictionary_splat")]
+
+
+def _keyword_args(call: Node, source: bytes) -> List[Tuple[str, Node]]:
+    args = call.child_by_field_name("arguments")
+    if args is None:
+        return []
+    out: List[Tuple[str, Node]] = []
+    for c in args.children:
+        if c.type == "keyword_argument":
+            name = c.child_by_field_name("name")
+            val = c.child_by_field_name("value")
+            if name is not None and val is not None:
+                out.append((_text(name, source), val))
+    return out
+
+
+def _callee_simple_name(call: Node, source: bytes) -> Optional[str]:
+    """Final identifier of the call target: ``f`` / ``m.f`` / ``a.b.f`` → ``f``."""
+    fn = call.child_by_field_name("function")
+    if fn is None:
+        return None
+    if fn.type == "identifier":
+        return _text(fn, source)
+    if fn.type == "attribute":
+        attr = fn.child_by_field_name("attribute")
+        if attr is not None and attr.type == "identifier":
+            return _text(attr, source)
+    return None
+
+
+class PythonTaintStrategy(TaintStrategy):
+    """Python's `TaintStrategy` (P2.19/P4.6)."""
+
+    def param_names(self, func_node: Node, source: bytes) -> List[str]:
+        return _param_names(func_node, source)
+
+    def _source_prefix(self, text: str) -> Optional[str]:
+        """Return the taint-source label if *text* begins with a known source."""
+        t = text.strip()
+        for src in self.config.taint_sources:
+            if t.startswith(src):
+                return src
+        return None
+
+    def _sink_info(self, call: Node, source: bytes) -> Optional[Tuple[str, str, Node]]:
+        """If *call* is a taint sink, return ``(callable_text, category, arg_node)``.
+
+        ``arg_node`` is the first positional argument (the data the sink consumes).
+        """
+        fn = call.child_by_field_name("function")
+        if fn is None:
+            return None
+        func_text = _text(fn, source).strip()
+        category = self.config.taint_sinks.get(func_text)
+        if category is None and fn.type == "attribute":
+            attr = fn.child_by_field_name("attribute")
+            if attr is not None:
+                category = self.config.sink_methods.get(_text(attr, source))
+        if category is None:
+            return None
+        pos_args = _positional_args(call)
+        if not pos_args:
+            return None
+        return func_text, category, pos_args[0]
+
+    def analyze_one(
+        self,
+        func_node: Node,
+        source: bytes,
+        function_id: str,
+        file_path: str,
+        seed_params: Set[str],
+    ) -> Tuple[Set[str], List[TaintPath], List[_OutCall]]:
+        body = func_node.child_by_field_name("body")
+        if body is None:
+            return set(), [], []
+
+        tainted: Set[str] = set(seed_params)
+        source_of: Dict[str, str] = {p: "parameter" for p in seed_params}
+
+        # ── collect assignment records (lhs names, rhs identifier reads, src label) ──
+        assigns: List[Tuple[List[str], Set[str], Optional[str]]] = []
+
+        def _record_assign(lhs: Optional[Node], rhs: Optional[Node]) -> None:
+            if lhs is None or rhs is None:
+                return
+            names = [n for n in _lhs_names(lhs, source) if n]
+            if not names:
+                return
+            assigns.append((names, _ids_in(rhs, source),
+                            self._source_prefix(_text(rhs, source))))
+
+        stack = [body]
+        while stack:
+            node = stack.pop()
+            t = node.type
+            if t == "function_definition" and node is not body:
+                continue  # nested function — separate analysis
+            if t == "assignment":
+                _record_assign(node.child_by_field_name("left"),
+                               node.child_by_field_name("right"))
+            elif t == "augmented_assignment":
+                lhs = node.child_by_field_name("left")
+                rhs = node.child_by_field_name("right")
+                if lhs is not None and rhs is not None:
+                    names = [n for n in _lhs_names(lhs, source) if n]
+                    rhs_ids = _ids_in(rhs, source) | set(names)  # x += y keeps x
+                    assigns.append((names, rhs_ids, self._source_prefix(_text(rhs, source))))
+            elif t == "named_expression":
+                _record_assign(node.child_by_field_name("name"),
+                               node.child_by_field_name("value"))
+            elif t == "for_statement":
+                # `for x in <iter>` taints x when the iterable is tainted
+                _record_assign(node.child_by_field_name("left"),
+                               node.child_by_field_name("right"))
+            elif t == "with_item":
+                _record_assign(node.child_by_field_name("alias"),
+                               node.child_by_field_name("value"))
+            stack.extend(node.children)
+
+        # ── propagate to fixpoint ───────────────────────────────────────────────
+        changed = True
+        while changed:
+            changed = False
+            for names, rhs_ids, lbl in assigns:
+                tainted_rhs = lbl or any(i in tainted for i in rhs_ids)
+                if not tainted_rhs:
+                    continue
+                label = lbl or next(
+                    (source_of.get(i) for i in rhs_ids if i in tainted), None
+                )
+                for n in names:
+                    if n not in tainted:
+                        tainted.add(n)
+                        source_of[n] = label or "tainted"
+                        changed = True
+
+        # ── sink detection ──────────────────────────────────────────────────────
+        paths: List[TaintPath] = []
+        seen_sites: Set[Tuple[int, int]] = set()
+        for call in _collect_calls(body):
+            info = self._sink_info(call, source)
+            if info is None:
+                continue
+            callable_text, category, arg_node = info
+            site = _pos(call)
+            site_key = (site.line, site.column)
+
+            direct = self._source_prefix(_text(arg_node, source))
+            if direct:
+                binding: Optional[str] = None
+                label = direct
+                cross = False
+            else:
+                arg_ids = _ids_in(arg_node, source)
+                hit = next((i for i in arg_ids if i in tainted), None)
+                if hit is None:
+                    continue
+                binding = hit
+                label = source_of.get(hit, "tainted")
+                cross = label == "parameter"
+
+            if site_key in seen_sites:
+                continue
+            seen_sites.add(site_key)
+
+            chain = [label]
+            if binding:
+                chain.append(binding)
+            chain.append(callable_text)
+            paths.append(TaintPath(
+                function_id=function_id,
+                file_path=file_path,
+                sink=callable_text,
+                sink_category=category,
+                source_label=label,
+                binding_name=binding,
+                sink_site=site,
+                chain=chain,
+                confidence=0.7 if cross else 0.85,
+                cross_function=cross,
+            ))
+
+        # ── outgoing tainted calls (for cross-function propagation) ─────────────
+        out_calls: List[_OutCall] = []
+        for call in _collect_calls(body):
+            callee = _callee_simple_name(call, source)
+            if not callee:
+                continue
+            pos_t: Set[int] = set()
+            for idx, arg in enumerate(_positional_args(call)):
+                if self._source_prefix(_text(arg, source)) or (_ids_in(arg, source) & tainted):
+                    pos_t.add(idx)
+            kw_t: Set[str] = set()
+            for kw_name, val in _keyword_args(call, source):
+                if self._source_prefix(_text(val, source)) or (_ids_in(val, source) & tainted):
+                    kw_t.add(kw_name)
+            if pos_t or kw_t:
+                out_calls.append(_OutCall(
+                    callee, pos_t, kw_t,
+                    site_line=call.start_point[0] + 1,
+                    site_column=call.start_point[1],
+                ))
+
+        return tainted, paths, out_calls
+
+
+def default_python_taint_strategy(config: CFGConfig) -> PythonTaintStrategy:
+    return PythonTaintStrategy(config)

package/adl_drift/lang/registry.py

@@ -0,0 +1,117 @@
+"""Backend registry — dispatch by name / file extension / project root.
+
+The engine layer (scanner, cli/commands, server/handlers) asks this registry
+for a `LanguageBackend` instead of importing a concrete implementation
+(P4.0/P4.3, PROBLEMS.md). File discovery is still Python-only until P4.8.
+
+A simple dict-keyed registry is deliberate — no plugin discovery, no entry
+points. Backends register themselves via `register_backend()`.
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Dict, List, Optional
+
+from adl_drift.lang.base import LanguageBackend
+from adl_drift.utils.fswalk import iter_tree
+
+_REGISTRY: Dict[str, LanguageBackend] = {}
+
+
+def register_backend(backend: LanguageBackend) -> None:
+    """Register (or replace) a backend under `backend.name`."""
+    _REGISTRY[backend.name] = backend
+
+
+def get_backend(name: str) -> LanguageBackend:
+    """Look up a backend by name (e.g. "python", "java").
+
+    Raises `KeyError` if no backend is registered under that name — there is
+    no silent Python fallback, so an unregistered language fails loudly.
+    """
+    try:
+        return _REGISTRY[name]
+    except KeyError:
+        raise KeyError(
+            f"No language backend registered for {name!r}. "
+            f"Registered backends: {sorted(_REGISTRY)}"
+        ) from None
+
+
+def backend_for_file(path: Path) -> Optional[LanguageBackend]:
+    """Dispatch a file path to the backend claiming its extension.
+
+    Returns `None` when no registered backend claims the extension (e.g.
+    `.java` today, since no Java backend is registered yet) — callers decide
+    whether that's an error or a file to skip.
+    """
+    suffix = Path(path).suffix
+    for backend in _REGISTRY.values():
+        if suffix in backend.file_extensions:
+            return backend
+    return None
+
+
+def detect_backends(root: Path) -> List[LanguageBackend]:
+    """Scan a project root for which registered languages are present.
+
+    A backend is considered present if any of its `project_markers` exist
+    directly under `root`, or if any file with one of its
+    `file_extensions` exists anywhere under `root`.
+
+    MCP_GRAPH_CACHE_PLAN.md Lớp 2: backends without a root marker are
+    probed together in one ``os.walk`` that stops as soon as every pending
+    backend has been seen, instead of one full ``rglob`` per backend per
+    extension (proving a language is *absent* used to cost a whole-tree
+    walk each). ``rglob`` matched directory entries too, so directory names
+    count as well.
+    """
+    root = Path(root)
+    backends = list(_REGISTRY.values())
+    present = {
+        backend.name for backend in backends
+        if any((root / marker).exists() for marker in backend.project_markers)
+    }
+    pending: Dict[str, str] = {
+        ext: backend.name
+        for backend in backends if backend.name not in present
+        for ext in backend.file_extensions
+    }
+    if pending:
+        for _dirpath, dirs, others in iter_tree(str(root)):
+            for entry in dirs + others:
+                for ext, backend_name in list(pending.items()):
+                    if entry.name.endswith(ext):
+                        present.add(backend_name)
+                        pending = {
+                            e: b for e, b in pending.items() if b != backend_name
+                        }
+            if not pending:
+                break
+    return [backend for backend in backends if backend.name in present]
+
+
+def _register_default_backends() -> None:
+    """Best-effort registration of bundled backends.
+
+    Wrapped in try/except so importing `adl_drift.lang.registry` never
+    fails outright even if the shim's optional dependencies are missing —
+    matches this module's job as pure dispatch scaffolding, not a hard
+    dependency on any concrete backend.
+    """
+    try:
+        from adl_drift.lang.python.backend import PythonBackend
+        if "python" not in _REGISTRY:
+            register_backend(PythonBackend())
+    except Exception:
+        pass
+    try:
+        from adl_drift.lang.java.backend import JavaBackend
+        if "java" not in _REGISTRY:
+            register_backend(JavaBackend())
+    except Exception:
+        pass
+
+
+_register_default_backends()

package/adl_drift/parser/__init__.py

@@ -0,0 +1 @@
+"""Tree-sitter parser wrapper for Python source files."""

package/adl_drift/parser/cache.py

@@ -0,0 +1,38 @@
+"""Minimal in-memory parse cache (SHA256-keyed). Disk persistence stubbed."""
+
+from __future__ import annotations
+
+import hashlib
+from pathlib import Path
+from typing import Any, Dict, Optional
+
+
+class ParseCache:
+    def __init__(self, cache_dir: Optional[Path] = None) -> None:
+        self.cache_dir = cache_dir
+        self._mem: Dict[str, Any] = {}
+
+    @staticmethod
+    def _file_hash(path: Path) -> str:
+        return hashlib.sha256(path.read_bytes()).hexdigest()
+
+    def get(self, path: Path) -> Optional[Any]:
+        try:
+            key = self._file_hash(path)
+        except OSError:
+            return None
+        return self._mem.get(key)
+
+    def put(self, path: Path, value: Any) -> None:
+        try:
+            key = self._file_hash(path)
+        except OSError:
+            return
+        self._mem[key] = value
+
+    def invalidate(self, path: Path) -> None:
+        try:
+            key = self._file_hash(path)
+        except OSError:
+            return
+        self._mem.pop(key, None)

package/adl_drift/parser/core.py

@@ -0,0 +1,33 @@
+"""Tree-sitter Python parser wrapper.
+
+Generalized into `lang/base.py::LanguageParser` (P4.1, PROBLEMS.md) — the
+grammar-agnostic parsing logic (error-node collection, byte-offset ->
+`SourcePosition` conversion, `node_text`) now lives there, constructed with a
+`tree_sitter.Language` + query map instead of a hardcoded global. This module
+re-exports the Python-configured parser under its original names
+(`PythonParser`, `ParseResult`, `PYTHON_LANGUAGE`, `QUERIES`) so existing call
+sites (`DriftCommands`, `MCPHandlers`, `scanner/parallel.py::_make_components`,
+and everything else importing from here) keep working unchanged.
+"""
+
+from __future__ import annotations
+
+from adl_drift.lang.base import LanguageParser, ParseResult
+from adl_drift.lang.python.parser import PYTHON_LANGUAGE, QUERIES
+
+__all__ = ["ParseResult", "PythonParser", "PYTHON_LANGUAGE", "QUERIES"]
+
+
+class PythonParser(LanguageParser):
+    """Python-configured `LanguageParser`.
+
+    Kept as a subclass (rather than a bare factory function) so direct
+    `PythonParser()` construction keeps working for every existing caller.
+    """
+
+    def __init__(self) -> None:
+        if PYTHON_LANGUAGE is None:
+            raise RuntimeError(
+                "tree-sitter-python not installed. pip install tree-sitter tree-sitter-python"
+            )
+        super().__init__(PYTHON_LANGUAGE, QUERIES)

package/adl_drift/parser/extractor.py

@@ -0,0 +1,17 @@
+"""Symbol table extractor — walks CST and produces Symbol records.
+
+Moved to `lang/python/extractor.py` (P4.2, PROBLEMS.md) as part of
+generalizing symbol extraction behind the `LanguageBackend` contract: the
+shared parse/dedupe/`AnalysisResult`-assembly plumbing now lives in
+`lang/base.py::SymbolExtractor`, and the Python-specific CST walk lives in
+`lang/python/extractor.py`. This module re-exports the Python-configured
+extractor under its original name so existing call sites (`DriftCommands`,
+`MCPHandlers`, `scanner/parallel.py::_make_components`, and everything else
+importing from here) keep working unchanged.
+"""
+
+from __future__ import annotations
+
+from adl_drift.lang.python.extractor import SymbolExtractor
+
+__all__ = ["SymbolExtractor"]