adl-toolkit 0.0.0-stage → 0.1.3

package/adl_drift/lang/java/type_provenance.py

@@ -0,0 +1,342 @@
+"""Java import/declared-type provenance (P4.6 F-04, PROBLEMS.md).
+
+Confirms a bare type/annotation *simple* name (`RequestParam`, `Statement`)
+actually resolves to a specific FQN (`org.springframework...RequestParam`,
+`java.sql.Statement`) via the compilation unit's own `import` declarations
+and local type/variable declarations, rather than trusting simple-name text
+alone — a same-named method/annotation on an unrelated, project-owned class
+must not be mistaken for the real JDK/framework API.
+
+This is P4.3's `lang/java/detectors/base.py::JavaDetector` machinery
+(`_imports`/`_declared_types`/`_type_has_provenance`/`_annotation_is`, used
+by `security.py`/`data_access.py`/`routes.py`/`additional.py`/`tests.py`)
+extracted to free functions so P4.6's Java taint layer (`lang/java/cfg.py`/
+`lang/java/taint.py`) can reuse the exact same, already-tested logic instead
+of re-deriving its own — `JavaDetector`'s methods below are now thin
+delegating wrappers, unchanged in signature or behavior for their many
+existing call sites.
+
+Unlike `JavaDetector`'s binary `_type_has_provenance`/`_annotation_is`
+(good enough for a single-file pattern detector, which simply doesn't flag
+when unsure), the taint layer needs a *three-way* verdict so it can keep
+recall on an unresolved case instead of dropping it silently:
+  - `"confirmed"`  — exact FQN literal, or simple name + a matching import
+    with nothing else in scope claiming that simple name.
+  - `"rejected"`   — a *different* type/annotation is definitely in scope
+    under this simple name (a local declaration, or a conflicting import) —
+    this is provably not `qualified_name`.
+  - `"ambiguous"`  — the simple name matches and nothing contradicts it, but
+    no import positively confirms it either (e.g. no import statement at
+    all). Still plausibly right — just not proven — so callers that care
+    about recall should keep treating it as a match, at reduced confidence.
+"""
+
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass, field
+from typing import Dict, List, Literal, Optional, Set, Tuple
+
+from tree_sitter import Node
+
+from adl_drift.lang.java.scopes import (
+    TYPE_DECLARATIONS,
+    TypeScopes,
+    declaration_scope,
+    declared_type_scopes,
+    type_declared_at,
+)
+
+DeclaredTypes = Dict[str, List[Tuple[int, int, str]]]
+# `(scope_start, scope_end, direct_supertypes)` per declaration, keyed by
+# simple name — scope-aware for the same reason `DeclaredTypes` is: two local
+# classes can share a simple name across sibling methods (PROBLEMS.md P4.6
+# F-02), and a flat `name -> supertypes` table would let one leak into the
+# other's `assignability_provenance` lookup.
+LocalSupertypes = Dict[str, List[Tuple[int, int, List[str]]]]
+Provenance = Literal["confirmed", "ambiguous", "rejected"]
+
+_DECLARATION_TYPES = ("field_declaration", "local_variable_declaration")
+_SCOPE_TYPES = (*TYPE_DECLARATIONS, "method_declaration", "constructor_declaration", "block", "lambda_expression")
+
+
+def _walk(node: Node):
+    yield node
+    for child in node.children:
+        yield from _walk(child)
+
+
+def _text(node: Optional[Node], source: bytes) -> str:
+    if node is None:
+        return ""
+    return source[node.start_byte:node.end_byte].decode("utf-8", errors="replace").strip()
+
+
+def imports_of(root: Node, source: bytes) -> Set[str]:
+    """`{"pkg.Simple", "pkg.*", ...}` for every `import`/`import static` in
+    the compilation unit (regex over each `import_declaration`'s own text —
+    cheap, and every import statement has a fixed, simple grammar)."""
+    imports: Set[str] = set()
+    for node in root.named_children:
+        if node.type != "import_declaration":
+            continue
+        match = re.match(r"import\s+(?:static\s+)?([\w.]+)(\.\*)?\s*;", _text(node, source))
+        if match:
+            imports.add(match.group(1) + (match.group(2) or ""))
+    return imports
+
+
+def has_import(imports: Set[str], qualified_name: str) -> bool:
+    package, _, _ = qualified_name.rpartition(".")
+    return qualified_name in imports or f"{package}.*" in imports
+
+
+def simple_type(type_name: str) -> str:
+    return re.sub(r"<.*>", "", type_name).rsplit(".", 1)[-1].replace("[]", "")
+
+
+def type_provenance(
+    raw_type: str, imports: Set[str], local_types: TypeScopes, qualified_name: str, at_byte: int,
+) -> Provenance:
+    raw_type = re.sub(r"<.*>", "", raw_type).replace("[]", "").strip()
+    simple = qualified_name.rsplit(".", 1)[-1]
+    if raw_type == qualified_name:
+        return "confirmed"
+    if raw_type != simple:
+        return "rejected"
+    if type_declared_at(local_types, simple, at_byte):
+        return "rejected"
+    conflicting = any(
+        imported.endswith(f".{simple}") and imported != qualified_name
+        for imported in imports
+    )
+    if conflicting:
+        return "rejected"
+    if qualified_name.startswith("java.lang.") or has_import(imports, qualified_name):
+        return "confirmed"
+    return "ambiguous"
+
+
+def type_has_provenance(
+    raw_type: str, imports: Set[str], local_types: TypeScopes, qualified_name: str, at_byte: int,
+) -> bool:
+    return type_provenance(raw_type, imports, local_types, qualified_name, at_byte) == "confirmed"
+
+
+def annotation_provenance(
+    name: str, at_byte: int, imports: Set[str], local_types: TypeScopes, qualified_name: str,
+) -> Provenance:
+    simple = qualified_name.rsplit(".", 1)[-1]
+    if name == qualified_name:
+        return "confirmed"
+    if name != simple:
+        return "rejected"
+    if type_declared_at(local_types, simple, at_byte):
+        return "rejected"
+    conflicting = any(
+        imported.endswith(f".{simple}") and imported != qualified_name
+        for imported in imports
+    )
+    if conflicting:
+        return "rejected"
+    if has_import(imports, qualified_name):
+        return "confirmed"
+    return "ambiguous"
+
+
+def annotation_is(
+    name: str, at_byte: int, imports: Set[str], local_types: TypeScopes, qualified_name: str,
+) -> bool:
+    return annotation_provenance(name, at_byte, imports, local_types, qualified_name) == "confirmed"
+
+
+def _declaration_scope(node: Node) -> Node:
+    parent = node.parent
+    while parent is not None and parent.type not in _SCOPE_TYPES:
+        parent = parent.parent
+    return parent or node
+
+
+def _typed_declaration_scope(node: Node) -> Optional[Node]:
+    if node.type == "enhanced_for_statement":
+        return node.child_by_field_name("body")
+    if node.type == "catch_formal_parameter":
+        return node.parent.child_by_field_name("body") if node.parent else None
+    statement = node.parent.parent if node.parent is not None else None
+    return statement.child_by_field_name("body") if statement is not None else None
+
+
+def declared_types_of(root: Node, source: bytes) -> DeclaredTypes:
+    """`{var_name: [(scope_start, scope_end, type_text), ...]}` for every
+    local variable, field, formal parameter, enhanced-`for`/`catch`/
+    try-with-resources binding in the compilation unit."""
+    types: DeclaredTypes = {}
+    for node in _walk(root):
+        if node.type in _DECLARATION_TYPES:
+            type_name = _text(node.child_by_field_name("type"), source)
+            scope = _declaration_scope(node)
+            start = scope.start_byte if node.type == "field_declaration" else node.start_byte
+            for child in node.children:
+                if child.type == "variable_declarator":
+                    name = _text(child.child_by_field_name("name"), source)
+                    if name:
+                        types.setdefault(name, []).append((start, scope.end_byte, type_name))
+        elif node.type == "formal_parameter":
+            name = _text(node.child_by_field_name("name"), source)
+            if name:
+                scope = _declaration_scope(node)
+                types.setdefault(name, []).append((
+                    scope.start_byte, scope.end_byte,
+                    _text(node.child_by_field_name("type"), source),
+                ))
+        elif node.type in {"enhanced_for_statement", "catch_formal_parameter", "resource"}:
+            name = _text(node.child_by_field_name("name"), source)
+            type_node = node.child_by_field_name("type")
+            if type_node is None and node.type == "catch_formal_parameter":
+                type_node = next(
+                    (child for child in node.named_children if child.type == "catch_type"), None,
+                )
+            scope = _typed_declaration_scope(node)
+            if name and type_node is not None and scope is not None:
+                start = node.start_byte if node.type == "resource" else scope.start_byte
+                types.setdefault(name, []).append((start, scope.end_byte, _text(type_node, source)))
+    return types
+
+
+def declared_type_of(receiver: str, types: DeclaredTypes, at_byte: int) -> str:
+    matches = [
+        (end - start, -start, type_name)
+        for start, end, type_name in types.get(receiver, [])
+        if start <= at_byte < end
+    ]
+    return min(matches)[2] if matches else ""
+
+
+def _type_names_in(node: Optional[Node], source: bytes) -> List[str]:
+    if node is None:
+        return []
+    out: List[str] = []
+    for child in _walk(node):
+        if child.type in ("type_identifier", "scoped_type_identifier"):
+            text = _text(child, source)
+            if text:
+                out.append(text)
+    return out
+
+
+def local_supertypes_of(root: Node, source: bytes) -> LocalSupertypes:
+    """Direct `extends`/`implements` type names for local Java declarations,
+    scoped identically to `declared_type_scopes` (`declaration_scope`) so a
+    lookup at a given `at_byte` only ever sees the declaration actually in
+    scope there.
+
+    Records an entry for *every* type declaration, even one with no
+    `extends`/`implements` (an empty `direct` list) — such a declaration
+    still needs a place in the table so `_supertypes_at` can pick it as the
+    innermost match and correctly shadow an unrelated outer/enclosing
+    declaration of the same simple name (a supertype-less local class named
+    the same as a top-level class that *does* implement something) — F-05,
+    PROBLEMS.md P4.6. Omitting the empty-`direct` case left nothing in the
+    table to shadow with, so the outer declaration's supertypes leaked
+    through untouched."""
+    supertypes: LocalSupertypes = {}
+    for node in _walk(root):
+        if node.type not in TYPE_DECLARATIONS:
+            continue
+        name = _text(node.child_by_field_name("name"), source)
+        if not name:
+            continue
+        direct: List[str] = []
+        direct.extend(_type_names_in(node.child_by_field_name("superclass"), source))
+        direct.extend(_type_names_in(node.child_by_field_name("interfaces"), source))
+        for child in node.children:
+            if child.type == "extends_interfaces":
+                direct.extend(_type_names_in(child, source))
+        start, end = declaration_scope(node, root)
+        supertypes.setdefault(name, []).append((start, end, direct))
+    return supertypes
+
+
+def _supertypes_at(local_supertypes: LocalSupertypes, name: str, at_byte: int) -> List[str]:
+    """Only the *innermost* enclosing declaration's supertypes — mirroring
+    `declared_type_of`'s smallest-range selection. A flat union across every
+    scope containing `at_byte` is wrong whenever one declaration's scope
+    nests inside another's: a top-level class is (correctly) visible for the
+    entire compilation unit (`declaration_scope`'s documented `root` case),
+    so a same-named local class declared anywhere inside a method would
+    still satisfy that top-level scope's range check too. Without narrowing
+    to the innermost match, the unrelated top-level class's `extends`/
+    `implements` leaked into the local class's own `assignability_provenance`
+    lookup (F-05, PROBLEMS.md P4.6) — the same false-positive shape F-02
+    fixed for sibling-method local classes, surviving here for outer/local
+    shadowing."""
+    matches = [
+        (end - start, -start, direct)
+        for start, end, direct in local_supertypes.get(name, ())
+        if start <= at_byte < end
+    ]
+    if not matches:
+        return []
+    matches.sort(key=lambda m: (m[0], m[1]))
+    return matches[0][2]
+
+
+def assignability_provenance(
+    raw_type: str,
+    imports: Set[str],
+    local_types: TypeScopes,
+    local_supertypes: LocalSupertypes,
+    qualified_name: str,
+    at_byte: int,
+) -> Provenance:
+    """Like `type_provenance`, but follows local `extends`/`implements`.
+
+    This stays compilation-unit local: external library hierarchies are not
+    guessed here, because callers need to choose where a low-confidence
+    external subtype fallback is acceptable.
+    """
+    direct = type_provenance(raw_type, imports, local_types, qualified_name, at_byte)
+    if direct == "confirmed":
+        return "confirmed"
+
+    start = simple_type(raw_type)
+    if not type_declared_at(local_types, start, at_byte):
+        return direct
+
+    seen: Set[str] = set()
+    stack = list(_supertypes_at(local_supertypes, start, at_byte))
+    while stack:
+        current = stack.pop()
+        current_simple = simple_type(current)
+        if current_simple in seen:
+            continue
+        seen.add(current_simple)
+        verdict = type_provenance(current, imports, local_types, qualified_name, at_byte)
+        if verdict == "confirmed":
+            return "confirmed"
+        if verdict == "ambiguous":
+            return "ambiguous"
+        stack.extend(_supertypes_at(local_supertypes, current_simple, at_byte))
+    return direct
+
+
+@dataclass
+class JavaTypeContext:
+    """Per-compilation-unit provenance data — computed once per file
+    (`build_type_context`) and reused for every function/call-site/
+    annotation lookup within it, rather than re-walking the whole file's
+    CST on every single check."""
+
+    imports: Set[str] = field(default_factory=set)
+    local_types: TypeScopes = field(default_factory=dict)
+    declared_types: DeclaredTypes = field(default_factory=dict)
+    local_supertypes: LocalSupertypes = field(default_factory=dict)
+
+
+def build_type_context(root: Node, source: bytes) -> JavaTypeContext:
+    return JavaTypeContext(
+        imports=imports_of(root, source),
+        local_types=declared_type_scopes(root, source),
+        declared_types=declared_types_of(root, source),
+        local_supertypes=local_supertypes_of(root, source),
+    )

package/adl_drift/lang/python/__init__.py

@@ -0,0 +1,8 @@
+"""Landing spot for the Python `LanguageBackend`.
+
+`backend.py` is a thin shim over the existing top-level Python modules
+(`parser/`, `detectors/`, `utils/config.py`) so the `LanguageBackend`
+contract has one real, working implementation. It is NOT the P4.1-P4.7
+migration: those modules still live at their current locations, unmoved and
+unmodified. Refactoring them to live under this package is P4.1-P4.7.
+"""

package/adl_drift/lang/python/backend.py

@@ -0,0 +1,94 @@
+"""Thin `LanguageBackend` shim over the existing Python pipeline.
+
+`parser()` is wired to the real migrated parser (`lang/python/parser.py`,
+P4.1). `resolver_config()`/`cfg_config()` are wired to the real P4.4/P4.6
+Python policy (`lang/python/resolver.py::default_python_resolver_config()`,
+`lang/python/cfg.py::default_python_cfg_config()`) — the same objects
+`graph/builder.py`'s pre-P4.4/P4.6 lazy fallback already defaults to, so
+callers get byte-identical behavior whether or not they go through this
+method. Everything else still wraps the current top-level modules
+(`parser.extractor.SymbolExtractor`, `detectors.*`, `utils.config.
+ProjectConfig`) as-is, unmodified.
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import List
+
+from adl_drift.detectors.async_patterns import AsyncPatternDetector
+from adl_drift.detectors.base import BaseDetector
+from adl_drift.detectors.config import ConfigDetector
+from adl_drift.detectors.data_access import DataAccessDetector
+from adl_drift.detectors.decorators import DecoratorDetector
+from adl_drift.detectors.dynamic_patterns import DynamicPatternDetector
+from adl_drift.detectors.errors import ErrorPatternDetector
+from adl_drift.detectors.imports import ImportDetector
+from adl_drift.detectors.logging_patterns import LoggingDetector
+from adl_drift.detectors.routes import DjangoURLDetector, RouteDetector
+from adl_drift.detectors.security import SecurityDetector
+from adl_drift.detectors.tests import TestDetector
+from adl_drift.detectors.type_conventions import TypeConventionDetector
+from adl_drift.lang.base import CFGConfig, LanguageParser, ResolverConfig
+from adl_drift.lang.python.cfg import default_python_cfg_config
+from adl_drift.lang.python.parser import create_parser
+from adl_drift.lang.python.resolver import default_python_resolver_config
+from adl_drift.parser.extractor import SymbolExtractor
+from adl_drift.utils.config import ProjectConfig as PythonProjectConfig
+
+
+class PythonBackend:
+    """`LanguageBackend` for Python, satisfied structurally (no ABC needed)."""
+
+    name = "python"
+    file_extensions = (".py",)
+    project_markers = ("pyproject.toml", "setup.py", "setup.cfg")
+
+    def parser(self) -> LanguageParser:
+        return create_parser()
+
+    def extractor(self) -> SymbolExtractor:
+        return SymbolExtractor(self.parser())
+
+    def detectors(self) -> List[BaseDetector]:
+        parser = self.parser()
+        return [
+            RouteDetector(parser),
+            DjangoURLDetector(parser),
+            ErrorPatternDetector(parser),
+            DataAccessDetector(parser),
+            DecoratorDetector(parser),
+            AsyncPatternDetector(parser),
+            ImportDetector(parser),
+            SecurityDetector(parser),
+            TestDetector(parser),
+            ConfigDetector(parser),
+            LoggingDetector(parser),
+            TypeConventionDetector(parser),
+            DynamicPatternDetector(parser),
+        ]
+
+    def resolver_config(self) -> ResolverConfig:
+        return default_python_resolver_config()
+
+    def cfg_config(self) -> CFGConfig:
+        return default_python_cfg_config()
+
+    def project_config(self, root: Path) -> PythonProjectConfig:
+        return PythonProjectConfig(root)
+
+    def module_name(self, file: Path, root: Path) -> str:
+        """Mirrors `CallGraphBuilder._path_to_module` without depending on
+        it, so this shim stays decoupled from the (language-coupled, P4.4/
+        P4.5) graph builder internals.
+        """
+        file = Path(file)
+        root = Path(root)
+        try:
+            rel = file.relative_to(root)
+        except ValueError:
+            rel = file
+        parts = rel.with_suffix("").parts
+        if parts and parts[-1] == "__init__":
+            parts = parts[:-1]
+        return ".".join(parts)