adl-toolkit 0.0.0-stage → 0.1.3

package/adl_drift/lang/python/extractor.py

@@ -0,0 +1,454 @@
+"""Symbol table extractor — walks the Python CST and produces Symbol records.
+
+Moved out of `parser/extractor.py` verbatim (P4.2, PROBLEMS.md): every
+node-type string, helper method, and behavior is unchanged. `extract()`,
+`extract_string()`, the depth-first walk, the variable/attribute dedupe key,
+and `AnalysisResult` assembly now come from `lang.base.SymbolExtractor`
+(shared with the Java extractor) instead of being defined here directly;
+`_build()` — the Python-specific node-type dispatch — is this class's own
+override, same as before.
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import List, Optional
+
+from tree_sitter import Node
+
+from adl_drift.lang.base import LanguageParser, ParseResult
+from adl_drift.lang.base import SymbolExtractor as _BaseSymbolExtractor
+from adl_drift.parser.literal_eval import find_exec_expansions
+from adl_drift.utils.types import (
+    LOCALS_MARKER,
+    AnalysisResult,
+    ImportRecord,
+    Symbol,
+)
+
+
+class SymbolExtractor(_BaseSymbolExtractor):
+    """Extracts function/class symbols + import strings from a parsed file."""
+
+    def __init__(self, parser: LanguageParser) -> None:
+        super().__init__(parser)
+
+    def _build(self, parse: ParseResult, file_path: Path) -> AnalysisResult:
+        symbols: List[Symbol] = []
+        imports: List[str] = []
+        import_records: List[ImportRecord] = []
+        seen_var_keys: set = set()  # (scope_chain, parent or "", name) — dedupe variables/attributes
+
+        for node in self._walk(parse.tree.root_node):
+            if node.type == "function_definition":
+                sym = self._extract_function(node, parse.source_bytes, file_path)
+                if sym:
+                    symbols.append(sym)
+            elif node.type == "class_definition":
+                sym = self._extract_class(node, parse.source_bytes, file_path)
+                if sym:
+                    symbols.append(sym)
+            elif node.type in ("import_statement", "import_from_statement"):
+                imports.append(self.parser.node_text(node, parse.source_bytes).strip())
+                import_records.extend(
+                    self._extract_import_records(node, parse.source_bytes)
+                )
+            elif node.type == "assignment":
+                for sym in self._extract_assignment(node, parse.source_bytes, file_path):
+                    key = self._dedupe_key(sym)
+                    if key in seen_var_keys:
+                        continue
+                    seen_var_keys.add(key)
+                    symbols.append(sym)
+
+        # P2.16: expand exec/eval with literal templates
+        for sym in self._expand_exec_symbols(parse, file_path):
+            key = self._dedupe_key(sym)
+            if key not in seen_var_keys:
+                seen_var_keys.add(key)
+                symbols.append(sym)
+
+        return self._finalize(parse, file_path, symbols, imports, import_records)
+
+    def _extract_import_records(self, node: Node, source: bytes) -> List[ImportRecord]:
+        """P2.1: structured import extraction by walking the CST.
+
+        Handles plain / aliased / from / relative / parenthesized-multiline /
+        wildcard imports. The CST walk also reaches conditional imports
+        (``if TYPE_CHECKING:`` blocks), which the old regex path missed.
+        """
+        records: List[ImportRecord] = []
+
+        if node.type == "import_statement":
+            for child in node.children:
+                if child.type == "dotted_name":
+                    records.append(ImportRecord(module=self.parser.node_text(child, source)))
+                elif child.type == "aliased_import":
+                    name_node = child.child_by_field_name("name")
+                    alias_node = child.child_by_field_name("alias")
+                    if name_node is not None:
+                        records.append(ImportRecord(
+                            module=self.parser.node_text(name_node, source),
+                            alias=self.parser.node_text(alias_node, source) if alias_node else None,
+                        ))
+            return records
+
+        # import_from_statement
+        mod_node = node.child_by_field_name("module_name")
+        module = ""
+        level = 0
+        if mod_node is not None:
+            if mod_node.type == "relative_import":
+                for sub in mod_node.children:
+                    if sub.type == "import_prefix":
+                        level = len(self.parser.node_text(sub, source))
+                    elif sub.type == "dotted_name":
+                        module = self.parser.node_text(sub, source)
+            else:
+                module = self.parser.node_text(mod_node, source)
+
+        seen_import_kw = False
+        for child in node.children:
+            if child.type == "import":
+                seen_import_kw = True
+                continue
+            if not seen_import_kw:
+                continue
+            if child.type == "dotted_name":
+                records.append(ImportRecord(
+                    module=module,
+                    name=self.parser.node_text(child, source),
+                    level=level,
+                ))
+            elif child.type == "aliased_import":
+                name_node = child.child_by_field_name("name")
+                alias_node = child.child_by_field_name("alias")
+                if name_node is not None:
+                    records.append(ImportRecord(
+                        module=module,
+                        name=self.parser.node_text(name_node, source),
+                        alias=self.parser.node_text(alias_node, source) if alias_node else None,
+                        level=level,
+                    ))
+            elif child.type == "wildcard_import":
+                records.append(ImportRecord(
+                    module=module, name="*", level=level, is_wildcard=True,
+                ))
+        return records
+
+    def _extract_function(self, node: Node, source: bytes, file_path: Path) -> Optional[Symbol]:
+        name_node = node.child_by_field_name("name")
+        if not name_node:
+            return None
+        name = self.parser.node_text(name_node, source)
+        params_node = node.child_by_field_name("parameters")
+        signature = (
+            f"def {name}{self.parser.node_text(params_node, source)}" if params_node else f"def {name}()"
+        )
+        return_type_node = node.child_by_field_name("return_type")
+        return_type = (
+            self.parser.node_text(return_type_node, source).strip() if return_type_node else None
+        )
+        if return_type:
+            signature = f"{signature} -> {return_type}"
+        param_types = self._extract_param_types(params_node, source) if params_node else {}
+        is_async = self._is_async(node, source)
+        decorators = self._extract_decorators(node, source)
+        docstring = self._extract_docstring(node, source)
+        parent = self._find_parent_class(node, source)
+        scope_chain = self._scope_chain(node, source)
+        kind = "function"
+        is_test = self._looks_like_test(name, decorators, file_path)
+        return Symbol(
+            name=name,
+            kind=kind,
+            file_path=file_path,
+            position=self.parser.node_position(node),
+            decorators=decorators,
+            docstring=docstring,
+            signature=signature,
+            is_async=is_async,
+            parent=parent,
+            param_types=param_types,
+            return_type=return_type,
+            is_test=is_test,
+            scope_chain=scope_chain,
+        )
+
+    @staticmethod
+    def _looks_like_test(name: str, decorators: List[str], file_path: Path) -> bool:
+        if any("pytest.fixture" in d or "fixture" == d.lstrip("@") for d in decorators):
+            return True
+        if name.startswith("test_") or name == "setUp" or name == "tearDown":
+            fname = file_path.name
+            if fname.startswith("test_") or fname.endswith("_test.py") or "tests" in file_path.parts:
+                return True
+            return name.startswith("test_")
+        return False
+
+    def _extract_param_types(self, params_node: Node, source: bytes) -> dict:
+        """Walk `parameters` CST children, return {param_name: annotation_text}.
+
+        Handles: typed_parameter, typed_default_parameter, and splat variants.
+        Untyped params produce no entry.
+        """
+        out: dict = {}
+        for child in params_node.children:
+            ct = child.type
+            if ct == "typed_parameter":
+                type_node = child.child_by_field_name("type")
+                if type_node is None:
+                    continue
+                name = None
+                for sub in child.children:
+                    if sub.type == "identifier":
+                        name = self.parser.node_text(sub, source)
+                        break
+                    if sub.type in ("list_splat_pattern", "dictionary_splat_pattern"):
+                        for sub2 in sub.children:
+                            if sub2.type == "identifier":
+                                name = self.parser.node_text(sub2, source)
+                                break
+                        break
+                if name:
+                    out[name] = self.parser.node_text(type_node, source).strip()
+            elif ct == "typed_default_parameter":
+                name_node = child.child_by_field_name("name")
+                type_node = child.child_by_field_name("type")
+                if name_node is not None and type_node is not None:
+                    out[self.parser.node_text(name_node, source)] = (
+                        self.parser.node_text(type_node, source).strip()
+                    )
+        return out
+
+    def _extract_class(self, node: Node, source: bytes, file_path: Path) -> Optional[Symbol]:
+        name_node = node.child_by_field_name("name")
+        if not name_node:
+            return None
+        name = self.parser.node_text(name_node, source)
+        decorators = self._extract_decorators(node, source)
+        docstring = self._extract_docstring(node, source)
+        parent = self._find_parent_class(node, source)
+        scope_chain = self._scope_chain(node, source)
+        superclasses = self._extract_superclasses(node, source)
+        return Symbol(
+            name=name,
+            kind="class",
+            file_path=file_path,
+            position=self.parser.node_position(node),
+            decorators=decorators,
+            docstring=docstring,
+            signature=f"class {name}",
+            parent=parent,
+            superclasses=superclasses,
+            scope_chain=scope_chain,
+        )
+
+    def _extract_superclasses(self, node: Node, source: bytes) -> List[str]:
+        sc_node = node.child_by_field_name("superclasses")
+        if not sc_node:
+            return []
+        out: List[str] = []
+        for child in sc_node.children:
+            if child.type in ("identifier", "attribute"):
+                out.append(self.parser.node_text(child, source).strip())
+            elif child.type == "keyword_argument":
+                continue  # metaclass=..., skip
+            elif child.type == "subscript":
+                # Generic[T] etc. — take the base identifier/attribute
+                base = child.child_by_field_name("value")
+                if base is not None:
+                    out.append(self.parser.node_text(base, source).strip())
+        return out
+
+    def _extract_decorators(self, node: Node, source: bytes) -> List[str]:
+        out: List[str] = []
+        sibling = node.prev_sibling
+        while sibling and sibling.type == "decorator":
+            out.insert(0, self.parser.node_text(sibling, source).strip())
+            sibling = sibling.prev_sibling
+        for child in node.children:
+            if child.type == "decorator":
+                txt = self.parser.node_text(child, source).strip()
+                if txt not in out:
+                    out.append(txt)
+        return out
+
+    def _extract_docstring(self, node: Node, source: bytes) -> Optional[str]:
+        body = node.child_by_field_name("body")
+        if not body:
+            return None
+        for child in body.children:
+            if child.type == "expression_statement":
+                for sub in child.children:
+                    if sub.type == "string":
+                        raw = self.parser.node_text(sub, source).strip()
+                        return raw.strip("'\"")
+                break
+        return None
+
+    def _is_async(self, node: Node, source: bytes) -> bool:
+        text = source[node.start_byte:min(node.end_byte, node.start_byte + 16)].decode(
+            "utf-8", errors="replace"
+        )
+        return text.lstrip().startswith("async ")
+
+    def _extract_assignment(self, node: Node, source: bytes, file_path: Path) -> List[Symbol]:
+        """Emit Symbol for module-level vars and `self.X = ...` attributes in __init__.
+
+        Skips: local variables inside functions, augmented assignments, walrus.
+        """
+        left = node.child_by_field_name("left")
+        if left is None:
+            return []
+
+        # Walk ancestors to determine scope
+        parent_func: Optional[str] = None
+        parent_class: Optional[str] = None
+        class_node: Optional[Node] = None
+        p = node.parent
+        while p is not None:
+            if p.type == "function_definition" and parent_func is None:
+                nn = p.child_by_field_name("name")
+                if nn is not None:
+                    parent_func = self.parser.node_text(nn, source)
+            elif p.type == "class_definition" and parent_class is None:
+                nn = p.child_by_field_name("name")
+                if nn is not None:
+                    parent_class = self.parser.node_text(nn, source)
+                    class_node = p
+            p = p.parent
+
+        ann_node = node.child_by_field_name("type")
+        annotation = self.parser.node_text(ann_node, source).strip() if ann_node is not None else None
+        pos = self.parser.node_position(node)
+
+        # Case 1: module-level assignment (no enclosing function/class)
+        if parent_func is None and parent_class is None:
+            out: List[Symbol] = []
+            for name in self._target_names(left, source):
+                out.append(Symbol(
+                    name=name,
+                    kind="variable",
+                    file_path=file_path,
+                    position=pos,
+                    signature=annotation,
+                ))
+            return out
+
+        # Case 2: self.X = ... inside __init__ → instance attribute on enclosing class
+        if parent_func == "__init__" and parent_class and left.type == "attribute":
+            obj = left.child_by_field_name("object")
+            attr = left.child_by_field_name("attribute")
+            if obj is not None and attr is not None and \
+               self.parser.node_text(obj, source) == "self":
+                # P1.10 (P3.13 fold-in): attribute ids must be scope-aware —
+                # the chain of the *enclosing class* plus the class name, so a
+                # function-nested class's attributes don't compute the same id
+                # as a same-named top-level class's attributes.
+                attr_chain = (
+                    self._scope_chain(class_node, source) + [parent_class]
+                    if class_node is not None else []
+                )
+                return [Symbol(
+                    name=self.parser.node_text(attr, source),
+                    kind="attribute",
+                    file_path=file_path,
+                    position=pos,
+                    parent=parent_class,
+                    signature=annotation,
+                    scope_chain=attr_chain,
+                )]
+        return []
+
+    def _target_names(self, target: Node, source: bytes) -> List[str]:
+        """Collect identifier names from an assignment target.
+
+        Handles `x = ...`, `x: int = ...`, and tuple/list unpacking.
+        Skips attribute and subscript targets at module level.
+        """
+        t = target.type
+        if t == "identifier":
+            return [self.parser.node_text(target, source)]
+        if t in ("typed_parameter",):
+            return []
+        if t in ("pattern_list", "tuple_pattern", "list_pattern"):
+            names: List[str] = []
+            for child in target.children:
+                names.extend(self._target_names(child, source))
+            return names
+        # Strip annotation wrapper: tree-sitter uses different node names by version;
+        # the annotated identifier itself remains type "identifier" via child_by_field_name.
+        return []
+
+    def _expand_exec_symbols(self, parse: ParseResult, file_path: Path) -> List[Symbol]:
+        """P2.16: expand exec/eval with literal templates and extract synthesized symbols.
+
+        Walks the CST looking for exec("...") / eval("...") calls and for-loops that
+        unroll exec over a literal iterable. Re-parses each recovered source string and
+        extracts symbols from it, marking them metadata={"synthetic": "exec"}.
+
+        Non-literal inputs abort silently — no symbols emitted for those sites.
+        """
+        try:
+            expansions = find_exec_expansions(parse.tree.root_node, parse.source_bytes)
+        except Exception:
+            return []
+
+        out: List[Symbol] = []
+        exec_pos = self.parser.node_position(parse.tree.root_node)  # file-level fallback
+
+        for src_str in expansions:
+            try:
+                sub_parse = self.parser.parse_string(src_str, file_path=file_path)
+            except Exception:
+                continue
+            for node in self._walk(sub_parse.tree.root_node):
+                sym: Optional[Symbol] = None
+                if node.type == "function_definition":
+                    sym = self._extract_function(node, sub_parse.source_bytes, file_path)
+                elif node.type == "class_definition":
+                    sym = self._extract_class(node, sub_parse.source_bytes, file_path)
+                if sym is not None:
+                    sym.metadata["synthetic"] = "exec"
+                    sym.position = exec_pos  # attribute back to the exec call site
+                    out.append(sym)
+        return out
+
+    def _scope_chain(self, node: Node, source: bytes) -> List[str]:
+        """P2.17/P1.10: full lexical scope chain for a symbol — outermost
+        enclosing class/function names first, with ``LOCALS_MARKER`` inserted
+        after every *function* segment (CPython's __qualname__ semantics).
+
+        Class-path ids stay unchanged (`m.Foo.bar`), while closures get
+        `m.outer.<locals>.bar` — so a class `Foo`'s method can no longer
+        collide with a same-named def nested inside a function `Foo`:
+          outer1.<locals>.app → scope_chain=["outer1", "<locals>"]
+          Foo.bar (method)    → scope_chain=["Foo"]
+
+        ``symbol.parent`` (nearest enclosing class) is kept unchanged for
+        self/cls resolution; ``scope_chain`` is the separate uniqueness key.
+        """
+        chain: List[str] = []
+        p = node.parent
+        while p is not None:
+            if p.type in ("class_definition", "function_definition"):
+                nn = p.child_by_field_name("name")
+                if nn is not None:
+                    if p.type == "function_definition":
+                        # reversed below: marker must follow the function name
+                        chain.append(LOCALS_MARKER)
+                    chain.append(self.parser.node_text(nn, source))
+            p = p.parent
+        chain.reverse()  # innermost last → outermost first
+        return chain
+
+    def _find_parent_class(self, node: Node, source: bytes) -> Optional[str]:
+        parent = node.parent
+        while parent:
+            if parent.type == "class_definition":
+                name_node = parent.child_by_field_name("name")
+                if name_node:
+                    return self.parser.node_text(name_node, source)
+            parent = parent.parent
+        return None

package/adl_drift/lang/python/parser.py

@@ -0,0 +1,41 @@
+"""Python tree-sitter grammar + query map (P4.1, PROBLEMS.md).
+
+Moved out of `parser/core.py` verbatim: the `Language` instantiation from
+`tree-sitter-python` and the `QUERIES` map are unchanged. `parser/core.py`
+now builds its `PythonParser` by delegating to `create_parser()` here
+instead of owning the grammar global directly.
+"""
+
+from __future__ import annotations
+
+from typing import Optional
+
+from tree_sitter import Language
+
+from adl_drift.lang.base import LanguageParser
+
+try:
+    import tree_sitter_python as tspython
+
+    PYTHON_LANGUAGE: Optional[Language] = Language(tspython.language())
+except Exception:  # pragma: no cover
+    PYTHON_LANGUAGE = None
+
+
+QUERIES = {
+    "function_def": "(function_definition name: (identifier) @name) @func",
+    "class_def": "(class_definition name: (identifier) @name) @class",
+    "decorator": "(decorator) @dec",
+    "call": "(call function: (_) @target) @call",
+    "import": "(import_statement) @imp",
+    "import_from": "(import_from_statement) @imp",
+}
+
+
+def create_parser() -> LanguageParser:
+    """Build a `LanguageParser` configured for Python."""
+    if PYTHON_LANGUAGE is None:
+        raise RuntimeError(
+            "tree-sitter-python not installed. pip install tree-sitter tree-sitter-python"
+        )
+    return LanguageParser(PYTHON_LANGUAGE, QUERIES)