adl-toolkit 0.0.0-stage → 0.1.1

package/adl_drift/parser/literal_eval.py

@@ -0,0 +1,326 @@
+"""Literal-template expander for exec/eval calls (P2.16).
+
+Strictly literal-only evaluation — any name lookup outside the provided
+environment aborts the expansion. Never executes user code; only folds
+string constants and unrolls literal for-loops.
+
+Public API
+----------
+eval_literal_str(node, source, env=None) -> Optional[str]
+    Try to evaluate a CST string/concat node to its Python str value.
+
+find_exec_expansions(root_node, source) -> List[str]
+    Walk a module root and return all expandable exec/eval source strings.
+    Handles two patterns:
+      1. Direct literal:  exec("def foo(): pass")
+      2. For-loop unroll: for n in ("a","b"): exec(f"def get_{n}(): ...")
+"""
+
+from __future__ import annotations
+
+import ast
+import warnings
+from typing import Any, Dict, List, Optional
+
+from tree_sitter import Node
+
+
+def eval_literal_str(
+    node: Node,
+    source: bytes,
+    env: Optional[Dict[str, Any]] = None,
+) -> Optional[str]:
+    """Try to evaluate a CST string/concat/binary-concat node to a str.
+
+    Returns the string value, or None if any non-literal sub-expression is
+    encountered.
+    """
+    return _eval_str_node(node, source, env or {})
+
+
+def find_exec_expansions(root_node: Node, source: bytes) -> List[str]:
+    """Walk a module root and return all expandable exec/eval source strings.
+
+    Non-literal inputs are silently skipped — only fully-recoverable
+    templates contribute to the result.
+    """
+    results: List[str] = []
+    _walk_for_execs(root_node, source, {}, results)
+    return results
+
+
+# ---------------------------------------------------------------------------
+# String evaluation
+# ---------------------------------------------------------------------------
+
+def _eval_str_node(node: Node, source: bytes, env: Dict[str, Any]) -> Optional[str]:
+    t = node.type
+
+    if t == "string":
+        return _eval_single_string(node, source, env)
+
+    if t == "concatenated_string":
+        # Adjacent string literals: "a" "b" -> "ab"
+        parts: List[str] = []
+        for child in node.children:
+            if child.type == "string":
+                val = _eval_single_string(child, source, env)
+                if val is None:
+                    return None
+                parts.append(val)
+        return "".join(parts)
+
+    if t == "binary_operator":
+        # "a" + "b"  string concatenation via + operator
+        left = node.child_by_field_name("left")
+        right = node.child_by_field_name("right")
+        if left is None or right is None:
+            return None
+        # Check operator is +
+        has_plus = any(
+            child.type == "+"
+            or (child.type not in (left.type, right.type, "comment")
+                and source[child.start_byte:child.end_byte] == b"+")
+            for child in node.children
+            if child.id != left.id and child.id != right.id
+        )
+        # Simpler: just check raw operator text
+        op_raw = source[left.end_byte:right.start_byte].decode("utf-8", errors="replace").strip()
+        if "+" not in op_raw:
+            return None
+        lv = _eval_str_node(left, source, env)
+        rv = _eval_str_node(right, source, env)
+        if lv is None or rv is None:
+            return None
+        return lv + rv
+
+    if t == "identifier":
+        name = source[node.start_byte:node.end_byte].decode("utf-8", errors="replace")
+        val = env.get(name)
+        return str(val) if val is not None else None
+
+    return None
+
+
+def _eval_single_string(node: Node, source: bytes, env: Dict[str, Any]) -> Optional[str]:
+    """Evaluate one `string` CST node."""
+    # Detect f-string by presence of interpolation children
+    is_fstring = any(c.type == "interpolation" for c in node.children)
+
+    if not is_fstring:
+        # Plain string — delegate to ast.literal_eval (handles escapes, quotes)
+        raw = source[node.start_byte:node.end_byte].decode("utf-8", errors="replace")
+        try:
+            with warnings.catch_warnings():
+                warnings.simplefilter("ignore", SyntaxWarning)
+                val = ast.literal_eval(raw)
+            return val if isinstance(val, str) else None
+        except (ValueError, SyntaxError):
+            return None
+
+    # F-string — walk CST children, substituting env vars
+    return _eval_fstring(node, source, env)
+
+
+def _eval_fstring(node: Node, source: bytes, env: Dict[str, Any]) -> Optional[str]:
+    """Reconstruct an f-string value from its CST children using env."""
+    parts: List[str] = []
+    for child in node.children:
+        ct = child.type
+        if ct in ("string_start", "string_end"):
+            continue
+        if ct == "string_content":
+            parts.append(source[child.start_byte:child.end_byte].decode("utf-8", errors="replace"))
+        elif ct == "escape_sequence":
+            raw = source[child.start_byte:child.end_byte].decode("utf-8", errors="replace")
+            try:
+                # Decode standard Python escape sequences
+                decoded = raw.encode("raw_unicode_escape").decode("unicode_escape")
+                parts.append(decoded)
+            except Exception:
+                parts.append(raw)
+        elif ct == "interpolation":
+            val = _eval_interpolation(child, source, env)
+            if val is None:
+                return None
+            parts.append(str(val))
+        else:
+            # Unknown structure (format spec, conversion, nested f-string) → abort
+            return None
+    return "".join(parts)
+
+
+def _eval_interpolation(node: Node, source: bytes, env: Dict[str, Any]) -> Optional[Any]:
+    """Evaluate a single {expr} interpolation."""
+    for child in node.children:
+        ct = child.type
+        if ct in ("{", "}", "!", "format_spec", "type_conversion"):
+            continue
+        # The expression inside the braces
+        return _eval_expr(child, source, env)
+    return None
+
+
+def _eval_expr(node: Node, source: bytes, env: Dict[str, Any]) -> Optional[Any]:
+    """Evaluate a simple expression to a Python value for f-string substitution."""
+    t = node.type
+    if t == "identifier":
+        name = source[node.start_byte:node.end_byte].decode("utf-8", errors="replace")
+        return env.get(name)  # None if unknown name → abort
+
+    if t in ("string", "concatenated_string"):
+        return _eval_str_node(node, source, env)
+
+    if t == "integer":
+        raw = source[node.start_byte:node.end_byte].decode("utf-8", errors="replace")
+        try:
+            return int(raw, 0)
+        except ValueError:
+            return None
+
+    if t == "float":
+        raw = source[node.start_byte:node.end_byte].decode("utf-8", errors="replace")
+        try:
+            return float(raw)
+        except ValueError:
+            return None
+
+    # Any other expression type (call, subscript, attribute, etc.) → abort
+    return None
+
+
+# ---------------------------------------------------------------------------
+# Iterable evaluation (for for-loop unrolling)
+# ---------------------------------------------------------------------------
+
+def _eval_iterable(node: Node, source: bytes) -> Optional[List[Any]]:
+    """Evaluate a literal tuple or list to a Python list of values."""
+    if node.type not in ("tuple", "list"):
+        return None
+    items: List[Any] = []
+    for child in node.children:
+        if child.type in (",", "(", ")", "[", "]"):
+            continue
+        val = _eval_elem(child, source)
+        if val is None:
+            return None
+        items.append(val)
+    return items if items else None
+
+
+def _eval_elem(node: Node, source: bytes) -> Optional[Any]:
+    """Evaluate a single tuple/list element (no env — pure literal)."""
+    t = node.type
+    if t in ("string", "concatenated_string"):
+        return _eval_str_node(node, source, {})
+    if t == "integer":
+        raw = source[node.start_byte:node.end_byte].decode("utf-8", errors="replace")
+        try:
+            return int(raw, 0)
+        except ValueError:
+            return None
+    if t == "float":
+        raw = source[node.start_byte:node.end_byte].decode("utf-8", errors="replace")
+        try:
+            return float(raw)
+        except ValueError:
+            return None
+    # Nested tuple/list not supported (single-level only per P2.16 spec)
+    return None
+
+
+# ---------------------------------------------------------------------------
+# Tree-walking exec/eval finder
+# ---------------------------------------------------------------------------
+
+def _walk_for_execs(
+    node: Node,
+    source: bytes,
+    env: Dict[str, Any],
+    results: List[str],
+) -> None:
+    """Recursively walk CST, collecting expandable exec/eval strings."""
+    t = node.type
+
+    if t == "for_statement":
+        _handle_for_statement(node, source, env, results)
+        return  # children already visited inside
+
+    if t == "call":
+        _handle_exec_call(node, source, env, results)
+        # Don't return — there may be other calls in siblings
+
+    for child in node.children:
+        _walk_for_execs(child, source, env, results)
+
+
+def _handle_for_statement(
+    node: Node,
+    source: bytes,
+    env: Dict[str, Any],
+    results: List[str],
+) -> None:
+    """Unroll: for var in (lit1, lit2, ...): body_with_exec"""
+    left = node.child_by_field_name("left")
+    right = node.child_by_field_name("right")
+    body = node.child_by_field_name("body")
+
+    if left is None or right is None or body is None:
+        return
+
+    # Only handle simple identifier loop variables (no tuple unpacking)
+    if left.type != "identifier":
+        return
+
+    var_name = source[left.start_byte:left.end_byte].decode("utf-8", errors="replace")
+
+    iterable = _eval_iterable(right, source)
+    if not iterable:
+        return
+
+    for val in iterable:
+        new_env = dict(env)
+        new_env[var_name] = val
+        _walk_for_execs(body, source, new_env, results)
+
+
+def _handle_exec_call(
+    node: Node,
+    source: bytes,
+    env: Dict[str, Any],
+    results: List[str],
+) -> None:
+    """Expand: exec("literal") or eval("literal") → collect source string."""
+    func = node.child_by_field_name("function")
+    if func is None:
+        return
+
+    func_name = source[func.start_byte:func.end_byte].decode("utf-8", errors="replace").strip()
+    if func_name not in ("exec", "eval"):
+        return
+
+    args = node.child_by_field_name("arguments")
+    if args is None:
+        return
+
+    code_arg = _nth_positional_arg(args, 0)
+    if code_arg is None:
+        return
+
+    code_str = eval_literal_str(code_arg, source, env)
+    if code_str and code_str.strip():
+        results.append(code_str)
+
+
+def _nth_positional_arg(args_node: Node, n: int) -> Optional[Node]:
+    """Return the n-th positional argument, skipping *args/**kwargs/keywords."""
+    pos = 0
+    for child in args_node.children:
+        if child.type in (",", "(", ")"):
+            continue
+        if child.type in ("keyword_argument", "dictionary_splat_argument", "list_splat_argument"):
+            continue
+        if pos == n:
+            return child
+        pos += 1
+    return None

package/adl_drift/scanner/__init__.py

@@ -0,0 +1 @@
+"""Scanner modules for incremental and full project scans."""

package/adl_drift/scanner/discovery.py

@@ -0,0 +1,338 @@
+"""Backend-owned source discovery."""
+
+from __future__ import annotations
+
+import os
+import time
+from pathlib import Path
+from typing import Iterable, NamedTuple
+
+from adl_drift.lang.base import LanguageBackend
+from adl_drift.lang.registry import backend_for_file, detect_backends, get_backend
+from adl_drift.utils.fswalk import iter_tree
+
+
+EXCLUDED_DIRS = {
+    ".venv", "venv", ".env", "env", "__pycache__", ".git", ".gradle",
+    ".idea", ".tox", ".mypy_cache", ".pytest_cache", "node_modules",
+    ".eggs", "site-packages",
+}
+BUILD_OUTPUT_DIRS = {"target", "build", "out", "dist"}
+GENERATED_DIRS = {"generated", "generated-sources", "generated-test-sources"}
+
+
+def select_backend(
+    path: Path, candidates: Iterable[LanguageBackend] | None = None
+) -> LanguageBackend:
+    """Compatibility helper for callers that require one backend.
+
+    ``candidates`` is a ``detect_backends(path)`` result the caller already
+    has, so the project tree is not walked again for the same answer.
+    """
+    path = Path(path)
+    if path.is_file():
+        return backend_for_file(path) or get_backend("python")
+    candidates = detect_backends(path) if candidates is None else list(candidates)
+    with_sources = [
+        backend for backend in candidates
+        if any(any(path.rglob(f"*{ext}")) for ext in backend.file_extensions)
+    ]
+    return (with_sources or candidates or [get_backend("python")])[0]
+
+
+def discover_file_groups(
+    path: Path,
+    *,
+    include_generated: bool | None = None,
+    backends: Iterable[LanguageBackend] | None = None,
+    configs: dict[str, object] | None = None,
+) -> dict[str, list[Path]]:
+    """Discover every supported source and group paths by backend name.
+
+    ``backends`` (a ``detect_backends(path)`` result) and ``configs``
+    (``backend name -> backend.project_config(path)``) let a caller that
+    already computed them for this same ``path`` skip the tree walk and the
+    build-file parse they cost; both must have been computed for ``path``.
+    """
+    path = Path(path)
+    if path.is_file():
+        backend = backend_for_file(path)
+        return {backend.name: [path]} if backend else {}
+    configs = configs or {}
+    return {
+        backend.name: files
+        for backend in (detect_backends(path) if backends is None else backends)
+        if (files := discover_files(
+            path, backend, include_generated=include_generated,
+            config=configs.get(backend.name),
+        ))
+    }
+
+
+class _DiscoveryEntry(NamedTuple):
+    taken_at: float
+    include_generated: bool
+    dir_mtimes: dict[str, int]
+    build_file_stats: dict[str, tuple[int, int]]
+    groups: dict[str, list[Path]]
+
+
+class CachedDiscovery:
+    """``discover_file_groups(path)`` memo for a long-lived process, validated
+    by directory mtimes and build-file stats instead of re-walking the tree.
+
+    Creating, deleting, renaming or re-linking any file or directory updates
+    the mtime of the directory holding it, so re-stat()ing every directory
+    the previous discovery could have visited proves the walks would list
+    the same entries. Content edits never touch a directory mtime and are
+    invisible here by design — the graph's per-file stat fingerprint
+    (``CortexMemory._source_state``) catches those. The one place discovery
+    reads file *contents* is the build files a backend's ``project_config``
+    parses (source roots, modules); those are re-validated by their own
+    ``(st_mtime_ns, st_size)``. Same racy-window rule as
+    MCP_GRAPH_CACHE_PLAN.md Lớp 1: a directory or build file modified within
+    ``racy_window`` seconds of the snapshot is never trusted.
+    """
+
+    def __init__(self, path: Path, *, racy_window: float) -> None:
+        self.path = Path(path)
+        self.racy_window = racy_window
+        self._entry: _DiscoveryEntry | None = None
+
+    @staticmethod
+    def _include_generated() -> bool:
+        return os.environ.get("DRIFT_INCLUDE_GENERATED") == "1"
+
+    def groups(self) -> dict[str, list[Path]]:
+        entry = self._entry
+        if entry is not None and self._still_valid(entry):
+            return {name: list(files) for name, files in entry.groups.items()}
+        if self.path.is_file():
+            return discover_file_groups(self.path)
+        self._entry = entry = self._discover()
+        return {name: list(files) for name, files in entry.groups.items()}
+
+    def _discover(self) -> _DiscoveryEntry:
+        taken_at = time.time()
+        include_generated = self._include_generated()
+        backends = detect_backends(self.path)
+        configs = {backend.name: backend.project_config(self.path) for backend in backends}
+        groups = discover_file_groups(
+            self.path, include_generated=include_generated,
+            backends=backends, configs=configs,
+        )
+        # Every walk starts at ``path`` or at a config's project root (which
+        # may lie above ``path``); track the outermost of those trees.
+        roots = {self.path.resolve()}
+        build_file_stats: dict[str, tuple[int, int]] = {}
+        for config in configs.values():
+            project_path = getattr(config, "project_path", None)
+            if project_path is not None:
+                roots.add(Path(project_path).resolve())
+            build_file_stats.update(getattr(config, "build_file_stats", {}) or {})
+        outermost = [
+            root for root in roots
+            if not any(other != root and _under(root, {other}) for other in roots)
+        ]
+        dir_mtimes: dict[str, int] = {}
+        for root in outermost:
+            try:
+                dir_mtimes[str(root)] = os.stat(root).st_mtime_ns
+            except OSError:
+                continue
+            for _dirpath, dirs, _others in iter_tree(str(root)):
+                for entry in dirs:
+                    try:
+                        dir_mtimes[entry.path] = entry.stat(follow_symlinks=False).st_mtime_ns
+                    except OSError:
+                        continue
+        return _DiscoveryEntry(taken_at, include_generated, dir_mtimes, build_file_stats, groups)
+
+    def _still_valid(self, entry: _DiscoveryEntry) -> bool:
+        if entry.include_generated != self._include_generated():
+            return False
+        racy_before = (entry.taken_at - self.racy_window) * 1e9
+        try:
+            for dirpath, mtime in entry.dir_mtimes.items():
+                if mtime >= racy_before or os.stat(dirpath).st_mtime_ns != mtime:
+                    return False
+            for filepath, (mtime, size) in entry.build_file_stats.items():
+                stat = os.stat(filepath)
+                if mtime >= racy_before or (stat.st_mtime_ns, stat.st_size) != (mtime, size):
+                    return False
+        except OSError:
+            return False
+        return True
+
+
+# ``Path.relative_to`` compares case-insensitively on Windows; normalise both
+# sides the same way so the string check keeps that behaviour (identity on
+# POSIX). Module attribute so tests can simulate the Windows rule.
+_normcase = os.path.normcase
+
+
+def _root_prefixes(roots: Iterable[Path]) -> tuple[tuple[str, str], ...]:
+    """Precompute (exact, prefix) string forms for containment checks.
+
+    ``Path.relative_to`` re-parses both paths on every call; a corpus with
+    many build modules multiplies that by every output root per candidate
+    file. Comparing resolved path strings is equivalent for the resolved
+    inputs used here and avoids the pathlib overhead.
+    """
+    prefixes = []
+    for root in roots:
+        text = _normcase(str(root))
+        prefixes.append((text, text if text.endswith(os.sep) else text + os.sep))
+    return tuple(prefixes)
+
+
+def _under_prefixes(path_text: str, prefixes: tuple[tuple[str, str], ...]) -> bool:
+    path_text = _normcase(path_text)
+    for exact, prefix in prefixes:
+        if path_text == exact or path_text.startswith(prefix):
+            return True
+    return False
+
+
+def _under(path: Path, roots: set[Path]) -> bool:
+    return _under_prefixes(str(path), _root_prefixes(roots))
+
+
+# Kept under the old private name for tests; the walker itself lives in
+# utils so lang/registry.py can share it without importing the scanner.
+_iter_tree = iter_tree
+
+
+def _project_dirs(project_root: Path, markers: Iterable[str]) -> set[Path]:
+    """Directories under ``project_root`` holding a project marker.
+
+    One walk for every marker at once instead of one full ``rglob`` per
+    marker (Java has five). ``rglob(marker)`` matched directory entries
+    too, so directory names are checked as well. ``project_root`` is
+    already resolved and the walk never descends symlinked directories, so
+    ``dirpath`` is the resolved parent the old ``marker_path.parent.resolve()``
+    produced.
+    """
+    marker_names = set(markers)
+    # Gradle permits ``project.buildFileName`` to point at an arbitrary
+    # ``*.gradle``/``*.gradle.kts`` file.  Spring projects commonly discover
+    # those files dynamically from ``settings.gradle`` (rather than spelling
+    # every project in a static ``include(...)`` call), so exact-name matching
+    # alone collapses all such projects into the repository-root namespace.
+    # Enable the suffix match only for backends that advertised Gradle's
+    # conventional build markers; other language backends keep exact matching.
+    accepts_custom_gradle = bool(
+        marker_names & {"build.gradle", "build.gradle.kts"}
+    )
+    dirs = {project_root}
+    if not marker_names:
+        return dirs
+    for dirpath, subdirs, others in iter_tree(str(project_root)):
+        if any(entry.name in marker_names for entry in subdirs) or any(
+            entry.name in marker_names
+            or (
+                accepts_custom_gradle
+                and entry.name not in {"settings.gradle", "settings.gradle.kts"}
+                and entry.name.endswith((".gradle", ".gradle.kts"))
+            )
+            for entry in others
+        ):
+            dirs.add(Path(dirpath))
+    return dirs
+
+
+def discover_files(
+    path: Path,
+    backend: LanguageBackend,
+    *,
+    include_generated: bool | None = None,
+    config: object | None = None,
+) -> list[Path]:
+    """Find backend sources under configured roots, excluding build output.
+
+    ``config`` is ``backend.project_config(path)`` when the caller already
+    holds it (see ``discover_file_groups``); otherwise it is computed here.
+
+    MCP_GRAPH_CACHE_PLAN.md Lớp 2: one walk per (outermost) scan root with
+    ``EXCLUDED_DIRS``/build-output subtrees pruned as they are met, and no
+    ``Path.resolve()`` per candidate — the scan roots are resolved once and
+    the walk never enters a symlinked directory, so a candidate's resolved
+    path differs from the candidate only when the file itself is a symlink.
+    The result set is the one the previous per-marker/per-extension
+    ``rglob`` implementation produced, except that a *directory* whose name
+    ends in a source extension is no longer reported as a source file.
+    """
+    path = Path(path)
+    if path.is_file():
+        return [path] if path.suffix in backend.file_extensions else []
+
+    if include_generated is None:
+        include_generated = os.environ.get("DRIFT_INCLUDE_GENERATED") == "1"
+
+    if config is None:
+        config = backend.project_config(path)
+    roots: Iterable[Path] = getattr(config, "source_roots", ()) or (path,)
+    found: set[Path] = set()
+    root = path.resolve()
+    project_root = Path(getattr(config, "project_path", root)).resolve()
+    output_roots = {
+        project_dir / name
+        for project_dir in _project_dirs(project_root, backend.project_markers)
+        for name in BUILD_OUTPUT_DIRS
+    }
+    output_prefixes = _root_prefixes(output_roots)
+
+    scan_roots: set[Path] = set()
+    for source_root in roots:
+        source_root = Path(source_root).resolve()
+        if not source_root.is_dir() or not _under(source_root, {project_root}):
+            continue
+        if _under(source_root, {root}):
+            scan_root = source_root
+        elif _under(root, {source_root}):
+            scan_root = root
+        else:
+            continue
+        if not include_generated and any(
+            part.lower() in GENERATED_DIRS
+            for part in source_root.relative_to(project_root).parts
+        ):
+            continue
+        scan_roots.add(scan_root)
+    # A scan root nested inside another is reached by the outer walk; the
+    # per-file checks below are identical either way, so walk it once.
+    outermost = [
+        scan_root for scan_root in scan_roots
+        if not any(other != scan_root and _under(scan_root, {other}) for other in scan_roots)
+    ]
+
+    extensions = tuple(backend.file_extensions)
+    for scan_root in outermost:
+        for _dirpath, dirs, others in _iter_tree(str(scan_root)):
+            # Prune what the per-file checks would reject anyway: every path
+            # below carries the excluded part / lies under the output root.
+            dirs[:] = [
+                entry for entry in dirs
+                if entry.name not in EXCLUDED_DIRS
+                and not _under_prefixes(entry.path, output_prefixes)
+            ]
+            for entry in others:
+                if not entry.name.endswith(extensions):
+                    continue
+                candidate = Path(entry.path)
+                try:
+                    if entry.is_symlink():
+                        if entry.is_dir():
+                            continue  # symlink to a directory is not a source file
+                        resolved = candidate.resolve()
+                    else:
+                        resolved = candidate
+                    parts = resolved.relative_to(root).parts[:-1]
+                except (OSError, ValueError):
+                    continue
+                if any(part in EXCLUDED_DIRS for part in parts):
+                    continue
+                if _under_prefixes(str(resolved), output_prefixes):
+                    continue
+                found.add(candidate)
+    return sorted(found, key=str)