adl-toolkit 0.0.0-stage → 0.1.1

package/adl_drift/scanner/incremental.py

@@ -0,0 +1,261 @@
+"""Incremental scanning support.
+
+Tracks file content hashes and caches AnalysisResults so that only changed
+files need to be re-parsed on subsequent scans.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+from pathlib import Path
+from typing import Any, Dict, List, Optional, Set, Tuple
+
+from adl_drift.cortex.memory import CortexMemory
+from adl_drift.detectors.base import DetectorRegistry
+from adl_drift.utils.types import AnalysisResult
+
+
+def java_declaration_fingerprint(result: Any) -> Tuple[Any, ...]:
+    """Stable Java API surface used by incremental invalidation.
+
+    A method-body-only edit cannot change inbound resolution, while a
+    changed declaration, hierarchy, annotation, or modifier can. The
+    latter therefore falls back to a full project rebuild until a richer
+    dependency index can prove a smaller closure safe.
+
+    Shared by ``DriftCommands.build_graph`` and ``MCPHandlers._build_full_graph``
+    (MCP_INCREMENTAL_REBUILD_PLAN.md Lớp 1) so the two entry points cannot
+    drift apart on what counts as a structural change.
+    """
+    symbols = []
+    for symbol in result.symbols:
+        if symbol.kind not in {"class", "function", "attribute"}:
+            continue
+        symbols.append((
+            symbol.kind, symbol.name, symbol.parent,
+            tuple(symbol.scope_chain), symbol.signature,
+            tuple(sorted(symbol.param_types.items())), symbol.return_type,
+            tuple(symbol.superclasses), tuple(symbol.implements),
+            tuple(symbol.decorators), tuple(symbol.modifiers),
+            tuple(symbol.throws),
+        ))
+    # ``parent`` is optional, so compare a stable representation rather
+    # than relying on Python to order ``None`` and ``str`` in rare nested
+    # declaration layouts.
+    return result.module_name, tuple(sorted(symbols, key=repr))
+
+
+def java_incremental_scope(
+    root: Path,
+    scanner: "IncrementalScanner",
+    changed: List[Path],
+    deleted: List[Path],
+) -> Tuple[Optional[Set[Path]], str]:
+    """Return the safely invalidated Java-file closure, or ``None``.
+
+    The persisted graph already supplies stable nodes and inbound edges.
+    For body-only edits their declaration fingerprints are identical, so
+    only the changed compilation units need new outgoing resolution. A
+    structural edit could make previously unresolved callers resolve (or
+    change overload/DI/hierarchy selection), hence its safe closure is the
+    whole Java project for now — signalled by ``None`` plus the reason.
+    """
+    from adl_drift.lang.registry import get_backend  # local: avoid import cycles
+
+    java_changed = [path for path in changed if path.suffix == ".java"]
+    if any(path.suffix == ".java" for path in deleted):
+        return None, "java file deleted"
+    backend = get_backend("java")
+    parser, extractor = backend.parser(), backend.extractor()
+    for path in java_changed:
+        previous = scanner.load_cached_result(root, path)
+        if previous is None:
+            return None, f"no cached declaration surface for {path.name}"
+        try:
+            current = extractor._build(parser.parse_file(path), path)
+        except Exception as exc:
+            return None, f"cannot parse {path.name}: {type(exc).__name__}"
+        if java_declaration_fingerprint(previous) != java_declaration_fingerprint(current):
+            return None, f"declaration surface changed in {path.name}"
+    return set(java_changed), "method-body-only Java edit"
+
+
+class IncrementalScanner:
+    """Manages incremental scanning for a project.
+
+    Works with ``CortexMemory.file_metadata`` to avoid re-parsing
+    unchanged files.
+    """
+
+    def __init__(
+        self,
+        memory: CortexMemory,
+        parser: object,
+        extractor: object,
+        registry: DetectorRegistry,
+    ) -> None:
+        self.memory = memory
+        self.parser = parser
+        self.extractor = extractor
+        self.registry = registry
+
+    @staticmethod
+    def _file_hash(path: Path) -> str:
+        """SHA256 of file contents."""
+        return hashlib.sha256(path.read_bytes()).hexdigest()
+
+    @staticmethod
+    def _resolve_paths(files: List[Path]) -> List[Tuple[Path, str]]:
+        """``(path, str(path.resolve()))`` for each file, one lstat per distinct
+        path component instead of one per component per file.
+
+        ``Path.resolve()`` lstat()s every component of every path, so a corpus
+        of a few hundred deep paths costs tens of thousands of syscalls on each
+        no-change query (MCP_GRAPH_CACHE_PLAN.md Lớp 2). Directories are
+        resolved once and shared by every file beneath them; only a component
+        that is itself a symlink falls back to a real ``resolve()``. Relative
+        paths and ``..`` components keep the plain ``resolve()`` so the result
+        is identical to it in every case.
+        """
+        cache: Dict[Path, Path] = {}
+
+        def real(path: Path) -> Path:
+            hit = cache.get(path)
+            if hit is None:
+                if path.is_symlink():
+                    hit = path.resolve()
+                else:
+                    parent = path.parent
+                    hit = path if parent == path else real(parent) / path.name
+                cache[path] = hit
+            return hit
+
+        return [
+            (f, str(f.resolve() if not f.is_absolute() or ".." in f.parts else real(f)))
+            for f in files
+        ]
+
+    def classify_files(
+        self,
+        project_path: Path,
+        files: List[Path],
+        extensions: Optional[Set[str]] = None,
+    ) -> Tuple[List[Path], List[Path], List[Path]]:
+        """Classify files as changed, unchanged, or deleted.
+
+        Returns:
+            (changed_files, unchanged_files, deleted_files)
+        """
+        stored = self.memory.get_all_file_metadata(project_path)
+        stored_by_path: Dict[str, Dict] = {row["file_path"]: row for row in stored}
+        resolved_paths = self._resolve_paths(files)
+        current_paths: Set[str] = {fpath for _, fpath in resolved_paths}
+
+        changed: List[Path] = []
+        unchanged: List[Path] = []
+
+        for f, fpath in resolved_paths:
+            row = stored_by_path.get(fpath)
+            if row is None:
+                changed.append(f)
+                continue
+
+            current_hash = self._file_hash(f)
+            if current_hash != row["content_hash"]:
+                changed.append(f)
+            else:
+                unchanged.append(f)
+
+        deleted: List[Path] = [
+            Path(row["file_path"])
+            for fpath, row in stored_by_path.items()
+            if fpath not in current_paths
+            and (extensions is None or Path(fpath).suffix in extensions)
+        ]
+
+        return changed, unchanged, deleted
+
+    def load_cached_result(
+        self,
+        project_path: Path,
+        file_path: Path,
+    ) -> Optional[AnalysisResult]:
+        """Load a cached ``AnalysisResult`` from memory."""
+        row = self.memory.get_file_metadata(project_path, file_path)
+        if row and row.get("analysis_json"):
+            return AnalysisResult.from_dict(json.loads(row["analysis_json"]))
+        return None
+
+    def load_cached_results(
+        self,
+        project_path: Path,
+        file_paths: List[Path],
+    ) -> Dict[Path, AnalysisResult]:
+        """Load cached ``AnalysisResult`` objects for many files at once.
+
+        One query replaces a connection-per-file round trip; a large corpus
+        reloads every unchanged file on each incremental update.
+        """
+        if not file_paths:
+            return {}
+        rows = {
+            row["file_path"]: row
+            for row in self.memory.get_all_file_metadata(project_path)
+        }
+        loaded: Dict[Path, AnalysisResult] = {}
+        for file_path in file_paths:
+            row = rows.get(str(file_path.resolve()))
+            if row and row.get("analysis_json"):
+                loaded[file_path] = AnalysisResult.from_dict(json.loads(row["analysis_json"]))
+        return loaded
+
+    def store_result(
+        self,
+        project_path: Path,
+        file_path: Path,
+        result: AnalysisResult,
+    ) -> None:
+        """Store an ``AnalysisResult`` and its content hash in memory."""
+        self.store_results(project_path, [(file_path, result)])
+
+    def store_results(
+        self,
+        project_path: Path,
+        entries: List[Tuple[Path, AnalysisResult]],
+    ) -> None:
+        """Store multiple ``AnalysisResult``s and their content hashes in one
+        connection/transaction (see ``CortexMemory.store_files_metadata``).
+
+        Callers must only flush this after every file in ``entries`` has
+        already had its patterns persisted (``_append_patterns``/
+        ``store_patterns``), same ordering requirement as the previous
+        one-call-per-file ``store_result`` — batching here just means the
+        flush point moves later, never earlier.
+        """
+        if not entries:
+            return
+        self.memory.store_files_metadata(project_path, [
+            (
+                file_path,
+                self._file_hash(file_path),
+                file_path.stat().st_mtime,
+                json.dumps(result.to_dict()),
+            )
+            for file_path, result in entries
+        ])
+
+    def prune_deleted_files(
+        self,
+        project_path: Path,
+        current_files: Set[str],
+        extensions: Optional[Set[str]] = None,
+    ) -> None:
+        """Remove metadata for files that no longer exist."""
+        if extensions is not None:
+            current_files |= {
+                row["file_path"]
+                for row in self.memory.get_all_file_metadata(project_path)
+                if Path(row["file_path"]).suffix not in extensions
+            }
+        self.memory.prune_file_metadata(project_path, current_files)

package/adl_drift/scanner/parallel.py

@@ -0,0 +1,224 @@
+"""Parallel file analysis using ProcessPoolExecutor.
+
+Worker processes initialize their own parser + extractor + detector registry
+once (via ``initializer``), then process file paths from a queue. Tree-sitter
+``Tree`` objects are not picklable, so workers return ``AnalysisResult`` and
+``List[Pattern]`` only — callers that need the CST (e.g. ``resolve_calls``)
+must re-parse in the main process.
+
+Degradation ladder (P2.9): processes → threads → sequential.
+- Spawned child processes (detected via multiprocessing) never attempt a pool
+  at all — prevents recursive spawn storms from unguarded host ``__main__``
+  modules and from Drift being embedded inside someone else's worker pool.
+- If the process pool cannot start or dies mid-run, the remaining files are
+  analyzed on a thread pool (tree-sitter parsing releases the GIL, so threads
+  still buy real parallelism for the parse-heavy part).
+- If even threads are unavailable, plain sequential in-process analysis.
+Every tier yields the identical (file_path, result, patterns, error) stream.
+"""
+
+from __future__ import annotations
+
+import logging
+import multiprocessing
+import os
+import sys
+import threading
+from concurrent.futures import ProcessPoolExecutor, ThreadPoolExecutor
+from concurrent.futures.process import BrokenProcessPool
+from pathlib import Path
+from pickle import PicklingError
+from typing import Iterator, List, Optional, Tuple
+
+from adl_drift.utils.types import AnalysisResult, Pattern
+
+logger = logging.getLogger(__name__)
+
+# Pool failures that mean "this parallelism tier is unavailable here", not
+# "the scan is broken" (P2.9). RuntimeError: spawn bootstrap guard (unguarded
+# host __main__ on Windows) or thread-creation failure. BrokenProcessPool:
+# worker died. OSError: process/handle limits. PicklingError: unpicklable arg
+# in a hostile embedding context.
+_POOL_FAILURES = (RuntimeError, BrokenProcessPool, OSError, PicklingError)
+
+
+# Worker-process globals (one set per worker).
+_PARSER = None
+_EXTRACTOR = None
+_REGISTRY = None
+_BACKEND_NAME = None
+
+# Thread-fallback components (one set per thread — tree-sitter Parser is not
+# thread-safe, so threads must not share a parser instance).
+_TLS = threading.local()
+
+
+def _in_spawned_child() -> bool:
+    """True when running inside a multiprocessing child process.
+
+    Covers two cases (P2.9):
+    - ``_inheriting``: this process is mid-bootstrap, re-importing the host's
+      ``__main__`` (the exact context Python's spawn guard protects).
+    - ``parent_process() is not None``: we are a worker — either Drift's own
+      or one belonging to whatever app embedded us. Nested pools multiply
+      processes; never start one from in here.
+    """
+    if getattr(multiprocessing.current_process(), "_inheriting", False):
+        return True
+    return multiprocessing.parent_process() is not None
+
+
+def _make_components(backend_name: str = "python"):
+    """Build a fresh (parser, extractor, registry) set."""
+    from adl_drift.detectors.base import DetectorRegistry
+    from adl_drift.lang.registry import get_backend
+
+    backend = get_backend(backend_name)
+    return backend.parser(), backend.extractor(), DetectorRegistry(backend.detectors())
+
+
+def _init_components(backend_name: str = "python") -> None:
+    global _PARSER, _EXTRACTOR, _REGISTRY, _BACKEND_NAME
+    _PARSER, _EXTRACTOR, _REGISTRY = _make_components(backend_name)
+    _BACKEND_NAME = backend_name
+
+
+def _init_worker(backend_name: str = "python") -> None:
+    # Workers must never write to the real stdout: under the MCP stdio
+    # transport it is the JSON-RPC channel, and one stray print() from a
+    # detector or import side-effect corrupts the protocol framing (P2.9).
+    sys.stdout = sys.stderr
+    _init_components(backend_name)
+
+
+def _analyze_one(
+    parser, extractor, registry, args: Tuple[str, Optional[str]]
+) -> Tuple[str, Optional[AnalysisResult], List[Pattern], Optional[str]]:
+    file_path_str, category_filter = args
+    file_path = Path(file_path_str)
+    try:
+        pr = parser.parse_file(file_path)
+        patterns = registry.run_all(pr.tree, pr.source_bytes, file_path)
+        ar = extractor._build(pr, file_path)
+        if category_filter:
+            patterns = [p for p in patterns if p.category.name.lower() == category_filter.lower()]
+        return (file_path_str, ar, patterns, None)
+    except Exception as exc:  # noqa: BLE001 — surfaced to caller
+        return (file_path_str, None, [], f"{type(exc).__name__}: {exc}")
+
+
+def _scan_one(args: Tuple[str, Optional[str]]) -> Tuple[str, Optional[AnalysisResult], List[Pattern], Optional[str]]:
+    """Process-worker entry: uses the per-process globals set by _init_worker."""
+    return _analyze_one(_PARSER, _EXTRACTOR, _REGISTRY, args)
+
+
+def _scan_one_threaded(
+    args: Tuple[str, Optional[str], str]
+) -> Tuple[str, Optional[AnalysisResult], List[Pattern], Optional[str]]:
+    """Thread-fallback entry: per-thread components (parser is not thread-safe)."""
+    file_path, category_filter, backend_name = args
+    if getattr(_TLS, "backend_name", None) != backend_name:
+        _TLS.parser, _TLS.extractor, _TLS.registry = _make_components(backend_name)
+        _TLS.backend_name = backend_name
+    return _analyze_one(
+        _TLS.parser, _TLS.extractor, _TLS.registry, (file_path, category_filter)
+    )
+
+
+def _default_workers() -> int:
+    env = os.environ.get("DRIFT_PARALLEL_WORKERS")
+    if env:
+        try:
+            n = int(env)
+            if n >= 1:
+                return n
+        except ValueError:
+            pass
+    cpu = os.cpu_count() or 1
+    return min(cpu, 8)
+
+
+# Below this file count, parallel overhead (spawn + import) loses to sequential.
+PARALLEL_THRESHOLD = 12
+
+
+def should_parallelize(file_count: int, max_workers: Optional[int] = None) -> bool:
+    if max_workers == 1:
+        return False
+    if os.environ.get("DRIFT_PARALLEL") == "0":
+        return False
+    if _in_spawned_child():
+        return False
+    return file_count >= PARALLEL_THRESHOLD
+
+
+def parallel_analyze(
+    files: List[Path],
+    backend_name: str = "python",
+    category_filter: Optional[str] = None,
+    max_workers: Optional[int] = None,
+) -> Iterator[Tuple[Path, Optional[AnalysisResult], List[Pattern], Optional[str]]]:
+    """Analyze files with worker-local components for ``backend_name``.
+
+    Yields (file_path, AnalysisResult|None, patterns, error).
+
+    P2.9 degradation ladder — every tier produces the identical stream:
+    1. Process pool (default). Skipped entirely inside spawned children.
+    2. Thread pool, if the process pool cannot start or dies mid-run. A
+       ``done`` set ensures files already yielded are not re-analyzed.
+    3. Sequential, if even the thread pool is unavailable.
+    """
+    if not files:
+        return
+    workers = max_workers or _default_workers()
+    args = [(str(f), category_filter) for f in files]
+    done: set = set()
+
+    # Tier 1: processes. Never from inside a spawned child — creating a pool
+    # there either trips Python's spawn-bootstrap guard or nests pools.
+    if not _in_spawned_child():
+        try:
+            with ProcessPoolExecutor(
+                max_workers=workers,
+                initializer=_init_worker,
+                initargs=(backend_name,),
+            ) as pool:
+                # chunksize tuned so each worker gets a healthy batch but no straggler dominates.
+                chunksize = max(1, len(args) // (workers * 4) or 1)
+                for file_path_str, ar, patterns, err in pool.map(_scan_one, args, chunksize=chunksize):
+                    done.add(file_path_str)
+                    yield (Path(file_path_str), ar, patterns, err)
+            return
+        except _POOL_FAILURES as exc:
+            logger.warning(
+                "Process pool unavailable (%s: %s) — falling back to threads "
+                "for %d remaining file(s).",
+                type(exc).__name__, exc, len(args) - len(done),
+            )
+    else:
+        logger.debug("In a spawned child process — skipping process pool.")
+
+    # Tier 2: threads. Parsing releases the GIL, so this still parallelizes
+    # the heavy part. Per-thread components via _TLS.
+    pending = [(a[0], a[1], backend_name) for a in args if a[0] not in done]
+    try:
+        with ThreadPoolExecutor(max_workers=min(workers, 4)) as tpool:
+            for file_path_str, ar, patterns, err in tpool.map(_scan_one_threaded, pending):
+                done.add(file_path_str)
+                yield (Path(file_path_str), ar, patterns, err)
+        return
+    except _POOL_FAILURES as exc:
+        logger.warning(
+            "Thread pool unavailable (%s: %s) — finishing sequentially "
+            "for %d remaining file(s).",
+            type(exc).__name__, exc, len(args) - len(done),
+        )
+
+    # Tier 3: sequential in this process (no stdout swap — we may be main).
+    if _PARSER is None or _BACKEND_NAME != backend_name:
+        _init_components(backend_name)
+    for arg in args:
+        if arg[0] in done:
+            continue
+        file_path_str, ar, patterns, err = _analyze_one(_PARSER, _EXTRACTOR, _REGISTRY, arg)
+        yield (Path(file_path_str), ar, patterns, err)

package/adl_drift/server/__init__.py

@@ -0,0 +1 @@
+"""MCP server implementation."""