adl-toolkit 0.0.0-stage → 0.1.1

package/adl_drift/cortex/patterns.py

@@ -0,0 +1,89 @@
+"""Pattern management and approval workflow.
+
+Handles the lifecycle of detected patterns:
+1. Detection (auto)
+2. Review (human)
+3. Approval (human marks as intentional)
+4. Enforcement (subsequent scans check against approved patterns)
+"""
+
+from __future__ import annotations
+
+from typing import List, Dict, Any, Optional
+from adl_drift.utils.types import Pattern, PatternCategory
+from adl_drift.cortex.memory import CortexMemory
+
+
+class PatternManager:
+    """Manages the pattern approval workflow.
+
+    Workflow:
+    1. Detectors find patterns
+    2. User reviews via `adl-drift patterns`
+    3. User approves via `adl-drift approve <id>`
+    4. Future scans flag deviations from approved patterns
+    """
+
+    def __init__(self, memory: CortexMemory):
+        self.memory = memory
+
+    def process_new_patterns(self, patterns: List[Pattern]) -> Dict[str, Any]:
+        """Process newly detected patterns.
+
+        Returns:
+            Summary of new, known, and conflicting patterns.
+        """
+        new_patterns = []
+        known_patterns = []
+        conflicts = []
+
+        for pattern in patterns:
+            pattern_id = pattern.compute_hash()
+
+            # Check if we already know this pattern
+            existing = self.memory.get_patterns(category=pattern.category.name)
+            existing_ids = {p["id"] for p in existing}
+
+            if pattern_id in existing_ids:
+                known_patterns.append(pattern)
+            else:
+                new_patterns.append(pattern)
+                # Auto-store for review
+                self.memory.store_pattern(pattern)
+
+        return {
+            "new": len(new_patterns),
+            "known": len(known_patterns),
+            "conflicts": len(conflicts),
+            "new_patterns": [p.name for p in new_patterns],
+        }
+
+    def approve(self, pattern_id: str) -> bool:
+        """Approve a pattern as intentional."""
+        self.memory.approve_pattern(pattern_id)
+        return True
+
+    def check_conformance(self, patterns: List[Pattern]) -> List[Dict[str, Any]]:
+        """Check if current patterns conform to approved patterns.
+
+        Flags deviations: e.g., if approved pattern is "use @login_required
+        on all routes" but a new route lacks it.
+        """
+        violations = []
+        approved = self.memory.get_patterns(approved_only=True)
+        approved_set = {p["id"] for p in approved}
+
+        for pattern in patterns:
+            pid = pattern.compute_hash()
+            if pid not in approved_set:
+                # Check for similar approved patterns
+                similar = self.memory.search_similar(pattern.name, limit=3)
+                if similar:
+                    violations.append({
+                        "pattern": pattern.name,
+                        "file": str(pattern.file_path),
+                        "line": pattern.position.line,
+                        "message": f"Pattern not in approved set. Similar approved: {similar[0]['name']}",
+                    })
+
+        return violations

package/adl_drift/cortex/telemetry.py

@@ -0,0 +1,192 @@
+"""Shared tool-call telemetry recorder (MONITOR).
+
+Both the MCP handler path (``server/handlers.py``) and the CLI command path
+(``cli/commands.py``) route their token-savings accounting through one
+:class:`TelemetryRecorder` so the centralized monitor sees a single, consistent
+metric regardless of how drift was invoked (PM.1 / PM.2).
+
+Token-savings model (per-call-site — honest, query-shape-aware):
+
+The "saved" number is what an agent would *avoid* spending by asking drift
+instead of locating + reading the code by hand. The manual alternative to a
+drift answer is to **grep-locate and read each distinct call-site / reference
+the answer surfaces** — not each *file*. Per-file accounting (the earlier PM.1
+model) mis-scored both ways: it overstated when one answer spanned many files,
+and *understated* high-fan-in queries where dozens of call-sites live in a few
+files (an agent would still have to read each site). So we count per site:
+
+* a **locate cost** of :data:`GREP_TOKENS_PER_SITE` ≈ one grep match line that
+  points the agent at the site; plus
+* a **read window** of :data:`READ_WINDOW_PER_SITE` ≈ the call-site line and its
+  immediate context the agent would read to confirm the dependency.
+
+A "site" is any node id the answer points at (callers, callees, refs, matches,
+the target itself). Sites are **deduplicated across calls** within a recorder's
+lifetime (one MCP session / one CLI process): once an agent has read a site it
+stays in context, so a later call referencing it saves nothing more.
+
+The result is an honest upper bound that tracks the actual fan-out of each
+answer instead of a flat per-file guess. The monitor labels it as such.
+"""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+from typing import Any, Dict, List, Optional
+
+from adl_drift.lang.registry import backend_for_file
+
+
+class TelemetryRecorder:
+    """Accumulates per-call token accounting and writes ``tool_invocations`` rows."""
+
+    # Per distinct call-site / reference an answer surfaces, the manual
+    # alternative costs a grep locate + a focused read of that site.
+    GREP_TOKENS_PER_SITE = 20   # one grep match line that locates the site
+    READ_WINDOW_PER_SITE = 80   # ~20 lines: the site + immediate context
+    _MAX_DEPTH = 8
+
+    def __init__(self, memory: Any, project_root: Path):
+        self.memory = memory
+        self.project_root = Path(project_root)
+        # Cross-call dedup, reset only when a fresh recorder is constructed:
+        # call-sites already "read" earlier in this session (the baseline unit),
+        # and files referenced (informational `files_referenced` stat).
+        self._counted_sites: set = set()
+        self._counted_files: set = set()
+
+    def record(
+        self,
+        tool: str,
+        params: Dict[str, Any],
+        result: Any,
+        *,
+        is_error: bool = False,
+        duration_ms: float = 0.0,
+        source: str = "mcp",
+    ) -> None:
+        """Record one tool call. Best-effort — never raises into the caller."""
+        try:
+            self._record(tool, params, result, is_error, duration_ms, source)
+        except Exception:
+            pass
+
+    def _record(
+        self,
+        tool: str,
+        params: Dict[str, Any],
+        result: Any,
+        is_error: bool,
+        duration_ms: float,
+        source: str,
+    ) -> None:
+        params = params or {}
+        response_json = json.dumps(result, default=str)
+        response_tokens = max(1, len(response_json) // 4)
+
+        files: set = set()
+        sites: set = set()
+        confidences: List[float] = []
+        self._collect_refs(result, files, confidences, sites, depth=0)
+
+        # Baseline = grep-locate + focused read of each distinct call-site the
+        # answer surfaces. Sites the agent already saw earlier this session stay
+        # in context, so only newly-seen ones cost anything (cross-call dedup).
+        new_sites = [s for s in sites if s not in self._counted_sites]
+        baseline_tokens = (
+            self.GREP_TOKENS_PER_SITE + self.READ_WINDOW_PER_SITE
+        ) * len(new_sites)
+        saved = max(baseline_tokens - response_tokens, 0)
+        self._counted_sites.update(new_sites)
+
+        # `files_referenced` stays an informational count of distinct source files
+        # an answer spans (canonicalized so relative/absolute forms collapse).
+        canon = {self._canonical(f) for f in files}
+        new_files = [f for f in canon if f not in self._counted_files]
+        self._counted_files.update(new_files)
+
+        hi = sum(1 for c in confidences if c >= 0.85)
+        md = sum(1 for c in confidences if 0.6 <= c < 0.85)
+        lo = sum(1 for c in confidences if c < 0.6)
+        cmin = min(confidences) if confidences else None
+        cmax = max(confidences) if confidences else None
+        cavg = (sum(confidences) / len(confidences)) if confidences else None
+
+        target = (params.get("symbol") or params.get("query")
+                  or params.get("node_id") or params.get("path")
+                  or params.get("topic") or params.get("category")
+                  or params.get("from"))
+        args = {k: v for k, v in params.items() if k != "request_id"}
+        args["source"] = source
+
+        self.memory.store_invocation({
+            "project_path": str(self.project_root),
+            "tool": tool,
+            "target": str(target) if target is not None else None,
+            "args": args,
+            "response_tokens": response_tokens,
+            "baseline_tokens": baseline_tokens,
+            "saved_tokens": saved,
+            "files_referenced": len(new_files),
+            "result_count": self._result_count(result),
+            "confidence_min": cmin, "confidence_avg": cavg, "confidence_max": cmax,
+            "high_conf": hi, "med_conf": md, "low_conf": lo,
+            "duration_ms": duration_ms,
+            "is_error": is_error,
+        })
+
+    # Node ids are `module:qualname` — they contain ':' and never whitespace or
+    # parentheses (which appear in signatures / snippets / docstrings).
+    @staticmethod
+    def _looks_like_site(s: str) -> bool:
+        return ":" in s and not any(c in s for c in " ()\n\t")
+
+    def _collect_refs(self, obj: Any, files: set, confidences: List[float],
+                      sites: set, depth: int) -> None:
+        """Recurse a response payload collecting backend file refs, confidences,
+        and call-site node ids (the baseline unit)."""
+        if depth > self._MAX_DEPTH:
+            return
+        if isinstance(obj, dict):
+            for k, v in obj.items():
+                if k == "confidence" and isinstance(v, (int, float)):
+                    confidences.append(float(v))
+                    continue
+                if k == "id" and isinstance(v, str):
+                    sites.add(v)  # explicit node id — always a site
+                    continue
+                # node-id dict keys (e.g. transitive_callers' {id: depth} map)
+                if isinstance(k, str) and self._looks_like_site(k):
+                    sites.add(k)
+                self._collect_refs(v, files, confidences, sites, depth + 1)
+        elif isinstance(obj, list):
+            for item in obj:
+                self._collect_refs(item, files, confidences, sites, depth + 1)
+        elif isinstance(obj, str):
+            if backend_for_file(Path(obj)) is not None:
+                files.add(obj)
+            elif self._looks_like_site(obj):
+                sites.add(obj)
+
+    def _canonical(self, ref: str) -> str:
+        """Resolve a source ref (relative or absolute) to one stable key per file."""
+        for candidate in (Path(ref), self.project_root / ref):
+            try:
+                if candidate.is_file():
+                    return str(candidate.resolve())
+            except (OSError, ValueError):
+                continue
+        return ref
+
+    @staticmethod
+    def _result_count(result: Any) -> int:
+        if isinstance(result, dict):
+            if isinstance(result.get("count"), int):
+                return result["count"]
+            for v in result.values():
+                if isinstance(v, list):
+                    return len(v)
+        elif isinstance(result, list):
+            return len(result)
+        return 0

package/adl_drift/detectors/__init__.py

@@ -0,0 +1 @@
+"""Pattern detectors for Python code analysis."""

package/adl_drift/detectors/async_patterns.py

@@ -0,0 +1,115 @@
+"""Async/await pattern detectors.
+
+Finds async functions, await expressions, asyncio usage, event loops.
+"""
+
+from __future__ import annotations
+
+from typing import List, Dict, Any
+from pathlib import Path
+from tree_sitter import Node, Tree
+
+from adl_drift.detectors.base import BaseDetector
+from adl_drift.utils.types import Pattern, PatternCategory
+from adl_drift.parser.core import PythonParser
+
+
+class AsyncPatternDetector(BaseDetector):
+    """Detects async/await patterns."""
+
+    def __init__(self, parser: PythonParser):
+        super().__init__(parser)
+
+    @property
+    def category(self) -> PatternCategory:
+        return PatternCategory.ASYNC
+
+    def detect(self, tree: Tree, source_bytes: bytes, file_path: Path) -> List[Pattern]:
+        patterns = []
+
+        for node in self._walk(tree.root_node):
+            if node.type == "function_definition":
+                if self._is_async(node):
+                    patterns.append(self._make_pattern(
+                        name="async_function",
+                        description="Async function definition",
+                        node=node,
+                        file_path=file_path,
+                        confidence=0.99,
+                        metadata={"type": "async_def"}
+                    ))
+            elif node.type == "await" and node.child_count > 0:
+                # Skip the bare `await` keyword token; only emit for the
+                # expression-level node (which has children).
+                patterns.append(self._make_pattern(
+                    name="await_expression",
+                    description="Await expression",
+                    node=node,
+                    file_path=file_path,
+                    confidence=0.99,
+                    metadata={"type": "await"}
+                ))
+            elif node.type == "for_statement":
+                if self._is_async_for(node):
+                    patterns.append(self._make_pattern(
+                        name="async_for",
+                        description="Async for loop",
+                        node=node,
+                        file_path=file_path,
+                        confidence=0.95,
+                        metadata={"type": "async_for"}
+                    ))
+            elif node.type == "with_statement":
+                if self._is_async_with(node):
+                    patterns.append(self._make_pattern(
+                        name="async_with",
+                        description="Async context manager",
+                        node=node,
+                        file_path=file_path,
+                        confidence=0.95,
+                        metadata={"type": "async_with"}
+                    ))
+            elif node.type == "call":
+                patterns.extend(self._analyze_asyncio_calls(node, source_bytes, file_path))
+
+        return patterns
+
+    def _is_async(self, node: Node) -> bool:
+        return any(child.type == "async" for child in node.children)
+
+    def _is_async_for(self, node: Node) -> bool:
+        return any(child.type == "async" for child in node.children)
+
+    def _is_async_with(self, node: Node) -> bool:
+        return any(child.type == "async" for child in node.children)
+
+    def _analyze_asyncio_calls(self, node: Node, source: bytes, path: Path) -> List[Pattern]:
+        patterns = []
+        text = self.parser.node_text(node, source)
+
+        asyncio_funcs = {
+            "create_task": "Creating async task",
+            "gather": "Gathering async results",
+            "run": "Running event loop",
+            "sleep": "Async sleep",
+            "wait_for": "Waiting with timeout",
+            "ensure_future": "Ensuring future",
+        }
+
+        for func, desc in asyncio_funcs.items():
+            if f"asyncio.{func}" in text or f"{func}(" in text:
+                patterns.append(self._make_pattern(
+                    name=f"asyncio_{func}",
+                    description=desc,
+                    node=node,
+                    file_path=path,
+                    confidence=0.85,
+                    metadata={"type": "asyncio", "function": func}
+                ))
+
+        return patterns
+
+    def _walk(self, node):
+        yield node
+        for child in node.children:
+            yield from self._walk(child)

package/adl_drift/detectors/base.py

@@ -0,0 +1,121 @@
+"""Base detector class - all detectors inherit from this.
+
+Equivalent to the detector trait system in Rust.
+Each detector is a plugin that scans the CST for specific patterns.
+"""
+
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+from collections.abc import Iterable
+from typing import List, Optional, Dict, Any
+from pathlib import Path
+
+from tree_sitter import Node, Tree
+
+from adl_drift.lang.base import LanguageParser
+from adl_drift.utils.types import Pattern, PatternCategory, Symbol
+
+
+class BaseDetector(ABC):
+    """Abstract base for all pattern detectors.
+
+    In Rust this would be a trait:
+    ```rust
+    trait Detector {
+        fn category(&self) -> PatternCategory;
+        fn detect(&self, tree: &Tree, source: &[u8]) -> Vec<Pattern>;
+    }
+    ```
+    """
+
+    def __init__(self, parser: LanguageParser):
+        self.parser = parser
+
+    @property
+    @abstractmethod
+    def category(self) -> PatternCategory:
+        """The pattern category this detector handles."""
+        pass
+
+    @abstractmethod
+    def detect(self, tree: Tree, source_bytes: bytes, file_path: Path) -> List[Pattern]:
+        """Run detection on a parsed file.
+
+        Args:
+            tree: The tree-sitter CST
+            source_bytes: Raw source for text extraction
+            file_path: Path to the file being analyzed
+
+        Returns:
+            List of detected patterns
+        """
+        pass
+
+    def _make_pattern(
+        self,
+        name: str,
+        description: str,
+        node: Node,
+        file_path: Path,
+        confidence: float,
+        symbol: Optional[Symbol] = None,
+        metadata: Optional[Dict[str, Any]] = None,
+    ) -> Pattern:
+        """Helper to create a Pattern from a CST node."""
+        return Pattern(
+            category=self.category,
+            name=name,
+            description=description,
+            position=self.parser.node_position(node),
+            file_path=file_path,
+            confidence=confidence,
+            symbol=symbol,
+            metadata=metadata or {},
+        )
+
+    def _node_matches_type(self, node: Node, *types: str) -> bool:
+        """Check if node type matches any of the given types."""
+        return node.type in types
+
+    def _get_child_by_field(self, node: Node, field: str) -> Optional[Node]:
+        """Safely get a child node by field name."""
+        try:
+            return node.child_by_field_name(field)
+        except Exception:
+            return None
+
+
+class DetectorRegistry:
+    """Registry of all available detectors.
+
+    Equivalent to a static registry or plugin system in Rust.
+    """
+
+    def __init__(self, detectors: Iterable[BaseDetector] = ()):
+        self._detectors: Dict[PatternCategory, List[BaseDetector]] = {}
+        for detector in detectors:
+            self.register(detector)
+
+    def register(self, detector: BaseDetector) -> None:
+        """Register a detector instance."""
+        cat = detector.category
+        if cat not in self._detectors:
+            self._detectors[cat] = []
+        self._detectors[cat].append(detector)
+
+    def get_for_category(self, category: PatternCategory) -> List[BaseDetector]:
+        """Get all detectors for a specific category."""
+        return self._detectors.get(category, [])
+
+    def get_all(self) -> List[BaseDetector]:
+        """Get all registered detectors."""
+        return [d for detectors in self._detectors.values() for d in detectors]
+
+    def run_all(self, tree: Tree, source_bytes: bytes, file_path: Path) -> List[Pattern]:
+        """Run all detectors and collect patterns."""
+        all_patterns = []
+        for detector in self.get_all():
+            patterns = detector.detect(tree, source_bytes, file_path)
+            all_patterns.extend(patterns)
+        return all_patterns

package/adl_drift/detectors/config.py

@@ -0,0 +1,122 @@
+"""Config / environment variable detector."""
+
+from __future__ import annotations
+
+import re
+from pathlib import Path
+from typing import List, Optional
+
+from tree_sitter import Node, Tree
+
+from adl_drift.detectors.base import BaseDetector
+from adl_drift.utils.types import Pattern, PatternCategory
+
+
+UPPER_SNAKE = re.compile(r"^[A-Z][A-Z0-9_]*$")
+
+
+class ConfigDetector(BaseDetector):
+    """Captures env-var reads, settings access, and UPPER_SNAKE constants."""
+
+    ENV_FUNCS = {"os.environ.get", "os.getenv"}
+
+    @property
+    def category(self) -> PatternCategory:
+        return PatternCategory.CONFIG
+
+    def detect(self, tree: Tree, source_bytes: bytes, file_path: Path) -> List[Pattern]:
+        patterns: List[Pattern] = []
+        is_dotenv_loader = False
+
+        for node in self._walk(tree.root_node):
+            if node.type == "call":
+                p = self._check_env_call(node, source_bytes, file_path)
+                if p is not None:
+                    patterns.append(p)
+            elif node.type == "subscript":
+                p = self._check_environ_subscript(node, source_bytes, file_path)
+                if p is not None:
+                    patterns.append(p)
+            elif node.type == "assignment":
+                p = self._check_module_constant(node, source_bytes, file_path)
+                if p is not None:
+                    patterns.append(p)
+            elif node.type in ("import_statement", "import_from_statement"):
+                text = self.parser.node_text(node, source_bytes)
+                if "dotenv" in text:
+                    is_dotenv_loader = True
+
+        if is_dotenv_loader:
+            patterns.append(self._make_pattern(
+                name="dotenv_loader",
+                description="Module loads .env via python-dotenv",
+                node=tree.root_node, file_path=file_path, confidence=0.95,
+                metadata={"role": "config_loader"},
+            ))
+        return patterns
+
+    def _check_env_call(self, node: Node, source: bytes, file_path: Path) -> Optional[Pattern]:
+        func = node.child_by_field_name("function")
+        if func is None:
+            return None
+        func_text = self.parser.node_text(func, source).strip()
+        if func_text not in self.ENV_FUNCS:
+            return None
+        key = self._first_string_arg(node, source)
+        return self._make_pattern(
+            name="env_read",
+            description=f"env-var read via {func_text}({key or '?'})",
+            node=node, file_path=file_path, confidence=0.9,
+            metadata={"function": func_text, "key": key},
+        )
+
+    def _check_environ_subscript(self, node: Node, source: bytes, file_path: Path) -> Optional[Pattern]:
+        value = node.child_by_field_name("value")
+        if value is None:
+            return None
+        if self.parser.node_text(value, source).strip() != "os.environ":
+            return None
+        subscript = node.child_by_field_name("subscript")
+        key = None
+        if subscript is not None and subscript.type == "string":
+            key = self.parser.node_text(subscript, source).strip("'\"")
+        return self._make_pattern(
+            name="env_read",
+            description=f"env-var read via os.environ[{key or '?'}]",
+            node=node, file_path=file_path, confidence=0.9,
+            metadata={"function": "os.environ[]", "key": key},
+        )
+
+    def _check_module_constant(self, node: Node, source: bytes, file_path: Path) -> Optional[Pattern]:
+        # only module-level
+        p = node.parent
+        while p is not None:
+            if p.type in ("function_definition", "class_definition"):
+                return None
+            p = p.parent
+        left = node.child_by_field_name("left")
+        if left is None or left.type != "identifier":
+            return None
+        name = self.parser.node_text(left, source)
+        if not UPPER_SNAKE.match(name) or len(name) < 2:
+            return None
+        return self._make_pattern(
+            name="module_constant",
+            description=f"module-level constant: {name}",
+            node=node, file_path=file_path, confidence=0.7,
+            metadata={"name": name},
+        )
+
+    def _first_string_arg(self, call_node: Node, source: bytes) -> Optional[str]:
+        args = call_node.child_by_field_name("arguments")
+        if args is None:
+            return None
+        for c in args.children:
+            if c.type == "string":
+                return self.parser.node_text(c, source).strip("'\"")
+        return None
+
+    def _walk(self, node: Node):
+        yield node
+        for child in node.children:
+            yield from self._walk(child)