devlensio 0.6.2 → 1.0.1

package/extractors/python/devlens_extractors_python/parser/functions.py

@@ -0,0 +1,163 @@
+"""Function extraction — FUNCTION/METHOD nodes + metadata.calls + ormOps.
+
+The scope rule lives here: calls inside nested functions belong to the
+inner function, not the outer one.
+
+ORM call patterns are COLLECTED here at parse time (one AST walk) and
+RESOLVED later by edges/orm_edges.py via lookup maps — no re-parsing.
+"""
+from __future__ import annotations
+import ast
+
+from ..contract import code_node
+
+NESTED_SCOPES = (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef, ast.Lambda)
+
+# ── ORM patterns (parse-time facts; resolution lives in edges/orm_edges.py) ──
+DJANGO_QUERY_METHODS = {
+    "get", "filter", "all", "exclude", "first", "last", "count", "exists",
+    "values", "values_list", "only", "defer", "select_related",
+    "prefetch_related", "aggregate", "annotate", "distinct", "order_by",
+    "reverse", "raw", "iterator", "latest", "earliest", "in_bulk",
+}
+DJANGO_WRITE_METHODS = {
+    "create", "update", "get_or_create", "update_or_create",
+    "bulk_create", "bulk_update", "delete",
+}
+SQLA_STATEMENT_FUNCS = {"select", "insert", "update", "delete"}
+SESSION_MUTATORS = {"add", "add_all", "merge", "delete"}   # session.add(obj) etc.
+INSTANCE_MUTATORS = {"save", "delete"}                     # obj.save() / obj.delete()
+
+
+def _walk_scope(body: list[ast.stmt], visit) -> None:
+    """Walk statements, applying the scope rule (never descend into nested
+    FunctionDef/ClassDef/Lambda). Shared by extract_calls + extract_orm_ops."""
+    def walk(node: ast.AST) -> None:
+        if isinstance(node, NESTED_SCOPES):
+            return
+        visit(node)
+        for child in ast.iter_child_nodes(node):
+            walk(child)
+    for stmt in body:
+        walk(stmt)
+
+
+def call_name(func: ast.AST) -> str | None:
+    """'foo()' → 'foo' · 'obj.method()' → 'obj.method' · 'requests.get()' → 'requests.get'.
+    Subscript callees (foo[0]()) and other exotic forms → None."""
+    if isinstance(func, ast.Name):
+        return func.id
+    if isinstance(func, ast.Attribute):
+        parts: list[str] = []
+        node = func
+        while isinstance(node, ast.Attribute):
+            parts.append(node.attr)
+            node = node.value
+        if isinstance(node, ast.Name):
+            parts.append(node.id)
+            return ".".join(reversed(parts))
+    return None
+
+
+def extract_calls(body: list[ast.stmt]) -> list[str]:
+    """Collect every called name in a function body, deduped."""
+    calls: list[str] = []
+
+    def visit(node: ast.AST) -> None:
+        if isinstance(node, ast.Call):
+            name = call_name(node.func)
+            if name:
+                calls.append(name)
+
+    _walk_scope(body, visit)
+    return list(dict.fromkeys(calls))   # dedupe, keep first-seen order
+
+
+def _match_orm_call(call: ast.Call) -> dict | None:
+    """One Call → ORM op dict {kind, pattern, target?, arg?, root?} or None.
+
+    Patterns:
+      X.objects.<query|write>(...)      django_objects   (target = X)
+      var.query(X)                      sqlalchemy_query (target = X)
+      var.get(X, pk)                    session_get      (target = X)
+      var.add/delete/merge(X)           session_mutate   (arg = instance var)
+      inst.save()/delete()              instance_mutate  (arg = var, heuristic)
+      select/insert/update/delete(X)    sqlalchemy_<fn>  (root verified at resolve)
+    """
+    func = call.func
+
+    # X.objects.<method>(...) — Django ORM queryset/manager
+    if isinstance(func, ast.Attribute) and isinstance(func.value, ast.Attribute) \
+            and func.value.attr == "objects" and isinstance(func.value.value, ast.Name):
+        m = func.attr
+        if m in DJANGO_QUERY_METHODS:
+            return {"kind": "read", "pattern": "django_objects", "target": func.value.value.id}
+        if m in DJANGO_WRITE_METHODS:
+            return {"kind": "write", "pattern": "django_objects", "target": func.value.value.id}
+        return None
+
+    # var.<attr>(...) — session / instance patterns
+    if isinstance(func, ast.Attribute) and isinstance(func.value, ast.Name):
+        var, m = func.value.id, func.attr
+        if m == "query" and call.args and isinstance(call.args[0], ast.Name):
+            return {"kind": "read", "pattern": "sqlalchemy_query", "target": call.args[0].id}
+        if m == "get" and len(call.args) >= 2 and isinstance(call.args[0], ast.Name):
+            return {"kind": "read", "pattern": "session_get", "target": call.args[0].id}
+        if m in SESSION_MUTATORS and call.args and isinstance(call.args[0], ast.Name):
+            return {"kind": "write", "pattern": "session_mutate", "arg": call.args[0].id}
+        if m in INSTANCE_MUTATORS and not call.args:
+            return {"kind": "write", "pattern": "instance_mutate", "arg": var}
+        return None
+
+    # select(X) / insert(X) / update(X) / delete(X) — SQLAlchemy 2.0 statements
+    if isinstance(func, ast.Name) and func.id in SQLA_STATEMENT_FUNCS \
+            and call.args and isinstance(call.args[0], ast.Name):
+        kind = "read" if func.id == "select" else "write"
+        return {"kind": kind, "pattern": f"sqlalchemy_{func.id}",
+                "target": call.args[0].id, "root": func.id}
+    return None
+
+
+def extract_orm_ops(body: list[ast.stmt]) -> list[dict]:
+    """Collect ORM access patterns in a function body (scope-guarded)."""
+    ops: list[dict] = []
+
+    def visit(node: ast.AST) -> None:
+        if isinstance(node, ast.Call):
+            op = _match_orm_call(node)
+            if op:
+                ops.append(op)
+
+    _walk_scope(body, visit)
+    return ops
+
+
+def extract_function(node: ast.FunctionDef | ast.AsyncFunctionDef, rel_path: str,
+                     source: str, parent_class: str | None = None) -> dict:
+    is_async = isinstance(node, ast.AsyncFunctionDef)
+    name = f"{parent_class}.{node.name}" if parent_class else node.name
+
+    raw = ast.get_source_segment(source, node) or ""
+    params = [a.arg for a in node.args.posonlyargs + node.args.args + node.args.kwonlyargs]
+    decorators = [ast.unparse(d) for d in node.decorator_list]
+
+    # hasErrorHandling/throws use ast.walk deliberately — mirrors the JS
+    # extractor, which scans the whole function subtree for try/raise.
+    metadata = {
+        "params": params,
+        "calls": extract_calls(node.body),
+        "ormOps": extract_orm_ops(node.body),
+        "isAsync": is_async,
+        "hasErrorHandling": any(isinstance(n, (ast.Try, ast.TryStar)) for n in ast.walk(node)),
+        "throws": any(isinstance(n, ast.Raise) for n in ast.walk(node)),
+        "lineCount": (node.end_lineno or node.lineno) - node.lineno,
+        "decorators": decorators,
+    }
+    if parent_class:
+        metadata["parentClass"] = parent_class
+
+    return code_node(
+        rel_path, name,
+        "METHOD" if parent_class else "FUNCTION",
+        node.lineno, node.end_lineno or node.lineno, raw, metadata,
+    )

package/extractors/python/devlens_extractors_python/parser/walker.py

@@ -0,0 +1,30 @@
+import os
+import re
+from pathlib import Path
+
+# Directories to Ingore
+IGNORE_DIRS = {
+     ".git", "__pycache__", ".venv", "venv", "env", "node_modules",
+    "dist", "build", ".next", "coverage", ".mypy_cache", ".pytest_cache",
+    ".tox", ".idea", ".vscode", ".eggs", "site-packages", "migrations",
+}
+
+_TEST_FILE_RE = re.compile(r"(?:^|/)(?:test_[^/]+|[^/]+_test)\.py$")
+
+
+def walk_python_files(repo_path: str):
+    """Yield relative paths of every .py file, pruning ignored dirs.
+
+    dirnames[:] = [...] is the in-place prune that makes os.walk skip
+    those directories entirely (same role as the JS parser's IGNORE_DIRS)."""
+    root = Path(repo_path)
+    for dirpath, dirnames, filenames in os.walk(root):
+        dirnames[:] = [d for d in dirnames if d not in IGNORE_DIRS]
+        for fn in sorted(filenames):
+            if fn.endswith(".py"):
+                rel = Path(dirpath).relative_to(root).as_posix()
+                yield f"{rel}/{fn}" if rel != "." else fn
+
+def is_test_file(rel_path: str) -> bool:
+    """test_foo.py or foo_test.py, at any depth."""
+    return bool(_TEST_FILE_RE.search(rel_path))

package/extractors/python/devlens_extractors_python/third_party.py

@@ -0,0 +1,103 @@
+"""Third-party node creation — [pip]/pkg and [pip]/pkg::name nodes.
+
+Mirrors the JS side's [npm]/... convention (src/graph/thirdPartyLibs.ts +
+importEdges.ts): one package node per import, per-name method nodes for
+named imports, all deduped in a single registry.
+"""
+from __future__ import annotations
+
+import sys
+
+# Exact stdlib module set from the running interpreter (Python 3.10+).
+STDLIB_MODULES = frozenset(sys.stdlib_module_names)
+
+# Runtime libraries — real architectural decisions (mirrors RUNTIME_INCLUSION_SET)
+RUNTIME_PACKAGES = {
+    "fastapi", "flask", "django", "starlette", "uvicorn", "gunicorn",
+    "sqlalchemy", "pydantic", "alembic", "psycopg2", "psycopg", "asyncpg",
+    "pymongo", "motor", "redis", "celery", "requests", "httpx", "aiohttp",
+    "numpy", "pandas", "scipy", "scikit-learn", "tensorflow", "torch",
+    "pyjwt", "passlib", "bcrypt", "cryptography", "boto3", "stripe",
+    "openai", "anthropic", "langchain", "sentry-sdk", "loguru",
+    "beautifulsoup4", "scrapy", "pillow", "opencv-python", "matplotlib",
+}
+
+# Dev tooling — low signal (mirrors the JS devtool category)
+DEVTOOL_PACKAGES = {
+    "pytest", "black", "flake8", "mypy", "ruff", "isort", "coverage",
+    "tox", "nox", "pre-commit", "sphinx", "mkdocs", "hypothesis",
+    "responses", "vcrpy", "faker", "factory-boy", "freezegun",
+}
+
+class ThirdPartyRegistry:
+    """Creates and caches [pip]/... nodes. One registry per extraction —
+    the same package imported in 20 files yields ONE node.
+
+    `allowed` gates third-party inclusion: only packages in the set get
+    nodes. Empty set → NO third-party nodes at all (the engine controls
+    this via options.includedThirdPartyLibs)."""
+
+    def __init__(self, raw_dependencies: dict[str, str],
+                 allowed: set[str] | None = None):
+        self._nodes: dict[str, dict] = {}
+        self.raw_dependencies = raw_dependencies
+        self.allowed = allowed if allowed is not None else set()
+
+    def _permitted(self, pkg: str) -> bool:
+        return pkg in self.allowed
+
+    def _base_metadata(self, pkg: str) -> dict:
+        version = self.raw_dependencies.get(pkg, "unknown")
+        if pkg in RUNTIME_PACKAGES:
+            category = "runtime"
+        elif pkg in DEVTOOL_PACKAGES:
+            category = "devtool"
+        else:
+            category = "unknown"
+        return {"isThirdParty": True, "packageVersion": version, "category": category}
+
+    def package_node(self, pkg: str) -> dict | None:
+        """[pip]/pkg node, or None when the package is not permitted."""
+        if not self._permitted(pkg):
+            return None
+        node_id = f"[pip]/{pkg}"
+        if node_id not in self._nodes:
+            self._nodes[node_id] = {
+                "id": node_id,
+                "name": pkg,
+                "type": "THIRD_PARTY",
+                "filePath": node_id,
+                "startLine": 0,
+                "endLine": 0,
+                "metadata": self._base_metadata(pkg),
+            }
+        return self._nodes[node_id]
+
+    def method_node(self, pkg: str, name: str) -> dict | None:
+        """Named import member: from flask import Flask → [pip]/flask::Flask.
+        None when the package is not permitted."""
+        if not self._permitted(pkg):
+            return None
+        node_id = f"[pip]/{pkg}::{name}"
+        if node_id not in self._nodes:
+            pkg_node = self.package_node(pkg)
+            if pkg_node is None:
+                return None
+            self._nodes[node_id] = {
+                "id": node_id,
+                "name": f"{pkg}.{name}",
+                "type": "THIRD_PARTY",
+                "filePath": f"[pip]/{pkg}",
+                "startLine": 0,
+                "endLine": 0,
+                "metadata": {
+                    **pkg_node["metadata"],
+                    "parentPackageId": pkg_node["id"],
+                    "methodName": name,
+                },
+            }
+        return self._nodes[node_id]
+
+    @property
+    def nodes(self) -> list[dict]:
+        return list(self._nodes.values())

package/extractors/python/pyproject.toml

@@ -0,0 +1,16 @@
+[build-system]
+requires = ["setuptools>=68"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "devlens-extractors-python"
+version = "0.1.0"
+description = "DevLens Python extractor — AST-based code analysis producing the DevLens JSON contract"
+requires-python = ">=3.11"
+dependencies = []
+
+[project.optional-dependencies]
+dev = ["pytest>=7"]
+
+[tool.setuptools.packages.find]
+include = ["devlens_extractors_python*"]

package/extractors/python/setup.mjs

@@ -0,0 +1,54 @@
+#!/usr/bin/env node
+// DevLens Python extractor — venv bootstrap (cross-platform: Linux/macOS/Windows).
+//
+// Runs on `npm install` (postinstall) on the INSTALLING machine: a venv is
+// machine-specific (absolute paths, platform binaries) so it can never live
+// in the npm tarball — it must be created here. Node is guaranteed present
+// (it's an npm package), so this works everywhere bash isn't.
+//
+// Idempotent + non-fatal:
+//   - python3 missing          → warn, exit 0 (Python analysis unavailable)
+//   - .venv already present    → skip
+//   - otherwise                → create venv + editable-install the package
+
+import { existsSync, mkdirSync } from "node:fs";
+import { dirname, join } from "node:path";
+import { fileURLToPath } from "node:url";
+import { spawnSync } from "node:child_process";
+
+const HERE = dirname(fileURLToPath(import.meta.url));
+const IS_WIN = process.platform === "win32";
+const VENV_PYTHON = join(HERE, ".venv", IS_WIN ? "Scripts/python.exe" : "bin/python");
+
+function which(cmd) {
+  const r = spawnSync(IS_WIN ? "where" : "which", [cmd], { stdio: "pipe" });
+  return r.status === 0;
+}
+
+// python3 on Windows is usually `python` (the Store alias is python3 too,
+// but `python` is the safe bet on CI images).
+const pythonCmd = which("python3") ? "python3" : which("python") ? "python" : null;
+
+if (!pythonCmd) {
+  console.error("devlens: python3 not found — Python extraction unavailable (install Python 3.11+)");
+  process.exit(0);
+}
+
+if (existsSync(VENV_PYTHON)) {
+  console.error("devlens: python extractor venv already present — skipping");
+  process.exit(0);
+}
+
+console.error(`devlens: creating python extractor venv at ${join(HERE, ".venv")}`);
+mkdirSync(join(HERE, ".venv"), { recursive: true });
+let r = spawnSync(pythonCmd, ["-m", "venv", ".venv"], { cwd: HERE, stdio: "inherit" });
+if (r.status !== 0) {
+  console.error("devlens: `python -m venv` failed — Python extraction unavailable");
+  process.exit(0);   // non-fatal: don't break npm install
+}
+r = spawnSync(VENV_PYTHON, ["-m", "pip", "install", "--quiet", "-e", "."], { cwd: HERE, stdio: "inherit" });
+if (r.status !== 0) {
+  console.error("devlens: pip install failed — Python extraction unavailable");
+  process.exit(0);
+}
+console.error("devlens: python extractor ready");

package/extractors/rust/Cargo.toml

@@ -0,0 +1,29 @@
+[package]
+name = "devlens_rust_extractor"
+version = "0.1.0"
+edition = "2021"
+description = "DevLens Rust codebase graph extractor (JSON over stdin/stdout)"
+license = "MIT"
+
+# 100% pure-Rust dependency tree — no C code. This is what makes the
+# no-sudo cross-compile matrix possible (rust-lld links musl/windows-gnu,
+# zig links darwin). Do NOT add any crate with C/cc dependencies.
+[dependencies]
+syn = { version = "2", features = ["full", "visit", "extra-traits"] }
+proc-macro2 = { version = "1", features = ["span-locations"] }
+quote = "1"
+serde = { version = "1", features = ["derive"] }
+serde_json = "1"
+toml = "0.8"
+sha2 = "0.10"
+
+# Static-binary hygiene: panic=abort avoids unwinding runtime bloat; strip
+# symbols in release. lto keeps the binary lean.
+[profile.release]
+panic = "abort"
+lto = true
+codegen-units = 1
+strip = "symbols"
+
+[profile.release.package."*"]
+opt-level = 3

package/extractors/rust/build.mjs

@@ -0,0 +1,85 @@
+#!/usr/bin/env node
+// DevLens Rust extractor — cross-compiled static binaries (Node script, so it
+// runs on Windows too; never bash).
+//
+// The Rust extractor ships as per-platform static binaries inside the
+// devlensio tarball (files whitelist). Node is guaranteed on every machine
+// that installs an npm package; the Rust toolchain is a build-time dep (same
+// class as JDK for java / Go toolchain for go).
+//
+// Cross-compile matrix (all sudo-free — the dep tree is 100% pure Rust):
+//   linux-amd64/arm64 → musl targets linked by rust-lld: true static
+//                       binaries (no glibc), zero runtime deps — matches
+//                       Go's CGO_ENABLED=0 claim
+//   windows-amd64      → x86_64-pc-windows-gnu via cargo-zigbuild: zig
+//                       bundles the mingw-w64 import libs (rust-lld alone
+//                       can't see kernel32.lib etc. without a system mingw)
+//   darwin-amd64/arm64 → cargo-zigbuild: zig is the Mach-O linker, no macOS
+//                       SDK needed for pure-Rust crates
+//
+// Prereqs on the publishing machine: rustup (stable + targets added),
+// cargo-zigbuild, and a zig binary (found via env ZIG, ~/zig/zig-*/,
+// or PATH).
+//
+// Usage: node build.mjs   (idempotent — run from this directory)
+
+import { execFileSync } from "node:child_process";
+import { existsSync, mkdirSync, copyFileSync, readdirSync, rmSync } from "node:fs";
+import { dirname, join } from "node:path";
+import { fileURLToPath } from "node:url";
+import { homedir } from "node:os";
+
+const HERE = dirname(fileURLToPath(import.meta.url));
+
+// [platform dir, cargo target triple, build kind]
+const TARGETS = [
+  ["linux-amd64", "x86_64-unknown-linux-musl", "rust-lld"],
+  ["linux-arm64", "aarch64-unknown-linux-musl", "rust-lld"],
+  ["windows-amd64", "x86_64-pc-windows-gnu", "zig"],
+  ["darwin-amd64", "x86_64-apple-darwin", "zig"],
+  ["darwin-arm64", "aarch64-apple-darwin", "zig"],
+];
+
+const cargo =
+  process.env.CARGO ||
+  (existsSync(join(homedir(), ".cargo", "bin", "cargo"))
+    ? join(homedir(), ".cargo", "bin", "cargo")
+    : "cargo");
+
+function findZig() {
+  if (process.env.ZIG && existsSync(process.env.ZIG)) return process.env.ZIG;
+  const zigDir = join(homedir(), "zig");
+  if (existsSync(zigDir)) {
+    for (const entry of readdirSync(zigDir).sort().reverse()) {
+      const cand = join(zigDir, entry, "zig");
+      if (existsSync(cand)) return cand;
+    }
+  }
+  return "zig"; // fall back to PATH
+}
+
+function run(cmd, args, opts = {}) {
+  execFileSync(cmd, args, { stdio: "inherit", ...opts });
+}
+
+const zig = findZig();
+for (const [platformDir, triple, kind] of TARGETS) {
+  const outDir = join(HERE, "bin", platformDir);
+  mkdirSync(outDir, { recursive: true });
+  const exe = platformDir.startsWith("windows") ? "devlens_rust_extractor.exe" : "devlens_rust_extractor";
+  console.log(`==> building ${platformDir} (${triple}, ${kind})`);
+  const env = { ...process.env };
+  let args;
+  if (kind === "rust-lld") {
+    // plain cargo build: rust-lld is the linker for pure-Rust musl targets
+    env.RUSTFLAGS = "-C linker=rust-lld";
+    args = ["build", "--target", triple, "--release"];
+  } else {
+    // zigbuild: zig must be on PATH (it spawns `zig`)
+    env.PATH = `${dirname(zig)}${process.platform === "win32" ? ";" : ":"}${env.PATH}`;
+    args = ["zigbuild", "--target", triple, "--release"];
+  }
+  run(cargo, args, { cwd: HERE, env });
+  copyFileSync(join(HERE, "target", triple, "release", exe), join(outDir, exe));
+}
+console.log("==> Done: extractors/rust/bin/<platform>/devlens_rust_extractor[.exe]");