devlensio 0.6.1 → 1.0.0

package/extractors/rust/src/extractor.rs

@@ -0,0 +1,199 @@
+// extractor.rs — pipeline orchestration (skeleton; modules wired in as they
+// land — order mirrors go/extractor.go):
+//
+//   walk → parse (facts) → module_map → lookup → imports (+[crate] nodes) →
+//   calls → routes → orm → inheritance → tests → enrich →
+//   dedupe → deterministic sort → result.
+//
+// Edge detectors resolve from facts via lookup maps — never re-walk ASTs
+// (playbook rule).
+
+use crate::contract::{ExtractorError, ExtractorResult, Stats};
+use crate::fingerprint;
+use crate::parser::ParsedFile;
+use serde_json::Value;
+
+/// All parsed facts for the repo — the single source consumed by every edge
+/// detector (mirrors go's parsedRepo).
+pub struct ParsedRepo {
+    pub files: Vec<ParsedFile>,
+    pub skipped: i64,
+}
+
+/// Options — the engine forwards includeThirdPartyLibs (both spellings).
+#[derive(Default, Clone)]
+pub struct Options {
+    /// Third-party crates the user selected in the frontend. Absent/empty →
+    /// ZERO third-party nodes (gate, not default-on).
+    pub include_third_party_libs: Vec<String>,
+}
+
+impl Options {
+    pub fn from_json(options: &Value) -> Self {
+        let mut o = Options::default();
+        for key in ["includeThirdPartyLibs", "includedThirdPartyLibs"] {
+            if let Some(v) = options.get(key).and_then(|v| v.as_array()) {
+                for item in v {
+                    if let Some(s) = item.as_str() {
+                        o.include_third_party_libs.push(s.to_string());
+                    }
+                }
+            }
+            if !o.include_third_party_libs.is_empty() {
+                break;
+            }
+        }
+        o
+    }
+}
+
+pub fn run(repo_path: &str, opts: &Options) -> ExtractorResult {
+    let _ = opts;
+
+    let mut errors: Vec<ExtractorError> = vec![];
+
+    // ── fingerprint (Cargo.toml only; no stdlib web framework in Rust, so no
+    // post-walk upgrades like Go's net-http) ──
+    let mf = fingerprint::parse_cargo_toml(repo_path);
+    let fingerprint = fingerprint::fingerprint_from_manifest(&mf, repo_path);
+
+    // ── walk + parse (facts; non-fatal per-file errors) ──
+    let mut parsed = ParsedRepo {
+        files: vec![],
+        skipped: 0,
+    };
+    for rel in crate::walker::collect_rs_files(repo_path) {
+        let abs = format!("{}/{}", repo_path, rel);
+        match std::fs::read_to_string(&abs) {
+            Ok(source) => match crate::parser::parse_file(&rel, &source) {
+                Ok(pf) => parsed.files.push(pf),
+                Err(e) => {
+                    parsed.skipped += 1;
+                    errors.push(ExtractorError {
+                        file: rel.clone(),
+                        error: format!("parse error: {}", e),
+                    });
+                }
+            },
+            Err(e) => {
+                parsed.skipped += 1;
+                errors.push(ExtractorError {
+                    file: rel.clone(),
+                    error: format!("read error: {}", e),
+                });
+            }
+        }
+    }
+
+    // ── module map + lookup (ONE shared index) ──
+    let module_map = crate::module_map::ModuleMap::build(&parsed.files);
+
+    // file + code nodes (test files = leaf: testCases only)
+    let file_nodes = crate::nodes::collect_file_nodes(&parsed.files);
+    let mut all_nodes: Vec<crate::contract::CodeNode> = file_nodes;
+    all_nodes.extend(crate::nodes::collect_code_nodes(&parsed.files));
+
+    let mut lookup = crate::lookup::build_lookup_maps(&parsed.files, &all_nodes);
+
+    // third-party registry — gate from options (absent/empty → ZERO [crate])
+    let mut tp = crate::thirdparty::ThirdPartyRegistry::new(&opts.include_third_party_libs);
+
+    // ── edges, in dependency order ──
+    let mut all_edges: Vec<crate::contract::CodeEdge> = vec![];
+
+    // IMPORTS + [crate] nodes
+    let imports = crate::imports::detect_imports(&parsed, &mf, &module_map, &mut tp, opts);
+    all_edges.extend(imports.edges);
+
+    // CALLS
+    // detect_calls writes metadata.resolvedCalls back into lookup.node_by_id
+    // (it clones→mutates→reinserts). all_nodes was built BEFORE that, so we
+    // must sync the resolvedCalls forward — else the output nodes carry calls
+    // (parser-time) but never resolvedCalls, and the CALLS parity census reads 0.
+    let calls = crate::calls::detect_calls(&parsed, &module_map, &mut lookup, &mf, &mut tp, opts);
+    all_edges.extend(calls.edges);
+    for n in &mut all_nodes {
+        if let Some(reg) = lookup.node_by_id.get(&n.id) {
+            if let Some(rc) = reg.metadata.get("resolvedCalls") {
+                n.metadata.insert("resolvedCalls".to_string(), rc.clone());
+            }
+        }
+    }
+
+    // ROUTE nodes + HANDLES edges + BackendRouteNodes
+    let routes_out = crate::routes::detect_routes(
+        &parsed,
+        &module_map,
+        &lookup,
+        &fingerprint.framework,
+        opts,
+        &mf,
+        &mut tp,
+    );
+    all_nodes.extend(routes_out.route_nodes);
+    all_edges.extend(routes_out.handles_edges);
+
+    // EXTENDS / IMPLEMENTS
+    let inheritance =
+        crate::inheritance::detect_inheritance(&parsed, &mf, &module_map, &lookup, &mut tp, opts);
+    all_edges.extend(inheritance.edges);
+    all_nodes.extend(inheritance.third_party_nodes);
+
+    // READS_FROM / WRITES_TO (Diesel)
+    all_edges.extend(crate::orm_edges::detect_orm_edges(&parsed, &lookup, opts));
+
+    // TESTS edges
+    all_edges.extend(crate::tests::detect_tests(
+        &parsed,
+        &module_map,
+        &lookup,
+        opts,
+    ));
+
+    // third-party nodes — the registry is the SINGLE source (imports/calls
+    // registered into it; collected last, go parity)
+    all_nodes.extend(tp.all_nodes());
+
+    // ── dedupe + deterministic sort ──
+    let edges = dedupe_and_sort_edges(all_edges);
+    all_nodes.sort_by(|a, b| a.id.cmp(&b.id));
+    let mut routes = routes_out.backend_routes;
+    routes.sort_by(|a, b| {
+        (&a.http_method, &a.url_path, &a.node_id).cmp(&(&b.http_method, &b.url_path, &b.node_id))
+    });
+
+    // semantic enrichment (isModel/isHandler) — needs the deduped edges
+    crate::enrich::enrich_nodes(&parsed, &mut all_nodes, &edges);
+    all_nodes.sort_by(|a, b| a.id.cmp(&b.id));
+
+    let total_nodes = all_nodes.len() as i64;
+    ExtractorResult {
+        fingerprint,
+        nodes: all_nodes,
+        edges,
+        routes,
+        stats: Stats {
+            total_files: parsed.files.len() as i64,
+            total_nodes,
+            skipped_files: parsed.skipped,
+        },
+        errors,
+    }
+}
+
+/// Dedupe (from, type, to) then sort by (from, type, to) — deterministic.
+fn dedupe_and_sort_edges(edges: Vec<crate::contract::CodeEdge>) -> Vec<crate::contract::CodeEdge> {
+    let mut seen: std::collections::HashSet<(String, String, String)> =
+        std::collections::HashSet::new();
+    let mut out = vec![];
+    for e in edges {
+        let key = (e.from.clone(), e.edge_type.clone(), e.to.clone());
+        if seen.contains(&key) {
+            continue;
+        }
+        seen.insert(key);
+        out.push(e);
+    }
+    out.sort_by(|a, b| (&a.from, &a.edge_type, &a.to).cmp(&(&b.from, &b.edge_type, &b.to)));
+    out
+}

package/extractors/rust/src/fingerprint.rs

@@ -0,0 +1,238 @@
+// fingerprint.rs — Cargo.toml parsing → framework / projectType / databases /
+// rawDependencies. Manifest is PARSED, never executed (security rule, same as
+// setup.py/pom.xml/go.mod handling in the other phases).
+//
+// Workspace-aware: [workspace] members' Cargo.tomls and the root
+// [workspace.dependencies] are unioned into one dependency view.
+
+use crate::contract::Fingerprint;
+use std::collections::BTreeMap;
+use std::path::Path;
+
+/// Parsed manifest facts — consumed by fingerprint AND import resolution
+/// (dependency names = the external-crate set for [crate]/ gating).
+pub struct ParsedManifest {
+    /// union of [dependencies] + [dev-dependencies] + [workspace.dependencies]
+    /// (name → version; version may be empty for path/git deps)
+    pub dependencies: BTreeMap<String, String>,
+    /// member crate directories (relative, from [workspace].members globs)
+    pub workspace_members: Vec<String>,
+}
+
+impl ParsedManifest {
+    pub fn is_external_crate(&self, name: &str) -> bool {
+        // strip a leading crate-name prefix (e.g. `axum::extract` → `axum`)
+        let root = name.split("::").next().unwrap_or(name);
+        if self.dependencies.contains_key(root) {
+            return true;
+        }
+        // Cargo normalization: the `actix-web` dependency provides the
+        // `actix_web` crate (hyphens → underscores in source)
+        let underscored = root.replace('-', "_");
+        if underscored != root && self.dependencies.contains_key(&underscored) {
+            return true;
+        }
+        self.dependencies
+            .keys()
+            .any(|k| k.replace('-', "_") == underscored)
+    }
+}
+
+/// Parse Cargo.toml at repoPath (if present). Never fails hard — a missing or
+/// malformed manifest yields an empty ParsedManifest (fingerprint degrades to
+/// unknown, documented non-fatal).
+pub fn parse_cargo_toml(repo_path: &str) -> ParsedManifest {
+    let mut mf = ParsedManifest {
+        dependencies: BTreeMap::new(),
+        workspace_members: vec![],
+    };
+    let root_toml = Path::new(repo_path).join("Cargo.toml");
+    if !root_toml.exists() {
+        return mf;
+    }
+    let text = match std::fs::read_to_string(&root_toml) {
+        Ok(t) => t,
+        Err(_) => return mf,
+    };
+    let root: toml::Value = match toml::from_str(&text) {
+        Ok(v) => v,
+        Err(_) => return mf,
+    };
+
+    // root package deps
+    collect_deps(&root, &mut mf.dependencies);
+
+    // workspace: root [workspace.dependencies] + member crates
+    let mut member_dirs: Vec<String> = vec![];
+    if let Some(ws) = root.get("workspace") {
+        if let Some(wd) = ws.get("dependencies").and_then(|d| d.as_table()) {
+            for (name, spec) in wd {
+                insert_dep(&mut mf.dependencies, name, spec);
+            }
+        }
+        if let Some(members) = ws.get("members").and_then(|m| m.as_array()) {
+            for m in members {
+                if let Some(s) = m.as_str() {
+                    expand_members(s, repo_path, &mut member_dirs);
+                }
+            }
+        }
+    }
+    mf.workspace_members = member_dirs;
+
+    // member crates' own [dependencies]
+    for dir in &mf.workspace_members {
+        let member_toml = Path::new(repo_path).join(dir).join("Cargo.toml");
+        if let Ok(t) = std::fs::read_to_string(&member_toml) {
+            if let Ok(v) = toml::from_str::<toml::Value>(&t) {
+                collect_deps(&v, &mut mf.dependencies);
+            }
+        }
+    }
+    mf
+}
+
+fn collect_deps(v: &toml::Value, out: &mut BTreeMap<String, String>) {
+    for section in ["dependencies", "dev-dependencies"] {
+        if let Some(t) = v.get(section).and_then(|d| d.as_table()) {
+            for (name, spec) in t {
+                insert_dep(out, name, spec);
+            }
+        }
+    }
+}
+
+/// dep spec forms: `foo = "1.0"` (string), `foo = { version = "1.0", ... }`
+/// (inline table), `[dependencies.foo]` (table header) — all land as
+/// Value::String or Value::Table in the toml crate.
+fn insert_dep(out: &mut BTreeMap<String, String>, name: &str, spec: &toml::Value) {
+    match spec {
+        toml::Value::String(s) => {
+            out.insert(name.to_string(), s.clone());
+        }
+        toml::Value::Table(t) => {
+            let ver = t
+                .get("version")
+                .and_then(|v| v.as_str())
+                .unwrap_or("")
+                .to_string();
+            out.insert(name.to_string(), ver);
+        }
+        toml::Value::Array(a) => {
+            // multi-spec form: `foo = ["1.0", { ... }]` — take first string
+            for item in a {
+                if let toml::Value::String(s) = item {
+                    out.insert(name.to_string(), s.clone());
+                    return;
+                }
+            }
+            out.insert(name.to_string(), String::new());
+        }
+        _ => {
+            out.insert(name.to_string(), String::new());
+        }
+    }
+}
+
+/// `members = ["crates/*", "core", "libs/foo"]` — expand `*` against the
+/// filesystem (a glob segment matches exactly one directory level).
+fn expand_members(glob: &str, repo_path: &str, out: &mut Vec<String>) {
+    let mut segments: Vec<&str> = glob.split('/').collect();
+    let mut current_dirs: Vec<String> = vec![String::new()];
+    while let Some(seg) = segments.first() {
+        let seg = *seg;
+        segments.remove(0);
+        let mut next: Vec<String> = vec![];
+        for base in &current_dirs {
+            if seg == "*" {
+                let dir = Path::new(repo_path).join(base);
+                if let Ok(entries) = std::fs::read_dir(&dir) {
+                    for e in entries.flatten() {
+                        let name = e.file_name().to_string_lossy().to_string();
+                        if name.starts_with('.') {
+                            continue;
+                        }
+                        if e.file_type().map(|t| t.is_dir()).unwrap_or(false) {
+                            next.push(if base.is_empty() {
+                                name
+                            } else {
+                                format!("{}/{}", base, name)
+                            });
+                        }
+                    }
+                }
+            } else {
+                next.push(if base.is_empty() {
+                    seg.to_string()
+                } else {
+                    format!("{}/{}", base, seg)
+                });
+            }
+        }
+        current_dirs = next;
+    }
+    for d in current_dirs {
+        // only actual crates (has Cargo.toml)
+        if Path::new(repo_path).join(&d).join("Cargo.toml").exists() {
+            out.push(d);
+        }
+    }
+}
+
+// ── framework / database / projectType detection ──
+
+const FRAMEWORKS: [(&str, &str); 5] = [
+    ("axum", "axum"),
+    ("actix-web", "actix-web"),
+    ("rocket", "rocket"),
+    ("warp", "warp"),
+    ("poem", "poem"),
+];
+
+const DATABASES: [(&str, &str); 8] = [
+    ("diesel", "diesel"),
+    ("sqlx", "sqlx"),
+    ("postgres", "postgresql"),
+    ("tokio-postgres", "postgresql"),
+    ("rusqlite", "sqlite"),
+    ("sqlite", "sqlite"),
+    ("mysql", "mysql"),
+    ("mongodb", "mongo"),
+];
+
+pub fn fingerprint_from_manifest(mf: &ParsedManifest, repo_path: &str) -> Fingerprint {
+    let mut fp = Fingerprint::new();
+
+    for (crate_name, fw) in FRAMEWORKS {
+        if mf.is_external_crate(crate_name) {
+            fp.framework = fw.to_string();
+            fp.router = fw.to_string();
+            fp.project_type = "backend".to_string();
+            break;
+        }
+    }
+
+    let mut dbs: Vec<String> = vec![];
+    for (crate_name, db) in DATABASES {
+        if mf.is_external_crate(crate_name) && !dbs.contains(&db.to_string()) {
+            dbs.push(db.to_string());
+        }
+    }
+    fp.databases = dbs;
+
+    // projectType: main.rs → backend (bin crate); lib.rs only → library.
+    if fp.project_type == "unknown" {
+        let has_main = Path::new(repo_path).join("src/main.rs").exists();
+        let has_lib = Path::new(repo_path).join("src/lib.rs").exists();
+        fp.project_type = if has_main {
+            "backend".to_string()
+        } else if has_lib {
+            "library".to_string()
+        } else {
+            "unknown".to_string()
+        };
+    }
+
+    fp.raw_dependencies = mf.dependencies.clone();
+    fp
+}

package/extractors/rust/src/imports.rs

@@ -0,0 +1,101 @@
+// imports.rs — IMPORTS edges from `use` declarations + [crate] third-party
+// nodes (mirrors go/imports.go).
+//
+// Resolution (documented Rust semantics — the module tree, not a sys.path
+// simulation):
+//   crate::a::b  → absolute;  super::/self::  → relative;  bare → relative
+//   then crate root. Longest module prefix wins; the remainder is an item.
+// Targets are FILES (Python model — no MODULE nodes in V1):
+//   `use crate::models::user::User;` → IMPORTS edge to src/models/user.rs
+// External first segments (a Cargo.toml dep, not std, not local) →
+//   gated [crate]/name[::member] node + IMPORTS edge to it. Nodes are
+//   registered into the shared ThirdPartyRegistry (the pipeline collects
+//   tp.all_nodes() last — go parity); this module only emits edges.
+
+use crate::contract::CodeEdge;
+use crate::extractor::{Options, ParsedRepo};
+use crate::fingerprint::ParsedManifest;
+use crate::module_map::ModuleMap;
+use crate::thirdparty::ThirdPartyRegistry;
+
+pub struct ImportsOutput {
+    pub edges: Vec<CodeEdge>,
+}
+
+/// Emit IMPORTS edges for every use declaration.
+pub fn detect_imports(
+    repo: &ParsedRepo,
+    mf: &ParsedManifest,
+    module_map: &ModuleMap,
+    tp: &mut ThirdPartyRegistry,
+    _opts: &Options,
+) -> ImportsOutput {
+    let mut edges = vec![];
+    let mut seen: std::collections::HashSet<(String, String)> = std::collections::HashSet::new();
+
+    for pf in &repo.files {
+        if pf.is_test_file {
+            // test files ARE leaf nodes, but they still import — emit edges
+            // from the TEST node (same file id — parentFile covers it)
+        }
+        let from = format!("file::{}", pf.rel_path);
+        let base_module = crate::walker::module_path_for_file(&pf.rel_path);
+        for u in &pf.uses {
+            // resolve the target path against the module map (use-semantics:
+            // crate-root absolute for bare paths — edition 2018)
+            if let Some((file, rest)) = module_map.resolve_use(&u.target, &base_module) {
+                // skip self-imports (`use super::*` inside a same-file mod,
+                // `use crate::foo::*` from within foo.rs) — noise edges
+                if file == pf.rel_path {
+                    continue;
+                }
+                let to = format!("file::{}", file);
+                let key = (from.clone(), to.clone());
+                if !seen.contains(&key) {
+                    seen.insert(key);
+                    edges.push(crate::contract::edge(&from, &to, "IMPORTS"));
+                }
+                // rest (an item) is refined by calls/tests via symbol maps
+                let _ = rest;
+                continue;
+            }
+            // external crate or unresolved
+            let first = crate::thirdparty::first_segment(&u.target);
+            if crate::thirdparty::is_std(first) {
+                continue; // skip tier — no node, no edge
+            }
+            if !crate::thirdparty::is_external_crate(first, mf) {
+                continue; // unresolvable (typo / not a dep) — metadata only
+            }
+            // external: [crate]/name (+ member when the use names one) —
+            // registration happens inside the registry (gated)
+            if tp.crate_node(first).is_some() {
+                push_import(
+                    &mut edges,
+                    &mut seen,
+                    &from,
+                    &crate::lookup::third_party_id(first),
+                );
+            }
+            let rest: Vec<&str> = u.target.split("::").skip(1).collect();
+            if !rest.is_empty() && !u.glob {
+                let member = rest.join("::");
+                let _ = tp.member_node(first, &member);
+            }
+        }
+    }
+    ImportsOutput { edges }
+}
+
+fn push_import(
+    edges: &mut Vec<CodeEdge>,
+    seen: &mut std::collections::HashSet<(String, String)>,
+    from: &str,
+    to: &str,
+) {
+    let key = (from.to_string(), to.to_string());
+    if !seen.contains(&key) {
+        seen.insert(key);
+        edges.push(crate::contract::edge(from, to, "IMPORTS"));
+    }
+}