devlensio 0.6.1 → 1.0.0

package/extractors/go/typeload.go

@@ -0,0 +1,178 @@
+// typeload.go — go/types type-checking, fully self-contained.
+//
+// REVISED from the plan: go/packages is NOT stdlib (it lives in
+// golang.org/x/tools) and would drag in a build-time network dep + a runtime
+// dependency on the `go` command. Instead we type-check the module's own
+// packages in-process:
+//
+//   - local packages: checked in dependency order (Go import graphs are
+//     acyclic) from the already-parsed ASTs
+//   - std packages: importer.Default() (compiled-in export data; degrades to
+//     import errors on machines without a GOROOT — harmless, errors ignored)
+//   - external packages: import error → unresolved (degraded) — calls to them
+//     resolve via name-based [mod] lazy edges anyway
+//
+// All checker errors are swallowed (we're type-RESOLVING, not verifying).
+// The result is per-file *types.Info: Selections/Uses give exact local call
+// targets, and package scopes give exact type objects for IMPLEMENTS.
+//
+// The binary stays static and offline — no `go` command needed at runtime.
+
+package main
+
+import (
+	"go/ast"
+	"go/importer"
+	"go/token"
+	"go/types"
+	"sort"
+)
+
+type TypeInfo struct {
+	Mode     string             // "typed" — informational
+	Infos    map[string]*types.Info // rel file path → type info for that file
+	Pkgs     map[string]*types.Package // import path → checked package
+	Checks   int
+	Failures int
+}
+
+// loadTypes — type-check every package in dependency order.
+func loadTypes(pr *parsedRepo, addError errFn) *TypeInfo {
+	ti := &TypeInfo{
+		Mode:  "typed",
+		Infos: map[string]*types.Info{},
+		Pkgs:  map[string]*types.Package{},
+	}
+
+	// Check units: a package's own files (non-test + same-package test files)
+	// and — for *_test.go files declaring a DIFFERENT package name
+	// (package foo_test) — a synthetic package with import path <dir>_test.
+	type unit struct {
+		path  string
+		files []*ParsedFile
+		fset  *token.FileSet
+	}
+	units := []*unit{}
+	byPath := map[string]*unit{}
+	for _, pkg := range pr.packages {
+		own := &unit{path: pkg.ImportPath, fset: pkg.fset}
+		ext := &unit{path: pkg.ImportPath + "_test", fset: pkg.fset}
+		byPath[own.path] = own
+		byPath[ext.path] = ext
+		for _, pf := range pkg.Files {
+			if pf.IsTest && pf.PkgName != pkg.Name {
+				ext.files = append(ext.files, pf)
+			} else {
+				own.files = append(own.files, pf)
+			}
+		}
+		units = append(units, own, ext)
+	}
+
+	// internal deps per unit (imports that resolve to local packages)
+	depPaths := func(u *unit) []string {
+		seen := map[string]bool{}
+		var deps []string
+		for _, pf := range u.files {
+			for _, imp := range pf.Imports {
+				if _, ok := byPath[imp.Path]; ok && !seen[imp.Path] {
+					seen[imp.Path] = true
+					deps = append(deps, imp.Path)
+				}
+			}
+		}
+		sort.Strings(deps)
+		return deps
+	}
+
+	// importer: local packages in-memory; std via importer.Default(); external
+	// → error (degraded, fine).
+	stdImp := importer.Default()
+	imp := importerFunc(func(path string) (*types.Package, error) {
+		if p, ok := ti.Pkgs[path]; ok {
+			return p, nil
+		}
+		if _, local := byPath[path]; local {
+			return nil, errNotCheckedYet
+		}
+		return stdImp.Import(path)
+	})
+
+	// topological order over the internal import graph
+	done := map[string]bool{}
+	var order []*unit
+	for len(order) < len(units) {
+		progress := false
+		for _, u := range units {
+			if done[u.path] || len(u.files) == 0 {
+				done[u.path] = true
+				continue
+			}
+			ready := true
+			for _, d := range depPaths(u) {
+				if !done[d] {
+					ready = false
+					break
+				}
+			}
+			if ready {
+				done[u.path] = true
+				order = append(order, u)
+				progress = true
+			}
+		}
+		if !progress {
+			// cycle / unresolvable internal dep — check what we can
+			for _, u := range units {
+				if !done[u.path] && len(u.files) > 0 {
+					done[u.path] = true
+					order = append(order, u)
+				}
+			}
+			break
+		}
+	}
+
+	for _, u := range order {
+		if len(u.files) == 0 {
+			continue
+		}
+		files := make([]*ast.File, 0, len(u.files))
+		for _, pf := range u.files {
+			files = append(files, pf.AST)
+		}
+		info := &types.Info{
+			Types:      map[ast.Expr]types.TypeAndValue{},
+			Uses:       map[*ast.Ident]types.Object{},
+			Selections: map[*ast.SelectorExpr]*types.Selection{},
+			Defs:       map[*ast.Ident]types.Object{},
+			Implicits:  map[ast.Node]types.Object{},
+		}
+		conf := types.Config{
+			Importer: imp,
+			Error:    func(error) {}, // swallow — resolution, not verification
+		}
+		pkg, _ := conf.Check(u.path, u.fset, files, info)
+		if pkg == nil {
+			ti.Failures++
+			continue
+		}
+		ti.Pkgs[u.path] = pkg
+		ti.Checks++
+		for _, pf := range u.files {
+			ti.Infos[pf.RelPath] = info
+		}
+	}
+	return ti
+}
+
+var errNotCheckedYet = &errNotChecked{}
+
+type errNotChecked struct{}
+
+func (*errNotChecked) Error() string { return "package not checked yet" }
+
+// importerFunc adapts a function to types.Importer.
+type importerFunc func(path string) (*types.Package, error)
+
+func (f importerFunc) Import(path string) (*types.Package, error) { return f(path) }

package/extractors/go/walker.go

@@ -0,0 +1,67 @@
+// walker.go — file discovery: prune IGNORE dirs at the frontier, collect
+// .go files, group them into packages (one package = one directory).
+
+package main
+
+import (
+	"io/fs"
+	"os"
+	"path/filepath"
+	"sort"
+	"strings"
+)
+
+// IGNORE_DIRS — pruned at the frontier (never descended into). vendor/ holds
+// third-party copies (go list resolves types there; we never walk it for
+// nodes). bin/dist/build/out/target are build outputs.
+var ignoreDirs = map[string]bool{
+	".git": true, "node_modules": true, "vendor": true, "bin": true,
+	"dist": true, "build": true, "out": true, "target": true,
+	"coverage": true, ".idea": true, ".vscode": true, ".cache": true,
+}
+
+// goFiles walks repoPath and returns relative paths of all .go files,
+// pruned at the frontier (never descend into pruned dirs).
+func goFiles(repoPath string) []string {
+	var files []string
+	filepath.WalkDir(repoPath, func(p string, d fs.DirEntry, err error) error {
+		if err != nil {
+			return nil // unreadable entry — skip silently
+		}
+		if d.IsDir() {
+			base := d.Name()
+			if p != repoPath && (ignoreDirs[base] || strings.HasPrefix(base, ".")) {
+				return filepath.SkipDir
+			}
+			return nil
+		}
+		if strings.HasSuffix(d.Name(), ".go") {
+			rel, _ := filepath.Rel(repoPath, p)
+			files = append(files, filepath.ToSlash(rel))
+		}
+		return nil
+	})
+	sort.Strings(files)
+	return files
+}
+
+// groupByDir → rel dir → []rel file paths (deterministic order).
+func groupByDir(files []string) map[string][]string {
+	groups := map[string][]string{}
+	for _, f := range files {
+		dir := "."
+		if i := strings.LastIndex(f, "/"); i >= 0 {
+			dir = f[:i]
+		}
+		groups[dir] = append(groups[dir], f)
+	}
+	for _, list := range groups {
+		sort.Strings(list)
+	}
+	return groups
+}
+
+func dirExists(repoPath, rel string) bool {
+	info, err := os.Stat(filepath.Join(repoPath, rel))
+	return err == nil && info.IsDir()
+}

package/extractors/java/build.mjs

@@ -0,0 +1,90 @@
+#!/usr/bin/env node
+// DevLens Java extractor — fat-jar build (cross-platform: Linux/macOS/Windows).
+//
+// Node is guaranteed on every machine that installs an npm package, so this
+// replaces the bash build.sh: downloads JavaParser + Gson from Maven Central,
+// compiles src/, and merges EVERYTHING into one self-contained jar so the
+// engine can run `java -jar devlens_java_extractor.jar` with no classpath.
+//
+// Usage: node build.mjs   (idempotent — run from this directory)
+
+import { execFileSync } from "node:child_process";
+import { existsSync, mkdirSync, readdirSync, rmSync, writeFileSync, readFileSync } from "node:fs";
+import { dirname, join } from "node:path";
+import { fileURLToPath } from "node:url";
+
+const HERE = dirname(fileURLToPath(import.meta.url));
+const VERSIONS = {
+  "javaparser-core.jar": "com/github/javaparser/javaparser-core/3.28.2/javaparser-core-3.28.2.jar",
+  "javaparser-symbol-solver-core.jar": "com/github/javaparser/javaparser-symbol-solver-core/3.28.2/javaparser-symbol-solver-core-3.28.2.jar",
+  "gson.jar": "com/google/code/gson/gson/2.14.0/gson-2.14.0.jar",
+  "javassist.jar": "org/javassist/javassist/3.32.0-GA/javassist-3.32.0-GA.jar",
+  "guava.jar": "com/google/guava/guava/33.6.0-jre/guava-33.6.0-jre.jar",
+  "checker-qual.jar": "org/checkerframework/checker-qual/4.2.2/checker-qual-4.2.2.jar",
+};
+const MAVEN = "https://repo1.maven.org/maven2/";
+
+const libDir = join(HERE, "lib");
+const classesDir = join(HERE, "classes");
+mkdirSync(libDir, { recursive: true });
+mkdirSync(classesDir, { recursive: true });
+
+function run(cmd, args, opts = {}) {
+  execFileSync(cmd, args, { stdio: "inherit", ...opts });
+}
+
+// ── 1. fetch dependencies (cache on disk; idempotent) ────────────────
+console.log("==> Fetching dependencies (Maven Central)");
+for (const [jar, path] of Object.entries(VERSIONS)) {
+  const dest = join(libDir, jar);
+  if (existsSync(dest) && readFileSync(dest).length > 0) {
+    console.log(`  cached ${jar}`);
+    continue;
+  }
+  console.log(`  downloading ${jar}`);
+  const res = await fetch(MAVEN + path);
+  if (!res.ok) throw new Error(`download failed (${res.status}): ${MAVEN + path}`);
+  writeFileSync(dest, Buffer.from(await res.arrayBuffer()));
+}
+
+// ── 2. compile ───────────────────────────────────────────────────────
+console.log("==> Compiling");
+const sources = [];
+function collect(dir) {
+  for (const entry of readdirSync(dir, { withFileTypes: true })) {
+    const p = join(dir, entry.name);
+    if (entry.isDirectory()) collect(p);
+    else if (entry.name.endsWith(".java")) sources.push(p);
+  }
+}
+collect(join(HERE, "src"));
+run(process.env.JAVAC || "javac",
+  ["--release", "17", "-cp", join(libDir, "*"), "-d", classesDir, ...sources]);
+
+// ── 3. assemble fat jar ──────────────────────────────────────────────
+console.log("==> Assembling fat jar");
+const fatDir = join(HERE, "fat");
+rmSync(fatDir, { recursive: true, force: true });
+mkdirSync(fatDir, { recursive: true });
+for (const jar of readdirSync(libDir).filter((f) => f.endsWith(".jar"))) {
+  run("jar", ["xf", join(libDir, jar)], { cwd: fatDir });
+}
+// strip other jars' META-INF signatures + multi-release leftovers are fine
+for (const sig of [".SF", ".DSA", ".RSA"]) {
+  for (const f of readdirSync(join(fatDir, "META-INF"))) {
+    if (f.endsWith(sig)) rmSync(join(fatDir, "META-INF", f));
+  }
+}
+for (const entry of readdirSync(classesDir)) {
+  run(process.platform === "win32" ? "xcopy" : "cp",
+    process.platform === "win32"
+      ? [join(classesDir, entry), join(fatDir, entry), "/E", "/Y"]
+      : ["-r", join(classesDir, entry), fatDir],
+    process.platform === "win32" ? { shell: true } : {});
+}
+run("jar", ["cfe", join(HERE, "devlens_java_extractor.jar"),
+  "devlens.extractor.Main", "-C", fatDir, "."]);
+rmSync(fatDir, { recursive: true, force: true });
+
+const size = (readFileSync(join(HERE, "devlens_java_extractor.jar")).length / 1e6).toFixed(1);
+console.log(`==> Done: ${size}M devlens_java_extractor.jar`);

package/extractors/java/src/devlens/extractor/Contract.java

@@ -0,0 +1,171 @@
+package devlens.extractor;
+
+import java.nio.charset.StandardCharsets;
+import java.security.MessageDigest;
+import java.util.ArrayList;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Contract — the exact JSON shape DevLens Engine expects (mirrors
+ * contract.py from the Python extractor). Every key here is camelCase to
+ * match the TypeScript interfaces (CodeNode / CodeEdge / BackendRouteNode).
+ * This class is the single source of truth for node/edge shapes; other
+ * modules never spell keys by hand.
+ */
+public final class Contract {
+
+    private Contract() {}
+
+    /** SHA-256 of the raw code, first 16 hex chars (mirrors contract.py). */
+    public static String codeHash(String rawCode) {
+        try {
+            MessageDigest md = MessageDigest.getInstance("SHA-256");
+            byte[] digest = md.digest(rawCode.getBytes(StandardCharsets.UTF_8));
+            StringBuilder sb = new StringBuilder(16);
+            for (int i = 0; i < 8; i++) {
+                sb.append(String.format("%02x", digest[i]));
+            }
+            return sb.toString();
+        } catch (Exception e) {
+            return "";
+        }
+    }
+
+    /** FILE / TEST node — id format: file::rel/path/File.java. */
+    public static Map<String, Object> fileNode(String relPath, int endLine,
+                                               String nodeType, String language) {
+        Map<String, Object> metadata = new LinkedHashMap<>();
+        metadata.put("nodeCount", 0);
+        metadata.put("childNodeIds", new ArrayList<String>());
+        metadata.put("language", language);
+
+        Map<String, Object> node = new LinkedHashMap<>();
+        node.put("id", "file::" + relPath);
+        node.put("name", relPath.substring(relPath.lastIndexOf('/') + 1));
+        node.put("type", nodeType);
+        node.put("filePath", relPath);
+        node.put("startLine", 1);
+        node.put("endLine", endLine);
+        node.put("parentFile", "file::" + relPath);
+        node.put("metadata", metadata);
+        return node;
+    }
+
+    /**
+     * CLASS / INTERFACE / ENUM / METHOD node — id format:
+     * rel/path/File.java::TypeName or rel/path/File.java::Type.method
+     */
+    public static Map<String, Object> codeNode(String relPath, String name, String nodeType,
+                                               int startLine, int endLine, String rawCode,
+                                               Map<String, Object> metadata, String language) {
+        Map<String, Object> node = new LinkedHashMap<>();
+        node.put("id", relPath + "::" + name);
+        node.put("name", name);
+        node.put("type", nodeType);
+        node.put("filePath", relPath);
+        node.put("startLine", startLine);
+        node.put("endLine", endLine);
+        node.put("rawCode", rawCode);
+        node.put("codeHash", codeHash(rawCode));
+        node.put("parentFile", "file::" + relPath);
+
+        Map<String, Object> meta = new LinkedHashMap<>();
+        meta.put("language", language);
+        if (metadata != null) {
+            meta.putAll(metadata);
+        }
+        node.put("metadata", meta);
+        return node;
+    }
+
+    /** THIRD_PARTY node — id format: [mvn]/prefix or [mvn]/prefix::Member. */
+    public static Map<String, Object> thirdPartyNode(String id, String name,
+                                                     Map<String, Object> metadata) {
+        Map<String, Object> node = new LinkedHashMap<>();
+        node.put("id", id);
+        node.put("name", name);
+        node.put("type", "THIRD_PARTY");
+        node.put("filePath", id);
+        node.put("startLine", 0);
+        node.put("endLine", 0);
+        node.put("metadata", metadata == null ? new LinkedHashMap<String, Object>() : metadata);
+        return node;
+    }
+
+    /** ROUTE code node — id format: rel::METHOD /path (mirrors python common.py). */
+    public static Map<String, Object> routeNode(String relPath, String httpMethod,
+                                                String urlPath, boolean isDynamic,
+                                                List<String> params, String framework,
+                                                String handlerName) {
+        Map<String, Object> metadata = new LinkedHashMap<>();
+        metadata.put("urlPath", urlPath);
+        metadata.put("httpMethod", httpMethod);
+        metadata.put("isDynamic", isDynamic);
+        metadata.put("params", params == null ? new ArrayList<String>() : params);
+        metadata.put("framework", framework);
+        metadata.put("handlerName", handlerName);
+        metadata.put("routeKind", "backend");
+
+        Map<String, Object> node = new LinkedHashMap<>();
+        node.put("id", relPath + "::" + httpMethod + " " + urlPath);
+        node.put("name", httpMethod + " " + urlPath);
+        node.put("type", "ROUTE");
+        node.put("filePath", relPath);
+        node.put("startLine", 0);
+        node.put("endLine", 0);
+        node.put("parentFile", "file::" + relPath);
+        node.put("metadata", metadata);
+        return node;
+    }
+
+    /** BackendRouteNode dict for the `routes` array (engine BackendRouteNode). */
+    public static Map<String, Object> backendRoute(String urlPath, String filePath,
+                                                   String httpMethod, String framework,
+                                                   boolean isDynamic, List<String> params,
+                                                   String handlerName, String nodeId) {
+        Map<String, Object> route = new LinkedHashMap<>();
+        route.put("type", "BACKEND_ROUTE");
+        route.put("urlPath", urlPath);
+        route.put("filePath", filePath);
+        route.put("httpMethod", httpMethod);
+        route.put("framework", framework);
+        route.put("isDynamic", isDynamic);
+        if (params != null && !params.isEmpty()) {
+            route.put("params", params);
+        }
+        if (handlerName != null) {
+            route.put("handlerName", handlerName);
+        }
+        if (nodeId != null) {
+            route.put("nodeId", nodeId);
+        }
+        return route;
+    }
+
+    /** Edge — {type, from, to, metadata?} (engine CodeEdge). */
+    public static Map<String, Object> edge(String type, String from, String to,
+                                           Map<String, Object> metadata) {
+        Map<String, Object> e = new LinkedHashMap<>();
+        e.put("type", type);
+        e.put("from", from);
+        e.put("to", to);
+        if (metadata != null && !metadata.isEmpty()) {
+            e.put("metadata", metadata);
+        }
+        return e;
+    }
+
+    /** Deterministic edge-key for dedupe: from|type|to. */
+    public static String edgeKey(Map<String, Object> edge) {
+        return edge.get("from") + "|" + edge.get("type") + "|" + edge.get("to");
+    }
+
+    public static Map<String, Object> error(String file, String message) {
+        Map<String, Object> err = new LinkedHashMap<>();
+        err.put("file", file);
+        err.put("error", message);
+        return err;
+    }
+}