devlensio 0.6.1 → 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +89 -31
- package/dist/extractors/detectLanguage.d.ts +2 -0
- package/dist/extractors/detectLanguage.js +33 -0
- package/dist/extractors/index.d.ts +6 -0
- package/dist/extractors/index.js +116 -0
- package/dist/extractors/runner.d.ts +7 -0
- package/dist/extractors/runner.js +194 -0
- package/dist/extractors/types.d.ts +37 -0
- package/dist/extractors/types.js +8 -0
- package/dist/graph/buildLookup.d.ts +1 -0
- package/dist/graph/buildLookup.js +2 -1
- package/dist/graph/edges/callEdges.js +19 -5
- package/dist/graph/edges/callEdges.test.d.ts +1 -0
- package/dist/graph/edges/callEdges.test.js +200 -0
- package/dist/graph/edges/importEdges.js +12 -0
- package/dist/graph/edges/inheritanceEdges.d.ts +3 -0
- package/dist/graph/edges/inheritanceEdges.js +73 -0
- package/dist/graph/edges/inheritanceEdges.test.d.ts +1 -0
- package/dist/graph/edges/inheritanceEdges.test.js +140 -0
- package/dist/graph/index.js +4 -0
- package/dist/parser/classes.test.d.ts +1 -0
- package/dist/parser/classes.test.js +360 -0
- package/dist/parser/extractors/classes.d.ts +5 -0
- package/dist/parser/extractors/classes.js +241 -0
- package/dist/parser/index.d.ts +2 -0
- package/dist/parser/index.js +7 -1
- package/dist/pipeline/index.d.ts +2 -1
- package/dist/pipeline/index.js +36 -48
- package/dist/scoring/index.js +8 -0
- package/dist/scoring/index.test.js +43 -0
- package/dist/scoring/nodeScorer.js +2 -0
- package/dist/scoring/pruneDisconnected.d.ts +8 -0
- package/dist/scoring/pruneDisconnected.js +66 -0
- package/dist/scoring/pruneDisconnected.test.d.ts +1 -0
- package/dist/scoring/pruneDisconnected.test.js +123 -0
- package/dist/summarizer/prompts.d.ts +1 -1
- package/dist/types.d.ts +5 -5
- package/extractors/go/bin/darwin-amd64/devlens_go_extractor +0 -0
- package/extractors/go/bin/darwin-arm64/devlens_go_extractor +0 -0
- package/extractors/go/bin/linux-amd64/devlens_go_extractor +0 -0
- package/extractors/go/bin/linux-arm64/devlens_go_extractor +0 -0
- package/extractors/go/bin/windows-amd64/devlens_go_extractor.exe +0 -0
- package/extractors/go/build.mjs +43 -0
- package/extractors/go/calls.go +289 -0
- package/extractors/go/contract.go +140 -0
- package/extractors/go/extractor.go +161 -0
- package/extractors/go/fingerprint.go +167 -0
- package/extractors/go/go.mod +3 -0
- package/extractors/go/imports.go +90 -0
- package/extractors/go/inheritance.go +138 -0
- package/extractors/go/lookup.go +106 -0
- package/extractors/go/main.go +57 -0
- package/extractors/go/nodes.go +177 -0
- package/extractors/go/orm_edges.go +277 -0
- package/extractors/go/parser.go +469 -0
- package/extractors/go/routes.go +521 -0
- package/extractors/go/tests.go +45 -0
- package/extractors/go/thirdparty.go +138 -0
- package/extractors/go/typeload.go +178 -0
- package/extractors/go/walker.go +67 -0
- package/extractors/java/build.mjs +90 -0
- package/extractors/java/devlens_java_extractor.jar +0 -0
- package/extractors/java/src/devlens/extractor/Contract.java +171 -0
- package/extractors/java/src/devlens/extractor/Extractor.java +262 -0
- package/extractors/java/src/devlens/extractor/ExtractorResult.java +12 -0
- package/extractors/java/src/devlens/extractor/Fingerprint.java +239 -0
- package/extractors/java/src/devlens/extractor/LookupMaps.java +123 -0
- package/extractors/java/src/devlens/extractor/Main.java +66 -0
- package/extractors/java/src/devlens/extractor/Parser.java +522 -0
- package/extractors/java/src/devlens/extractor/SourceWalker.java +82 -0
- package/extractors/java/src/devlens/extractor/ThirdParty.java +141 -0
- package/extractors/java/src/devlens/extractor/TypeSolverFactory.java +43 -0
- package/extractors/java/src/devlens/extractor/edges/Calls.java +266 -0
- package/extractors/java/src/devlens/extractor/edges/Enrich.java +54 -0
- package/extractors/java/src/devlens/extractor/edges/Imports.java +154 -0
- package/extractors/java/src/devlens/extractor/edges/Inheritance.java +79 -0
- package/extractors/java/src/devlens/extractor/edges/OrmEdges.java +209 -0
- package/extractors/java/src/devlens/extractor/edges/Routes.java +143 -0
- package/extractors/java/src/devlens/extractor/edges/Tests.java +53 -0
- package/extractors/python/devlens_extractors_python/__init__.py +3 -0
- package/extractors/python/devlens_extractors_python/__main__.py +33 -0
- package/extractors/python/devlens_extractors_python/contract.py +94 -0
- package/extractors/python/devlens_extractors_python/edges/__init__.py +28 -0
- package/extractors/python/devlens_extractors_python/edges/calls.py +112 -0
- package/extractors/python/devlens_extractors_python/edges/enrich.py +44 -0
- package/extractors/python/devlens_extractors_python/edges/imports.py +155 -0
- package/extractors/python/devlens_extractors_python/edges/inheritance.py +97 -0
- package/extractors/python/devlens_extractors_python/edges/orm_edges.py +225 -0
- package/extractors/python/devlens_extractors_python/edges/routes/__init__.py +28 -0
- package/extractors/python/devlens_extractors_python/edges/routes/common.py +79 -0
- package/extractors/python/devlens_extractors_python/edges/routes/decorators.py +212 -0
- package/extractors/python/devlens_extractors_python/edges/routes/django_urls.py +213 -0
- package/extractors/python/devlens_extractors_python/edges/routes/drf.py +128 -0
- package/extractors/python/devlens_extractors_python/edges/tests.py +53 -0
- package/extractors/python/devlens_extractors_python/extractor.py +107 -0
- package/extractors/python/devlens_extractors_python/fingerprint.py +201 -0
- package/extractors/python/devlens_extractors_python/lookup.py +72 -0
- package/extractors/python/devlens_extractors_python/parser/__init__.py +72 -0
- package/extractors/python/devlens_extractors_python/parser/classes.py +109 -0
- package/extractors/python/devlens_extractors_python/parser/functions.py +163 -0
- package/extractors/python/devlens_extractors_python/parser/walker.py +30 -0
- package/extractors/python/devlens_extractors_python/third_party.py +103 -0
- package/extractors/python/pyproject.toml +16 -0
- package/extractors/python/setup.mjs +54 -0
- package/extractors/rust/Cargo.toml +29 -0
- package/extractors/rust/bin/darwin-amd64/devlens_rust_extractor +0 -0
- package/extractors/rust/bin/darwin-arm64/devlens_rust_extractor +0 -0
- package/extractors/rust/bin/linux-amd64/devlens_rust_extractor +0 -0
- package/extractors/rust/bin/linux-arm64/devlens_rust_extractor +0 -0
- package/extractors/rust/bin/windows-amd64/devlens_rust_extractor.exe +0 -0
- package/extractors/rust/build.mjs +85 -0
- package/extractors/rust/src/calls.rs +295 -0
- package/extractors/rust/src/contract.rs +236 -0
- package/extractors/rust/src/enrich.rs +107 -0
- package/extractors/rust/src/extractor.rs +199 -0
- package/extractors/rust/src/fingerprint.rs +238 -0
- package/extractors/rust/src/imports.rs +101 -0
- package/extractors/rust/src/inheritance.rs +233 -0
- package/extractors/rust/src/lookup.rs +226 -0
- package/extractors/rust/src/main.rs +53 -0
- package/extractors/rust/src/module_map.rs +131 -0
- package/extractors/rust/src/nodes.rs +174 -0
- package/extractors/rust/src/orm_edges.rs +125 -0
- package/extractors/rust/src/parser.rs +859 -0
- package/extractors/rust/src/routes.rs +1265 -0
- package/extractors/rust/src/tests.rs +109 -0
- package/extractors/rust/src/thirdparty.rs +114 -0
- package/extractors/rust/src/walker.rs +82 -0
- package/package.json +22 -4
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
package devlens.extractor;
|
|
2
|
+
|
|
3
|
+
import com.google.gson.Gson;
|
|
4
|
+
import com.google.gson.GsonBuilder;
|
|
5
|
+
import com.google.gson.JsonArray;
|
|
6
|
+
import com.google.gson.JsonElement;
|
|
7
|
+
import com.google.gson.JsonObject;
|
|
8
|
+
import com.google.gson.JsonParser;
|
|
9
|
+
|
|
10
|
+
import java.io.BufferedReader;
|
|
11
|
+
import java.io.InputStreamReader;
|
|
12
|
+
import java.nio.charset.StandardCharsets;
|
|
13
|
+
import java.util.ArrayList;
|
|
14
|
+
import java.util.List;
|
|
15
|
+
|
|
16
|
+
/**
|
|
17
|
+
* Entry point — JSON over stdin/stdout contract:
|
|
18
|
+
* stdin: {"repoPath": "/abs/path", "options": {"includedThirdPartyLibs": [...]}}
|
|
19
|
+
* stdout: {"fingerprint": ..., "nodes": [...], "edges": [...], "routes": [...],
|
|
20
|
+
* "stats": {...}, "errors": [...]}
|
|
21
|
+
*
|
|
22
|
+
* JSON only on stdout; anything human-readable goes to stderr. Non-zero exit
|
|
23
|
+
* only on fatal errors (parse problems are collected into errors[] instead).
|
|
24
|
+
*/
|
|
25
|
+
public final class Main {
|
|
26
|
+
|
|
27
|
+
public static void main(String[] args) {
|
|
28
|
+
try {
|
|
29
|
+
StringBuilder input = new StringBuilder();
|
|
30
|
+
try (BufferedReader reader = new BufferedReader(
|
|
31
|
+
new InputStreamReader(System.in, StandardCharsets.UTF_8))) {
|
|
32
|
+
String line;
|
|
33
|
+
while ((line = reader.readLine()) != null) {
|
|
34
|
+
input.append(line).append('\n');
|
|
35
|
+
}
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
JsonObject json = JsonParser.parseString(input.toString()).getAsJsonObject();
|
|
39
|
+
String repoPath = json.get("repoPath").getAsString();
|
|
40
|
+
|
|
41
|
+
// The engine's real contract key is `includeThirdPartyLibs` (no "d");
|
|
42
|
+
// contract.html wrote `includedThirdPartyLibs` — accept BOTH spellings.
|
|
43
|
+
List<String> allowed = new ArrayList<>();
|
|
44
|
+
JsonObject options = json.has("options") && json.get("options").isJsonObject()
|
|
45
|
+
? json.getAsJsonObject("options") : new JsonObject();
|
|
46
|
+
JsonElement libs = options.has("includeThirdPartyLibs")
|
|
47
|
+
? options.get("includeThirdPartyLibs")
|
|
48
|
+
: options.get("includedThirdPartyLibs");
|
|
49
|
+
if (libs != null && libs.isJsonArray()) {
|
|
50
|
+
for (JsonElement e : libs.getAsJsonArray()) {
|
|
51
|
+
allowed.add(e.getAsString());
|
|
52
|
+
}
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
ExtractorResult result = new Extractor(repoPath, allowed).run();
|
|
56
|
+
|
|
57
|
+
Gson gson = new GsonBuilder().serializeNulls().disableHtmlEscaping().create();
|
|
58
|
+
System.out.println(gson.toJson(result.json()));
|
|
59
|
+
System.exit(result.fatal() ? 1 : 0);
|
|
60
|
+
} catch (Throwable t) {
|
|
61
|
+
System.err.println("java extractor fatal error: " + t.getMessage());
|
|
62
|
+
t.printStackTrace();
|
|
63
|
+
System.exit(1);
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
}
|
|
@@ -0,0 +1,522 @@
|
|
|
1
|
+
package devlens.extractor;
|
|
2
|
+
|
|
3
|
+
import com.github.javaparser.StaticJavaParser;
|
|
4
|
+
import com.github.javaparser.ast.CompilationUnit;
|
|
5
|
+
import com.github.javaparser.ast.ImportDeclaration;
|
|
6
|
+
import com.github.javaparser.ast.Node;
|
|
7
|
+
import com.github.javaparser.ast.body.AnnotationDeclaration;
|
|
8
|
+
import com.github.javaparser.ast.body.BodyDeclaration;
|
|
9
|
+
import com.github.javaparser.ast.body.ClassOrInterfaceDeclaration;
|
|
10
|
+
import com.github.javaparser.ast.body.ConstructorDeclaration;
|
|
11
|
+
import com.github.javaparser.ast.body.EnumDeclaration;
|
|
12
|
+
import com.github.javaparser.ast.body.FieldDeclaration;
|
|
13
|
+
import com.github.javaparser.ast.body.MethodDeclaration;
|
|
14
|
+
import com.github.javaparser.ast.body.Parameter;
|
|
15
|
+
import com.github.javaparser.ast.body.RecordDeclaration;
|
|
16
|
+
import com.github.javaparser.ast.body.TypeDeclaration;
|
|
17
|
+
import com.github.javaparser.ast.body.VariableDeclarator;
|
|
18
|
+
import com.github.javaparser.ast.nodeTypes.NodeWithMembers;
|
|
19
|
+
import com.github.javaparser.ast.type.ReferenceType;
|
|
20
|
+
import com.github.javaparser.ast.expr.AnnotationExpr;
|
|
21
|
+
import com.github.javaparser.ast.expr.Expression;
|
|
22
|
+
import com.github.javaparser.ast.expr.FieldAccessExpr;
|
|
23
|
+
import com.github.javaparser.ast.expr.MethodCallExpr;
|
|
24
|
+
import com.github.javaparser.ast.expr.NameExpr;
|
|
25
|
+
import com.github.javaparser.ast.expr.NormalAnnotationExpr;
|
|
26
|
+
import com.github.javaparser.ast.expr.ObjectCreationExpr;
|
|
27
|
+
import com.github.javaparser.ast.expr.SingleMemberAnnotationExpr;
|
|
28
|
+
import com.github.javaparser.ast.expr.StringLiteralExpr;
|
|
29
|
+
import com.github.javaparser.ast.nodeTypes.NodeWithAnnotations;
|
|
30
|
+
import com.github.javaparser.ast.stmt.BlockStmt;
|
|
31
|
+
import com.github.javaparser.resolution.declarations.ResolvedMethodDeclaration;
|
|
32
|
+
|
|
33
|
+
import java.io.IOException;
|
|
34
|
+
import java.nio.file.Files;
|
|
35
|
+
import java.nio.file.Path;
|
|
36
|
+
import java.util.ArrayList;
|
|
37
|
+
import java.util.LinkedHashMap;
|
|
38
|
+
import java.util.LinkedHashSet;
|
|
39
|
+
import java.util.List;
|
|
40
|
+
import java.util.Map;
|
|
41
|
+
import java.util.Set;
|
|
42
|
+
|
|
43
|
+
/**
|
|
44
|
+
* Parser — parse-time FACT collection (mirrors the Python parser package).
|
|
45
|
+
*
|
|
46
|
+
* One AST walk per file produces a ParsedFile of facts: imports, types
|
|
47
|
+
* (class/interface/enum/record), fields, methods, call sites. Edge detectors
|
|
48
|
+
* NEVER re-walk the AST — they resolve facts via the shared LookupMaps.
|
|
49
|
+
*
|
|
50
|
+
* Facts (not decisions): "who calls what" is recorded here as strings +
|
|
51
|
+
* best-effort symbol-solver targets; deciding which of those become CALLS
|
|
52
|
+
* edges happens later in edges/Calls.java.
|
|
53
|
+
*/
|
|
54
|
+
public final class Parser {
|
|
55
|
+
|
|
56
|
+
private Parser() {}
|
|
57
|
+
|
|
58
|
+
// ─────────────────────────── fact model ───────────────────────────
|
|
59
|
+
|
|
60
|
+
public static final class ImportInfo {
|
|
61
|
+
public String name; // com.foo.Bar | com.foo (wildcard)
|
|
62
|
+
public boolean isStatic;
|
|
63
|
+
public boolean isWildcard;
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
public static final class FieldInfo {
|
|
67
|
+
public String name;
|
|
68
|
+
public String type; // simple type name (generics stripped)
|
|
69
|
+
public List<String> annotations = new ArrayList<>();
|
|
70
|
+
public int line;
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
public static final class CallInfo {
|
|
74
|
+
public String name; // full display string: userService.getUser
|
|
75
|
+
public String methodName; // simple method name: getUser
|
|
76
|
+
public String receiverName; // root identifier of the scope (or null)
|
|
77
|
+
public int line;
|
|
78
|
+
public String resolvedTarget; // FQCN.method when the symbol solver hit, else null
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
public static final class MethodInfo {
|
|
82
|
+
public String name; // getName | <init> for constructors
|
|
83
|
+
public boolean isConstructor;
|
|
84
|
+
public boolean isStatic;
|
|
85
|
+
public boolean isAbstract;
|
|
86
|
+
public List<String> annotations = new ArrayList<>();
|
|
87
|
+
public Map<String, String> annotationValues = new LinkedHashMap<>();
|
|
88
|
+
public String returnType = "";
|
|
89
|
+
public List<String> params = new ArrayList<>();
|
|
90
|
+
public List<String> throwsTypes = new ArrayList<>();
|
|
91
|
+
public int startLine;
|
|
92
|
+
public int endLine;
|
|
93
|
+
public String rawCode;
|
|
94
|
+
public List<CallInfo> calls = new ArrayList<>();
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
public static final class TypeInfo {
|
|
98
|
+
public String dottedName; // Outer.Inner for nested types
|
|
99
|
+
public String kind; // class | interface | enum | record | annotation
|
|
100
|
+
public int startLine;
|
|
101
|
+
public int endLine;
|
|
102
|
+
public String rawCode;
|
|
103
|
+
public List<String> annotations = new ArrayList<>();
|
|
104
|
+
public Map<String, String> annotationValues = new LinkedHashMap<>();
|
|
105
|
+
public String extendedType = ""; // simple name (generics stripped)
|
|
106
|
+
public List<String> implementedTypes = new ArrayList<>();
|
|
107
|
+
public List<FieldInfo> fields = new ArrayList<>();
|
|
108
|
+
public List<MethodInfo> methods = new ArrayList<>();
|
|
109
|
+
public List<String> typeParameters = new ArrayList<>();
|
|
110
|
+
public List<TypeInfo> nested = new ArrayList<>();
|
|
111
|
+
public boolean isInterface;
|
|
112
|
+
public boolean isEnum;
|
|
113
|
+
public boolean isRecord;
|
|
114
|
+
public boolean isAbstract;
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
public static final class ParsedFile {
|
|
118
|
+
public String relPath;
|
|
119
|
+
public String raw;
|
|
120
|
+
public int endLine;
|
|
121
|
+
public boolean isTest;
|
|
122
|
+
public String packageName = "";
|
|
123
|
+
public List<ImportInfo> imports = new ArrayList<>();
|
|
124
|
+
public List<TypeInfo> types = new ArrayList<>();
|
|
125
|
+
public Set<String> usedTypeNames = new LinkedHashSet<>();
|
|
126
|
+
public List<String> errors = new ArrayList<>();
|
|
127
|
+
|
|
128
|
+
/** Top-level + nested types flattened (nested get dotted names). */
|
|
129
|
+
public List<TypeInfo> allTypes() {
|
|
130
|
+
List<TypeInfo> out = new ArrayList<>();
|
|
131
|
+
for (TypeInfo t : types) {
|
|
132
|
+
out.add(t);
|
|
133
|
+
out.addAll(t.nested);
|
|
134
|
+
}
|
|
135
|
+
return out;
|
|
136
|
+
}
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
// ─────────────────────────── parsing ───────────────────────────
|
|
140
|
+
|
|
141
|
+
/** Parse one file into facts. Parse problems are non-fatal (errors[]). */
|
|
142
|
+
public static ParsedFile parseFile(Path repoRoot, String relPath) throws IOException {
|
|
143
|
+
String raw = Files.readString(repoRoot.resolve(relPath));
|
|
144
|
+
ParsedFile pf = new ParsedFile();
|
|
145
|
+
pf.relPath = relPath;
|
|
146
|
+
pf.raw = raw;
|
|
147
|
+
pf.endLine = countLines(raw);
|
|
148
|
+
pf.isTest = SourceWalker.isTestFile(relPath);
|
|
149
|
+
try {
|
|
150
|
+
CompilationUnit cu = StaticJavaParser.parse(raw);
|
|
151
|
+
cu.getPackageDeclaration().ifPresent(p -> pf.packageName = p.getNameAsString());
|
|
152
|
+
for (ImportDeclaration imp : cu.getImports()) {
|
|
153
|
+
ImportInfo ii = new ImportInfo();
|
|
154
|
+
ii.isStatic = imp.isStatic();
|
|
155
|
+
ii.isWildcard = imp.isAsterisk();
|
|
156
|
+
ii.name = imp.getNameAsString();
|
|
157
|
+
pf.imports.add(ii);
|
|
158
|
+
}
|
|
159
|
+
for (TypeDeclaration<?> td : cu.getTypes()) {
|
|
160
|
+
pf.types.add(extractType(td, pf, null));
|
|
161
|
+
}
|
|
162
|
+
} catch (Exception e) {
|
|
163
|
+
pf.errors.add(e.getMessage() == null ? e.toString() : e.getMessage());
|
|
164
|
+
}
|
|
165
|
+
return pf;
|
|
166
|
+
}
|
|
167
|
+
|
|
168
|
+
private static TypeInfo extractType(TypeDeclaration<?> td, ParsedFile pf, String outerDotted) {
|
|
169
|
+
TypeInfo t = new TypeInfo();
|
|
170
|
+
String simple = td.getNameAsString();
|
|
171
|
+
t.dottedName = outerDotted == null ? simple : outerDotted + "." + simple;
|
|
172
|
+
t.startLine = lineOf(td, true);
|
|
173
|
+
t.endLine = lineOf(td, false);
|
|
174
|
+
t.rawCode = slice(pf.raw, t.startLine, t.endLine);
|
|
175
|
+
|
|
176
|
+
if (td instanceof EnumDeclaration ed) {
|
|
177
|
+
t.kind = "enum";
|
|
178
|
+
t.isEnum = true;
|
|
179
|
+
t.implementedTypes = typeNames(ed.getImplementedTypes());
|
|
180
|
+
extractFields(ed, t, pf);
|
|
181
|
+
extractMethods(ed, t, pf);
|
|
182
|
+
extractNested(ed, t, pf, t.dottedName);
|
|
183
|
+
} else if (td instanceof RecordDeclaration rd) {
|
|
184
|
+
t.kind = "record";
|
|
185
|
+
t.isRecord = true;
|
|
186
|
+
for (Parameter p : rd.getParameters()) { // record components → fields
|
|
187
|
+
FieldInfo f = new FieldInfo();
|
|
188
|
+
f.name = p.getNameAsString();
|
|
189
|
+
f.type = simpleTypeName(p.getTypeAsString());
|
|
190
|
+
f.line = t.startLine;
|
|
191
|
+
t.fields.add(f);
|
|
192
|
+
}
|
|
193
|
+
t.implementedTypes = typeNames(rd.getImplementedTypes());
|
|
194
|
+
extractMethods(rd, t, pf);
|
|
195
|
+
extractNested(rd, t, pf, t.dottedName);
|
|
196
|
+
} else if (td instanceof AnnotationDeclaration) {
|
|
197
|
+
t.kind = "annotation";
|
|
198
|
+
} else if (td instanceof ClassOrInterfaceDeclaration c) {
|
|
199
|
+
if (c.isInterface()) {
|
|
200
|
+
t.kind = "interface";
|
|
201
|
+
t.isInterface = true;
|
|
202
|
+
} else {
|
|
203
|
+
t.kind = "class";
|
|
204
|
+
}
|
|
205
|
+
t.isAbstract = c.isAbstract();
|
|
206
|
+
t.typeParameters.addAll(c.getTypeParameters().stream()
|
|
207
|
+
.map(tp -> tp.getNameAsString()).toList());
|
|
208
|
+
c.getExtendedTypes().forEach(ext -> t.extendedType = simpleTypeName(ext.getNameAsString()));
|
|
209
|
+
t.implementedTypes.addAll(typeNames(c.getImplementedTypes()));
|
|
210
|
+
extractFields(c, t, pf);
|
|
211
|
+
extractMethods(c, t, pf);
|
|
212
|
+
extractNested(c, t, pf, t.dottedName);
|
|
213
|
+
}
|
|
214
|
+
|
|
215
|
+
collectAnnotations(td, t.annotations, t.annotationValues, pf);
|
|
216
|
+
return t;
|
|
217
|
+
}
|
|
218
|
+
|
|
219
|
+
private static void extractFields(NodeWithMembers<?> node, TypeInfo t, ParsedFile pf) {
|
|
220
|
+
for (BodyDeclaration<?> member : node.getMembers()) {
|
|
221
|
+
if (member instanceof FieldDeclaration fd) {
|
|
222
|
+
for (VariableDeclarator v : fd.getVariables()) {
|
|
223
|
+
FieldInfo f = new FieldInfo();
|
|
224
|
+
f.name = v.getNameAsString();
|
|
225
|
+
f.type = simpleTypeName(v.getTypeAsString());
|
|
226
|
+
f.line = lineOf(fd, true);
|
|
227
|
+
f.annotations.addAll(annotationNames(fd.getAnnotations()));
|
|
228
|
+
t.fields.add(f);
|
|
229
|
+
}
|
|
230
|
+
}
|
|
231
|
+
}
|
|
232
|
+
}
|
|
233
|
+
|
|
234
|
+
private static void extractMethods(NodeWithMembers<?> node, TypeInfo t, ParsedFile pf) {
|
|
235
|
+
for (BodyDeclaration<?> member : node.getMembers()) {
|
|
236
|
+
if (member instanceof MethodDeclaration md) {
|
|
237
|
+
t.methods.add(extractMethod(md, pf, t.isInterface));
|
|
238
|
+
} else if (member instanceof ConstructorDeclaration cd) {
|
|
239
|
+
t.methods.add(extractConstructor(cd, pf));
|
|
240
|
+
}
|
|
241
|
+
}
|
|
242
|
+
}
|
|
243
|
+
|
|
244
|
+
private static void extractNested(NodeWithMembers<?> node, TypeInfo t, ParsedFile pf, String outerDotted) {
|
|
245
|
+
for (BodyDeclaration<?> member : node.getMembers()) {
|
|
246
|
+
if (member instanceof TypeDeclaration<?> nested) {
|
|
247
|
+
t.nested.add(extractType(nested, pf, outerDotted));
|
|
248
|
+
}
|
|
249
|
+
}
|
|
250
|
+
}
|
|
251
|
+
|
|
252
|
+
private static MethodInfo extractConstructor(ConstructorDeclaration cd, ParsedFile pf) {
|
|
253
|
+
MethodInfo m = new MethodInfo();
|
|
254
|
+
m.name = "<init>";
|
|
255
|
+
m.isConstructor = true;
|
|
256
|
+
m.returnType = cd.getNameAsString();
|
|
257
|
+
m.startLine = lineOf(cd, true);
|
|
258
|
+
m.endLine = lineOf(cd, false);
|
|
259
|
+
m.rawCode = slice(pf.raw, m.startLine, m.endLine);
|
|
260
|
+
collectAnnotations(cd, m.annotations, m.annotationValues, pf);
|
|
261
|
+
for (Parameter p : cd.getParameters()) {
|
|
262
|
+
m.params.add(p.getTypeAsString() + " " + p.getNameAsString());
|
|
263
|
+
}
|
|
264
|
+
m.calls = collectCalls(cd.getBody(), m);
|
|
265
|
+
return m;
|
|
266
|
+
}
|
|
267
|
+
|
|
268
|
+
private static MethodInfo extractMethod(MethodDeclaration md, ParsedFile pf, boolean inInterface) {
|
|
269
|
+
MethodInfo m = new MethodInfo();
|
|
270
|
+
m.name = md.getNameAsString();
|
|
271
|
+
m.isStatic = md.isStatic();
|
|
272
|
+
m.isAbstract = inInterface || md.isAbstract();
|
|
273
|
+
m.returnType = simpleTypeName(md.getTypeAsString());
|
|
274
|
+
m.startLine = lineOf(md, true);
|
|
275
|
+
m.endLine = lineOf(md, false);
|
|
276
|
+
m.rawCode = slice(pf.raw, m.startLine, m.endLine);
|
|
277
|
+
collectAnnotations(md, m.annotations, m.annotationValues, pf);
|
|
278
|
+
for (Parameter p : md.getParameters()) {
|
|
279
|
+
m.params.add(p.getTypeAsString() + " " + p.getNameAsString());
|
|
280
|
+
}
|
|
281
|
+
m.throwsTypes.addAll(md.getThrownExceptions().stream()
|
|
282
|
+
.map(t -> simpleTypeName(t.asString())).toList());
|
|
283
|
+
m.calls = collectCalls(md.getBody().orElse(null), m);
|
|
284
|
+
return m;
|
|
285
|
+
}
|
|
286
|
+
|
|
287
|
+
/**
|
|
288
|
+
* Call collection — SCOPE RULE: only the method's own body. We visit
|
|
289
|
+
* MethodCallExprs but never descend into nested type declarations /
|
|
290
|
+
* lambdas attributed to another owner (same guard as Python's
|
|
291
|
+
* NESTED_SCOPES).
|
|
292
|
+
*/
|
|
293
|
+
private static List<CallInfo> collectCalls(BlockStmt body, MethodInfo owner) {
|
|
294
|
+
List<CallInfo> calls = new ArrayList<>();
|
|
295
|
+
if (body == null) {
|
|
296
|
+
return calls;
|
|
297
|
+
}
|
|
298
|
+
body.walk(MethodCallExpr.class, call -> {
|
|
299
|
+
CallInfo ci = new CallInfo();
|
|
300
|
+
ci.methodName = call.getNameAsString();
|
|
301
|
+
ci.line = lineOf(call, true);
|
|
302
|
+
Expression scope = call.getScope().orElse(null);
|
|
303
|
+
if (scope != null) {
|
|
304
|
+
ci.receiverName = rootName(scope);
|
|
305
|
+
ci.name = scopeName(scope) + "." + call.getNameAsString();
|
|
306
|
+
} else {
|
|
307
|
+
ci.receiverName = null;
|
|
308
|
+
ci.name = call.getNameAsString();
|
|
309
|
+
}
|
|
310
|
+
// Best-effort symbol-solver target: pkg.Class.method (or null).
|
|
311
|
+
try {
|
|
312
|
+
ResolvedMethodDeclaration r = call.resolve();
|
|
313
|
+
ci.resolvedTarget = r.getPackageName() + "." + r.getClassName() + "." + r.getName();
|
|
314
|
+
} catch (Throwable ignored) {
|
|
315
|
+
// unresolved (external lib, local variable, dynamic) — fallback ladder
|
|
316
|
+
}
|
|
317
|
+
calls.add(ci);
|
|
318
|
+
});
|
|
319
|
+
return calls;
|
|
320
|
+
}
|
|
321
|
+
|
|
322
|
+
// ─────────────────────────── helpers ───────────────────────────
|
|
323
|
+
|
|
324
|
+
private static int lineOf(Node n, boolean start) {
|
|
325
|
+
return n.getRange().map(r -> start ? r.begin.line : r.end.line).orElse(0);
|
|
326
|
+
}
|
|
327
|
+
|
|
328
|
+
private static int countLines(String raw) {
|
|
329
|
+
int n = 1;
|
|
330
|
+
for (int i = 0; i < raw.length(); i++) {
|
|
331
|
+
if (raw.charAt(i) == '\n') n++;
|
|
332
|
+
}
|
|
333
|
+
return n;
|
|
334
|
+
}
|
|
335
|
+
|
|
336
|
+
/** Exact source slice by 1-based inclusive line range. */
|
|
337
|
+
public static String slice(String raw, int start, int end) {
|
|
338
|
+
if (start < 1) start = 1;
|
|
339
|
+
String[] lines = raw.split("\n", -1);
|
|
340
|
+
if (end > lines.length) end = lines.length;
|
|
341
|
+
StringBuilder sb = new StringBuilder();
|
|
342
|
+
for (int i = start - 1; i < end; i++) {
|
|
343
|
+
sb.append(lines[i]).append('\n');
|
|
344
|
+
}
|
|
345
|
+
return sb.toString();
|
|
346
|
+
}
|
|
347
|
+
|
|
348
|
+
/** Strip generics + qualifiers: java.util.List<User> → List. */
|
|
349
|
+
public static String simpleTypeName(String type) {
|
|
350
|
+
String t = type;
|
|
351
|
+
int generic = t.indexOf('<');
|
|
352
|
+
if (generic >= 0) t = t.substring(0, generic);
|
|
353
|
+
int dot = t.lastIndexOf('.');
|
|
354
|
+
if (dot >= 0) t = t.substring(dot + 1);
|
|
355
|
+
return t.trim();
|
|
356
|
+
}
|
|
357
|
+
|
|
358
|
+
private static List<String> typeNames(List<? extends com.github.javaparser.ast.type.Type> types) {
|
|
359
|
+
return types.stream().map(t -> simpleTypeName(t.asString())).toList();
|
|
360
|
+
}
|
|
361
|
+
|
|
362
|
+
private static List<String> annotationNames(List<AnnotationExpr> anns) {
|
|
363
|
+
List<String> names = new ArrayList<>();
|
|
364
|
+
for (AnnotationExpr a : anns) {
|
|
365
|
+
names.add(a.getNameAsString());
|
|
366
|
+
}
|
|
367
|
+
return names;
|
|
368
|
+
}
|
|
369
|
+
|
|
370
|
+
private static void collectAnnotations(NodeWithAnnotations<?> node,
|
|
371
|
+
List<String> names,
|
|
372
|
+
Map<String, String> values,
|
|
373
|
+
ParsedFile pf) {
|
|
374
|
+
for (AnnotationExpr a : node.getAnnotations()) {
|
|
375
|
+
names.add(a.getNameAsString());
|
|
376
|
+
String value = annotationValue(a, values, pf);
|
|
377
|
+
if (value != null) {
|
|
378
|
+
values.put(a.getNameAsString(), value);
|
|
379
|
+
}
|
|
380
|
+
}
|
|
381
|
+
}
|
|
382
|
+
|
|
383
|
+
/**
|
|
384
|
+
* First string literal from an annotation — handles the common forms:
|
|
385
|
+
* @GetMapping("/users") → /users
|
|
386
|
+
* @RequestMapping(value="/x") → /x
|
|
387
|
+
* @RequestMapping(path="/x") → /x
|
|
388
|
+
* @RequestMapping({"/a","/b"}) → /a (first element)
|
|
389
|
+
* @Entity → null
|
|
390
|
+
*/
|
|
391
|
+
private static String annotationValue(AnnotationExpr a, Map<String, String> values, ParsedFile pf) {
|
|
392
|
+
if (a instanceof SingleMemberAnnotationExpr single) {
|
|
393
|
+
return stringOf(single.getMemberValue());
|
|
394
|
+
}
|
|
395
|
+
if (a instanceof NormalAnnotationExpr normal) {
|
|
396
|
+
String valueResult = null;
|
|
397
|
+
for (var pair : normal.getPairs()) {
|
|
398
|
+
String key = pair.getNameAsString();
|
|
399
|
+
if ((key.equals("value") || key.equals("path")) && valueResult == null) {
|
|
400
|
+
valueResult = stringOf(pair.getValue());
|
|
401
|
+
}
|
|
402
|
+
// @RequestMapping(method = RequestMethod.POST) — note: must NOT
|
|
403
|
+
// early-return on the path pair; method pairs come after it.
|
|
404
|
+
if (key.equals("method")) {
|
|
405
|
+
String v = enumNameOf(pair.getValue());
|
|
406
|
+
if (v != null) {
|
|
407
|
+
values.put(a.getNameAsString() + ".method", v);
|
|
408
|
+
}
|
|
409
|
+
}
|
|
410
|
+
}
|
|
411
|
+
return valueResult;
|
|
412
|
+
}
|
|
413
|
+
return null;
|
|
414
|
+
}
|
|
415
|
+
|
|
416
|
+
/**
|
|
417
|
+
* Verb of a method= member: RequestMethod.POST → POST (FieldAccessExpr),
|
|
418
|
+
* or a bare static-imported POST (NameExpr) — realworld uses both styles.
|
|
419
|
+
*/
|
|
420
|
+
private static String enumNameOf(Expression e) {
|
|
421
|
+
if (e instanceof FieldAccessExpr fa) {
|
|
422
|
+
return fa.getNameAsString();
|
|
423
|
+
}
|
|
424
|
+
if (e instanceof NameExpr n) {
|
|
425
|
+
return n.getNameAsString();
|
|
426
|
+
}
|
|
427
|
+
if (e instanceof com.github.javaparser.ast.expr.ArrayInitializerExpr arr) {
|
|
428
|
+
for (Expression v : arr.getValues()) {
|
|
429
|
+
String s = enumNameOf(v);
|
|
430
|
+
if (s != null) {
|
|
431
|
+
return s;
|
|
432
|
+
}
|
|
433
|
+
}
|
|
434
|
+
}
|
|
435
|
+
return null;
|
|
436
|
+
}
|
|
437
|
+
|
|
438
|
+
private static String stringOf(Expression e) {
|
|
439
|
+
if (e instanceof StringLiteralExpr s) {
|
|
440
|
+
return s.getValue();
|
|
441
|
+
}
|
|
442
|
+
if (e instanceof com.github.javaparser.ast.expr.ArrayInitializerExpr arr) {
|
|
443
|
+
for (Expression v : arr.getValues()) {
|
|
444
|
+
if (v instanceof StringLiteralExpr s) {
|
|
445
|
+
return s.getValue();
|
|
446
|
+
}
|
|
447
|
+
}
|
|
448
|
+
}
|
|
449
|
+
return null;
|
|
450
|
+
}
|
|
451
|
+
|
|
452
|
+
/**
|
|
453
|
+
* Root identifier of a scope chain for field/param lookup:
|
|
454
|
+
* userService.getX() → userService
|
|
455
|
+
* this.owners.save() → owners (this. prefix stripped — field lookup)
|
|
456
|
+
* config.props.save() → config (outer field; deep chains usually miss)
|
|
457
|
+
* service.getUser().x → null (method-call chain — unresolvable)
|
|
458
|
+
*/
|
|
459
|
+
private static String rootName(Expression scope) {
|
|
460
|
+
Expression cur = scope;
|
|
461
|
+
String lastSegment = null;
|
|
462
|
+
while (cur != null) {
|
|
463
|
+
if (cur instanceof NameExpr n) {
|
|
464
|
+
return n.getNameAsString();
|
|
465
|
+
}
|
|
466
|
+
if (cur instanceof FieldAccessExpr fa) {
|
|
467
|
+
lastSegment = fa.getNameAsString();
|
|
468
|
+
cur = fa.getScope();
|
|
469
|
+
if (cur instanceof com.github.javaparser.ast.expr.ThisExpr
|
|
470
|
+
|| cur instanceof com.github.javaparser.ast.expr.SuperExpr) {
|
|
471
|
+
return lastSegment;
|
|
472
|
+
}
|
|
473
|
+
continue;
|
|
474
|
+
}
|
|
475
|
+
return lastSegment; // method-call chain or other expr
|
|
476
|
+
}
|
|
477
|
+
return lastSegment;
|
|
478
|
+
}
|
|
479
|
+
|
|
480
|
+
/** Display string of a scope: userService, userService.a.b, or the method call name. */
|
|
481
|
+
private static String scopeName(Expression scope) {
|
|
482
|
+
if (scope instanceof NameExpr n) {
|
|
483
|
+
return n.getNameAsString();
|
|
484
|
+
}
|
|
485
|
+
if (scope instanceof FieldAccessExpr fa) {
|
|
486
|
+
return fa.toString();
|
|
487
|
+
}
|
|
488
|
+
if (scope instanceof MethodCallExpr mc) {
|
|
489
|
+
return mc.getNameAsString() + "()";
|
|
490
|
+
}
|
|
491
|
+
return scope.toString();
|
|
492
|
+
}
|
|
493
|
+
|
|
494
|
+
/** Used simple type names — feeds wildcard import resolution. */
|
|
495
|
+
public static void collectUsedTypes(ParsedFile pf, TypeInfo t) {
|
|
496
|
+
pf.usedTypeNames.add(t.extendedType);
|
|
497
|
+
pf.usedTypeNames.addAll(t.implementedTypes);
|
|
498
|
+
pf.usedTypeNames.addAll(t.typeParameters);
|
|
499
|
+
pf.usedTypeNames.addAll(t.annotations);
|
|
500
|
+
for (FieldInfo f : t.fields) {
|
|
501
|
+
pf.usedTypeNames.add(f.type);
|
|
502
|
+
pf.usedTypeNames.addAll(f.annotations);
|
|
503
|
+
}
|
|
504
|
+
for (MethodInfo m : t.methods) {
|
|
505
|
+
pf.usedTypeNames.add(m.returnType);
|
|
506
|
+
pf.usedTypeNames.addAll(m.annotations);
|
|
507
|
+
for (String p : m.params) {
|
|
508
|
+
pf.usedTypeNames.add(simpleTypeName(p));
|
|
509
|
+
}
|
|
510
|
+
pf.usedTypeNames.addAll(m.throwsTypes);
|
|
511
|
+
for (CallInfo c : m.calls) {
|
|
512
|
+
if (c.receiverName != null) {
|
|
513
|
+
pf.usedTypeNames.add(c.receiverName);
|
|
514
|
+
}
|
|
515
|
+
}
|
|
516
|
+
}
|
|
517
|
+
pf.usedTypeNames.remove("");
|
|
518
|
+
for (TypeInfo nested : t.nested) {
|
|
519
|
+
collectUsedTypes(pf, nested);
|
|
520
|
+
}
|
|
521
|
+
}
|
|
522
|
+
}
|
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
package devlens.extractor;
|
|
2
|
+
|
|
3
|
+
import java.io.IOException;
|
|
4
|
+
import java.nio.file.FileVisitResult;
|
|
5
|
+
import java.nio.file.Files;
|
|
6
|
+
import java.nio.file.Path;
|
|
7
|
+
import java.nio.file.SimpleFileVisitor;
|
|
8
|
+
import java.nio.file.attribute.BasicFileAttributes;
|
|
9
|
+
import java.util.ArrayList;
|
|
10
|
+
import java.util.Collections;
|
|
11
|
+
import java.util.List;
|
|
12
|
+
import java.util.Set;
|
|
13
|
+
|
|
14
|
+
/**
|
|
15
|
+
* SourceWalker — finds every .java file under the repo, pruning ignore
|
|
16
|
+
* directories AT THE FRONTIER (SKIP_SUBTREE before descending — the same
|
|
17
|
+
* "cut branches before you walk them" pattern as Python's dirnames[:] = []).
|
|
18
|
+
*/
|
|
19
|
+
public final class SourceWalker {
|
|
20
|
+
|
|
21
|
+
private static final Set<String> IGNORE_DIRS = Set.of(
|
|
22
|
+
"target", ".git", "build", "out", "node_modules", ".gradle",
|
|
23
|
+
".idea", ".vscode", "dist", ".mvn", ".settings", "bin", "obj", ".cache");
|
|
24
|
+
|
|
25
|
+
private SourceWalker() {}
|
|
26
|
+
|
|
27
|
+
/** All .java files relative to repoRoot, sorted (deterministic). */
|
|
28
|
+
public static List<String> walkJavaFiles(Path repoRoot) throws IOException {
|
|
29
|
+
List<String> rels = new ArrayList<>();
|
|
30
|
+
Files.walkFileTree(repoRoot, new SimpleFileVisitor<>() {
|
|
31
|
+
@Override
|
|
32
|
+
public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) {
|
|
33
|
+
if (!dir.equals(repoRoot) && IGNORE_DIRS.contains(dir.getFileName().toString())) {
|
|
34
|
+
return FileVisitResult.SKIP_SUBTREE;
|
|
35
|
+
}
|
|
36
|
+
return FileVisitResult.CONTINUE;
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
@Override
|
|
40
|
+
public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) {
|
|
41
|
+
if (file.getFileName().toString().endsWith(".java")) {
|
|
42
|
+
rels.add(repoRoot.relativize(file).toString().replace('\\', '/'));
|
|
43
|
+
}
|
|
44
|
+
return FileVisitResult.CONTINUE;
|
|
45
|
+
}
|
|
46
|
+
});
|
|
47
|
+
Collections.sort(rels);
|
|
48
|
+
return rels;
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
/** Test file = under src/test/ OR named *Test.java / *Tests.java / Test*.java. */
|
|
52
|
+
public static boolean isTestFile(String relPath) {
|
|
53
|
+
String lower = relPath.toLowerCase();
|
|
54
|
+
if (lower.contains("/src/test/")) {
|
|
55
|
+
return true;
|
|
56
|
+
}
|
|
57
|
+
String name = lower.substring(lower.lastIndexOf('/') + 1);
|
|
58
|
+
return name.startsWith("test")
|
|
59
|
+
|| name.endsWith("test.java")
|
|
60
|
+
|| name.endsWith("tests.java");
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
/**
|
|
64
|
+
* Maven-standard layout roots. Falls back to the repo root itself for
|
|
65
|
+
* plain layouts (single package dir, no src/main/java).
|
|
66
|
+
*/
|
|
67
|
+
public static List<Path> sourceRoots(Path repoRoot) {
|
|
68
|
+
List<Path> roots = new ArrayList<>();
|
|
69
|
+
Path main = repoRoot.resolve("src/main/java");
|
|
70
|
+
Path test = repoRoot.resolve("src/test/java");
|
|
71
|
+
if (Files.isDirectory(main)) {
|
|
72
|
+
roots.add(main);
|
|
73
|
+
}
|
|
74
|
+
if (Files.isDirectory(test)) {
|
|
75
|
+
roots.add(test);
|
|
76
|
+
}
|
|
77
|
+
if (roots.isEmpty()) {
|
|
78
|
+
roots.add(repoRoot);
|
|
79
|
+
}
|
|
80
|
+
return roots;
|
|
81
|
+
}
|
|
82
|
+
}
|