devlensio 0.6.2 → 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +89 -31
- package/dist/extractors/detectLanguage.d.ts +2 -0
- package/dist/extractors/detectLanguage.js +33 -0
- package/dist/extractors/index.d.ts +6 -0
- package/dist/extractors/index.js +116 -0
- package/dist/extractors/runner.d.ts +7 -0
- package/dist/extractors/runner.js +194 -0
- package/dist/extractors/types.d.ts +37 -0
- package/dist/extractors/types.js +8 -0
- package/dist/graph/buildLookup.d.ts +1 -0
- package/dist/graph/buildLookup.js +2 -1
- package/dist/graph/edges/callEdges.js +19 -5
- package/dist/graph/edges/callEdges.test.d.ts +1 -0
- package/dist/graph/edges/callEdges.test.js +200 -0
- package/dist/graph/edges/importEdges.js +12 -0
- package/dist/graph/edges/inheritanceEdges.d.ts +3 -0
- package/dist/graph/edges/inheritanceEdges.js +73 -0
- package/dist/graph/edges/inheritanceEdges.test.d.ts +1 -0
- package/dist/graph/edges/inheritanceEdges.test.js +140 -0
- package/dist/graph/index.js +4 -0
- package/dist/parser/classes.test.d.ts +1 -0
- package/dist/parser/classes.test.js +360 -0
- package/dist/parser/extractors/classes.d.ts +5 -0
- package/dist/parser/extractors/classes.js +241 -0
- package/dist/parser/index.d.ts +2 -0
- package/dist/parser/index.js +7 -1
- package/dist/pipeline/index.d.ts +2 -1
- package/dist/pipeline/index.js +30 -42
- package/dist/scoring/index.js +8 -0
- package/dist/scoring/index.test.js +43 -0
- package/dist/scoring/nodeScorer.js +2 -0
- package/dist/scoring/pruneDisconnected.d.ts +8 -0
- package/dist/scoring/pruneDisconnected.js +66 -0
- package/dist/scoring/pruneDisconnected.test.d.ts +1 -0
- package/dist/scoring/pruneDisconnected.test.js +123 -0
- package/dist/summarizer/prompts.d.ts +1 -1
- package/dist/types.d.ts +5 -5
- package/extractors/go/bin/darwin-amd64/devlens_go_extractor +0 -0
- package/extractors/go/bin/darwin-arm64/devlens_go_extractor +0 -0
- package/extractors/go/bin/linux-amd64/devlens_go_extractor +0 -0
- package/extractors/go/bin/linux-arm64/devlens_go_extractor +0 -0
- package/extractors/go/bin/windows-amd64/devlens_go_extractor.exe +0 -0
- package/extractors/go/build.mjs +43 -0
- package/extractors/go/calls.go +289 -0
- package/extractors/go/contract.go +140 -0
- package/extractors/go/extractor.go +161 -0
- package/extractors/go/fingerprint.go +167 -0
- package/extractors/go/go.mod +3 -0
- package/extractors/go/imports.go +90 -0
- package/extractors/go/inheritance.go +138 -0
- package/extractors/go/lookup.go +106 -0
- package/extractors/go/main.go +57 -0
- package/extractors/go/nodes.go +177 -0
- package/extractors/go/orm_edges.go +277 -0
- package/extractors/go/parser.go +469 -0
- package/extractors/go/routes.go +521 -0
- package/extractors/go/tests.go +45 -0
- package/extractors/go/thirdparty.go +138 -0
- package/extractors/go/typeload.go +178 -0
- package/extractors/go/walker.go +67 -0
- package/extractors/java/build.mjs +90 -0
- package/extractors/java/devlens_java_extractor.jar +0 -0
- package/extractors/java/src/devlens/extractor/Contract.java +171 -0
- package/extractors/java/src/devlens/extractor/Extractor.java +262 -0
- package/extractors/java/src/devlens/extractor/ExtractorResult.java +12 -0
- package/extractors/java/src/devlens/extractor/Fingerprint.java +239 -0
- package/extractors/java/src/devlens/extractor/LookupMaps.java +123 -0
- package/extractors/java/src/devlens/extractor/Main.java +66 -0
- package/extractors/java/src/devlens/extractor/Parser.java +522 -0
- package/extractors/java/src/devlens/extractor/SourceWalker.java +82 -0
- package/extractors/java/src/devlens/extractor/ThirdParty.java +141 -0
- package/extractors/java/src/devlens/extractor/TypeSolverFactory.java +43 -0
- package/extractors/java/src/devlens/extractor/edges/Calls.java +266 -0
- package/extractors/java/src/devlens/extractor/edges/Enrich.java +54 -0
- package/extractors/java/src/devlens/extractor/edges/Imports.java +154 -0
- package/extractors/java/src/devlens/extractor/edges/Inheritance.java +79 -0
- package/extractors/java/src/devlens/extractor/edges/OrmEdges.java +209 -0
- package/extractors/java/src/devlens/extractor/edges/Routes.java +143 -0
- package/extractors/java/src/devlens/extractor/edges/Tests.java +53 -0
- package/extractors/python/devlens_extractors_python/__init__.py +3 -0
- package/extractors/python/devlens_extractors_python/__main__.py +33 -0
- package/extractors/python/devlens_extractors_python/contract.py +94 -0
- package/extractors/python/devlens_extractors_python/edges/__init__.py +28 -0
- package/extractors/python/devlens_extractors_python/edges/calls.py +112 -0
- package/extractors/python/devlens_extractors_python/edges/enrich.py +44 -0
- package/extractors/python/devlens_extractors_python/edges/imports.py +155 -0
- package/extractors/python/devlens_extractors_python/edges/inheritance.py +97 -0
- package/extractors/python/devlens_extractors_python/edges/orm_edges.py +225 -0
- package/extractors/python/devlens_extractors_python/edges/routes/__init__.py +28 -0
- package/extractors/python/devlens_extractors_python/edges/routes/common.py +79 -0
- package/extractors/python/devlens_extractors_python/edges/routes/decorators.py +212 -0
- package/extractors/python/devlens_extractors_python/edges/routes/django_urls.py +213 -0
- package/extractors/python/devlens_extractors_python/edges/routes/drf.py +128 -0
- package/extractors/python/devlens_extractors_python/edges/tests.py +53 -0
- package/extractors/python/devlens_extractors_python/extractor.py +107 -0
- package/extractors/python/devlens_extractors_python/fingerprint.py +201 -0
- package/extractors/python/devlens_extractors_python/lookup.py +72 -0
- package/extractors/python/devlens_extractors_python/parser/__init__.py +72 -0
- package/extractors/python/devlens_extractors_python/parser/classes.py +109 -0
- package/extractors/python/devlens_extractors_python/parser/functions.py +163 -0
- package/extractors/python/devlens_extractors_python/parser/walker.py +30 -0
- package/extractors/python/devlens_extractors_python/third_party.py +103 -0
- package/extractors/python/pyproject.toml +16 -0
- package/extractors/python/setup.mjs +54 -0
- package/extractors/rust/Cargo.toml +29 -0
- package/extractors/rust/bin/darwin-amd64/devlens_rust_extractor +0 -0
- package/extractors/rust/bin/darwin-arm64/devlens_rust_extractor +0 -0
- package/extractors/rust/bin/linux-amd64/devlens_rust_extractor +0 -0
- package/extractors/rust/bin/linux-arm64/devlens_rust_extractor +0 -0
- package/extractors/rust/bin/windows-amd64/devlens_rust_extractor.exe +0 -0
- package/extractors/rust/build.mjs +85 -0
- package/extractors/rust/src/calls.rs +295 -0
- package/extractors/rust/src/contract.rs +236 -0
- package/extractors/rust/src/enrich.rs +107 -0
- package/extractors/rust/src/extractor.rs +199 -0
- package/extractors/rust/src/fingerprint.rs +238 -0
- package/extractors/rust/src/imports.rs +101 -0
- package/extractors/rust/src/inheritance.rs +233 -0
- package/extractors/rust/src/lookup.rs +226 -0
- package/extractors/rust/src/main.rs +53 -0
- package/extractors/rust/src/module_map.rs +131 -0
- package/extractors/rust/src/nodes.rs +174 -0
- package/extractors/rust/src/orm_edges.rs +125 -0
- package/extractors/rust/src/parser.rs +859 -0
- package/extractors/rust/src/routes.rs +1265 -0
- package/extractors/rust/src/tests.rs +109 -0
- package/extractors/rust/src/thirdparty.rs +114 -0
- package/extractors/rust/src/walker.rs +82 -0
- package/package.json +22 -4
|
@@ -0,0 +1,262 @@
|
|
|
1
|
+
package devlens.extractor;
|
|
2
|
+
|
|
3
|
+
import devlens.extractor.edges.Calls;
|
|
4
|
+
import devlens.extractor.edges.Enrich;
|
|
5
|
+
import devlens.extractor.edges.Imports;
|
|
6
|
+
import devlens.extractor.edges.Inheritance;
|
|
7
|
+
import devlens.extractor.edges.OrmEdges;
|
|
8
|
+
import devlens.extractor.edges.Routes;
|
|
9
|
+
import devlens.extractor.edges.Tests;
|
|
10
|
+
|
|
11
|
+
import java.io.IOException;
|
|
12
|
+
import java.nio.file.Path;
|
|
13
|
+
import java.util.ArrayList;
|
|
14
|
+
import java.util.Comparator;
|
|
15
|
+
import java.util.HashSet;
|
|
16
|
+
import java.util.LinkedHashMap;
|
|
17
|
+
import java.util.List;
|
|
18
|
+
import java.util.Map;
|
|
19
|
+
import java.util.Set;
|
|
20
|
+
|
|
21
|
+
/**
|
|
22
|
+
* Extractor — the pipeline orchestrator (mirrors extractor.py).
|
|
23
|
+
*
|
|
24
|
+
* Pipeline order (each stage consumes the shared LookupMaps, never re-walks
|
|
25
|
+
* the AST):
|
|
26
|
+
* 1. walk + parse every .java file → nodes (test files stay leaf nodes)
|
|
27
|
+
* 2. shared lookup maps + imports → IMPORTS edges + symbol maps + [mvn] nodes
|
|
28
|
+
* 3. calls → CALLS edges (lazy [mvn]/pkg::member nodes)
|
|
29
|
+
* 4. routes → ROUTE nodes + BackendRouteNodes + HANDLES edges
|
|
30
|
+
* 5. ORM → model metadata + READS_FROM/WRITES_TO edges
|
|
31
|
+
* 6. inheritance → EXTENDS / IMPLEMENTS
|
|
32
|
+
* 7. tests → TESTS edges
|
|
33
|
+
* 8. enrich → semantic metadata
|
|
34
|
+
* 9. collect third-party nodes (after call resolution), dedupe + sort
|
|
35
|
+
*/
|
|
36
|
+
public final class Extractor {
|
|
37
|
+
|
|
38
|
+
private static final String LANGUAGE = "java";
|
|
39
|
+
|
|
40
|
+
private final String repoPath;
|
|
41
|
+
private final List<String> allowedThirdPartyLibs;
|
|
42
|
+
|
|
43
|
+
public Extractor(String repoPath, List<String> allowedThirdPartyLibs) {
|
|
44
|
+
this.repoPath = repoPath;
|
|
45
|
+
this.allowedThirdPartyLibs = allowedThirdPartyLibs == null
|
|
46
|
+
? new ArrayList<>() : allowedThirdPartyLibs;
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
public ExtractorResult run() {
|
|
50
|
+
Path root = Path.of(repoPath);
|
|
51
|
+
TypeSolverFactory.configure(root);
|
|
52
|
+
|
|
53
|
+
Fingerprint fp = Fingerprint.detect(root);
|
|
54
|
+
List<Map<String, Object>> nodes = new ArrayList<>();
|
|
55
|
+
List<Map<String, Object>> edgesOut = new ArrayList<>();
|
|
56
|
+
List<Map<String, Object>> errors = new ArrayList<>();
|
|
57
|
+
List<Parser.ParsedFile> parsedFiles = new ArrayList<>();
|
|
58
|
+
Map<String, List<Parser.CallInfo>> methodCallFacts = new LinkedHashMap<>();
|
|
59
|
+
Map<String, List<String>> testCases = new LinkedHashMap<>();
|
|
60
|
+
int totalFiles = 0;
|
|
61
|
+
int skipped = 0;
|
|
62
|
+
|
|
63
|
+
// ── 1. walk + parse ─────────────────────────────────────────────
|
|
64
|
+
List<String> relPaths;
|
|
65
|
+
try {
|
|
66
|
+
relPaths = SourceWalker.walkJavaFiles(root);
|
|
67
|
+
} catch (IOException e) {
|
|
68
|
+
return new ExtractorResult(errorResult(fp, e.getMessage()), true);
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
for (String rel : relPaths) {
|
|
72
|
+
totalFiles++;
|
|
73
|
+
try {
|
|
74
|
+
Parser.ParsedFile pf = Parser.parseFile(root, rel);
|
|
75
|
+
parsedFiles.add(pf);
|
|
76
|
+
for (Parser.TypeInfo t : pf.types) {
|
|
77
|
+
Parser.collectUsedTypes(pf, t);
|
|
78
|
+
}
|
|
79
|
+
if (pf.isTest) {
|
|
80
|
+
List<String> cases = new ArrayList<>();
|
|
81
|
+
for (Parser.TypeInfo t : pf.allTypes()) {
|
|
82
|
+
for (Parser.MethodInfo m : t.methods) {
|
|
83
|
+
if (m.annotations.contains("Test")
|
|
84
|
+
|| m.name.toLowerCase().startsWith("test")) {
|
|
85
|
+
cases.add(m.name);
|
|
86
|
+
}
|
|
87
|
+
}
|
|
88
|
+
}
|
|
89
|
+
java.util.Collections.sort(cases);
|
|
90
|
+
testCases.put(rel, cases);
|
|
91
|
+
}
|
|
92
|
+
} catch (IOException e) {
|
|
93
|
+
skipped++;
|
|
94
|
+
errors.add(Contract.error(rel, e.getMessage()));
|
|
95
|
+
}
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
// ── 2. build nodes + call facts ─────────────────────────────────
|
|
99
|
+
for (Parser.ParsedFile pf : parsedFiles) {
|
|
100
|
+
nodes.add(Contract.fileNode(pf.relPath, pf.endLine, pf.isTest ? "TEST" : "FILE", LANGUAGE));
|
|
101
|
+
if (pf.isTest) {
|
|
102
|
+
continue; // test files are LEAF nodes — children feed metadata only
|
|
103
|
+
}
|
|
104
|
+
List<String> childIds = new ArrayList<>();
|
|
105
|
+
for (Parser.TypeInfo t : pf.allTypes()) {
|
|
106
|
+
Map<String, Object> typeNode = buildTypeNode(pf, t);
|
|
107
|
+
nodes.add(typeNode);
|
|
108
|
+
childIds.add((String) typeNode.get("id"));
|
|
109
|
+
@SuppressWarnings("unchecked")
|
|
110
|
+
List<String> childMethods = (List<String>) ((Map<String, Object>) typeNode.get("metadata")).get("childMethodIds");
|
|
111
|
+
for (Parser.MethodInfo m : t.methods) {
|
|
112
|
+
Map<String, Object> methodNode = buildMethodNode(pf, t, m);
|
|
113
|
+
nodes.add(methodNode);
|
|
114
|
+
childMethods.add(t.dottedName + "." + m.name);
|
|
115
|
+
methodCallFacts.put((String) methodNode.get("id"), m.calls);
|
|
116
|
+
}
|
|
117
|
+
}
|
|
118
|
+
// attach children to the FILE node
|
|
119
|
+
for (Map<String, Object> n : nodes) {
|
|
120
|
+
if (n.get("id").equals("file::" + pf.relPath)) {
|
|
121
|
+
@SuppressWarnings("unchecked")
|
|
122
|
+
List<String> cn = (List<String>) ((Map<String, Object>) n.get("metadata")).get("childNodeIds");
|
|
123
|
+
cn.addAll(childIds);
|
|
124
|
+
((Map<String, Object>) n.get("metadata")).put("nodeCount", childIds.size());
|
|
125
|
+
break;
|
|
126
|
+
}
|
|
127
|
+
}
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
// ── 3. shared lookup (built ONCE, consumed by every detector) ───
|
|
131
|
+
LookupMaps lookup = LookupMaps.build(parsedFiles, nodes);
|
|
132
|
+
lookup.methodCallFacts.putAll(methodCallFacts);
|
|
133
|
+
lookup.testCases.putAll(testCases);
|
|
134
|
+
ThirdParty tp = new ThirdParty(fp.rawDependencies, allowedThirdPartyLibs);
|
|
135
|
+
|
|
136
|
+
// ── 4. edges ───────────────────────────────────────────────────
|
|
137
|
+
Imports.resolve(parsedFiles, lookup, tp, edgesOut);
|
|
138
|
+
OrmEdges.detect(lookup, edgesOut); // models/repos BEFORE calls (repo CALLS target)
|
|
139
|
+
Calls.resolve(lookup, edgesOut);
|
|
140
|
+
Routes.RouteResult routeResult = Routes.resolve(parsedFiles, lookup, fp.framework);
|
|
141
|
+
edgesOut.addAll(routeResult.handlesEdges);
|
|
142
|
+
nodes.addAll(routeResult.routeNodes);
|
|
143
|
+
OrmEdges.consumerEdges(lookup, edgesOut); // repo-call → R/W edges
|
|
144
|
+
Inheritance.resolve(parsedFiles, lookup, tp, edgesOut);
|
|
145
|
+
Tests.resolve(lookup, edgesOut);
|
|
146
|
+
Enrich.enrich(lookup);
|
|
147
|
+
|
|
148
|
+
// ── 5. third-party nodes AFTER edge resolution (lazy members) ───
|
|
149
|
+
nodes.addAll(tp.allNodes());
|
|
150
|
+
|
|
151
|
+
// ── 6. dedupe edges (from|type|to) + deterministic sort ─────────
|
|
152
|
+
Set<String> seenEdges = new HashSet<>();
|
|
153
|
+
List<Map<String, Object>> uniqueEdges = new ArrayList<>();
|
|
154
|
+
for (Map<String, Object> e : edgesOut) {
|
|
155
|
+
if (seenEdges.add(Contract.edgeKey(e))) {
|
|
156
|
+
uniqueEdges.add(e);
|
|
157
|
+
}
|
|
158
|
+
}
|
|
159
|
+
edgesOut = uniqueEdges;
|
|
160
|
+
nodes.sort(Comparator.comparing(n -> (String) n.get("id")));
|
|
161
|
+
edgesOut.sort(Comparator.comparing(n -> Contract.edgeKey(n)));
|
|
162
|
+
routeResult.routes.sort(Comparator
|
|
163
|
+
.comparing((Map<String, Object> r) -> (String) r.get("httpMethod"))
|
|
164
|
+
.thenComparing(r -> (String) r.get("urlPath")));
|
|
165
|
+
|
|
166
|
+
Map<String, Object> result = new LinkedHashMap<>();
|
|
167
|
+
result.put("fingerprint", fp.toDict());
|
|
168
|
+
result.put("nodes", nodes);
|
|
169
|
+
result.put("edges", edgesOut);
|
|
170
|
+
result.put("routes", routeResult.routes);
|
|
171
|
+
Map<String, Object> stats = new LinkedHashMap<>();
|
|
172
|
+
stats.put("totalFiles", totalFiles);
|
|
173
|
+
stats.put("totalNodes", nodes.size());
|
|
174
|
+
stats.put("skippedFiles", skipped);
|
|
175
|
+
result.put("stats", stats);
|
|
176
|
+
result.put("errors", errors);
|
|
177
|
+
return new ExtractorResult(result, false);
|
|
178
|
+
}
|
|
179
|
+
|
|
180
|
+
private Map<String, Object> errorResult(Fingerprint fp, String message) {
|
|
181
|
+
Map<String, Object> result = new LinkedHashMap<>();
|
|
182
|
+
result.put("fingerprint", fp.toDict());
|
|
183
|
+
result.put("nodes", new ArrayList<Map<String, Object>>());
|
|
184
|
+
result.put("edges", new ArrayList<Map<String, Object>>());
|
|
185
|
+
result.put("routes", new ArrayList<Map<String, Object>>());
|
|
186
|
+
Map<String, Object> stats = new LinkedHashMap<>();
|
|
187
|
+
stats.put("totalFiles", 0);
|
|
188
|
+
stats.put("totalNodes", 0);
|
|
189
|
+
stats.put("skippedFiles", 0);
|
|
190
|
+
result.put("stats", stats);
|
|
191
|
+
List<Map<String, Object>> errors = new ArrayList<>();
|
|
192
|
+
errors.add(Contract.error("", message));
|
|
193
|
+
result.put("errors", errors);
|
|
194
|
+
return result;
|
|
195
|
+
}
|
|
196
|
+
|
|
197
|
+
private static String nodeTypeFor(Parser.TypeInfo t) {
|
|
198
|
+
return switch (t.kind) {
|
|
199
|
+
case "interface" -> "INTERFACE";
|
|
200
|
+
case "enum" -> "ENUM";
|
|
201
|
+
default -> "CLASS"; // class | record | annotation
|
|
202
|
+
};
|
|
203
|
+
}
|
|
204
|
+
|
|
205
|
+
private static Map<String, Object> buildTypeNode(Parser.ParsedFile pf, Parser.TypeInfo t) {
|
|
206
|
+
Map<String, Object> metadata = new LinkedHashMap<>();
|
|
207
|
+
metadata.put("kind", t.kind);
|
|
208
|
+
metadata.put("annotations", t.annotations);
|
|
209
|
+
metadata.put("isAbstract", t.isAbstract);
|
|
210
|
+
if (!t.typeParameters.isEmpty()) {
|
|
211
|
+
metadata.put("typeParameters", t.typeParameters);
|
|
212
|
+
}
|
|
213
|
+
if (t.extendedType != null && !t.extendedType.isEmpty()) {
|
|
214
|
+
metadata.put("extendsType", t.extendedType);
|
|
215
|
+
}
|
|
216
|
+
if (!t.implementedTypes.isEmpty()) {
|
|
217
|
+
metadata.put("implementsTypes", t.implementedTypes);
|
|
218
|
+
}
|
|
219
|
+
List<Map<String, Object>> fields = new ArrayList<>();
|
|
220
|
+
for (Parser.FieldInfo f : t.fields) {
|
|
221
|
+
Map<String, Object> fm = new LinkedHashMap<>();
|
|
222
|
+
fm.put("name", f.name);
|
|
223
|
+
fm.put("type", f.type);
|
|
224
|
+
if (!f.annotations.isEmpty()) {
|
|
225
|
+
fm.put("annotations", f.annotations);
|
|
226
|
+
}
|
|
227
|
+
fields.add(fm);
|
|
228
|
+
}
|
|
229
|
+
if (!fields.isEmpty()) {
|
|
230
|
+
metadata.put("fields", fields);
|
|
231
|
+
}
|
|
232
|
+
metadata.put("childMethodIds", new ArrayList<String>());
|
|
233
|
+
return Contract.codeNode(pf.relPath, t.dottedName, nodeTypeFor(t),
|
|
234
|
+
t.startLine, t.endLine, t.rawCode, metadata, LANGUAGE);
|
|
235
|
+
}
|
|
236
|
+
|
|
237
|
+
private static Map<String, Object> buildMethodNode(Parser.ParsedFile pf, Parser.TypeInfo t, Parser.MethodInfo m) {
|
|
238
|
+
Map<String, Object> metadata = new LinkedHashMap<>();
|
|
239
|
+
metadata.put("parentClass", t.dottedName);
|
|
240
|
+
metadata.put("isConstructor", m.isConstructor);
|
|
241
|
+
metadata.put("isStatic", m.isStatic);
|
|
242
|
+
metadata.put("isAbstract", m.isAbstract);
|
|
243
|
+
if (!m.annotations.isEmpty()) {
|
|
244
|
+
metadata.put("annotations", m.annotations);
|
|
245
|
+
}
|
|
246
|
+
if (!m.params.isEmpty()) {
|
|
247
|
+
metadata.put("params", m.params);
|
|
248
|
+
}
|
|
249
|
+
if (!m.returnType.isEmpty()) {
|
|
250
|
+
metadata.put("returnType", m.returnType);
|
|
251
|
+
}
|
|
252
|
+
if (!m.throwsTypes.isEmpty()) {
|
|
253
|
+
metadata.put("throws", m.throwsTypes);
|
|
254
|
+
}
|
|
255
|
+
if (!m.calls.isEmpty()) {
|
|
256
|
+
List<String> callStrings = m.calls.stream().map(c -> c.name).distinct().sorted().toList();
|
|
257
|
+
metadata.put("calls", callStrings);
|
|
258
|
+
}
|
|
259
|
+
return Contract.codeNode(pf.relPath, t.dottedName + "." + m.name, "METHOD",
|
|
260
|
+
m.startLine, m.endLine, m.rawCode, metadata, LANGUAGE);
|
|
261
|
+
}
|
|
262
|
+
}
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
package devlens.extractor;
|
|
2
|
+
|
|
3
|
+
import java.util.Map;
|
|
4
|
+
|
|
5
|
+
/**
|
|
6
|
+
* Immutable container for the extractor's output + a fatal flag.
|
|
7
|
+
* `fatal` is true only when the pipeline itself blew up (the engine turns
|
|
8
|
+
* that into a non-zero exit); per-file parse problems go into errors[] and
|
|
9
|
+
* are non-fatal by design.
|
|
10
|
+
*/
|
|
11
|
+
public record ExtractorResult(Map<String, Object> json, boolean fatal) {
|
|
12
|
+
}
|
|
@@ -0,0 +1,239 @@
|
|
|
1
|
+
package devlens.extractor;
|
|
2
|
+
|
|
3
|
+
import org.w3c.dom.Document;
|
|
4
|
+
import org.w3c.dom.Element;
|
|
5
|
+
import org.w3c.dom.Node;
|
|
6
|
+
import org.w3c.dom.NodeList;
|
|
7
|
+
|
|
8
|
+
import javax.xml.parsers.DocumentBuilder;
|
|
9
|
+
import javax.xml.parsers.DocumentBuilderFactory;
|
|
10
|
+
import java.io.ByteArrayInputStream;
|
|
11
|
+
import java.nio.charset.StandardCharsets;
|
|
12
|
+
import java.nio.file.Files;
|
|
13
|
+
import java.nio.file.Path;
|
|
14
|
+
import java.util.ArrayList;
|
|
15
|
+
import java.util.LinkedHashMap;
|
|
16
|
+
import java.util.LinkedHashSet;
|
|
17
|
+
import java.util.List;
|
|
18
|
+
import java.util.Map;
|
|
19
|
+
import java.util.Set;
|
|
20
|
+
import java.util.TreeMap;
|
|
21
|
+
import java.util.regex.Matcher;
|
|
22
|
+
import java.util.regex.Pattern;
|
|
23
|
+
|
|
24
|
+
/**
|
|
25
|
+
* Fingerprint — manifest → framework / projectType / databases / rawDependencies.
|
|
26
|
+
*
|
|
27
|
+
* pom.xml is parsed with the JDK's secure DOM (never executed — same rule as
|
|
28
|
+
* ast-parsing setup.py). build.gradle has no structured format, so it gets a
|
|
29
|
+
* documented LOW-fidelity regex pass; pom wins when both exist.
|
|
30
|
+
*
|
|
31
|
+
* rawDependencies: g:a → version (TreeMap = deterministic output).
|
|
32
|
+
*/
|
|
33
|
+
public final class Fingerprint {
|
|
34
|
+
|
|
35
|
+
public String language = "java";
|
|
36
|
+
public String projectType = "unknown";
|
|
37
|
+
public String framework = "unknown";
|
|
38
|
+
public String router = "none";
|
|
39
|
+
public List<String> stateManagement = new ArrayList<>();
|
|
40
|
+
public List<String> dataFetching = new ArrayList<>();
|
|
41
|
+
public List<String> databases = new ArrayList<>();
|
|
42
|
+
public Map<String, String> rawDependencies = new TreeMap<>();
|
|
43
|
+
|
|
44
|
+
/** Deps seen (g:a) — drives framework/database classification. */
|
|
45
|
+
private final Set<String> deps = new LinkedHashSet<>();
|
|
46
|
+
private boolean hasSpringBootPlugin;
|
|
47
|
+
|
|
48
|
+
public static Fingerprint detect(Path repoPath) {
|
|
49
|
+
Fingerprint fp = new Fingerprint();
|
|
50
|
+
Path pom = repoPath.resolve("pom.xml");
|
|
51
|
+
Path gradle = repoPath.resolve("build.gradle");
|
|
52
|
+
Path gradleKts = repoPath.resolve("build.gradle.kts");
|
|
53
|
+
if (Files.isRegularFile(pom)) {
|
|
54
|
+
fp.parsePom(pom);
|
|
55
|
+
} else if (Files.isRegularFile(gradle)) {
|
|
56
|
+
fp.parseGradle(gradle);
|
|
57
|
+
} else if (Files.isRegularFile(gradleKts)) {
|
|
58
|
+
fp.parseGradle(gradleKts);
|
|
59
|
+
}
|
|
60
|
+
fp.classify();
|
|
61
|
+
return fp;
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
// ─────────────────────────── pom.xml ───────────────────────────
|
|
65
|
+
|
|
66
|
+
private void parsePom(Path pom) {
|
|
67
|
+
try {
|
|
68
|
+
String xml = Files.readString(pom, StandardCharsets.UTF_8);
|
|
69
|
+
DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
|
|
70
|
+
dbf.setFeature("http://apache.org/xml/features/disallow-doctype-decl", true);
|
|
71
|
+
dbf.setFeature("http://xml.org/sax/features/external-general-entities", false);
|
|
72
|
+
dbf.setFeature("http://xml.org/sax/features/external-parameter-entities", false);
|
|
73
|
+
dbf.setXIncludeAware(false);
|
|
74
|
+
dbf.setExpandEntityReferences(false);
|
|
75
|
+
DocumentBuilder builder = dbf.newDocumentBuilder();
|
|
76
|
+
Document doc = builder.parse(new ByteArrayInputStream(xml.getBytes(StandardCharsets.UTF_8)));
|
|
77
|
+
|
|
78
|
+
String groupId = text(doc, "project > groupId");
|
|
79
|
+
String artifactId = text(doc, "project > artifactId");
|
|
80
|
+
String version = text(doc, "project > version");
|
|
81
|
+
if ((groupId == null || version == null) && hasElement(doc, "project > parent")) {
|
|
82
|
+
// version/groupId often live on the parent (spring-boot-starter-parent)
|
|
83
|
+
if (groupId == null) groupId = text(doc, "project > parent > groupId");
|
|
84
|
+
if (version == null) version = text(doc, "project > parent > version");
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
NodeList depNodes = doc.getElementsByTagName("dependency");
|
|
88
|
+
for (int i = 0; i < depNodes.getLength(); i++) {
|
|
89
|
+
Element dep = (Element) depNodes.item(i);
|
|
90
|
+
String g = childText(dep, "groupId");
|
|
91
|
+
String a = childText(dep, "artifactId");
|
|
92
|
+
String v = childText(dep, "version");
|
|
93
|
+
if (g != null && a != null) {
|
|
94
|
+
String key = g + ":" + a;
|
|
95
|
+
deps.add(key);
|
|
96
|
+
// versions are usually inherited from the parent (spring-boot-starter-parent)
|
|
97
|
+
rawDependencies.put(key, v != null && !v.isBlank() ? v : "unknown");
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
NodeList pluginNodes = doc.getElementsByTagName("plugin");
|
|
102
|
+
for (int i = 0; i < pluginNodes.getLength(); i++) {
|
|
103
|
+
Element plugin = (Element) pluginNodes.item(i);
|
|
104
|
+
String g = childText(plugin, "groupId");
|
|
105
|
+
String a = childText(plugin, "artifactId");
|
|
106
|
+
if ("org.springframework.boot".equals(g) && "spring-boot-maven-plugin".equals(a)) {
|
|
107
|
+
hasSpringBootPlugin = true;
|
|
108
|
+
}
|
|
109
|
+
}
|
|
110
|
+
} catch (Exception e) {
|
|
111
|
+
// unparseable pom → stay "unknown" (non-fatal)
|
|
112
|
+
}
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
// ─────────────────────────── build.gradle ───────────────────────────
|
|
116
|
+
|
|
117
|
+
private static final Pattern DEP_PATTERN = Pattern.compile(
|
|
118
|
+
"(?:implementation|api|compileOnly|runtimeOnly|testImplementation|testRuntimeOnly|classpath)" +
|
|
119
|
+
"\\s*\\(?\\s*['\"]([^:'\"]+):([^:'\"]+):([^'\"]+)['\"]");
|
|
120
|
+
private static final Pattern PLUGIN_PATTERN = Pattern.compile(
|
|
121
|
+
"id\\s*['\"]([^'\"]+)['\"]");
|
|
122
|
+
|
|
123
|
+
private void parseGradle(Path gradle) {
|
|
124
|
+
try {
|
|
125
|
+
String text = Files.readString(gradle, StandardCharsets.UTF_8);
|
|
126
|
+
Matcher dm = DEP_PATTERN.matcher(text);
|
|
127
|
+
while (dm.find()) {
|
|
128
|
+
String key = dm.group(1) + ":" + dm.group(2);
|
|
129
|
+
deps.add(key);
|
|
130
|
+
rawDependencies.put(key, dm.group(3));
|
|
131
|
+
}
|
|
132
|
+
Matcher pm = PLUGIN_PATTERN.matcher(text);
|
|
133
|
+
while (pm.find()) {
|
|
134
|
+
if ("org.springframework.boot".equals(pm.group(1))) {
|
|
135
|
+
hasSpringBootPlugin = true;
|
|
136
|
+
}
|
|
137
|
+
}
|
|
138
|
+
} catch (Exception e) {
|
|
139
|
+
// non-fatal
|
|
140
|
+
}
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
// ─────────────────────────── classification ───────────────────────────
|
|
144
|
+
|
|
145
|
+
private static final Map<String, String> DATABASE_DRIVERS = Map.ofEntries(
|
|
146
|
+
Map.entry("org.postgresql:postgresql", "postgresql"),
|
|
147
|
+
Map.entry("com.h2database:h2", "h2"),
|
|
148
|
+
Map.entry("com.mysql:mysql-connector-j", "mysql"),
|
|
149
|
+
Map.entry("mysql:mysql-connector-java", "mysql"),
|
|
150
|
+
Map.entry("org.mariadb.jdbc:mariadb-java-client", "mariadb"),
|
|
151
|
+
Map.entry("org.xerial:sqlite-jdbc", "sqlite"),
|
|
152
|
+
Map.entry("com.microsoft.sqlserver:mssql-jdbc", "sqlserver"),
|
|
153
|
+
Map.entry("org.mongodb:mongodb-driver-sync", "mongodb"),
|
|
154
|
+
Map.entry("org.mongodb:mongodb-driver-core", "mongodb"));
|
|
155
|
+
|
|
156
|
+
private void classify() {
|
|
157
|
+
boolean springBoot = hasSpringBootPlugin || deps.stream().anyMatch(d -> d.startsWith("org.springframework.boot:"));
|
|
158
|
+
boolean springMvc = deps.stream().anyMatch(d ->
|
|
159
|
+
d.equals("org.springframework:spring-web") || d.equals("org.springframework:spring-webmvc"));
|
|
160
|
+
if (springBoot) {
|
|
161
|
+
framework = "spring-boot";
|
|
162
|
+
} else if (springMvc) {
|
|
163
|
+
framework = "spring-mvc";
|
|
164
|
+
}
|
|
165
|
+
|
|
166
|
+
boolean jpa = deps.stream().anyMatch(d ->
|
|
167
|
+
d.startsWith("org.springframework.boot:spring-boot-starter-data-jpa")
|
|
168
|
+
|| d.startsWith("jakarta.persistence:")
|
|
169
|
+
|| d.startsWith("javax.persistence:")
|
|
170
|
+
|| d.startsWith("org.hibernate.orm:")
|
|
171
|
+
|| d.equals("org.hibernate:hibernate-core"));
|
|
172
|
+
if (jpa) {
|
|
173
|
+
databases.add("jpa");
|
|
174
|
+
}
|
|
175
|
+
for (Map.Entry<String, String> e : DATABASE_DRIVERS.entrySet()) {
|
|
176
|
+
if (deps.contains(e.getKey()) && !databases.contains(e.getValue())) {
|
|
177
|
+
databases.add(e.getValue());
|
|
178
|
+
}
|
|
179
|
+
}
|
|
180
|
+
if (deps.stream().anyMatch(d -> d.startsWith("org.springframework.boot:spring-boot-starter-data-redis"))
|
|
181
|
+
|| deps.contains("redis.clients:jedis")) {
|
|
182
|
+
databases.add("redis");
|
|
183
|
+
}
|
|
184
|
+
java.util.Collections.sort(databases);
|
|
185
|
+
|
|
186
|
+
if (springBoot || springMvc) {
|
|
187
|
+
projectType = "backend";
|
|
188
|
+
} else if (!rawDependencies.isEmpty()) {
|
|
189
|
+
projectType = "library";
|
|
190
|
+
}
|
|
191
|
+
}
|
|
192
|
+
|
|
193
|
+
// ─────────────────────────── output ───────────────────────────
|
|
194
|
+
|
|
195
|
+
public Map<String, Object> toDict() {
|
|
196
|
+
Map<String, Object> out = new LinkedHashMap<>();
|
|
197
|
+
out.put("language", language);
|
|
198
|
+
out.put("projectType", projectType);
|
|
199
|
+
out.put("framework", framework);
|
|
200
|
+
out.put("router", router);
|
|
201
|
+
out.put("stateManagement", stateManagement);
|
|
202
|
+
out.put("dataFetching", dataFetching);
|
|
203
|
+
out.put("databases", databases);
|
|
204
|
+
out.put("rawDependencies", rawDependencies);
|
|
205
|
+
return out;
|
|
206
|
+
}
|
|
207
|
+
|
|
208
|
+
// ─────────────────────────── xml helpers ───────────────────────────
|
|
209
|
+
|
|
210
|
+
private static String text(Document doc, String path) {
|
|
211
|
+
Node n = doc.getDocumentElement();
|
|
212
|
+
for (String part : path.split(" > ")) {
|
|
213
|
+
if (!(n instanceof Element el)) return null;
|
|
214
|
+
n = firstChildElement(el, part);
|
|
215
|
+
if (n == null) return null;
|
|
216
|
+
}
|
|
217
|
+
return n.getTextContent() == null ? null : n.getTextContent().trim();
|
|
218
|
+
}
|
|
219
|
+
|
|
220
|
+
private static boolean hasElement(Document doc, String path) {
|
|
221
|
+
return text(doc, path) != null;
|
|
222
|
+
}
|
|
223
|
+
|
|
224
|
+
private static Element firstChildElement(Element parent, String name) {
|
|
225
|
+
NodeList children = parent.getChildNodes();
|
|
226
|
+
for (int i = 0; i < children.getLength(); i++) {
|
|
227
|
+
Node n = children.item(i);
|
|
228
|
+
if (n instanceof Element e && name.equals(e.getTagName())) {
|
|
229
|
+
return e;
|
|
230
|
+
}
|
|
231
|
+
}
|
|
232
|
+
return null;
|
|
233
|
+
}
|
|
234
|
+
|
|
235
|
+
private static String childText(Element parent, String name) {
|
|
236
|
+
Element e = firstChildElement(parent, name);
|
|
237
|
+
return e == null ? null : (e.getTextContent() == null ? null : e.getTextContent().trim());
|
|
238
|
+
}
|
|
239
|
+
}
|
|
@@ -0,0 +1,123 @@
|
|
|
1
|
+
package devlens.extractor;
|
|
2
|
+
|
|
3
|
+
import java.util.ArrayList;
|
|
4
|
+
import java.util.HashMap;
|
|
5
|
+
import java.util.LinkedHashMap;
|
|
6
|
+
import java.util.List;
|
|
7
|
+
import java.util.Map;
|
|
8
|
+
|
|
9
|
+
/**
|
|
10
|
+
* LookupMaps — ONE shared index built once from the parsed files + nodes,
|
|
11
|
+
* consumed by every edge detector (mirrors lookup.py). Edge detectors never
|
|
12
|
+
* re-walk ASTs; they answer "which node is this name?" through these maps.
|
|
13
|
+
*
|
|
14
|
+
* Java specifics:
|
|
15
|
+
* typeMap FQCN (pkg + dottedName) → relPath — import resolution is
|
|
16
|
+
* exact because Java imports are fully qualified
|
|
17
|
+
* typeDottedMap FQCN → dottedName (Outer.Inner) — needed to rebuild node ids
|
|
18
|
+
* symbolMaps per-file {simple alias → node id} — written by Imports,
|
|
19
|
+
* consumed by Calls / Inheritance / Tests
|
|
20
|
+
*/
|
|
21
|
+
public final class LookupMaps {
|
|
22
|
+
|
|
23
|
+
public final Map<String, List<String>> nodesByName = new HashMap<>();
|
|
24
|
+
public final Map<String, Map<String, String>> nodesByFile = new HashMap<>();
|
|
25
|
+
public final Map<String, Map<String, Object>> nodeById = new LinkedHashMap<>();
|
|
26
|
+
public final Map<String, Map<String, Object>> fileNodesByPath = new HashMap<>();
|
|
27
|
+
public final Map<String, Map<String, String>> symbolMaps = new HashMap<>();
|
|
28
|
+
public final Map<String, String> typeMap = new HashMap<>();
|
|
29
|
+
public final Map<String, String> typeDottedMap = new HashMap<>();
|
|
30
|
+
public final List<String> methodNodes = new ArrayList<>();
|
|
31
|
+
/** method node id → parse-time call facts (CallInfo) */
|
|
32
|
+
public final Map<String, List<Parser.CallInfo>> methodCallFacts = new LinkedHashMap<>();
|
|
33
|
+
/** test file relPath → test method names (metadata.testCases) */
|
|
34
|
+
public final Map<String, List<String>> testCases = new HashMap<>();
|
|
35
|
+
|
|
36
|
+
public static LookupMaps build(List<Parser.ParsedFile> parsedFiles,
|
|
37
|
+
List<Map<String, Object>> nodes) {
|
|
38
|
+
LookupMaps lm = new LookupMaps();
|
|
39
|
+
for (Map<String, Object> node : nodes) {
|
|
40
|
+
String id = (String) node.get("id");
|
|
41
|
+
String type = (String) node.get("type");
|
|
42
|
+
lm.nodeById.put(id, node);
|
|
43
|
+
if ("FILE".equals(type) || "TEST".equals(type)) {
|
|
44
|
+
lm.fileNodesByPath.put((String) node.get("filePath"), node);
|
|
45
|
+
continue;
|
|
46
|
+
}
|
|
47
|
+
String name = (String) node.get("name");
|
|
48
|
+
if (name != null) {
|
|
49
|
+
lm.nodesByName.computeIfAbsent(name, k -> new ArrayList<>()).add(id);
|
|
50
|
+
}
|
|
51
|
+
if ("METHOD".equals(type)) {
|
|
52
|
+
lm.methodNodes.add(id);
|
|
53
|
+
} else if (isTypeNode(type)) {
|
|
54
|
+
String filePath = (String) node.get("filePath");
|
|
55
|
+
lm.nodesByFile.computeIfAbsent(filePath, k -> new HashMap<>())
|
|
56
|
+
.put(name, id);
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
for (Parser.ParsedFile pf : parsedFiles) {
|
|
60
|
+
for (Parser.TypeInfo t : pf.allTypes()) {
|
|
61
|
+
String fqcn = pf.packageName == null || pf.packageName.isEmpty()
|
|
62
|
+
? t.dottedName : pf.packageName + "." + t.dottedName;
|
|
63
|
+
lm.typeMap.put(fqcn, pf.relPath);
|
|
64
|
+
lm.typeDottedMap.put(fqcn, t.dottedName);
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
return lm;
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
private static boolean isTypeNode(String type) {
|
|
71
|
+
return "CLASS".equals(type) || "INTERFACE".equals(type) || "ENUM".equals(type);
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
public Map<String, String> symbolMap(String relPath) {
|
|
75
|
+
return symbolMaps.computeIfAbsent(relPath, k -> new HashMap<>());
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
/**
|
|
79
|
+
* Simple-name disambiguation for fallback resolution: among the candidate
|
|
80
|
+
* node ids sharing this simple name, pick the one whose file shares the
|
|
81
|
+
* longest directory prefix with the caller (same file wins).
|
|
82
|
+
*/
|
|
83
|
+
public String closestByName(String name, String callerRelPath) {
|
|
84
|
+
List<String> candidates = nodesByName.get(name);
|
|
85
|
+
if (candidates == null || candidates.isEmpty()) {
|
|
86
|
+
return null;
|
|
87
|
+
}
|
|
88
|
+
if (candidates.size() == 1) {
|
|
89
|
+
return candidates.get(0);
|
|
90
|
+
}
|
|
91
|
+
String best = candidates.get(0);
|
|
92
|
+
int bestScore = -1;
|
|
93
|
+
for (String id : candidates) {
|
|
94
|
+
Map<String, Object> node = nodeById.get(id);
|
|
95
|
+
if (node == null) {
|
|
96
|
+
continue;
|
|
97
|
+
}
|
|
98
|
+
String filePath = (String) node.get("filePath");
|
|
99
|
+
int score = commonDirPrefixLength(filePath, callerRelPath);
|
|
100
|
+
if (score > bestScore) {
|
|
101
|
+
bestScore = score;
|
|
102
|
+
best = id;
|
|
103
|
+
}
|
|
104
|
+
}
|
|
105
|
+
return best;
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
private static int commonDirPrefixLength(String a, String b) {
|
|
109
|
+
if (a.equals(b)) {
|
|
110
|
+
return Integer.MAX_VALUE / 2;
|
|
111
|
+
}
|
|
112
|
+
String[] aa = a.split("/");
|
|
113
|
+
String[] bb = b.split("/");
|
|
114
|
+
int n = 0;
|
|
115
|
+
for (int i = 0; i < Math.min(aa.length, bb.length) - 1; i++) {
|
|
116
|
+
if (!aa[i].equals(bb[i])) {
|
|
117
|
+
break;
|
|
118
|
+
}
|
|
119
|
+
n++;
|
|
120
|
+
}
|
|
121
|
+
return n;
|
|
122
|
+
}
|
|
123
|
+
}
|