@guanlin4924/devflow-pi 0.1.76 → 0.1.77
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/plugins/devflow-intel/skills/workflow-code-index/scripts/adapters/base.py +20 -0
- package/plugins/devflow-intel/skills/workflow-code-index/scripts/adapters/generic_text.py +1 -1
- package/plugins/devflow-intel/skills/workflow-code-index/scripts/adapters/go_ast.py +20 -0
- package/plugins/devflow-intel/skills/workflow-code-index/scripts/adapters/helpers/DevflowJavaAstHelper.java +96 -0
- package/plugins/devflow-intel/skills/workflow-code-index/scripts/adapters/helpers/go_ast_helper.go +96 -0
- package/plugins/devflow-intel/skills/workflow-code-index/scripts/adapters/helpers/php_ast_helper.php +91 -0
- package/plugins/devflow-intel/skills/workflow-code-index/scripts/adapters/helpers/typescript_ast_helper.cjs +123 -0
- package/plugins/devflow-intel/skills/workflow-code-index/scripts/adapters/java_ast.py +37 -0
- package/plugins/devflow-intel/skills/workflow-code-index/scripts/adapters/native_process.py +328 -0
- package/plugins/devflow-intel/skills/workflow-code-index/scripts/adapters/php_ast.py +55 -0
- package/plugins/devflow-intel/skills/workflow-code-index/scripts/adapters/python_ast.py +10 -0
- package/plugins/devflow-intel/skills/workflow-code-index/scripts/adapters/registry.py +66 -50
- package/plugins/devflow-intel/skills/workflow-code-index/scripts/adapters/structured_scope.py +1 -1
- package/plugins/devflow-intel/skills/workflow-code-index/scripts/adapters/typescript_ast.py +31 -0
- package/plugins/devflow-intel/skills/workflow-code-index/scripts/code_index.py +5 -8
- package/src/constants.ts +1 -1
package/package.json
CHANGED
|
@@ -12,6 +12,9 @@ class ParseResult:
|
|
|
12
12
|
# in a separate linker pass.
|
|
13
13
|
entities: List[Dict] = field(default_factory=list)
|
|
14
14
|
flows: List[Dict] = field(default_factory=list)
|
|
15
|
+
# Parser that actually produced this result. This can differ from the
|
|
16
|
+
# selected adapter when a native parser fails and a fallback is used.
|
|
17
|
+
actual_parser: str = ""
|
|
15
18
|
|
|
16
19
|
|
|
17
20
|
class Adapter:
|
|
@@ -29,6 +32,23 @@ class Adapter:
|
|
|
29
32
|
def version(self) -> str:
|
|
30
33
|
return "0"
|
|
31
34
|
|
|
35
|
+
def parser_for_file(self, codebase, file_record) -> str:
|
|
36
|
+
"""Return the parser that will actually produce this file's facts.
|
|
37
|
+
|
|
38
|
+
Native adapters override this when their runtime dependency is not
|
|
39
|
+
available and a declared fallback will be used instead. Keeping this
|
|
40
|
+
separate from ``parser_id`` prevents fallback output from being
|
|
41
|
+
labelled as native AST output.
|
|
42
|
+
"""
|
|
43
|
+
return self.parser_id
|
|
44
|
+
|
|
45
|
+
def capability(self) -> Dict:
|
|
46
|
+
return {
|
|
47
|
+
"parser": self.parser_id,
|
|
48
|
+
"native": False,
|
|
49
|
+
"available": True,
|
|
50
|
+
}
|
|
51
|
+
|
|
32
52
|
@property
|
|
33
53
|
def parser_id(self) -> str:
|
|
34
54
|
return f"{self.name}@{self.version()}"
|
|
@@ -155,7 +155,7 @@ class GenericTextAdapter(Adapter):
|
|
|
155
155
|
}
|
|
156
156
|
)
|
|
157
157
|
|
|
158
|
-
return ParseResult(symbols=symbols, relations=relations, diagnostics=diagnostics)
|
|
158
|
+
return ParseResult(symbols=symbols, relations=relations, diagnostics=diagnostics, actual_parser=self.parser_id)
|
|
159
159
|
|
|
160
160
|
def _extract_symbols(self, codebase: str, language: str, path: str, text: str, content_hash: str) -> List[Dict]:
|
|
161
161
|
symbols: List[Dict] = []
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
import shutil
|
|
2
|
+
from typing import List, Tuple
|
|
3
|
+
|
|
4
|
+
from adapters.native_process import HELPER_DIR, NativeProcessAdapter
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
class GoAstAdapter(NativeProcessAdapter):
|
|
8
|
+
name = "go-parser-ast"
|
|
9
|
+
languages = ("go",)
|
|
10
|
+
native_description = "Go standard library go parser AST; declarations, imports, and calls"
|
|
11
|
+
|
|
12
|
+
def version(self) -> str:
|
|
13
|
+
return "1"
|
|
14
|
+
|
|
15
|
+
def check_available(self, codebase=None) -> Tuple[bool, str]:
|
|
16
|
+
go = shutil.which("go")
|
|
17
|
+
return (True, "go/parser available") if go else (False, "go executable not found")
|
|
18
|
+
|
|
19
|
+
def command(self, codebase=None) -> List[str]:
|
|
20
|
+
return [shutil.which("go") or "go", "run", str(HELPER_DIR / "go_ast_helper.go")]
|
|
@@ -0,0 +1,96 @@
|
|
|
1
|
+
import java.io.*;
|
|
2
|
+
import java.net.URI;
|
|
3
|
+
import java.nio.charset.StandardCharsets;
|
|
4
|
+
import java.util.*;
|
|
5
|
+
import javax.tools.*;
|
|
6
|
+
import javax.lang.model.element.Modifier;
|
|
7
|
+
import com.sun.source.tree.*;
|
|
8
|
+
import com.sun.source.util.*;
|
|
9
|
+
|
|
10
|
+
public final class DevflowJavaAstHelper {
|
|
11
|
+
private static String json(String value) {
|
|
12
|
+
if (value == null) return "\"\"";
|
|
13
|
+
StringBuilder out = new StringBuilder("\"");
|
|
14
|
+
for (char c : value.toCharArray()) {
|
|
15
|
+
switch (c) {
|
|
16
|
+
case '\\': out.append("\\\\"); break;
|
|
17
|
+
case '"': out.append("\\\""); break;
|
|
18
|
+
case '\n': out.append("\\n"); break;
|
|
19
|
+
case '\r': out.append("\\r"); break;
|
|
20
|
+
case '\t': out.append("\\t"); break;
|
|
21
|
+
default:
|
|
22
|
+
if (c < 32) out.append(String.format("\\u%04x", (int)c)); else out.append(c);
|
|
23
|
+
}
|
|
24
|
+
}
|
|
25
|
+
return out.append('"').toString();
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
private static final class Source extends SimpleJavaFileObject {
|
|
29
|
+
final String text;
|
|
30
|
+
Source(String path, String text) { super(URI.create("string:///" + path.replace(' ', '_')), Kind.SOURCE); this.text = text; }
|
|
31
|
+
public CharSequence getCharContent(boolean ignoreEncodingErrors) { return text; }
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
private static final class Decl {
|
|
35
|
+
String key, parent, kind, name, signature, nativeKind, visibility; int start, end;
|
|
36
|
+
String toJson() { return "{\"key\":"+json(key)+",\"parent\":"+json(parent)+",\"kind\":"+json(kind)+",\"name\":"+json(name)+",\"line_start\":"+start+",\"line_end\":"+end+",\"signature\":"+json(signature)+",\"native_kind\":"+json(nativeKind)+",\"visibility\":"+json(visibility)+"}"; }
|
|
37
|
+
}
|
|
38
|
+
private static final class Rel {
|
|
39
|
+
String owner, type, target, confidence, resolution; int line;
|
|
40
|
+
String toJson() { return "{\"owner\":"+json(owner)+",\"type\":"+json(type)+",\"target\":"+json(target)+",\"line\":"+line+",\"confidence\":"+json(confidence)+",\"metadata\":{\"resolution\":"+json(resolution)+"}}"; }
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
private static String parse(String path, String sourceText) throws Exception {
|
|
44
|
+
JavaCompiler compiler = ToolProvider.getSystemJavaCompiler();
|
|
45
|
+
if (compiler == null) throw new IllegalStateException("JDK compiler API unavailable");
|
|
46
|
+
DiagnosticCollector<JavaFileObject> diagnostics = new DiagnosticCollector<>();
|
|
47
|
+
JavacTask task = (JavacTask) compiler.getTask(null, null, diagnostics, List.of("-proc:none"), null, List.of(new Source(path, sourceText)));
|
|
48
|
+
CompilationUnitTree unit = task.parse().iterator().next();
|
|
49
|
+
Trees trees = Trees.instance(task); SourcePositions positions = trees.getSourcePositions(); LineMap lines = unit.getLineMap();
|
|
50
|
+
List<Decl> declarations = new ArrayList<>(); List<Rel> relations = new ArrayList<>();
|
|
51
|
+
new TreePathScanner<Void, String>() {
|
|
52
|
+
int next = 1;
|
|
53
|
+
int line(Tree tree) { long p = positions.getStartPosition(unit, tree); return p < 0 ? 1 : (int)lines.getLineNumber(p); }
|
|
54
|
+
int end(Tree tree) { long p = positions.getEndPosition(unit, tree); return p < 0 ? line(tree) : (int)lines.getLineNumber(Math.max(0, p - 1)); }
|
|
55
|
+
String visibility(Set<Modifier> modifiers) { if (modifiers.contains(Modifier.PUBLIC)) return "public"; if (modifiers.contains(Modifier.PROTECTED)) return "protected"; if (modifiers.contains(Modifier.PRIVATE)) return "private"; return ""; }
|
|
56
|
+
String addDecl(Tree tree, String parent, String kind, String name, String signature, Set<Modifier> modifiers) {
|
|
57
|
+
Decl d = new Decl(); d.key = String.valueOf(next++); d.parent = parent == null ? "" : parent; d.kind = kind; d.name = name; d.signature = signature; d.nativeKind = tree.getKind().name(); d.visibility = visibility(modifiers); d.start = line(tree); d.end = Math.max(d.start, end(tree)); declarations.add(d); return d.key;
|
|
58
|
+
}
|
|
59
|
+
void addRel(String owner, String type, String target, Tree tree, String confidence, String resolution) { Rel r = new Rel(); r.owner=owner==null?"":owner; r.type=type; r.target=target; r.line=line(tree); r.confidence=confidence; r.resolution=resolution; relations.add(r); }
|
|
60
|
+
@Override public Void visitClass(ClassTree node, String owner) {
|
|
61
|
+
String kind = node.getKind() == Tree.Kind.INTERFACE || node.getKind() == Tree.Kind.ANNOTATION_TYPE ? "interface" : "class";
|
|
62
|
+
String key = addDecl(node, owner, kind, node.getSimpleName().toString(), node.getSimpleName()+"()", node.getModifiers().getFlags());
|
|
63
|
+
if (node.getExtendsClause() != null) addRel(key, "extends", "type:"+node.getExtendsClause(), node.getExtendsClause(), "inferred", "syntax_name");
|
|
64
|
+
for (Tree inherited : node.getImplementsClause()) addRel(key, "implements", "type:"+inherited, inherited, "inferred", "syntax_name");
|
|
65
|
+
return super.visitClass(node, key);
|
|
66
|
+
}
|
|
67
|
+
@Override public Void visitMethod(MethodTree node, String owner) {
|
|
68
|
+
String name = node.getName().toString(); String kind = name.toLowerCase().startsWith("test") ? "test" : "method";
|
|
69
|
+
String key = addDecl(node, owner, kind, name, name+"("+String.join(", ", node.getParameters().stream().map(v -> v.getName().toString()).toList())+")", node.getModifiers().getFlags());
|
|
70
|
+
return super.visitMethod(node, key);
|
|
71
|
+
}
|
|
72
|
+
@Override public Void visitImport(ImportTree node, String owner) { addRel(owner, "imports", "import:"+node.getQualifiedIdentifier(), node, "exact", "import_tree"); return super.visitImport(node, owner); }
|
|
73
|
+
@Override public Void visitMethodInvocation(MethodInvocationTree node, String owner) {
|
|
74
|
+
String target=node.getMethodSelect().toString(); boolean receiver=target.contains("."); addRel(owner, "calls", "call:"+target, node, receiver?"heuristic":"inferred", receiver?"receiver_unresolved":"lexical_name"); return super.visitMethodInvocation(node, owner);
|
|
75
|
+
}
|
|
76
|
+
@Override public Void visitNewClass(NewClassTree node, String owner) { addRel(owner, "calls", "call:"+node.getIdentifier(), node, "inferred", "constructor_syntax"); return super.visitNewClass(node, owner); }
|
|
77
|
+
}.scan(unit, "");
|
|
78
|
+
StringBuilder out = new StringBuilder("{\"ok\":true,\"declarations\":[");
|
|
79
|
+
for (int i=0;i<declarations.size();i++) { if(i>0) out.append(','); out.append(declarations.get(i).toJson()); }
|
|
80
|
+
out.append("],\"relations\":["); for(int i=0;i<relations.size();i++){if(i>0)out.append(',');out.append(relations.get(i).toJson());}
|
|
81
|
+
out.append("],\"diagnostics\":["); boolean first=true; for(Diagnostic<? extends JavaFileObject> d: diagnostics.getDiagnostics()){if(!first)out.append(',');first=false;out.append("{\"line\":").append(Math.max(1,d.getLineNumber())).append(",\"severity\":\"warning\",\"kind\":\"parse_failed\",\"message\":").append(json(d.getMessage(Locale.ROOT))).append('}');}
|
|
82
|
+
return out.append("]}").toString();
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
public static void main(String[] args) throws Exception {
|
|
86
|
+
BufferedReader reader = new BufferedReader(new InputStreamReader(System.in, StandardCharsets.UTF_8)); String line;
|
|
87
|
+
while ((line = reader.readLine()) != null) {
|
|
88
|
+
try {
|
|
89
|
+
String[] parts=line.split("\\t",2); if(parts.length!=2)throw new IllegalArgumentException("invalid request");
|
|
90
|
+
String path=new String(Base64.getDecoder().decode(parts[0]),StandardCharsets.UTF_8); String source=new String(Base64.getDecoder().decode(parts[1]),StandardCharsets.UTF_8);
|
|
91
|
+
System.out.println(parse(path,source));
|
|
92
|
+
} catch(Exception error) { System.out.println("{\"ok\":false,\"error\":"+json(error.toString())+"}"); }
|
|
93
|
+
System.out.flush();
|
|
94
|
+
}
|
|
95
|
+
}
|
|
96
|
+
}
|
package/plugins/devflow-intel/skills/workflow-code-index/scripts/adapters/helpers/go_ast_helper.go
ADDED
|
@@ -0,0 +1,96 @@
|
|
|
1
|
+
package main
|
|
2
|
+
|
|
3
|
+
import (
|
|
4
|
+
"bufio"
|
|
5
|
+
"encoding/json"
|
|
6
|
+
"fmt"
|
|
7
|
+
"go/ast"
|
|
8
|
+
"go/parser"
|
|
9
|
+
"go/token"
|
|
10
|
+
"os"
|
|
11
|
+
"strings"
|
|
12
|
+
)
|
|
13
|
+
|
|
14
|
+
type Request struct { Path string `json:"path"`; Source string `json:"source"` }
|
|
15
|
+
type Declaration struct { Key string `json:"key"`; Parent string `json:"parent"`; Kind string `json:"kind"`; Name string `json:"name"`; LineStart int `json:"line_start"`; LineEnd int `json:"line_end"`; Signature string `json:"signature"`; NativeKind string `json:"native_kind"` }
|
|
16
|
+
type Relation struct { Owner string `json:"owner"`; Type string `json:"type"`; Target string `json:"target"`; Line int `json:"line"`; Confidence string `json:"confidence"`; Metadata map[string]any `json:"metadata"` }
|
|
17
|
+
type Diagnostic struct { Line int `json:"line"`; Severity string `json:"severity"`; Kind string `json:"kind"`; Message string `json:"message"` }
|
|
18
|
+
type Response struct { Ok bool `json:"ok"`; Error string `json:"error,omitempty"`; Declarations []Declaration `json:"declarations,omitempty"`; Relations []Relation `json:"relations,omitempty"`; Diagnostics []Diagnostic `json:"diagnostics,omitempty"` }
|
|
19
|
+
|
|
20
|
+
func expressionName(expr ast.Expr) string {
|
|
21
|
+
switch value := expr.(type) {
|
|
22
|
+
case *ast.Ident: return value.Name
|
|
23
|
+
case *ast.SelectorExpr:
|
|
24
|
+
parent := expressionName(value.X)
|
|
25
|
+
if parent != "" { return parent + "." + value.Sel.Name }
|
|
26
|
+
return value.Sel.Name
|
|
27
|
+
case *ast.IndexExpr: return expressionName(value.X)
|
|
28
|
+
case *ast.IndexListExpr: return expressionName(value.X)
|
|
29
|
+
}
|
|
30
|
+
return ""
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
func parseRequest(request Request) Response {
|
|
34
|
+
fset := token.NewFileSet()
|
|
35
|
+
file, err := parser.ParseFile(fset, request.Path, request.Source, parser.AllErrors|parser.ParseComments)
|
|
36
|
+
response := Response{Ok: true, Declarations: []Declaration{}, Relations: []Relation{}, Diagnostics: []Diagnostic{}}
|
|
37
|
+
if err != nil {
|
|
38
|
+
line := 1
|
|
39
|
+
if list, ok := err.(interface{ Error() string }); ok { _ = list }
|
|
40
|
+
response.Diagnostics = append(response.Diagnostics, Diagnostic{Line: line, Severity: "warning", Kind: "parse_failed", Message: err.Error()})
|
|
41
|
+
}
|
|
42
|
+
if file == nil { response.Ok = false; response.Error = "go/parser produced no syntax tree"; return response }
|
|
43
|
+
keys := map[ast.Node]string{}
|
|
44
|
+
next := 1
|
|
45
|
+
for _, declaration := range file.Decls {
|
|
46
|
+
switch node := declaration.(type) {
|
|
47
|
+
case *ast.FuncDecl:
|
|
48
|
+
key := fmt.Sprintf("%d", next); next++
|
|
49
|
+
keys[node] = key
|
|
50
|
+
kind := "function"
|
|
51
|
+
if node.Recv != nil { kind = "method" }
|
|
52
|
+
if strings.HasPrefix(strings.ToLower(node.Name.Name), "test") { kind = "test" }
|
|
53
|
+
response.Declarations = append(response.Declarations, Declaration{Key:key, Kind:kind, Name:node.Name.Name, LineStart:fset.Position(node.Pos()).Line, LineEnd:fset.Position(node.End()).Line, Signature:node.Name.Name+"()", NativeKind:"FuncDecl"})
|
|
54
|
+
case *ast.GenDecl:
|
|
55
|
+
for _, spec := range node.Specs {
|
|
56
|
+
if typeSpec, ok := spec.(*ast.TypeSpec); ok {
|
|
57
|
+
key := fmt.Sprintf("%d", next); next++
|
|
58
|
+
keys[typeSpec] = key
|
|
59
|
+
kind := "class"
|
|
60
|
+
native := "TypeSpec"
|
|
61
|
+
switch typeSpec.Type.(type) { case *ast.StructType: kind="struct"; native="StructType"; case *ast.InterfaceType: kind="interface"; native="InterfaceType" }
|
|
62
|
+
response.Declarations = append(response.Declarations, Declaration{Key:key, Kind:kind, Name:typeSpec.Name.Name, LineStart:fset.Position(typeSpec.Pos()).Line, LineEnd:fset.Position(typeSpec.End()).Line, Signature:typeSpec.Name.Name, NativeKind:native})
|
|
63
|
+
}
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
for _, imp := range file.Imports {
|
|
68
|
+
target := strings.Trim(imp.Path.Value, "\"")
|
|
69
|
+
response.Relations = append(response.Relations, Relation{Type:"imports", Target:"import:"+target, Line:fset.Position(imp.Pos()).Line, Confidence:"exact", Metadata:map[string]any{"resolution":"import_spec"}})
|
|
70
|
+
}
|
|
71
|
+
for _, declaration := range file.Decls {
|
|
72
|
+
owner := keys[declaration]
|
|
73
|
+
ast.Inspect(declaration, func(node ast.Node) bool {
|
|
74
|
+
call, ok := node.(*ast.CallExpr)
|
|
75
|
+
if !ok { return true }
|
|
76
|
+
name := expressionName(call.Fun)
|
|
77
|
+
if name == "" { return true }
|
|
78
|
+
confidence := "inferred"; resolution := "lexical_name"
|
|
79
|
+
if strings.Contains(name, ".") { confidence = "heuristic"; resolution = "receiver_unresolved" }
|
|
80
|
+
response.Relations = append(response.Relations, Relation{Owner:owner, Type:"calls", Target:"call:"+name, Line:fset.Position(call.Pos()).Line, Confidence:confidence, Metadata:map[string]any{"resolution":resolution}})
|
|
81
|
+
return true
|
|
82
|
+
})
|
|
83
|
+
}
|
|
84
|
+
return response
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
func main() {
|
|
88
|
+
scanner := bufio.NewScanner(os.Stdin)
|
|
89
|
+
buffer := make([]byte, 64*1024); scanner.Buffer(buffer, 64*1024*1024)
|
|
90
|
+
encoder := json.NewEncoder(os.Stdout)
|
|
91
|
+
for scanner.Scan() {
|
|
92
|
+
var request Request
|
|
93
|
+
if err := json.Unmarshal(scanner.Bytes(), &request); err != nil { encoder.Encode(Response{Ok:false, Error:err.Error()}); continue }
|
|
94
|
+
encoder.Encode(parseRequest(request))
|
|
95
|
+
}
|
|
96
|
+
}
|
package/plugins/devflow-intel/skills/workflow-code-index/scripts/adapters/helpers/php_ast_helper.php
ADDED
|
@@ -0,0 +1,91 @@
|
|
|
1
|
+
<?php
|
|
2
|
+
declare(strict_types=1);
|
|
3
|
+
|
|
4
|
+
$autoload = $argv[1] ?? '';
|
|
5
|
+
if (!$autoload || !is_file($autoload)) {
|
|
6
|
+
fwrite(STDOUT, json_encode(['ok' => false, 'error' => 'nikic/PHP-Parser autoload file not found']) . "\n");
|
|
7
|
+
exit(2);
|
|
8
|
+
}
|
|
9
|
+
require_once $autoload;
|
|
10
|
+
|
|
11
|
+
function node_name($node): string {
|
|
12
|
+
if ($node instanceof PhpParser\Node\Name || $node instanceof PhpParser\Node\Identifier || $node instanceof PhpParser\Node\VarLikeIdentifier) {
|
|
13
|
+
return $node->toString();
|
|
14
|
+
}
|
|
15
|
+
if ($node instanceof PhpParser\Node\Expr\Variable && is_string($node->name)) return '$' . $node->name;
|
|
16
|
+
if (is_string($node)) return $node;
|
|
17
|
+
return '';
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
function relation_target($node): string {
|
|
21
|
+
if ($node instanceof PhpParser\Node\Name || $node instanceof PhpParser\Node\Identifier) return $node->toString();
|
|
22
|
+
if ($node instanceof PhpParser\Node\Expr\Variable) return node_name($node);
|
|
23
|
+
if ($node instanceof PhpParser\Node\Expr\PropertyFetch || $node instanceof PhpParser\Node\Expr\StaticPropertyFetch) {
|
|
24
|
+
return trim(relation_target($node->var ?? $node->class) . '->' . node_name($node->name), '->');
|
|
25
|
+
}
|
|
26
|
+
return '';
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
function parser_instance() {
|
|
30
|
+
$factory = new PhpParser\ParserFactory();
|
|
31
|
+
if (method_exists($factory, 'createForNewestSupportedVersion')) return $factory->createForNewestSupportedVersion();
|
|
32
|
+
return $factory->create(PhpParser\ParserFactory::PREFER_PHP7);
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
function parse_source(string $source): array {
|
|
36
|
+
$declarations = []; $relations = []; $diagnostics = []; $next = 1;
|
|
37
|
+
try { $nodes = parser_instance()->parse($source) ?? []; }
|
|
38
|
+
catch (PhpParser\Error $error) {
|
|
39
|
+
return ['ok' => true, 'declarations' => [], 'relations' => [], 'diagnostics' => [[
|
|
40
|
+
'line' => max(1, $error->getStartLine()), 'severity' => 'warning', 'kind' => 'parse_failed', 'message' => $error->getMessage()
|
|
41
|
+
]]];
|
|
42
|
+
}
|
|
43
|
+
$walk = function ($node, string $owner = '') use (&$walk, &$declarations, &$relations, &$next): void {
|
|
44
|
+
if (!$node instanceof PhpParser\Node) return;
|
|
45
|
+
$current = $owner; $kind = null; $name = '';
|
|
46
|
+
if ($node instanceof PhpParser\Node\Stmt\Class_) { $kind = 'class'; $name = node_name($node->name) ?: 'anonymous'; }
|
|
47
|
+
elseif ($node instanceof PhpParser\Node\Stmt\Interface_) { $kind = 'interface'; $name = node_name($node->name); }
|
|
48
|
+
elseif ($node instanceof PhpParser\Node\Stmt\Trait_) { $kind = 'trait'; $name = node_name($node->name); }
|
|
49
|
+
elseif ($node instanceof PhpParser\Node\Stmt\Function_) { $kind = str_starts_with(strtolower(node_name($node->name)), 'test') ? 'test' : 'function'; $name = node_name($node->name); }
|
|
50
|
+
elseif ($node instanceof PhpParser\Node\Stmt\ClassMethod) { $kind = str_starts_with(strtolower(node_name($node->name)), 'test') ? 'test' : 'method'; $name = node_name($node->name); }
|
|
51
|
+
elseif ($node instanceof PhpParser\Node\Expr\Closure || $node instanceof PhpParser\Node\Expr\ArrowFunction) { $kind = 'function'; $name = 'anonymous'; }
|
|
52
|
+
if ($kind !== null) {
|
|
53
|
+
$current = (string)$next++;
|
|
54
|
+
$params = property_exists($node, 'params') ? array_map(fn($p) => node_name($p->var), $node->params) : [];
|
|
55
|
+
$visibility = method_exists($node, 'isPrivate') && $node->isPrivate() ? 'private' : (method_exists($node, 'isProtected') && $node->isProtected() ? 'protected' : (method_exists($node, 'isPublic') && $node->isPublic() ? 'public' : ''));
|
|
56
|
+
$declarations[] = ['key'=>$current,'parent'=>$owner,'kind'=>$kind,'name'=>$name,'line_start'=>max(1,$node->getStartLine()),'line_end'=>max(1,$node->getEndLine()),'signature'=>$name.'('.implode(', ',$params).')','visibility'=>$visibility,'native_kind'=>$node->getType()];
|
|
57
|
+
if ($node instanceof PhpParser\Node\Stmt\Class_) {
|
|
58
|
+
if ($node->extends) $relations[] = ['owner'=>$current,'type'=>'extends','target'=>'type:'.node_name($node->extends),'line'=>$node->extends->getStartLine(),'confidence'=>'inferred','metadata'=>['resolution'=>'syntax_name']];
|
|
59
|
+
foreach ($node->implements as $type) $relations[] = ['owner'=>$current,'type'=>'implements','target'=>'type:'.node_name($type),'line'=>$type->getStartLine(),'confidence'=>'inferred','metadata'=>['resolution'=>'syntax_name']];
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
if ($node instanceof PhpParser\Node\Stmt\Use_ || $node instanceof PhpParser\Node\Stmt\GroupUse) {
|
|
63
|
+
foreach ($node->uses as $use) $relations[] = ['owner'=>$owner,'type'=>'imports','target'=>'import:'.node_name($use->name),'line'=>$use->getStartLine(),'confidence'=>'exact','metadata'=>['resolution'=>'use_tree']];
|
|
64
|
+
} elseif ($node instanceof PhpParser\Node\Expr\FuncCall) {
|
|
65
|
+
$target = node_name($node->name); if ($target) $relations[] = ['owner'=>$owner,'type'=>'calls','target'=>'call:'.$target,'line'=>$node->getStartLine(),'confidence'=>'inferred','metadata'=>['resolution'=>'syntax_name']];
|
|
66
|
+
} elseif ($node instanceof PhpParser\Node\Expr\MethodCall || $node instanceof PhpParser\Node\Expr\NullsafeMethodCall) {
|
|
67
|
+
$target = node_name($node->name); if ($target) $relations[] = ['owner'=>$owner,'type'=>'calls','target'=>'call:'.$target,'line'=>$node->getStartLine(),'confidence'=>'heuristic','metadata'=>['resolution'=>'receiver_unresolved']];
|
|
68
|
+
} elseif ($node instanceof PhpParser\Node\Expr\StaticCall) {
|
|
69
|
+
$target = node_name($node->class).'::'.node_name($node->name); if ($target !== '::') $relations[] = ['owner'=>$owner,'type'=>'calls','target'=>'call:'.$target,'line'=>$node->getStartLine(),'confidence'=>'heuristic','metadata'=>['resolution'=>'static_receiver_unresolved']];
|
|
70
|
+
} elseif ($node instanceof PhpParser\Node\Expr\New_) {
|
|
71
|
+
$target = node_name($node->class); if ($target) $relations[] = ['owner'=>$owner,'type'=>'calls','target'=>'call:'.$target,'line'=>$node->getStartLine(),'confidence'=>'inferred','metadata'=>['resolution'=>'constructor_syntax']];
|
|
72
|
+
}
|
|
73
|
+
foreach ($node->getSubNodeNames() as $field) {
|
|
74
|
+
$value = $node->$field;
|
|
75
|
+
if ($value instanceof PhpParser\Node) $walk($value, $current);
|
|
76
|
+
elseif (is_array($value)) foreach ($value as $child) if ($child instanceof PhpParser\Node) $walk($child, $current);
|
|
77
|
+
}
|
|
78
|
+
};
|
|
79
|
+
foreach ($nodes as $node) $walk($node, '');
|
|
80
|
+
return ['ok'=>true,'declarations'=>$declarations,'relations'=>$relations,'diagnostics'=>$diagnostics];
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
while (($line = fgets(STDIN)) !== false) {
|
|
84
|
+
try {
|
|
85
|
+
$request = json_decode($line, true, 512, JSON_THROW_ON_ERROR);
|
|
86
|
+
echo json_encode(parse_source((string)($request['source'] ?? '')), JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES) . "\n";
|
|
87
|
+
} catch (Throwable $error) {
|
|
88
|
+
echo json_encode(['ok'=>false,'error'=>$error->getMessage()], JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES) . "\n";
|
|
89
|
+
}
|
|
90
|
+
flush();
|
|
91
|
+
}
|
|
@@ -0,0 +1,123 @@
|
|
|
1
|
+
const readline = require("readline");
|
|
2
|
+
|
|
3
|
+
let ts;
|
|
4
|
+
try {
|
|
5
|
+
ts = require("typescript");
|
|
6
|
+
} catch (error) {
|
|
7
|
+
process.stdout.write(JSON.stringify({ ok: false, error: `typescript compiler API unavailable: ${error.message}` }) + "\n");
|
|
8
|
+
process.exit(2);
|
|
9
|
+
}
|
|
10
|
+
|
|
11
|
+
function scriptKind(path) {
|
|
12
|
+
if (path.endsWith(".tsx")) return ts.ScriptKind.TSX;
|
|
13
|
+
if (path.endsWith(".jsx")) return ts.ScriptKind.JSX;
|
|
14
|
+
if (path.endsWith(".js") || path.endsWith(".mjs") || path.endsWith(".cjs")) return ts.ScriptKind.JS;
|
|
15
|
+
return ts.ScriptKind.TS;
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
function parse(input) {
|
|
19
|
+
const source = ts.createSourceFile(input.path, input.source, ts.ScriptTarget.Latest, true, scriptKind(input.path));
|
|
20
|
+
const declarations = [];
|
|
21
|
+
const relations = [];
|
|
22
|
+
let nextKey = 1;
|
|
23
|
+
|
|
24
|
+
function line(node) {
|
|
25
|
+
return source.getLineAndCharacterOfPosition(node.getStart(source, false)).line + 1;
|
|
26
|
+
}
|
|
27
|
+
function endLine(node) {
|
|
28
|
+
return source.getLineAndCharacterOfPosition(node.end).line + 1;
|
|
29
|
+
}
|
|
30
|
+
function nodeName(node) {
|
|
31
|
+
if (node.name && ts.isIdentifier(node.name)) return node.name.text;
|
|
32
|
+
if (node.name && ts.isStringLiteralLike(node.name)) return node.name.text;
|
|
33
|
+
return "anonymous";
|
|
34
|
+
}
|
|
35
|
+
function signature(node, name) {
|
|
36
|
+
if (node.parameters) {
|
|
37
|
+
const parameters = node.parameters.map((p) => p.name.getText(source));
|
|
38
|
+
return `${name}(${parameters.join(", ")})`;
|
|
39
|
+
}
|
|
40
|
+
return `${name}()`;
|
|
41
|
+
}
|
|
42
|
+
function visibility(node) {
|
|
43
|
+
const modifiers = node.modifiers || [];
|
|
44
|
+
if (modifiers.some((m) => m.kind === ts.SyntaxKind.PrivateKeyword)) return "private";
|
|
45
|
+
if (modifiers.some((m) => m.kind === ts.SyntaxKind.ProtectedKeyword)) return "protected";
|
|
46
|
+
if (modifiers.some((m) => m.kind === ts.SyntaxKind.PublicKeyword)) return "public";
|
|
47
|
+
return "";
|
|
48
|
+
}
|
|
49
|
+
function declarationKind(node) {
|
|
50
|
+
if (ts.isClassDeclaration(node) || ts.isEnumDeclaration(node)) return "class";
|
|
51
|
+
if (ts.isInterfaceDeclaration(node)) return "interface";
|
|
52
|
+
if (ts.isModuleDeclaration(node)) return "module";
|
|
53
|
+
if (ts.isMethodDeclaration(node) || ts.isMethodSignature(node) || ts.isConstructorDeclaration(node)) return "method";
|
|
54
|
+
if (ts.isFunctionDeclaration(node) || ts.isFunctionExpression(node) || ts.isArrowFunction(node)) return "function";
|
|
55
|
+
return null;
|
|
56
|
+
}
|
|
57
|
+
function callTarget(expression) {
|
|
58
|
+
if (ts.isIdentifier(expression)) return { target: expression.text, confidence: "inferred", resolution: "lexical_name" };
|
|
59
|
+
if (ts.isPropertyAccessExpression(expression)) return { target: expression.getText(source), confidence: "heuristic", resolution: "receiver_unresolved" };
|
|
60
|
+
if (ts.isElementAccessExpression(expression)) return { target: expression.getText(source), confidence: "heuristic", resolution: "dynamic_element" };
|
|
61
|
+
return null;
|
|
62
|
+
}
|
|
63
|
+
function addRelation(owner, type, target, at, confidence, metadata = {}) {
|
|
64
|
+
relations.push({ owner: owner || "", type, target, line: at, confidence, metadata });
|
|
65
|
+
}
|
|
66
|
+
function visit(node, owner = "") {
|
|
67
|
+
let current = owner;
|
|
68
|
+
let kind = declarationKind(node);
|
|
69
|
+
let name = nodeName(node);
|
|
70
|
+
if (ts.isVariableDeclaration(node) && node.initializer && (ts.isArrowFunction(node.initializer) || ts.isFunctionExpression(node.initializer))) {
|
|
71
|
+
kind = "function";
|
|
72
|
+
name = node.name.getText(source);
|
|
73
|
+
}
|
|
74
|
+
if (kind) {
|
|
75
|
+
const key = String(nextKey++);
|
|
76
|
+
declarations.push({
|
|
77
|
+
key,
|
|
78
|
+
parent: owner,
|
|
79
|
+
kind: name.toLowerCase().startsWith("test") && kind === "function" ? "test" : kind,
|
|
80
|
+
name,
|
|
81
|
+
line_start: line(node),
|
|
82
|
+
line_end: endLine(node),
|
|
83
|
+
signature: signature(node, name),
|
|
84
|
+
visibility: visibility(node),
|
|
85
|
+
native_kind: ts.SyntaxKind[node.kind],
|
|
86
|
+
});
|
|
87
|
+
current = key;
|
|
88
|
+
const heritage = node.heritageClauses || [];
|
|
89
|
+
for (const clause of heritage) {
|
|
90
|
+
const type = clause.token === ts.SyntaxKind.ImplementsKeyword ? "implements" : "extends";
|
|
91
|
+
for (const inherited of clause.types) {
|
|
92
|
+
addRelation(current, type, `type:${inherited.expression.getText(source)}`, line(inherited), "inferred", { resolution: "syntax_name" });
|
|
93
|
+
}
|
|
94
|
+
}
|
|
95
|
+
}
|
|
96
|
+
if (ts.isImportDeclaration(node) && node.moduleSpecifier && ts.isStringLiteralLike(node.moduleSpecifier)) {
|
|
97
|
+
addRelation(owner, "imports", `import:${node.moduleSpecifier.text}`, line(node), "exact", { resolution: "module_specifier" });
|
|
98
|
+
} else if (ts.isExportDeclaration(node) && node.moduleSpecifier && ts.isStringLiteralLike(node.moduleSpecifier)) {
|
|
99
|
+
addRelation(owner, "imports", `import:${node.moduleSpecifier.text}`, line(node), "exact", { resolution: "export_module_specifier" });
|
|
100
|
+
} else if (ts.isCallExpression(node)) {
|
|
101
|
+
const target = callTarget(node.expression);
|
|
102
|
+
if (target) addRelation(owner, "calls", `call:${target.target}`, line(node), target.confidence, { resolution: target.resolution });
|
|
103
|
+
}
|
|
104
|
+
ts.forEachChild(node, (child) => visit(child, current));
|
|
105
|
+
}
|
|
106
|
+
visit(source, "");
|
|
107
|
+
const diagnostics = source.parseDiagnostics.map((diagnostic) => ({
|
|
108
|
+
line: diagnostic.start == null ? 1 : source.getLineAndCharacterOfPosition(diagnostic.start).line + 1,
|
|
109
|
+
severity: "warning",
|
|
110
|
+
kind: "parse_failed",
|
|
111
|
+
message: ts.flattenDiagnosticMessageText(diagnostic.messageText, " "),
|
|
112
|
+
}));
|
|
113
|
+
return { ok: true, declarations, relations, diagnostics };
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
const rl = readline.createInterface({ input: process.stdin, crlfDelay: Infinity });
|
|
117
|
+
rl.on("line", (line) => {
|
|
118
|
+
try {
|
|
119
|
+
process.stdout.write(JSON.stringify(parse(JSON.parse(line))) + "\n");
|
|
120
|
+
} catch (error) {
|
|
121
|
+
process.stdout.write(JSON.stringify({ ok: false, error: error.stack || error.message || String(error) }) + "\n");
|
|
122
|
+
}
|
|
123
|
+
});
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
import shutil
|
|
2
|
+
import subprocess
|
|
3
|
+
import tempfile
|
|
4
|
+
from pathlib import Path
|
|
5
|
+
from typing import List, Tuple
|
|
6
|
+
|
|
7
|
+
from adapters.native_process import HELPER_DIR, NativeProcessAdapter
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
class JavaAstAdapter(NativeProcessAdapter):
|
|
11
|
+
name = "javac-tree-ast"
|
|
12
|
+
languages = ("java",)
|
|
13
|
+
native_description = "JDK javac com.sun.source Tree API; declarations, imports, and calls"
|
|
14
|
+
transport = "base64-tsv"
|
|
15
|
+
|
|
16
|
+
def __init__(self, fallback):
|
|
17
|
+
super().__init__(fallback)
|
|
18
|
+
self.classes = Path(tempfile.gettempdir()) / "devflow-java-ast-helper-v1"
|
|
19
|
+
|
|
20
|
+
def version(self) -> str:
|
|
21
|
+
return "1"
|
|
22
|
+
|
|
23
|
+
def check_available(self, codebase=None) -> Tuple[bool, str]:
|
|
24
|
+
javac, java = shutil.which("javac"), shutil.which("java")
|
|
25
|
+
if not javac or not java:
|
|
26
|
+
return False, "JDK javac/java executables not found"
|
|
27
|
+
source = HELPER_DIR / "DevflowJavaAstHelper.java"
|
|
28
|
+
target = self.classes / "DevflowJavaAstHelper.class"
|
|
29
|
+
if not target.exists() or target.stat().st_mtime < source.stat().st_mtime:
|
|
30
|
+
self.classes.mkdir(parents=True, exist_ok=True)
|
|
31
|
+
result = subprocess.run([javac, "-d", str(self.classes), str(source)], stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, check=False)
|
|
32
|
+
if result.returncode != 0:
|
|
33
|
+
return False, f"javac Tree API helper compilation failed: {result.stderr.strip()[:500]}"
|
|
34
|
+
return True, "JDK javac Tree API available"
|
|
35
|
+
|
|
36
|
+
def command(self, codebase=None) -> List[str]:
|
|
37
|
+
return [shutil.which("java") or "java", "-cp", str(self.classes), "DevflowJavaAstHelper"]
|
|
@@ -0,0 +1,328 @@
|
|
|
1
|
+
import base64
|
|
2
|
+
import json
|
|
3
|
+
import os
|
|
4
|
+
import select
|
|
5
|
+
import shutil
|
|
6
|
+
import subprocess
|
|
7
|
+
from pathlib import Path
|
|
8
|
+
from typing import Dict, Iterable, List, Optional, Tuple
|
|
9
|
+
|
|
10
|
+
from adapters.base import Adapter, ParseResult
|
|
11
|
+
from adapters.generic_text import _relation, _symbol
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
HELPER_DIR = Path(__file__).resolve().parent / "helpers"
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
class NativeProcessAdapter(Adapter):
|
|
18
|
+
"""Base class for language-native parsers exposed through JSONL helpers.
|
|
19
|
+
|
|
20
|
+
Helpers stay alive for the lifetime of one index scan, which avoids the
|
|
21
|
+
process-startup cost for every source file. They return syntax facts only;
|
|
22
|
+
this class converts those facts into the stable shared index schema.
|
|
23
|
+
"""
|
|
24
|
+
|
|
25
|
+
languages: Tuple[str, ...] = ()
|
|
26
|
+
native_description = "language-native syntax parser"
|
|
27
|
+
transport = "json"
|
|
28
|
+
|
|
29
|
+
def __init__(self, fallback: Adapter):
|
|
30
|
+
self.fallback = fallback
|
|
31
|
+
self._processes: Dict[str, subprocess.Popen] = {}
|
|
32
|
+
self._availability: Dict[str, Tuple[bool, str]] = {}
|
|
33
|
+
|
|
34
|
+
def detect(self, codebase, files, manifests) -> int:
|
|
35
|
+
return 120 if any(self._language_for_path(path) in self.languages for path in files) else 0
|
|
36
|
+
|
|
37
|
+
def supports(self, language: str) -> bool:
|
|
38
|
+
return language in self.languages
|
|
39
|
+
|
|
40
|
+
def check_available(self, codebase=None) -> Tuple[bool, str]:
|
|
41
|
+
raise NotImplementedError
|
|
42
|
+
|
|
43
|
+
def command(self, codebase=None) -> List[str]:
|
|
44
|
+
raise NotImplementedError
|
|
45
|
+
|
|
46
|
+
def _availability_for(self, codebase=None) -> Tuple[bool, str]:
|
|
47
|
+
key = self._availability_key(codebase)
|
|
48
|
+
if key not in self._availability:
|
|
49
|
+
self._availability[key] = self.check_available(codebase)
|
|
50
|
+
return self._availability[key]
|
|
51
|
+
|
|
52
|
+
@staticmethod
|
|
53
|
+
def _availability_key(codebase=None) -> str:
|
|
54
|
+
if codebase is None:
|
|
55
|
+
return "<global>"
|
|
56
|
+
root = getattr(codebase, "root", None)
|
|
57
|
+
if root is not None:
|
|
58
|
+
try:
|
|
59
|
+
return str(Path(root).resolve())
|
|
60
|
+
except OSError:
|
|
61
|
+
return str(root)
|
|
62
|
+
return f"<codebase:{getattr(codebase, 'name', id(codebase))}>"
|
|
63
|
+
|
|
64
|
+
def parser_for_file(self, codebase, file_record) -> str:
|
|
65
|
+
available, _ = self._availability_for(codebase)
|
|
66
|
+
return self.parser_id if available else self.fallback.parser_for_file(codebase, file_record)
|
|
67
|
+
|
|
68
|
+
def capability(self) -> Dict:
|
|
69
|
+
codebase_results = [value for key, value in self._availability.items() if key != "<global>"]
|
|
70
|
+
if codebase_results:
|
|
71
|
+
available = any(value[0] for value in codebase_results)
|
|
72
|
+
reasons = sorted({value[1] for value in codebase_results})
|
|
73
|
+
reason = "; ".join(reasons)
|
|
74
|
+
else:
|
|
75
|
+
available, reason = self._availability_for(None)
|
|
76
|
+
return {
|
|
77
|
+
"parser": self.parser_id,
|
|
78
|
+
"languages": list(self.languages),
|
|
79
|
+
"native": True,
|
|
80
|
+
"available": available,
|
|
81
|
+
"reason": reason,
|
|
82
|
+
"fallback": self.fallback.parser_id,
|
|
83
|
+
"scope": self.native_description,
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
def parse_file(self, codebase, file_record, text) -> ParseResult:
|
|
87
|
+
if not self.supports(file_record["language"]):
|
|
88
|
+
return self.fallback.parse_file(codebase, file_record, text)
|
|
89
|
+
available, reason = self._availability_for(codebase)
|
|
90
|
+
if not available:
|
|
91
|
+
return self._fallback_result(codebase, file_record, text, reason)
|
|
92
|
+
try:
|
|
93
|
+
payload = self._request(codebase, file_record, text)
|
|
94
|
+
except Exception as error:
|
|
95
|
+
self._stop_process(codebase)
|
|
96
|
+
return self._fallback_result(codebase, file_record, text, f"native parser failed: {error}", kind="parse_failed")
|
|
97
|
+
if not payload.get("ok", False):
|
|
98
|
+
message = payload.get("error") or "native parser returned no result"
|
|
99
|
+
return self._fallback_result(codebase, file_record, text, message, kind="parse_failed")
|
|
100
|
+
return self._convert(codebase, file_record, text, payload)
|
|
101
|
+
|
|
102
|
+
def _fallback_result(self, codebase, file_record, text, reason: str, kind: str = "adapter_missing") -> ParseResult:
|
|
103
|
+
result = self.fallback.parse_file(codebase, file_record, text)
|
|
104
|
+
result.actual_parser = result.actual_parser or self.fallback.parser_for_file(codebase, file_record)
|
|
105
|
+
result.diagnostics.append({
|
|
106
|
+
"codebase": codebase.name,
|
|
107
|
+
"path": file_record["path"],
|
|
108
|
+
"line": 1,
|
|
109
|
+
"severity": "warning",
|
|
110
|
+
"kind": kind,
|
|
111
|
+
"message": f"native AST unavailable for {file_record['language']}; {self.fallback.parser_id} fallback used: {reason}",
|
|
112
|
+
"parser": self.parser_id,
|
|
113
|
+
"metadata": {
|
|
114
|
+
"capability": "native_unavailable",
|
|
115
|
+
"native_parser": self.parser_id,
|
|
116
|
+
"fallback_parser": self.fallback.parser_id,
|
|
117
|
+
},
|
|
118
|
+
})
|
|
119
|
+
return result
|
|
120
|
+
|
|
121
|
+
def _start_process(self, codebase=None) -> subprocess.Popen:
|
|
122
|
+
command = self.command(codebase)
|
|
123
|
+
key = self._process_key(codebase, command)
|
|
124
|
+
current = self._processes.get(key)
|
|
125
|
+
if current is not None and current.poll() is None:
|
|
126
|
+
return current
|
|
127
|
+
process = subprocess.Popen(
|
|
128
|
+
command,
|
|
129
|
+
stdin=subprocess.PIPE,
|
|
130
|
+
stdout=subprocess.PIPE,
|
|
131
|
+
stderr=subprocess.DEVNULL,
|
|
132
|
+
text=True,
|
|
133
|
+
encoding="utf-8",
|
|
134
|
+
bufsize=1,
|
|
135
|
+
env={**os.environ, "NO_COLOR": "1"},
|
|
136
|
+
)
|
|
137
|
+
self._processes[key] = process
|
|
138
|
+
return process
|
|
139
|
+
|
|
140
|
+
def _process_key(self, codebase=None, command: Optional[List[str]] = None) -> str:
|
|
141
|
+
actual_command = command if command is not None else self.command(codebase)
|
|
142
|
+
return f"{self._availability_key(codebase)}\0{json.dumps(actual_command, ensure_ascii=False)}"
|
|
143
|
+
|
|
144
|
+
def _request(self, codebase, file_record, text) -> Dict:
|
|
145
|
+
process = self._start_process(codebase)
|
|
146
|
+
if self.transport == "base64-tsv":
|
|
147
|
+
request = "\t".join([
|
|
148
|
+
base64.b64encode(file_record["path"].encode("utf-8")).decode("ascii"),
|
|
149
|
+
base64.b64encode(text.encode("utf-8")).decode("ascii"),
|
|
150
|
+
])
|
|
151
|
+
else:
|
|
152
|
+
request = json.dumps({"path": file_record["path"], "source": text}, ensure_ascii=False)
|
|
153
|
+
assert process.stdin is not None and process.stdout is not None
|
|
154
|
+
process.stdin.write(request + "\n")
|
|
155
|
+
process.stdin.flush()
|
|
156
|
+
readable, _, _ = select.select([process.stdout], [], [], 30.0)
|
|
157
|
+
if not readable:
|
|
158
|
+
raise TimeoutError("native parser timed out after 30 seconds")
|
|
159
|
+
response = process.stdout.readline()
|
|
160
|
+
if not response:
|
|
161
|
+
raise RuntimeError(f"native parser exited with status {process.poll()}")
|
|
162
|
+
return json.loads(response)
|
|
163
|
+
|
|
164
|
+
def _convert(self, codebase, record, text: str, payload: Dict) -> ParseResult:
|
|
165
|
+
path = record["path"]
|
|
166
|
+
language = record["language"]
|
|
167
|
+
content_hash = record["content_hash"]
|
|
168
|
+
file_symbol = _symbol(
|
|
169
|
+
codebase.name, language, "file", Path(path).name, path, 1,
|
|
170
|
+
max(1, len(text.splitlines())), content_hash,
|
|
171
|
+
qualified_name=path, parser=self.parser_id,
|
|
172
|
+
)
|
|
173
|
+
symbols = [file_symbol]
|
|
174
|
+
by_key: Dict[str, Dict] = {"": file_symbol}
|
|
175
|
+
pending = list(payload.get("declarations") or [])
|
|
176
|
+
# Helpers emit parents before children, but tolerate out-of-order data.
|
|
177
|
+
while pending:
|
|
178
|
+
remaining = []
|
|
179
|
+
progressed = False
|
|
180
|
+
for declaration in pending:
|
|
181
|
+
parent_key = str(declaration.get("parent") or "")
|
|
182
|
+
parent = by_key.get(parent_key)
|
|
183
|
+
if parent is None:
|
|
184
|
+
remaining.append(declaration)
|
|
185
|
+
continue
|
|
186
|
+
kind = declaration.get("kind") or "function"
|
|
187
|
+
name = declaration.get("name") or "anonymous"
|
|
188
|
+
signature = declaration.get("signature") or ""
|
|
189
|
+
qualified = declaration.get("qualified_name") or f"{parent['qualified_name']}::{name}"
|
|
190
|
+
symbol = _symbol(
|
|
191
|
+
codebase.name, language, kind, name, path,
|
|
192
|
+
max(1, int(declaration.get("line_start") or 1)),
|
|
193
|
+
max(1, int(declaration.get("line_end") or declaration.get("line_start") or 1)),
|
|
194
|
+
content_hash, qualified_name=qualified, signature=signature[:500],
|
|
195
|
+
parser=self.parser_id, visibility=declaration.get("visibility") or "",
|
|
196
|
+
)
|
|
197
|
+
symbol["container_id"] = parent["id"]
|
|
198
|
+
symbol["metadata"].update({"scope_resolution": "native_ast", "native_kind": declaration.get("native_kind", "")})
|
|
199
|
+
symbols.append(symbol)
|
|
200
|
+
by_key[str(declaration.get("key"))] = symbol
|
|
201
|
+
progressed = True
|
|
202
|
+
if not progressed:
|
|
203
|
+
# Malformed helper output must not create dangling containers.
|
|
204
|
+
for declaration in remaining:
|
|
205
|
+
declaration["parent"] = ""
|
|
206
|
+
pending = remaining
|
|
207
|
+
if pending and all(str(item.get("parent") or "") == "" for item in pending):
|
|
208
|
+
continue
|
|
209
|
+
break
|
|
210
|
+
pending = remaining
|
|
211
|
+
|
|
212
|
+
relations = []
|
|
213
|
+
for key, symbol in by_key.items():
|
|
214
|
+
if not key:
|
|
215
|
+
continue
|
|
216
|
+
relations.append(_relation(
|
|
217
|
+
symbol["container_id"] or file_symbol["id"], "contains", path,
|
|
218
|
+
symbol["line_start"], self.parser_id, "exact", to_id=symbol["id"],
|
|
219
|
+
metadata={"extraction": "native_ast"},
|
|
220
|
+
))
|
|
221
|
+
for item in payload.get("relations") or []:
|
|
222
|
+
owner = by_key.get(str(item.get("owner") or ""), file_symbol)
|
|
223
|
+
target_key = str(item.get("target_key") or "")
|
|
224
|
+
target = by_key.get(target_key) if target_key else None
|
|
225
|
+
relations.append(_relation(
|
|
226
|
+
owner["id"], item.get("type") or "calls", path,
|
|
227
|
+
max(1, int(item.get("line") or 1)), self.parser_id,
|
|
228
|
+
item.get("confidence") or "heuristic",
|
|
229
|
+
to_id=target["id"] if target else "",
|
|
230
|
+
to_external="" if target else str(item.get("target") or ""),
|
|
231
|
+
metadata={"extraction": "native_ast", **(item.get("metadata") or {})},
|
|
232
|
+
))
|
|
233
|
+
diagnostics = []
|
|
234
|
+
for item in payload.get("diagnostics") or []:
|
|
235
|
+
diagnostics.append({
|
|
236
|
+
"codebase": codebase.name,
|
|
237
|
+
"path": path,
|
|
238
|
+
"line": max(1, int(item.get("line") or 1)),
|
|
239
|
+
"severity": item.get("severity") or "warning",
|
|
240
|
+
"kind": item.get("kind") or "parse_failed",
|
|
241
|
+
"message": item.get("message") or "native parser diagnostic",
|
|
242
|
+
"parser": self.parser_id,
|
|
243
|
+
})
|
|
244
|
+
return ParseResult(symbols=symbols, relations=relations, diagnostics=diagnostics, actual_parser=self.parser_id)
|
|
245
|
+
|
|
246
|
+
def _stop_process(self, codebase=None) -> None:
|
|
247
|
+
if codebase is None:
|
|
248
|
+
processes = list(self._processes.values())
|
|
249
|
+
self._processes.clear()
|
|
250
|
+
else:
|
|
251
|
+
prefix = f"{self._availability_key(codebase)}\0"
|
|
252
|
+
keys = [key for key in self._processes if key.startswith(prefix)]
|
|
253
|
+
processes = [self._processes.pop(key) for key in keys]
|
|
254
|
+
for process in processes:
|
|
255
|
+
self._close_process(process)
|
|
256
|
+
|
|
257
|
+
@staticmethod
|
|
258
|
+
def _close_process(process: subprocess.Popen) -> None:
|
|
259
|
+
try:
|
|
260
|
+
if process.stdin:
|
|
261
|
+
process.stdin.close()
|
|
262
|
+
process.terminate()
|
|
263
|
+
process.wait(timeout=1)
|
|
264
|
+
except Exception:
|
|
265
|
+
process.kill()
|
|
266
|
+
finally:
|
|
267
|
+
if process.stdout:
|
|
268
|
+
process.stdout.close()
|
|
269
|
+
|
|
270
|
+
def __del__(self):
|
|
271
|
+
self._stop_process()
|
|
272
|
+
|
|
273
|
+
@staticmethod
|
|
274
|
+
def _language_for_path(path: Path) -> str:
|
|
275
|
+
from adapters.generic_text import language_for_path
|
|
276
|
+
return language_for_path(path)
|
|
277
|
+
|
|
278
|
+
|
|
279
|
+
def executable(name: str) -> Tuple[bool, str]:
|
|
280
|
+
path = shutil.which(name)
|
|
281
|
+
return (bool(path), path or f"{name} executable not found")
|
|
282
|
+
|
|
283
|
+
|
|
284
|
+
class UnavailableNativeAdapter(Adapter):
|
|
285
|
+
"""Explicit native capability gap with a provenance-safe fallback."""
|
|
286
|
+
|
|
287
|
+
name = "native-unavailable"
|
|
288
|
+
|
|
289
|
+
def __init__(self, language: str, expected_parser: str, fallback: Adapter, reason: str):
|
|
290
|
+
self.language = language
|
|
291
|
+
self.expected_parser = expected_parser
|
|
292
|
+
self.fallback = fallback
|
|
293
|
+
self.reason = reason
|
|
294
|
+
|
|
295
|
+
def version(self) -> str:
|
|
296
|
+
return "1"
|
|
297
|
+
|
|
298
|
+
def parser_for_file(self, codebase, file_record) -> str:
|
|
299
|
+
return self.fallback.parser_for_file(codebase, file_record)
|
|
300
|
+
|
|
301
|
+
def parse_file(self, codebase, file_record, text) -> ParseResult:
|
|
302
|
+
result = self.fallback.parse_file(codebase, file_record, text)
|
|
303
|
+
result.actual_parser = result.actual_parser or self.fallback.parser_for_file(codebase, file_record)
|
|
304
|
+
result.diagnostics.append({
|
|
305
|
+
"codebase": codebase.name,
|
|
306
|
+
"path": file_record["path"],
|
|
307
|
+
"line": 1,
|
|
308
|
+
"severity": "warning",
|
|
309
|
+
"kind": "adapter_missing",
|
|
310
|
+
"message": f"native AST unavailable for {self.language}; {self.fallback.parser_id} fallback used: {self.reason}",
|
|
311
|
+
"parser": self.parser_id,
|
|
312
|
+
"metadata": {
|
|
313
|
+
"capability": "native_unavailable",
|
|
314
|
+
"expected_parser": self.expected_parser,
|
|
315
|
+
"fallback_parser": self.fallback.parser_id,
|
|
316
|
+
},
|
|
317
|
+
})
|
|
318
|
+
return result
|
|
319
|
+
|
|
320
|
+
def capability(self) -> Dict:
|
|
321
|
+
return {
|
|
322
|
+
"parser": self.expected_parser,
|
|
323
|
+
"languages": [self.language],
|
|
324
|
+
"native": True,
|
|
325
|
+
"available": False,
|
|
326
|
+
"reason": self.reason,
|
|
327
|
+
"fallback": self.fallback.parser_id,
|
|
328
|
+
}
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
import os
|
|
2
|
+
import shutil
|
|
3
|
+
import subprocess
|
|
4
|
+
from pathlib import Path
|
|
5
|
+
from typing import List, Optional, Tuple
|
|
6
|
+
|
|
7
|
+
from adapters.native_process import HELPER_DIR, NativeProcessAdapter
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
class PhpAstAdapter(NativeProcessAdapter):
|
|
11
|
+
name = "nikic-php-parser-ast"
|
|
12
|
+
languages = ("php",)
|
|
13
|
+
native_description = "nikic PHP-Parser syntax tree; declarations, imports, and calls"
|
|
14
|
+
|
|
15
|
+
def __init__(self, fallback):
|
|
16
|
+
super().__init__(fallback)
|
|
17
|
+
self.autoload: Optional[Path] = None
|
|
18
|
+
self.autoload_by_root = {}
|
|
19
|
+
|
|
20
|
+
def version(self) -> str:
|
|
21
|
+
return "1"
|
|
22
|
+
|
|
23
|
+
def _autoload_candidates(self, codebase=None):
|
|
24
|
+
configured = os.environ.get("DEVFLOW_PHP_PARSER_AUTOLOAD")
|
|
25
|
+
if configured:
|
|
26
|
+
yield Path(configured)
|
|
27
|
+
if codebase is not None:
|
|
28
|
+
yield Path(codebase.root) / "vendor" / "autoload.php"
|
|
29
|
+
for parent in Path(codebase.root).parents:
|
|
30
|
+
yield parent / "vendor" / "autoload.php"
|
|
31
|
+
for parent in Path(__file__).resolve().parents:
|
|
32
|
+
yield parent / "vendor" / "autoload.php"
|
|
33
|
+
|
|
34
|
+
def check_available(self, codebase=None) -> Tuple[bool, str]:
|
|
35
|
+
php = shutil.which("php")
|
|
36
|
+
if not php:
|
|
37
|
+
return False, "php executable not found"
|
|
38
|
+
for candidate in self._autoload_candidates(codebase):
|
|
39
|
+
if not candidate.is_file():
|
|
40
|
+
continue
|
|
41
|
+
probe = subprocess.run(
|
|
42
|
+
[php, "-r", f"require {str(candidate)!r}; exit(class_exists('PhpParser\\\\ParserFactory') ? 0 : 3);"],
|
|
43
|
+
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, check=False,
|
|
44
|
+
)
|
|
45
|
+
if probe.returncode == 0:
|
|
46
|
+
self.autoload = candidate
|
|
47
|
+
self.autoload_by_root[self._availability_key(codebase)] = candidate
|
|
48
|
+
return True, f"nikic/PHP-Parser available via {candidate}"
|
|
49
|
+
return False, "nikic/PHP-Parser is not installed; set DEVFLOW_PHP_PARSER_AUTOLOAD or install nikic/php-parser"
|
|
50
|
+
|
|
51
|
+
def command(self, codebase=None) -> List[str]:
|
|
52
|
+
autoload = self.autoload_by_root.get(self._availability_key(codebase), self.autoload)
|
|
53
|
+
if autoload is None:
|
|
54
|
+
raise RuntimeError("nikic/PHP-Parser autoload path is unavailable")
|
|
55
|
+
return [shutil.which("php") or "php", str(HELPER_DIR / "php_ast_helper.php"), str(autoload)]
|
|
@@ -18,6 +18,15 @@ class PythonAstAdapter(Adapter):
|
|
|
18
18
|
def version(self) -> str:
|
|
19
19
|
return "1"
|
|
20
20
|
|
|
21
|
+
def capability(self) -> Dict:
|
|
22
|
+
return {
|
|
23
|
+
"parser": self.parser_id,
|
|
24
|
+
"languages": ["python"],
|
|
25
|
+
"native": True,
|
|
26
|
+
"available": True,
|
|
27
|
+
"scope": "Python standard library ast syntax tree; declarations, imports, and calls",
|
|
28
|
+
}
|
|
29
|
+
|
|
21
30
|
def parse_file(self, codebase, file_record, text) -> ParseResult:
|
|
22
31
|
if file_record["language"] != "python":
|
|
23
32
|
return self.fallback.parse_file(codebase, file_record, text)
|
|
@@ -44,6 +53,7 @@ class PythonAstAdapter(Adapter):
|
|
|
44
53
|
symbols=[file_symbol, *visitor.symbols],
|
|
45
54
|
relations=visitor.contains + visitor.relations,
|
|
46
55
|
diagnostics=[],
|
|
56
|
+
actual_parser=self.parser_id,
|
|
47
57
|
)
|
|
48
58
|
|
|
49
59
|
|
|
@@ -1,6 +1,41 @@
|
|
|
1
1
|
from .generic_text import GenericTextAdapter
|
|
2
|
+
from .go_ast import GoAstAdapter
|
|
3
|
+
from .java_ast import JavaAstAdapter
|
|
4
|
+
from .native_process import UnavailableNativeAdapter
|
|
5
|
+
from .php_ast import PhpAstAdapter
|
|
2
6
|
from .python_ast import PythonAstAdapter
|
|
3
7
|
from .structured_scope import StructuredScopeAdapter
|
|
8
|
+
from .typescript_ast import TypeScriptAstAdapter
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
UNAVAILABLE_NATIVE = {
|
|
12
|
+
"ruby": ("prism-ast", "Ruby Prism integration is not installed"),
|
|
13
|
+
"rust": ("rust-analyzer-ast", "rust-analyzer/syn integration is not installed"),
|
|
14
|
+
"kotlin": ("kotlin-psi-ast", "Kotlin PSI/compiler integration is not installed"),
|
|
15
|
+
"c": ("clang-ast", "Clang libTooling integration is not installed"),
|
|
16
|
+
"cpp": ("clang-ast", "Clang libTooling integration is not installed"),
|
|
17
|
+
"csharp": ("roslyn-ast", "Roslyn integration is not installed"),
|
|
18
|
+
"swift": ("swift-syntax-ast", "SwiftSyntax integration is not installed"),
|
|
19
|
+
"dart": ("dart-analyzer-ast", "Dart analyzer integration is not installed"),
|
|
20
|
+
"scala": ("scalameta-ast", "Scalameta integration is not installed"),
|
|
21
|
+
"lua": ("tree-sitter-lua-ast", "tree-sitter-lua integration is not installed"),
|
|
22
|
+
"shell": ("tree-sitter-bash-ast", "tree-sitter-bash integration is not installed"),
|
|
23
|
+
"vue": ("vue-compiler-ast", "Vue compiler integration is not installed"),
|
|
24
|
+
"svelte": ("svelte-compiler-ast", "Svelte compiler integration is not installed"),
|
|
25
|
+
"sql": ("sqlglot-ast", "SQLGlot integration is not installed"),
|
|
26
|
+
"html": ("html-parser-ast", "HTML syntax adapter is not installed"),
|
|
27
|
+
"css": ("postcss-ast", "PostCSS integration is not installed"),
|
|
28
|
+
"json": ("json-syntax-tree", "JSON syntax adapter is not installed"),
|
|
29
|
+
"yaml": ("yaml-syntax-tree", "YAML syntax adapter is not installed"),
|
|
30
|
+
"toml": ("toml-syntax-tree", "TOML syntax adapter is not installed"),
|
|
31
|
+
"xml": ("xml-syntax-tree", "XML syntax adapter is not installed"),
|
|
32
|
+
"graphql": ("graphql-ast", "GraphQL parser integration is not installed"),
|
|
33
|
+
"protobuf": ("protobuf-parser-ast", "Protobuf parser integration is not installed"),
|
|
34
|
+
"hcl": ("hcl-parser-ast", "HCL parser integration is not installed"),
|
|
35
|
+
"terraform": ("hcl-parser-ast", "HCL parser integration is not installed"),
|
|
36
|
+
"dockerfile": ("dockerfile-parser-ast", "Dockerfile parser integration is not installed"),
|
|
37
|
+
"markdown": ("markdown-parser-ast", "Markdown syntax adapter is not installed"),
|
|
38
|
+
}
|
|
4
39
|
|
|
5
40
|
|
|
6
41
|
class AdapterRegistry:
|
|
@@ -8,61 +43,42 @@ class AdapterRegistry:
|
|
|
8
43
|
self.python = PythonAstAdapter()
|
|
9
44
|
self.structured = StructuredScopeAdapter()
|
|
10
45
|
self.generic = GenericTextAdapter()
|
|
11
|
-
self.
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
46
|
+
self.typescript = TypeScriptAstAdapter(self.structured)
|
|
47
|
+
self.go = GoAstAdapter(self.structured)
|
|
48
|
+
self.php = PhpAstAdapter(self.structured)
|
|
49
|
+
self.java = JavaAstAdapter(self.structured)
|
|
50
|
+
self.native_adapters = [self.python, self.typescript, self.go, self.php, self.java]
|
|
51
|
+
self.adapters = [*self.native_adapters, self.structured, self.generic]
|
|
17
52
|
self.language_adapters = {
|
|
18
53
|
"python": self.python,
|
|
19
|
-
"typescript": self.
|
|
20
|
-
"javascript": self.
|
|
21
|
-
"
|
|
22
|
-
"
|
|
23
|
-
"
|
|
24
|
-
"php": self.structured,
|
|
25
|
-
"java": self.structured,
|
|
26
|
-
"kotlin": self.structured,
|
|
27
|
-
"c": self.structured,
|
|
28
|
-
"cpp": self.structured,
|
|
29
|
-
"csharp": self.structured,
|
|
30
|
-
"swift": self.structured,
|
|
31
|
-
"dart": self.structured,
|
|
32
|
-
"scala": self.structured,
|
|
33
|
-
"lua": self.structured,
|
|
34
|
-
"shell": self.structured,
|
|
35
|
-
"vue": self.structured,
|
|
36
|
-
"svelte": self.structured,
|
|
37
|
-
# Data, configuration, and template types are explicitly registered too. They
|
|
38
|
-
# currently use the safe text adapter until a native parser is
|
|
39
|
-
# available; this keeps their parser provenance explicit and the
|
|
40
|
-
# unified output contract unchanged.
|
|
41
|
-
"sql": self.generic,
|
|
42
|
-
"html": self.generic,
|
|
43
|
-
"css": self.generic,
|
|
44
|
-
"json": self.generic,
|
|
45
|
-
"yaml": self.generic,
|
|
46
|
-
"toml": self.generic,
|
|
47
|
-
"xml": self.generic,
|
|
48
|
-
"graphql": self.generic,
|
|
49
|
-
"protobuf": self.generic,
|
|
50
|
-
"hcl": self.generic,
|
|
51
|
-
"terraform": self.generic,
|
|
52
|
-
"dockerfile": self.generic,
|
|
53
|
-
"markdown": self.generic,
|
|
54
|
+
"typescript": self.typescript,
|
|
55
|
+
"javascript": self.typescript,
|
|
56
|
+
"go": self.go,
|
|
57
|
+
"php": self.php,
|
|
58
|
+
"java": self.java,
|
|
54
59
|
}
|
|
60
|
+
structured_languages = {
|
|
61
|
+
"ruby", "rust", "kotlin", "c", "cpp", "csharp", "swift",
|
|
62
|
+
"dart", "scala", "lua", "shell", "vue", "svelte",
|
|
63
|
+
}
|
|
64
|
+
for language, (expected, reason) in UNAVAILABLE_NATIVE.items():
|
|
65
|
+
fallback = self.structured if language in structured_languages else self.generic
|
|
66
|
+
self.language_adapters[language] = UnavailableNativeAdapter(language, expected, fallback, reason)
|
|
55
67
|
|
|
56
68
|
def adapter_for_file(self, codebase, file_record):
|
|
57
69
|
language = file_record.get("language")
|
|
58
|
-
return self.language_adapters.get(
|
|
70
|
+
return self.language_adapters.get(
|
|
71
|
+
language,
|
|
72
|
+
UnavailableNativeAdapter(language or "unknown", "unregistered-native-ast", self.generic, "no native adapter is registered"),
|
|
73
|
+
)
|
|
59
74
|
|
|
60
75
|
def adapters_for(self, codebase, files, manifests):
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
76
|
+
return [self.adapter_for_file(codebase, {"language": self._language(path)}) for path in files]
|
|
77
|
+
|
|
78
|
+
def capabilities(self):
|
|
79
|
+
return {language: adapter.capability() for language, adapter in sorted(self.language_adapters.items())}
|
|
80
|
+
|
|
81
|
+
@staticmethod
|
|
82
|
+
def _language(path):
|
|
83
|
+
from .generic_text import language_for_path
|
|
84
|
+
return language_for_path(path)
|
package/plugins/devflow-intel/skills/workflow-code-index/scripts/adapters/structured_scope.py
CHANGED
|
@@ -127,7 +127,7 @@ class StructuredScopeAdapter(Adapter):
|
|
|
127
127
|
for target in _extract_calls(line, record["language"]):
|
|
128
128
|
receiver = bool(re.search(rf"\.\s*{re.escape(target)}\s*\(", line))
|
|
129
129
|
relations.append(_relation(owner, "calls", record["path"], number, self.parser_id, "heuristic", to_external=f"call:{target}", metadata={"resolution": "receiver_unresolved" if receiver else "name_only", "allow_unique_name_resolution": not receiver, "dynamic_limitations": True}))
|
|
130
|
-
return ParseResult(symbols=[file_symbol, *symbols], relations=relations, diagnostics=[])
|
|
130
|
+
return ParseResult(symbols=[file_symbol, *symbols], relations=relations, diagnostics=[], actual_parser=self.parser_id)
|
|
131
131
|
|
|
132
132
|
|
|
133
133
|
def _without_strings_and_comments(line: str) -> str:
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
import shutil
|
|
2
|
+
import subprocess
|
|
3
|
+
from pathlib import Path
|
|
4
|
+
from typing import List, Tuple
|
|
5
|
+
|
|
6
|
+
from adapters.native_process import HELPER_DIR, NativeProcessAdapter
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
class TypeScriptAstAdapter(NativeProcessAdapter):
|
|
10
|
+
name = "typescript-compiler-ast"
|
|
11
|
+
languages = ("typescript", "javascript")
|
|
12
|
+
native_description = "TypeScript Compiler API syntax tree; declarations, imports, and calls"
|
|
13
|
+
|
|
14
|
+
def version(self) -> str:
|
|
15
|
+
return "1"
|
|
16
|
+
|
|
17
|
+
def check_available(self, codebase=None) -> Tuple[bool, str]:
|
|
18
|
+
node = shutil.which("node")
|
|
19
|
+
if not node:
|
|
20
|
+
return False, "node executable not found"
|
|
21
|
+
helper = HELPER_DIR / "typescript_ast_helper.cjs"
|
|
22
|
+
probe = subprocess.run(
|
|
23
|
+
[node, "-e", "require.resolve('typescript')"], cwd=str(helper.parent),
|
|
24
|
+
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, check=False,
|
|
25
|
+
)
|
|
26
|
+
if probe.returncode != 0:
|
|
27
|
+
return False, "typescript compiler API module not found"
|
|
28
|
+
return True, "typescript compiler API available"
|
|
29
|
+
|
|
30
|
+
def command(self, codebase=None) -> List[str]:
|
|
31
|
+
return [shutil.which("node") or "node", str(HELPER_DIR / "typescript_ast_helper.cjs")]
|
|
@@ -99,7 +99,7 @@ def _publish_snapshot(staging: Path, output: Path, counts: Dict[str, int], works
|
|
|
99
99
|
os.close(directory_fd)
|
|
100
100
|
|
|
101
101
|
|
|
102
|
-
def _workspace_state(root: Path, files: List[dict], changed: Set[str], discovered_paths: Iterable[str] | None = None) -> Dict[str, object]:
|
|
102
|
+
def _workspace_state(root: Path, files: List[dict], changed: Set[str], discovered_paths: Iterable[str] | None = None, capabilities: Dict[str, object] | None = None) -> Dict[str, object]:
|
|
103
103
|
indexed_paths = sorted(record["path"] for record in files)
|
|
104
104
|
discovered = sorted(set(discovered_paths or indexed_paths))
|
|
105
105
|
source_manifest_hash = _hash_lines(f"{record['path']}\0{record['content_hash']}" for record in files)
|
|
@@ -112,11 +112,7 @@ def _workspace_state(root: Path, files: List[dict], changed: Set[str], discovere
|
|
|
112
112
|
"dirtyPathsAtBuild": sorted(changed),
|
|
113
113
|
"extractors": extractors,
|
|
114
114
|
"extractorSetHash": _hash_lines(extractors),
|
|
115
|
-
"capabilities": {
|
|
116
|
-
"python-ast@1": {"scope": "exact syntax tree", "targets": "lexical names; dynamic dispatch unresolved"},
|
|
117
|
-
"structured-scope@1": {"scope": "inferred lexical braces/end blocks", "targets": "name-only heuristic"},
|
|
118
|
-
"generic-text@2": {"scope": "file-level", "targets": "heuristic only"},
|
|
119
|
-
},
|
|
115
|
+
"capabilities": capabilities or {},
|
|
120
116
|
}
|
|
121
117
|
|
|
122
118
|
|
|
@@ -290,8 +286,9 @@ def command_scan(args) -> int:
|
|
|
290
286
|
"parser": "pending",
|
|
291
287
|
}
|
|
292
288
|
adapter = registry.adapter_for_file(codebase, file_record)
|
|
293
|
-
file_record["parser"] = adapter.
|
|
289
|
+
file_record["parser"] = adapter.parser_for_file(codebase, file_record)
|
|
294
290
|
result = adapter.parse_file(codebase, file_record, text)
|
|
291
|
+
file_record["parser"] = result.actual_parser or file_record["parser"]
|
|
295
292
|
files.append(file_record)
|
|
296
293
|
symbols.extend(result.symbols)
|
|
297
294
|
relations.extend(result.relations)
|
|
@@ -319,7 +316,7 @@ def command_scan(args) -> int:
|
|
|
319
316
|
staging = Path(tempfile.mkdtemp(prefix=f".{output.name}.build-", dir=str(output.parent)))
|
|
320
317
|
try:
|
|
321
318
|
generation = str(uuid.uuid4())
|
|
322
|
-
workspace_state = _workspace_state(root, files, changed, discovered_paths)
|
|
319
|
+
workspace_state = _workspace_state(root, files, changed, discovered_paths, registry.capabilities())
|
|
323
320
|
workspace_state["generatedAt"] = datetime.now(timezone.utc).isoformat()
|
|
324
321
|
workspace_state["semanticExtractor"] = SEMANTIC_EXTRACTOR_VERSION
|
|
325
322
|
workspace_state["extractorSetHash"] = _hash_lines([*workspace_state.get("extractors", []), SEMANTIC_EXTRACTOR_VERSION])
|
package/src/constants.ts
CHANGED