flowseeker 0.1.7 → 0.1.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.env.example +7 -0
- package/CHANGELOG.md +131 -108
- package/README.md +288 -221
- package/dist/cli/flowCommand.js +175 -0
- package/dist/cli/main.js +1794 -0
- package/dist/cli/mcpServer.js +7 -1
- package/dist/cli/runEvaluation.js +178 -2
- package/dist/config/defaultConfig.js +11 -1
- package/dist/config/env.js +118 -0
- package/dist/config/loadConfig.js +18 -1
- package/dist/config/loadConfigFromPath.js +3 -1
- package/dist/extension.js +23 -0
- package/dist/gateway/embeddingProviders.js +852 -0
- package/dist/index/cacheStore.js +43 -0
- package/dist/index/configRouteDiscoveryProbe.js +288 -0
- package/dist/index/embeddingIndex.js +193 -0
- package/dist/index/graphIndex.js +460 -0
- package/dist/index/indexWatcher.js +86 -0
- package/dist/index/structuredExtractor.js +303 -12
- package/dist/index/treeSitterExtractor.js +264 -0
- package/dist/index/vectorStore.js +41 -0
- package/dist/index/workspaceIndex.js +591 -26
- package/dist/mcp/mcpTools.js +51 -0
- package/dist/pipeline/contextPack.js +3 -3
- package/dist/pipeline/deterministicReranker.js +358 -0
- package/dist/pipeline/evaluationMetrics.js +14 -2
- package/dist/pipeline/fileGroups.js +7 -1
- package/dist/pipeline/fileScanner.js +93 -11
- package/dist/pipeline/llmReranker.js +151 -0
- package/dist/pipeline/nodeScan.js +91 -12
- package/dist/pipeline/ranker.js +875 -16
- package/dist/pipeline/retrievalFusion.js +41 -0
- package/dist/pipeline/runHeadless.js +35 -0
- package/dist/pipeline/solvePacket.js +549 -42
- package/dist/pipeline/subsystem.js +21 -0
- package/docs/demo-screenshot-checklist.md +86 -0
- package/docs/marketplace-copy.md +92 -0
- package/docs/mcp-onboarding.md +191 -0
- package/package.json +633 -561
|
@@ -3,6 +3,7 @@ Object.defineProperty(exports, "__esModule", { value: true });
|
|
|
3
3
|
exports.detectLanguage = detectLanguage;
|
|
4
4
|
exports.extractStructuredNodes = extractStructuredNodes;
|
|
5
5
|
exports.getDeclarationSnippet = getDeclarationSnippet;
|
|
6
|
+
const treeSitterExtractor_1 = require("./treeSitterExtractor");
|
|
6
7
|
const maxStructuredLineChars = 12000;
|
|
7
8
|
const languagePatterns = {
|
|
8
9
|
typescript: {
|
|
@@ -21,25 +22,37 @@ const languagePatterns = {
|
|
|
21
22
|
comment: /\/\*\*[\s\S]*?\*\/|\/\/.*$/gm,
|
|
22
23
|
annotation: /@\w+/g
|
|
23
24
|
},
|
|
25
|
+
dart: {
|
|
26
|
+
extensions: [".dart"],
|
|
27
|
+
// class, enum, extension, mixin, abstract class, factory
|
|
28
|
+
declaration: /^\s*(?:abstract\s+)?(?:class|enum|extension|mixin)\s+(\w+)|^\s*(?:const\s+)?(?:final\s+)?(?:static\s+)?(?:Future\s*<\s*\w+\s*>\s+)?(?:void|bool|int|double|String|num|dynamic|Widget)\??\s+(\w+)\s*\([^)]*\)\s*\{?|^\s*(?:Future\s*<\s*\w+\s*>\s+)?\w+\??\s+(\w+)\s*\([^)]*\)\s*\{?/gm,
|
|
29
|
+
call: /(\w+(?:\.\w+)*)\s*\(/g,
|
|
30
|
+
stringLiteral: /(["'])((?:\\.|(?!\1).)*?)\1/g,
|
|
31
|
+
comment: /\/\*[\s\S]*?\*\/|\/\/.*$/gm,
|
|
32
|
+
annotation: /@\w+(?:\([^)]*\))?/g
|
|
33
|
+
},
|
|
24
34
|
python: {
|
|
25
35
|
extensions: [".py"],
|
|
26
|
-
|
|
36
|
+
// async def, def, class; also capture decorators and Django URL patterns
|
|
37
|
+
declaration: /^\s*(?:async\s+)?(?:def|class)\s+(\w+)|^\s*@\w+(?:\.\w+)*(?:\([^)]*\))?\s*\n?\s*(?:async\s+)?(?:def|class)\s+(\w+)/gm,
|
|
27
38
|
call: /(\w+(?:\.\w+)*)\s*\(/g,
|
|
28
39
|
stringLiteral: /(["'])((?:\\.|(?!\1).)*?)\1/g,
|
|
29
40
|
comment: /#.*$/gm,
|
|
30
|
-
annotation: /@\w
|
|
41
|
+
annotation: /@\w+(?:\.\w+)*(?:\([^)]*\))?/g
|
|
31
42
|
},
|
|
32
43
|
php: {
|
|
33
44
|
extensions: [".php"],
|
|
34
|
-
|
|
35
|
-
|
|
45
|
+
// class, interface, trait; also catch Laravel-named classes and methods
|
|
46
|
+
declaration: /^\s*(?:(?:abstract|final)\s+)?(?:class|interface|trait)\s+(\w+)|^\s*(?:(?:public|private|protected|static)\s+)*function\s+(\w+)/gm,
|
|
47
|
+
call: /(\w+(?:->\w+)*(?:::\w+)*)\s*\(/g,
|
|
36
48
|
stringLiteral: /(["'])((?:\\.|(?!\1).)*?)\1/g,
|
|
37
49
|
comment: /\/\*\*[\s\S]*?\*\/|\/\/.*$|#.*$/gm,
|
|
38
50
|
annotation: /#\[[\s\S]*?\]|@\w+/g
|
|
39
51
|
},
|
|
40
52
|
java: {
|
|
41
53
|
extensions: [".java"],
|
|
42
|
-
|
|
54
|
+
// class, interface, enum; also capture Spring-annotated methods and constructors
|
|
55
|
+
declaration: /^\s*(?:(?:public|private|protected|static|final|abstract)\s+)*(?:class|interface|enum)\s+(\w+)|^\s*(?:@\w+(?:\([^)]*\))?\s*)*(?:(?:public|private|protected|static|final|abstract)\s+)*(?:<[^>]+>\s+)?(?:\w+(?:<[^>]+>)?)\s+(\w+)\s*\([^)]*\)\s*(?:throws\s+\w+(?:\s*,\s*\w+)*)?\s*\{?/gm,
|
|
43
56
|
call: /(\w+(?:\.\w+)*)\s*\(/g,
|
|
44
57
|
stringLiteral: /"((?:\\.|[^"\\])*)"/g,
|
|
45
58
|
comment: /\/\*\*[\s\S]*?\*\/|\/\/.*$/gm,
|
|
@@ -47,7 +60,8 @@ const languagePatterns = {
|
|
|
47
60
|
},
|
|
48
61
|
go: {
|
|
49
62
|
extensions: [".go"],
|
|
50
|
-
|
|
63
|
+
// func declarations, methods with pointer/value receivers, type struct/interface
|
|
64
|
+
declaration: /^\s*func\s+(?:\(\w+\s+\*?\w+\)\s+)?(\w+)\s*\(|^\s*type\s+(\w+)\s+(?:struct|interface)\s*\{/gm,
|
|
51
65
|
call: /(\w+(?:\.\w+)*)\s*\(/g,
|
|
52
66
|
stringLiteral: /"((?:\\.|[^"\\])*)"|`[^`]*`/g,
|
|
53
67
|
comment: /\/\/.*$/gm,
|
|
@@ -72,7 +86,7 @@ const languagePatterns = {
|
|
|
72
86
|
rust: {
|
|
73
87
|
extensions: [".rs"],
|
|
74
88
|
declaration: /^\s*(?:pub\s+)?(?:async\s+)?fn\s+(\w+)/gm,
|
|
75
|
-
call: /(\w+(
|
|
89
|
+
call: /(\w+(?:::\w+)*(?:\.\w+)*)\s*\(/g,
|
|
76
90
|
stringLiteral: /"((?:\\.|[^"\\])*)"/g,
|
|
77
91
|
comment: /\/\/.*$|\/\/!.*$/gm,
|
|
78
92
|
annotation: /#\[[\s\S]*?\]/g
|
|
@@ -95,7 +109,15 @@ function detectLanguage(filePath) {
|
|
|
95
109
|
}
|
|
96
110
|
return undefined;
|
|
97
111
|
}
|
|
98
|
-
function extractStructuredNodes(content, language) {
|
|
112
|
+
function extractStructuredNodes(content, language, relativePath = "") {
|
|
113
|
+
// Try tree-sitter parser first for supported languages
|
|
114
|
+
if ((0, treeSitterExtractor_1.supportsTreeSitterLanguage)(language)) {
|
|
115
|
+
const tsResult = (0, treeSitterExtractor_1.extractWithTreeSitter)(content, language, relativePath);
|
|
116
|
+
if (tsResult) {
|
|
117
|
+
return convertTreeSitterResult(tsResult, relativePath).map((n) => ({ ...n, extractorKind: "tree-sitter" }));
|
|
118
|
+
}
|
|
119
|
+
}
|
|
120
|
+
// Fall back to structured regex
|
|
99
121
|
const patterns = languagePatterns[language];
|
|
100
122
|
if (!patterns) {
|
|
101
123
|
return [];
|
|
@@ -103,11 +125,127 @@ function extractStructuredNodes(content, language) {
|
|
|
103
125
|
const lines = content.split(/\r?\n/);
|
|
104
126
|
const nodes = [];
|
|
105
127
|
nodes.push(...extractDeclarations(lines, patterns.declaration, language));
|
|
128
|
+
nodes.push(...extractMethodDeclarations(lines, language));
|
|
106
129
|
nodes.push(...extractCalls(lines, patterns.call));
|
|
107
130
|
nodes.push(...extractStringLiterals(lines, patterns.stringLiteral));
|
|
108
131
|
nodes.push(...extractComments(lines, patterns.comment));
|
|
109
132
|
nodes.push(...extractAnnotations(lines, patterns.annotation));
|
|
110
|
-
|
|
133
|
+
nodes.push(...extractLanguageSpecific(lines, language));
|
|
134
|
+
return dedupeNodes(assignParentFqns(nodes)).map((n) => ({ ...n, extractorKind: "structured-regex" }));
|
|
135
|
+
}
|
|
136
|
+
function extractLanguageSpecific(lines, language) {
|
|
137
|
+
const nodes = [];
|
|
138
|
+
// Dart: extract import statements
|
|
139
|
+
if (language === "dart") {
|
|
140
|
+
const importRe = /^\s*import\s+['"]([^'"]+)['"]\s*;?/;
|
|
141
|
+
const exportRe = /^\s*export\s+['"]([^'"]+)['"]\s*;?/;
|
|
142
|
+
const partRe = /^\s*part\s+['"]([^'"]+)['"]\s*;?/;
|
|
143
|
+
lines.forEach((line, index) => {
|
|
144
|
+
if (line.length > maxStructuredLineChars)
|
|
145
|
+
return;
|
|
146
|
+
let m = importRe.exec(line) || exportRe.exec(line) || partRe.exec(line);
|
|
147
|
+
if (m) {
|
|
148
|
+
nodes.push({ kind: "import", name: m[1], range: { startLine: index + 1, endLine: index + 1 }, text: line.trim() });
|
|
149
|
+
}
|
|
150
|
+
});
|
|
151
|
+
}
|
|
152
|
+
// Go: extract var/const blocks and import blocks
|
|
153
|
+
if (language === "go") {
|
|
154
|
+
lines.forEach((line, index) => {
|
|
155
|
+
if (line.length > maxStructuredLineChars)
|
|
156
|
+
return;
|
|
157
|
+
// Package-level var or const
|
|
158
|
+
if (/^\s*var\s+(\w+)/.test(line)) {
|
|
159
|
+
const m = line.match(/^\s*var\s+(\w+)/);
|
|
160
|
+
if (m)
|
|
161
|
+
nodes.push({ kind: "declaration", name: m[1], fqn: m[1], symbolKind: "constant", range: { startLine: index + 1, endLine: index + 1 }, text: line.trim() });
|
|
162
|
+
}
|
|
163
|
+
if (/^\s*const\s+(\w+)/.test(line)) {
|
|
164
|
+
const m = line.match(/^\s*const\s+(\w+)/);
|
|
165
|
+
if (m)
|
|
166
|
+
nodes.push({ kind: "declaration", name: m[1], fqn: m[1], symbolKind: "constant", range: { startLine: index + 1, endLine: index + 1 }, text: line.trim() });
|
|
167
|
+
}
|
|
168
|
+
});
|
|
169
|
+
}
|
|
170
|
+
// PHP: extract use (namespace import) statements
|
|
171
|
+
if (language === "php") {
|
|
172
|
+
const useRe = /^\s*use\s+([\w\\]+?)(?:\s+as\s+\w+)?\s*;/;
|
|
173
|
+
const nsRe = /^\s*namespace\s+([\w\\]+)\s*;/;
|
|
174
|
+
lines.forEach((line, index) => {
|
|
175
|
+
if (line.length > maxStructuredLineChars)
|
|
176
|
+
return;
|
|
177
|
+
let m = useRe.exec(line);
|
|
178
|
+
if (m) {
|
|
179
|
+
const fqn = m[1].trim();
|
|
180
|
+
const shortName = fqn.split("\\").pop() || fqn;
|
|
181
|
+
nodes.push({ kind: "import", name: shortName, fqn, range: { startLine: index + 1, endLine: index + 1 }, text: line.trim() });
|
|
182
|
+
return;
|
|
183
|
+
}
|
|
184
|
+
m = nsRe.exec(line);
|
|
185
|
+
if (m) {
|
|
186
|
+
nodes.push({ kind: "declaration", name: m[1], fqn: m[1], symbolKind: "class", range: { startLine: index + 1, endLine: index + 1 }, text: line.trim() });
|
|
187
|
+
}
|
|
188
|
+
});
|
|
189
|
+
}
|
|
190
|
+
// Java: extract import statements
|
|
191
|
+
if (language === "java") {
|
|
192
|
+
const importRe = /^\s*import\s+(?:static\s+)?([\w.]+(?:\.\*)?)\s*;/;
|
|
193
|
+
lines.forEach((line, index) => {
|
|
194
|
+
if (line.length > maxStructuredLineChars)
|
|
195
|
+
return;
|
|
196
|
+
const m = importRe.exec(line);
|
|
197
|
+
if (m) {
|
|
198
|
+
const pkg = m[1].trim();
|
|
199
|
+
const shortName = pkg.split(".").pop() || pkg;
|
|
200
|
+
nodes.push({ kind: "import", name: shortName, fqn: pkg, range: { startLine: index + 1, endLine: index + 1 }, text: line.trim() });
|
|
201
|
+
}
|
|
202
|
+
});
|
|
203
|
+
}
|
|
204
|
+
// Python: extract Django URL patterns and import statements
|
|
205
|
+
if (language === "python") {
|
|
206
|
+
const importRe = /^\s*(?:from\s+(\S+)\s+import\s+\S+|import\s+(\S+))/;
|
|
207
|
+
lines.forEach((line, index) => {
|
|
208
|
+
if (line.length > maxStructuredLineChars)
|
|
209
|
+
return;
|
|
210
|
+
const m = importRe.exec(line);
|
|
211
|
+
if (m) {
|
|
212
|
+
const name = m[1] || m[2];
|
|
213
|
+
if (name) {
|
|
214
|
+
const shortName = name.split(".").pop() || name;
|
|
215
|
+
nodes.push({ kind: "import", name: shortName, fqn: name, range: { startLine: index + 1, endLine: index + 1 }, text: line.trim() });
|
|
216
|
+
}
|
|
217
|
+
}
|
|
218
|
+
});
|
|
219
|
+
}
|
|
220
|
+
// Go: extract import blocks
|
|
221
|
+
if (language === "go") {
|
|
222
|
+
const importRe = /^\s*"([^"]+)"\s*\/\/?\s*(\w+)?/;
|
|
223
|
+
let inImportBlock = false;
|
|
224
|
+
lines.forEach((line, index) => {
|
|
225
|
+
if (/^\s*import\s*\(/.test(line)) {
|
|
226
|
+
inImportBlock = true;
|
|
227
|
+
return;
|
|
228
|
+
}
|
|
229
|
+
if (inImportBlock && /^\s*\)/.test(line)) {
|
|
230
|
+
inImportBlock = false;
|
|
231
|
+
return;
|
|
232
|
+
}
|
|
233
|
+
if (/^\s*import\s+"([^"]+)"/.test(line)) {
|
|
234
|
+
const m = line.match(/^\s*import\s+"([^"]+)"/);
|
|
235
|
+
if (m)
|
|
236
|
+
nodes.push({ kind: "import", name: m[1].split("/").pop() || m[1], fqn: m[1], range: { startLine: index + 1, endLine: index + 1 }, text: line.trim() });
|
|
237
|
+
return;
|
|
238
|
+
}
|
|
239
|
+
if (inImportBlock) {
|
|
240
|
+
const m = importRe.exec(line.trim());
|
|
241
|
+
if (m) {
|
|
242
|
+
const alias = m[2] || m[1].split("/").pop() || m[1];
|
|
243
|
+
nodes.push({ kind: "import", name: alias, fqn: m[1], range: { startLine: index + 1, endLine: index + 1 }, text: line.trim() });
|
|
244
|
+
}
|
|
245
|
+
}
|
|
246
|
+
});
|
|
247
|
+
}
|
|
248
|
+
return nodes;
|
|
111
249
|
}
|
|
112
250
|
function extractDeclarations(lines, pattern, language) {
|
|
113
251
|
const nodes = [];
|
|
@@ -118,13 +256,16 @@ function extractDeclarations(lines, pattern, language) {
|
|
|
118
256
|
pattern.lastIndex = 0;
|
|
119
257
|
let match = pattern.exec(line);
|
|
120
258
|
while (match) {
|
|
121
|
-
const name = match[1];
|
|
259
|
+
const name = match[1] ?? match[2];
|
|
122
260
|
const isExported = /\bexport\b/.test(line) || /\bpub\b/.test(line) || /\bpublic\b/.test(line);
|
|
123
261
|
const nodeKind = isClassDecl(line, language) ? "declaration" : "declaration";
|
|
262
|
+
const declarationKind = inferSymbolKind(line, language);
|
|
124
263
|
nodes.push({
|
|
125
264
|
kind: nodeKind,
|
|
126
265
|
name,
|
|
127
|
-
|
|
266
|
+
fqn: name,
|
|
267
|
+
symbolKind: declarationKind,
|
|
268
|
+
range: { startLine: index + 1, endLine: expandDeclarationEnd(lines, index, language) },
|
|
128
269
|
text: line.trim(),
|
|
129
270
|
parent: isExported ? "exported" : undefined
|
|
130
271
|
});
|
|
@@ -133,6 +274,34 @@ function extractDeclarations(lines, pattern, language) {
|
|
|
133
274
|
});
|
|
134
275
|
return nodes;
|
|
135
276
|
}
|
|
277
|
+
function extractMethodDeclarations(lines, language) {
|
|
278
|
+
if (!methodLanguages.has(language)) {
|
|
279
|
+
return [];
|
|
280
|
+
}
|
|
281
|
+
const nodes = [];
|
|
282
|
+
const pattern = /^\s*(?:(?:public|private|protected|static|async|readonly|override)\s+)*(\w+)\s*\([^)]*\)\s*(?::[^={]+)?\s*[{:]?/;
|
|
283
|
+
lines.forEach((line, index) => {
|
|
284
|
+
if (line.length > maxStructuredLineChars || /\b(function|if|for|while|switch|catch|return|class|interface|type|enum)\b/.test(line)) {
|
|
285
|
+
return;
|
|
286
|
+
}
|
|
287
|
+
const match = line.match(pattern);
|
|
288
|
+
const name = match?.[1];
|
|
289
|
+
if (!name || commonNonMethodNames.has(name)) {
|
|
290
|
+
return;
|
|
291
|
+
}
|
|
292
|
+
nodes.push({
|
|
293
|
+
kind: "declaration",
|
|
294
|
+
name,
|
|
295
|
+
fqn: name,
|
|
296
|
+
symbolKind: "method",
|
|
297
|
+
range: { startLine: index + 1, endLine: expandDeclarationEnd(lines, index, language) },
|
|
298
|
+
text: line.trim()
|
|
299
|
+
});
|
|
300
|
+
});
|
|
301
|
+
return nodes;
|
|
302
|
+
}
|
|
303
|
+
const methodLanguages = new Set(["typescript", "javascript", "python", "php", "java", "csharp", "kotlin", "dart"]);
|
|
304
|
+
const commonNonMethodNames = new Set(["if", "for", "while", "switch", "catch", "return", "function", "class", "interface", "type", "enum"]);
|
|
136
305
|
function isClassDecl(line, language) {
|
|
137
306
|
const classKeywords = {
|
|
138
307
|
typescript: /\bclass\b/,
|
|
@@ -141,10 +310,75 @@ function isClassDecl(line, language) {
|
|
|
141
310
|
php: /\bclass\b/,
|
|
142
311
|
java: /\bclass\b/,
|
|
143
312
|
csharp: /\bclass\b/,
|
|
144
|
-
kotlin: /\bclass\b
|
|
313
|
+
kotlin: /\bclass\b/,
|
|
314
|
+
dart: /\bclass\b/,
|
|
315
|
+
go: /\bstruct\b|\binterface\b/
|
|
145
316
|
};
|
|
146
317
|
return classKeywords[language]?.test(line) ?? false;
|
|
147
318
|
}
|
|
319
|
+
function inferSymbolKind(line, language) {
|
|
320
|
+
if (/\bclass\b/.test(line)) {
|
|
321
|
+
return "class";
|
|
322
|
+
}
|
|
323
|
+
if (/\binterface\b|\btrait\b/.test(line)) {
|
|
324
|
+
return "interface";
|
|
325
|
+
}
|
|
326
|
+
if (/\btype\b/.test(line)) {
|
|
327
|
+
return "type";
|
|
328
|
+
}
|
|
329
|
+
if (/\bstruct\b/.test(line)) {
|
|
330
|
+
return "type";
|
|
331
|
+
}
|
|
332
|
+
if (/\benum\b/.test(line)) {
|
|
333
|
+
return "constant";
|
|
334
|
+
}
|
|
335
|
+
if (/\bextension\b/.test(line)) {
|
|
336
|
+
return "class";
|
|
337
|
+
}
|
|
338
|
+
if (/\bmixin\b/.test(line) && language === "dart") {
|
|
339
|
+
return "class";
|
|
340
|
+
}
|
|
341
|
+
if (/\bconst\b/.test(line) && /=>|function\s*\(/.test(line)) {
|
|
342
|
+
return "function";
|
|
343
|
+
}
|
|
344
|
+
if (/\bconst\b/.test(line)) {
|
|
345
|
+
return "constant";
|
|
346
|
+
}
|
|
347
|
+
if (/\bfunction\b|\bdef\b|\bfunc\b|\bfn\b/.test(line)) {
|
|
348
|
+
return "function";
|
|
349
|
+
}
|
|
350
|
+
return "function";
|
|
351
|
+
}
|
|
352
|
+
function expandDeclarationEnd(lines, startIndex, language) {
|
|
353
|
+
if (language === "python") {
|
|
354
|
+
const baseIndent = leadingSpaces(lines[startIndex]);
|
|
355
|
+
for (let index = startIndex + 1; index < Math.min(lines.length, startIndex + 200); index += 1) {
|
|
356
|
+
const line = lines[index];
|
|
357
|
+
if (line.trim() && leadingSpaces(line) <= baseIndent) {
|
|
358
|
+
return index;
|
|
359
|
+
}
|
|
360
|
+
}
|
|
361
|
+
return Math.min(lines.length, startIndex + 80);
|
|
362
|
+
}
|
|
363
|
+
let depth = 0;
|
|
364
|
+
for (let index = startIndex; index < Math.min(lines.length, startIndex + 240); index += 1) {
|
|
365
|
+
for (const char of lines[index]) {
|
|
366
|
+
if (char === "{") {
|
|
367
|
+
depth += 1;
|
|
368
|
+
}
|
|
369
|
+
else if (char === "}") {
|
|
370
|
+
depth -= 1;
|
|
371
|
+
}
|
|
372
|
+
}
|
|
373
|
+
if (index > startIndex && depth <= 0) {
|
|
374
|
+
return index + 1;
|
|
375
|
+
}
|
|
376
|
+
}
|
|
377
|
+
return startIndex + 1;
|
|
378
|
+
}
|
|
379
|
+
function leadingSpaces(line) {
|
|
380
|
+
return line.match(/^\s*/)?.[0].length ?? 0;
|
|
381
|
+
}
|
|
148
382
|
function extractCalls(lines, pattern) {
|
|
149
383
|
const nodes = [];
|
|
150
384
|
const callNames = new Set();
|
|
@@ -235,6 +469,29 @@ function extractAnnotations(lines, pattern) {
|
|
|
235
469
|
});
|
|
236
470
|
return nodes.slice(0, 40);
|
|
237
471
|
}
|
|
472
|
+
function assignParentFqns(nodes) {
|
|
473
|
+
const declarations = nodes
|
|
474
|
+
.filter((node) => node.kind === "declaration" && node.name)
|
|
475
|
+
.sort((left, right) => left.range.startLine - right.range.startLine || right.range.endLine - left.range.endLine);
|
|
476
|
+
return nodes.map((node) => {
|
|
477
|
+
if (node.kind !== "declaration" || !node.name || node.symbolKind === "class" || node.symbolKind === "interface") {
|
|
478
|
+
return node;
|
|
479
|
+
}
|
|
480
|
+
const parent = declarations
|
|
481
|
+
.filter((candidate) => candidate !== node && (candidate.symbolKind === "class" || candidate.symbolKind === "interface"))
|
|
482
|
+
.filter((candidate) => candidate.range.startLine <= node.range.startLine && candidate.range.endLine >= node.range.endLine)
|
|
483
|
+
.sort((left, right) => (right.range.startLine - left.range.startLine) || (left.range.endLine - right.range.endLine))[0];
|
|
484
|
+
if (!parent?.name) {
|
|
485
|
+
return node;
|
|
486
|
+
}
|
|
487
|
+
return {
|
|
488
|
+
...node,
|
|
489
|
+
parentFqn: parent.fqn ?? parent.name,
|
|
490
|
+
fqn: `${parent.fqn ?? parent.name}.${node.name}`,
|
|
491
|
+
symbolKind: node.symbolKind === "function" ? "method" : node.symbolKind
|
|
492
|
+
};
|
|
493
|
+
});
|
|
494
|
+
}
|
|
238
495
|
function dedupeNodes(nodes) {
|
|
239
496
|
const seen = new Set();
|
|
240
497
|
const result = [];
|
|
@@ -253,4 +510,38 @@ function getDeclarationSnippet(content, node, contextLines = 5) {
|
|
|
253
510
|
const end = Math.min(lines.length, node.range.endLine + contextLines);
|
|
254
511
|
return lines.slice(start, end).join("\n");
|
|
255
512
|
}
|
|
513
|
+
function convertTreeSitterResult(result, relativePath) {
|
|
514
|
+
if (!result)
|
|
515
|
+
return [];
|
|
516
|
+
const nodes = [];
|
|
517
|
+
const kindMap = {
|
|
518
|
+
function: "function",
|
|
519
|
+
method: "method",
|
|
520
|
+
class: "class",
|
|
521
|
+
interface: "interface",
|
|
522
|
+
type: "type",
|
|
523
|
+
constant: "constant",
|
|
524
|
+
declaration: "declaration"
|
|
525
|
+
};
|
|
526
|
+
for (const symbol of result.symbols) {
|
|
527
|
+
nodes.push({
|
|
528
|
+
kind: "declaration",
|
|
529
|
+
name: symbol.name,
|
|
530
|
+
fqn: symbol.fqn,
|
|
531
|
+
symbolKind: kindMap[symbol.kind] ?? "declaration",
|
|
532
|
+
range: { startLine: symbol.line, endLine: symbol.endLine },
|
|
533
|
+
text: symbol.name,
|
|
534
|
+
parentFqn: symbol.parentFqn
|
|
535
|
+
});
|
|
536
|
+
}
|
|
537
|
+
for (const call of result.calls) {
|
|
538
|
+
nodes.push({
|
|
539
|
+
kind: "call",
|
|
540
|
+
name: call.to,
|
|
541
|
+
range: { startLine: 1, endLine: 1 },
|
|
542
|
+
text: call.to
|
|
543
|
+
});
|
|
544
|
+
}
|
|
545
|
+
return dedupeNodes(nodes);
|
|
546
|
+
}
|
|
256
547
|
//# sourceMappingURL=structuredExtractor.js.map
|
|
@@ -0,0 +1,264 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.supportsTreeSitterLanguage = supportsTreeSitterLanguage;
|
|
4
|
+
exports.extractWithTreeSitter = extractWithTreeSitter;
|
|
5
|
+
let tsLanguage;
|
|
6
|
+
let tsxLanguage;
|
|
7
|
+
let loadAttempted = false;
|
|
8
|
+
function loadTreeSitter() {
|
|
9
|
+
if (loadAttempted)
|
|
10
|
+
return tsLanguage !== undefined;
|
|
11
|
+
loadAttempted = true;
|
|
12
|
+
try {
|
|
13
|
+
const Parser = require("tree-sitter");
|
|
14
|
+
const TS = require("tree-sitter-typescript");
|
|
15
|
+
tsLanguage = TS.typescript;
|
|
16
|
+
tsxLanguage = TS.tsx;
|
|
17
|
+
return true;
|
|
18
|
+
}
|
|
19
|
+
catch {
|
|
20
|
+
return false;
|
|
21
|
+
}
|
|
22
|
+
}
|
|
23
|
+
function supportsTreeSitterLanguage(language) {
|
|
24
|
+
if (!loadTreeSitter())
|
|
25
|
+
return false;
|
|
26
|
+
return language === "typescript" || language === "javascript" || language === "tsx" || language === "jsx";
|
|
27
|
+
}
|
|
28
|
+
function extractWithTreeSitter(content, language, relativePath) {
|
|
29
|
+
if (!loadTreeSitter())
|
|
30
|
+
return undefined;
|
|
31
|
+
if (!supportsTreeSitterLanguage(language))
|
|
32
|
+
return undefined;
|
|
33
|
+
try {
|
|
34
|
+
const Parser = require("tree-sitter");
|
|
35
|
+
const parser = new Parser();
|
|
36
|
+
const lang = language === "tsx" || language === "jsx" ? tsxLanguage : tsLanguage;
|
|
37
|
+
if (!lang)
|
|
38
|
+
return undefined;
|
|
39
|
+
parser.setLanguage(lang);
|
|
40
|
+
const tree = parser.parse(content);
|
|
41
|
+
const root = tree.rootNode;
|
|
42
|
+
const symbols = [];
|
|
43
|
+
const calls = [];
|
|
44
|
+
const imports = [];
|
|
45
|
+
extractFromNode(root, content, relativePath, symbols, calls, imports, "");
|
|
46
|
+
return {
|
|
47
|
+
extractorKind: "tree-sitter",
|
|
48
|
+
language,
|
|
49
|
+
parserVersion: "tree-sitter-1",
|
|
50
|
+
symbols: symbols.slice(0, 200),
|
|
51
|
+
imports: imports.slice(0, 100),
|
|
52
|
+
calls: calls.slice(0, 200)
|
|
53
|
+
};
|
|
54
|
+
}
|
|
55
|
+
catch (err) {
|
|
56
|
+
return undefined;
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
function extractFromNode(node, content, relativePath, symbols, calls, imports, parentFqn) {
|
|
60
|
+
if (!node || !node.type)
|
|
61
|
+
return;
|
|
62
|
+
try {
|
|
63
|
+
switch (node.type) {
|
|
64
|
+
case "function_declaration":
|
|
65
|
+
case "generator_function_declaration": {
|
|
66
|
+
const name = childText(node, "identifier") || childText(node, "name");
|
|
67
|
+
if (name) {
|
|
68
|
+
const fqn = parentFqn ? `${parentFqn}.${name}` : name;
|
|
69
|
+
symbols.push({
|
|
70
|
+
id: `${relativePath}:${fqn}:${node.startPosition.row + 1}`,
|
|
71
|
+
name,
|
|
72
|
+
fqn,
|
|
73
|
+
kind: parentFqn ? "method" : "function",
|
|
74
|
+
line: node.startPosition.row + 1,
|
|
75
|
+
endLine: node.endPosition.row + 1,
|
|
76
|
+
exported: false,
|
|
77
|
+
parentFqn: parentFqn || undefined
|
|
78
|
+
});
|
|
79
|
+
extractChildren(node, content, relativePath, symbols, calls, imports, fqn);
|
|
80
|
+
return;
|
|
81
|
+
}
|
|
82
|
+
break;
|
|
83
|
+
}
|
|
84
|
+
case "class_declaration": {
|
|
85
|
+
const name = childText(node, "type_identifier") || childText(node, "identifier") || childText(node, "name");
|
|
86
|
+
if (name) {
|
|
87
|
+
const fqn = parentFqn ? `${parentFqn}.${name}` : name;
|
|
88
|
+
symbols.push({
|
|
89
|
+
id: `${relativePath}:${fqn}:${node.startPosition.row + 1}`,
|
|
90
|
+
name,
|
|
91
|
+
fqn,
|
|
92
|
+
kind: "class",
|
|
93
|
+
line: node.startPosition.row + 1,
|
|
94
|
+
endLine: node.endPosition.row + 1,
|
|
95
|
+
exported: false,
|
|
96
|
+
parentFqn: parentFqn || undefined
|
|
97
|
+
});
|
|
98
|
+
extractChildren(node, content, relativePath, symbols, calls, imports, fqn);
|
|
99
|
+
return;
|
|
100
|
+
}
|
|
101
|
+
break;
|
|
102
|
+
}
|
|
103
|
+
case "method_definition": {
|
|
104
|
+
const name = childText(node, "property_identifier") || childText(node, "name");
|
|
105
|
+
if (name && parentFqn) {
|
|
106
|
+
const fqn = `${parentFqn}.${name}`;
|
|
107
|
+
symbols.push({
|
|
108
|
+
id: `${relativePath}:${fqn}:${node.startPosition.row + 1}`,
|
|
109
|
+
name,
|
|
110
|
+
fqn,
|
|
111
|
+
kind: "method",
|
|
112
|
+
line: node.startPosition.row + 1,
|
|
113
|
+
endLine: node.endPosition.row + 1,
|
|
114
|
+
exported: false,
|
|
115
|
+
parentFqn
|
|
116
|
+
});
|
|
117
|
+
extractChildren(node, content, relativePath, symbols, calls, imports, fqn);
|
|
118
|
+
return;
|
|
119
|
+
}
|
|
120
|
+
break;
|
|
121
|
+
}
|
|
122
|
+
case "variable_declarator": {
|
|
123
|
+
const name = childText(node, "identifier") || childText(node, "name");
|
|
124
|
+
const value = findChild(node, "arrow_function") || findChild(node, "function_expression");
|
|
125
|
+
if (name && value) {
|
|
126
|
+
const fqn = parentFqn ? `${parentFqn}.${name}` : name;
|
|
127
|
+
symbols.push({
|
|
128
|
+
id: `${relativePath}:${fqn}:${node.startPosition.row + 1}`,
|
|
129
|
+
name,
|
|
130
|
+
fqn,
|
|
131
|
+
kind: parentFqn ? "method" : "function",
|
|
132
|
+
line: node.startPosition.row + 1,
|
|
133
|
+
endLine: node.endPosition.row + 1,
|
|
134
|
+
exported: false,
|
|
135
|
+
parentFqn: parentFqn || undefined
|
|
136
|
+
});
|
|
137
|
+
extractChildren(node, content, relativePath, symbols, calls, imports, fqn);
|
|
138
|
+
return;
|
|
139
|
+
}
|
|
140
|
+
break;
|
|
141
|
+
}
|
|
142
|
+
case "call_expression": {
|
|
143
|
+
const funcName = extractCallName(node);
|
|
144
|
+
if (funcName) {
|
|
145
|
+
calls.push({
|
|
146
|
+
id: `${relativePath}:call:${funcName}:${node.startPosition.row + 1}`,
|
|
147
|
+
type: "callee",
|
|
148
|
+
from: parentFqn || relativePath,
|
|
149
|
+
to: funcName,
|
|
150
|
+
fromPath: relativePath,
|
|
151
|
+
toPath: "",
|
|
152
|
+
weight: 0.65,
|
|
153
|
+
reason: `calls ${funcName}`
|
|
154
|
+
});
|
|
155
|
+
}
|
|
156
|
+
break;
|
|
157
|
+
}
|
|
158
|
+
case "import_statement": {
|
|
159
|
+
const source = childText(node, "string")?.replace(/^["']|["']$/g, "");
|
|
160
|
+
if (source) {
|
|
161
|
+
const specifiers = collectImportSpecifiers(node);
|
|
162
|
+
imports.push({
|
|
163
|
+
source,
|
|
164
|
+
importedNames: specifiers,
|
|
165
|
+
resolvedPath: undefined,
|
|
166
|
+
relativePath,
|
|
167
|
+
line: node.startPosition.row + 1
|
|
168
|
+
});
|
|
169
|
+
}
|
|
170
|
+
return; // Don't recurse into imports
|
|
171
|
+
}
|
|
172
|
+
case "export_statement": {
|
|
173
|
+
// Mark the exported declaration
|
|
174
|
+
const decl = findChild(node, "function_declaration") || findChild(node, "class_declaration") ||
|
|
175
|
+
findChild(node, "variable_declaration") || findChild(node, "lexical_declaration");
|
|
176
|
+
if (decl) {
|
|
177
|
+
extractFromNode(decl, content, relativePath, symbols, calls, imports, parentFqn);
|
|
178
|
+
// Tag last symbol as exported
|
|
179
|
+
if (symbols.length > 0) {
|
|
180
|
+
symbols[symbols.length - 1] = { ...symbols[symbols.length - 1], exported: true };
|
|
181
|
+
}
|
|
182
|
+
}
|
|
183
|
+
return;
|
|
184
|
+
}
|
|
185
|
+
}
|
|
186
|
+
}
|
|
187
|
+
catch {
|
|
188
|
+
// Skip malformed nodes
|
|
189
|
+
}
|
|
190
|
+
// Recurse into children for unhandled node types
|
|
191
|
+
extractChildren(node, content, relativePath, symbols, calls, imports, parentFqn);
|
|
192
|
+
}
|
|
193
|
+
function extractChildren(node, content, relativePath, symbols, calls, imports, parentFqn) {
|
|
194
|
+
if (!node.children)
|
|
195
|
+
return;
|
|
196
|
+
for (const child of node.children) {
|
|
197
|
+
extractFromNode(child, content, relativePath, symbols, calls, imports, parentFqn);
|
|
198
|
+
}
|
|
199
|
+
}
|
|
200
|
+
function childText(node, type) {
|
|
201
|
+
if (!node.children)
|
|
202
|
+
return undefined;
|
|
203
|
+
for (const child of node.children) {
|
|
204
|
+
if (child.type === type)
|
|
205
|
+
return child.text;
|
|
206
|
+
}
|
|
207
|
+
return undefined;
|
|
208
|
+
}
|
|
209
|
+
function findChild(node, type) {
|
|
210
|
+
if (!node.children)
|
|
211
|
+
return undefined;
|
|
212
|
+
for (const child of node.children) {
|
|
213
|
+
if (child.type === type)
|
|
214
|
+
return child;
|
|
215
|
+
}
|
|
216
|
+
if (node.namedChildren) {
|
|
217
|
+
for (const child of node.namedChildren) {
|
|
218
|
+
if (child.type === type)
|
|
219
|
+
return child;
|
|
220
|
+
}
|
|
221
|
+
}
|
|
222
|
+
return undefined;
|
|
223
|
+
}
|
|
224
|
+
function extractCallName(node) {
|
|
225
|
+
if (!node.children)
|
|
226
|
+
return undefined;
|
|
227
|
+
for (const child of node.children) {
|
|
228
|
+
if (child.type === "identifier")
|
|
229
|
+
return child.text;
|
|
230
|
+
if (child.type === "member_expression") {
|
|
231
|
+
// e.g. obj.method() -> return "method"
|
|
232
|
+
const parts = [];
|
|
233
|
+
for (const mc of child.children) {
|
|
234
|
+
if (mc.type === "property_identifier")
|
|
235
|
+
parts.push(mc.text);
|
|
236
|
+
else if (mc.type === "identifier")
|
|
237
|
+
parts.push(mc.text);
|
|
238
|
+
}
|
|
239
|
+
if (parts.length > 0)
|
|
240
|
+
return parts[parts.length - 1];
|
|
241
|
+
}
|
|
242
|
+
}
|
|
243
|
+
return undefined;
|
|
244
|
+
}
|
|
245
|
+
function collectImportSpecifiers(node) {
|
|
246
|
+
const names = [];
|
|
247
|
+
if (!node.children)
|
|
248
|
+
return names;
|
|
249
|
+
for (const child of node.children) {
|
|
250
|
+
if (child.type === "import_specifier" || child.type === "named_imports") {
|
|
251
|
+
if (child.children) {
|
|
252
|
+
for (const spec of child.children) {
|
|
253
|
+
if (spec.type === "identifier")
|
|
254
|
+
names.push(spec.text);
|
|
255
|
+
}
|
|
256
|
+
}
|
|
257
|
+
}
|
|
258
|
+
if (child.type === "namespace_import") {
|
|
259
|
+
names.push("*");
|
|
260
|
+
}
|
|
261
|
+
}
|
|
262
|
+
return names;
|
|
263
|
+
}
|
|
264
|
+
//# sourceMappingURL=treeSitterExtractor.js.map
|