graph-indexer 1.1.0 → 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +412 -0
- package/README.md +228 -350
- package/SECURITY.md +68 -0
- package/TRADEMARK.md +31 -0
- package/cli-ui.mjs +78 -0
- package/config.mjs +269 -0
- package/daemon-ctl.mjs +175 -0
- package/daemon-lock.mjs +69 -0
- package/embedders/python/mlx_embed_server.py +61 -0
- package/embedders/python/requirements-mlx.txt +5 -0
- package/embedders/setup-mlx.mjs +110 -0
- package/embeddings.mjs +489 -0
- package/engine/binary.mjs +399 -0
- package/{core-engine.mjs → engine/memory.mjs} +359 -249
- package/engine/sqlite.mjs +863 -0
- package/enrichment.mjs +426 -0
- package/git-signals.mjs +248 -0
- package/indexer.mjs +216 -77
- package/init.mjs +1604 -170
- package/layout.mjs +123 -0
- package/mcp/format.mjs +61 -0
- package/mcp/tools.mjs +1157 -0
- package/mcp/topology.mjs +551 -0
- package/mcp-server.mjs +118 -573
- package/package.json +56 -16
- package/parse/extractor.mjs +617 -0
- package/parse/imports.mjs +365 -0
- package/parse/languages.mjs +89 -0
- package/parse/metadata.mjs +655 -0
- package/parse/routes.mjs +327 -0
- package/prompts/CORE.md +86 -0
- package/prompts/DOMAIN_TEMPLATE.md +46 -0
- package/prompts/INTEGRATION.md +123 -0
- package/prompts/frameworks/ANDROID.md +17 -0
- package/prompts/frameworks/ASPNET_CORE.md +19 -0
- package/prompts/frameworks/FASTAPI_DJANGO.md +19 -0
- package/prompts/frameworks/LARAVEL_SYMFONY.md +22 -0
- package/prompts/frameworks/NODE_EXPRESS_NESTJS.md +18 -0
- package/prompts/frameworks/RAILS.md +20 -0
- package/prompts/frameworks/REACT.md +20 -0
- package/prompts/frameworks/SPRING_BOOT.md +19 -0
- package/prompts/languages/BASH.md +22 -0
- package/prompts/languages/C.md +21 -0
- package/prompts/languages/CSHARP.md +24 -0
- package/prompts/languages/CSS_SCSS.md +22 -0
- package/prompts/languages/GO.md +23 -0
- package/prompts/languages/JAVA.md +23 -0
- package/prompts/languages/JAVASCRIPT_TYPESCRIPT.md +24 -0
- package/prompts/languages/KOTLIN.md +24 -0
- package/prompts/languages/PHP.md +23 -0
- package/prompts/languages/PYTHON.md +23 -0
- package/prompts/languages/RUBY.md +23 -0
- package/prompts/languages/RUST.md +21 -0
- package/prompts/languages/SWIFT.md +21 -0
- package/search-core.mjs +627 -0
- package/storage.mjs +68 -0
- package/watch-daemon.mjs +204 -69
- package/PROMPT.md +0 -143
- package/parser-utils.mjs +0 -861
package/parser-utils.mjs
DELETED
|
@@ -1,861 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* @file parser-utils.mjs
|
|
3
|
-
* @description Shared AST parsing, language registry, and embedding utilities.
|
|
4
|
-
* @author MaquinaTech <https://github.com/MaquinaTech>
|
|
5
|
-
* @copyright (c) 2026 MaquinaTech. All rights reserved.
|
|
6
|
-
* @license MIT
|
|
7
|
-
* Copyright (c) 2026 MaquinaTech. All rights reserved.
|
|
8
|
-
* Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
9
|
-
* of this software and associated documentation files (the "Software"), to deal
|
|
10
|
-
* in the Software without restriction, including without limitation the rights
|
|
11
|
-
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
12
|
-
* copies of the Software, and to permit persons to whom the Software is
|
|
13
|
-
* furnished to do so, subject to the following conditions: The above copyright
|
|
14
|
-
* notice and this permission notice shall be included in all copies or
|
|
15
|
-
* substantial portions of the Software.
|
|
16
|
-
*/
|
|
17
|
-
import fs from 'fs';
|
|
18
|
-
import path from 'path';
|
|
19
|
-
import { createHash } from 'crypto';
|
|
20
|
-
import ignore from 'ignore';
|
|
21
|
-
import Parser from 'tree-sitter';
|
|
22
|
-
const { Query } = Parser; // Native parser query helper
|
|
23
|
-
|
|
24
|
-
import { truncateForEmbedding } from './core-engine.mjs';
|
|
25
|
-
|
|
26
|
-
export const MAX_FILE_SIZE_BYTES = 500000;
|
|
27
|
-
export const OLLAMA_HOST = process.env.OLLAMA_HOST || "http://localhost:11434";
|
|
28
|
-
|
|
29
|
-
// ─── Dynamic Language Loading ─────────────────────────────────────────────────
|
|
30
|
-
|
|
31
|
-
function _loadProjectConfig() {
|
|
32
|
-
const configPath = path.join(process.env.MCP_PROJECT_ROOT || process.cwd(), '.graph-indexer.json');
|
|
33
|
-
try {
|
|
34
|
-
if (fs.existsSync(configPath)) return JSON.parse(fs.readFileSync(configPath, 'utf-8'));
|
|
35
|
-
} catch { /* ignore */ }
|
|
36
|
-
return null;
|
|
37
|
-
}
|
|
38
|
-
|
|
39
|
-
async function _tryLang(pkg, enabledLangs, key) {
|
|
40
|
-
if (enabledLangs && !enabledLangs.includes(key)) return null;
|
|
41
|
-
try {
|
|
42
|
-
return (await import(pkg)).default;
|
|
43
|
-
} catch {
|
|
44
|
-
process.stderr.write(`[graph-indexer] WARNING: ${pkg} not installed — ${key} files will be skipped\n`);
|
|
45
|
-
return null;
|
|
46
|
-
}
|
|
47
|
-
}
|
|
48
|
-
|
|
49
|
-
const _cfg = _loadProjectConfig();
|
|
50
|
-
const _enabled = _cfg?.languages ?? null; // null = all languages
|
|
51
|
-
|
|
52
|
-
const [
|
|
53
|
-
TypeScript, JavaScript, CSS, Python, Rust,
|
|
54
|
-
Go, PHP, Java, Kotlin, CSharp, Ruby
|
|
55
|
-
] = await Promise.all([
|
|
56
|
-
_tryLang('tree-sitter-typescript', _enabled, 'typescript'),
|
|
57
|
-
_tryLang('tree-sitter-javascript', _enabled, 'javascript'),
|
|
58
|
-
_tryLang('tree-sitter-css', _enabled, 'css'),
|
|
59
|
-
_tryLang('tree-sitter-python', _enabled, 'python'),
|
|
60
|
-
_tryLang('tree-sitter-rust', _enabled, 'rust'),
|
|
61
|
-
_tryLang('tree-sitter-go', _enabled, 'go'),
|
|
62
|
-
_tryLang('tree-sitter-php', _enabled, 'php'),
|
|
63
|
-
_tryLang('tree-sitter-java', _enabled, 'java'),
|
|
64
|
-
_tryLang('tree-sitter-kotlin', _enabled, 'kotlin'),
|
|
65
|
-
_tryLang('tree-sitter-c-sharp', _enabled, 'csharp'),
|
|
66
|
-
_tryLang('tree-sitter-ruby', _enabled, 'ruby'),
|
|
67
|
-
]);
|
|
68
|
-
|
|
69
|
-
const LANGUAGE_MAP = {
|
|
70
|
-
...(TypeScript ? { '.ts': TypeScript.typescript, '.tsx': TypeScript.tsx } : {}),
|
|
71
|
-
...(JavaScript ? { '.js': JavaScript, '.jsx': JavaScript, '.mjs': JavaScript, '.cjs': JavaScript } : {}),
|
|
72
|
-
...(CSS ? { '.css': CSS, '.scss': CSS } : {}),
|
|
73
|
-
...(Python ? { '.py': Python } : {}),
|
|
74
|
-
...(Rust ? { '.rs': Rust } : {}),
|
|
75
|
-
...(Go ? { '.go': Go } : {}),
|
|
76
|
-
...(PHP ? { '.php': PHP.php } : {}),
|
|
77
|
-
...(Java ? { '.java': Java } : {}),
|
|
78
|
-
...(Kotlin ? { '.kt': Kotlin, '.kts': Kotlin } : {}),
|
|
79
|
-
...(CSharp ? { '.cs': CSharp } : {}),
|
|
80
|
-
...(Ruby ? { '.rb': Ruby } : {}),
|
|
81
|
-
};
|
|
82
|
-
|
|
83
|
-
// 🥇 ULTRA-GENERIC QUERIES: Immune to grammar changes between TS/JS/TSX
|
|
84
|
-
const LANGUAGE_QUERIES = {
|
|
85
|
-
ts: `
|
|
86
|
-
(class_declaration) @chunk
|
|
87
|
-
(function_declaration) @chunk
|
|
88
|
-
(method_definition) @chunk
|
|
89
|
-
(lexical_declaration) @chunk
|
|
90
|
-
(expression_statement) @chunk
|
|
91
|
-
(export_statement) @chunk
|
|
92
|
-
`,
|
|
93
|
-
css: `(rule_set) @chunk`,
|
|
94
|
-
py: `
|
|
95
|
-
(function_definition) @chunk
|
|
96
|
-
(class_definition) @chunk
|
|
97
|
-
(expression_statement) @chunk
|
|
98
|
-
`,
|
|
99
|
-
rs: `
|
|
100
|
-
(function_item) @chunk
|
|
101
|
-
(struct_item) @chunk
|
|
102
|
-
(enum_item) @chunk
|
|
103
|
-
(trait_item) @chunk
|
|
104
|
-
(impl_item) @chunk
|
|
105
|
-
`,
|
|
106
|
-
go: `
|
|
107
|
-
(function_declaration) @chunk
|
|
108
|
-
(method_declaration) @chunk
|
|
109
|
-
(type_declaration) @chunk
|
|
110
|
-
`,
|
|
111
|
-
php: `
|
|
112
|
-
(function_definition) @chunk
|
|
113
|
-
(class_declaration) @chunk
|
|
114
|
-
(expression_statement) @chunk
|
|
115
|
-
`,
|
|
116
|
-
java: `
|
|
117
|
-
(method_declaration) @chunk
|
|
118
|
-
(class_declaration) @chunk
|
|
119
|
-
(interface_declaration) @chunk
|
|
120
|
-
(constructor_declaration) @chunk
|
|
121
|
-
(enum_declaration) @chunk
|
|
122
|
-
`,
|
|
123
|
-
kotlin: `
|
|
124
|
-
(function_declaration) @chunk
|
|
125
|
-
(class_declaration) @chunk
|
|
126
|
-
(object_declaration) @chunk
|
|
127
|
-
(companion_object) @chunk
|
|
128
|
-
(secondary_constructor) @chunk
|
|
129
|
-
`,
|
|
130
|
-
cs: `
|
|
131
|
-
(method_declaration) @chunk
|
|
132
|
-
(class_declaration) @chunk
|
|
133
|
-
(interface_declaration) @chunk
|
|
134
|
-
(constructor_declaration) @chunk
|
|
135
|
-
(enum_declaration) @chunk
|
|
136
|
-
(property_declaration) @chunk
|
|
137
|
-
`,
|
|
138
|
-
rb: `
|
|
139
|
-
(method) @chunk
|
|
140
|
-
(singleton_method) @chunk
|
|
141
|
-
(class) @chunk
|
|
142
|
-
(module) @chunk
|
|
143
|
-
`
|
|
144
|
-
};
|
|
145
|
-
|
|
146
|
-
export const EXTENSIONS = new Set(Object.keys(LANGUAGE_MAP));
|
|
147
|
-
|
|
148
|
-
// Definition of container node types to avoid nested duplicates
|
|
149
|
-
const CONTAINERS = new Set([
|
|
150
|
-
'class_declaration', 'function_declaration', 'method_definition',
|
|
151
|
-
'lexical_declaration', 'expression_statement', 'export_statement',
|
|
152
|
-
'function_definition', 'class_definition', 'rule_set',
|
|
153
|
-
'function_item', 'struct_item', 'enum_item', 'trait_item', 'impl_item',
|
|
154
|
-
'method_declaration', 'type_declaration',
|
|
155
|
-
// Java / C#
|
|
156
|
-
'interface_declaration', 'constructor_declaration', 'enum_declaration',
|
|
157
|
-
// Kotlin
|
|
158
|
-
'object_declaration', 'companion_object', 'secondary_constructor',
|
|
159
|
-
// C#
|
|
160
|
-
'property_declaration',
|
|
161
|
-
// Ruby
|
|
162
|
-
'method', 'singleton_method', 'module'
|
|
163
|
-
]);
|
|
164
|
-
|
|
165
|
-
export function getParserForFile(ext) {
|
|
166
|
-
const language = LANGUAGE_MAP[ext];
|
|
167
|
-
if (!language) return null;
|
|
168
|
-
const parser = new Parser();
|
|
169
|
-
parser.setLanguage(language);
|
|
170
|
-
return parser;
|
|
171
|
-
}
|
|
172
|
-
|
|
173
|
-
export function buildIgnoreFilter(rootPath) {
|
|
174
|
-
const ig = ignore();
|
|
175
|
-
ig.add(['node_modules', '.git', 'dist', 'build', '.next', 'coverage', '*.tmp', 'vendor', '.venv']);
|
|
176
|
-
const gitignorePath = path.join(rootPath, '.gitignore');
|
|
177
|
-
if (fs.existsSync(gitignorePath)) ig.add(fs.readFileSync(gitignorePath, 'utf-8'));
|
|
178
|
-
return ig;
|
|
179
|
-
}
|
|
180
|
-
|
|
181
|
-
export function generateChunkHash(text) {
|
|
182
|
-
return createHash('sha256').update(text).digest('hex');
|
|
183
|
-
}
|
|
184
|
-
|
|
185
|
-
export function extractFileSkeleton(rootNode, content) {
|
|
186
|
-
const signatures = [];
|
|
187
|
-
function walk(node) {
|
|
188
|
-
if (node.type.includes('declaration') || node.type.includes('definition')) {
|
|
189
|
-
let name = node.childForFieldName?.("name")?.text || "anonymous";
|
|
190
|
-
signatures.push(`- [${node.type}] ${name} (lines ${node.startPosition.row + 1}-${node.endPosition.row + 1})`);
|
|
191
|
-
}
|
|
192
|
-
node.children.forEach(walk);
|
|
193
|
-
}
|
|
194
|
-
walk(rootNode);
|
|
195
|
-
return signatures.join('\n');
|
|
196
|
-
}
|
|
197
|
-
|
|
198
|
-
export function extractImportsFromAST(rootNode, ext) {
|
|
199
|
-
const imports = new Set();
|
|
200
|
-
function walk(node) {
|
|
201
|
-
// ── JavaScript / TypeScript ──────────────────────────────────────────
|
|
202
|
-
if (['.ts', '.tsx', '.js', '.jsx', '.mjs', '.cjs'].includes(ext)) {
|
|
203
|
-
if (node.type === 'import_statement') {
|
|
204
|
-
const source = node.children.find(c => c.type === 'string');
|
|
205
|
-
if (source) imports.add(source.text.replace(/['"]/g, ''));
|
|
206
|
-
} else if (node.type === 'call_expression' && node.children[0]?.text === 'require') {
|
|
207
|
-
const arg = node.children[1]?.children?.find(c => c.type === 'string');
|
|
208
|
-
if (arg) imports.add(arg.text.replace(/['"]/g, ''));
|
|
209
|
-
}
|
|
210
|
-
}
|
|
211
|
-
// ── Python ──────────────────────────────────────────────────────────
|
|
212
|
-
else if (ext === '.py') {
|
|
213
|
-
if (node.type === 'import_statement') {
|
|
214
|
-
// import foo.bar.baz → store as foo/bar/baz
|
|
215
|
-
for (const child of node.children) {
|
|
216
|
-
if (child.type === 'dotted_name') imports.add(child.text.replace(/\./g, '/'));
|
|
217
|
-
}
|
|
218
|
-
} else if (node.type === 'import_from_statement') {
|
|
219
|
-
// from .sibling import X → relative: emit as ./sibling
|
|
220
|
-
// from ..pkg import Y → relative: emit as ../pkg
|
|
221
|
-
// from foo.bar import Z → absolute: store foo/bar
|
|
222
|
-
const relNode = node.children.find(c => c.type === 'relative_import');
|
|
223
|
-
if (relNode) {
|
|
224
|
-
// dots count = depth; remainder is the module path after dots
|
|
225
|
-
const raw = relNode.text; // e.g. '.' or '..utils'
|
|
226
|
-
const dots = raw.match(/^\.+/)?.[0] ?? '.';
|
|
227
|
-
const mod = raw.slice(dots.length);
|
|
228
|
-
const prefix = dots.length === 1 ? './' : '../'.repeat(dots.length - 1);
|
|
229
|
-
imports.add(mod ? prefix + mod.replace(/\./g, '/') : prefix.slice(0, -1) || '.');
|
|
230
|
-
} else {
|
|
231
|
-
const mod = node.children.find(c => c.type === 'dotted_name');
|
|
232
|
-
if (mod) imports.add(mod.text.replace(/\./g, '/'));
|
|
233
|
-
}
|
|
234
|
-
}
|
|
235
|
-
}
|
|
236
|
-
// ── Rust ────────────────────────────────────────────────────────────
|
|
237
|
-
else if (ext === '.rs') {
|
|
238
|
-
if (node.type === 'use_declaration') {
|
|
239
|
-
// Collect the first path text from the argument subtree
|
|
240
|
-
const arg = node.childForFieldName?.('argument') ||
|
|
241
|
-
node.children.find(c => !['use', ';', 'pub'].includes(c.type));
|
|
242
|
-
if (arg) imports.add(arg.text.split('::').slice(0, 3).join('::'));
|
|
243
|
-
}
|
|
244
|
-
}
|
|
245
|
-
// ── Go ──────────────────────────────────────────────────────────────
|
|
246
|
-
else if (ext === '.go') {
|
|
247
|
-
if (node.type === 'import_spec') {
|
|
248
|
-
const pathNode = node.children.find(c =>
|
|
249
|
-
c.type === 'interpreted_string_literal' || c.type === 'raw_string_literal');
|
|
250
|
-
if (pathNode) imports.add(pathNode.text.replace(/['"`]/g, ''));
|
|
251
|
-
}
|
|
252
|
-
}
|
|
253
|
-
// ── PHP ─────────────────────────────────────────────────────────────
|
|
254
|
-
else if (ext === '.php') {
|
|
255
|
-
if (node.type === 'include_expression' || node.type === 'require_expression' ||
|
|
256
|
-
node.type === 'include_once_expression' || node.type === 'require_once_expression') {
|
|
257
|
-
const strNode = node.children.find(c =>
|
|
258
|
-
c.type === 'string' || c.type === 'encapsed_string');
|
|
259
|
-
if (strNode) imports.add(strNode.text.replace(/['"]/g, ''));
|
|
260
|
-
}
|
|
261
|
-
} // ── Java ─────────────────────────────────────────────────────────────
|
|
262
|
-
else if (ext === '.java') {
|
|
263
|
-
if (node.type === 'import_declaration') {
|
|
264
|
-
const scopedId = node.children.find(c => c.type === 'scoped_identifier' || c.type === 'identifier');
|
|
265
|
-
if (scopedId) imports.add(scopedId.text.replace(/\./g, '/'));
|
|
266
|
-
}
|
|
267
|
-
}
|
|
268
|
-
// ── Kotlin ───────────────────────────────────────────────────────────
|
|
269
|
-
else if (ext === '.kt' || ext === '.kts') {
|
|
270
|
-
if (node.type === 'import_header') {
|
|
271
|
-
const path = node.children.find(c => c.type === 'identifier' || c.type === 'user_type' || c.isNamed);
|
|
272
|
-
const raw = node.text.replace(/^import\s+/, '').replace(/\s*\.\*\s*$/, '').trim();
|
|
273
|
-
if (raw) imports.add(raw.replace(/\./g, '/'));
|
|
274
|
-
}
|
|
275
|
-
}
|
|
276
|
-
// ── C# ───────────────────────────────────────────────────────────────
|
|
277
|
-
else if (ext === '.cs') {
|
|
278
|
-
if (node.type === 'using_directive') {
|
|
279
|
-
const ns = node.children.find(c => c.type === 'qualified_name' || c.type === 'identifier' || c.type === 'name_equals');
|
|
280
|
-
if (ns) {
|
|
281
|
-
const raw = ns.text.replace(/\s*=\s*.*$/, '').trim();
|
|
282
|
-
if (raw) imports.add(raw.replace(/\./g, '/'));
|
|
283
|
-
}
|
|
284
|
-
}
|
|
285
|
-
}
|
|
286
|
-
// ── Ruby ──────────────────────────────────────────────────────────────
|
|
287
|
-
else if (ext === '.rb') {
|
|
288
|
-
if (node.type === 'call' || node.type === 'method_call') {
|
|
289
|
-
const method = node.childForFieldName?.('method') || node.children[0];
|
|
290
|
-
if (method && (method.text === 'require' || method.text === 'require_relative')) {
|
|
291
|
-
const args = node.childForFieldName?.('arguments') || node.children.find(c => c.type === 'argument_list');
|
|
292
|
-
const strArg = args?.children?.find(c => c.type === 'string' || c.type === 'simple_string');
|
|
293
|
-
if (strArg) imports.add(strArg.text.replace(/['"]/g, ''));
|
|
294
|
-
}
|
|
295
|
-
}
|
|
296
|
-
}
|
|
297
|
-
node.children.forEach(walk);
|
|
298
|
-
}
|
|
299
|
-
walk(rootNode);
|
|
300
|
-
return Array.from(imports);
|
|
301
|
-
}
|
|
302
|
-
|
|
303
|
-
export function extractSemanticChunks(rootNode, relPath, sourceCode, ext) {
|
|
304
|
-
const chunks = [];
|
|
305
|
-
const parser = getParserForFile(ext);
|
|
306
|
-
if (!parser) return chunks;
|
|
307
|
-
|
|
308
|
-
const JS_LIKE = ['.ts', '.tsx', '.js', '.jsx', '.mjs', '.cjs'];
|
|
309
|
-
const EXT_TO_LANG = {
|
|
310
|
-
'.java': 'java', '.kt': 'kotlin', '.kts': 'kotlin', '.cs': 'cs',
|
|
311
|
-
'.rb': 'rb'
|
|
312
|
-
};
|
|
313
|
-
const langKey = JS_LIKE.includes(ext) ? 'ts'
|
|
314
|
-
: (EXT_TO_LANG[ext] || (LANGUAGE_QUERIES[ext.slice(1)] ? ext.slice(1) : null));
|
|
315
|
-
if (!langKey || !LANGUAGE_QUERIES[langKey]) return chunks;
|
|
316
|
-
|
|
317
|
-
// 🥇 HEADER INHERITANCE: Extract top-of-file/global comments for module context
|
|
318
|
-
let fileDocstring = "";
|
|
319
|
-
let cursorNode = rootNode.children[0];
|
|
320
|
-
while (cursorNode) {
|
|
321
|
-
if (cursorNode.type === 'comment') fileDocstring += cursorNode.text + "\n";
|
|
322
|
-
else if (cursorNode.type !== 'import_statement' && cursorNode.type !== 'expression_statement') break;
|
|
323
|
-
cursorNode = cursorNode.nextSibling;
|
|
324
|
-
}
|
|
325
|
-
fileDocstring = fileDocstring.trim();
|
|
326
|
-
|
|
327
|
-
try {
|
|
328
|
-
const query = new Query(parser.getLanguage(), LANGUAGE_QUERIES[langKey]);
|
|
329
|
-
const matches = query.matches(rootNode);
|
|
330
|
-
const processedNodes = new Set();
|
|
331
|
-
|
|
332
|
-
for (const match of matches) {
|
|
333
|
-
let chunkNode = null;
|
|
334
|
-
for (const capture of match.captures) {
|
|
335
|
-
if (capture.name === 'chunk') chunkNode = capture.node;
|
|
336
|
-
}
|
|
337
|
-
|
|
338
|
-
if (!chunkNode || processedNodes.has(chunkNode.id)) continue;
|
|
339
|
-
processedNodes.add(chunkNode.id);
|
|
340
|
-
|
|
341
|
-
// Filter out very small fragments (simple variables, etc.)
|
|
342
|
-
if (chunkNode.endPosition.row - chunkNode.startPosition.row < 2) continue;
|
|
343
|
-
|
|
344
|
-
// 🥇 DEDUPLICATION LOGIC: Ignore nodes that are nested inside other container nodes.
|
|
345
|
-
// Stop at the actual tree root (parent === null) to be language-agnostic:
|
|
346
|
-
// Python root = 'module', JS root = 'program', Go root = 'source_file', Ruby root = 'program'.
|
|
347
|
-
// Stopping at 'program' alone was falsely marking top-level Python classes as nested
|
|
348
|
-
// because Ruby's 'module' keyword (also in CONTAINERS) shares the name with Python's root.
|
|
349
|
-
let isNested = false;
|
|
350
|
-
let currentParent = chunkNode.parent;
|
|
351
|
-
while (currentParent && currentParent.parent !== null) {
|
|
352
|
-
if (CONTAINERS.has(currentParent.type)) {
|
|
353
|
-
isNested = true;
|
|
354
|
-
break;
|
|
355
|
-
}
|
|
356
|
-
currentParent = currentParent.parent;
|
|
357
|
-
}
|
|
358
|
-
if (isNested) continue;
|
|
359
|
-
|
|
360
|
-
// 🥇 ROBUST NAME EXTRACTION (JS logic)
|
|
361
|
-
let nameText = "anonymous";
|
|
362
|
-
const nameNode = chunkNode.childForFieldName?.("name");
|
|
363
|
-
|
|
364
|
-
if (nameNode) {
|
|
365
|
-
nameText = nameNode.text;
|
|
366
|
-
} else if (chunkNode.type === "export_statement") {
|
|
367
|
-
const decl = chunkNode.children.find(c => ['lexical_declaration', 'function_declaration', 'class_declaration'].includes(c.type));
|
|
368
|
-
if (decl) {
|
|
369
|
-
if (decl.type === "lexical_declaration") {
|
|
370
|
-
const varDecl = decl.children.find(c => c.type === "variable_declarator");
|
|
371
|
-
nameText = varDecl?.children.find(c => c.type === "identifier")?.text || "anonymous";
|
|
372
|
-
} else {
|
|
373
|
-
nameText = decl.childForFieldName?.("name")?.text || "anonymous";
|
|
374
|
-
}
|
|
375
|
-
} else {
|
|
376
|
-
const defaultChild = chunkNode.children.find(c => c.type === 'identifier' || c.type === 'call_expression');
|
|
377
|
-
if (defaultChild) {
|
|
378
|
-
if (defaultChild.type === 'identifier') {
|
|
379
|
-
nameText = `default_${defaultChild.text}`;
|
|
380
|
-
} else if (defaultChild.type === 'call_expression') {
|
|
381
|
-
const funcName = defaultChild.childForFieldName?.("function")?.text || defaultChild.children[0]?.text;
|
|
382
|
-
const argNode = defaultChild.childForFieldName?.("arguments")?.children?.find(c => c.type === 'identifier');
|
|
383
|
-
nameText = argNode ? `default_${funcName}_${argNode.text}` : `default_${funcName}`;
|
|
384
|
-
}
|
|
385
|
-
} else {
|
|
386
|
-
nameText = "default_export";
|
|
387
|
-
}
|
|
388
|
-
}
|
|
389
|
-
} else if (chunkNode.type === "lexical_declaration") {
|
|
390
|
-
const decl = chunkNode.children.find(c => c.type === "variable_declarator");
|
|
391
|
-
nameText = decl?.children.find(c => c.type === "identifier")?.text || "anonymous";
|
|
392
|
-
} else if (chunkNode.type === "expression_statement") {
|
|
393
|
-
const callExp = chunkNode.children.find(c => c.type === "call_expression");
|
|
394
|
-
const assignExp = chunkNode.children.find(c => c.type === "assignment_expression");
|
|
395
|
-
if (callExp) {
|
|
396
|
-
const funcName = callExp.childForFieldName?.("function")?.text || callExp.children[0]?.text;
|
|
397
|
-
const argsNode = callExp.childForFieldName?.("arguments");
|
|
398
|
-
const stringArg = argsNode?.children?.find(c => c.type === "string" || c.type === "template_string");
|
|
399
|
-
nameText = stringArg ? `${funcName}_${stringArg.text.replace(/['"`]/g, '')}` : (funcName || "anonymous");
|
|
400
|
-
} else if (assignExp) {
|
|
401
|
-
nameText = assignExp.childForFieldName?.("left")?.text || "anonymous";
|
|
402
|
-
}
|
|
403
|
-
} else {
|
|
404
|
-
const idNode = chunkNode.children.find(c => c.type === "identifier" || c.type === "name" || c.type === "property_identifier");
|
|
405
|
-
nameText = idNode?.text || "anonymous";
|
|
406
|
-
}
|
|
407
|
-
|
|
408
|
-
if (nameText === "anonymous" || nameText === "default_export") {
|
|
409
|
-
nameText = `${path.basename(relPath, path.extname(relPath))}_${chunkNode.type}`;
|
|
410
|
-
}
|
|
411
|
-
|
|
412
|
-
let docstring = "";
|
|
413
|
-
let prev = chunkNode.previousSibling;
|
|
414
|
-
while (prev && (prev.type === 'comment' || !prev.isNamed)) {
|
|
415
|
-
if (prev.type === 'comment') docstring = prev.text + "\n" + docstring;
|
|
416
|
-
prev = prev.previousSibling;
|
|
417
|
-
}
|
|
418
|
-
docstring = docstring.trim();
|
|
419
|
-
|
|
420
|
-
if (!docstring && fileDocstring) {
|
|
421
|
-
docstring = `[File Context]: ${fileDocstring}`;
|
|
422
|
-
}
|
|
423
|
-
|
|
424
|
-
const snippet = chunkNode.text.slice(0, 3000);
|
|
425
|
-
const hash = generateChunkHash(docstring + snippet);
|
|
426
|
-
const outgoingCalls = extractCalls(chunkNode);
|
|
427
|
-
|
|
428
|
-
// 🥇 PARAMETER / TYPE / CLASS CONTEXT ENRICHMENT (improves recall on undocumented code)
|
|
429
|
-
const params = extractParams(chunkNode, ext);
|
|
430
|
-
const returnType = extractReturnType(chunkNode, ext);
|
|
431
|
-
const classContext = extractClassContext(chunkNode);
|
|
432
|
-
const typeRefs = extractTypeAnnotations(chunkNode, ext);
|
|
433
|
-
|
|
434
|
-
const id = createHash('sha256')
|
|
435
|
-
.update(`${relPath}::${chunkNode.startPosition.row}::${chunkNode.startPosition.column}`)
|
|
436
|
-
.digest('hex').slice(0, 24);
|
|
437
|
-
|
|
438
|
-
chunks.push({
|
|
439
|
-
id, file_path: relPath, node_type: chunkNode.type, name: nameText,
|
|
440
|
-
docstring: docstring, code_snippet: snippet, content_hash: hash,
|
|
441
|
-
start_line: chunkNode.startPosition.row + 1, end_line: chunkNode.endPosition.row + 1,
|
|
442
|
-
calls: outgoingCalls,
|
|
443
|
-
params, return_type: returnType, class_context: classContext,
|
|
444
|
-
type_refs: typeRefs,
|
|
445
|
-
});
|
|
446
|
-
}
|
|
447
|
-
} catch (e) {
|
|
448
|
-
// Visible protective log for developers
|
|
449
|
-
process.stderr.write(`\n[parser-utils] 💥 Query Error in ${relPath}: ${e.message}\n`);
|
|
450
|
-
}
|
|
451
|
-
return chunks;
|
|
452
|
-
}
|
|
453
|
-
|
|
454
|
-
// ─── Barrel export resolution ─────────────────────────────────────────────────
|
|
455
|
-
|
|
456
|
-
// Module-level cache: barrelAbsPath → Map<exportedName, sourceRelPath>
|
|
457
|
-
const _barrelCache = new Map();
|
|
458
|
-
|
|
459
|
-
/**
|
|
460
|
-
* Parses a barrel file (index.ts / index.js) and returns a map of
|
|
461
|
-
* exportedName → sourceFilePath (relative to projectRoot).
|
|
462
|
-
* e.g. { useAuthStore: 'src/stores/authStore.ts' }
|
|
463
|
-
*/
|
|
464
|
-
export function resolveBarrelExports(barrelAbsPath, projectRoot) {
|
|
465
|
-
if (_barrelCache.has(barrelAbsPath)) return _barrelCache.get(barrelAbsPath);
|
|
466
|
-
|
|
467
|
-
const result = new Map();
|
|
468
|
-
_barrelCache.set(barrelAbsPath, result);
|
|
469
|
-
|
|
470
|
-
const ext = path.extname(barrelAbsPath);
|
|
471
|
-
if (!['.ts', '.tsx', '.js', '.jsx', '.mjs', '.cjs'].includes(ext)) return result;
|
|
472
|
-
|
|
473
|
-
let content;
|
|
474
|
-
try { content = fs.readFileSync(barrelAbsPath, 'utf-8'); } catch { return result; }
|
|
475
|
-
|
|
476
|
-
const parser = getParserForFile(ext);
|
|
477
|
-
if (!parser) return result;
|
|
478
|
-
|
|
479
|
-
let tree;
|
|
480
|
-
try { tree = parser.parse((offset) => offset < content.length ? content.slice(offset, offset + 4096) : null); } catch { return result; }
|
|
481
|
-
|
|
482
|
-
const barrelDir = path.dirname(barrelAbsPath);
|
|
483
|
-
|
|
484
|
-
function walk(node) {
|
|
485
|
-
// export { X, Y as Z } from './source'
|
|
486
|
-
if (node.type === 'export_statement') {
|
|
487
|
-
const fromNode = node.children.find(c => c.type === 'string');
|
|
488
|
-
if (!fromNode) { node.children.forEach(walk); return; }
|
|
489
|
-
|
|
490
|
-
const rawSource = fromNode.text.replace(/['"]/g, '');
|
|
491
|
-
if (!rawSource.startsWith('.')) { node.children.forEach(walk); return; }
|
|
492
|
-
|
|
493
|
-
// Resolve the source file
|
|
494
|
-
const absSource = path.resolve(barrelDir, rawSource);
|
|
495
|
-
let finalAbs = null;
|
|
496
|
-
if (EXTENSIONS.has(path.extname(absSource)) && fs.existsSync(absSource)) {
|
|
497
|
-
finalAbs = absSource;
|
|
498
|
-
} else {
|
|
499
|
-
for (const e of EXTENSIONS) {
|
|
500
|
-
if (fs.existsSync(absSource + e)) { finalAbs = absSource + e; break; }
|
|
501
|
-
const idx = path.join(absSource, 'index' + e);
|
|
502
|
-
if (fs.existsSync(idx)) { finalAbs = idx; break; }
|
|
503
|
-
}
|
|
504
|
-
}
|
|
505
|
-
if (!finalAbs) { node.children.forEach(walk); return; }
|
|
506
|
-
|
|
507
|
-
const relSource = path.relative(projectRoot, finalAbs).replace(/\\/g, '/');
|
|
508
|
-
|
|
509
|
-
// Walk named exports
|
|
510
|
-
const namedExports = node.children.find(c => c.type === 'named_imports' || c.type === 'export_clause');
|
|
511
|
-
if (namedExports) {
|
|
512
|
-
for (const child of namedExports.children) {
|
|
513
|
-
if (child.type === 'import_specifier' || child.type === 'export_specifier') {
|
|
514
|
-
// `alias as exported` or just `name`
|
|
515
|
-
const names = child.children.filter(c => c.type === 'identifier');
|
|
516
|
-
if (names.length > 0) {
|
|
517
|
-
// The exported name is the last identifier (the alias if present)
|
|
518
|
-
result.set(names[names.length - 1].text, relSource);
|
|
519
|
-
}
|
|
520
|
-
}
|
|
521
|
-
}
|
|
522
|
-
}
|
|
523
|
-
|
|
524
|
-
// export * from './source' → map the source file itself
|
|
525
|
-
const starNode = node.children.find(c => c.text === '*');
|
|
526
|
-
if (starNode) {
|
|
527
|
-
result.set('*', relSource);
|
|
528
|
-
}
|
|
529
|
-
}
|
|
530
|
-
node.children.forEach(walk);
|
|
531
|
-
}
|
|
532
|
-
walk(tree.rootNode);
|
|
533
|
-
return result;
|
|
534
|
-
}
|
|
535
|
-
|
|
536
|
-
// ─── Go module-name cache (reads go.mod once per project root) ──────────────
|
|
537
|
-
const _goModCache = new Map();
|
|
538
|
-
function _readGoModuleName(projectRoot) {
|
|
539
|
-
if (_goModCache.has(projectRoot)) return _goModCache.get(projectRoot);
|
|
540
|
-
const modFile = path.join(projectRoot, 'go.mod');
|
|
541
|
-
let name = null;
|
|
542
|
-
if (fs.existsSync(modFile)) {
|
|
543
|
-
try {
|
|
544
|
-
const first = fs.readFileSync(modFile, 'utf-8').split('\n').find(l => l.trimStart().startsWith('module '));
|
|
545
|
-
if (first) name = first.trim().replace(/^module\s+/, '').split(/\s/)[0];
|
|
546
|
-
} catch { /* ignore */ }
|
|
547
|
-
}
|
|
548
|
-
_goModCache.set(projectRoot, name);
|
|
549
|
-
return name;
|
|
550
|
-
}
|
|
551
|
-
|
|
552
|
-
export function resolveLocalImports(rawImports, fromFileRelPath, projectRoot) {
|
|
553
|
-
const fileDir = path.dirname(path.join(projectRoot, fromFileRelPath));
|
|
554
|
-
const ext = path.extname(fromFileRelPath);
|
|
555
|
-
const resolved = [];
|
|
556
|
-
for (const raw of rawImports) {
|
|
557
|
-
// ── Dot-relative (JS/TS/Python relative) ────────────────────────────
|
|
558
|
-
if (raw.startsWith('.')) {
|
|
559
|
-
const absResolved = path.resolve(fileDir, raw);
|
|
560
|
-
const existingExt = path.extname(absResolved);
|
|
561
|
-
let finalAbs = null;
|
|
562
|
-
if (existingExt && EXTENSIONS.has(existingExt) && fs.existsSync(absResolved)) {
|
|
563
|
-
finalAbs = absResolved;
|
|
564
|
-
} else {
|
|
565
|
-
for (const e of EXTENSIONS) {
|
|
566
|
-
if (fs.existsSync(absResolved + e)) { finalAbs = absResolved + e; break; }
|
|
567
|
-
const idx = path.join(absResolved, 'index' + e);
|
|
568
|
-
if (fs.existsSync(idx)) { finalAbs = idx; break; }
|
|
569
|
-
// Python: also try __init__.py for package directories
|
|
570
|
-
if (e === '.py') {
|
|
571
|
-
const init = path.join(absResolved, '__init__.py');
|
|
572
|
-
if (fs.existsSync(init)) { finalAbs = init; break; }
|
|
573
|
-
}
|
|
574
|
-
}
|
|
575
|
-
}
|
|
576
|
-
if (finalAbs) {
|
|
577
|
-
const relPath = path.relative(projectRoot, finalAbs).replace(/\\/g, '/');
|
|
578
|
-
const baseName = path.basename(finalAbs, path.extname(finalAbs));
|
|
579
|
-
// 🥇 BARREL RESOLUTION: If the resolved file is an index file, expand barrel exports
|
|
580
|
-
if (baseName === 'index' && ['.ts', '.tsx', '.js', '.jsx', '.mjs', '.cjs'].includes(path.extname(finalAbs))) {
|
|
581
|
-
const barrelMap = resolveBarrelExports(finalAbs, projectRoot);
|
|
582
|
-
if (barrelMap.size > 0) {
|
|
583
|
-
// Add all unique source files referenced by this barrel
|
|
584
|
-
const sources = new Set(barrelMap.values());
|
|
585
|
-
for (const src of sources) {
|
|
586
|
-
if (!resolved.includes(src)) resolved.push(src);
|
|
587
|
-
}
|
|
588
|
-
} else {
|
|
589
|
-
// Barrel has no re-exports — keep the barrel file itself
|
|
590
|
-
if (!resolved.includes(relPath)) resolved.push(relPath);
|
|
591
|
-
}
|
|
592
|
-
} else {
|
|
593
|
-
if (!resolved.includes(relPath)) resolved.push(relPath);
|
|
594
|
-
}
|
|
595
|
-
}
|
|
596
|
-
}
|
|
597
|
-
// ── Go intra-module: github.com/owner/repo/sub → sub/*.go ──────────
|
|
598
|
-
// Go import paths use the module name as prefix; map them to local dirs.
|
|
599
|
-
else if (ext === '.go') {
|
|
600
|
-
const modName = _readGoModuleName(projectRoot);
|
|
601
|
-
if (modName && raw.startsWith(modName + '/')) {
|
|
602
|
-
const subPkg = raw.slice(modName.length + 1); // e.g. 'render'
|
|
603
|
-
const absDir = path.join(projectRoot, subPkg);
|
|
604
|
-
if (fs.existsSync(absDir)) {
|
|
605
|
-
try {
|
|
606
|
-
const goFiles = fs.readdirSync(absDir)
|
|
607
|
-
.filter(f => f.endsWith('.go') && !f.includes('_test'))
|
|
608
|
-
.slice(0, 5); // cap: take a representative sample
|
|
609
|
-
for (const gof of goFiles) {
|
|
610
|
-
const rel = path.relative(projectRoot, path.join(absDir, gof)).replace(/\\/g, '/');
|
|
611
|
-
if (!resolved.includes(rel)) resolved.push(rel);
|
|
612
|
-
}
|
|
613
|
-
} catch { /* directory unreadable — skip */ }
|
|
614
|
-
}
|
|
615
|
-
}
|
|
616
|
-
}
|
|
617
|
-
// ── Rust crate-local: crate::module::item → src/module.rs ──────────
|
|
618
|
-
else if (ext === '.rs' && raw.startsWith('crate::')) {
|
|
619
|
-
const parts = raw.slice('crate::'.length).split('::').filter(Boolean);
|
|
620
|
-
for (let depth = parts.length; depth >= 1; depth--) {
|
|
621
|
-
const subPath = parts.slice(0, depth).join('/');
|
|
622
|
-
const candidates = [
|
|
623
|
-
path.join(projectRoot, 'src', subPath + '.rs'),
|
|
624
|
-
path.join(projectRoot, 'src', subPath, 'mod.rs'),
|
|
625
|
-
];
|
|
626
|
-
let found = false;
|
|
627
|
-
for (const c of candidates) {
|
|
628
|
-
if (fs.existsSync(c)) {
|
|
629
|
-
resolved.push(path.relative(projectRoot, c).replace(/\\/g, '/'));
|
|
630
|
-
found = true; break;
|
|
631
|
-
}
|
|
632
|
-
}
|
|
633
|
-
if (found) break;
|
|
634
|
-
}
|
|
635
|
-
}
|
|
636
|
-
}
|
|
637
|
-
return resolved;
|
|
638
|
-
}
|
|
639
|
-
|
|
640
|
-
/**
|
|
641
|
-
* Build the text payload sent to the embedding model for a chunk.
|
|
642
|
-
*
|
|
643
|
-
* Shared by the bootstrap indexer (indexer.mjs) and the watch daemon
|
|
644
|
-
* (watch-daemon.mjs) so a chunk yields the SAME embedding regardless of which
|
|
645
|
-
* path embedded it first. This matters because the cache key (content_hash) is
|
|
646
|
-
* derived from code + docstring only — it does NOT include this payload — so two
|
|
647
|
-
* divergent payloads for the same hash would silently produce inconsistent
|
|
648
|
-
* embeddings across a full re-index vs. an incremental update.
|
|
649
|
-
*
|
|
650
|
-
* @param {object} chunk Semantic chunk: { file_path, node_type, name, docstring, type_refs, code_snippet }.
|
|
651
|
-
* @param {string[]} depRelPaths Resolved local imports of the chunk's file (project-relative paths).
|
|
652
|
-
* @returns {string}
|
|
653
|
-
*/
|
|
654
|
-
export function buildEmbeddingPayload(chunk, depRelPaths = []) {
|
|
655
|
-
const neighbors = depRelPaths
|
|
656
|
-
.map(d => path.basename(d, path.extname(d)))
|
|
657
|
-
.filter(Boolean);
|
|
658
|
-
const topologicalContext = neighbors.length
|
|
659
|
-
? `This code architectural neighborhood connects with: ${neighbors.join(', ')}.`
|
|
660
|
-
: '';
|
|
661
|
-
return [
|
|
662
|
-
`File Location: ${chunk.file_path}`,
|
|
663
|
-
`Symbol Name: ${chunk.node_type} -> ${chunk.name}`,
|
|
664
|
-
chunk.docstring ? `Developer Documentation: ${chunk.docstring}` : '',
|
|
665
|
-
chunk.type_refs?.length ? `Type References: ${chunk.type_refs.join(', ')}` : '',
|
|
666
|
-
topologicalContext,
|
|
667
|
-
`--- Source Code ---`,
|
|
668
|
-
chunk.code_snippet,
|
|
669
|
-
].filter(Boolean).join('\n');
|
|
670
|
-
}
|
|
671
|
-
|
|
672
|
-
export async function getLocalEmbedding(text, graceful = true) {
|
|
673
|
-
if (process.env.INDEXER_EMBEDDINGS === 'off') return null; // lexical-only mode
|
|
674
|
-
const MAX_RETRIES = 3;
|
|
675
|
-
const safeText = "search_query: " + truncateForEmbedding(text);
|
|
676
|
-
for (let attempt = 0; attempt <= MAX_RETRIES; attempt++) {
|
|
677
|
-
try {
|
|
678
|
-
const res = await fetch(`${OLLAMA_HOST}/api/embeddings`, {
|
|
679
|
-
method: "POST", headers: { "Content-Type": "application/json" },
|
|
680
|
-
body: JSON.stringify({ model: "nomic-embed-text", prompt: safeText }),
|
|
681
|
-
signal: AbortSignal.timeout(15000),
|
|
682
|
-
});
|
|
683
|
-
if (!res.ok) throw new Error(`HTTP ${res.status}`);
|
|
684
|
-
const data = await res.json();
|
|
685
|
-
return data.embedding;
|
|
686
|
-
} catch (err) {
|
|
687
|
-
if (attempt < MAX_RETRIES) await new Promise(r => setTimeout(r, 500 * 2 ** attempt));
|
|
688
|
-
else if (!graceful) throw err;
|
|
689
|
-
}
|
|
690
|
-
}
|
|
691
|
-
return null;
|
|
692
|
-
}
|
|
693
|
-
|
|
694
|
-
export async function getLocalEmbeddingsBatch(texts, graceful = true) {
|
|
695
|
-
if (!texts || texts.length === 0) return [];
|
|
696
|
-
if (process.env.INDEXER_EMBEDDINGS === 'off') return null; // lexical-only mode
|
|
697
|
-
const MAX_RETRIES = 3;
|
|
698
|
-
const safeTexts = texts.map(t => "search_document: " + (t.length > 8000 ? t.slice(0, 8000) : t));
|
|
699
|
-
|
|
700
|
-
for (let attempt = 0; attempt <= MAX_RETRIES; attempt++) {
|
|
701
|
-
try {
|
|
702
|
-
const res = await fetch(`${OLLAMA_HOST}/api/embed`, {
|
|
703
|
-
method: "POST", headers: { "Content-Type": "application/json" },
|
|
704
|
-
body: JSON.stringify({ model: "nomic-embed-text", input: safeTexts }),
|
|
705
|
-
signal: AbortSignal.timeout(60000),
|
|
706
|
-
});
|
|
707
|
-
if (!res.ok) throw new Error(`HTTP ${res.status}`);
|
|
708
|
-
const data = await res.json();
|
|
709
|
-
return data.embeddings;
|
|
710
|
-
} catch (err) {
|
|
711
|
-
if (attempt < MAX_RETRIES) await new Promise(r => setTimeout(r, 500 * 2 ** attempt));
|
|
712
|
-
else if (!graceful) throw err;
|
|
713
|
-
}
|
|
714
|
-
}
|
|
715
|
-
return null;
|
|
716
|
-
}
|
|
717
|
-
|
|
718
|
-
// ─── Enrichment helpers (param names, return type, class context) ─────────────
|
|
719
|
-
|
|
720
|
-
export function extractParams(chunkNode, ext) {
|
|
721
|
-
const params = [];
|
|
722
|
-
const JS_LIKE = ['.ts', '.tsx', '.js', '.jsx', '.mjs', '.cjs'];
|
|
723
|
-
const paramTypes = JS_LIKE.includes(ext)
|
|
724
|
-
? ['required_parameter', 'optional_parameter', 'formal_parameters', 'identifier']
|
|
725
|
-
: ['parameter', 'formal_parameter', 'identifier'];
|
|
726
|
-
|
|
727
|
-
function walkParams(node) {
|
|
728
|
-
if (node.type === 'formal_parameters' || node.type === 'parameters' || node.type === 'parameter_list') {
|
|
729
|
-
for (const child of node.children) {
|
|
730
|
-
// TS: required_parameter / optional_parameter have an identifier child
|
|
731
|
-
if (child.type === 'required_parameter' || child.type === 'optional_parameter' || child.type === 'formal_parameter') {
|
|
732
|
-
const id = child.childForFieldName?.('pattern') || child.childForFieldName?.('name') ||
|
|
733
|
-
child.children.find(c => c.type === 'identifier');
|
|
734
|
-
if (id) params.push(id.text);
|
|
735
|
-
// also grab type annotation text
|
|
736
|
-
const typeAnnotation = child.childForFieldName?.('type');
|
|
737
|
-
if (typeAnnotation) {
|
|
738
|
-
const typeText = typeAnnotation.text.replace(/^:\s*/, '').trim();
|
|
739
|
-
if (typeText) params.push(typeText);
|
|
740
|
-
}
|
|
741
|
-
} else if (child.type === 'identifier') {
|
|
742
|
-
params.push(child.text);
|
|
743
|
-
}
|
|
744
|
-
}
|
|
745
|
-
}
|
|
746
|
-
for (const child of node.children) walkParams(child);
|
|
747
|
-
}
|
|
748
|
-
|
|
749
|
-
// Only walk the direct params node to avoid deep recursion into body
|
|
750
|
-
const paramsNode = chunkNode.childForFieldName?.('parameters') || chunkNode.childForFieldName?.('formal_parameters');
|
|
751
|
-
if (paramsNode) walkParams(paramsNode);
|
|
752
|
-
return [...new Set(params)].filter(p => p && p.length > 1).slice(0, 15);
|
|
753
|
-
}
|
|
754
|
-
|
|
755
|
-
export function extractReturnType(chunkNode, ext) {
|
|
756
|
-
const JS_LIKE = ['.ts', '.tsx', '.js', '.jsx', '.mjs', '.cjs'];
|
|
757
|
-
if (JS_LIKE.includes(ext)) {
|
|
758
|
-
const retTypeNode = chunkNode.childForFieldName?.('return_type');
|
|
759
|
-
if (retTypeNode) return retTypeNode.text.replace(/^:\s*/, '').trim().slice(0, 80);
|
|
760
|
-
}
|
|
761
|
-
return '';
|
|
762
|
-
}
|
|
763
|
-
|
|
764
|
-
export function extractClassContext(chunkNode) {
|
|
765
|
-
let parent = chunkNode.parent;
|
|
766
|
-
while (parent) {
|
|
767
|
-
if (parent.type === 'class_declaration' || parent.type === 'class_definition' ||
|
|
768
|
-
parent.type === 'class_body' || parent.type === 'impl_item') {
|
|
769
|
-
const nameNode = parent.childForFieldName?.('name');
|
|
770
|
-
if (nameNode) return nameNode.text;
|
|
771
|
-
}
|
|
772
|
-
parent = parent.parent;
|
|
773
|
-
}
|
|
774
|
-
return '';
|
|
775
|
-
}
|
|
776
|
-
|
|
777
|
-
/**
|
|
778
|
-
* Frontier 2: Extract TypeScript/Python type annotation names from a chunk node.
|
|
779
|
-
* Returns simple type names (e.g. ['User', 'AuthToken', 'PaymentService'])
|
|
780
|
-
* used to enrich the inverted index and the type_refs chunk field.
|
|
781
|
-
*/
|
|
782
|
-
export function extractTypeAnnotations(chunkNode, ext) {
|
|
783
|
-
const types = new Set();
|
|
784
|
-
const JS_LIKE = ['.ts', '.tsx', '.js', '.jsx', '.mjs', '.cjs'];
|
|
785
|
-
if (!JS_LIKE.includes(ext) && ext !== '.py') return [];
|
|
786
|
-
|
|
787
|
-
function walk(node) {
|
|
788
|
-
// TypeScript: type_annotation nodes contain the type text
|
|
789
|
-
if (node.type === 'type_annotation') {
|
|
790
|
-
const typeText = node.text.replace(/^:\s*/, '').trim();
|
|
791
|
-
// Extract simple identifiers from the type (skip primitives)
|
|
792
|
-
const PRIMITIVES = new Set(['string','number','boolean','void','any','unknown','never','null','undefined','object','symbol','bigint']);
|
|
793
|
-
for (const match of typeText.matchAll(/\b([A-Z][A-Za-z0-9]*)\b/g)) {
|
|
794
|
-
if (!PRIMITIVES.has(match[1].toLowerCase())) types.add(match[1]);
|
|
795
|
-
}
|
|
796
|
-
}
|
|
797
|
-
// TypeScript: generic_type, predefined_type (string, number…) skip, named types keep
|
|
798
|
-
else if (node.type === 'type_identifier' || node.type === 'generic_type') {
|
|
799
|
-
const name = node.children[0]?.text || node.text;
|
|
800
|
-
if (name && /^[A-Z]/.test(name)) types.add(name);
|
|
801
|
-
}
|
|
802
|
-
// Python: type comments or annotations (annotation nodes)
|
|
803
|
-
else if (node.type === 'annotation' && ext === '.py') {
|
|
804
|
-
const typeText = node.text.replace(/^->\s*|^:\s*/, '').trim();
|
|
805
|
-
for (const match of typeText.matchAll(/\b([A-Z][A-Za-z0-9]*)\b/g)) {
|
|
806
|
-
types.add(match[1]);
|
|
807
|
-
}
|
|
808
|
-
}
|
|
809
|
-
for (const child of node.children) walk(child);
|
|
810
|
-
}
|
|
811
|
-
walk(chunkNode);
|
|
812
|
-
return Array.from(types).slice(0, 20);
|
|
813
|
-
}
|
|
814
|
-
|
|
815
|
-
export function extractCalls(rootNode) {
|
|
816
|
-
const calls = new Set();
|
|
817
|
-
function walk(node) {
|
|
818
|
-
// JavaScript / TypeScript / Go / Rust: call_expression
|
|
819
|
-
if (node.type === 'call_expression') {
|
|
820
|
-
const funcNode = node.childForFieldName?.('function') || node.children[0];
|
|
821
|
-
if (funcNode) {
|
|
822
|
-
if (funcNode.type === 'identifier') calls.add(funcNode.text);
|
|
823
|
-
else if (funcNode.type === 'member_expression' || funcNode.type === 'property_identifier') {
|
|
824
|
-
const prop = funcNode.childForFieldName?.('property');
|
|
825
|
-
if (prop) calls.add(prop.text);
|
|
826
|
-
else calls.add(funcNode.text.split('.').pop());
|
|
827
|
-
}
|
|
828
|
-
}
|
|
829
|
-
}
|
|
830
|
-
// Python: `call` node type (different name from call_expression)
|
|
831
|
-
else if (node.type === 'call') {
|
|
832
|
-
const funcNode = node.childForFieldName?.('function') || node.children[0];
|
|
833
|
-
if (funcNode) {
|
|
834
|
-
if (funcNode.type === 'identifier') calls.add(funcNode.text);
|
|
835
|
-
else if (funcNode.type === 'attribute') {
|
|
836
|
-
const attr = funcNode.childForFieldName?.('attribute');
|
|
837
|
-
if (attr) calls.add(attr.text);
|
|
838
|
-
}
|
|
839
|
-
}
|
|
840
|
-
}
|
|
841
|
-
// Rust: macro_invocation (e.g. vec!, println!, format!)
|
|
842
|
-
else if (node.type === 'macro_invocation') {
|
|
843
|
-
const macroNode = node.childForFieldName?.('macro') || node.children[0];
|
|
844
|
-
if (macroNode && macroNode.type === 'identifier') calls.add(macroNode.text + '!');
|
|
845
|
-
}
|
|
846
|
-
// Java / C#: method_invocation
|
|
847
|
-
else if (node.type === 'method_invocation') {
|
|
848
|
-
const nameNode = node.childForFieldName?.('name') || node.children.find(c => c.type === 'identifier');
|
|
849
|
-
if (nameNode) calls.add(nameNode.text);
|
|
850
|
-
}
|
|
851
|
-
// Ruby: method_call / call
|
|
852
|
-
else if (node.type === 'method_call') {
|
|
853
|
-
const method = node.childForFieldName?.('method') || node.children.find(c => c.type === 'identifier');
|
|
854
|
-
if (method && method.type === 'identifier') calls.add(method.text);
|
|
855
|
-
}
|
|
856
|
-
node.children.forEach(walk);
|
|
857
|
-
}
|
|
858
|
-
walk(rootNode);
|
|
859
|
-
const noise = new Set(['require', 'console', 'log', 'expect', 'test', 'it', 'describe', 'setTimeout', 'print', 'println!']);
|
|
860
|
-
return Array.from(calls).filter(c => c && c.length > 2 && !noise.has(c));
|
|
861
|
-
}
|