graph-indexer 1.2.0 → 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +412 -0
- package/README.md +216 -596
- package/SECURITY.md +68 -0
- package/TRADEMARK.md +31 -0
- package/cli-ui.mjs +78 -0
- package/config.mjs +179 -28
- package/daemon-ctl.mjs +175 -0
- package/daemon-lock.mjs +69 -0
- package/embedders/python/mlx_embed_server.py +61 -0
- package/embedders/python/requirements-mlx.txt +5 -0
- package/embedders/setup-mlx.mjs +110 -0
- package/embeddings.mjs +489 -0
- package/engine/binary.mjs +399 -0
- package/{core-engine.mjs → engine/memory.mjs} +107 -440
- package/{sqlite-store.mjs → engine/sqlite.mjs} +137 -42
- package/enrichment.mjs +111 -32
- package/git-signals.mjs +248 -0
- package/indexer.mjs +151 -42
- package/init.mjs +1603 -171
- package/layout.mjs +123 -0
- package/mcp/format.mjs +61 -0
- package/mcp/tools.mjs +1157 -0
- package/mcp/topology.mjs +551 -0
- package/mcp-server.mjs +55 -24
- package/package.json +44 -13
- package/parse/extractor.mjs +617 -0
- package/parse/imports.mjs +365 -0
- package/parse/languages.mjs +89 -0
- package/parse/metadata.mjs +655 -0
- package/parse/routes.mjs +327 -0
- package/prompts/CORE.md +86 -0
- package/prompts/DOMAIN_TEMPLATE.md +46 -0
- package/prompts/INTEGRATION.md +123 -0
- package/prompts/frameworks/ANDROID.md +17 -0
- package/prompts/frameworks/ASPNET_CORE.md +19 -0
- package/prompts/frameworks/FASTAPI_DJANGO.md +19 -0
- package/prompts/frameworks/LARAVEL_SYMFONY.md +22 -0
- package/prompts/frameworks/NODE_EXPRESS_NESTJS.md +18 -0
- package/prompts/frameworks/RAILS.md +20 -0
- package/prompts/frameworks/REACT.md +20 -0
- package/prompts/frameworks/SPRING_BOOT.md +19 -0
- package/prompts/languages/BASH.md +22 -0
- package/prompts/languages/C.md +21 -0
- package/prompts/languages/CSHARP.md +24 -0
- package/prompts/languages/CSS_SCSS.md +22 -0
- package/prompts/languages/GO.md +23 -0
- package/prompts/languages/JAVA.md +23 -0
- package/prompts/languages/JAVASCRIPT_TYPESCRIPT.md +24 -0
- package/prompts/languages/KOTLIN.md +24 -0
- package/prompts/languages/PHP.md +23 -0
- package/prompts/languages/PYTHON.md +23 -0
- package/prompts/languages/RUBY.md +23 -0
- package/prompts/languages/RUST.md +21 -0
- package/prompts/languages/SWIFT.md +21 -0
- package/search-core.mjs +254 -43
- package/storage.mjs +28 -3
- package/watch-daemon.mjs +69 -23
- package/PROMPT.md +0 -182
- package/mcp-tools.mjs +0 -545
- package/parser-utils.mjs +0 -1170
package/parser-utils.mjs
DELETED
|
@@ -1,1170 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* @file parser-utils.mjs
|
|
3
|
-
* @description Shared AST parsing, language registry, and embedding utilities.
|
|
4
|
-
* @author MaquinaTech <https://github.com/MaquinaTech>
|
|
5
|
-
* @copyright (c) 2026 MaquinaTech. All rights reserved.
|
|
6
|
-
* @license MIT
|
|
7
|
-
* Copyright (c) 2026 MaquinaTech. All rights reserved.
|
|
8
|
-
* Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
9
|
-
* of this software and associated documentation files (the "Software"), to deal
|
|
10
|
-
* in the Software without restriction, including without limitation the rights
|
|
11
|
-
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
12
|
-
* copies of the Software, and to permit persons to whom the Software is
|
|
13
|
-
* furnished to do so, subject to the following conditions: The above copyright
|
|
14
|
-
* notice and this permission notice shall be included in all copies or
|
|
15
|
-
* substantial portions of the Software.
|
|
16
|
-
*/
|
|
17
|
-
import fs from 'fs';
|
|
18
|
-
import path from 'path';
|
|
19
|
-
import { createHash } from 'crypto';
|
|
20
|
-
import ignore from 'ignore';
|
|
21
|
-
import Parser from 'tree-sitter';
|
|
22
|
-
const { Query } = Parser; // Native parser query helper
|
|
23
|
-
|
|
24
|
-
import { truncateForEmbedding } from './core-engine.mjs';
|
|
25
|
-
|
|
26
|
-
export const MAX_FILE_SIZE_BYTES = 500000;
|
|
27
|
-
export const OLLAMA_HOST = process.env.OLLAMA_HOST || "http://localhost:11434";
|
|
28
|
-
|
|
29
|
-
// Resolves the Ollama host at call time so callers that set ollamaHost via
|
|
30
|
-
// .graph-indexer.json don't have to pass it through every call chain.
|
|
31
|
-
// Priority: caller override → OLLAMA_HOST env var → PROJECT .graph-indexer.json
|
|
32
|
-
// (MCP_PROJECT_ROOT or cwd — NOT the package directory: when graph-indexer is
|
|
33
|
-
// installed as a dependency, the user's config lives in their project root) →
|
|
34
|
-
// default. Mirrors config.mjs precedence so every entry point agrees.
|
|
35
|
-
//
|
|
36
|
-
// Note: OLLAMA_HOST in the shell is Ollama's binding address (e.g. "0.0.0.0:11435"), not an
|
|
37
|
-
// HTTP client URL. We normalise bare "host:port" strings by adding http:// and translating
|
|
38
|
-
// 0.0.0.0 → localhost so fetches work in both formats.
|
|
39
|
-
let _cachedHost = null;
|
|
40
|
-
let _cachedEmbedModel = null;
|
|
41
|
-
function _normalizeOllamaHost(raw) {
|
|
42
|
-
if (!raw) return null;
|
|
43
|
-
if (raw.startsWith('http://') || raw.startsWith('https://')) return raw;
|
|
44
|
-
return 'http://' + raw.replace(/^0\.0\.0\.0/, 'localhost');
|
|
45
|
-
}
|
|
46
|
-
function _readProjectConfig() {
|
|
47
|
-
const root = process.env.MCP_PROJECT_ROOT || process.cwd();
|
|
48
|
-
try {
|
|
49
|
-
return JSON.parse(fs.readFileSync(path.join(root, '.graph-indexer.json'), 'utf8'));
|
|
50
|
-
} catch { return null; }
|
|
51
|
-
}
|
|
52
|
-
function _resolveOllamaHost(override) {
|
|
53
|
-
if (override) return _normalizeOllamaHost(override) || 'http://localhost:11434';
|
|
54
|
-
if (_cachedHost) return _cachedHost;
|
|
55
|
-
if (process.env.OLLAMA_HOST) {
|
|
56
|
-
_cachedHost = _normalizeOllamaHost(process.env.OLLAMA_HOST) || 'http://localhost:11434';
|
|
57
|
-
return _cachedHost;
|
|
58
|
-
}
|
|
59
|
-
const cfg = _readProjectConfig();
|
|
60
|
-
_cachedHost = _normalizeOllamaHost(cfg?.ollamaHost) || 'http://localhost:11434';
|
|
61
|
-
return _cachedHost;
|
|
62
|
-
}
|
|
63
|
-
function _resolveEmbedModel(override) {
|
|
64
|
-
if (override) return override;
|
|
65
|
-
if (_cachedEmbedModel) return _cachedEmbedModel;
|
|
66
|
-
const cfg = _readProjectConfig();
|
|
67
|
-
_cachedEmbedModel = cfg?.embedModel || 'nomic-embed-text';
|
|
68
|
-
return _cachedEmbedModel;
|
|
69
|
-
}
|
|
70
|
-
|
|
71
|
-
// ─── Dynamic Language Loading ─────────────────────────────────────────────────
|
|
72
|
-
|
|
73
|
-
function _loadProjectConfig() {
|
|
74
|
-
const configPath = path.join(process.env.MCP_PROJECT_ROOT || process.cwd(), '.graph-indexer.json');
|
|
75
|
-
try {
|
|
76
|
-
if (fs.existsSync(configPath)) return JSON.parse(fs.readFileSync(configPath, 'utf-8'));
|
|
77
|
-
} catch { /* ignore */ }
|
|
78
|
-
return null;
|
|
79
|
-
}
|
|
80
|
-
|
|
81
|
-
async function _tryLang(pkg, enabledLangs, key) {
|
|
82
|
-
if (enabledLangs && !enabledLangs.includes(key)) return null;
|
|
83
|
-
try {
|
|
84
|
-
return (await import(pkg)).default;
|
|
85
|
-
} catch {
|
|
86
|
-
process.stderr.write(`[graph-indexer] WARNING: ${pkg} not installed — ${key} files will be skipped\n`);
|
|
87
|
-
return null;
|
|
88
|
-
}
|
|
89
|
-
}
|
|
90
|
-
|
|
91
|
-
const _cfg = _loadProjectConfig();
|
|
92
|
-
const _enabled = _cfg?.languages ?? null; // null = all languages
|
|
93
|
-
|
|
94
|
-
const [
|
|
95
|
-
TypeScript, JavaScript, CSS, Python, Rust,
|
|
96
|
-
Go, PHP, Java, Kotlin, CSharp, Ruby
|
|
97
|
-
] = await Promise.all([
|
|
98
|
-
_tryLang('tree-sitter-typescript', _enabled, 'typescript'),
|
|
99
|
-
_tryLang('tree-sitter-javascript', _enabled, 'javascript'),
|
|
100
|
-
_tryLang('tree-sitter-css', _enabled, 'css'),
|
|
101
|
-
_tryLang('tree-sitter-python', _enabled, 'python'),
|
|
102
|
-
_tryLang('tree-sitter-rust', _enabled, 'rust'),
|
|
103
|
-
_tryLang('tree-sitter-go', _enabled, 'go'),
|
|
104
|
-
_tryLang('tree-sitter-php', _enabled, 'php'),
|
|
105
|
-
_tryLang('tree-sitter-java', _enabled, 'java'),
|
|
106
|
-
_tryLang('tree-sitter-kotlin', _enabled, 'kotlin'),
|
|
107
|
-
_tryLang('tree-sitter-c-sharp', _enabled, 'csharp'),
|
|
108
|
-
_tryLang('tree-sitter-ruby', _enabled, 'ruby'),
|
|
109
|
-
]);
|
|
110
|
-
|
|
111
|
-
const LANGUAGE_MAP = {
|
|
112
|
-
...(TypeScript ? { '.ts': TypeScript.typescript, '.tsx': TypeScript.tsx } : {}),
|
|
113
|
-
...(JavaScript ? { '.js': JavaScript, '.jsx': JavaScript, '.mjs': JavaScript, '.cjs': JavaScript } : {}),
|
|
114
|
-
...(CSS ? { '.css': CSS, '.scss': CSS } : {}),
|
|
115
|
-
...(Python ? { '.py': Python } : {}),
|
|
116
|
-
...(Rust ? { '.rs': Rust } : {}),
|
|
117
|
-
...(Go ? { '.go': Go } : {}),
|
|
118
|
-
...(PHP ? { '.php': PHP.php } : {}),
|
|
119
|
-
...(Java ? { '.java': Java } : {}),
|
|
120
|
-
...(Kotlin ? { '.kt': Kotlin, '.kts': Kotlin } : {}),
|
|
121
|
-
...(CSharp ? { '.cs': CSharp } : {}),
|
|
122
|
-
...(Ruby ? { '.rb': Ruby } : {}),
|
|
123
|
-
};
|
|
124
|
-
|
|
125
|
-
// 🥇 ULTRA-GENERIC QUERIES: Immune to grammar changes between TS/JS/TSX
|
|
126
|
-
const LANGUAGE_QUERIES = {
|
|
127
|
-
ts: `
|
|
128
|
-
(class_declaration) @chunk
|
|
129
|
-
(function_declaration) @chunk
|
|
130
|
-
(method_definition) @chunk
|
|
131
|
-
(lexical_declaration) @chunk
|
|
132
|
-
(expression_statement) @chunk
|
|
133
|
-
(export_statement) @chunk
|
|
134
|
-
`,
|
|
135
|
-
css: `(rule_set) @chunk`,
|
|
136
|
-
py: `
|
|
137
|
-
(function_definition) @chunk
|
|
138
|
-
(class_definition) @chunk
|
|
139
|
-
(expression_statement) @chunk
|
|
140
|
-
`,
|
|
141
|
-
rs: `
|
|
142
|
-
(function_item) @chunk
|
|
143
|
-
(struct_item) @chunk
|
|
144
|
-
(enum_item) @chunk
|
|
145
|
-
(trait_item) @chunk
|
|
146
|
-
(impl_item) @chunk
|
|
147
|
-
`,
|
|
148
|
-
go: `
|
|
149
|
-
(function_declaration) @chunk
|
|
150
|
-
(method_declaration) @chunk
|
|
151
|
-
(type_declaration) @chunk
|
|
152
|
-
`,
|
|
153
|
-
php: `
|
|
154
|
-
(function_definition) @chunk
|
|
155
|
-
(class_declaration) @chunk
|
|
156
|
-
(expression_statement) @chunk
|
|
157
|
-
`,
|
|
158
|
-
java: `
|
|
159
|
-
(method_declaration) @chunk
|
|
160
|
-
(class_declaration) @chunk
|
|
161
|
-
(interface_declaration) @chunk
|
|
162
|
-
(constructor_declaration) @chunk
|
|
163
|
-
(enum_declaration) @chunk
|
|
164
|
-
`,
|
|
165
|
-
kotlin: `
|
|
166
|
-
(function_declaration) @chunk
|
|
167
|
-
(class_declaration) @chunk
|
|
168
|
-
(object_declaration) @chunk
|
|
169
|
-
(companion_object) @chunk
|
|
170
|
-
(secondary_constructor) @chunk
|
|
171
|
-
`,
|
|
172
|
-
cs: `
|
|
173
|
-
(method_declaration) @chunk
|
|
174
|
-
(class_declaration) @chunk
|
|
175
|
-
(interface_declaration) @chunk
|
|
176
|
-
(constructor_declaration) @chunk
|
|
177
|
-
(enum_declaration) @chunk
|
|
178
|
-
(property_declaration) @chunk
|
|
179
|
-
`,
|
|
180
|
-
rb: `
|
|
181
|
-
(method) @chunk
|
|
182
|
-
(singleton_method) @chunk
|
|
183
|
-
(class) @chunk
|
|
184
|
-
(module) @chunk
|
|
185
|
-
`
|
|
186
|
-
};
|
|
187
|
-
|
|
188
|
-
export const EXTENSIONS = new Set(Object.keys(LANGUAGE_MAP));
|
|
189
|
-
|
|
190
|
-
// Definition of container node types to avoid nested duplicates
|
|
191
|
-
const CONTAINERS = new Set([
|
|
192
|
-
'class_declaration', 'function_declaration', 'method_definition',
|
|
193
|
-
'lexical_declaration', 'expression_statement', 'export_statement',
|
|
194
|
-
'function_definition', 'class_definition', 'rule_set',
|
|
195
|
-
'function_item', 'struct_item', 'enum_item', 'trait_item', 'impl_item',
|
|
196
|
-
'method_declaration', 'type_declaration',
|
|
197
|
-
// Java / C#
|
|
198
|
-
'interface_declaration', 'constructor_declaration', 'enum_declaration',
|
|
199
|
-
// Kotlin
|
|
200
|
-
'object_declaration', 'companion_object', 'secondary_constructor',
|
|
201
|
-
// C#
|
|
202
|
-
'property_declaration',
|
|
203
|
-
// Ruby
|
|
204
|
-
'method', 'singleton_method', 'module'
|
|
205
|
-
]);
|
|
206
|
-
|
|
207
|
-
export function getParserForFile(ext) {
|
|
208
|
-
const language = LANGUAGE_MAP[ext];
|
|
209
|
-
if (!language) return null;
|
|
210
|
-
const parser = new Parser();
|
|
211
|
-
parser.setLanguage(language);
|
|
212
|
-
return parser;
|
|
213
|
-
}
|
|
214
|
-
|
|
215
|
-
export function buildIgnoreFilter(rootPath) {
|
|
216
|
-
const ig = ignore();
|
|
217
|
-
ig.add(['node_modules', '.git', 'dist', 'build', '.next', 'coverage', '*.tmp', 'vendor', '.venv']);
|
|
218
|
-
const gitignorePath = path.join(rootPath, '.gitignore');
|
|
219
|
-
if (fs.existsSync(gitignorePath)) ig.add(fs.readFileSync(gitignorePath, 'utf-8'));
|
|
220
|
-
return ig;
|
|
221
|
-
}
|
|
222
|
-
|
|
223
|
-
export function generateChunkHash(text) {
|
|
224
|
-
return createHash('sha256').update(text).digest('hex');
|
|
225
|
-
}
|
|
226
|
-
|
|
227
|
-
export function extractFileSkeleton(rootNode, content) {
|
|
228
|
-
const signatures = [];
|
|
229
|
-
function walk(node) {
|
|
230
|
-
if (node.type.includes('declaration') || node.type.includes('definition')) {
|
|
231
|
-
let name = node.childForFieldName?.("name")?.text || "anonymous";
|
|
232
|
-
signatures.push(`- [${node.type}] ${name} (lines ${node.startPosition.row + 1}-${node.endPosition.row + 1})`);
|
|
233
|
-
}
|
|
234
|
-
node.children.forEach(walk);
|
|
235
|
-
}
|
|
236
|
-
walk(rootNode);
|
|
237
|
-
return signatures.join('\n');
|
|
238
|
-
}
|
|
239
|
-
|
|
240
|
-
export function extractImportsFromAST(rootNode, ext) {
|
|
241
|
-
const imports = new Set();
|
|
242
|
-
function walk(node) {
|
|
243
|
-
// ── JavaScript / TypeScript ──────────────────────────────────────────
|
|
244
|
-
if (['.ts', '.tsx', '.js', '.jsx', '.mjs', '.cjs'].includes(ext)) {
|
|
245
|
-
if (node.type === 'import_statement') {
|
|
246
|
-
const source = node.children.find(c => c.type === 'string');
|
|
247
|
-
if (source) imports.add(source.text.replace(/['"]/g, ''));
|
|
248
|
-
} else if (node.type === 'call_expression' && node.children[0]?.text === 'require') {
|
|
249
|
-
const arg = node.children[1]?.children?.find(c => c.type === 'string');
|
|
250
|
-
if (arg) imports.add(arg.text.replace(/['"]/g, ''));
|
|
251
|
-
}
|
|
252
|
-
}
|
|
253
|
-
// ── Python ──────────────────────────────────────────────────────────
|
|
254
|
-
else if (ext === '.py') {
|
|
255
|
-
if (node.type === 'import_statement') {
|
|
256
|
-
// import foo.bar.baz → store as foo/bar/baz
|
|
257
|
-
for (const child of node.children) {
|
|
258
|
-
if (child.type === 'dotted_name') imports.add(child.text.replace(/\./g, '/'));
|
|
259
|
-
}
|
|
260
|
-
} else if (node.type === 'import_from_statement') {
|
|
261
|
-
// from .sibling import X → relative: emit as ./sibling
|
|
262
|
-
// from ..pkg import Y → relative: emit as ../pkg
|
|
263
|
-
// from foo.bar import Z → absolute: store foo/bar
|
|
264
|
-
const relNode = node.children.find(c => c.type === 'relative_import');
|
|
265
|
-
if (relNode) {
|
|
266
|
-
// dots count = depth; remainder is the module path after dots
|
|
267
|
-
const raw = relNode.text; // e.g. '.' or '..utils'
|
|
268
|
-
const dots = raw.match(/^\.+/)?.[0] ?? '.';
|
|
269
|
-
const mod = raw.slice(dots.length);
|
|
270
|
-
const prefix = dots.length === 1 ? './' : '../'.repeat(dots.length - 1);
|
|
271
|
-
imports.add(mod ? prefix + mod.replace(/\./g, '/') : prefix.slice(0, -1) || '.');
|
|
272
|
-
} else {
|
|
273
|
-
const mod = node.children.find(c => c.type === 'dotted_name');
|
|
274
|
-
if (mod) imports.add(mod.text.replace(/\./g, '/'));
|
|
275
|
-
}
|
|
276
|
-
}
|
|
277
|
-
}
|
|
278
|
-
// ── Rust ────────────────────────────────────────────────────────────
|
|
279
|
-
else if (ext === '.rs') {
|
|
280
|
-
if (node.type === 'use_declaration') {
|
|
281
|
-
// Collect the first path text from the argument subtree
|
|
282
|
-
const arg = node.childForFieldName?.('argument') ||
|
|
283
|
-
node.children.find(c => !['use', ';', 'pub'].includes(c.type));
|
|
284
|
-
if (arg) imports.add(arg.text.split('::').slice(0, 3).join('::'));
|
|
285
|
-
}
|
|
286
|
-
}
|
|
287
|
-
// ── Go ──────────────────────────────────────────────────────────────
|
|
288
|
-
else if (ext === '.go') {
|
|
289
|
-
if (node.type === 'import_spec') {
|
|
290
|
-
const pathNode = node.children.find(c =>
|
|
291
|
-
c.type === 'interpreted_string_literal' || c.type === 'raw_string_literal');
|
|
292
|
-
if (pathNode) imports.add(pathNode.text.replace(/['"`]/g, ''));
|
|
293
|
-
}
|
|
294
|
-
}
|
|
295
|
-
// ── PHP ─────────────────────────────────────────────────────────────
|
|
296
|
-
else if (ext === '.php') {
|
|
297
|
-
if (node.type === 'include_expression' || node.type === 'require_expression' ||
|
|
298
|
-
node.type === 'include_once_expression' || node.type === 'require_once_expression') {
|
|
299
|
-
const strNode = node.children.find(c =>
|
|
300
|
-
c.type === 'string' || c.type === 'encapsed_string');
|
|
301
|
-
if (strNode) imports.add(strNode.text.replace(/['"]/g, ''));
|
|
302
|
-
}
|
|
303
|
-
} // ── Java ─────────────────────────────────────────────────────────────
|
|
304
|
-
else if (ext === '.java') {
|
|
305
|
-
if (node.type === 'import_declaration') {
|
|
306
|
-
const scopedId = node.children.find(c => c.type === 'scoped_identifier' || c.type === 'identifier');
|
|
307
|
-
if (scopedId) imports.add(scopedId.text.replace(/\./g, '/'));
|
|
308
|
-
}
|
|
309
|
-
}
|
|
310
|
-
// ── Kotlin ───────────────────────────────────────────────────────────
|
|
311
|
-
else if (ext === '.kt' || ext === '.kts') {
|
|
312
|
-
if (node.type === 'import_header') {
|
|
313
|
-
const path = node.children.find(c => c.type === 'identifier' || c.type === 'user_type' || c.isNamed);
|
|
314
|
-
const raw = node.text.replace(/^import\s+/, '').replace(/\s*\.\*\s*$/, '').trim();
|
|
315
|
-
if (raw) imports.add(raw.replace(/\./g, '/'));
|
|
316
|
-
}
|
|
317
|
-
}
|
|
318
|
-
// ── C# ───────────────────────────────────────────────────────────────
|
|
319
|
-
else if (ext === '.cs') {
|
|
320
|
-
if (node.type === 'using_directive') {
|
|
321
|
-
const ns = node.children.find(c => c.type === 'qualified_name' || c.type === 'identifier' || c.type === 'name_equals');
|
|
322
|
-
if (ns) {
|
|
323
|
-
const raw = ns.text.replace(/\s*=\s*.*$/, '').trim();
|
|
324
|
-
if (raw) imports.add(raw.replace(/\./g, '/'));
|
|
325
|
-
}
|
|
326
|
-
}
|
|
327
|
-
}
|
|
328
|
-
// ── Ruby ──────────────────────────────────────────────────────────────
|
|
329
|
-
else if (ext === '.rb') {
|
|
330
|
-
if (node.type === 'call' || node.type === 'method_call') {
|
|
331
|
-
const method = node.childForFieldName?.('method') || node.children[0];
|
|
332
|
-
if (method && (method.text === 'require' || method.text === 'require_relative')) {
|
|
333
|
-
const args = node.childForFieldName?.('arguments') || node.children.find(c => c.type === 'argument_list');
|
|
334
|
-
const strArg = args?.children?.find(c => c.type === 'string' || c.type === 'simple_string');
|
|
335
|
-
if (strArg) imports.add(strArg.text.replace(/['"]/g, ''));
|
|
336
|
-
}
|
|
337
|
-
}
|
|
338
|
-
}
|
|
339
|
-
node.children.forEach(walk);
|
|
340
|
-
}
|
|
341
|
-
walk(rootNode);
|
|
342
|
-
return Array.from(imports);
|
|
343
|
-
}
|
|
344
|
-
|
|
345
|
-
// ─── God-class defence ────────────────────────────────────────────────────────
|
|
346
|
-
|
|
347
|
-
/**
|
|
348
|
-
* Token-safe skeleton for a class that exceeds GOD_CLASS_LINES.
|
|
349
|
-
*
|
|
350
|
-
* The agent's PROMPT.md promises get_chunk() costs ~300 tokens. A 2 000-line
|
|
351
|
-
* "god class" stored as one chunk violates that contract. This skeleton keeps
|
|
352
|
-
* only the first HEADER_LINES of the class (signature + opening brace) and
|
|
353
|
-
* appends a one-line summary — enough for name resolution and embeddings, while
|
|
354
|
-
* the real bodies are reachable via the individual method chunks that the
|
|
355
|
-
* god-class split produces.
|
|
356
|
-
*/
|
|
357
|
-
function buildGodClassSkeleton(classNode) {
|
|
358
|
-
const allLines = classNode.text.split('\n');
|
|
359
|
-
const HEADER_LINES = 15;
|
|
360
|
-
const header = allLines.slice(0, HEADER_LINES).join('\n');
|
|
361
|
-
return (
|
|
362
|
-
`${header}\n` +
|
|
363
|
-
` // ⚠ [Large class: ${allLines.length} lines — ` +
|
|
364
|
-
`methods are indexed as individual searchable chunks. ` +
|
|
365
|
-
`Use search_code() to find specific methods.]\n}`
|
|
366
|
-
);
|
|
367
|
-
}
|
|
368
|
-
|
|
369
|
-
export function extractSemanticChunks(rootNode, relPath, sourceCode, ext) {
|
|
370
|
-
const chunks = [];
|
|
371
|
-
const parser = getParserForFile(ext);
|
|
372
|
-
if (!parser) return chunks;
|
|
373
|
-
|
|
374
|
-
const JS_LIKE = ['.ts', '.tsx', '.js', '.jsx', '.mjs', '.cjs'];
|
|
375
|
-
const EXT_TO_LANG = {
|
|
376
|
-
'.java': 'java', '.kt': 'kotlin', '.kts': 'kotlin', '.cs': 'cs',
|
|
377
|
-
'.rb': 'rb'
|
|
378
|
-
};
|
|
379
|
-
const langKey = JS_LIKE.includes(ext) ? 'ts'
|
|
380
|
-
: (EXT_TO_LANG[ext] || (LANGUAGE_QUERIES[ext.slice(1)] ? ext.slice(1) : null));
|
|
381
|
-
if (!langKey || !LANGUAGE_QUERIES[langKey]) return chunks;
|
|
382
|
-
|
|
383
|
-
// 🥇 HEADER INHERITANCE: Extract top-of-file/global comments for module context
|
|
384
|
-
let fileDocstring = "";
|
|
385
|
-
let cursorNode = rootNode.children[0];
|
|
386
|
-
while (cursorNode) {
|
|
387
|
-
if (cursorNode.type === 'comment') fileDocstring += cursorNode.text + "\n";
|
|
388
|
-
else if (cursorNode.type !== 'import_statement' && cursorNode.type !== 'expression_statement') break;
|
|
389
|
-
cursorNode = cursorNode.nextSibling;
|
|
390
|
-
}
|
|
391
|
-
fileDocstring = fileDocstring.trim();
|
|
392
|
-
|
|
393
|
-
try {
|
|
394
|
-
const query = new Query(parser.getLanguage(), LANGUAGE_QUERIES[langKey]);
|
|
395
|
-
const matches = query.matches(rootNode);
|
|
396
|
-
const processedNodes = new Set();
|
|
397
|
-
|
|
398
|
-
// ── God-class pre-pass ─────────────────────────────────────────────────
|
|
399
|
-
// Identify class-type container nodes whose line count exceeds the threshold.
|
|
400
|
-
// Their methods are allowed through the isNested filter below so each method
|
|
401
|
-
// becomes its own independent, searchable chunk; the class node itself gets a
|
|
402
|
-
// compact skeleton instead of a truncated body dump.
|
|
403
|
-
//
|
|
404
|
-
// Cross-language: class_declaration (TS/JS/Java/C#), class_definition (Python),
|
|
405
|
-
// impl_item (Rust impl blocks), class (Ruby), object_declaration (Kotlin).
|
|
406
|
-
// TS special case: exported classes live inside export_statement — we mark the
|
|
407
|
-
// inner class_declaration so the method-isNested check works correctly.
|
|
408
|
-
const GOD_CLASS_LINES = 200;
|
|
409
|
-
const GOD_CLASS_NODE_TYPES = new Set([
|
|
410
|
-
'class_declaration', 'class_definition',
|
|
411
|
-
'impl_item', // Rust impl blocks
|
|
412
|
-
'class', // Ruby
|
|
413
|
-
'object_declaration', // Kotlin
|
|
414
|
-
]);
|
|
415
|
-
const oversizedClassIds = new Set(); // node IDs whose direct methods are un-nested
|
|
416
|
-
|
|
417
|
-
for (const match of matches) {
|
|
418
|
-
for (const capture of match.captures) {
|
|
419
|
-
if (capture.name !== 'chunk') continue;
|
|
420
|
-
const n = capture.node;
|
|
421
|
-
const nLines = n.endPosition.row - n.startPosition.row + 1;
|
|
422
|
-
if (nLines <= GOD_CLASS_LINES) continue;
|
|
423
|
-
if (GOD_CLASS_NODE_TYPES.has(n.type)) {
|
|
424
|
-
oversizedClassIds.add(n.id);
|
|
425
|
-
}
|
|
426
|
-
// TS: large export_statement wrapping a class — mark the inner class_declaration
|
|
427
|
-
// so method_definition.parent chain finds the oversized class node, not the
|
|
428
|
-
// export_statement (which isn't what the isNested walk stops at).
|
|
429
|
-
if (n.type === 'export_statement') {
|
|
430
|
-
for (let ci = 0; ci < n.namedChildCount; ci++) {
|
|
431
|
-
const c = n.namedChild(ci);
|
|
432
|
-
if (GOD_CLASS_NODE_TYPES.has(c.type)) {
|
|
433
|
-
oversizedClassIds.add(c.id);
|
|
434
|
-
break;
|
|
435
|
-
}
|
|
436
|
-
}
|
|
437
|
-
}
|
|
438
|
-
}
|
|
439
|
-
}
|
|
440
|
-
|
|
441
|
-
for (const match of matches) {
|
|
442
|
-
let chunkNode = null;
|
|
443
|
-
for (const capture of match.captures) {
|
|
444
|
-
if (capture.name === 'chunk') chunkNode = capture.node;
|
|
445
|
-
}
|
|
446
|
-
|
|
447
|
-
if (!chunkNode || processedNodes.has(chunkNode.id)) continue;
|
|
448
|
-
processedNodes.add(chunkNode.id);
|
|
449
|
-
|
|
450
|
-
// Filter out very small fragments (simple variables, etc.)
|
|
451
|
-
if (chunkNode.endPosition.row - chunkNode.startPosition.row < 2) continue;
|
|
452
|
-
|
|
453
|
-
// 🥇 DEDUPLICATION LOGIC: Ignore nodes that are nested inside other container nodes.
|
|
454
|
-
// Stop at the actual tree root (parent === null) to be language-agnostic:
|
|
455
|
-
// Python root = 'module', JS root = 'program', Go root = 'source_file', Ruby root = 'program'.
|
|
456
|
-
// Stopping at 'program' alone was falsely marking top-level Python classes as nested
|
|
457
|
-
// because Ruby's 'module' keyword (also in CONTAINERS) shares the name with Python's root.
|
|
458
|
-
//
|
|
459
|
-
// God-class exception: if the first CONTAINERS ancestor is an oversized class, the
|
|
460
|
-
// node is a direct method of that class — allow it through as its own chunk.
|
|
461
|
-
let isNested = false;
|
|
462
|
-
let currentParent = chunkNode.parent;
|
|
463
|
-
while (currentParent && currentParent.parent !== null) {
|
|
464
|
-
if (CONTAINERS.has(currentParent.type)) {
|
|
465
|
-
if (!oversizedClassIds.has(currentParent.id)) {
|
|
466
|
-
isNested = true;
|
|
467
|
-
}
|
|
468
|
-
break;
|
|
469
|
-
}
|
|
470
|
-
currentParent = currentParent.parent;
|
|
471
|
-
}
|
|
472
|
-
if (isNested) continue;
|
|
473
|
-
|
|
474
|
-
// 🥇 ROBUST NAME EXTRACTION (JS logic)
|
|
475
|
-
let nameText = "anonymous";
|
|
476
|
-
const nameNode = chunkNode.childForFieldName?.("name");
|
|
477
|
-
|
|
478
|
-
if (nameNode) {
|
|
479
|
-
nameText = nameNode.text;
|
|
480
|
-
} else if (chunkNode.type === "export_statement") {
|
|
481
|
-
const decl = chunkNode.children.find(c => ['lexical_declaration', 'function_declaration', 'class_declaration'].includes(c.type));
|
|
482
|
-
if (decl) {
|
|
483
|
-
if (decl.type === "lexical_declaration") {
|
|
484
|
-
const varDecl = decl.children.find(c => c.type === "variable_declarator");
|
|
485
|
-
nameText = varDecl?.children.find(c => c.type === "identifier")?.text || "anonymous";
|
|
486
|
-
} else {
|
|
487
|
-
nameText = decl.childForFieldName?.("name")?.text || "anonymous";
|
|
488
|
-
}
|
|
489
|
-
} else {
|
|
490
|
-
const defaultChild = chunkNode.children.find(c => c.type === 'identifier' || c.type === 'call_expression');
|
|
491
|
-
if (defaultChild) {
|
|
492
|
-
if (defaultChild.type === 'identifier') {
|
|
493
|
-
nameText = `default_${defaultChild.text}`;
|
|
494
|
-
} else if (defaultChild.type === 'call_expression') {
|
|
495
|
-
const funcName = defaultChild.childForFieldName?.("function")?.text || defaultChild.children[0]?.text;
|
|
496
|
-
const argNode = defaultChild.childForFieldName?.("arguments")?.children?.find(c => c.type === 'identifier');
|
|
497
|
-
nameText = argNode ? `default_${funcName}_${argNode.text}` : `default_${funcName}`;
|
|
498
|
-
}
|
|
499
|
-
} else {
|
|
500
|
-
nameText = "default_export";
|
|
501
|
-
}
|
|
502
|
-
}
|
|
503
|
-
} else if (chunkNode.type === "lexical_declaration") {
|
|
504
|
-
const decl = chunkNode.children.find(c => c.type === "variable_declarator");
|
|
505
|
-
nameText = decl?.children.find(c => c.type === "identifier")?.text || "anonymous";
|
|
506
|
-
} else if (chunkNode.type === "expression_statement") {
|
|
507
|
-
const callExp = chunkNode.children.find(c => c.type === "call_expression");
|
|
508
|
-
const assignExp = chunkNode.children.find(c => c.type === "assignment_expression");
|
|
509
|
-
if (callExp) {
|
|
510
|
-
const funcName = callExp.childForFieldName?.("function")?.text || callExp.children[0]?.text;
|
|
511
|
-
const argsNode = callExp.childForFieldName?.("arguments");
|
|
512
|
-
const stringArg = argsNode?.children?.find(c => c.type === "string" || c.type === "template_string");
|
|
513
|
-
nameText = stringArg ? `${funcName}_${stringArg.text.replace(/['"`]/g, '')}` : (funcName || "anonymous");
|
|
514
|
-
} else if (assignExp) {
|
|
515
|
-
nameText = assignExp.childForFieldName?.("left")?.text || "anonymous";
|
|
516
|
-
}
|
|
517
|
-
} else if (chunkNode.type === "type_declaration") {
|
|
518
|
-
// Go: `type X struct {…}` / `type X interface {…}` / `type X = Y`.
|
|
519
|
-
// The identifier lives on the nested type_spec / type_alias node, not on
|
|
520
|
-
// the type_declaration itself, so childForFieldName("name") above is null.
|
|
521
|
-
// Without this branch every Go struct/interface collapses to the useless
|
|
522
|
-
// synthetic name `<file>_type_declaration`, making core types (e.g. Gin's
|
|
523
|
-
// RouterGroup, Engine, Context) unsearchable by name and invisible to the
|
|
524
|
-
// 2.0× name-boost in searchHybrid.
|
|
525
|
-
const spec = chunkNode.namedChildren?.find(c => c.type === "type_spec" || c.type === "type_alias");
|
|
526
|
-
nameText = spec?.childForFieldName?.("name")?.text
|
|
527
|
-
|| spec?.children?.find(c => c.type === "type_identifier")?.text
|
|
528
|
-
|| "anonymous";
|
|
529
|
-
} else {
|
|
530
|
-
// Generic fallback: search direct children for an identifier-like node.
|
|
531
|
-
// Includes type_identifier (Go/TS), constant (Ruby class/module names) and
|
|
532
|
-
// field_identifier so nested-name grammars don't fall through to anonymous.
|
|
533
|
-
const idNode = chunkNode.children.find(c =>
|
|
534
|
-
c.type === "identifier" || c.type === "name" || c.type === "property_identifier"
|
|
535
|
-
|| c.type === "type_identifier" || c.type === "constant" || c.type === "field_identifier");
|
|
536
|
-
nameText = idNode?.text || "anonymous";
|
|
537
|
-
}
|
|
538
|
-
|
|
539
|
-
if (nameText === "anonymous" || nameText === "default_export") {
|
|
540
|
-
nameText = `${path.basename(relPath, path.extname(relPath))}_${chunkNode.type}`;
|
|
541
|
-
}
|
|
542
|
-
|
|
543
|
-
let docstring = "";
|
|
544
|
-
let prev = chunkNode.previousSibling;
|
|
545
|
-
while (prev && (prev.type === 'comment' || !prev.isNamed)) {
|
|
546
|
-
if (prev.type === 'comment') docstring = prev.text + "\n" + docstring;
|
|
547
|
-
prev = prev.previousSibling;
|
|
548
|
-
}
|
|
549
|
-
docstring = docstring.trim();
|
|
550
|
-
|
|
551
|
-
if (!docstring && fileDocstring) {
|
|
552
|
-
docstring = `[File Context]: ${fileDocstring}`;
|
|
553
|
-
}
|
|
554
|
-
|
|
555
|
-
// For oversized class containers — or TS export_statements wrapping one —
|
|
556
|
-
// store a compact skeleton so get_chunk() returns a bounded token count.
|
|
557
|
-
// All metadata fields (calls, params, type_refs, decorators, extends) are
|
|
558
|
-
// still extracted from the FULL AST node, so ranking and topology are intact.
|
|
559
|
-
const isOversizedClass = oversizedClassIds.has(chunkNode.id);
|
|
560
|
-
const wrapsOversizedClass = !isOversizedClass && chunkNode.type === 'export_statement' && (() => {
|
|
561
|
-
for (let ci = 0; ci < chunkNode.namedChildCount; ci++) {
|
|
562
|
-
if (oversizedClassIds.has(chunkNode.namedChild(ci).id)) return true;
|
|
563
|
-
}
|
|
564
|
-
return false;
|
|
565
|
-
})();
|
|
566
|
-
const snippet = (isOversizedClass || wrapsOversizedClass)
|
|
567
|
-
? buildGodClassSkeleton(isOversizedClass ? chunkNode : (() => {
|
|
568
|
-
for (let ci = 0; ci < chunkNode.namedChildCount; ci++) {
|
|
569
|
-
const c = chunkNode.namedChild(ci);
|
|
570
|
-
if (oversizedClassIds.has(c.id)) return c;
|
|
571
|
-
}
|
|
572
|
-
return chunkNode;
|
|
573
|
-
})())
|
|
574
|
-
: chunkNode.text.slice(0, 3000);
|
|
575
|
-
const hash = generateChunkHash(docstring + snippet);
|
|
576
|
-
const outgoingCalls = extractCalls(chunkNode);
|
|
577
|
-
|
|
578
|
-
// 🥇 PARAMETER / TYPE / CLASS CONTEXT ENRICHMENT (improves recall on undocumented code)
|
|
579
|
-
const params = extractParams(chunkNode, ext);
|
|
580
|
-
const returnType = extractReturnType(chunkNode, ext);
|
|
581
|
-
const classContext = extractClassContext(chunkNode);
|
|
582
|
-
const typeRefs = extractTypeAnnotations(chunkNode, ext);
|
|
583
|
-
const decorators = extractDecorators(chunkNode);
|
|
584
|
-
const heritage = extractHeritage(chunkNode, ext);
|
|
585
|
-
|
|
586
|
-
const id = createHash('sha256')
|
|
587
|
-
.update(`${relPath}::${chunkNode.startPosition.row}::${chunkNode.startPosition.column}`)
|
|
588
|
-
.digest('hex').slice(0, 24);
|
|
589
|
-
|
|
590
|
-
chunks.push({
|
|
591
|
-
id, file_path: relPath, node_type: chunkNode.type, name: nameText,
|
|
592
|
-
docstring: docstring, code_snippet: snippet, content_hash: hash,
|
|
593
|
-
start_line: chunkNode.startPosition.row + 1, end_line: chunkNode.endPosition.row + 1,
|
|
594
|
-
calls: outgoingCalls,
|
|
595
|
-
params, return_type: returnType, class_context: classContext,
|
|
596
|
-
type_refs: typeRefs, decorators, extends: heritage,
|
|
597
|
-
});
|
|
598
|
-
}
|
|
599
|
-
} catch (e) {
|
|
600
|
-
// Visible protective log for developers
|
|
601
|
-
process.stderr.write(`\n[parser-utils] 💥 Query Error in ${relPath}: ${e.message}\n`);
|
|
602
|
-
}
|
|
603
|
-
|
|
604
|
-
// ── Python public re-exports ────────────────────────────────────────────────
|
|
605
|
-
// PEP 484 convention: `from starlette.background import BackgroundTasks as
|
|
606
|
-
// BackgroundTasks` re-exports a symbol as public API. Files like
|
|
607
|
-
// fastapi/background.py consist ONLY of such lines and previously produced
|
|
608
|
-
// zero chunks — resolve_symbol('BackgroundTasks') found nothing and agents
|
|
609
|
-
// hit a dead end. Each explicit re-export becomes a small chunk that names
|
|
610
|
-
// the symbol and points at its source module. (JS/TS barrels already chunk
|
|
611
|
-
// via the export_statement capture.)
|
|
612
|
-
if (ext === '.py') {
|
|
613
|
-
for (const node of rootNode.children) {
|
|
614
|
-
if (node.type !== 'import_from_statement') continue;
|
|
615
|
-
const moduleName = node.childForFieldName?.('module_name')?.text || '';
|
|
616
|
-
for (const child of node.children) {
|
|
617
|
-
if (child.type !== 'aliased_import') continue;
|
|
618
|
-
const orig = child.childForFieldName?.('name')?.text || '';
|
|
619
|
-
const alias = child.childForFieldName?.('alias')?.text || '';
|
|
620
|
-
if (!alias || orig.split('.').pop() !== alias) continue; // only `import X as X`
|
|
621
|
-
const snippet = node.text.slice(0, 300);
|
|
622
|
-
const docstring = `Public re-export of ${alias} from ${moduleName}.`;
|
|
623
|
-
const id = createHash('sha256')
|
|
624
|
-
.update(`${relPath}::${node.startPosition.row}::${child.startPosition.column}`)
|
|
625
|
-
.digest('hex').slice(0, 24);
|
|
626
|
-
chunks.push({
|
|
627
|
-
id, file_path: relPath, node_type: 're_export', name: alias,
|
|
628
|
-
docstring, code_snippet: snippet,
|
|
629
|
-
content_hash: generateChunkHash(docstring + snippet),
|
|
630
|
-
start_line: node.startPosition.row + 1, end_line: node.endPosition.row + 1,
|
|
631
|
-
calls: [], params: [], return_type: '', class_context: '',
|
|
632
|
-
type_refs: [], decorators: [], extends: [],
|
|
633
|
-
});
|
|
634
|
-
}
|
|
635
|
-
}
|
|
636
|
-
}
|
|
637
|
-
return chunks;
|
|
638
|
-
}
|
|
639
|
-
|
|
640
|
-
// ─── Barrel export resolution ─────────────────────────────────────────────────
|
|
641
|
-
|
|
642
|
-
// Module-level cache: barrelAbsPath → Map<exportedName, sourceRelPath>
|
|
643
|
-
const _barrelCache = new Map();
|
|
644
|
-
|
|
645
|
-
/**
|
|
646
|
-
* Parses a barrel file (index.ts / index.js) and returns a map of
|
|
647
|
-
* exportedName → sourceFilePath (relative to projectRoot).
|
|
648
|
-
* e.g. { useAuthStore: 'src/stores/authStore.ts' }
|
|
649
|
-
*/
|
|
650
|
-
export function resolveBarrelExports(barrelAbsPath, projectRoot) {
|
|
651
|
-
if (_barrelCache.has(barrelAbsPath)) return _barrelCache.get(barrelAbsPath);
|
|
652
|
-
|
|
653
|
-
const result = new Map();
|
|
654
|
-
_barrelCache.set(barrelAbsPath, result);
|
|
655
|
-
|
|
656
|
-
const ext = path.extname(barrelAbsPath);
|
|
657
|
-
if (!['.ts', '.tsx', '.js', '.jsx', '.mjs', '.cjs'].includes(ext)) return result;
|
|
658
|
-
|
|
659
|
-
let content;
|
|
660
|
-
try { content = fs.readFileSync(barrelAbsPath, 'utf-8'); } catch { return result; }
|
|
661
|
-
|
|
662
|
-
const parser = getParserForFile(ext);
|
|
663
|
-
if (!parser) return result;
|
|
664
|
-
|
|
665
|
-
let tree;
|
|
666
|
-
try { tree = parser.parse((offset) => offset < content.length ? content.slice(offset, offset + 4096) : null); } catch { return result; }
|
|
667
|
-
|
|
668
|
-
const barrelDir = path.dirname(barrelAbsPath);
|
|
669
|
-
|
|
670
|
-
function walk(node) {
|
|
671
|
-
// export { X, Y as Z } from './source'
|
|
672
|
-
if (node.type === 'export_statement') {
|
|
673
|
-
const fromNode = node.children.find(c => c.type === 'string');
|
|
674
|
-
if (!fromNode) { node.children.forEach(walk); return; }
|
|
675
|
-
|
|
676
|
-
const rawSource = fromNode.text.replace(/['"]/g, '');
|
|
677
|
-
if (!rawSource.startsWith('.')) { node.children.forEach(walk); return; }
|
|
678
|
-
|
|
679
|
-
// Resolve the source file
|
|
680
|
-
const absSource = path.resolve(barrelDir, rawSource);
|
|
681
|
-
let finalAbs = null;
|
|
682
|
-
if (EXTENSIONS.has(path.extname(absSource)) && fs.existsSync(absSource)) {
|
|
683
|
-
finalAbs = absSource;
|
|
684
|
-
} else {
|
|
685
|
-
for (const e of EXTENSIONS) {
|
|
686
|
-
if (fs.existsSync(absSource + e)) { finalAbs = absSource + e; break; }
|
|
687
|
-
const idx = path.join(absSource, 'index' + e);
|
|
688
|
-
if (fs.existsSync(idx)) { finalAbs = idx; break; }
|
|
689
|
-
}
|
|
690
|
-
}
|
|
691
|
-
if (!finalAbs) { node.children.forEach(walk); return; }
|
|
692
|
-
|
|
693
|
-
const relSource = path.relative(projectRoot, finalAbs).replace(/\\/g, '/');
|
|
694
|
-
|
|
695
|
-
// Walk named exports
|
|
696
|
-
const namedExports = node.children.find(c => c.type === 'named_imports' || c.type === 'export_clause');
|
|
697
|
-
if (namedExports) {
|
|
698
|
-
for (const child of namedExports.children) {
|
|
699
|
-
if (child.type === 'import_specifier' || child.type === 'export_specifier') {
|
|
700
|
-
// `alias as exported` or just `name`
|
|
701
|
-
const names = child.children.filter(c => c.type === 'identifier');
|
|
702
|
-
if (names.length > 0) {
|
|
703
|
-
// The exported name is the last identifier (the alias if present)
|
|
704
|
-
result.set(names[names.length - 1].text, relSource);
|
|
705
|
-
}
|
|
706
|
-
}
|
|
707
|
-
}
|
|
708
|
-
}
|
|
709
|
-
|
|
710
|
-
// export * from './source' → map the source file itself
|
|
711
|
-
const starNode = node.children.find(c => c.text === '*');
|
|
712
|
-
if (starNode) {
|
|
713
|
-
result.set('*', relSource);
|
|
714
|
-
}
|
|
715
|
-
}
|
|
716
|
-
node.children.forEach(walk);
|
|
717
|
-
}
|
|
718
|
-
walk(tree.rootNode);
|
|
719
|
-
return result;
|
|
720
|
-
}
|
|
721
|
-
|
|
722
|
-
// ─── Go module-name cache (reads go.mod once per project root) ──────────────
|
|
723
|
-
const _goModCache = new Map();
|
|
724
|
-
function _readGoModuleName(projectRoot) {
|
|
725
|
-
if (_goModCache.has(projectRoot)) return _goModCache.get(projectRoot);
|
|
726
|
-
const modFile = path.join(projectRoot, 'go.mod');
|
|
727
|
-
let name = null;
|
|
728
|
-
if (fs.existsSync(modFile)) {
|
|
729
|
-
try {
|
|
730
|
-
const first = fs.readFileSync(modFile, 'utf-8').split('\n').find(l => l.trimStart().startsWith('module '));
|
|
731
|
-
if (first) name = first.trim().replace(/^module\s+/, '').split(/\s/)[0];
|
|
732
|
-
} catch { /* ignore */ }
|
|
733
|
-
}
|
|
734
|
-
_goModCache.set(projectRoot, name);
|
|
735
|
-
return name;
|
|
736
|
-
}
|
|
737
|
-
|
|
738
|
-
export function resolveLocalImports(rawImports, fromFileRelPath, projectRoot) {
|
|
739
|
-
const fileDir = path.dirname(path.join(projectRoot, fromFileRelPath));
|
|
740
|
-
const ext = path.extname(fromFileRelPath);
|
|
741
|
-
const resolved = [];
|
|
742
|
-
for (const raw of rawImports) {
|
|
743
|
-
// ── Dot-relative (JS/TS/Python relative) ────────────────────────────
|
|
744
|
-
if (raw.startsWith('.')) {
|
|
745
|
-
const absResolved = path.resolve(fileDir, raw);
|
|
746
|
-
const existingExt = path.extname(absResolved);
|
|
747
|
-
let finalAbs = null;
|
|
748
|
-
if (existingExt && EXTENSIONS.has(existingExt) && fs.existsSync(absResolved)) {
|
|
749
|
-
finalAbs = absResolved;
|
|
750
|
-
} else {
|
|
751
|
-
for (const e of EXTENSIONS) {
|
|
752
|
-
if (fs.existsSync(absResolved + e)) { finalAbs = absResolved + e; break; }
|
|
753
|
-
const idx = path.join(absResolved, 'index' + e);
|
|
754
|
-
if (fs.existsSync(idx)) { finalAbs = idx; break; }
|
|
755
|
-
// Python: also try __init__.py for package directories
|
|
756
|
-
if (e === '.py') {
|
|
757
|
-
const init = path.join(absResolved, '__init__.py');
|
|
758
|
-
if (fs.existsSync(init)) { finalAbs = init; break; }
|
|
759
|
-
}
|
|
760
|
-
}
|
|
761
|
-
}
|
|
762
|
-
if (finalAbs) {
|
|
763
|
-
const relPath = path.relative(projectRoot, finalAbs).replace(/\\/g, '/');
|
|
764
|
-
const baseName = path.basename(finalAbs, path.extname(finalAbs));
|
|
765
|
-
// 🥇 BARREL RESOLUTION: If the resolved file is an index file, expand barrel exports
|
|
766
|
-
if (baseName === 'index' && ['.ts', '.tsx', '.js', '.jsx', '.mjs', '.cjs'].includes(path.extname(finalAbs))) {
|
|
767
|
-
const barrelMap = resolveBarrelExports(finalAbs, projectRoot);
|
|
768
|
-
if (barrelMap.size > 0) {
|
|
769
|
-
// Add all unique source files referenced by this barrel
|
|
770
|
-
const sources = new Set(barrelMap.values());
|
|
771
|
-
for (const src of sources) {
|
|
772
|
-
if (!resolved.includes(src)) resolved.push(src);
|
|
773
|
-
}
|
|
774
|
-
} else {
|
|
775
|
-
// Barrel has no re-exports — keep the barrel file itself
|
|
776
|
-
if (!resolved.includes(relPath)) resolved.push(relPath);
|
|
777
|
-
}
|
|
778
|
-
} else {
|
|
779
|
-
if (!resolved.includes(relPath)) resolved.push(relPath);
|
|
780
|
-
}
|
|
781
|
-
}
|
|
782
|
-
}
|
|
783
|
-
// ── Go intra-module: github.com/owner/repo/sub → sub/*.go ──────────
|
|
784
|
-
// Go import paths use the module name as prefix; map them to local dirs.
|
|
785
|
-
else if (ext === '.go') {
|
|
786
|
-
const modName = _readGoModuleName(projectRoot);
|
|
787
|
-
if (modName && raw.startsWith(modName + '/')) {
|
|
788
|
-
const subPkg = raw.slice(modName.length + 1); // e.g. 'render'
|
|
789
|
-
const absDir = path.join(projectRoot, subPkg);
|
|
790
|
-
if (fs.existsSync(absDir)) {
|
|
791
|
-
try {
|
|
792
|
-
const goFiles = fs.readdirSync(absDir)
|
|
793
|
-
.filter(f => f.endsWith('.go') && !f.includes('_test'))
|
|
794
|
-
.slice(0, 5); // cap: take a representative sample
|
|
795
|
-
for (const gof of goFiles) {
|
|
796
|
-
const rel = path.relative(projectRoot, path.join(absDir, gof)).replace(/\\/g, '/');
|
|
797
|
-
if (!resolved.includes(rel)) resolved.push(rel);
|
|
798
|
-
}
|
|
799
|
-
} catch { /* directory unreadable — skip */ }
|
|
800
|
-
}
|
|
801
|
-
}
|
|
802
|
-
}
|
|
803
|
-
// ── Rust crate-local: crate::module::item → src/module.rs ──────────
|
|
804
|
-
else if (ext === '.rs' && raw.startsWith('crate::')) {
|
|
805
|
-
const parts = raw.slice('crate::'.length).split('::').filter(Boolean);
|
|
806
|
-
for (let depth = parts.length; depth >= 1; depth--) {
|
|
807
|
-
const subPath = parts.slice(0, depth).join('/');
|
|
808
|
-
const candidates = [
|
|
809
|
-
path.join(projectRoot, 'src', subPath + '.rs'),
|
|
810
|
-
path.join(projectRoot, 'src', subPath, 'mod.rs'),
|
|
811
|
-
];
|
|
812
|
-
let found = false;
|
|
813
|
-
for (const c of candidates) {
|
|
814
|
-
if (fs.existsSync(c)) {
|
|
815
|
-
resolved.push(path.relative(projectRoot, c).replace(/\\/g, '/'));
|
|
816
|
-
found = true; break;
|
|
817
|
-
}
|
|
818
|
-
}
|
|
819
|
-
if (found) break;
|
|
820
|
-
}
|
|
821
|
-
}
|
|
822
|
-
}
|
|
823
|
-
return resolved;
|
|
824
|
-
}
|
|
825
|
-
|
|
826
|
-
/**
|
|
827
|
-
* Build the text payload sent to the embedding model for a chunk.
|
|
828
|
-
*
|
|
829
|
-
* Shared by the bootstrap indexer (indexer.mjs) and the watch daemon
|
|
830
|
-
* (watch-daemon.mjs) so a chunk yields the SAME embedding regardless of which
|
|
831
|
-
* path embedded it first. This matters because the cache key (content_hash) is
|
|
832
|
-
* derived from code + docstring only — it does NOT include this payload — so two
|
|
833
|
-
* divergent payloads for the same hash would silently produce inconsistent
|
|
834
|
-
* embeddings across a full re-index vs. an incremental update.
|
|
835
|
-
*
|
|
836
|
-
* @param {object} chunk Semantic chunk: { file_path, node_type, name, docstring, type_refs, code_snippet }.
|
|
837
|
-
* @param {string[]} depRelPaths Resolved local imports of the chunk's file (project-relative paths).
|
|
838
|
-
* @returns {string}
|
|
839
|
-
*/
|
|
840
|
-
export function buildEmbeddingPayload(chunk, depRelPaths = []) {
|
|
841
|
-
const neighbors = depRelPaths
|
|
842
|
-
.map(d => path.basename(d, path.extname(d)))
|
|
843
|
-
.filter(Boolean);
|
|
844
|
-
const topologicalContext = neighbors.length
|
|
845
|
-
? `This code architectural neighborhood connects with: ${neighbors.join(', ')}.`
|
|
846
|
-
: '';
|
|
847
|
-
// NOTE: decorators and inheritance edges are NOT added here (A/B-tested: neutral
|
|
848
|
-
// on vector, regression on BM25 — surfaced as metadata only).
|
|
849
|
-
// LLM summary leads the payload when available: declarative voice aligns with
|
|
850
|
-
// nomic-embed-text's search_document: training objective and anchors the embedding
|
|
851
|
-
// toward developer query vocabulary. Questions/hyde are intentionally excluded from
|
|
852
|
-
// the vector payload — they add stopword noise and dilute the code's semantic
|
|
853
|
-
// fingerprint. Concept keywords (chunk.hyde = concepts.join(' ')) go to BM25 only
|
|
854
|
-
// via buildLexicalDocument, keeping both retrieval channels clean.
|
|
855
|
-
return [
|
|
856
|
-
chunk.summary || '', // semantic lead: LLM-generated declarative summary (opt-in)
|
|
857
|
-
`File Location: ${chunk.file_path}`,
|
|
858
|
-
`Symbol Name: ${chunk.node_type} -> ${chunk.name}`,
|
|
859
|
-
chunk.docstring ? `Developer Documentation: ${chunk.docstring}` : '',
|
|
860
|
-
chunk.type_refs?.length ? `Type References: ${chunk.type_refs.join(', ')}` : '',
|
|
861
|
-
topologicalContext,
|
|
862
|
-
`--- Source Code ---`,
|
|
863
|
-
chunk.code_snippet,
|
|
864
|
-
].filter(Boolean).join('\n');
|
|
865
|
-
}
|
|
866
|
-
|
|
867
|
-
export async function getLocalEmbedding(text, graceful = true, { ollamaHost, model } = {}) {
|
|
868
|
-
if (process.env.INDEXER_EMBEDDINGS === 'off') return null; // lexical-only mode
|
|
869
|
-
const host = _resolveOllamaHost(ollamaHost);
|
|
870
|
-
const embedModel = _resolveEmbedModel(model);
|
|
871
|
-
const MAX_RETRIES = 3;
|
|
872
|
-
const safeText = "search_query: " + truncateForEmbedding(text);
|
|
873
|
-
for (let attempt = 0; attempt <= MAX_RETRIES; attempt++) {
|
|
874
|
-
try {
|
|
875
|
-
const res = await fetch(`${host}/api/embeddings`, {
|
|
876
|
-
method: "POST", headers: { "Content-Type": "application/json" },
|
|
877
|
-
body: JSON.stringify({ model: embedModel, prompt: safeText }),
|
|
878
|
-
signal: AbortSignal.timeout(15000),
|
|
879
|
-
});
|
|
880
|
-
if (!res.ok) throw new Error(`HTTP ${res.status}`);
|
|
881
|
-
const data = await res.json();
|
|
882
|
-
return data.embedding;
|
|
883
|
-
} catch (err) {
|
|
884
|
-
if (attempt < MAX_RETRIES) await new Promise(r => setTimeout(r, 500 * 2 ** attempt));
|
|
885
|
-
else if (!graceful) throw err;
|
|
886
|
-
}
|
|
887
|
-
}
|
|
888
|
-
return null;
|
|
889
|
-
}
|
|
890
|
-
|
|
891
|
-
export async function getLocalEmbeddingsBatch(texts, graceful = true, { ollamaHost, model } = {}) {
|
|
892
|
-
if (!texts || texts.length === 0) return [];
|
|
893
|
-
if (process.env.INDEXER_EMBEDDINGS === 'off') return null; // lexical-only mode
|
|
894
|
-
const host = _resolveOllamaHost(ollamaHost);
|
|
895
|
-
const embedModel = _resolveEmbedModel(model);
|
|
896
|
-
const MAX_RETRIES = 3;
|
|
897
|
-
const safeTexts = texts.map(t => "search_document: " + (t.length > 8000 ? t.slice(0, 8000) : t));
|
|
898
|
-
|
|
899
|
-
for (let attempt = 0; attempt <= MAX_RETRIES; attempt++) {
|
|
900
|
-
try {
|
|
901
|
-
const res = await fetch(`${host}/api/embed`, {
|
|
902
|
-
method: "POST", headers: { "Content-Type": "application/json" },
|
|
903
|
-
body: JSON.stringify({ model: embedModel, input: safeTexts }),
|
|
904
|
-
signal: AbortSignal.timeout(60000),
|
|
905
|
-
});
|
|
906
|
-
if (!res.ok) throw new Error(`HTTP ${res.status}`);
|
|
907
|
-
const data = await res.json();
|
|
908
|
-
return data.embeddings;
|
|
909
|
-
} catch (err) {
|
|
910
|
-
if (attempt < MAX_RETRIES) await new Promise(r => setTimeout(r, 500 * 2 ** attempt));
|
|
911
|
-
else if (!graceful) throw err;
|
|
912
|
-
}
|
|
913
|
-
}
|
|
914
|
-
return null;
|
|
915
|
-
}
|
|
916
|
-
|
|
917
|
-
// ─── Enrichment helpers (param names, return type, class context) ─────────────
|
|
918
|
-
|
|
919
|
-
export function extractParams(chunkNode, ext) {
|
|
920
|
-
const params = [];
|
|
921
|
-
const JS_LIKE = ['.ts', '.tsx', '.js', '.jsx', '.mjs', '.cjs'];
|
|
922
|
-
const paramTypes = JS_LIKE.includes(ext)
|
|
923
|
-
? ['required_parameter', 'optional_parameter', 'formal_parameters', 'identifier']
|
|
924
|
-
: ['parameter', 'formal_parameter', 'identifier'];
|
|
925
|
-
|
|
926
|
-
function walkParams(node) {
|
|
927
|
-
if (node.type === 'formal_parameters' || node.type === 'parameters' || node.type === 'parameter_list') {
|
|
928
|
-
for (const child of node.children) {
|
|
929
|
-
// TS: required_parameter / optional_parameter have an identifier child
|
|
930
|
-
if (child.type === 'required_parameter' || child.type === 'optional_parameter' || child.type === 'formal_parameter') {
|
|
931
|
-
const id = child.childForFieldName?.('pattern') || child.childForFieldName?.('name') ||
|
|
932
|
-
child.children.find(c => c.type === 'identifier');
|
|
933
|
-
if (id) params.push(id.text);
|
|
934
|
-
// also grab type annotation text
|
|
935
|
-
const typeAnnotation = child.childForFieldName?.('type');
|
|
936
|
-
if (typeAnnotation) {
|
|
937
|
-
const typeText = typeAnnotation.text.replace(/^:\s*/, '').trim();
|
|
938
|
-
if (typeText) params.push(typeText);
|
|
939
|
-
}
|
|
940
|
-
} else if (child.type === 'identifier') {
|
|
941
|
-
params.push(child.text);
|
|
942
|
-
}
|
|
943
|
-
}
|
|
944
|
-
}
|
|
945
|
-
for (const child of node.children) walkParams(child);
|
|
946
|
-
}
|
|
947
|
-
|
|
948
|
-
// Only walk the direct params node to avoid deep recursion into body
|
|
949
|
-
const paramsNode = chunkNode.childForFieldName?.('parameters') || chunkNode.childForFieldName?.('formal_parameters');
|
|
950
|
-
if (paramsNode) walkParams(paramsNode);
|
|
951
|
-
return [...new Set(params)].filter(p => p && p.length > 1).slice(0, 15);
|
|
952
|
-
}
|
|
953
|
-
|
|
954
|
-
export function extractReturnType(chunkNode, ext) {
|
|
955
|
-
const JS_LIKE = ['.ts', '.tsx', '.js', '.jsx', '.mjs', '.cjs'];
|
|
956
|
-
if (JS_LIKE.includes(ext)) {
|
|
957
|
-
const retTypeNode = chunkNode.childForFieldName?.('return_type');
|
|
958
|
-
if (retTypeNode) return retTypeNode.text.replace(/^:\s*/, '').trim().slice(0, 80);
|
|
959
|
-
}
|
|
960
|
-
return '';
|
|
961
|
-
}
|
|
962
|
-
|
|
963
|
-
export function extractClassContext(chunkNode) {
|
|
964
|
-
let parent = chunkNode.parent;
|
|
965
|
-
while (parent) {
|
|
966
|
-
if (parent.type === 'class_declaration' || parent.type === 'class_definition' ||
|
|
967
|
-
parent.type === 'class_body' || parent.type === 'impl_item') {
|
|
968
|
-
const nameNode = parent.childForFieldName?.('name');
|
|
969
|
-
if (nameNode) return nameNode.text;
|
|
970
|
-
}
|
|
971
|
-
parent = parent.parent;
|
|
972
|
-
}
|
|
973
|
-
return '';
|
|
974
|
-
}
|
|
975
|
-
|
|
976
|
-
/**
|
|
977
|
-
* Extract decorator / annotation names applied to a chunk (and, for class chunks,
|
|
978
|
-
* to the methods inside it). Decorators encode what a symbol *is* in modern
|
|
979
|
-
* frameworks — `@Controller`, `@Injectable`, `@Get`, `@Entity` (TS: NestJS,
|
|
980
|
-
* Angular, TypeORM) and `@app.route`, `@pytest.fixture`, `@dataclass`,
|
|
981
|
-
* `@property` (Python) — yet as raw snippet text they are diluted to a single
|
|
982
|
-
* low-weight token inside a large class body. Surfacing them as a dedicated field
|
|
983
|
-
* lets a class annotated `@Controller` be retrieved by "controller" and a method
|
|
984
|
-
* annotated `@Get` by "get/route", independent of language.
|
|
985
|
-
*
|
|
986
|
-
* Generalises by node type only (the tree-sitter `decorator` node is shared across
|
|
987
|
-
* TS/JS/Python grammars) — no framework-specific names are hardcoded. Callee
|
|
988
|
-
* arguments are stripped: `@Controller('cats')` -> 'Controller',
|
|
989
|
-
* `@app.route('/x')` -> 'app.route', `@UseGuards(AuthGuard)` -> 'UseGuards'.
|
|
990
|
-
*
|
|
991
|
-
* @returns {string[]} unique decorator callee names (max 24)
|
|
992
|
-
*/
|
|
993
|
-
export function extractDecorators(chunkNode) {
|
|
994
|
-
const names = new Set();
|
|
995
|
-
const addDecorator = (decoNode) => {
|
|
996
|
-
let t = (decoNode.text || '').trim().replace(/^@/, '');
|
|
997
|
-
t = t.split('(')[0]; // drop call arguments: @Get(':id') -> Get
|
|
998
|
-
t = t.split(/[\s\n{]/)[0].trim(); // first token only
|
|
999
|
-
if (t && t.length <= 64) names.add(t);
|
|
1000
|
-
};
|
|
1001
|
-
|
|
1002
|
-
// (1) Decorators that PRECEDE the chunk as siblings. Python wraps a decorated
|
|
1003
|
-
// symbol in `decorated_definition` ([decorator…, def]); some grammars place
|
|
1004
|
-
// class decorators as leading siblings rather than children.
|
|
1005
|
-
let prev = chunkNode.previousSibling;
|
|
1006
|
-
while (prev) {
|
|
1007
|
-
if (prev.type === 'decorator') addDecorator(prev);
|
|
1008
|
-
else if (prev.isNamed && prev.type !== 'comment') break;
|
|
1009
|
-
prev = prev.previousSibling;
|
|
1010
|
-
}
|
|
1011
|
-
|
|
1012
|
-
// (2) Decorators within the chunk's subtree. A captured TS class chunk (the
|
|
1013
|
-
// enclosing export_statement / class_declaration) carries its own class
|
|
1014
|
-
// decorators plus the @Get/@Post/@Inject decorators on its methods.
|
|
1015
|
-
// Bounded traversal so a large class body cannot inflate indexing time.
|
|
1016
|
-
let budget = 800;
|
|
1017
|
-
const stack = [chunkNode];
|
|
1018
|
-
while (stack.length && budget-- > 0) {
|
|
1019
|
-
const n = stack.pop();
|
|
1020
|
-
for (let i = 0; i < n.namedChildCount; i++) {
|
|
1021
|
-
const child = n.namedChild(i);
|
|
1022
|
-
if (child.type === 'decorator') addDecorator(child);
|
|
1023
|
-
else stack.push(child);
|
|
1024
|
-
}
|
|
1025
|
-
}
|
|
1026
|
-
|
|
1027
|
-
return Array.from(names).slice(0, 24);
|
|
1028
|
-
}
|
|
1029
|
-
|
|
1030
|
-
/**
|
|
1031
|
-
* Extract the base classes and implemented interfaces of a class chunk — the
|
|
1032
|
-
* inheritance edge that links a concept to its implementations
|
|
1033
|
-
* (`class ValidationPipe extends BasePipe implements PipeTransform`). Surfacing
|
|
1034
|
-
* this lets an agent move from an abstract type to the concrete classes that
|
|
1035
|
-
* realise it, and feeds the semantic embedding so an implementation is retrievable
|
|
1036
|
-
* by the interface it fulfils.
|
|
1037
|
-
*
|
|
1038
|
-
* Generalises across `extends`/`implements` (TS/JS) and base-class argument lists
|
|
1039
|
-
* (Python) by node type. Returns parent type names, e.g.
|
|
1040
|
-
* ['BasePipe', 'PipeTransform', 'OnInit'].
|
|
1041
|
-
*
|
|
1042
|
-
* @returns {string[]} base/interface names (max 12)
|
|
1043
|
-
*/
|
|
1044
|
-
export function extractHeritage(chunkNode, ext) {
|
|
1045
|
-
const bases = new Set();
|
|
1046
|
-
const JS_LIKE = ['.ts', '.tsx', '.js', '.jsx', '.mjs', '.cjs'];
|
|
1047
|
-
|
|
1048
|
-
const addTypeNames = (node) => {
|
|
1049
|
-
const stack = [node];
|
|
1050
|
-
let budget = 200;
|
|
1051
|
-
while (stack.length && budget-- > 0) {
|
|
1052
|
-
const n = stack.pop();
|
|
1053
|
-
if ((n.type === 'type_identifier' || n.type === 'identifier') && /^[A-Za-z_$]/.test(n.text)) {
|
|
1054
|
-
bases.add(n.text);
|
|
1055
|
-
}
|
|
1056
|
-
for (let i = 0; i < n.namedChildCount; i++) stack.push(n.namedChild(i));
|
|
1057
|
-
}
|
|
1058
|
-
};
|
|
1059
|
-
|
|
1060
|
-
if (JS_LIKE.includes(ext)) {
|
|
1061
|
-
// Walk the chunk subtree for extends/implements clauses, but never descend
|
|
1062
|
-
// into class_body — base types of the class only, not its method internals.
|
|
1063
|
-
const stack = [chunkNode];
|
|
1064
|
-
let budget = 400;
|
|
1065
|
-
while (stack.length && budget-- > 0) {
|
|
1066
|
-
const n = stack.pop();
|
|
1067
|
-
if (n.type === 'extends_clause' || n.type === 'implements_clause') { addTypeNames(n); continue; }
|
|
1068
|
-
for (let i = 0; i < n.namedChildCount; i++) {
|
|
1069
|
-
const c = n.namedChild(i);
|
|
1070
|
-
if (c.type !== 'class_body' && c.type !== 'statement_block') stack.push(c);
|
|
1071
|
-
}
|
|
1072
|
-
}
|
|
1073
|
-
} else if (ext === '.py') {
|
|
1074
|
-
const sc = chunkNode.childForFieldName?.('superclasses');
|
|
1075
|
-
if (sc) {
|
|
1076
|
-
for (let i = 0; i < sc.namedChildCount; i++) {
|
|
1077
|
-
const c = sc.namedChild(i);
|
|
1078
|
-
// skip keyword args like metaclass=… (keyword_argument node)
|
|
1079
|
-
if (c.type === 'identifier' || c.type === 'attribute') bases.add(c.text);
|
|
1080
|
-
}
|
|
1081
|
-
}
|
|
1082
|
-
}
|
|
1083
|
-
return Array.from(bases).slice(0, 12);
|
|
1084
|
-
}
|
|
1085
|
-
|
|
1086
|
-
/**
|
|
1087
|
-
* Frontier 2: Extract TypeScript/Python type annotation names from a chunk node.
|
|
1088
|
-
* Returns simple type names (e.g. ['User', 'AuthToken', 'PaymentService'])
|
|
1089
|
-
* used to enrich the inverted index and the type_refs chunk field.
|
|
1090
|
-
*/
|
|
1091
|
-
export function extractTypeAnnotations(chunkNode, ext) {
|
|
1092
|
-
const types = new Set();
|
|
1093
|
-
const JS_LIKE = ['.ts', '.tsx', '.js', '.jsx', '.mjs', '.cjs'];
|
|
1094
|
-
if (!JS_LIKE.includes(ext) && ext !== '.py') return [];
|
|
1095
|
-
|
|
1096
|
-
function walk(node) {
|
|
1097
|
-
// TypeScript: type_annotation nodes contain the type text
|
|
1098
|
-
if (node.type === 'type_annotation') {
|
|
1099
|
-
const typeText = node.text.replace(/^:\s*/, '').trim();
|
|
1100
|
-
// Extract simple identifiers from the type (skip primitives)
|
|
1101
|
-
const PRIMITIVES = new Set(['string','number','boolean','void','any','unknown','never','null','undefined','object','symbol','bigint']);
|
|
1102
|
-
for (const match of typeText.matchAll(/\b([A-Z][A-Za-z0-9]*)\b/g)) {
|
|
1103
|
-
if (!PRIMITIVES.has(match[1].toLowerCase())) types.add(match[1]);
|
|
1104
|
-
}
|
|
1105
|
-
}
|
|
1106
|
-
// TypeScript: generic_type, predefined_type (string, number…) skip, named types keep
|
|
1107
|
-
else if (node.type === 'type_identifier' || node.type === 'generic_type') {
|
|
1108
|
-
const name = node.children[0]?.text || node.text;
|
|
1109
|
-
if (name && /^[A-Z]/.test(name)) types.add(name);
|
|
1110
|
-
}
|
|
1111
|
-
// Python: type comments or annotations (annotation nodes)
|
|
1112
|
-
else if (node.type === 'annotation' && ext === '.py') {
|
|
1113
|
-
const typeText = node.text.replace(/^->\s*|^:\s*/, '').trim();
|
|
1114
|
-
for (const match of typeText.matchAll(/\b([A-Z][A-Za-z0-9]*)\b/g)) {
|
|
1115
|
-
types.add(match[1]);
|
|
1116
|
-
}
|
|
1117
|
-
}
|
|
1118
|
-
for (const child of node.children) walk(child);
|
|
1119
|
-
}
|
|
1120
|
-
walk(chunkNode);
|
|
1121
|
-
return Array.from(types).slice(0, 20);
|
|
1122
|
-
}
|
|
1123
|
-
|
|
1124
|
-
export function extractCalls(rootNode) {
|
|
1125
|
-
const calls = new Set();
|
|
1126
|
-
function walk(node) {
|
|
1127
|
-
// JavaScript / TypeScript / Go / Rust: call_expression
|
|
1128
|
-
if (node.type === 'call_expression') {
|
|
1129
|
-
const funcNode = node.childForFieldName?.('function') || node.children[0];
|
|
1130
|
-
if (funcNode) {
|
|
1131
|
-
if (funcNode.type === 'identifier') calls.add(funcNode.text);
|
|
1132
|
-
else if (funcNode.type === 'member_expression' || funcNode.type === 'property_identifier') {
|
|
1133
|
-
const prop = funcNode.childForFieldName?.('property');
|
|
1134
|
-
if (prop) calls.add(prop.text);
|
|
1135
|
-
else calls.add(funcNode.text.split('.').pop());
|
|
1136
|
-
}
|
|
1137
|
-
}
|
|
1138
|
-
}
|
|
1139
|
-
// Python: `call` node type (different name from call_expression)
|
|
1140
|
-
else if (node.type === 'call') {
|
|
1141
|
-
const funcNode = node.childForFieldName?.('function') || node.children[0];
|
|
1142
|
-
if (funcNode) {
|
|
1143
|
-
if (funcNode.type === 'identifier') calls.add(funcNode.text);
|
|
1144
|
-
else if (funcNode.type === 'attribute') {
|
|
1145
|
-
const attr = funcNode.childForFieldName?.('attribute');
|
|
1146
|
-
if (attr) calls.add(attr.text);
|
|
1147
|
-
}
|
|
1148
|
-
}
|
|
1149
|
-
}
|
|
1150
|
-
// Rust: macro_invocation (e.g. vec!, println!, format!)
|
|
1151
|
-
else if (node.type === 'macro_invocation') {
|
|
1152
|
-
const macroNode = node.childForFieldName?.('macro') || node.children[0];
|
|
1153
|
-
if (macroNode && macroNode.type === 'identifier') calls.add(macroNode.text + '!');
|
|
1154
|
-
}
|
|
1155
|
-
// Java / C#: method_invocation
|
|
1156
|
-
else if (node.type === 'method_invocation') {
|
|
1157
|
-
const nameNode = node.childForFieldName?.('name') || node.children.find(c => c.type === 'identifier');
|
|
1158
|
-
if (nameNode) calls.add(nameNode.text);
|
|
1159
|
-
}
|
|
1160
|
-
// Ruby: method_call / call
|
|
1161
|
-
else if (node.type === 'method_call') {
|
|
1162
|
-
const method = node.childForFieldName?.('method') || node.children.find(c => c.type === 'identifier');
|
|
1163
|
-
if (method && method.type === 'identifier') calls.add(method.text);
|
|
1164
|
-
}
|
|
1165
|
-
node.children.forEach(walk);
|
|
1166
|
-
}
|
|
1167
|
-
walk(rootNode);
|
|
1168
|
-
const noise = new Set(['require', 'console', 'log', 'expect', 'test', 'it', 'describe', 'setTimeout', 'print', 'println!']);
|
|
1169
|
-
return Array.from(calls).filter(c => c && c.length > 2 && !noise.has(c));
|
|
1170
|
-
}
|