graph-indexer 1.2.0 → 2.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (70) hide show
  1. package/CHANGELOG.md +854 -0
  2. package/README.md +258 -595
  3. package/SECURITY.md +85 -0
  4. package/TRADEMARK.md +31 -0
  5. package/cli-ui.mjs +78 -0
  6. package/colbert.mjs +168 -0
  7. package/config.mjs +408 -32
  8. package/daemon-ctl.mjs +175 -0
  9. package/daemon-lock.mjs +69 -0
  10. package/embedders/python/mlx_embed_server.py +61 -0
  11. package/embedders/python/requirements-mlx.txt +5 -0
  12. package/embedders/setup-mlx.mjs +110 -0
  13. package/embeddings.mjs +571 -0
  14. package/engine/binary.mjs +437 -0
  15. package/{core-engine.mjs → engine/memory.mjs} +321 -441
  16. package/{sqlite-store.mjs → engine/sqlite.mjs} +413 -45
  17. package/enrichment.mjs +206 -32
  18. package/git-signals.mjs +248 -0
  19. package/indexer.mjs +399 -45
  20. package/init.mjs +1626 -171
  21. package/layout.mjs +123 -0
  22. package/mcp/centrality.mjs +117 -0
  23. package/mcp/format.mjs +61 -0
  24. package/mcp/resolver.mjs +116 -0
  25. package/mcp/symbolgraph.mjs +137 -0
  26. package/mcp/taint.mjs +225 -0
  27. package/mcp/tools.mjs +1582 -0
  28. package/mcp/topology.mjs +676 -0
  29. package/mcp-server.mjs +80 -25
  30. package/package.json +66 -27
  31. package/parse/extractor.mjs +628 -0
  32. package/parse/imports.mjs +365 -0
  33. package/parse/interprocedural.mjs +134 -0
  34. package/parse/languages.mjs +253 -0
  35. package/parse/metadata.mjs +701 -0
  36. package/parse/routes.mjs +327 -0
  37. package/parse/scip.mjs +391 -0
  38. package/parse/taint-patterns.mjs +119 -0
  39. package/prompts/CORE.md +91 -0
  40. package/prompts/DOMAIN_TEMPLATE.md +46 -0
  41. package/prompts/INTEGRATION.md +123 -0
  42. package/prompts/frameworks/ANDROID.md +17 -0
  43. package/prompts/frameworks/ASPNET_CORE.md +19 -0
  44. package/prompts/frameworks/FASTAPI_DJANGO.md +20 -0
  45. package/prompts/frameworks/LARAVEL_SYMFONY.md +22 -0
  46. package/prompts/frameworks/NODE_EXPRESS_NESTJS.md +19 -0
  47. package/prompts/frameworks/RAILS.md +20 -0
  48. package/prompts/frameworks/REACT.md +20 -0
  49. package/prompts/frameworks/SPRING_BOOT.md +20 -0
  50. package/prompts/languages/BASH.md +22 -0
  51. package/prompts/languages/C.md +21 -0
  52. package/prompts/languages/CSHARP.md +24 -0
  53. package/prompts/languages/CSS_SCSS.md +22 -0
  54. package/prompts/languages/GO.md +24 -0
  55. package/prompts/languages/JAVA.md +23 -0
  56. package/prompts/languages/JAVASCRIPT_TYPESCRIPT.md +24 -0
  57. package/prompts/languages/KOTLIN.md +24 -0
  58. package/prompts/languages/PHP.md +23 -0
  59. package/prompts/languages/PYTHON.md +23 -0
  60. package/prompts/languages/RUBY.md +23 -0
  61. package/prompts/languages/RUST.md +21 -0
  62. package/prompts/languages/SWIFT.md +21 -0
  63. package/seal.mjs +400 -0
  64. package/search-core.mjs +405 -44
  65. package/search-sparse.mjs +183 -0
  66. package/storage.mjs +33 -3
  67. package/watch-daemon.mjs +75 -24
  68. package/PROMPT.md +0 -182
  69. package/mcp-tools.mjs +0 -545
  70. package/parser-utils.mjs +0 -1170
package/parser-utils.mjs DELETED
@@ -1,1170 +0,0 @@
1
- /**
2
- * @file parser-utils.mjs
3
- * @description Shared AST parsing, language registry, and embedding utilities.
4
- * @author MaquinaTech <https://github.com/MaquinaTech>
5
- * @copyright (c) 2026 MaquinaTech. All rights reserved.
6
- * @license MIT
7
- * Copyright (c) 2026 MaquinaTech. All rights reserved.
8
- * Permission is hereby granted, free of charge, to any person obtaining a copy
9
- * of this software and associated documentation files (the "Software"), to deal
10
- * in the Software without restriction, including without limitation the rights
11
- * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
12
- * copies of the Software, and to permit persons to whom the Software is
13
- * furnished to do so, subject to the following conditions: The above copyright
14
- * notice and this permission notice shall be included in all copies or
15
- * substantial portions of the Software.
16
- */
17
- import fs from 'fs';
18
- import path from 'path';
19
- import { createHash } from 'crypto';
20
- import ignore from 'ignore';
21
- import Parser from 'tree-sitter';
22
- const { Query } = Parser; // Native parser query helper
23
-
24
- import { truncateForEmbedding } from './core-engine.mjs';
25
-
26
- export const MAX_FILE_SIZE_BYTES = 500000;
27
- export const OLLAMA_HOST = process.env.OLLAMA_HOST || "http://localhost:11434";
28
-
29
- // Resolves the Ollama host at call time so callers that set ollamaHost via
30
- // .graph-indexer.json don't have to pass it through every call chain.
31
- // Priority: caller override → OLLAMA_HOST env var → PROJECT .graph-indexer.json
32
- // (MCP_PROJECT_ROOT or cwd — NOT the package directory: when graph-indexer is
33
- // installed as a dependency, the user's config lives in their project root) →
34
- // default. Mirrors config.mjs precedence so every entry point agrees.
35
- //
36
- // Note: OLLAMA_HOST in the shell is Ollama's binding address (e.g. "0.0.0.0:11435"), not an
37
- // HTTP client URL. We normalise bare "host:port" strings by adding http:// and translating
38
- // 0.0.0.0 → localhost so fetches work in both formats.
39
- let _cachedHost = null;
40
- let _cachedEmbedModel = null;
41
- function _normalizeOllamaHost(raw) {
42
- if (!raw) return null;
43
- if (raw.startsWith('http://') || raw.startsWith('https://')) return raw;
44
- return 'http://' + raw.replace(/^0\.0\.0\.0/, 'localhost');
45
- }
46
- function _readProjectConfig() {
47
- const root = process.env.MCP_PROJECT_ROOT || process.cwd();
48
- try {
49
- return JSON.parse(fs.readFileSync(path.join(root, '.graph-indexer.json'), 'utf8'));
50
- } catch { return null; }
51
- }
52
- function _resolveOllamaHost(override) {
53
- if (override) return _normalizeOllamaHost(override) || 'http://localhost:11434';
54
- if (_cachedHost) return _cachedHost;
55
- if (process.env.OLLAMA_HOST) {
56
- _cachedHost = _normalizeOllamaHost(process.env.OLLAMA_HOST) || 'http://localhost:11434';
57
- return _cachedHost;
58
- }
59
- const cfg = _readProjectConfig();
60
- _cachedHost = _normalizeOllamaHost(cfg?.ollamaHost) || 'http://localhost:11434';
61
- return _cachedHost;
62
- }
63
- function _resolveEmbedModel(override) {
64
- if (override) return override;
65
- if (_cachedEmbedModel) return _cachedEmbedModel;
66
- const cfg = _readProjectConfig();
67
- _cachedEmbedModel = cfg?.embedModel || 'nomic-embed-text';
68
- return _cachedEmbedModel;
69
- }
70
-
71
- // ─── Dynamic Language Loading ─────────────────────────────────────────────────
72
-
73
- function _loadProjectConfig() {
74
- const configPath = path.join(process.env.MCP_PROJECT_ROOT || process.cwd(), '.graph-indexer.json');
75
- try {
76
- if (fs.existsSync(configPath)) return JSON.parse(fs.readFileSync(configPath, 'utf-8'));
77
- } catch { /* ignore */ }
78
- return null;
79
- }
80
-
81
- async function _tryLang(pkg, enabledLangs, key) {
82
- if (enabledLangs && !enabledLangs.includes(key)) return null;
83
- try {
84
- return (await import(pkg)).default;
85
- } catch {
86
- process.stderr.write(`[graph-indexer] WARNING: ${pkg} not installed — ${key} files will be skipped\n`);
87
- return null;
88
- }
89
- }
90
-
91
- const _cfg = _loadProjectConfig();
92
- const _enabled = _cfg?.languages ?? null; // null = all languages
93
-
94
- const [
95
- TypeScript, JavaScript, CSS, Python, Rust,
96
- Go, PHP, Java, Kotlin, CSharp, Ruby
97
- ] = await Promise.all([
98
- _tryLang('tree-sitter-typescript', _enabled, 'typescript'),
99
- _tryLang('tree-sitter-javascript', _enabled, 'javascript'),
100
- _tryLang('tree-sitter-css', _enabled, 'css'),
101
- _tryLang('tree-sitter-python', _enabled, 'python'),
102
- _tryLang('tree-sitter-rust', _enabled, 'rust'),
103
- _tryLang('tree-sitter-go', _enabled, 'go'),
104
- _tryLang('tree-sitter-php', _enabled, 'php'),
105
- _tryLang('tree-sitter-java', _enabled, 'java'),
106
- _tryLang('tree-sitter-kotlin', _enabled, 'kotlin'),
107
- _tryLang('tree-sitter-c-sharp', _enabled, 'csharp'),
108
- _tryLang('tree-sitter-ruby', _enabled, 'ruby'),
109
- ]);
110
-
111
- const LANGUAGE_MAP = {
112
- ...(TypeScript ? { '.ts': TypeScript.typescript, '.tsx': TypeScript.tsx } : {}),
113
- ...(JavaScript ? { '.js': JavaScript, '.jsx': JavaScript, '.mjs': JavaScript, '.cjs': JavaScript } : {}),
114
- ...(CSS ? { '.css': CSS, '.scss': CSS } : {}),
115
- ...(Python ? { '.py': Python } : {}),
116
- ...(Rust ? { '.rs': Rust } : {}),
117
- ...(Go ? { '.go': Go } : {}),
118
- ...(PHP ? { '.php': PHP.php } : {}),
119
- ...(Java ? { '.java': Java } : {}),
120
- ...(Kotlin ? { '.kt': Kotlin, '.kts': Kotlin } : {}),
121
- ...(CSharp ? { '.cs': CSharp } : {}),
122
- ...(Ruby ? { '.rb': Ruby } : {}),
123
- };
124
-
125
- // 🥇 ULTRA-GENERIC QUERIES: Immune to grammar changes between TS/JS/TSX
126
- const LANGUAGE_QUERIES = {
127
- ts: `
128
- (class_declaration) @chunk
129
- (function_declaration) @chunk
130
- (method_definition) @chunk
131
- (lexical_declaration) @chunk
132
- (expression_statement) @chunk
133
- (export_statement) @chunk
134
- `,
135
- css: `(rule_set) @chunk`,
136
- py: `
137
- (function_definition) @chunk
138
- (class_definition) @chunk
139
- (expression_statement) @chunk
140
- `,
141
- rs: `
142
- (function_item) @chunk
143
- (struct_item) @chunk
144
- (enum_item) @chunk
145
- (trait_item) @chunk
146
- (impl_item) @chunk
147
- `,
148
- go: `
149
- (function_declaration) @chunk
150
- (method_declaration) @chunk
151
- (type_declaration) @chunk
152
- `,
153
- php: `
154
- (function_definition) @chunk
155
- (class_declaration) @chunk
156
- (expression_statement) @chunk
157
- `,
158
- java: `
159
- (method_declaration) @chunk
160
- (class_declaration) @chunk
161
- (interface_declaration) @chunk
162
- (constructor_declaration) @chunk
163
- (enum_declaration) @chunk
164
- `,
165
- kotlin: `
166
- (function_declaration) @chunk
167
- (class_declaration) @chunk
168
- (object_declaration) @chunk
169
- (companion_object) @chunk
170
- (secondary_constructor) @chunk
171
- `,
172
- cs: `
173
- (method_declaration) @chunk
174
- (class_declaration) @chunk
175
- (interface_declaration) @chunk
176
- (constructor_declaration) @chunk
177
- (enum_declaration) @chunk
178
- (property_declaration) @chunk
179
- `,
180
- rb: `
181
- (method) @chunk
182
- (singleton_method) @chunk
183
- (class) @chunk
184
- (module) @chunk
185
- `
186
- };
187
-
188
- export const EXTENSIONS = new Set(Object.keys(LANGUAGE_MAP));
189
-
190
- // Definition of container node types to avoid nested duplicates
191
- const CONTAINERS = new Set([
192
- 'class_declaration', 'function_declaration', 'method_definition',
193
- 'lexical_declaration', 'expression_statement', 'export_statement',
194
- 'function_definition', 'class_definition', 'rule_set',
195
- 'function_item', 'struct_item', 'enum_item', 'trait_item', 'impl_item',
196
- 'method_declaration', 'type_declaration',
197
- // Java / C#
198
- 'interface_declaration', 'constructor_declaration', 'enum_declaration',
199
- // Kotlin
200
- 'object_declaration', 'companion_object', 'secondary_constructor',
201
- // C#
202
- 'property_declaration',
203
- // Ruby
204
- 'method', 'singleton_method', 'module'
205
- ]);
206
-
207
- export function getParserForFile(ext) {
208
- const language = LANGUAGE_MAP[ext];
209
- if (!language) return null;
210
- const parser = new Parser();
211
- parser.setLanguage(language);
212
- return parser;
213
- }
214
-
215
- export function buildIgnoreFilter(rootPath) {
216
- const ig = ignore();
217
- ig.add(['node_modules', '.git', 'dist', 'build', '.next', 'coverage', '*.tmp', 'vendor', '.venv']);
218
- const gitignorePath = path.join(rootPath, '.gitignore');
219
- if (fs.existsSync(gitignorePath)) ig.add(fs.readFileSync(gitignorePath, 'utf-8'));
220
- return ig;
221
- }
222
-
223
- export function generateChunkHash(text) {
224
- return createHash('sha256').update(text).digest('hex');
225
- }
226
-
227
- export function extractFileSkeleton(rootNode, content) {
228
- const signatures = [];
229
- function walk(node) {
230
- if (node.type.includes('declaration') || node.type.includes('definition')) {
231
- let name = node.childForFieldName?.("name")?.text || "anonymous";
232
- signatures.push(`- [${node.type}] ${name} (lines ${node.startPosition.row + 1}-${node.endPosition.row + 1})`);
233
- }
234
- node.children.forEach(walk);
235
- }
236
- walk(rootNode);
237
- return signatures.join('\n');
238
- }
239
-
240
- export function extractImportsFromAST(rootNode, ext) {
241
- const imports = new Set();
242
- function walk(node) {
243
- // ── JavaScript / TypeScript ──────────────────────────────────────────
244
- if (['.ts', '.tsx', '.js', '.jsx', '.mjs', '.cjs'].includes(ext)) {
245
- if (node.type === 'import_statement') {
246
- const source = node.children.find(c => c.type === 'string');
247
- if (source) imports.add(source.text.replace(/['"]/g, ''));
248
- } else if (node.type === 'call_expression' && node.children[0]?.text === 'require') {
249
- const arg = node.children[1]?.children?.find(c => c.type === 'string');
250
- if (arg) imports.add(arg.text.replace(/['"]/g, ''));
251
- }
252
- }
253
- // ── Python ──────────────────────────────────────────────────────────
254
- else if (ext === '.py') {
255
- if (node.type === 'import_statement') {
256
- // import foo.bar.baz → store as foo/bar/baz
257
- for (const child of node.children) {
258
- if (child.type === 'dotted_name') imports.add(child.text.replace(/\./g, '/'));
259
- }
260
- } else if (node.type === 'import_from_statement') {
261
- // from .sibling import X → relative: emit as ./sibling
262
- // from ..pkg import Y → relative: emit as ../pkg
263
- // from foo.bar import Z → absolute: store foo/bar
264
- const relNode = node.children.find(c => c.type === 'relative_import');
265
- if (relNode) {
266
- // dots count = depth; remainder is the module path after dots
267
- const raw = relNode.text; // e.g. '.' or '..utils'
268
- const dots = raw.match(/^\.+/)?.[0] ?? '.';
269
- const mod = raw.slice(dots.length);
270
- const prefix = dots.length === 1 ? './' : '../'.repeat(dots.length - 1);
271
- imports.add(mod ? prefix + mod.replace(/\./g, '/') : prefix.slice(0, -1) || '.');
272
- } else {
273
- const mod = node.children.find(c => c.type === 'dotted_name');
274
- if (mod) imports.add(mod.text.replace(/\./g, '/'));
275
- }
276
- }
277
- }
278
- // ── Rust ────────────────────────────────────────────────────────────
279
- else if (ext === '.rs') {
280
- if (node.type === 'use_declaration') {
281
- // Collect the first path text from the argument subtree
282
- const arg = node.childForFieldName?.('argument') ||
283
- node.children.find(c => !['use', ';', 'pub'].includes(c.type));
284
- if (arg) imports.add(arg.text.split('::').slice(0, 3).join('::'));
285
- }
286
- }
287
- // ── Go ──────────────────────────────────────────────────────────────
288
- else if (ext === '.go') {
289
- if (node.type === 'import_spec') {
290
- const pathNode = node.children.find(c =>
291
- c.type === 'interpreted_string_literal' || c.type === 'raw_string_literal');
292
- if (pathNode) imports.add(pathNode.text.replace(/['"`]/g, ''));
293
- }
294
- }
295
- // ── PHP ─────────────────────────────────────────────────────────────
296
- else if (ext === '.php') {
297
- if (node.type === 'include_expression' || node.type === 'require_expression' ||
298
- node.type === 'include_once_expression' || node.type === 'require_once_expression') {
299
- const strNode = node.children.find(c =>
300
- c.type === 'string' || c.type === 'encapsed_string');
301
- if (strNode) imports.add(strNode.text.replace(/['"]/g, ''));
302
- }
303
- } // ── Java ─────────────────────────────────────────────────────────────
304
- else if (ext === '.java') {
305
- if (node.type === 'import_declaration') {
306
- const scopedId = node.children.find(c => c.type === 'scoped_identifier' || c.type === 'identifier');
307
- if (scopedId) imports.add(scopedId.text.replace(/\./g, '/'));
308
- }
309
- }
310
- // ── Kotlin ───────────────────────────────────────────────────────────
311
- else if (ext === '.kt' || ext === '.kts') {
312
- if (node.type === 'import_header') {
313
- const path = node.children.find(c => c.type === 'identifier' || c.type === 'user_type' || c.isNamed);
314
- const raw = node.text.replace(/^import\s+/, '').replace(/\s*\.\*\s*$/, '').trim();
315
- if (raw) imports.add(raw.replace(/\./g, '/'));
316
- }
317
- }
318
- // ── C# ───────────────────────────────────────────────────────────────
319
- else if (ext === '.cs') {
320
- if (node.type === 'using_directive') {
321
- const ns = node.children.find(c => c.type === 'qualified_name' || c.type === 'identifier' || c.type === 'name_equals');
322
- if (ns) {
323
- const raw = ns.text.replace(/\s*=\s*.*$/, '').trim();
324
- if (raw) imports.add(raw.replace(/\./g, '/'));
325
- }
326
- }
327
- }
328
- // ── Ruby ──────────────────────────────────────────────────────────────
329
- else if (ext === '.rb') {
330
- if (node.type === 'call' || node.type === 'method_call') {
331
- const method = node.childForFieldName?.('method') || node.children[0];
332
- if (method && (method.text === 'require' || method.text === 'require_relative')) {
333
- const args = node.childForFieldName?.('arguments') || node.children.find(c => c.type === 'argument_list');
334
- const strArg = args?.children?.find(c => c.type === 'string' || c.type === 'simple_string');
335
- if (strArg) imports.add(strArg.text.replace(/['"]/g, ''));
336
- }
337
- }
338
- }
339
- node.children.forEach(walk);
340
- }
341
- walk(rootNode);
342
- return Array.from(imports);
343
- }
344
-
345
- // ─── God-class defence ────────────────────────────────────────────────────────
346
-
347
- /**
348
- * Token-safe skeleton for a class that exceeds GOD_CLASS_LINES.
349
- *
350
- * The agent's PROMPT.md promises get_chunk() costs ~300 tokens. A 2 000-line
351
- * "god class" stored as one chunk violates that contract. This skeleton keeps
352
- * only the first HEADER_LINES of the class (signature + opening brace) and
353
- * appends a one-line summary — enough for name resolution and embeddings, while
354
- * the real bodies are reachable via the individual method chunks that the
355
- * god-class split produces.
356
- */
357
- function buildGodClassSkeleton(classNode) {
358
- const allLines = classNode.text.split('\n');
359
- const HEADER_LINES = 15;
360
- const header = allLines.slice(0, HEADER_LINES).join('\n');
361
- return (
362
- `${header}\n` +
363
- ` // ⚠ [Large class: ${allLines.length} lines — ` +
364
- `methods are indexed as individual searchable chunks. ` +
365
- `Use search_code() to find specific methods.]\n}`
366
- );
367
- }
368
-
369
- export function extractSemanticChunks(rootNode, relPath, sourceCode, ext) {
370
- const chunks = [];
371
- const parser = getParserForFile(ext);
372
- if (!parser) return chunks;
373
-
374
- const JS_LIKE = ['.ts', '.tsx', '.js', '.jsx', '.mjs', '.cjs'];
375
- const EXT_TO_LANG = {
376
- '.java': 'java', '.kt': 'kotlin', '.kts': 'kotlin', '.cs': 'cs',
377
- '.rb': 'rb'
378
- };
379
- const langKey = JS_LIKE.includes(ext) ? 'ts'
380
- : (EXT_TO_LANG[ext] || (LANGUAGE_QUERIES[ext.slice(1)] ? ext.slice(1) : null));
381
- if (!langKey || !LANGUAGE_QUERIES[langKey]) return chunks;
382
-
383
- // 🥇 HEADER INHERITANCE: Extract top-of-file/global comments for module context
384
- let fileDocstring = "";
385
- let cursorNode = rootNode.children[0];
386
- while (cursorNode) {
387
- if (cursorNode.type === 'comment') fileDocstring += cursorNode.text + "\n";
388
- else if (cursorNode.type !== 'import_statement' && cursorNode.type !== 'expression_statement') break;
389
- cursorNode = cursorNode.nextSibling;
390
- }
391
- fileDocstring = fileDocstring.trim();
392
-
393
- try {
394
- const query = new Query(parser.getLanguage(), LANGUAGE_QUERIES[langKey]);
395
- const matches = query.matches(rootNode);
396
- const processedNodes = new Set();
397
-
398
- // ── God-class pre-pass ─────────────────────────────────────────────────
399
- // Identify class-type container nodes whose line count exceeds the threshold.
400
- // Their methods are allowed through the isNested filter below so each method
401
- // becomes its own independent, searchable chunk; the class node itself gets a
402
- // compact skeleton instead of a truncated body dump.
403
- //
404
- // Cross-language: class_declaration (TS/JS/Java/C#), class_definition (Python),
405
- // impl_item (Rust impl blocks), class (Ruby), object_declaration (Kotlin).
406
- // TS special case: exported classes live inside export_statement — we mark the
407
- // inner class_declaration so the method-isNested check works correctly.
408
- const GOD_CLASS_LINES = 200;
409
- const GOD_CLASS_NODE_TYPES = new Set([
410
- 'class_declaration', 'class_definition',
411
- 'impl_item', // Rust impl blocks
412
- 'class', // Ruby
413
- 'object_declaration', // Kotlin
414
- ]);
415
- const oversizedClassIds = new Set(); // node IDs whose direct methods are un-nested
416
-
417
- for (const match of matches) {
418
- for (const capture of match.captures) {
419
- if (capture.name !== 'chunk') continue;
420
- const n = capture.node;
421
- const nLines = n.endPosition.row - n.startPosition.row + 1;
422
- if (nLines <= GOD_CLASS_LINES) continue;
423
- if (GOD_CLASS_NODE_TYPES.has(n.type)) {
424
- oversizedClassIds.add(n.id);
425
- }
426
- // TS: large export_statement wrapping a class — mark the inner class_declaration
427
- // so method_definition.parent chain finds the oversized class node, not the
428
- // export_statement (which isn't what the isNested walk stops at).
429
- if (n.type === 'export_statement') {
430
- for (let ci = 0; ci < n.namedChildCount; ci++) {
431
- const c = n.namedChild(ci);
432
- if (GOD_CLASS_NODE_TYPES.has(c.type)) {
433
- oversizedClassIds.add(c.id);
434
- break;
435
- }
436
- }
437
- }
438
- }
439
- }
440
-
441
- for (const match of matches) {
442
- let chunkNode = null;
443
- for (const capture of match.captures) {
444
- if (capture.name === 'chunk') chunkNode = capture.node;
445
- }
446
-
447
- if (!chunkNode || processedNodes.has(chunkNode.id)) continue;
448
- processedNodes.add(chunkNode.id);
449
-
450
- // Filter out very small fragments (simple variables, etc.)
451
- if (chunkNode.endPosition.row - chunkNode.startPosition.row < 2) continue;
452
-
453
- // 🥇 DEDUPLICATION LOGIC: Ignore nodes that are nested inside other container nodes.
454
- // Stop at the actual tree root (parent === null) to be language-agnostic:
455
- // Python root = 'module', JS root = 'program', Go root = 'source_file', Ruby root = 'program'.
456
- // Stopping at 'program' alone was falsely marking top-level Python classes as nested
457
- // because Ruby's 'module' keyword (also in CONTAINERS) shares the name with Python's root.
458
- //
459
- // God-class exception: if the first CONTAINERS ancestor is an oversized class, the
460
- // node is a direct method of that class — allow it through as its own chunk.
461
- let isNested = false;
462
- let currentParent = chunkNode.parent;
463
- while (currentParent && currentParent.parent !== null) {
464
- if (CONTAINERS.has(currentParent.type)) {
465
- if (!oversizedClassIds.has(currentParent.id)) {
466
- isNested = true;
467
- }
468
- break;
469
- }
470
- currentParent = currentParent.parent;
471
- }
472
- if (isNested) continue;
473
-
474
- // 🥇 ROBUST NAME EXTRACTION (JS logic)
475
- let nameText = "anonymous";
476
- const nameNode = chunkNode.childForFieldName?.("name");
477
-
478
- if (nameNode) {
479
- nameText = nameNode.text;
480
- } else if (chunkNode.type === "export_statement") {
481
- const decl = chunkNode.children.find(c => ['lexical_declaration', 'function_declaration', 'class_declaration'].includes(c.type));
482
- if (decl) {
483
- if (decl.type === "lexical_declaration") {
484
- const varDecl = decl.children.find(c => c.type === "variable_declarator");
485
- nameText = varDecl?.children.find(c => c.type === "identifier")?.text || "anonymous";
486
- } else {
487
- nameText = decl.childForFieldName?.("name")?.text || "anonymous";
488
- }
489
- } else {
490
- const defaultChild = chunkNode.children.find(c => c.type === 'identifier' || c.type === 'call_expression');
491
- if (defaultChild) {
492
- if (defaultChild.type === 'identifier') {
493
- nameText = `default_${defaultChild.text}`;
494
- } else if (defaultChild.type === 'call_expression') {
495
- const funcName = defaultChild.childForFieldName?.("function")?.text || defaultChild.children[0]?.text;
496
- const argNode = defaultChild.childForFieldName?.("arguments")?.children?.find(c => c.type === 'identifier');
497
- nameText = argNode ? `default_${funcName}_${argNode.text}` : `default_${funcName}`;
498
- }
499
- } else {
500
- nameText = "default_export";
501
- }
502
- }
503
- } else if (chunkNode.type === "lexical_declaration") {
504
- const decl = chunkNode.children.find(c => c.type === "variable_declarator");
505
- nameText = decl?.children.find(c => c.type === "identifier")?.text || "anonymous";
506
- } else if (chunkNode.type === "expression_statement") {
507
- const callExp = chunkNode.children.find(c => c.type === "call_expression");
508
- const assignExp = chunkNode.children.find(c => c.type === "assignment_expression");
509
- if (callExp) {
510
- const funcName = callExp.childForFieldName?.("function")?.text || callExp.children[0]?.text;
511
- const argsNode = callExp.childForFieldName?.("arguments");
512
- const stringArg = argsNode?.children?.find(c => c.type === "string" || c.type === "template_string");
513
- nameText = stringArg ? `${funcName}_${stringArg.text.replace(/['"`]/g, '')}` : (funcName || "anonymous");
514
- } else if (assignExp) {
515
- nameText = assignExp.childForFieldName?.("left")?.text || "anonymous";
516
- }
517
- } else if (chunkNode.type === "type_declaration") {
518
- // Go: `type X struct {…}` / `type X interface {…}` / `type X = Y`.
519
- // The identifier lives on the nested type_spec / type_alias node, not on
520
- // the type_declaration itself, so childForFieldName("name") above is null.
521
- // Without this branch every Go struct/interface collapses to the useless
522
- // synthetic name `<file>_type_declaration`, making core types (e.g. Gin's
523
- // RouterGroup, Engine, Context) unsearchable by name and invisible to the
524
- // 2.0× name-boost in searchHybrid.
525
- const spec = chunkNode.namedChildren?.find(c => c.type === "type_spec" || c.type === "type_alias");
526
- nameText = spec?.childForFieldName?.("name")?.text
527
- || spec?.children?.find(c => c.type === "type_identifier")?.text
528
- || "anonymous";
529
- } else {
530
- // Generic fallback: search direct children for an identifier-like node.
531
- // Includes type_identifier (Go/TS), constant (Ruby class/module names) and
532
- // field_identifier so nested-name grammars don't fall through to anonymous.
533
- const idNode = chunkNode.children.find(c =>
534
- c.type === "identifier" || c.type === "name" || c.type === "property_identifier"
535
- || c.type === "type_identifier" || c.type === "constant" || c.type === "field_identifier");
536
- nameText = idNode?.text || "anonymous";
537
- }
538
-
539
- if (nameText === "anonymous" || nameText === "default_export") {
540
- nameText = `${path.basename(relPath, path.extname(relPath))}_${chunkNode.type}`;
541
- }
542
-
543
- let docstring = "";
544
- let prev = chunkNode.previousSibling;
545
- while (prev && (prev.type === 'comment' || !prev.isNamed)) {
546
- if (prev.type === 'comment') docstring = prev.text + "\n" + docstring;
547
- prev = prev.previousSibling;
548
- }
549
- docstring = docstring.trim();
550
-
551
- if (!docstring && fileDocstring) {
552
- docstring = `[File Context]: ${fileDocstring}`;
553
- }
554
-
555
- // For oversized class containers — or TS export_statements wrapping one —
556
- // store a compact skeleton so get_chunk() returns a bounded token count.
557
- // All metadata fields (calls, params, type_refs, decorators, extends) are
558
- // still extracted from the FULL AST node, so ranking and topology are intact.
559
- const isOversizedClass = oversizedClassIds.has(chunkNode.id);
560
- const wrapsOversizedClass = !isOversizedClass && chunkNode.type === 'export_statement' && (() => {
561
- for (let ci = 0; ci < chunkNode.namedChildCount; ci++) {
562
- if (oversizedClassIds.has(chunkNode.namedChild(ci).id)) return true;
563
- }
564
- return false;
565
- })();
566
- const snippet = (isOversizedClass || wrapsOversizedClass)
567
- ? buildGodClassSkeleton(isOversizedClass ? chunkNode : (() => {
568
- for (let ci = 0; ci < chunkNode.namedChildCount; ci++) {
569
- const c = chunkNode.namedChild(ci);
570
- if (oversizedClassIds.has(c.id)) return c;
571
- }
572
- return chunkNode;
573
- })())
574
- : chunkNode.text.slice(0, 3000);
575
- const hash = generateChunkHash(docstring + snippet);
576
- const outgoingCalls = extractCalls(chunkNode);
577
-
578
- // 🥇 PARAMETER / TYPE / CLASS CONTEXT ENRICHMENT (improves recall on undocumented code)
579
- const params = extractParams(chunkNode, ext);
580
- const returnType = extractReturnType(chunkNode, ext);
581
- const classContext = extractClassContext(chunkNode);
582
- const typeRefs = extractTypeAnnotations(chunkNode, ext);
583
- const decorators = extractDecorators(chunkNode);
584
- const heritage = extractHeritage(chunkNode, ext);
585
-
586
- const id = createHash('sha256')
587
- .update(`${relPath}::${chunkNode.startPosition.row}::${chunkNode.startPosition.column}`)
588
- .digest('hex').slice(0, 24);
589
-
590
- chunks.push({
591
- id, file_path: relPath, node_type: chunkNode.type, name: nameText,
592
- docstring: docstring, code_snippet: snippet, content_hash: hash,
593
- start_line: chunkNode.startPosition.row + 1, end_line: chunkNode.endPosition.row + 1,
594
- calls: outgoingCalls,
595
- params, return_type: returnType, class_context: classContext,
596
- type_refs: typeRefs, decorators, extends: heritage,
597
- });
598
- }
599
- } catch (e) {
600
- // Visible protective log for developers
601
- process.stderr.write(`\n[parser-utils] 💥 Query Error in ${relPath}: ${e.message}\n`);
602
- }
603
-
604
- // ── Python public re-exports ────────────────────────────────────────────────
605
- // PEP 484 convention: `from starlette.background import BackgroundTasks as
606
- // BackgroundTasks` re-exports a symbol as public API. Files like
607
- // fastapi/background.py consist ONLY of such lines and previously produced
608
- // zero chunks — resolve_symbol('BackgroundTasks') found nothing and agents
609
- // hit a dead end. Each explicit re-export becomes a small chunk that names
610
- // the symbol and points at its source module. (JS/TS barrels already chunk
611
- // via the export_statement capture.)
612
- if (ext === '.py') {
613
- for (const node of rootNode.children) {
614
- if (node.type !== 'import_from_statement') continue;
615
- const moduleName = node.childForFieldName?.('module_name')?.text || '';
616
- for (const child of node.children) {
617
- if (child.type !== 'aliased_import') continue;
618
- const orig = child.childForFieldName?.('name')?.text || '';
619
- const alias = child.childForFieldName?.('alias')?.text || '';
620
- if (!alias || orig.split('.').pop() !== alias) continue; // only `import X as X`
621
- const snippet = node.text.slice(0, 300);
622
- const docstring = `Public re-export of ${alias} from ${moduleName}.`;
623
- const id = createHash('sha256')
624
- .update(`${relPath}::${node.startPosition.row}::${child.startPosition.column}`)
625
- .digest('hex').slice(0, 24);
626
- chunks.push({
627
- id, file_path: relPath, node_type: 're_export', name: alias,
628
- docstring, code_snippet: snippet,
629
- content_hash: generateChunkHash(docstring + snippet),
630
- start_line: node.startPosition.row + 1, end_line: node.endPosition.row + 1,
631
- calls: [], params: [], return_type: '', class_context: '',
632
- type_refs: [], decorators: [], extends: [],
633
- });
634
- }
635
- }
636
- }
637
- return chunks;
638
- }
639
-
640
- // ─── Barrel export resolution ─────────────────────────────────────────────────
641
-
642
- // Module-level cache: barrelAbsPath → Map<exportedName, sourceRelPath>
643
- const _barrelCache = new Map();
644
-
645
- /**
646
- * Parses a barrel file (index.ts / index.js) and returns a map of
647
- * exportedName → sourceFilePath (relative to projectRoot).
648
- * e.g. { useAuthStore: 'src/stores/authStore.ts' }
649
- */
650
- export function resolveBarrelExports(barrelAbsPath, projectRoot) {
651
- if (_barrelCache.has(barrelAbsPath)) return _barrelCache.get(barrelAbsPath);
652
-
653
- const result = new Map();
654
- _barrelCache.set(barrelAbsPath, result);
655
-
656
- const ext = path.extname(barrelAbsPath);
657
- if (!['.ts', '.tsx', '.js', '.jsx', '.mjs', '.cjs'].includes(ext)) return result;
658
-
659
- let content;
660
- try { content = fs.readFileSync(barrelAbsPath, 'utf-8'); } catch { return result; }
661
-
662
- const parser = getParserForFile(ext);
663
- if (!parser) return result;
664
-
665
- let tree;
666
- try { tree = parser.parse((offset) => offset < content.length ? content.slice(offset, offset + 4096) : null); } catch { return result; }
667
-
668
- const barrelDir = path.dirname(barrelAbsPath);
669
-
670
- function walk(node) {
671
- // export { X, Y as Z } from './source'
672
- if (node.type === 'export_statement') {
673
- const fromNode = node.children.find(c => c.type === 'string');
674
- if (!fromNode) { node.children.forEach(walk); return; }
675
-
676
- const rawSource = fromNode.text.replace(/['"]/g, '');
677
- if (!rawSource.startsWith('.')) { node.children.forEach(walk); return; }
678
-
679
- // Resolve the source file
680
- const absSource = path.resolve(barrelDir, rawSource);
681
- let finalAbs = null;
682
- if (EXTENSIONS.has(path.extname(absSource)) && fs.existsSync(absSource)) {
683
- finalAbs = absSource;
684
- } else {
685
- for (const e of EXTENSIONS) {
686
- if (fs.existsSync(absSource + e)) { finalAbs = absSource + e; break; }
687
- const idx = path.join(absSource, 'index' + e);
688
- if (fs.existsSync(idx)) { finalAbs = idx; break; }
689
- }
690
- }
691
- if (!finalAbs) { node.children.forEach(walk); return; }
692
-
693
- const relSource = path.relative(projectRoot, finalAbs).replace(/\\/g, '/');
694
-
695
- // Walk named exports
696
- const namedExports = node.children.find(c => c.type === 'named_imports' || c.type === 'export_clause');
697
- if (namedExports) {
698
- for (const child of namedExports.children) {
699
- if (child.type === 'import_specifier' || child.type === 'export_specifier') {
700
- // `alias as exported` or just `name`
701
- const names = child.children.filter(c => c.type === 'identifier');
702
- if (names.length > 0) {
703
- // The exported name is the last identifier (the alias if present)
704
- result.set(names[names.length - 1].text, relSource);
705
- }
706
- }
707
- }
708
- }
709
-
710
- // export * from './source' → map the source file itself
711
- const starNode = node.children.find(c => c.text === '*');
712
- if (starNode) {
713
- result.set('*', relSource);
714
- }
715
- }
716
- node.children.forEach(walk);
717
- }
718
- walk(tree.rootNode);
719
- return result;
720
- }
721
-
722
- // ─── Go module-name cache (reads go.mod once per project root) ──────────────
723
- const _goModCache = new Map();
724
- function _readGoModuleName(projectRoot) {
725
- if (_goModCache.has(projectRoot)) return _goModCache.get(projectRoot);
726
- const modFile = path.join(projectRoot, 'go.mod');
727
- let name = null;
728
- if (fs.existsSync(modFile)) {
729
- try {
730
- const first = fs.readFileSync(modFile, 'utf-8').split('\n').find(l => l.trimStart().startsWith('module '));
731
- if (first) name = first.trim().replace(/^module\s+/, '').split(/\s/)[0];
732
- } catch { /* ignore */ }
733
- }
734
- _goModCache.set(projectRoot, name);
735
- return name;
736
- }
737
-
738
- export function resolveLocalImports(rawImports, fromFileRelPath, projectRoot) {
739
- const fileDir = path.dirname(path.join(projectRoot, fromFileRelPath));
740
- const ext = path.extname(fromFileRelPath);
741
- const resolved = [];
742
- for (const raw of rawImports) {
743
- // ── Dot-relative (JS/TS/Python relative) ────────────────────────────
744
- if (raw.startsWith('.')) {
745
- const absResolved = path.resolve(fileDir, raw);
746
- const existingExt = path.extname(absResolved);
747
- let finalAbs = null;
748
- if (existingExt && EXTENSIONS.has(existingExt) && fs.existsSync(absResolved)) {
749
- finalAbs = absResolved;
750
- } else {
751
- for (const e of EXTENSIONS) {
752
- if (fs.existsSync(absResolved + e)) { finalAbs = absResolved + e; break; }
753
- const idx = path.join(absResolved, 'index' + e);
754
- if (fs.existsSync(idx)) { finalAbs = idx; break; }
755
- // Python: also try __init__.py for package directories
756
- if (e === '.py') {
757
- const init = path.join(absResolved, '__init__.py');
758
- if (fs.existsSync(init)) { finalAbs = init; break; }
759
- }
760
- }
761
- }
762
- if (finalAbs) {
763
- const relPath = path.relative(projectRoot, finalAbs).replace(/\\/g, '/');
764
- const baseName = path.basename(finalAbs, path.extname(finalAbs));
765
- // 🥇 BARREL RESOLUTION: If the resolved file is an index file, expand barrel exports
766
- if (baseName === 'index' && ['.ts', '.tsx', '.js', '.jsx', '.mjs', '.cjs'].includes(path.extname(finalAbs))) {
767
- const barrelMap = resolveBarrelExports(finalAbs, projectRoot);
768
- if (barrelMap.size > 0) {
769
- // Add all unique source files referenced by this barrel
770
- const sources = new Set(barrelMap.values());
771
- for (const src of sources) {
772
- if (!resolved.includes(src)) resolved.push(src);
773
- }
774
- } else {
775
- // Barrel has no re-exports — keep the barrel file itself
776
- if (!resolved.includes(relPath)) resolved.push(relPath);
777
- }
778
- } else {
779
- if (!resolved.includes(relPath)) resolved.push(relPath);
780
- }
781
- }
782
- }
783
- // ── Go intra-module: github.com/owner/repo/sub → sub/*.go ──────────
784
- // Go import paths use the module name as prefix; map them to local dirs.
785
- else if (ext === '.go') {
786
- const modName = _readGoModuleName(projectRoot);
787
- if (modName && raw.startsWith(modName + '/')) {
788
- const subPkg = raw.slice(modName.length + 1); // e.g. 'render'
789
- const absDir = path.join(projectRoot, subPkg);
790
- if (fs.existsSync(absDir)) {
791
- try {
792
- const goFiles = fs.readdirSync(absDir)
793
- .filter(f => f.endsWith('.go') && !f.includes('_test'))
794
- .slice(0, 5); // cap: take a representative sample
795
- for (const gof of goFiles) {
796
- const rel = path.relative(projectRoot, path.join(absDir, gof)).replace(/\\/g, '/');
797
- if (!resolved.includes(rel)) resolved.push(rel);
798
- }
799
- } catch { /* directory unreadable — skip */ }
800
- }
801
- }
802
- }
803
- // ── Rust crate-local: crate::module::item → src/module.rs ──────────
804
- else if (ext === '.rs' && raw.startsWith('crate::')) {
805
- const parts = raw.slice('crate::'.length).split('::').filter(Boolean);
806
- for (let depth = parts.length; depth >= 1; depth--) {
807
- const subPath = parts.slice(0, depth).join('/');
808
- const candidates = [
809
- path.join(projectRoot, 'src', subPath + '.rs'),
810
- path.join(projectRoot, 'src', subPath, 'mod.rs'),
811
- ];
812
- let found = false;
813
- for (const c of candidates) {
814
- if (fs.existsSync(c)) {
815
- resolved.push(path.relative(projectRoot, c).replace(/\\/g, '/'));
816
- found = true; break;
817
- }
818
- }
819
- if (found) break;
820
- }
821
- }
822
- }
823
- return resolved;
824
- }
825
-
826
- /**
827
- * Build the text payload sent to the embedding model for a chunk.
828
- *
829
- * Shared by the bootstrap indexer (indexer.mjs) and the watch daemon
830
- * (watch-daemon.mjs) so a chunk yields the SAME embedding regardless of which
831
- * path embedded it first. This matters because the cache key (content_hash) is
832
- * derived from code + docstring only — it does NOT include this payload — so two
833
- * divergent payloads for the same hash would silently produce inconsistent
834
- * embeddings across a full re-index vs. an incremental update.
835
- *
836
- * @param {object} chunk Semantic chunk: { file_path, node_type, name, docstring, type_refs, code_snippet }.
837
- * @param {string[]} depRelPaths Resolved local imports of the chunk's file (project-relative paths).
838
- * @returns {string}
839
- */
840
- export function buildEmbeddingPayload(chunk, depRelPaths = []) {
841
- const neighbors = depRelPaths
842
- .map(d => path.basename(d, path.extname(d)))
843
- .filter(Boolean);
844
- const topologicalContext = neighbors.length
845
- ? `This code architectural neighborhood connects with: ${neighbors.join(', ')}.`
846
- : '';
847
- // NOTE: decorators and inheritance edges are NOT added here (A/B-tested: neutral
848
- // on vector, regression on BM25 — surfaced as metadata only).
849
- // LLM summary leads the payload when available: declarative voice aligns with
850
- // nomic-embed-text's search_document: training objective and anchors the embedding
851
- // toward developer query vocabulary. Questions/hyde are intentionally excluded from
852
- // the vector payload — they add stopword noise and dilute the code's semantic
853
- // fingerprint. Concept keywords (chunk.hyde = concepts.join(' ')) go to BM25 only
854
- // via buildLexicalDocument, keeping both retrieval channels clean.
855
- return [
856
- chunk.summary || '', // semantic lead: LLM-generated declarative summary (opt-in)
857
- `File Location: ${chunk.file_path}`,
858
- `Symbol Name: ${chunk.node_type} -> ${chunk.name}`,
859
- chunk.docstring ? `Developer Documentation: ${chunk.docstring}` : '',
860
- chunk.type_refs?.length ? `Type References: ${chunk.type_refs.join(', ')}` : '',
861
- topologicalContext,
862
- `--- Source Code ---`,
863
- chunk.code_snippet,
864
- ].filter(Boolean).join('\n');
865
- }
866
-
867
- export async function getLocalEmbedding(text, graceful = true, { ollamaHost, model } = {}) {
868
- if (process.env.INDEXER_EMBEDDINGS === 'off') return null; // lexical-only mode
869
- const host = _resolveOllamaHost(ollamaHost);
870
- const embedModel = _resolveEmbedModel(model);
871
- const MAX_RETRIES = 3;
872
- const safeText = "search_query: " + truncateForEmbedding(text);
873
- for (let attempt = 0; attempt <= MAX_RETRIES; attempt++) {
874
- try {
875
- const res = await fetch(`${host}/api/embeddings`, {
876
- method: "POST", headers: { "Content-Type": "application/json" },
877
- body: JSON.stringify({ model: embedModel, prompt: safeText }),
878
- signal: AbortSignal.timeout(15000),
879
- });
880
- if (!res.ok) throw new Error(`HTTP ${res.status}`);
881
- const data = await res.json();
882
- return data.embedding;
883
- } catch (err) {
884
- if (attempt < MAX_RETRIES) await new Promise(r => setTimeout(r, 500 * 2 ** attempt));
885
- else if (!graceful) throw err;
886
- }
887
- }
888
- return null;
889
- }
890
-
891
- export async function getLocalEmbeddingsBatch(texts, graceful = true, { ollamaHost, model } = {}) {
892
- if (!texts || texts.length === 0) return [];
893
- if (process.env.INDEXER_EMBEDDINGS === 'off') return null; // lexical-only mode
894
- const host = _resolveOllamaHost(ollamaHost);
895
- const embedModel = _resolveEmbedModel(model);
896
- const MAX_RETRIES = 3;
897
- const safeTexts = texts.map(t => "search_document: " + (t.length > 8000 ? t.slice(0, 8000) : t));
898
-
899
- for (let attempt = 0; attempt <= MAX_RETRIES; attempt++) {
900
- try {
901
- const res = await fetch(`${host}/api/embed`, {
902
- method: "POST", headers: { "Content-Type": "application/json" },
903
- body: JSON.stringify({ model: embedModel, input: safeTexts }),
904
- signal: AbortSignal.timeout(60000),
905
- });
906
- if (!res.ok) throw new Error(`HTTP ${res.status}`);
907
- const data = await res.json();
908
- return data.embeddings;
909
- } catch (err) {
910
- if (attempt < MAX_RETRIES) await new Promise(r => setTimeout(r, 500 * 2 ** attempt));
911
- else if (!graceful) throw err;
912
- }
913
- }
914
- return null;
915
- }
916
-
917
- // ─── Enrichment helpers (param names, return type, class context) ─────────────
918
-
919
- export function extractParams(chunkNode, ext) {
920
- const params = [];
921
- const JS_LIKE = ['.ts', '.tsx', '.js', '.jsx', '.mjs', '.cjs'];
922
- const paramTypes = JS_LIKE.includes(ext)
923
- ? ['required_parameter', 'optional_parameter', 'formal_parameters', 'identifier']
924
- : ['parameter', 'formal_parameter', 'identifier'];
925
-
926
- function walkParams(node) {
927
- if (node.type === 'formal_parameters' || node.type === 'parameters' || node.type === 'parameter_list') {
928
- for (const child of node.children) {
929
- // TS: required_parameter / optional_parameter have an identifier child
930
- if (child.type === 'required_parameter' || child.type === 'optional_parameter' || child.type === 'formal_parameter') {
931
- const id = child.childForFieldName?.('pattern') || child.childForFieldName?.('name') ||
932
- child.children.find(c => c.type === 'identifier');
933
- if (id) params.push(id.text);
934
- // also grab type annotation text
935
- const typeAnnotation = child.childForFieldName?.('type');
936
- if (typeAnnotation) {
937
- const typeText = typeAnnotation.text.replace(/^:\s*/, '').trim();
938
- if (typeText) params.push(typeText);
939
- }
940
- } else if (child.type === 'identifier') {
941
- params.push(child.text);
942
- }
943
- }
944
- }
945
- for (const child of node.children) walkParams(child);
946
- }
947
-
948
- // Only walk the direct params node to avoid deep recursion into body
949
- const paramsNode = chunkNode.childForFieldName?.('parameters') || chunkNode.childForFieldName?.('formal_parameters');
950
- if (paramsNode) walkParams(paramsNode);
951
- return [...new Set(params)].filter(p => p && p.length > 1).slice(0, 15);
952
- }
953
-
954
- export function extractReturnType(chunkNode, ext) {
955
- const JS_LIKE = ['.ts', '.tsx', '.js', '.jsx', '.mjs', '.cjs'];
956
- if (JS_LIKE.includes(ext)) {
957
- const retTypeNode = chunkNode.childForFieldName?.('return_type');
958
- if (retTypeNode) return retTypeNode.text.replace(/^:\s*/, '').trim().slice(0, 80);
959
- }
960
- return '';
961
- }
962
-
963
- export function extractClassContext(chunkNode) {
964
- let parent = chunkNode.parent;
965
- while (parent) {
966
- if (parent.type === 'class_declaration' || parent.type === 'class_definition' ||
967
- parent.type === 'class_body' || parent.type === 'impl_item') {
968
- const nameNode = parent.childForFieldName?.('name');
969
- if (nameNode) return nameNode.text;
970
- }
971
- parent = parent.parent;
972
- }
973
- return '';
974
- }
975
-
976
- /**
977
- * Extract decorator / annotation names applied to a chunk (and, for class chunks,
978
- * to the methods inside it). Decorators encode what a symbol *is* in modern
979
- * frameworks — `@Controller`, `@Injectable`, `@Get`, `@Entity` (TS: NestJS,
980
- * Angular, TypeORM) and `@app.route`, `@pytest.fixture`, `@dataclass`,
981
- * `@property` (Python) — yet as raw snippet text they are diluted to a single
982
- * low-weight token inside a large class body. Surfacing them as a dedicated field
983
- * lets a class annotated `@Controller` be retrieved by "controller" and a method
984
- * annotated `@Get` by "get/route", independent of language.
985
- *
986
- * Generalises by node type only (the tree-sitter `decorator` node is shared across
987
- * TS/JS/Python grammars) — no framework-specific names are hardcoded. Callee
988
- * arguments are stripped: `@Controller('cats')` -> 'Controller',
989
- * `@app.route('/x')` -> 'app.route', `@UseGuards(AuthGuard)` -> 'UseGuards'.
990
- *
991
- * @returns {string[]} unique decorator callee names (max 24)
992
- */
993
- export function extractDecorators(chunkNode) {
994
- const names = new Set();
995
- const addDecorator = (decoNode) => {
996
- let t = (decoNode.text || '').trim().replace(/^@/, '');
997
- t = t.split('(')[0]; // drop call arguments: @Get(':id') -> Get
998
- t = t.split(/[\s\n{]/)[0].trim(); // first token only
999
- if (t && t.length <= 64) names.add(t);
1000
- };
1001
-
1002
- // (1) Decorators that PRECEDE the chunk as siblings. Python wraps a decorated
1003
- // symbol in `decorated_definition` ([decorator…, def]); some grammars place
1004
- // class decorators as leading siblings rather than children.
1005
- let prev = chunkNode.previousSibling;
1006
- while (prev) {
1007
- if (prev.type === 'decorator') addDecorator(prev);
1008
- else if (prev.isNamed && prev.type !== 'comment') break;
1009
- prev = prev.previousSibling;
1010
- }
1011
-
1012
- // (2) Decorators within the chunk's subtree. A captured TS class chunk (the
1013
- // enclosing export_statement / class_declaration) carries its own class
1014
- // decorators plus the @Get/@Post/@Inject decorators on its methods.
1015
- // Bounded traversal so a large class body cannot inflate indexing time.
1016
- let budget = 800;
1017
- const stack = [chunkNode];
1018
- while (stack.length && budget-- > 0) {
1019
- const n = stack.pop();
1020
- for (let i = 0; i < n.namedChildCount; i++) {
1021
- const child = n.namedChild(i);
1022
- if (child.type === 'decorator') addDecorator(child);
1023
- else stack.push(child);
1024
- }
1025
- }
1026
-
1027
- return Array.from(names).slice(0, 24);
1028
- }
1029
-
1030
- /**
1031
- * Extract the base classes and implemented interfaces of a class chunk — the
1032
- * inheritance edge that links a concept to its implementations
1033
- * (`class ValidationPipe extends BasePipe implements PipeTransform`). Surfacing
1034
- * this lets an agent move from an abstract type to the concrete classes that
1035
- * realise it, and feeds the semantic embedding so an implementation is retrievable
1036
- * by the interface it fulfils.
1037
- *
1038
- * Generalises across `extends`/`implements` (TS/JS) and base-class argument lists
1039
- * (Python) by node type. Returns parent type names, e.g.
1040
- * ['BasePipe', 'PipeTransform', 'OnInit'].
1041
- *
1042
- * @returns {string[]} base/interface names (max 12)
1043
- */
1044
- export function extractHeritage(chunkNode, ext) {
1045
- const bases = new Set();
1046
- const JS_LIKE = ['.ts', '.tsx', '.js', '.jsx', '.mjs', '.cjs'];
1047
-
1048
- const addTypeNames = (node) => {
1049
- const stack = [node];
1050
- let budget = 200;
1051
- while (stack.length && budget-- > 0) {
1052
- const n = stack.pop();
1053
- if ((n.type === 'type_identifier' || n.type === 'identifier') && /^[A-Za-z_$]/.test(n.text)) {
1054
- bases.add(n.text);
1055
- }
1056
- for (let i = 0; i < n.namedChildCount; i++) stack.push(n.namedChild(i));
1057
- }
1058
- };
1059
-
1060
- if (JS_LIKE.includes(ext)) {
1061
- // Walk the chunk subtree for extends/implements clauses, but never descend
1062
- // into class_body — base types of the class only, not its method internals.
1063
- const stack = [chunkNode];
1064
- let budget = 400;
1065
- while (stack.length && budget-- > 0) {
1066
- const n = stack.pop();
1067
- if (n.type === 'extends_clause' || n.type === 'implements_clause') { addTypeNames(n); continue; }
1068
- for (let i = 0; i < n.namedChildCount; i++) {
1069
- const c = n.namedChild(i);
1070
- if (c.type !== 'class_body' && c.type !== 'statement_block') stack.push(c);
1071
- }
1072
- }
1073
- } else if (ext === '.py') {
1074
- const sc = chunkNode.childForFieldName?.('superclasses');
1075
- if (sc) {
1076
- for (let i = 0; i < sc.namedChildCount; i++) {
1077
- const c = sc.namedChild(i);
1078
- // skip keyword args like metaclass=… (keyword_argument node)
1079
- if (c.type === 'identifier' || c.type === 'attribute') bases.add(c.text);
1080
- }
1081
- }
1082
- }
1083
- return Array.from(bases).slice(0, 12);
1084
- }
1085
-
1086
- /**
1087
- * Frontier 2: Extract TypeScript/Python type annotation names from a chunk node.
1088
- * Returns simple type names (e.g. ['User', 'AuthToken', 'PaymentService'])
1089
- * used to enrich the inverted index and the type_refs chunk field.
1090
- */
1091
- export function extractTypeAnnotations(chunkNode, ext) {
1092
- const types = new Set();
1093
- const JS_LIKE = ['.ts', '.tsx', '.js', '.jsx', '.mjs', '.cjs'];
1094
- if (!JS_LIKE.includes(ext) && ext !== '.py') return [];
1095
-
1096
- function walk(node) {
1097
- // TypeScript: type_annotation nodes contain the type text
1098
- if (node.type === 'type_annotation') {
1099
- const typeText = node.text.replace(/^:\s*/, '').trim();
1100
- // Extract simple identifiers from the type (skip primitives)
1101
- const PRIMITIVES = new Set(['string','number','boolean','void','any','unknown','never','null','undefined','object','symbol','bigint']);
1102
- for (const match of typeText.matchAll(/\b([A-Z][A-Za-z0-9]*)\b/g)) {
1103
- if (!PRIMITIVES.has(match[1].toLowerCase())) types.add(match[1]);
1104
- }
1105
- }
1106
- // TypeScript: generic_type, predefined_type (string, number…) skip, named types keep
1107
- else if (node.type === 'type_identifier' || node.type === 'generic_type') {
1108
- const name = node.children[0]?.text || node.text;
1109
- if (name && /^[A-Z]/.test(name)) types.add(name);
1110
- }
1111
- // Python: type comments or annotations (annotation nodes)
1112
- else if (node.type === 'annotation' && ext === '.py') {
1113
- const typeText = node.text.replace(/^->\s*|^:\s*/, '').trim();
1114
- for (const match of typeText.matchAll(/\b([A-Z][A-Za-z0-9]*)\b/g)) {
1115
- types.add(match[1]);
1116
- }
1117
- }
1118
- for (const child of node.children) walk(child);
1119
- }
1120
- walk(chunkNode);
1121
- return Array.from(types).slice(0, 20);
1122
- }
1123
-
1124
- export function extractCalls(rootNode) {
1125
- const calls = new Set();
1126
- function walk(node) {
1127
- // JavaScript / TypeScript / Go / Rust: call_expression
1128
- if (node.type === 'call_expression') {
1129
- const funcNode = node.childForFieldName?.('function') || node.children[0];
1130
- if (funcNode) {
1131
- if (funcNode.type === 'identifier') calls.add(funcNode.text);
1132
- else if (funcNode.type === 'member_expression' || funcNode.type === 'property_identifier') {
1133
- const prop = funcNode.childForFieldName?.('property');
1134
- if (prop) calls.add(prop.text);
1135
- else calls.add(funcNode.text.split('.').pop());
1136
- }
1137
- }
1138
- }
1139
- // Python: `call` node type (different name from call_expression)
1140
- else if (node.type === 'call') {
1141
- const funcNode = node.childForFieldName?.('function') || node.children[0];
1142
- if (funcNode) {
1143
- if (funcNode.type === 'identifier') calls.add(funcNode.text);
1144
- else if (funcNode.type === 'attribute') {
1145
- const attr = funcNode.childForFieldName?.('attribute');
1146
- if (attr) calls.add(attr.text);
1147
- }
1148
- }
1149
- }
1150
- // Rust: macro_invocation (e.g. vec!, println!, format!)
1151
- else if (node.type === 'macro_invocation') {
1152
- const macroNode = node.childForFieldName?.('macro') || node.children[0];
1153
- if (macroNode && macroNode.type === 'identifier') calls.add(macroNode.text + '!');
1154
- }
1155
- // Java / C#: method_invocation
1156
- else if (node.type === 'method_invocation') {
1157
- const nameNode = node.childForFieldName?.('name') || node.children.find(c => c.type === 'identifier');
1158
- if (nameNode) calls.add(nameNode.text);
1159
- }
1160
- // Ruby: method_call / call
1161
- else if (node.type === 'method_call') {
1162
- const method = node.childForFieldName?.('method') || node.children.find(c => c.type === 'identifier');
1163
- if (method && method.type === 'identifier') calls.add(method.text);
1164
- }
1165
- node.children.forEach(walk);
1166
- }
1167
- walk(rootNode);
1168
- const noise = new Set(['require', 'console', 'log', 'expect', 'test', 'it', 'describe', 'setTimeout', 'print', 'println!']);
1169
- return Array.from(calls).filter(c => c && c.length > 2 && !noise.has(c));
1170
- }