graph-indexer 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +677 -0
- package/PROMPT.md +191 -0
- package/README.md +512 -0
- package/core-engine.mjs +405 -0
- package/indexer.mjs +180 -0
- package/mcp-server.mjs +295 -0
- package/package.json +81 -0
- package/parser-utils.mjs +458 -0
- package/watch-daemon.mjs +136 -0
package/parser-utils.mjs
ADDED
|
@@ -0,0 +1,458 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @file parser-utils.mjs
|
|
3
|
+
* @description Shared AST parsing, language registry, and embedding utilities.
|
|
4
|
+
* @author MaquinaTech <https://github.com/MaquinaTech>
|
|
5
|
+
* @copyright (c) 2026 MaquinaTech. All rights reserved.
|
|
6
|
+
* @license GPL-3.0-only
|
|
7
|
+
* * This program is free software: you can redistribute it and/or modify
|
|
8
|
+
* it under the terms of the GNU General Public License as published by
|
|
9
|
+
* the Free Software Foundation, either version 3 of the License, or
|
|
10
|
+
* (at your option) any later version.
|
|
11
|
+
*
|
|
12
|
+
* This program is distributed in the hope that it will be useful,
|
|
13
|
+
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
14
|
+
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
15
|
+
* GNU General Public License for more details.
|
|
16
|
+
*/
|
|
17
|
+
import fs from 'fs';
|
|
18
|
+
import path from 'path';
|
|
19
|
+
import { createHash } from 'crypto';
|
|
20
|
+
import ignore from 'ignore';
|
|
21
|
+
import Parser from 'tree-sitter';
|
|
22
|
+
const { Query } = Parser; // ImportaciΓ³n robusta nativa
|
|
23
|
+
|
|
24
|
+
import TypeScript from 'tree-sitter-typescript';
|
|
25
|
+
import JavaScript from 'tree-sitter-javascript';
|
|
26
|
+
import CSS from 'tree-sitter-css';
|
|
27
|
+
import Python from 'tree-sitter-python';
|
|
28
|
+
import Rust from 'tree-sitter-rust';
|
|
29
|
+
import Go from 'tree-sitter-go';
|
|
30
|
+
import PHP from 'tree-sitter-php';
|
|
31
|
+
import { truncateForEmbedding } from './core-engine.mjs';
|
|
32
|
+
|
|
33
|
+
export const MAX_FILE_SIZE_BYTES = 500000;
|
|
34
|
+
export const OLLAMA_HOST = process.env.OLLAMA_HOST || "http://localhost:11434";
|
|
35
|
+
|
|
36
|
+
const LANGUAGE_MAP = {
|
|
37
|
+
'.ts': TypeScript.typescript, '.tsx': TypeScript.tsx,
|
|
38
|
+
'.js': JavaScript, '.jsx': JavaScript, '.mjs': JavaScript, '.cjs': JavaScript,
|
|
39
|
+
'.css': CSS, '.scss': CSS,
|
|
40
|
+
'.py': Python, '.rs': Rust, '.go': Go,
|
|
41
|
+
'.php': PHP.php,
|
|
42
|
+
};
|
|
43
|
+
|
|
44
|
+
// π₯ ULTRA-GENERIC QUERIES: Immune to grammar changes between TS/JS/TSX
|
|
45
|
+
const LANGUAGE_QUERIES = {
|
|
46
|
+
ts: `
|
|
47
|
+
(class_declaration) @chunk
|
|
48
|
+
(function_declaration) @chunk
|
|
49
|
+
(method_definition) @chunk
|
|
50
|
+
(lexical_declaration) @chunk
|
|
51
|
+
(expression_statement) @chunk
|
|
52
|
+
(export_statement) @chunk
|
|
53
|
+
`,
|
|
54
|
+
css: `(rule_set) @chunk`,
|
|
55
|
+
py: `
|
|
56
|
+
(function_definition) @chunk
|
|
57
|
+
(class_definition) @chunk
|
|
58
|
+
(expression_statement) @chunk
|
|
59
|
+
`,
|
|
60
|
+
rs: `
|
|
61
|
+
(function_item) @chunk
|
|
62
|
+
(struct_item) @chunk
|
|
63
|
+
(enum_item) @chunk
|
|
64
|
+
(trait_item) @chunk
|
|
65
|
+
(impl_item) @chunk
|
|
66
|
+
`,
|
|
67
|
+
go: `
|
|
68
|
+
(function_declaration) @chunk
|
|
69
|
+
(method_declaration) @chunk
|
|
70
|
+
(type_declaration) @chunk
|
|
71
|
+
`,
|
|
72
|
+
php: `
|
|
73
|
+
(function_definition) @chunk
|
|
74
|
+
(class_declaration) @chunk
|
|
75
|
+
(expression_statement) @chunk
|
|
76
|
+
`
|
|
77
|
+
};
|
|
78
|
+
|
|
79
|
+
export const EXTENSIONS = new Set(Object.keys(LANGUAGE_MAP));
|
|
80
|
+
|
|
81
|
+
// Definition of container node types to avoid nested duplicates
|
|
82
|
+
const CONTAINERS = new Set([
|
|
83
|
+
'class_declaration', 'function_declaration', 'method_definition',
|
|
84
|
+
'lexical_declaration', 'expression_statement', 'export_statement',
|
|
85
|
+
'function_definition', 'class_definition', 'rule_set',
|
|
86
|
+
'function_item', 'struct_item', 'enum_item', 'trait_item', 'impl_item',
|
|
87
|
+
'method_declaration', 'type_declaration'
|
|
88
|
+
]);
|
|
89
|
+
|
|
90
|
+
export function getParserForFile(ext) {
|
|
91
|
+
const language = LANGUAGE_MAP[ext];
|
|
92
|
+
if (!language) return null;
|
|
93
|
+
const parser = new Parser();
|
|
94
|
+
parser.setLanguage(language);
|
|
95
|
+
return parser;
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
export function buildIgnoreFilter(rootPath) {
|
|
99
|
+
const ig = ignore();
|
|
100
|
+
ig.add(['node_modules', '.git', 'dist', 'build', '.next', 'coverage', '*.tmp', 'vendor', '.venv']);
|
|
101
|
+
const gitignorePath = path.join(rootPath, '.gitignore');
|
|
102
|
+
if (fs.existsSync(gitignorePath)) ig.add(fs.readFileSync(gitignorePath, 'utf-8'));
|
|
103
|
+
return ig;
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
export function generateChunkHash(text) {
|
|
107
|
+
return createHash('sha256').update(text).digest('hex');
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
export function extractFileSkeleton(rootNode, content) {
|
|
111
|
+
const signatures = [];
|
|
112
|
+
function walk(node) {
|
|
113
|
+
if (node.type.includes('declaration') || node.type.includes('definition')) {
|
|
114
|
+
let name = node.childForFieldName?.("name")?.text || "anonymous";
|
|
115
|
+
signatures.push(`- [${node.type}] ${name} (lines ${node.startPosition.row + 1}-${node.endPosition.row + 1})`);
|
|
116
|
+
}
|
|
117
|
+
node.children.forEach(walk);
|
|
118
|
+
}
|
|
119
|
+
walk(rootNode);
|
|
120
|
+
return signatures.join('\n');
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
export function extractImportsFromAST(rootNode, ext) {
|
|
124
|
+
const imports = new Set();
|
|
125
|
+
function walk(node) {
|
|
126
|
+
// ββ JavaScript / TypeScript ββββββββββββββββββββββββββββββββββββββββββ
|
|
127
|
+
if (['.ts', '.tsx', '.js', '.jsx', '.mjs', '.cjs'].includes(ext)) {
|
|
128
|
+
if (node.type === 'import_statement') {
|
|
129
|
+
const source = node.children.find(c => c.type === 'string');
|
|
130
|
+
if (source) imports.add(source.text.replace(/['"]/g, ''));
|
|
131
|
+
} else if (node.type === 'call_expression' && node.children[0]?.text === 'require') {
|
|
132
|
+
const arg = node.children[1]?.children?.find(c => c.type === 'string');
|
|
133
|
+
if (arg) imports.add(arg.text.replace(/['"]/g, ''));
|
|
134
|
+
}
|
|
135
|
+
}
|
|
136
|
+
// ββ Python ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
|
137
|
+
else if (ext === '.py') {
|
|
138
|
+
if (node.type === 'import_statement') {
|
|
139
|
+
// import foo.bar.baz β store as foo/bar/baz
|
|
140
|
+
for (const child of node.children) {
|
|
141
|
+
if (child.type === 'dotted_name') imports.add(child.text.replace(/\./g, '/'));
|
|
142
|
+
}
|
|
143
|
+
} else if (node.type === 'import_from_statement') {
|
|
144
|
+
// from .sibling import X β relative: emit as ./sibling
|
|
145
|
+
// from ..pkg import Y β relative: emit as ../pkg
|
|
146
|
+
// from foo.bar import Z β absolute: store foo/bar
|
|
147
|
+
const relNode = node.children.find(c => c.type === 'relative_import');
|
|
148
|
+
if (relNode) {
|
|
149
|
+
// dots count = depth; remainder is the module path after dots
|
|
150
|
+
const raw = relNode.text; // e.g. '.' or '..utils'
|
|
151
|
+
const dots = raw.match(/^\.+/)?.[0] ?? '.';
|
|
152
|
+
const mod = raw.slice(dots.length);
|
|
153
|
+
const prefix = dots.length === 1 ? './' : '../'.repeat(dots.length - 1);
|
|
154
|
+
imports.add(mod ? prefix + mod.replace(/\./g, '/') : prefix.slice(0, -1) || '.');
|
|
155
|
+
} else {
|
|
156
|
+
const mod = node.children.find(c => c.type === 'dotted_name');
|
|
157
|
+
if (mod) imports.add(mod.text.replace(/\./g, '/'));
|
|
158
|
+
}
|
|
159
|
+
}
|
|
160
|
+
}
|
|
161
|
+
// ββ Rust ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
|
162
|
+
else if (ext === '.rs') {
|
|
163
|
+
if (node.type === 'use_declaration') {
|
|
164
|
+
// Collect the first path text from the argument subtree
|
|
165
|
+
const arg = node.childForFieldName?.('argument') ||
|
|
166
|
+
node.children.find(c => !['use', ';', 'pub'].includes(c.type));
|
|
167
|
+
if (arg) imports.add(arg.text.split('::').slice(0, 3).join('::'));
|
|
168
|
+
}
|
|
169
|
+
}
|
|
170
|
+
// ββ Go ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
|
171
|
+
else if (ext === '.go') {
|
|
172
|
+
if (node.type === 'import_spec') {
|
|
173
|
+
const pathNode = node.children.find(c =>
|
|
174
|
+
c.type === 'interpreted_string_literal' || c.type === 'raw_string_literal');
|
|
175
|
+
if (pathNode) imports.add(pathNode.text.replace(/['"`]/g, ''));
|
|
176
|
+
}
|
|
177
|
+
}
|
|
178
|
+
// ββ PHP βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
|
179
|
+
else if (ext === '.php') {
|
|
180
|
+
if (node.type === 'include_expression' || node.type === 'require_expression' ||
|
|
181
|
+
node.type === 'include_once_expression' || node.type === 'require_once_expression') {
|
|
182
|
+
const strNode = node.children.find(c =>
|
|
183
|
+
c.type === 'string' || c.type === 'encapsed_string');
|
|
184
|
+
if (strNode) imports.add(strNode.text.replace(/['"]/g, ''));
|
|
185
|
+
}
|
|
186
|
+
}
|
|
187
|
+
node.children.forEach(walk);
|
|
188
|
+
}
|
|
189
|
+
walk(rootNode);
|
|
190
|
+
return Array.from(imports);
|
|
191
|
+
}
|
|
192
|
+
|
|
193
|
+
export function extractSemanticChunks(rootNode, relPath, sourceCode, ext) {
|
|
194
|
+
const chunks = [];
|
|
195
|
+
const parser = getParserForFile(ext);
|
|
196
|
+
if (!parser) return chunks;
|
|
197
|
+
|
|
198
|
+
const JS_LIKE = ['.ts', '.tsx', '.js', '.jsx', '.mjs', '.cjs'];
|
|
199
|
+
const langKey = JS_LIKE.includes(ext) ? 'ts' : (LANGUAGE_QUERIES[ext.slice(1)] ? ext.slice(1) : null);
|
|
200
|
+
if (!langKey || !LANGUAGE_QUERIES[langKey]) return chunks;
|
|
201
|
+
|
|
202
|
+
// π₯ HEADER INHERITANCE: Extract top-of-file/global comments for module context
|
|
203
|
+
let fileDocstring = "";
|
|
204
|
+
let cursorNode = rootNode.children[0];
|
|
205
|
+
while (cursorNode) {
|
|
206
|
+
if (cursorNode.type === 'comment') fileDocstring += cursorNode.text + "\n";
|
|
207
|
+
else if (cursorNode.type !== 'import_statement' && cursorNode.type !== 'expression_statement') break;
|
|
208
|
+
cursorNode = cursorNode.nextSibling;
|
|
209
|
+
}
|
|
210
|
+
fileDocstring = fileDocstring.trim();
|
|
211
|
+
|
|
212
|
+
try {
|
|
213
|
+
const query = new Query(parser.getLanguage(), LANGUAGE_QUERIES[langKey]);
|
|
214
|
+
const matches = query.matches(rootNode);
|
|
215
|
+
const processedNodes = new Set();
|
|
216
|
+
|
|
217
|
+
for (const match of matches) {
|
|
218
|
+
let chunkNode = null;
|
|
219
|
+
for (const capture of match.captures) {
|
|
220
|
+
if (capture.name === 'chunk') chunkNode = capture.node;
|
|
221
|
+
}
|
|
222
|
+
|
|
223
|
+
if (!chunkNode || processedNodes.has(chunkNode.id)) continue;
|
|
224
|
+
processedNodes.add(chunkNode.id);
|
|
225
|
+
|
|
226
|
+
// Filter out very small fragments (simple variables, etc.)
|
|
227
|
+
if (chunkNode.endPosition.row - chunkNode.startPosition.row < 2) continue;
|
|
228
|
+
|
|
229
|
+
// π₯ DEDUPLICATION LOGIC: Ignore nodes that are nested inside other container nodes
|
|
230
|
+
let isNested = false;
|
|
231
|
+
let currentParent = chunkNode.parent;
|
|
232
|
+
while (currentParent && currentParent.type !== 'program') {
|
|
233
|
+
if (CONTAINERS.has(currentParent.type)) {
|
|
234
|
+
isNested = true;
|
|
235
|
+
break;
|
|
236
|
+
}
|
|
237
|
+
currentParent = currentParent.parent;
|
|
238
|
+
}
|
|
239
|
+
if (isNested) continue;
|
|
240
|
+
|
|
241
|
+
// π₯ ROBUST NAME EXTRACTION (JS logic)
|
|
242
|
+
let nameText = "anonymous";
|
|
243
|
+
const nameNode = chunkNode.childForFieldName?.("name");
|
|
244
|
+
|
|
245
|
+
if (nameNode) {
|
|
246
|
+
nameText = nameNode.text;
|
|
247
|
+
} else if (chunkNode.type === "export_statement") {
|
|
248
|
+
const decl = chunkNode.children.find(c => ['lexical_declaration', 'function_declaration', 'class_declaration'].includes(c.type));
|
|
249
|
+
if (decl) {
|
|
250
|
+
if (decl.type === "lexical_declaration") {
|
|
251
|
+
const varDecl = decl.children.find(c => c.type === "variable_declarator");
|
|
252
|
+
nameText = varDecl?.children.find(c => c.type === "identifier")?.text || "anonymous";
|
|
253
|
+
} else {
|
|
254
|
+
nameText = decl.childForFieldName?.("name")?.text || "anonymous";
|
|
255
|
+
}
|
|
256
|
+
} else {
|
|
257
|
+
const defaultChild = chunkNode.children.find(c => c.type === 'identifier' || c.type === 'call_expression');
|
|
258
|
+
if (defaultChild) {
|
|
259
|
+
if (defaultChild.type === 'identifier') {
|
|
260
|
+
nameText = `default_${defaultChild.text}`;
|
|
261
|
+
} else if (defaultChild.type === 'call_expression') {
|
|
262
|
+
const funcName = defaultChild.childForFieldName?.("function")?.text || defaultChild.children[0]?.text;
|
|
263
|
+
const argNode = defaultChild.childForFieldName?.("arguments")?.children?.find(c => c.type === 'identifier');
|
|
264
|
+
nameText = argNode ? `default_${funcName}_${argNode.text}` : `default_${funcName}`;
|
|
265
|
+
}
|
|
266
|
+
} else {
|
|
267
|
+
nameText = "default_export";
|
|
268
|
+
}
|
|
269
|
+
}
|
|
270
|
+
} else if (chunkNode.type === "lexical_declaration") {
|
|
271
|
+
const decl = chunkNode.children.find(c => c.type === "variable_declarator");
|
|
272
|
+
nameText = decl?.children.find(c => c.type === "identifier")?.text || "anonymous";
|
|
273
|
+
} else if (chunkNode.type === "expression_statement") {
|
|
274
|
+
const callExp = chunkNode.children.find(c => c.type === "call_expression");
|
|
275
|
+
const assignExp = chunkNode.children.find(c => c.type === "assignment_expression");
|
|
276
|
+
if (callExp) {
|
|
277
|
+
const funcName = callExp.childForFieldName?.("function")?.text || callExp.children[0]?.text;
|
|
278
|
+
const argsNode = callExp.childForFieldName?.("arguments");
|
|
279
|
+
const stringArg = argsNode?.children?.find(c => c.type === "string" || c.type === "template_string");
|
|
280
|
+
nameText = stringArg ? `${funcName}_${stringArg.text.replace(/['"`]/g, '')}` : (funcName || "anonymous");
|
|
281
|
+
} else if (assignExp) {
|
|
282
|
+
nameText = assignExp.childForFieldName?.("left")?.text || "anonymous";
|
|
283
|
+
}
|
|
284
|
+
} else {
|
|
285
|
+
const idNode = chunkNode.children.find(c => c.type === "identifier" || c.type === "name" || c.type === "property_identifier");
|
|
286
|
+
nameText = idNode?.text || "anonymous";
|
|
287
|
+
}
|
|
288
|
+
|
|
289
|
+
if (nameText === "anonymous" || nameText === "default_export") {
|
|
290
|
+
nameText = `${path.basename(relPath, path.extname(relPath))}_${chunkNode.type}`;
|
|
291
|
+
}
|
|
292
|
+
|
|
293
|
+
let docstring = "";
|
|
294
|
+
let prev = chunkNode.previousSibling;
|
|
295
|
+
while (prev && (prev.type === 'comment' || !prev.isNamed)) {
|
|
296
|
+
if (prev.type === 'comment') docstring = prev.text + "\n" + docstring;
|
|
297
|
+
prev = prev.previousSibling;
|
|
298
|
+
}
|
|
299
|
+
docstring = docstring.trim();
|
|
300
|
+
|
|
301
|
+
if (!docstring && fileDocstring) {
|
|
302
|
+
docstring = `[File Context]: ${fileDocstring}`;
|
|
303
|
+
}
|
|
304
|
+
|
|
305
|
+
const snippet = chunkNode.text.slice(0, 3000);
|
|
306
|
+
const hash = generateChunkHash(docstring + snippet);
|
|
307
|
+
const outgoingCalls = extractCalls(chunkNode);
|
|
308
|
+
|
|
309
|
+
const id = createHash('sha256')
|
|
310
|
+
.update(`${relPath}::${chunkNode.startPosition.row}::${chunkNode.startPosition.column}`)
|
|
311
|
+
.digest('hex').slice(0, 24);
|
|
312
|
+
|
|
313
|
+
chunks.push({
|
|
314
|
+
id, file_path: relPath, node_type: chunkNode.type, name: nameText,
|
|
315
|
+
docstring: docstring, code_snippet: snippet, content_hash: hash,
|
|
316
|
+
start_line: chunkNode.startPosition.row + 1, end_line: chunkNode.endPosition.row + 1,
|
|
317
|
+
calls: outgoingCalls
|
|
318
|
+
});
|
|
319
|
+
}
|
|
320
|
+
} catch (e) {
|
|
321
|
+
// Visible protective log for developers
|
|
322
|
+
process.stderr.write(`\n[parser-utils] π₯ Query Error in ${relPath}: ${e.message}\n`);
|
|
323
|
+
}
|
|
324
|
+
return chunks;
|
|
325
|
+
}
|
|
326
|
+
|
|
327
|
+
export function resolveLocalImports(rawImports, fromFileRelPath, projectRoot) {
|
|
328
|
+
const fileDir = path.dirname(path.join(projectRoot, fromFileRelPath));
|
|
329
|
+
const ext = path.extname(fromFileRelPath);
|
|
330
|
+
const resolved = [];
|
|
331
|
+
for (const raw of rawImports) {
|
|
332
|
+
// ββ Dot-relative (JS/TS/Python relative) ββββββββββββββββββββββββββββ
|
|
333
|
+
if (raw.startsWith('.')) {
|
|
334
|
+
const absResolved = path.resolve(fileDir, raw);
|
|
335
|
+
const existingExt = path.extname(absResolved);
|
|
336
|
+
let finalAbs = null;
|
|
337
|
+
if (existingExt && EXTENSIONS.has(existingExt) && fs.existsSync(absResolved)) {
|
|
338
|
+
finalAbs = absResolved;
|
|
339
|
+
} else {
|
|
340
|
+
for (const e of EXTENSIONS) {
|
|
341
|
+
if (fs.existsSync(absResolved + e)) { finalAbs = absResolved + e; break; }
|
|
342
|
+
const idx = path.join(absResolved, 'index' + e);
|
|
343
|
+
if (fs.existsSync(idx)) { finalAbs = idx; break; }
|
|
344
|
+
// Python: also try __init__.py for package directories
|
|
345
|
+
if (e === '.py') {
|
|
346
|
+
const init = path.join(absResolved, '__init__.py');
|
|
347
|
+
if (fs.existsSync(init)) { finalAbs = init; break; }
|
|
348
|
+
}
|
|
349
|
+
}
|
|
350
|
+
}
|
|
351
|
+
if (finalAbs) resolved.push(path.relative(projectRoot, finalAbs).replace(/\\/g, '/'));
|
|
352
|
+
}
|
|
353
|
+
// ββ Rust crate-local: crate::module::item β src/module.rs ββββββββββ
|
|
354
|
+
else if (ext === '.rs' && raw.startsWith('crate::')) {
|
|
355
|
+
const parts = raw.slice('crate::'.length).split('::').filter(Boolean);
|
|
356
|
+
for (let depth = parts.length; depth >= 1; depth--) {
|
|
357
|
+
const subPath = parts.slice(0, depth).join('/');
|
|
358
|
+
const candidates = [
|
|
359
|
+
path.join(projectRoot, 'src', subPath + '.rs'),
|
|
360
|
+
path.join(projectRoot, 'src', subPath, 'mod.rs'),
|
|
361
|
+
];
|
|
362
|
+
let found = false;
|
|
363
|
+
for (const c of candidates) {
|
|
364
|
+
if (fs.existsSync(c)) {
|
|
365
|
+
resolved.push(path.relative(projectRoot, c).replace(/\\/g, '/'));
|
|
366
|
+
found = true; break;
|
|
367
|
+
}
|
|
368
|
+
}
|
|
369
|
+
if (found) break;
|
|
370
|
+
}
|
|
371
|
+
}
|
|
372
|
+
}
|
|
373
|
+
return resolved;
|
|
374
|
+
}
|
|
375
|
+
|
|
376
|
+
export async function getLocalEmbedding(text, graceful = true) {
|
|
377
|
+
if (process.env.INDEXER_EMBEDDINGS === 'off') return null; // lexical-only mode
|
|
378
|
+
const MAX_RETRIES = 3;
|
|
379
|
+
const safeText = "search_query: " + truncateForEmbedding(text);
|
|
380
|
+
for (let attempt = 0; attempt <= MAX_RETRIES; attempt++) {
|
|
381
|
+
try {
|
|
382
|
+
const res = await fetch(`${OLLAMA_HOST}/api/embeddings`, {
|
|
383
|
+
method: "POST", headers: { "Content-Type": "application/json" },
|
|
384
|
+
body: JSON.stringify({ model: "nomic-embed-text", prompt: safeText }),
|
|
385
|
+
signal: AbortSignal.timeout(15000),
|
|
386
|
+
});
|
|
387
|
+
if (!res.ok) throw new Error(`HTTP ${res.status}`);
|
|
388
|
+
const data = await res.json();
|
|
389
|
+
return data.embedding;
|
|
390
|
+
} catch (err) {
|
|
391
|
+
if (attempt < MAX_RETRIES) await new Promise(r => setTimeout(r, 500 * 2 ** attempt));
|
|
392
|
+
else if (!graceful) throw err;
|
|
393
|
+
}
|
|
394
|
+
}
|
|
395
|
+
return null;
|
|
396
|
+
}
|
|
397
|
+
|
|
398
|
+
export async function getLocalEmbeddingsBatch(texts, graceful = true) {
|
|
399
|
+
if (!texts || texts.length === 0) return [];
|
|
400
|
+
if (process.env.INDEXER_EMBEDDINGS === 'off') return null; // lexical-only mode
|
|
401
|
+
const MAX_RETRIES = 3;
|
|
402
|
+
const safeTexts = texts.map(t => "search_document: " + (t.length > 8000 ? t.slice(0, 8000) : t));
|
|
403
|
+
|
|
404
|
+
for (let attempt = 0; attempt <= MAX_RETRIES; attempt++) {
|
|
405
|
+
try {
|
|
406
|
+
const res = await fetch(`${OLLAMA_HOST}/api/embed`, {
|
|
407
|
+
method: "POST", headers: { "Content-Type": "application/json" },
|
|
408
|
+
body: JSON.stringify({ model: "nomic-embed-text", input: safeTexts }),
|
|
409
|
+
signal: AbortSignal.timeout(60000),
|
|
410
|
+
});
|
|
411
|
+
if (!res.ok) throw new Error(`HTTP ${res.status}`);
|
|
412
|
+
const data = await res.json();
|
|
413
|
+
return data.embeddings;
|
|
414
|
+
} catch (err) {
|
|
415
|
+
if (attempt < MAX_RETRIES) await new Promise(r => setTimeout(r, 500 * 2 ** attempt));
|
|
416
|
+
else if (!graceful) throw err;
|
|
417
|
+
}
|
|
418
|
+
}
|
|
419
|
+
return null;
|
|
420
|
+
}
|
|
421
|
+
|
|
422
|
+
export function extractCalls(rootNode) {
|
|
423
|
+
const calls = new Set();
|
|
424
|
+
function walk(node) {
|
|
425
|
+
// JavaScript / TypeScript / Go / Rust: call_expression
|
|
426
|
+
if (node.type === 'call_expression') {
|
|
427
|
+
const funcNode = node.childForFieldName?.('function') || node.children[0];
|
|
428
|
+
if (funcNode) {
|
|
429
|
+
if (funcNode.type === 'identifier') calls.add(funcNode.text);
|
|
430
|
+
else if (funcNode.type === 'member_expression' || funcNode.type === 'property_identifier') {
|
|
431
|
+
const prop = funcNode.childForFieldName?.('property');
|
|
432
|
+
if (prop) calls.add(prop.text);
|
|
433
|
+
else calls.add(funcNode.text.split('.').pop());
|
|
434
|
+
}
|
|
435
|
+
}
|
|
436
|
+
}
|
|
437
|
+
// Python: `call` node type (different name from call_expression)
|
|
438
|
+
else if (node.type === 'call') {
|
|
439
|
+
const funcNode = node.childForFieldName?.('function') || node.children[0];
|
|
440
|
+
if (funcNode) {
|
|
441
|
+
if (funcNode.type === 'identifier') calls.add(funcNode.text);
|
|
442
|
+
else if (funcNode.type === 'attribute') {
|
|
443
|
+
const attr = funcNode.childForFieldName?.('attribute');
|
|
444
|
+
if (attr) calls.add(attr.text);
|
|
445
|
+
}
|
|
446
|
+
}
|
|
447
|
+
}
|
|
448
|
+
// Rust: macro_invocation (e.g. vec!, println!, format!)
|
|
449
|
+
else if (node.type === 'macro_invocation') {
|
|
450
|
+
const macroNode = node.childForFieldName?.('macro') || node.children[0];
|
|
451
|
+
if (macroNode && macroNode.type === 'identifier') calls.add(macroNode.text + '!');
|
|
452
|
+
}
|
|
453
|
+
node.children.forEach(walk);
|
|
454
|
+
}
|
|
455
|
+
walk(rootNode);
|
|
456
|
+
const noise = new Set(['require', 'console', 'log', 'expect', 'test', 'it', 'describe', 'setTimeout', 'print', 'println!']);
|
|
457
|
+
return Array.from(calls).filter(c => c && c.length > 2 && !noise.has(c));
|
|
458
|
+
}
|
package/watch-daemon.mjs
ADDED
|
@@ -0,0 +1,136 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
/**
|
|
3
|
+
* @file watch-daemon.mjs
|
|
4
|
+
* @description Native FileSystem Watcher Daemon to maintain an in-memory graph index. Utilizes Tree-sitter for AST parsing and local embedding generation.
|
|
5
|
+
* @author MaquinaTech <https://github.com/MaquinaTech>
|
|
6
|
+
* @copyright (c) 2026 MaquinaTech. All rights reserved.
|
|
7
|
+
* @license GPL-3.0-only
|
|
8
|
+
* This program is free software: you can redistribute it and/or modify
|
|
9
|
+
* it under the terms of the GNU General Public License as published by
|
|
10
|
+
* the Free Software Foundation, either version 3 of the License, or
|
|
11
|
+
* (at your option) any later version.
|
|
12
|
+
*
|
|
13
|
+
* This program is distributed with the hope that it will be beneficial,
|
|
14
|
+
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
15
|
+
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
16
|
+
* GNU General Public License for more details.
|
|
17
|
+
*/
|
|
18
|
+
|
|
19
|
+
import fs from 'fs';
|
|
20
|
+
import path from 'path';
|
|
21
|
+
import chokidar from 'chokidar';
|
|
22
|
+
import { MemoryGraphIndex } from './core-engine.mjs';
|
|
23
|
+
import {
|
|
24
|
+
MAX_FILE_SIZE_BYTES, EXTENSIONS, getParserForFile, buildIgnoreFilter,
|
|
25
|
+
extractImportsFromAST, extractSemanticChunks, resolveLocalImports, getLocalEmbeddingsBatch
|
|
26
|
+
} from './parser-utils.mjs';
|
|
27
|
+
|
|
28
|
+
const PROJECT_ROOT = process.env.MCP_PROJECT_ROOT || process.cwd();
|
|
29
|
+
const INDEX_PATH = path.join(PROJECT_ROOT, 'code-index.json');
|
|
30
|
+
|
|
31
|
+
const ignoreFilter = buildIgnoreFilter(PROJECT_ROOT);
|
|
32
|
+
const db = new MemoryGraphIndex(INDEX_PATH);
|
|
33
|
+
db.load();
|
|
34
|
+
|
|
35
|
+
async function processFileChange(absolutePath) {
|
|
36
|
+
const filename = path.relative(PROJECT_ROOT, absolutePath).replace(/\\/g, '/');
|
|
37
|
+
if (ignoreFilter.ignores(filename)) return;
|
|
38
|
+
if (filename.includes('.bundle.') || filename.includes('.min.')) return;
|
|
39
|
+
|
|
40
|
+
try {
|
|
41
|
+
if (!fs.existsSync(absolutePath)) {
|
|
42
|
+
for (const [id, chunk] of db.chunks.entries()) {
|
|
43
|
+
if (chunk.file_path === filename) {
|
|
44
|
+
db._removeLexical(id);
|
|
45
|
+
db.chunks.delete(id);
|
|
46
|
+
db.removeVector(id);
|
|
47
|
+
}
|
|
48
|
+
}
|
|
49
|
+
db.updateFileGraph(filename, []);
|
|
50
|
+
db.saveDebounced();
|
|
51
|
+
process.stderr.write(`[daemon] ποΈ Purged: ${filename}\n`);
|
|
52
|
+
return;
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
const stats = fs.statSync(absolutePath);
|
|
56
|
+
if (stats.size > MAX_FILE_SIZE_BYTES) return;
|
|
57
|
+
|
|
58
|
+
let content = fs.readFileSync(absolutePath, 'utf-8');
|
|
59
|
+
if (!content.trim()) return;
|
|
60
|
+
|
|
61
|
+
const ext = path.extname(absolutePath);
|
|
62
|
+
const parser = getParserForFile(ext);
|
|
63
|
+
if (!parser) return;
|
|
64
|
+
|
|
65
|
+
const tree = parser.parse(content);
|
|
66
|
+
const imports = resolveLocalImports(extractImportsFromAST(tree.rootNode, ext), filename, PROJECT_ROOT);
|
|
67
|
+
db.updateFileGraph(filename, imports);
|
|
68
|
+
|
|
69
|
+
const newChunks = extractSemanticChunks(tree.rootNode, filename, content, ext);
|
|
70
|
+
|
|
71
|
+
for (const [id, chunk] of db.chunks.entries()) {
|
|
72
|
+
if (chunk.file_path === filename) {
|
|
73
|
+
db._removeLexical(id);
|
|
74
|
+
db.chunks.delete(id);
|
|
75
|
+
db.removeVector(id); // invalidates flat matrix so next search rebuilds it
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
const chunksToEmbed = newChunks.filter(c => !db.embeddingCache.has(c.content_hash));
|
|
80
|
+
|
|
81
|
+
if (chunksToEmbed.length > 0) {
|
|
82
|
+
const textsToEmbed = chunksToEmbed.map(c => {
|
|
83
|
+
const dependencies = db.graph.dependencies[c.file_path] || [];
|
|
84
|
+
const cleanDeps = dependencies.map(d => path.basename(d, path.extname(d)));
|
|
85
|
+
const topologicalContext = cleanDeps.length > 0
|
|
86
|
+
? `This code architectural neighborhood connects with: ${cleanDeps.join(', ')}.`
|
|
87
|
+
: '';
|
|
88
|
+
|
|
89
|
+
// π₯ STRICT PAYLOAD PARITY: Eliminate real-time bleed
|
|
90
|
+
return [
|
|
91
|
+
`File Location: ${c.file_path}`,
|
|
92
|
+
`Symbol Name: ${c.node_type} -> ${c.name}`,
|
|
93
|
+
c.docstring ? `Developer Documentation: ${c.docstring}` : '',
|
|
94
|
+
topologicalContext,
|
|
95
|
+
`--- Source Code ---`,
|
|
96
|
+
c.code_snippet
|
|
97
|
+
].filter(Boolean).join('\n');
|
|
98
|
+
});
|
|
99
|
+
|
|
100
|
+
const embeddingsMatrix = await getLocalEmbeddingsBatch(textsToEmbed, true);
|
|
101
|
+
if (embeddingsMatrix) {
|
|
102
|
+
chunksToEmbed.forEach((chunk, j) => {
|
|
103
|
+
if (embeddingsMatrix[j]) {
|
|
104
|
+
db.embeddingCache.set(chunk.content_hash, new Float32Array(embeddingsMatrix[j]));
|
|
105
|
+
}
|
|
106
|
+
});
|
|
107
|
+
}
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
for (const chunk of newChunks) {
|
|
111
|
+
if (db.embeddingCache.has(chunk.content_hash)) {
|
|
112
|
+
db.addVector(chunk.id, db.embeddingCache.get(chunk.content_hash));
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
// Unified lexical re-indexing of neighborhood
|
|
116
|
+
const cleanDeps = imports.map(d => d.split('/').pop().split('.')[0]);
|
|
117
|
+
const enrichedContext = `${chunk.name} ${chunk.docstring || ''} ${cleanDeps.join(' ')} ${(chunk.calls || []).join(' ')} ${chunk.code_snippet}`;
|
|
118
|
+
|
|
119
|
+
db._indexLexical(chunk.id, enrichedContext);
|
|
120
|
+
db.chunks.set(chunk.id, chunk);
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
db.saveDebounced();
|
|
124
|
+
process.stderr.write(`[daemon] π Synced: ${filename} (Cache Hit: ${newChunks.length - chunksToEmbed.length}/${newChunks.length})\n`);
|
|
125
|
+
} catch (err) {
|
|
126
|
+
process.stderr.write(`[daemon] β Error en ${filename}: ${err.message}\n`);
|
|
127
|
+
}
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
process.stderr.write(`π Chokidar Watcher Daemon iniciado en: ${PROJECT_ROOT}\n`);
|
|
131
|
+
const watcher = chokidar.watch(PROJECT_ROOT, {
|
|
132
|
+
ignored: /(^|[\/\\])\../, persistent: true,
|
|
133
|
+
awaitWriteFinish: { stabilityThreshold: 300, pollInterval: 100 }
|
|
134
|
+
});
|
|
135
|
+
watcher.on('add', processFileChange).on('change', processFileChange).on('unlink', (p) => processFileChange(p));
|
|
136
|
+
watcher.on('error', (err) => process.stderr.write(`[daemon] π₯ OS Watcher panic: ${err.message}\n`));
|