devsmind-mcp 2.0.2 → 2.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -21
- package/README.md +176 -73
- package/dist/cli/index.js +46 -4
- package/dist/cli/index.js.map +1 -1
- package/dist/cli/init.js +2 -1
- package/dist/cli/init.js.map +1 -1
- package/dist/cli/rule.js +31 -8
- package/dist/cli/rule.js.map +1 -1
- package/dist/cli/runner.d.ts +24 -0
- package/dist/cli/runner.js +502 -376
- package/dist/cli/runner.js.map +1 -1
- package/dist/db/database.d.ts +66 -0
- package/dist/db/database.js +329 -28
- package/dist/db/database.js.map +1 -1
- package/dist/db/edges.d.ts +42 -0
- package/dist/db/edges.js +162 -0
- package/dist/db/edges.js.map +1 -0
- package/dist/db/indexer.d.ts +3 -3
- package/dist/db/indexer.js +8 -8
- package/dist/db/indexer.js.map +1 -1
- package/dist/db/schema.js +39 -39
- package/dist/db/staging.d.ts +39 -0
- package/dist/db/staging.js +137 -0
- package/dist/db/staging.js.map +1 -0
- package/dist/mcp/server.js +168 -200
- package/dist/mcp/server.js.map +1 -1
- package/dist/mcp/visualizer_2d.html +635 -635
- package/dist/mcp/visualizer_3d.html +613 -628
- package/dist/utils/ast.d.ts +38 -0
- package/dist/utils/ast.js +1079 -0
- package/dist/utils/ast.js.map +1 -0
- package/package.json +4 -4
package/dist/cli/runner.js
CHANGED
|
@@ -32,6 +32,9 @@ var __importStar = (this && this.__importStar) || (function () {
|
|
|
32
32
|
return result;
|
|
33
33
|
};
|
|
34
34
|
})();
|
|
35
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
36
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
37
|
+
};
|
|
35
38
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
36
39
|
exports.runBackgroundIndexing = runBackgroundIndexing;
|
|
37
40
|
exports.runBackgroundReindexing = runBackgroundReindexing;
|
|
@@ -40,10 +43,14 @@ const path = __importStar(require("path"));
|
|
|
40
43
|
const http = __importStar(require("http"));
|
|
41
44
|
const https = __importStar(require("https"));
|
|
42
45
|
const crypto = __importStar(require("crypto"));
|
|
46
|
+
const prompts_1 = __importDefault(require("prompts"));
|
|
43
47
|
const database_1 = require("../db/database");
|
|
44
48
|
const indexer_1 = require("../db/indexer");
|
|
45
49
|
const scanner_1 = require("../utils/scanner");
|
|
50
|
+
const config_1 = require("../utils/config");
|
|
46
51
|
const json_1 = require("../utils/json");
|
|
52
|
+
const ast_1 = require("../utils/ast");
|
|
53
|
+
const edges_1 = require("../db/edges");
|
|
47
54
|
function makeHttpRequest(urlStr, method, headers, body) {
|
|
48
55
|
return new Promise((resolve, reject) => {
|
|
49
56
|
const isHttps = urlStr.startsWith('https');
|
|
@@ -84,6 +91,21 @@ function makeHttpRequest(urlStr, method, headers, body) {
|
|
|
84
91
|
function sleep(ms) {
|
|
85
92
|
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
86
93
|
}
|
|
94
|
+
// ── LLM request pacing ───────────────────────────────────────────────────
|
|
95
|
+
// Off by default — requests fire as fast as possible and 429s are handled by
|
|
96
|
+
// the retry/backoff in extractNodesFromCode. Pass --rpm to proactively space
|
|
97
|
+
// out requests and stay under a known quota instead of reacting after the fact.
|
|
98
|
+
let lastLlmCallAt = 0;
|
|
99
|
+
async function throttleRpm(rpm) {
|
|
100
|
+
if (!rpm || rpm <= 0)
|
|
101
|
+
return;
|
|
102
|
+
const minIntervalMs = 60000 / rpm;
|
|
103
|
+
const wait = lastLlmCallAt + minIntervalMs - Date.now();
|
|
104
|
+
if (wait > 0) {
|
|
105
|
+
await sleep(wait);
|
|
106
|
+
}
|
|
107
|
+
lastLlmCallAt = Date.now();
|
|
108
|
+
}
|
|
87
109
|
// ── Progress Display ─────────────────────────────────────────────────────
|
|
88
110
|
const SPINNER_FRAMES = ['⠋', '⠙', '⠹', '⠸', '⠼', '⠴', '⠦', '⠧', '⠇', '⠏'];
|
|
89
111
|
const BAR_WIDTH = 28;
|
|
@@ -406,55 +428,6 @@ CRITICAL RULES:
|
|
|
406
428
|
}
|
|
407
429
|
return (0, json_1.safeJsonParse)(text, {});
|
|
408
430
|
}
|
|
409
|
-
async function resolveConnectionsWithVertex(model, token, projectId, location, sourceNodeId, code, candidateNodeIds) {
|
|
410
|
-
const url = `https://${location}-aiplatform.googleapis.com/v1/projects/${projectId}/locations/${location}/publishers/google/models/${model}:generateContent`;
|
|
411
|
-
const systemPrompt = `You are a codebase indexing assistant. Your job is to analyze the source code of a specific code entity and identify which other known code entities from the provided candidate list it calls or references.
|
|
412
|
-
Return ONLY a valid JSON object matching the schema:
|
|
413
|
-
{
|
|
414
|
-
"connections": [
|
|
415
|
-
"target_node_id_1",
|
|
416
|
-
"target_node_id_2"
|
|
417
|
-
]
|
|
418
|
-
}
|
|
419
|
-
CRITICAL RULES:
|
|
420
|
-
1. ONLY return target node IDs that are present in the provided list of known candidates. Do NOT invent new node IDs.
|
|
421
|
-
2. DO NOT include connections to third-party libraries, language built-ins, or the source node itself.
|
|
422
|
-
3. DO NOT wrap JSON in markdown blocks (e.g. no \`\`\`json). Return raw JSON.
|
|
423
|
-
4. If no connections are found, return an empty array.`;
|
|
424
|
-
const payload = {
|
|
425
|
-
contents: [
|
|
426
|
-
{
|
|
427
|
-
role: 'user',
|
|
428
|
-
parts: [
|
|
429
|
-
{
|
|
430
|
-
text: `Source Node ID: ${sourceNodeId}\n\nSource Code:\n${code}\n\nCandidate Target Node IDs in the Codebase:\n${JSON.stringify(candidateNodeIds, null, 2)}`
|
|
431
|
-
}
|
|
432
|
-
]
|
|
433
|
-
}
|
|
434
|
-
],
|
|
435
|
-
systemInstruction: {
|
|
436
|
-
parts: [
|
|
437
|
-
{
|
|
438
|
-
text: systemPrompt
|
|
439
|
-
}
|
|
440
|
-
]
|
|
441
|
-
},
|
|
442
|
-
generationConfig: {
|
|
443
|
-
responseMimeType: 'application/json'
|
|
444
|
-
}
|
|
445
|
-
};
|
|
446
|
-
const responseText = await makeHttpRequest(url, 'POST', {
|
|
447
|
-
'Content-Type': 'application/json',
|
|
448
|
-
'Authorization': `Bearer ${token}`
|
|
449
|
-
}, JSON.stringify(payload));
|
|
450
|
-
const parsed = (0, json_1.safeJsonParse)(responseText, {});
|
|
451
|
-
const text = parsed.candidates?.[0]?.content?.parts?.[0]?.text;
|
|
452
|
-
if (!text) {
|
|
453
|
-
return [];
|
|
454
|
-
}
|
|
455
|
-
const result = (0, json_1.safeJsonParse)(text, {});
|
|
456
|
-
return result.connections || [];
|
|
457
|
-
}
|
|
458
431
|
async function extractWithGemini(model, key, filePath, code) {
|
|
459
432
|
const url = `https://generativelanguage.googleapis.com/v1beta/models/${model}:generateContent?key=${key}`;
|
|
460
433
|
const systemPrompt = `You are a codebase indexing assistant. Your job is to analyze the source code file provided and extract all code structures (functions, methods, classes, controllers, services, interfaces, schema models, types) defined in the file.
|
|
@@ -543,149 +516,209 @@ CRITICAL RULES:
|
|
|
543
516
|
}
|
|
544
517
|
return (0, json_1.safeJsonParse)(text, {});
|
|
545
518
|
}
|
|
546
|
-
async function
|
|
547
|
-
|
|
548
|
-
const
|
|
549
|
-
|
|
550
|
-
|
|
551
|
-
|
|
552
|
-
|
|
553
|
-
|
|
554
|
-
|
|
555
|
-
|
|
556
|
-
|
|
557
|
-
|
|
558
|
-
|
|
559
|
-
|
|
560
|
-
|
|
561
|
-
|
|
562
|
-
|
|
563
|
-
|
|
564
|
-
|
|
565
|
-
|
|
566
|
-
|
|
567
|
-
}
|
|
568
|
-
]
|
|
519
|
+
async function extractNodesFromCode(provider, modelName, key, url, filePath, code, getVertexToken, vertexProjectId, vertexLocation, progress, chunkSize, chunkOverlap, rpm) {
|
|
520
|
+
// Chunking is opt-in: with no --chunk-size, the whole file always goes in one call.
|
|
521
|
+
const maxLines = chunkSize;
|
|
522
|
+
const overlap = chunkOverlap ?? 50;
|
|
523
|
+
const lines = code.split('\n');
|
|
524
|
+
const executeExtraction = async (codeChunk) => {
|
|
525
|
+
let retries = 5;
|
|
526
|
+
let backoffMs = 10000;
|
|
527
|
+
while (retries > 0) {
|
|
528
|
+
try {
|
|
529
|
+
await throttleRpm(rpm);
|
|
530
|
+
if (provider === 'gemini') {
|
|
531
|
+
return await extractWithGemini(modelName, key, filePath, codeChunk);
|
|
532
|
+
}
|
|
533
|
+
else if (provider === 'vertex') {
|
|
534
|
+
const token = await getVertexToken();
|
|
535
|
+
return await extractWithVertex(modelName, token, vertexProjectId, vertexLocation, filePath, codeChunk);
|
|
536
|
+
}
|
|
537
|
+
else {
|
|
538
|
+
return await extractWithOllama(url, modelName, filePath, codeChunk);
|
|
539
|
+
}
|
|
569
540
|
}
|
|
570
|
-
|
|
571
|
-
|
|
572
|
-
|
|
573
|
-
|
|
574
|
-
text: systemPrompt
|
|
541
|
+
catch (err) {
|
|
542
|
+
retries--;
|
|
543
|
+
if (retries === 0) {
|
|
544
|
+
throw err;
|
|
575
545
|
}
|
|
576
|
-
|
|
577
|
-
|
|
578
|
-
|
|
579
|
-
|
|
546
|
+
const errMsg = err.message;
|
|
547
|
+
if (errMsg.includes('429')) {
|
|
548
|
+
progress.updateStatus(`Rate limited (429). Retrying in ${backoffMs / 1000}s...`);
|
|
549
|
+
await sleep(backoffMs);
|
|
550
|
+
backoffMs *= 2;
|
|
551
|
+
}
|
|
552
|
+
else {
|
|
553
|
+
progress.updateStatus(`API error. Retrying in 2s...`);
|
|
554
|
+
await sleep(2000);
|
|
555
|
+
}
|
|
556
|
+
}
|
|
580
557
|
}
|
|
558
|
+
return {};
|
|
581
559
|
};
|
|
582
|
-
|
|
583
|
-
|
|
584
|
-
const text = parsed.candidates?.[0]?.content?.parts?.[0]?.text;
|
|
585
|
-
if (!text) {
|
|
586
|
-
return [];
|
|
560
|
+
if (!maxLines || lines.length <= maxLines) {
|
|
561
|
+
return await executeExtraction(code);
|
|
587
562
|
}
|
|
588
|
-
const
|
|
589
|
-
|
|
590
|
-
|
|
591
|
-
|
|
592
|
-
|
|
593
|
-
const
|
|
594
|
-
|
|
595
|
-
|
|
596
|
-
|
|
597
|
-
|
|
598
|
-
|
|
599
|
-
|
|
600
|
-
}
|
|
601
|
-
CRITICAL RULES:
|
|
602
|
-
1. ONLY return target node IDs that are present in the provided list of known candidates. Do NOT invent new node IDs.
|
|
603
|
-
2. Return a clean, valid JSON object.`;
|
|
604
|
-
const userPrompt = `Source Node ID: ${sourceNodeId}\n\nSource Code:\n${code}\n\nCandidate Target Node IDs:\n${JSON.stringify(candidateNodeIds, null, 2)}`;
|
|
605
|
-
const payload = {
|
|
606
|
-
model,
|
|
607
|
-
messages: [
|
|
608
|
-
{ role: 'system', content: systemPrompt },
|
|
609
|
-
{ role: 'user', content: userPrompt }
|
|
610
|
-
],
|
|
611
|
-
stream: false,
|
|
612
|
-
format: 'json'
|
|
613
|
-
};
|
|
614
|
-
const responseText = await makeHttpRequest(endpoint, 'POST', { 'Content-Type': 'application/json' }, JSON.stringify(payload));
|
|
615
|
-
const parsed = (0, json_1.safeJsonParse)(responseText, {});
|
|
616
|
-
const text = parsed.message?.content;
|
|
617
|
-
if (!text) {
|
|
618
|
-
return [];
|
|
563
|
+
const relPath = path.relative(process.cwd(), filePath);
|
|
564
|
+
progress.log(`\x1B[90m[${relPath}] Large file (${lines.length} lines) - parsing in chunks to prevent LLM output truncation...\x1B[0m`);
|
|
565
|
+
const chunks = [];
|
|
566
|
+
let start = 0;
|
|
567
|
+
// Guard against overlap >= chunkSize, which would make the step <= 0 and loop forever.
|
|
568
|
+
const step = Math.max(1, maxLines - overlap);
|
|
569
|
+
while (start < lines.length) {
|
|
570
|
+
const end = Math.min(start + maxLines, lines.length);
|
|
571
|
+
chunks.push(lines.slice(start, end).join('\n'));
|
|
572
|
+
if (end === lines.length)
|
|
573
|
+
break;
|
|
574
|
+
start += step;
|
|
619
575
|
}
|
|
620
|
-
const
|
|
621
|
-
|
|
622
|
-
}
|
|
623
|
-
|
|
624
|
-
|
|
625
|
-
|
|
626
|
-
|
|
627
|
-
|
|
628
|
-
|
|
629
|
-
|
|
630
|
-
|
|
631
|
-
|
|
632
|
-
|
|
633
|
-
|
|
576
|
+
const seenNodeIds = new Map();
|
|
577
|
+
for (let i = 0; i < chunks.length; i++) {
|
|
578
|
+
progress.updateStatus(`Sending chunk ${i + 1}/${chunks.length} to AI…`);
|
|
579
|
+
const chunkCode = chunks[i];
|
|
580
|
+
try {
|
|
581
|
+
const chunkResult = await executeExtraction(chunkCode);
|
|
582
|
+
if (chunkResult.nodes && Array.isArray(chunkResult.nodes)) {
|
|
583
|
+
for (const node of chunkResult.nodes) {
|
|
584
|
+
if (!node.node_id)
|
|
585
|
+
continue;
|
|
586
|
+
const existing = seenNodeIds.get(node.node_id);
|
|
587
|
+
if (!existing || (node.code_snapshot && (!existing.code_snapshot || node.code_snapshot.length > existing.code_snapshot.length))) {
|
|
588
|
+
seenNodeIds.set(node.node_id, node);
|
|
589
|
+
}
|
|
590
|
+
}
|
|
591
|
+
}
|
|
592
|
+
}
|
|
593
|
+
catch (err) {
|
|
594
|
+
progress.log(`\x1B[31mError extracting chunk ${i + 1}/${chunks.length}: ${err.message}\x1B[0m`);
|
|
595
|
+
throw err;
|
|
596
|
+
}
|
|
597
|
+
if (i < chunks.length - 1) {
|
|
598
|
+
if (provider === 'gemini' || provider === 'vertex') {
|
|
599
|
+
await sleep(1500);
|
|
600
|
+
}
|
|
601
|
+
else {
|
|
602
|
+
await sleep(100);
|
|
603
|
+
}
|
|
604
|
+
}
|
|
605
|
+
}
|
|
606
|
+
return { nodes: Array.from(seenNodeIds.values()) };
|
|
634
607
|
}
|
|
635
608
|
async function runBackgroundIndexing(opts) {
|
|
636
609
|
const resolvedDevmind = path.resolve(opts.devmindPath);
|
|
610
|
+
const chunkSize = opts.chunkSize;
|
|
611
|
+
const chunkOverlap = opts.chunkOverlap;
|
|
612
|
+
const fromScratch = !!opts.fromScratch;
|
|
613
|
+
const nodesOnly = !!opts.nodesOnly;
|
|
614
|
+
const edgesOnly = !!opts.edgesOnly;
|
|
615
|
+
const rpm = opts.rpm;
|
|
616
|
+
// Repo scoping: restrict the whole operation to the named repos. Standalone-only.
|
|
617
|
+
const scopedRepos = opts.repos && opts.repos.length ? opts.repos : null;
|
|
618
|
+
const inScope = (nodeId) => !scopedRepos || scopedRepos.some(r => nodeId.startsWith(`{${r}}/`));
|
|
619
|
+
if (scopedRepos) {
|
|
620
|
+
let ctx;
|
|
621
|
+
try {
|
|
622
|
+
ctx = (0, config_1.loadProjectContext)(resolvedDevmind);
|
|
623
|
+
}
|
|
624
|
+
catch (err) {
|
|
625
|
+
console.error(`❌ Error: ${err.message}`);
|
|
626
|
+
process.exit(1);
|
|
627
|
+
}
|
|
628
|
+
if (ctx.config.mode !== 'standalone') {
|
|
629
|
+
console.error('❌ Error: --repos only works in standalone mode (embedded projects share one root, so per-repo scoping does not apply).');
|
|
630
|
+
process.exit(1);
|
|
631
|
+
}
|
|
632
|
+
if (fromScratch) {
|
|
633
|
+
console.error('❌ Error: --from-scratch wipes the entire graph, so it cannot be combined with --repos. Drop one of them.');
|
|
634
|
+
process.exit(1);
|
|
635
|
+
}
|
|
636
|
+
const known = new Set(ctx.config.repos.map(r => r.name));
|
|
637
|
+
const unknown = scopedRepos.filter(r => !known.has(r));
|
|
638
|
+
if (unknown.length) {
|
|
639
|
+
console.error(`❌ Error: unknown repo name(s): ${unknown.join(', ')}`);
|
|
640
|
+
console.error(` Valid repos: ${[...known].join(', ')}`);
|
|
641
|
+
process.exit(1);
|
|
642
|
+
}
|
|
643
|
+
console.log(` Scope : repos = ${scopedRepos.join(', ')}`);
|
|
644
|
+
}
|
|
645
|
+
// Scoped runs use a SEPARATE scratchpad so they can't clobber (or mark "complete") the
|
|
646
|
+
// global session — otherwise a later full `index --run` would see the scoped run's
|
|
647
|
+
// "complete" and skip every un-indexed repo.
|
|
648
|
+
const padFile = scopedRepos ? 'index_scratchpad.scoped.json' : undefined;
|
|
649
|
+
// Missing-node detection: resolveConnectionsLocally reports references that resolve to a
|
|
650
|
+
// real repo file with no node (a Phase-1 extraction gap). Deduped by (file, symbol); these
|
|
651
|
+
// are auto-created from the AST and reported at the end of every edge-resolution run.
|
|
652
|
+
const missingRefs = new Map();
|
|
653
|
+
const onMissing = (rec) => {
|
|
654
|
+
const key = rec.targetFile + '' + rec.name;
|
|
655
|
+
let e = missingRefs.get(key);
|
|
656
|
+
if (!e) {
|
|
657
|
+
e = { file: rec.targetFile, symbol: rec.name, referenced_by: new Set() };
|
|
658
|
+
missingRefs.set(key, e);
|
|
659
|
+
}
|
|
660
|
+
e.referenced_by.add(rec.sourceNodeId);
|
|
661
|
+
};
|
|
637
662
|
console.log(`\n🧠 DevsMind Background Indexer`);
|
|
638
663
|
console.log(` Brain directory : ${resolvedDevmind}`);
|
|
639
664
|
console.log(` Provider : ${opts.provider}`);
|
|
665
|
+
console.log(` Connections : local AST resolution (always)`);
|
|
666
|
+
console.log(` Chunking : ${chunkSize ? `${chunkSize} lines (overlap: ${chunkOverlap ?? 50})` : 'off — whole file per call'}`);
|
|
667
|
+
console.log(` Rate limit : ${rpm ? `${rpm} req/min` : 'unthrottled'}`);
|
|
640
668
|
let modelName = opts.model || '';
|
|
641
669
|
let vertexSaData = null;
|
|
642
670
|
let vertexToken = null;
|
|
643
671
|
let vertexProjectId = '';
|
|
644
672
|
let vertexLocation = 'us-central1';
|
|
645
|
-
|
|
646
|
-
|
|
647
|
-
|
|
648
|
-
if (
|
|
649
|
-
|
|
650
|
-
process.
|
|
651
|
-
|
|
652
|
-
|
|
653
|
-
|
|
654
|
-
|
|
655
|
-
|
|
656
|
-
const inputKey = opts.key || process.env.GOOGLE_APPLICATION_CREDENTIALS || process.env.VERTEX_API_KEY || process.env.GEMINI_API_KEY || '';
|
|
657
|
-
if (!inputKey) {
|
|
658
|
-
console.error('❌ Error: Vertex AI requires a Service Account JSON path or Bearer Token. Pass --key or set GOOGLE_APPLICATION_CREDENTIALS / VERTEX_API_KEY environment variable.');
|
|
659
|
-
process.exit(1);
|
|
673
|
+
// --edges-only never calls the LLM (Phase 1 is skipped entirely), so it shouldn't
|
|
674
|
+
// require provider credentials at all.
|
|
675
|
+
if (!edgesOnly) {
|
|
676
|
+
if (opts.provider === 'gemini') {
|
|
677
|
+
modelName = modelName || 'gemini-2.0-flash';
|
|
678
|
+
const apiKey = opts.key || process.env.GEMINI_API_KEY || '';
|
|
679
|
+
if (!apiKey) {
|
|
680
|
+
console.error('❌ Error: Gemini API key is required. Pass --key or set GEMINI_API_KEY environment variable.');
|
|
681
|
+
process.exit(1);
|
|
682
|
+
}
|
|
683
|
+
opts.key = apiKey;
|
|
660
684
|
}
|
|
661
|
-
|
|
662
|
-
|
|
663
|
-
|
|
685
|
+
else if (opts.provider === 'vertex') {
|
|
686
|
+
modelName = modelName || 'gemini-1.5-flash';
|
|
687
|
+
const inputKey = opts.key || process.env.GOOGLE_APPLICATION_CREDENTIALS || process.env.VERTEX_API_KEY || process.env.GEMINI_API_KEY || '';
|
|
688
|
+
if (!inputKey) {
|
|
689
|
+
console.error('❌ Error: Vertex AI requires a Service Account JSON path or Bearer Token. Pass --key or set GOOGLE_APPLICATION_CREDENTIALS / VERTEX_API_KEY environment variable.');
|
|
690
|
+
process.exit(1);
|
|
664
691
|
}
|
|
665
|
-
|
|
666
|
-
|
|
692
|
+
try {
|
|
693
|
+
if (inputKey.trim().startsWith('{')) {
|
|
694
|
+
vertexSaData = JSON.parse(inputKey);
|
|
695
|
+
}
|
|
696
|
+
else if (fs.existsSync(inputKey)) {
|
|
697
|
+
vertexSaData = JSON.parse(fs.readFileSync(inputKey, 'utf-8'));
|
|
698
|
+
}
|
|
699
|
+
}
|
|
700
|
+
catch (e) {
|
|
701
|
+
// Treat as raw token
|
|
702
|
+
}
|
|
703
|
+
vertexProjectId = vertexSaData?.project_id || process.env.GCP_PROJECT_ID || process.env.VERTEX_PROJECT_ID || '';
|
|
704
|
+
vertexLocation = process.env.GCP_LOCATION || process.env.VERTEX_LOCATION || 'us-central1';
|
|
705
|
+
if (!vertexSaData && !inputKey.startsWith('ya29.')) {
|
|
706
|
+
console.error('❌ Error: Vertex key must be a valid Service Account JSON file path, inline JSON, or raw OAuth access token starting with "ya29."');
|
|
707
|
+
process.exit(1);
|
|
708
|
+
}
|
|
709
|
+
if (!vertexProjectId) {
|
|
710
|
+
console.error('❌ Error: Vertex Project ID could not be determined. Please set GCP_PROJECT_ID environment variable or specify it in your service account JSON.');
|
|
711
|
+
process.exit(1);
|
|
712
|
+
}
|
|
713
|
+
if (!vertexSaData) {
|
|
714
|
+
vertexToken = inputKey; // Raw Bearer token
|
|
667
715
|
}
|
|
668
716
|
}
|
|
669
|
-
|
|
670
|
-
|
|
671
|
-
|
|
672
|
-
vertexProjectId = vertexSaData?.project_id || process.env.GCP_PROJECT_ID || process.env.VERTEX_PROJECT_ID || '';
|
|
673
|
-
vertexLocation = process.env.GCP_LOCATION || process.env.VERTEX_LOCATION || 'us-central1';
|
|
674
|
-
if (!vertexSaData && !inputKey.startsWith('ya29.')) {
|
|
675
|
-
console.error('❌ Error: Vertex key must be a valid Service Account JSON file path, inline JSON, or raw OAuth access token starting with "ya29."');
|
|
676
|
-
process.exit(1);
|
|
677
|
-
}
|
|
678
|
-
if (!vertexProjectId) {
|
|
679
|
-
console.error('❌ Error: Vertex Project ID could not be determined. Please set GCP_PROJECT_ID environment variable or specify it in your service account JSON.');
|
|
680
|
-
process.exit(1);
|
|
681
|
-
}
|
|
682
|
-
if (!vertexSaData) {
|
|
683
|
-
vertexToken = inputKey; // Raw Bearer token
|
|
717
|
+
else {
|
|
718
|
+
modelName = modelName || 'qwen2.5-coder';
|
|
719
|
+
opts.url = opts.url || 'http://localhost:11434';
|
|
684
720
|
}
|
|
685
|
-
|
|
686
|
-
else {
|
|
687
|
-
modelName = modelName || 'qwen2.5-coder';
|
|
688
|
-
opts.url = opts.url || 'http://localhost:11434';
|
|
721
|
+
console.log(` Model : ${modelName}`);
|
|
689
722
|
}
|
|
690
723
|
const getVertexToken = async () => {
|
|
691
724
|
if (vertexToken)
|
|
@@ -695,21 +728,127 @@ async function runBackgroundIndexing(opts) {
|
|
|
695
728
|
}
|
|
696
729
|
throw new Error('No Vertex credentials available');
|
|
697
730
|
};
|
|
698
|
-
console.log(` Model : ${modelName}`);
|
|
699
731
|
// 1. Open DB
|
|
700
732
|
const dbFile = path.join(resolvedDevmind, 'brain.db');
|
|
701
733
|
const db = new database_1.DevMindDatabase(dbFile);
|
|
702
|
-
//
|
|
703
|
-
|
|
734
|
+
// --from-scratch: wipe everything (with confirmation) before proceeding.
|
|
735
|
+
if (fromScratch) {
|
|
736
|
+
if (!opts.yes) {
|
|
737
|
+
const confirmResult = await (0, prompts_1.default)({
|
|
738
|
+
type: 'confirm',
|
|
739
|
+
name: 'yes',
|
|
740
|
+
message: '🚨 WARNING: --from-scratch will permanently delete ALL nodes, connections, history, and the committed graph/ and history/ folders, then reindex from zero. Are you absolutely sure?',
|
|
741
|
+
initial: false
|
|
742
|
+
});
|
|
743
|
+
if (!confirmResult.yes) {
|
|
744
|
+
console.log('Aborted — nothing was changed.');
|
|
745
|
+
db.close();
|
|
746
|
+
return;
|
|
747
|
+
}
|
|
748
|
+
}
|
|
749
|
+
console.log('💥 Wiping all nodes, connections, history, and graph/history folders...');
|
|
750
|
+
db.resetAll();
|
|
751
|
+
const scratchpadFile = path.join(resolvedDevmind, 'index_scratchpad.json');
|
|
752
|
+
if (fs.existsSync(scratchpadFile)) {
|
|
753
|
+
fs.unlinkSync(scratchpadFile);
|
|
754
|
+
}
|
|
755
|
+
}
|
|
756
|
+
// --edges-only: skip Phase 1 entirely, wipe existing edges, rebuild them fresh
|
|
757
|
+
// across every currently-active node using the AST resolver. Requires nodes to
|
|
758
|
+
// already exist (from a prior full index or a --nodes-only run).
|
|
759
|
+
if (edgesOnly) {
|
|
760
|
+
// All nodes stay in the candidate pool (targets can live in any repo/file), but when
|
|
761
|
+
// scoped we only rebuild edges ORIGINATING from the named repos' nodes.
|
|
762
|
+
const allNodes = db.listNodes();
|
|
763
|
+
if (allNodes.length === 0) {
|
|
764
|
+
console.error('❌ Error: --edges-only requires nodes to already exist. Run without --edges-only first (or with --nodes-only) to extract nodes.');
|
|
765
|
+
db.close();
|
|
766
|
+
process.exit(1);
|
|
767
|
+
}
|
|
768
|
+
const existingNodes = scopedRepos ? allNodes.filter(n => inScope(n.id)) : allNodes;
|
|
769
|
+
if (existingNodes.length === 0) {
|
|
770
|
+
console.error(`❌ Error: no nodes found for repo(s): ${scopedRepos?.join(', ')}. Extract nodes first.`);
|
|
771
|
+
db.close();
|
|
772
|
+
process.exit(1);
|
|
773
|
+
}
|
|
774
|
+
let edgePad = (0, indexer_1.readScratchpad)(resolvedDevmind, padFile);
|
|
775
|
+
let resumeIndex = 0;
|
|
776
|
+
// Scoped runs never resume the shared scratchpad (its counts describe a different set).
|
|
777
|
+
if (!scopedRepos && edgePad && edgePad.phase === 2 && edgePad.status === 'in_progress' && edgePad.nodes_total === existingNodes.length) {
|
|
778
|
+
resumeIndex = edgePad.nodes_done || 0;
|
|
779
|
+
console.log(`↻ Resuming edge rebuild from node ${resumeIndex + 1}/${existingNodes.length}`);
|
|
780
|
+
}
|
|
781
|
+
else {
|
|
782
|
+
if (scopedRepos) {
|
|
783
|
+
console.log(`🧹 Clearing connections for ${existingNodes.length} node(s) in scope...`);
|
|
784
|
+
db.clearConnectionsForSources(existingNodes.map(n => n.id));
|
|
785
|
+
}
|
|
786
|
+
else {
|
|
787
|
+
console.log('🧹 Clearing existing connections...');
|
|
788
|
+
db.clearAllConnections();
|
|
789
|
+
}
|
|
790
|
+
edgePad = (0, indexer_1.createScratchpad)(resolvedDevmind, 0, padFile);
|
|
791
|
+
edgePad.phase = 2;
|
|
792
|
+
edgePad.nodes_total = existingNodes.length;
|
|
793
|
+
(0, indexer_1.writeScratchpad)(resolvedDevmind, edgePad, padFile);
|
|
794
|
+
}
|
|
795
|
+
const edgeProgress = new ProgressDisplay();
|
|
796
|
+
const allNodeIds = allNodes.map(n => n.id);
|
|
797
|
+
edgeProgress.startPhase(2, 'AI Connection Resolution', existingNodes.length, resumeIndex);
|
|
798
|
+
for (let i = resumeIndex; i < existingNodes.length; i++) {
|
|
799
|
+
const node = existingNodes[i];
|
|
800
|
+
edgeProgress.beginItem(node.id);
|
|
801
|
+
const latestCode = db.getLatestCode(node.id);
|
|
802
|
+
if (!latestCode || !latestCode.code_snapshot || latestCode.code_snapshot.trim().length === 0) {
|
|
803
|
+
edgePad.nodes_done = i + 1;
|
|
804
|
+
(0, indexer_1.writeScratchpad)(resolvedDevmind, edgePad, padFile);
|
|
805
|
+
edgeProgress.skipItem('no code snapshot');
|
|
806
|
+
continue;
|
|
807
|
+
}
|
|
808
|
+
edgeProgress.updateStatus(`Resolving connections locally via AST…`);
|
|
809
|
+
const connections = (0, ast_1.resolveConnectionsLocally)(node.id, node.file_path, allNodes, resolvedDevmind, onMissing);
|
|
810
|
+
let addedCount = 0;
|
|
811
|
+
for (const targetId of connections) {
|
|
812
|
+
if (allNodeIds.includes(targetId)) {
|
|
813
|
+
edgeProgress.log(`Linked: \x1B[36m${node.id}\x1B[0m → \x1B[36m${targetId}\x1B[0m`);
|
|
814
|
+
db.addConnection(node.id, targetId);
|
|
815
|
+
addedCount++;
|
|
816
|
+
}
|
|
817
|
+
}
|
|
818
|
+
edgePad.nodes_done = i + 1;
|
|
819
|
+
edgePad.connections_created += addedCount;
|
|
820
|
+
(0, indexer_1.writeScratchpad)(resolvedDevmind, edgePad, padFile);
|
|
821
|
+
edgeProgress.completeItem(`${edgePad.connections_created} connection(s) created so far`);
|
|
822
|
+
}
|
|
823
|
+
edgeProgress.finishPhase(`Phase 2 done — ${edgePad.connections_created} connection(s) linked across ${existingNodes.length} node(s)`);
|
|
824
|
+
(0, edges_1.finalizeMissingNodes)(resolvedDevmind, db, missingRefs);
|
|
825
|
+
edgePad.status = 'complete';
|
|
826
|
+
edgePad.updated_at = new Date().toISOString();
|
|
827
|
+
(0, indexer_1.writeScratchpad)(resolvedDevmind, edgePad, padFile);
|
|
828
|
+
db.vacuum();
|
|
829
|
+
db.close();
|
|
830
|
+
console.log('');
|
|
831
|
+
console.log('\x1B[1m\x1B[32m ✔ Edge rebuild complete!\x1B[0m');
|
|
832
|
+
console.log(` └─ Connections : \x1B[33m${edgePad.connections_created}\x1B[0m`);
|
|
833
|
+
console.log('');
|
|
834
|
+
return;
|
|
835
|
+
}
|
|
836
|
+
// 2. Scan for repos & files (restricted to scoped repos when --repos is given)
|
|
837
|
+
const scanResult = (0, scanner_1.scanRepoFiles)(resolvedDevmind);
|
|
838
|
+
const repos = scopedRepos
|
|
839
|
+
? scanResult.repos.filter(r => scopedRepos.includes(r.repo_name))
|
|
840
|
+
: scanResult.repos;
|
|
841
|
+
const total_files = repos.reduce((sum, r) => sum + r.files.length, 0);
|
|
704
842
|
if (total_files === 0) {
|
|
705
843
|
console.log('⚠️ No files found to index. Make sure config.json repositories are configured properly.');
|
|
706
844
|
db.close();
|
|
707
845
|
return;
|
|
708
846
|
}
|
|
709
|
-
// 3. Read or create scratchpad
|
|
710
|
-
|
|
711
|
-
|
|
712
|
-
|
|
847
|
+
// 3. Read or create scratchpad. Scoped runs always start a fresh scratchpad — they are
|
|
848
|
+
// targeted re-runs, so they must not be blocked by (or resume) a prior global session.
|
|
849
|
+
let pad = (0, indexer_1.readScratchpad)(resolvedDevmind, padFile);
|
|
850
|
+
if (scopedRepos || !pad) {
|
|
851
|
+
pad = (0, indexer_1.createScratchpad)(resolvedDevmind, total_files, padFile);
|
|
713
852
|
}
|
|
714
853
|
else if (pad.status === 'complete') {
|
|
715
854
|
console.log('✅ Indexing is already completed!');
|
|
@@ -754,48 +893,21 @@ async function runBackgroundIndexing(opts) {
|
|
|
754
893
|
if (code.trim().length === 0) {
|
|
755
894
|
pad.files_done++;
|
|
756
895
|
pad.last_file_indexed = fileObj.absolutePath;
|
|
757
|
-
(0, indexer_1.writeScratchpad)(resolvedDevmind, pad);
|
|
896
|
+
(0, indexer_1.writeScratchpad)(resolvedDevmind, pad, padFile);
|
|
758
897
|
progress.skipItem('empty file');
|
|
759
898
|
continue;
|
|
760
899
|
}
|
|
761
900
|
const fileLines = code.split('\n').length;
|
|
762
901
|
progress.updateStatus(`Reading ${fileLines} lines — sending to AI…`);
|
|
763
902
|
let result = {};
|
|
764
|
-
|
|
765
|
-
|
|
766
|
-
|
|
767
|
-
|
|
768
|
-
|
|
769
|
-
|
|
770
|
-
|
|
771
|
-
|
|
772
|
-
const token = await getVertexToken();
|
|
773
|
-
result = await extractWithVertex(modelName, token, vertexProjectId, vertexLocation, fileObj.absolutePath, code);
|
|
774
|
-
}
|
|
775
|
-
else {
|
|
776
|
-
result = await extractWithOllama(opts.url, modelName, fileObj.absolutePath, code);
|
|
777
|
-
}
|
|
778
|
-
break;
|
|
779
|
-
}
|
|
780
|
-
catch (err) {
|
|
781
|
-
retries--;
|
|
782
|
-
if (retries === 0) {
|
|
783
|
-
progress.finishPhase(`Paused — API error. Run again to resume.`);
|
|
784
|
-
console.error(`❌ ${err.message}`);
|
|
785
|
-
db.close();
|
|
786
|
-
process.exit(1);
|
|
787
|
-
}
|
|
788
|
-
const errMsg = err.message;
|
|
789
|
-
if (errMsg.includes('429')) {
|
|
790
|
-
progress.updateStatus(`Rate limited (429). Retrying in ${backoffMs / 1000}s...`);
|
|
791
|
-
await sleep(backoffMs);
|
|
792
|
-
backoffMs *= 2;
|
|
793
|
-
}
|
|
794
|
-
else {
|
|
795
|
-
progress.updateStatus(`API error. Retrying in 2s...`);
|
|
796
|
-
await sleep(2000);
|
|
797
|
-
}
|
|
798
|
-
}
|
|
903
|
+
try {
|
|
904
|
+
result = await extractNodesFromCode(opts.provider, modelName, opts.key, opts.url, fileObj.absolutePath, code, getVertexToken, vertexProjectId, vertexLocation, progress, chunkSize, chunkOverlap, rpm);
|
|
905
|
+
}
|
|
906
|
+
catch (err) {
|
|
907
|
+
progress.finishPhase(`Paused — API error. Run again to resume.`);
|
|
908
|
+
console.error(`❌ ${err.message}`);
|
|
909
|
+
db.close();
|
|
910
|
+
process.exit(1);
|
|
799
911
|
}
|
|
800
912
|
let newNodesCount = 0;
|
|
801
913
|
const totalNodesFound = result.nodes?.length ?? 0;
|
|
@@ -854,29 +966,44 @@ async function runBackgroundIndexing(opts) {
|
|
|
854
966
|
if (isRepoDone && !pad.repos_done.includes(fileObj.repoName)) {
|
|
855
967
|
pad.repos_done.push(fileObj.repoName);
|
|
856
968
|
}
|
|
857
|
-
(0, indexer_1.writeScratchpad)(resolvedDevmind, pad);
|
|
969
|
+
(0, indexer_1.writeScratchpad)(resolvedDevmind, pad, padFile);
|
|
858
970
|
progress.completeItem(`${pad.nodes_created} node(s) found so far`);
|
|
859
971
|
if (opts.provider === 'gemini' || opts.provider === 'vertex')
|
|
860
972
|
await sleep(2000);
|
|
861
973
|
else
|
|
862
974
|
await sleep(200);
|
|
863
975
|
}
|
|
864
|
-
// Transition to Phase 2
|
|
865
976
|
const activeNodes = db.listNodes();
|
|
866
|
-
pad.phase = 2;
|
|
867
|
-
pad.nodes_total = activeNodes.length;
|
|
868
|
-
pad.nodes_done = 0;
|
|
869
|
-
pad.updated_at = new Date().toISOString();
|
|
870
|
-
(0, indexer_1.writeScratchpad)(resolvedDevmind, pad);
|
|
871
977
|
progress.finishPhase(`Phase 1 done — ${activeNodes.length} node(s) extracted from ${pad.files_done} file(s)`);
|
|
978
|
+
if (nodesOnly) {
|
|
979
|
+
pad.status = 'complete';
|
|
980
|
+
pad.updated_at = new Date().toISOString();
|
|
981
|
+
(0, indexer_1.writeScratchpad)(resolvedDevmind, pad, padFile);
|
|
982
|
+
}
|
|
983
|
+
else {
|
|
984
|
+
// Transition to Phase 2
|
|
985
|
+
pad.phase = 2;
|
|
986
|
+
pad.nodes_total = activeNodes.length;
|
|
987
|
+
pad.nodes_done = 0;
|
|
988
|
+
pad.updated_at = new Date().toISOString();
|
|
989
|
+
(0, indexer_1.writeScratchpad)(resolvedDevmind, pad, padFile);
|
|
990
|
+
}
|
|
872
991
|
}
|
|
873
992
|
// =========================================================================
|
|
874
993
|
// PHASE 2: AI CONNECTION RESOLUTION / LINKING
|
|
875
994
|
// =========================================================================
|
|
876
|
-
if (pad.phase === 2) {
|
|
877
|
-
const
|
|
878
|
-
const allNodeIds =
|
|
995
|
+
if (pad.phase === 2 && !nodesOnly) {
|
|
996
|
+
const allNodes = db.listNodes();
|
|
997
|
+
const allNodeIds = allNodes.map(n => n.id);
|
|
998
|
+
// Candidates are always all nodes; when scoped we only (re)build edges from the
|
|
999
|
+
// named repos' nodes and clear just those first so we don't wipe other repos' edges.
|
|
1000
|
+
const activeNodes = scopedRepos ? allNodes.filter(n => inScope(n.id)) : allNodes;
|
|
879
1001
|
const resumeIndex = pad.nodes_done || 0;
|
|
1002
|
+
if (scopedRepos && resumeIndex === 0) {
|
|
1003
|
+
console.log(`🧹 Clearing connections for ${activeNodes.length} node(s) in scope...`);
|
|
1004
|
+
db.clearConnectionsForSources(activeNodes.map(n => n.id));
|
|
1005
|
+
}
|
|
1006
|
+
pad.nodes_total = activeNodes.length;
|
|
880
1007
|
// Use total node count and resume offset so bar shows true progress
|
|
881
1008
|
progress.startPhase(2, 'AI Connection Resolution', activeNodes.length, resumeIndex);
|
|
882
1009
|
let nodeIndex = resumeIndex;
|
|
@@ -887,56 +1014,12 @@ async function runBackgroundIndexing(opts) {
|
|
|
887
1014
|
if (!latestCode || !latestCode.code_snapshot || latestCode.code_snapshot.trim().length === 0) {
|
|
888
1015
|
pad.nodes_done = nodeIndex + 1;
|
|
889
1016
|
pad.updated_at = new Date().toISOString();
|
|
890
|
-
(0, indexer_1.writeScratchpad)(resolvedDevmind, pad);
|
|
1017
|
+
(0, indexer_1.writeScratchpad)(resolvedDevmind, pad, padFile);
|
|
891
1018
|
progress.skipItem('no code snapshot');
|
|
892
1019
|
continue;
|
|
893
1020
|
}
|
|
894
|
-
|
|
895
|
-
const
|
|
896
|
-
if (filteredCandidates.length === 0) {
|
|
897
|
-
pad.nodes_done = nodeIndex + 1;
|
|
898
|
-
pad.updated_at = new Date().toISOString();
|
|
899
|
-
(0, indexer_1.writeScratchpad)(resolvedDevmind, pad);
|
|
900
|
-
progress.skipItem('no matching candidates');
|
|
901
|
-
continue;
|
|
902
|
-
}
|
|
903
|
-
let connections = [];
|
|
904
|
-
let retries = 5;
|
|
905
|
-
let backoffMs = 10000;
|
|
906
|
-
while (retries > 0) {
|
|
907
|
-
try {
|
|
908
|
-
if (opts.provider === 'gemini') {
|
|
909
|
-
connections = await resolveConnectionsWithGemini(modelName, opts.key, node.id, latestCode.code_snapshot, filteredCandidates);
|
|
910
|
-
}
|
|
911
|
-
else if (opts.provider === 'vertex') {
|
|
912
|
-
const token = await getVertexToken();
|
|
913
|
-
connections = await resolveConnectionsWithVertex(modelName, token, vertexProjectId, vertexLocation, node.id, latestCode.code_snapshot, filteredCandidates);
|
|
914
|
-
}
|
|
915
|
-
else {
|
|
916
|
-
connections = await resolveConnectionsWithOllama(opts.url, modelName, node.id, latestCode.code_snapshot, filteredCandidates);
|
|
917
|
-
}
|
|
918
|
-
break;
|
|
919
|
-
}
|
|
920
|
-
catch (err) {
|
|
921
|
-
retries--;
|
|
922
|
-
if (retries === 0) {
|
|
923
|
-
progress.finishPhase('Paused — API error. Run again to resume.');
|
|
924
|
-
console.error(`❌ ${err.message}`);
|
|
925
|
-
db.close();
|
|
926
|
-
process.exit(1);
|
|
927
|
-
}
|
|
928
|
-
const errMsg = err.message;
|
|
929
|
-
if (errMsg.includes('429')) {
|
|
930
|
-
progress.updateStatus(`Rate limited (429). Retrying in ${backoffMs / 1000}s...`);
|
|
931
|
-
await sleep(backoffMs);
|
|
932
|
-
backoffMs *= 2;
|
|
933
|
-
}
|
|
934
|
-
else {
|
|
935
|
-
progress.updateStatus(`API error. Retrying in 2s...`);
|
|
936
|
-
await sleep(2000);
|
|
937
|
-
}
|
|
938
|
-
}
|
|
939
|
-
}
|
|
1021
|
+
progress.updateStatus(`Resolving connections locally via AST…`);
|
|
1022
|
+
const connections = (0, ast_1.resolveConnectionsLocally)(node.id, node.file_path, allNodes, resolvedDevmind, onMissing);
|
|
940
1023
|
let addedCount = 0;
|
|
941
1024
|
for (const targetId of connections) {
|
|
942
1025
|
if (allNodeIds.includes(targetId)) {
|
|
@@ -948,19 +1031,16 @@ async function runBackgroundIndexing(opts) {
|
|
|
948
1031
|
pad.nodes_done = nodeIndex + 1;
|
|
949
1032
|
pad.connections_created += addedCount;
|
|
950
1033
|
pad.updated_at = new Date().toISOString();
|
|
951
|
-
(0, indexer_1.writeScratchpad)(resolvedDevmind, pad);
|
|
1034
|
+
(0, indexer_1.writeScratchpad)(resolvedDevmind, pad, padFile);
|
|
952
1035
|
progress.completeItem(`${pad.connections_created} connection(s) created so far`);
|
|
953
|
-
if (opts.provider === 'gemini' || opts.provider === 'vertex')
|
|
954
|
-
await sleep(2000);
|
|
955
|
-
else
|
|
956
|
-
await sleep(200);
|
|
957
1036
|
}
|
|
958
1037
|
progress.finishPhase(`Phase 2 done — ${pad.connections_created} connection(s) linked across ${pad.nodes_total} node(s)`);
|
|
1038
|
+
(0, edges_1.finalizeMissingNodes)(resolvedDevmind, db, missingRefs);
|
|
959
1039
|
}
|
|
960
1040
|
// Mark indexing session as fully complete
|
|
961
1041
|
pad.status = 'complete';
|
|
962
1042
|
pad.updated_at = new Date().toISOString();
|
|
963
|
-
(0, indexer_1.writeScratchpad)(resolvedDevmind, pad);
|
|
1043
|
+
(0, indexer_1.writeScratchpad)(resolvedDevmind, pad, padFile);
|
|
964
1044
|
db.vacuum();
|
|
965
1045
|
db.close();
|
|
966
1046
|
console.log('');
|
|
@@ -972,9 +1052,16 @@ async function runBackgroundIndexing(opts) {
|
|
|
972
1052
|
}
|
|
973
1053
|
async function runBackgroundReindexing(opts) {
|
|
974
1054
|
const resolvedDevmind = path.resolve(opts.devmindPath);
|
|
1055
|
+
const chunkSize = opts.chunkSize;
|
|
1056
|
+
const chunkOverlap = opts.chunkOverlap;
|
|
1057
|
+
const rpm = opts.rpm;
|
|
1058
|
+
const fillGaps = !!opts.fillGaps;
|
|
975
1059
|
console.log(`\n🧠 DevsMind Background Reindexer`);
|
|
976
1060
|
console.log(` Brain directory : ${resolvedDevmind}`);
|
|
977
1061
|
console.log(` Provider : ${opts.provider}`);
|
|
1062
|
+
console.log(` Connections : local AST resolution (always)`);
|
|
1063
|
+
console.log(` Chunking : ${chunkSize ? `${chunkSize} lines (overlap: ${chunkOverlap ?? 50})` : 'off — whole file per call'}`);
|
|
1064
|
+
console.log(` Rate limit : ${rpm ? `${rpm} req/min` : 'unthrottled'}`);
|
|
978
1065
|
let modelName = opts.model || '';
|
|
979
1066
|
let vertexSaData = null;
|
|
980
1067
|
let vertexToken = null;
|
|
@@ -1051,36 +1138,62 @@ async function runBackgroundReindexing(opts) {
|
|
|
1051
1138
|
db.close();
|
|
1052
1139
|
return;
|
|
1053
1140
|
}
|
|
1054
|
-
// 4.
|
|
1055
|
-
const lastReindexVal = db.getSystemMeta('last_reindex_at');
|
|
1056
|
-
const lastReindexTime = lastReindexVal ? new Date(lastReindexVal).getTime() : 0;
|
|
1057
|
-
console.log(` Last reindex : ${lastReindexVal ? new Date(lastReindexVal).toLocaleString() : 'Never'}`);
|
|
1058
|
-
// 5. Detect modified or newly added files
|
|
1141
|
+
// 4 & 5. Select files to process.
|
|
1059
1142
|
const modifiedFiles = [];
|
|
1060
|
-
|
|
1061
|
-
|
|
1062
|
-
|
|
1063
|
-
|
|
1064
|
-
if (
|
|
1143
|
+
if (fillGaps) {
|
|
1144
|
+
console.log(' Mode : gap-fill (files with zero graph nodes)');
|
|
1145
|
+
for (const repo of repos) {
|
|
1146
|
+
for (const f of repo.files) {
|
|
1147
|
+
if (db.getNodesByFilePath(f).length === 0) {
|
|
1065
1148
|
modifiedFiles.push({ repoName: repo.repo_name, absolutePath: f });
|
|
1066
1149
|
}
|
|
1067
1150
|
}
|
|
1068
|
-
catch (err) {
|
|
1069
|
-
// ignore errors
|
|
1070
|
-
}
|
|
1071
1151
|
}
|
|
1152
|
+
if (modifiedFiles.length === 0) {
|
|
1153
|
+
console.log('\n✅ No gaps found — every indexable file already has at least one graph node.');
|
|
1154
|
+
db.close();
|
|
1155
|
+
return;
|
|
1156
|
+
}
|
|
1157
|
+
console.log(`\n📝 Found ${modifiedFiles.length} file(s) with zero nodes (never indexed, or dropped by a prior failed run).`);
|
|
1072
1158
|
}
|
|
1073
|
-
|
|
1074
|
-
|
|
1075
|
-
|
|
1076
|
-
|
|
1077
|
-
|
|
1159
|
+
else {
|
|
1160
|
+
const lastReindexVal = db.getSystemMeta('last_reindex_at');
|
|
1161
|
+
const lastReindexTime = lastReindexVal ? new Date(lastReindexVal).getTime() : 0;
|
|
1162
|
+
console.log(` Last reindex : ${lastReindexVal ? new Date(lastReindexVal).toLocaleString() : 'Never'}`);
|
|
1163
|
+
for (const repo of repos) {
|
|
1164
|
+
for (const f of repo.files) {
|
|
1165
|
+
try {
|
|
1166
|
+
const stat = fs.statSync(f);
|
|
1167
|
+
if (stat.mtimeMs > lastReindexTime) {
|
|
1168
|
+
modifiedFiles.push({ repoName: repo.repo_name, absolutePath: f });
|
|
1169
|
+
}
|
|
1170
|
+
}
|
|
1171
|
+
catch (err) {
|
|
1172
|
+
// ignore errors
|
|
1173
|
+
}
|
|
1174
|
+
}
|
|
1175
|
+
}
|
|
1176
|
+
if (modifiedFiles.length === 0) {
|
|
1177
|
+
console.log('\n✅ Code graph is already up to date. No modified files detected.');
|
|
1178
|
+
db.setSystemMeta('last_reindex_at', new Date().toISOString());
|
|
1179
|
+
db.close();
|
|
1180
|
+
return;
|
|
1181
|
+
}
|
|
1182
|
+
console.log(`\n📝 Detected ${modifiedFiles.length} modified/new file(s) since last reindex.`);
|
|
1078
1183
|
}
|
|
1079
|
-
console.log(`\n📝 Detected ${modifiedFiles.length} modified/new file(s) since last reindex.`);
|
|
1080
1184
|
// 6. Extraction & Upserting of modified nodes
|
|
1081
1185
|
const progress = new ProgressDisplay();
|
|
1082
|
-
progress.startPhase(1, 'Incremental Node Extraction', modifiedFiles.length, 0);
|
|
1186
|
+
progress.startPhase(1, fillGaps ? 'Gap-Fill Node Extraction' : 'Incremental Node Extraction', modifiedFiles.length, 0);
|
|
1083
1187
|
const newOrUpdatedNodeIds = [];
|
|
1188
|
+
// Source nodes (in possibly-unmodified files) that had edges pointing INTO the modified
|
|
1189
|
+
// files. deprecateNode below deletes those inbound edges, so we capture their sources here
|
|
1190
|
+
// and re-resolve them after Phase 2 to rebuild the "used-by" edges (else every reindex
|
|
1191
|
+
// silently strips incoming edges from unmodified callers).
|
|
1192
|
+
const inboundSourceIds = new Set();
|
|
1193
|
+
// Gap-fill mode only: files that failed extraction after retries. Logged and skipped
|
|
1194
|
+
// instead of aborting the whole run, so a persistently-failing file never blocks the
|
|
1195
|
+
// rest of the gaps — the run is safe to repeat until this list is empty.
|
|
1196
|
+
const stillFailedFiles = [];
|
|
1084
1197
|
for (let fileIndex = 0; fileIndex < modifiedFiles.length; fileIndex++) {
|
|
1085
1198
|
const fileObj = modifiedFiles[fileIndex];
|
|
1086
1199
|
const relPath = path.relative(process.cwd(), fileObj.absolutePath);
|
|
@@ -1093,9 +1206,12 @@ async function runBackgroundReindexing(opts) {
|
|
|
1093
1206
|
progress.skipItem(`read error: ${err.message}`);
|
|
1094
1207
|
continue;
|
|
1095
1208
|
}
|
|
1096
|
-
// Deprecate existing nodes for this file path before parsing
|
|
1209
|
+
// Deprecate existing nodes for this file path before parsing. Capture inbound-edge
|
|
1210
|
+
// sources FIRST (deprecateNode deletes edges in both directions).
|
|
1097
1211
|
const oldNodes = db.getNodesByFilePath(fileObj.absolutePath);
|
|
1098
1212
|
for (const oldNode of oldNodes) {
|
|
1213
|
+
for (const src of db.getInboundSources(oldNode.id))
|
|
1214
|
+
inboundSourceIds.add(src);
|
|
1099
1215
|
db.deprecateNode(oldNode.id);
|
|
1100
1216
|
}
|
|
1101
1217
|
if (code.trim().length === 0) {
|
|
@@ -1105,41 +1221,19 @@ async function runBackgroundReindexing(opts) {
|
|
|
1105
1221
|
const fileLines = code.split('\n').length;
|
|
1106
1222
|
progress.updateStatus(`Reading ${fileLines} lines — sending to AI…`);
|
|
1107
1223
|
let result = {};
|
|
1108
|
-
|
|
1109
|
-
|
|
1110
|
-
|
|
1111
|
-
|
|
1112
|
-
|
|
1113
|
-
|
|
1114
|
-
}
|
|
1115
|
-
|
|
1116
|
-
const token = await getVertexToken();
|
|
1117
|
-
result = await extractWithVertex(modelName, token, vertexProjectId, vertexLocation, fileObj.absolutePath, code);
|
|
1118
|
-
}
|
|
1119
|
-
else {
|
|
1120
|
-
result = await extractWithOllama(opts.url, modelName, fileObj.absolutePath, code);
|
|
1121
|
-
}
|
|
1122
|
-
break;
|
|
1123
|
-
}
|
|
1124
|
-
catch (err) {
|
|
1125
|
-
retries--;
|
|
1126
|
-
if (retries === 0) {
|
|
1127
|
-
progress.finishPhase(`Paused — API error.`);
|
|
1128
|
-
console.error(`❌ ${err.message}`);
|
|
1129
|
-
db.close();
|
|
1130
|
-
process.exit(1);
|
|
1131
|
-
}
|
|
1132
|
-
const errMsg = err.message;
|
|
1133
|
-
if (errMsg.includes('429')) {
|
|
1134
|
-
progress.updateStatus(`Rate limited (429). Retrying in ${backoffMs / 1000}s...`);
|
|
1135
|
-
await sleep(backoffMs);
|
|
1136
|
-
backoffMs *= 2;
|
|
1137
|
-
}
|
|
1138
|
-
else {
|
|
1139
|
-
progress.updateStatus(`API error. Retrying in 2s...`);
|
|
1140
|
-
await sleep(2000);
|
|
1141
|
-
}
|
|
1224
|
+
try {
|
|
1225
|
+
result = await extractNodesFromCode(opts.provider, modelName, opts.key, opts.url, fileObj.absolutePath, code, getVertexToken, vertexProjectId, vertexLocation, progress, chunkSize, chunkOverlap, rpm);
|
|
1226
|
+
}
|
|
1227
|
+
catch (err) {
|
|
1228
|
+
if (fillGaps) {
|
|
1229
|
+
stillFailedFiles.push(relPath);
|
|
1230
|
+
progress.skipItem(`extraction failed: ${err.message}`);
|
|
1231
|
+
continue;
|
|
1142
1232
|
}
|
|
1233
|
+
progress.finishPhase(`Paused — API error.`);
|
|
1234
|
+
console.error(`❌ ${err.message}`);
|
|
1235
|
+
db.close();
|
|
1236
|
+
process.exit(1);
|
|
1143
1237
|
}
|
|
1144
1238
|
let fileNodesCount = 0;
|
|
1145
1239
|
if (result.nodes && Array.isArray(result.nodes)) {
|
|
@@ -1190,6 +1284,52 @@ async function runBackgroundReindexing(opts) {
|
|
|
1190
1284
|
await sleep(200);
|
|
1191
1285
|
}
|
|
1192
1286
|
progress.finishPhase(`Phase 1 done — parsed ${modifiedFiles.length} file(s), found ${newOrUpdatedNodeIds.length} new/updated node(s)`);
|
|
1287
|
+
if (fillGaps) {
|
|
1288
|
+
// Full graph-wide edge rebuild instead of the incremental Phase 2/2b: a newly-added
|
|
1289
|
+
// node can be the TARGET of edges from files that were already indexed (the resolver
|
|
1290
|
+
// couldn't create those edges earlier because the target didn't exist yet), so only
|
|
1291
|
+
// re-resolving the new nodes' own outbound edges isn't enough. This is local AST
|
|
1292
|
+
// resolution (no LLM calls), so rebuilding it across the whole graph is cheap and
|
|
1293
|
+
// safe to repeat.
|
|
1294
|
+
const activeNodes = db.listNodes();
|
|
1295
|
+
const allNodeIds = new Set(activeNodes.map(n => n.id));
|
|
1296
|
+
console.log('\n🧹 Clearing existing connections for a full rebuild...');
|
|
1297
|
+
db.clearAllConnections();
|
|
1298
|
+
progress.startPhase(2, 'Full Graph Edge Rebuild', activeNodes.length, 0);
|
|
1299
|
+
let totalConnections = 0;
|
|
1300
|
+
for (const node of activeNodes) {
|
|
1301
|
+
progress.beginItem(node.id);
|
|
1302
|
+
if (!node.file_path) {
|
|
1303
|
+
progress.skipItem('no file_path');
|
|
1304
|
+
continue;
|
|
1305
|
+
}
|
|
1306
|
+
progress.updateStatus(`Resolving connections locally via AST…`);
|
|
1307
|
+
const connections = (0, ast_1.resolveConnectionsLocally)(node.id, node.file_path, activeNodes, resolvedDevmind);
|
|
1308
|
+
let added = 0;
|
|
1309
|
+
for (const targetId of connections) {
|
|
1310
|
+
if (allNodeIds.has(targetId)) {
|
|
1311
|
+
db.addConnection(node.id, targetId);
|
|
1312
|
+
added++;
|
|
1313
|
+
}
|
|
1314
|
+
}
|
|
1315
|
+
totalConnections += added;
|
|
1316
|
+
progress.completeItem(`${added} connection(s)`);
|
|
1317
|
+
}
|
|
1318
|
+
progress.finishPhase(`Phase 2 done — ${totalConnections} connection(s) rebuilt across ${activeNodes.length} node(s)`);
|
|
1319
|
+
db.vacuum();
|
|
1320
|
+
db.close();
|
|
1321
|
+
console.log('\n\x1B[1m\x1B[32m ✔ Gap-fill complete!\x1B[0m');
|
|
1322
|
+
console.log(` └─ Files backfilled : ${modifiedFiles.length - stillFailedFiles.length}/${modifiedFiles.length}`);
|
|
1323
|
+
console.log(` └─ Nodes added : ${newOrUpdatedNodeIds.length}`);
|
|
1324
|
+
console.log(` └─ Connections : ${totalConnections}`);
|
|
1325
|
+
if (stillFailedFiles.length > 0) {
|
|
1326
|
+
console.log(`\n\x1B[33m⚠️ ${stillFailedFiles.length} file(s) still failed extraction — re-run --fill-gaps to retry them:\x1B[0m`);
|
|
1327
|
+
for (const f of stillFailedFiles)
|
|
1328
|
+
console.log(` - ${f}`);
|
|
1329
|
+
}
|
|
1330
|
+
console.log('');
|
|
1331
|
+
return;
|
|
1332
|
+
}
|
|
1193
1333
|
// Phase 2: Resolving connections for modified nodes
|
|
1194
1334
|
if (newOrUpdatedNodeIds.length > 0) {
|
|
1195
1335
|
const activeNodes = db.listNodes();
|
|
@@ -1203,48 +1343,11 @@ async function runBackgroundReindexing(opts) {
|
|
|
1203
1343
|
progress.skipItem('no code snapshot');
|
|
1204
1344
|
continue;
|
|
1205
1345
|
}
|
|
1206
|
-
|
|
1207
|
-
const filteredCandidates = candidates.filter(id => id !== nodeId);
|
|
1208
|
-
if (filteredCandidates.length === 0) {
|
|
1209
|
-
progress.skipItem('no matching candidates');
|
|
1210
|
-
continue;
|
|
1211
|
-
}
|
|
1346
|
+
progress.updateStatus(`Resolving connections locally via AST…`);
|
|
1212
1347
|
let connections = [];
|
|
1213
|
-
|
|
1214
|
-
|
|
1215
|
-
|
|
1216
|
-
try {
|
|
1217
|
-
if (opts.provider === 'gemini') {
|
|
1218
|
-
connections = await resolveConnectionsWithGemini(modelName, opts.key, nodeId, latestCode.code_snapshot, filteredCandidates);
|
|
1219
|
-
}
|
|
1220
|
-
else if (opts.provider === 'vertex') {
|
|
1221
|
-
const token = await getVertexToken();
|
|
1222
|
-
connections = await resolveConnectionsWithVertex(modelName, token, vertexProjectId, vertexLocation, nodeId, latestCode.code_snapshot, filteredCandidates);
|
|
1223
|
-
}
|
|
1224
|
-
else {
|
|
1225
|
-
connections = await resolveConnectionsWithOllama(opts.url, modelName, nodeId, latestCode.code_snapshot, filteredCandidates);
|
|
1226
|
-
}
|
|
1227
|
-
break;
|
|
1228
|
-
}
|
|
1229
|
-
catch (err) {
|
|
1230
|
-
retries--;
|
|
1231
|
-
if (retries === 0) {
|
|
1232
|
-
progress.finishPhase('Paused — API error.');
|
|
1233
|
-
console.error(`❌ ${err.message}`);
|
|
1234
|
-
db.close();
|
|
1235
|
-
process.exit(1);
|
|
1236
|
-
}
|
|
1237
|
-
const errMsg = err.message;
|
|
1238
|
-
if (errMsg.includes('429')) {
|
|
1239
|
-
progress.updateStatus(`Rate limited (429). Retrying in ${backoffMs / 1000}s...`);
|
|
1240
|
-
await sleep(backoffMs);
|
|
1241
|
-
backoffMs *= 2;
|
|
1242
|
-
}
|
|
1243
|
-
else {
|
|
1244
|
-
progress.updateStatus(`API error. Retrying in 2s...`);
|
|
1245
|
-
await sleep(2000);
|
|
1246
|
-
}
|
|
1247
|
-
}
|
|
1348
|
+
const nodeObj = db.getNode(nodeId);
|
|
1349
|
+
if (nodeObj && nodeObj.file_path) {
|
|
1350
|
+
connections = (0, ast_1.resolveConnectionsLocally)(nodeId, nodeObj.file_path, activeNodes, resolvedDevmind);
|
|
1248
1351
|
}
|
|
1249
1352
|
let connectionsAdded = 0;
|
|
1250
1353
|
for (const targetId of connections) {
|
|
@@ -1255,13 +1358,36 @@ async function runBackgroundReindexing(opts) {
|
|
|
1255
1358
|
}
|
|
1256
1359
|
}
|
|
1257
1360
|
progress.completeItem(`${connectionsAdded} connection(s) created`);
|
|
1258
|
-
if (opts.provider === 'gemini' || opts.provider === 'vertex')
|
|
1259
|
-
await sleep(2000);
|
|
1260
|
-
else
|
|
1261
|
-
await sleep(200);
|
|
1262
1361
|
}
|
|
1263
1362
|
progress.finishPhase('Phase 2 done — finished reindexing connections');
|
|
1264
1363
|
}
|
|
1364
|
+
// Phase 2b: rebuild inbound edges. Callers in unmodified files had their edges into the
|
|
1365
|
+
// modified files deleted by deprecateNode; re-resolve those callers so their links to the
|
|
1366
|
+
// modified files' NEW nodes are restored (addConnection is idempotent for unchanged edges).
|
|
1367
|
+
const reresolveSources = [...inboundSourceIds].filter(id => !newOrUpdatedNodeIds.includes(id) && db.getNode(id));
|
|
1368
|
+
if (reresolveSources.length > 0) {
|
|
1369
|
+
const activeNodes = db.listNodes();
|
|
1370
|
+
const allNodeIds = new Set(activeNodes.map(n => n.id));
|
|
1371
|
+
progress.startPhase(2, 'Inbound Edge Rebuild', reresolveSources.length, 0);
|
|
1372
|
+
for (const srcId of reresolveSources) {
|
|
1373
|
+
progress.beginItem(srcId);
|
|
1374
|
+
const srcNode = db.getNode(srcId);
|
|
1375
|
+
if (!srcNode || !srcNode.file_path) {
|
|
1376
|
+
progress.skipItem('missing');
|
|
1377
|
+
continue;
|
|
1378
|
+
}
|
|
1379
|
+
const conns = (0, ast_1.resolveConnectionsLocally)(srcId, srcNode.file_path, activeNodes, resolvedDevmind);
|
|
1380
|
+
let added = 0;
|
|
1381
|
+
for (const targetId of conns) {
|
|
1382
|
+
if (allNodeIds.has(targetId)) {
|
|
1383
|
+
db.addConnection(srcId, targetId);
|
|
1384
|
+
added++;
|
|
1385
|
+
}
|
|
1386
|
+
}
|
|
1387
|
+
progress.completeItem(`${added} connection(s)`);
|
|
1388
|
+
}
|
|
1389
|
+
progress.finishPhase('Phase 2b done — inbound edges rebuilt');
|
|
1390
|
+
}
|
|
1265
1391
|
// Update last_reindex_at
|
|
1266
1392
|
db.setSystemMeta('last_reindex_at', new Date().toISOString());
|
|
1267
1393
|
db.vacuum();
|