@rpcajr/smart-graph-indexer 1.0.0

package/src/index.ts

@@ -0,0 +1,216 @@
+#!/usr/bin/env node
+import { Command } from 'commander';
+import { IndexerEngine, createMCPServer } from './mcp_server.js';
+import { generateGraphHtml } from './ui.js';
+import { StdioServerTransport } from '@modelcontextprotocol/sdk/server/stdio.js';
+import { ListRootsResultSchema } from '@modelcontextprotocol/sdk/types.js';
+import * as path from 'path';
+import * as fs from 'fs';
+import { exec } from 'child_process';
+
+const program = new Command();
+
+program
+  .name('graph-indexer')
+  .description('Smart Graph Code Indexer MCP CLI')
+  .version('1.0.0');
+
+program
+  .command('start')
+  .description('Starts the MCP server via stdio')
+  .option('-p, --path <directory>', 'Target project directory to index', '.')
+  .action(async (options) => {
+    try {
+      const projectPath = path.resolve(options.path);
+      
+
+      console.error(`[Start] Initializing SmartGraphIndexerMCP for: ${projectPath}`);
+      
+      const engine = new IndexerEngine(projectPath);
+      const server = createMCPServer(engine);
+      const transport = new StdioServerTransport();
+      
+      await server.connect(transport);
+      console.error('[Start] MCP Server successfully connected via Stdio.');
+
+      // Dynamic workspace path discovery via client roots list
+      let resolvedPath = projectPath;
+      let rootsResult = "none";
+      let rootsError = "none";
+      try {
+        const response = await server.request({ method: 'roots/list' }, ListRootsResultSchema);
+        rootsResult = JSON.stringify(response);
+        if (response && response.roots && response.roots.length > 0) {
+          const rootUri = response.roots[0].uri;
+          if (rootUri.startsWith('file:///')) {
+            // Remove file:/// prefix
+            let pathFromUri = rootUri.substring(8);
+            pathFromUri = decodeURIComponent(pathFromUri);
+            
+            // Normalize Windows/Unix path
+            resolvedPath = path.resolve(pathFromUri);
+            engine.updateProjectPath(resolvedPath);
+          }
+        }
+      } catch (err: any) {
+        rootsError = err?.message || String(err);
+      }
+
+
+      await engine.startIndexing();
+
+      // Handle termination gracefully
+      process.on('SIGINT', async () => {
+        console.error('[Shutdown] Closing watcher...');
+        await engine.shutdown();
+        process.exit(0);
+      });
+      process.on('SIGTERM', async () => {
+        console.error('[Shutdown] Closing watcher...');
+        await engine.shutdown();
+        process.exit(0);
+      });
+    } catch (err) {
+      console.error('Critical error starting the MCP server:', err);
+      process.exit(1);
+    }
+  });
+
+program
+  .command('index')
+  .description('Performs only the initial indexing of the project and exports a JSON structural report')
+  .option('-p, --path <directory>', 'Target project directory to index', '.')
+  .action(async (options) => {
+    try {
+      const projectPath = path.resolve(options.path);
+      console.log(`[Index] Indexing project at: ${projectPath}`);
+      
+      const engine = new IndexerEngine(projectPath);
+      await engine.initialize();
+      
+      await (engine as any).scanDir(projectPath);
+      
+      const report = engine.getReport();
+      console.log(JSON.stringify(report, null, 2));
+      process.exit(0);
+    } catch (err) {
+      console.error('Critical error during indexing:', err);
+      process.exit(1);
+    }
+  });
+
+program
+  .command('view')
+  .description('Maps the project and opens an interactive web graph of dependencies')
+  .option('-p, --path <directory>', 'Target project directory to map', '.')
+  .option('-w, --watch', 'Monitors filesystem changes in real time and regenerates the graph automatically')
+  .action(async (options) => {
+    try {
+      const projectPath = path.resolve(options.path);
+      console.log(`[View] Mapping project dependencies at: ${projectPath}`);
+      
+      let engine: IndexerEngine;
+      
+      const writeHtml = () => {
+        const htmlContent = generateGraphHtml(engine);
+        const mcpDir = path.join(projectPath, '.graph-indexer');
+        if (!fs.existsSync(mcpDir)) {
+          fs.mkdirSync(mcpDir, { recursive: true });
+        }
+        const htmlPath = path.join(mcpDir, 'graph.html');
+        fs.writeFileSync(htmlPath, htmlContent, 'utf-8');
+        console.log(`[View] Interactive graph updated at: ${htmlPath}`);
+      };
+
+      if (options.watch) {
+        engine = new IndexerEngine(projectPath, () => {
+          writeHtml();
+        });
+      } else {
+        engine = new IndexerEngine(projectPath);
+      }
+
+      await engine.initialize();
+      await (engine as any).scanDir(projectPath);
+
+      // Write initial HTML
+      writeHtml();
+
+      const mcpDir = path.join(projectPath, '.graph-indexer');
+      const htmlPath = path.join(mcpDir, 'graph.html');
+
+      // Open in the default browser based on platform
+      console.log('[View] Opening interactive graph in default browser...');
+      const platform = process.platform;
+      const fileUrl = `file:///${htmlPath.replace(/\\/g, '/')}`;
+
+      if (platform === 'win32') {
+        exec(`start "" "${fileUrl}"`);
+      } else if (platform === 'darwin') {
+        exec(`open "${fileUrl}"`);
+      } else {
+        exec(`xdg-open "${fileUrl}"`);
+      }
+
+      if (options.watch) {
+        console.log('[View] Real-time monitoring (--watch) active. Press Ctrl+C to terminate.');
+        engine.startWatcher();
+        
+        process.on('SIGINT', async () => {
+          console.log('[Shutdown] Closing watcher...');
+          await engine.shutdown();
+          process.exit(0);
+        });
+      } else {
+        process.exit(0);
+      }
+    } catch (err) {
+      console.error('Critical error generating graph view:', err);
+      process.exit(1);
+    }
+  });
+
+program
+  .command('init')
+  .description('Initializes the current project directory for SmartGraphIndexer by creating the .graph-indexer config folder')
+  .option('-p, --path <directory>', 'Target directory to initialize', '.')
+  .action(async (options) => {
+    try {
+      const projectPath = path.resolve(options.path);
+      const mcpDir = path.join(projectPath, '.graph-indexer');
+      
+      if (fs.existsSync(mcpDir)) {
+        console.log(`[Init] Project is already initialized at: ${projectPath} (.graph-indexer directory exists).`);
+        process.exit(0);
+      }
+
+      fs.mkdirSync(mcpDir, { recursive: true });
+      
+      // Create initial empty stats file
+      const statsPath = path.join(mcpDir, 'stats.json');
+      const initialStats = {
+        totalCalls: 0,
+        totalTokensSaved: 0,
+        totalTokensActual: 0,
+        totalTokensAlternative: 0,
+        efficiencyPercentage: 0,
+        callsByType: {
+          search_symbols: { calls: 0, saved: 0, actual: 0, alternative: 0, totalTimeMs: 0 },
+          get_file_skeleton: { calls: 0, saved: 0, actual: 0, alternative: 0, totalTimeMs: 0 },
+          get_dependencies: { calls: 0, saved: 0, actual: 0, alternative: 0, totalTimeMs: 0 },
+          read_symbol_body: { calls: 0, saved: 0, actual: 0, alternative: 0, totalTimeMs: 0 }
+        },
+        history: []
+      };
+      fs.writeFileSync(statsPath, JSON.stringify(initialStats, null, 2), 'utf-8');
+
+      console.log(`[Init] Successfully initialized SmartGraphIndexer workspace at: ${projectPath}`);
+      console.log('[Init] The MCP server will now index and analyze this project.');
+      process.exit(0);
+    } catch (err) {
+      console.error('Critical error during initialization:', err);
+      process.exit(1);
+    }
+  });
+
+program.parse(process.argv);

package/src/mcp_server.ts

@@ -0,0 +1,454 @@
+import * as fs from 'fs';
+import * as path from 'path';
+import { Server } from '@modelcontextprotocol/sdk/server/index.js';
+import { StdioServerTransport } from '@modelcontextprotocol/sdk/server/stdio.js';
+import {
+  CallToolRequestSchema,
+  ListToolsRequestSchema,
+} from '@modelcontextprotocol/sdk/types.js';
+import { ASTParser, FileSkeleton, SymbolSkeleton } from './parser.js';
+import { DependencyGraph } from './graph.js';
+import { VectorStore } from './vector.js';
+import { GitAwareWatcher } from './watcher.js';
+import { StatsManager } from './stats.js';
+
+export class IndexerEngine {
+  public projectPath: string;
+  private parser: ASTParser;
+  private graph: DependencyGraph;
+  private vectorStore: VectorStore;
+  private watcher: GitAwareWatcher;
+  private fileSkeletons: Map<string, FileSkeleton> = new Map();
+  private isLoaded = false;
+  private stats: StatsManager;
+
+  constructor(projectPath: string, onUpdate?: () => void) {
+    this.projectPath = path.resolve(projectPath);
+    this.stats = new StatsManager(this.projectPath);
+    this.parser = new ASTParser();
+    this.graph = new DependencyGraph();
+    this.vectorStore = new VectorStore();
+    this.watcher = new GitAwareWatcher({
+      projectPath: this.projectPath,
+      onChange: async (changedFiles) => {
+        console.error(`[Watcher] Files modified, re-indexing: ${changedFiles.join(', ')}`);
+        for (const file of changedFiles) {
+          await this.indexFile(file);
+        }
+        if (onUpdate) {
+          onUpdate();
+        }
+      },
+    });
+  }
+
+  public updateProjectPath(newPath: string, onUpdate?: () => void) {
+    this.projectPath = path.resolve(newPath);
+    this.stats = new StatsManager(this.projectPath);
+    this.watcher = new GitAwareWatcher({
+      projectPath: this.projectPath,
+      onChange: async (changedFiles) => {
+        console.error(`[Watcher] Files modified, re-indexing: ${changedFiles.join(', ')}`);
+        for (const file of changedFiles) {
+          await this.indexFile(file);
+        }
+        if (onUpdate) {
+          onUpdate();
+        }
+      },
+    });
+  }
+
+  public startWatcher() {
+    this.watcher.start();
+  }
+
+  public async initialize() {
+    const wasmDir = path.join(__dirname, 'wasm');
+    await this.parser.init(wasmDir);
+    await this.vectorStore.init();
+  }
+
+  /**
+   * Scans and indexes the entire directory recursively.
+   */
+  public async startIndexing() {
+    await this.initialize();
+    
+    // Check if the workspace is initialized for SmartGraphIndexer (.graph-indexer folder exists)
+    const mcpDir = path.join(this.projectPath, '.graph-indexer');
+    if (!fs.existsSync(mcpDir)) {
+      console.error(`[Indexer] .graph-indexer directory not found at: ${this.projectPath}. SmartGraphIndexerMCP is dormant for this workspace.`);
+      this.isLoaded = true;
+      return; // Do not scan or watch
+    }
+
+    const deltas = await this.watcher.getGitDeltas();
+    console.error(`[Indexer] Git status deltas detected: ${deltas.length} files.`);
+
+    // Recursive scan
+    await this.scanDir(this.projectPath);
+    this.isLoaded = true;
+
+    // Start watching filesystem
+    this.watcher.start();
+    console.error('[Indexer] Full initial indexing completed. File watcher active.');
+  }
+
+  private async scanDir(dir: string) {
+    const entries = fs.readdirSync(dir, { withFileTypes: true });
+
+    for (const entry of entries) {
+      const fullPath = path.join(dir, entry.name);
+      const relativePath = path.relative(this.projectPath, fullPath).replace(/\\/g, '/');
+
+      if (this.watcher.isIgnored(relativePath)) {
+        continue;
+      }
+
+      if (entry.isDirectory()) {
+        await this.scanDir(fullPath);
+      } else if (entry.isFile()) {
+        const ext = path.extname(entry.name).toLowerCase();
+        const supported = [
+          '.cs', '.ts', '.tsx', '.js', '.jsx', '.py',
+          '.java', '.go', '.rs', '.rb',
+          '.cpp', '.hpp', '.cc', '.cxx', '.h', '.c'
+        ];
+        if (supported.includes(ext)) {
+          await this.indexFile(relativePath);
+        }
+      }
+    }
+  }
+
+  /**
+   * Indexes a single file.
+   */
+  public async indexFile(relativeFilePath: string) {
+    const normalizedPath = relativeFilePath.replace(/\\/g, '/');
+    const fullPath = path.resolve(this.projectPath, normalizedPath);
+    if (!fs.existsSync(fullPath)) {
+      this.fileSkeletons.delete(normalizedPath);
+      this.graph.removeFile(normalizedPath);
+      this.vectorStore.removeFileSymbols(normalizedPath);
+      return;
+    }
+
+    try {
+      const content = fs.readFileSync(fullPath, 'utf-8');
+      const hash = this.parser.getHash(content);
+
+      const cached = this.fileSkeletons.get(normalizedPath);
+      if (cached && cached.file_hash === hash) {
+        return;
+      }
+
+      const skeleton = this.parser.parseFile(normalizedPath, content);
+      this.fileSkeletons.set(normalizedPath, skeleton);
+
+      this.graph.updateFileDependencies(normalizedPath, skeleton.dependencies);
+
+      this.vectorStore.removeFileSymbols(normalizedPath);
+      for (const symbol of skeleton.symbols) {
+        await this.vectorStore.addSymbol(normalizedPath, symbol);
+      }
+    } catch (err) {
+      console.error(`Error indexing file ${normalizedPath}:`, err);
+    }
+  }
+
+  /**
+   * Gracefully updates a file on demand (for JIT read operations).
+   */
+  public async ensureFileUpToDate(relativeFilePath: string): Promise<FileSkeleton | undefined> {
+    await this.indexFile(relativeFilePath);
+    return this.fileSkeletons.get(relativeFilePath);
+  }
+
+  public getFileSkeleton(relativeFilePath: string): FileSkeleton | undefined {
+    return this.fileSkeletons.get(relativeFilePath);
+  }
+
+  public getGraph() {
+    return this.graph;
+  }
+
+  public getVectorStore() {
+    return this.vectorStore;
+  }
+
+  public getStatsManager(): StatsManager {
+    return this.stats;
+  }
+
+  public async shutdown() {
+    await this.watcher.stop();
+  }
+
+  public getReport() {
+    return {
+      totalFiles: this.fileSkeletons.size,
+      files: Array.from(this.fileSkeletons.keys()),
+      symbolsCount: this.vectorStore.getAllSymbols().length,
+      dependencyGraph: this.graph.serialize(),
+      tokenStats: this.stats.getStats(),
+    };
+  }
+}
+
+export function createMCPServer(engine: IndexerEngine): Server {
+  const server = new Server(
+    {
+      name: 'SmartGraphIndexerMCP',
+      version: '1.0.0',
+    },
+    {
+      capabilities: {
+        tools: {},
+      },
+    }
+  );
+
+  // 1. List available tools
+  server.setRequestHandler(ListToolsRequestSchema, async () => {
+    return {
+      tools: [
+        {
+          name: 'search_symbols',
+          description: 'Performs hybrid search (semantic via vectors + lexical via text) over the indexed signatures database.',
+          inputSchema: {
+            type: 'object',
+            properties: {
+              query: {
+                type: 'string',
+                description: 'Search term or functional description of the logic you want to find.',
+              },
+              limit: {
+                type: 'integer',
+                description: 'Maximum number of matches to return.',
+                default: 5,
+              },
+            },
+            required: ['query'],
+          },
+        },
+        {
+          name: 'get_file_skeleton',
+          description: 'Returns the complete structure (skeleton) of a specific file without inner implementations.',
+          inputSchema: {
+            type: 'object',
+            properties: {
+              file_path: {
+                type: 'string',
+                description: 'Relative path of the file in the project.',
+              },
+            },
+            required: ['file_path'],
+          },
+        },
+        {
+          name: 'get_dependencies',
+          description: 'Queries the in-memory graph and returns imports/dependants of a specific file.',
+          inputSchema: {
+            type: 'object',
+            properties: {
+              file_path: {
+                type: 'string',
+                description: 'Relative path of the source file.',
+              },
+              direction: {
+                type: 'string',
+                enum: ['incoming', 'outgoing', 'both'],
+                description: 'Direction of dependencies to query.',
+                default: 'both',
+              },
+            },
+            required: ['file_path'],
+          },
+        },
+        {
+          name: 'read_symbol_body',
+          description: 'JIT Extraction. Returns the actual, complete source code of a specific method or class based on coordinates.',
+          inputSchema: {
+            type: 'object',
+            properties: {
+              file_path: {
+                type: 'string',
+                description: 'Relative path of the file.',
+              },
+              symbol_name: {
+                type: 'string',
+                description: 'Exact name of the class or method to read.',
+              },
+            },
+            required: ['file_path', 'symbol_name'],
+          },
+        },
+        {
+          name: 'get_token_savings',
+          description: 'Returns token savings statistics accumulated in the project by using this MCP.',
+          inputSchema: {
+            type: 'object',
+            properties: {},
+          },
+        },
+      ],
+    };
+  });
+
+  // 2. Handle tool calls
+  server.setRequestHandler(CallToolRequestSchema, async (request) => {
+    const { name, arguments: args } = request.params;
+    const startTime = Date.now();
+
+    try {
+      if (name === 'search_symbols') {
+        const query = String(args?.query);
+        const limit = Number(args?.limit ?? 5);
+        const results = await engine.getVectorStore().search(query, limit);
+
+        const actualPayload = JSON.stringify(results, null, 2);
+        let alternativeText = '';
+        for (const res of results) {
+          const fullPath = path.resolve(engine.projectPath, res.symbol.filePath);
+          if (fs.existsSync(fullPath)) {
+            alternativeText += fs.readFileSync(fullPath, 'utf-8');
+          }
+        }
+        
+        const execTimeMs = Date.now() - startTime;
+        engine.getStatsManager().recordCall('search_symbols', actualPayload, alternativeText, `Query: "${query}"`, execTimeMs);
+
+        return {
+          content: [
+            {
+              type: 'text',
+              text: actualPayload,
+            },
+          ],
+        };
+      }
+
+      if (name === 'get_file_skeleton') {
+        const filePath = String(args?.file_path).replace(/\\/g, '/');
+        const skeleton = await engine.ensureFileUpToDate(filePath);
+
+        if (!skeleton) {
+          throw new Error(`File ${filePath} not found or could not be parsed.`);
+        }
+
+        const actualPayload = JSON.stringify(skeleton, null, 2);
+        const fullPath = path.resolve(engine.projectPath, filePath);
+        const alternativeText = fs.existsSync(fullPath) ? fs.readFileSync(fullPath, 'utf-8') : '';
+        
+        const execTimeMs = Date.now() - startTime;
+        engine.getStatsManager().recordCall('get_file_skeleton', actualPayload, alternativeText, filePath, execTimeMs);
+
+        return {
+          content: [
+            {
+              type: 'text',
+              text: actualPayload,
+            },
+          ],
+        };
+      }
+
+      if (name === 'get_dependencies') {
+        const filePath = String(args?.file_path).replace(/\\/g, '/');
+        const direction = String(args?.direction ?? 'both');
+        const graph = engine.getGraph();
+
+        const result: Record<string, string[]> = {};
+        if (direction === 'outgoing' || direction === 'both') {
+          result.outgoing = graph.getOutgoing(filePath);
+        }
+        if (direction === 'incoming' || direction === 'both') {
+          result.incoming = graph.getIncoming(filePath);
+        }
+
+        const actualPayload = JSON.stringify(result, null, 2);
+        const fullPath = path.resolve(engine.projectPath, filePath);
+        const alternativeText = fs.existsSync(fullPath) ? fs.readFileSync(fullPath, 'utf-8') : '';
+        
+        const execTimeMs = Date.now() - startTime;
+        engine.getStatsManager().recordCall('get_dependencies', actualPayload, alternativeText, filePath, execTimeMs);
+
+        return {
+          content: [
+            {
+              type: 'text',
+              text: actualPayload,
+            },
+          ],
+        };
+      }
+
+      if (name === 'read_symbol_body') {
+        const filePath = String(args?.file_path).replace(/\\/g, '/');
+        const symbolName = String(args?.symbol_name);
+
+        const skeleton = await engine.ensureFileUpToDate(filePath);
+        if (!skeleton) {
+          throw new Error(`File ${filePath} not found or could not be parsed.`);
+        }
+
+        const symbol = skeleton.symbols.find(s => s.name === symbolName);
+        if (!symbol) {
+          throw new Error(`Symbol ${symbolName} not found in ${filePath}.`);
+        }
+
+        const fullPath = path.resolve(engine.projectPath, filePath);
+        if (!fs.existsSync(fullPath)) {
+          throw new Error(`File ${filePath} does not exist on disk.`);
+        }
+
+        const fileLines = fs.readFileSync(fullPath, 'utf-8').split('\n');
+        const start = Math.max(0, symbol.start_line - 1);
+        const end = Math.min(fileLines.length, symbol.end_line);
+        const codeSlice = fileLines.slice(start, end).join('\n');
+
+        const fullContent = fileLines.join('\n');
+        
+        const execTimeMs = Date.now() - startTime;
+        engine.getStatsManager().recordCall('read_symbol_body', codeSlice, fullContent, `${filePath} -> ${symbolName}`, execTimeMs);
+
+        return {
+          content: [
+            {
+              type: 'text',
+              text: codeSlice,
+            },
+          ],
+        };
+      }
+
+      if (name === 'get_token_savings') {
+        const stats = engine.getStatsManager().getStats();
+        return {
+          content: [
+            {
+              type: 'text',
+              text: JSON.stringify(stats, null, 2),
+            },
+          ],
+        };
+      }
+
+      throw new Error(`Unknown tool: ${name}`);
+    } catch (err: any) {
+      return {
+        isError: true,
+        content: [
+          {
+            type: 'text',
+            text: err?.message || String(err),
+          },
+        ],
+      };
+    }
+  });
+
+  return server;
+}