clast-ai 0.1.0

package/src/parser/index.ts

@@ -0,0 +1,226 @@
+import TreeSitter from 'web-tree-sitter';
+import { createHash } from 'node:crypto';
+import { readFileSync, readdirSync, statSync } from 'node:fs';
+import { join, extname, relative } from 'node:path';
+import { getLanguageForExtension, getGrammarWasmPath, getSupportedExtensions } from './languages.js';
+import { extractFromTree } from './extractor.js';
+import { resolveCallEdges } from './call-resolver.js';
+import type { ClastDatabase } from '../db/index.js';
+import type { ClastConfig } from '../config/index.js';
+import type { AstNodeRow } from '../db/types.js';
+
+export class ParserManager {
+  private parser: TreeSitter | null = null;
+  private languages = new Map<string, TreeSitter.Language>();
+  private config: ClastConfig;
+  private initialized = false;
+
+  constructor(config: ClastConfig) {
+    this.config = config;
+  }
+
+  async init(): Promise<void> {
+    if (this.initialized) return;
+    await TreeSitter.init();
+    this.parser = new TreeSitter();
+    this.initialized = true;
+  }
+
+  private async getLanguage(langName: string): Promise<TreeSitter.Language | null> {
+    if (this.languages.has(langName)) {
+      return this.languages.get(langName)!;
+    }
+
+    const wasmPath = getGrammarWasmPath(langName);
+    if (!wasmPath) {
+      console.error(`[clast] No WASM grammar found for ${langName}`);
+      return null;
+    }
+
+    try {
+      const language = await TreeSitter.Language.load(wasmPath);
+      this.languages.set(langName, language);
+      return language;
+    } catch (err) {
+      console.error(`[clast] Failed to load grammar for ${langName}:`, err);
+      return null;
+    }
+  }
+
+  async parseSource(source: string, langName: string): Promise<TreeSitter.Tree | null> {
+    await this.init();
+    if (!this.parser) return null;
+
+    const language = await this.getLanguage(langName);
+    if (!language) return null;
+
+    this.parser.setLanguage(language);
+    return this.parser.parse(source);
+  }
+
+  async parseAndStore(filePath: string, projectRoot: string, db: ClastDatabase): Promise<boolean> {
+    const ext = extname(filePath);
+    const langEntry = getLanguageForExtension(ext);
+    if (!langEntry) return false;
+
+    const relPath = relative(projectRoot, filePath).replace(/\\/g, '/');
+
+    let source: string;
+    try {
+      source = readFileSync(filePath, 'utf-8');
+    } catch {
+      return false;
+    }
+
+    const hash = createHash('sha256').update(source).digest('hex');
+    const existingHash = db.queries.getFileHash(relPath);
+    if (existingHash === hash) return false;
+
+    const tree = await this.parseSource(source, langEntry.name);
+    if (!tree) return false;
+
+    const result = extractFromTree(
+      tree,
+      relPath,
+      langEntry.name,
+      source,
+      this.config.maxBodySize
+    );
+
+    // Delete old data for this file and upsert file record BEFORE inserting nodes
+    // (nodes have a FK to files.file_path)
+    db.queries.deleteNodesForFile(relPath);
+    db.queries.upsertFile(relPath, hash, langEntry.name, result.nodes.length);
+
+    // Insert nodes
+    const nodeRows: Omit<AstNodeRow, 'id'>[] = result.nodes.map(n => ({
+      file_path: n.filePath,
+      name: n.name,
+      type: n.type,
+      signature: n.signature,
+      start_line: n.startLine,
+      end_line: n.endLine,
+      start_col: n.startCol,
+      end_col: n.endCol,
+      parent_id: n.parentId,
+      language: n.language,
+      body_text: n.bodyText,
+    }));
+
+    const nodeIds = db.queries.insertNodes(nodeRows);
+
+    // Insert edges
+    const edgeRows = result.edges
+      .filter(e => nodeIds[e.callerNodeIndex] !== undefined)
+      .map(e => ({
+        caller_id: nodeIds[e.callerNodeIndex]!,
+        callee_id: null as number | null,
+        callee_name: e.calleeName,
+        file_path: relPath,
+        line: e.line,
+      }));
+
+    if (edgeRows.length > 0) {
+      db.queries.insertEdges(edgeRows);
+    }
+
+    // Insert comments
+    for (const c of result.comments) {
+      const dbId = nodeIds[c.nodeIndex];
+      if (dbId !== undefined) {
+        db.queries.insertComment(dbId, c.text, c.source);
+      }
+    }
+
+    // Save DB periodically
+    db.save();
+
+    return true;
+  }
+
+  async fullIndex(
+    projectRoot: string,
+    db: ClastDatabase,
+    onProgress?: (file: string, current: number, total: number) => void
+  ): Promise<{ filesProcessed: number; nodesFound: number; edgesResolved: number; durationMs: number }> {
+    await this.init();
+
+    const start = Date.now();
+    const supportedExts = new Set(getSupportedExtensions(this.config.languages));
+    const files = this.collectFiles(projectRoot, supportedExts);
+
+    let filesProcessed = 0;
+    for (let i = 0; i < files.length; i++) {
+      const file = files[i]!;
+      onProgress?.(file, i + 1, files.length);
+      if (await this.parseAndStore(file, projectRoot, db)) {
+        filesProcessed++;
+      }
+    }
+
+    const edgesResolved = resolveCallEdges(db.queries);
+
+    // Final save
+    db.save();
+
+    const status = db.queries.getStatus();
+    return {
+      filesProcessed,
+      nodesFound: status.total_nodes,
+      edgesResolved,
+      durationMs: Date.now() - start,
+    };
+  }
+
+  async incrementalUpdate(filePath: string, projectRoot: string, db: ClastDatabase): Promise<boolean> {
+    const parsed = await this.parseAndStore(filePath, projectRoot, db);
+    if (parsed) {
+      resolveCallEdges(db.queries);
+      db.save();
+    }
+    return parsed;
+  }
+
+  removeFile(filePath: string, projectRoot: string, db: ClastDatabase): void {
+    const relPath = relative(projectRoot, filePath).replace(/\\/g, '/');
+    db.queries.deleteFile(relPath);
+    db.save();
+  }
+
+  private collectFiles(dir: string, supportedExts: Set<string>, depth: number = 0): string[] {
+    if (depth > 20) return [];
+
+    const files: string[] = [];
+    let entries: string[];
+
+    try {
+      entries = readdirSync(dir);
+    } catch {
+      return files;
+    }
+
+    for (const entry of entries) {
+      if (this.config.ignoredPaths.includes(entry)) continue;
+      if (entry.startsWith('.')) continue;
+
+      const fullPath = join(dir, entry);
+      let stat;
+      try {
+        stat = statSync(fullPath);
+      } catch {
+        continue;
+      }
+
+      if (stat.isDirectory()) {
+        files.push(...this.collectFiles(fullPath, supportedExts, depth + 1));
+      } else if (stat.isFile()) {
+        const ext = extname(entry);
+        if (supportedExts.has(ext)) {
+          files.push(fullPath);
+        }
+      }
+    }
+
+    return files;
+  }
+}

package/src/parser/languages.ts

@@ -0,0 +1,86 @@
+import { join, dirname } from 'node:path';
+import { existsSync } from 'node:fs';
+import { fileURLToPath } from 'node:url';
+import { createRequire } from 'node:module';
+
+interface LanguageEntry {
+  name: string;
+  extensions: string[];
+  wasmFile: string;
+}
+
+const LANGUAGES: LanguageEntry[] = [
+  { name: 'typescript', extensions: ['.ts'], wasmFile: 'tree-sitter-typescript.wasm' },
+  { name: 'tsx', extensions: ['.tsx'], wasmFile: 'tree-sitter-tsx.wasm' },
+  { name: 'javascript', extensions: ['.js', '.jsx', '.mjs', '.cjs'], wasmFile: 'tree-sitter-javascript.wasm' },
+  { name: 'python', extensions: ['.py', '.pyw'], wasmFile: 'tree-sitter-python.wasm' },
+  { name: 'java', extensions: ['.java'], wasmFile: 'tree-sitter-java.wasm' },
+  { name: 'csharp', extensions: ['.cs'], wasmFile: 'tree-sitter-c_sharp.wasm' },
+  { name: 'go', extensions: ['.go'], wasmFile: 'tree-sitter-go.wasm' },
+  { name: 'rust', extensions: ['.rs'], wasmFile: 'tree-sitter-rust.wasm' },
+  { name: 'c', extensions: ['.c', '.h'], wasmFile: 'tree-sitter-c.wasm' },
+  { name: 'cpp', extensions: ['.cpp', '.hpp', '.cc', '.hh', '.cxx', '.hxx'], wasmFile: 'tree-sitter-cpp.wasm' },
+  { name: 'ruby', extensions: ['.rb'], wasmFile: 'tree-sitter-ruby.wasm' },
+  { name: 'php', extensions: ['.php'], wasmFile: 'tree-sitter-php.wasm' },
+];
+
+// Extension → language entry mapping
+const EXT_MAP = new Map<string, LanguageEntry>();
+for (const lang of LANGUAGES) {
+  for (const ext of lang.extensions) {
+    EXT_MAP.set(ext, lang);
+  }
+}
+
+export function getLanguageForExtension(ext: string): LanguageEntry | undefined {
+  return EXT_MAP.get(ext.toLowerCase());
+}
+
+export function getGrammarWasmPath(langName: string): string | null {
+  const entry = LANGUAGES.find(l => l.name === langName);
+  if (!entry) return null;
+
+  // Try to resolve from tree-sitter-wasms npm package first
+  const require = createRequire(import.meta.url);
+  try {
+    const wasmsDir = dirname(require.resolve('tree-sitter-wasms/package.json'));
+    const npmPath = join(wasmsDir, 'out', entry.wasmFile);
+    if (existsSync(npmPath)) return npmPath;
+  } catch {
+    // tree-sitter-wasms not installed, try fallbacks
+  }
+
+  const __dirname = dirname(fileURLToPath(import.meta.url));
+
+  // Check grammars/ directory relative to project
+  const candidates = [
+    // Custom grammars dir (env override)
+    process.env.CLAST_GRAMMARS_DIR ? join(process.env.CLAST_GRAMMARS_DIR, entry.wasmFile) : null,
+    // Plugin root
+    process.env.CLAUDE_PLUGIN_ROOT ? join(process.env.CLAUDE_PLUGIN_ROOT, 'grammars', entry.wasmFile) : null,
+    // Relative to dist/
+    join(__dirname, '..', '..', 'grammars', entry.wasmFile),
+    // CWD
+    join(process.cwd(), 'grammars', entry.wasmFile),
+  ].filter(Boolean) as string[];
+
+  for (const candidate of candidates) {
+    if (existsSync(candidate)) return candidate;
+  }
+
+  return null;
+}
+
+export function getSupportedExtensions(enabledLanguages?: string[]): string[] {
+  const result: string[] = [];
+  for (const lang of LANGUAGES) {
+    if (!enabledLanguages || enabledLanguages.includes(lang.name)) {
+      result.push(...lang.extensions);
+    }
+  }
+  return result;
+}
+
+export function getAllLanguageNames(): string[] {
+  return LANGUAGES.map(l => l.name);
+}

package/src/server/index.ts

@@ -0,0 +1,109 @@
+#!/usr/bin/env node
+
+import { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
+import { StdioServerTransport } from '@modelcontextprotocol/sdk/server/stdio.js';
+import { resolve } from 'node:path';
+
+import { loadConfig, resolveDbPath } from '../config/index.js';
+import { ClastDatabase } from '../db/index.js';
+import { ParserManager } from '../parser/index.js';
+import { FileWatcher } from '../watcher/index.js';
+import { LlmClient } from '../llm/index.js';
+import { CommentGenerator } from '../llm/comment-generator.js';
+import { registerAllTools } from './register-tools.js';
+
+async function main() {
+  // Determine project root
+  const projectRoot = resolve(
+    process.env.CLAST_PROJECT_DIR ??
+    process.env.CLAUDE_PROJECT_DIR ??
+    process.cwd()
+  );
+
+  console.error(`[clast] Project root: ${projectRoot}`);
+
+  // Load configuration
+  const config = loadConfig(projectRoot);
+  console.error(`[clast] Config loaded. Languages: ${config.languages.join(', ')}`);
+
+  // Open database (async — sql.js uses WASM)
+  const dbPath = resolveDbPath(projectRoot, config);
+  console.error(`[clast] Database: ${dbPath}`);
+  const db = await ClastDatabase.open(dbPath);
+
+  // Create parser (async — web-tree-sitter uses WASM)
+  const parser = new ParserManager(config);
+  await parser.init();
+
+  // Create watcher
+  const watcher = new FileWatcher(projectRoot, config, db, parser);
+
+  // Create LLM client + comment generator
+  const llmClient = new LlmClient(config.llm.endpoint, config.llm.model, config.llm.apiKey);
+  const commentGen = new CommentGenerator(llmClient, db.queries, config);
+
+  if (llmClient.isConfigured) {
+    console.error(`[clast] LLM configured: ${config.llm.model} @ ${config.llm.endpoint}`);
+  } else {
+    console.error('[clast] LLM not configured (comment generation disabled)');
+  }
+
+  // Create MCP server
+  const server = new McpServer({
+    name: 'clast',
+    version: '0.1.0',
+  });
+
+  // Register all tools
+  registerAllTools(server, db, parser, watcher, commentGen, projectRoot);
+
+  // Perform initial index
+  console.error('[clast] Starting initial index...');
+  const indexResult = await parser.fullIndex(projectRoot, db, (file, current, total) => {
+    if (current % 50 === 0 || current === total) {
+      console.error(`[clast] Indexing: ${current}/${total} files`);
+    }
+  });
+  console.error(
+    `[clast] Index complete: ${indexResult.filesProcessed} files, ` +
+    `${indexResult.nodesFound} nodes, ${indexResult.edgesResolved} edges resolved ` +
+    `(${indexResult.durationMs}ms)`
+  );
+
+  // Start file watcher
+  watcher.start();
+  watcher.on('indexed', ({ filePath, nodesFound }: { filePath: string; nodesFound: number }) => {
+    console.error(`[clast] Updated: ${filePath} (${nodesFound} nodes)`);
+  });
+  watcher.on('deleted', ({ filePath }: { filePath: string }) => {
+    console.error(`[clast] Removed: ${filePath}`);
+  });
+  watcher.on('error', ({ filePath, error }: { filePath: string; error: Error }) => {
+    console.error(`[clast] Error processing ${filePath}:`, error.message);
+  });
+
+  if (watcher.isActive) {
+    console.error('[clast] File watcher active');
+  }
+
+  // Connect to stdio transport
+  const transport = new StdioServerTransport();
+  await server.connect(transport);
+  console.error('[clast] MCP server running on stdio');
+
+  // Graceful shutdown
+  const shutdown = () => {
+    console.error('[clast] Shutting down...');
+    watcher.stop();
+    db.close();
+    process.exit(0);
+  };
+
+  process.on('SIGINT', shutdown);
+  process.on('SIGTERM', shutdown);
+}
+
+main().catch((err) => {
+  console.error('[clast] Fatal error:', err);
+  process.exit(1);
+});

package/src/server/register-tools.ts

@@ -0,0 +1,30 @@
+import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
+import type { ClastDatabase } from '../db/index.js';
+import type { ParserManager } from '../parser/index.js';
+import type { FileWatcher } from '../watcher/index.js';
+import type { CommentGenerator } from '../llm/comment-generator.js';
+
+import { registerSearchTool } from './tools/clast-search.js';
+import { registerCallGraphTool } from './tools/clast-call-graph.js';
+import { registerFileSummaryTool } from './tools/clast-file-summary.js';
+import { registerGetContextTool } from './tools/clast-get-context.js';
+import { registerStatusTool } from './tools/clast-status.js';
+import { registerReindexTool } from './tools/clast-reindex.js';
+import { registerCommentTool } from './tools/clast-comment.js';
+
+export function registerAllTools(
+  server: McpServer,
+  db: ClastDatabase,
+  parser: ParserManager,
+  watcher: FileWatcher,
+  commentGen: CommentGenerator,
+  projectRoot: string
+): void {
+  registerSearchTool(server, db);
+  registerCallGraphTool(server, db);
+  registerFileSummaryTool(server, db);
+  registerGetContextTool(server, db);
+  registerStatusTool(server, db, watcher);
+  registerReindexTool(server, db, parser, projectRoot);
+  registerCommentTool(server, db, commentGen);
+}

package/src/server/tools/clast-call-graph.ts

@@ -0,0 +1,82 @@
+import { z } from 'zod';
+import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
+import type { ClastDatabase } from '../../db/index.js';
+
+export function registerCallGraphTool(server: McpServer, db: ClastDatabase): void {
+  server.tool(
+    'clast_call_graph',
+    'Get the call graph for a function or method: what it calls (callees) and what calls it (callers). Supports configurable traversal depth.',
+    {
+      name: z.string().describe('Function or method name to trace'),
+      depth: z.number().int().min(1).max(5).default(2).describe('How many levels deep to traverse'),
+      direction: z.enum(['callers', 'callees', 'both']).default('both').describe('Direction to traverse'),
+    },
+    async ({ name, depth, direction }) => {
+      const node = db.queries.getNodeByName(name);
+      if (node.length === 0) {
+        return {
+          content: [{
+            type: 'text' as const,
+            text: JSON.stringify({ error: `No symbol found with name "${name}"` }),
+          }],
+        };
+      }
+
+      const result: Record<string, unknown> = {
+        symbol: {
+          name: node[0]!.name,
+          type: node[0]!.type,
+          filePath: node[0]!.file_path,
+          startLine: node[0]!.start_line,
+          endLine: node[0]!.end_line,
+          signature: node[0]!.signature,
+        },
+      };
+
+      if (direction === 'callers' || direction === 'both') {
+        const callers = db.queries.getCallersOf(name, depth);
+        result.callers = callers.map(n => ({
+          name: n.name,
+          type: n.type,
+          filePath: n.file_path,
+          startLine: n.start_line,
+          signature: n.signature,
+        }));
+      }
+
+      if (direction === 'callees' || direction === 'both') {
+        const callees = db.queries.getCalleesOf(name, depth);
+        result.callees = callees.map(n => ({
+          name: n.name,
+          type: n.type,
+          filePath: n.file_path,
+          startLine: n.start_line,
+          signature: n.signature,
+        }));
+      }
+
+      // Also include direct call edge details for the primary node
+      if (node[0]) {
+        const edges = db.queries.getCallEdgesForNode(node[0].id);
+        result.directCalls = {
+          outgoing: edges.outgoing.map(e => ({
+            calleeName: e.callee_name,
+            line: e.line,
+            resolved: e.callee_id !== null,
+          })),
+          incoming: edges.incoming.map(e => ({
+            callerId: e.caller_id,
+            line: e.line,
+          })),
+        };
+      }
+
+      return {
+        content: [{
+          type: 'text' as const,
+          text: JSON.stringify(result, null, 2),
+        }],
+      };
+    }
+  );
+}

package/src/server/tools/clast-comment.ts

@@ -0,0 +1,76 @@
+import { z } from 'zod';
+import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
+import type { ClastDatabase } from '../../db/index.js';
+import type { CommentGenerator } from '../../llm/comment-generator.js';
+
+export function registerCommentTool(
+  server: McpServer,
+  db: ClastDatabase,
+  commentGen: CommentGenerator
+): void {
+  server.tool(
+    'clast_comment',
+    'Get or generate documentation comments for a symbol. By default returns existing comments. Set generate=true to create a new comment via LLM if none exists.',
+    {
+      name: z.string().describe('Symbol name to get/generate comment for'),
+      generate: z.boolean().default(false).describe('If true and no comment exists, generate one via LLM'),
+    },
+    async ({ name, generate }) => {
+      const nodes = db.queries.getNodeByName(name);
+      if (nodes.length === 0) {
+        return {
+          content: [{
+            type: 'text' as const,
+            text: JSON.stringify({ error: `No symbol found with name "${name}"` }),
+          }],
+        };
+      }
+
+      const results = [];
+      for (const node of nodes) {
+        const comments = db.queries.getCommentsForNode(node.id);
+        const original = comments.find(c => c.source === 'original');
+        const generated = comments.find(c => c.source === 'generated');
+
+        const entry: Record<string, unknown> = {
+          name: node.name,
+          type: node.type,
+          filePath: node.file_path,
+          startLine: node.start_line,
+        };
+
+        if (original) {
+          entry.comment = original.text;
+          entry.source = 'original';
+        } else if (generated) {
+          entry.comment = generated.text;
+          entry.source = 'generated';
+        } else if (generate) {
+          if (!commentGen.isAvailable) {
+            entry.error = 'LLM not configured. Set llm.model in clast.config.json.';
+          } else {
+            const result = await commentGen.generateForNode(node.id);
+            if (result) {
+              entry.comment = result;
+              entry.source = 'generated';
+            } else {
+              entry.error = 'Failed to generate comment';
+            }
+          }
+        } else {
+          entry.comment = null;
+          entry.source = null;
+        }
+
+        results.push(entry);
+      }
+
+      return {
+        content: [{
+          type: 'text' as const,
+          text: JSON.stringify(results, null, 2),
+        }],
+      };
+    }
+  );
+}

package/src/server/tools/clast-file-summary.ts

@@ -0,0 +1,59 @@
+import { z } from 'zod';
+import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
+import type { ClastDatabase } from '../../db/index.js';
+
+export function registerFileSummaryTool(server: McpServer, db: ClastDatabase): void {
+  server.tool(
+    'clast_file_summary',
+    'Get a structured summary of all symbols in a file, grouped by type (imports, classes, functions, exports, etc.). Use instead of reading the whole file to understand its structure.',
+    {
+      filePath: z.string().describe('Relative file path (forward slashes, e.g. "src/parser/index.ts")'),
+    },
+    async ({ filePath }) => {
+      // Normalize path
+      const normalized = filePath.replace(/\\/g, '/');
+      const nodes = db.queries.getNodesByFile(normalized);
+
+      if (nodes.length === 0) {
+        return {
+          content: [{
+            type: 'text' as const,
+            text: JSON.stringify({ error: `No indexed data for file "${normalized}". Try clast_reindex first.` }),
+          }],
+        };
+      }
+
+      // Group by type
+      const grouped: Record<string, Array<{
+        name: string;
+        signature: string;
+        startLine: number;
+        endLine: number;
+      }>> = {};
+
+      for (const node of nodes) {
+        const group = grouped[node.type] ??= [];
+        group.push({
+          name: node.name,
+          signature: node.signature,
+          startLine: node.start_line,
+          endLine: node.end_line,
+        });
+      }
+
+      const summary = {
+        filePath: normalized,
+        language: nodes[0]?.language,
+        totalSymbols: nodes.length,
+        symbols: grouped,
+      };
+
+      return {
+        content: [{
+          type: 'text' as const,
+          text: JSON.stringify(summary, null, 2),
+        }],
+      };
+    }
+  );
+}