clast-ai 0.1.2 → 0.2.0

package/src/parser/index.ts

@@ -1,226 +1,126 @@
-import TreeSitter from 'web-tree-sitter';
-import { createHash } from 'node:crypto';
-import { readFileSync, readdirSync, statSync } from 'node:fs';
-import { join, extname, relative } from 'node:path';
-import { getLanguageForExtension, getGrammarWasmPath, getSupportedExtensions } from './languages.js';
-import { extractFromTree } from './extractor.js';
-import { resolveCallEdges } from './call-resolver.js';
-import type { ClastDatabase } from '../db/index.js';
-import type { ClastConfig } from '../config/index.js';
-import type { AstNodeRow } from '../db/types.js';
-
-export class ParserManager {
-  private parser: TreeSitter | null = null;
-  private languages = new Map<string, TreeSitter.Language>();
-  private config: ClastConfig;
-  private initialized = false;
-
-  constructor(config: ClastConfig) {
-    this.config = config;
-  }
-
-  async init(): Promise<void> {
-    if (this.initialized) return;
-    await TreeSitter.init();
-    this.parser = new TreeSitter();
-    this.initialized = true;
-  }
-
-  private async getLanguage(langName: string): Promise<TreeSitter.Language | null> {
-    if (this.languages.has(langName)) {
-      return this.languages.get(langName)!;
-    }
-
-    const wasmPath = getGrammarWasmPath(langName);
-    if (!wasmPath) {
-      console.error(`[clast] No WASM grammar found for ${langName}`);
-      return null;
-    }
-
-    try {
-      const language = await TreeSitter.Language.load(wasmPath);
-      this.languages.set(langName, language);
-      return language;
-    } catch (err) {
-      console.error(`[clast] Failed to load grammar for ${langName}:`, err);
-      return null;
-    }
+import { Parser, Language } from 'web-tree-sitter';
+import { existsSync } from 'node:fs';
+import { dirname, extname, join } from 'node:path';
+import { createRequire } from 'node:module';
+import { fileURLToPath } from 'node:url';
+import type { Extraction } from '../types.js';
+import type { LangSpec } from './spec.js';
+import { compileSpec, extract, type CompiledSpec } from './extractor.js';
+import { specById, specForExtension } from './registry.js';
+
+let initPromise: Promise<void> | null = null;
+
+function grammarPath(grammar: string): string | null {
+  const file = `tree-sitter-${grammar}.wasm`;
+  const candidates: string[] = [];
+  if (process.env.CLAST_GRAMMARS_DIR) candidates.push(join(process.env.CLAST_GRAMMARS_DIR, file));
+  try {
+    const require = createRequire(import.meta.url);
+    const pkg = dirname(require.resolve('@repomix/tree-sitter-wasms/package.json'));
+    candidates.push(join(pkg, 'out', file));
+  } catch {
+    // package not resolvable; fall through to local grammar dirs
   }
-
-  async parseSource(source: string, langName: string): Promise<TreeSitter.Tree | null> {
-    await this.init();
-    if (!this.parser) return null;
-
-    const language = await this.getLanguage(langName);
-    if (!language) return null;
-
-    this.parser.setLanguage(language);
-    return this.parser.parse(source);
+  const here = dirname(fileURLToPath(import.meta.url));
+  candidates.push(join(here, '..', '..', 'grammars', file));
+  for (const c of candidates) {
+    if (existsSync(c)) return c;
   }
+  return null;
+}
 
-  async parseAndStore(filePath: string, projectRoot: string, db: ClastDatabase): Promise<boolean> {
-    const ext = extname(filePath);
-    const langEntry = getLanguageForExtension(ext);
-    if (!langEntry) return false;
-
-    const relPath = relative(projectRoot, filePath).replace(/\\/g, '/');
-
-    let source: string;
-    try {
-      source = readFileSync(filePath, 'utf-8');
-    } catch {
-      return false;
-    }
-
-    const hash = createHash('sha256').update(source).digest('hex');
-    const existingHash = db.queries.getFileHash(relPath);
-    if (existingHash === hash) return false;
-
-    const tree = await this.parseSource(source, langEntry.name);
-    if (!tree) return false;
-
-    const result = extractFromTree(
-      tree,
-      relPath,
-      langEntry.name,
-      source,
-      this.config.maxBodySize
-    );
-
-    // Delete old data for this file and upsert file record BEFORE inserting nodes
-    // (nodes have a FK to files.file_path)
-    db.queries.deleteNodesForFile(relPath);
-    db.queries.upsertFile(relPath, hash, langEntry.name, result.nodes.length);
-
-    // Insert nodes
-    const nodeRows: Omit<AstNodeRow, 'id'>[] = result.nodes.map(n => ({
-      file_path: n.filePath,
-      name: n.name,
-      type: n.type,
-      signature: n.signature,
-      start_line: n.startLine,
-      end_line: n.endLine,
-      start_col: n.startCol,
-      end_col: n.endCol,
-      parent_id: n.parentId,
-      language: n.language,
-      body_text: n.bodyText,
-    }));
-
-    const nodeIds = db.queries.insertNodes(nodeRows);
-
-    // Insert edges
-    const edgeRows = result.edges
-      .filter(e => nodeIds[e.callerNodeIndex] !== undefined)
-      .map(e => ({
-        caller_id: nodeIds[e.callerNodeIndex]!,
-        callee_id: null as number | null,
-        callee_name: e.calleeName,
-        file_path: relPath,
-        line: e.line,
-      }));
-
-    if (edgeRows.length > 0) {
-      db.queries.insertEdges(edgeRows);
-    }
-
-    // Insert comments
-    for (const c of result.comments) {
-      const dbId = nodeIds[c.nodeIndex];
-      if (dbId !== undefined) {
-        db.queries.insertComment(dbId, c.text, c.source);
-      }
-    }
-
-    // Save DB periodically
-    db.save();
+/**
+ * Owns one tree-sitter parser and lazily loaded grammars. One instance per
+ * thread (the main thread and each indexing worker have their own).
+ */
+export class SourceParser {
+  private parser: Parser | null = null;
+  private compiled = new Map<string, CompiledSpec | null>();
 
-    return true;
+  async init(): Promise<void> {
+    initPromise ??= Parser.init();
+    await initPromise;
+    this.parser ??= new Parser();
   }
 
-  async fullIndex(
-    projectRoot: string,
-    db: ClastDatabase,
-    onProgress?: (file: string, current: number, total: number) => void
-  ): Promise<{ filesProcessed: number; nodesFound: number; edgesResolved: number; durationMs: number }> {
+  /** Load (once) and compile the spec's grammar. Returns null if unavailable. */
+  async load(spec: LangSpec): Promise<CompiledSpec | null> {
+    const cached = this.compiled.get(spec.id);
+    if (cached !== undefined) return cached;
     await this.init();
-
-    const start = Date.now();
-    const supportedExts = new Set(getSupportedExtensions(this.config.languages));
-    const files = this.collectFiles(projectRoot, supportedExts);
-
-    let filesProcessed = 0;
-    for (let i = 0; i < files.length; i++) {
-      const file = files[i]!;
-      onProgress?.(file, i + 1, files.length);
-      if (await this.parseAndStore(file, projectRoot, db)) {
-        filesProcessed++;
+    let result: CompiledSpec | null = null;
+    const path = grammarPath(spec.grammar);
+    if (!path) {
+      console.error(`[clast] No grammar found for ${spec.id} (tree-sitter-${spec.grammar}.wasm)`);
+    } else {
+      try {
+        const language = await Language.load(path);
+        result = compileSpec(spec, language);
+        if (result.unknownTypes.length && process.env.CLAST_DEBUG) {
+          console.error(`[clast] ${spec.id}: node types not in grammar: ${result.unknownTypes.join(', ')}`);
+        }
+      } catch (err) {
+        console.error(`[clast] Failed to load grammar for ${spec.id}:`, err);
       }
     }
-
-    const edgesResolved = resolveCallEdges(db.queries);
-
-    // Final save
-    db.save();
-
-    const status = db.queries.getStatus();
-    return {
-      filesProcessed,
-      nodesFound: status.total_nodes,
-      edgesResolved,
-      durationMs: Date.now() - start,
-    };
+    this.compiled.set(spec.id, result);
+    return result;
   }
 
-  async incrementalUpdate(filePath: string, projectRoot: string, db: ClastDatabase): Promise<boolean> {
-    const parsed = await this.parseAndStore(filePath, projectRoot, db);
-    if (parsed) {
-      resolveCallEdges(db.queries);
-      db.save();
+  /** Resolve the spec for a path (+ content sniffing), or undefined. */
+  specFor(path: string, source?: string): LangSpec | undefined {
+    const spec = specForExtension(extname(path));
+    if (spec && spec.sniff && source !== undefined) {
+      const alt = spec.sniff(source, path);
+      if (alt) return specById(alt) ?? spec;
     }
-    return parsed;
-  }
-
-  removeFile(filePath: string, projectRoot: string, db: ClastDatabase): void {
-    const relPath = relative(projectRoot, filePath).replace(/\\/g, '/');
-    db.queries.deleteFile(relPath);
-    db.save();
+    return spec;
   }
 
-  private collectFiles(dir: string, supportedExts: Set<string>, depth: number = 0): string[] {
-    if (depth > 20) return [];
-
-    const files: string[] = [];
-    let entries: string[];
-
+  /** Parse and extract a single source text. Returns null for unsupported files. */
+  async extract(path: string, source: string): Promise<Extraction | null> {
+    const spec = this.specFor(path, source);
+    if (!spec) return null;
+    const compiled = await this.load(spec);
+    if (!compiled || !this.parser) return null;
+    this.parser.setLanguage(compiled.language);
+    const tree = this.parser.parse(source);
+    if (!tree) return null;
     try {
-      entries = readdirSync(dir);
-    } catch {
-      return files;
+      return extract(tree, source, compiled);
+    } finally {
+      tree.delete();
     }
+  }
 
-    for (const entry of entries) {
-      if (this.config.ignoredPaths.includes(entry)) continue;
-      if (entry.startsWith('.')) continue;
-
-      const fullPath = join(dir, entry);
-      let stat;
-      try {
-        stat = statSync(fullPath);
-      } catch {
-        continue;
+  /** Debug helper: S-expression-ish dump of the tree with field names. */
+  async dumpTree(path: string, source: string, maxDepth = 40): Promise<string> {
+    const spec = this.specFor(path, source);
+    if (!spec) return `unsupported file type: ${path}`;
+    const compiled = await this.load(spec);
+    if (!compiled || !this.parser) return `grammar unavailable for ${spec.id}`;
+    this.parser.setLanguage(compiled.language);
+    const tree = this.parser.parse(source);
+    if (!tree) return 'parse failed';
+    const out: string[] = [];
+    const cursor = tree.walk();
+    const walk = (depth: number): void => {
+      if (cursor.nodeIsNamed) {
+        const f = cursor.currentFieldName;
+        const pos = cursor.startPosition;
+        let line = `${'  '.repeat(depth)}${f ? f + ': ' : ''}${cursor.nodeType} [${pos.row + 1}:${pos.column}]`;
+        const node = cursor.currentNode;
+        if (node.namedChildCount === 0) line += ` ${JSON.stringify(node.text.slice(0, 60))}`;
+        out.push(line);
       }
-
-      if (stat.isDirectory()) {
-        files.push(...this.collectFiles(fullPath, supportedExts, depth + 1));
-      } else if (stat.isFile()) {
-        const ext = extname(entry);
-        if (supportedExts.has(ext)) {
-          files.push(fullPath);
-        }
+      if (depth < maxDepth && cursor.gotoFirstChild()) {
+        do walk(depth + (cursor.nodeIsNamed ? 1 : 1)); while (cursor.gotoNextSibling());
+        cursor.gotoParent();
       }
-    }
-
-    return files;
+    };
+    walk(0);
+    cursor.delete();
+    tree.delete();
+    return out.join('\n');
   }
 }