clast-ai 0.1.0

package/src/llm/comment-generator.ts

@@ -0,0 +1,131 @@
+import { LlmClient } from './index.js';
+import { buildFunctionCommentPrompt, buildClassCommentPrompt } from './prompts.js';
+import type { Queries } from '../db/queries.js';
+import type { AstNodeRow } from '../db/types.js';
+import type { ClastConfig } from '../config/index.js';
+
+export class CommentGenerator {
+  private client: LlmClient;
+  private queries: Queries;
+  private config: ClastConfig;
+
+  constructor(client: LlmClient, queries: Queries, config: ClastConfig) {
+    this.client = client;
+    this.queries = queries;
+    this.config = config;
+  }
+
+  get isAvailable(): boolean {
+    return this.client.isConfigured;
+  }
+
+  /**
+   * Generate a comment for a specific node.
+   * Returns the generated comment text, or null if generation is disabled/failed.
+   */
+  async generateForNode(nodeId: number): Promise<string | null> {
+    if (!this.client.isConfigured) return null;
+
+    const node = this.queries.getNodeById(nodeId);
+    if (!node) return null;
+
+    // Check if original comment exists
+    const existingComments = this.queries.getCommentsForNode(nodeId);
+    const hasOriginal = existingComments.some(c => c.source === 'original');
+
+    if (hasOriginal && !this.config.llm.alwaysGenerate) {
+      // Original comment exists and we're not forcing generation
+      return existingComments.find(c => c.source === 'original')?.text ?? null;
+    }
+
+    try {
+      const prompt = this.buildPrompt(node);
+      const response = await this.client.complete([
+        { role: 'system', content: prompt.system },
+        { role: 'user', content: prompt.user },
+      ]);
+
+      const comment = response.content.trim();
+      if (!comment) return null;
+
+      // Remove existing generated comment if any
+      this.queries.deleteCommentsBySource(nodeId, 'generated');
+
+      // Store the generated comment
+      this.queries.insertComment(nodeId, comment, 'generated');
+
+      return comment;
+    } catch (err) {
+      console.error(`[clast] Failed to generate comment for node ${nodeId}:`, err);
+      return null;
+    }
+  }
+
+  /**
+   * Batch generate comments for nodes without any comments.
+   * Returns counts of successes and failures.
+   */
+  async batchGenerate(
+    limit: number = 100,
+    onProgress?: (current: number, total: number) => void
+  ): Promise<{ success: number; failed: number; skipped: number }> {
+    if (!this.client.isConfigured) {
+      return { success: 0, failed: 0, skipped: 0 };
+    }
+
+    const nodes = this.queries.getNodesWithoutComments(limit);
+    let success = 0;
+    let failed = 0;
+    let skipped = 0;
+
+    // Process with concurrency limit
+    const concurrency = this.config.llm.maxConcurrent;
+    const chunks = chunkArray(nodes, concurrency);
+
+    let processed = 0;
+    for (const chunk of chunks) {
+      const results = await Promise.allSettled(
+        chunk.map(node => this.generateForNode(node.id))
+      );
+
+      for (const result of results) {
+        processed++;
+        onProgress?.(processed, nodes.length);
+
+        if (result.status === 'fulfilled') {
+          if (result.value) {
+            success++;
+          } else {
+            skipped++;
+          }
+        } else {
+          failed++;
+        }
+      }
+    }
+
+    return { success, failed, skipped };
+  }
+
+  private buildPrompt(node: AstNodeRow): { system: string; user: string } {
+    if (node.type === 'class' || node.type === 'interface' || node.type === 'struct' || node.type === 'trait') {
+      // Get method names for context
+      const children = this.queries.searchNodes(node.name, 'method', 50);
+      const methodNames = children
+        .filter(c => c.file_path === node.file_path)
+        .map(c => c.name);
+
+      return buildClassCommentPrompt(node.name, node.signature, methodNames, node.language);
+    }
+
+    return buildFunctionCommentPrompt(node.signature, node.body_text, node.language);
+  }
+}
+
+function chunkArray<T>(arr: T[], size: number): T[][] {
+  const chunks: T[][] = [];
+  for (let i = 0; i < arr.length; i += size) {
+    chunks.push(arr.slice(i, i + size));
+  }
+  return chunks;
+}

package/src/llm/index.ts

@@ -0,0 +1,80 @@
+export interface ChatMessage {
+  role: 'system' | 'user' | 'assistant';
+  content: string;
+}
+
+export interface LlmResponse {
+  content: string;
+  model: string;
+  usage?: { prompt_tokens: number; completion_tokens: number };
+}
+
+export class LlmClient {
+  private endpoint: string;
+  private model: string;
+  private apiKey: string;
+
+  constructor(endpoint: string, model: string, apiKey: string = '') {
+    // Ensure endpoint doesn't have trailing slash
+    this.endpoint = endpoint.replace(/\/$/, '');
+    this.model = model;
+    this.apiKey = apiKey;
+  }
+
+  get isConfigured(): boolean {
+    return this.model.length > 0;
+  }
+
+  async complete(
+    messages: ChatMessage[],
+    options: { temperature?: number; maxTokens?: number } = {}
+  ): Promise<LlmResponse> {
+    const { temperature = 0.3, maxTokens = 500 } = options;
+
+    const url = `${this.endpoint}/chat/completions`;
+    const headers: Record<string, string> = {
+      'Content-Type': 'application/json',
+    };
+
+    if (this.apiKey) {
+      headers['Authorization'] = `Bearer ${this.apiKey}`;
+    }
+
+    const controller = new AbortController();
+    const timeout = setTimeout(() => controller.abort(), 30000);
+
+    try {
+      const response = await fetch(url, {
+        method: 'POST',
+        headers,
+        body: JSON.stringify({
+          model: this.model,
+          messages,
+          temperature,
+          max_tokens: maxTokens,
+        }),
+        signal: controller.signal,
+      });
+
+      if (!response.ok) {
+        const body = await response.text().catch(() => '');
+        throw new Error(`LLM API error ${response.status}: ${body}`);
+      }
+
+      const data = await response.json() as {
+        choices: Array<{ message: { content: string } }>;
+        model: string;
+        usage?: { prompt_tokens: number; completion_tokens: number };
+      };
+
+      const content = data.choices?.[0]?.message?.content ?? '';
+      return {
+        content,
+        model: data.model ?? this.model,
+        usage: data.usage,
+      };
+    } finally {
+      clearTimeout(timeout);
+    }
+  }
+}

package/src/llm/prompts.ts

@@ -0,0 +1,95 @@
+const LANGUAGE_COMMENT_STYLES: Record<string, { docPrefix: string; example: string }> = {
+  typescript: {
+    docPrefix: '/**',
+    example: '/** Calculates the total price including tax. */',
+  },
+  tsx: {
+    docPrefix: '/**',
+    example: '/** Calculates the total price including tax. */',
+  },
+  javascript: {
+    docPrefix: '/**',
+    example: '/** Calculates the total price including tax. */',
+  },
+  python: {
+    docPrefix: '"""',
+    example: '"""Calculates the total price including tax."""',
+  },
+  java: {
+    docPrefix: '/**',
+    example: '/** Calculates the total price including tax. */',
+  },
+  csharp: {
+    docPrefix: '///',
+    example: '/// <summary>Calculates the total price including tax.</summary>',
+  },
+  go: {
+    docPrefix: '//',
+    example: '// CalculateTotal calculates the total price including tax.',
+  },
+  rust: {
+    docPrefix: '///',
+    example: '/// Calculates the total price including tax.',
+  },
+  c: {
+    docPrefix: '/**',
+    example: '/** Calculates the total price including tax. */',
+  },
+  cpp: {
+    docPrefix: '/**',
+    example: '/** Calculates the total price including tax. */',
+  },
+  ruby: {
+    docPrefix: '#',
+    example: '# Calculates the total price including tax.',
+  },
+  php: {
+    docPrefix: '/**',
+    example: '/** Calculates the total price including tax. */',
+  },
+};
+
+export function buildFunctionCommentPrompt(
+  signature: string,
+  bodyText: string,
+  language: string
+): { system: string; user: string } {
+  const style = LANGUAGE_COMMENT_STYLES[language] ?? LANGUAGE_COMMENT_STYLES['typescript']!;
+
+  const system = `You are a code documentation expert. Generate a concise, clear documentation comment for the given function or method. Use the ${language} documentation style. Output ONLY the comment text (no code fences, no extra explanation). Keep it brief: 1-3 sentences describing what the function does, its parameters, and return value if applicable.`;
+
+  const user = `Language: ${language}
+Comment style example: ${style.example}
+
+Signature:
+${signature}
+
+Body:
+${bodyText.slice(0, 1500)}
+
+Generate a documentation comment for this function.`;
+
+  return { system, user };
+}
+
+export function buildClassCommentPrompt(
+  name: string,
+  signature: string,
+  methodNames: string[],
+  language: string
+): { system: string; user: string } {
+  const style = LANGUAGE_COMMENT_STYLES[language] ?? LANGUAGE_COMMENT_STYLES['typescript']!;
+
+  const system = `You are a code documentation expert. Generate a concise, clear documentation comment for the given class or type. Use the ${language} documentation style. Output ONLY the comment text (no code fences, no extra explanation). Keep it brief: 1-3 sentences describing the class purpose and responsibilities.`;
+
+  const user = `Language: ${language}
+Comment style example: ${style.example}
+
+Class: ${name}
+Signature: ${signature}
+Methods: ${methodNames.join(', ') || 'none'}
+
+Generate a documentation comment for this class.`;
+
+  return { system, user };
+}

package/src/parser/call-resolver.ts

@@ -0,0 +1,17 @@
+import type { Queries } from '../db/queries.js';
+
+/**
+ * Resolves unresolved call edges in the database.
+ *
+ * After all files are parsed and nodes inserted, call edges may have
+ * callee_id = null because the callee wasn't inserted yet. This resolver
+ * attempts to match callee_name to ast_nodes.name.
+ *
+ * Resolution strategy (best-effort):
+ * 1. Exact name match within the same file
+ * 2. Exact name match across all files
+ * 3. Leave unresolved if no match
+ */
+export function resolveCallEdges(queries: Queries): number {
+  return queries.resolveEdges();
+}

package/src/parser/extractor.ts

@@ -0,0 +1,303 @@
+import type TreeSitter from 'web-tree-sitter';
+import type { AstNode, AstNodeType } from '../types.js';
+
+// Node types that represent extractable symbols per language
+const FUNCTION_TYPES = new Set([
+  'function_declaration',
+  'function_definition',
+  'method_definition',
+  'method_declaration',
+  'arrow_function',
+  'function_item',           // Rust
+  'func_declaration',        // Go
+  'function_expression',
+]);
+
+const CLASS_TYPES = new Set([
+  'class_declaration',
+  'class_definition',
+  'struct_item',             // Rust
+  'struct_specifier',        // C/C++
+  'impl_item',              // Rust
+  'type_declaration',        // Go
+]);
+
+const INTERFACE_TYPES = new Set([
+  'interface_declaration',
+  'trait_item',              // Rust
+]);
+
+const IMPORT_TYPES = new Set([
+  'import_statement',
+  'import_declaration',
+  'use_declaration',         // Rust
+  'preproc_include',         // C/C++ (#include)
+]);
+
+const EXPORT_TYPES = new Set([
+  'export_statement',
+  'export_declaration',
+]);
+
+const ENUM_TYPES = new Set([
+  'enum_declaration',
+  'enum_item',               // Rust
+  'enum_specifier',          // C/C++
+]);
+
+const CALL_TYPES = new Set([
+  'call_expression',
+  'new_expression',
+  'method_invocation',       // Java
+  'invocation_expression',   // C#
+]);
+
+function classifyNodeType(treeSitterType: string): AstNodeType | null {
+  if (FUNCTION_TYPES.has(treeSitterType)) return 'function';
+  if (CLASS_TYPES.has(treeSitterType)) return 'class';
+  if (INTERFACE_TYPES.has(treeSitterType)) return 'interface';
+  if (IMPORT_TYPES.has(treeSitterType)) return 'import';
+  if (EXPORT_TYPES.has(treeSitterType)) return 'export';
+  if (ENUM_TYPES.has(treeSitterType)) return 'enum';
+  return null;
+}
+
+function extractName(node: TreeSitter.SyntaxNode): string {
+  const nameNode =
+    node.childForFieldName('name') ??
+    node.childForFieldName('declarator')?.childForFieldName('name') ??
+    node.childForFieldName('declarator');
+
+  if (nameNode) {
+    return nameNode.text;
+  }
+
+  // Arrow functions assigned to variables
+  if (node.type === 'arrow_function' && node.parent) {
+    const parentName = node.parent.childForFieldName('name');
+    if (parentName) return parentName.text;
+  }
+
+  // Exports
+  if (node.type === 'export_statement') {
+    const decl = node.childForFieldName('declaration');
+    if (decl) return extractName(decl);
+    return '<export>';
+  }
+
+  // Imports
+  if (IMPORT_TYPES.has(node.type)) {
+    const source = node.childForFieldName('source') ?? node.childForFieldName('path');
+    if (source) return source.text.replace(/['"]/g, '');
+    return node.text.slice(0, 80);
+  }
+
+  return '<anonymous>';
+}
+
+function extractSignature(node: TreeSitter.SyntaxNode, maxLength: number = 200): string {
+  const text = node.text;
+  const braceIdx = text.indexOf('{');
+  const parenCloseIdx = text.indexOf(')');
+
+  let end = text.length;
+  if (braceIdx > 0) {
+    end = braceIdx;
+  } else if (parenCloseIdx > 0) {
+    end = parenCloseIdx + 1;
+  }
+
+  const sig = text.slice(0, end).trim();
+  return sig.length > maxLength ? sig.slice(0, maxLength) + '...' : sig;
+}
+
+function extractPrecedingComment(
+  node: TreeSitter.SyntaxNode,
+  source: string
+): string | null {
+  const lines = source.split('\n');
+  const nodeLine = node.startPosition.row;
+
+  const commentLines: string[] = [];
+  let checkLine = nodeLine - 1;
+
+  while (checkLine >= 0) {
+    const line = lines[checkLine]?.trim() ?? '';
+
+    if (
+      line.startsWith('//') ||
+      line.startsWith('#') ||
+      line.startsWith('*') ||
+      line.startsWith('/*') ||
+      line.startsWith('///') ||
+      line.startsWith('/**')
+    ) {
+      commentLines.unshift(line);
+      checkLine--;
+    } else if (line === '' && commentLines.length === 0) {
+      checkLine--;
+    } else {
+      break;
+    }
+  }
+
+  if (commentLines.length === 0) return null;
+
+  const cleaned = commentLines
+    .map(l => l.replace(/^\/\/\/?\s?|^\/\*\*?\s?|^\*\/?\s?|^#\s?/g, '').trim())
+    .filter(l => l.length > 0)
+    .join('\n');
+
+  return cleaned || null;
+}
+
+function extractPythonDocstring(node: TreeSitter.SyntaxNode): string | null {
+  const body = node.childForFieldName('body');
+  if (!body) return null;
+
+  const firstChild = body.firstChild;
+  if (!firstChild) return null;
+
+  if (
+    firstChild.type === 'expression_statement' &&
+    firstChild.firstChild?.type === 'string'
+  ) {
+    const text = firstChild.firstChild.text;
+    return text.replace(/^["']{3}|["']{3}$/g, '').trim();
+  }
+
+  return null;
+}
+
+function extractCallName(node: TreeSitter.SyntaxNode): string | null {
+  const fn = node.childForFieldName('function') ?? node.childForFieldName('method');
+  if (!fn) {
+    const firstChild = node.firstChild;
+    if (firstChild) {
+      if (firstChild.type === 'member_expression' || firstChild.type === 'field_expression') {
+        const prop = firstChild.childForFieldName('property') ?? firstChild.childForFieldName('field');
+        return prop?.text ?? firstChild.text;
+      }
+      return firstChild.text;
+    }
+    return null;
+  }
+
+  if (fn.type === 'member_expression' || fn.type === 'field_expression') {
+    const prop = fn.childForFieldName('property') ?? fn.childForFieldName('field');
+    return prop?.text ?? fn.text;
+  }
+
+  return fn.text;
+}
+
+export interface ExtractionResult {
+  nodes: AstNode[];
+  edges: Array<{ callerNodeIndex: number; calleeName: string; line: number }>;
+  comments: Array<{ nodeIndex: number; text: string; source: 'original' }>;
+}
+
+export function extractFromTree(
+  tree: TreeSitter.Tree,
+  filePath: string,
+  language: string,
+  source: string,
+  maxBodySize: number = 2000
+): ExtractionResult {
+  const nodes: AstNode[] = [];
+  const edges: Array<{ callerNodeIndex: number; calleeName: string; line: number }> = [];
+  const comments: Array<{ nodeIndex: number; text: string; source: 'original' }> = [];
+
+  const nodeIdToIndex = new Map<number, number>();
+
+  function findEnclosingFunctionIndex(node: TreeSitter.SyntaxNode): number | null {
+    let current = node.parent;
+    while (current) {
+      const type = classifyNodeType(current.type);
+      if (type === 'function') {
+        return nodeIdToIndex.get(current.id) ?? null;
+      }
+      current = current.parent;
+    }
+    return null;
+  }
+
+  function walk(node: TreeSitter.SyntaxNode, parentIndex: number | null): void {
+    const nodeType = classifyNodeType(node.type);
+    let currentIndex = parentIndex;
+
+    if (nodeType && nodeType !== 'import' && nodeType !== 'export') {
+      const name = extractName(node);
+      const signature = extractSignature(node);
+      const bodyText = node.text.length > maxBodySize
+        ? node.text.slice(0, maxBodySize) + '...'
+        : node.text;
+
+      const astNode: AstNode = {
+        filePath,
+        name,
+        type: nodeType === 'function' && parentIndex !== null ? 'method' : nodeType,
+        signature,
+        startLine: node.startPosition.row + 1,
+        endLine: node.endPosition.row + 1,
+        startCol: node.startPosition.column,
+        endCol: node.endPosition.column,
+        parentId: null,
+        language,
+        bodyText,
+      };
+
+      currentIndex = nodes.length;
+      nodes.push(astNode);
+      nodeIdToIndex.set(node.id, currentIndex);
+
+      const comment = language === 'python'
+        ? (extractPythonDocstring(node) ?? extractPrecedingComment(node, source))
+        : extractPrecedingComment(node, source);
+
+      if (comment) {
+        comments.push({ nodeIndex: currentIndex, text: comment, source: 'original' });
+      }
+    } else if (nodeType === 'import' || nodeType === 'export') {
+      const name = extractName(node);
+      nodes.push({
+        filePath,
+        name,
+        type: nodeType,
+        signature: node.text.slice(0, 200),
+        startLine: node.startPosition.row + 1,
+        endLine: node.endPosition.row + 1,
+        startCol: node.startPosition.column,
+        endCol: node.endPosition.column,
+        parentId: null,
+        language,
+        bodyText: '',
+      });
+    }
+
+    // Extract call expressions
+    if (CALL_TYPES.has(node.type)) {
+      const calleeName = extractCallName(node);
+      const enclosingFunc = findEnclosingFunctionIndex(node);
+      if (calleeName && enclosingFunc !== null) {
+        edges.push({
+          callerNodeIndex: enclosingFunc,
+          calleeName,
+          line: node.startPosition.row + 1,
+        });
+      }
+    }
+
+    // Recurse into children
+    for (let i = 0; i < node.childCount; i++) {
+      const child = node.child(i);
+      if (child) {
+        walk(child, currentIndex);
+      }
+    }
+  }
+
+  walk(tree.rootNode, null);
+
+  return { nodes, edges, comments };
+}