clast-ai 0.1.2 → 0.2.1

package/src/indexer/scan.ts

@@ -0,0 +1,226 @@
+import ignoreFactory, { type Ignore } from 'ignore';
+import { readdirSync, readFileSync, existsSync } from 'node:fs';
+import { stat } from 'node:fs/promises';
+import { extname, join } from 'node:path';
+import type { ClastConfig } from '../config/index.js';
+import { supportedExtensions } from '../parser/registry.js';
+
+// `ignore` is CJS; under NodeNext the default import may be the module object.
+const createIgnore: () => Ignore =
+  (ignoreFactory as unknown as { default?: () => Ignore }).default ?? (ignoreFactory as unknown as () => Ignore);
+
+const BINARY_EXTENSIONS = new Set([
+  '.png', '.jpg', '.jpeg', '.gif', '.bmp', '.ico', '.webp', '.tiff', '.psd', '.svgz', '.pdf', '.zip', '.gz',
+  '.tgz', '.bz2', '.xz', '.7z', '.rar', '.tar', '.jar', '.war', '.class', '.exe', '.dll', '.so', '.dylib', '.a',
+  '.o', '.obj', '.lib', '.pdb', '.wasm', '.woff', '.woff2', '.ttf', '.otf', '.eot', '.mp3', '.mp4', '.mov',
+  '.avi', '.wav', '.ogg', '.flac', '.webm', '.bin', '.dat', '.db', '.sqlite', '.pyc', '.pyo', '.node', '.snap',
+  '.map', '.min.js', '.icns', '.cur', '.keystore', '.p12', '.pfx', '.der', '.nupkg', '.dmg', '.iso', '.blend',
+]);
+const LOCK_FILES = new Set([
+  'package-lock.json', 'yarn.lock', 'pnpm-lock.yaml', 'Cargo.lock', 'composer.lock', 'Gemfile.lock', 'poetry.lock',
+  'go.sum', 'bun.lockb', 'npm-shrinkwrap.json',
+]);
+
+export interface ScannedFile {
+  abs: string;
+  /** Project-relative path with forward slashes. */
+  rel: string;
+  mtime: number;
+  size: number;
+}
+
+interface Rule {
+  /** Directory prefix this rule set is relative to ('' or 'a/b/'). */
+  base: string;
+  ig: Ignore;
+}
+
+/**
+ * Ignore decisions shared by the scanner and the watcher: config patterns plus
+ * (optionally) every .gitignore from the project root down to the file.
+ */
+export class IgnoreRules {
+  private readonly root: string;
+  private readonly useGitignore: boolean;
+  private readonly rootRule: Rule;
+  private readonly dirRules = new Map<string, Rule | null>();
+  readonly extensions: Set<string>;
+  readonly maxFileSize: number;
+
+  constructor(root: string, config: ClastConfig) {
+    this.root = root;
+    this.useGitignore = config.respectGitignore;
+    this.rootRule = { base: '', ig: createIgnore().add(config.ignoredPaths) };
+    this.extensions = supportedExtensions(config.languages);
+    this.maxFileSize = config.maxFileSize;
+    this.maxFiles = config.maxFiles;
+  }
+
+  /** Forget cached .gitignore contents (call when a .gitignore changes). */
+  invalidate(): void {
+    this.dirRules.clear();
+  }
+
+  /** Rule for the .gitignore inside `dirRel` ('' or 'a/b/'), if any. */
+  private ruleFor(dirRel: string): Rule | null {
+    if (!this.useGitignore) return null;
+    let rule = this.dirRules.get(dirRel);
+    if (rule !== undefined) return rule;
+    rule = null;
+    const file = join(this.root, dirRel, '.gitignore');
+    try {
+      if (existsSync(file)) {
+        rule = { base: dirRel, ig: createIgnore().add(readFileSync(file, 'utf-8')) };
+      }
+    } catch {
+      rule = null;
+    }
+    this.dirRules.set(dirRel, rule);
+    return rule;
+  }
+
+  private static test(rules: Rule[], rel: string, isDir: boolean): boolean {
+    let ignored = false;
+    for (const r of rules) {
+      const sub = rel.slice(r.base.length) + (isDir ? '/' : '');
+      if (!sub) continue;
+      const t = r.ig.test(sub);
+      if (t.ignored) ignored = true;
+      else if (t.unignored) ignored = false;
+    }
+    return ignored;
+  }
+
+  /** Full check for an arbitrary project-relative path (used by the watcher). */
+  isIgnored(rel: string): boolean {
+    const parts = rel.split('/');
+    const rules: Rule[] = [this.rootRule];
+    const rootGit = this.ruleFor('');
+    if (rootGit) rules.push(rootGit);
+    let prefix = '';
+    for (let i = 0; i < parts.length; i++) {
+      const name = parts[i]!;
+      const isDir = i < parts.length - 1;
+      if (isDir && name.startsWith('.')) return true;
+      const path = prefix + name;
+      if (IgnoreRules.test(rules, path, isDir)) return true;
+      if (isDir) {
+        prefix = path + '/';
+        const r = this.ruleFor(prefix);
+        if (r) rules.push(r);
+      }
+    }
+    return false;
+  }
+
+  isSupported(rel: string): boolean {
+    return this.extensions.has(extname(rel).toLowerCase());
+  }
+
+  /** Files skipped by the last scan because they exceed maxFileSize. */
+  skippedLarge = 0;
+  /** True when the last scan stopped at the maxFiles cap. */
+  truncated = false;
+  private readonly maxFiles: number;
+
+  /**
+   * Project-relative paths of text-like files that are NOT indexed (config,
+   * docs, data, unsupported languages) but pass the ignore rules. Used by
+   * text search so it can replace a plain grep.
+   */
+  otherTextFiles(limit = 60000): string[] {
+    const out: string[] = [];
+    const visit = (dirRel: string, parentRules: Rule[], depth: number): void => {
+      if (depth > 40 || out.length >= limit) return;
+      const rule = this.ruleFor(dirRel);
+      const rules = rule ? [...parentRules, rule] : parentRules;
+      let entries;
+      try {
+        entries = readdirSync(join(this.root, dirRel), { withFileTypes: true });
+      } catch {
+        return;
+      }
+      for (const e of entries) {
+        const name = e.name;
+        if (e.isDirectory()) {
+          if (name.startsWith('.') && name !== '.github') continue;
+          const rel = dirRel + name;
+          if (IgnoreRules.test(rules, rel, true)) continue;
+          visit(rel + '/', rules, depth + 1);
+        } else if (e.isFile()) {
+          const ext = extname(name).toLowerCase();
+          if (this.extensions.has(ext) || BINARY_EXTENSIONS.has(ext) || LOCK_FILES.has(name)) continue;
+          const rel = dirRel + name;
+          if (IgnoreRules.test(rules, rel, false)) continue;
+          out.push(rel);
+          if (out.length >= limit) return;
+        }
+      }
+    };
+    visit('', [this.rootRule], 0);
+    return out;
+  }
+
+  /** Walk the project and return every indexable file. */
+  async scan(): Promise<ScannedFile[]> {
+    const candidates: string[] = [];
+    this.skippedLarge = 0;
+    this.truncated = false;
+    const rules: Rule[] = [this.rootRule];
+    this.walk('', rules, candidates, 0);
+
+    // stat() dominates a warm start (especially on Windows); run it on the
+    // libuv thread pool instead of serially on the main thread.
+    const out: ScannedFile[] = new Array(candidates.length);
+    let next = 0;
+    const worker = async (): Promise<void> => {
+      while (next < candidates.length) {
+        const i = next++;
+        const rel = candidates[i]!;
+        const abs = join(this.root, rel);
+        try {
+          const st = await stat(abs);
+          if (st.size <= this.maxFileSize) out[i] = { abs, rel, mtime: st.mtimeMs, size: st.size };
+          else this.skippedLarge++;
+        } catch {
+          // vanished between readdir and stat
+        }
+      }
+    };
+    await Promise.all(Array.from({ length: Math.min(32, candidates.length) }, worker));
+    return out.filter(Boolean);
+  }
+
+  private walk(dirRel: string, parentRules: Rule[], out: string[], depth: number): void {
+    if (depth > 40) return;
+    if (out.length >= this.maxFiles) {
+      this.truncated = true;
+      return;
+    }
+    const rule = this.ruleFor(dirRel);
+    const rules = rule ? [...parentRules, rule] : parentRules;
+
+    let entries;
+    try {
+      entries = readdirSync(join(this.root, dirRel), { withFileTypes: true });
+    } catch {
+      return;
+    }
+
+    for (const e of entries) {
+      const name = e.name;
+      if (e.isDirectory()) {
+        if (name.startsWith('.')) continue;
+        const rel = dirRel + name;
+        if (IgnoreRules.test(rules, rel, true)) continue;
+        this.walk(rel + '/', rules, out, depth + 1);
+      } else if (e.isFile()) {
+        if (!this.extensions.has(extname(name).toLowerCase())) continue;
+        const rel = dirRel + name;
+        if (IgnoreRules.test(rules, rel, false)) continue;
+        out.push(rel);
+      }
+      // Symlinks are skipped: they cause cycles and duplicate symbols.
+    }
+  }
+}

package/src/indexer/worker.ts

@@ -0,0 +1,24 @@
+import { parentPort, workerData } from 'node:worker_threads';
+import { SourceParser } from '../parser/index.js';
+import { processFile, type FileResult, type FileTask } from './process.js';
+
+/**
+ * Indexing worker: receives batches of files, reads + parses them, and posts
+ * the extraction results back. The main thread owns the database.
+ */
+const { maxFileSize } = workerData as { maxFileSize: number };
+const parser = new SourceParser();
+
+if (parentPort) {
+  const port = parentPort;
+  port.on('message', async (tasks: FileTask[] | null) => {
+    if (tasks === null) {
+      process.exit(0);
+    }
+    const results: FileResult[] = [];
+    for (const task of tasks) {
+      results.push(await processFile(parser, task, maxFileSize));
+    }
+    port.postMessage(results);
+  });
+}

package/src/llm/comment-generator.ts

@@ -1,131 +1,80 @@
 import { LlmClient } from './index.js';
 import { buildFunctionCommentPrompt, buildClassCommentPrompt } from './prompts.js';
-import type { Queries } from '../db/queries.js';
-import type { AstNodeRow } from '../db/types.js';
+import type { ClastDb } from '../db/index.js';
 import type { ClastConfig } from '../config/index.js';
-
+import { CONTAINER_KINDS, type SymbolRow } from '../types.js';
+import type { LineCache } from '../query/format.js';
+
+/**
+ * Generates documentation for undocumented symbols with an OpenAI-compatible
+ * endpoint. Generated text is stored by (path, qualified name) so it survives
+ * re-indexing of the file.
+ */
 export class CommentGenerator {
   private client: LlmClient;
-  private queries: Queries;
-  private config: ClastConfig;
 
-  constructor(client: LlmClient, queries: Queries, config: ClastConfig) {
-    this.client = client;
-    this.queries = queries;
-    this.config = config;
+  constructor(
+    private readonly db: ClastDb,
+    private readonly config: ClastConfig,
+    private readonly lines: LineCache
+  ) {
+    this.client = new LlmClient(config.llm.endpoint, config.llm.model, config.llm.apiKey);
   }
 
   get isAvailable(): boolean {
     return this.client.isConfigured;
   }
 
+  /** Previously generated doc for a symbol, if any. */
+  stored(sym: SymbolRow): string | null {
+    const row = this.db.get<{ text: string }>(
+      `SELECT text FROM gen_docs WHERE path = ? AND qname = ?`, sym.path, sym.qname
+    );
+    return row?.text ?? null;
+  }
+
   /**
-   * Generate a comment for a specific node.
-   * Returns the generated comment text, or null if generation is disabled/failed.
+   * Returns the doc for a symbol: the original comment when present (unless
+   * alwaysGenerate), else a stored or freshly generated one. Null on failure.
    */
-  async generateForNode(nodeId: number): Promise<string | null> {
+  async generate(sym: SymbolRow, force = false): Promise<string | null> {
+    if (sym.doc && !this.config.llm.alwaysGenerate && !force) return sym.doc;
+    if (!force) {
+      const existing = this.stored(sym);
+      if (existing) return existing;
+    }
     if (!this.client.isConfigured) return null;
 
-    const node = this.queries.getNodeById(nodeId);
-    if (!node) return null;
-
-    // Check if original comment exists
-    const existingComments = this.queries.getCommentsForNode(nodeId);
-    const hasOriginal = existingComments.some(c => c.source === 'original');
-
-    if (hasOriginal && !this.config.llm.alwaysGenerate) {
-      // Original comment exists and we're not forcing generation
-      return existingComments.find(c => c.source === 'original')?.text ?? null;
+    let prompt: { system: string; user: string };
+    if (CONTAINER_KINDS.has(sym.kind)) {
+      const members = this.db
+        .all<{ name: string }>(
+          `SELECT name FROM symbols WHERE parent_id = ? AND kind IN ('method', 'function') LIMIT 60`, sym.id
+        )
+        .map(m => m.name);
+      prompt = buildClassCommentPrompt(sym.name, sym.signature, members, sym.lang);
+    } else {
+      const src = this.lines.lines(sym.path);
+      const body = src ? src.slice(sym.start_line - 1, Math.min(sym.end_line, sym.start_line + 80)).join('\n') : '';
+      prompt = buildFunctionCommentPrompt(sym.signature, body, sym.lang);
     }
 
     try {
-      const prompt = this.buildPrompt(node);
       const response = await this.client.complete([
         { role: 'system', content: prompt.system },
         { role: 'user', content: prompt.user },
       ]);
-
-      const comment = response.content.trim();
-      if (!comment) return null;
-
-      // Remove existing generated comment if any
-      this.queries.deleteCommentsBySource(nodeId, 'generated');
-
-      // Store the generated comment
-      this.queries.insertComment(nodeId, comment, 'generated');
-
-      return comment;
+      const text = response.content.trim();
+      if (!text) return null;
+      this.db.run(
+        `INSERT INTO gen_docs (path, qname, text, created_at) VALUES (?, ?, ?, ?)
+         ON CONFLICT(path, qname) DO UPDATE SET text = excluded.text, created_at = excluded.created_at`,
+        sym.path, sym.qname, text, Date.now()
+      );
+      return text;
     } catch (err) {
-      console.error(`[clast] Failed to generate comment for node ${nodeId}:`, err);
+      console.error(`[clast] comment generation failed for ${sym.qname}:`, err instanceof Error ? err.message : err);
       return null;
     }
   }
-
-  /**
-   * Batch generate comments for nodes without any comments.
-   * Returns counts of successes and failures.
-   */
-  async batchGenerate(
-    limit: number = 100,
-    onProgress?: (current: number, total: number) => void
-  ): Promise<{ success: number; failed: number; skipped: number }> {
-    if (!this.client.isConfigured) {
-      return { success: 0, failed: 0, skipped: 0 };
-    }
-
-    const nodes = this.queries.getNodesWithoutComments(limit);
-    let success = 0;
-    let failed = 0;
-    let skipped = 0;
-
-    // Process with concurrency limit
-    const concurrency = this.config.llm.maxConcurrent;
-    const chunks = chunkArray(nodes, concurrency);
-
-    let processed = 0;
-    for (const chunk of chunks) {
-      const results = await Promise.allSettled(
-        chunk.map(node => this.generateForNode(node.id))
-      );
-
-      for (const result of results) {
-        processed++;
-        onProgress?.(processed, nodes.length);
-
-        if (result.status === 'fulfilled') {
-          if (result.value) {
-            success++;
-          } else {
-            skipped++;
-          }
-        } else {
-          failed++;
-        }
-      }
-    }
-
-    return { success, failed, skipped };
-  }
-
-  private buildPrompt(node: AstNodeRow): { system: string; user: string } {
-    if (node.type === 'class' || node.type === 'interface' || node.type === 'struct' || node.type === 'trait') {
-      // Get method names for context
-      const children = this.queries.searchNodes(node.name, 'method', 50);
-      const methodNames = children
-        .filter(c => c.file_path === node.file_path)
-        .map(c => c.name);
-
-      return buildClassCommentPrompt(node.name, node.signature, methodNames, node.language);
-    }
-
-    return buildFunctionCommentPrompt(node.signature, node.body_text, node.language);
-  }
-}
-
-function chunkArray<T>(arr: T[], size: number): T[][] {
-  const chunks: T[][] = [];
-  for (let i = 0; i < arr.length; i += size) {
-    chunks.push(arr.slice(i, i + size));
-  }
-  return chunks;
 }

package/src/llm/index.ts

@@ -1,80 +1,80 @@
-export interface ChatMessage {
-  role: 'system' | 'user' | 'assistant';
-  content: string;
-}
-
-export interface LlmResponse {
-  content: string;
-  model: string;
-  usage?: { prompt_tokens: number; completion_tokens: number };
-}
-
-export class LlmClient {
-  private endpoint: string;
-  private model: string;
-  private apiKey: string;
-
-  constructor(endpoint: string, model: string, apiKey: string = '') {
-    // Ensure endpoint doesn't have trailing slash
-    this.endpoint = endpoint.replace(/\/$/, '');
-    this.model = model;
-    this.apiKey = apiKey;
-  }
-
-  get isConfigured(): boolean {
-    return this.model.length > 0;
-  }
-
-  async complete(
-    messages: ChatMessage[],
-    options: { temperature?: number; maxTokens?: number } = {}
-  ): Promise<LlmResponse> {
-    const { temperature = 0.3, maxTokens = 500 } = options;
-
-    const url = `${this.endpoint}/chat/completions`;
-    const headers: Record<string, string> = {
-      'Content-Type': 'application/json',
-    };
-
-    if (this.apiKey) {
-      headers['Authorization'] = `Bearer ${this.apiKey}`;
-    }
-
-    const controller = new AbortController();
-    const timeout = setTimeout(() => controller.abort(), 30000);
-
-    try {
-      const response = await fetch(url, {
-        method: 'POST',
-        headers,
-        body: JSON.stringify({
-          model: this.model,
-          messages,
-          temperature,
-          max_tokens: maxTokens,
-        }),
-        signal: controller.signal,
-      });
-
-      if (!response.ok) {
-        const body = await response.text().catch(() => '');
-        throw new Error(`LLM API error ${response.status}: ${body}`);
-      }
-
-      const data = await response.json() as {
-        choices: Array<{ message: { content: string } }>;
-        model: string;
-        usage?: { prompt_tokens: number; completion_tokens: number };
-      };
-
-      const content = data.choices?.[0]?.message?.content ?? '';
-      return {
-        content,
-        model: data.model ?? this.model,
-        usage: data.usage,
-      };
-    } finally {
-      clearTimeout(timeout);
-    }
-  }
-}
+export interface ChatMessage {
+  role: 'system' | 'user' | 'assistant';
+  content: string;
+}
+
+export interface LlmResponse {
+  content: string;
+  model: string;
+  usage?: { prompt_tokens: number; completion_tokens: number };
+}
+
+export class LlmClient {
+  private endpoint: string;
+  private model: string;
+  private apiKey: string;
+
+  constructor(endpoint: string, model: string, apiKey: string = '') {
+    // Ensure endpoint doesn't have trailing slash
+    this.endpoint = endpoint.replace(/\/$/, '');
+    this.model = model;
+    this.apiKey = apiKey;
+  }
+
+  get isConfigured(): boolean {
+    return this.model.length > 0;
+  }
+
+  async complete(
+    messages: ChatMessage[],
+    options: { temperature?: number; maxTokens?: number } = {}
+  ): Promise<LlmResponse> {
+    const { temperature = 0.3, maxTokens = 500 } = options;
+
+    const url = `${this.endpoint}/chat/completions`;
+    const headers: Record<string, string> = {
+      'Content-Type': 'application/json',
+    };
+
+    if (this.apiKey) {
+      headers['Authorization'] = `Bearer ${this.apiKey}`;
+    }
+
+    const controller = new AbortController();
+    const timeout = setTimeout(() => controller.abort(), 30000);
+
+    try {
+      const response = await fetch(url, {
+        method: 'POST',
+        headers,
+        body: JSON.stringify({
+          model: this.model,
+          messages,
+          temperature,
+          max_tokens: maxTokens,
+        }),
+        signal: controller.signal,
+      });
+
+      if (!response.ok) {
+        const body = await response.text().catch(() => '');
+        throw new Error(`LLM API error ${response.status}: ${body}`);
+      }
+
+      const data = await response.json() as {
+        choices: Array<{ message: { content: string } }>;
+        model: string;
+        usage?: { prompt_tokens: number; completion_tokens: number };
+      };
+
+      const content = data.choices?.[0]?.message?.content ?? '';
+      return {
+        content,
+        model: data.model ?? this.model,
+        usage: data.usage,
+      };
+    } finally {
+      clearTimeout(timeout);
+    }
+  }
+}