clast-ai 0.1.2 → 0.2.0

package/src/indexer/imports.ts

@@ -0,0 +1,241 @@
+import { posix } from 'node:path';
+import type { ClastDb } from '../db/index.js';
+
+const JS_LANGS = new Set(['typescript', 'tsx', 'javascript', 'vue']);
+const EXT_RE = /\.[A-Za-z0-9]+$/;
+
+interface ImportRow {
+  rowid: number;
+  file_id: number;
+  source: string;
+  path: string;
+  lang: string;
+}
+
+/**
+ * In-memory lookup from path shapes to file ids, used to resolve import
+ * specifiers to indexed files.
+ */
+class PathIndex {
+  /** Exact path -> id. */
+  readonly exact = new Map<string, number>();
+  /** Path without extension -> ids. */
+  readonly noExt = new Map<string, number[]>();
+  /** Last path segment (without extension) -> [pathNoExt, id][]. */
+  readonly byBase = new Map<string, Array<[string, number, string]>>();
+
+  constructor(files: Array<{ id: number; path: string }>) {
+    for (const f of files) {
+      this.exact.set(f.path, f.id);
+      const stem = f.path.replace(EXT_RE, '');
+      const list = this.noExt.get(stem);
+      if (list) list.push(f.id);
+      else this.noExt.set(stem, [f.id]);
+      const base = stem.slice(stem.lastIndexOf('/') + 1);
+      const bl = this.byBase.get(base);
+      if (bl) bl.push([stem, f.id, f.path]);
+      else this.byBase.set(base, [[stem, f.id, f.path]]);
+    }
+  }
+
+  /** Resolve a concrete project-relative path, trying extensions and index files. */
+  lookup(p: string): number | undefined {
+    const direct = this.exact.get(p);
+    if (direct !== undefined) return direct;
+    const stemmed = this.noExt.get(p);
+    if (stemmed) return stemmed[0];
+    // foo.js -> foo.ts (ESM-style TypeScript imports)
+    if (EXT_RE.test(p)) {
+      const alt = this.noExt.get(p.replace(EXT_RE, ''));
+      if (alt) return alt[0];
+    }
+    for (const idx of ['/index', '/__init__', '/mod']) {
+      const hit = this.noExt.get(p + idx);
+      if (hit) return hit[0];
+    }
+    return undefined;
+  }
+
+  /**
+   * Resolve a non-relative specifier by path suffix. When several files match,
+   * prefer the one closest to the importing file; give up if still ambiguous.
+   */
+  suffix(spec: string, fromPath: string): number | undefined {
+    const stem = spec.replace(EXT_RE, '');
+    const base = stem.slice(stem.lastIndexOf('/') + 1);
+    const candidates = this.byBase.get(base) ?? (EXT_RE.test(spec) ? undefined : this.byBase.get(spec));
+    const hits: Array<[string, number, string]> = [];
+    const hasExt = spec !== stem;
+    for (const c of candidates ?? []) {
+      if (c[0] === stem || c[0].endsWith('/' + stem)) {
+        if (hasExt && !c[2].endsWith(spec)) continue;
+        hits.push(c);
+      }
+    }
+    if (hits.length === 0) {
+      // Package directory: a/b -> a/b/index.*, a/b/__init__.py, a/b/mod.rs
+      for (const idx of ['index', '__init__', 'mod']) {
+        for (const c of this.byBase.get(idx) ?? []) {
+          const dir = c[0].slice(0, -(idx.length + 1));
+          if (dir === stem || dir.endsWith('/' + stem)) hits.push(c);
+        }
+        if (hits.length) break;
+      }
+    }
+    if (hits.length === 0) return undefined;
+    if (hits.length === 1) return hits[0]![1];
+    let best: number | undefined;
+    let bestScore = -1;
+    let tie = false;
+    for (const h of hits) {
+      const score = commonPrefixSegments(h[0], fromPath);
+      if (score > bestScore) { bestScore = score; best = h[1]; tie = false; }
+      else if (score === bestScore) tie = true;
+    }
+    return tie ? undefined : best;
+  }
+}
+
+function commonPrefixSegments(a: string, b: string): number {
+  const as = a.split('/');
+  const bs = b.split('/');
+  let n = 0;
+  while (n < as.length && n < bs.length && as[n] === bs[n]) n++;
+  return n;
+}
+
+function resolveOne(row: ImportRow, index: PathIndex): number | undefined {
+  const dir = posix.dirname(row.path);
+  let src = row.source;
+
+  if (JS_LANGS.has(row.lang)) {
+    src = src.replace(/[?#].*$/, '');
+    if (src.startsWith('.')) {
+      const target = posix.normalize(posix.join(dir, src)).replace(/\/$/, '');
+      // require('..') / import '.' name a directory: its index file.
+      if (target === '.' || target === '') return index.lookup('index');
+      return index.lookup(target);
+    }
+    if (src.startsWith('/')) return index.lookup(src.slice(1));
+    // tsconfig-style aliases: "@/x", "~/x", "src/x", bare "vs/base/common/x"
+    const stripped = src.replace(/^(@|~)\//, '');
+    if (!stripped.includes('/')) return undefined; // bare package name
+    return index.lookup(stripped) ?? index.lookup('src/' + stripped) ?? index.suffix(stripped, row.path);
+  }
+
+  if (row.lang === 'python') {
+    const m = /^(\.+)(.*)$/.exec(src);
+    if (m) {
+      let base = dir;
+      for (let i = 1; i < m[1]!.length; i++) base = posix.dirname(base);
+      const rest = m[2]!.replace(/\./g, '/');
+      return index.lookup(posix.normalize(posix.join(base, rest || '.')));
+    }
+    const p = src.replace(/\./g, '/');
+    const direct = index.lookup(p) ?? index.lookup('src/' + p) ?? index.lookup('lib/' + p);
+    if (direct !== undefined) return direct;
+    // A bare top-level module (`import datetime`) that is not at a source
+    // root is almost always the standard library or a dependency.
+    if (!p.includes('/')) return undefined;
+    return index.suffix(p, row.path);
+  }
+
+  if (row.lang === 'c' || row.lang === 'cpp' || row.lang === 'objc') {
+    // <system.h> includes only resolve to an exact project path (a public
+    // include dir), never by suffix: <math.h> is not some vendored test/math.h.
+    const sys = /^<(.*)>$/.exec(src);
+    if (sys) {
+      const p = sys[1]!;
+      return index.lookup(p) ?? index.lookup('include/' + p) ?? index.lookup('src/' + p);
+    }
+    return (
+      index.lookup(posix.normalize(posix.join(dir, src))) ??
+      index.lookup(src) ??
+      index.suffix(src, row.path)
+    );
+  }
+
+  if (row.lang === 'rust') {
+    let parts = src.replace(/\{.*$/s, '').replace(/::\*?$/, '').split('::').filter(Boolean);
+    let base = '';
+    if (parts[0] === 'crate') parts = parts.slice(1);
+    else if (parts[0] === 'self') { parts = parts.slice(1); base = dir; }
+    else if (parts[0] === 'super') {
+      base = dir;
+      while (parts[0] === 'super') { parts = parts.slice(1); base = posix.dirname(base); }
+    }
+    // Trailing segments are usually items inside the module: try shorter paths.
+    for (let n = parts.length; n >= 1 && n >= parts.length - 2; n--) {
+      const p = parts.slice(0, n).join('/');
+      const hit = base ? index.lookup(posix.join(base, p)) : index.suffix(p, row.path);
+      if (hit !== undefined) return hit;
+    }
+    return undefined;
+  }
+
+  if (row.lang === 'go') return undefined; // Go imports name packages (directories), not files.
+
+  // Java / C# / PHP / Ruby / Kotlin / Swift / Dart ...: dotted or slashed names.
+  if (row.lang === 'ruby' || row.lang === 'dart') {
+    const clean = src.replace(/^package:[^/]+\//, '');
+    return (
+      index.lookup(posix.normalize(posix.join(dir, clean))) ??
+      index.lookup('lib/' + clean) ??
+      index.suffix(clean, row.path)
+    );
+  }
+  const p = src.replace(/^\\/, '').replace(/[.\\]|::/g, '/').replace(/\/\*$/, '');
+  const hit = index.suffix(p, row.path);
+  if (hit !== undefined || row.lang !== 'java' || !p.includes('/')) return hit;
+  // `import static pkg.Utils.mkEntry;` / `import pkg.Outer.Inner;`: the last segment is a member of the class file.
+  return index.suffix(p.slice(0, p.lastIndexOf('/')), row.path);
+}
+
+/**
+ * Resolve import specifiers to indexed files. With `fileIds`, only imports of
+ * those files are (re)resolved; otherwise every import is.
+ */
+export function resolveImports(db: ClastDb, fileIds?: number[], newPaths?: string[]): number {
+  const files = db.all<{ id: number; path: string }>(`SELECT id, path FROM files`);
+  const index = new PathIndex(files);
+
+  const base = `SELECT i.rowid AS rowid, i.file_id, i.source, f.path, f.lang
+                FROM imports i JOIN files f ON f.id = i.file_id`;
+  let rows: ImportRow[];
+  if (fileIds) {
+    rows = [];
+    for (const id of fileIds) rows.push(...db.all<ImportRow>(`${base} WHERE i.file_id = ?`, id));
+    // Files that just appeared may satisfy imports that could not be resolved before:
+    // retry only the unresolved imports that mention such a file's name.
+    const seen = new Set(rows.map(r => r.rowid));
+    for (const p of newPaths ?? []) {
+      const stem = p.slice(p.lastIndexOf('/') + 1).replace(/\.[A-Za-z0-9]+$/, '');
+      if (stem.length < 2) continue;
+      const like = '%' + stem.replace(/[\\%_]/g, c => '\\' + c) + '%';
+      for (const r of db.all<ImportRow>(`${base} WHERE i.resolved_id IS NULL AND i.source LIKE ? ESCAPE '\\'`, like)) {
+        if (!seen.has(r.rowid)) {
+          seen.add(r.rowid);
+          rows.push(r);
+        }
+      }
+    }
+  } else {
+    rows = db.all<ImportRow>(base);
+  }
+
+  let resolved = 0;
+  db.transaction(() => {
+    for (const row of rows) {
+      let target: number | undefined;
+      try {
+        target = resolveOne(row, index);
+      } catch {
+        target = undefined;
+      }
+      if (target === row.file_id) target = undefined;
+      db.run(`UPDATE imports SET resolved_id = ? WHERE rowid = ?`, target ?? null, row.rowid);
+      if (target !== undefined) resolved++;
+    }
+  });
+  return resolved;
+}

package/src/indexer/index.ts

@@ -0,0 +1,369 @@
+import { Worker } from 'node:worker_threads';
+import { availableParallelism } from 'node:os';
+import { existsSync, statSync } from 'node:fs';
+import { fileURLToPath } from 'node:url';
+import { isAbsolute, join, relative } from 'node:path';
+import type { ClastConfig } from '../config/index.js';
+import type { ClastDb } from '../db/index.js';
+import type { FileRow } from '../types.js';
+import { SourceParser } from '../parser/index.js';
+import { IgnoreRules, type ScannedFile } from './scan.js';
+import { processFile, type FileResult, type FileTask } from './process.js';
+import { resolveImports } from './imports.js';
+
+export interface IndexResult {
+  scanned: number;
+  parsed: number;
+  unchanged: number;
+  removed: number;
+  durationMs: number;
+  workers: number;
+}
+
+export interface IndexProgress {
+  phase: 'idle' | 'scanning' | 'parsing' | 'finalizing';
+  done: number;
+  total: number;
+}
+
+const WORKER_THRESHOLD = 48;
+const TASK_BATCH = 12;
+const WRITE_BATCH = 150;
+const BULK_THRESHOLD = 400;
+const BULK_WRITE_BATCH = 800;
+
+/**
+ * Keeps the database in sync with the working tree: bulk (re)indexing with a
+ * worker pool, single-file updates, and on-demand freshness checks.
+ */
+export class Indexer {
+  readonly root: string;
+  readonly rules: IgnoreRules;
+  readonly parser = new SourceParser();
+  progress: IndexProgress = { phase: 'idle', done: 0, total: 0 };
+  lastResult: IndexResult | null = null;
+  /** True while a first-time index is being loaded (secondary indexes are not built yet). */
+  coldBuild = false;
+  private running: Promise<IndexResult> | null = null;
+  private structureChanged = false;
+
+  constructor(
+    root: string,
+    private readonly config: ClastConfig,
+    private readonly db: ClastDb
+  ) {
+    this.root = root;
+    this.rules = new IgnoreRules(root, config);
+  }
+
+  get isIndexing(): boolean {
+    return this.running !== null;
+  }
+
+  /** Resolves when the in-flight bulk index (if any) is finished. */
+  async whenIdle(timeoutMs?: number): Promise<boolean> {
+    if (!this.running) return true;
+    if (timeoutMs === undefined) {
+      await this.running.catch(() => undefined);
+      return true;
+    }
+    let timer: ReturnType<typeof setTimeout> | undefined;
+    const timedOut = new Promise<boolean>(res => { timer = setTimeout(() => res(false), timeoutMs); });
+    const done = this.running.then(() => true, () => true);
+    const result = await Promise.race([done, timedOut]);
+    clearTimeout(timer);
+    return result;
+  }
+
+  toRel(path: string): string {
+    const rel = isAbsolute(path) ? relative(this.root, path) : path;
+    return rel.replace(/\\/g, '/').replace(/^\.\//, '');
+  }
+
+  /** Bring the whole index up to date. Concurrent calls share one run. */
+  indexAll(opts: { force?: boolean } = {}): Promise<IndexResult> {
+    if (this.running) return this.running;
+    this.running = this.runIndexAll(opts).finally(() => {
+      this.running = null;
+      this.progress = { phase: 'idle', done: 0, total: 0 };
+    });
+    return this.running;
+  }
+
+  private async runIndexAll(opts: { force?: boolean }): Promise<IndexResult> {
+    const start = performance.now();
+    this.progress = { phase: 'scanning', done: 0, total: 0 };
+    this.rules.invalidate();
+    const scanned = await this.rules.scan();
+
+    // A forced rebuild starts from an empty index so it can use the cold bulk path.
+    if (opts.force) this.db.clearIndex();
+    const known = new Map<string, FileRow>();
+    for (const row of this.db.allFiles()) known.set(row.path, row);
+
+    const tasks: FileTask[] = [];
+    const seen = new Set<string>();
+    for (const f of scanned) {
+      seen.add(f.rel);
+      const row = known.get(f.rel);
+      if (!opts.force && row && row.mtime === f.mtime && row.size === f.size) continue;
+      tasks.push({ abs: f.abs, rel: f.rel, knownHash: opts.force ? undefined : row?.hash });
+    }
+
+    let removed = 0;
+    const gone = [...known.keys()].filter(p => !seen.has(p));
+    if (gone.length) {
+      this.db.transaction(() => {
+        for (const p of gone) if (this.db.deleteFile(p)) removed++;
+      });
+      this.structureChanged = true;
+    }
+
+    const bulk = tasks.length >= BULK_THRESHOLD;
+    const cold = known.size === 0;
+    const writeBatch = bulk ? BULK_WRITE_BATCH : WRITE_BATCH;
+    if (bulk) this.db.beginBulk(cold);
+    this.coldBuild = bulk && cold;
+
+    this.progress = { phase: 'parsing', done: 0, total: tasks.length };
+    let parsed = 0;
+    let unchanged = 0;
+    const touchedIds: number[] = [];
+    let pending: FileResult[] = [];
+
+    let writeMs = 0;
+    const flush = (): void => {
+      if (pending.length === 0) return;
+      const batch = pending;
+      pending = [];
+      const t0 = performance.now();
+      this.db.transaction(() => {
+        for (const r of batch) {
+          const outcome = this.apply(r);
+          if (outcome.parsed) { parsed++; touchedIds.push(outcome.fileId!); }
+          else if (r.status === 'unchanged') unchanged++;
+        }
+      });
+      writeMs += performance.now() - t0;
+    };
+
+    const completed = new Set<string>();
+    const onResult = (r: FileResult): void => {
+      pending.push(r);
+      completed.add(r.rel);
+      this.progress.done++;
+      if (pending.length >= writeBatch) flush();
+    };
+    const inline = async (list: FileTask[]): Promise<void> => {
+      for (const task of list) {
+        onResult(await processFile(this.parser, task, this.config.maxFileSize));
+        // Yield so the MCP transport stays responsive during a cold index.
+        if (this.progress.done % 25 === 0) await new Promise<void>(res => setImmediate(res));
+      }
+    };
+
+    let workerCount = this.workerCount(tasks.length);
+    let endBulkMs = 0;
+    try {
+      if (workerCount > 1) {
+        try {
+          await this.runPool(tasks, workerCount, onResult);
+        } catch (err) {
+          // A worker died (out of memory, a crashing grammar): finish the rest on this thread.
+          console.error(`[clast] parser worker failed (${err instanceof Error ? err.message : err}); continuing without workers`);
+          workerCount = 1;
+          await inline(tasks.filter(t => !completed.has(t.rel)));
+        }
+      } else {
+        await inline(tasks);
+      }
+      flush();
+    } finally {
+      // Always leave bulk mode: it restores the journal and rebuilds the secondary indexes,
+      // without which every query would scan whole tables.
+      const tBulk = performance.now();
+      if (bulk) this.db.endBulk();
+      endBulkMs = performance.now() - tBulk;
+      this.coldBuild = false;
+    }
+
+    const parseDoneMs = performance.now() - start;
+    this.progress.phase = 'finalizing';
+    if (parsed > 0 || removed > 0 || this.structureChanged) {
+      // New or removed files can change how *other* files' imports resolve.
+      const full = removed > 0 || this.structureChanged || parsed > 200;
+      resolveImports(this.db, full ? undefined : touchedIds);
+      this.structureChanged = false;
+      try { this.db.db.exec('PRAGMA optimize'); } catch { /* ignore */ }
+    }
+
+    const result: IndexResult = {
+      scanned: scanned.length,
+      parsed,
+      unchanged,
+      removed,
+      durationMs: Math.round(performance.now() - start),
+      workers: workerCount,
+    };
+    if (process.env.CLAST_DEBUG) {
+      console.error(`[clast] timing: total ${result.durationMs}ms, parse+write phase ${Math.round(parseDoneMs)}ms, db writes ${Math.round(writeMs)}ms, index build ${Math.round(endBulkMs)}ms`);
+    }
+    this.lastResult = result;
+    this.db.setMeta('last_full_index', String(Date.now()));
+    return result;
+  }
+
+  /** Write one processing result. Must run inside a transaction. */
+  private apply(r: FileResult): { parsed: boolean; fileId?: number } {
+    if (r.status === 'ok' && r.extraction) {
+      const isNew = !this.db.fileByPath(r.rel);
+      const fileId = this.db.writeFile({ path: r.rel, hash: r.hash, mtime: r.mtime, size: r.size }, r.extraction);
+      if (isNew) this.structureChanged = true;
+      return { parsed: true, fileId };
+    }
+    if (r.status === 'unchanged') {
+      const row = this.db.fileByPath(r.rel);
+      if (row) this.db.touchFile(row.id, r.mtime, r.size);
+      return { parsed: false };
+    }
+    // skip / missing: make sure nothing stale remains.
+    if (this.db.deleteFile(r.rel)) this.structureChanged = true;
+    return { parsed: false };
+  }
+
+  private workerCount(taskCount: number): number {
+    if (taskCount < WORKER_THRESHOLD) return 1;
+    if (!existsSync(workerPath())) return 1;
+    const configured = this.config.workers;
+    if (configured === 1) return 1;
+    const auto = Math.max(1, Math.min(12, availableParallelism() - 1));
+    const n = configured > 0 ? configured : auto;
+    return Math.max(1, Math.min(n, Math.ceil(taskCount / TASK_BATCH)));
+  }
+
+  private runPool(tasks: FileTask[], count: number, onResult: (r: FileResult) => void): Promise<void> {
+    return new Promise<void>((resolveAll, rejectAll) => {
+      let next = 0;
+      let active = 0;
+      let failed = false;
+      const workers: Worker[] = [];
+
+      const finish = (err?: Error): void => {
+        if (failed) return;
+        if (err) {
+          failed = true;
+          for (const w of workers) void w.terminate();
+          rejectAll(err);
+          return;
+        }
+        if (active === 0 && next >= tasks.length) {
+          for (const w of workers) w.postMessage(null);
+          resolveAll();
+        }
+      };
+
+      const dispatch = (w: Worker): void => {
+        if (next >= tasks.length) return finish();
+        const batch = tasks.slice(next, next + TASK_BATCH);
+        next += batch.length;
+        active++;
+        w.postMessage(batch);
+      };
+
+      for (let i = 0; i < count; i++) {
+        const w = new Worker(workerPath(), { workerData: { maxFileSize: this.config.maxFileSize } });
+        workers.push(w);
+        w.on('message', (results: FileResult[]) => {
+          active--;
+          try {
+            for (const r of results) onResult(r);
+          } catch (err) {
+            return finish(err as Error);
+          }
+          dispatch(w);
+        });
+        w.on('error', err => finish(err));
+        // Two batches in flight per worker so it never idles while the main
+        // thread is busy writing a transaction.
+        dispatch(w);
+        dispatch(w);
+      }
+    });
+  }
+
+  /**
+   * Re-index a single file inline. Returns true if the index changed.
+   * Accepts absolute or project-relative paths.
+   */
+  async indexFile(path: string, opts: { force?: boolean } = {}): Promise<boolean> {
+    const rel = this.toRel(path);
+    if (rel.startsWith('..')) return false;
+    const abs = join(this.root, rel);
+    if (!this.rules.isSupported(rel) || this.rules.isIgnored(rel)) {
+      return this.removeFile(rel);
+    }
+    const row = this.db.fileByPath(rel);
+    const result = await processFile(
+      this.parser,
+      { abs, rel, knownHash: opts.force ? undefined : row?.hash },
+      this.config.maxFileSize
+    );
+    let changed = false;
+    let fileId: number | undefined;
+    this.db.transaction(() => {
+      const outcome = this.apply(result);
+      changed = outcome.parsed || (result.status !== 'unchanged' && !!row);
+      fileId = outcome.fileId;
+    });
+    if (fileId !== undefined) {
+      // A new file can also satisfy other files' previously unresolved imports;
+      // only those that mention its name are retried (not the whole project).
+      resolveImports(this.db, [fileId], this.structureChanged ? [rel] : undefined);
+      this.structureChanged = false;
+    }
+    return changed;
+  }
+
+  removeFile(path: string): boolean {
+    const rel = this.toRel(path);
+    let removed = false;
+    this.db.transaction(() => {
+      removed = this.db.deleteFile(rel);
+    });
+    return removed;
+  }
+
+  /**
+   * Make sure the given indexed files reflect what is on disk right now
+   * (cheap stat check, inline re-parse when stale). Query paths call this
+   * before returning line numbers or source text.
+   */
+  async ensureFresh(paths: Iterable<string>): Promise<number> {
+    let updated = 0;
+    const seen = new Set<string>();
+    for (const rel of paths) {
+      if (seen.has(rel)) continue;
+      seen.add(rel);
+      const row = this.db.fileByPath(rel);
+      if (!row) continue;
+      let st;
+      try {
+        st = statSync(join(this.root, rel));
+      } catch {
+        if (this.removeFile(rel)) updated++;
+        continue;
+      }
+      if (st.mtimeMs === row.mtime && st.size === row.size) continue;
+      if (await this.indexFile(rel)) updated++;
+    }
+    return updated;
+  }
+
+  scan(): Promise<ScannedFile[]> {
+    return this.rules.scan();
+  }
+}
+
+function workerPath(): string {
+  return fileURLToPath(new URL('./worker.js', import.meta.url));
+}

package/src/indexer/process.ts

@@ -0,0 +1,63 @@
+import { createHash } from 'node:crypto';
+import { readFileSync, statSync } from 'node:fs';
+import type { Extraction } from '../types.js';
+import type { SourceParser } from '../parser/index.js';
+
+export interface FileTask {
+  abs: string;
+  rel: string;
+  /** Hash currently stored in the index, if any. */
+  knownHash?: string;
+}
+
+export interface FileResult {
+  rel: string;
+  /**
+   * ok: parsed, `extraction` is set. unchanged: content hash matches the index.
+   * skip: not indexable (binary, too large, unsupported). missing: file is gone.
+   */
+  status: 'ok' | 'unchanged' | 'skip' | 'missing';
+  hash: string;
+  mtime: number;
+  size: number;
+  extraction?: Extraction;
+}
+
+/** Read, hash and (if changed) parse one file. Never throws. */
+export async function processFile(
+  parser: SourceParser,
+  task: FileTask,
+  maxFileSize: number
+): Promise<FileResult> {
+  const base = { rel: task.rel, hash: '', mtime: 0, size: 0 };
+  let buf: Buffer;
+  try {
+    const st = statSync(task.abs);
+    base.mtime = st.mtimeMs;
+    base.size = st.size;
+    if (!st.isFile()) return { ...base, status: 'missing' };
+    if (st.size > maxFileSize) return { ...base, status: 'skip' };
+    buf = readFileSync(task.abs);
+  } catch {
+    return { ...base, status: 'missing' };
+  }
+
+  // Binary sniff: NUL byte in the first block.
+  const head = buf.subarray(0, 4096);
+  if (head.includes(0)) return { ...base, status: 'skip' };
+
+  base.hash = createHash('sha1').update(buf).digest('hex');
+  if (task.knownHash && task.knownHash === base.hash) return { ...base, status: 'unchanged' };
+
+  let source = buf.toString('utf-8');
+  if (source.charCodeAt(0) === 0xfeff) source = source.slice(1);
+
+  try {
+    const extraction = await parser.extract(task.rel, source);
+    if (!extraction) return { ...base, status: 'skip' };
+    return { ...base, status: 'ok', extraction };
+  } catch (err) {
+    if (process.env.CLAST_DEBUG) console.error(`[clast] parse failed for ${task.rel}:`, err);
+    return { ...base, status: 'skip' };
+  }
+}