graph-indexer 3.0.1 → 3.1.0

package/src/index/indexer.mjs

@@ -19,6 +19,7 @@ import { extractFile } from '../parse/extract.mjs';
 import { specForPath, LANGUAGES } from '../parse/languages.mjs';
 import { tokenString, codeTokens } from '../search/tokenize.mjs';
 import { indexFingerprint } from './fingerprint.mjs';
+import { ParsePool, poolSize } from './parsepool.mjs';
 
 const SPECS = Object.fromEntries(LANGUAGES.map(l => [l.id, l]));
 const TYPE_KINDS = new Set(['class', 'interface', 'struct', 'enum', 'trait', 'type', 'object', 'module', 'impl']);
@@ -59,7 +60,7 @@ export class Indexer {
         this.resolveAll = s.getMeta('resolve_pending') === '1';
         const files = s.all('SELECT id, path, lang, package, is_test FROM files');
         for (const f of files) this.table.addFile({ id: f.id, path: f.path, lang: f.lang, package: f.package, isTest: !!f.is_test });
-        for (const r of s.all('SELECT id, file_id, name, qname, kind, parent_id, owner, type, bases, exported, is_static, start_line, end_line FROM symbols')) {
+        for (const r of s.all('SELECT id, file_id, name, qname, kind, parent_id, owner, type, bases, decorators, exported, is_static, start_line, end_line FROM symbols')) {
             this.table.addSym(rowToSym(r));
         }
         // the interfaces a Go type satisfies implicitly are stored only as `~structural` edges
@@ -80,6 +81,7 @@ export class Indexer {
         for (const r of s.all('SELECT file_id, owner_qname, name, type FROM fields')) (fields.get(r.file_id) ?? fields.set(r.file_id, []).get(r.file_id)).push(r);
         for (const [fid, rows] of fields) this.table.setFieldTypes(fid, rows);
         this.modules = new ModuleResolver(this.root, files.map(f => f.path));
+        this.resolver.modules = this.modules;
         this.loaded = true;
     }
 
@@ -152,7 +154,7 @@ export class Indexer {
                 for (const rel of removed) {
                     const f = s.get('SELECT id FROM files WHERE path = ?', rel);
                     if (!f) continue;
-                    for (const r of s.all('SELECT id FROM symbols WHERE file_id = ?', f.id)) { removedSymIds.push(r.id); s.run('DELETE FROM fts WHERE rowid = ?', r.id); }
+                    for (const r of s.all('SELECT id FROM symbols WHERE file_id = ?', f.id)) { removedSymIds.push(r.id); s.run('DELETE FROM fts WHERE rowid = ?', r.id); s.run('DELETE FROM doc_fts WHERE rowid = ?', r.id); }
                     s.run('DELETE FROM file_fts WHERE rowid = ?', f.id);
                     for (const t of ['symbols', 'refs', 'imports', 'fields']) s.run(`DELETE FROM ${t} WHERE file_id = ?`, f.id);
                     s.run('DELETE FROM files WHERE id = ?', f.id);
@@ -162,17 +164,20 @@ export class Indexer {
             });
         }
 
-        // parse changed/new files (async extraction), write in batches
+        // parse changed/new files, write in batches; with many files a pool of threads parses the
+        // next batch while this one is written
         if (toParse.length || removed.length) s.setMeta('resolve_pending', '1');
         const BATCH = 200;
-        let done = 0;
-        for (let i = 0; i < toParse.length; i += BATCH) {
-            const batch = toParse.slice(i, i + BATCH);
-            const parsed = [];
+        const threads = poolSize(toParse.length);
+        const pool = threads ? new ParsePool(threads) : null;
+        // a file the parser could not read at all is one unreadable region, so no answer calls it complete
+        const failed = (e, rel, source) => { this.log(`parse failed for ${rel}: ${e.message}`); return { symbols: [], refs: [], imports: [], fields: [], fileInfo: null, errors: 1, errorLines: [[1, source.split('\n').length]] }; };
+        // read and hash a batch now; its extraction starts now with the pool, or when awaited without
+        const start = (batch) => {
+            const items = [];
             for (const item of batch) {
-                const abs = path.join(this.root, item.rel);
                 let source;
-                try { source = fs.readFileSync(abs, 'utf8'); } catch { continue; }
+                try { source = fs.readFileSync(path.join(this.root, item.rel), 'utf8'); } catch { continue; }
                 const hash = sha1(source);
                 if (item.prev && item.prev.hash === hash) {
                     s.run('UPDATE files SET size = ?, mtime_ms = ? WHERE id = ?', item.st.size, Math.floor(item.st.mtimeMs), item.prev.id);
@@ -181,26 +186,48 @@ export class Indexer {
                 }
                 const spec = specForPath(item.rel);
                 if (!spec) continue;
-                let extraction;
-                if (looksMinified(source)) extraction = { symbols: [], refs: [], imports: [], fields: [], fileInfo: null, errors: 0 };
-                else {
-                    try { extraction = await extractFile(spec, source, item.rel); }
-                    catch (e) { this.log(`parse failed for ${item.rel}: ${e.message}`); extraction = { symbols: [], refs: [], imports: [], fields: [], fileInfo: null, errors: 1 }; }
-                }
-                parsed.push({ ...item, source, hash, spec, extraction });
+                const minified = looksMinified(source);
+                const inline = () => extractFile(spec, source, item.rel).catch(e => failed(e, item.rel, source));
+                const job = minified ? null : pool ? pool.extract(item.rel, source).catch(inline) : null;
+                items.push({ ...item, source, hash, spec, minified, job });
             }
-            s.tx(() => {
-                for (const p of parsed) {
-                    const r = this.#writeFile(p);
-                    touchedFileIds.push(r.fileId);
-                    removedSymIds.push(...r.removedIds);
-                    for (const n of r.addedNames) addedNames.add(n);
-                    if (p.prev) changed++; else added++;
+            return items;
+        };
+        let done = 0;
+        try {
+            // with the pool, a few batches are in flight so no thread waits; sources held stay bounded
+            const ahead = pool ? 4 : 0;
+            const queued = [];
+            let started = 0;
+            const fill = () => { while (started < toParse.length && queued.length <= ahead) { queued.push(start(toParse.slice(started, started + BATCH))); started += BATCH; } };
+            for (let i = 0; i < toParse.length; i += BATCH) {
+                fill();
+                const items = queued.shift();
+                fill();
+                const parsed = [];
+                for (const it of items) {
+                    let extraction;
+                    if (it.minified) extraction = { symbols: [], refs: [], imports: [], fields: [], fileInfo: null, errors: 0 };
+                    else if (it.job) extraction = await it.job;
+                    else {
+                        try { extraction = await extractFile(it.spec, it.source, it.rel); }
+                        catch (e) { extraction = failed(e, it.rel, it.source); }
+                    }
+                    parsed.push({ rel: it.rel, st: it.st, prev: it.prev, source: it.source, hash: it.hash, spec: it.spec, extraction });
                 }
-            });
-            done += batch.length;
-            onProgress?.(done, toParse.length);
-        }
+                s.tx(() => {
+                    for (const p of parsed) {
+                        const r = this.#writeFile(p);
+                        touchedFileIds.push(r.fileId);
+                        removedSymIds.push(...r.removedIds);
+                        for (const n of r.addedNames) addedNames.add(n);
+                        if (p.prev) changed++; else added++;
+                    }
+                });
+                done += Math.min(BATCH, toParse.length - i);
+                onProgress?.(done, toParse.length);
+            }
+        } finally { await pool?.close(); }
 
         // resolve: refs of touched files + refs that pointed at removed symbols + refs that may bind to new names
         this.resolver.resetMemo();
@@ -235,7 +262,7 @@ export class Indexer {
 
     #writeFile({ rel, st, prev, source, hash, spec, extraction }) {
         const s = this.store;
-        const { symbols, refs, imports, fields, fileInfo, errors } = extraction;
+        const { symbols, refs, imports, fields, fileInfo, errors, errorLines } = extraction;
         const lines = lineStarts(source);
         const isTest = spec.testFile?.test(rel) ? 1 : 0;
         const pkg = fileInfo?.package ?? null;
@@ -244,15 +271,15 @@ export class Indexer {
         if (prev) {
             fileId = prev.id;
             oldSyms = s.all('SELECT id, qname, kind, ordinal FROM symbols WHERE file_id = ?', fileId);
-            for (const o of oldSyms) s.run('DELETE FROM fts WHERE rowid = ?', o.id);
+            for (const o of oldSyms) { s.run('DELETE FROM fts WHERE rowid = ?', o.id); s.run('DELETE FROM doc_fts WHERE rowid = ?', o.id); }
             s.run('DELETE FROM file_fts WHERE rowid = ?', fileId);
             for (const t of ['symbols', 'refs', 'imports', 'fields']) s.run(`DELETE FROM ${t} WHERE file_id = ?`, fileId);
-            s.run('UPDATE files SET lang = ?, size = ?, mtime_ms = ?, hash = ?, lines = ?, package = ?, is_test = ?, parse_errors = ?, indexed_at = ? WHERE id = ?',
-                spec.id, st.size, Math.floor(st.mtimeMs), hash, lines.length, pkg, isTest, errors, Date.now(), fileId);
+            s.run('UPDATE files SET lang = ?, size = ?, mtime_ms = ?, hash = ?, lines = ?, package = ?, is_test = ?, parse_errors = ?, parse_error_lines = ?, indexed_at = ? WHERE id = ?',
+                spec.id, st.size, Math.floor(st.mtimeMs), hash, lines.length, pkg, isTest, errors, errorLines?.length ? JSON.stringify(errorLines) : null, Date.now(), fileId);
             this.table.removeFile(fileId);
         } else {
-            const r = s.run('INSERT INTO files (path, lang, size, mtime_ms, hash, lines, package, is_test, parse_errors, indexed_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)',
-                rel, spec.id, st.size, Math.floor(st.mtimeMs), hash, lines.length, pkg, isTest, errors, Date.now());
+            const r = s.run('INSERT INTO files (path, lang, size, mtime_ms, hash, lines, package, is_test, parse_errors, parse_error_lines, indexed_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)',
+                rel, spec.id, st.size, Math.floor(st.mtimeMs), hash, lines.length, pkg, isTest, errors, errorLines?.length ? JSON.stringify(errorLines) : null, Date.now());
             fileId = Number(r.lastInsertRowid);
         }
         this.table.addFile({ id: fileId, path: rel, lang: spec.id, package: pkg, isTest: !!isTest });
@@ -281,10 +308,18 @@ export class Indexer {
             const id = reuse ?? Number(r.lastInsertRowid);
             ids[i] = id;
             if (reuse != null) reused.add(reuse); else addedNames.push(sym.name);
-            this.table.addSym({ id, fileId, name: sym.name, qname: sym.qname, kind: sym.kind, parentId, owner: sym.owner, type: sym.type, bases: sym.bases ?? [], exported: !!sym.exported, isStatic: !!sym.isStatic, startLine: sym.startLine, endLine: sym.endLine });
+            this.table.addSym({ id, fileId, name: sym.name, qname: sym.qname, kind: sym.kind, parentId, owner: sym.owner, type: sym.type, bases: sym.bases ?? [], exported: !!sym.exported, isStatic: !!sym.isStatic, startLine: sym.startLine, endLine: sym.endLine, byRef: !!sym.decorators?.[0]?.startsWith('impl &') });
         }
-        // FTS documents
-        for (let i = 0; i < symbols.length; i++) {
+        // FTS documents; documentation goes to its own table and has no file document
+        if (spec.docs) {
+            for (let i = 0; i < symbols.length; i++) {
+                const sym = symbols[i];
+                const a = lines[sym.startLine - 1] ?? 0;
+                const b = sym.endLine >= lines.length ? source.length : lines[sym.endLine];
+                s.run('INSERT INTO doc_fts (rowid, name, body) VALUES (?, ?, ?)', ids[i], tokenString(sym.qname.replace(/ › /g, ' ')), codeTokens(source.slice(a, b), { maxTokens: BODY_TOKEN_CAP }).join(' '));
+            }
+        }
+        for (let i = 0; i < symbols.length && !spec.docs; i++) {
             const sym = symbols[i];
             let body;
             if (TYPE_KINDS.has(sym.kind)) {
@@ -305,16 +340,16 @@ export class Indexer {
                 ids[i], tokenString(sym.name), tokenString(sym.qname), tokenString(sym.sig ?? ''), tokenString(sym.doc ?? ''), pathTokens, body);
         }
         // file document (hierarchical ranking: which files talk about the query at all)
-        s.run('INSERT INTO file_fts (rowid, path, body) VALUES (?, ?, ?)', fileId, pathTokens, codeTokens(source, { maxTokens: FILE_TOKEN_CAP }).join(' '));
+        if (!spec.docs) s.run('INSERT INTO file_fts (rowid, path, body) VALUES (?, ?, ?)', fileId, pathTokens, codeTokens(source, { maxTokens: FILE_TOKEN_CAP }).join(' '));
         // refs (unresolved for now)
         for (const r of refs) {
-            s.run('INSERT INTO refs (file_id, src_id, name, kind, line, col, recv, recv_type, argc) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)',
-                fileId, r.symIdx >= 0 ? ids[r.symIdx] : null, r.name, r.kind, r.line, r.col, r.recv || null, r.recvType || null, r.argc ?? null);
+            s.run('INSERT INTO refs (file_id, src_id, name, kind, line, col, recv, recv_type, argc, arg_types) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)',
+                fileId, r.symIdx >= 0 ? ids[r.symIdx] : null, r.name, r.kind, r.line, r.col, r.recv || null, r.recvType || null, r.argc ?? null, r.argTypes ? JSON.stringify(r.argTypes) : null);
         }
         // imports (module resolution needs only the file list)
         const impRows = [];
         for (const imp of imports) {
-            const target = this.modules.resolve(spec.id, rel, imp.source);
+            const target = this.modules.resolveImport(spec.id, rel, imp);
             let targetFileId = null;
             if (target?.file) targetFileId = s.get('SELECT id FROM files WHERE path = ?', target.file)?.id ?? null;
             const row = { source: imp.source ?? null, imported: imp.imported ?? null, local: imp.local ?? null, reexport: imp.reexport ?? null, wildcard: imp.wildcard ? 1 : 0, line: imp.line ?? null, target_file_id: targetFileId, target_dir: target?.dir ?? null };
@@ -446,7 +481,7 @@ export class Indexer {
         s.tx(() => {
             for (let i = 0; i < refIds.length; i += 900) {
                 const chunk = refIds.slice(i, i + 900);
-                const rows = s.all(`SELECT id, file_id, src_id, name, kind, recv, recv_type, argc FROM refs WHERE id IN (${chunk.map(() => '?').join(',')})`, ...chunk);
+                const rows = s.all(`SELECT id, file_id, src_id, name, kind, recv, recv_type, argc, arg_types FROM refs WHERE id IN (${chunk.map(() => '?').join(',')})`, ...chunk);
                 for (const r of rows) {
                     const res = this.resolver.resolve(r);
                     if (res.id == null && (r.kind === 'value' || r.kind === 'read' || r.kind === 'type' || res.external)) { s.run('DELETE FROM refs WHERE id = ?', r.id); continue; }
@@ -465,6 +500,7 @@ function rowToSym(r) {
     return {
         id: r.id, fileId: r.file_id, name: r.name, qname: r.qname, kind: r.kind, parentId: r.parent_id ?? null, owner: r.owner ?? null,
         type: r.type ?? null, bases: r.bases ? JSON.parse(r.bases) : [], exported: !!r.exported, isStatic: !!r.is_static, startLine: r.start_line, endLine: r.end_line,
+        byRef: r.decorators ? /^\["impl &/.test(r.decorators) : false,
     };
 }
 

package/src/index/modules.mjs

@@ -70,6 +70,32 @@ export class ModuleResolver {
             const scope = this.#tsScope(rel);
             if (scope) this.tsScopes.set(dir, scope);
         }
+        // package.json "imports" (subpath imports, `#/*` → `./src/*`): aliases of that package's files
+        for (const [, { dir, json }] of this.packages) {
+            const imp = json.imports;
+            if (!imp || typeof imp !== 'object') continue;
+            const at = (p) => path.posix.normalize(dir === '.' ? p : `${dir}/${p}`);
+            const target = (v) => typeof v === 'string' ? v : (v && typeof v === 'object' ? (v.types ?? v.import ?? v.default ?? v.node ?? null) : null);
+            const paths = Object.entries(imp).map(([pat, v]) => [pat, target(v)]).filter(([pat, t]) => typeof t === 'string' && t.startsWith('.') && pat.startsWith('#'))
+                .map(([pat, t]) => ({ pat, targets: [at(t)] }));
+            if (!paths.length) continue;
+            const scope = this.tsScopes.get(dir);
+            this.tsScopes.set(dir, scope ? { ...scope, paths: [...scope.paths, ...paths] } : { paths, baseUrl: null });
+        }
+        // SvelteKit: `$lib` (and `kit.alias`) map into the project; they live in a generated
+        // .svelte-kit/tsconfig.json that a repository does not carry, so read svelte.config itself
+        for (const rel of [...this.#findManifests('svelte.config.js'), ...this.#findManifests('svelte.config.ts')]) {
+            const dir = path.posix.dirname(rel), at = (p) => path.posix.normalize(dir === '.' ? p : `${dir}/${p}`);
+            let text = '';
+            try { text = fs.readFileSync(path.join(this.root, rel), 'utf8'); } catch { /* unreadable */ }
+            const lib = /\bfiles\s*:\s*\{[^}]*\blib\s*:\s*['"]([^'"]+)['"]/.exec(text)?.[1] ?? 'src/lib';
+            const aliases = [['$lib', lib]];
+            const block = /\balias\s*:\s*\{([^}]*)\}/.exec(text)?.[1] ?? '';
+            for (const m of block.matchAll(/['"]?([$@#~\w/-]+?)(?:\/\*)?['"]?\s*:\s*['"]([^'"]+?)(?:\/\*)?['"]/g)) aliases.push([m[1], m[2]]);
+            const paths = aliases.flatMap(([a, t]) => [{ pat: a, targets: [at(t)] }, { pat: `${a}/*`, targets: [`${at(t)}/*`] }]);
+            const scope = this.tsScopes.get(dir);
+            this.tsScopes.set(dir, scope ? { ...scope, paths: [...scope.paths, ...paths] } : { paths, baseUrl: null });
+        }
         // Go module path
         this.goModules = [];
         for (const rel of this.#findManifests('go.mod')) {
@@ -79,6 +105,34 @@ export class ModuleResolver {
             } catch { /* unreadable */ }
         }
         this.goModules.sort((a, b) => b.module.length - a.module.length);
+        // Rust crates of the repository (a workspace's members, or one package): the name other
+        // crates, tests and examples use for each library (`serde_json::Value`, `tokio::sync`)
+        this.rustCrates = new Map();
+        this.rustPackages = [];
+        for (const rel of this.#findManifests('Cargo.toml')) {
+            let text;
+            try { text = fs.readFileSync(path.join(this.root, rel), 'utf8'); } catch { continue; }
+            const dir = path.posix.dirname(rel), at = (p) => (dir === '.' ? p : `${dir}/${p}`);
+            if (/^\[package\]\s*$/m.test(text)) this.rustPackages.push(dir);
+            const table = (name) => { const m = new RegExp(`^\\[${name}\\]\\s*$([\\s\\S]*?)(?=^\\[|(?![\\s\\S]))`, 'm').exec(text); return m?.[1] ?? ''; };
+            const key = (body, k) => new RegExp(`^\\s*${k}\\s*=\\s*"([^"]+)"`, 'm').exec(body)?.[1];
+            const pkg = key(table('package'), 'name'), lib = table('lib');
+            const libFile = path.posix.normalize(at(key(lib, 'path') ?? 'src/lib.rs'));
+            if (!pkg || !this.files.has(libFile)) continue;
+            this.rustCrates.set((key(lib, 'name') ?? pkg).replace(/-/g, '_'), libFile);
+        }
+    }
+
+    /**
+     * Resolve one import (`imp.source`, `imp.imported`). In Rust, `use a::b::c` resolves to the
+     * module `c` when c is a module (a file `c.rs` or `c/mod.rs`), not to the module that declares it.
+     */
+    resolveImport(lang, fromFile, imp) {
+        if (lang === 'rust' && imp.imported && !['self', '*', 'super', 'crate'].includes(imp.imported) && /^[a-z_][a-z0-9_]*$/.test(imp.imported)) {
+            const sub = this.resolve(lang, fromFile, imp.source ? `${imp.source}::${imp.imported}` : imp.imported);
+            if (sub?.file && new RegExp(`(^|/)${imp.imported}(\\.rs|/mod\\.rs)$`).test(sub.file)) return sub;
+        }
+        return this.resolve(lang, fromFile, imp.source);
     }
 
     /** compilerOptions of a tsconfig file, following relative `extends` (a child's paths replace its parent's). */
@@ -178,7 +232,7 @@ export class ModuleResolver {
         if (!source) return null;
         const fromDir = path.posix.dirname(fromFile);
         switch (lang) {
-            case 'javascript': case 'typescript': case 'tsx': return this.#resolveEcma(fromDir, source);
+            case 'javascript': case 'typescript': case 'tsx': case 'vue': case 'svelte': return this.#resolveEcma(fromDir, source);
             case 'python': return this.#resolvePython(fromDir, source);
             case 'go': return this.#resolveGo(source);
             case 'c': case 'cpp': return this.#resolveInclude(fromDir, source);
@@ -247,7 +301,10 @@ export class ModuleResolver {
         const rel = mod.replace(/\./g, '/');
         const idx = this.#suffixIndex();
         const cands = [...(idx.get(rel + '.py') ?? []), ...(idx.get(rel + '/__init__.py') ?? [])]
-            .filter(f => f === rel + '.py' || f === rel + '/__init__.py' || f.endsWith('/' + rel + '.py') || f.endsWith('/' + rel + '/__init__.py'));
+            .filter(f => f === rel + '.py' || f === rel + '/__init__.py' || f.endsWith('/' + rel + '.py') || f.endsWith('/' + rel + '/__init__.py'))
+            // an absolute import starts at an import root: `import json` is not `flask/json/`, since
+            // `flask/` is a package (it has an __init__.py) and only reaches it as `flask.json`
+            .filter(f => { const root = f.slice(0, f.length - (f.endsWith('/__init__.py') ? rel.length + 12 : rel.length + 3)).replace(/\/$/, ''); return !root || !this.files.has(root + '/__init__.py'); });
         if (cands.length) {
             cands.sort((a, b) => a.length - b.length);
             const f = cands[0];
@@ -305,16 +362,33 @@ export class ModuleResolver {
         if (segs[0] === 'crate') { baseDir = crateRoot; segs.shift(); }
         else if (segs[0] === 'self') { baseDir = this.#rustModDir(fromFile); segs.shift(); }
         else if (segs[0] === 'super') { baseDir = path.posix.dirname(this.#rustModDir(fromFile)); segs.shift(); while (segs[0] === 'super') { baseDir = path.posix.dirname(baseDir); segs.shift(); } }
+        else if (this.rustCrates?.has(segs[0])) {
+            // another crate of the repository, by name; its root module is its lib.rs
+            const lib = this.rustCrates.get(segs.shift());
+            if (!segs.length) return { file: lib };
+            baseDir = path.posix.dirname(lib);
+        }
+        // a child module of this one (2018 paths: `mod runtime; pub use runtime::Runtime;`)
+        else if (['.rs', '/mod.rs'].some(e => this.files.has(path.posix.join(this.#rustModDir(fromFile), segs[0]) + e))) baseDir = this.#rustModDir(fromFile);
         else return null; // external crate
         for (let n = segs.length; n > 0; n--) {
             const p = path.posix.join(baseDir, ...segs.slice(0, n));
             if (this.files.has(p + '.rs')) return { file: p + '.rs' };
             if (this.files.has(p + '/mod.rs')) return { file: p + '/mod.rs' };
         }
+        // the module itself: `dir/mod.rs`, or `dir.rs` next to its directory (2018 layout)
+        if (this.files.has(baseDir + '.rs') && !['lib.rs', 'main.rs', 'mod.rs'].some(f => this.files.has(path.posix.join(baseDir, f)))) return { file: baseDir + '.rs' };
         const lib = ['lib.rs', 'main.rs', 'mod.rs'].map(f => path.posix.join(baseDir, f)).find(f => this.files.has(f));
         return lib ? { file: lib } : null;
     }
 
+    /** The directory of the Cargo package a file belongs to (its nearest `Cargo.toml` with a `[package]`), or null. */
+    rustPackageOf(file) {
+        let best = null;
+        for (const d of this.rustPackages ?? []) if ((d === '.' || file.startsWith(d + '/')) && (best == null || d.length > best.length)) best = d;
+        return best;
+    }
+
     #rustCrateRoot(fromFile) {
         let d = path.posix.dirname(fromFile);
         while (d && d !== '.') {

package/src/index/parse-worker.mjs

@@ -0,0 +1,12 @@
+/** A parsing thread of the ParsePool: extracts one file per message (src/index/parsepool.mjs). */
+import { parentPort } from 'node:worker_threads';
+import { extractFile } from '../parse/extract.mjs';
+import { specForPath } from '../parse/languages.mjs';
+
+parentPort.on('message', async ({ id, rel, source }) => {
+    try {
+        parentPort.postMessage({ id, extraction: await extractFile(specForPath(rel), source, rel) });
+    } catch (e) {
+        parentPort.postMessage({ id, error: e.message });
+    }
+});

package/src/index/parsepool.mjs

@@ -0,0 +1,73 @@
+/**
+ * Parsing on several threads. Extraction (tree-sitter parse + queries) is most of a full index's
+ * time and each file is independent, so a pool of worker threads parses while the main thread
+ * writes the previous batch to SQLite. Each worker loads the WASM runtime and the grammars it meets
+ * once; results come back by structured clone, the same objects `extractFile` returns.
+ *
+ * A file the pool fails on is parsed again inline by the caller, so a thread error never loses it.
+ * Only worth it for many files: small updates (a watcher event, a query-time check) parse inline.
+ * GRAPH_INDEXER_WORKERS sets the number of threads (0 turns the pool off).
+ */
+import os from 'node:os';
+import { Worker } from 'node:worker_threads';
+
+const WORKER = new URL('./parse-worker.mjs', import.meta.url);
+export const POOL_MIN_FILES = 48;
+
+/** Threads to use for `n` files to parse: 0 when parsing inline is as fast. */
+export function poolSize(n) {
+    const env = process.env.GRAPH_INDEXER_WORKERS;
+    const want = env != null && env !== '' ? Math.max(0, Number(env) | 0) : Math.min(8, Math.max(0, (os.availableParallelism?.() ?? os.cpus().length) - 1));
+    if (n < POOL_MIN_FILES || want < 1) return 0;
+    return Math.min(want, Math.ceil(n / 16));
+}
+
+export class ParsePool {
+    constructor(size) {
+        this.workers = [];
+        this.idle = [];
+        this.queue = [];
+        this.pending = new Map();
+        this.nextId = 0;
+        for (let i = 0; i < size; i++) {
+            const w = new Worker(WORKER);
+            w.on('message', ({ id, extraction, error }) => {
+                const p = this.pending.get(id);
+                this.pending.delete(id);
+                if (error) p.reject(new Error(error)); else p.resolve(extraction);
+                this.#next(w);
+            });
+            // a thread that dies takes the pool with it: every pending and later file is parsed inline
+            w.on('error', (e) => this.#break(e));
+            w.on('exit', (code) => { if (code !== 0 && !this.closing) this.#break(new Error(`parse thread exited with code ${code}`)); });
+            this.workers.push(w);
+            this.idle.push(w);
+        }
+    }
+
+    /** Extraction of one file, as `extractFile(specForPath(rel), source, rel)` would return it. */
+    extract(rel, source) {
+        if (this.broken) return Promise.reject(this.broken);
+        return new Promise((resolve, reject) => {
+            const id = this.nextId++;
+            this.pending.set(id, { resolve, reject });
+            const task = { id, rel, source };
+            const w = this.idle.pop();
+            if (w) w.postMessage(task); else this.queue.push(task);
+        });
+    }
+
+    #next(w) {
+        const task = this.queue.shift();
+        if (task) w.postMessage(task); else this.idle.push(w);
+    }
+
+    #break(e) {
+        this.broken ??= e;
+        for (const p of this.pending.values()) p.reject(e);
+        this.pending.clear();
+        this.queue = [];
+    }
+
+    async close() { this.closing = true; await Promise.all(this.workers.map(w => w.terminate())); this.workers = []; }
+}