graph-indexer 2.0.0 → 2.1.0

package/parse/languages.mjs

@@ -1,86 +1,250 @@
 /**
  * @file parse/languages.mjs
- * @description Language registry: dynamic Tree-sitter grammar loading, LANGUAGE_MAP,
- *              EXTENSIONS, and getParserForFile.
+ * @description Language registry: Tree-sitter grammar loading, LANGUAGE_MAP,
+ *              EXTENSIONS, and getParserForFile. Grammar packages are NOT bundled
+ *              dependencies of graph-indexer (they were pulled via npm's
+ *              `optionalDependencies`, which installs eagerly regardless of the
+ *              name — that shipped all 16 grammars, ~250 MB, to every install even
+ *              when a repo only uses one or two languages). Instead, `GRAMMAR_PACKAGES`
+ *              is the single-sourced registry of what each language needs, and
+ *              `ensureLanguagesReady()` resolves each package: ambient node_modules
+ *              first (covers this repo's own devDependencies, or a project that
+ *              added a grammar directly), then a scoped `.graph-indexer/node_modules`
+ *              install dir (works for ANY project — Node or not, npx-run or not,
+ *              since it's keyed off `--repo`, not ambient resolution), auto-
+ *              installing missing packages there on first use.
  * @author MaquinaTech <https://github.com/MaquinaTech>
  * @copyright (c) 2026 MaquinaTech. All rights reserved.
  * @license MIT
  */
 import fs from 'fs';
 import path from 'path';
-import ignore from 'ignore';
+import { createRequire } from 'module';
+import { pathToFileURL } from 'url';
+import { spawnSync } from 'child_process';
 import Parser from 'tree-sitter';
+import { dataDir } from '../layout.mjs';
 
 export const OLLAMA_HOST = process.env.OLLAMA_HOST || "http://localhost:11434";
 
-// ─── Dynamic Language Loading ─────────────────────────────────────────────────
+// ─── Grammar registry (single source of truth for versions) ───────────────────
+// Keyed by the same language keys used by init.mjs's LANGUAGES list and
+// `.graph-indexer.json`'s `languages` array. 'css' covers two packages (CSS and
+// SCSS share one user-facing "CSS / SCSS" selection).
+export const GRAMMAR_PACKAGES = {
+    typescript: [{ pkg: 'tree-sitter-typescript', version: '0.21.2' }],
+    javascript: [{ pkg: 'tree-sitter-javascript', version: '0.21.4' }],
+    css: [
+        { pkg: 'tree-sitter-css', version: '0.21.1' },
+        { pkg: 'tree-sitter-scss', version: '1.0.0' },
+    ],
+    python: [{ pkg: 'tree-sitter-python', version: '0.21.0' }],
+    rust: [{ pkg: 'tree-sitter-rust', version: '0.21.0' }],
+    go: [{ pkg: 'tree-sitter-go', version: '0.21.0' }],
+    php: [{ pkg: 'tree-sitter-php', version: '0.23.1' }],
+    java: [{ pkg: 'tree-sitter-java', version: '0.21.0' }],
+    kotlin: [{ pkg: 'tree-sitter-kotlin', version: '0.3.8' }],
+    csharp: [{ pkg: 'tree-sitter-c-sharp', version: '0.21.3' }],
+    ruby: [{ pkg: 'tree-sitter-ruby', version: '0.23.1' }],
+    c: [{ pkg: 'tree-sitter-c', version: '0.21.4' }],
+    bash: [{ pkg: 'tree-sitter-bash', version: '0.23.3' }],
+    swift: [{ pkg: 'tree-sitter-swift', version: '0.5.0' }],
+};
 
-function _loadProjectConfig() {
-    const configPath = path.join(process.env.MCP_PROJECT_ROOT || process.cwd(), '.graph-indexer.json');
+function _loadProjectConfig(projectRoot) {
+    const configPath = path.join(projectRoot || process.env.MCP_PROJECT_ROOT || process.cwd(), '.graph-indexer.json');
     try {
         if (fs.existsSync(configPath)) return JSON.parse(fs.readFileSync(configPath, 'utf-8'));
     } catch { /* ignore */ }
     return null;
 }
 
-async function _tryLang(pkg, enabledLangs, key) {
-    if (enabledLangs && !enabledLangs.includes(key)) return null;
+// ─── Package resolution ────────────────────────────────────────────────────────
+
+/** Ambient resolution: relative to THIS module's own location (covers devDependencies
+ *  in this repo's checkout, or a grammar a project installed directly). */
+async function _tryAmbient(pkg) {
+    try { return (await import(pkg)).default; }
+    catch { return null; }
+}
+
+/** Resolve `pkg` from a scoped install dir (its own node_modules), without relying
+ *  on ambient ancestor-directory resolution — required for the npx/global-install
+ *  case, where graph-indexer's own location shares no node_modules ancestry with
+ *  the target `--repo`. */
+async function _importFrom(pkg, dir) {
     try {
-        return (await import(pkg)).default;
-    } catch {
-        process.stderr.write(`[graph-indexer] WARNING: ${pkg} not installed — ${key} files will be skipped\n`);
-        return null;
+        const req = createRequire(path.join(dir, '__grammar_resolve__.js'));
+        const resolved = req.resolve(pkg);
+        return (await import(pathToFileURL(resolved).href)).default;
+    } catch { return null; }
+}
+
+// A platform-incompatible grammar (e.g. no Xcode toolchain for a native binding)
+// would otherwise re-attempt its slow, doomed build on every single `idx-index`
+// run. Once a spec fails, it's remembered here so later runs skip straight to the
+// "not installed — skipped" warning instead of re-running node-gyp. Delete this
+// file (or the whole node_modules dir) to retry after fixing the toolchain.
+function _failedCachePath(dir) { return path.join(dir, '.install-failed.json'); }
+function _readFailedCache(dir) {
+    try { return new Set(JSON.parse(fs.readFileSync(_failedCachePath(dir), 'utf-8'))); }
+    catch { return new Set(); }
+}
+function _writeFailedCache(dir, failedSet) {
+    try { fs.writeFileSync(_failedCachePath(dir), JSON.stringify([...failedSet])); } catch { /* best effort */ }
+}
+
+// Installed ONE package per `npm install` call — not batched. A single native
+// build failure (e.g. no Xcode toolchain for a grammar with a compiled binding)
+// must only skip THAT language, not roll back the whole batch: npm treats a
+// multi-spec `install` as one transaction, so one failing package would otherwise
+// take every other requested language down with it (this is exactly the
+// per-package graceful-degradation `optionalDependencies` used to give us for
+// free). Returns the subset of `specs` that failed (including previously-known
+// failures, which are skipped without a retry).
+function _installGrammars(dir, specs, log) {
+    fs.mkdirSync(dir, { recursive: true });
+    const pkgPath = path.join(dir, 'package.json');
+    if (!fs.existsSync(pkgPath)) fs.writeFileSync(pkgPath, JSON.stringify({ name: 'graph-indexer-grammars', private: true }) + '\n');
+    const npmBin = process.platform === 'win32' ? 'npm.cmd' : 'npm';
+    const knownBad = _readFailedCache(dir);
+    const failed = [];
+    let knownBadChanged = false;
+    for (const spec of specs) {
+        if (knownBad.has(spec)) {
+            log(`[graph-indexer] WARNING: ${spec} previously failed to install — skipping retry (delete ${path.relative(process.cwd(), dir) || dir}/.install-failed.json to retry)`);
+            failed.push(spec);
+            continue;
+        }
+        log(`[graph-indexer] installing ${spec} (first use only, cached in ${path.relative(process.cwd(), dir) || dir}/node_modules)…`);
+        const res = spawnSync(npmBin, ['install', '--no-audit', '--no-fund', '--no-package-lock', spec], { cwd: dir, stdio: 'inherit' });
+        if (res.status !== 0) {
+            log(`[graph-indexer] WARNING: ${spec} failed to install (exit ${res.status}) — this platform may not support it; skipping`);
+            failed.push(spec);
+            knownBad.add(spec);
+            knownBadChanged = true;
+        }
     }
+    if (knownBadChanged) _writeFailedCache(dir, knownBad);
+    return failed;
 }
 
-const _cfg = _loadProjectConfig();
-const _enabled = _cfg?.languages ?? null; // null = all languages
-
-const [
-    TypeScript, JavaScript, CSS, SCSS, Python, Rust,
-    Go, PHP, Java, Kotlin, CSharp, Ruby, C, Bash, Swift
-] = await Promise.all([
-    _tryLang('tree-sitter-typescript', _enabled, 'typescript'),
-    _tryLang('tree-sitter-javascript', _enabled, 'javascript'),
-    _tryLang('tree-sitter-css', _enabled, 'css'),
-    // SCSS shares the 'css' enable-key — init records .css and .scss under one
-    // 'css' language, so the SCSS grammar must load whenever CSS is enabled.
-    _tryLang('tree-sitter-scss', _enabled, (_enabled && _enabled.includes('css')) ? 'css' : 'scss'),
-    _tryLang('tree-sitter-python', _enabled, 'python'),
-    _tryLang('tree-sitter-rust', _enabled, 'rust'),
-    _tryLang('tree-sitter-go', _enabled, 'go'),
-    _tryLang('tree-sitter-php', _enabled, 'php'),
-    _tryLang('tree-sitter-java', _enabled, 'java'),
-    _tryLang('tree-sitter-kotlin', _enabled, 'kotlin'),
-    _tryLang('tree-sitter-c-sharp', _enabled, 'csharp'),
-    _tryLang('tree-sitter-ruby', _enabled, 'ruby'),
-    _tryLang('tree-sitter-c', _enabled, 'c'),
-    _tryLang('tree-sitter-bash', _enabled, 'bash'),
-    _tryLang('tree-sitter-swift', _enabled, 'swift'),
-]);
-
-export const LANGUAGE_MAP = {
-    ...(TypeScript ? { '.ts': TypeScript.typescript, '.tsx': TypeScript.tsx } : {}),
-    ...(JavaScript ? { '.js': JavaScript, '.jsx': JavaScript, '.mjs': JavaScript, '.cjs': JavaScript } : {}),
-    ...(CSS ? { '.css': CSS } : {}),
-    ...(SCSS ? { '.scss': SCSS } : (CSS ? { '.scss': CSS } : {})),
-    ...(Python ? { '.py': Python } : {}),
-    ...(Rust ? { '.rs': Rust } : {}),
-    ...(Go ? { '.go': Go } : {}),
-    ...(PHP ? { '.php': PHP.php } : {}),
-    ...(Java ? { '.java': Java } : {}),
-    ...(Kotlin ? { '.kt': Kotlin, '.kts': Kotlin } : {}),
-    ...(CSharp ? { '.cs': CSharp } : {}),
-    ...(Ruby ? { '.rb': Ruby } : {}),
-    ...(C ? { '.c': C, '.h': C } : {}),
-    // Shebang-only extensionless scripts are not keyed.
-    ...(Bash ? { '.sh': Bash, '.bash': Bash } : {}),
-    ...(Swift ? { '.swift': Swift } : {}),
-};
+// ─── Lazy, memoized language loading ───────────────────────────────────────────
+// LANGUAGE_MAP/EXTENSIONS are mutated IN PLACE (not reassigned) so every consumer
+// that imported these bindings sees the same populated object/set once
+// ensureLanguagesReady() resolves — no need to re-import or re-bind anything.
+export const LANGUAGE_MAP = {};
+export const EXTENSIONS = new Set();
+
+let _readyPromise = null;
+
+function _applyLanguageMap(modules) {
+    const primary = (key) => modules[key]?.[0]?.mod || null;
+    const byPkg = (key, pkg) => modules[key]?.find(m => m.pkg === pkg)?.mod || null;
+
+    const TypeScript = primary('typescript');
+    const JavaScript = primary('javascript');
+    const CSS = byPkg('css', 'tree-sitter-css');
+    const SCSS = byPkg('css', 'tree-sitter-scss');
+    const Python = primary('python');
+    const Rust = primary('rust');
+    const Go = primary('go');
+    const PHP = primary('php');
+    const Java = primary('java');
+    const Kotlin = primary('kotlin');
+    const CSharp = primary('csharp');
+    const Ruby = primary('ruby');
+    const C = primary('c');
+    const Bash = primary('bash');
+    const Swift = primary('swift');
+
+    const map = {
+        ...(TypeScript ? { '.ts': TypeScript.typescript, '.tsx': TypeScript.tsx } : {}),
+        ...(JavaScript ? { '.js': JavaScript, '.jsx': JavaScript, '.mjs': JavaScript, '.cjs': JavaScript } : {}),
+        ...(CSS ? { '.css': CSS } : {}),
+        ...(SCSS ? { '.scss': SCSS } : (CSS ? { '.scss': CSS } : {})),
+        ...(Python ? { '.py': Python } : {}),
+        ...(Rust ? { '.rs': Rust } : {}),
+        ...(Go ? { '.go': Go } : {}),
+        ...(PHP ? { '.php': PHP.php } : {}),
+        ...(Java ? { '.java': Java } : {}),
+        ...(Kotlin ? { '.kt': Kotlin, '.kts': Kotlin } : {}),
+        ...(CSharp ? { '.cs': CSharp } : {}),
+        ...(Ruby ? { '.rb': Ruby } : {}),
+        ...(C ? { '.c': C, '.h': C } : {}),
+        // Shebang-only extensionless scripts are not keyed.
+        ...(Bash ? { '.sh': Bash, '.bash': Bash } : {}),
+        ...(Swift ? { '.swift': Swift } : {}),
+    };
+    for (const k of Object.keys(LANGUAGE_MAP)) delete LANGUAGE_MAP[k];
+    Object.assign(LANGUAGE_MAP, map);
+    EXTENSIONS.clear();
+    for (const k of Object.keys(map)) EXTENSIONS.add(k);
+}
 
-export const EXTENSIONS = new Set(Object.keys(LANGUAGE_MAP));
+async function _doLoadLanguages({ projectRoot, enabledLangs, autoInstall, log }) {
+    const grammarsRoot = dataDir(projectRoot); // packages land in <grammarsRoot>/node_modules
+    const keys = enabledLangs && enabledLangs.length ? enabledLangs : Object.keys(GRAMMAR_PACKAGES);
 
+    const modules = {};
+    let missing = [];
+    for (const key of keys) {
+        const specs = GRAMMAR_PACKAGES[key];
+        if (!specs) continue;
+        modules[key] = [];
+        for (const { pkg, version } of specs) {
+            const mod = (await _tryAmbient(pkg)) || (await _importFrom(pkg, grammarsRoot));
+            if (mod) modules[key].push({ pkg, mod });
+            else missing.push({ key, pkg, version });
+        }
+    }
+
+    if (missing.length && autoInstall) {
+        const specs = [...new Set(missing.map(m => `${m.pkg}@${m.version}`))];
+        const failedSpecs = new Set(_installGrammars(grammarsRoot, specs, log));
+        const stillMissing = [];
+        for (const m of missing) {
+            if (failedSpecs.has(`${m.pkg}@${m.version}`)) { stillMissing.push(m); continue; }
+            const mod = await _importFrom(m.pkg, grammarsRoot);
+            if (mod) modules[m.key].push({ pkg: m.pkg, mod });
+            else stillMissing.push(m);
+        }
+        missing = stillMissing;
+    }
+
+    for (const m of missing) {
+        log(`[graph-indexer] WARNING: ${m.pkg} not installed — ${m.key} files will be skipped`);
+    }
+
+    _applyLanguageMap(modules);
+    return { missing };
+}
+
+/**
+ * Resolve (and, unless `autoInstall` is false, install) every requested language's
+ * grammar, then populate LANGUAGE_MAP/EXTENSIONS. Idempotent + memoized per process
+ * — the first call's options win; call this once, early, before any getParserForFile.
+ * `enabledLangs` defaults to `.graph-indexer.json`'s `languages` field (null = all 14).
+ */
+export function ensureLanguagesReady({
+    projectRoot = process.env.MCP_PROJECT_ROOT || process.cwd(),
+    enabledLangs = _loadProjectConfig(projectRoot)?.languages ?? null,
+    autoInstall = true,
+    log = (msg) => process.stderr.write(msg + '\n'),
+} = {}) {
+    if (!_readyPromise) _readyPromise = _doLoadLanguages({ projectRoot, enabledLangs, autoInstall, log });
+    return _readyPromise;
+}
+
+/** Test-only: forces the next ensureLanguagesReady() call to reload from scratch. */
+export function _resetLanguagesReady() { _readyPromise = null; }
+
+let _warnedNotReady = false;
 export function getParserForFile(ext) {
+    if (!_readyPromise && !_warnedNotReady) {
+        _warnedNotReady = true;
+        process.stderr.write('[graph-indexer] WARNING: getParserForFile() called before ensureLanguagesReady() — no grammars loaded\n');
+    }
     const language = LANGUAGE_MAP[ext];
     if (!language) return null;
     const parser = new Parser();

package/parse/metadata.mjs

@@ -534,6 +534,52 @@ function _inferReceiverType(objNode, bindings) {
     return _classifyValueNode(objNode); // inline: new Repo().save(), getStore().save()
 }
 
+const _FN_NODE_TYPES = new Set([
+    'function_declaration', 'function_definition', 'method_definition', 'method_declaration',
+    'arrow_function', 'function', 'function_expression', 'lambda', 'method', 'singleton_method',
+    'generator_function', 'generator_function_declaration',
+]);
+
+/**
+ * Classify what a function chunk RETURNS, for the opt-in inter-procedural fixpoint
+ * (parse/interprocedural.mjs). Looks at this function's own return expression(s):
+ *   • { type }    — returns `new Repo()` (a concrete type, no annotation needed).
+ *   • { viaCall } — returns a bare factory call `makeRepo()` whose type is resolved
+ *                   later by propagating along the call graph.
+ *   • null        — nothing safe (no return, mixed returns, opaque expression).
+ * Does NOT descend into nested functions/lambdas (only THIS function's returns).
+ * This is a transient signal (attached as `_return_via`) consumed at index time and
+ * stripped before serialization — it never persists in the index.
+ */
+export function extractReturnVia(chunkNode) {
+    const found = [];
+    const consider = (expr) => { const v = _classifyValueNode(expr); if (v) found.push(v); };
+
+    // Expression-body arrow / lambda: `() => new Repo()` has no return_statement.
+    if (chunkNode.type === 'arrow_function' || chunkNode.type === 'lambda') {
+        const body = chunkNode.childForFieldName?.('body');
+        if (body && body.type !== 'statement_block' && body.type !== 'block') consider(body);
+    }
+
+    (function walk(node, depth) {
+        for (const child of node.children) {
+            if (depth > 0 && _FN_NODE_TYPES.has(child.type)) continue; // skip nested functions
+            if (child.type === 'return_statement') {
+                consider(child.namedChildren?.[0] || child.childForFieldName?.('argument'));
+            }
+            walk(child, depth + 1);
+        }
+    })(chunkNode, 0);
+
+    if (found.length === 0) return null;
+    const types = new Set(found.filter(f => f.type).map(f => f.type));
+    const vias = new Set(found.filter(f => f.viaCall).map(f => f.viaCall));
+    // Conservative: only a single unambiguous classification is usable.
+    if (types.size === 1 && vias.size === 0) return { type: [...types][0] };
+    if (vias.size === 1 && types.size === 0) return { viaCall: [...vias][0] };
+    return null;
+}
+
 /**
  * Walk a subtree and collect every call site as { name, recv } (receiver hint), plus
  * an optional `recv_type` / `recv_via_call` when the receiver's type is recoverable