homegraph 1.2.0 → 1.4.1

package/dist/resolution/c-fnptr-synthesizer.js

@@ -85,6 +85,7 @@ exports.cFnPointerDispatchEdges = cFnPointerDispatchEdges;
  * key + a real-function gate; a project with no fn-pointer dispatch is a no-op.
  */
 const path = __importStar(require("node:path"));
+const lru_cache_1 = require("./lru-cache");
 const strip_comments_1 = require("./strip-comments");
 const C_CPP_EXT = /\.(c|h|cc|cpp|cxx|hpp|hh|hxx|cppm|ipp|inl|tcc)$/i;
 const FN_KINDS = new Set(['function', 'method']);
@@ -337,13 +338,30 @@ const C_TYPE_KEYWORDS = new Set([
 const INCLUDE_RE = /#[ \t]*include[ \t]+"([^"\n]+)"/g;
 /** Included files worth scanning for registration tables (e.g. a generated `.def`). */
 const INCLUDABLE_EXT = /\.(def|inc|h|hh|hpp|hxx|c|cc|cpp|cxx|ipp|tcc|tbl)$/i;
-function cFnPointerDispatchEdges(queries, ctx) {
+async function cFnPointerDispatchEdges(_queries, ctx, onYield, onFraction) {
+    let scannedFiles = 0;
     const files = ctx.getAllFiles().filter((f) => C_CPP_EXT.test(f));
     if (files.length === 0)
         return [];
-    // Cache raw + stripped source per file (read once, reused across passes).
-    // Raw is needed for `#include "…"` directives — strip blanks string contents.
-    const rawCache = new Map();
+    // Within-pass progress: this is the pass that parks the "Linking dynamic
+    // dispatch" bar on C-heavy repos, so it reports a real fraction of its
+    // dominant work. `files` is swept once per file loop below (passes A, C, D,
+    // E — pass B is node-bound and comparatively brief), reported at the same
+    // per-16-files cadence as the cooperative yield.
+    const FILE_SWEEPS = 4;
+    const tick = async () => {
+        if ((++scannedFiles & 15) === 0) {
+            onFraction?.(scannedFiles / (files.length * FILE_SWEEPS));
+            await onYield();
+        }
+    };
+    // Cache raw + stripped source per file, LRU-BOUNDED. The old unbounded Maps
+    // retained every C/C++ file's raw AND stripped text for the whole pass —
+    // multiple GB on the Linux kernel, one of the two OOM culprits in #1212.
+    // Every sweep below iterates in `files` order, and node-kind scans return
+    // rows in file-commit order, so access is near-sequential and a small LRU
+    // hits; a miss just re-reads + re-strips.
+    const rawCache = new lru_cache_1.LRUCache(128);
     const raw = (file) => {
         if (rawCache.has(file))
             return rawCache.get(file);
@@ -351,10 +369,14 @@ function cFnPointerDispatchEdges(queries, ctx) {
         rawCache.set(file, r);
         return r;
     };
-    const srcCache = new Map();
+    const srcCache = new lru_cache_1.LRUCache(128);
     const src = (file) => {
-        if (srcCache.has(file))
-            return srcCache.get(file);
+        // A cached '' (empty or unreadable file) returns '' where the miss path
+        // returns null for unreadable — every caller falsy-checks, so the two are
+        // interchangeable.
+        const hit = srcCache.get(file);
+        if (hit !== undefined)
+            return hit;
         const r = raw(file);
         const s = r == null ? '' : (0, strip_comments_1.stripCommentsForRegex)(r, 'c');
         srcCache.set(file, s);
@@ -380,6 +402,7 @@ function cFnPointerDispatchEdges(queries, ctx) {
     const fnPtrTypedefs = new Set();
     const fnTypeTypedefs = new Set();
     for (const file of files) {
+        await tick();
         const s = src(file);
         if (!s || !s.includes('typedef'))
             continue;
@@ -472,10 +495,12 @@ function cFnPointerDispatchEdges(queries, ctx) {
         if (fields.some((f) => f.isFnPtr))
             structLayout.set(name, fields);
     };
-    for (const st of ctx.getNodesByKind('struct')) {
+    for (const st of (ctx.iterateNodesByKind?.('struct') ?? ctx.getNodesByKind('struct'))) {
+        if ((++scannedFiles & 255) === 0)
+            await onYield();
         if (!C_CPP_EXT.test(st.filePath))
             continue;
-        const s = srcCache.get(st.filePath) ?? src(st.filePath);
+        const s = src(st.filePath);
         if (!s)
             continue;
         const body = sliceLines(s, st.startLine, st.endLine);
@@ -490,12 +515,9 @@ function cFnPointerDispatchEdges(queries, ctx) {
     // up by `#ifdef`) is discovered later during the unit scan. The `reg.size === 0`
     // guard after registration still short-circuits when nothing bridges.
     const fnPtrFieldOf = (struct, field) => !!structLayout.get(struct)?.some((f) => f.name === field && f.isFnPtr);
-    // C/C++ function + method nodes, materialized once (bounded by C/C++ files).
-    const cFns = [];
-    for (const fn of iterateFns(queries)) {
-        if (C_CPP_EXT.test(fn.filePath))
-            cFns.push(fn);
-    }
+    // C/C++ function + method nodes are STREAMED per sweep (see passes D/E) —
+    // the old materialized `cFns` array held every function node on the repo
+    // (O(nodes) memory, part of the #1212 kernel OOM).
     // ---- function-name → node resolution (prefer a function in the same file) ----
     const resolveFn = (name, preferFile) => {
         const cands = ctx.getNodesByName(name).filter((n) => FN_KINDS.has(n.kind));
@@ -511,14 +533,14 @@ function cFnPointerDispatchEdges(queries, ctx) {
         return cands[0];
     };
     // ---- Pass C: registrations — Map<"struct.field", Set<funcNodeId>> ----
+    // Ids only — retaining the full Node per registration (the old `idToNode`)
+    // was write-only dead weight at O(registrations) memory.
     const reg = new Map();
-    const idToNode = new Map();
     const addReg = (struct, field, fn) => {
         const key = `${struct}.${field}`;
         if (!reg.has(key))
             reg.set(key, new Set());
         reg.get(key).add(fn.id);
-        idToNode.set(fn.id, fn);
     };
     // Bare arrays-of-fn-pointers (no struct): array VARIABLE name → per-file sets
     // of registered function ids. Multi-entry because a file-scope `static` table
@@ -537,7 +559,6 @@ function cFnPointerDispatchEdges(queries, ctx) {
             entries.push(e);
         }
         e.ids.add(fn.id);
-        idToNode.set(fn.id, fn);
     };
     // A struct value `{ … }` (one element) — register its function entries to the
     // struct's fields, by `.field = fn` designators or by positional slot.
@@ -612,7 +633,8 @@ function cFnPointerDispatchEdges(queries, ctx) {
         }
     };
     // Per-file macro + include parsing (any file, indexed or not), cached.
-    const fnMacroCache = new Map();
+    // Derived per-file caches, LRU-bounded like the content caches (#1212).
+    const fnMacroCache = new lru_cache_1.LRUCache(256);
     const fileFnMacros = (file) => {
         let m = fnMacroCache.get(file);
         if (!m) {
@@ -621,7 +643,7 @@ function cFnPointerDispatchEdges(queries, ctx) {
         }
         return m;
     };
-    const objMacroCache = new Map();
+    const objMacroCache = new lru_cache_1.LRUCache(256);
     const fileObjMacros = (file) => {
         let m = objMacroCache.get(file);
         if (!m) {
@@ -630,7 +652,7 @@ function cFnPointerDispatchEdges(queries, ctx) {
         }
         return m;
     };
-    const definedCache = new Map();
+    const definedCache = new lru_cache_1.LRUCache(256);
     const fileDefinedNames = (file) => {
         let d = definedCache.get(file);
         if (!d) {
@@ -639,7 +661,7 @@ function cFnPointerDispatchEdges(queries, ctx) {
         }
         return d;
     };
-    const includeCache = new Map();
+    const includeCache = new lru_cache_1.LRUCache(1024);
     const localIncludesOf = (file) => {
         let out = includeCache.get(file);
         if (out)
@@ -681,45 +703,7 @@ function cFnPointerDispatchEdges(queries, ctx) {
             buildEnv(inc, depth - 1, seen, fn, obj, def);
     };
     const indexedSet = new Set(files);
-    const units = [];
     const seenInclude = new Set();
-    for (const file of files) {
-        const env = new Map();
-        const objEnv = new Map();
-        const defined = new Set();
-        buildEnv(file, 2, new Set(), env, objEnv, defined);
-        const s = src(file);
-        if (s)
-            units.push({ text: s, file, env, objEnv });
-        for (const target of localIncludesOf(file)) {
-            if (seenInclude.has(`${file}>${target}`))
-                continue;
-            const incSrc = src(target);
-            if (!incSrc)
-                continue;
-            if (indexedSet.has(target)) {
-                // Re-scan an indexed header only when this includer unlocks guarded code.
-                const ownDef = fileDefinedNames(target);
-                const adds = [...defined].some((n) => !ownDef.has(n));
-                if (!adds || !/#\s*if/.test(incSrc))
-                    continue;
-            }
-            seenInclude.add(`${file}>${target}`);
-            // The include is pasted into the includer — evaluate its conditionals in
-            // the includer's defined set (a no-op when it has none). Re-parse the
-            // included file's OWN macros from that resolved text so a macro it defines
-            // conditionally (vim's `EXCMD`, whose plain last-wins parse picks the enum
-            // arm) overrides with the ARM THAT IS ACTUALLY ACTIVE here.
-            const text = evalConditionals(incSrc, defined);
-            const incEnv = new Map(env);
-            for (const [k, v] of parseFunctionMacros(text))
-                incEnv.set(k, v);
-            const incObjEnv = new Map(objEnv);
-            for (const [k, v] of parseObjectMacros(text))
-                incObjEnv.set(k, v);
-            units.push({ text, file: target, env: incEnv, objEnv: incObjEnv });
-        }
-    }
     // Global variable → struct type, for resolving a dispatch through a file-scope
     // table by subscript (`cmdnames[i].cmd_func(…)`).
     const globalVarType = new Map();
@@ -760,10 +744,13 @@ function cFnPointerDispatchEdges(queries, ctx) {
     // (`zend_rc_dtor_func_t t[]`) needs none. The typedef-set membership gate
     // (below) is what separates this from a plain data/struct array.
     const ARRAY_TABLE_RE = /(?:^|[;{}])\s*(?:(?:static|const|extern|register|volatile)\s+)*(\w+)\s+(\*\s*)?(\w+)\s*\[[^\]]*\]\s*=\s*\{/g;
-    for (const unit of units) {
+    // Process ONE unit's text and discard it. The old shape built every unit up
+    // front (`const units: Unit[]`) — the full text of every C file plus its
+    // expanded includes held simultaneously, gigabytes on the kernel (#1212).
+    const processUnit = (unit) => {
         const s = unit.text;
         if (!s || !s.includes('{'))
-            continue;
+            return;
         INLINE_STRUCT_RE.lastIndex = 0;
         let im;
         while ((im = INLINE_STRUCT_RE.exec(s))) {
@@ -793,7 +780,7 @@ function cFnPointerDispatchEdges(queries, ctx) {
             }
         }
         if (!s.includes('='))
-            continue;
+            return;
         INIT_RE.lastIndex = 0;
         let m;
         while ((m = INIT_RE.exec(s))) {
@@ -829,6 +816,46 @@ function cFnPointerDispatchEdges(queries, ctx) {
             registerArrayValue(am[3], s.slice(open + 1, close), unit.file, unit.env);
             ARRAY_TABLE_RE.lastIndex = close;
         }
+    };
+    // ---- Pass C: registrations — stream each file (and its qualifying local
+    // includes) through processUnit, one at a time.
+    for (const file of files) {
+        await tick();
+        const env = new Map();
+        const objEnv = new Map();
+        const defined = new Set();
+        buildEnv(file, 2, new Set(), env, objEnv, defined);
+        const s = src(file);
+        if (s)
+            processUnit({ text: s, file, env, objEnv });
+        for (const target of localIncludesOf(file)) {
+            if (seenInclude.has(`${file}>${target}`))
+                continue;
+            const incSrc = src(target);
+            if (!incSrc)
+                continue;
+            if (indexedSet.has(target)) {
+                // Re-scan an indexed header only when this includer unlocks guarded code.
+                const ownDef = fileDefinedNames(target);
+                const adds = [...defined].some((n) => !ownDef.has(n));
+                if (!adds || !/#\s*if/.test(incSrc))
+                    continue;
+            }
+            seenInclude.add(`${file}>${target}`);
+            // The include is pasted into the includer — evaluate its conditionals in
+            // the includer's defined set (a no-op when it has none). Re-parse the
+            // included file's OWN macros from that resolved text so a macro it defines
+            // conditionally (vim's `EXCMD`, whose plain last-wins parse picks the enum
+            // arm) overrides with the ARM THAT IS ACTUALLY ACTIVE here.
+            const text = evalConditionals(incSrc, defined);
+            const incEnv = new Map(env);
+            for (const [k, v] of parseFunctionMacros(text))
+                incEnv.set(k, v);
+            const incObjEnv = new Map(objEnv);
+            for (const [k, v] of parseObjectMacros(text))
+                incObjEnv.set(k, v);
+            processUnit({ text, file: target, env: incEnv, objEnv: incObjEnv });
+        }
     }
     // ---- receiver-type resolution within a function's source ----
     // `(?:struct )?TYPE [*]recv` declared in the params or body → TYPE (if a known
@@ -882,21 +909,26 @@ function cFnPointerDispatchEdges(queries, ctx) {
     // a fixpoint so a hook slot inherits a registry field's handlers.
     const FIELD_ASSIGN_RE = /(\w+)\s*(?:->|\.)\s*(\w+)\s*=\s*(\w+)\s*(?:->|\.)\s*(\w+)/g;
     const propagations = [];
-    for (const fn of cFns) {
-        const s = srcCache.get(fn.filePath);
-        if (!s)
-            continue;
-        const body = sliceLines(s, fn.startLine, fn.endLine);
-        if (!body.includes('='))
+    for (const file of files) {
+        await tick();
+        const s = src(file);
+        if (!s || !s.includes('='))
             continue;
-        FIELD_ASSIGN_RE.lastIndex = 0;
-        let m;
-        while ((m = FIELD_ASSIGN_RE.exec(body))) {
-            const [, lrecv, lfield, rrecv, rfield] = m;
-            const lt = recvTypeIn(body, lrecv);
-            const rt = recvTypeIn(body, rrecv);
-            if (lt && rt && fnPtrFieldOf(lt, lfield) && fnPtrFieldOf(rt, rfield)) {
-                propagations.push({ to: `${lt}.${lfield}`, from: `${rt}.${rfield}` });
+        for (const fn of ctx.getNodesInFile(file)) {
+            if (!FN_KINDS.has(fn.kind))
+                continue;
+            const body = sliceLines(s, fn.startLine, fn.endLine);
+            if (!body.includes('='))
+                continue;
+            FIELD_ASSIGN_RE.lastIndex = 0;
+            let m;
+            while ((m = FIELD_ASSIGN_RE.exec(body))) {
+                const [, lrecv, lfield, rrecv, rfield] = m;
+                const lt = recvTypeIn(body, lrecv);
+                const rt = recvTypeIn(body, rrecv);
+                if (lt && rt && fnPtrFieldOf(lt, lfield) && fnPtrFieldOf(rt, rfield)) {
+                    propagations.push({ to: `${lt}.${lfield}`, from: `${rt}.${rfield}` });
+                }
             }
         }
     }
@@ -934,76 +966,42 @@ function cFnPointerDispatchEdges(queries, ctx) {
     const ARRAY_DISPATCH_RE = /(?:\(\s*\*\s*)?\b(\w+)\s*\[[^\][]*\]\s*\)?\s*\(/g;
     const edges = [];
     const seen = new Set();
-    for (const fn of cFns) {
-        const s = srcCache.get(fn.filePath);
+    for (const file of files) {
+        await tick();
+        const s = src(file);
         if (!s)
             continue;
-        const body = sliceLines(s, fn.startLine, fn.endLine);
-        DISPATCH_RE.lastIndex = 0;
-        let m;
-        let added = 0;
-        while ((m = DISPATCH_RE.exec(body)) && added < FANOUT_CAP) {
-            const baseChain = m[1].replace(/\s*(?:->|\.)\s*$/, '').trim(); // receiver, minus the trailing arrow
-            const field = m[2];
-            const owners = fieldToStructs.get(field);
-            if (!owners || owners.size === 0)
-                continue;
-            // 1) resolve the receiver chain's struct type precisely (handles c->cmd->proc);
-            // 2) else the last segment as a simple local/param of a fn-pointer-bearing struct;
-            // 3) else fall back to a field name that belongs to exactly one struct.
-            let struct = resolveChainType(body, baseChain);
-            if (!struct || !owners.has(struct)) {
-                const lastSeg = baseChain.replace(/\s*\[[^\]]*\]/g, '').split(/\s*(?:->|\.)\s*/).pop();
-                const t = recvTypeIn(body, lastSeg);
-                struct = t && owners.has(t) ? t : null;
-            }
-            if (!struct || !owners.has(struct))
-                struct = owners.size === 1 ? [...owners][0] : null;
-            if (!struct)
+        for (const fn of ctx.getNodesInFile(file)) {
+            if (!FN_KINDS.has(fn.kind))
                 continue;
-            const targets = reg.get(`${struct}.${field}`);
-            if (!targets)
-                continue;
-            const line = fn.startLine + body.slice(0, m.index).split('\n').length - 1;
-            for (const tid of targets) {
-                if (tid === fn.id)
-                    continue;
-                const key = `${fn.id}>${tid}`;
-                if (seen.has(key))
+            const body = sliceLines(s, fn.startLine, fn.endLine);
+            DISPATCH_RE.lastIndex = 0;
+            let m;
+            let added = 0;
+            while ((m = DISPATCH_RE.exec(body)) && added < FANOUT_CAP) {
+                const baseChain = m[1].replace(/\s*(?:->|\.)\s*$/, '').trim(); // receiver, minus the trailing arrow
+                const field = m[2];
+                const owners = fieldToStructs.get(field);
+                if (!owners || owners.size === 0)
                     continue;
-                seen.add(key);
-                edges.push({
-                    source: fn.id,
-                    target: tid,
-                    kind: 'calls',
-                    line,
-                    provenance: 'heuristic',
-                    metadata: {
-                        synthesizedBy: 'fn-pointer-dispatch',
-                        via: `${struct}.${field}`,
-                        registeredAt: `${fn.filePath}:${line}`,
-                    },
-                });
-                if (++added >= FANOUT_CAP)
-                    break;
-            }
-        }
-        // ---- bare array-of-fn-pointers dispatch (`tbl[i](…)`) ----
-        if (arrayReg.size && added < FANOUT_CAP) {
-            ARRAY_DISPATCH_RE.lastIndex = 0;
-            while ((m = ARRAY_DISPATCH_RE.exec(body)) && added < FANOUT_CAP) {
-                const entries = arrayReg.get(m[1]);
-                if (!entries)
+                // 1) resolve the receiver chain's struct type precisely (handles c->cmd->proc);
+                // 2) else the last segment as a simple local/param of a fn-pointer-bearing struct;
+                // 3) else fall back to a field name that belongs to exactly one struct.
+                let struct = resolveChainType(body, baseChain);
+                if (!struct || !owners.has(struct)) {
+                    const lastSeg = baseChain.replace(/\s*\[[^\]]*\]/g, '').split(/\s*(?:->|\.)\s*/).pop();
+                    const t = recvTypeIn(body, lastSeg);
+                    struct = t && owners.has(t) ? t : null;
+                }
+                if (!struct || !owners.has(struct))
+                    struct = owners.size === 1 ? [...owners][0] : null;
+                if (!struct)
                     continue;
-                // Same-file table wins on a name collision (two file-local `opcodes`);
-                // a unique name resolves cross-file; otherwise ambiguous — bail.
-                const ids = entries.length === 1
-                    ? entries[0].ids
-                    : (entries.find((e) => e.file === fn.filePath)?.ids ?? null);
-                if (!ids)
+                const targets = reg.get(`${struct}.${field}`);
+                if (!targets)
                     continue;
                 const line = fn.startLine + body.slice(0, m.index).split('\n').length - 1;
-                for (const tid of ids) {
+                for (const tid of targets) {
                     if (tid === fn.id)
                         continue;
                     const key = `${fn.id}>${tid}`;
@@ -1018,7 +1016,7 @@ function cFnPointerDispatchEdges(queries, ctx) {
                         provenance: 'heuristic',
                         metadata: {
                             synthesizedBy: 'fn-pointer-dispatch',
-                            via: `${m[1]}[]`,
+                            via: `${struct}.${field}`,
                             registeredAt: `${fn.filePath}:${line}`,
                         },
                     });
@@ -1026,13 +1024,47 @@ function cFnPointerDispatchEdges(queries, ctx) {
                         break;
                 }
             }
+            // ---- bare array-of-fn-pointers dispatch (`tbl[i](…)`) ----
+            if (arrayReg.size && added < FANOUT_CAP) {
+                ARRAY_DISPATCH_RE.lastIndex = 0;
+                while ((m = ARRAY_DISPATCH_RE.exec(body)) && added < FANOUT_CAP) {
+                    const entries = arrayReg.get(m[1]);
+                    if (!entries)
+                        continue;
+                    // Same-file table wins on a name collision (two file-local `opcodes`);
+                    // a unique name resolves cross-file; otherwise ambiguous — bail.
+                    const ids = entries.length === 1
+                        ? entries[0].ids
+                        : (entries.find((e) => e.file === fn.filePath)?.ids ?? null);
+                    if (!ids)
+                        continue;
+                    const line = fn.startLine + body.slice(0, m.index).split('\n').length - 1;
+                    for (const tid of ids) {
+                        if (tid === fn.id)
+                            continue;
+                        const key = `${fn.id}>${tid}`;
+                        if (seen.has(key))
+                            continue;
+                        seen.add(key);
+                        edges.push({
+                            source: fn.id,
+                            target: tid,
+                            kind: 'calls',
+                            line,
+                            provenance: 'heuristic',
+                            metadata: {
+                                synthesizedBy: 'fn-pointer-dispatch',
+                                via: `${m[1]}[]`,
+                                registeredAt: `${fn.filePath}:${line}`,
+                            },
+                        });
+                        if (++added >= FANOUT_CAP)
+                            break;
+                    }
+                }
+            }
         }
     }
     return edges;
 }
-/** C/C++ function + method nodes, streamed (memory-safe on symbol-dense repos). */
-function* iterateFns(queries) {
-    yield* queries.iterateNodesByKind('function');
-    yield* queries.iterateNodesByKind('method');
-}
 //# sourceMappingURL=c-fnptr-synthesizer.js.map