homegraph 1.5.0 → 1.5.2

package/dist/resolution/callback-synthesizer.d.ts

@@ -54,6 +54,7 @@ export declare function synthesizeCallbackEdges(queries: QueryBuilder, ctx: Reso
         edges: Edge[];
         ms: number;
     }>;
-} | null, backpressure?: () => Promise<void> | null): Promise<number>;
+    size?: number;
+} | null, backpressure?: () => Promise<void> | null, dbPath?: string): Promise<number>;
 export {};
 //# sourceMappingURL=callback-synthesizer.d.ts.map

package/dist/resolution/callback-synthesizer.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"callback-synthesizer.d.ts","sourceRoot":"","sources":["../../src/resolution/callback-synthesizer.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;GAsBG;AACH,OAAO,KAAK,EAAE,IAAI,EAAkB,MAAM,UAAU,CAAC;AACrD,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,eAAe,CAAC;AAClD,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,SAAS,CAAC;AASjD,OAAO,EAAiB,KAAK,UAAU,EAAE,MAAM,qBAAqB,CAAC;AA64GrE,mGAAmG;AACnG,KAAK,OAAO,GAAG,CAAC,GAAG,EAAE,EAAE,MAAM,EAAE,KAAK,OAAO,CAAC;AAE5C;;;;;;;GAOG;AACH,MAAM,WAAW,YAAY;IAC3B,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,CAAC,GAAG,EAAE,OAAO,KAAK,OAAO,CAAC;IAChC,GAAG,EAAE,CACH,OAAO,EAAE,YAAY,EACrB,GAAG,EAAE,iBAAiB,EACtB,WAAW,EAAE,UAAU,EACvB,WAAW,CAAC,EAAE,CAAC,QAAQ,EAAE,MAAM,KAAK,IAAI,KACrC,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC;CACtB;AA6FD;;;;;;GAMG;AACH,eAAO,MAAM,YAAY,EAAE,YAAY,EA6DtC,CAAC;AAIF,eAAO,MAAM,oBAAoB,QAA0C,CAAC;AAC5E,wBAAsB,uBAAuB,CAC3C,OAAO,EAAE,YAAY,EACrB,GAAG,EAAE,iBAAiB,EACtB,UAAU,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,KAAK,IAAI,EAIlD,IAAI,CAAC,EAAE;IAAE,YAAY,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC;QAAE,KAAK,EAAE,IAAI,EAAE,CAAC;QAAC,EAAE,EAAE,MAAM,CAAA;KAAE,CAAC,CAAA;CAAE,GAAG,IAAI,EAIpF,YAAY,CAAC,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,GACxC,OAAO,CAAC,MAAM,CAAC,CAmLjB"}
+{"version":3,"file":"callback-synthesizer.d.ts","sourceRoot":"","sources":["../../src/resolution/callback-synthesizer.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;GAsBG;AACH,OAAO,KAAK,EAAE,IAAI,EAAkB,MAAM,UAAU,CAAC;AACrD,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,eAAe,CAAC;AAClD,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,SAAS,CAAC;AAWjD,OAAO,EAAiB,KAAK,UAAU,EAAE,MAAM,qBAAqB,CAAC;AA64GrE,mGAAmG;AACnG,KAAK,OAAO,GAAG,CAAC,GAAG,EAAE,EAAE,MAAM,EAAE,KAAK,OAAO,CAAC;AAE5C;;;;;;;GAOG;AACH,MAAM,WAAW,YAAY;IAC3B,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,CAAC,GAAG,EAAE,OAAO,KAAK,OAAO,CAAC;IAChC,GAAG,EAAE,CACH,OAAO,EAAE,YAAY,EACrB,GAAG,EAAE,iBAAiB,EACtB,WAAW,EAAE,UAAU,EACvB,WAAW,CAAC,EAAE,CAAC,QAAQ,EAAE,MAAM,KAAK,IAAI,KACrC,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC;CACtB;AAsGD;;;;;;GAMG;AACH,eAAO,MAAM,YAAY,EAAE,YAAY,EAiEtC,CAAC;AAIF,eAAO,MAAM,oBAAoB,QAA0C,CAAC;AAC5E,wBAAsB,uBAAuB,CAC3C,OAAO,EAAE,YAAY,EACrB,GAAG,EAAE,iBAAiB,EACtB,UAAU,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,KAAK,IAAI,EAIlD,IAAI,CAAC,EAAE;IAAE,YAAY,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC;QAAE,KAAK,EAAE,IAAI,EAAE,CAAC;QAAC,EAAE,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IAAC,IAAI,CAAC,EAAE,MAAM,CAAA;CAAE,GAAG,IAAI,EAInG,YAAY,CAAC,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,EAGzC,MAAM,CAAC,EAAE,MAAM,GACd,OAAO,CAAC,MAAM,CAAC,CAwOjB"}

package/dist/resolution/callback-synthesizer.js

@@ -1,4 +1,37 @@
 "use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.SYNTH_PROGRESS_STEPS = exports.SYNTH_PASSES = void 0;
 exports.synthesizeCallbackEdges = synthesizeCallbackEdges;
@@ -6,6 +39,8 @@ const generated_detection_1 = require("../extraction/generated-detection");
 const strip_comments_1 = require("./strip-comments");
 const c_fnptr_synthesizer_1 = require("./c-fnptr-synthesizer");
 const goframe_synthesizer_1 = require("./goframe-synthesizer");
+const memory_budget_1 = require("./memory-budget");
+const fs = __importStar(require("fs"));
 const arkts_entry_1 = require("./frameworks/arkts-entry");
 const cooperative_yield_1 = require("./cooperative-yield");
 const REGISTRAR_NAME = /^(on[A-Z]\w*|subscribe|addListener|addEventListener|register|watch|listen|addCallback)$/;
@@ -3818,9 +3853,7 @@ function arktsEntryEdges(queries, ctx) {
             }
         }
     }
-    for (const cls of [...queries.getNodesByKind('class'), ...queries.getNodesByKind('struct')]) {
-        if (cls.language !== 'arkts')
-            continue;
+    for (const cls of iterateArktsTypeNodes(queries, ['class', 'struct'])) {
         const methods = queries
             .getOutgoingEdges(cls.id, ['contains'])
             .map((e) => queries.getNodeById(e.target))
@@ -3831,18 +3864,18 @@ function arktsEntryEdges(queries, ctx) {
             add(onCreate.id, onWindowStageCreate.id, 'arkts-lifecycle', { via: 'ability' });
         }
     }
-    for (const comp of [
-        ...queries.getNodesByKind('component'),
-        ...queries.getNodesByKind('class'),
-        ...queries.getNodesByKind('struct'),
-    ]) {
-        if (comp.language !== 'arkts')
-            continue;
-        const owner = comp.kind === 'component'
-            ? queries.getNodesByKind('struct').find((c) => c.filePath === comp.filePath && c.name === comp.name) ??
-                queries.getNodesByKind('class').find((c) => c.filePath === comp.filePath && c.name === comp.name) ??
-                comp
-            : comp;
+    for (const comp of iterateArktsTypeNodes(queries, ['component', 'class', 'struct'])) {
+        let owner = comp;
+        if (comp.kind === 'component') {
+            for (const c of queries.getNodesByFile(comp.filePath)) {
+                if ((c.kind === 'struct' || c.kind === 'class') &&
+                    c.language === 'arkts' &&
+                    c.name === comp.name) {
+                    owner = c;
+                    break;
+                }
+            }
+        }
         const methods = queries
             .getOutgoingEdges(owner.id, ['contains'])
             .map((e) => queries.getNodeById(e.target))
@@ -3859,6 +3892,15 @@ function arktsEntryEdges(queries, ctx) {
     }
     return edges;
 }
+/** Stream ArkTS class/struct/component nodes without materializing full-kind arrays. */
+function* iterateArktsTypeNodes(queries, kinds) {
+    for (const kind of kinds) {
+        for (const n of queries.iterateNodesByKind(kind)) {
+            if (n.language === 'arkts')
+                yield n;
+        }
+    }
+}
 const ALWAYS = () => true;
 /**
  * The independent passes, in MERGE ORDER — the first-seen dedup in
@@ -3875,7 +3917,7 @@ exports.SYNTH_PASSES = [
     { name: 'jsxEdges', gate: ALWAYS, run: (_q, c, y) => reactJsxChildEdges(c, y) },
     { name: 'vueEdges', gate: (has) => has('vue'), run: (_q, c, y) => vueTemplateEdges(c, y) },
     { name: 'svelteKitEdges', gate: (has) => has('svelte'), run: (_q, c, y) => svelteKitLoadEdges(c, y) },
-    { name: 'pascalEdges', gate: ALWAYS, run: (_q, c, y) => pascalFormEdges(c, y) },
+    { name: 'pascalEdges', gate: (has) => has('pascal'), run: (_q, c, y) => pascalFormEdges(c, y) },
     { name: 'flutterEdges', gate: (has) => has('dart'), run: (q, c, y) => flutterBuildEdges(q, c, y) },
     { name: 'arkuiStateEdges', gate: (has) => has('arkts'), run: (q, c, y) => arkuiStateBuildEdges(q, c, y) },
     { name: 'arkuiEmitter', gate: (has) => has('arkts'), run: (_q, c, y) => arkuiEmitterEdges(c, y) },
@@ -3890,7 +3932,11 @@ exports.SYNTH_PASSES = [
     { name: 'kotlinExpectActual', gate: (has) => has('kotlin'), run: (q, _c, y) => kotlinExpectActualEdges(q, y) },
     { name: 'goGrpcEdges', gate: (has) => has('go'), run: (q, _c, y) => goGrpcStubImplEdges(q, y) },
     { name: 'rnEventEdgesList', gate: (has) => has(...JS_FAMILY), run: (_q, c, y) => rnEventEdges(c, y) },
-    { name: 'fabricNativeEdges', gate: ALWAYS, run: (_q, c, y) => fabricNativeImplEdges(c, y) },
+    {
+        name: 'fabricNativeEdges',
+        gate: (has) => has(...JS_FAMILY),
+        run: (_q, c, y) => fabricNativeImplEdges(c, y),
+    },
     // Expo module nodes (`expo-module:` ids) are emitted only from .swift/.kt
     // files, and a pair needs BOTH platforms — so without both languages the
     // pass's only collection loop is provably empty (it was streaming every
@@ -3907,7 +3953,7 @@ exports.SYNTH_PASSES = [
     },
     { name: 'ginEdges', gate: (has) => has('go'), run: (q, c, y) => ginMiddlewareChainEdges(q, c, y) },
     { name: 'thunkEdges', gate: (has) => has(...JS_FAMILY), run: (q, c, y) => reduxThunkEdges(q, c, y) },
-    { name: 'registryEdges', gate: ALWAYS, run: (_q, c, y) => objectRegistryEdges(c, y) },
+    { name: 'registryEdges', gate: (has) => has(...JS_FAMILY), run: (_q, c, y) => objectRegistryEdges(c, y) },
     { name: 'rtkEdges', gate: (has) => has(...JS_FAMILY), run: (q, c, y) => rtkQueryEdges(q, c, y) },
     { name: 'piniaEdges', gate: (has) => has('vue', ...JS_FAMILY), run: (_q, c, y) => piniaStoreEdges(c, y) },
     { name: 'vuexEdges', gate: (has) => has('vue', ...JS_FAMILY), run: (_q, c, y) => vuexDispatchEdges(c, y) },
@@ -3940,7 +3986,10 @@ pool,
 // WAL-valve writer backstop (WalCheckpointValve.backpressure), called at
 // pool-idle points in the edge-insert loops below — the passes themselves
 // only read; every write in this function happens with the pool idle.
-backpressure) {
+backpressure, 
+// On-disk DB path for the same per-pass cost estimate the resolver pool uses.
+// Optional — falls back to a node/edge-count heuristic when absent.
+dbPath) {
     // Each sub-pass below is a whole-graph scan, and there are ~30 of them, all
     // running synchronously on the indexer's main thread. Their AGGREGATE can run
     // for well over a minute on a large repo — long enough for the #850 liveness
@@ -4033,9 +4082,8 @@ backpressure) {
     // persist until that merge — so every pass sees the same committed
     // post-resolution DB state whether it runs sequentially here or on a resolver
     // pool worker. With a live pool (already booted on ≥150k-ref repos), passes
-    // fan out across its read-only workers and the per-pass wall-clock comes from
-    // the worker; a pass that fails on a worker falls back to running on the main
-    // thread, so a worker crash isolates to a retry instead of failing synthesis.
+    // fan across workers up to a memory-budget concurrency (not unbounded
+    // Promise.all) — tight soft `--mem` → serial; large budgets still parallelize.
     const passEdges = new Array(exports.SYNTH_PASSES.length).fill(NONE);
     const markPass = (label, dt) => {
         if (process.env.HOMEGRAPH_SYNTH_TIMINGS && (dt > 250 || process.env.HOMEGRAPH_SYNTH_TIMINGS === 'all')) {
@@ -4065,42 +4113,94 @@ backpressure) {
     // main-thread retry keeps coverage. Skipping loses only that pass's
     // synthesized edges; the index still completes.
     const MAIN_RETRY_MAX_NODES = 1_500_000;
-    const graphNodes = queries.getNodeAndEdgeCount().nodes;
-    if (pool && gatedIn.length > 1) {
-        await Promise.all(gatedIn.map(async (i) => {
-            const pass = exports.SYNTH_PASSES[i];
-            try {
-                const out = await pool.runSynthPass(pass.name);
-                passEdges[i] = out.edges;
-                markPass(pass.name, out.ms);
-            }
-            catch (err) {
-                if (graphNodes > MAIN_RETRY_MAX_NODES) {
-                    // Worker died at a scale where the main-thread retry is a process
-                    // OOM risk: skip the pass, keep the index alive, and say so.
-                    console.error(`[synthesis] pass '${pass.name}' failed on a worker at ${graphNodes} nodes — skipped (edges from this pass are absent): ${err instanceof Error ? err.message : String(err)}`);
-                    markPass(`${pass.name} (skipped at scale)`, 0);
-                    return;
-                }
-                // Worker-side failure (crash, OOM, unknown pass after a version
-                // mismatch): retry this one pass on the main thread.
-                await runPassOnMain(i);
-            }
-        }));
+    const counts = queries.getNodeAndEdgeCount();
+    const graphNodes = counts.nodes;
+    let dbSizeBytes = 0;
+    if (dbPath) {
+        try {
+            dbSizeBytes = fs.statSync(dbPath).size;
+        }
+        catch { /* fresh/missing — heuristic below */ }
     }
-    else {
+    if (dbSizeBytes <= 0) {
+        dbSizeBytes = Math.max((counts.nodes + counts.edges) * 200, 64 * 1024 * 1024);
+    }
+    const poolWorkers = pool?.size ?? 0;
+    const budget = (0, memory_budget_1.synthMemoryBudgetBytes)();
+    const concurrency = (0, memory_budget_1.resolveSynthConcurrency)({
+        explicit: process.env.HOMEGRAPH_SYNTH_CONCURRENCY,
+        memoryBudget: budget,
+        dbSizeBytes,
+        poolWorkers,
+    });
+    if (process.env.HOMEGRAPH_SYNTH_TIMINGS) {
+        console.error(`[synth-timing] concurrency=${concurrency} (pool=${poolWorkers} budget=${Math.round(budget / 1024 / 1024)}MB db=${Math.round(dbSizeBytes / 1024 / 1024)}MB)`);
+    }
+    /**
+     * Fold one pass into the ordered first-wins list and drop its array. Used on
+     * the serial path so we never hold every pass's Edge[] at once — but we must
+     * NOT insert into the DB yet: later passes assume the same committed
+     * post-resolution graph (the parallel path's invariant).
+     */
+    const mergedSerial = [];
+    const seenKeys = new Set();
+    const foldPassEdges = (edges) => {
+        for (const e of edges) {
+            const key = `${e.source}>${e.target}`;
+            if (seenKeys.has(key))
+                continue;
+            seenKeys.add(key);
+            mergedSerial.push(e);
+        }
+    };
+    const runPassOnPool = async (i) => {
+        const pass = exports.SYNTH_PASSES[i];
+        try {
+            const out = await pool.runSynthPass(pass.name);
+            passEdges[i] = out.edges;
+            markPass(pass.name, out.ms);
+        }
+        catch (err) {
+            if (graphNodes > MAIN_RETRY_MAX_NODES) {
+                console.error(`[synthesis] pass '${pass.name}' failed on a worker at ${graphNodes} nodes — skipped (edges from this pass are absent): ${err instanceof Error ? err.message : String(err)}`);
+                markPass(`${pass.name} (skipped at scale)`, 0);
+                return;
+            }
+            await runPassOnMain(i);
+        }
+    };
+    // Serial (concurrency 1): registry order + fold-as-you-go (no mid-pass DB
+    // writes). Parallel: limited fan-out across the pool, hold results until
+    // ordered merge.
+    let serialFold = false;
+    if (!pool || concurrency <= 1 || gatedIn.length <= 1) {
+        serialFold = true;
         for (const i of gatedIn) {
             await runPassOnMain(i);
+            foldPassEdges(passEdges[i]);
+            passEdges[i] = NONE;
         }
     }
-    const merged = [];
-    const seen = new Set();
-    for (const e of passEdges.flat()) {
-        const key = `${e.source}>${e.target}`;
-        if (seen.has(key))
-            continue;
-        seen.add(key);
-        merged.push(e);
+    else {
+        let next = 0;
+        const runners = Array.from({ length: Math.min(concurrency, gatedIn.length) }, async () => {
+            while (next < gatedIn.length) {
+                const idx = next++;
+                await runPassOnPool(gatedIn[idx]);
+            }
+        });
+        await Promise.all(runners);
+    }
+    const merged = serialFold ? mergedSerial : [];
+    if (!serialFold) {
+        const seen = new Set();
+        for (const e of passEdges.flat()) {
+            const key = `${e.source}>${e.target}`;
+            if (seen.has(key))
+                continue;
+            seen.add(key);
+            merged.push(e);
+        }
     }
     __mark('dedupe-merge');
     // Chunked insert with yields: on the Linux kernel the merged synthesized