gitnexus 1.6.10-rc.2 → 1.6.10-rc.21

package/dist/core/ingestion/community-processor.js

@@ -14,6 +14,7 @@ import Graph from 'graphology';
 import { createRequire } from 'node:module';
 import { fileURLToPath } from 'node:url';
 import { dirname, resolve } from 'node:path';
+import { Worker } from 'node:worker_threads';
 const __filename = fileURLToPath(import.meta.url);
 const __dirname = dirname(__filename);
 // Navigate to package root (works from both src/ and dist/)
@@ -37,6 +38,20 @@ function createSeededRng(seed) {
         return ((t ^ (t >>> 14)) >>> 0) / 4294967296;
     };
 }
+const COMMUNITY_ENGINE_ENV = 'GITNEXUS_COMMUNITY_ENGINE';
+const DEFAULT_COMMUNITY_ENGINE = 'graphology';
+const LEIDEN_TIMEOUT_MS = 60_000;
+const ICEBUG_TIMEOUT_MS = 60_000;
+const MIN_CONFIDENCE_LARGE = 0.5;
+export const resolveCommunityDetectionEngine = (raw = process.env[COMMUNITY_ENGINE_ENV]) => {
+    if (raw === undefined || raw.trim() === '')
+        return DEFAULT_COMMUNITY_ENGINE;
+    const normalized = raw.trim().toLowerCase();
+    if (normalized === 'graphology' || normalized === 'icebug' || normalized === 'auto') {
+        return normalized;
+    }
+    return DEFAULT_COMMUNITY_ENGINE;
+};
 // ============================================================================
 // COMMUNITY COLORS (for visualization)
 // ============================================================================
@@ -66,59 +81,28 @@ export const getCommunityColor = (communityIndex) => {
  * This runs AFTER all relationships (CALLS, IMPORTS, etc.) have been built.
  * It uses primarily CALLS edges to cluster code that works together.
  */
-export const processCommunities = async (knowledgeGraph, onProgress) => {
+export const processCommunities = async (knowledgeGraph, onProgress, options = {}) => {
     onProgress?.('Building graph for community detection...', 0);
-    // Pre-check total symbol count to determine large-graph mode before building
-    let symbolCount = 0;
-    knowledgeGraph.forEachNode((node) => {
-        if (node.label === 'Function' ||
-            node.label === 'Class' ||
-            node.label === 'Method' ||
-            node.label === 'Interface') {
-            symbolCount++;
-        }
-    });
-    const isLarge = symbolCount > 10_000;
-    const graph = buildGraphologyGraph(knowledgeGraph, isLarge);
+    const engineRequested = options.engine ?? resolveCommunityDetectionEngine();
+    const projection = buildCommunityProjection(knowledgeGraph);
+    const graph = buildGraphologyGraph(projection);
     if (graph.order === 0) {
         return {
             communities: [],
             memberships: [],
-            stats: { totalCommunities: 0, modularity: 0, nodesProcessed: 0 },
+            stats: {
+                totalCommunities: 0,
+                modularity: 0,
+                nodesProcessed: 0,
+                engine: DEFAULT_COMMUNITY_ENGINE,
+                engineRequested,
+            },
         };
     }
     const nodeCount = graph.order;
     const edgeCount = graph.size;
-    onProgress?.(`Running Leiden on ${nodeCount} nodes, ${edgeCount} edges${isLarge ? ` (filtered from ${symbolCount} symbols)` : ''}...`, 30);
-    // Large graphs: higher resolution + capped iterations (matching Python leidenalg default of 2).
-    // The first 2 iterations capture ~95%+ of modularity; additional iterations have diminishing returns.
-    // Timeout: abort after 60s for pathological graph structures.
-    const LEIDEN_TIMEOUT_MS = 60_000;
-    let details;
-    try {
-        details = await Promise.race([
-            Promise.resolve(leiden.detailed(graph, {
-                resolution: isLarge ? 2.0 : 1.0,
-                maxIterations: isLarge ? 3 : 0,
-                rng: createSeededRng(LEIDEN_SEED),
-            })),
-            new Promise((_, reject) => setTimeout(() => reject(new Error('Leiden timeout')), LEIDEN_TIMEOUT_MS)),
-        ]);
-    }
-    catch (e) {
-        if (e.message === 'Leiden timeout') {
-            onProgress?.('Community detection timed out, using fallback...', 60);
-            // Fallback: assign all nodes to community 0
-            const communities = {};
-            graph.forEachNode((node) => {
-                communities[node] = 0;
-            });
-            details = { communities, count: 1, modularity: 0 };
-        }
-        else {
-            throw e;
-        }
-    }
+    onProgress?.(`Running Leiden on ${nodeCount} nodes, ${edgeCount} edges${projection.isLarge ? ` (filtered from ${projection.symbolCount} symbols)` : ''}...`, 30);
+    const details = await runCommunityEngine(graph, projection, engineRequested, options, onProgress);
     onProgress?.(`Found ${details.count} communities...`, 60);
     // Step 3: Create community nodes with heuristic labels
     const communityNodes = createCommunityNodes(details.communities, details.count, graph, knowledgeGraph);
@@ -139,27 +123,32 @@ export const processCommunities = async (knowledgeGraph, onProgress) => {
             totalCommunities: details.count,
             modularity: details.modularity,
             nodesProcessed: graph.order,
+            engine: details.engine,
+            engineRequested: details.engineRequested,
+            fallbackReason: details.fallbackReason,
         },
     };
 };
 // ============================================================================
-// HELPER: Build graphology graph from knowledge graph
+// HELPER: Build community projection from knowledge graph
 // ============================================================================
 /**
- * Build a graphology graph containing only symbol nodes and clustering edges.
+ * Build a community projection containing only symbol nodes and clustering edges.
  * For large graphs (>10K symbols), filter out low-confidence fuzzy-global edges
  * and degree-1 nodes that add noise and massively increase Leiden runtime.
  */
-const MIN_CONFIDENCE_LARGE = 0.5;
-const buildGraphologyGraph = (knowledgeGraph, isLarge) => {
-    const GraphCtor = Graph;
-    const graph = new GraphCtor({ type: 'undirected', allowSelfLoops: false });
-    const symbolTypes = new Set(['Function', 'Class', 'Method', 'Interface']);
-    const clusteringRelTypes = new Set(['CALLS', 'EXTENDS', 'IMPLEMENTS']);
+export const buildCommunityProjection = (knowledgeGraph) => {
+    let symbolCount = 0;
+    knowledgeGraph.forEachNode((node) => {
+        if (isCommunitySymbol(node)) {
+            symbolCount++;
+        }
+    });
+    const isLarge = symbolCount > 10_000;
     const connectedNodes = new Set();
     const nodeDegree = new Map();
     knowledgeGraph.forEachRelationship((rel) => {
-        if (!clusteringRelTypes.has(rel.type) || rel.sourceId === rel.targetId)
+        if (!isClusteringRelationship(rel.type) || rel.sourceId === rel.targetId)
             return;
         if (isLarge && rel.confidence < MIN_CONFIDENCE_LARGE)
             return;
@@ -168,34 +157,298 @@ const buildGraphologyGraph = (knowledgeGraph, isLarge) => {
         nodeDegree.set(rel.sourceId, (nodeDegree.get(rel.sourceId) || 0) + 1);
         nodeDegree.set(rel.targetId, (nodeDegree.get(rel.targetId) || 0) + 1);
     });
+    const nodes = [];
+    const nodeIndexById = new Map();
     knowledgeGraph.forEachNode((node) => {
-        if (!symbolTypes.has(node.label) || !connectedNodes.has(node.id))
+        if (!isCommunitySymbol(node) || !connectedNodes.has(node.id))
             return;
         // For large graphs, skip degree-1 nodes — they just become singletons or
         // get absorbed into their single neighbor's community, but cost iteration time.
         if (isLarge && (nodeDegree.get(node.id) || 0) < 2)
             return;
-        graph.addNode(node.id, {
+        nodeIndexById.set(node.id, nodes.length);
+        nodes.push({
+            id: node.id,
             name: node.properties.name,
             filePath: node.properties.filePath,
             type: node.label,
         });
     });
+    const seenEdges = new Set();
+    const edges = [];
     knowledgeGraph.forEachRelationship((rel) => {
-        if (!clusteringRelTypes.has(rel.type))
+        if (!isClusteringRelationship(rel.type) || rel.sourceId === rel.targetId)
             return;
         if (isLarge && rel.confidence < MIN_CONFIDENCE_LARGE)
             return;
-        if (graph.hasNode(rel.sourceId) &&
-            graph.hasNode(rel.targetId) &&
-            rel.sourceId !== rel.targetId) {
-            if (!graph.hasEdge(rel.sourceId, rel.targetId)) {
-                graph.addEdge(rel.sourceId, rel.targetId);
-            }
-        }
+        const sourceIndex = nodeIndexById.get(rel.sourceId);
+        const targetIndex = nodeIndexById.get(rel.targetId);
+        if (sourceIndex === undefined || targetIndex === undefined || sourceIndex === targetIndex)
+            return;
+        const [a, b] = sourceIndex < targetIndex ? [sourceIndex, targetIndex] : [targetIndex, sourceIndex];
+        const edgeKey = `${a}:${b}`;
+        if (seenEdges.has(edgeKey))
+            return;
+        seenEdges.add(edgeKey);
+        edges.push([a, b]);
     });
+    return { nodes, edges, symbolCount, isLarge };
+};
+export const buildCommunityCsr = (projection) => {
+    const adjacency = Array.from({ length: projection.nodes.length }, () => new Set());
+    for (const [sourceIndex, targetIndex] of projection.edges) {
+        adjacency[sourceIndex].add(targetIndex);
+        adjacency[targetIndex].add(sourceIndex);
+    }
+    const edgeTraversalCount = adjacency.reduce((count, neighbors) => count + neighbors.size, 0);
+    const indptr = new BigUint64Array(projection.nodes.length + 1);
+    const indices = new BigUint64Array(edgeTraversalCount);
+    let cursor = 0;
+    for (let nodeIndex = 0; nodeIndex < adjacency.length; nodeIndex++) {
+        indptr[nodeIndex] = BigInt(cursor);
+        const neighbors = [...adjacency[nodeIndex]].sort((a, b) => a - b);
+        for (const neighbor of neighbors) {
+            indices[cursor++] = BigInt(neighbor);
+        }
+    }
+    indptr[projection.nodes.length] = BigInt(cursor);
+    return { indptr, indices };
+};
+export const buildGraphologyGraph = (projection) => {
+    const GraphCtor = Graph;
+    const graph = new GraphCtor({ type: 'undirected', allowSelfLoops: false });
+    for (const node of projection.nodes) {
+        graph.addNode(node.id, {
+            name: node.name,
+            filePath: node.filePath,
+            type: node.type,
+        });
+    }
+    for (const [sourceIndex, targetIndex] of projection.edges) {
+        graph.addEdge(projection.nodes[sourceIndex].id, projection.nodes[targetIndex].id);
+    }
     return graph;
 };
+const isCommunitySymbol = (node) => node.label === 'Function' ||
+    node.label === 'Class' ||
+    node.label === 'Method' ||
+    node.label === 'Interface';
+const isClusteringRelationship = (type) => type === 'CALLS' || type === 'EXTENDS' || type === 'IMPLEMENTS';
+const runCommunityEngine = async (graph, projection, engineRequested, options, onProgress) => {
+    if (engineRequested === 'graphology') {
+        return runGraphologyLeiden(graph, projection.isLarge, engineRequested);
+    }
+    try {
+        return await runIcebugLeiden(projection, engineRequested, options);
+    }
+    catch (error) {
+        const fallbackReason = error instanceof Error ? error.message : String(error);
+        onProgress?.(`Icebug community engine unavailable, falling back to Graphology: ${fallbackReason}`, 35);
+        const fallback = await runGraphologyLeiden(graph, projection.isLarge, engineRequested);
+        return { ...fallback, fallbackReason };
+    }
+};
+const runGraphologyLeiden = async (graph, isLarge, engineRequested) => {
+    try {
+        const details = await Promise.race([
+            Promise.resolve(leiden.detailed(graph, {
+                resolution: isLarge ? 2.0 : 1.0,
+                maxIterations: isLarge ? 3 : 0,
+                rng: createSeededRng(LEIDEN_SEED),
+            })),
+            new Promise((_, reject) => setTimeout(() => reject(new Error('Leiden timeout')), LEIDEN_TIMEOUT_MS)),
+        ]);
+        return { ...details, engine: 'graphology', engineRequested };
+    }
+    catch (e) {
+        if (e.message !== 'Leiden timeout') {
+            throw e;
+        }
+        // Fallback: assign all nodes to community 0
+        const communities = {};
+        graph.forEachNode((node) => {
+            communities[node] = 0;
+        });
+        return {
+            communities,
+            count: 1,
+            modularity: 0,
+            engine: 'graphology',
+            engineRequested,
+            fallbackReason: 'Graphology Leiden timeout',
+        };
+    }
+};
+const runIcebugLeiden = async (projection, engineRequested, options) => {
+    const csr = buildCommunityCsr(projection);
+    const nativeResult = await runIcebugWorker(projection.nodes.length, csr, options);
+    const partition = nativeResult.partition;
+    if (!Number.isFinite(nativeResult.modularity)) {
+        throw new Error('optional icebug modularity was not finite');
+    }
+    if (partition.length !== projection.nodes.length ||
+        partition.some((community) => !Number.isSafeInteger(community))) {
+        throw new Error(`optional icebug partition was malformed for ${projection.nodes.length} projected nodes`);
+    }
+    const communities = normalizePartition(projection, partition);
+    return {
+        communities,
+        count: new Set(Object.values(communities)).size,
+        modularity: nativeResult.modularity,
+        engine: 'icebug',
+        engineRequested,
+    };
+};
+const runIcebugWorker = (nodeCount, csr, options) => {
+    const threads = options.icebug?.threads ?? 1;
+    if (!Number.isSafeInteger(threads) || threads !== 1) {
+        throw new Error('optional icebug engine currently requires deterministic threads=1');
+    }
+    if (options.icebug?.randomize === true) {
+        throw new Error('optional icebug engine currently requires randomize=false');
+    }
+    const worker = new Worker(ICEBUG_WORKER_SOURCE, {
+        eval: true,
+        workerData: {
+            nodeCount,
+            indices: csr.indices,
+            indptr: csr.indptr,
+            threads,
+            seed: options.icebug?.seed ?? LEIDEN_SEED,
+            iterations: options.icebug?.iterations ?? 4,
+            gamma: options.icebug?.gamma ?? 1.0,
+            randomize: options.icebug?.randomize ?? false,
+        },
+    });
+    return new Promise((resolve, reject) => {
+        let settled = false;
+        const timeout = setTimeout(() => {
+            settled = true;
+            void worker.terminate();
+            reject(new Error(`optional icebug community engine timed out after ${ICEBUG_TIMEOUT_MS}ms`));
+        }, ICEBUG_TIMEOUT_MS);
+        worker.once('message', (message) => {
+            settled = true;
+            clearTimeout(timeout);
+            void worker.terminate();
+            if (message.ok === true) {
+                resolve(message);
+            }
+            else {
+                reject(new Error(message.error));
+            }
+        });
+        worker.once('error', (error) => {
+            settled = true;
+            clearTimeout(timeout);
+            void worker.terminate();
+            reject(error);
+        });
+        worker.once('exit', (code) => {
+            if (settled)
+                return;
+            clearTimeout(timeout);
+            if (code === 0) {
+                reject(new Error('optional icebug worker exited before returning a partition'));
+                return;
+            }
+            reject(new Error(`optional icebug worker exited with code ${code}`));
+        });
+    });
+};
+const ICEBUG_WORKER_SOURCE = `
+const { parentPort, workerData } = require('node:worker_threads');
+
+const isNumericArrayLike = (value) =>
+  typeof value === 'object' &&
+  value !== null &&
+  'length' in value &&
+  typeof value.length === 'number';
+
+const readPartition = (runner) => {
+  const candidates = [
+    typeof runner.getPartition === 'function' ? runner.getPartition() : runner.partition,
+    typeof runner.getCommunities === 'function' ? runner.getCommunities() : undefined,
+    typeof runner.getMembership === 'function' ? runner.getMembership() : undefined,
+    typeof runner.getMemberships === 'function' ? runner.getMemberships() : undefined,
+  ];
+
+  for (const candidate of candidates) {
+    if (isNumericArrayLike(candidate)) {
+      return Array.from(candidate, Number);
+    }
+  }
+
+  throw new Error('optional icebug ParallelLeidenView did not expose a partition array');
+};
+
+const readModularity = (runner) => {
+  if (typeof runner.getModularity === 'function') return runner.getModularity();
+  if (typeof runner.modularity === 'function') return runner.modularity();
+  if (typeof runner.modularity === 'number') return runner.modularity;
+  return 0;
+};
+
+(async () => {
+  const imported = await import('icebug');
+  const icebug = imported.default ?? imported;
+  const fromCSR = icebug.Graph?.fromCSR;
+  const ParallelLeidenView = icebug.community?.ParallelLeidenView;
+  if (!fromCSR || !ParallelLeidenView) {
+    throw new Error('optional icebug module does not expose Graph.fromCSR/ParallelLeidenView');
+  }
+
+  if (typeof icebug.setNumberOfThreads !== 'function' || typeof icebug.setSeed !== 'function') {
+    throw new Error('optional icebug module does not expose deterministic thread/seed controls');
+  }
+  icebug.setNumberOfThreads(workerData.threads);
+  icebug.setSeed(workerData.seed, false);
+
+  const nativeGraph = fromCSR(workerData.nodeCount, false, workerData.indices, workerData.indptr);
+  let runner;
+  try {
+    runner = new ParallelLeidenView(nativeGraph, {
+      iterations: workerData.iterations,
+      gamma: workerData.gamma,
+      randomize: workerData.randomize,
+    });
+  } catch {
+    runner = new ParallelLeidenView(
+      nativeGraph,
+      workerData.iterations,
+      workerData.gamma,
+      workerData.randomize,
+    );
+  }
+
+  if (typeof runner.run !== 'function') {
+    throw new Error('optional icebug ParallelLeidenView does not expose run()');
+  }
+
+  runner.run();
+  parentPort.postMessage({
+    ok: true,
+    partition: readPartition(runner),
+    modularity: readModularity(runner),
+  });
+})().catch((error) => {
+  parentPort.postMessage({ ok: false, error: error instanceof Error ? error.message : String(error) });
+});
+`;
+const normalizePartition = (projection, partition) => {
+    const remap = new Map();
+    const communities = {};
+    for (let index = 0; index < projection.nodes.length; index++) {
+        const rawCommunity = String(partition[index]);
+        let communityId = remap.get(rawCommunity);
+        if (communityId === undefined) {
+            communityId = remap.size;
+            remap.set(rawCommunity, communityId);
+        }
+        communities[projection.nodes[index].id] = communityId;
+    }
+    return communities;
+};
 // ============================================================================
 // HELPER: Create community nodes with heuristic labels
 // ============================================================================

package/dist/core/ingestion/emit-references.js

@@ -44,6 +44,7 @@
  * ancestor scope, and if THAT produces nothing either the edge is
  * skipped (with a count returned in `EmitStats.skippedNoCaller`).
  */
+import { toZeroBasedLine } from './utils/line-base.js';
 /**
  * Drain `referenceIndex.bySourceScope` into graph edges.
  *
@@ -95,8 +96,8 @@ export function emitScopeGraph(input) {
             properties: {
                 name: scope.kind,
                 filePath: scope.filePath,
-                startLine: scope.range.startLine,
-                endLine: scope.range.endLine,
+                startLine: toZeroBasedLine(scope.range.startLine),
+                endLine: toZeroBasedLine(scope.range.endLine),
                 description: `Scope: ${scope.kind}`,
             },
         });