gitnexus 1.0.0

package/dist/core/embeddings/types.js

@@ -0,0 +1,32 @@
+/**
+ * Embedding Pipeline Types
+ *
+ * Type definitions for the embedding generation and semantic search system.
+ */
+/**
+ * Node labels that should be embedded for semantic search
+ * These are code elements that benefit from semantic matching
+ */
+export const EMBEDDABLE_LABELS = [
+    'Function',
+    'Class',
+    'Method',
+    'Interface',
+    'File',
+];
+/**
+ * Check if a label should be embedded
+ */
+export const isEmbeddableLabel = (label) => EMBEDDABLE_LABELS.includes(label);
+/**
+ * Default embedding configuration
+ * Uses snowflake-arctic-embed-xs for browser efficiency
+ * Tries WebGPU first (fast), user can choose WASM fallback if unavailable
+ */
+export const DEFAULT_EMBEDDING_CONFIG = {
+    modelId: 'Snowflake/snowflake-arctic-embed-xs',
+    batchSize: 16,
+    dimensions: 384,
+    device: 'auto',
+    maxSnippetLength: 500,
+};

package/dist/core/graph/graph.d.ts

@@ -0,0 +1,2 @@
+import { KnowledgeGraph } from './types.js';
+export declare const createKnowledgeGraph: () => KnowledgeGraph;

package/dist/core/graph/graph.js

@@ -0,0 +1,61 @@
+export const createKnowledgeGraph = () => {
+    const nodeMap = new Map();
+    const relationshipMap = new Map();
+    const addNode = (node) => {
+        if (!nodeMap.has(node.id)) {
+            nodeMap.set(node.id, node);
+        }
+    };
+    const addRelationship = (relationship) => {
+        if (!relationshipMap.has(relationship.id)) {
+            relationshipMap.set(relationship.id, relationship);
+        }
+    };
+    /**
+     * Remove a single node and all relationships involving it
+     */
+    const removeNode = (nodeId) => {
+        if (!nodeMap.has(nodeId))
+            return false;
+        nodeMap.delete(nodeId);
+        // Remove all relationships involving this node
+        for (const [relId, rel] of relationshipMap) {
+            if (rel.sourceId === nodeId || rel.targetId === nodeId) {
+                relationshipMap.delete(relId);
+            }
+        }
+        return true;
+    };
+    /**
+     * Remove all nodes (and their relationships) belonging to a file
+     */
+    const removeNodesByFile = (filePath) => {
+        let removed = 0;
+        for (const [nodeId, node] of nodeMap) {
+            if (node.properties?.filePath === filePath) {
+                removeNode(nodeId);
+                removed++;
+            }
+        }
+        return removed;
+    };
+    return {
+        get nodes() {
+            return Array.from(nodeMap.values());
+        },
+        get relationships() {
+            return Array.from(relationshipMap.values());
+        },
+        // O(1) count getters - avoid creating arrays just for length
+        get nodeCount() {
+            return nodeMap.size;
+        },
+        get relationshipCount() {
+            return relationshipMap.size;
+        },
+        addNode,
+        addRelationship,
+        removeNode,
+        removeNodesByFile,
+    };
+};

package/dist/core/graph/types.d.ts

@@ -0,0 +1,50 @@
+export type NodeLabel = 'Project' | 'Package' | 'Module' | 'Folder' | 'File' | 'Class' | 'Function' | 'Method' | 'Variable' | 'Interface' | 'Enum' | 'Decorator' | 'Import' | 'Type' | 'CodeElement' | 'Community' | 'Process';
+export type NodeProperties = {
+    name: string;
+    filePath: string;
+    startLine?: number;
+    endLine?: number;
+    language?: string;
+    isExported?: boolean;
+    heuristicLabel?: string;
+    cohesion?: number;
+    symbolCount?: number;
+    keywords?: string[];
+    description?: string;
+    enrichedBy?: 'heuristic' | 'llm';
+    processType?: 'intra_community' | 'cross_community';
+    stepCount?: number;
+    communities?: string[];
+    entryPointId?: string;
+    terminalId?: string;
+    entryPointScore?: number;
+    entryPointReason?: string;
+};
+export type RelationshipType = 'CONTAINS' | 'CALLS' | 'INHERITS' | 'OVERRIDES' | 'IMPORTS' | 'USES' | 'DEFINES' | 'DECORATES' | 'IMPLEMENTS' | 'EXTENDS' | 'MEMBER_OF' | 'STEP_IN_PROCESS';
+export interface GraphNode {
+    id: string;
+    label: NodeLabel;
+    properties: NodeProperties;
+}
+export interface GraphRelationship {
+    id: string;
+    sourceId: string;
+    targetId: string;
+    type: RelationshipType;
+    /** Confidence score 0-1 (1.0 = certain, lower = uncertain resolution) */
+    confidence: number;
+    /** Resolution reason: 'import-resolved', 'same-file', 'fuzzy-global', or empty for non-CALLS */
+    reason: string;
+    /** Step number for STEP_IN_PROCESS relationships (1-indexed) */
+    step?: number;
+}
+export interface KnowledgeGraph {
+    nodes: GraphNode[];
+    relationships: GraphRelationship[];
+    nodeCount: number;
+    relationshipCount: number;
+    addNode: (node: GraphNode) => void;
+    addRelationship: (relationship: GraphRelationship) => void;
+    removeNode: (nodeId: string) => boolean;
+    removeNodesByFile: (filePath: string) => number;
+}

package/dist/core/graph/types.js

@@ -0,0 +1 @@
+export {};

package/dist/core/ingestion/ast-cache.d.ts

@@ -0,0 +1,11 @@
+import Parser from 'tree-sitter';
+export interface ASTCache {
+    get: (filePath: string) => Parser.Tree | undefined;
+    set: (filePath: string, tree: Parser.Tree) => void;
+    clear: () => void;
+    stats: () => {
+        size: number;
+        maxSize: number;
+    };
+}
+export declare const createASTCache: (maxSize?: number) => ASTCache;

package/dist/core/ingestion/ast-cache.js

@@ -0,0 +1,34 @@
+import { LRUCache } from 'lru-cache';
+export const createASTCache = (maxSize = 50) => {
+    // Initialize the cache with a 'dispose' handler
+    // This is the magic: When an item is evicted (dropped), this runs automatically.
+    const cache = new LRUCache({
+        max: maxSize,
+        dispose: (tree) => {
+            try {
+                // NOTE: web-tree-sitter has tree.delete(); native tree-sitter trees are GC-managed.
+                // Keep this try/catch so we don't crash on either runtime.
+                tree.delete?.();
+            }
+            catch (e) {
+                console.warn('Failed to delete tree from WASM memory', e);
+            }
+        }
+    });
+    return {
+        get: (filePath) => {
+            const tree = cache.get(filePath);
+            return tree; // Returns undefined if not found
+        },
+        set: (filePath, tree) => {
+            cache.set(filePath, tree);
+        },
+        clear: () => {
+            cache.clear();
+        },
+        stats: () => ({
+            size: cache.size,
+            maxSize: maxSize
+        })
+    };
+};

package/dist/core/ingestion/call-processor.d.ts

@@ -0,0 +1,8 @@
+import { KnowledgeGraph } from '../graph/types.js';
+import { ASTCache } from './ast-cache.js';
+import { SymbolTable } from './symbol-table.js';
+import { ImportMap } from './import-processor.js';
+export declare const processCalls: (graph: KnowledgeGraph, files: {
+    path: string;
+    content: string;
+}[], astCache: ASTCache, symbolTable: SymbolTable, importMap: ImportMap, onProgress?: (current: number, total: number) => void) => Promise<void>;

package/dist/core/ingestion/call-processor.js

@@ -0,0 +1,269 @@
+import Parser from 'tree-sitter';
+import { loadParser, loadLanguage } from '../tree-sitter/parser-loader.js';
+import { LANGUAGE_QUERIES } from './tree-sitter-queries.js';
+import { generateId } from '../../lib/utils.js';
+import { getLanguageFromFilename, yieldToEventLoop } from './utils.js';
+/**
+ * Node types that represent function/method definitions across languages.
+ * Used to find the enclosing function for a call site.
+ */
+const FUNCTION_NODE_TYPES = new Set([
+    // TypeScript/JavaScript
+    'function_declaration',
+    'arrow_function',
+    'function_expression',
+    'method_definition',
+    'generator_function_declaration',
+    // Python
+    'function_definition',
+    // Common async variants
+    'async_function_declaration',
+    'async_arrow_function',
+    // Java
+    'method_declaration',
+    'constructor_declaration',
+    // C/C++
+    // 'function_definition' already included above
+    // Go
+    // 'method_declaration' already included from Java
+    // C#
+    'local_function_statement',
+    // Rust
+    'function_item',
+    'impl_item', // Methods inside impl blocks
+]);
+/**
+ * Walk up the AST from a node to find the enclosing function/method.
+ * Returns null if the call is at module/file level (top-level code).
+ */
+const findEnclosingFunction = (node, filePath, symbolTable) => {
+    let current = node.parent;
+    while (current) {
+        if (FUNCTION_NODE_TYPES.has(current.type)) {
+            // Found enclosing function - try to get its name
+            let funcName = null;
+            let label = 'Function';
+            // Different node types have different name locations
+            if (current.type === 'function_declaration' ||
+                current.type === 'function_definition' ||
+                current.type === 'async_function_declaration' ||
+                current.type === 'generator_function_declaration' ||
+                current.type === 'function_item') { // Rust function
+                // Named function: function foo() {}
+                const nameNode = current.childForFieldName?.('name') ||
+                    current.children?.find((c) => c.type === 'identifier' || c.type === 'property_identifier');
+                funcName = nameNode?.text;
+            }
+            else if (current.type === 'impl_item') {
+                // Rust method inside impl block: wrapper around function_item or const_item
+                // We need to look inside for the function_item
+                const funcItem = current.children?.find((c) => c.type === 'function_item');
+                if (funcItem) {
+                    const nameNode = funcItem.childForFieldName?.('name') ||
+                        funcItem.children?.find((c) => c.type === 'identifier');
+                    funcName = nameNode?.text;
+                    label = 'Method';
+                }
+            }
+            else if (current.type === 'method_definition') {
+                // Method: foo() {} inside class (JS/TS)
+                const nameNode = current.childForFieldName?.('name') ||
+                    current.children?.find((c) => c.type === 'property_identifier');
+                funcName = nameNode?.text;
+                label = 'Method';
+            }
+            else if (current.type === 'method_declaration') {
+                // Java method: public void foo() {}
+                const nameNode = current.childForFieldName?.('name') ||
+                    current.children?.find((c) => c.type === 'identifier');
+                funcName = nameNode?.text;
+                label = 'Method';
+            }
+            else if (current.type === 'constructor_declaration') {
+                // Java constructor: public ClassName() {}
+                const nameNode = current.childForFieldName?.('name') ||
+                    current.children?.find((c) => c.type === 'identifier');
+                funcName = nameNode?.text;
+                label = 'Method'; // Treat constructors as methods for process detection
+            }
+            else if (current.type === 'arrow_function' || current.type === 'function_expression') {
+                // Arrow/expression: const foo = () => {} - check parent variable declarator
+                const parent = current.parent;
+                if (parent?.type === 'variable_declarator') {
+                    const nameNode = parent.childForFieldName?.('name') ||
+                        parent.children?.find((c) => c.type === 'identifier');
+                    funcName = nameNode?.text;
+                }
+            }
+            if (funcName) {
+                // Look up the function in symbol table to get its node ID
+                // Try exact match first
+                const nodeId = symbolTable.lookupExact(filePath, funcName);
+                if (nodeId)
+                    return nodeId;
+                // Try construct ID manually if lookup fails (common for non-exported internal functions)
+                // Format should match what parsing-processor generates: "Function:path/to/file:funcName"
+                // Check if we already have a node with this ID in the symbol table to be safe
+                const generatedId = generateId(label, `${filePath}:${funcName}`);
+                // Ideally we should verify this ID exists, but strictly speaking if we are inside it,
+                // it SHOULD exist. Returning it is better than falling back to File.
+                return generatedId;
+            }
+            // Couldn't determine function name - try parent (might be nested)
+        }
+        current = current.parent;
+    }
+    return null; // Top-level call (not inside any function)
+};
+export const processCalls = async (graph, files, astCache, symbolTable, importMap, onProgress) => {
+    const parser = await loadParser();
+    for (let i = 0; i < files.length; i++) {
+        const file = files[i];
+        onProgress?.(i + 1, files.length);
+        if (i % 20 === 0)
+            await yieldToEventLoop();
+        // 1. Check language support first
+        const language = getLanguageFromFilename(file.path);
+        if (!language)
+            continue;
+        const queryStr = LANGUAGE_QUERIES[language];
+        if (!queryStr)
+            continue;
+        // 2. ALWAYS load the language before querying (parser is stateful)
+        await loadLanguage(language, file.path);
+        // 3. Get AST (Try Cache First)
+        let tree = astCache.get(file.path);
+        let wasReparsed = false;
+        if (!tree) {
+            // Cache Miss: Re-parse
+            // Use larger bufferSize for files > 32KB
+            try {
+                tree = parser.parse(file.content, undefined, { bufferSize: 1024 * 256 });
+            }
+            catch (parseError) {
+                // Skip files that can't be parsed
+                continue;
+            }
+            wasReparsed = true;
+        }
+        let query;
+        let matches;
+        try {
+            const language = parser.getLanguage();
+            query = new Parser.Query(language, queryStr);
+            matches = query.matches(tree.rootNode);
+        }
+        catch (queryError) {
+            console.warn(`Query error for ${file.path}:`, queryError);
+            if (wasReparsed)
+                tree.delete?.();
+            continue;
+        }
+        // 3. Process each call match
+        matches.forEach(match => {
+            const captureMap = {};
+            match.captures.forEach(c => captureMap[c.name] = c.node);
+            // Only process @call captures
+            if (!captureMap['call'])
+                return;
+            const nameNode = captureMap['call.name'];
+            if (!nameNode)
+                return;
+            const calledName = nameNode.text;
+            // Skip common built-ins and noise
+            if (isBuiltInOrNoise(calledName))
+                return;
+            // 4. Resolve the target using priority strategy (returns confidence)
+            const resolved = resolveCallTarget(calledName, file.path, symbolTable, importMap);
+            if (!resolved)
+                return;
+            // 5. Find the enclosing function (caller)
+            const callNode = captureMap['call'];
+            const enclosingFuncId = findEnclosingFunction(callNode, file.path, symbolTable);
+            // Use enclosing function as source, fallback to file for top-level calls
+            const sourceId = enclosingFuncId || generateId('File', file.path);
+            const relId = generateId('CALLS', `${sourceId}:${calledName}->${resolved.nodeId}`);
+            graph.addRelationship({
+                id: relId,
+                sourceId,
+                targetId: resolved.nodeId,
+                type: 'CALLS',
+                confidence: resolved.confidence,
+                reason: resolved.reason,
+            });
+        });
+        // Cleanup if re-parsed
+        if (wasReparsed) {
+            tree.delete?.();
+        }
+    }
+};
+/**
+ * Resolve a function call to its target node ID using priority strategy:
+ * A. Check imported files first (highest confidence)
+ * B. Check local file definitions
+ * C. Fuzzy global search (lowest confidence)
+ *
+ * Returns confidence score so agents know what to trust.
+ */
+const resolveCallTarget = (calledName, currentFile, symbolTable, importMap) => {
+    // Strategy A: Check imported files (HIGH confidence - we know the import chain)
+    const importedFiles = importMap.get(currentFile);
+    if (importedFiles) {
+        for (const importedFile of importedFiles) {
+            const nodeId = symbolTable.lookupExact(importedFile, calledName);
+            if (nodeId) {
+                return { nodeId, confidence: 0.9, reason: 'import-resolved' };
+            }
+        }
+    }
+    // Strategy B: Check local file (HIGH confidence - same file definition)
+    const localNodeId = symbolTable.lookupExact(currentFile, calledName);
+    if (localNodeId) {
+        return { nodeId: localNodeId, confidence: 0.85, reason: 'same-file' };
+    }
+    // Strategy C: Fuzzy global search (LOW confidence - just matching by name)
+    const fuzzyMatches = symbolTable.lookupFuzzy(calledName);
+    if (fuzzyMatches.length > 0) {
+        // Lower confidence if multiple matches exist (more ambiguous)
+        const confidence = fuzzyMatches.length === 1 ? 0.5 : 0.3;
+        return { nodeId: fuzzyMatches[0].nodeId, confidence, reason: 'fuzzy-global' };
+    }
+    return null;
+};
+/**
+ * Filter out common built-in functions and noise
+ * that shouldn't be tracked as calls
+ */
+const isBuiltInOrNoise = (name) => {
+    const builtIns = new Set([
+        // JavaScript/TypeScript built-ins
+        'console', 'log', 'warn', 'error', 'info', 'debug',
+        'setTimeout', 'setInterval', 'clearTimeout', 'clearInterval',
+        'parseInt', 'parseFloat', 'isNaN', 'isFinite',
+        'encodeURI', 'decodeURI', 'encodeURIComponent', 'decodeURIComponent',
+        'JSON', 'parse', 'stringify',
+        'Object', 'Array', 'String', 'Number', 'Boolean', 'Symbol', 'BigInt',
+        'Map', 'Set', 'WeakMap', 'WeakSet',
+        'Promise', 'resolve', 'reject', 'then', 'catch', 'finally',
+        'Math', 'Date', 'RegExp', 'Error',
+        'require', 'import', 'export',
+        'fetch', 'Response', 'Request',
+        // React hooks and common functions
+        'useState', 'useEffect', 'useCallback', 'useMemo', 'useRef', 'useContext',
+        'useReducer', 'useLayoutEffect', 'useImperativeHandle', 'useDebugValue',
+        'createElement', 'createContext', 'createRef', 'forwardRef', 'memo', 'lazy',
+        // Common array/object methods
+        'map', 'filter', 'reduce', 'forEach', 'find', 'findIndex', 'some', 'every',
+        'includes', 'indexOf', 'slice', 'splice', 'concat', 'join', 'split',
+        'push', 'pop', 'shift', 'unshift', 'sort', 'reverse',
+        'keys', 'values', 'entries', 'assign', 'freeze', 'seal',
+        'hasOwnProperty', 'toString', 'valueOf',
+        // Python built-ins
+        'print', 'len', 'range', 'str', 'int', 'float', 'list', 'dict', 'set', 'tuple',
+        'open', 'read', 'write', 'close', 'append', 'extend', 'update',
+        'super', 'type', 'isinstance', 'issubclass', 'getattr', 'setattr', 'hasattr',
+        'enumerate', 'zip', 'sorted', 'reversed', 'min', 'max', 'sum', 'abs',
+    ]);
+    return builtIns.has(name);
+};

package/dist/core/ingestion/cluster-enricher.d.ts

@@ -0,0 +1,38 @@
+/**
+ * Cluster Enricher
+ *
+ * LLM-based enrichment for community clusters.
+ * Generates semantic names, keywords, and descriptions using an LLM.
+ */
+import { CommunityNode } from './community-processor.js';
+export interface ClusterEnrichment {
+    name: string;
+    keywords: string[];
+    description: string;
+}
+export interface EnrichmentResult {
+    enrichments: Map<string, ClusterEnrichment>;
+    tokensUsed: number;
+}
+export interface LLMClient {
+    generate: (prompt: string) => Promise<string>;
+}
+export interface ClusterMemberInfo {
+    name: string;
+    filePath: string;
+    type: string;
+}
+/**
+ * Enrich clusters with LLM-generated names, keywords, and descriptions
+ *
+ * @param communities - Community nodes to enrich
+ * @param memberMap - Map of communityId -> member info
+ * @param llmClient - LLM client for generation
+ * @param onProgress - Progress callback
+ */
+export declare const enrichClusters: (communities: CommunityNode[], memberMap: Map<string, ClusterMemberInfo[]>, llmClient: LLMClient, onProgress?: (current: number, total: number) => void) => Promise<EnrichmentResult>;
+/**
+ * Enrich multiple clusters in a single LLM call (batch mode)
+ * More efficient for token usage but requires larger context window
+ */
+export declare const enrichClustersBatch: (communities: CommunityNode[], memberMap: Map<string, ClusterMemberInfo[]>, llmClient: LLMClient, batchSize?: number, onProgress?: (current: number, total: number) => void) => Promise<EnrichmentResult>;