gitnexus 1.1.8 → 1.2.0

package/dist/core/ingestion/workers/parse-worker.js

@@ -0,0 +1,451 @@
+import { parentPort } from 'node:worker_threads';
+import Parser from 'tree-sitter';
+import JavaScript from 'tree-sitter-javascript';
+import TypeScript from 'tree-sitter-typescript';
+import Python from 'tree-sitter-python';
+import Java from 'tree-sitter-java';
+import C from 'tree-sitter-c';
+import CPP from 'tree-sitter-cpp';
+import CSharp from 'tree-sitter-c-sharp';
+import Go from 'tree-sitter-go';
+import Rust from 'tree-sitter-rust';
+import { SupportedLanguages } from '../../../config/supported-languages.js';
+import { LANGUAGE_QUERIES } from '../tree-sitter-queries.js';
+import { getLanguageFromFilename } from '../utils.js';
+import { generateId } from '../../../lib/utils.js';
+// ============================================================================
+// Worker-local parser + language map
+// ============================================================================
+const parser = new Parser();
+const languageMap = {
+    [SupportedLanguages.JavaScript]: JavaScript,
+    [SupportedLanguages.TypeScript]: TypeScript.typescript,
+    [`${SupportedLanguages.TypeScript}:tsx`]: TypeScript.tsx,
+    [SupportedLanguages.Python]: Python,
+    [SupportedLanguages.Java]: Java,
+    [SupportedLanguages.C]: C,
+    [SupportedLanguages.CPlusPlus]: CPP,
+    [SupportedLanguages.CSharp]: CSharp,
+    [SupportedLanguages.Go]: Go,
+    [SupportedLanguages.Rust]: Rust,
+};
+const setLanguage = (language, filePath) => {
+    const key = language === SupportedLanguages.TypeScript && filePath.endsWith('.tsx')
+        ? `${language}:tsx`
+        : language;
+    const lang = languageMap[key];
+    if (!lang)
+        throw new Error(`Unsupported language: ${language}`);
+    parser.setLanguage(lang);
+};
+// ============================================================================
+// Export detection (copied — needs AST parent traversal, can't cross threads)
+// ============================================================================
+const isNodeExported = (node, name, language) => {
+    let current = node;
+    switch (language) {
+        case 'javascript':
+        case 'typescript':
+            while (current) {
+                const type = current.type;
+                if (type === 'export_statement' ||
+                    type === 'export_specifier' ||
+                    type === 'lexical_declaration' && current.parent?.type === 'export_statement') {
+                    return true;
+                }
+                if (current.text?.startsWith('export ')) {
+                    return true;
+                }
+                current = current.parent;
+            }
+            return false;
+        case 'python':
+            return !name.startsWith('_');
+        case 'java':
+            while (current) {
+                if (current.parent) {
+                    const parent = current.parent;
+                    for (let i = 0; i < parent.childCount; i++) {
+                        const child = parent.child(i);
+                        if (child?.type === 'modifiers' && child.text?.includes('public')) {
+                            return true;
+                        }
+                    }
+                    if (parent.type === 'method_declaration' || parent.type === 'constructor_declaration') {
+                        if (parent.text?.trimStart().startsWith('public')) {
+                            return true;
+                        }
+                    }
+                }
+                current = current.parent;
+            }
+            return false;
+        case 'csharp':
+            while (current) {
+                if (current.type === 'modifier' || current.type === 'modifiers') {
+                    if (current.text?.includes('public'))
+                        return true;
+                }
+                current = current.parent;
+            }
+            return false;
+        case 'go':
+            if (name.length === 0)
+                return false;
+            const first = name[0];
+            return first === first.toUpperCase() && first !== first.toLowerCase();
+        case 'rust':
+            while (current) {
+                if (current.type === 'visibility_modifier') {
+                    if (current.text?.includes('pub'))
+                        return true;
+                }
+                current = current.parent;
+            }
+            return false;
+        case 'c':
+        case 'cpp':
+            return false;
+        default:
+            return false;
+    }
+};
+// ============================================================================
+// Enclosing function detection (for call extraction)
+// ============================================================================
+const FUNCTION_NODE_TYPES = new Set([
+    'function_declaration', 'arrow_function', 'function_expression',
+    'method_definition', 'generator_function_declaration',
+    'function_definition', 'async_function_declaration', 'async_arrow_function',
+    'method_declaration', 'constructor_declaration',
+    'local_function_statement', 'function_item', 'impl_item',
+]);
+/** Walk up AST to find enclosing function, return its generateId or null for top-level */
+const findEnclosingFunctionId = (node, filePath) => {
+    let current = node.parent;
+    while (current) {
+        if (FUNCTION_NODE_TYPES.has(current.type)) {
+            let funcName = null;
+            let label = 'Function';
+            if (['function_declaration', 'function_definition', 'async_function_declaration',
+                'generator_function_declaration', 'function_item'].includes(current.type)) {
+                const nameNode = current.childForFieldName?.('name') ||
+                    current.children?.find((c) => c.type === 'identifier' || c.type === 'property_identifier');
+                funcName = nameNode?.text;
+            }
+            else if (current.type === 'impl_item') {
+                const funcItem = current.children?.find((c) => c.type === 'function_item');
+                if (funcItem) {
+                    const nameNode = funcItem.childForFieldName?.('name') ||
+                        funcItem.children?.find((c) => c.type === 'identifier');
+                    funcName = nameNode?.text;
+                    label = 'Method';
+                }
+            }
+            else if (current.type === 'method_definition') {
+                const nameNode = current.childForFieldName?.('name') ||
+                    current.children?.find((c) => c.type === 'property_identifier');
+                funcName = nameNode?.text;
+                label = 'Method';
+            }
+            else if (current.type === 'method_declaration' || current.type === 'constructor_declaration') {
+                const nameNode = current.childForFieldName?.('name') ||
+                    current.children?.find((c) => c.type === 'identifier');
+                funcName = nameNode?.text;
+                label = 'Method';
+            }
+            else if (current.type === 'arrow_function' || current.type === 'function_expression') {
+                const parent = current.parent;
+                if (parent?.type === 'variable_declarator') {
+                    const nameNode = parent.childForFieldName?.('name') ||
+                        parent.children?.find((c) => c.type === 'identifier');
+                    funcName = nameNode?.text;
+                }
+            }
+            if (funcName) {
+                return generateId(label, `${filePath}:${funcName}`);
+            }
+        }
+        current = current.parent;
+    }
+    return null;
+};
+const BUILT_INS = new Set([
+    'console', 'log', 'warn', 'error', 'info', 'debug',
+    'setTimeout', 'setInterval', 'clearTimeout', 'clearInterval',
+    'parseInt', 'parseFloat', 'isNaN', 'isFinite',
+    'encodeURI', 'decodeURI', 'encodeURIComponent', 'decodeURIComponent',
+    'JSON', 'parse', 'stringify',
+    'Object', 'Array', 'String', 'Number', 'Boolean', 'Symbol', 'BigInt',
+    'Map', 'Set', 'WeakMap', 'WeakSet',
+    'Promise', 'resolve', 'reject', 'then', 'catch', 'finally',
+    'Math', 'Date', 'RegExp', 'Error',
+    'require', 'import', 'export', 'fetch', 'Response', 'Request',
+    'useState', 'useEffect', 'useCallback', 'useMemo', 'useRef', 'useContext',
+    'useReducer', 'useLayoutEffect', 'useImperativeHandle', 'useDebugValue',
+    'createElement', 'createContext', 'createRef', 'forwardRef', 'memo', 'lazy',
+    'map', 'filter', 'reduce', 'forEach', 'find', 'findIndex', 'some', 'every',
+    'includes', 'indexOf', 'slice', 'splice', 'concat', 'join', 'split',
+    'push', 'pop', 'shift', 'unshift', 'sort', 'reverse',
+    'keys', 'values', 'entries', 'assign', 'freeze', 'seal',
+    'hasOwnProperty', 'toString', 'valueOf',
+    'print', 'len', 'range', 'str', 'int', 'float', 'list', 'dict', 'set', 'tuple',
+    'open', 'read', 'write', 'close', 'append', 'extend', 'update',
+    'super', 'type', 'isinstance', 'issubclass', 'getattr', 'setattr', 'hasattr',
+    'enumerate', 'zip', 'sorted', 'reversed', 'min', 'max', 'sum', 'abs',
+]);
+// ============================================================================
+// Label detection from capture map
+// ============================================================================
+const getLabelFromCaptures = (captureMap) => {
+    // Skip imports (handled separately) and calls
+    if (captureMap['import'] || captureMap['call'])
+        return null;
+    if (!captureMap['name'])
+        return null;
+    if (captureMap['definition.function'])
+        return 'Function';
+    if (captureMap['definition.class'])
+        return 'Class';
+    if (captureMap['definition.interface'])
+        return 'Interface';
+    if (captureMap['definition.method'])
+        return 'Method';
+    if (captureMap['definition.struct'])
+        return 'Struct';
+    if (captureMap['definition.enum'])
+        return 'Enum';
+    if (captureMap['definition.namespace'])
+        return 'Namespace';
+    if (captureMap['definition.module'])
+        return 'Module';
+    if (captureMap['definition.trait'])
+        return 'Trait';
+    if (captureMap['definition.impl'])
+        return 'Impl';
+    if (captureMap['definition.type'])
+        return 'TypeAlias';
+    if (captureMap['definition.const'])
+        return 'Const';
+    if (captureMap['definition.static'])
+        return 'Static';
+    if (captureMap['definition.typedef'])
+        return 'Typedef';
+    if (captureMap['definition.macro'])
+        return 'Macro';
+    if (captureMap['definition.union'])
+        return 'Union';
+    if (captureMap['definition.property'])
+        return 'Property';
+    if (captureMap['definition.record'])
+        return 'Record';
+    if (captureMap['definition.delegate'])
+        return 'Delegate';
+    if (captureMap['definition.annotation'])
+        return 'Annotation';
+    if (captureMap['definition.constructor'])
+        return 'Constructor';
+    if (captureMap['definition.template'])
+        return 'Template';
+    return 'CodeElement';
+};
+// ============================================================================
+// Process a batch of files
+// ============================================================================
+const processBatch = (files, onProgress) => {
+    const result = {
+        nodes: [],
+        relationships: [],
+        symbols: [],
+        imports: [],
+        calls: [],
+        heritage: [],
+        fileCount: 0,
+    };
+    // Group by language to minimize setLanguage calls
+    const byLanguage = new Map();
+    for (const file of files) {
+        const lang = getLanguageFromFilename(file.path);
+        if (!lang)
+            continue;
+        let list = byLanguage.get(lang);
+        if (!list) {
+            list = [];
+            byLanguage.set(lang, list);
+        }
+        list.push(file);
+    }
+    let totalProcessed = 0;
+    let lastReported = 0;
+    const PROGRESS_INTERVAL = 100; // report every 100 files
+    const onFileProcessed = onProgress ? () => {
+        totalProcessed++;
+        if (totalProcessed - lastReported >= PROGRESS_INTERVAL) {
+            lastReported = totalProcessed;
+            onProgress(totalProcessed);
+        }
+    } : undefined;
+    for (const [language, langFiles] of byLanguage) {
+        const queryString = LANGUAGE_QUERIES[language];
+        if (!queryString)
+            continue;
+        // Track if we need to handle tsx separately
+        const tsxFiles = [];
+        const regularFiles = [];
+        if (language === SupportedLanguages.TypeScript) {
+            for (const f of langFiles) {
+                if (f.path.endsWith('.tsx')) {
+                    tsxFiles.push(f);
+                }
+                else {
+                    regularFiles.push(f);
+                }
+            }
+        }
+        else {
+            regularFiles.push(...langFiles);
+        }
+        // Process regular files for this language
+        if (regularFiles.length > 0) {
+            setLanguage(language, regularFiles[0].path);
+            processFileGroup(regularFiles, language, queryString, result, onFileProcessed);
+        }
+        // Process tsx files separately (different grammar)
+        if (tsxFiles.length > 0) {
+            setLanguage(language, tsxFiles[0].path);
+            processFileGroup(tsxFiles, language, queryString, result, onFileProcessed);
+        }
+    }
+    return result;
+};
+const processFileGroup = (files, language, queryString, result, onFileProcessed) => {
+    let query;
+    try {
+        const lang = parser.getLanguage();
+        query = new Parser.Query(lang, queryString);
+    }
+    catch {
+        return;
+    }
+    for (const file of files) {
+        let tree;
+        try {
+            tree = parser.parse(file.content, undefined, { bufferSize: 1024 * 256 });
+        }
+        catch {
+            continue;
+        }
+        result.fileCount++;
+        onFileProcessed?.();
+        let matches;
+        try {
+            matches = query.matches(tree.rootNode);
+        }
+        catch {
+            continue;
+        }
+        for (const match of matches) {
+            const captureMap = {};
+            for (const c of match.captures) {
+                captureMap[c.name] = c.node;
+            }
+            // Extract import paths before skipping
+            if (captureMap['import'] && captureMap['import.source']) {
+                const rawImportPath = captureMap['import.source'].text.replace(/['"<>]/g, '');
+                result.imports.push({
+                    filePath: file.path,
+                    rawImportPath,
+                    language: language,
+                });
+                continue;
+            }
+            // Extract call sites
+            if (captureMap['call']) {
+                const callNameNode = captureMap['call.name'];
+                if (callNameNode) {
+                    const calledName = callNameNode.text;
+                    if (!BUILT_INS.has(calledName)) {
+                        const callNode = captureMap['call'];
+                        const sourceId = findEnclosingFunctionId(callNode, file.path)
+                            || generateId('File', file.path);
+                        result.calls.push({ filePath: file.path, calledName, sourceId });
+                    }
+                }
+                continue;
+            }
+            // Extract heritage (extends/implements)
+            if (captureMap['heritage.class']) {
+                if (captureMap['heritage.extends']) {
+                    result.heritage.push({
+                        filePath: file.path,
+                        className: captureMap['heritage.class'].text,
+                        parentName: captureMap['heritage.extends'].text,
+                        kind: 'extends',
+                    });
+                }
+                if (captureMap['heritage.implements']) {
+                    result.heritage.push({
+                        filePath: file.path,
+                        className: captureMap['heritage.class'].text,
+                        parentName: captureMap['heritage.implements'].text,
+                        kind: 'implements',
+                    });
+                }
+                if (captureMap['heritage.trait']) {
+                    result.heritage.push({
+                        filePath: file.path,
+                        className: captureMap['heritage.class'].text,
+                        parentName: captureMap['heritage.trait'].text,
+                        kind: 'trait-impl',
+                    });
+                }
+                if (captureMap['heritage.extends'] || captureMap['heritage.implements'] || captureMap['heritage.trait']) {
+                    continue;
+                }
+            }
+            const nodeLabel = getLabelFromCaptures(captureMap);
+            if (!nodeLabel)
+                continue;
+            const nameNode = captureMap['name'];
+            const nodeName = nameNode.text;
+            const nodeId = generateId(nodeLabel, `${file.path}:${nodeName}`);
+            result.nodes.push({
+                id: nodeId,
+                label: nodeLabel,
+                properties: {
+                    name: nodeName,
+                    filePath: file.path,
+                    startLine: nameNode.startPosition.row,
+                    endLine: nameNode.endPosition.row,
+                    language: language,
+                    isExported: isNodeExported(nameNode, nodeName, language),
+                },
+            });
+            result.symbols.push({
+                filePath: file.path,
+                name: nodeName,
+                nodeId,
+                type: nodeLabel,
+            });
+            const fileId = generateId('File', file.path);
+            const relId = generateId('DEFINES', `${fileId}->${nodeId}`);
+            result.relationships.push({
+                id: relId,
+                sourceId: fileId,
+                targetId: nodeId,
+                type: 'DEFINES',
+                confidence: 1.0,
+                reason: '',
+            });
+        }
+    }
+};
+// ============================================================================
+// Worker message handler
+// ============================================================================
+parentPort.on('message', (files) => {
+    const result = processBatch(files, (filesProcessed) => {
+        parentPort.postMessage({ type: 'progress', filesProcessed });
+    });
+    parentPort.postMessage({ type: 'result', data: result });
+});

package/dist/core/ingestion/workers/worker-pool.d.ts

@@ -0,0 +1,22 @@
+export interface WorkerPool {
+    /**
+     * Dispatch items across workers. Items are split into chunks (one per worker),
+     * each worker processes its chunk, and results are concatenated back in order.
+     *
+     * @param onProgress - Called with cumulative files processed across all workers
+     */
+    dispatch<TInput, TResult>(items: TInput[], onProgress?: (filesProcessed: number) => void): Promise<TResult[]>;
+    /**
+     * Terminate all workers. Must be called when done.
+     */
+    terminate(): Promise<void>;
+    /** Number of workers in the pool */
+    readonly size: number;
+}
+/**
+ * Create a pool of worker threads.
+ *
+ * @param workerUrl - URL to the worker script (use `new URL('./parse-worker.js', import.meta.url)`)
+ * @param poolSize - Number of workers (defaults to cpus - 1, minimum 1)
+ */
+export declare const createWorkerPool: (workerUrl: URL, poolSize?: number) => WorkerPool;

package/dist/core/ingestion/workers/worker-pool.js

@@ -0,0 +1,65 @@
+import { Worker } from 'node:worker_threads';
+import os from 'node:os';
+/**
+ * Create a pool of worker threads.
+ *
+ * @param workerUrl - URL to the worker script (use `new URL('./parse-worker.js', import.meta.url)`)
+ * @param poolSize - Number of workers (defaults to cpus - 1, minimum 1)
+ */
+export const createWorkerPool = (workerUrl, poolSize) => {
+    const size = poolSize ?? Math.max(1, os.cpus().length - 1);
+    const workers = [];
+    for (let i = 0; i < size; i++) {
+        workers.push(new Worker(workerUrl));
+    }
+    const dispatch = (items, onProgress) => {
+        if (items.length === 0)
+            return Promise.resolve([]);
+        // Split items into one chunk per worker
+        const chunkSize = Math.ceil(items.length / size);
+        const chunks = [];
+        for (let i = 0; i < items.length; i += chunkSize) {
+            chunks.push(items.slice(i, i + chunkSize));
+        }
+        // Track per-worker progress for cumulative reporting
+        const workerProgress = new Array(chunks.length).fill(0);
+        // Send one chunk to each worker, collect results
+        const promises = chunks.map((chunk, i) => {
+            const worker = workers[i];
+            return new Promise((resolve, reject) => {
+                const handler = (msg) => {
+                    if (msg && msg.type === 'progress') {
+                        // Intermediate progress from worker
+                        workerProgress[i] = msg.filesProcessed;
+                        if (onProgress) {
+                            const total = workerProgress.reduce((a, b) => a + b, 0);
+                            onProgress(total);
+                        }
+                    }
+                    else if (msg && msg.type === 'result') {
+                        // Final result
+                        worker.removeListener('message', handler);
+                        resolve(msg.data);
+                    }
+                    else {
+                        // Legacy: treat any non-typed message as result (backward compat)
+                        worker.removeListener('message', handler);
+                        resolve(msg);
+                    }
+                };
+                worker.on('message', handler);
+                worker.once('error', (err) => {
+                    worker.removeListener('message', handler);
+                    reject(err);
+                });
+                worker.postMessage(chunk);
+            });
+        });
+        return Promise.all(promises);
+    };
+    const terminate = async () => {
+        await Promise.all(workers.map(w => w.terminate()));
+        workers.length = 0;
+    };
+    return { dispatch, terminate, size };
+};

package/dist/core/kuzu/kuzu-adapter.d.ts

@@ -4,10 +4,12 @@ export declare const initKuzu: (dbPath: string) => Promise<{
     db: kuzu.Database;
     conn: kuzu.Connection;
 }>;
-export declare const loadGraphToKuzu: (graph: KnowledgeGraph, fileContents: Map<string, string>, storagePath: string) => Promise<{
+export type KuzuProgressCallback = (message: string) => void;
+export declare const loadGraphToKuzu: (graph: KnowledgeGraph, fileContents: Map<string, string>, storagePath: string, onProgress?: KuzuProgressCallback) => Promise<{
     success: boolean;
     insertedRels: number;
     skippedRels: number;
+    warnings: string[];
 }>;
 /**
  * Insert a single node to KuzuDB
@@ -35,6 +37,18 @@ export declare const getKuzuStats: () => Promise<{
     nodes: number;
     edges: number;
 }>;
+/**
+ * Load cached embeddings from KuzuDB before a rebuild.
+ * Returns all embedding vectors so they can be re-inserted after the graph is reloaded,
+ * avoiding expensive re-embedding of unchanged nodes.
+ */
+export declare const loadCachedEmbeddings: () => Promise<{
+    embeddingNodeIds: Set<string>;
+    embeddings: Array<{
+        nodeId: string;
+        embedding: number[];
+    }>;
+}>;
 export declare const closeKuzu: () => Promise<void>;
 export declare const isKuzuReady: () => boolean;
 /**