gitnexus 1.2.8 → 1.3.0

package/dist/core/kuzu/csv-generator.js

@@ -1,59 +1,47 @@
 /**
  * CSV Generator for KuzuDB Hybrid Schema
  *
- * Generates separate CSV files for each node table and one relation CSV.
- * This enables efficient bulk loading via COPY FROM for hybrid schema.
+ * Streams CSV rows directly to disk files in a single pass over graph nodes.
+ * File contents are lazy-read from disk per-node to avoid holding the entire
+ * repo in RAM. Rows are buffered (FLUSH_EVERY) before writing to minimize
+ * per-row Promise overhead.
  *
  * RFC 4180 Compliant:
  * - Fields containing commas, double quotes, or newlines are enclosed in double quotes
  * - Double quotes within fields are escaped by doubling them ("")
  * - All fields are consistently quoted for safety with code content
  */
+import fs from 'fs/promises';
+import { createWriteStream } from 'fs';
+import path from 'path';
+/** Flush buffered rows to disk every N rows */
+const FLUSH_EVERY = 500;
 // ============================================================================
 // CSV ESCAPE UTILITIES
 // ============================================================================
-/**
- * Sanitize string to ensure valid UTF-8 and safe CSV content for KuzuDB
- * Removes or replaces invalid characters that would break CSV parsing.
- *
- * Critical: KuzuDB's native CSV parser on Windows can misinterpret \r\n
- * inside quoted fields. We normalize all line endings to \n only.
- */
 const sanitizeUTF8 = (str) => {
     return str
-        .replace(/\r\n/g, '\n') // Normalize Windows line endings first
-        .replace(/\r/g, '\n') // Normalize remaining \r to \n
-        .replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, '') // Remove control chars except \t \n
-        .replace(/[\uD800-\uDFFF]/g, '') // Remove surrogate pairs (invalid standalone)
-        .replace(/[\uFFFE\uFFFF]/g, ''); // Remove BOM and special chars
+        .replace(/\r\n/g, '\n')
+        .replace(/\r/g, '\n')
+        .replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, '')
+        .replace(/[\uD800-\uDFFF]/g, '')
+        .replace(/[\uFFFE\uFFFF]/g, '');
 };
-/**
- * RFC 4180 compliant CSV field escaping
- * ALWAYS wraps in double quotes for safety with code content
- */
 const escapeCSVField = (value) => {
-    if (value === undefined || value === null) {
+    if (value === undefined || value === null)
         return '""';
-    }
     let str = String(value);
     str = sanitizeUTF8(str);
     return `"${str.replace(/"/g, '""')}"`;
 };
-/**
- * Escape a numeric value (no quotes needed for numbers)
- */
 const escapeCSVNumber = (value, defaultValue = -1) => {
-    if (value === undefined || value === null) {
+    if (value === undefined || value === null)
         return String(defaultValue);
-    }
     return String(value);
 };
 // ============================================================================
-// CONTENT EXTRACTION
+// CONTENT EXTRACTION (lazy — reads from disk on demand)
 // ============================================================================
-/**
- * Check if content looks like binary data
- */
 const isBinaryContent = (content) => {
     if (!content || content.length === 0)
         return false;
@@ -61,181 +49,261 @@ const isBinaryContent = (content) => {
     let nonPrintable = 0;
     for (let i = 0; i < sample.length; i++) {
         const code = sample.charCodeAt(i);
-        if ((code < 9) || (code > 13 && code < 32) || code === 127) {
+        if ((code < 9) || (code > 13 && code < 32) || code === 127)
             nonPrintable++;
-        }
     }
     return (nonPrintable / sample.length) > 0.1;
 };
 /**
- * Extract code content for a node
+ * LRU content cache — avoids re-reading the same source file for every
+ * symbol defined in it. Sized generously so most files stay cached during
+ * the single-pass node iteration.
  */
-const extractContent = (node, fileContents) => {
+class FileContentCache {
+    cache = new Map();
+    accessOrder = [];
+    maxSize;
+    repoPath;
+    constructor(repoPath, maxSize = 3000) {
+        this.repoPath = repoPath;
+        this.maxSize = maxSize;
+    }
+    async get(relativePath) {
+        if (!relativePath)
+            return '';
+        const cached = this.cache.get(relativePath);
+        if (cached !== undefined)
+            return cached;
+        try {
+            const fullPath = path.join(this.repoPath, relativePath);
+            const content = await fs.readFile(fullPath, 'utf-8');
+            this.set(relativePath, content);
+            return content;
+        }
+        catch {
+            this.set(relativePath, '');
+            return '';
+        }
+    }
+    set(key, value) {
+        if (this.cache.size >= this.maxSize) {
+            const oldest = this.accessOrder.shift();
+            if (oldest)
+                this.cache.delete(oldest);
+        }
+        this.cache.set(key, value);
+        this.accessOrder.push(key);
+    }
+}
+const extractContent = async (node, contentCache) => {
     const filePath = node.properties.filePath;
-    const content = fileContents.get(filePath);
+    const content = await contentCache.get(filePath);
     if (!content)
         return '';
     if (node.label === 'Folder')
         return '';
     if (isBinaryContent(content))
         return '[Binary file - content not stored]';
-    // For File nodes, return content (limited)
     if (node.label === 'File') {
         const MAX_FILE_CONTENT = 10000;
-        if (content.length > MAX_FILE_CONTENT) {
-            return content.slice(0, MAX_FILE_CONTENT) + '\n... [truncated]';
-        }
-        return content;
+        return content.length > MAX_FILE_CONTENT
+            ? content.slice(0, MAX_FILE_CONTENT) + '\n... [truncated]'
+            : content;
     }
-    // For code elements, extract the relevant lines with context
     const startLine = node.properties.startLine;
     const endLine = node.properties.endLine;
     if (startLine === undefined || endLine === undefined)
         return '';
     const lines = content.split('\n');
-    const contextLines = 2;
-    const start = Math.max(0, startLine - contextLines);
-    const end = Math.min(lines.length - 1, endLine + contextLines);
+    const start = Math.max(0, startLine - 2);
+    const end = Math.min(lines.length - 1, endLine + 2);
     const snippet = lines.slice(start, end + 1).join('\n');
     const MAX_SNIPPET = 5000;
-    if (snippet.length > MAX_SNIPPET) {
-        return snippet.slice(0, MAX_SNIPPET) + '\n... [truncated]';
-    }
-    return snippet;
+    return snippet.length > MAX_SNIPPET
+        ? snippet.slice(0, MAX_SNIPPET) + '\n... [truncated]'
+        : snippet;
 };
 // ============================================================================
-// NODE CSV GENERATORS
+// BUFFERED CSV WRITER
 // ============================================================================
-/**
- * Generate CSV for File nodes
- * Headers: id,name,filePath,content
- */
-const generateFileCSV = (nodes, fileContents) => {
-    const headers = ['id', 'name', 'filePath', 'content'];
-    const rows = [headers.join(',')];
-    const seenIds = new Set();
-    for (const node of nodes) {
-        if (node.label !== 'File')
-            continue;
-        // Skip duplicates
-        if (seenIds.has(node.id))
-            continue;
-        seenIds.add(node.id);
-        const content = extractContent(node, fileContents);
-        rows.push([
-            escapeCSVField(node.id),
-            escapeCSVField(node.properties.name || ''),
-            escapeCSVField(node.properties.filePath || ''),
-            escapeCSVField(content),
-        ].join(','));
+class BufferedCSVWriter {
+    ws;
+    buffer = [];
+    rows = 0;
+    constructor(filePath, header) {
+        this.ws = createWriteStream(filePath, 'utf-8');
+        // Large repos flush many times — raise listener cap to avoid MaxListenersExceededWarning
+        this.ws.setMaxListeners(50);
+        this.buffer.push(header);
     }
-    return rows.join('\n');
-};
-/**
- * Generate CSV for Folder nodes
- * Headers: id,name,filePath
- */
-const generateFolderCSV = (nodes) => {
-    const headers = ['id', 'name', 'filePath'];
-    const rows = [headers.join(',')];
-    for (const node of nodes) {
-        if (node.label !== 'Folder')
-            continue;
-        rows.push([
-            escapeCSVField(node.id),
-            escapeCSVField(node.properties.name || ''),
-            escapeCSVField(node.properties.filePath || ''),
-        ].join(','));
+    addRow(row) {
+        this.buffer.push(row);
+        this.rows++;
+        if (this.buffer.length >= FLUSH_EVERY) {
+            return this.flush();
+        }
+        return Promise.resolve();
     }
-    return rows.join('\n');
-};
-/**
- * Generate CSV for code element nodes (Function, Class, Interface, Method, CodeElement)
- * Headers: id,name,filePath,startLine,endLine,isExported,content
- */
-const generateCodeElementCSV = (nodes, label, fileContents) => {
-    const headers = ['id', 'name', 'filePath', 'startLine', 'endLine', 'isExported', 'content'];
-    const rows = [headers.join(',')];
-    for (const node of nodes) {
-        if (node.label !== label)
-            continue;
-        const content = extractContent(node, fileContents);
-        rows.push([
-            escapeCSVField(node.id),
-            escapeCSVField(node.properties.name || ''),
-            escapeCSVField(node.properties.filePath || ''),
-            escapeCSVNumber(node.properties.startLine, -1),
-            escapeCSVNumber(node.properties.endLine, -1),
-            node.properties.isExported ? 'true' : 'false',
-            escapeCSVField(content),
-        ].join(','));
+    flush() {
+        if (this.buffer.length === 0)
+            return Promise.resolve();
+        const chunk = this.buffer.join('\n') + '\n';
+        this.buffer.length = 0;
+        return new Promise((resolve, reject) => {
+            const ok = this.ws.write(chunk);
+            if (ok)
+                resolve();
+            else
+                this.ws.once('drain', resolve);
+        });
     }
-    return rows.join('\n');
-};
-/**
- * Generate CSV for Community nodes (from Leiden algorithm)
- * Headers: id,label,heuristicLabel,keywords,description,enrichedBy,cohesion,symbolCount
- */
-const generateCommunityCSV = (nodes) => {
-    const headers = ['id', 'label', 'heuristicLabel', 'keywords', 'description', 'enrichedBy', 'cohesion', 'symbolCount'];
-    const rows = [headers.join(',')];
-    for (const node of nodes) {
-        if (node.label !== 'Community')
-            continue;
-        // Handle keywords array - convert to KuzuDB array format
-        const keywords = node.properties.keywords || [];
-        const keywordsStr = `[${keywords.map((k) => `'${k.replace(/'/g, "''")}'`).join(',')}]`;
-        rows.push([
-            escapeCSVField(node.id),
-            escapeCSVField(node.properties.name || ''), // label is stored in name
-            escapeCSVField(node.properties.heuristicLabel || ''),
-            keywordsStr, // Array format for KuzuDB
-            escapeCSVField(node.properties.description || ''),
-            escapeCSVField(node.properties.enrichedBy || 'heuristic'),
-            escapeCSVNumber(node.properties.cohesion, 0),
-            escapeCSVNumber(node.properties.symbolCount, 0),
-        ].join(','));
+    async finish() {
+        await this.flush();
+        return new Promise((resolve, reject) => {
+            this.ws.end(() => resolve());
+            this.ws.on('error', reject);
+        });
     }
-    return rows.join('\n');
-};
+}
 /**
- * Generate CSV for Process nodes
- * Headers: id,label,heuristicLabel,processType,stepCount,communities,entryPointId,terminalId
+ * Stream all CSV data directly to disk files.
+ * Iterates graph nodes exactly ONCE — routes each node to the right writer.
+ * File contents are lazy-read from disk with a generous LRU cache.
  */
-const generateProcessCSV = (nodes) => {
-    const headers = ['id', 'label', 'heuristicLabel', 'processType', 'stepCount', 'communities', 'entryPointId', 'terminalId'];
-    const rows = [headers.join(',')];
-    for (const node of nodes) {
-        if (node.label !== 'Process')
-            continue;
-        // Handle communities array (string[])
-        const communities = node.properties.communities || [];
-        const communitiesStr = `[${communities.map((c) => `'${c.replace(/'/g, "''")}'`).join(',')}]`;
-        rows.push([
-            escapeCSVField(node.id),
-            escapeCSVField(node.properties.name || ''), // label stores name
-            escapeCSVField(node.properties.heuristicLabel || ''),
-            escapeCSVField(node.properties.processType || ''),
-            escapeCSVNumber(node.properties.stepCount, 0),
-            escapeCSVField(communitiesStr), // Needs CSV escaping because it contains commas!
-            escapeCSVField(node.properties.entryPointId || ''),
-            escapeCSVField(node.properties.terminalId || ''),
-        ].join(','));
+export const streamAllCSVsToDisk = async (graph, repoPath, csvDir) => {
+    // Remove stale CSVs from previous crashed runs, then recreate
+    try {
+        await fs.rm(csvDir, { recursive: true, force: true });
     }
-    return rows.join('\n');
-};
-/**
- * Generate CSV for the single CodeRelation table
- * Headers: from,to,type,confidence,reason
- *
- * confidence: 0-1 score for CALLS edges (how sure are we about the target?)
- * reason: 'import-resolved' | 'same-file' | 'fuzzy-global' (or empty for non-CALLS)
- */
-const generateRelationCSV = (graph) => {
-    const headers = ['from', 'to', 'type', 'confidence', 'reason', 'step'];
-    const rows = [headers.join(',')];
-    for (const rel of graph.relationships) {
-        rows.push([
+    catch { }
+    await fs.mkdir(csvDir, { recursive: true });
+    // We open ~30 concurrent write-streams; raise process limit to suppress
+    // MaxListenersExceededWarning (restored after all streams finish).
+    const prevMax = process.getMaxListeners();
+    process.setMaxListeners(prevMax + 40);
+    const contentCache = new FileContentCache(repoPath);
+    // Create writers for every node type up-front
+    const fileWriter = new BufferedCSVWriter(path.join(csvDir, 'file.csv'), 'id,name,filePath,content');
+    const folderWriter = new BufferedCSVWriter(path.join(csvDir, 'folder.csv'), 'id,name,filePath');
+    const codeElementHeader = 'id,name,filePath,startLine,endLine,isExported,content';
+    const functionWriter = new BufferedCSVWriter(path.join(csvDir, 'function.csv'), codeElementHeader);
+    const classWriter = new BufferedCSVWriter(path.join(csvDir, 'class.csv'), codeElementHeader);
+    const interfaceWriter = new BufferedCSVWriter(path.join(csvDir, 'interface.csv'), codeElementHeader);
+    const methodWriter = new BufferedCSVWriter(path.join(csvDir, 'method.csv'), codeElementHeader);
+    const codeElemWriter = new BufferedCSVWriter(path.join(csvDir, 'codeelement.csv'), codeElementHeader);
+    const communityWriter = new BufferedCSVWriter(path.join(csvDir, 'community.csv'), 'id,label,heuristicLabel,keywords,description,enrichedBy,cohesion,symbolCount');
+    const processWriter = new BufferedCSVWriter(path.join(csvDir, 'process.csv'), 'id,label,heuristicLabel,processType,stepCount,communities,entryPointId,terminalId');
+    // Multi-language node types share the same CSV shape (no isExported column)
+    const multiLangHeader = 'id,name,filePath,startLine,endLine,content';
+    const MULTI_LANG_TYPES = ['Struct', 'Enum', 'Macro', 'Typedef', 'Union', 'Namespace', 'Trait', 'Impl',
+        'TypeAlias', 'Const', 'Static', 'Property', 'Record', 'Delegate', 'Annotation', 'Constructor', 'Template', 'Module'];
+    const multiLangWriters = new Map();
+    for (const t of MULTI_LANG_TYPES) {
+        multiLangWriters.set(t, new BufferedCSVWriter(path.join(csvDir, `${t.toLowerCase()}.csv`), multiLangHeader));
+    }
+    const codeWriterMap = {
+        'Function': functionWriter,
+        'Class': classWriter,
+        'Interface': interfaceWriter,
+        'Method': methodWriter,
+        'CodeElement': codeElemWriter,
+    };
+    const seenFileIds = new Set();
+    // --- SINGLE PASS over all nodes ---
+    for (const node of graph.iterNodes()) {
+        switch (node.label) {
+            case 'File': {
+                if (seenFileIds.has(node.id))
+                    break;
+                seenFileIds.add(node.id);
+                const content = await extractContent(node, contentCache);
+                await fileWriter.addRow([
+                    escapeCSVField(node.id),
+                    escapeCSVField(node.properties.name || ''),
+                    escapeCSVField(node.properties.filePath || ''),
+                    escapeCSVField(content),
+                ].join(','));
+                break;
+            }
+            case 'Folder':
+                await folderWriter.addRow([
+                    escapeCSVField(node.id),
+                    escapeCSVField(node.properties.name || ''),
+                    escapeCSVField(node.properties.filePath || ''),
+                ].join(','));
+                break;
+            case 'Community': {
+                const keywords = node.properties.keywords || [];
+                const keywordsStr = `[${keywords.map((k) => `'${k.replace(/'/g, "''")}'`).join(',')}]`;
+                await communityWriter.addRow([
+                    escapeCSVField(node.id),
+                    escapeCSVField(node.properties.name || ''),
+                    escapeCSVField(node.properties.heuristicLabel || ''),
+                    keywordsStr,
+                    escapeCSVField(node.properties.description || ''),
+                    escapeCSVField(node.properties.enrichedBy || 'heuristic'),
+                    escapeCSVNumber(node.properties.cohesion, 0),
+                    escapeCSVNumber(node.properties.symbolCount, 0),
+                ].join(','));
+                break;
+            }
+            case 'Process': {
+                const communities = node.properties.communities || [];
+                const communitiesStr = `[${communities.map((c) => `'${c.replace(/'/g, "''")}'`).join(',')}]`;
+                await processWriter.addRow([
+                    escapeCSVField(node.id),
+                    escapeCSVField(node.properties.name || ''),
+                    escapeCSVField(node.properties.heuristicLabel || ''),
+                    escapeCSVField(node.properties.processType || ''),
+                    escapeCSVNumber(node.properties.stepCount, 0),
+                    escapeCSVField(communitiesStr),
+                    escapeCSVField(node.properties.entryPointId || ''),
+                    escapeCSVField(node.properties.terminalId || ''),
+                ].join(','));
+                break;
+            }
+            default: {
+                // Code element nodes (Function, Class, Interface, Method, CodeElement)
+                const writer = codeWriterMap[node.label];
+                if (writer) {
+                    const content = await extractContent(node, contentCache);
+                    await writer.addRow([
+                        escapeCSVField(node.id),
+                        escapeCSVField(node.properties.name || ''),
+                        escapeCSVField(node.properties.filePath || ''),
+                        escapeCSVNumber(node.properties.startLine, -1),
+                        escapeCSVNumber(node.properties.endLine, -1),
+                        node.properties.isExported ? 'true' : 'false',
+                        escapeCSVField(content),
+                    ].join(','));
+                }
+                else {
+                    // Multi-language node types (Struct, Impl, Trait, Macro, etc.)
+                    const mlWriter = multiLangWriters.get(node.label);
+                    if (mlWriter) {
+                        const content = await extractContent(node, contentCache);
+                        await mlWriter.addRow([
+                            escapeCSVField(node.id),
+                            escapeCSVField(node.properties.name || ''),
+                            escapeCSVField(node.properties.filePath || ''),
+                            escapeCSVNumber(node.properties.startLine, -1),
+                            escapeCSVNumber(node.properties.endLine, -1),
+                            escapeCSVField(content),
+                        ].join(','));
+                    }
+                }
+                break;
+            }
+        }
+    }
+    // Finish all node writers
+    const allWriters = [fileWriter, folderWriter, functionWriter, classWriter, interfaceWriter, methodWriter, codeElemWriter, communityWriter, processWriter, ...multiLangWriters.values()];
+    await Promise.all(allWriters.map(w => w.finish()));
+    // --- Stream relationship CSV ---
+    const relCsvPath = path.join(csvDir, 'relations.csv');
+    const relWriter = new BufferedCSVWriter(relCsvPath, 'from,to,type,confidence,reason,step');
+    for (const rel of graph.iterRelationships()) {
+        await relWriter.addRow([
             escapeCSVField(rel.sourceId),
             escapeCSVField(rel.targetId),
             escapeCSVField(rel.type),
@@ -244,29 +312,23 @@ const generateRelationCSV = (graph) => {
             escapeCSVNumber(rel.step, 0),
         ].join(','));
     }
-    return rows.join('\n');
-};
-// ============================================================================
-// MAIN CSV GENERATION FUNCTION
-// ============================================================================
-/**
- * Generate all CSV data for hybrid schema bulk loading
- * Returns Maps of node table name -> CSV content, and single relation CSV
- */
-export const generateAllCSVs = (graph, fileContents) => {
-    const nodes = Array.from(graph.nodes);
-    // Generate node CSVs
-    const nodeCSVs = new Map();
-    nodeCSVs.set('File', generateFileCSV(nodes, fileContents));
-    nodeCSVs.set('Folder', generateFolderCSV(nodes));
-    nodeCSVs.set('Function', generateCodeElementCSV(nodes, 'Function', fileContents));
-    nodeCSVs.set('Class', generateCodeElementCSV(nodes, 'Class', fileContents));
-    nodeCSVs.set('Interface', generateCodeElementCSV(nodes, 'Interface', fileContents));
-    nodeCSVs.set('Method', generateCodeElementCSV(nodes, 'Method', fileContents));
-    nodeCSVs.set('CodeElement', generateCodeElementCSV(nodes, 'CodeElement', fileContents));
-    nodeCSVs.set('Community', generateCommunityCSV(nodes));
-    nodeCSVs.set('Process', generateProcessCSV(nodes));
-    // Generate single relation CSV
-    const relCSV = generateRelationCSV(graph);
-    return { nodes: nodeCSVs, relCSV };
+    await relWriter.finish();
+    // Build result map — only include tables that have rows
+    const nodeFiles = new Map();
+    const tableMap = [
+        ['File', fileWriter], ['Folder', folderWriter],
+        ['Function', functionWriter], ['Class', classWriter],
+        ['Interface', interfaceWriter], ['Method', methodWriter],
+        ['CodeElement', codeElemWriter],
+        ['Community', communityWriter], ['Process', processWriter],
+        ...Array.from(multiLangWriters.entries()).map(([name, w]) => [name, w]),
+    ];
+    for (const [name, writer] of tableMap) {
+        if (writer.rows > 0) {
+            nodeFiles.set(name, { csvPath: path.join(csvDir, `${name.toLowerCase()}.csv`), rows: writer.rows });
+        }
+    }
+    // Restore original process listener limit
+    process.setMaxListeners(prevMax);
+    return { nodeFiles, relCsvPath, relRows: relWriter.rows };
 };

package/dist/core/kuzu/kuzu-adapter.d.ts

@@ -5,7 +5,7 @@ export declare const initKuzu: (dbPath: string) => Promise<{
     conn: kuzu.Connection;
 }>;
 export type KuzuProgressCallback = (message: string) => void;
-export declare const loadGraphToKuzu: (graph: KnowledgeGraph, fileContents: Map<string, string>, storagePath: string, onProgress?: KuzuProgressCallback) => Promise<{
+export declare const loadGraphToKuzu: (graph: KnowledgeGraph, repoPath: string, storagePath: string, onProgress?: KuzuProgressCallback) => Promise<{
     success: boolean;
     insertedRels: number;
     skippedRels: number;
@@ -61,9 +61,6 @@ export declare const deleteNodesForFile: (filePath: string, dbPath?: string) =>
     deletedNodes: number;
 }>;
 export declare const getEmbeddingTableName: () => string;
-/**
- * Load the FTS extension (required before using FTS functions)
- */
 export declare const loadFTSExtension: () => Promise<void>;
 /**
  * Create a full-text search index on a table