@soulcraft/brainy 4.1.3 → 4.2.0

package/dist/import/InstancePool.js

@@ -0,0 +1,231 @@
+/**
+ * InstancePool - Shared instance management for memory efficiency
+ *
+ * Production-grade instance pooling to prevent memory leaks during imports.
+ * Critical for scaling to billions of entities.
+ *
+ * Problem: Creating new NLP/Extractor instances in loops → memory leak
+ * Solution: Reuse shared instances across entire import session
+ *
+ * Memory savings:
+ * - Without pooling: 100K rows × 50MB per instance = 5TB RAM (OOM!)
+ * - With pooling: 50MB total (shared across all rows)
+ */
+import { NaturalLanguageProcessor } from '../neural/naturalLanguageProcessor.js';
+import { NeuralEntityExtractor } from '../neural/entityExtractor.js';
+/**
+ * InstancePool - Manages shared instances for memory efficiency
+ *
+ * Lifecycle:
+ * 1. Create pool at import start
+ * 2. Reuse instances across all rows
+ * 3. Pool is garbage collected when import completes
+ *
+ * Thread safety: Not thread-safe (single import session per pool)
+ */
+export class InstancePool {
+    constructor(brain) {
+        // Shared instances (created lazily)
+        this.nlpInstance = null;
+        this.extractorInstance = null;
+        // Initialization state
+        this.nlpInitialized = false;
+        this.initializationPromise = null;
+        // Statistics
+        this.stats = {
+            nlpReuses: 0,
+            extractorReuses: 0,
+            creationTime: 0
+        };
+        this.brain = brain;
+    }
+    /**
+     * Get shared NaturalLanguageProcessor instance
+     *
+     * Lazy initialization - created on first access
+     * All subsequent calls return same instance
+     *
+     * @returns Shared NLP instance
+     */
+    async getNLP() {
+        if (!this.nlpInstance) {
+            const startTime = Date.now();
+            this.nlpInstance = new NaturalLanguageProcessor(this.brain);
+            this.stats.creationTime += Date.now() - startTime;
+        }
+        // Ensure initialized before returning
+        if (!this.nlpInitialized) {
+            await this.ensureNLPInitialized();
+        }
+        this.stats.nlpReuses++;
+        return this.nlpInstance;
+    }
+    /**
+     * Get shared NeuralEntityExtractor instance
+     *
+     * Lazy initialization - created on first access
+     * All subsequent calls return same instance
+     *
+     * @returns Shared extractor instance
+     */
+    getExtractor() {
+        if (!this.extractorInstance) {
+            const startTime = Date.now();
+            this.extractorInstance = new NeuralEntityExtractor(this.brain);
+            this.stats.creationTime += Date.now() - startTime;
+        }
+        this.stats.extractorReuses++;
+        return this.extractorInstance;
+    }
+    /**
+     * Get shared NLP instance (synchronous, may return uninitialized)
+     *
+     * Use when you need NLP synchronously and will handle initialization yourself.
+     * Prefer getNLP() for async code.
+     *
+     * @returns Shared NLP instance (possibly uninitialized)
+     */
+    getNLPSync() {
+        if (!this.nlpInstance) {
+            this.nlpInstance = new NaturalLanguageProcessor(this.brain);
+        }
+        this.stats.nlpReuses++;
+        return this.nlpInstance;
+    }
+    /**
+     * Initialize all instances upfront
+     *
+     * Call at start of import to avoid lazy initialization overhead
+     * during processing. Improves predictability and first-row performance.
+     *
+     * @returns Promise that resolves when all instances are ready
+     */
+    async init() {
+        // Prevent duplicate initialization
+        if (this.initializationPromise) {
+            return this.initializationPromise;
+        }
+        this.initializationPromise = this.initializeInternal();
+        return this.initializationPromise;
+    }
+    /**
+     * Internal initialization implementation
+     */
+    async initializeInternal() {
+        const startTime = Date.now();
+        // Create instances
+        if (!this.nlpInstance) {
+            this.nlpInstance = new NaturalLanguageProcessor(this.brain);
+        }
+        if (!this.extractorInstance) {
+            this.extractorInstance = new NeuralEntityExtractor(this.brain);
+        }
+        // Initialize NLP (loads pattern library)
+        await this.ensureNLPInitialized();
+        this.stats.creationTime = Date.now() - startTime;
+    }
+    /**
+     * Ensure NLP is initialized (loads 220 patterns)
+     *
+     * Handles concurrent initialization requests safely
+     */
+    async ensureNLPInitialized() {
+        if (this.nlpInitialized) {
+            return;
+        }
+        if (!this.nlpInstance) {
+            throw new Error('NLP instance not created yet');
+        }
+        await this.nlpInstance.init();
+        this.nlpInitialized = true;
+    }
+    /**
+     * Check if instances are initialized
+     *
+     * @returns True if NLP is initialized and ready to use
+     */
+    isInitialized() {
+        return this.nlpInitialized && this.nlpInstance !== null;
+    }
+    /**
+     * Get pool statistics
+     *
+     * Useful for performance monitoring and memory leak detection
+     *
+     * @returns Statistics about instance reuse
+     */
+    getStats() {
+        return {
+            ...this.stats,
+            nlpCreated: this.nlpInstance !== null,
+            extractorCreated: this.extractorInstance !== null,
+            initialized: this.isInitialized(),
+            // Memory savings estimate
+            memorySaved: this.calculateMemorySaved()
+        };
+    }
+    /**
+     * Calculate estimated memory saved by pooling
+     *
+     * Assumes ~50MB per NLP instance, ~10MB per extractor instance
+     *
+     * @returns Estimated memory saved in bytes
+     */
+    calculateMemorySaved() {
+        const nlpSize = 50 * 1024 * 1024; // 50MB per instance
+        const extractorSize = 10 * 1024 * 1024; // 10MB per instance
+        // Without pooling: size × reuses
+        // With pooling: size × 1
+        // Saved: size × (reuses - 1)
+        const nlpSaved = nlpSize * Math.max(0, this.stats.nlpReuses - 1);
+        const extractorSaved = extractorSize * Math.max(0, this.stats.extractorReuses - 1);
+        return nlpSaved + extractorSaved;
+    }
+    /**
+     * Reset statistics (useful for testing)
+     */
+    resetStats() {
+        this.stats = {
+            nlpReuses: 0,
+            extractorReuses: 0,
+            creationTime: 0
+        };
+    }
+    /**
+     * Get string representation (for debugging)
+     */
+    toString() {
+        const stats = this.getStats();
+        return `InstancePool(nlp=${stats.nlpCreated}, extractor=${stats.extractorCreated}, initialized=${stats.initialized}, nlpReuses=${stats.nlpReuses}, extractorReuses=${stats.extractorReuses})`;
+    }
+    /**
+     * Cleanup method (for explicit resource management)
+     *
+     * Note: Usually not needed - pool is garbage collected when import completes.
+     * Use only if you need explicit cleanup for some reason.
+     */
+    cleanup() {
+        // Clear references to allow garbage collection
+        this.nlpInstance = null;
+        this.extractorInstance = null;
+        this.nlpInitialized = false;
+        this.initializationPromise = null;
+    }
+}
+/**
+ * Create a new instance pool
+ *
+ * Convenience factory function
+ *
+ * @param brain Brainy instance
+ * @param autoInit Whether to initialize instances immediately
+ * @returns Instance pool
+ */
+export async function createInstancePool(brain, autoInit = true) {
+    const pool = new InstancePool(brain);
+    if (autoInit) {
+        await pool.init();
+    }
+    return pool;
+}
+//# sourceMappingURL=InstancePool.js.map

package/dist/importers/SmartCSVImporter.d.ts

@@ -101,6 +101,7 @@ export declare class SmartCSVImporter {
     private brain;
     private extractor;
     private nlp;
+    private relationshipExtractor;
     private csvHandler;
     constructor(brain: Brainy);
     /**
@@ -124,7 +125,7 @@ export declare class SmartCSVImporter {
      */
     private mapTypeString;
     /**
-     * Infer relationship type from context
+     * Infer relationship type from context using SmartRelationshipExtractor
      */
     private inferRelationship;
     /**

package/dist/importers/SmartCSVImporter.js

@@ -12,6 +12,7 @@
  */
 import { NeuralEntityExtractor } from '../neural/entityExtractor.js';
 import { NaturalLanguageProcessor } from '../neural/naturalLanguageProcessor.js';
+import { SmartRelationshipExtractor } from '../neural/SmartRelationshipExtractor.js';
 import { NounType, VerbType } from '../types/graphTypes.js';
 import { CSVHandler } from '../augmentations/intelligentImport/handlers/csvHandler.js';
 /**
@@ -22,6 +23,7 @@ export class SmartCSVImporter {
         this.brain = brain;
         this.extractor = new NeuralEntityExtractor(brain);
         this.nlp = new NaturalLanguageProcessor(brain);
+        this.relationshipExtractor = new SmartRelationshipExtractor(brain);
         this.csvHandler = new CSVHandler();
     }
     /**
@@ -266,29 +268,16 @@ export class SmartCSVImporter {
         return mapping[normalized] || NounType.Thing;
     }
     /**
-     * Infer relationship type from context
+     * Infer relationship type from context using SmartRelationshipExtractor
      */
-    async inferRelationship(fromTerm, toTerm, context) {
-        const lowerContext = context.toLowerCase();
-        // Pattern-based relationship detection
-        const patterns = [
-            [new RegExp(`${toTerm}.*of.*${fromTerm}`, 'i'), VerbType.PartOf],
-            [new RegExp(`${fromTerm}.*contains.*${toTerm}`, 'i'), VerbType.Contains],
-            [new RegExp(`located in.*${toTerm}`, 'i'), VerbType.LocatedAt],
-            [new RegExp(`ruled by.*${toTerm}`, 'i'), VerbType.Owns],
-            [new RegExp(`capital.*${toTerm}`, 'i'), VerbType.Contains],
-            [new RegExp(`created by.*${toTerm}`, 'i'), VerbType.CreatedBy],
-            [new RegExp(`authored by.*${toTerm}`, 'i'), VerbType.CreatedBy],
-            [new RegExp(`part of.*${toTerm}`, 'i'), VerbType.PartOf],
-            [new RegExp(`related to.*${toTerm}`, 'i'), VerbType.RelatedTo]
-        ];
-        for (const [pattern, verbType] of patterns) {
-            if (pattern.test(lowerContext)) {
-                return verbType;
-            }
-        }
-        // Default to RelatedTo
-        return VerbType.RelatedTo;
+    async inferRelationship(fromTerm, toTerm, context, fromType, toType) {
+        // Use SmartRelationshipExtractor for robust relationship classification
+        const result = await this.relationshipExtractor.infer(fromTerm, toTerm, context, {
+            subjectType: fromType,
+            objectType: toType
+        });
+        // Return inferred type or fallback to RelatedTo
+        return result?.type || VerbType.RelatedTo;
     }
     /**
      * Generate consistent entity ID from name

package/dist/importers/SmartDOCXImporter.d.ts

@@ -0,0 +1,125 @@
+/**
+ * Smart DOCX Importer
+ *
+ * Extracts entities and relationships from Word documents using:
+ * - Mammoth parser for DOCX → HTML/text conversion
+ * - Heading extraction for document structure
+ * - Table extraction for structured data
+ * - NeuralEntityExtractor for entity extraction from paragraphs
+ * - NaturalLanguageProcessor for relationship inference
+ * - Hierarchical relationship creation based on heading hierarchy
+ *
+ * v4.2.0: New format handler
+ * NO MOCKS - Production-ready implementation
+ */
+import { Brainy } from '../brainy.js';
+import { NounType, VerbType } from '../types/graphTypes.js';
+export interface SmartDOCXOptions {
+    /** Enable neural entity extraction from paragraphs */
+    enableNeuralExtraction?: boolean;
+    /** Enable hierarchical relationship creation based on headings */
+    enableHierarchicalRelationships?: boolean;
+    /** Enable concept extraction for tagging */
+    enableConceptExtraction?: boolean;
+    /** Confidence threshold for entities (0-1) */
+    confidenceThreshold?: number;
+    /** Minimum paragraph length to process */
+    minParagraphLength?: number;
+    /** Progress callback */
+    onProgress?: (stats: {
+        processed: number;
+        entities: number;
+        relationships: number;
+    }) => void;
+}
+export interface ExtractedDOCXEntity {
+    /** Entity ID */
+    id: string;
+    /** Entity name */
+    name: string;
+    /** Entity type */
+    type: NounType;
+    /** Entity description/context */
+    description: string;
+    /** Confidence score */
+    confidence: number;
+    /** Weight/importance score */
+    weight?: number;
+    /** Section/heading context */
+    section: string | null;
+    /** Paragraph index in document */
+    paragraphIndex: number;
+    /** Metadata */
+    metadata: Record<string, any>;
+}
+export interface ExtractedDOCXRelationship {
+    from: string;
+    to: string;
+    type: VerbType;
+    confidence: number;
+    weight?: number;
+    evidence: string;
+}
+export interface SmartDOCXResult {
+    /** Total paragraphs processed */
+    paragraphsProcessed: number;
+    /** Entities extracted */
+    entitiesExtracted: number;
+    /** Relationships inferred */
+    relationshipsInferred: number;
+    /** All extracted entities */
+    entities: ExtractedDOCXEntity[];
+    /** All relationships */
+    relationships: ExtractedDOCXRelationship[];
+    /** Entity ID mapping (index -> ID) */
+    entityMap: Map<string, string>;
+    /** Processing time in ms */
+    processingTime: number;
+    /** Document structure */
+    structure: {
+        headings: Array<{
+            level: number;
+            text: string;
+            index: number;
+        }>;
+        paragraphCount: number;
+        tableCount: number;
+    };
+    /** Extraction statistics */
+    stats: {
+        byType: Record<string, number>;
+        bySection: Record<string, number>;
+        byConfidence: {
+            high: number;
+            medium: number;
+            low: number;
+        };
+    };
+}
+/**
+ * SmartDOCXImporter - Extracts structured knowledge from Word documents
+ */
+export declare class SmartDOCXImporter {
+    private brain;
+    private extractor;
+    private nlp;
+    private relationshipExtractor;
+    private mammothLoaded;
+    constructor(brain: Brainy);
+    /**
+     * Initialize the importer
+     */
+    init(): Promise<void>;
+    /**
+     * Extract entities and relationships from DOCX buffer
+     */
+    extract(buffer: Buffer, options?: SmartDOCXOptions): Promise<SmartDOCXResult>;
+    /**
+     * Extract entities and relationships from parsed DOCX content
+     */
+    private extractFromContent;
+    /**
+     * Parse document structure from HTML
+     */
+    private parseStructure;
+}

package/dist/importers/SmartDOCXImporter.js

@@ -0,0 +1,227 @@
+/**
+ * Smart DOCX Importer
+ *
+ * Extracts entities and relationships from Word documents using:
+ * - Mammoth parser for DOCX → HTML/text conversion
+ * - Heading extraction for document structure
+ * - Table extraction for structured data
+ * - NeuralEntityExtractor for entity extraction from paragraphs
+ * - NaturalLanguageProcessor for relationship inference
+ * - Hierarchical relationship creation based on heading hierarchy
+ *
+ * v4.2.0: New format handler
+ * NO MOCKS - Production-ready implementation
+ */
+import { NeuralEntityExtractor } from '../neural/entityExtractor.js';
+import { NaturalLanguageProcessor } from '../neural/naturalLanguageProcessor.js';
+import { SmartRelationshipExtractor } from '../neural/SmartRelationshipExtractor.js';
+import { VerbType } from '../types/graphTypes.js';
+// Dynamic import for mammoth (ESM compatibility)
+let mammoth;
+/**
+ * SmartDOCXImporter - Extracts structured knowledge from Word documents
+ */
+export class SmartDOCXImporter {
+    constructor(brain) {
+        this.mammothLoaded = false;
+        this.brain = brain;
+        this.extractor = new NeuralEntityExtractor(brain);
+        this.nlp = new NaturalLanguageProcessor(brain);
+        this.relationshipExtractor = new SmartRelationshipExtractor(brain);
+    }
+    /**
+     * Initialize the importer
+     */
+    async init() {
+        await this.nlp.init();
+        // Lazy load mammoth
+        if (!this.mammothLoaded) {
+            try {
+                mammoth = await import('mammoth');
+                this.mammothLoaded = true;
+            }
+            catch (error) {
+                throw new Error(`Failed to load mammoth parser: ${error.message}`);
+            }
+        }
+    }
+    /**
+     * Extract entities and relationships from DOCX buffer
+     */
+    async extract(buffer, options = {}) {
+        const startTime = Date.now();
+        // Ensure mammoth is loaded
+        if (!this.mammothLoaded) {
+            await this.init();
+        }
+        // Extract raw text for entity extraction
+        const textResult = await mammoth.extractRawText({ buffer });
+        // Extract HTML for structure analysis (headings, tables)
+        const htmlResult = await mammoth.convertToHtml({ buffer });
+        // Process the document
+        const result = await this.extractFromContent(textResult.value, htmlResult.value, options);
+        result.processingTime = Date.now() - startTime;
+        return result;
+    }
+    /**
+     * Extract entities and relationships from parsed DOCX content
+     */
+    async extractFromContent(rawText, html, options) {
+        const opts = {
+            enableNeuralExtraction: options.enableNeuralExtraction !== false,
+            enableHierarchicalRelationships: options.enableHierarchicalRelationships !== false,
+            enableConceptExtraction: options.enableConceptExtraction !== false,
+            confidenceThreshold: options.confidenceThreshold || 0.6,
+            minParagraphLength: options.minParagraphLength || 20
+        };
+        const entities = [];
+        const relationships = [];
+        const entityMap = new Map();
+        const stats = {
+            byType: {},
+            bySection: {},
+            byConfidence: { high: 0, medium: 0, low: 0 }
+        };
+        // Parse document structure from HTML
+        const structure = this.parseStructure(html);
+        // Split into paragraphs
+        const paragraphs = rawText.split(/\n\n+/).filter(p => p.trim().length >= opts.minParagraphLength);
+        let currentSection = 'Introduction';
+        let headingIndex = 0;
+        // Process each paragraph
+        for (let i = 0; i < paragraphs.length; i++) {
+            const paragraph = paragraphs[i].trim();
+            // Check if this paragraph is a heading
+            if (headingIndex < structure.headings.length) {
+                const heading = structure.headings[headingIndex];
+                if (paragraph.startsWith(heading.text) || heading.text.includes(paragraph.substring(0, 50))) {
+                    currentSection = heading.text;
+                    headingIndex++;
+                    stats.bySection[currentSection] = 0;
+                    continue;
+                }
+            }
+            // Extract entities from paragraph
+            if (opts.enableNeuralExtraction) {
+                const extractedEntities = await this.extractor.extract(paragraph, {
+                    confidence: opts.confidenceThreshold
+                });
+                for (const extracted of extractedEntities) {
+                    const entityId = `para${i}:${extracted.text}`;
+                    const entity = {
+                        id: entityId,
+                        name: extracted.text,
+                        type: extracted.type,
+                        description: paragraph,
+                        confidence: extracted.confidence,
+                        weight: extracted.weight || 1.0,
+                        section: currentSection,
+                        paragraphIndex: i,
+                        metadata: {
+                            position: extracted.position,
+                            headingContext: currentSection
+                        }
+                    };
+                    entities.push(entity);
+                    entityMap.set(entityId, entityId);
+                    // Update stats
+                    stats.byType[entity.type] = (stats.byType[entity.type] || 0) + 1;
+                    stats.bySection[currentSection] = (stats.bySection[currentSection] || 0) + 1;
+                    if (entity.confidence > 0.8)
+                        stats.byConfidence.high++;
+                    else if (entity.confidence >= 0.6)
+                        stats.byConfidence.medium++;
+                    else
+                        stats.byConfidence.low++;
+                }
+            }
+            // Report progress
+            if (options.onProgress && i % 10 === 0) {
+                options.onProgress({
+                    processed: i,
+                    entities: entities.length,
+                    relationships: relationships.length
+                });
+            }
+        }
+        // Create hierarchical relationships based on sections
+        if (opts.enableHierarchicalRelationships) {
+            const entitiesBySection = new Map();
+            for (const entity of entities) {
+                const section = entity.section || 'Unknown';
+                if (!entitiesBySection.has(section)) {
+                    entitiesBySection.set(section, []);
+                }
+                entitiesBySection.get(section).push(entity);
+            }
+            // Create relationships within sections
+            for (const [section, sectionEntities] of entitiesBySection) {
+                for (let i = 0; i < sectionEntities.length - 1; i++) {
+                    for (let j = i + 1; j < Math.min(i + 3, sectionEntities.length); j++) {
+                        const entityA = sectionEntities[i];
+                        const entityB = sectionEntities[j];
+                        // Infer relationship type using SmartRelationshipExtractor
+                        // Combine entity descriptions for better context
+                        const context = `In section "${section}": ${entityA.description.substring(0, 150)}... ${entityB.description.substring(0, 150)}...`;
+                        const inferredRelationship = await this.relationshipExtractor.infer(entityA.name, entityB.name, context, {
+                            subjectType: entityA.type,
+                            objectType: entityB.type
+                        });
+                        relationships.push({
+                            from: entityA.id,
+                            to: entityB.id,
+                            type: inferredRelationship?.type || VerbType.RelatedTo, // Fallback to RelatedTo for co-occurrence
+                            confidence: inferredRelationship?.confidence || 0.7,
+                            weight: inferredRelationship?.weight || 0.8,
+                            evidence: inferredRelationship?.evidence || `Both entities appear in section: ${section}`
+                        });
+                    }
+                }
+            }
+        }
+        // Final progress report
+        if (options.onProgress) {
+            options.onProgress({
+                processed: paragraphs.length,
+                entities: entities.length,
+                relationships: relationships.length
+            });
+        }
+        return {
+            paragraphsProcessed: paragraphs.length,
+            entitiesExtracted: entities.length,
+            relationshipsInferred: relationships.length,
+            entities,
+            relationships,
+            entityMap,
+            processingTime: 0, // Will be set by caller
+            structure,
+            stats
+        };
+    }
+    /**
+     * Parse document structure from HTML
+     */
+    parseStructure(html) {
+        const headings = [];
+        // Extract headings (h1-h6)
+        const headingRegex = /<h([1-6])>(.*?)<\/h\1>/gi;
+        let match;
+        let index = 0;
+        while ((match = headingRegex.exec(html)) !== null) {
+            const level = parseInt(match[1]);
+            const text = match[2].replace(/<[^>]+>/g, '').trim(); // Strip HTML tags
+            headings.push({ level, text, index: index++ });
+        }
+        // Count paragraphs
+        const paragraphCount = (html.match(/<p>/g) || []).length;
+        // Count tables
+        const tableCount = (html.match(/<table>/g) || []).length;
+        return {
+            headings,
+            paragraphCount,
+            tableCount
+        };
+    }
+}
+//# sourceMappingURL=SmartDOCXImporter.js.map

package/dist/importers/SmartExcelImporter.d.ts

@@ -88,6 +88,16 @@ export interface SmartExcelResult {
             low: number;
         };
     };
+    /** Sheet-specific data for VFS extraction (v4.2.0) */
+    sheets?: Array<{
+        name: string;
+        rows: ExtractedRow[];
+        stats: {
+            rowCount: number;
+            entityCount: number;
+            relationshipCount: number;
+        };
+    }>;
 }
 /**
  * SmartExcelImporter - Extracts structured knowledge from Excel files
@@ -96,6 +106,7 @@ export declare class SmartExcelImporter {
     private brain;
     private extractor;
     private nlp;
+    private relationshipExtractor;
     private excelHandler;
     constructor(brain: Brainy);
     /**
@@ -119,7 +130,7 @@ export declare class SmartExcelImporter {
      */
     private mapTypeString;
     /**
-     * Infer relationship type from context
+     * Infer relationship type from context using SmartRelationshipExtractor
      */
     private inferRelationship;
     /**