@soulcraft/brainy 4.1.3 → 4.2.0

package/dist/importers/SmartExcelImporter.js

@@ -10,6 +10,7 @@
  */
 import { NeuralEntityExtractor } from '../neural/entityExtractor.js';
 import { NaturalLanguageProcessor } from '../neural/naturalLanguageProcessor.js';
+import { SmartRelationshipExtractor } from '../neural/SmartRelationshipExtractor.js';
 import { NounType, VerbType } from '../types/graphTypes.js';
 import { ExcelHandler } from '../augmentations/intelligentImport/handlers/excelHandler.js';
 /**
@@ -20,6 +21,7 @@ export class SmartExcelImporter {
         this.brain = brain;
         this.extractor = new NeuralEntityExtractor(brain);
         this.nlp = new NaturalLanguageProcessor(brain);
+        this.relationshipExtractor = new SmartRelationshipExtractor(brain);
         this.excelHandler = new ExcelHandler();
     }
     /**
@@ -131,7 +133,9 @@ export class SmartExcelImporter {
                 if (opts.enableRelationshipInference) {
                     // Extract relationships from definition text
                     for (const relEntity of relatedEntities) {
-                        const verbType = await this.inferRelationship(term, relEntity.text, definition);
+                        const verbType = await this.inferRelationship(term, relEntity.text, definition, mainEntityType, // Pass subject type hint
+                        relEntity.type // Pass object type hint
+                        );
                         relationships.push({
                             from: entityId,
                             to: relEntity.text,
@@ -145,10 +149,13 @@ export class SmartExcelImporter {
                         const terms = relatedTerms.split(/[,;]/).map(t => t.trim()).filter(Boolean);
                         for (const relTerm of terms) {
                             if (relTerm.toLowerCase() !== term.toLowerCase()) {
+                                // Use SmartRelationshipExtractor even for explicit relationships
+                                const verbType = await this.inferRelationship(term, relTerm, `${term} related to ${relTerm}. ${definition}`, // Combine for better context
+                                mainEntityType);
                                 relationships.push({
                                     from: entityId,
                                     to: relTerm,
-                                    type: VerbType.RelatedTo,
+                                    type: verbType,
                                     confidence: 0.9,
                                     evidence: `Explicitly listed in "Related" column`
                                 });
@@ -203,6 +210,26 @@ export class SmartExcelImporter {
                 phase: 'extracting'
             });
         }
+        // Group rows by sheet for VFS extraction (v4.2.0)
+        const sheetGroups = new Map();
+        extractedRows.forEach((extractedRow, index) => {
+            const originalRow = rows[index];
+            const sheetName = originalRow._sheet || 'Sheet1';
+            if (!sheetGroups.has(sheetName)) {
+                sheetGroups.set(sheetName, []);
+            }
+            sheetGroups.get(sheetName).push(extractedRow);
+        });
+        // Build sheet-specific statistics
+        const sheets = Array.from(sheetGroups.entries()).map(([name, sheetRows]) => ({
+            name,
+            rows: sheetRows,
+            stats: {
+                rowCount: sheetRows.length,
+                entityCount: sheetRows.reduce((sum, row) => sum + 1 + row.relatedEntities.length, 0),
+                relationshipCount: sheetRows.reduce((sum, row) => sum + row.relationships.length, 0)
+            }
+        }));
         return {
             rowsProcessed: rows.length,
             entitiesExtracted: extractedRows.reduce((sum, row) => sum + 1 + row.relatedEntities.length, 0),
@@ -210,7 +237,8 @@ export class SmartExcelImporter {
             rows: extractedRows,
             entityMap,
             processingTime: Date.now() - startTime,
-            stats
+            stats,
+            sheets
         };
     }
     /**
@@ -270,29 +298,16 @@ export class SmartExcelImporter {
         return mapping[normalized] || NounType.Thing;
     }
     /**
-     * Infer relationship type from context
+     * Infer relationship type from context using SmartRelationshipExtractor
      */
-    async inferRelationship(fromTerm, toTerm, context) {
-        const lowerContext = context.toLowerCase();
-        // Pattern-based relationship detection
-        const patterns = [
-            [new RegExp(`${toTerm}.*of.*${fromTerm}`, 'i'), VerbType.PartOf],
-            [new RegExp(`${fromTerm}.*contains.*${toTerm}`, 'i'), VerbType.Contains],
-            [new RegExp(`located in.*${toTerm}`, 'i'), VerbType.LocatedAt],
-            [new RegExp(`ruled by.*${toTerm}`, 'i'), VerbType.Owns],
-            [new RegExp(`capital.*${toTerm}`, 'i'), VerbType.Contains],
-            [new RegExp(`created by.*${toTerm}`, 'i'), VerbType.CreatedBy],
-            [new RegExp(`authored by.*${toTerm}`, 'i'), VerbType.CreatedBy],
-            [new RegExp(`part of.*${toTerm}`, 'i'), VerbType.PartOf],
-            [new RegExp(`related to.*${toTerm}`, 'i'), VerbType.RelatedTo]
-        ];
-        for (const [pattern, verbType] of patterns) {
-            if (pattern.test(lowerContext)) {
-                return verbType;
-            }
-        }
-        // Default to RelatedTo
-        return VerbType.RelatedTo;
+    async inferRelationship(fromTerm, toTerm, context, fromType, toType) {
+        // Use SmartRelationshipExtractor for robust relationship classification
+        const result = await this.relationshipExtractor.infer(fromTerm, toTerm, context, {
+            subjectType: fromType,
+            objectType: toType
+        });
+        // Return inferred type or fallback to RelatedTo
+        return result?.type || VerbType.RelatedTo;
     }
     /**
      * Generate consistent entity ID from name

package/dist/importers/SmartJSONImporter.d.ts

@@ -95,6 +95,7 @@ export declare class SmartJSONImporter {
     private brain;
     private extractor;
     private nlp;
+    private relationshipExtractor;
     constructor(brain: Brainy);
     /**
      * Initialize the importer

package/dist/importers/SmartJSONImporter.js

@@ -11,6 +11,7 @@
  */
 import { NeuralEntityExtractor } from '../neural/entityExtractor.js';
 import { NaturalLanguageProcessor } from '../neural/naturalLanguageProcessor.js';
+import { SmartRelationshipExtractor } from '../neural/SmartRelationshipExtractor.js';
 import { NounType, VerbType } from '../types/graphTypes.js';
 /**
  * SmartJSONImporter - Extracts structured knowledge from JSON files
@@ -20,6 +21,7 @@ export class SmartJSONImporter {
         this.brain = brain;
         this.extractor = new NeuralEntityExtractor(brain);
         this.nlp = new NaturalLanguageProcessor(brain);
+        this.relationshipExtractor = new SmartRelationshipExtractor(brain);
     }
     /**
      * Initialize the importer
@@ -120,12 +122,20 @@ export class SmartJSONImporter {
                 // Create hierarchical relationship if parent exists
                 if (options.enableHierarchicalRelationships && parentPath && entityMap.has(parentPath)) {
                     const parentId = entityMap.get(parentPath);
+                    // Extract parent and child names from paths
+                    const parentName = parentPath.split('.').pop()?.replace(/\[(\d+)\]/, 'item $1') || 'parent';
+                    const childName = entity.name;
+                    // Infer relationship type using SmartRelationshipExtractor
+                    const context = `Hierarchical JSON structure: ${parentName} contains ${childName}. Parent path: ${parentPath}, Child path: ${path}`;
+                    const inferredRelationship = await this.relationshipExtractor.infer(parentName, childName, context, {
+                        objectType: entity.type // Pass child entity type as hint
+                    });
                     relationships.push({
                         from: parentId,
                         to: entity.id,
-                        type: VerbType.Contains,
-                        confidence: 0.95,
-                        evidence: `Hierarchical relationship: ${parentPath} contains ${path}`
+                        type: inferredRelationship?.type || VerbType.Contains, // Fallback to Contains for hierarchical relationships
+                        confidence: inferredRelationship?.confidence || 0.95,
+                        evidence: inferredRelationship?.evidence || `Hierarchical relationship: ${parentPath} contains ${path}`
                     });
                 }
             }
@@ -165,12 +175,21 @@ export class SmartJSONImporter {
                     // Link to parent if exists
                     if (options.enableHierarchicalRelationships && parentPath && entityMap.has(parentPath)) {
                         const parentId = entityMap.get(parentPath);
+                        // Extract parent name from path
+                        const parentName = parentPath.split('.').pop()?.replace(/\[(\d+)\]/, 'item $1') || 'parent';
+                        const childName = entity.name;
+                        // Infer relationship type using SmartRelationshipExtractor
+                        // Context: entity was extracted from string value within parent container
+                        const context = `Entity "${childName}" found in text value at path ${path} within parent "${parentName}". Full text: "${node.substring(0, 200)}..."`;
+                        const inferredRelationship = await this.relationshipExtractor.infer(parentName, childName, context, {
+                            objectType: entity.type // Pass extracted entity type as hint
+                        });
                         relationships.push({
                             from: parentId,
                             to: entity.id,
-                            type: VerbType.RelatedTo,
-                            confidence: extracted.confidence * 0.9,
-                            evidence: `Found in: ${path}`
+                            type: inferredRelationship?.type || VerbType.RelatedTo, // Fallback to RelatedTo for text extraction
+                            confidence: inferredRelationship?.confidence || (extracted.confidence * 0.9),
+                            evidence: inferredRelationship?.evidence || `Found in: ${path}`
                         });
                     }
                 }

package/dist/importers/SmartMarkdownImporter.d.ts

@@ -107,6 +107,7 @@ export declare class SmartMarkdownImporter {
     private brain;
     private extractor;
     private nlp;
+    private relationshipExtractor;
     constructor(brain: Brainy);
     /**
      * Initialize the importer
@@ -145,7 +146,7 @@ export declare class SmartMarkdownImporter {
      */
     private entitiesAreRelated;
     /**
-     * Infer relationship type from context
+     * Infer relationship type from context using SmartRelationshipExtractor
      */
     private inferRelationship;
     /**

package/dist/importers/SmartMarkdownImporter.js

@@ -11,6 +11,7 @@
  */
 import { NeuralEntityExtractor } from '../neural/entityExtractor.js';
 import { NaturalLanguageProcessor } from '../neural/naturalLanguageProcessor.js';
+import { SmartRelationshipExtractor } from '../neural/SmartRelationshipExtractor.js';
 import { NounType, VerbType } from '../types/graphTypes.js';
 /**
  * SmartMarkdownImporter - Extracts structured knowledge from Markdown files
@@ -20,6 +21,7 @@ export class SmartMarkdownImporter {
         this.brain = brain;
         this.extractor = new NeuralEntityExtractor(brain);
         this.nlp = new NaturalLanguageProcessor(brain);
+        this.relationshipExtractor = new SmartRelationshipExtractor(brain);
     }
     /**
      * Initialize the importer
@@ -321,23 +323,16 @@ export class SmartMarkdownImporter {
         return Math.abs(index1 - index2) < 300;
     }
     /**
-     * Infer relationship type from context
+     * Infer relationship type from context using SmartRelationshipExtractor
      */
-    async inferRelationship(fromEntity, toEntity, context) {
-        const lowerContext = context.toLowerCase();
-        const patterns = [
-            [new RegExp(`${toEntity}.*of.*${fromEntity}`, 'i'), VerbType.PartOf],
-            [new RegExp(`${fromEntity}.*contains.*${toEntity}`, 'i'), VerbType.Contains],
-            [new RegExp(`${fromEntity}.*in.*${toEntity}`, 'i'), VerbType.LocatedAt],
-            [new RegExp(`${fromEntity}.*created.*${toEntity}`, 'i'), VerbType.Creates],
-            [new RegExp(`${fromEntity}.*and.*${toEntity}`, 'i'), VerbType.RelatedTo]
-        ];
-        for (const [pattern, verbType] of patterns) {
-            if (pattern.test(lowerContext)) {
-                return verbType;
-            }
-        }
-        return VerbType.RelatedTo;
+    async inferRelationship(fromEntity, toEntity, context, fromType, toType) {
+        // Use SmartRelationshipExtractor for robust relationship classification
+        const result = await this.relationshipExtractor.infer(fromEntity, toEntity, context, {
+            subjectType: fromType,
+            objectType: toType
+        });
+        // Return inferred type or fallback to RelatedTo
+        return result?.type || VerbType.RelatedTo;
     }
     /**
      * Generate consistent entity ID

package/dist/importers/SmartPDFImporter.d.ts

@@ -111,6 +111,7 @@ export declare class SmartPDFImporter {
     private brain;
     private extractor;
     private nlp;
+    private relationshipExtractor;
     private pdfHandler;
     constructor(brain: Brainy);
     /**
@@ -142,7 +143,7 @@ export declare class SmartPDFImporter {
      */
     private extractRelationshipContext;
     /**
-     * Infer relationship type from context
+     * Infer relationship type from context using SmartRelationshipExtractor
      */
     private inferRelationship;
     /**

package/dist/importers/SmartPDFImporter.js

@@ -11,6 +11,7 @@
  */
 import { NeuralEntityExtractor } from '../neural/entityExtractor.js';
 import { NaturalLanguageProcessor } from '../neural/naturalLanguageProcessor.js';
+import { SmartRelationshipExtractor } from '../neural/SmartRelationshipExtractor.js';
 import { VerbType } from '../types/graphTypes.js';
 import { PDFHandler } from '../augmentations/intelligentImport/handlers/pdfHandler.js';
 /**
@@ -21,6 +22,7 @@ export class SmartPDFImporter {
         this.brain = brain;
         this.extractor = new NeuralEntityExtractor(brain);
         this.nlp = new NaturalLanguageProcessor(brain);
+        this.relationshipExtractor = new SmartRelationshipExtractor(brain);
         this.pdfHandler = new PDFHandler();
     }
     /**
@@ -272,29 +274,16 @@ export class SmartPDFImporter {
         return text.substring(start, end + 100).trim();
     }
     /**
-     * Infer relationship type from context
+     * Infer relationship type from context using SmartRelationshipExtractor
      */
-    async inferRelationship(fromEntity, toEntity, context) {
-        const lowerContext = context.toLowerCase();
-        // Pattern-based relationship detection
-        const patterns = [
-            [new RegExp(`${toEntity}.*of.*${fromEntity}`, 'i'), VerbType.PartOf],
-            [new RegExp(`${fromEntity}.*contains.*${toEntity}`, 'i'), VerbType.Contains],
-            [new RegExp(`${fromEntity}.*in.*${toEntity}`, 'i'), VerbType.LocatedAt],
-            [new RegExp(`${fromEntity}.*by.*${toEntity}`, 'i'), VerbType.CreatedBy],
-            [new RegExp(`${fromEntity}.*created.*${toEntity}`, 'i'), VerbType.Creates],
-            [new RegExp(`${fromEntity}.*authored.*${toEntity}`, 'i'), VerbType.CreatedBy],
-            [new RegExp(`${fromEntity}.*part of.*${toEntity}`, 'i'), VerbType.PartOf],
-            [new RegExp(`${fromEntity}.*related to.*${toEntity}`, 'i'), VerbType.RelatedTo],
-            [new RegExp(`${fromEntity}.*and.*${toEntity}`, 'i'), VerbType.RelatedTo]
-        ];
-        for (const [pattern, verbType] of patterns) {
-            if (pattern.test(lowerContext)) {
-                return verbType;
-            }
-        }
-        // Default to RelatedTo
-        return VerbType.RelatedTo;
+    async inferRelationship(fromEntity, toEntity, context, fromType, toType) {
+        // Use SmartRelationshipExtractor for robust relationship classification
+        const result = await this.relationshipExtractor.infer(fromEntity, toEntity, context, {
+            subjectType: fromType,
+            objectType: toType
+        });
+        // Return inferred type or fallback to RelatedTo
+        return result?.type || VerbType.RelatedTo;
     }
     /**
      * Generate consistent entity ID

package/dist/importers/SmartYAMLImporter.d.ts

@@ -0,0 +1,121 @@
+/**
+ * Smart YAML Importer
+ *
+ * Extracts entities and relationships from YAML files using:
+ * - YAML parsing to JSON-like structure
+ * - Recursive traversal of nested structures
+ * - NeuralEntityExtractor for entity extraction from text values
+ * - NaturalLanguageProcessor for relationship inference
+ * - Hierarchical relationship creation (parent-child, contains, etc.)
+ *
+ * v4.2.0: New format handler
+ * NO MOCKS - Production-ready implementation
+ */
+import { Brainy } from '../brainy.js';
+import { NounType, VerbType } from '../types/graphTypes.js';
+export interface SmartYAMLOptions {
+    /** Enable neural entity extraction from string values */
+    enableNeuralExtraction?: boolean;
+    /** Enable hierarchical relationship creation */
+    enableHierarchicalRelationships?: boolean;
+    /** Enable concept extraction for tagging */
+    enableConceptExtraction?: boolean;
+    /** Confidence threshold for entities (0-1) */
+    confidenceThreshold?: number;
+    /** Maximum depth to traverse */
+    maxDepth?: number;
+    /** Minimum string length to process for entity extraction */
+    minStringLength?: number;
+    /** Keys that indicate entity names */
+    nameKeys?: string[];
+    /** Keys that indicate entity descriptions */
+    descriptionKeys?: string[];
+    /** Keys that indicate entity types */
+    typeKeys?: string[];
+    /** Progress callback */
+    onProgress?: (stats: {
+        processed: number;
+        entities: number;
+        relationships: number;
+    }) => void;
+}
+export interface ExtractedYAMLEntity {
+    /** Entity ID */
+    id: string;
+    /** Entity name */
+    name: string;
+    /** Entity type */
+    type: NounType;
+    /** Entity description/value */
+    description: string;
+    /** Confidence score */
+    confidence: number;
+    /** Weight/importance score */
+    weight?: number;
+    /** YAML path to this entity */
+    path: string;
+    /** Parent path in YAML hierarchy */
+    parentPath: string | null;
+    /** Metadata */
+    metadata: Record<string, any>;
+}
+export interface ExtractedYAMLRelationship {
+    from: string;
+    to: string;
+    type: VerbType;
+    confidence: number;
+    weight?: number;
+    evidence: string;
+}
+export interface SmartYAMLResult {
+    /** Total nodes processed */
+    nodesProcessed: number;
+    /** Entities extracted */
+    entitiesExtracted: number;
+    /** Relationships inferred */
+    relationshipsInferred: number;
+    /** All extracted entities */
+    entities: ExtractedYAMLEntity[];
+    /** All relationships */
+    relationships: ExtractedYAMLRelationship[];
+    /** Entity ID mapping (path -> ID) */
+    entityMap: Map<string, string>;
+    /** Processing time in ms */
+    processingTime: number;
+    /** Extraction statistics */
+    stats: {
+        byType: Record<string, number>;
+        byDepth: Record<number, number>;
+        byConfidence: {
+            high: number;
+            medium: number;
+            low: number;
+        };
+    };
+}
+/**
+ * SmartYAMLImporter - Extracts structured knowledge from YAML files
+ */
+export declare class SmartYAMLImporter {
+    private brain;
+    private extractor;
+    private nlp;
+    private relationshipExtractor;
+    constructor(brain: Brainy);
+    /**
+     * Initialize the importer
+     */
+    init(): Promise<void>;
+    /**
+     * Extract entities and relationships from YAML string or buffer
+     */
+    extract(yamlContent: string | Buffer, options?: SmartYAMLOptions): Promise<SmartYAMLResult>;
+    /**
+     * Extract entities and relationships from parsed YAML data
+     */
+    private extractFromData;
+    /**
+     * Extract an entity from a YAML object node
+     */
+    private extractEntityFromObject;
+}