@soulcraft/brainy 3.27.1 → 3.28.0

package/dist/importers/SmartExcelImporter.d.ts

@@ -0,0 +1,131 @@
+/**
+ * Smart Excel Importer
+ *
+ * Extracts entities and relationships from Excel files using:
+ * - NeuralEntityExtractor for entity extraction
+ * - NaturalLanguageProcessor for relationship inference
+ * - brain.extractConcepts() for tagging
+ *
+ * NO MOCKS - Production-ready implementation
+ */
+import { Brainy } from '../brainy.js';
+import { NounType, VerbType } from '../types/graphTypes.js';
+import type { FormatHandlerOptions } from '../augmentations/intelligentImport/types.js';
+export interface SmartExcelOptions extends FormatHandlerOptions {
+    /** Enable neural entity extraction */
+    enableNeuralExtraction?: boolean;
+    /** Enable relationship inference from text */
+    enableRelationshipInference?: boolean;
+    /** Enable concept extraction for tagging */
+    enableConceptExtraction?: boolean;
+    /** Confidence threshold for entities (0-1) */
+    confidenceThreshold?: number;
+    /** Column name patterns to detect */
+    termColumn?: string;
+    definitionColumn?: string;
+    typeColumn?: string;
+    relatedColumn?: string;
+    /** Progress callback */
+    onProgress?: (stats: {
+        processed: number;
+        total: number;
+        entities: number;
+        relationships: number;
+    }) => void;
+}
+export interface ExtractedRow {
+    /** Main entity from this row */
+    entity: {
+        id: string;
+        name: string;
+        type: NounType;
+        description: string;
+        confidence: number;
+        metadata: Record<string, any>;
+    };
+    /** Additional entities extracted from definition */
+    relatedEntities: Array<{
+        name: string;
+        type: NounType;
+        confidence: number;
+    }>;
+    /** Inferred relationships */
+    relationships: Array<{
+        from: string;
+        to: string;
+        type: VerbType;
+        confidence: number;
+        evidence: string;
+    }>;
+    /** Extracted concepts */
+    concepts?: string[];
+}
+export interface SmartExcelResult {
+    /** Total rows processed */
+    rowsProcessed: number;
+    /** Entities extracted (includes main + related) */
+    entitiesExtracted: number;
+    /** Relationships inferred */
+    relationshipsInferred: number;
+    /** All extracted data */
+    rows: ExtractedRow[];
+    /** Entity ID mapping (name -> ID) */
+    entityMap: Map<string, string>;
+    /** Processing time in ms */
+    processingTime: number;
+    /** Extraction statistics */
+    stats: {
+        byType: Record<string, number>;
+        byConfidence: {
+            high: number;
+            medium: number;
+            low: number;
+        };
+    };
+}
+/**
+ * SmartExcelImporter - Extracts structured knowledge from Excel files
+ */
+export declare class SmartExcelImporter {
+    private brain;
+    private extractor;
+    private nlp;
+    private excelHandler;
+    constructor(brain: Brainy);
+    /**
+     * Initialize the importer
+     */
+    init(): Promise<void>;
+    /**
+     * Extract entities and relationships from Excel file
+     */
+    extract(buffer: Buffer, options?: SmartExcelOptions): Promise<SmartExcelResult>;
+    /**
+     * Detect column names from first row
+     */
+    private detectColumns;
+    /**
+     * Get value from row using column name
+     */
+    private getColumnValue;
+    /**
+     * Map type string to NounType
+     */
+    private mapTypeString;
+    /**
+     * Infer relationship type from context
+     */
+    private inferRelationship;
+    /**
+     * Generate consistent entity ID from name
+     */
+    private generateEntityId;
+    /**
+     * Update statistics
+     */
+    private updateStats;
+    /**
+     * Create empty result
+     */
+    private emptyResult;
+}

package/dist/importers/SmartExcelImporter.js

@@ -0,0 +1,302 @@
+/**
+ * Smart Excel Importer
+ *
+ * Extracts entities and relationships from Excel files using:
+ * - NeuralEntityExtractor for entity extraction
+ * - NaturalLanguageProcessor for relationship inference
+ * - brain.extractConcepts() for tagging
+ *
+ * NO MOCKS - Production-ready implementation
+ */
+import { NeuralEntityExtractor } from '../neural/entityExtractor.js';
+import { NaturalLanguageProcessor } from '../neural/naturalLanguageProcessor.js';
+import { NounType, VerbType } from '../types/graphTypes.js';
+import { ExcelHandler } from '../augmentations/intelligentImport/handlers/excelHandler.js';
+/**
+ * SmartExcelImporter - Extracts structured knowledge from Excel files
+ */
+export class SmartExcelImporter {
+    constructor(brain) {
+        this.brain = brain;
+        this.extractor = new NeuralEntityExtractor(brain);
+        this.nlp = new NaturalLanguageProcessor(brain);
+        this.excelHandler = new ExcelHandler();
+    }
+    /**
+     * Initialize the importer
+     */
+    async init() {
+        await this.nlp.init();
+    }
+    /**
+     * Extract entities and relationships from Excel file
+     */
+    async extract(buffer, options = {}) {
+        const startTime = Date.now();
+        // Set defaults
+        const opts = {
+            enableNeuralExtraction: true,
+            enableRelationshipInference: true,
+            enableConceptExtraction: true,
+            confidenceThreshold: 0.6,
+            termColumn: 'term|name|title|concept',
+            definitionColumn: 'definition|description|desc|details',
+            typeColumn: 'type|category|kind',
+            relatedColumn: 'related|see also|links',
+            onProgress: () => { },
+            ...options
+        };
+        // Parse Excel using existing handler
+        const processedData = await this.excelHandler.process(buffer, options);
+        const rows = processedData.data;
+        if (rows.length === 0) {
+            return this.emptyResult(startTime);
+        }
+        // Detect column names
+        const columns = this.detectColumns(rows[0], opts);
+        // Process each row
+        const extractedRows = [];
+        const entityMap = new Map();
+        const stats = {
+            byType: {},
+            byConfidence: { high: 0, medium: 0, low: 0 }
+        };
+        for (let i = 0; i < rows.length; i++) {
+            const row = rows[i];
+            // Extract data from row
+            const term = this.getColumnValue(row, columns.term) || `Entity_${i}`;
+            const definition = this.getColumnValue(row, columns.definition) || '';
+            const type = this.getColumnValue(row, columns.type);
+            const relatedTerms = this.getColumnValue(row, columns.related);
+            // Extract entities from definition
+            let relatedEntities = [];
+            if (opts.enableNeuralExtraction && definition) {
+                relatedEntities = await this.extractor.extract(definition, {
+                    confidence: opts.confidenceThreshold * 0.8, // Lower threshold for related entities
+                    neuralMatching: true,
+                    cache: { enabled: true }
+                });
+                // Filter out the main term from related entities
+                relatedEntities = relatedEntities.filter(e => e.text.toLowerCase() !== term.toLowerCase());
+            }
+            // Determine main entity type
+            const mainEntityType = type ?
+                this.mapTypeString(type) :
+                (relatedEntities.length > 0 ? relatedEntities[0].type : NounType.Thing);
+            // Generate entity ID
+            const entityId = this.generateEntityId(term);
+            entityMap.set(term.toLowerCase(), entityId);
+            // Extract concepts
+            let concepts = [];
+            if (opts.enableConceptExtraction && definition) {
+                try {
+                    concepts = await this.brain.extractConcepts(definition, { limit: 10 });
+                }
+                catch (error) {
+                    // Concept extraction is optional
+                    concepts = [];
+                }
+            }
+            // Create main entity
+            const mainEntity = {
+                id: entityId,
+                name: term,
+                type: mainEntityType,
+                description: definition,
+                confidence: 0.95, // Main entity from row has high confidence
+                metadata: {
+                    source: 'excel',
+                    row: i + 1,
+                    originalData: row,
+                    concepts,
+                    extractedAt: Date.now()
+                }
+            };
+            // Track statistics
+            this.updateStats(stats, mainEntityType, mainEntity.confidence);
+            // Infer relationships
+            const relationships = [];
+            if (opts.enableRelationshipInference) {
+                // Extract relationships from definition text
+                for (const relEntity of relatedEntities) {
+                    const verbType = await this.inferRelationship(term, relEntity.text, definition);
+                    relationships.push({
+                        from: entityId,
+                        to: relEntity.text, // Use entity name directly, will be resolved later
+                        type: verbType,
+                        confidence: relEntity.confidence,
+                        evidence: `Extracted from: "${definition.substring(0, 100)}..."`
+                    });
+                }
+                // Parse explicit "Related Terms" column
+                if (relatedTerms) {
+                    const terms = relatedTerms.split(/[,;]/).map(t => t.trim()).filter(Boolean);
+                    for (const relTerm of terms) {
+                        // Ensure we don't create self-relationships
+                        if (relTerm.toLowerCase() !== term.toLowerCase()) {
+                            relationships.push({
+                                from: entityId,
+                                to: relTerm, // Use term name directly
+                                type: VerbType.RelatedTo,
+                                confidence: 0.9, // Explicit relationships have high confidence
+                                evidence: `Explicitly listed in "Related" column`
+                            });
+                        }
+                    }
+                }
+            }
+            // Add extracted row
+            extractedRows.push({
+                entity: mainEntity,
+                relatedEntities: relatedEntities.map(e => ({
+                    name: e.text,
+                    type: e.type,
+                    confidence: e.confidence
+                })),
+                relationships,
+                concepts
+            });
+            // Report progress
+            opts.onProgress({
+                processed: i + 1,
+                total: rows.length,
+                entities: extractedRows.length + relatedEntities.length,
+                relationships: relationships.length
+            });
+        }
+        return {
+            rowsProcessed: rows.length,
+            entitiesExtracted: extractedRows.reduce((sum, row) => sum + 1 + row.relatedEntities.length, 0),
+            relationshipsInferred: extractedRows.reduce((sum, row) => sum + row.relationships.length, 0),
+            rows: extractedRows,
+            entityMap,
+            processingTime: Date.now() - startTime,
+            stats
+        };
+    }
+    /**
+     * Detect column names from first row
+     */
+    detectColumns(firstRow, options) {
+        const columnNames = Object.keys(firstRow);
+        const matchColumn = (pattern) => {
+            const regex = new RegExp(pattern, 'i');
+            return columnNames.find(col => regex.test(col)) || null;
+        };
+        return {
+            term: matchColumn(options.termColumn || 'term|name'),
+            definition: matchColumn(options.definitionColumn || 'definition|description'),
+            type: matchColumn(options.typeColumn || 'type|category'),
+            related: matchColumn(options.relatedColumn || 'related|see also')
+        };
+    }
+    /**
+     * Get value from row using column name
+     */
+    getColumnValue(row, columnName) {
+        if (!columnName)
+            return '';
+        const value = row[columnName];
+        if (value === null || value === undefined)
+            return '';
+        return String(value).trim();
+    }
+    /**
+     * Map type string to NounType
+     */
+    mapTypeString(typeString) {
+        const normalized = typeString.toLowerCase().trim();
+        const mapping = {
+            'person': NounType.Person,
+            'character': NounType.Person,
+            'people': NounType.Person,
+            'place': NounType.Location,
+            'location': NounType.Location,
+            'geography': NounType.Location,
+            'organization': NounType.Organization,
+            'org': NounType.Organization,
+            'company': NounType.Organization,
+            'concept': NounType.Concept,
+            'idea': NounType.Concept,
+            'theory': NounType.Concept,
+            'event': NounType.Event,
+            'occurrence': NounType.Event,
+            'product': NounType.Product,
+            'item': NounType.Product,
+            'thing': NounType.Thing,
+            'document': NounType.Document,
+            'file': NounType.File,
+            'project': NounType.Project
+        };
+        return mapping[normalized] || NounType.Thing;
+    }
+    /**
+     * Infer relationship type from context
+     */
+    async inferRelationship(fromTerm, toTerm, context) {
+        const lowerContext = context.toLowerCase();
+        // Pattern-based relationship detection
+        const patterns = [
+            [new RegExp(`${toTerm}.*of.*${fromTerm}`, 'i'), VerbType.PartOf],
+            [new RegExp(`${fromTerm}.*contains.*${toTerm}`, 'i'), VerbType.Contains],
+            [new RegExp(`located in.*${toTerm}`, 'i'), VerbType.LocatedAt],
+            [new RegExp(`ruled by.*${toTerm}`, 'i'), VerbType.Owns],
+            [new RegExp(`capital.*${toTerm}`, 'i'), VerbType.Contains],
+            [new RegExp(`created by.*${toTerm}`, 'i'), VerbType.CreatedBy],
+            [new RegExp(`authored by.*${toTerm}`, 'i'), VerbType.CreatedBy],
+            [new RegExp(`part of.*${toTerm}`, 'i'), VerbType.PartOf],
+            [new RegExp(`related to.*${toTerm}`, 'i'), VerbType.RelatedTo]
+        ];
+        for (const [pattern, verbType] of patterns) {
+            if (pattern.test(lowerContext)) {
+                return verbType;
+            }
+        }
+        // Default to RelatedTo
+        return VerbType.RelatedTo;
+    }
+    /**
+     * Generate consistent entity ID from name
+     */
+    generateEntityId(name) {
+        // Create deterministic ID based on normalized name
+        const normalized = name.toLowerCase().trim().replace(/\s+/g, '_');
+        return `ent_${normalized}_${Date.now()}`;
+    }
+    /**
+     * Update statistics
+     */
+    updateStats(stats, type, confidence) {
+        // Track by type
+        const typeName = String(type);
+        stats.byType[typeName] = (stats.byType[typeName] || 0) + 1;
+        // Track by confidence
+        if (confidence > 0.8) {
+            stats.byConfidence.high++;
+        }
+        else if (confidence >= 0.6) {
+            stats.byConfidence.medium++;
+        }
+        else {
+            stats.byConfidence.low++;
+        }
+    }
+    /**
+     * Create empty result
+     */
+    emptyResult(startTime) {
+        return {
+            rowsProcessed: 0,
+            entitiesExtracted: 0,
+            relationshipsInferred: 0,
+            rows: [],
+            entityMap: new Map(),
+            processingTime: Date.now() - startTime,
+            stats: {
+                byType: {},
+                byConfidence: { high: 0, medium: 0, low: 0 }
+            }
+        };
+    }
+}
+//# sourceMappingURL=SmartExcelImporter.js.map

package/dist/importers/SmartImportOrchestrator.d.ts

@@ -0,0 +1,125 @@
+/**
+ * Smart Import Orchestrator
+ *
+ * Coordinates the entire smart import pipeline:
+ * 1. Extract entities/relationships using SmartExcelImporter
+ * 2. Create entities and relationships in Brainy
+ * 3. Organize into VFS structure using VFSStructureGenerator
+ *
+ * NO MOCKS - Production-ready implementation
+ */
+import { Brainy } from '../brainy.js';
+import { SmartExcelOptions, SmartExcelResult } from './SmartExcelImporter.js';
+import { SmartPDFOptions } from './SmartPDFImporter.js';
+import { SmartCSVOptions } from './SmartCSVImporter.js';
+import { SmartJSONOptions } from './SmartJSONImporter.js';
+import { SmartMarkdownOptions } from './SmartMarkdownImporter.js';
+export interface SmartImportOptions extends SmartExcelOptions {
+    /** Create VFS structure */
+    createVFSStructure?: boolean;
+    /** VFS root path */
+    vfsRootPath?: string;
+    /** VFS grouping strategy */
+    vfsGroupBy?: 'type' | 'sheet' | 'flat' | 'custom';
+    /** Create entities in Brainy */
+    createEntities?: boolean;
+    /** Create relationships in Brainy */
+    createRelationships?: boolean;
+    /** Source filename */
+    filename?: string;
+}
+export interface SmartImportProgress {
+    phase: 'parsing' | 'extracting' | 'creating' | 'organizing' | 'complete';
+    message: string;
+    processed: number;
+    total: number;
+    entities: number;
+    relationships: number;
+}
+export interface SmartImportResult {
+    success: boolean;
+    /** Extraction results */
+    extraction: SmartExcelResult;
+    /** Created entity IDs */
+    entityIds: string[];
+    /** Created relationship IDs */
+    relationshipIds: string[];
+    /** VFS structure created */
+    vfsStructure?: {
+        rootPath: string;
+        directories: string[];
+        files: number;
+    };
+    /** Overall statistics */
+    stats: {
+        rowsProcessed: number;
+        entitiesCreated: number;
+        relationshipsCreated: number;
+        filesCreated: number;
+        totalTime: number;
+    };
+    /** Any errors encountered */
+    errors: string[];
+}
+/**
+ * SmartImportOrchestrator - Main entry point for smart imports
+ */
+export declare class SmartImportOrchestrator {
+    private brain;
+    private excelImporter;
+    private pdfImporter;
+    private csvImporter;
+    private jsonImporter;
+    private markdownImporter;
+    private vfsGenerator;
+    constructor(brain: Brainy);
+    /**
+     * Initialize the orchestrator
+     */
+    init(): Promise<void>;
+    /**
+     * Import Excel file with full pipeline
+     */
+    importExcel(buffer: Buffer, options?: SmartImportOptions, onProgress?: (progress: SmartImportProgress) => void): Promise<SmartImportResult>;
+    /**
+     * Import PDF file with full pipeline
+     */
+    importPDF(buffer: Buffer, options?: SmartImportOptions & SmartPDFOptions, onProgress?: (progress: SmartImportProgress) => void): Promise<SmartImportResult>;
+    /**
+     * Import CSV file with full pipeline
+     */
+    importCSV(buffer: Buffer, options?: SmartImportOptions & SmartCSVOptions, onProgress?: (progress: SmartImportProgress) => void): Promise<SmartImportResult>;
+    /**
+     * Import JSON data with full pipeline
+     */
+    importJSON(data: any, options?: SmartImportOptions & SmartJSONOptions, onProgress?: (progress: SmartImportProgress) => void): Promise<SmartImportResult>;
+    /**
+     * Import Markdown content with full pipeline
+     */
+    importMarkdown(markdown: string, options?: SmartImportOptions & SmartMarkdownOptions, onProgress?: (progress: SmartImportProgress) => void): Promise<SmartImportResult>;
+    /**
+     * Helper: Create entities and relationships from extraction result
+     */
+    private createEntitiesAndRelationships;
+    /**
+     * Helper: Convert PDF result to Excel-like format
+     */
+    private convertPDFToExcelFormat;
+    /**
+     * Helper: Convert JSON result to Excel-like format
+     */
+    private convertJSONToExcelFormat;
+    /**
+     * Helper: Convert Markdown result to Excel-like format
+     */
+    private convertMarkdownToExcelFormat;
+    /**
+     * Get import statistics
+     */
+    getImportStatistics(vfsRootPath: string): Promise<{
+        entitiesInGraph: number;
+        relationshipsInGraph: number;
+        filesInVFS: number;
+        lastImport?: Date;
+    }>;
+}