@soulcraft/brainy 3.27.1 → 3.28.0

package/dist/import/ImportHistory.js

@@ -0,0 +1,183 @@
+/**
+ * Import History & Rollback (Phase 4)
+ *
+ * Tracks all imports with:
+ * - Complete metadata and provenance
+ * - Entity and relationship tracking
+ * - Rollback capability
+ * - Import statistics
+ *
+ * NO MOCKS - Production-ready implementation
+ */
+/**
+ * ImportHistory - Track and manage import history with rollback
+ */
+export class ImportHistory {
+    constructor(brain, historyFile = '/.brainy/import_history.json') {
+        this.brain = brain;
+        this.history = new Map();
+        this.historyFile = historyFile;
+    }
+    /**
+     * Initialize history (load from VFS if exists)
+     */
+    async init() {
+        try {
+            const vfs = this.brain.vfs();
+            await vfs.init();
+            // Try to load existing history
+            const content = await vfs.readFile(this.historyFile);
+            const data = JSON.parse(content.toString('utf-8'));
+            this.history = new Map(Object.entries(data));
+        }
+        catch (error) {
+            // No existing history or VFS not available, start fresh
+            this.history = new Map();
+        }
+    }
+    /**
+     * Record an import
+     */
+    async recordImport(importId, source, result) {
+        const entry = {
+            importId,
+            timestamp: Date.now(),
+            source,
+            result,
+            entities: result.entities.map(e => e.id),
+            relationships: result.relationships.map(r => r.id),
+            vfsPaths: result.vfs.files.map(f => f.path),
+            status: result.stats.entitiesExtracted > 0 ? 'success' : 'partial'
+        };
+        this.history.set(importId, entry);
+        // Persist to VFS
+        await this.persist();
+    }
+    /**
+     * Get import history
+     */
+    getHistory() {
+        return Array.from(this.history.values()).sort((a, b) => b.timestamp - a.timestamp);
+    }
+    /**
+     * Get specific import
+     */
+    getImport(importId) {
+        return this.history.get(importId) || null;
+    }
+    /**
+     * Rollback an import (delete all entities, relationships, VFS files)
+     */
+    async rollback(importId) {
+        const entry = this.history.get(importId);
+        if (!entry) {
+            throw new Error(`Import ${importId} not found in history`);
+        }
+        const result = {
+            success: true,
+            entitiesDeleted: 0,
+            relationshipsDeleted: 0,
+            vfsFilesDeleted: 0,
+            errors: []
+        };
+        // Delete relationships first
+        for (const relId of entry.relationships) {
+            try {
+                await this.brain.unrelate(relId);
+                result.relationshipsDeleted++;
+            }
+            catch (error) {
+                result.errors.push(`Failed to delete relationship ${relId}: ${error instanceof Error ? error.message : String(error)}`);
+            }
+        }
+        // Delete entities
+        for (const entityId of entry.entities) {
+            try {
+                await this.brain.delete(entityId);
+                result.entitiesDeleted++;
+            }
+            catch (error) {
+                result.errors.push(`Failed to delete entity ${entityId}: ${error instanceof Error ? error.message : String(error)}`);
+            }
+        }
+        // Delete VFS files
+        try {
+            const vfs = this.brain.vfs();
+            await vfs.init();
+            for (const vfsPath of entry.vfsPaths) {
+                try {
+                    await vfs.unlink(vfsPath);
+                    result.vfsFilesDeleted++;
+                }
+                catch (error) {
+                    // File might not exist or VFS unavailable
+                    result.errors.push(`Failed to delete VFS file ${vfsPath}: ${error instanceof Error ? error.message : String(error)}`);
+                }
+            }
+            // Try to delete VFS root directory if empty
+            try {
+                const rootPath = entry.result.vfs.rootPath;
+                const contents = await vfs.readdir(rootPath);
+                if (contents.length === 0) {
+                    await vfs.rmdir(rootPath);
+                }
+            }
+            catch (error) {
+                // Ignore errors for directory cleanup
+            }
+        }
+        catch (error) {
+            result.errors.push(`VFS cleanup failed: ${error instanceof Error ? error.message : String(error)}`);
+        }
+        // Remove from history
+        this.history.delete(importId);
+        // Persist updated history
+        await this.persist();
+        result.success = result.errors.length === 0;
+        return result;
+    }
+    /**
+     * Get import statistics
+     */
+    getStatistics() {
+        const history = Array.from(this.history.values());
+        return {
+            totalImports: history.length,
+            totalEntities: history.reduce((sum, h) => sum + h.entities.length, 0),
+            totalRelationships: history.reduce((sum, h) => sum + h.relationships.length, 0),
+            byFormat: history.reduce((acc, h) => {
+                acc[h.source.format] = (acc[h.source.format] || 0) + 1;
+                return acc;
+            }, {}),
+            byStatus: history.reduce((acc, h) => {
+                acc[h.status] = (acc[h.status] || 0) + 1;
+                return acc;
+            }, {})
+        };
+    }
+    /**
+     * Persist history to VFS
+     */
+    async persist() {
+        try {
+            const vfs = this.brain.vfs();
+            await vfs.init();
+            // Ensure directory exists
+            const dir = this.historyFile.substring(0, this.historyFile.lastIndexOf('/'));
+            try {
+                await vfs.mkdir(dir, { recursive: true });
+            }
+            catch (error) {
+                // Directory might exist
+            }
+            // Convert Map to object for JSON
+            const data = Object.fromEntries(this.history);
+            await vfs.writeFile(this.historyFile, JSON.stringify(data, null, 2));
+        }
+        catch (error) {
+            // VFS might not be available, continue without persistence
+            console.warn('Failed to persist import history:', error instanceof Error ? error.message : String(error));
+        }
+    }
+}
+//# sourceMappingURL=ImportHistory.js.map

package/dist/import/index.d.ts

@@ -0,0 +1,16 @@
+/**
+ * Unified Import System
+ *
+ * Single entry point for importing any file format into Brainy with:
+ * - Auto-detection of formats
+ * - Dual storage (VFS + Knowledge Graph)
+ * - Shared entities across imports (deduplication)
+ * - Simple, powerful API
+ */
+export { ImportCoordinator } from './ImportCoordinator.js';
+export { FormatDetector, SupportedFormat, DetectionResult } from './FormatDetector.js';
+export { EntityDeduplicator } from './EntityDeduplicator.js';
+export { ImportHistory } from './ImportHistory.js';
+export type { ImportSource, ImportOptions, ImportProgress, ImportResult } from './ImportCoordinator.js';
+export type { EntityCandidate, DuplicateMatch, EntityDeduplicationOptions, MergeResult } from './EntityDeduplicator.js';
+export type { ImportHistoryEntry, RollbackResult } from './ImportHistory.js';

package/dist/import/index.js

@@ -0,0 +1,14 @@
+/**
+ * Unified Import System
+ *
+ * Single entry point for importing any file format into Brainy with:
+ * - Auto-detection of formats
+ * - Dual storage (VFS + Knowledge Graph)
+ * - Shared entities across imports (deduplication)
+ * - Simple, powerful API
+ */
+export { ImportCoordinator } from './ImportCoordinator.js';
+export { FormatDetector } from './FormatDetector.js';
+export { EntityDeduplicator } from './EntityDeduplicator.js';
+export { ImportHistory } from './ImportHistory.js';
+//# sourceMappingURL=index.js.map

package/dist/importers/SmartCSVImporter.d.ts

@@ -0,0 +1,136 @@
+/**
+ * Smart CSV Importer
+ *
+ * Extracts entities and relationships from CSV files using:
+ * - NeuralEntityExtractor for entity extraction
+ * - NaturalLanguageProcessor for relationship inference
+ * - brain.extractConcepts() for tagging
+ *
+ * Very similar to SmartExcelImporter but handles CSV-specific features
+ *
+ * NO MOCKS - Production-ready implementation
+ */
+import { Brainy } from '../brainy.js';
+import { NounType, VerbType } from '../types/graphTypes.js';
+import type { FormatHandlerOptions } from '../augmentations/intelligentImport/types.js';
+export interface SmartCSVOptions extends FormatHandlerOptions {
+    /** Enable neural entity extraction */
+    enableNeuralExtraction?: boolean;
+    /** Enable relationship inference from text */
+    enableRelationshipInference?: boolean;
+    /** Enable concept extraction for tagging */
+    enableConceptExtraction?: boolean;
+    /** Confidence threshold for entities (0-1) */
+    confidenceThreshold?: number;
+    /** Column name patterns to detect */
+    termColumn?: string;
+    definitionColumn?: string;
+    typeColumn?: string;
+    relatedColumn?: string;
+    /** CSV-specific options */
+    csvDelimiter?: string;
+    csvHeaders?: boolean;
+    /** Progress callback */
+    onProgress?: (stats: {
+        processed: number;
+        total: number;
+        entities: number;
+        relationships: number;
+    }) => void;
+}
+export interface ExtractedRow {
+    /** Main entity from this row */
+    entity: {
+        id: string;
+        name: string;
+        type: NounType;
+        description: string;
+        confidence: number;
+        metadata: Record<string, any>;
+    };
+    /** Additional entities extracted from definition */
+    relatedEntities: Array<{
+        name: string;
+        type: NounType;
+        confidence: number;
+    }>;
+    /** Inferred relationships */
+    relationships: Array<{
+        from: string;
+        to: string;
+        type: VerbType;
+        confidence: number;
+        evidence: string;
+    }>;
+    /** Extracted concepts */
+    concepts?: string[];
+}
+export interface SmartCSVResult {
+    /** Total rows processed */
+    rowsProcessed: number;
+    /** Entities extracted (includes main + related) */
+    entitiesExtracted: number;
+    /** Relationships inferred */
+    relationshipsInferred: number;
+    /** All extracted data */
+    rows: ExtractedRow[];
+    /** Entity ID mapping (name -> ID) */
+    entityMap: Map<string, string>;
+    /** Processing time in ms */
+    processingTime: number;
+    /** Extraction statistics */
+    stats: {
+        byType: Record<string, number>;
+        byConfidence: {
+            high: number;
+            medium: number;
+            low: number;
+        };
+    };
+}
+/**
+ * SmartCSVImporter - Extracts structured knowledge from CSV files
+ */
+export declare class SmartCSVImporter {
+    private brain;
+    private extractor;
+    private nlp;
+    private csvHandler;
+    constructor(brain: Brainy);
+    /**
+     * Initialize the importer
+     */
+    init(): Promise<void>;
+    /**
+     * Extract entities and relationships from CSV file
+     */
+    extract(buffer: Buffer, options?: SmartCSVOptions): Promise<SmartCSVResult>;
+    /**
+     * Detect column names from first row
+     */
+    private detectColumns;
+    /**
+     * Get value from row using column name
+     */
+    private getColumnValue;
+    /**
+     * Map type string to NounType
+     */
+    private mapTypeString;
+    /**
+     * Infer relationship type from context
+     */
+    private inferRelationship;
+    /**
+     * Generate consistent entity ID from name
+     */
+    private generateEntityId;
+    /**
+     * Update statistics
+     */
+    private updateStats;
+    /**
+     * Create empty result
+     */
+    private emptyResult;
+}

package/dist/importers/SmartCSVImporter.js

@@ -0,0 +1,308 @@
+/**
+ * Smart CSV Importer
+ *
+ * Extracts entities and relationships from CSV files using:
+ * - NeuralEntityExtractor for entity extraction
+ * - NaturalLanguageProcessor for relationship inference
+ * - brain.extractConcepts() for tagging
+ *
+ * Very similar to SmartExcelImporter but handles CSV-specific features
+ *
+ * NO MOCKS - Production-ready implementation
+ */
+import { NeuralEntityExtractor } from '../neural/entityExtractor.js';
+import { NaturalLanguageProcessor } from '../neural/naturalLanguageProcessor.js';
+import { NounType, VerbType } from '../types/graphTypes.js';
+import { CSVHandler } from '../augmentations/intelligentImport/handlers/csvHandler.js';
+/**
+ * SmartCSVImporter - Extracts structured knowledge from CSV files
+ */
+export class SmartCSVImporter {
+    constructor(brain) {
+        this.brain = brain;
+        this.extractor = new NeuralEntityExtractor(brain);
+        this.nlp = new NaturalLanguageProcessor(brain);
+        this.csvHandler = new CSVHandler();
+    }
+    /**
+     * Initialize the importer
+     */
+    async init() {
+        await this.nlp.init();
+    }
+    /**
+     * Extract entities and relationships from CSV file
+     */
+    async extract(buffer, options = {}) {
+        const startTime = Date.now();
+        // Set defaults
+        const opts = {
+            enableNeuralExtraction: true,
+            enableRelationshipInference: true,
+            enableConceptExtraction: true,
+            confidenceThreshold: 0.6,
+            termColumn: 'term|name|title|concept|entity',
+            definitionColumn: 'definition|description|desc|details|text',
+            typeColumn: 'type|category|kind|class',
+            relatedColumn: 'related|see also|links|references',
+            csvDelimiter: undefined,
+            csvHeaders: true,
+            onProgress: () => { },
+            ...options
+        };
+        // Parse CSV using existing handler
+        const processedData = await this.csvHandler.process(buffer, {
+            ...options,
+            csvDelimiter: opts.csvDelimiter,
+            csvHeaders: opts.csvHeaders
+        });
+        const rows = processedData.data;
+        if (rows.length === 0) {
+            return this.emptyResult(startTime);
+        }
+        // Detect column names
+        const columns = this.detectColumns(rows[0], opts);
+        // Process each row
+        const extractedRows = [];
+        const entityMap = new Map();
+        const stats = {
+            byType: {},
+            byConfidence: { high: 0, medium: 0, low: 0 }
+        };
+        for (let i = 0; i < rows.length; i++) {
+            const row = rows[i];
+            // Extract data from row
+            const term = this.getColumnValue(row, columns.term) || `Entity_${i}`;
+            const definition = this.getColumnValue(row, columns.definition) || '';
+            const type = this.getColumnValue(row, columns.type);
+            const relatedTerms = this.getColumnValue(row, columns.related);
+            // Extract entities from definition
+            let relatedEntities = [];
+            if (opts.enableNeuralExtraction && definition) {
+                relatedEntities = await this.extractor.extract(definition, {
+                    confidence: opts.confidenceThreshold * 0.8, // Lower threshold for related entities
+                    neuralMatching: true,
+                    cache: { enabled: true }
+                });
+                // Filter out the main term from related entities
+                relatedEntities = relatedEntities.filter(e => e.text.toLowerCase() !== term.toLowerCase());
+            }
+            // Determine main entity type
+            const mainEntityType = type ?
+                this.mapTypeString(type) :
+                (relatedEntities.length > 0 ? relatedEntities[0].type : NounType.Thing);
+            // Generate entity ID
+            const entityId = this.generateEntityId(term);
+            entityMap.set(term.toLowerCase(), entityId);
+            // Extract concepts
+            let concepts = [];
+            if (opts.enableConceptExtraction && definition) {
+                try {
+                    concepts = await this.brain.extractConcepts(definition, { limit: 10 });
+                }
+                catch (error) {
+                    concepts = [];
+                }
+            }
+            // Create main entity
+            const mainEntity = {
+                id: entityId,
+                name: term,
+                type: mainEntityType,
+                description: definition,
+                confidence: 0.95, // Main entity from row has high confidence
+                metadata: {
+                    source: 'csv',
+                    row: i + 1,
+                    originalData: row,
+                    concepts,
+                    extractedAt: Date.now()
+                }
+            };
+            // Track statistics
+            this.updateStats(stats, mainEntityType, mainEntity.confidence);
+            // Infer relationships
+            const relationships = [];
+            if (opts.enableRelationshipInference) {
+                // Extract relationships from definition text
+                for (const relEntity of relatedEntities) {
+                    const verbType = await this.inferRelationship(term, relEntity.text, definition);
+                    relationships.push({
+                        from: entityId,
+                        to: relEntity.text,
+                        type: verbType,
+                        confidence: relEntity.confidence,
+                        evidence: `Extracted from: "${definition.substring(0, 100)}..."`
+                    });
+                }
+                // Parse explicit "Related" column
+                if (relatedTerms) {
+                    const terms = relatedTerms.split(/[,;|]/).map(t => t.trim()).filter(Boolean);
+                    for (const relTerm of terms) {
+                        // Ensure we don't create self-relationships
+                        if (relTerm.toLowerCase() !== term.toLowerCase()) {
+                            relationships.push({
+                                from: entityId,
+                                to: relTerm,
+                                type: VerbType.RelatedTo,
+                                confidence: 0.9,
+                                evidence: `Explicitly listed in "Related" column`
+                            });
+                        }
+                    }
+                }
+            }
+            // Add extracted row
+            extractedRows.push({
+                entity: mainEntity,
+                relatedEntities: relatedEntities.map(e => ({
+                    name: e.text,
+                    type: e.type,
+                    confidence: e.confidence
+                })),
+                relationships,
+                concepts
+            });
+            // Report progress
+            opts.onProgress({
+                processed: i + 1,
+                total: rows.length,
+                entities: extractedRows.length + relatedEntities.length,
+                relationships: relationships.length
+            });
+        }
+        return {
+            rowsProcessed: rows.length,
+            entitiesExtracted: extractedRows.reduce((sum, row) => sum + 1 + row.relatedEntities.length, 0),
+            relationshipsInferred: extractedRows.reduce((sum, row) => sum + row.relationships.length, 0),
+            rows: extractedRows,
+            entityMap,
+            processingTime: Date.now() - startTime,
+            stats
+        };
+    }
+    /**
+     * Detect column names from first row
+     */
+    detectColumns(firstRow, options) {
+        const columnNames = Object.keys(firstRow);
+        const matchColumn = (pattern) => {
+            const regex = new RegExp(pattern, 'i');
+            return columnNames.find(col => regex.test(col)) || null;
+        };
+        return {
+            term: matchColumn(options.termColumn || 'term|name'),
+            definition: matchColumn(options.definitionColumn || 'definition|description'),
+            type: matchColumn(options.typeColumn || 'type|category'),
+            related: matchColumn(options.relatedColumn || 'related|see also')
+        };
+    }
+    /**
+     * Get value from row using column name
+     */
+    getColumnValue(row, columnName) {
+        if (!columnName)
+            return '';
+        const value = row[columnName];
+        if (value === null || value === undefined)
+            return '';
+        return String(value).trim();
+    }
+    /**
+     * Map type string to NounType
+     */
+    mapTypeString(typeString) {
+        const normalized = typeString.toLowerCase().trim();
+        const mapping = {
+            'person': NounType.Person,
+            'character': NounType.Person,
+            'people': NounType.Person,
+            'place': NounType.Location,
+            'location': NounType.Location,
+            'geography': NounType.Location,
+            'organization': NounType.Organization,
+            'org': NounType.Organization,
+            'company': NounType.Organization,
+            'concept': NounType.Concept,
+            'idea': NounType.Concept,
+            'theory': NounType.Concept,
+            'event': NounType.Event,
+            'occurrence': NounType.Event,
+            'product': NounType.Product,
+            'item': NounType.Product,
+            'thing': NounType.Thing,
+            'document': NounType.Document,
+            'file': NounType.File,
+            'project': NounType.Project
+        };
+        return mapping[normalized] || NounType.Thing;
+    }
+    /**
+     * Infer relationship type from context
+     */
+    async inferRelationship(fromTerm, toTerm, context) {
+        const lowerContext = context.toLowerCase();
+        // Pattern-based relationship detection
+        const patterns = [
+            [new RegExp(`${toTerm}.*of.*${fromTerm}`, 'i'), VerbType.PartOf],
+            [new RegExp(`${fromTerm}.*contains.*${toTerm}`, 'i'), VerbType.Contains],
+            [new RegExp(`located in.*${toTerm}`, 'i'), VerbType.LocatedAt],
+            [new RegExp(`ruled by.*${toTerm}`, 'i'), VerbType.Owns],
+            [new RegExp(`capital.*${toTerm}`, 'i'), VerbType.Contains],
+            [new RegExp(`created by.*${toTerm}`, 'i'), VerbType.CreatedBy],
+            [new RegExp(`authored by.*${toTerm}`, 'i'), VerbType.CreatedBy],
+            [new RegExp(`part of.*${toTerm}`, 'i'), VerbType.PartOf],
+            [new RegExp(`related to.*${toTerm}`, 'i'), VerbType.RelatedTo]
+        ];
+        for (const [pattern, verbType] of patterns) {
+            if (pattern.test(lowerContext)) {
+                return verbType;
+            }
+        }
+        // Default to RelatedTo
+        return VerbType.RelatedTo;
+    }
+    /**
+     * Generate consistent entity ID from name
+     */
+    generateEntityId(name) {
+        const normalized = name.toLowerCase().trim().replace(/\s+/g, '_');
+        return `ent_${normalized}_${Date.now()}`;
+    }
+    /**
+     * Update statistics
+     */
+    updateStats(stats, type, confidence) {
+        // Track by type
+        const typeName = String(type);
+        stats.byType[typeName] = (stats.byType[typeName] || 0) + 1;
+        // Track by confidence
+        if (confidence > 0.8) {
+            stats.byConfidence.high++;
+        }
+        else if (confidence >= 0.6) {
+            stats.byConfidence.medium++;
+        }
+        else {
+            stats.byConfidence.low++;
+        }
+    }
+    /**
+     * Create empty result
+     */
+    emptyResult(startTime) {
+        return {
+            rowsProcessed: 0,
+            entitiesExtracted: 0,
+            relationshipsInferred: 0,
+            rows: [],
+            entityMap: new Map(),
+            processingTime: Date.now() - startTime,
+            stats: {
+                byType: {},
+                byConfidence: { high: 0, medium: 0, low: 0 }
+            }
+        };
+    }
+}
+//# sourceMappingURL=SmartCSVImporter.js.map