@soulcraft/brainy 3.27.1 → 3.29.0

package/dist/import/ImportCoordinator.js

@@ -0,0 +1,498 @@
+/**
+ * Import Coordinator
+ *
+ * Unified import orchestrator that:
+ * - Auto-detects file formats
+ * - Routes to appropriate handlers
+ * - Coordinates dual storage (VFS + Graph)
+ * - Provides simple, unified API
+ *
+ * NO MOCKS - Production-ready implementation
+ */
+import { FormatDetector } from './FormatDetector.js';
+import { EntityDeduplicator } from './EntityDeduplicator.js';
+import { ImportHistory } from './ImportHistory.js';
+import { SmartExcelImporter } from '../importers/SmartExcelImporter.js';
+import { SmartPDFImporter } from '../importers/SmartPDFImporter.js';
+import { SmartCSVImporter } from '../importers/SmartCSVImporter.js';
+import { SmartJSONImporter } from '../importers/SmartJSONImporter.js';
+import { SmartMarkdownImporter } from '../importers/SmartMarkdownImporter.js';
+import { VFSStructureGenerator } from '../importers/VFSStructureGenerator.js';
+import { NounType } from '../types/graphTypes.js';
+import { v4 as uuidv4 } from '../universal/uuid.js';
+import * as fs from 'fs';
+import * as path from 'path';
+/**
+ * ImportCoordinator - Main entry point for all imports
+ */
+export class ImportCoordinator {
+    constructor(brain) {
+        this.brain = brain;
+        this.detector = new FormatDetector();
+        this.deduplicator = new EntityDeduplicator(brain);
+        this.history = new ImportHistory(brain);
+        this.excelImporter = new SmartExcelImporter(brain);
+        this.pdfImporter = new SmartPDFImporter(brain);
+        this.csvImporter = new SmartCSVImporter(brain);
+        this.jsonImporter = new SmartJSONImporter(brain);
+        this.markdownImporter = new SmartMarkdownImporter(brain);
+        this.vfsGenerator = new VFSStructureGenerator(brain);
+    }
+    /**
+     * Initialize all importers
+     */
+    async init() {
+        await this.excelImporter.init();
+        await this.pdfImporter.init();
+        await this.csvImporter.init();
+        await this.jsonImporter.init();
+        await this.markdownImporter.init();
+        await this.vfsGenerator.init();
+        await this.history.init();
+    }
+    /**
+     * Get import history
+     */
+    getHistory() {
+        return this.history;
+    }
+    /**
+     * Import from any source with auto-detection
+     */
+    async import(source, options = {}) {
+        const startTime = Date.now();
+        const importId = uuidv4();
+        // Normalize source
+        const normalizedSource = this.normalizeSource(source, options.format);
+        // Report detection stage
+        options.onProgress?.({
+            stage: 'detecting',
+            message: 'Detecting format...'
+        });
+        // Detect format
+        const detection = options.format
+            ? { format: options.format, confidence: 1.0, evidence: ['Explicitly specified'] }
+            : this.detectFormat(normalizedSource);
+        if (!detection) {
+            throw new Error('Unable to detect file format. Please specify format explicitly.');
+        }
+        // Report extraction stage
+        options.onProgress?.({
+            stage: 'extracting',
+            message: `Extracting entities from ${detection.format}...`
+        });
+        // Extract entities and relationships
+        const extractionResult = await this.extract(normalizedSource, detection.format, options);
+        // Set defaults
+        const opts = {
+            vfsPath: options.vfsPath || `/imports/${Date.now()}`,
+            groupBy: options.groupBy || 'type',
+            createEntities: options.createEntities !== false,
+            createRelationships: options.createRelationships !== false,
+            preserveSource: options.preserveSource !== false,
+            enableDeduplication: options.enableDeduplication !== false,
+            deduplicationThreshold: options.deduplicationThreshold || 0.85,
+            ...options
+        };
+        // Report VFS storage stage
+        options.onProgress?.({
+            stage: 'storing-vfs',
+            message: 'Creating VFS structure...'
+        });
+        // Normalize extraction result to unified format
+        const normalizedResult = this.normalizeExtractionResult(extractionResult, detection.format);
+        // Create VFS structure
+        const vfsResult = await this.vfsGenerator.generate(normalizedResult, {
+            rootPath: opts.vfsPath,
+            groupBy: opts.groupBy,
+            customGrouping: opts.customGrouping,
+            preserveSource: opts.preserveSource,
+            sourceBuffer: normalizedSource.type === 'buffer' ? normalizedSource.data : undefined,
+            sourceFilename: normalizedSource.filename || `import.${detection.format}`,
+            createRelationshipFile: true,
+            createMetadataFile: true
+        });
+        // Report graph storage stage
+        options.onProgress?.({
+            stage: 'storing-graph',
+            message: 'Creating knowledge graph...'
+        });
+        // Create entities and relationships in graph
+        const graphResult = await this.createGraphEntities(normalizedResult, vfsResult, opts);
+        // Report complete
+        options.onProgress?.({
+            stage: 'complete',
+            message: 'Import complete',
+            entities: graphResult.entities.length,
+            relationships: graphResult.relationships.length
+        });
+        const result = {
+            importId,
+            format: detection.format,
+            formatConfidence: detection.confidence,
+            vfs: {
+                rootPath: vfsResult.rootPath,
+                directories: vfsResult.directories,
+                files: vfsResult.files
+            },
+            entities: graphResult.entities,
+            relationships: graphResult.relationships,
+            stats: {
+                entitiesExtracted: extractionResult.entitiesExtracted,
+                relationshipsInferred: extractionResult.relationshipsInferred,
+                vfsFilesCreated: vfsResult.files.length,
+                graphNodesCreated: graphResult.entities.length,
+                graphEdgesCreated: graphResult.relationships.length,
+                entitiesMerged: graphResult.merged || 0,
+                entitiesNew: graphResult.newEntities || 0,
+                processingTime: Date.now() - startTime
+            }
+        };
+        // Record in history if enabled
+        if (options.enableHistory !== false) {
+            await this.history.recordImport(importId, {
+                type: normalizedSource.type === 'path' ? 'file' : normalizedSource.type,
+                filename: normalizedSource.filename,
+                format: detection.format
+            }, result);
+        }
+        return result;
+    }
+    /**
+     * Normalize source to ImportSource
+     */
+    normalizeSource(source, formatHint) {
+        // Buffer
+        if (Buffer.isBuffer(source)) {
+            return {
+                type: 'buffer',
+                data: source
+            };
+        }
+        // String - could be path or content
+        if (typeof source === 'string') {
+            // Check if it's a file path
+            if (this.isFilePath(source)) {
+                const buffer = fs.readFileSync(source);
+                return {
+                    type: 'path',
+                    data: buffer,
+                    filename: path.basename(source)
+                };
+            }
+            // Otherwise treat as content
+            return {
+                type: 'string',
+                data: source
+            };
+        }
+        // Object
+        if (typeof source === 'object' && source !== null) {
+            return {
+                type: 'object',
+                data: source
+            };
+        }
+        throw new Error('Invalid source type. Expected Buffer, string, or object.');
+    }
+    /**
+     * Check if string is a file path
+     */
+    isFilePath(str) {
+        // Check if file exists
+        try {
+            return fs.existsSync(str) && fs.statSync(str).isFile();
+        }
+        catch {
+            return false;
+        }
+    }
+    /**
+     * Detect format from source
+     */
+    detectFormat(source) {
+        switch (source.type) {
+            case 'buffer':
+            case 'path':
+                const buffer = source.data;
+                let result = this.detector.detectFromBuffer(buffer);
+                // Try filename hint if buffer detection fails
+                if (!result && source.filename) {
+                    result = this.detector.detectFromPath(source.filename);
+                }
+                return result;
+            case 'string':
+                return this.detector.detectFromString(source.data);
+            case 'object':
+                return this.detector.detectFromObject(source.data);
+        }
+    }
+    /**
+     * Extract entities using format-specific importer
+     */
+    async extract(source, format, options) {
+        const extractOptions = {
+            enableNeuralExtraction: options.enableNeuralExtraction !== false,
+            enableRelationshipInference: options.enableRelationshipInference !== false,
+            enableConceptExtraction: options.enableConceptExtraction !== false,
+            confidenceThreshold: options.confidenceThreshold || 0.6,
+            onProgress: (stats) => {
+                options.onProgress?.({
+                    stage: 'extracting',
+                    message: `Extracting entities from ${format}...`,
+                    processed: stats.processed,
+                    total: stats.total,
+                    entities: stats.entities,
+                    relationships: stats.relationships
+                });
+            }
+        };
+        switch (format) {
+            case 'excel':
+                const buffer = source.type === 'buffer' || source.type === 'path'
+                    ? source.data
+                    : Buffer.from(JSON.stringify(source.data));
+                return await this.excelImporter.extract(buffer, extractOptions);
+            case 'pdf':
+                const pdfBuffer = source.data;
+                return await this.pdfImporter.extract(pdfBuffer, extractOptions);
+            case 'csv':
+                const csvBuffer = source.type === 'buffer' || source.type === 'path'
+                    ? source.data
+                    : Buffer.from(source.data);
+                return await this.csvImporter.extract(csvBuffer, extractOptions);
+            case 'json':
+                const jsonData = source.type === 'object'
+                    ? source.data
+                    : source.type === 'string'
+                        ? source.data
+                        : source.data.toString('utf8');
+                return await this.jsonImporter.extract(jsonData, extractOptions);
+            case 'markdown':
+                const mdContent = source.type === 'string'
+                    ? source.data
+                    : source.data.toString('utf8');
+                return await this.markdownImporter.extract(mdContent, extractOptions);
+            default:
+                throw new Error(`Unsupported format: ${format}`);
+        }
+    }
+    /**
+     * Create entities and relationships in knowledge graph
+     */
+    async createGraphEntities(extractionResult, vfsResult, options) {
+        const entities = [];
+        const relationships = [];
+        let mergedCount = 0;
+        let newCount = 0;
+        if (!options.createEntities) {
+            return { entities, relationships, merged: 0, newEntities: 0 };
+        }
+        // Extract rows/sections/entities from result (unified across formats)
+        const rows = extractionResult.rows || extractionResult.sections || extractionResult.entities || [];
+        // Create entities in graph
+        for (const row of rows) {
+            const entity = row.entity || row;
+            // Find corresponding VFS file
+            const vfsFile = vfsResult.files.find((f) => f.entityId === entity.id);
+            // Create or merge entity
+            try {
+                const importSource = vfsResult.rootPath;
+                let entityId;
+                let wasMerged = false;
+                if (options.enableDeduplication) {
+                    // Use deduplicator to check for existing entities
+                    const mergeResult = await this.deduplicator.createOrMerge({
+                        id: entity.id,
+                        name: entity.name,
+                        type: entity.type,
+                        description: entity.description || entity.name,
+                        confidence: entity.confidence,
+                        metadata: {
+                            ...entity.metadata,
+                            vfsPath: vfsFile?.path,
+                            importedFrom: 'import-coordinator'
+                        }
+                    }, importSource, {
+                        similarityThreshold: options.deduplicationThreshold || 0.85,
+                        strictTypeMatching: true,
+                        enableFuzzyMatching: true
+                    });
+                    entityId = mergeResult.mergedEntityId;
+                    wasMerged = mergeResult.wasMerged;
+                    if (wasMerged) {
+                        mergedCount++;
+                    }
+                    else {
+                        newCount++;
+                    }
+                }
+                else {
+                    // Direct creation without deduplication
+                    entityId = await this.brain.add({
+                        data: entity.description || entity.name,
+                        type: entity.type,
+                        metadata: {
+                            ...entity.metadata,
+                            name: entity.name,
+                            confidence: entity.confidence,
+                            vfsPath: vfsFile?.path,
+                            importedAt: Date.now(),
+                            importedFrom: 'import-coordinator',
+                            imports: [importSource]
+                        }
+                    });
+                    newCount++;
+                }
+                // Update entity ID in extraction result
+                entity.id = entityId;
+                entities.push({
+                    id: entityId,
+                    name: entity.name,
+                    type: entity.type,
+                    vfsPath: vfsFile?.path
+                });
+                // Create relationships if enabled
+                if (options.createRelationships && row.relationships) {
+                    for (const rel of row.relationships) {
+                        try {
+                            // Find or create target entity
+                            let targetEntityId;
+                            // Check if target already exists in our entities list
+                            const existingTarget = entities.find(e => e.name.toLowerCase() === rel.to.toLowerCase());
+                            if (existingTarget) {
+                                targetEntityId = existingTarget.id;
+                            }
+                            else {
+                                // Try to find in other extracted entities
+                                for (const otherRow of rows) {
+                                    const otherEntity = otherRow.entity || otherRow;
+                                    if (rel.to.toLowerCase().includes(otherEntity.name.toLowerCase()) ||
+                                        otherEntity.name.toLowerCase().includes(rel.to.toLowerCase())) {
+                                        targetEntityId = otherEntity.id;
+                                        break;
+                                    }
+                                }
+                                // If still not found, create placeholder entity
+                                if (!targetEntityId) {
+                                    targetEntityId = await this.brain.add({
+                                        data: rel.to,
+                                        type: NounType.Thing,
+                                        metadata: {
+                                            name: rel.to,
+                                            placeholder: true,
+                                            inferredFrom: entity.name,
+                                            importedAt: Date.now()
+                                        }
+                                    });
+                                    entities.push({
+                                        id: targetEntityId,
+                                        name: rel.to,
+                                        type: NounType.Thing
+                                    });
+                                }
+                            }
+                            // Create relationship using brain.relate()
+                            const relId = await this.brain.relate({
+                                from: entityId,
+                                to: targetEntityId,
+                                type: rel.type,
+                                metadata: {
+                                    confidence: rel.confidence,
+                                    evidence: rel.evidence,
+                                    importedAt: Date.now()
+                                }
+                            });
+                            relationships.push({
+                                id: relId,
+                                from: entityId,
+                                to: targetEntityId,
+                                type: rel.type
+                            });
+                        }
+                        catch (error) {
+                            // Skip relationship creation errors (entity might not exist, etc.)
+                            continue;
+                        }
+                    }
+                }
+            }
+            catch (error) {
+                // Skip entity creation errors (might already exist, etc.)
+                continue;
+            }
+        }
+        return {
+            entities,
+            relationships,
+            merged: mergedCount,
+            newEntities: newCount
+        };
+    }
+    /**
+     * Normalize extraction result to unified format (Excel-like structure)
+     */
+    normalizeExtractionResult(result, format) {
+        // Excel and CSV already have the right format
+        if (format === 'excel' || format === 'csv') {
+            return result;
+        }
+        // PDF: sections -> rows
+        if (format === 'pdf') {
+            const rows = result.sections.flatMap((section) => section.entities.map((entity) => ({
+                entity,
+                relatedEntities: [],
+                relationships: section.relationships.filter((r) => r.from === entity.id),
+                concepts: section.concepts || []
+            })));
+            return {
+                rowsProcessed: result.sectionsProcessed,
+                entitiesExtracted: result.entitiesExtracted,
+                relationshipsInferred: result.relationshipsInferred,
+                rows,
+                entityMap: result.entityMap,
+                processingTime: result.processingTime,
+                stats: result.stats
+            };
+        }
+        // JSON: entities -> rows
+        if (format === 'json') {
+            const rows = result.entities.map((entity) => ({
+                entity,
+                relatedEntities: [],
+                relationships: result.relationships.filter((r) => r.from === entity.id),
+                concepts: entity.metadata?.concepts || []
+            }));
+            return {
+                rowsProcessed: result.nodesProcessed,
+                entitiesExtracted: result.entitiesExtracted,
+                relationshipsInferred: result.relationshipsInferred,
+                rows,
+                entityMap: result.entityMap,
+                processingTime: result.processingTime,
+                stats: result.stats
+            };
+        }
+        // Markdown: sections -> rows
+        if (format === 'markdown') {
+            const rows = result.sections.flatMap((section) => section.entities.map((entity) => ({
+                entity,
+                relatedEntities: [],
+                relationships: section.relationships.filter((r) => r.from === entity.id),
+                concepts: section.concepts || []
+            })));
+            return {
+                rowsProcessed: result.sectionsProcessed,
+                entitiesExtracted: result.entitiesExtracted,
+                relationshipsInferred: result.relationshipsInferred,
+                rows,
+                entityMap: result.entityMap,
+                processingTime: result.processingTime,
+                stats: result.stats
+            };
+        }
+        // Fallback: return as-is
+        return result;
+    }
+}
+//# sourceMappingURL=ImportCoordinator.js.map

package/dist/import/ImportHistory.d.ts

@@ -0,0 +1,92 @@
+/**
+ * Import History & Rollback (Phase 4)
+ *
+ * Tracks all imports with:
+ * - Complete metadata and provenance
+ * - Entity and relationship tracking
+ * - Rollback capability
+ * - Import statistics
+ *
+ * NO MOCKS - Production-ready implementation
+ */
+import { Brainy } from '../brainy.js';
+import type { ImportResult } from './ImportCoordinator.js';
+export interface ImportHistoryEntry {
+    /** Unique import ID */
+    importId: string;
+    /** Import timestamp */
+    timestamp: number;
+    /** Source information */
+    source: {
+        type: 'file' | 'buffer' | 'object' | 'string';
+        filename?: string;
+        format: string;
+    };
+    /** Import results */
+    result: ImportResult;
+    /** Entities created in this import */
+    entities: string[];
+    /** Relationships created in this import */
+    relationships: string[];
+    /** VFS paths created */
+    vfsPaths: string[];
+    /** Import status */
+    status: 'success' | 'partial' | 'failed';
+    /** Error messages (if any) */
+    errors?: string[];
+}
+export interface RollbackResult {
+    /** Was rollback successful */
+    success: boolean;
+    /** Entities deleted */
+    entitiesDeleted: number;
+    /** Relationships deleted */
+    relationshipsDeleted: number;
+    /** VFS files deleted */
+    vfsFilesDeleted: number;
+    /** Errors encountered */
+    errors: string[];
+}
+/**
+ * ImportHistory - Track and manage import history with rollback
+ */
+export declare class ImportHistory {
+    private brain;
+    private history;
+    private historyFile;
+    constructor(brain: Brainy, historyFile?: string);
+    /**
+     * Initialize history (load from VFS if exists)
+     */
+    init(): Promise<void>;
+    /**
+     * Record an import
+     */
+    recordImport(importId: string, source: ImportHistoryEntry['source'], result: ImportResult): Promise<void>;
+    /**
+     * Get import history
+     */
+    getHistory(): ImportHistoryEntry[];
+    /**
+     * Get specific import
+     */
+    getImport(importId: string): ImportHistoryEntry | null;
+    /**
+     * Rollback an import (delete all entities, relationships, VFS files)
+     */
+    rollback(importId: string): Promise<RollbackResult>;
+    /**
+     * Get import statistics
+     */
+    getStatistics(): {
+        totalImports: number;
+        totalEntities: number;
+        totalRelationships: number;
+        byFormat: Record<string, number>;
+        byStatus: Record<string, number>;
+    };
+    /**
+     * Persist history to VFS
+     */
+    private persist;
+}