@soulcraft/brainy 3.27.1 → 3.28.0

package/dist/importers/SmartPDFImporter.js

@@ -0,0 +1,337 @@
+/**
+ * Smart PDF Importer
+ *
+ * Extracts entities and relationships from PDF files using:
+ * - NeuralEntityExtractor for entity extraction
+ * - NaturalLanguageProcessor for relationship inference
+ * - brain.extractConcepts() for tagging
+ * - Section-based organization (by page or detected structure)
+ *
+ * NO MOCKS - Production-ready implementation
+ */
+import { NeuralEntityExtractor } from '../neural/entityExtractor.js';
+import { NaturalLanguageProcessor } from '../neural/naturalLanguageProcessor.js';
+import { VerbType } from '../types/graphTypes.js';
+import { PDFHandler } from '../augmentations/intelligentImport/handlers/pdfHandler.js';
+/**
+ * SmartPDFImporter - Extracts structured knowledge from PDF files
+ */
+export class SmartPDFImporter {
+    constructor(brain) {
+        this.brain = brain;
+        this.extractor = new NeuralEntityExtractor(brain);
+        this.nlp = new NaturalLanguageProcessor(brain);
+        this.pdfHandler = new PDFHandler();
+    }
+    /**
+     * Initialize the importer
+     */
+    async init() {
+        await this.nlp.init();
+    }
+    /**
+     * Extract entities and relationships from PDF file
+     */
+    async extract(buffer, options = {}) {
+        const startTime = Date.now();
+        // Set defaults
+        const opts = {
+            enableNeuralExtraction: true,
+            enableRelationshipInference: true,
+            enableConceptExtraction: true,
+            confidenceThreshold: 0.6,
+            minParagraphLength: 50,
+            extractFromTables: true,
+            groupBy: 'document',
+            onProgress: () => { },
+            ...options
+        };
+        // Parse PDF using existing handler
+        const processedData = await this.pdfHandler.process(buffer, options);
+        const data = processedData.data;
+        const pdfMetadata = processedData.metadata.additionalInfo?.pdfMetadata || {};
+        if (data.length === 0) {
+            return this.emptyResult(startTime, pdfMetadata);
+        }
+        // Group data by page or combine into single document
+        const grouped = this.groupData(data, opts);
+        // Process each group
+        const sections = [];
+        const entityMap = new Map();
+        const stats = {
+            byType: {},
+            byConfidence: { high: 0, medium: 0, low: 0 },
+            bySource: { paragraphs: 0, tables: 0 }
+        };
+        let processedCount = 0;
+        const totalGroups = grouped.length;
+        for (const group of grouped) {
+            const sectionResult = await this.processSection(group, opts, stats, entityMap);
+            sections.push(sectionResult);
+            processedCount++;
+            opts.onProgress({
+                processed: processedCount,
+                total: totalGroups,
+                entities: sections.reduce((sum, s) => sum + s.entities.length, 0),
+                relationships: sections.reduce((sum, s) => sum + s.relationships.length, 0)
+            });
+        }
+        const pagesProcessed = new Set(data.map(d => d._page)).size;
+        return {
+            sectionsProcessed: sections.length,
+            pagesProcessed,
+            entitiesExtracted: sections.reduce((sum, s) => sum + s.entities.length, 0),
+            relationshipsInferred: sections.reduce((sum, s) => sum + s.relationships.length, 0),
+            sections,
+            entityMap,
+            processingTime: Date.now() - startTime,
+            stats,
+            pdfMetadata: {
+                pageCount: pdfMetadata.pageCount || pagesProcessed,
+                title: pdfMetadata.title,
+                author: pdfMetadata.author,
+                subject: pdfMetadata.subject
+            }
+        };
+    }
+    /**
+     * Group data by strategy
+     */
+    groupData(data, options) {
+        if (options.groupBy === 'page') {
+            // Group by page
+            const pageGroups = new Map();
+            for (const item of data) {
+                const page = item._page || 1;
+                if (!pageGroups.has(page)) {
+                    pageGroups.set(page, []);
+                }
+                pageGroups.get(page).push(item);
+            }
+            return Array.from(pageGroups.entries()).map(([page, items]) => ({
+                id: `page_${page}`,
+                type: 'page',
+                items
+            }));
+        }
+        else {
+            // Single document group
+            return [{
+                    id: 'document',
+                    type: 'paragraph',
+                    items: data
+                }];
+        }
+    }
+    /**
+     * Process a single section
+     */
+    async processSection(group, options, stats, entityMap) {
+        // Combine all text from the group
+        const texts = [];
+        for (const item of group.items) {
+            if (item._type === 'paragraph') {
+                const text = item.text || '';
+                if (text.length >= (options.minParagraphLength || 50)) {
+                    texts.push(text);
+                    stats.bySource.paragraphs++;
+                }
+            }
+            else if (item._type === 'table_row' && options.extractFromTables) {
+                // For table rows, combine all column values
+                const values = Object.entries(item)
+                    .filter(([key]) => !key.startsWith('_'))
+                    .map(([_, value]) => String(value))
+                    .filter(Boolean);
+                if (values.length > 0) {
+                    texts.push(values.join(' '));
+                    stats.bySource.tables++;
+                }
+            }
+        }
+        const combinedText = texts.join('\n\n');
+        // Extract entities if enabled
+        let extractedEntities = [];
+        if (options.enableNeuralExtraction && combinedText.length > 0) {
+            extractedEntities = await this.extractor.extract(combinedText, {
+                confidence: options.confidenceThreshold || 0.6,
+                neuralMatching: true,
+                cache: { enabled: true }
+            });
+        }
+        // Extract concepts if enabled
+        let concepts = [];
+        if (options.enableConceptExtraction && combinedText.length > 0) {
+            try {
+                concepts = await this.brain.extractConcepts(combinedText, { limit: 15 });
+            }
+            catch (error) {
+                concepts = [];
+            }
+        }
+        // Create entity objects
+        const entities = extractedEntities.map(e => {
+            const entityId = this.generateEntityId(e.text, group.id);
+            entityMap.set(e.text.toLowerCase(), entityId);
+            // Update statistics
+            this.updateStats(stats, e.type, e.confidence);
+            return {
+                id: entityId,
+                name: e.text,
+                type: e.type,
+                description: this.extractContextAroundEntity(combinedText, e.text),
+                confidence: e.confidence,
+                metadata: {
+                    source: 'pdf',
+                    section: group.id,
+                    sectionType: group.type,
+                    extractedAt: Date.now()
+                }
+            };
+        });
+        // Infer relationships if enabled
+        const relationships = [];
+        if (options.enableRelationshipInference && entities.length > 1) {
+            // Find relationships between entities in this section
+            for (let i = 0; i < entities.length; i++) {
+                for (let j = i + 1; j < entities.length; j++) {
+                    const entity1 = entities[i];
+                    const entity2 = entities[j];
+                    // Check if entities appear near each other in text
+                    if (this.entitiesAreRelated(combinedText, entity1.name, entity2.name)) {
+                        const verbType = await this.inferRelationship(entity1.name, entity2.name, combinedText);
+                        const context = this.extractRelationshipContext(combinedText, entity1.name, entity2.name);
+                        relationships.push({
+                            from: entity1.id,
+                            to: entity2.id,
+                            type: verbType,
+                            confidence: Math.min(entity1.confidence, entity2.confidence) * 0.9,
+                            evidence: context
+                        });
+                    }
+                }
+            }
+        }
+        return {
+            sectionId: group.id,
+            sectionType: group.type,
+            entities,
+            relationships,
+            concepts,
+            text: combinedText.substring(0, 1000) // Store first 1000 chars as preview
+        };
+    }
+    /**
+     * Extract context around an entity mention
+     */
+    extractContextAroundEntity(text, entityName, contextLength = 200) {
+        const index = text.toLowerCase().indexOf(entityName.toLowerCase());
+        if (index === -1)
+            return text.substring(0, contextLength);
+        const start = Math.max(0, index - contextLength / 2);
+        const end = Math.min(text.length, index + entityName.length + contextLength / 2);
+        return text.substring(start, end).trim();
+    }
+    /**
+     * Check if two entities are related based on proximity in text
+     */
+    entitiesAreRelated(text, entity1, entity2) {
+        const lowerText = text.toLowerCase();
+        const index1 = lowerText.indexOf(entity1.toLowerCase());
+        const index2 = lowerText.indexOf(entity2.toLowerCase());
+        if (index1 === -1 || index2 === -1)
+            return false;
+        // Entities are related if they appear within 500 characters of each other
+        return Math.abs(index1 - index2) < 500;
+    }
+    /**
+     * Extract context showing relationship between entities
+     */
+    extractRelationshipContext(text, entity1, entity2) {
+        const lowerText = text.toLowerCase();
+        const index1 = lowerText.indexOf(entity1.toLowerCase());
+        const index2 = lowerText.indexOf(entity2.toLowerCase());
+        if (index1 === -1 || index2 === -1)
+            return '';
+        const start = Math.min(index1, index2);
+        const end = Math.max(index1 + entity1.length, index2 + entity2.length);
+        return text.substring(start, end + 100).trim();
+    }
+    /**
+     * Infer relationship type from context
+     */
+    async inferRelationship(fromEntity, toEntity, context) {
+        const lowerContext = context.toLowerCase();
+        // Pattern-based relationship detection
+        const patterns = [
+            [new RegExp(`${toEntity}.*of.*${fromEntity}`, 'i'), VerbType.PartOf],
+            [new RegExp(`${fromEntity}.*contains.*${toEntity}`, 'i'), VerbType.Contains],
+            [new RegExp(`${fromEntity}.*in.*${toEntity}`, 'i'), VerbType.LocatedAt],
+            [new RegExp(`${fromEntity}.*by.*${toEntity}`, 'i'), VerbType.CreatedBy],
+            [new RegExp(`${fromEntity}.*created.*${toEntity}`, 'i'), VerbType.Creates],
+            [new RegExp(`${fromEntity}.*authored.*${toEntity}`, 'i'), VerbType.CreatedBy],
+            [new RegExp(`${fromEntity}.*part of.*${toEntity}`, 'i'), VerbType.PartOf],
+            [new RegExp(`${fromEntity}.*related to.*${toEntity}`, 'i'), VerbType.RelatedTo],
+            [new RegExp(`${fromEntity}.*and.*${toEntity}`, 'i'), VerbType.RelatedTo]
+        ];
+        for (const [pattern, verbType] of patterns) {
+            if (pattern.test(lowerContext)) {
+                return verbType;
+            }
+        }
+        // Default to RelatedTo
+        return VerbType.RelatedTo;
+    }
+    /**
+     * Generate consistent entity ID
+     */
+    generateEntityId(name, section) {
+        const normalized = name.toLowerCase().trim().replace(/\s+/g, '_');
+        const sectionNorm = section.replace(/\s+/g, '_');
+        return `ent_${normalized}_${sectionNorm}_${Date.now()}`;
+    }
+    /**
+     * Update statistics
+     */
+    updateStats(stats, type, confidence) {
+        // Track by type
+        const typeName = String(type);
+        stats.byType[typeName] = (stats.byType[typeName] || 0) + 1;
+        // Track by confidence
+        if (confidence > 0.8) {
+            stats.byConfidence.high++;
+        }
+        else if (confidence >= 0.6) {
+            stats.byConfidence.medium++;
+        }
+        else {
+            stats.byConfidence.low++;
+        }
+    }
+    /**
+     * Create empty result
+     */
+    emptyResult(startTime, pdfMetadata = {}) {
+        return {
+            sectionsProcessed: 0,
+            pagesProcessed: 0,
+            entitiesExtracted: 0,
+            relationshipsInferred: 0,
+            sections: [],
+            entityMap: new Map(),
+            processingTime: Date.now() - startTime,
+            stats: {
+                byType: {},
+                byConfidence: { high: 0, medium: 0, low: 0 },
+                bySource: { paragraphs: 0, tables: 0 }
+            },
+            pdfMetadata: {
+                pageCount: pdfMetadata.pageCount || 0,
+                title: pdfMetadata.title,
+                author: pdfMetadata.author,
+                subject: pdfMetadata.subject
+            }
+        };
+    }
+}
+//# sourceMappingURL=SmartPDFImporter.js.map

package/dist/importers/VFSStructureGenerator.d.ts

@@ -0,0 +1,82 @@
+/**
+ * VFS Structure Generator
+ *
+ * Organizes imported entities into structured VFS directories
+ * - Type-based grouping (Place/, Character/, Concept/)
+ * - Metadata files (_metadata.json, _relationships.json)
+ * - Source file preservation
+ *
+ * NO MOCKS - Production-ready implementation
+ */
+import { Brainy } from '../brainy.js';
+import type { SmartExcelResult } from './SmartExcelImporter.js';
+export interface VFSStructureOptions {
+    /** Root path in VFS for import */
+    rootPath: string;
+    /** Grouping strategy */
+    groupBy: 'type' | 'sheet' | 'flat' | 'custom';
+    /** Custom grouping function */
+    customGrouping?: (entity: any) => string;
+    /** Preserve source file */
+    preserveSource?: boolean;
+    /** Source file buffer (if preserving) */
+    sourceBuffer?: Buffer;
+    /** Source filename */
+    sourceFilename?: string;
+    /** Create relationship file */
+    createRelationshipFile?: boolean;
+    /** Create metadata file */
+    createMetadataFile?: boolean;
+}
+export interface VFSStructureResult {
+    /** Root path created */
+    rootPath: string;
+    /** Directories created */
+    directories: string[];
+    /** Files created */
+    files: Array<{
+        path: string;
+        entityId?: string;
+        type: 'entity' | 'metadata' | 'source' | 'relationships';
+    }>;
+    /** Total operations */
+    operations: number;
+    /** Time taken in ms */
+    duration: number;
+}
+/**
+ * VFSStructureGenerator - Organizes imported data into VFS
+ */
+export declare class VFSStructureGenerator {
+    private brain;
+    private vfs;
+    constructor(brain: Brainy);
+    /**
+     * Initialize the generator
+     */
+    init(): Promise<void>;
+    /**
+     * Generate VFS structure from import result
+     */
+    generate(importResult: SmartExcelResult, options: VFSStructureOptions): Promise<VFSStructureResult>;
+    /**
+     * Group entities by strategy
+     */
+    private groupEntities;
+    /**
+     * Get directory name for entity type
+     */
+    private getTypeGroupName;
+    /**
+     * Sanitize filename
+     */
+    private sanitizeFilename;
+    /**
+     * Get file extension
+     */
+    private getExtension;
+    /**
+     * Group items by property
+     */
+    private groupByType;
+}

package/dist/importers/VFSStructureGenerator.js

@@ -0,0 +1,260 @@
+/**
+ * VFS Structure Generator
+ *
+ * Organizes imported entities into structured VFS directories
+ * - Type-based grouping (Place/, Character/, Concept/)
+ * - Metadata files (_metadata.json, _relationships.json)
+ * - Source file preservation
+ *
+ * NO MOCKS - Production-ready implementation
+ */
+import { VirtualFileSystem } from '../vfs/VirtualFileSystem.js';
+import { NounType } from '../types/graphTypes.js';
+/**
+ * VFSStructureGenerator - Organizes imported data into VFS
+ */
+export class VFSStructureGenerator {
+    constructor(brain) {
+        this.brain = brain;
+        this.vfs = new VirtualFileSystem(brain);
+    }
+    /**
+     * Initialize the generator
+     */
+    async init() {
+        // Always ensure VFS is initialized
+        try {
+            // Check if VFS is initialized by trying to access root
+            await this.vfs.stat('/');
+        }
+        catch (error) {
+            // VFS not initialized, initialize it
+            await this.vfs.init();
+        }
+    }
+    /**
+     * Generate VFS structure from import result
+     */
+    async generate(importResult, options) {
+        const startTime = Date.now();
+        const result = {
+            rootPath: options.rootPath,
+            directories: [],
+            files: [],
+            operations: 0,
+            duration: 0
+        };
+        // Ensure VFS is initialized
+        await this.init();
+        // Create root directory
+        try {
+            await this.vfs.mkdir(options.rootPath, { recursive: true });
+            result.directories.push(options.rootPath);
+            result.operations++;
+        }
+        catch (error) {
+            // Directory might already exist, that's fine
+            if (error.code !== 'EEXIST') {
+                throw error;
+            }
+            result.directories.push(options.rootPath);
+        }
+        // Preserve source file if requested
+        if (options.preserveSource && options.sourceBuffer && options.sourceFilename) {
+            const sourcePath = `${options.rootPath}/_source${this.getExtension(options.sourceFilename)}`;
+            await this.vfs.writeFile(sourcePath, options.sourceBuffer);
+            result.files.push({
+                path: sourcePath,
+                type: 'source'
+            });
+            result.operations++;
+        }
+        // Group entities
+        const groups = this.groupEntities(importResult, options);
+        // Create directories and files for each group
+        for (const [groupName, entities] of groups.entries()) {
+            const groupPath = `${options.rootPath}/${groupName}`;
+            // Create group directory
+            try {
+                await this.vfs.mkdir(groupPath, { recursive: true });
+                result.directories.push(groupPath);
+                result.operations++;
+            }
+            catch (error) {
+                // Directory might already exist
+                if (error.code !== 'EEXIST') {
+                    throw error;
+                }
+                result.directories.push(groupPath);
+            }
+            // Create entity files
+            for (const extracted of entities) {
+                const sanitizedName = this.sanitizeFilename(extracted.entity.name);
+                const entityPath = `${groupPath}/${sanitizedName}.json`;
+                // Create entity JSON
+                const entityJson = {
+                    id: extracted.entity.id,
+                    name: extracted.entity.name,
+                    type: extracted.entity.type,
+                    description: extracted.entity.description,
+                    confidence: extracted.entity.confidence,
+                    metadata: extracted.entity.metadata,
+                    concepts: extracted.concepts || [],
+                    relatedEntities: extracted.relatedEntities,
+                    relationships: extracted.relationships.map(rel => ({
+                        from: rel.from,
+                        to: rel.to,
+                        type: rel.type,
+                        confidence: rel.confidence,
+                        evidence: rel.evidence
+                    }))
+                };
+                await this.vfs.writeFile(entityPath, JSON.stringify(entityJson, null, 2));
+                result.files.push({
+                    path: entityPath,
+                    entityId: extracted.entity.id,
+                    type: 'entity'
+                });
+                result.operations++;
+            }
+        }
+        // Create relationships file
+        if (options.createRelationshipFile !== false) {
+            const relationshipsPath = `${options.rootPath}/_relationships.json`;
+            const allRelationships = importResult.rows.flatMap(row => row.relationships);
+            const relationshipsJson = {
+                source: options.sourceFilename || 'unknown',
+                count: allRelationships.length,
+                relationships: allRelationships,
+                stats: {
+                    byType: this.groupByType(allRelationships, 'type'),
+                    byConfidence: {
+                        high: allRelationships.filter(r => r.confidence > 0.8).length,
+                        medium: allRelationships.filter(r => r.confidence >= 0.6 && r.confidence <= 0.8).length,
+                        low: allRelationships.filter(r => r.confidence < 0.6).length
+                    }
+                }
+            };
+            await this.vfs.writeFile(relationshipsPath, JSON.stringify(relationshipsJson, null, 2));
+            result.files.push({
+                path: relationshipsPath,
+                type: 'relationships'
+            });
+            result.operations++;
+        }
+        // Create metadata file
+        if (options.createMetadataFile !== false) {
+            const metadataPath = `${options.rootPath}/_metadata.json`;
+            const metadataJson = {
+                import: {
+                    timestamp: new Date().toISOString(),
+                    source: {
+                        filename: options.sourceFilename || 'unknown',
+                        format: 'excel'
+                    },
+                    options: {
+                        groupBy: options.groupBy,
+                        preserveSource: options.preserveSource
+                    },
+                    stats: {
+                        rowsProcessed: importResult.rowsProcessed,
+                        entitiesExtracted: importResult.entitiesExtracted,
+                        relationshipsInferred: importResult.relationshipsInferred,
+                        processingTime: importResult.processingTime,
+                        byType: importResult.stats.byType,
+                        byConfidence: importResult.stats.byConfidence
+                    }
+                },
+                structure: {
+                    rootPath: options.rootPath,
+                    groupingStrategy: options.groupBy,
+                    directories: result.directories,
+                    fileCount: result.files.length
+                }
+            };
+            await this.vfs.writeFile(metadataPath, JSON.stringify(metadataJson, null, 2));
+            result.files.push({
+                path: metadataPath,
+                type: 'metadata'
+            });
+            result.operations++;
+        }
+        result.duration = Date.now() - startTime;
+        return result;
+    }
+    /**
+     * Group entities by strategy
+     */
+    groupEntities(importResult, options) {
+        const groups = new Map();
+        for (const extracted of importResult.rows) {
+            let groupName;
+            switch (options.groupBy) {
+                case 'type':
+                    groupName = this.getTypeGroupName(extracted.entity.type);
+                    break;
+                case 'flat':
+                    groupName = 'entities';
+                    break;
+                case 'custom':
+                    groupName = options.customGrouping ?
+                        options.customGrouping(extracted.entity) :
+                        'entities';
+                    break;
+                default:
+                    groupName = 'entities';
+            }
+            if (!groups.has(groupName)) {
+                groups.set(groupName, []);
+            }
+            groups.get(groupName).push(extracted);
+        }
+        return groups;
+    }
+    /**
+     * Get directory name for entity type
+     */
+    getTypeGroupName(type) {
+        const typeMap = {
+            [NounType.Person]: 'Characters',
+            [NounType.Location]: 'Places',
+            [NounType.Organization]: 'Organizations',
+            [NounType.Concept]: 'Concepts',
+            [NounType.Event]: 'Events',
+            [NounType.Product]: 'Items',
+            [NounType.Document]: 'Documents',
+            [NounType.Project]: 'Projects',
+            [NounType.Thing]: 'Other'
+        };
+        return typeMap[type] || 'Other';
+    }
+    /**
+     * Sanitize filename
+     */
+    sanitizeFilename(name) {
+        return name
+            .replace(/[<>:"/\\|?*]/g, '_') // Replace invalid chars
+            .replace(/\s+/g, '_') // Replace spaces
+            .replace(/_{2,}/g, '_') // Collapse multiple underscores
+            .substring(0, 200); // Limit length
+    }
+    /**
+     * Get file extension
+     */
+    getExtension(filename) {
+        const lastDot = filename.lastIndexOf('.');
+        return lastDot !== -1 ? filename.substring(lastDot) : '.bin';
+    }
+    /**
+     * Group items by property
+     */
+    groupByType(items, property) {
+        const groups = {};
+        for (const item of items) {
+            const key = String(item[property]);
+            groups[key] = (groups[key] || 0) + 1;
+        }
+        return groups;
+    }
+}
+//# sourceMappingURL=VFSStructureGenerator.js.map