@soulcraft/brainy 3.21.0 → 3.22.0

package/CHANGELOG.md

@@ -2,6 +2,11 @@
 
 All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines.
 
+### [3.22.0](https://github.com/soulcraftlabs/brainy/compare/v3.21.0...v3.22.0) (2025-10-01)
+
+- feat: add intelligent import for CSV, Excel, and PDF files (814cbb4)
+
+
 ### [3.21.0](https://github.com/soulcraftlabs/brainy/compare/v3.20.5...v3.21.0) (2025-10-01)
 
 - feat: add progress tracking, entity caching, and relationship confidence (2f9d512)

package/README.md

@@ -457,6 +457,41 @@ npm run download-models        # Download Q8 model
 npm run download-models:q8     # Download Q8 model
 ```
 
+## 🚀 Import Anything - Files, Data, URLs
+
+Brainy's universal import intelligently handles **any data format**:
+
+```javascript
+// Import CSV with auto-detection
+await brain.import('customers.csv')
+// ✨ Auto-detects: encoding, delimiter, types, creates entities!
+
+// Import Excel workbooks with multi-sheet support
+await brain.import('sales-data.xlsx', {
+  excelSheets: ['Q1', 'Q2']  // or 'all' for all sheets
+})
+// ✨ Processes all sheets, preserves structure, infers types!
+
+// Import PDF documents with table extraction
+await brain.import('research-paper.pdf', {
+  pdfExtractTables: true
+})
+// ✨ Extracts text, detects tables, preserves metadata!
+
+// Import JSON/YAML data
+await brain.import([
+  { name: 'Alice', role: 'Engineer' },
+  { name: 'Bob', role: 'Designer' }
+])
+// ✨ Automatically creates Person entities with relationships!
+
+// Import from URLs (auto-fetched)
+await brain.import('https://api.example.com/data.json')
+// ✨ Auto-detects URL, fetches, parses, processes!
+```
+
+**📖 [Complete Import Guide →](docs/guides/import-anything.md)** | **[Live Example →](examples/import-excel-pdf-csv.ts)**
+
 ## 📚 Core API
 
 ### `search()` - Vector Similarity
@@ -513,6 +548,11 @@ await brain.deleteVerb(verbId)
 // Bulk operations
 await brain.import(arrayOfData)
 const exported = await brain.export({format: 'json'})
+
+// Import from CSV, Excel, PDF files (auto-detected)
+await brain.import('customers.csv')      // CSV with encoding detection
+await brain.import('sales-report.xlsx')  // Excel with multi-sheet support
+await brain.import('research.pdf')       // PDF with table extraction
 ```
 
 ## 🌐 Distributed System (NEW!)

package/dist/augmentations/defaultAugmentations.d.ts

@@ -13,6 +13,7 @@ import { CacheAugmentation } from './cacheAugmentation.js';
 import { MetricsAugmentation } from './metricsAugmentation.js';
 import { MonitoringAugmentation } from './monitoringAugmentation.js';
 import { UniversalDisplayAugmentation } from './universalDisplayAugmentation.js';
+import { IntelligentImportAugmentation } from './intelligentImport/index.js';
 /**
  * Create default augmentations for zero-config operation
  * Returns an array of augmentations to be registered
@@ -25,6 +26,7 @@ export declare function createDefaultAugmentations(config?: {
     metrics?: boolean | Record<string, any>;
     monitoring?: boolean | Record<string, any>;
     display?: boolean | Record<string, any>;
+    intelligentImport?: boolean | Record<string, any>;
 }): BaseAugmentation[];
 /**
  * Get augmentation by name with type safety
@@ -54,4 +56,8 @@ export declare const AugmentationHelpers: {
      * Get display augmentation
      */
     getDisplay(brain: Brainy): UniversalDisplayAugmentation | null;
+    /**
+     * Get intelligent import augmentation
+     */
+    getIntelligentImport(brain: Brainy): IntelligentImportAugmentation | null;
 };

package/dist/augmentations/defaultAugmentations.js

@@ -11,6 +11,7 @@ import { CacheAugmentation } from './cacheAugmentation.js';
 import { MetricsAugmentation } from './metricsAugmentation.js';
 import { MonitoringAugmentation } from './monitoringAugmentation.js';
 import { UniversalDisplayAugmentation } from './universalDisplayAugmentation.js';
+import { IntelligentImportAugmentation } from './intelligentImport/index.js';
 /**
  * Create default augmentations for zero-config operation
  * Returns an array of augmentations to be registered
@@ -20,6 +21,11 @@ import { UniversalDisplayAugmentation } from './universalDisplayAugmentation.js'
  */
 export function createDefaultAugmentations(config = {}) {
     const augmentations = [];
+    // Intelligent Import augmentation (CSV, Excel, PDF)
+    if (config.intelligentImport !== false) {
+        const importConfig = typeof config.intelligentImport === 'object' ? config.intelligentImport : {};
+        augmentations.push(new IntelligentImportAugmentation(importConfig));
+    }
     // Cache augmentation (was SearchCache)
     if (config.cache !== false) {
         const cacheConfig = typeof config.cache === 'object' ? config.cache : {};
@@ -88,6 +94,12 @@ export const AugmentationHelpers = {
      */
     getDisplay(brain) {
         return getAugmentation(brain, 'display');
+    },
+    /**
+     * Get intelligent import augmentation
+     */
+    getIntelligentImport(brain) {
+        return getAugmentation(brain, 'intelligent-import');
     }
 };
 //# sourceMappingURL=defaultAugmentations.js.map

package/dist/augmentations/intelligentImport/IntelligentImportAugmentation.d.ts

@@ -0,0 +1,51 @@
+/**
+ * Intelligent Import Augmentation
+ *
+ * Automatically detects and processes CSV, Excel, and PDF files with:
+ * - Format detection and routing
+ * - Lazy-loaded handlers
+ * - Intelligent entity and relationship extraction
+ * - Integration with NeuralImport augmentation
+ */
+import { BaseAugmentation } from '../brainyAugmentation.js';
+import { FormatHandler, IntelligentImportConfig } from './types.js';
+export declare class IntelligentImportAugmentation extends BaseAugmentation {
+    readonly name = "intelligent-import";
+    readonly timing: "before";
+    readonly metadata: {
+        reads: "*";
+        writes: string[];
+    };
+    readonly operations: any[];
+    readonly priority = 75;
+    protected config: IntelligentImportConfig;
+    private handlers;
+    private initialized;
+    constructor(config?: Partial<IntelligentImportConfig>);
+    protected onInitialize(): Promise<void>;
+    execute<T = any>(operation: string, params: any, next: () => Promise<T>): Promise<T>;
+    /**
+     * Check if we should process this operation
+     */
+    private shouldProcess;
+    /**
+     * Extract file data from various param formats
+     */
+    private extractFileData;
+    /**
+     * Detect which handler can process this file
+     */
+    private detectHandler;
+    /**
+     * Get handler by format name
+     */
+    getHandler(format: string): FormatHandler | undefined;
+    /**
+     * Get all registered handlers
+     */
+    getHandlers(): FormatHandler[];
+    /**
+     * Get supported formats
+     */
+    getSupportedFormats(): string[];
+}

package/dist/augmentations/intelligentImport/IntelligentImportAugmentation.js

@@ -0,0 +1,185 @@
+/**
+ * Intelligent Import Augmentation
+ *
+ * Automatically detects and processes CSV, Excel, and PDF files with:
+ * - Format detection and routing
+ * - Lazy-loaded handlers
+ * - Intelligent entity and relationship extraction
+ * - Integration with NeuralImport augmentation
+ */
+import { BaseAugmentation } from '../brainyAugmentation.js';
+import { CSVHandler } from './handlers/csvHandler.js';
+import { ExcelHandler } from './handlers/excelHandler.js';
+import { PDFHandler } from './handlers/pdfHandler.js';
+export class IntelligentImportAugmentation extends BaseAugmentation {
+    constructor(config = {}) {
+        super(config);
+        this.name = 'intelligent-import';
+        this.timing = 'before';
+        this.metadata = {
+            reads: '*',
+            writes: ['_intelligentImport', '_processedFormat', '_extractedData']
+        };
+        this.operations = ['import', 'importFile', 'importFromFile', 'importFromURL', 'all'];
+        this.priority = 75; // Before NeuralImport (80), after validation
+        this.handlers = new Map();
+        this.initialized = false;
+        this.config = {
+            enableCSV: true,
+            enableExcel: true,
+            enablePDF: true,
+            maxFileSize: 100 * 1024 * 1024, // 100MB default
+            enableCache: true,
+            cacheTTL: 24 * 60 * 60 * 1000, // 24 hours
+            ...config
+        };
+    }
+    async onInitialize() {
+        // Initialize handlers based on config
+        if (this.config.enableCSV) {
+            this.handlers.set('csv', new CSVHandler());
+        }
+        if (this.config.enableExcel) {
+            this.handlers.set('excel', new ExcelHandler());
+        }
+        if (this.config.enablePDF) {
+            this.handlers.set('pdf', new PDFHandler());
+        }
+        this.initialized = true;
+        this.log(`Initialized with ${this.handlers.size} format handlers (CSV: ${this.config.enableCSV}, Excel: ${this.config.enableExcel}, PDF: ${this.config.enablePDF})`);
+    }
+    async execute(operation, params, next) {
+        // Only process import operations
+        if (!this.shouldProcess(operation, params)) {
+            return next();
+        }
+        try {
+            // Extract file data from params
+            const fileData = this.extractFileData(params);
+            if (!fileData) {
+                return next();
+            }
+            // Check file size limit
+            if (this.config.maxFileSize && fileData.data.length > this.config.maxFileSize) {
+                this.log(`File too large (${fileData.data.length} bytes), skipping intelligent import`, 'warn');
+                return next();
+            }
+            // Detect format and get appropriate handler
+            const handler = this.detectHandler(fileData.data, fileData.filename);
+            if (!handler) {
+                // Not a supported format, pass through
+                return next();
+            }
+            this.log(`Processing ${fileData.filename || 'file'} with ${handler.format} handler`);
+            // Process the file
+            const processed = await handler.process(fileData.data, {
+                filename: fileData.filename,
+                ext: fileData.ext,
+                ...this.config.csvDefaults,
+                ...this.config.excelDefaults,
+                ...this.config.pdfDefaults,
+                ...params.options
+            });
+            // Enrich params with processed data
+            params._intelligentImport = true;
+            params._processedFormat = processed.format;
+            params._extractedData = processed.data;
+            params._metadata = {
+                ...params._metadata,
+                intelligentImport: processed.metadata
+            };
+            // If this is an import operation, transform params to include the structured data
+            if (processed.data.length > 0) {
+                // Store processed data for the neural import augmentation to use
+                params.data = processed.data;
+                params.metadata = params._metadata;
+            }
+            this.log(`Extracted ${processed.data.length} items from ${processed.format} file`);
+            return next();
+        }
+        catch (error) {
+            this.log(`Intelligent import processing failed: ${error instanceof Error ? error.message : String(error)}`, 'warn');
+            // Fall through to normal import on error
+            return next();
+        }
+    }
+    /**
+     * Check if we should process this operation
+     */
+    shouldProcess(operation, params) {
+        // Only process if we have handlers initialized
+        if (!this.initialized || this.handlers.size === 0) {
+            return false;
+        }
+        // Check operation type
+        const validOps = ['import', 'importFile', 'importFromFile', 'importFromURL'];
+        if (!validOps.some(op => operation.includes(op))) {
+            return false;
+        }
+        // Must have some data
+        if (!params || (!params.source && !params.data && !params.filePath && !params.url)) {
+            return false;
+        }
+        return true;
+    }
+    /**
+     * Extract file data from various param formats
+     */
+    extractFileData(params) {
+        // From source parameter
+        if (params.source) {
+            if (Buffer.isBuffer(params.source)) {
+                return { data: params.source, filename: params.filename };
+            }
+            if (typeof params.source === 'string') {
+                return { data: Buffer.from(params.source), filename: params.filename };
+            }
+        }
+        // From data parameter
+        if (params.data) {
+            if (Buffer.isBuffer(params.data)) {
+                return { data: params.data, filename: params.filename };
+            }
+            if (typeof params.data === 'string') {
+                return { data: Buffer.from(params.data), filename: params.filename };
+            }
+        }
+        // From file path (would need to read - but that should be handled by UniversalImportAPI)
+        if (params.filePath && typeof params.filePath === 'string') {
+            const ext = params.filePath.split('.').pop();
+            return null; // File reading handled elsewhere
+        }
+        return null;
+    }
+    /**
+     * Detect which handler can process this file
+     */
+    detectHandler(data, filename) {
+        // Try each handler's canHandle method
+        for (const handler of this.handlers.values()) {
+            if (handler.canHandle(data) || (filename && handler.canHandle({ filename }))) {
+                return handler;
+            }
+        }
+        return null;
+    }
+    /**
+     * Get handler by format name
+     */
+    getHandler(format) {
+        return this.handlers.get(format.toLowerCase());
+    }
+    /**
+     * Get all registered handlers
+     */
+    getHandlers() {
+        return Array.from(this.handlers.values());
+    }
+    /**
+     * Get supported formats
+     */
+    getSupportedFormats() {
+        return Array.from(this.handlers.keys());
+    }
+}
+//# sourceMappingURL=IntelligentImportAugmentation.js.map

package/dist/augmentations/intelligentImport/handlers/base.d.ts

@@ -0,0 +1,49 @@
+/**
+ * Base Format Handler
+ * Abstract class providing common functionality for all format handlers
+ */
+import { FormatHandler, FormatHandlerOptions, ProcessedData } from '../types.js';
+export declare abstract class BaseFormatHandler implements FormatHandler {
+    abstract readonly format: string;
+    abstract process(data: Buffer | string, options: FormatHandlerOptions): Promise<ProcessedData>;
+    abstract canHandle(data: Buffer | string | {
+        filename?: string;
+        ext?: string;
+    }): boolean;
+    /**
+     * Detect file extension from various inputs
+     */
+    protected detectExtension(data: Buffer | string | {
+        filename?: string;
+        ext?: string;
+    }): string | null;
+    /**
+     * Extract extension from filename
+     */
+    protected getExtension(filename: string): string;
+    /**
+     * Infer field types from data
+     * Analyzes multiple rows to determine the most appropriate type
+     */
+    protected inferFieldTypes(data: Array<Record<string, any>>): Record<string, string>;
+    /**
+     * Infer type of a single value
+     */
+    protected inferType(value: any): string;
+    /**
+     * Check if string looks like a date
+     */
+    protected isDateString(value: string): boolean;
+    /**
+     * Sanitize field names for use as object keys
+     */
+    protected sanitizeFieldName(name: string): string;
+    /**
+     * Convert value to appropriate type
+     */
+    protected convertValue(value: any, type: string): any;
+    /**
+     * Create metadata object with common fields
+     */
+    protected createMetadata(rowCount: number, fields: string[], processingTime: number, extra?: Record<string, any>): ProcessedData['metadata'];
+}

package/dist/augmentations/intelligentImport/handlers/base.js

@@ -0,0 +1,149 @@
+/**
+ * Base Format Handler
+ * Abstract class providing common functionality for all format handlers
+ */
+export class BaseFormatHandler {
+    /**
+     * Detect file extension from various inputs
+     */
+    detectExtension(data) {
+        if (typeof data === 'object' && 'filename' in data && data.filename) {
+            return this.getExtension(data.filename);
+        }
+        if (typeof data === 'object' && 'ext' in data && data.ext) {
+            return data.ext.toLowerCase().replace(/^\./, '');
+        }
+        return null;
+    }
+    /**
+     * Extract extension from filename
+     */
+    getExtension(filename) {
+        const match = filename.match(/\.([^.]+)$/);
+        return match ? match[1].toLowerCase() : '';
+    }
+    /**
+     * Infer field types from data
+     * Analyzes multiple rows to determine the most appropriate type
+     */
+    inferFieldTypes(data) {
+        if (data.length === 0)
+            return {};
+        const types = {};
+        const firstRow = data[0];
+        const sampleSize = Math.min(10, data.length);
+        for (const key of Object.keys(firstRow)) {
+            // Check first few rows to get more accurate type
+            const sampleTypes = new Set();
+            for (let i = 0; i < sampleSize; i++) {
+                const value = data[i][key];
+                const type = this.inferType(value);
+                sampleTypes.add(type);
+            }
+            // If we see both integer and float, use float
+            if (sampleTypes.has('float') || (sampleTypes.has('integer') && sampleTypes.has('float'))) {
+                types[key] = 'float';
+            }
+            else if (sampleTypes.has('integer')) {
+                types[key] = 'integer';
+            }
+            else if (sampleTypes.has('date')) {
+                types[key] = 'date';
+            }
+            else if (sampleTypes.has('boolean')) {
+                types[key] = 'boolean';
+            }
+            else {
+                types[key] = 'string';
+            }
+        }
+        return types;
+    }
+    /**
+     * Infer type of a single value
+     */
+    inferType(value) {
+        if (value === null || value === undefined || value === '')
+            return 'string';
+        if (typeof value === 'number')
+            return 'number';
+        if (typeof value === 'boolean')
+            return 'boolean';
+        if (typeof value === 'string') {
+            // Check if it's a number
+            if (/^-?\d+$/.test(value))
+                return 'integer';
+            if (/^-?\d+\.\d+$/.test(value))
+                return 'float';
+            // Check if it's a date
+            if (this.isDateString(value))
+                return 'date';
+            // Check if it's a boolean
+            if (/^(true|false|yes|no|y|n)$/i.test(value))
+                return 'boolean';
+        }
+        return 'string';
+    }
+    /**
+     * Check if string looks like a date
+     */
+    isDateString(value) {
+        // ISO 8601
+        if (/^\d{4}-\d{2}-\d{2}/.test(value))
+            return true;
+        // Common date formats
+        if (/^\d{1,2}\/\d{1,2}\/\d{2,4}$/.test(value))
+            return true;
+        if (/^\d{1,2}-\d{1,2}-\d{2,4}$/.test(value))
+            return true;
+        return false;
+    }
+    /**
+     * Sanitize field names for use as object keys
+     */
+    sanitizeFieldName(name) {
+        return name
+            .trim()
+            .replace(/[^a-zA-Z0-9_\s-]/g, '')
+            .replace(/\s+/g, '_')
+            .replace(/-+/g, '_')
+            .replace(/_+/g, '_')
+            .replace(/^_|_$/g, '')
+            || 'field';
+    }
+    /**
+     * Convert value to appropriate type
+     */
+    convertValue(value, type) {
+        if (value === null || value === undefined || value === '')
+            return null;
+        switch (type) {
+            case 'integer':
+                return parseInt(String(value), 10);
+            case 'float':
+            case 'number':
+                return parseFloat(String(value));
+            case 'boolean':
+                if (typeof value === 'boolean')
+                    return value;
+                const str = String(value).toLowerCase();
+                return ['true', 'yes', 'y', '1'].includes(str);
+            case 'date':
+                return new Date(value);
+            default:
+                return value;
+        }
+    }
+    /**
+     * Create metadata object with common fields
+     */
+    createMetadata(rowCount, fields, processingTime, extra = {}) {
+        return {
+            rowCount,
+            fields,
+            processingTime,
+            ...extra
+        };
+    }
+}
+//# sourceMappingURL=base.js.map

package/dist/augmentations/intelligentImport/handlers/csvHandler.d.ts

@@ -0,0 +1,34 @@
+/**
+ * CSV Format Handler
+ * Handles CSV files with:
+ * - Automatic encoding detection
+ * - Automatic delimiter detection
+ * - Streaming for large files
+ * - Type inference
+ */
+import { BaseFormatHandler } from './base.js';
+import { FormatHandlerOptions, ProcessedData } from '../types.js';
+export declare class CSVHandler extends BaseFormatHandler {
+    readonly format = "csv";
+    canHandle(data: Buffer | string | {
+        filename?: string;
+        ext?: string;
+    }): boolean;
+    process(data: Buffer | string, options: FormatHandlerOptions): Promise<ProcessedData>;
+    /**
+     * Check if text looks like CSV
+     */
+    private looksLikeCSV;
+    /**
+     * Detect CSV delimiter
+     */
+    private detectDelimiter;
+    /**
+     * Detect encoding safely (with fallback)
+     */
+    private detectEncodingSafe;
+    /**
+     * Normalize encoding names to Node.js-supported encodings
+     */
+    private normalizeEncoding;
+}