@soulcraft/brainy 4.1.3 → 4.2.0

package/dist/import/ImportCoordinator.js

@@ -17,6 +17,8 @@ import { SmartPDFImporter } from '../importers/SmartPDFImporter.js';
 import { SmartCSVImporter } from '../importers/SmartCSVImporter.js';
 import { SmartJSONImporter } from '../importers/SmartJSONImporter.js';
 import { SmartMarkdownImporter } from '../importers/SmartMarkdownImporter.js';
+import { SmartYAMLImporter } from '../importers/SmartYAMLImporter.js';
+import { SmartDOCXImporter } from '../importers/SmartDOCXImporter.js';
 import { VFSStructureGenerator } from '../importers/VFSStructureGenerator.js';
 import { NounType } from '../types/graphTypes.js';
 import { v4 as uuidv4 } from '../universal/uuid.js';
@@ -36,6 +38,8 @@ export class ImportCoordinator {
         this.csvImporter = new SmartCSVImporter(brain);
         this.jsonImporter = new SmartJSONImporter(brain);
         this.markdownImporter = new SmartMarkdownImporter(brain);
+        this.yamlImporter = new SmartYAMLImporter(brain);
+        this.docxImporter = new SmartDOCXImporter(brain);
         this.vfsGenerator = new VFSStructureGenerator(brain);
     }
     /**
@@ -47,6 +51,8 @@ export class ImportCoordinator {
         await this.csvImporter.init();
         await this.jsonImporter.init();
         await this.markdownImporter.init();
+        await this.yamlImporter.init();
+        await this.docxImporter.init();
         await this.vfsGenerator.init();
         await this.history.init();
     }
@@ -58,12 +64,15 @@ export class ImportCoordinator {
     }
     /**
      * Import from any source with auto-detection
+     * v4.2.0: Now supports URL imports with authentication
      */
     async import(source, options = {}) {
         const startTime = Date.now();
         const importId = uuidv4();
-        // Normalize source
-        const normalizedSource = this.normalizeSource(source, options.format);
+        // Validate options (v4.0.0+: Reject deprecated v3.x options)
+        this.validateOptions(options);
+        // Normalize source (v4.2.0: handles URL fetching)
+        const normalizedSource = await this.normalizeSource(source, options.format);
         // Report detection stage
         options.onProgress?.({
             stage: 'detecting',
@@ -168,8 +177,16 @@ export class ImportCoordinator {
     }
     /**
      * Normalize source to ImportSource
+     * v4.2.0: Now async to support URL fetching
      */
-    normalizeSource(source, formatHint) {
+    async normalizeSource(source, formatHint) {
+        // If already an ImportSource, handle URL fetching if needed
+        if (this.isImportSource(source)) {
+            if (source.type === 'url') {
+                return await this.fetchUrl(source);
+            }
+            return source;
+        }
         // Buffer
         if (Buffer.isBuffer(source)) {
             return {
@@ -177,8 +194,15 @@ export class ImportCoordinator {
                 data: source
             };
         }
-        // String - could be path or content
+        // String - could be URL, path, or content
         if (typeof source === 'string') {
+            // Check if it's a URL
+            if (this.isUrl(source)) {
+                return await this.fetchUrl({
+                    type: 'url',
+                    data: source
+                });
+            }
             // Check if it's a file path
             if (this.isFilePath(source)) {
                 const buffer = fs.readFileSync(source);
@@ -201,7 +225,73 @@ export class ImportCoordinator {
                 data: source
             };
         }
-        throw new Error('Invalid source type. Expected Buffer, string, or object.');
+        throw new Error('Invalid source type. Expected Buffer, string, object, or ImportSource.');
+    }
+    /**
+     * Check if value is an ImportSource object
+     */
+    isImportSource(value) {
+        return value && typeof value === 'object' && 'type' in value && 'data' in value;
+    }
+    /**
+     * Check if string is a URL
+     */
+    isUrl(str) {
+        try {
+            const url = new URL(str);
+            return url.protocol === 'http:' || url.protocol === 'https:';
+        }
+        catch {
+            return false;
+        }
+    }
+    /**
+     * Fetch content from URL
+     * v4.2.0: Supports authentication and custom headers
+     */
+    async fetchUrl(source) {
+        const url = typeof source.data === 'string' ? source.data : String(source.data);
+        // Build headers
+        const headers = {
+            'User-Agent': 'Brainy/4.2.0',
+            ...(source.headers || {})
+        };
+        // Add basic auth if provided
+        if (source.auth) {
+            const credentials = Buffer.from(`${source.auth.username}:${source.auth.password}`).toString('base64');
+            headers['Authorization'] = `Basic ${credentials}`;
+        }
+        try {
+            const response = await fetch(url, { headers });
+            if (!response.ok) {
+                throw new Error(`HTTP ${response.status}: ${response.statusText}`);
+            }
+            // Get filename from URL or Content-Disposition header
+            const contentDisposition = response.headers.get('content-disposition');
+            let filename = source.filename;
+            if (contentDisposition) {
+                const match = contentDisposition.match(/filename=["']?([^"';]+)["']?/);
+                if (match)
+                    filename = match[1];
+            }
+            if (!filename) {
+                filename = new URL(url).pathname.split('/').pop() || 'download';
+            }
+            // Get content type for format hint
+            const contentType = response.headers.get('content-type');
+            // Convert response to buffer
+            const arrayBuffer = await response.arrayBuffer();
+            const buffer = Buffer.from(arrayBuffer);
+            return {
+                type: 'buffer',
+                data: buffer,
+                filename,
+                headers: { 'content-type': contentType || 'application/octet-stream' }
+            };
+        }
+        catch (error) {
+            throw new Error(`Failed to fetch URL ${url}: ${error.message}`);
+        }
     }
     /**
      * Check if string is a file path
@@ -233,6 +323,12 @@ export class ImportCoordinator {
                 return this.detector.detectFromString(source.data);
             case 'object':
                 return this.detector.detectFromObject(source.data);
+            case 'url':
+                // URL sources are converted to buffers in normalizeSource()
+                // This should never be reached, but included for type safety
+                return null;
+            default:
+                return null;
         }
     }
     /**
@@ -288,6 +384,18 @@ export class ImportCoordinator {
                     ? source.data
                     : source.data.toString('utf8');
                 return await this.markdownImporter.extract(mdContent, extractOptions);
+            case 'yaml':
+                const yamlContent = source.type === 'string'
+                    ? source.data
+                    : source.type === 'buffer' || source.type === 'path'
+                        ? source.data.toString('utf8')
+                        : JSON.stringify(source.data);
+                return await this.yamlImporter.extract(yamlContent, extractOptions);
+            case 'docx':
+                const docxBuffer = source.type === 'buffer' || source.type === 'path'
+                    ? source.data
+                    : Buffer.from(JSON.stringify(source.data));
+                return await this.docxImporter.extract(docxBuffer, extractOptions);
             default:
                 throw new Error(`Unsupported format: ${format}`);
         }
@@ -305,6 +413,17 @@ export class ImportCoordinator {
         }
         // Extract rows/sections/entities from result (unified across formats)
         const rows = extractionResult.rows || extractionResult.sections || extractionResult.entities || [];
+        // Progressive flush interval - adjusts based on current count (v4.2.0+)
+        // Starts at 100, increases to 1000 at 1K entities, then 5000 at 10K
+        // This works for both known totals (files) and unknown totals (streaming APIs)
+        let currentFlushInterval = 100; // Start with frequent updates for better UX
+        let entitiesSinceFlush = 0;
+        let totalFlushes = 0;
+        console.log(`📊 Streaming Import: Progressive flush intervals\n` +
+            `   Starting interval: Every ${currentFlushInterval} entities\n` +
+            `   Auto-adjusts: 100 → 1000 (at 1K entities) → 5000 (at 10K entities)\n` +
+            `   Benefits: Live queries, crash resilience, frequent early updates\n` +
+            `   Works with: Known totals (files) and unknown totals (streaming APIs)`);
         // Smart deduplication auto-disable for large imports (prevents O(n²) performance)
         const DEDUPLICATION_AUTO_DISABLE_THRESHOLD = 100;
         let actuallyEnableDeduplication = options.enableDeduplication;
@@ -428,8 +547,9 @@ export class ImportCoordinator {
                                 from: entityId,
                                 to: targetEntityId,
                                 type: rel.type,
+                                confidence: rel.confidence, // v4.2.0: Top-level field
+                                weight: rel.weight || 1.0, // v4.2.0: Top-level field
                                 metadata: {
-                                    confidence: rel.confidence,
                                     evidence: rel.evidence,
                                     importedAt: Date.now()
                                 }
@@ -441,12 +561,58 @@ export class ImportCoordinator {
                         }
                     }
                 }
+                // Streaming import: Progressive flush with dynamic interval adjustment (v4.2.0+)
+                entitiesSinceFlush++;
+                if (entitiesSinceFlush >= currentFlushInterval) {
+                    const flushStart = Date.now();
+                    await this.brain.flush();
+                    const flushDuration = Date.now() - flushStart;
+                    totalFlushes++;
+                    // Reset counter
+                    entitiesSinceFlush = 0;
+                    // Recalculate flush interval based on current entity count
+                    const newInterval = this.getProgressiveFlushInterval(entities.length);
+                    if (newInterval !== currentFlushInterval) {
+                        console.log(`📊 Flush interval adjusted: ${currentFlushInterval} → ${newInterval}\n` +
+                            `   Reason: Reached ${entities.length} entities (threshold for next tier)\n` +
+                            `   Impact: ${newInterval > currentFlushInterval ? 'Fewer' : 'More'} flushes = ${newInterval > currentFlushInterval ? 'Better performance' : 'More frequent updates'}`);
+                        currentFlushInterval = newInterval;
+                    }
+                    // Notify progress callback that data is now queryable
+                    await options.onProgress?.({
+                        stage: 'storing-graph',
+                        message: `Flushed indexes (${entities.length}/${rows.length} entities, ${flushDuration}ms)`,
+                        processed: entities.length,
+                        total: rows.length,
+                        entities: entities.length,
+                        queryable: true // ← Indexes are flushed, data is queryable!
+                    });
+                }
             }
             catch (error) {
                 // Skip entity creation errors (might already exist, etc.)
                 continue;
             }
         }
+        // Final flush for any remaining entities
+        if (entitiesSinceFlush > 0) {
+            const flushStart = Date.now();
+            await this.brain.flush();
+            const flushDuration = Date.now() - flushStart;
+            totalFlushes++;
+            console.log(`✅ Import complete: ${entities.length} entities processed\n` +
+                `   Total flushes: ${totalFlushes}\n` +
+                `   Final flush: ${flushDuration}ms\n` +
+                `   Average overhead: ~${((totalFlushes * 50) / (entities.length * 100) * 100).toFixed(2)}%`);
+            await options.onProgress?.({
+                stage: 'storing-graph',
+                message: `Final flush complete (${entities.length} entities)`,
+                processed: entities.length,
+                total: rows.length,
+                entities: entities.length,
+                queryable: true
+            });
+        }
         // Batch create all relationships using brain.relateMany() for performance
         if (options.createRelationships && relationships.length > 0) {
             try {
@@ -555,8 +721,182 @@ export class ImportCoordinator {
                 stats: result.stats
             };
         }
+        // YAML: entities -> rows (v4.2.0)
+        if (format === 'yaml') {
+            const rows = result.entities.map((entity) => ({
+                entity,
+                relatedEntities: [],
+                relationships: result.relationships.filter((r) => r.from === entity.id),
+                concepts: entity.metadata?.concepts || []
+            }));
+            return {
+                rowsProcessed: result.nodesProcessed,
+                entitiesExtracted: result.entitiesExtracted,
+                relationshipsInferred: result.relationshipsInferred,
+                rows,
+                entityMap: result.entityMap,
+                processingTime: result.processingTime,
+                stats: result.stats
+            };
+        }
+        // DOCX: entities -> rows (v4.2.0)
+        if (format === 'docx') {
+            const rows = result.entities.map((entity) => ({
+                entity,
+                relatedEntities: [],
+                relationships: result.relationships.filter((r) => r.from === entity.id),
+                concepts: entity.metadata?.concepts || []
+            }));
+            return {
+                rowsProcessed: result.paragraphsProcessed,
+                entitiesExtracted: result.entitiesExtracted,
+                relationshipsInferred: result.relationshipsInferred,
+                rows,
+                entityMap: result.entityMap,
+                processingTime: result.processingTime,
+                stats: result.stats
+            };
+        }
         // Fallback: return as-is
         return result;
     }
+    /**
+     * Validate options and reject deprecated v3.x options (v4.0.0+)
+     * Throws clear errors with migration guidance
+     */
+    validateOptions(options) {
+        const invalidOptions = [];
+        // Check for v3.x deprecated options
+        if ('extractRelationships' in options) {
+            invalidOptions.push({
+                old: 'extractRelationships',
+                new: 'enableRelationshipInference',
+                message: 'Option renamed for clarity in v4.x - explicitly indicates AI-powered relationship inference'
+            });
+        }
+        if ('autoDetect' in options) {
+            invalidOptions.push({
+                old: 'autoDetect',
+                new: '(removed)',
+                message: 'Auto-detection is now always enabled - no need to specify this option'
+            });
+        }
+        if ('createFileStructure' in options) {
+            invalidOptions.push({
+                old: 'createFileStructure',
+                new: 'vfsPath',
+                message: 'Use vfsPath to explicitly specify the virtual filesystem directory path'
+            });
+        }
+        if ('excelSheets' in options) {
+            invalidOptions.push({
+                old: 'excelSheets',
+                new: '(removed)',
+                message: 'All sheets are now processed automatically - no configuration needed'
+            });
+        }
+        if ('pdfExtractTables' in options) {
+            invalidOptions.push({
+                old: 'pdfExtractTables',
+                new: '(removed)',
+                message: 'Table extraction is now automatic for PDF imports'
+            });
+        }
+        // If invalid options found, throw error with detailed message
+        if (invalidOptions.length > 0) {
+            const errorMessage = this.buildValidationErrorMessage(invalidOptions);
+            throw new Error(errorMessage);
+        }
+    }
+    /**
+     * Build detailed error message for invalid options
+     * Respects LOG_LEVEL for verbosity (detailed in dev, concise in prod)
+     */
+    buildValidationErrorMessage(invalidOptions) {
+        // Check environment for verbosity level
+        const verbose = process.env.LOG_LEVEL === 'debug' ||
+            process.env.LOG_LEVEL === 'verbose' ||
+            process.env.NODE_ENV === 'development' ||
+            process.env.NODE_ENV === 'dev';
+        if (verbose) {
+            // DETAILED mode (development)
+            const optionDetails = invalidOptions
+                .map((opt) => `
+  ❌ ${opt.old}
+     → Use: ${opt.new}
+     → Why: ${opt.message}`)
+                .join('\n');
+            return `
+❌ Invalid import options detected (Brainy v4.x breaking changes)
+
+The following v3.x options are no longer supported:
+${optionDetails}
+
+📖 Migration Guide: https://brainy.dev/docs/guides/migrating-to-v4
+💡 Quick Fix Examples:
+
+   Before (v3.x):
+   await brain.import(file, {
+     extractRelationships: true,
+     createFileStructure: true
+   })
+
+   After (v4.x):
+   await brain.import(file, {
+     enableRelationshipInference: true,
+     vfsPath: '/imports/my-data'
+   })
+
+🔗 Full API docs: https://brainy.dev/docs/api/import
+      `.trim();
+        }
+        else {
+            // CONCISE mode (production)
+            const optionsList = invalidOptions.map((o) => `'${o.old}'`).join(', ');
+            return `Invalid import options: ${optionsList}. See https://brainy.dev/docs/guides/migrating-to-v4`;
+        }
+    }
+    /**
+     * Get progressive flush interval based on CURRENT entity count (v4.2.0+)
+     *
+     * Unlike adaptive intervals (which require knowing total count upfront),
+     * progressive intervals adjust dynamically as import proceeds.
+     *
+     * Thresholds:
+     * - 0-999 entities:   Flush every 100   (frequent updates for better UX)
+     * - 1K-9.9K entities: Flush every 1000  (balanced performance/responsiveness)
+     * - 10K+ entities:    Flush every 5000  (performance focused, minimal overhead)
+     *
+     * Benefits:
+     * - Works with known totals (file imports)
+     * - Works with unknown totals (streaming APIs, database cursors)
+     * - Frequent updates early when user is watching
+     * - Efficient processing later when performance matters
+     * - Low overhead (~0.3% for large imports)
+     * - No configuration required
+     *
+     * Example:
+     * - Import with 50K entities:
+     *   - Flushes at: 100, 200, ..., 900 (9 flushes with interval=100)
+     *   - Interval increases to 1000 at entity #1000
+     *   - Flushes at: 1000, 2000, ..., 9000 (9 more flushes)
+     *   - Interval increases to 5000 at entity #10000
+     *   - Flushes at: 10000, 15000, ..., 50000 (8 more flushes)
+     *   - Total: ~26 flushes = ~1.3s overhead = 0.026% of import time
+     *
+     * @param currentEntityCount - Current number of entities imported so far
+     * @returns Current optimal flush interval
+     */
+    getProgressiveFlushInterval(currentEntityCount) {
+        if (currentEntityCount < 1000) {
+            return 100; // Frequent updates for small imports and early stages
+        }
+        else if (currentEntityCount < 10000) {
+            return 1000; // Balanced interval for medium-sized imports
+        }
+        else {
+            return 5000; // Performance-focused interval for large imports
+        }
+    }
 }
 //# sourceMappingURL=ImportCoordinator.js.map

package/dist/import/InstancePool.d.ts

@@ -0,0 +1,136 @@
+/**
+ * InstancePool - Shared instance management for memory efficiency
+ *
+ * Production-grade instance pooling to prevent memory leaks during imports.
+ * Critical for scaling to billions of entities.
+ *
+ * Problem: Creating new NLP/Extractor instances in loops → memory leak
+ * Solution: Reuse shared instances across entire import session
+ *
+ * Memory savings:
+ * - Without pooling: 100K rows × 50MB per instance = 5TB RAM (OOM!)
+ * - With pooling: 50MB total (shared across all rows)
+ */
+import { Brainy } from '../brainy.js';
+import { NaturalLanguageProcessor } from '../neural/naturalLanguageProcessor.js';
+import { NeuralEntityExtractor } from '../neural/entityExtractor.js';
+/**
+ * InstancePool - Manages shared instances for memory efficiency
+ *
+ * Lifecycle:
+ * 1. Create pool at import start
+ * 2. Reuse instances across all rows
+ * 3. Pool is garbage collected when import completes
+ *
+ * Thread safety: Not thread-safe (single import session per pool)
+ */
+export declare class InstancePool {
+    private brain;
+    private nlpInstance;
+    private extractorInstance;
+    private nlpInitialized;
+    private initializationPromise;
+    private stats;
+    constructor(brain: Brainy);
+    /**
+     * Get shared NaturalLanguageProcessor instance
+     *
+     * Lazy initialization - created on first access
+     * All subsequent calls return same instance
+     *
+     * @returns Shared NLP instance
+     */
+    getNLP(): Promise<NaturalLanguageProcessor>;
+    /**
+     * Get shared NeuralEntityExtractor instance
+     *
+     * Lazy initialization - created on first access
+     * All subsequent calls return same instance
+     *
+     * @returns Shared extractor instance
+     */
+    getExtractor(): NeuralEntityExtractor;
+    /**
+     * Get shared NLP instance (synchronous, may return uninitialized)
+     *
+     * Use when you need NLP synchronously and will handle initialization yourself.
+     * Prefer getNLP() for async code.
+     *
+     * @returns Shared NLP instance (possibly uninitialized)
+     */
+    getNLPSync(): NaturalLanguageProcessor;
+    /**
+     * Initialize all instances upfront
+     *
+     * Call at start of import to avoid lazy initialization overhead
+     * during processing. Improves predictability and first-row performance.
+     *
+     * @returns Promise that resolves when all instances are ready
+     */
+    init(): Promise<void>;
+    /**
+     * Internal initialization implementation
+     */
+    private initializeInternal;
+    /**
+     * Ensure NLP is initialized (loads 220 patterns)
+     *
+     * Handles concurrent initialization requests safely
+     */
+    private ensureNLPInitialized;
+    /**
+     * Check if instances are initialized
+     *
+     * @returns True if NLP is initialized and ready to use
+     */
+    isInitialized(): boolean;
+    /**
+     * Get pool statistics
+     *
+     * Useful for performance monitoring and memory leak detection
+     *
+     * @returns Statistics about instance reuse
+     */
+    getStats(): {
+        nlpCreated: boolean;
+        extractorCreated: boolean;
+        initialized: boolean;
+        memorySaved: number;
+        nlpReuses: number;
+        extractorReuses: number;
+        creationTime: number;
+    };
+    /**
+     * Calculate estimated memory saved by pooling
+     *
+     * Assumes ~50MB per NLP instance, ~10MB per extractor instance
+     *
+     * @returns Estimated memory saved in bytes
+     */
+    private calculateMemorySaved;
+    /**
+     * Reset statistics (useful for testing)
+     */
+    resetStats(): void;
+    /**
+     * Get string representation (for debugging)
+     */
+    toString(): string;
+    /**
+     * Cleanup method (for explicit resource management)
+     *
+     * Note: Usually not needed - pool is garbage collected when import completes.
+     * Use only if you need explicit cleanup for some reason.
+     */
+    cleanup(): void;
+}
+/**
+ * Create a new instance pool
+ *
+ * Convenience factory function
+ *
+ * @param brain Brainy instance
+ * @param autoInit Whether to initialize instances immediately
+ * @returns Instance pool
+ */
+export declare function createInstancePool(brain: Brainy, autoInit?: boolean): Promise<InstancePool>;