@soulcraft/brainy 4.3.2 → 4.5.0

package/dist/cli/commands/core.js

@@ -109,23 +109,37 @@ export const coreCommands = {
                 spinner.text = `Using detected type: ${nounType}`;
             }
             // Add with explicit type
-            const result = await brain.add({
+            const addParams = {
                 data: text,
                 type: nounType,
                 metadata
-            });
+            };
+            // v4.3.x: Add confidence and weight if provided
+            if (options.confidence) {
+                addParams.confidence = parseFloat(options.confidence);
+            }
+            if (options.weight) {
+                addParams.weight = parseFloat(options.weight);
+            }
+            const result = await brain.add(addParams);
             spinner.succeed('Added successfully');
             if (!options.json) {
                 console.log(chalk.green(`✓ Added with ID: ${result}`));
                 if (options.type) {
                     console.log(chalk.dim(`  Type: ${options.type}`));
                 }
+                if (options.confidence) {
+                    console.log(chalk.dim(`  Confidence: ${options.confidence}`));
+                }
+                if (options.weight) {
+                    console.log(chalk.dim(`  Weight: ${options.weight}`));
+                }
                 if (Object.keys(metadata).length > 0) {
                     console.log(chalk.dim(`  Metadata: ${JSON.stringify(metadata)}`));
                 }
             }
             else {
-                formatOutput({ id: result, metadata }, options);
+                formatOutput({ id: result, metadata, confidence: addParams.confidence, weight: addParams.weight }, options);
             }
         }
         catch (error) {
@@ -260,6 +274,10 @@ export const coreCommands = {
             if (options.includeRelations) {
                 searchParams.includeRelations = true;
             }
+            // Include VFS files (v4.4.0 - find excludes VFS by default)
+            if (options.includeVfs) {
+                searchParams.includeVFS = true;
+            }
             // Triple Intelligence Fusion - custom weighting
             if (options.fusion || options.vectorWeight || options.graphWeight || options.fieldWeight) {
                 searchParams.fusion = {

package/dist/cli/commands/import.js

@@ -120,19 +120,25 @@ export const importCommands = {
                     }]);
                 options.recursive = answer.recursive;
             }
-            spinner = ora('Initializing neural import...').start();
+            spinner = ora('Initializing import...').start();
             const brain = getBrainy();
-            // Load UniversalImportAPI
-            const { UniversalImportAPI } = await import('../../api/UniversalImportAPI.js');
-            const universalImport = new UniversalImportAPI(brain);
-            await universalImport.init();
-            spinner.text = 'Processing import...';
             // Handle different source types
             let result;
             if (isURL) {
-                // URL import
+                // URL import - fetch first
                 spinner.text = `Fetching from ${source}...`;
-                result = await universalImport.importFromURL(source);
+                const response = await fetch(source);
+                const buffer = Buffer.from(await response.arrayBuffer());
+                spinner.text = 'Importing...';
+                result = await brain.import(buffer, {
+                    enableNeuralExtraction: options.extractEntities !== false,
+                    enableRelationshipInference: options.detectRelationships !== false,
+                    enableConceptExtraction: options.extractConcepts || false,
+                    confidenceThreshold: options.confidence ? parseFloat(options.confidence) : 0.6,
+                    onProgress: options.progress ? (p) => {
+                        spinner.text = `${p.message}${p.entities ? ` (${p.entities} entities)` : ''}`;
+                    } : undefined
+                });
             }
             else if (isDirectory) {
                 // Directory import - process each file
@@ -163,31 +169,42 @@ export const importCommands = {
                 };
                 await collectFiles(source);
                 spinner.succeed(`Found ${files.length} files`);
-                // Process files in batches
-                const batchSize = options.batchSize ? parseInt(options.batchSize) : 100;
+                // Process files with progress
                 let totalEntities = 0;
                 let totalRelationships = 0;
                 let filesProcessed = 0;
-                for (let i = 0; i < files.length; i += batchSize) {
-                    const batch = files.slice(i, i + batchSize);
-                    if (options.progress) {
-                        spinner = ora(`Processing batch ${Math.floor(i / batchSize) + 1}/${Math.ceil(files.length / batchSize)} (${filesProcessed}/${files.length} files)...`).start();
-                    }
-                    for (const file of batch) {
-                        try {
-                            const fileResult = await universalImport.importFromFile(file);
-                            totalEntities += fileResult.stats.entitiesCreated;
-                            totalRelationships += fileResult.stats.relationshipsCreated;
-                            filesProcessed++;
+                for (const file of files) {
+                    try {
+                        if (options.progress) {
+                            spinner = ora(`[${filesProcessed + 1}/${files.length}] Importing ${file}...`).start();
                         }
-                        catch (error) {
-                            if (options.verbose) {
-                                console.log(chalk.yellow(`⚠️  Failed to import ${file}: ${error.message}`));
-                            }
+                        const fileResult = await brain.import(file, {
+                            enableNeuralExtraction: options.extractEntities !== false,
+                            enableRelationshipInference: options.detectRelationships !== false,
+                            enableConceptExtraction: options.extractConcepts || false,
+                            confidenceThreshold: options.confidence ? parseFloat(options.confidence) : 0.6,
+                            onProgress: options.progress ? (p) => {
+                                spinner.text = `[${filesProcessed + 1}/${files.length}] ${p.message}`;
+                            } : undefined
+                        });
+                        totalEntities += fileResult.entities.length;
+                        totalRelationships += fileResult.relationships.length;
+                        filesProcessed++;
+                        if (options.progress) {
+                            spinner.succeed(`[${filesProcessed}/${files.length}] ${file}`);
+                        }
+                    }
+                    catch (error) {
+                        if (options.verbose) {
+                            if (spinner)
+                                spinner.fail(`Failed: ${file}`);
+                            console.log(chalk.yellow(`⚠️  ${error.message}`));
                         }
                     }
                 }
                 result = {
+                    entities: [],
+                    relationships: [],
                     stats: {
                         filesProcessed,
                         entitiesCreated: totalEntities,
@@ -195,11 +212,23 @@ export const importCommands = {
                         totalProcessed: filesProcessed
                     }
                 };
-                spinner.succeed('Directory import complete');
+                spinner = ora().succeed(`Directory import complete: ${filesProcessed} files`);
             }
             else {
-                // File import
-                result = await universalImport.importFromFile(source);
+                // File import with progress
+                result = await brain.import(source, {
+                    format: options.format,
+                    enableNeuralExtraction: options.extractEntities !== false,
+                    enableRelationshipInference: options.detectRelationships !== false,
+                    enableConceptExtraction: options.extractConcepts || false,
+                    confidenceThreshold: options.confidence ? parseFloat(options.confidence) : 0.6,
+                    onProgress: options.progress ? (p) => {
+                        spinner.text = `${p.message}${p.entities ? ` (${p.entities} entities, ${p.relationships || 0} relationships)` : ''}`;
+                        if (p.throughput && p.eta) {
+                            spinner.text += ` - ${p.throughput.toFixed(1)}/sec, ETA: ${Math.round(p.eta / 1000)}s`;
+                        }
+                    } : undefined
+                });
             }
             spinner.succeed('Import complete');
             // Post-processing: extract concepts if requested
@@ -270,15 +299,21 @@ export const importCommands = {
             if (!options.json && !options.quiet) {
                 console.log(chalk.cyan('\n📊 Import Results:\n'));
                 console.log(chalk.bold('Statistics:'));
-                console.log(`  Entities created: ${chalk.green(result.stats.entitiesCreated)}`);
-                if (result.stats.relationshipsCreated > 0) {
-                    console.log(`  Relationships created: ${chalk.green(result.stats.relationshipsCreated)}`);
+                const entitiesCount = result.stats?.entitiesCreated || result.entities?.length || 0;
+                const relationshipsCount = result.stats?.relationshipsCreated || result.relationships?.length || 0;
+                console.log(`  Entities created: ${chalk.green(entitiesCount)}`);
+                if (relationshipsCount > 0) {
+                    console.log(`  Relationships created: ${chalk.green(relationshipsCount)}`);
                 }
-                if (result.stats.filesProcessed) {
+                if (result.stats?.filesProcessed) {
                     console.log(`  Files processed: ${chalk.green(result.stats.filesProcessed)}`);
                 }
-                console.log(`  Average confidence: ${chalk.yellow((result.stats.averageConfidence * 100).toFixed(1))}%`);
-                console.log(`  Processing time: ${chalk.dim(result.stats.processingTimeMs)}ms`);
+                if (result.stats?.averageConfidence) {
+                    console.log(`  Average confidence: ${chalk.yellow((result.stats.averageConfidence * 100).toFixed(1))}%`);
+                }
+                if (result.stats?.processingTimeMs) {
+                    console.log(`  Processing time: ${chalk.dim(result.stats.processingTimeMs)}ms`);
+                }
                 if (options.verbose && result.entities && result.entities.length > 0) {
                     console.log(chalk.bold('\n📦 Imported Entities (first 10):'));
                     result.entities.slice(0, 10).forEach((entity, i) => {

package/dist/importers/SmartCSVImporter.js

@@ -53,10 +53,44 @@ export class SmartCSVImporter {
             ...options
         };
         // Parse CSV using existing handler
+        // v4.5.0: Pass progress hooks to handler for file parsing progress
         const processedData = await this.csvHandler.process(buffer, {
             ...options,
             csvDelimiter: opts.csvDelimiter,
-            csvHeaders: opts.csvHeaders
+            csvHeaders: opts.csvHeaders,
+            totalBytes: buffer.length,
+            progressHooks: {
+                onBytesProcessed: (bytes) => {
+                    // Handler reports bytes processed during parsing
+                    opts.onProgress?.({
+                        processed: 0,
+                        total: 0,
+                        entities: 0,
+                        relationships: 0,
+                        phase: `Parsing CSV (${Math.round((bytes / buffer.length) * 100)}%)`
+                    });
+                },
+                onCurrentItem: (message) => {
+                    // Handler reports current processing step
+                    opts.onProgress?.({
+                        processed: 0,
+                        total: 0,
+                        entities: 0,
+                        relationships: 0,
+                        phase: message
+                    });
+                },
+                onDataExtracted: (count, total) => {
+                    // Handler reports rows extracted
+                    opts.onProgress?.({
+                        processed: 0,
+                        total: total || count,
+                        entities: 0,
+                        relationships: 0,
+                        phase: `Extracted ${count} rows`
+                    });
+                }
+            }
         });
         const rows = processedData.data;
         if (rows.length === 0) {

package/dist/importers/SmartDOCXImporter.js

@@ -54,10 +54,22 @@ export class SmartDOCXImporter {
         if (!this.mammothLoaded) {
             await this.init();
         }
+        // v4.5.0: Report parsing start
+        options.onProgress?.({
+            processed: 0,
+            entities: 0,
+            relationships: 0
+        });
         // Extract raw text for entity extraction
         const textResult = await mammoth.extractRawText({ buffer });
         // Extract HTML for structure analysis (headings, tables)
         const htmlResult = await mammoth.convertToHtml({ buffer });
+        // v4.5.0: Report parsing complete
+        options.onProgress?.({
+            processed: 0,
+            entities: 0,
+            relationships: 0
+        });
         // Process the document
         const result = await this.extractFromContent(textResult.value, htmlResult.value, options);
         result.processingTime = Date.now() - startTime;

package/dist/importers/SmartExcelImporter.js

@@ -61,7 +61,43 @@ export class SmartExcelImporter {
             ...options
         };
         // Parse Excel using existing handler
-        const processedData = await this.excelHandler.process(buffer, options);
+        // v4.5.0: Pass progress hooks to handler for file parsing progress
+        const processedData = await this.excelHandler.process(buffer, {
+            ...options,
+            totalBytes: buffer.length,
+            progressHooks: {
+                onBytesProcessed: (bytes) => {
+                    // Handler reports bytes processed during parsing
+                    opts.onProgress?.({
+                        processed: 0,
+                        total: 0,
+                        entities: 0,
+                        relationships: 0,
+                        phase: `Parsing Excel (${Math.round((bytes / buffer.length) * 100)}%)`
+                    });
+                },
+                onCurrentItem: (message) => {
+                    // Handler reports current processing step (e.g., "Reading sheet: Sales (1/3)")
+                    opts.onProgress?.({
+                        processed: 0,
+                        total: 0,
+                        entities: 0,
+                        relationships: 0,
+                        phase: message
+                    });
+                },
+                onDataExtracted: (count, total) => {
+                    // Handler reports rows extracted
+                    opts.onProgress?.({
+                        processed: 0,
+                        total: total || count,
+                        entities: 0,
+                        relationships: 0,
+                        phase: `Extracted ${count} rows from Excel`
+                    });
+                }
+            }
+        });
         const rows = processedData.data;
         if (rows.length === 0) {
             return this.emptyResult(startTime);

package/dist/importers/SmartJSONImporter.js

@@ -48,6 +48,12 @@ export class SmartJSONImporter {
             onProgress: () => { },
             ...options
         };
+        // v4.5.0: Report parsing start
+        opts.onProgress({
+            processed: 0,
+            entities: 0,
+            relationships: 0
+        });
         // Parse JSON if string
         let jsonData;
         if (typeof data === 'string') {
@@ -61,6 +67,12 @@ export class SmartJSONImporter {
         else {
             jsonData = data;
         }
+        // v4.5.0: Report parsing complete, starting traversal
+        opts.onProgress({
+            processed: 0,
+            entities: 0,
+            relationships: 0
+        });
         // Traverse and extract
         const entities = [];
         const relationships = [];
@@ -82,6 +94,12 @@ export class SmartJSONImporter {
                 });
             }
         });
+        // v4.5.0: Report completion
+        opts.onProgress({
+            processed: nodesProcessed,
+            entities: entities.length,
+            relationships: relationships.length
+        });
         return {
             nodesProcessed,
             entitiesExtracted: entities.length,

package/dist/importers/SmartMarkdownImporter.js

@@ -46,8 +46,22 @@ export class SmartMarkdownImporter {
             onProgress: () => { },
             ...options
         };
+        // v4.5.0: Report parsing start
+        opts.onProgress({
+            processed: 0,
+            total: 0,
+            entities: 0,
+            relationships: 0
+        });
         // Parse markdown into sections
         const parsedSections = this.parseMarkdown(markdown, opts);
+        // v4.5.0: Report parsing complete
+        opts.onProgress({
+            processed: 0,
+            total: parsedSections.length,
+            entities: 0,
+            relationships: 0
+        });
         // Process each section
         const sections = [];
         const entityMap = new Map();
@@ -69,10 +83,19 @@ export class SmartMarkdownImporter {
                 relationships: sections.reduce((sum, s) => sum + s.relationships.length, 0)
             });
         }
+        // v4.5.0: Report completion
+        const totalEntities = sections.reduce((sum, s) => sum + s.entities.length, 0);
+        const totalRelationships = sections.reduce((sum, s) => sum + s.relationships.length, 0);
+        opts.onProgress({
+            processed: sections.length,
+            total: sections.length,
+            entities: totalEntities,
+            relationships: totalRelationships
+        });
         return {
             sectionsProcessed: sections.length,
-            entitiesExtracted: sections.reduce((sum, s) => sum + s.entities.length, 0),
-            relationshipsInferred: sections.reduce((sum, s) => sum + s.relationships.length, 0),
+            entitiesExtracted: totalEntities,
+            relationshipsInferred: totalRelationships,
             sections,
             entityMap,
             processingTime: Date.now() - startTime,

package/dist/importers/SmartPDFImporter.js

@@ -49,7 +49,43 @@ export class SmartPDFImporter {
             ...options
         };
         // Parse PDF using existing handler
-        const processedData = await this.pdfHandler.process(buffer, options);
+        // v4.5.0: Pass progress hooks to handler for file parsing progress
+        const processedData = await this.pdfHandler.process(buffer, {
+            ...options,
+            totalBytes: buffer.length,
+            progressHooks: {
+                onBytesProcessed: (bytes) => {
+                    // Handler reports bytes processed during parsing
+                    opts.onProgress?.({
+                        processed: 0,
+                        total: 0,
+                        entities: 0,
+                        relationships: 0,
+                        phase: `Parsing PDF (${Math.round((bytes / buffer.length) * 100)}%)`
+                    });
+                },
+                onCurrentItem: (message) => {
+                    // Handler reports current processing step (e.g., "Processing page 5 of 23")
+                    opts.onProgress?.({
+                        processed: 0,
+                        total: 0,
+                        entities: 0,
+                        relationships: 0,
+                        phase: message
+                    });
+                },
+                onDataExtracted: (count, total) => {
+                    // Handler reports items extracted (paragraphs + tables)
+                    opts.onProgress?.({
+                        processed: 0,
+                        total: total || count,
+                        entities: 0,
+                        relationships: 0,
+                        phase: `Extracted ${count} items from PDF`
+                    });
+                }
+            }
+        });
         const data = processedData.data;
         const pdfMetadata = processedData.metadata.additionalInfo?.pdfMetadata || {};
         if (data.length === 0) {

package/dist/importers/SmartYAMLImporter.js

@@ -37,6 +37,12 @@ export class SmartYAMLImporter {
      */
     async extract(yamlContent, options = {}) {
         const startTime = Date.now();
+        // v4.5.0: Report parsing start
+        options.onProgress?.({
+            processed: 0,
+            entities: 0,
+            relationships: 0
+        });
         // Parse YAML to JavaScript object
         const yamlString = typeof yamlContent === 'string'
             ? yamlContent
@@ -48,6 +54,12 @@ export class SmartYAMLImporter {
         catch (error) {
             throw new Error(`Failed to parse YAML: ${error.message}`);
         }
+        // v4.5.0: Report parsing complete
+        options.onProgress?.({
+            processed: 0,
+            entities: 0,
+            relationships: 0
+        });
         // Process as JSON-like structure
         const result = await this.extractFromData(data, options);
         result.processingTime = Date.now() - startTime;

package/dist/types/brainy.types.d.ts

@@ -146,6 +146,7 @@ export interface FindParams<T = any> {
     mode?: SearchMode;
     explain?: boolean;
     includeRelations?: boolean;
+    includeVFS?: boolean;
     service?: string;
     fusion?: {
         strategy?: 'adaptive' | 'weighted' | 'progressive';
@@ -183,6 +184,7 @@ export interface SimilarParams<T = any> {
     type?: NounType | NounType[];
     where?: Partial<T>;
     service?: string;
+    includeVFS?: boolean;
 }
 /**
  * Parameters for getting relationships
@@ -317,6 +319,102 @@ export interface BatchResult<T = any> {
     total: number;
     duration: number;
 }
+/**
+ * Import stage enumeration
+ */
+export type ImportStage = 'detecting' | 'reading' | 'parsing' | 'extracting' | 'indexing' | 'completing';
+/**
+ * Overall import status
+ */
+export type ImportStatus = 'starting' | 'processing' | 'completing' | 'done';
+/**
+ * Comprehensive import progress information
+ *
+ * Provides multi-dimensional progress tracking:
+ * - Bytes processed (always deterministic)
+ * - Entities extracted and indexed
+ * - Stage-specific progress
+ * - Time estimates
+ * - Performance metrics
+ *
+ * @since v4.5.0
+ */
+export interface ImportProgress {
+    overall_progress: number;
+    overall_status: ImportStatus;
+    stage: ImportStage;
+    stage_progress: number;
+    stage_message: string;
+    bytes_processed: number;
+    total_bytes: number;
+    bytes_percentage: number;
+    bytes_per_second?: number;
+    entities_extracted: number;
+    entities_indexed: number;
+    entities_per_second?: number;
+    estimated_total_entities?: number;
+    estimation_confidence?: number;
+    elapsed_ms: number;
+    estimated_remaining_ms?: number;
+    estimated_total_ms?: number;
+    current_item?: string;
+    current_file?: string;
+    file_number?: number;
+    total_files?: number;
+    metrics?: {
+        parsing_rate_mbps?: number;
+        extraction_rate_entities_per_sec?: number;
+        indexing_rate_entities_per_sec?: number;
+        memory_usage_mb?: number;
+        peak_memory_mb?: number;
+    };
+    current: number;
+    total: number;
+}
+/**
+ * Import progress callback - backwards compatible
+ *
+ * Supports both legacy (current, total) and new (ImportProgress object) signatures
+ */
+export type ImportProgressCallback = ((progress: ImportProgress) => void) | ((current: number, total: number) => void);
+/**
+ * Stage weight configuration for overall progress calculation
+ *
+ * These weights reflect the typical time distribution across stages.
+ * Extraction is typically the slowest stage (60% of time).
+ */
+export interface StageWeights {
+    detecting: number;
+    reading: number;
+    parsing: number;
+    extracting: number;
+    indexing: number;
+    completing: number;
+}
+/**
+ * Import result statistics
+ */
+export interface ImportStats {
+    graphNodesCreated: number;
+    graphEdgesCreated: number;
+    vfsFilesCreated: number;
+    duration: number;
+    bytesProcessed: number;
+    averageRate: number;
+    peakMemoryMB?: number;
+}
+/**
+ * Import operation result
+ */
+export interface ImportResult {
+    success: boolean;
+    stats: ImportStats;
+    errors?: Array<{
+        stage: ImportStage;
+        message: string;
+        error?: any;
+    }>;
+}
 /**
  * Graph traversal parameters
  */