@soulcraft/brainy 4.4.0 → 4.5.1

package/dist/importers/SmartExcelImporter.js

@@ -61,7 +61,43 @@ export class SmartExcelImporter {
             ...options
         };
         // Parse Excel using existing handler
-        const processedData = await this.excelHandler.process(buffer, options);
+        // v4.5.0: Pass progress hooks to handler for file parsing progress
+        const processedData = await this.excelHandler.process(buffer, {
+            ...options,
+            totalBytes: buffer.length,
+            progressHooks: {
+                onBytesProcessed: (bytes) => {
+                    // Handler reports bytes processed during parsing
+                    opts.onProgress?.({
+                        processed: 0,
+                        total: 0,
+                        entities: 0,
+                        relationships: 0,
+                        phase: `Parsing Excel (${Math.round((bytes / buffer.length) * 100)}%)`
+                    });
+                },
+                onCurrentItem: (message) => {
+                    // Handler reports current processing step (e.g., "Reading sheet: Sales (1/3)")
+                    opts.onProgress?.({
+                        processed: 0,
+                        total: 0,
+                        entities: 0,
+                        relationships: 0,
+                        phase: message
+                    });
+                },
+                onDataExtracted: (count, total) => {
+                    // Handler reports rows extracted
+                    opts.onProgress?.({
+                        processed: 0,
+                        total: total || count,
+                        entities: 0,
+                        relationships: 0,
+                        phase: `Extracted ${count} rows from Excel`
+                    });
+                }
+            }
+        });
         const rows = processedData.data;
         if (rows.length === 0) {
             return this.emptyResult(startTime);

package/dist/importers/SmartJSONImporter.js

@@ -48,6 +48,12 @@ export class SmartJSONImporter {
             onProgress: () => { },
             ...options
         };
+        // v4.5.0: Report parsing start
+        opts.onProgress({
+            processed: 0,
+            entities: 0,
+            relationships: 0
+        });
         // Parse JSON if string
         let jsonData;
         if (typeof data === 'string') {
@@ -61,6 +67,12 @@ export class SmartJSONImporter {
         else {
             jsonData = data;
         }
+        // v4.5.0: Report parsing complete, starting traversal
+        opts.onProgress({
+            processed: 0,
+            entities: 0,
+            relationships: 0
+        });
         // Traverse and extract
         const entities = [];
         const relationships = [];
@@ -82,6 +94,12 @@ export class SmartJSONImporter {
                 });
             }
         });
+        // v4.5.0: Report completion
+        opts.onProgress({
+            processed: nodesProcessed,
+            entities: entities.length,
+            relationships: relationships.length
+        });
         return {
             nodesProcessed,
             entitiesExtracted: entities.length,

package/dist/importers/SmartMarkdownImporter.js

@@ -46,8 +46,22 @@ export class SmartMarkdownImporter {
             onProgress: () => { },
             ...options
         };
+        // v4.5.0: Report parsing start
+        opts.onProgress({
+            processed: 0,
+            total: 0,
+            entities: 0,
+            relationships: 0
+        });
         // Parse markdown into sections
         const parsedSections = this.parseMarkdown(markdown, opts);
+        // v4.5.0: Report parsing complete
+        opts.onProgress({
+            processed: 0,
+            total: parsedSections.length,
+            entities: 0,
+            relationships: 0
+        });
         // Process each section
         const sections = [];
         const entityMap = new Map();
@@ -69,10 +83,19 @@ export class SmartMarkdownImporter {
                 relationships: sections.reduce((sum, s) => sum + s.relationships.length, 0)
             });
         }
+        // v4.5.0: Report completion
+        const totalEntities = sections.reduce((sum, s) => sum + s.entities.length, 0);
+        const totalRelationships = sections.reduce((sum, s) => sum + s.relationships.length, 0);
+        opts.onProgress({
+            processed: sections.length,
+            total: sections.length,
+            entities: totalEntities,
+            relationships: totalRelationships
+        });
         return {
             sectionsProcessed: sections.length,
-            entitiesExtracted: sections.reduce((sum, s) => sum + s.entities.length, 0),
-            relationshipsInferred: sections.reduce((sum, s) => sum + s.relationships.length, 0),
+            entitiesExtracted: totalEntities,
+            relationshipsInferred: totalRelationships,
             sections,
             entityMap,
             processingTime: Date.now() - startTime,

package/dist/importers/SmartPDFImporter.js

@@ -49,7 +49,43 @@ export class SmartPDFImporter {
             ...options
         };
         // Parse PDF using existing handler
-        const processedData = await this.pdfHandler.process(buffer, options);
+        // v4.5.0: Pass progress hooks to handler for file parsing progress
+        const processedData = await this.pdfHandler.process(buffer, {
+            ...options,
+            totalBytes: buffer.length,
+            progressHooks: {
+                onBytesProcessed: (bytes) => {
+                    // Handler reports bytes processed during parsing
+                    opts.onProgress?.({
+                        processed: 0,
+                        total: 0,
+                        entities: 0,
+                        relationships: 0,
+                        phase: `Parsing PDF (${Math.round((bytes / buffer.length) * 100)}%)`
+                    });
+                },
+                onCurrentItem: (message) => {
+                    // Handler reports current processing step (e.g., "Processing page 5 of 23")
+                    opts.onProgress?.({
+                        processed: 0,
+                        total: 0,
+                        entities: 0,
+                        relationships: 0,
+                        phase: message
+                    });
+                },
+                onDataExtracted: (count, total) => {
+                    // Handler reports items extracted (paragraphs + tables)
+                    opts.onProgress?.({
+                        processed: 0,
+                        total: total || count,
+                        entities: 0,
+                        relationships: 0,
+                        phase: `Extracted ${count} items from PDF`
+                    });
+                }
+            }
+        });
         const data = processedData.data;
         const pdfMetadata = processedData.metadata.additionalInfo?.pdfMetadata || {};
         if (data.length === 0) {

package/dist/importers/SmartYAMLImporter.js

@@ -37,6 +37,12 @@ export class SmartYAMLImporter {
      */
     async extract(yamlContent, options = {}) {
         const startTime = Date.now();
+        // v4.5.0: Report parsing start
+        options.onProgress?.({
+            processed: 0,
+            entities: 0,
+            relationships: 0
+        });
         // Parse YAML to JavaScript object
         const yamlString = typeof yamlContent === 'string'
             ? yamlContent
@@ -48,6 +54,12 @@ export class SmartYAMLImporter {
         catch (error) {
             throw new Error(`Failed to parse YAML: ${error.message}`);
         }
+        // v4.5.0: Report parsing complete
+        options.onProgress?.({
+            processed: 0,
+            entities: 0,
+            relationships: 0
+        });
         // Process as JSON-like structure
         const result = await this.extractFromData(data, options);
         result.processingTime = Date.now() - startTime;

package/dist/types/brainy.types.d.ts

@@ -266,6 +266,16 @@ export interface GetRelationsParams {
      * Only return relationships belonging to this service.
      */
     service?: string;
+    /**
+     * Include VFS relationships (v4.5.1)
+     *
+     * By default, getRelations() excludes VFS relationships (since v4.4.0).
+     * Set this to true when you need to traverse VFS structure.
+     *
+     * @default false
+     * @since v4.5.1
+     */
+    includeVFS?: boolean;
 }
 /**
  * Batch add parameters
@@ -319,6 +329,102 @@ export interface BatchResult<T = any> {
     total: number;
     duration: number;
 }
+/**
+ * Import stage enumeration
+ */
+export type ImportStage = 'detecting' | 'reading' | 'parsing' | 'extracting' | 'indexing' | 'completing';
+/**
+ * Overall import status
+ */
+export type ImportStatus = 'starting' | 'processing' | 'completing' | 'done';
+/**
+ * Comprehensive import progress information
+ *
+ * Provides multi-dimensional progress tracking:
+ * - Bytes processed (always deterministic)
+ * - Entities extracted and indexed
+ * - Stage-specific progress
+ * - Time estimates
+ * - Performance metrics
+ *
+ * @since v4.5.0
+ */
+export interface ImportProgress {
+    overall_progress: number;
+    overall_status: ImportStatus;
+    stage: ImportStage;
+    stage_progress: number;
+    stage_message: string;
+    bytes_processed: number;
+    total_bytes: number;
+    bytes_percentage: number;
+    bytes_per_second?: number;
+    entities_extracted: number;
+    entities_indexed: number;
+    entities_per_second?: number;
+    estimated_total_entities?: number;
+    estimation_confidence?: number;
+    elapsed_ms: number;
+    estimated_remaining_ms?: number;
+    estimated_total_ms?: number;
+    current_item?: string;
+    current_file?: string;
+    file_number?: number;
+    total_files?: number;
+    metrics?: {
+        parsing_rate_mbps?: number;
+        extraction_rate_entities_per_sec?: number;
+        indexing_rate_entities_per_sec?: number;
+        memory_usage_mb?: number;
+        peak_memory_mb?: number;
+    };
+    current: number;
+    total: number;
+}
+/**
+ * Import progress callback - backwards compatible
+ *
+ * Supports both legacy (current, total) and new (ImportProgress object) signatures
+ */
+export type ImportProgressCallback = ((progress: ImportProgress) => void) | ((current: number, total: number) => void);
+/**
+ * Stage weight configuration for overall progress calculation
+ *
+ * These weights reflect the typical time distribution across stages.
+ * Extraction is typically the slowest stage (60% of time).
+ */
+export interface StageWeights {
+    detecting: number;
+    reading: number;
+    parsing: number;
+    extracting: number;
+    indexing: number;
+    completing: number;
+}
+/**
+ * Import result statistics
+ */
+export interface ImportStats {
+    graphNodesCreated: number;
+    graphEdgesCreated: number;
+    vfsFilesCreated: number;
+    duration: number;
+    bytesProcessed: number;
+    averageRate: number;
+    peakMemoryMB?: number;
+}
+/**
+ * Import operation result
+ */
+export interface ImportResult {
+    success: boolean;
+    stats: ImportStats;
+    errors?: Array<{
+        stage: ImportStage;
+        message: string;
+        error?: any;
+    }>;
+}
 /**
  * Graph traversal parameters
  */

package/dist/utils/import-progress-tracker.d.ts

@@ -0,0 +1,140 @@
+/**
+ * Import Progress Tracker (v4.5.0)
+ *
+ * Comprehensive progress tracking for imports with:
+ * - Multi-dimensional progress (bytes, entities, stages, timing)
+ * - Smart estimation (entity count, time remaining)
+ * - Stage-specific metrics (bytes/sec vs entities/sec)
+ * - Throttled callbacks (avoid spam)
+ * - Weighted overall progress
+ *
+ * @since v4.5.0
+ */
+import { ImportProgress, ImportStage, StageWeights, ImportProgressCallback } from '../types/brainy.types.js';
+/**
+ * Progress tracker for imports
+ */
+export declare class ImportProgressTracker {
+    private readonly stageWeights;
+    private readonly throttleMs;
+    private readonly callback?;
+    private startTime;
+    private lastEmitTime;
+    private currentStage;
+    private completedStages;
+    private totalBytes;
+    private bytesProcessed;
+    private entitiesExtracted;
+    private entitiesIndexed;
+    private parseStartTime?;
+    private extractStartTime?;
+    private indexStartTime?;
+    private lastBytesCheckpoint;
+    private lastBytesCheckpointTime;
+    private lastEntitiesCheckpoint;
+    private lastEntitiesCheckpointTime;
+    private currentItem?;
+    private currentFile?;
+    private fileNumber?;
+    private totalFiles?;
+    private peakMemoryMB;
+    constructor(options?: {
+        totalBytes?: number;
+        stageWeights?: Partial<StageWeights>;
+        throttleMs?: number;
+        callback?: ImportProgressCallback;
+    });
+    /**
+     * Set total file size (if known later)
+     */
+    setTotalBytes(bytes: number): void;
+    /**
+     * Update current stage
+     */
+    setStage(stage: ImportStage, message?: string): void;
+    /**
+     * Update bytes processed
+     */
+    updateBytes(bytes: number): void;
+    /**
+     * Increment bytes processed
+     */
+    addBytes(bytes: number): void;
+    /**
+     * Update entities extracted
+     */
+    updateEntitiesExtracted(count: number): void;
+    /**
+     * Increment entities extracted
+     */
+    addEntitiesExtracted(count: number): void;
+    /**
+     * Update entities indexed
+     */
+    updateEntitiesIndexed(count: number): void;
+    /**
+     * Increment entities indexed
+     */
+    addEntitiesIndexed(count: number): void;
+    /**
+     * Set context information
+     */
+    setContext(context: {
+        currentItem?: string;
+        currentFile?: string;
+        fileNumber?: number;
+        totalFiles?: number;
+    }): void;
+    /**
+     * Set stage message
+     */
+    private setStageMessage;
+    /**
+     * Calculate stage progress (0-100 within current stage)
+     */
+    private calculateStageProgress;
+    /**
+     * Calculate overall progress (0-100 weighted across all stages)
+     */
+    private calculateOverallProgress;
+    /**
+     * Calculate bytes per second
+     */
+    private calculateBytesPerSecond;
+    /**
+     * Calculate entities per second
+     */
+    private calculateEntitiesPerSecond;
+    /**
+     * Estimate total entities
+     */
+    private estimateTotalEntities;
+    /**
+     * Estimate remaining time
+     */
+    private estimateRemainingTime;
+    /**
+     * Get current memory usage
+     */
+    private getCurrentMemoryMB;
+    /**
+     * Build complete progress object
+     */
+    private buildProgress;
+    /**
+     * Emit progress (throttled)
+     */
+    private emit;
+    /**
+     * Force emit (for completion or critical updates)
+     */
+    forceEmit(): void;
+    /**
+     * Get current progress (without emitting)
+     */
+    getProgress(): ImportProgress;
+    /**
+     * Mark import as complete
+     */
+    complete(): ImportProgress;
+}