@soulcraft/brainy 4.4.0 → 4.5.1

package/dist/utils/import-progress-tracker.js

@@ -0,0 +1,444 @@
+/**
+ * Import Progress Tracker (v4.5.0)
+ *
+ * Comprehensive progress tracking for imports with:
+ * - Multi-dimensional progress (bytes, entities, stages, timing)
+ * - Smart estimation (entity count, time remaining)
+ * - Stage-specific metrics (bytes/sec vs entities/sec)
+ * - Throttled callbacks (avoid spam)
+ * - Weighted overall progress
+ *
+ * @since v4.5.0
+ */
+/**
+ * Default stage weights (reflect typical time distribution)
+ */
+const DEFAULT_STAGE_WEIGHTS = {
+    detecting: 0.01, // 1% - very fast
+    reading: 0.05, // 5% - reading file
+    parsing: 0.10, // 10% - parsing structure
+    extracting: 0.60, // 60% - AI extraction (slowest!)
+    indexing: 0.20, // 20% - creating graph
+    completing: 0.04 // 4% - cleanup
+};
+/**
+ * Stage ordering for progress calculation
+ */
+const STAGE_ORDER = [
+    'detecting',
+    'reading',
+    'parsing',
+    'extracting',
+    'indexing',
+    'completing'
+];
+/**
+ * Progress tracker for imports
+ */
+export class ImportProgressTracker {
+    constructor(options = {}) {
+        this.lastEmitTime = 0;
+        this.currentStage = 'detecting';
+        this.completedStages = new Set();
+        // Metrics
+        this.totalBytes = 0;
+        this.bytesProcessed = 0;
+        this.entitiesExtracted = 0;
+        this.entitiesIndexed = 0;
+        // Estimation
+        this.lastBytesCheckpoint = 0;
+        this.lastBytesCheckpointTime = 0;
+        this.lastEntitiesCheckpoint = 0;
+        this.lastEntitiesCheckpointTime = 0;
+        // Memory tracking
+        this.peakMemoryMB = 0;
+        this.stageWeights = { ...DEFAULT_STAGE_WEIGHTS, ...options.stageWeights };
+        this.throttleMs = options.throttleMs ?? 100; // 100ms default
+        this.callback = options.callback;
+        this.totalBytes = options.totalBytes ?? 0;
+        this.startTime = Date.now();
+        this.lastBytesCheckpointTime = this.startTime;
+        this.lastEntitiesCheckpointTime = this.startTime;
+    }
+    /**
+     * Set total file size (if known later)
+     */
+    setTotalBytes(bytes) {
+        this.totalBytes = bytes;
+    }
+    /**
+     * Update current stage
+     */
+    setStage(stage, message) {
+        // Mark previous stage as complete
+        if (this.currentStage !== stage) {
+            this.completedStages.add(this.currentStage);
+        }
+        this.currentStage = stage;
+        if (message) {
+            this.setStageMessage(message);
+        }
+        // Track stage start times
+        const now = Date.now();
+        switch (stage) {
+            case 'parsing':
+                this.parseStartTime = now;
+                break;
+            case 'extracting':
+                this.extractStartTime = now;
+                break;
+            case 'indexing':
+                this.indexStartTime = now;
+                break;
+        }
+        // Force emit on stage change
+        this.emit(true);
+    }
+    /**
+     * Update bytes processed
+     */
+    updateBytes(bytes) {
+        this.bytesProcessed = bytes;
+        this.emit();
+    }
+    /**
+     * Increment bytes processed
+     */
+    addBytes(bytes) {
+        this.bytesProcessed += bytes;
+        this.emit();
+    }
+    /**
+     * Update entities extracted
+     */
+    updateEntitiesExtracted(count) {
+        this.entitiesExtracted = count;
+        this.emit();
+    }
+    /**
+     * Increment entities extracted
+     */
+    addEntitiesExtracted(count) {
+        this.entitiesExtracted += count;
+        this.emit();
+    }
+    /**
+     * Update entities indexed
+     */
+    updateEntitiesIndexed(count) {
+        this.entitiesIndexed = count;
+        this.emit();
+    }
+    /**
+     * Increment entities indexed
+     */
+    addEntitiesIndexed(count) {
+        this.entitiesIndexed += count;
+        this.emit();
+    }
+    /**
+     * Set context information
+     */
+    setContext(context) {
+        if (context.currentItem !== undefined)
+            this.currentItem = context.currentItem;
+        if (context.currentFile !== undefined)
+            this.currentFile = context.currentFile;
+        if (context.fileNumber !== undefined)
+            this.fileNumber = context.fileNumber;
+        if (context.totalFiles !== undefined)
+            this.totalFiles = context.totalFiles;
+        this.emit();
+    }
+    /**
+     * Set stage message
+     */
+    setStageMessage(message) {
+        this.currentItem = message;
+    }
+    /**
+     * Calculate stage progress (0-100 within current stage)
+     */
+    calculateStageProgress() {
+        switch (this.currentStage) {
+            case 'detecting':
+            case 'completing':
+                // These are quick, assume 100% once started
+                return 100;
+            case 'reading':
+            case 'parsing':
+                // Use bytes as proxy for progress
+                if (this.totalBytes === 0)
+                    return 0;
+                return Math.min(100, (this.bytesProcessed / this.totalBytes) * 100);
+            case 'extracting':
+                // Extraction progress is hard to estimate (AI is unpredictable)
+                // We can't reliably say % complete, so return 0
+                return 0;
+            case 'indexing':
+                // If we have estimated total entities, use that
+                if (this.entitiesExtracted > 0) {
+                    return Math.min(100, (this.entitiesIndexed / this.entitiesExtracted) * 100);
+                }
+                return 0;
+            default:
+                return 0;
+        }
+    }
+    /**
+     * Calculate overall progress (0-100 weighted across all stages)
+     */
+    calculateOverallProgress() {
+        // Calculate progress of completed stages
+        let completedWeight = 0;
+        for (const stage of this.completedStages) {
+            completedWeight += this.stageWeights[stage];
+        }
+        // Calculate progress of current stage
+        const stageProgress = this.calculateStageProgress();
+        const currentStageContribution = this.stageWeights[this.currentStage] * (stageProgress / 100);
+        // Overall = completed stages + current stage contribution
+        const overall = (completedWeight + currentStageContribution) * 100;
+        return Math.min(100, Math.max(0, overall));
+    }
+    /**
+     * Calculate bytes per second
+     */
+    calculateBytesPerSecond() {
+        const now = Date.now();
+        const elapsed = now - this.lastBytesCheckpointTime;
+        // Need at least 1 second of data
+        if (elapsed < 1000)
+            return undefined;
+        const bytesDelta = this.bytesProcessed - this.lastBytesCheckpoint;
+        const bytesPerSec = (bytesDelta / elapsed) * 1000;
+        // Update checkpoint
+        this.lastBytesCheckpoint = this.bytesProcessed;
+        this.lastBytesCheckpointTime = now;
+        return bytesPerSec > 0 ? bytesPerSec : undefined;
+    }
+    /**
+     * Calculate entities per second
+     */
+    calculateEntitiesPerSecond() {
+        const now = Date.now();
+        const elapsed = now - this.lastEntitiesCheckpointTime;
+        // Need at least 1 second of data
+        if (elapsed < 1000)
+            return undefined;
+        // Use appropriate counter based on stage
+        const currentCount = this.currentStage === 'indexing'
+            ? this.entitiesIndexed
+            : this.entitiesExtracted;
+        const entitiesDelta = currentCount - this.lastEntitiesCheckpoint;
+        const entitiesPerSec = (entitiesDelta / elapsed) * 1000;
+        // Update checkpoint
+        this.lastEntitiesCheckpoint = currentCount;
+        this.lastEntitiesCheckpointTime = now;
+        return entitiesPerSec > 0 ? entitiesPerSec : undefined;
+    }
+    /**
+     * Estimate total entities
+     */
+    estimateTotalEntities() {
+        // Only estimate if we've processed some bytes and extracted some entities
+        if (this.bytesProcessed === 0 || this.entitiesExtracted === 0 || this.totalBytes === 0) {
+            return undefined;
+        }
+        // Estimate based on entities per byte
+        const bytesPercentage = this.bytesProcessed / this.totalBytes;
+        const estimatedTotal = Math.ceil(this.entitiesExtracted / bytesPercentage);
+        // Confidence increases with more data
+        const confidence = Math.min(0.95, bytesPercentage);
+        return { count: estimatedTotal, confidence };
+    }
+    /**
+     * Estimate remaining time
+     */
+    estimateRemainingTime() {
+        const now = Date.now();
+        const elapsed = now - this.startTime;
+        // Need at least 5 seconds of data for reasonable estimate
+        if (elapsed < 5000)
+            return undefined;
+        const overallProgress = this.calculateOverallProgress();
+        if (overallProgress === 0)
+            return undefined;
+        // Estimate total time based on current progress
+        const estimatedTotalMs = (elapsed / overallProgress) * 100;
+        const remainingMs = estimatedTotalMs - elapsed;
+        return remainingMs > 0 ? remainingMs : undefined;
+    }
+    /**
+     * Get current memory usage
+     */
+    getCurrentMemoryMB() {
+        if (typeof process === 'undefined' || !process.memoryUsage)
+            return undefined;
+        const usage = process.memoryUsage();
+        const currentMB = usage.heapUsed / 1024 / 1024;
+        // Track peak
+        this.peakMemoryMB = Math.max(this.peakMemoryMB, currentMB);
+        return currentMB;
+    }
+    /**
+     * Build complete progress object
+     */
+    buildProgress() {
+        const now = Date.now();
+        const elapsed = now - this.startTime;
+        const stageProgress = this.calculateStageProgress();
+        const overallProgress = this.calculateOverallProgress();
+        const bytesPerSec = this.calculateBytesPerSecond();
+        const entitiesPerSec = this.calculateEntitiesPerSecond();
+        const entityEstimate = this.estimateTotalEntities();
+        const remainingMs = this.estimateRemainingTime();
+        const currentMemoryMB = this.getCurrentMemoryMB();
+        // Determine overall status
+        let overallStatus;
+        if (overallProgress === 0) {
+            overallStatus = 'starting';
+        }
+        else if (overallProgress === 100) {
+            overallStatus = 'done';
+        }
+        else if (this.currentStage === 'completing') {
+            overallStatus = 'completing';
+        }
+        else {
+            overallStatus = 'processing';
+        }
+        // Stage message
+        let stageMessage;
+        if (this.currentItem) {
+            stageMessage = this.currentItem;
+        }
+        else {
+            // Default messages
+            switch (this.currentStage) {
+                case 'detecting':
+                    stageMessage = 'Detecting file format...';
+                    break;
+                case 'reading':
+                    stageMessage = 'Reading file...';
+                    break;
+                case 'parsing':
+                    stageMessage = 'Parsing file structure...';
+                    break;
+                case 'extracting':
+                    stageMessage = 'Extracting entities using AI...';
+                    break;
+                case 'indexing':
+                    stageMessage = 'Creating graph nodes...';
+                    break;
+                case 'completing':
+                    stageMessage = 'Finalizing import...';
+                    break;
+                default:
+                    stageMessage = 'Processing...';
+            }
+        }
+        // Calculate bytes percentage
+        const bytesPercentage = this.totalBytes > 0
+            ? (this.bytesProcessed / this.totalBytes) * 100
+            : 0;
+        // Build metrics object
+        const metrics = {
+            parsing_rate_mbps: this.currentStage === 'parsing' && bytesPerSec
+                ? bytesPerSec / 1000000
+                : undefined,
+            extraction_rate_entities_per_sec: this.currentStage === 'extracting'
+                ? entitiesPerSec
+                : undefined,
+            indexing_rate_entities_per_sec: this.currentStage === 'indexing'
+                ? entitiesPerSec
+                : undefined,
+            memory_usage_mb: currentMemoryMB,
+            peak_memory_mb: this.peakMemoryMB > 0 ? this.peakMemoryMB : undefined
+        };
+        const progress = {
+            // Overall
+            overall_progress: overallProgress,
+            overall_status: overallStatus,
+            // Stage
+            stage: this.currentStage,
+            stage_progress: stageProgress,
+            stage_message: stageMessage,
+            // Bytes
+            bytes_processed: this.bytesProcessed,
+            total_bytes: this.totalBytes,
+            bytes_percentage: bytesPercentage,
+            bytes_per_second: bytesPerSec,
+            // Entities
+            entities_extracted: this.entitiesExtracted,
+            entities_indexed: this.entitiesIndexed,
+            entities_per_second: entitiesPerSec,
+            estimated_total_entities: entityEstimate?.count,
+            estimation_confidence: entityEstimate?.confidence,
+            // Timing
+            elapsed_ms: elapsed,
+            estimated_remaining_ms: remainingMs,
+            estimated_total_ms: remainingMs ? elapsed + remainingMs : undefined,
+            // Context
+            current_item: this.currentItem,
+            current_file: this.currentFile,
+            file_number: this.fileNumber,
+            total_files: this.totalFiles,
+            // Metrics
+            metrics,
+            // Backwards compatibility
+            current: this.entitiesIndexed,
+            total: entityEstimate?.count ?? 0
+        };
+        return progress;
+    }
+    /**
+     * Emit progress (throttled)
+     */
+    emit(force = false) {
+        if (!this.callback)
+            return;
+        const now = Date.now();
+        const timeSinceLastEmit = now - this.lastEmitTime;
+        // Throttle unless forced
+        if (!force && timeSinceLastEmit < this.throttleMs) {
+            return;
+        }
+        const progress = this.buildProgress();
+        // Handle both callback types (legacy and new)
+        if (this.callback.length === 2) {
+            // Legacy callback: (current, total) => void
+            ;
+            this.callback(progress.current, progress.total);
+        }
+        else {
+            // New callback: (progress: ImportProgress) => void
+            ;
+            this.callback(progress);
+        }
+        this.lastEmitTime = now;
+    }
+    /**
+     * Force emit (for completion or critical updates)
+     */
+    forceEmit() {
+        this.emit(true);
+    }
+    /**
+     * Get current progress (without emitting)
+     */
+    getProgress() {
+        return this.buildProgress();
+    }
+    /**
+     * Mark import as complete
+     */
+    complete() {
+        this.currentStage = 'completing';
+        this.completedStages.add('completing');
+        const progress = this.buildProgress();
+        this.forceEmit();
+        return progress;
+    }
+}
+//# sourceMappingURL=import-progress-tracker.js.map

package/dist/vfs/PathResolver.js

@@ -135,7 +135,8 @@ export class PathResolver {
         // Get all relationships where parentId contains other entities
         const relations = await this.brain.getRelations({
             from: parentId,
-            type: VerbType.Contains
+            type: VerbType.Contains,
+            includeVFS: true // v4.5.1: Required to see VFS relationships
         });
         // Find the child with matching name
         for (const relation of relations) {
@@ -159,7 +160,8 @@ export class PathResolver {
         // Production-ready: Use graph relationships (VFS creates these in mkdir/writeFile)
         const relations = await this.brain.getRelations({
             from: dirId,
-            type: VerbType.Contains
+            type: VerbType.Contains,
+            includeVFS: true // v4.5.1: Required to see VFS relationships
         });
         const validChildren = [];
         const childNames = new Set();

package/dist/vfs/VirtualFileSystem.js

@@ -330,7 +330,8 @@ export class VirtualFileSystem {
                 await this.brain.relate({
                     from: parentId,
                     to: existingId,
-                    type: VerbType.Contains
+                    type: VerbType.Contains,
+                    metadata: { isVFS: true } // v4.5.1: Mark as VFS relationship
                 });
             }
         }
@@ -347,7 +348,8 @@ export class VirtualFileSystem {
             await this.brain.relate({
                 from: parentId,
                 to: entity,
-                type: VerbType.Contains
+                type: VerbType.Contains,
+                metadata: { isVFS: true } // v4.5.1: Mark as VFS relationship
             });
             // Update path resolver cache
             await this.pathResolver.createPath(path, entity);
@@ -612,7 +614,8 @@ export class VirtualFileSystem {
                 await this.brain.relate({
                     from: parentId,
                     to: entity,
-                    type: VerbType.Contains
+                    type: VerbType.Contains,
+                    metadata: { isVFS: true } // v4.5.1: Mark as VFS relationship
                 });
             }
             // Update path resolver cache
@@ -1410,7 +1413,12 @@ export class VirtualFileSystem {
             // Add to new parent
             if (newParentPath && newParentPath !== '/') {
                 const newParentId = await this.pathResolver.resolve(newParentPath);
-                await this.brain.relate({ from: newParentId, to: entityId, type: VerbType.Contains });
+                await this.brain.relate({
+                    from: newParentId,
+                    to: entityId,
+                    type: VerbType.Contains,
+                    metadata: { isVFS: true } // v4.5.1: Mark as VFS relationship
+                });
             }
         }
         // Update the entity
@@ -1475,7 +1483,12 @@ export class VirtualFileSystem {
         const parentPath = this.getParentPath(destPath);
         if (parentPath && parentPath !== '/') {
             const parentId = await this.pathResolver.resolve(parentPath);
-            await this.brain.relate({ from: parentId, to: newEntity, type: VerbType.Contains });
+            await this.brain.relate({
+                from: parentId,
+                to: newEntity,
+                type: VerbType.Contains,
+                metadata: { isVFS: true } // v4.5.1: Mark as VFS relationship
+            });
         }
         // Update path cache
         await this.pathResolver.createPath(destPath, newEntity);
@@ -1562,7 +1575,8 @@ export class VirtualFileSystem {
         await this.brain.relate({
             from: parentId,
             to: entity,
-            type: VerbType.Contains
+            type: VerbType.Contains,
+            metadata: { isVFS: true } // v4.5.1: Mark as VFS relationship
         });
         // Update path resolver cache
         await this.pathResolver.createPath(path, entity);
@@ -1732,7 +1746,8 @@ export class VirtualFileSystem {
         await this.brain.relate({
             from: fromEntityId,
             to: toEntityId,
-            type: type // Convert string to VerbType
+            type: type, // Convert string to VerbType
+            metadata: { isVFS: true } // v4.5.1: Mark as VFS relationship
         });
         // Invalidate caches for both paths
         this.invalidateCaches(from);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@soulcraft/brainy",
-  "version": "4.4.0",
+  "version": "4.5.1",
   "description": "Universal Knowledge Protocol™ - World's first Triple Intelligence database unifying vector, graph, and document search in one API. 31 nouns × 40 verbs for infinite expressiveness.",
   "main": "dist/index.js",
   "module": "dist/index.js",