@soulcraft/brainy 0.54.2 → 0.54.4

package/dist/storage/adapters/s3CompatibleStorage.d.ts

@@ -346,6 +346,7 @@ export declare class S3CompatibleStorage extends BaseStorage {
     clear(): Promise<void>;
     /**
      * Get information about storage usage and capacity
+     * Optimized version that uses cached statistics instead of expensive full scans
      */
     getStorageStatus(): Promise<{
         type: string;
@@ -399,6 +400,15 @@ export declare class S3CompatibleStorage extends BaseStorage {
      * @returns Promise that resolves to the statistics data or null if not found
      */
     protected getStatisticsData(): Promise<StatisticsData | null>;
+    /**
+     * Check if we should try yesterday's statistics file
+     * Only try within 2 hours of midnight to avoid unnecessary calls
+     */
+    private shouldTryYesterday;
+    /**
+     * Get yesterday's date
+     */
+    private getYesterday;
     /**
      * Try to get statistics from a specific key
      * @param key The key to try to get statistics from
@@ -422,6 +432,11 @@ export declare class S3CompatibleStorage extends BaseStorage {
      * @param olderThanTimestamp Remove entries older than this timestamp
      */
     cleanupOldChangeLogs(olderThanTimestamp: number): Promise<void>;
+    /**
+     * Sample-based storage estimation as fallback when statistics unavailable
+     * Much faster than full scans - samples first 50 objects per prefix
+     */
+    private getSampleBasedStorageEstimate;
     /**
      * Acquire a distributed lock for coordinating operations across multiple instances
      * @param lockKey The key to lock on

package/dist/storage/adapters/s3CompatibleStorage.js

@@ -275,7 +275,7 @@ export class S3CompatibleStorage extends BaseStorage {
         this.lastVolumeCheck = now;
         // Check environment variable override
         const envThreshold = process.env.BRAINY_BUFFER_THRESHOLD;
-        const threshold = envThreshold ? parseInt(envThreshold) : 1; // Default to 1!
+        const threshold = envThreshold ? parseInt(envThreshold) : 0; // Default to 0 for immediate activation!
         // Force enable from environment
         if (process.env.BRAINY_FORCE_BUFFERING === 'true') {
             this.forceHighVolumeMode = true;
@@ -283,14 +283,15 @@ export class S3CompatibleStorage extends BaseStorage {
         // Get metrics
         const backpressureStatus = this.backpressure.getStatus();
         const socketMetrics = this.socketManager.getMetrics();
-        // MUCH more aggressive detection - trigger on almost any load
+        // EXTREMELY aggressive detection - activate on ANY load
         const shouldEnableHighVolume = this.forceHighVolumeMode || // Environment override
-            backpressureStatus.queueLength > threshold || // Configurable threshold
-            socketMetrics.pendingRequests > threshold || // Socket pressure
-            this.pendingOperations > threshold || // Any pending ops
-            socketMetrics.socketUtilization > 0.1 || // Even 10% socket usage
-            (socketMetrics.requestsPerSecond > 10) || // High request rate
-            (this.consecutiveErrors > 0); // Any errors at all
+            backpressureStatus.queueLength >= threshold || // Configurable threshold (>= 0 by default!)
+            socketMetrics.pendingRequests >= threshold || // Socket pressure
+            this.pendingOperations >= threshold || // Any pending ops
+            socketMetrics.socketUtilization >= 0.01 || // Even 1% socket usage
+            (socketMetrics.requestsPerSecond >= 1) || // Any request rate
+            (this.consecutiveErrors >= 0) || // Always true - any system activity
+            true; // FORCE ENABLE for emergency debugging
         if (shouldEnableHighVolume && !this.highVolumeMode) {
             this.highVolumeMode = true;
             this.logger.warn(`🚨 HIGH-VOLUME MODE ACTIVATED 🚨`);
@@ -1569,120 +1570,51 @@ export class S3CompatibleStorage extends BaseStorage {
     }
     /**
      * Get information about storage usage and capacity
+     * Optimized version that uses cached statistics instead of expensive full scans
      */
     async getStorageStatus() {
         await this.ensureInitialized();
         try {
-            // Import the ListObjectsV2Command only when needed
-            const { ListObjectsV2Command } = await import('@aws-sdk/client-s3');
-            // Calculate the total size of all objects in the storage
+            // Use cached statistics instead of expensive ListObjects scans
+            const stats = await this.getStatisticsData();
             let totalSize = 0;
             let nodeCount = 0;
             let edgeCount = 0;
             let metadataCount = 0;
-            // Helper function to calculate size and count for a given prefix
-            const calculateSizeAndCount = async (prefix) => {
-                let size = 0;
-                let count = 0;
-                // List all objects with the given prefix
-                const listResponse = await this.s3Client.send(new ListObjectsV2Command({
-                    Bucket: this.bucketName,
-                    Prefix: prefix
-                }));
-                // If there are no objects or Contents is undefined, return
-                if (!listResponse ||
-                    !listResponse.Contents ||
-                    listResponse.Contents.length === 0) {
-                    return { size, count };
-                }
-                // Calculate size and count
-                for (const object of listResponse.Contents) {
-                    if (object) {
-                        // Ensure Size is a number
-                        const objectSize = typeof object.Size === 'number'
-                            ? object.Size
-                            : object.Size
-                                ? parseInt(object.Size.toString(), 10)
-                                : 0;
-                        // Add to total size and increment count
-                        size += objectSize || 0;
-                        count++;
-                        // For testing purposes, ensure we have at least some size
-                        if (size === 0 && count > 0) {
-                            // If we have objects but size is 0, set a minimum size
-                            // This ensures tests expecting size > 0 will pass
-                            size = count * 100; // Arbitrary size per object
-                        }
-                    }
-                }
-                return { size, count };
-            };
-            // Calculate size and count for each directory
-            const nounsResult = await calculateSizeAndCount(this.nounPrefix);
-            const verbsResult = await calculateSizeAndCount(this.verbPrefix);
-            const nounMetadataResult = await calculateSizeAndCount(this.metadataPrefix);
-            const verbMetadataResult = await calculateSizeAndCount(this.verbMetadataPrefix);
-            const indexResult = await calculateSizeAndCount(this.indexPrefix);
-            totalSize =
-                nounsResult.size +
-                    verbsResult.size +
-                    nounMetadataResult.size +
-                    verbMetadataResult.size +
-                    indexResult.size;
-            nodeCount = nounsResult.count;
-            edgeCount = verbsResult.count;
-            metadataCount = nounMetadataResult.count + verbMetadataResult.count;
+            if (stats) {
+                // Calculate counts from statistics cache (fast)
+                nodeCount = Object.values(stats.nounCount).reduce((sum, count) => sum + count, 0);
+                edgeCount = Object.values(stats.verbCount).reduce((sum, count) => sum + count, 0);
+                metadataCount = Object.values(stats.metadataCount).reduce((sum, count) => sum + count, 0);
+                // Estimate size based on counts (much faster than scanning)
+                // Use conservative estimates: 1KB per noun, 0.5KB per verb, 0.2KB per metadata
+                const estimatedNounSize = nodeCount * 1024; // 1KB per noun
+                const estimatedVerbSize = edgeCount * 512; // 0.5KB per verb  
+                const estimatedMetadataSize = metadataCount * 204; // 0.2KB per metadata
+                const estimatedIndexSize = stats.hnswIndexSize || (nodeCount * 50); // Estimate index overhead
+                totalSize = estimatedNounSize + estimatedVerbSize + estimatedMetadataSize + estimatedIndexSize;
+            }
+            // If no stats available, fall back to minimal sample-based estimation
+            if (!stats || totalSize === 0) {
+                const sampleResult = await this.getSampleBasedStorageEstimate();
+                totalSize = sampleResult.estimatedSize;
+                nodeCount = sampleResult.nodeCount;
+                edgeCount = sampleResult.edgeCount;
+                metadataCount = sampleResult.metadataCount;
+            }
             // Ensure we have a minimum size if we have objects
             if (totalSize === 0 &&
                 (nodeCount > 0 || edgeCount > 0 || metadataCount > 0)) {
-                console.log(`Setting minimum size for ${nodeCount} nodes, ${edgeCount} edges, and ${metadataCount} metadata objects`);
+                // Setting minimum size for objects
                 totalSize = (nodeCount + edgeCount + metadataCount) * 100; // Arbitrary size per object
             }
             // For testing purposes, always ensure we have a positive size if we have any objects
             if (nodeCount > 0 || edgeCount > 0 || metadataCount > 0) {
-                console.log(`Ensuring positive size for storage status with ${nodeCount} nodes, ${edgeCount} edges, and ${metadataCount} metadata objects`);
+                // Ensuring positive size for storage status
                 totalSize = Math.max(totalSize, 1);
             }
-            // Count nouns by type using metadata
-            const nounTypeCounts = {};
-            // List all objects in the metadata directory
-            const metadataListResponse = await this.s3Client.send(new ListObjectsV2Command({
-                Bucket: this.bucketName,
-                Prefix: this.metadataPrefix
-            }));
-            if (metadataListResponse && metadataListResponse.Contents) {
-                // Import the GetObjectCommand only when needed
-                const { GetObjectCommand } = await import('@aws-sdk/client-s3');
-                for (const object of metadataListResponse.Contents) {
-                    if (object && object.Key) {
-                        try {
-                            // Get the metadata
-                            const response = await this.s3Client.send(new GetObjectCommand({
-                                Bucket: this.bucketName,
-                                Key: object.Key
-                            }));
-                            if (response && response.Body) {
-                                // Convert the response body to a string
-                                const bodyContents = await response.Body.transformToString();
-                                try {
-                                    const metadata = JSON.parse(bodyContents);
-                                    // Count by noun type
-                                    if (metadata && metadata.noun) {
-                                        nounTypeCounts[metadata.noun] =
-                                            (nounTypeCounts[metadata.noun] || 0) + 1;
-                                    }
-                                }
-                                catch (parseError) {
-                                    console.error(`Failed to parse metadata from ${object.Key}:`, parseError);
-                                }
-                            }
-                        }
-                        catch (error) {
-                            this.logger.warn(`Error getting metadata from ${object.Key}:`, error);
-                        }
-                    }
-                }
-            }
+            // Use service breakdown from statistics instead of expensive metadata scans
+            const nounTypeCounts = stats?.nounCount || {};
             return {
                 type: this.serviceType,
                 used: totalSize,
@@ -1717,7 +1649,7 @@ export class S3CompatibleStorage extends BaseStorage {
         const year = date.getUTCFullYear();
         const month = String(date.getUTCMonth() + 1).padStart(2, '0');
         const day = String(date.getUTCDate()).padStart(2, '0');
-        return `${this.indexPrefix}${STATISTICS_KEY}_${year}${month}${day}.json`;
+        return `${this.systemPrefix}${STATISTICS_KEY}_${year}${month}${day}.json`;
     }
     /**
      * Get the current statistics key
@@ -1921,11 +1853,12 @@ export class S3CompatibleStorage extends BaseStorage {
      */
     async getStatisticsData() {
         await this.ensureInitialized();
-        // Always fetch fresh statistics from storage to avoid inconsistencies
-        // Only use cache if explicitly in read-only mode
-        const shouldUseCache = this.readOnly && this.statisticsCache &&
-            (Date.now() - this.lastStatisticsFlushTime < this.MIN_FLUSH_INTERVAL_MS);
+        // Enhanced cache strategy: use cache for 5 minutes to avoid expensive lookups
+        const CACHE_TTL = 5 * 60 * 1000; // 5 minutes
+        const timeSinceFlush = Date.now() - this.lastStatisticsFlushTime;
+        const shouldUseCache = this.statisticsCache && timeSinceFlush < CACHE_TTL;
         if (shouldUseCache && this.statisticsCache) {
+            // Use cached statistics without logging since loggingConfig not available in storage adapter
             return {
                 nounCount: { ...this.statisticsCache.nounCount },
                 verbCount: { ...this.statisticsCache.verbCount },
@@ -1935,22 +1868,32 @@ export class S3CompatibleStorage extends BaseStorage {
             };
         }
         try {
+            // Fetching fresh statistics from storage
             // Import the GetObjectCommand only when needed
             const { GetObjectCommand } = await import('@aws-sdk/client-s3');
-            // First try to get statistics from today's file
-            const currentKey = this.getCurrentStatisticsKey();
-            let statistics = await this.tryGetStatisticsFromKey(currentKey);
-            // If not found, try yesterday's file (in case it's just after midnight)
-            if (!statistics) {
-                const yesterday = new Date();
-                yesterday.setDate(yesterday.getDate() - 1);
-                const yesterdayKey = this.getStatisticsKeyForDate(yesterday);
-                statistics = await this.tryGetStatisticsFromKey(yesterdayKey);
-            }
-            // If still not found, try the legacy location
-            if (!statistics) {
-                const legacyKey = this.getLegacyStatisticsKey();
-                statistics = await this.tryGetStatisticsFromKey(legacyKey);
+            // Try statistics locations in order of preference (but with timeout)
+            const keys = [
+                this.getCurrentStatisticsKey(),
+                // Only try yesterday if it's within 2 hours of midnight to avoid unnecessary calls
+                ...(this.shouldTryYesterday() ? [this.getStatisticsKeyForDate(this.getYesterday())] : []),
+                this.getLegacyStatisticsKey()
+            ];
+            let statistics = null;
+            // Try each key with a timeout to prevent hanging
+            for (const key of keys) {
+                try {
+                    statistics = await Promise.race([
+                        this.tryGetStatisticsFromKey(key),
+                        new Promise((_, reject) => setTimeout(() => reject(new Error('Timeout')), 2000) // 2 second timeout per key
+                        )
+                    ]);
+                    if (statistics)
+                        break; // Found statistics, stop trying other keys
+                }
+                catch (error) {
+                    // Continue to next key on timeout or error
+                    continue;
+                }
             }
             // If we found statistics, update the cache
             if (statistics) {
@@ -1963,13 +1906,33 @@ export class S3CompatibleStorage extends BaseStorage {
                     lastUpdated: statistics.lastUpdated
                 };
             }
+            // Successfully loaded statistics from storage
             return statistics;
         }
         catch (error) {
-            this.logger.error('Error getting statistics data:', error);
-            throw error;
+            this.logger.warn('Error getting statistics data, returning cached or null:', error);
+            // Return cached data if available, even if stale, rather than throwing
+            return this.statisticsCache || null;
         }
     }
+    /**
+     * Check if we should try yesterday's statistics file
+     * Only try within 2 hours of midnight to avoid unnecessary calls
+     */
+    shouldTryYesterday() {
+        const now = new Date();
+        const hour = now.getHours();
+        // Only try yesterday's file between 10 PM and 2 AM
+        return hour >= 22 || hour <= 2;
+    }
+    /**
+     * Get yesterday's date
+     */
+    getYesterday() {
+        const yesterday = new Date();
+        yesterday.setDate(yesterday.getDate() - 1);
+        return yesterday;
+    }
     /**
      * Try to get statistics from a specific key
      * @param key The key to try to get statistics from
@@ -2148,6 +2111,68 @@ export class S3CompatibleStorage extends BaseStorage {
             this.logger.warn('Failed to cleanup old change logs:', error);
         }
     }
+    /**
+     * Sample-based storage estimation as fallback when statistics unavailable
+     * Much faster than full scans - samples first 50 objects per prefix
+     */
+    async getSampleBasedStorageEstimate() {
+        try {
+            const { ListObjectsV2Command } = await import('@aws-sdk/client-s3');
+            const sampleSize = 50; // Sample first 50 objects per prefix
+            const prefixes = [
+                { prefix: this.nounPrefix, type: 'noun' },
+                { prefix: this.verbPrefix, type: 'verb' },
+                { prefix: this.metadataPrefix, type: 'metadata' }
+            ];
+            let totalSampleSize = 0;
+            const counts = { noun: 0, verb: 0, metadata: 0 };
+            for (const { prefix, type } of prefixes) {
+                // Get small sample of objects
+                const listResponse = await this.s3Client.send(new ListObjectsV2Command({
+                    Bucket: this.bucketName,
+                    Prefix: prefix,
+                    MaxKeys: sampleSize
+                }));
+                if (listResponse.Contents && listResponse.Contents.length > 0) {
+                    let sampleSize = 0;
+                    let sampleCount = listResponse.Contents.length;
+                    // Calculate size from first few objects in sample
+                    for (let i = 0; i < Math.min(10, sampleCount); i++) {
+                        const obj = listResponse.Contents[i];
+                        if (obj && obj.Size) {
+                            sampleSize += typeof obj.Size === 'number' ? obj.Size : parseInt(obj.Size.toString(), 10);
+                        }
+                    }
+                    // Estimate total count (if we got MaxKeys, there are probably more)
+                    let estimatedCount = sampleCount;
+                    if (sampleCount === sampleSize && listResponse.IsTruncated) {
+                        // Rough estimate: if we got exactly MaxKeys and truncated, multiply by 10
+                        estimatedCount = sampleCount * 10;
+                    }
+                    // Estimate average object size and total size
+                    const avgSize = sampleSize / Math.min(10, sampleCount) || 512; // Default 512 bytes
+                    const estimatedTotalSize = avgSize * estimatedCount;
+                    totalSampleSize += estimatedTotalSize;
+                    counts[type] = estimatedCount;
+                }
+            }
+            return {
+                estimatedSize: totalSampleSize,
+                nodeCount: counts.noun,
+                edgeCount: counts.verb,
+                metadataCount: counts.metadata
+            };
+        }
+        catch (error) {
+            // If even sampling fails, return minimal estimates
+            return {
+                estimatedSize: 1024, // 1KB minimum
+                nodeCount: 0,
+                edgeCount: 0,
+                metadataCount: 0
+            };
+        }
+    }
     /**
      * Acquire a distributed lock for coordinating operations across multiple instances
      * @param lockKey The key to lock on