@soulcraft/brainy 0.60.0 → 0.61.1

package/dist/storage/adapters/s3CompatibleStorage.js

@@ -381,15 +381,23 @@ export class S3CompatibleStorage extends BaseStorage {
         // Get metrics
         const backpressureStatus = this.backpressure.getStatus();
         const socketMetrics = this.socketManager.getMetrics();
-        // EXTREMELY aggressive detection - activate on ANY load
-        const shouldEnableHighVolume = this.forceHighVolumeMode || // Environment override
-            backpressureStatus.queueLength >= threshold || // Configurable threshold (>= 0 by default!)
-            socketMetrics.pendingRequests >= threshold || // Socket pressure
-            this.pendingOperations >= threshold || // Any pending ops
-            socketMetrics.socketUtilization >= 0.01 || // Even 1% socket usage
-            (socketMetrics.requestsPerSecond >= 1) || // Any request rate
-            (this.consecutiveErrors >= 0) || // Always true - any system activity
-            true; // FORCE ENABLE for emergency debugging
+        // Reasonable high-volume detection - only activate under real load
+        const isTestEnvironment = process.env.NODE_ENV === 'test';
+        const explicitlyDisabled = process.env.BRAINY_FORCE_BUFFERING === 'false';
+        // Use reasonable thresholds instead of emergency aggressive ones
+        const reasonableThreshold = Math.max(threshold, 10); // At least 10 pending operations
+        const highSocketUtilization = 0.8; // 80% socket utilization 
+        const highRequestRate = 50; // 50 requests per second
+        const significantErrors = 5; // 5 consecutive errors
+        const shouldEnableHighVolume = !isTestEnvironment && // Disable in test environment
+            !explicitlyDisabled && // Allow explicit disabling
+            (this.forceHighVolumeMode || // Environment override
+                backpressureStatus.queueLength >= reasonableThreshold || // High queue backlog
+                socketMetrics.pendingRequests >= reasonableThreshold || // Many pending requests
+                this.pendingOperations >= reasonableThreshold || // Many pending ops
+                socketMetrics.socketUtilization >= highSocketUtilization || // High socket pressure
+                (socketMetrics.requestsPerSecond >= highRequestRate) || // High request rate
+                (this.consecutiveErrors >= significantErrors)); // Significant error pattern
         if (shouldEnableHighVolume && !this.highVolumeMode) {
             this.highVolumeMode = true;
             this.logger.warn(`🚨 HIGH-VOLUME MODE ACTIVATED 🚨`);
@@ -1278,8 +1286,42 @@ export class S3CompatibleStorage extends BaseStorage {
                 graphVerbs.push(graphVerb);
             }
         }
+        // Apply filtering at GraphVerb level since HNSWVerb filtering is not supported
+        let filteredGraphVerbs = graphVerbs;
+        if (options.filter) {
+            filteredGraphVerbs = graphVerbs.filter((graphVerb) => {
+                // Filter by sourceId
+                if (options.filter.sourceId) {
+                    const sourceIds = Array.isArray(options.filter.sourceId)
+                        ? options.filter.sourceId
+                        : [options.filter.sourceId];
+                    if (!sourceIds.includes(graphVerb.sourceId)) {
+                        return false;
+                    }
+                }
+                // Filter by targetId
+                if (options.filter.targetId) {
+                    const targetIds = Array.isArray(options.filter.targetId)
+                        ? options.filter.targetId
+                        : [options.filter.targetId];
+                    if (!targetIds.includes(graphVerb.targetId)) {
+                        return false;
+                    }
+                }
+                // Filter by verbType (maps to type field)
+                if (options.filter.verbType) {
+                    const verbTypes = Array.isArray(options.filter.verbType)
+                        ? options.filter.verbType
+                        : [options.filter.verbType];
+                    if (graphVerb.type && !verbTypes.includes(graphVerb.type)) {
+                        return false;
+                    }
+                }
+                return true;
+            });
+        }
         return {
-            items: graphVerbs,
+            items: filteredGraphVerbs,
             hasMore: result.hasMore,
             nextCursor: result.nextCursor
         };
@@ -1288,46 +1330,34 @@ export class S3CompatibleStorage extends BaseStorage {
      * Get verbs by source (internal implementation)
      */
     async getVerbsBySource_internal(sourceId) {
-        return this.getEdgesBySource(sourceId);
-    }
-    /**
-     * Get edges by source
-     */
-    async getEdgesBySource(sourceId) {
-        // This method is deprecated and would require loading metadata for each edge
-        // For now, return empty array since this is not efficiently implementable with new storage pattern
-        this.logger.trace('getEdgesBySource is deprecated and not efficiently supported in new storage pattern');
-        return [];
+        // Use the paginated approach to properly handle HNSWVerb to GraphVerb conversion
+        const result = await this.getVerbsWithPagination({
+            filter: { sourceId: [sourceId] },
+            limit: Number.MAX_SAFE_INTEGER // Get all matching results
+        });
+        return result.items;
     }
     /**
      * Get verbs by target (internal implementation)
      */
     async getVerbsByTarget_internal(targetId) {
-        return this.getEdgesByTarget(targetId);
-    }
-    /**
-     * Get edges by target
-     */
-    async getEdgesByTarget(targetId) {
-        // This method is deprecated and would require loading metadata for each edge
-        // For now, return empty array since this is not efficiently implementable with new storage pattern
-        this.logger.trace('getEdgesByTarget is deprecated and not efficiently supported in new storage pattern');
-        return [];
+        // Use the paginated approach to properly handle HNSWVerb to GraphVerb conversion
+        const result = await this.getVerbsWithPagination({
+            filter: { targetId: [targetId] },
+            limit: Number.MAX_SAFE_INTEGER // Get all matching results
+        });
+        return result.items;
     }
     /**
      * Get verbs by type (internal implementation)
      */
     async getVerbsByType_internal(type) {
-        return this.getEdgesByType(type);
-    }
-    /**
-     * Get edges by type
-     */
-    async getEdgesByType(type) {
-        // This method is deprecated and would require loading metadata for each edge
-        // For now, return empty array since this is not efficiently implementable with new storage pattern
-        this.logger.trace('getEdgesByType is deprecated and not efficiently supported in new storage pattern');
-        return [];
+        // Use the paginated approach to properly handle HNSWVerb to GraphVerb conversion
+        const result = await this.getVerbsWithPagination({
+            filter: { verbType: [type] },
+            limit: Number.MAX_SAFE_INTEGER // Get all matching results
+        });
+        return result.items;
     }
     /**
      * Delete a verb from storage (internal implementation)

package/dist/storage/baseStorage.d.ts

@@ -72,11 +72,10 @@ export declare abstract class BaseStorage extends BaseStorageAdapter {
      */
     protected convertHNSWVerbToGraphVerb(hnswVerb: HNSWVerb): Promise<GraphVerb | null>;
     /**
-     * Get all verbs from storage
-     * @returns Promise that resolves to an array of all HNSWVerbs
-     * @deprecated This method loads all data into memory and may cause performance issues. Use getVerbs() with pagination instead.
+     * Internal method for loading all verbs - used by performance optimizations
+     * @internal - Do not use directly, use getVerbs() with pagination instead
      */
-    getAllVerbs(): Promise<HNSWVerb[]>;
+    protected _loadAllVerbsForOptimization(): Promise<HNSWVerb[]>;
     /**
      * Get verbs by source
      */
@@ -90,11 +89,10 @@ export declare abstract class BaseStorage extends BaseStorageAdapter {
      */
     getVerbsByType(type: string): Promise<GraphVerb[]>;
     /**
-     * Get all nouns from storage
-     * @returns Promise that resolves to an array of all nouns
-     * @deprecated This method loads all data into memory and may cause performance issues. Use getNouns() with pagination instead.
+     * Internal method for loading all nouns - used by performance optimizations
+     * @internal - Do not use directly, use getNouns() with pagination instead
      */
-    getAllNouns(): Promise<HNSWNoun[]>;
+    protected _loadAllNounsForOptimization(): Promise<HNSWNoun[]>;
     /**
      * Get nouns with pagination and filtering
      * @param options Pagination and filtering options

package/dist/storage/baseStorage.js

@@ -181,27 +181,22 @@ export class BaseStorage extends BaseStorageAdapter {
         }
     }
     /**
-     * Get all verbs from storage
-     * @returns Promise that resolves to an array of all HNSWVerbs
-     * @deprecated This method loads all data into memory and may cause performance issues. Use getVerbs() with pagination instead.
+     * Internal method for loading all verbs - used by performance optimizations
+     * @internal - Do not use directly, use getVerbs() with pagination instead
      */
-    async getAllVerbs() {
+    async _loadAllVerbsForOptimization() {
         await this.ensureInitialized();
-        console.warn('getAllVerbs() is deprecated and may cause memory issues with large datasets. Consider using getVerbs() with pagination instead.');
-        // Get all verbs using the paginated method with a very large limit
+        // Only use this for internal optimizations when safe
         const result = await this.getVerbs({
-            pagination: {
-                limit: Number.MAX_SAFE_INTEGER
-            }
+            pagination: { limit: Number.MAX_SAFE_INTEGER }
         });
-        // Convert GraphVerbs back to HNSWVerbs since that's what this method should return
+        // Convert GraphVerbs back to HNSWVerbs for internal use
         const hnswVerbs = [];
         for (const graphVerb of result.items) {
-            // Create an HNSWVerb from the GraphVerb (reverse conversion)
             const hnswVerb = {
                 id: graphVerb.id,
                 vector: graphVerb.vector,
-                connections: new Map() // HNSWVerbs need connections, but GraphVerbs don't have them
+                connections: new Map()
             };
             hnswVerbs.push(hnswVerb);
         }
@@ -241,17 +236,14 @@ export class BaseStorage extends BaseStorageAdapter {
         return result.items;
     }
     /**
-     * Get all nouns from storage
-     * @returns Promise that resolves to an array of all nouns
-     * @deprecated This method loads all data into memory and may cause performance issues. Use getNouns() with pagination instead.
+     * Internal method for loading all nouns - used by performance optimizations
+     * @internal - Do not use directly, use getNouns() with pagination instead
      */
-    async getAllNouns() {
+    async _loadAllNounsForOptimization() {
         await this.ensureInitialized();
-        console.warn('getAllNouns() is deprecated and may cause memory issues with large datasets. Consider using getNouns() with pagination instead.');
+        // Only use this for internal optimizations when safe
         const result = await this.getNouns({
-            pagination: {
-                limit: Number.MAX_SAFE_INTEGER
-            }
+            pagination: { limit: Number.MAX_SAFE_INTEGER }
         });
         return result.items;
     }

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@soulcraft/brainy",
-  "version": "0.60.0",
-  "description": "A vector graph database using HNSW indexing with Origin Private File System storage",
+  "version": "0.61.1",
+  "description": "Multi-Dimensional AI Database - Vector similarity, graph relationships, metadata facets with HNSW indexing and OPFS storage",
   "main": "dist/index.js",
   "module": "dist/index.js",
   "types": "dist/index.d.ts",
@@ -158,13 +158,13 @@
   "publishConfig": {
     "access": "public"
   },
-  "homepage": "https://github.com/soulcraft-research/brainy",
+  "homepage": "https://github.com/soulcraftlabs/brainy",
   "bugs": {
-    "url": "https://github.com/soulcraft-research/brainy/issues"
+    "url": "https://github.com/soulcraftlabs/brainy/issues"
   },
   "repository": {
     "type": "git",
-    "url": "git+https://github.com/soulcraft-research/brainy.git"
+    "url": "git+https://github.com/soulcraftlabs/brainy.git"
   },
   "files": [
     "dist/**/*.js",