@soulcraft/brainy 0.53.0 → 0.54.1

package/dist/index.d.ts

@@ -10,8 +10,11 @@ export { euclideanDistance, cosineDistance, manhattanDistance, dotProductDistanc
 import { UniversalSentenceEncoder, TransformerEmbedding, createEmbeddingFunction, defaultEmbeddingFunction, batchEmbed, embeddingFunctions } from './utils/embedding.js';
 import { executeInThread, cleanupWorkerPools } from './utils/workerUtils.js';
 import { logger, LogLevel, configureLogger, createModuleLogger } from './utils/logger.js';
+import { getGlobalSocketManager, AdaptiveSocketManager } from './utils/adaptiveSocketManager.js';
+import { getGlobalBackpressure, AdaptiveBackpressure } from './utils/adaptiveBackpressure.js';
+import { getGlobalPerformanceMonitor, PerformanceMonitor } from './utils/performanceMonitor.js';
 import { isBrowser, isNode, isWebWorker, areWebWorkersAvailable, areWorkerThreadsAvailable, areWorkerThreadsAvailableSync, isThreadingAvailable, isThreadingAvailableAsync } from './utils/environment.js';
-export { UniversalSentenceEncoder, TransformerEmbedding, createEmbeddingFunction, defaultEmbeddingFunction, batchEmbed, embeddingFunctions, executeInThread, cleanupWorkerPools, isBrowser, isNode, isWebWorker, areWebWorkersAvailable, areWorkerThreadsAvailable, areWorkerThreadsAvailableSync, isThreadingAvailable, isThreadingAvailableAsync, logger, LogLevel, configureLogger, createModuleLogger };
+export { UniversalSentenceEncoder, TransformerEmbedding, createEmbeddingFunction, defaultEmbeddingFunction, batchEmbed, embeddingFunctions, executeInThread, cleanupWorkerPools, isBrowser, isNode, isWebWorker, areWebWorkersAvailable, areWorkerThreadsAvailable, areWorkerThreadsAvailableSync, isThreadingAvailable, isThreadingAvailableAsync, logger, LogLevel, configureLogger, createModuleLogger, getGlobalSocketManager, AdaptiveSocketManager, getGlobalBackpressure, AdaptiveBackpressure, getGlobalPerformanceMonitor, PerformanceMonitor };
 import { OPFSStorage, MemoryStorage, R2Storage, S3CompatibleStorage, createStorage } from './storage/storageFactory.js';
 export { OPFSStorage, MemoryStorage, R2Storage, S3CompatibleStorage, createStorage };
 export { FileSystemStorage } from './storage/adapters/fileSystemStorage.js';

package/dist/index.js

@@ -15,6 +15,10 @@ import { UniversalSentenceEncoder, TransformerEmbedding, createEmbeddingFunction
 import { executeInThread, cleanupWorkerPools } from './utils/workerUtils.js';
 // Export logging utilities
 import { logger, LogLevel, configureLogger, createModuleLogger } from './utils/logger.js';
+// Export performance and optimization utilities
+import { getGlobalSocketManager, AdaptiveSocketManager } from './utils/adaptiveSocketManager.js';
+import { getGlobalBackpressure, AdaptiveBackpressure } from './utils/adaptiveBackpressure.js';
+import { getGlobalPerformanceMonitor, PerformanceMonitor } from './utils/performanceMonitor.js';
 // Export environment utilities
 import { isBrowser, isNode, isWebWorker, areWebWorkersAvailable, areWorkerThreadsAvailable, areWorkerThreadsAvailableSync, isThreadingAvailable, isThreadingAvailableAsync } from './utils/environment.js';
 export { UniversalSentenceEncoder, TransformerEmbedding, createEmbeddingFunction, defaultEmbeddingFunction, batchEmbed, embeddingFunctions, 
@@ -23,7 +27,9 @@ executeInThread, cleanupWorkerPools,
 // Environment utilities
 isBrowser, isNode, isWebWorker, areWebWorkersAvailable, areWorkerThreadsAvailable, areWorkerThreadsAvailableSync, isThreadingAvailable, isThreadingAvailableAsync, 
 // Logging utilities
-logger, LogLevel, configureLogger, createModuleLogger };
+logger, LogLevel, configureLogger, createModuleLogger, 
+// Performance and optimization utilities
+getGlobalSocketManager, AdaptiveSocketManager, getGlobalBackpressure, AdaptiveBackpressure, getGlobalPerformanceMonitor, PerformanceMonitor };
 // Export storage adapters
 import { OPFSStorage, MemoryStorage, R2Storage, S3CompatibleStorage, createStorage } from './storage/storageFactory.js';
 export { OPFSStorage, MemoryStorage, R2Storage, S3CompatibleStorage, createStorage };

package/dist/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,kEAAkE;AAElE,iDAAiD;AACjD,OAAO,EAAE,UAAU,EAAoB,MAAM,iBAAiB,CAAA;AAE9D,OAAO,EAAE,UAAU,EAAE,CAAA;AAGrB,4CAA4C;AAC5C,OAAO,EACL,iBAAiB,EACjB,cAAc,EACd,iBAAiB,EACjB,kBAAkB,EAClB,aAAa,EACd,MAAM,kBAAkB,CAAA;AAEzB,OAAO,EACL,iBAAiB,EACjB,cAAc,EACd,iBAAiB,EACjB,kBAAkB,EAClB,aAAa,EACd,CAAA;AAED,iCAAiC;AACjC,OAAO,EACL,wBAAwB,EACxB,oBAAoB,EACpB,uBAAuB,EACvB,wBAAwB,EACxB,UAAU,EACV,kBAAkB,EACnB,MAAM,sBAAsB,CAAA;AAE7B,0BAA0B;AAC1B,OAAO,EAAE,eAAe,EAAE,kBAAkB,EAAE,MAAM,wBAAwB,CAAA;AAE5E,2BAA2B;AAC3B,OAAO,EACL,MAAM,EACN,QAAQ,EACR,eAAe,EACf,kBAAkB,EACnB,MAAM,mBAAmB,CAAA;AAE1B,+BAA+B;AAC/B,OAAO,EACL,SAAS,EACT,MAAM,EACN,WAAW,EACX,sBAAsB,EACtB,yBAAyB,EACzB,6BAA6B,EAC7B,oBAAoB,EACpB,yBAAyB,EAC1B,MAAM,wBAAwB,CAAA;AAE/B,OAAO,EACL,wBAAwB,EACxB,oBAAoB,EACpB,uBAAuB,EACvB,wBAAwB,EACxB,UAAU,EACV,kBAAkB;AAElB,mBAAmB;AACnB,eAAe,EACf,kBAAkB;AAElB,wBAAwB;AACxB,SAAS,EACT,MAAM,EACN,WAAW,EACX,sBAAsB,EACtB,yBAAyB,EACzB,6BAA6B,EAC7B,oBAAoB,EACpB,yBAAyB;AAEzB,oBAAoB;AACpB,MAAM,EACN,QAAQ,EACR,eAAe,EACf,kBAAkB,EACnB,CAAA;AAED,0BAA0B;AAC1B,OAAO,EACL,WAAW,EACX,aAAa,EACb,SAAS,EACT,mBAAmB,EACnB,aAAa,EACd,MAAM,6BAA6B,CAAA;AAEpC,OAAO,EACL,WAAW,EACX,aAAa,EACb,SAAS,EACT,mBAAmB,EACnB,aAAa,EACd,CAAA;AAED,yEAAyE;AACzE,OAAO,EAAE,iBAAiB,EAAE,MAAM,yCAAyC,CAAA;AAE3E,0BAA0B;AAC1B,OAAO,EACL,QAAQ,EACR,QAAQ,EACR,oBAAoB,EACpB,aAAa,EAGb,kBAAkB,EAClB,aAAa,EACb,aAAa,EACb,iBAAiB,EACjB,oBAAoB,EACpB,cAAc,EACd,uBAAuB,EACvB,wBAAwB,EAGzB,MAAM,eAAe,CAAA;AAEtB,0DAA0D;AAC1D,OAAO,EACL,kBAAkB,EAClB,kBAAkB,EAEnB,MAAM,yBAAyB,CAAA;AAEhC,8BAA8B;AAC9B,OAAO,EACL,uBAAuB,EACvB,mBAAmB,EACnB,mBAAmB,EACnB,sBAAsB,EAEvB,MAAM,0BAA0B,CAAA;AAEjC,OAAO;AACL,2BAA2B;AAC3B,QAAQ,EACR,QAAQ,EACR,oBAAoB,EACpB,aAAa,EACb,kBAAkB,EAClB,kBAAkB;AAElB,8DAA8D;AAC9D,kBAAkB,EAClB,aAAa,EACb,aAAa,EACb,iBAAiB,EACjB,oBAAoB,EACpB,cAAc,EACd,uBAAuB,EACvB,wBAAwB;AAExB,+BAA+B;AAC/B,uBAAuB,EACvB,mBAAmB,EACnB,mBAAmB,EACnB,sBAAsB,EACvB,CAAA;AAUD,sDAAsD;AACtD,OAAO,EACL,sBAAsB,EACtB,oBAAoB,EACpB,8BAA8B,EAC9B,sBAAsB,EACtB,sBAAsB,EACvB,MAAM,2BAA2B,CAAA;AAElC,OAAO,EACL,sBAAsB,EACtB,oBAAoB,EACpB,8BAA8B,EAC9B,sBAAsB,EACtB,sBAAsB,EACvB,CAAA;AAED,sDAAsD;AACtD,OAAO,EACL,4BAA4B,EAC5B,gCAAgC,EAChC,sCAAsC,EACvC,MAAM,iCAAiC,CAAA;AAMxC,OAAO,EACL,4BAA4B,EAC5B,gCAAgC,EAChC,sCAAsC,EACvC,CAAA;AAGD,sCAAsC;AACtC,OAAO,EACL,yBAAyB,EACzB,6BAA6B,EAC7B,uBAAuB,EACvB,wBAAwB,EACzB,MAAM,wCAAwC,CAAA;AAC/C,OAAO,EACL,4BAA4B,EAC5B,yBAAyB,EACzB,yBAAyB,EAC1B,MAAM,yCAAyC,CAAA;AAChD,OAAO,EACL,+BAA+B,EAC/B,kCAAkC,EAClC,+BAA+B,EAChC,MAAM,8CAA8C,CAAA;AAErD,kBAAkB;AAClB,OAAO,EACL,yBAAyB,EACzB,6BAA6B,EAC7B,uBAAuB,EACvB,wBAAwB,EACxB,4BAA4B,EAC5B,yBAAyB,EACzB,yBAAyB,EACzB,+BAA+B,EAC/B,kCAAkC,EAClC,+BAA+B,EAChC,CAAA;AAoBD,0CAA0C;AAC1C,OAAO,EAAE,SAAS,EAAE,MAAM,qBAAqB,CAAA;AAC/C,OAAO,EACL,kBAAkB,EAEnB,MAAM,8BAA8B,CAAA;AAErC,OAAO,EAAE,SAAS,EAAE,kBAAkB,EAAE,CAAA;AA6BxC,OAAO,EAAE,gBAAgB,EAAuB,MAAM,0BAA0B,CAAA;AAGhF,OAAO,EACL,gBAAgB,EASjB,CAAA;AA4CD,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAE,MAAM,uBAAuB,CAAA;AAgC1D,gCAAgC;AAChC,OAAO,EAAE,YAAY,EAAE,YAAY,EAAE,cAAc,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAA;AAEjG,OAAO,EACL,QAAQ,EACR,QAAQ,EACR,YAAY,EACZ,YAAY,EACZ,cAAc,EACd,cAAc,EACf,CAAA;AAED,iDAAiD;AACjD,OAAO,EACL,gBAAgB,EAChB,sBAAsB,EACtB,gBAAgB,EACjB,MAAM,gBAAgB,CAAA,CAAC,2BAA2B;AACnD,OAAO,EAOL,cAAc,EAGd,WAAW,EACZ,MAAM,qBAAqB,CAAA;AAE5B,OAAO;AACL,cAAc;AACd,gBAAgB,EAChB,sBAAsB,EACtB,gBAAgB;AAEhB,YAAY;AACZ,cAAc,EACd,WAAW,EACZ,CAAA"}
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,kEAAkE;AAElE,iDAAiD;AACjD,OAAO,EAAE,UAAU,EAAoB,MAAM,iBAAiB,CAAA;AAE9D,OAAO,EAAE,UAAU,EAAE,CAAA;AAGrB,4CAA4C;AAC5C,OAAO,EACL,iBAAiB,EACjB,cAAc,EACd,iBAAiB,EACjB,kBAAkB,EAClB,aAAa,EACd,MAAM,kBAAkB,CAAA;AAEzB,OAAO,EACL,iBAAiB,EACjB,cAAc,EACd,iBAAiB,EACjB,kBAAkB,EAClB,aAAa,EACd,CAAA;AAED,iCAAiC;AACjC,OAAO,EACL,wBAAwB,EACxB,oBAAoB,EACpB,uBAAuB,EACvB,wBAAwB,EACxB,UAAU,EACV,kBAAkB,EACnB,MAAM,sBAAsB,CAAA;AAE7B,0BAA0B;AAC1B,OAAO,EAAE,eAAe,EAAE,kBAAkB,EAAE,MAAM,wBAAwB,CAAA;AAE5E,2BAA2B;AAC3B,OAAO,EACL,MAAM,EACN,QAAQ,EACR,eAAe,EACf,kBAAkB,EACnB,MAAM,mBAAmB,CAAA;AAE1B,gDAAgD;AAChD,OAAO,EACL,sBAAsB,EACtB,qBAAqB,EACtB,MAAM,kCAAkC,CAAA;AAEzC,OAAO,EACL,qBAAqB,EACrB,oBAAoB,EACrB,MAAM,iCAAiC,CAAA;AAExC,OAAO,EACL,2BAA2B,EAC3B,kBAAkB,EACnB,MAAM,+BAA+B,CAAA;AAEtC,+BAA+B;AAC/B,OAAO,EACL,SAAS,EACT,MAAM,EACN,WAAW,EACX,sBAAsB,EACtB,yBAAyB,EACzB,6BAA6B,EAC7B,oBAAoB,EACpB,yBAAyB,EAC1B,MAAM,wBAAwB,CAAA;AAE/B,OAAO,EACL,wBAAwB,EACxB,oBAAoB,EACpB,uBAAuB,EACvB,wBAAwB,EACxB,UAAU,EACV,kBAAkB;AAElB,mBAAmB;AACnB,eAAe,EACf,kBAAkB;AAElB,wBAAwB;AACxB,SAAS,EACT,MAAM,EACN,WAAW,EACX,sBAAsB,EACtB,yBAAyB,EACzB,6BAA6B,EAC7B,oBAAoB,EACpB,yBAAyB;AAEzB,oBAAoB;AACpB,MAAM,EACN,QAAQ,EACR,eAAe,EACf,kBAAkB;AAElB,yCAAyC;AACzC,sBAAsB,EACtB,qBAAqB,EACrB,qBAAqB,EACrB,oBAAoB,EACpB,2BAA2B,EAC3B,kBAAkB,EACnB,CAAA;AAED,0BAA0B;AAC1B,OAAO,EACL,WAAW,EACX,aAAa,EACb,SAAS,EACT,mBAAmB,EACnB,aAAa,EACd,MAAM,6BAA6B,CAAA;AAEpC,OAAO,EACL,WAAW,EACX,aAAa,EACb,SAAS,EACT,mBAAmB,EACnB,aAAa,EACd,CAAA;AAED,yEAAyE;AACzE,OAAO,EAAE,iBAAiB,EAAE,MAAM,yCAAyC,CAAA;AAE3E,0BAA0B;AAC1B,OAAO,EACL,QAAQ,EACR,QAAQ,EACR,oBAAoB,EACpB,aAAa,EAGb,kBAAkB,EAClB,aAAa,EACb,aAAa,EACb,iBAAiB,EACjB,oBAAoB,EACpB,cAAc,EACd,uBAAuB,EACvB,wBAAwB,EAGzB,MAAM,eAAe,CAAA;AAEtB,0DAA0D;AAC1D,OAAO,EACL,kBAAkB,EAClB,kBAAkB,EAEnB,MAAM,yBAAyB,CAAA;AAEhC,8BAA8B;AAC9B,OAAO,EACL,uBAAuB,EACvB,mBAAmB,EACnB,mBAAmB,EACnB,sBAAsB,EAEvB,MAAM,0BAA0B,CAAA;AAEjC,OAAO;AACL,2BAA2B;AAC3B,QAAQ,EACR,QAAQ,EACR,oBAAoB,EACpB,aAAa,EACb,kBAAkB,EAClB,kBAAkB;AAElB,8DAA8D;AAC9D,kBAAkB,EAClB,aAAa,EACb,aAAa,EACb,iBAAiB,EACjB,oBAAoB,EACpB,cAAc,EACd,uBAAuB,EACvB,wBAAwB;AAExB,+BAA+B;AAC/B,uBAAuB,EACvB,mBAAmB,EACnB,mBAAmB,EACnB,sBAAsB,EACvB,CAAA;AAUD,sDAAsD;AACtD,OAAO,EACL,sBAAsB,EACtB,oBAAoB,EACpB,8BAA8B,EAC9B,sBAAsB,EACtB,sBAAsB,EACvB,MAAM,2BAA2B,CAAA;AAElC,OAAO,EACL,sBAAsB,EACtB,oBAAoB,EACpB,8BAA8B,EAC9B,sBAAsB,EACtB,sBAAsB,EACvB,CAAA;AAED,sDAAsD;AACtD,OAAO,EACL,4BAA4B,EAC5B,gCAAgC,EAChC,sCAAsC,EACvC,MAAM,iCAAiC,CAAA;AAMxC,OAAO,EACL,4BAA4B,EAC5B,gCAAgC,EAChC,sCAAsC,EACvC,CAAA;AAGD,sCAAsC;AACtC,OAAO,EACL,yBAAyB,EACzB,6BAA6B,EAC7B,uBAAuB,EACvB,wBAAwB,EACzB,MAAM,wCAAwC,CAAA;AAC/C,OAAO,EACL,4BAA4B,EAC5B,yBAAyB,EACzB,yBAAyB,EAC1B,MAAM,yCAAyC,CAAA;AAChD,OAAO,EACL,+BAA+B,EAC/B,kCAAkC,EAClC,+BAA+B,EAChC,MAAM,8CAA8C,CAAA;AAErD,kBAAkB;AAClB,OAAO,EACL,yBAAyB,EACzB,6BAA6B,EAC7B,uBAAuB,EACvB,wBAAwB,EACxB,4BAA4B,EAC5B,yBAAyB,EACzB,yBAAyB,EACzB,+BAA+B,EAC/B,kCAAkC,EAClC,+BAA+B,EAChC,CAAA;AAoBD,0CAA0C;AAC1C,OAAO,EAAE,SAAS,EAAE,MAAM,qBAAqB,CAAA;AAC/C,OAAO,EACL,kBAAkB,EAEnB,MAAM,8BAA8B,CAAA;AAErC,OAAO,EAAE,SAAS,EAAE,kBAAkB,EAAE,CAAA;AA6BxC,OAAO,EAAE,gBAAgB,EAAuB,MAAM,0BAA0B,CAAA;AAGhF,OAAO,EACL,gBAAgB,EASjB,CAAA;AA4CD,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAE,MAAM,uBAAuB,CAAA;AAgC1D,gCAAgC;AAChC,OAAO,EAAE,YAAY,EAAE,YAAY,EAAE,cAAc,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAA;AAEjG,OAAO,EACL,QAAQ,EACR,QAAQ,EACR,YAAY,EACZ,YAAY,EACZ,cAAc,EACd,cAAc,EACf,CAAA;AAED,iDAAiD;AACjD,OAAO,EACL,gBAAgB,EAChB,sBAAsB,EACtB,gBAAgB,EACjB,MAAM,gBAAgB,CAAA,CAAC,2BAA2B;AACnD,OAAO,EAOL,cAAc,EAGd,WAAW,EACZ,MAAM,qBAAqB,CAAA;AAE5B,OAAO;AACL,cAAc;AACd,gBAAgB,EAChB,sBAAsB,EACtB,gBAAgB;AAEhB,YAAY;AACZ,cAAc,EACd,WAAW,EACZ,CAAA"}

package/dist/storage/adapters/s3CompatibleStorage.d.ts

@@ -68,6 +68,14 @@ export declare class S3CompatibleStorage extends BaseStorage {
     private memoryCheckInterval;
     private consecutiveErrors;
     private lastErrorReset;
+    private socketManager;
+    private backpressure;
+    private nounWriteBuffer;
+    private verbWriteBuffer;
+    private requestCoalescer;
+    private highVolumeMode;
+    private lastVolumeCheck;
+    private volumeCheckInterval;
     private operationExecutors;
     private nounCacheManager;
     private verbCacheManager;
@@ -97,6 +105,42 @@ export declare class S3CompatibleStorage extends BaseStorage {
      * Initialize the storage adapter
      */
     init(): Promise<void>;
+    /**
+     * Initialize write buffers for high-volume scenarios
+     */
+    private initializeBuffers;
+    /**
+     * Initialize request coalescer
+     */
+    private initializeCoalescer;
+    /**
+     * Check if we should enable high-volume mode
+     */
+    private checkVolumeMode;
+    /**
+     * Bulk write nouns to S3
+     */
+    private bulkWriteNouns;
+    /**
+     * Bulk write verbs to S3
+     */
+    private bulkWriteVerbs;
+    /**
+     * Process coalesced batch of operations
+     */
+    private processCoalescedBatch;
+    /**
+     * Process bulk deletes
+     */
+    private processBulkDeletes;
+    /**
+     * Process bulk writes
+     */
+    private processBulkWrites;
+    /**
+     * Process bulk reads
+     */
+    private processBulkReads;
     /**
      * Dynamically adjust batch size based on memory pressure and error rates
      */

package/dist/storage/adapters/s3CompatibleStorage.js

@@ -9,6 +9,10 @@ import { StorageOperationExecutors } from '../../utils/operationUtils.js';
 import { BrainyError } from '../../errors/brainyError.js';
 import { CacheManager } from '../cacheManager.js';
 import { createModuleLogger } from '../../utils/logger.js';
+import { getGlobalSocketManager } from '../../utils/adaptiveSocketManager.js';
+import { getGlobalBackpressure } from '../../utils/adaptiveBackpressure.js';
+import { getWriteBuffer } from '../../utils/writeBuffer.js';
+import { getCoalescer } from '../../utils/requestCoalescer.js';
 // Export R2Storage as an alias for S3CompatibleStorage
 export { S3CompatibleStorage as R2Storage };
 /**
@@ -58,6 +62,19 @@ export class S3CompatibleStorage extends BaseStorage {
         this.memoryCheckInterval = 5000; // Check every 5 seconds
         this.consecutiveErrors = 0;
         this.lastErrorReset = Date.now();
+        // Adaptive socket manager for automatic optimization
+        this.socketManager = getGlobalSocketManager();
+        // Adaptive backpressure for automatic flow control
+        this.backpressure = getGlobalBackpressure();
+        // Write buffers for bulk operations
+        this.nounWriteBuffer = null;
+        this.verbWriteBuffer = null;
+        // Request coalescer for deduplication
+        this.requestCoalescer = null;
+        // High-volume mode detection
+        this.highVolumeMode = false;
+        this.lastVolumeCheck = 0;
+        this.volumeCheckInterval = 5000;
         // Module logger
         this.logger = createModuleLogger('S3Storage');
         // Node cache to avoid redundant API calls
@@ -104,16 +121,6 @@ export class S3CompatibleStorage extends BaseStorage {
         try {
             // Import AWS SDK modules only when needed
             const { S3Client } = await import('@aws-sdk/client-s3');
-            const { NodeHttpHandler } = await import('@smithy/node-http-handler');
-            const https = await import('https');
-            // Configure HTTP agent with high socket limits for high-volume scenarios
-            // This prevents socket exhaustion without requiring user configuration
-            const httpAgent = new https.Agent({
-                keepAlive: true,
-                maxSockets: 500, // Increased from default 50 to handle high volume
-                maxFreeSockets: 100, // Keep more sockets alive for reuse
-                timeout: 60000 // 60 second timeout
-            });
             // Configure the S3 client based on the service type
             const clientConfig = {
                 region: this.region,
@@ -121,12 +128,8 @@ export class S3CompatibleStorage extends BaseStorage {
                     accessKeyId: this.accessKeyId,
                     secretAccessKey: this.secretAccessKey
                 },
-                // Use custom HTTP handler with optimized socket management
-                requestHandler: new NodeHttpHandler({
-                    httpsAgent: httpAgent,
-                    connectionTimeout: 10000, // 10 second connection timeout
-                    socketTimeout: 60000 // 60 second socket timeout
-                }),
+                // Use adaptive socket manager for automatic optimization
+                requestHandler: this.socketManager.getHttpHandler(),
                 // Retry configuration for resilience
                 maxAttempts: 5, // Retry up to 5 times
                 retryMode: 'adaptive' // Use adaptive retry with backoff
@@ -222,6 +225,10 @@ export class S3CompatibleStorage extends BaseStorage {
             // Set storage adapters for cache managers
             this.nounCacheManager.setStorageAdapters(nounStorageAdapter, nounStorageAdapter);
             this.verbCacheManager.setStorageAdapters(verbStorageAdapter, verbStorageAdapter);
+            // Initialize write buffers for high-volume scenarios
+            this.initializeBuffers();
+            // Initialize request coalescer
+            this.initializeCoalescer();
             this.isInitialized = true;
             this.logger.info(`Initialized ${this.serviceType} storage with bucket ${this.bucketName}`);
         }
@@ -231,64 +238,248 @@ export class S3CompatibleStorage extends BaseStorage {
         }
     }
     /**
-     * Dynamically adjust batch size based on memory pressure and error rates
+     * Initialize write buffers for high-volume scenarios
      */
-    adjustBatchSize() {
+    initializeBuffers() {
+        const storageId = `${this.serviceType}-${this.bucketName}`;
+        // Create noun write buffer
+        this.nounWriteBuffer = getWriteBuffer(`${storageId}-nouns`, 'noun', async (items) => {
+            // Bulk write nouns to S3
+            await this.bulkWriteNouns(items);
+        });
+        // Create verb write buffer
+        this.verbWriteBuffer = getWriteBuffer(`${storageId}-verbs`, 'verb', async (items) => {
+            // Bulk write verbs to S3
+            await this.bulkWriteVerbs(items);
+        });
+    }
+    /**
+     * Initialize request coalescer
+     */
+    initializeCoalescer() {
+        const storageId = `${this.serviceType}-${this.bucketName}`;
+        this.requestCoalescer = getCoalescer(storageId, async (batch) => {
+            // Process coalesced operations
+            await this.processCoalescedBatch(batch);
+        });
+    }
+    /**
+     * Check if we should enable high-volume mode
+     */
+    checkVolumeMode() {
         const now = Date.now();
-        // Check memory pressure periodically
-        if (now - this.lastMemoryCheck > this.memoryCheckInterval) {
-            this.lastMemoryCheck = now;
-            const memUsage = process.memoryUsage();
-            const heapUsedPercent = (memUsage.heapUsed / memUsage.heapTotal) * 100;
-            // Adjust batch size based on memory pressure
-            if (heapUsedPercent > 80) {
-                // High memory pressure - reduce batch size
-                this.currentBatchSize = Math.max(1, Math.floor(this.currentBatchSize * 0.5));
-                this.maxConcurrentOperations = Math.max(10, Math.floor(this.maxConcurrentOperations * 0.5));
-                this.logger.warn(`High memory pressure (${heapUsedPercent.toFixed(1)}%), reducing batch size to ${this.currentBatchSize}`);
-            }
-            else if (heapUsedPercent < 50 && this.consecutiveErrors === 0) {
-                // Low memory pressure and no errors - gradually increase batch size
-                this.currentBatchSize = Math.min(this.baseBatchSize * 2, this.currentBatchSize + 1);
-                this.maxConcurrentOperations = Math.min(200, this.maxConcurrentOperations + 10);
+        if (now - this.lastVolumeCheck < this.volumeCheckInterval) {
+            return;
+        }
+        this.lastVolumeCheck = now;
+        // Check pending operations
+        const backpressureStatus = this.backpressure.getStatus();
+        const socketMetrics = this.socketManager.getMetrics();
+        const shouldEnableHighVolume = backpressureStatus.queueLength > 100 ||
+            socketMetrics.pendingRequests > 50 ||
+            this.pendingOperations > 20;
+        if (shouldEnableHighVolume && !this.highVolumeMode) {
+            this.highVolumeMode = true;
+            this.logger.info('Enabling high-volume mode for optimized batching');
+            // Adjust buffer parameters for high volume
+            if (this.nounWriteBuffer) {
+                this.nounWriteBuffer.adjustForLoad(backpressureStatus.queueLength);
+            }
+            if (this.verbWriteBuffer) {
+                this.verbWriteBuffer.adjustForLoad(backpressureStatus.queueLength);
+            }
+            if (this.requestCoalescer) {
+                this.requestCoalescer.adjustParameters(backpressureStatus.queueLength);
+            }
+        }
+        else if (!shouldEnableHighVolume && this.highVolumeMode) {
+            this.highVolumeMode = false;
+            this.logger.info('Disabling high-volume mode');
+        }
+    }
+    /**
+     * Bulk write nouns to S3
+     */
+    async bulkWriteNouns(items) {
+        const { PutObjectCommand } = await import('@aws-sdk/client-s3');
+        // Process in parallel with limited concurrency
+        const promises = [];
+        const batchSize = 10; // Process 10 at a time
+        const entries = Array.from(items.entries());
+        for (let i = 0; i < entries.length; i += batchSize) {
+            const batch = entries.slice(i, i + batchSize);
+            const batchPromise = Promise.all(batch.map(async ([id, node]) => {
+                const serializableNode = {
+                    ...node,
+                    connections: this.mapToObject(node.connections, (set) => Array.from(set))
+                };
+                const key = `${this.nounPrefix}${id}.json`;
+                const body = JSON.stringify(serializableNode, null, 2);
+                await this.s3Client.send(new PutObjectCommand({
+                    Bucket: this.bucketName,
+                    Key: key,
+                    Body: body,
+                    ContentType: 'application/json'
+                }));
+            })).then(() => { }); // Convert Promise<void[]> to Promise<void>
+            promises.push(batchPromise);
+        }
+        await Promise.all(promises);
+    }
+    /**
+     * Bulk write verbs to S3
+     */
+    async bulkWriteVerbs(items) {
+        const { PutObjectCommand } = await import('@aws-sdk/client-s3');
+        // Process in parallel with limited concurrency
+        const promises = [];
+        const batchSize = 10;
+        const entries = Array.from(items.entries());
+        for (let i = 0; i < entries.length; i += batchSize) {
+            const batch = entries.slice(i, i + batchSize);
+            const batchPromise = Promise.all(batch.map(async ([id, edge]) => {
+                const serializableEdge = {
+                    ...edge,
+                    connections: this.mapToObject(edge.connections, (set) => Array.from(set))
+                };
+                const key = `${this.verbPrefix}${id}.json`;
+                const body = JSON.stringify(serializableEdge, null, 2);
+                await this.s3Client.send(new PutObjectCommand({
+                    Bucket: this.bucketName,
+                    Key: key,
+                    Body: body,
+                    ContentType: 'application/json'
+                }));
+            })).then(() => { }); // Convert Promise<void[]> to Promise<void>
+            promises.push(batchPromise);
+        }
+        await Promise.all(promises);
+    }
+    /**
+     * Process coalesced batch of operations
+     */
+    async processCoalescedBatch(batch) {
+        // Group operations by type
+        const writes = [];
+        const reads = [];
+        const deletes = [];
+        for (const op of batch) {
+            if (op.type === 'write') {
+                writes.push(op);
+            }
+            else if (op.type === 'read') {
+                reads.push(op);
             }
+            else if (op.type === 'delete') {
+                deletes.push(op);
+            }
+        }
+        // Process in order: deletes, writes, reads
+        if (deletes.length > 0) {
+            await this.processBulkDeletes(deletes);
+        }
+        if (writes.length > 0) {
+            await this.processBulkWrites(writes);
         }
+        if (reads.length > 0) {
+            await this.processBulkReads(reads);
+        }
+    }
+    /**
+     * Process bulk deletes
+     */
+    async processBulkDeletes(deletes) {
+        const { DeleteObjectCommand } = await import('@aws-sdk/client-s3');
+        await Promise.all(deletes.map(async (op) => {
+            await this.s3Client.send(new DeleteObjectCommand({
+                Bucket: this.bucketName,
+                Key: op.key
+            }));
+        }));
+    }
+    /**
+     * Process bulk writes
+     */
+    async processBulkWrites(writes) {
+        const { PutObjectCommand } = await import('@aws-sdk/client-s3');
+        await Promise.all(writes.map(async (op) => {
+            await this.s3Client.send(new PutObjectCommand({
+                Bucket: this.bucketName,
+                Key: op.key,
+                Body: JSON.stringify(op.data),
+                ContentType: 'application/json'
+            }));
+        }));
+    }
+    /**
+     * Process bulk reads
+     */
+    async processBulkReads(reads) {
+        const { GetObjectCommand } = await import('@aws-sdk/client-s3');
+        await Promise.all(reads.map(async (op) => {
+            try {
+                const response = await this.s3Client.send(new GetObjectCommand({
+                    Bucket: this.bucketName,
+                    Key: op.key
+                }));
+                if (response.Body) {
+                    const data = await response.Body.transformToString();
+                    op.data = JSON.parse(data);
+                }
+            }
+            catch (error) {
+                op.data = null;
+            }
+        }));
+    }
+    /**
+     * Dynamically adjust batch size based on memory pressure and error rates
+     */
+    adjustBatchSize() {
+        // Let the adaptive socket manager handle batch size optimization
+        this.currentBatchSize = this.socketManager.getBatchSize();
+        // Get adaptive configuration for concurrent operations
+        const config = this.socketManager.getConfig();
+        this.maxConcurrentOperations = Math.min(config.maxSockets * 2, 500);
+        // Track metrics for the socket manager
+        const now = Date.now();
         // Reset error counter periodically if no recent errors
         if (now - this.lastErrorReset > 60000 && this.consecutiveErrors > 0) {
             this.consecutiveErrors = Math.max(0, this.consecutiveErrors - 1);
             this.lastErrorReset = now;
         }
-        // Adjust based on error rate
-        if (this.consecutiveErrors > 5) {
-            this.currentBatchSize = Math.max(1, Math.floor(this.currentBatchSize * 0.7));
-            this.maxConcurrentOperations = Math.max(10, Math.floor(this.maxConcurrentOperations * 0.7));
-            this.logger.warn(`High error rate, reducing batch size to ${this.currentBatchSize}`);
-        }
     }
     /**
      * Apply backpressure when system is under load
      */
     async applyBackpressure() {
-        // Wait if too many operations are pending
-        while (this.pendingOperations >= this.maxConcurrentOperations) {
-            const memUsage = process.memoryUsage();
-            const heapUsedPercent = (memUsage.heapUsed / memUsage.heapTotal) * 100;
-            if (heapUsedPercent > 85) {
-                // Critical memory pressure - wait longer
-                await new Promise(resolve => setTimeout(resolve, 100));
-            }
-            else {
-                // Normal backpressure - short wait
-                await new Promise(resolve => setImmediate(resolve));
-            }
+        // Generate unique request ID for tracking
+        const requestId = `${Date.now()}-${Math.random().toString(36).substr(2, 9)}`;
+        try {
+            // Use adaptive backpressure system
+            await this.backpressure.requestPermission(requestId, 1);
+            // Track with socket manager
+            this.socketManager.trackRequestStart(requestId);
+            this.pendingOperations++;
+            return requestId;
+        }
+        catch (error) {
+            // If backpressure rejects, throw a more informative error
+            const message = error instanceof Error ? error.message : String(error);
+            throw new Error(`System overloaded: ${message}`);
         }
-        this.pendingOperations++;
     }
     /**
      * Release backpressure after operation completes
      */
-    releaseBackpressure(success = true) {
+    releaseBackpressure(success = true, requestId) {
         this.pendingOperations = Math.max(0, this.pendingOperations - 1);
+        if (requestId) {
+            // Track with socket manager
+            this.socketManager.trackRequestComplete(requestId, success);
+            // Release from backpressure system
+            this.backpressure.releasePermission(requestId, success);
+        }
         if (!success) {
             this.consecutiveErrors++;
         }
@@ -303,8 +494,8 @@ export class S3CompatibleStorage extends BaseStorage {
      * Get current batch size for operations
      */
     getBatchSize() {
-        this.adjustBatchSize();
-        return this.currentBatchSize;
+        // Use adaptive socket manager's batch size
+        return this.socketManager.getBatchSize();
     }
     /**
      * Save a noun to storage (internal implementation)
@@ -317,8 +508,15 @@ export class S3CompatibleStorage extends BaseStorage {
      */
     async saveNode(node) {
         await this.ensureInitialized();
+        // Check if we should use high-volume mode
+        this.checkVolumeMode();
+        // Use write buffer in high-volume mode
+        if (this.highVolumeMode && this.nounWriteBuffer) {
+            await this.nounWriteBuffer.add(node.id, node);
+            return;
+        }
         // Apply backpressure before starting operation
-        await this.applyBackpressure();
+        const requestId = await this.applyBackpressure();
         try {
             this.logger.trace(`Saving node ${node.id}`);
             // Convert connections Map to a serializable format
@@ -368,11 +566,11 @@ export class S3CompatibleStorage extends BaseStorage {
                 this.logger.warn(`Failed to verify node ${node.id} was saved correctly:`, verifyError);
             }
             // Release backpressure on success
-            this.releaseBackpressure(true);
+            this.releaseBackpressure(true, requestId);
         }
         catch (error) {
             // Release backpressure on error
-            this.releaseBackpressure(false);
+            this.releaseBackpressure(false, requestId);
             this.logger.error(`Failed to save node ${node.id}:`, error);
             throw new Error(`Failed to save node ${node.id}: ${error}`);
         }
@@ -652,8 +850,15 @@ export class S3CompatibleStorage extends BaseStorage {
      */
     async saveEdge(edge) {
         await this.ensureInitialized();
+        // Check if we should use high-volume mode
+        this.checkVolumeMode();
+        // Use write buffer in high-volume mode
+        if (this.highVolumeMode && this.verbWriteBuffer) {
+            await this.verbWriteBuffer.add(edge.id, edge);
+            return;
+        }
         // Apply backpressure before starting operation
-        await this.applyBackpressure();
+        const requestId = await this.applyBackpressure();
         try {
             // Convert connections Map to a serializable format
             const serializableEdge = {
@@ -680,11 +885,11 @@ export class S3CompatibleStorage extends BaseStorage {
                 }
             });
             // Release backpressure on success
-            this.releaseBackpressure(true);
+            this.releaseBackpressure(true, requestId);
         }
         catch (error) {
             // Release backpressure on error
-            this.releaseBackpressure(false);
+            this.releaseBackpressure(false, requestId);
             this.logger.error(`Failed to save edge ${edge.id}:`, error);
             throw new Error(`Failed to save edge ${edge.id}: ${error}`);
         }
@@ -1024,7 +1229,7 @@ export class S3CompatibleStorage extends BaseStorage {
     async saveMetadata(id, metadata) {
         await this.ensureInitialized();
         // Apply backpressure before starting operation
-        await this.applyBackpressure();
+        const requestId = await this.applyBackpressure();
         try {
             // Import the PutObjectCommand only when needed
             const { PutObjectCommand } = await import('@aws-sdk/client-s3');
@@ -1065,11 +1270,11 @@ export class S3CompatibleStorage extends BaseStorage {
                 this.logger.warn(`Failed to verify metadata for ${id} was saved correctly:`, verifyError);
             }
             // Release backpressure on success
-            this.releaseBackpressure(true);
+            this.releaseBackpressure(true, requestId);
         }
         catch (error) {
             // Release backpressure on error
-            this.releaseBackpressure(false);
+            this.releaseBackpressure(false, requestId);
             this.logger.error(`Failed to save metadata for ${id}:`, error);
             throw new Error(`Failed to save metadata for ${id}: ${error}`);
         }