@soulcraft/brainy 0.54.0 → 0.54.2

package/dist/storage/adapters/s3CompatibleStorage.d.ts

@@ -70,6 +70,13 @@ export declare class S3CompatibleStorage extends BaseStorage {
     private lastErrorReset;
     private socketManager;
     private backpressure;
+    private nounWriteBuffer;
+    private verbWriteBuffer;
+    private requestCoalescer;
+    private highVolumeMode;
+    private lastVolumeCheck;
+    private volumeCheckInterval;
+    private forceHighVolumeMode;
     private operationExecutors;
     private nounCacheManager;
     private verbCacheManager;
@@ -99,6 +106,42 @@ export declare class S3CompatibleStorage extends BaseStorage {
      * Initialize the storage adapter
      */
     init(): Promise<void>;
+    /**
+     * Initialize write buffers for high-volume scenarios
+     */
+    private initializeBuffers;
+    /**
+     * Initialize request coalescer
+     */
+    private initializeCoalescer;
+    /**
+     * Check if we should enable high-volume mode
+     */
+    private checkVolumeMode;
+    /**
+     * Bulk write nouns to S3
+     */
+    private bulkWriteNouns;
+    /**
+     * Bulk write verbs to S3
+     */
+    private bulkWriteVerbs;
+    /**
+     * Process coalesced batch of operations
+     */
+    private processCoalescedBatch;
+    /**
+     * Process bulk deletes
+     */
+    private processBulkDeletes;
+    /**
+     * Process bulk writes
+     */
+    private processBulkWrites;
+    /**
+     * Process bulk reads
+     */
+    private processBulkReads;
     /**
      * Dynamically adjust batch size based on memory pressure and error rates
      */

package/dist/storage/adapters/s3CompatibleStorage.js

@@ -11,6 +11,8 @@ import { CacheManager } from '../cacheManager.js';
 import { createModuleLogger } from '../../utils/logger.js';
 import { getGlobalSocketManager } from '../../utils/adaptiveSocketManager.js';
 import { getGlobalBackpressure } from '../../utils/adaptiveBackpressure.js';
+import { getWriteBuffer } from '../../utils/writeBuffer.js';
+import { getCoalescer } from '../../utils/requestCoalescer.js';
 // Export R2Storage as an alias for S3CompatibleStorage
 export { S3CompatibleStorage as R2Storage };
 /**
@@ -64,6 +66,16 @@ export class S3CompatibleStorage extends BaseStorage {
         this.socketManager = getGlobalSocketManager();
         // Adaptive backpressure for automatic flow control
         this.backpressure = getGlobalBackpressure();
+        // Write buffers for bulk operations
+        this.nounWriteBuffer = null;
+        this.verbWriteBuffer = null;
+        // Request coalescer for deduplication
+        this.requestCoalescer = null;
+        // High-volume mode detection - MUCH more aggressive
+        this.highVolumeMode = false;
+        this.lastVolumeCheck = 0;
+        this.volumeCheckInterval = 1000; // Check every second, not 5
+        this.forceHighVolumeMode = false; // Environment variable override
         // Module logger
         this.logger = createModuleLogger('S3Storage');
         // Node cache to avoid redundant API calls
@@ -214,6 +226,10 @@ export class S3CompatibleStorage extends BaseStorage {
             // Set storage adapters for cache managers
             this.nounCacheManager.setStorageAdapters(nounStorageAdapter, nounStorageAdapter);
             this.verbCacheManager.setStorageAdapters(verbStorageAdapter, verbStorageAdapter);
+            // Initialize write buffers for high-volume scenarios
+            this.initializeBuffers();
+            // Initialize request coalescer
+            this.initializeCoalescer();
             this.isInitialized = true;
             this.logger.info(`Initialized ${this.serviceType} storage with bucket ${this.bucketName}`);
         }
@@ -222,6 +238,231 @@ export class S3CompatibleStorage extends BaseStorage {
             throw new Error(`Failed to initialize ${this.serviceType} storage: ${error}`);
         }
     }
+    /**
+     * Initialize write buffers for high-volume scenarios
+     */
+    initializeBuffers() {
+        const storageId = `${this.serviceType}-${this.bucketName}`;
+        // Create noun write buffer
+        this.nounWriteBuffer = getWriteBuffer(`${storageId}-nouns`, 'noun', async (items) => {
+            // Bulk write nouns to S3
+            await this.bulkWriteNouns(items);
+        });
+        // Create verb write buffer
+        this.verbWriteBuffer = getWriteBuffer(`${storageId}-verbs`, 'verb', async (items) => {
+            // Bulk write verbs to S3
+            await this.bulkWriteVerbs(items);
+        });
+    }
+    /**
+     * Initialize request coalescer
+     */
+    initializeCoalescer() {
+        const storageId = `${this.serviceType}-${this.bucketName}`;
+        this.requestCoalescer = getCoalescer(storageId, async (batch) => {
+            // Process coalesced operations
+            await this.processCoalescedBatch(batch);
+        });
+    }
+    /**
+     * Check if we should enable high-volume mode
+     */
+    checkVolumeMode() {
+        const now = Date.now();
+        if (now - this.lastVolumeCheck < this.volumeCheckInterval) {
+            return;
+        }
+        this.lastVolumeCheck = now;
+        // Check environment variable override
+        const envThreshold = process.env.BRAINY_BUFFER_THRESHOLD;
+        const threshold = envThreshold ? parseInt(envThreshold) : 1; // Default to 1!
+        // Force enable from environment
+        if (process.env.BRAINY_FORCE_BUFFERING === 'true') {
+            this.forceHighVolumeMode = true;
+        }
+        // Get metrics
+        const backpressureStatus = this.backpressure.getStatus();
+        const socketMetrics = this.socketManager.getMetrics();
+        // MUCH more aggressive detection - trigger on almost any load
+        const shouldEnableHighVolume = this.forceHighVolumeMode || // Environment override
+            backpressureStatus.queueLength > threshold || // Configurable threshold
+            socketMetrics.pendingRequests > threshold || // Socket pressure
+            this.pendingOperations > threshold || // Any pending ops
+            socketMetrics.socketUtilization > 0.1 || // Even 10% socket usage
+            (socketMetrics.requestsPerSecond > 10) || // High request rate
+            (this.consecutiveErrors > 0); // Any errors at all
+        if (shouldEnableHighVolume && !this.highVolumeMode) {
+            this.highVolumeMode = true;
+            this.logger.warn(`🚨 HIGH-VOLUME MODE ACTIVATED 🚨`);
+            this.logger.warn(`  Queue Length: ${backpressureStatus.queueLength}`);
+            this.logger.warn(`  Pending Requests: ${socketMetrics.pendingRequests}`);
+            this.logger.warn(`  Pending Operations: ${this.pendingOperations}`);
+            this.logger.warn(`  Socket Utilization: ${(socketMetrics.socketUtilization * 100).toFixed(1)}%`);
+            this.logger.warn(`  Requests/sec: ${socketMetrics.requestsPerSecond}`);
+            this.logger.warn(`  Consecutive Errors: ${this.consecutiveErrors}`);
+            this.logger.warn(`  Threshold: ${threshold}`);
+            // Adjust buffer parameters for high volume
+            const queueLength = Math.max(backpressureStatus.queueLength, socketMetrics.pendingRequests, 100);
+            if (this.nounWriteBuffer) {
+                this.nounWriteBuffer.adjustForLoad(queueLength);
+                const stats = this.nounWriteBuffer.getStats();
+                this.logger.warn(`  Noun Buffer: ${stats.bufferSize} items, ${stats.totalWrites} total writes`);
+            }
+            if (this.verbWriteBuffer) {
+                this.verbWriteBuffer.adjustForLoad(queueLength);
+                const stats = this.verbWriteBuffer.getStats();
+                this.logger.warn(`  Verb Buffer: ${stats.bufferSize} items, ${stats.totalWrites} total writes`);
+            }
+            if (this.requestCoalescer) {
+                this.requestCoalescer.adjustParameters(queueLength);
+                const sizes = this.requestCoalescer.getQueueSizes();
+                this.logger.warn(`  Coalescer: ${sizes.total} queued operations`);
+            }
+        }
+        else if (!shouldEnableHighVolume && this.highVolumeMode && !this.forceHighVolumeMode) {
+            this.highVolumeMode = false;
+            this.logger.info('✅ High-volume mode deactivated - load normalized');
+        }
+        // Log current status every 10 checks when in high-volume mode
+        if (this.highVolumeMode && (now % 10000) < this.volumeCheckInterval) {
+            this.logger.info(`📊 High-volume mode status: Queue=${backpressureStatus.queueLength}, Pending=${socketMetrics.pendingRequests}, Sockets=${(socketMetrics.socketUtilization * 100).toFixed(1)}%`);
+        }
+    }
+    /**
+     * Bulk write nouns to S3
+     */
+    async bulkWriteNouns(items) {
+        const { PutObjectCommand } = await import('@aws-sdk/client-s3');
+        // Process in parallel with limited concurrency
+        const promises = [];
+        const batchSize = 10; // Process 10 at a time
+        const entries = Array.from(items.entries());
+        for (let i = 0; i < entries.length; i += batchSize) {
+            const batch = entries.slice(i, i + batchSize);
+            const batchPromise = Promise.all(batch.map(async ([id, node]) => {
+                const serializableNode = {
+                    ...node,
+                    connections: this.mapToObject(node.connections, (set) => Array.from(set))
+                };
+                const key = `${this.nounPrefix}${id}.json`;
+                const body = JSON.stringify(serializableNode, null, 2);
+                await this.s3Client.send(new PutObjectCommand({
+                    Bucket: this.bucketName,
+                    Key: key,
+                    Body: body,
+                    ContentType: 'application/json'
+                }));
+            })).then(() => { }); // Convert Promise<void[]> to Promise<void>
+            promises.push(batchPromise);
+        }
+        await Promise.all(promises);
+    }
+    /**
+     * Bulk write verbs to S3
+     */
+    async bulkWriteVerbs(items) {
+        const { PutObjectCommand } = await import('@aws-sdk/client-s3');
+        // Process in parallel with limited concurrency
+        const promises = [];
+        const batchSize = 10;
+        const entries = Array.from(items.entries());
+        for (let i = 0; i < entries.length; i += batchSize) {
+            const batch = entries.slice(i, i + batchSize);
+            const batchPromise = Promise.all(batch.map(async ([id, edge]) => {
+                const serializableEdge = {
+                    ...edge,
+                    connections: this.mapToObject(edge.connections, (set) => Array.from(set))
+                };
+                const key = `${this.verbPrefix}${id}.json`;
+                const body = JSON.stringify(serializableEdge, null, 2);
+                await this.s3Client.send(new PutObjectCommand({
+                    Bucket: this.bucketName,
+                    Key: key,
+                    Body: body,
+                    ContentType: 'application/json'
+                }));
+            })).then(() => { }); // Convert Promise<void[]> to Promise<void>
+            promises.push(batchPromise);
+        }
+        await Promise.all(promises);
+    }
+    /**
+     * Process coalesced batch of operations
+     */
+    async processCoalescedBatch(batch) {
+        // Group operations by type
+        const writes = [];
+        const reads = [];
+        const deletes = [];
+        for (const op of batch) {
+            if (op.type === 'write') {
+                writes.push(op);
+            }
+            else if (op.type === 'read') {
+                reads.push(op);
+            }
+            else if (op.type === 'delete') {
+                deletes.push(op);
+            }
+        }
+        // Process in order: deletes, writes, reads
+        if (deletes.length > 0) {
+            await this.processBulkDeletes(deletes);
+        }
+        if (writes.length > 0) {
+            await this.processBulkWrites(writes);
+        }
+        if (reads.length > 0) {
+            await this.processBulkReads(reads);
+        }
+    }
+    /**
+     * Process bulk deletes
+     */
+    async processBulkDeletes(deletes) {
+        const { DeleteObjectCommand } = await import('@aws-sdk/client-s3');
+        await Promise.all(deletes.map(async (op) => {
+            await this.s3Client.send(new DeleteObjectCommand({
+                Bucket: this.bucketName,
+                Key: op.key
+            }));
+        }));
+    }
+    /**
+     * Process bulk writes
+     */
+    async processBulkWrites(writes) {
+        const { PutObjectCommand } = await import('@aws-sdk/client-s3');
+        await Promise.all(writes.map(async (op) => {
+            await this.s3Client.send(new PutObjectCommand({
+                Bucket: this.bucketName,
+                Key: op.key,
+                Body: JSON.stringify(op.data),
+                ContentType: 'application/json'
+            }));
+        }));
+    }
+    /**
+     * Process bulk reads
+     */
+    async processBulkReads(reads) {
+        const { GetObjectCommand } = await import('@aws-sdk/client-s3');
+        await Promise.all(reads.map(async (op) => {
+            try {
+                const response = await this.s3Client.send(new GetObjectCommand({
+                    Bucket: this.bucketName,
+                    Key: op.key
+                }));
+                if (response.Body) {
+                    const data = await response.Body.transformToString();
+                    op.data = JSON.parse(data);
+                }
+            }
+            catch (error) {
+                op.data = null;
+            }
+        }));
+    }
     /**
      * Dynamically adjust batch size based on memory pressure and error rates
      */
@@ -298,6 +539,17 @@ export class S3CompatibleStorage extends BaseStorage {
      */
     async saveNode(node) {
         await this.ensureInitialized();
+        // ALWAYS check if we should use high-volume mode (critical for detection)
+        this.checkVolumeMode();
+        // Use write buffer in high-volume mode
+        if (this.highVolumeMode && this.nounWriteBuffer) {
+            this.logger.trace(`📝 BUFFERING: Adding noun ${node.id} to write buffer (high-volume mode active)`);
+            await this.nounWriteBuffer.add(node.id, node);
+            return;
+        }
+        else if (!this.highVolumeMode) {
+            this.logger.trace(`📝 DIRECT WRITE: Saving noun ${node.id} directly (high-volume mode inactive)`);
+        }
         // Apply backpressure before starting operation
         const requestId = await this.applyBackpressure();
         try {
@@ -633,6 +885,17 @@ export class S3CompatibleStorage extends BaseStorage {
      */
     async saveEdge(edge) {
         await this.ensureInitialized();
+        // ALWAYS check if we should use high-volume mode (critical for detection)
+        this.checkVolumeMode();
+        // Use write buffer in high-volume mode
+        if (this.highVolumeMode && this.verbWriteBuffer) {
+            this.logger.trace(`📝 BUFFERING: Adding verb ${edge.id} to write buffer (high-volume mode active)`);
+            await this.verbWriteBuffer.add(edge.id, edge);
+            return;
+        }
+        else if (!this.highVolumeMode) {
+            this.logger.trace(`📝 DIRECT WRITE: Saving verb ${edge.id} directly (high-volume mode inactive)`);
+        }
         // Apply backpressure before starting operation
         const requestId = await this.applyBackpressure();
         try {