@soulcraft/brainy 4.10.3 → 4.11.0

package/dist/storage/adapters/gcsStorage.d.ts

@@ -9,7 +9,7 @@
  * 4. HMAC Keys (fallback for backward compatibility)
  */
 import { HNSWNoun, HNSWVerb, HNSWNounWithMetadata, HNSWVerbWithMetadata, StatisticsData } from '../../coreTypes.js';
-import { BaseStorage } from '../baseStorage.js';
+import { BaseStorage, StorageBatchConfig } from '../baseStorage.js';
 type HNSWNode = HNSWNoun;
 type Edge = HNSWVerb;
 /**
@@ -76,6 +76,21 @@ export declare class GcsStorage extends BaseStorage {
         };
         readOnly?: boolean;
     });
+    /**
+     * Get GCS-optimized batch configuration
+     *
+     * GCS has strict rate limits (~5000 writes/second per bucket) and benefits from:
+     * - Moderate batch sizes (50 items)
+     * - Sequential processing (not parallel)
+     * - Delays between batches (100ms)
+     *
+     * Note: Each entity write involves 2 operations (vector + metadata),
+     * so 800 ops/sec = ~400 entities/sec = ~2500 actual GCS writes/sec
+     *
+     * @returns GCS-optimized batch configuration
+     * @since v4.11.0
+     */
+    getBatchConfig(): StorageBatchConfig;
     /**
      * Initialize the storage adapter
      */

package/dist/storage/adapters/gcsStorage.js

@@ -92,6 +92,32 @@ export class GcsStorage extends BaseStorage {
             prodLog.info('🚀 High-volume mode FORCED via BRAINY_FORCE_HIGH_VOLUME environment variable');
         }
     }
+    /**
+     * Get GCS-optimized batch configuration
+     *
+     * GCS has strict rate limits (~5000 writes/second per bucket) and benefits from:
+     * - Moderate batch sizes (50 items)
+     * - Sequential processing (not parallel)
+     * - Delays between batches (100ms)
+     *
+     * Note: Each entity write involves 2 operations (vector + metadata),
+     * so 800 ops/sec = ~400 entities/sec = ~2500 actual GCS writes/sec
+     *
+     * @returns GCS-optimized batch configuration
+     * @since v4.11.0
+     */
+    getBatchConfig() {
+        return {
+            maxBatchSize: 50,
+            batchDelayMs: 100,
+            maxConcurrent: 50,
+            supportsParallelWrites: false, // Sequential is safer for GCS rate limits
+            rateLimit: {
+                operationsPerSecond: 800, // Conservative estimate for entity operations
+                burstCapacity: 200
+            }
+        };
+    }
     /**
      * Initialize the storage adapter
      */

package/dist/storage/adapters/memoryStorage.d.ts

@@ -3,7 +3,7 @@
  * In-memory storage adapter for environments where persistent storage is not available or needed
  */
 import { HNSWNoun, HNSWVerb, HNSWNounWithMetadata, HNSWVerbWithMetadata, StatisticsData } from '../../coreTypes.js';
-import { BaseStorage } from '../baseStorage.js';
+import { BaseStorage, StorageBatchConfig } from '../baseStorage.js';
 /**
  * In-memory storage adapter
  * Uses Maps to store data in memory
@@ -17,6 +17,19 @@ export declare class MemoryStorage extends BaseStorage {
     private get nounMetadata();
     private get verbMetadata();
     constructor();
+    /**
+     * Get Memory-optimized batch configuration
+     *
+     * Memory storage has no rate limits and can handle very high throughput:
+     * - Large batch sizes (1000 items)
+     * - No delays needed (0ms)
+     * - High concurrency (1000 operations)
+     * - Parallel processing maximizes throughput
+     *
+     * @returns Memory-optimized batch configuration
+     * @since v4.11.0
+     */
+    getBatchConfig(): StorageBatchConfig;
     /**
      * Initialize the storage adapter
      * Nothing to initialize for in-memory storage

package/dist/storage/adapters/memoryStorage.js

@@ -32,6 +32,30 @@ export class MemoryStorage extends BaseStorage {
         // Even in-memory operations need serialization to prevent async race conditions
         this.hnswLocks = new Map();
     }
+    /**
+     * Get Memory-optimized batch configuration
+     *
+     * Memory storage has no rate limits and can handle very high throughput:
+     * - Large batch sizes (1000 items)
+     * - No delays needed (0ms)
+     * - High concurrency (1000 operations)
+     * - Parallel processing maximizes throughput
+     *
+     * @returns Memory-optimized batch configuration
+     * @since v4.11.0
+     */
+    getBatchConfig() {
+        return {
+            maxBatchSize: 1000,
+            batchDelayMs: 0,
+            maxConcurrent: 1000,
+            supportsParallelWrites: true, // Memory loves parallel operations
+            rateLimit: {
+                operationsPerSecond: 100000, // Virtually unlimited
+                burstCapacity: 100000
+            }
+        };
+    }
     /**
      * Initialize the storage adapter
      * Nothing to initialize for in-memory storage

package/dist/storage/adapters/opfsStorage.d.ts

@@ -3,7 +3,7 @@
  * Provides persistent storage for the vector database using the Origin Private File System API
  */
 import { HNSWNoun, HNSWVerb, HNSWNounWithMetadata, HNSWVerbWithMetadata, StatisticsData } from '../../coreTypes.js';
-import { BaseStorage } from '../baseStorage.js';
+import { BaseStorage, StorageBatchConfig } from '../baseStorage.js';
 import '../../types/fileSystemTypes.js';
 type HNSWNode = HNSWNoun;
 /**
@@ -30,6 +30,19 @@ export declare class OPFSStorage extends BaseStorage {
     private activeLocks;
     private lockPrefix;
     constructor();
+    /**
+     * Get OPFS-optimized batch configuration
+     *
+     * OPFS (Origin Private File System) is browser-based storage with moderate performance:
+     * - Moderate batch sizes (100 items)
+     * - Small delays (10ms) for browser event loop
+     * - Limited concurrency (50 operations) - browser constraints
+     * - Sequential processing preferred for stability
+     *
+     * @returns OPFS-optimized batch configuration
+     * @since v4.11.0
+     */
+    getBatchConfig(): StorageBatchConfig;
     /**
      * Initialize the storage adapter
      */

package/dist/storage/adapters/opfsStorage.js

@@ -51,6 +51,30 @@ export class OPFSStorage extends BaseStorage {
                 'storage' in navigator &&
                 'getDirectory' in navigator.storage;
     }
+    /**
+     * Get OPFS-optimized batch configuration
+     *
+     * OPFS (Origin Private File System) is browser-based storage with moderate performance:
+     * - Moderate batch sizes (100 items)
+     * - Small delays (10ms) for browser event loop
+     * - Limited concurrency (50 operations) - browser constraints
+     * - Sequential processing preferred for stability
+     *
+     * @returns OPFS-optimized batch configuration
+     * @since v4.11.0
+     */
+    getBatchConfig() {
+        return {
+            maxBatchSize: 100,
+            batchDelayMs: 10,
+            maxConcurrent: 50,
+            supportsParallelWrites: false, // Sequential safer in browser
+            rateLimit: {
+                operationsPerSecond: 1000,
+                burstCapacity: 500
+            }
+        };
+    }
     /**
      * Initialize the storage adapter
      */

package/dist/storage/adapters/r2Storage.d.ts

@@ -12,7 +12,7 @@
  * Based on latest GCS and S3 implementations with R2-specific enhancements
  */
 import { HNSWNoun, HNSWVerb, HNSWNounWithMetadata, HNSWVerbWithMetadata, StatisticsData } from '../../coreTypes.js';
-import { BaseStorage } from '../baseStorage.js';
+import { BaseStorage, StorageBatchConfig } from '../baseStorage.js';
 type HNSWNode = HNSWNoun;
 type Edge = HNSWVerb;
 /**
@@ -75,6 +75,23 @@ export declare class R2Storage extends BaseStorage {
         };
         readOnly?: boolean;
     });
+    /**
+     * Get R2-optimized batch configuration
+     *
+     * Cloudflare R2 has S3-compatible characteristics with some advantages:
+     * - Zero egress fees (can cache more aggressively)
+     * - Global edge network
+     * - Similar throughput to S3
+     *
+     * R2 benefits from the same configuration as S3:
+     * - Larger batch sizes (100 items)
+     * - Parallel processing
+     * - Short delays (50ms)
+     *
+     * @returns R2-optimized batch configuration
+     * @since v4.11.0
+     */
+    getBatchConfig(): StorageBatchConfig;
     /**
      * Initialize the storage adapter
      */

package/dist/storage/adapters/r2Storage.js

@@ -94,6 +94,34 @@ export class R2Storage extends BaseStorage {
             prodLog.info('🚀 R2: High-volume mode FORCED via environment variable');
         }
     }
+    /**
+     * Get R2-optimized batch configuration
+     *
+     * Cloudflare R2 has S3-compatible characteristics with some advantages:
+     * - Zero egress fees (can cache more aggressively)
+     * - Global edge network
+     * - Similar throughput to S3
+     *
+     * R2 benefits from the same configuration as S3:
+     * - Larger batch sizes (100 items)
+     * - Parallel processing
+     * - Short delays (50ms)
+     *
+     * @returns R2-optimized batch configuration
+     * @since v4.11.0
+     */
+    getBatchConfig() {
+        return {
+            maxBatchSize: 100,
+            batchDelayMs: 50,
+            maxConcurrent: 100,
+            supportsParallelWrites: true, // R2 handles parallel writes like S3
+            rateLimit: {
+                operationsPerSecond: 3500, // Similar to S3 throughput
+                burstCapacity: 1000
+            }
+        };
+    }
     /**
      * Initialize the storage adapter
      */

package/dist/storage/adapters/s3CompatibleStorage.d.ts

@@ -4,7 +4,7 @@
  * including Amazon S3, Cloudflare R2, and Google Cloud Storage
  */
 import { Change, HNSWNoun, HNSWVerb, HNSWNounWithMetadata, HNSWVerbWithMetadata, StatisticsData } from '../../coreTypes.js';
-import { BaseStorage } from '../baseStorage.js';
+import { BaseStorage, StorageBatchConfig } from '../baseStorage.js';
 import { OperationConfig } from '../../utils/operationUtils.js';
 type HNSWNode = HNSWNoun;
 type Edge = HNSWVerb;
@@ -96,6 +96,20 @@ export declare class S3CompatibleStorage extends BaseStorage {
         };
         readOnly?: boolean;
     });
+    /**
+     * Get S3-optimized batch configuration
+     *
+     * S3 has higher throughput than GCS and handles parallel writes efficiently:
+     * - Larger batch sizes (100 items)
+     * - Parallel processing supported
+     * - Shorter delays between batches (50ms)
+     *
+     * S3 can handle ~3500 operations/second per bucket with good performance
+     *
+     * @returns S3-optimized batch configuration
+     * @since v4.11.0
+     */
+    getBatchConfig(): StorageBatchConfig;
     /**
      * Initialize the storage adapter
      */

package/dist/storage/adapters/s3CompatibleStorage.js

@@ -114,6 +114,31 @@ export class S3CompatibleStorage extends BaseStorage {
         this.nounCacheManager = new CacheManager(options.cacheConfig);
         this.verbCacheManager = new CacheManager(options.cacheConfig);
     }
+    /**
+     * Get S3-optimized batch configuration
+     *
+     * S3 has higher throughput than GCS and handles parallel writes efficiently:
+     * - Larger batch sizes (100 items)
+     * - Parallel processing supported
+     * - Shorter delays between batches (50ms)
+     *
+     * S3 can handle ~3500 operations/second per bucket with good performance
+     *
+     * @returns S3-optimized batch configuration
+     * @since v4.11.0
+     */
+    getBatchConfig() {
+        return {
+            maxBatchSize: 100,
+            batchDelayMs: 50,
+            maxConcurrent: 100,
+            supportsParallelWrites: true, // S3 handles parallel writes efficiently
+            rateLimit: {
+                operationsPerSecond: 3500, // S3 is more permissive than GCS
+                burstCapacity: 1000
+            }
+        };
+    }
     /**
      * Initialize the storage adapter
      */

package/dist/storage/baseStorage.d.ts

@@ -5,6 +5,30 @@
 import { GraphAdjacencyIndex } from '../graph/graphAdjacencyIndex.js';
 import { GraphVerb, HNSWNoun, HNSWVerb, NounMetadata, VerbMetadata, HNSWNounWithMetadata, HNSWVerbWithMetadata, StatisticsData } from '../coreTypes.js';
 import { BaseStorageAdapter } from './adapters/baseStorageAdapter.js';
+/**
+ * Storage adapter batch configuration profile
+ * Each storage adapter declares its optimal batch behavior for rate limiting
+ * and performance optimization
+ *
+ * @since v4.11.0
+ */
+export interface StorageBatchConfig {
+    /** Maximum items per batch */
+    maxBatchSize: number;
+    /** Delay between batches in milliseconds (for rate limiting) */
+    batchDelayMs: number;
+    /** Maximum concurrent operations this storage can handle */
+    maxConcurrent: number;
+    /** Whether storage can handle parallel writes efficiently */
+    supportsParallelWrites: boolean;
+    /** Rate limit characteristics of this storage adapter */
+    rateLimit: {
+        /** Approximate operations per second this storage can handle */
+        operationsPerSecond: number;
+        /** Maximum burst capacity before throttling occurs */
+        burstCapacity: number;
+    };
+}
 export declare const NOUNS_METADATA_DIR = "entities/nouns/metadata";
 export declare const VERBS_METADATA_DIR = "entities/verbs/metadata";
 export declare const SYSTEM_DIR = "_system";

package/dist/utils/adaptiveBackpressure.d.ts

@@ -27,33 +27,40 @@ export declare class AdaptiveBackpressure {
     private metrics;
     private config;
     private patterns;
-    private circuitState;
-    private circuitOpenTime;
-    private circuitFailures;
-    private circuitThreshold;
-    private circuitTimeout;
+    private circuits;
     private operationTimes;
     private completedOps;
     private errorOps;
     private lastAdaptation;
     /**
      * Request permission to proceed with an operation
+     * @param operationId Unique ID for this operation
+     * @param priority Priority level (higher = more important)
+     * @param operationType Type of operation (read or write) for circuit breaker isolation
      */
-    requestPermission(operationId: string, priority?: number): Promise<void>;
+    requestPermission(operationId: string, priority?: number, operationType?: 'read' | 'write'): Promise<void>;
     /**
      * Release permission after operation completes
+     * @param operationId Unique ID for this operation
+     * @param success Whether the operation succeeded
+     * @param operationType Type of operation (read or write) for circuit breaker tracking
      */
-    releasePermission(operationId: string, success?: boolean): void;
+    releasePermission(operationId: string, success?: boolean, operationType?: 'read' | 'write'): void;
     /**
-     * Check if circuit breaker is open
+     * Check if circuit breaker is open for a specific operation type
+     * @param circuit The circuit to check (read or write)
      */
     private isCircuitOpen;
     /**
-     * Open the circuit breaker
+     * Open the circuit breaker for a specific operation type
+     * @param circuit The circuit to open (read or write)
+     * @param operationType The operation type name for logging
      */
     private openCircuit;
     /**
-     * Close the circuit breaker
+     * Close the circuit breaker for a specific operation type
+     * @param circuit The circuit to close (read or write)
+     * @param operationType The operation type name for logging
      */
     private closeCircuit;
     /**

package/dist/utils/adaptiveBackpressure.js

@@ -32,12 +32,24 @@ export class AdaptiveBackpressure {
         };
         // Historical patterns for learning
         this.patterns = [];
-        // Circuit breaker state
-        this.circuitState = 'closed';
-        this.circuitOpenTime = 0;
-        this.circuitFailures = 0;
-        this.circuitThreshold = 5;
-        this.circuitTimeout = 30000; // 30 seconds
+        // Separate circuit breakers for read vs write operations
+        // This allows reads to continue even when writes are throttled
+        this.circuits = {
+            read: {
+                state: 'closed',
+                failures: 0,
+                openTime: 0,
+                threshold: 10, // More lenient for reads
+                timeout: 30000
+            },
+            write: {
+                state: 'closed',
+                failures: 0,
+                openTime: 0,
+                threshold: 5, // Stricter for writes
+                timeout: 30000
+            }
+        };
         // Performance tracking
         this.operationTimes = new Map();
         this.completedOps = [];
@@ -46,11 +58,22 @@ export class AdaptiveBackpressure {
     }
     /**
      * Request permission to proceed with an operation
+     * @param operationId Unique ID for this operation
+     * @param priority Priority level (higher = more important)
+     * @param operationType Type of operation (read or write) for circuit breaker isolation
      */
-    async requestPermission(operationId, priority = 1) {
-        // Check circuit breaker
-        if (this.isCircuitOpen()) {
-            throw new Error('Circuit breaker is open - system is recovering');
+    async requestPermission(operationId, priority = 1, operationType = 'write') {
+        const circuit = this.circuits[operationType];
+        // Check circuit breaker for this operation type
+        if (this.isCircuitOpen(circuit)) {
+            // KEY: Allow reads even if write circuit is open
+            if (operationType === 'read' && this.circuits.write.state === 'open') {
+                // Write circuit is open but read circuit is fine - allow read
+                this.activeOperations.add(operationId);
+                this.operationTimes.set(operationId, Date.now());
+                return;
+            }
+            throw new Error(`Circuit breaker is open for ${operationType} operations - system is recovering`);
         }
         // Fast path for low load
         if (this.activeOperations.size < this.maxConcurrent * 0.5 && this.queue.length === 0) {
@@ -84,8 +107,11 @@ export class AdaptiveBackpressure {
     }
     /**
      * Release permission after operation completes
+     * @param operationId Unique ID for this operation
+     * @param success Whether the operation succeeded
+     * @param operationType Type of operation (read or write) for circuit breaker tracking
      */
-    releasePermission(operationId, success = true) {
+    releasePermission(operationId, success = true, operationType = 'write') {
         // Remove from active operations
         this.activeOperations.delete(operationId);
         // Track completion time
@@ -99,19 +125,20 @@ export class AdaptiveBackpressure {
                 this.completedOps = this.completedOps.slice(-500);
             }
         }
-        // Track errors for circuit breaker
+        // Track errors for circuit breaker per operation type
+        const circuit = this.circuits[operationType];
         if (!success) {
             this.errorOps++;
-            this.circuitFailures++;
-            // Check if we should open circuit
-            if (this.circuitFailures >= this.circuitThreshold) {
-                this.openCircuit();
+            circuit.failures++;
+            // Check if we should open circuit for this operation type
+            if (circuit.failures >= circuit.threshold) {
+                this.openCircuit(circuit, operationType);
             }
         }
         else {
             // Reset circuit failures on success
-            if (this.circuitState === 'half-open') {
-                this.closeCircuit();
+            if (circuit.state === 'half-open') {
+                this.closeCircuit(circuit, operationType);
             }
         }
         // Process queue if there are waiting operations
@@ -129,13 +156,14 @@ export class AdaptiveBackpressure {
         this.adaptIfNeeded();
     }
     /**
-     * Check if circuit breaker is open
+     * Check if circuit breaker is open for a specific operation type
+     * @param circuit The circuit to check (read or write)
      */
-    isCircuitOpen() {
-        if (this.circuitState === 'open') {
+    isCircuitOpen(circuit) {
+        if (circuit.state === 'open') {
             // Check if timeout has passed
-            if (Date.now() - this.circuitOpenTime > this.circuitTimeout) {
-                this.circuitState = 'half-open';
+            if (Date.now() - circuit.openTime > circuit.timeout) {
+                circuit.state = 'half-open';
                 this.logger.info('Circuit breaker entering half-open state');
                 return false;
             }
@@ -144,26 +172,34 @@ export class AdaptiveBackpressure {
         return false;
     }
     /**
-     * Open the circuit breaker
+     * Open the circuit breaker for a specific operation type
+     * @param circuit The circuit to open (read or write)
+     * @param operationType The operation type name for logging
      */
-    openCircuit() {
-        if (this.circuitState !== 'open') {
-            this.circuitState = 'open';
-            this.circuitOpenTime = Date.now();
-            this.logger.warn('Circuit breaker opened due to high error rate');
-            // Reduce load immediately
-            this.maxConcurrent = Math.max(10, Math.floor(this.maxConcurrent * 0.3));
+    openCircuit(circuit, operationType) {
+        if (circuit.state !== 'open') {
+            circuit.state = 'open';
+            circuit.openTime = Date.now();
+            this.logger.warn(`Circuit breaker opened for ${operationType} operations due to high error rate`);
+            // Reduce load immediately for write operations
+            if (operationType === 'write') {
+                this.maxConcurrent = Math.max(10, Math.floor(this.maxConcurrent * 0.3));
+            }
         }
     }
     /**
-     * Close the circuit breaker
+     * Close the circuit breaker for a specific operation type
+     * @param circuit The circuit to close (read or write)
+     * @param operationType The operation type name for logging
      */
-    closeCircuit() {
-        this.circuitState = 'closed';
-        this.circuitFailures = 0;
-        this.logger.info('Circuit breaker closed - system recovered');
-        // Gradually increase capacity
-        this.maxConcurrent = Math.min(500, Math.floor(this.maxConcurrent * 1.5));
+    closeCircuit(circuit, operationType) {
+        circuit.state = 'closed';
+        circuit.failures = 0;
+        this.logger.info(`Circuit breaker closed for ${operationType} - system recovered`);
+        // Gradually increase capacity for write operations
+        if (operationType === 'write') {
+            this.maxConcurrent = Math.min(500, Math.floor(this.maxConcurrent * 1.5));
+        }
     }
     /**
      * Adapt configuration based on metrics
@@ -262,13 +298,8 @@ export class AdaptiveBackpressure {
             // Allow queue depth to be 10 seconds worth of throughput
             this.config.maxQueueDepth = Math.max(100, Math.min(10000, Math.floor(this.metrics.throughput * 10)));
         }
-        // Adapt circuit breaker threshold based on error patterns
-        if (this.metrics.errorRate < 0.01 && this.circuitThreshold > 5) {
-            this.circuitThreshold = Math.max(5, this.circuitThreshold - 1);
-        }
-        else if (this.metrics.errorRate > 0.05 && this.circuitThreshold < 20) {
-            this.circuitThreshold = Math.min(20, this.circuitThreshold + 1);
-        }
+        // Note: Circuit breaker thresholds are now fixed per operation type (read/write)
+        // and do not adapt dynamically to maintain predictable behavior
     }
     /**
      * Predict future load based on patterns
@@ -303,10 +334,24 @@ export class AdaptiveBackpressure {
      * Get current configuration and metrics
      */
     getStatus() {
+        // Combined circuit status for backward compatibility
+        let circuitStatus = 'closed';
+        if (this.circuits.read.state === 'open' && this.circuits.write.state === 'open') {
+            circuitStatus = 'open';
+        }
+        else if (this.circuits.write.state === 'open') {
+            circuitStatus = 'write-circuit-open';
+        }
+        else if (this.circuits.read.state === 'open') {
+            circuitStatus = 'read-circuit-open';
+        }
+        else if (this.circuits.read.state === 'half-open' || this.circuits.write.state === 'half-open') {
+            circuitStatus = 'half-open';
+        }
         return {
             config: { ...this.config },
             metrics: { ...this.metrics },
-            circuit: this.circuitState,
+            circuit: circuitStatus,
             maxConcurrent: this.maxConcurrent,
             activeOps: this.activeOperations.size,
             queueLength: this.queue.length
@@ -322,8 +367,13 @@ export class AdaptiveBackpressure {
         this.completedOps = [];
         this.errorOps = 0;
         this.patterns = [];
-        this.circuitState = 'closed';
-        this.circuitFailures = 0;
+        // Reset both circuit breakers
+        this.circuits.read.state = 'closed';
+        this.circuits.read.failures = 0;
+        this.circuits.read.openTime = 0;
+        this.circuits.write.state = 'closed';
+        this.circuits.write.failures = 0;
+        this.circuits.write.openTime = 0;
         this.maxConcurrent = 100;
         this.logger.info('Backpressure system reset to defaults');
     }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@soulcraft/brainy",
-  "version": "4.10.3",
+  "version": "4.11.0",
   "description": "Universal Knowledge Protocol™ - World's first Triple Intelligence database unifying vector, graph, and document search in one API. 31 nouns × 40 verbs for infinite expressiveness.",
   "main": "dist/index.js",
   "module": "dist/index.js",