@soulcraft/brainy 4.10.3 → 4.10.4

package/dist/storage/adapters/baseStorageAdapter.d.ts

@@ -3,6 +3,7 @@
  * Provides common functionality for all storage adapters, including statistics tracking
  */
 import { StatisticsData, StorageAdapter, HNSWNoun, HNSWVerb, HNSWNounWithMetadata, HNSWVerbWithMetadata, NounMetadata, VerbMetadata } from '../../coreTypes.js';
+import { StorageBatchConfig } from '../baseStorage.js';
 /**
  * Base class for storage adapters that implements statistics tracking
  */
@@ -50,6 +51,18 @@ export declare abstract class BaseStorageAdapter implements StorageAdapter {
         quota: number | null;
         details?: Record<string, any>;
     }>;
+    /**
+     * Get optimal batch configuration for this storage adapter
+     * Override in subclasses to provide storage-specific optimization
+     *
+     * This method allows each storage adapter to declare its optimal batch behavior
+     * for rate limiting and performance. The configuration is used by addMany(),
+     * relateMany(), and import operations to automatically adapt to storage capabilities.
+     *
+     * @returns Batch configuration optimized for this storage type
+     * @since v4.11.0
+     */
+    getBatchConfig(): StorageBatchConfig;
     /**
      * Get nouns with pagination and filtering
      * @param options Pagination and filtering options

package/dist/storage/adapters/baseStorageAdapter.js

@@ -60,6 +60,32 @@ export class BaseStorageAdapter {
         this.countPersistBatchSize = 10; // Operations before forcing persist (cloud storage)
         this.countPersistInterval = 5000; // Milliseconds before forcing persist (cloud storage)
     }
+    /**
+     * Get optimal batch configuration for this storage adapter
+     * Override in subclasses to provide storage-specific optimization
+     *
+     * This method allows each storage adapter to declare its optimal batch behavior
+     * for rate limiting and performance. The configuration is used by addMany(),
+     * relateMany(), and import operations to automatically adapt to storage capabilities.
+     *
+     * @returns Batch configuration optimized for this storage type
+     * @since v4.11.0
+     */
+    getBatchConfig() {
+        // Conservative defaults that work safely across all storage types
+        // Cloud storage adapters should override with higher throughput values
+        // Local storage adapters should override with no delays
+        return {
+            maxBatchSize: 50,
+            batchDelayMs: 100,
+            maxConcurrent: 50,
+            supportsParallelWrites: false,
+            rateLimit: {
+                operationsPerSecond: 100,
+                burstCapacity: 200
+            }
+        };
+    }
     /**
      * Save statistics data
      * @param statistics The statistics data to save

package/dist/storage/adapters/fileSystemStorage.d.ts

@@ -3,7 +3,7 @@
  * File system storage adapter for Node.js environments
  */
 import { HNSWNoun, HNSWVerb, HNSWNounWithMetadata, HNSWVerbWithMetadata, StatisticsData } from '../../coreTypes.js';
-import { BaseStorage } from '../baseStorage.js';
+import { BaseStorage, StorageBatchConfig } from '../baseStorage.js';
 type HNSWNode = HNSWNoun;
 type Edge = HNSWVerb;
 /**
@@ -39,6 +39,19 @@ export declare class FileSystemStorage extends BaseStorage {
         compression?: boolean;
         compressionLevel?: number;
     });
+    /**
+     * Get FileSystem-optimized batch configuration
+     *
+     * File system storage is I/O bound but not rate limited:
+     * - Large batch sizes (500 items)
+     * - No delays needed (0ms)
+     * - Moderate concurrency (100 operations) - limited by I/O threads
+     * - Parallel processing supported
+     *
+     * @returns FileSystem-optimized batch configuration
+     * @since v4.11.0
+     */
+    getBatchConfig(): StorageBatchConfig;
     /**
      * Initialize the storage adapter
      */

package/dist/storage/adapters/fileSystemStorage.js

@@ -70,6 +70,30 @@ export class FileSystemStorage extends BaseStorage {
         }
         // Defer path operations until init() when path module is guaranteed to be loaded
     }
+    /**
+     * Get FileSystem-optimized batch configuration
+     *
+     * File system storage is I/O bound but not rate limited:
+     * - Large batch sizes (500 items)
+     * - No delays needed (0ms)
+     * - Moderate concurrency (100 operations) - limited by I/O threads
+     * - Parallel processing supported
+     *
+     * @returns FileSystem-optimized batch configuration
+     * @since v4.11.0
+     */
+    getBatchConfig() {
+        return {
+            maxBatchSize: 500,
+            batchDelayMs: 0,
+            maxConcurrent: 100,
+            supportsParallelWrites: true, // Filesystem handles parallel I/O
+            rateLimit: {
+                operationsPerSecond: 5000, // Depends on disk speed
+                burstCapacity: 2000
+            }
+        };
+    }
     /**
      * Initialize the storage adapter
      */

package/dist/storage/adapters/gcsStorage.d.ts

@@ -9,7 +9,7 @@
  * 4. HMAC Keys (fallback for backward compatibility)
  */
 import { HNSWNoun, HNSWVerb, HNSWNounWithMetadata, HNSWVerbWithMetadata, StatisticsData } from '../../coreTypes.js';
-import { BaseStorage } from '../baseStorage.js';
+import { BaseStorage, StorageBatchConfig } from '../baseStorage.js';
 type HNSWNode = HNSWNoun;
 type Edge = HNSWVerb;
 /**
@@ -76,6 +76,21 @@ export declare class GcsStorage extends BaseStorage {
         };
         readOnly?: boolean;
     });
+    /**
+     * Get GCS-optimized batch configuration
+     *
+     * GCS has strict rate limits (~5000 writes/second per bucket) and benefits from:
+     * - Moderate batch sizes (50 items)
+     * - Sequential processing (not parallel)
+     * - Delays between batches (100ms)
+     *
+     * Note: Each entity write involves 2 operations (vector + metadata),
+     * so 800 ops/sec = ~400 entities/sec = ~2500 actual GCS writes/sec
+     *
+     * @returns GCS-optimized batch configuration
+     * @since v4.11.0
+     */
+    getBatchConfig(): StorageBatchConfig;
     /**
      * Initialize the storage adapter
      */

package/dist/storage/adapters/gcsStorage.js

@@ -92,6 +92,32 @@ export class GcsStorage extends BaseStorage {
             prodLog.info('🚀 High-volume mode FORCED via BRAINY_FORCE_HIGH_VOLUME environment variable');
         }
     }
+    /**
+     * Get GCS-optimized batch configuration
+     *
+     * GCS has strict rate limits (~5000 writes/second per bucket) and benefits from:
+     * - Moderate batch sizes (50 items)
+     * - Sequential processing (not parallel)
+     * - Delays between batches (100ms)
+     *
+     * Note: Each entity write involves 2 operations (vector + metadata),
+     * so 800 ops/sec = ~400 entities/sec = ~2500 actual GCS writes/sec
+     *
+     * @returns GCS-optimized batch configuration
+     * @since v4.11.0
+     */
+    getBatchConfig() {
+        return {
+            maxBatchSize: 50,
+            batchDelayMs: 100,
+            maxConcurrent: 50,
+            supportsParallelWrites: false, // Sequential is safer for GCS rate limits
+            rateLimit: {
+                operationsPerSecond: 800, // Conservative estimate for entity operations
+                burstCapacity: 200
+            }
+        };
+    }
     /**
      * Initialize the storage adapter
      */

package/dist/storage/adapters/memoryStorage.d.ts

@@ -3,7 +3,7 @@
  * In-memory storage adapter for environments where persistent storage is not available or needed
  */
 import { HNSWNoun, HNSWVerb, HNSWNounWithMetadata, HNSWVerbWithMetadata, StatisticsData } from '../../coreTypes.js';
-import { BaseStorage } from '../baseStorage.js';
+import { BaseStorage, StorageBatchConfig } from '../baseStorage.js';
 /**
  * In-memory storage adapter
  * Uses Maps to store data in memory
@@ -17,6 +17,19 @@ export declare class MemoryStorage extends BaseStorage {
     private get nounMetadata();
     private get verbMetadata();
     constructor();
+    /**
+     * Get Memory-optimized batch configuration
+     *
+     * Memory storage has no rate limits and can handle very high throughput:
+     * - Large batch sizes (1000 items)
+     * - No delays needed (0ms)
+     * - High concurrency (1000 operations)
+     * - Parallel processing maximizes throughput
+     *
+     * @returns Memory-optimized batch configuration
+     * @since v4.11.0
+     */
+    getBatchConfig(): StorageBatchConfig;
     /**
      * Initialize the storage adapter
      * Nothing to initialize for in-memory storage

package/dist/storage/adapters/memoryStorage.js

@@ -32,6 +32,30 @@ export class MemoryStorage extends BaseStorage {
         // Even in-memory operations need serialization to prevent async race conditions
         this.hnswLocks = new Map();
     }
+    /**
+     * Get Memory-optimized batch configuration
+     *
+     * Memory storage has no rate limits and can handle very high throughput:
+     * - Large batch sizes (1000 items)
+     * - No delays needed (0ms)
+     * - High concurrency (1000 operations)
+     * - Parallel processing maximizes throughput
+     *
+     * @returns Memory-optimized batch configuration
+     * @since v4.11.0
+     */
+    getBatchConfig() {
+        return {
+            maxBatchSize: 1000,
+            batchDelayMs: 0,
+            maxConcurrent: 1000,
+            supportsParallelWrites: true, // Memory loves parallel operations
+            rateLimit: {
+                operationsPerSecond: 100000, // Virtually unlimited
+                burstCapacity: 100000
+            }
+        };
+    }
     /**
      * Initialize the storage adapter
      * Nothing to initialize for in-memory storage

package/dist/storage/adapters/opfsStorage.d.ts

@@ -3,7 +3,7 @@
  * Provides persistent storage for the vector database using the Origin Private File System API
  */
 import { HNSWNoun, HNSWVerb, HNSWNounWithMetadata, HNSWVerbWithMetadata, StatisticsData } from '../../coreTypes.js';
-import { BaseStorage } from '../baseStorage.js';
+import { BaseStorage, StorageBatchConfig } from '../baseStorage.js';
 import '../../types/fileSystemTypes.js';
 type HNSWNode = HNSWNoun;
 /**
@@ -30,6 +30,19 @@ export declare class OPFSStorage extends BaseStorage {
     private activeLocks;
     private lockPrefix;
     constructor();
+    /**
+     * Get OPFS-optimized batch configuration
+     *
+     * OPFS (Origin Private File System) is browser-based storage with moderate performance:
+     * - Moderate batch sizes (100 items)
+     * - Small delays (10ms) for browser event loop
+     * - Limited concurrency (50 operations) - browser constraints
+     * - Sequential processing preferred for stability
+     *
+     * @returns OPFS-optimized batch configuration
+     * @since v4.11.0
+     */
+    getBatchConfig(): StorageBatchConfig;
     /**
      * Initialize the storage adapter
      */

package/dist/storage/adapters/opfsStorage.js

@@ -51,6 +51,30 @@ export class OPFSStorage extends BaseStorage {
                 'storage' in navigator &&
                 'getDirectory' in navigator.storage;
     }
+    /**
+     * Get OPFS-optimized batch configuration
+     *
+     * OPFS (Origin Private File System) is browser-based storage with moderate performance:
+     * - Moderate batch sizes (100 items)
+     * - Small delays (10ms) for browser event loop
+     * - Limited concurrency (50 operations) - browser constraints
+     * - Sequential processing preferred for stability
+     *
+     * @returns OPFS-optimized batch configuration
+     * @since v4.11.0
+     */
+    getBatchConfig() {
+        return {
+            maxBatchSize: 100,
+            batchDelayMs: 10,
+            maxConcurrent: 50,
+            supportsParallelWrites: false, // Sequential safer in browser
+            rateLimit: {
+                operationsPerSecond: 1000,
+                burstCapacity: 500
+            }
+        };
+    }
     /**
      * Initialize the storage adapter
      */

package/dist/storage/adapters/r2Storage.d.ts

@@ -12,7 +12,7 @@
  * Based on latest GCS and S3 implementations with R2-specific enhancements
  */
 import { HNSWNoun, HNSWVerb, HNSWNounWithMetadata, HNSWVerbWithMetadata, StatisticsData } from '../../coreTypes.js';
-import { BaseStorage } from '../baseStorage.js';
+import { BaseStorage, StorageBatchConfig } from '../baseStorage.js';
 type HNSWNode = HNSWNoun;
 type Edge = HNSWVerb;
 /**
@@ -75,6 +75,23 @@ export declare class R2Storage extends BaseStorage {
         };
         readOnly?: boolean;
     });
+    /**
+     * Get R2-optimized batch configuration
+     *
+     * Cloudflare R2 has S3-compatible characteristics with some advantages:
+     * - Zero egress fees (can cache more aggressively)
+     * - Global edge network
+     * - Similar throughput to S3
+     *
+     * R2 benefits from the same configuration as S3:
+     * - Larger batch sizes (100 items)
+     * - Parallel processing
+     * - Short delays (50ms)
+     *
+     * @returns R2-optimized batch configuration
+     * @since v4.11.0
+     */
+    getBatchConfig(): StorageBatchConfig;
     /**
      * Initialize the storage adapter
      */

package/dist/storage/adapters/r2Storage.js

@@ -94,6 +94,34 @@ export class R2Storage extends BaseStorage {
             prodLog.info('🚀 R2: High-volume mode FORCED via environment variable');
         }
     }
+    /**
+     * Get R2-optimized batch configuration
+     *
+     * Cloudflare R2 has S3-compatible characteristics with some advantages:
+     * - Zero egress fees (can cache more aggressively)
+     * - Global edge network
+     * - Similar throughput to S3
+     *
+     * R2 benefits from the same configuration as S3:
+     * - Larger batch sizes (100 items)
+     * - Parallel processing
+     * - Short delays (50ms)
+     *
+     * @returns R2-optimized batch configuration
+     * @since v4.11.0
+     */
+    getBatchConfig() {
+        return {
+            maxBatchSize: 100,
+            batchDelayMs: 50,
+            maxConcurrent: 100,
+            supportsParallelWrites: true, // R2 handles parallel writes like S3
+            rateLimit: {
+                operationsPerSecond: 3500, // Similar to S3 throughput
+                burstCapacity: 1000
+            }
+        };
+    }
     /**
      * Initialize the storage adapter
      */

package/dist/storage/adapters/s3CompatibleStorage.d.ts

@@ -4,7 +4,7 @@
  * including Amazon S3, Cloudflare R2, and Google Cloud Storage
  */
 import { Change, HNSWNoun, HNSWVerb, HNSWNounWithMetadata, HNSWVerbWithMetadata, StatisticsData } from '../../coreTypes.js';
-import { BaseStorage } from '../baseStorage.js';
+import { BaseStorage, StorageBatchConfig } from '../baseStorage.js';
 import { OperationConfig } from '../../utils/operationUtils.js';
 type HNSWNode = HNSWNoun;
 type Edge = HNSWVerb;
@@ -96,6 +96,20 @@ export declare class S3CompatibleStorage extends BaseStorage {
         };
         readOnly?: boolean;
     });
+    /**
+     * Get S3-optimized batch configuration
+     *
+     * S3 has higher throughput than GCS and handles parallel writes efficiently:
+     * - Larger batch sizes (100 items)
+     * - Parallel processing supported
+     * - Shorter delays between batches (50ms)
+     *
+     * S3 can handle ~3500 operations/second per bucket with good performance
+     *
+     * @returns S3-optimized batch configuration
+     * @since v4.11.0
+     */
+    getBatchConfig(): StorageBatchConfig;
     /**
      * Initialize the storage adapter
      */

package/dist/storage/adapters/s3CompatibleStorage.js

@@ -114,6 +114,31 @@ export class S3CompatibleStorage extends BaseStorage {
         this.nounCacheManager = new CacheManager(options.cacheConfig);
         this.verbCacheManager = new CacheManager(options.cacheConfig);
     }
+    /**
+     * Get S3-optimized batch configuration
+     *
+     * S3 has higher throughput than GCS and handles parallel writes efficiently:
+     * - Larger batch sizes (100 items)
+     * - Parallel processing supported
+     * - Shorter delays between batches (50ms)
+     *
+     * S3 can handle ~3500 operations/second per bucket with good performance
+     *
+     * @returns S3-optimized batch configuration
+     * @since v4.11.0
+     */
+    getBatchConfig() {
+        return {
+            maxBatchSize: 100,
+            batchDelayMs: 50,
+            maxConcurrent: 100,
+            supportsParallelWrites: true, // S3 handles parallel writes efficiently
+            rateLimit: {
+                operationsPerSecond: 3500, // S3 is more permissive than GCS
+                burstCapacity: 1000
+            }
+        };
+    }
     /**
      * Initialize the storage adapter
      */

package/dist/storage/baseStorage.d.ts

@@ -5,6 +5,30 @@
 import { GraphAdjacencyIndex } from '../graph/graphAdjacencyIndex.js';
 import { GraphVerb, HNSWNoun, HNSWVerb, NounMetadata, VerbMetadata, HNSWNounWithMetadata, HNSWVerbWithMetadata, StatisticsData } from '../coreTypes.js';
 import { BaseStorageAdapter } from './adapters/baseStorageAdapter.js';
+/**
+ * Storage adapter batch configuration profile
+ * Each storage adapter declares its optimal batch behavior for rate limiting
+ * and performance optimization
+ *
+ * @since v4.11.0
+ */
+export interface StorageBatchConfig {
+    /** Maximum items per batch */
+    maxBatchSize: number;
+    /** Delay between batches in milliseconds (for rate limiting) */
+    batchDelayMs: number;
+    /** Maximum concurrent operations this storage can handle */
+    maxConcurrent: number;
+    /** Whether storage can handle parallel writes efficiently */
+    supportsParallelWrites: boolean;
+    /** Rate limit characteristics of this storage adapter */
+    rateLimit: {
+        /** Approximate operations per second this storage can handle */
+        operationsPerSecond: number;
+        /** Maximum burst capacity before throttling occurs */
+        burstCapacity: number;
+    };
+}
 export declare const NOUNS_METADATA_DIR = "entities/nouns/metadata";
 export declare const VERBS_METADATA_DIR = "entities/verbs/metadata";
 export declare const SYSTEM_DIR = "_system";

package/dist/utils/adaptiveBackpressure.d.ts

@@ -27,33 +27,40 @@ export declare class AdaptiveBackpressure {
     private metrics;
     private config;
     private patterns;
-    private circuitState;
-    private circuitOpenTime;
-    private circuitFailures;
-    private circuitThreshold;
-    private circuitTimeout;
+    private circuits;
     private operationTimes;
     private completedOps;
     private errorOps;
     private lastAdaptation;
     /**
      * Request permission to proceed with an operation
+     * @param operationId Unique ID for this operation
+     * @param priority Priority level (higher = more important)
+     * @param operationType Type of operation (read or write) for circuit breaker isolation
      */
-    requestPermission(operationId: string, priority?: number): Promise<void>;
+    requestPermission(operationId: string, priority?: number, operationType?: 'read' | 'write'): Promise<void>;
     /**
      * Release permission after operation completes
+     * @param operationId Unique ID for this operation
+     * @param success Whether the operation succeeded
+     * @param operationType Type of operation (read or write) for circuit breaker tracking
      */
-    releasePermission(operationId: string, success?: boolean): void;
+    releasePermission(operationId: string, success?: boolean, operationType?: 'read' | 'write'): void;
     /**
-     * Check if circuit breaker is open
+     * Check if circuit breaker is open for a specific operation type
+     * @param circuit The circuit to check (read or write)
      */
     private isCircuitOpen;
     /**
-     * Open the circuit breaker
+     * Open the circuit breaker for a specific operation type
+     * @param circuit The circuit to open (read or write)
+     * @param operationType The operation type name for logging
      */
     private openCircuit;
     /**
-     * Close the circuit breaker
+     * Close the circuit breaker for a specific operation type
+     * @param circuit The circuit to close (read or write)
+     * @param operationType The operation type name for logging
      */
     private closeCircuit;
     /**