@soulcraft/brainy 0.37.0 → 0.39.0

package/dist/brainyData.d.ts

@@ -3,10 +3,12 @@
  * Main class that provides the vector database functionality
  */
 import { HNSWOptimizedConfig } from './hnsw/hnswIndexOptimized.js';
-import { DistanceFunction, GraphVerb, EmbeddingFunction, HNSWConfig, SearchResult, StorageAdapter, Vector, VectorDocument } from './coreTypes.js';
+import { DistanceFunction, GraphVerb, EmbeddingFunction, HNSWConfig, SearchResult, SearchCursor, PaginatedSearchResult, StorageAdapter, Vector, VectorDocument } from './coreTypes.js';
 import { NounType, VerbType } from './types/graphTypes.js';
 import { WebSocketConnection } from './types/augmentations.js';
 import { BrainyDataInterface } from './types/brainyDataInterface.js';
+import { DistributedConfig } from './types/distributedTypes.js';
+import { SearchCacheConfig } from './utils/searchCache.js';
 export interface BrainyDataConfig {
     /**
      * HNSW index configuration
@@ -121,6 +123,11 @@ export interface BrainyDataConfig {
          */
         verbose?: boolean;
     };
+    /**
+     * Search result caching configuration
+     * Improves performance for repeated queries
+     */
+    searchCache?: SearchCacheConfig;
     /**
      * Timeout configuration for async operations
      * Controls how long operations wait before timing out
@@ -195,6 +202,11 @@ export interface BrainyDataConfig {
          */
         updateIndex?: boolean;
     };
+    /**
+     * Distributed mode configuration
+     * Enables coordination across multiple Brainy instances
+     */
+    distributed?: DistributedConfig | boolean;
     /**
      * Cache configuration for optimizing search performance
      * Controls how the system caches data for faster access
@@ -279,6 +291,8 @@ export declare class BrainyData<T = any> implements BrainyDataInterface<T> {
     private _dimensions;
     private loggingConfig;
     private defaultService;
+    private searchCache;
+    private cacheAutoConfigurator;
     private timeoutConfig;
     private retryConfig;
     private cacheConfig;
@@ -289,6 +303,12 @@ export declare class BrainyData<T = any> implements BrainyDataInterface<T> {
     private remoteServerConfig;
     private serverSearchConduit;
     private serverConnection;
+    private distributedConfig;
+    private configManager;
+    private partitioner;
+    private operationalMode;
+    private domainDetector;
+    private healthMonitor;
     /**
      * Get the vector dimensions
      */
@@ -376,6 +396,20 @@ export declare class BrainyData<T = any> implements BrainyDataInterface<T> {
      * Loads existing data from storage if available
      */
     init(): Promise<void>;
+    /**
+     * Initialize distributed mode
+     * Sets up configuration management, partitioning, and operational modes
+     */
+    private initializeDistributedMode;
+    /**
+     * Handle distributed configuration updates
+     */
+    private handleDistributedConfigUpdate;
+    /**
+     * Get distributed health status
+     * @returns Health status if distributed mode is enabled
+     */
+    getHealthStatus(): any;
     /**
      * Connect to a remote Brainy server for search operations
      * @param serverUrl WebSocket URL of the remote Brainy server
@@ -475,6 +509,7 @@ export declare class BrainyData<T = any> implements BrainyDataInterface<T> {
     searchByNounTypes(queryVectorOrData: Vector | any, k?: number, nounTypes?: string[] | null, options?: {
         forceEmbed?: boolean;
         service?: string;
+        offset?: number;
     }): Promise<SearchResult<T>[]>;
     /**
      * Search for similar vectors
@@ -494,7 +529,31 @@ export declare class BrainyData<T = any> implements BrainyDataInterface<T> {
         verbDirection?: 'outgoing' | 'incoming' | 'both';
         service?: string;
         searchField?: string;
+        filter?: {
+            domain?: string;
+        };
+        offset?: number;
+        skipCache?: boolean;
     }): Promise<SearchResult<T>[]>;
+    /**
+     * Search with cursor-based pagination for better performance on large datasets
+     * @param queryVectorOrData Query vector or data to search for
+     * @param k Number of results to return
+     * @param options Additional options including cursor for pagination
+     * @returns Paginated search results with cursor for next page
+     */
+    searchWithCursor(queryVectorOrData: Vector | any, k?: number, options?: {
+        forceEmbed?: boolean;
+        nounTypes?: string[];
+        includeVerbs?: boolean;
+        service?: string;
+        searchField?: string;
+        filter?: {
+            domain?: string;
+        };
+        cursor?: SearchCursor;
+        skipCache?: boolean;
+    }): Promise<PaginatedSearchResult<T>>;
     /**
      * Search the local database for similar vectors
      * @param queryVectorOrData Query vector or data to search for
@@ -509,6 +568,11 @@ export declare class BrainyData<T = any> implements BrainyDataInterface<T> {
         service?: string;
         searchField?: string;
         priorityFields?: string[];
+        filter?: {
+            domain?: string;
+        };
+        offset?: number;
+        skipCache?: boolean;
     }): Promise<SearchResult<T>[]>;
     /**
      * Find entities similar to a given entity ID
@@ -683,6 +747,32 @@ export declare class BrainyData<T = any> implements BrainyDataInterface<T> {
      * Get the number of vectors in the database
      */
     size(): number;
+    /**
+     * Get search cache statistics for performance monitoring
+     * @returns Cache statistics including hit rate and memory usage
+     */
+    getCacheStats(): {
+        search: {
+            hits: number;
+            misses: number;
+            evictions: number;
+            hitRate: number;
+            size: number;
+            maxSize: number;
+            enabled: boolean;
+        };
+        searchMemoryUsage: number;
+    };
+    /**
+     * Clear search cache manually (useful for testing or memory management)
+     */
+    clearCache(): void;
+    /**
+     * Adapt cache configuration based on current performance metrics
+     * This method analyzes usage patterns and automatically optimizes cache settings
+     * @private
+     */
+    private adaptCacheConfiguration;
     /**
      * Get the number of nouns in the database (excluding verbs)
      * This is used for statistics reporting to match the expected behavior in tests
@@ -829,6 +919,7 @@ export declare class BrainyData<T = any> implements BrainyDataInterface<T> {
         storeResults?: boolean;
         service?: string;
         searchField?: string;
+        offset?: number;
     }): Promise<SearchResult<T>[]>;
     /**
      * Search both local and remote Brainy instances, combining the results
@@ -844,6 +935,7 @@ export declare class BrainyData<T = any> implements BrainyDataInterface<T> {
         localFirst?: boolean;
         service?: string;
         searchField?: string;
+        offset?: number;
     }): Promise<SearchResult<T>[]>;
     /**
      * Check if the instance is connected to a remote server
@@ -994,5 +1086,10 @@ export declare class BrainyData<T = any> implements BrainyDataInterface<T> {
         includeVerbs?: boolean;
         searchMode?: 'local' | 'remote' | 'combined';
     }): Promise<SearchResult<T>[]>;
+    /**
+     * Cleanup distributed resources
+     * Should be called when shutting down the instance
+     */
+    cleanup(): Promise<void>;
 }
 export { euclideanDistance, cosineDistance, manhattanDistance, dotProductDistance } from './utils/index.js';

package/dist/coreTypes.d.ts

@@ -22,6 +22,23 @@ export interface SearchResult<T = any> {
     vector: Vector;
     metadata?: T;
 }
+/**
+ * Cursor for pagination through search results
+ */
+export interface SearchCursor {
+    lastId: string;
+    lastScore: number;
+    position: number;
+}
+/**
+ * Paginated search result with cursor support
+ */
+export interface PaginatedSearchResult<T = any> {
+    results: SearchResult<T>[];
+    cursor?: SearchCursor;
+    hasMore: boolean;
+    totalEstimate?: number;
+}
 /**
  * Distance function for comparing vectors
  */

package/dist/distributed/configManager.d.ts

@@ -0,0 +1,97 @@
+/**
+ * Distributed Configuration Manager
+ * Manages shared configuration in S3 for distributed Brainy instances
+ */
+import { DistributedConfig, SharedConfig, InstanceInfo, InstanceRole } from '../types/distributedTypes.js';
+import { StorageAdapter } from '../coreTypes.js';
+export declare class DistributedConfigManager {
+    private config;
+    private instanceId;
+    private role;
+    private configPath;
+    private heartbeatInterval;
+    private configCheckInterval;
+    private instanceTimeout;
+    private storage;
+    private heartbeatTimer?;
+    private configWatchTimer?;
+    private lastConfigVersion;
+    private onConfigUpdate?;
+    constructor(storage: StorageAdapter, distributedConfig?: DistributedConfig, brainyMode?: {
+        readOnly?: boolean;
+        writeOnly?: boolean;
+    });
+    /**
+     * Initialize the distributed configuration
+     */
+    initialize(): Promise<SharedConfig>;
+    /**
+     * Load existing config or create new one
+     */
+    private loadOrCreateConfig;
+    /**
+     * Determine role based on configuration
+     * IMPORTANT: Role must be explicitly set - no automatic assignment based on order
+     */
+    private determineRole;
+    /**
+     * Check if an instance is still alive
+     */
+    private isInstanceAlive;
+    /**
+     * Register this instance in the shared config
+     */
+    private registerInstance;
+    /**
+     * Save configuration with version increment
+     */
+    private saveConfig;
+    /**
+     * Start heartbeat to keep instance alive in config
+     */
+    private startHeartbeat;
+    /**
+     * Update heartbeat and clean stale instances
+     */
+    private updateHeartbeat;
+    /**
+     * Start watching for config changes
+     */
+    private startConfigWatch;
+    /**
+     * Check for configuration updates
+     */
+    private checkForConfigUpdates;
+    /**
+     * Load configuration from storage
+     */
+    private loadConfig;
+    /**
+     * Get current configuration
+     */
+    getConfig(): SharedConfig | null;
+    /**
+     * Get instance role
+     */
+    getRole(): InstanceRole;
+    /**
+     * Get instance ID
+     */
+    getInstanceId(): string;
+    /**
+     * Set config update callback
+     */
+    setOnConfigUpdate(callback: (config: SharedConfig) => void): void;
+    /**
+     * Get all active instances of a specific role
+     */
+    getInstancesByRole(role: InstanceRole): InstanceInfo[];
+    /**
+     * Update instance metrics
+     */
+    updateMetrics(metrics: Partial<InstanceInfo['metrics']>): Promise<void>;
+    /**
+     * Cleanup resources
+     */
+    cleanup(): Promise<void>;
+}

package/dist/distributed/domainDetector.d.ts

@@ -0,0 +1,77 @@
+/**
+ * Domain Detector
+ * Automatically detects and manages data domains for logical separation
+ */
+import { DomainMetadata } from '../types/distributedTypes.js';
+export interface DomainPattern {
+    domain: string;
+    patterns: {
+        fields?: string[];
+        keywords?: string[];
+        regex?: RegExp;
+    };
+    priority?: number;
+}
+export declare class DomainDetector {
+    private domainPatterns;
+    private customPatterns;
+    private domainStats;
+    /**
+     * Detect domain from data object
+     * @param data - The data object to analyze
+     * @returns The detected domain and metadata
+     */
+    detectDomain(data: any): DomainMetadata;
+    /**
+     * Score a data object against a domain pattern
+     */
+    private scorePattern;
+    /**
+     * Extract domain-specific metadata
+     */
+    private extractDomainMetadata;
+    /**
+     * Calculate detection confidence
+     */
+    private calculateConfidence;
+    /**
+     * Categorize price ranges
+     */
+    private getPriceRange;
+    /**
+     * Categorize customer value
+     */
+    private getValueCategory;
+    /**
+     * Categorize amount ranges
+     */
+    private getAmountRange;
+    /**
+     * Add custom domain pattern
+     * @param pattern - Custom domain pattern to add
+     */
+    addCustomPattern(pattern: DomainPattern): void;
+    /**
+     * Remove custom domain pattern
+     * @param domain - Domain to remove pattern for
+     */
+    removeCustomPattern(domain: string): void;
+    /**
+     * Update domain statistics
+     */
+    private updateStats;
+    /**
+     * Get domain statistics
+     * @returns Map of domain to count
+     */
+    getDomainStats(): Map<string, number>;
+    /**
+     * Clear domain statistics
+     */
+    clearStats(): void;
+    /**
+     * Get all configured domains
+     * @returns Array of domain names
+     */
+    getConfiguredDomains(): string[];
+}

package/dist/distributed/hashPartitioner.d.ts

@@ -0,0 +1,77 @@
+/**
+ * Hash-based Partitioner
+ * Provides deterministic partitioning for distributed writes
+ */
+import { SharedConfig } from '../types/distributedTypes.js';
+export declare class HashPartitioner {
+    private partitionCount;
+    private partitionPrefix;
+    constructor(config: SharedConfig);
+    /**
+     * Get partition for a given vector ID using deterministic hashing
+     * @param vectorId - The unique identifier of the vector
+     * @returns The partition path
+     */
+    getPartition(vectorId: string): string;
+    /**
+     * Get partition with domain metadata (domain stored as metadata, not in path)
+     * @param vectorId - The unique identifier of the vector
+     * @param domain - The domain identifier (for metadata only)
+     * @returns The partition path
+     */
+    getPartitionWithDomain(vectorId: string, domain?: string): string;
+    /**
+     * Get all partition paths
+     * @returns Array of all partition paths
+     */
+    getAllPartitions(): string[];
+    /**
+     * Get partition index from partition path
+     * @param partitionPath - The partition path
+     * @returns The partition index
+     */
+    getPartitionIndex(partitionPath: string): number;
+    /**
+     * Hash a string to a number for consistent partitioning
+     * @param str - The string to hash
+     * @returns A positive integer hash
+     */
+    private hashString;
+    /**
+     * Get partitions for batch operations
+     * Groups vector IDs by their target partition
+     * @param vectorIds - Array of vector IDs
+     * @returns Map of partition to vector IDs
+     */
+    getPartitionsForBatch(vectorIds: string[]): Map<string, string[]>;
+}
+/**
+ * Affinity-based Partitioner
+ * Extends HashPartitioner to prefer certain partitions for a writer
+ * while maintaining correctness
+ */
+export declare class AffinityPartitioner extends HashPartitioner {
+    private preferredPartitions;
+    private instanceId;
+    constructor(config: SharedConfig, instanceId: string);
+    /**
+     * Calculate preferred partitions for this instance
+     */
+    private calculatePreferredPartitions;
+    /**
+     * Check if a partition is preferred for this instance
+     * @param partitionPath - The partition path
+     * @returns Whether this partition is preferred
+     */
+    isPreferredPartition(partitionPath: string): boolean;
+    /**
+     * Get all preferred partitions for this instance
+     * @returns Array of preferred partition paths
+     */
+    getPreferredPartitions(): string[];
+    /**
+     * Update preferred partitions based on new config
+     * @param config - The updated shared configuration
+     */
+    updatePreferences(config: SharedConfig): void;
+}

package/dist/distributed/healthMonitor.d.ts

@@ -0,0 +1,110 @@
+/**
+ * Health Monitor
+ * Monitors and reports instance health in distributed deployments
+ */
+import { DistributedConfigManager } from './configManager.js';
+export interface HealthMetrics {
+    vectorCount: number;
+    cacheHitRate: number;
+    memoryUsage: number;
+    cpuUsage?: number;
+    requestsPerSecond?: number;
+    averageLatency?: number;
+    errorRate?: number;
+}
+export interface HealthStatus {
+    status: 'healthy' | 'degraded' | 'unhealthy';
+    instanceId: string;
+    role: string;
+    uptime: number;
+    lastCheck: string;
+    metrics: HealthMetrics;
+    warnings?: string[];
+    errors?: string[];
+}
+export declare class HealthMonitor {
+    private configManager;
+    private startTime;
+    private requestCount;
+    private errorCount;
+    private totalLatency;
+    private cacheHits;
+    private cacheMisses;
+    private vectorCount;
+    private checkInterval;
+    private healthCheckTimer?;
+    private metricsWindow;
+    private latencyWindow;
+    private windowSize;
+    constructor(configManager: DistributedConfigManager);
+    /**
+     * Start health monitoring
+     */
+    start(): void;
+    /**
+     * Stop health monitoring
+     */
+    stop(): void;
+    /**
+     * Update health status and metrics
+     */
+    private updateHealth;
+    /**
+     * Collect current metrics
+     */
+    private collectMetrics;
+    /**
+     * Calculate cache hit rate
+     */
+    private calculateCacheHitRate;
+    /**
+     * Calculate requests per second
+     */
+    private calculateRPS;
+    /**
+     * Calculate average latency
+     */
+    private calculateAverageLatency;
+    /**
+     * Calculate error rate
+     */
+    private calculateErrorRate;
+    /**
+     * Get CPU usage (simplified)
+     */
+    private getCPUUsage;
+    /**
+     * Clean old entries from sliding windows
+     */
+    private cleanWindows;
+    /**
+     * Record a request
+     * @param latency - Request latency in milliseconds
+     * @param error - Whether the request resulted in an error
+     */
+    recordRequest(latency: number, error?: boolean): void;
+    /**
+     * Record cache access
+     * @param hit - Whether it was a cache hit
+     */
+    recordCacheAccess(hit: boolean): void;
+    /**
+     * Update vector count
+     * @param count - New vector count
+     */
+    updateVectorCount(count: number): void;
+    /**
+     * Get current health status
+     * @returns Health status object
+     */
+    getHealthStatus(): HealthStatus;
+    /**
+     * Get health check endpoint data
+     * @returns JSON-serializable health data
+     */
+    getHealthEndpointData(): Record<string, any>;
+    /**
+     * Reset metrics (useful for testing)
+     */
+    resetMetrics(): void;
+}

package/dist/distributed/index.d.ts

@@ -0,0 +1,10 @@
+/**
+ * Distributed module exports
+ */
+export { DistributedConfigManager } from './configManager.js';
+export { HashPartitioner, AffinityPartitioner } from './hashPartitioner.js';
+export { BaseOperationalMode, ReaderMode, WriterMode, HybridMode, OperationalModeFactory } from './operationalModes.js';
+export { DomainDetector } from './domainDetector.js';
+export { HealthMonitor } from './healthMonitor.js';
+export type { HealthMetrics, HealthStatus } from './healthMonitor.js';
+export type { DomainPattern } from './domainDetector.js';

package/dist/distributed/operationalModes.d.ts

@@ -0,0 +1,104 @@
+/**
+ * Operational Modes for Distributed Brainy
+ * Defines different modes with optimized caching strategies
+ */
+import { OperationalMode, CacheStrategy, InstanceRole } from '../types/distributedTypes.js';
+/**
+ * Base operational mode
+ */
+export declare abstract class BaseOperationalMode implements OperationalMode {
+    abstract canRead: boolean;
+    abstract canWrite: boolean;
+    abstract canDelete: boolean;
+    abstract cacheStrategy: CacheStrategy;
+    /**
+     * Validate operation is allowed in this mode
+     */
+    validateOperation(operation: 'read' | 'write' | 'delete'): void;
+}
+/**
+ * Read-only mode optimized for query performance
+ */
+export declare class ReaderMode extends BaseOperationalMode {
+    canRead: boolean;
+    canWrite: boolean;
+    canDelete: boolean;
+    cacheStrategy: CacheStrategy;
+    /**
+     * Get optimized cache configuration for readers
+     */
+    getCacheConfig(): {
+        hotCacheMaxSize: number;
+        hotCacheEvictionThreshold: number;
+        warmCacheTTL: number;
+        batchSize: number;
+        autoTune: boolean;
+        autoTuneInterval: number;
+        readOnly: boolean;
+    };
+}
+/**
+ * Write-only mode optimized for ingestion
+ */
+export declare class WriterMode extends BaseOperationalMode {
+    canRead: boolean;
+    canWrite: boolean;
+    canDelete: boolean;
+    cacheStrategy: CacheStrategy;
+    /**
+     * Get optimized cache configuration for writers
+     */
+    getCacheConfig(): {
+        hotCacheMaxSize: number;
+        hotCacheEvictionThreshold: number;
+        warmCacheTTL: number;
+        batchSize: number;
+        autoTune: boolean;
+        writeOnly: boolean;
+    };
+}
+/**
+ * Hybrid mode that can both read and write
+ */
+export declare class HybridMode extends BaseOperationalMode {
+    canRead: boolean;
+    canWrite: boolean;
+    canDelete: boolean;
+    cacheStrategy: CacheStrategy;
+    private readWriteRatio;
+    /**
+     * Get balanced cache configuration
+     */
+    getCacheConfig(): {
+        hotCacheMaxSize: number;
+        hotCacheEvictionThreshold: number;
+        warmCacheTTL: number;
+        batchSize: number;
+        autoTune: boolean;
+        autoTuneInterval: number;
+    };
+    /**
+     * Update cache strategy based on workload
+     * @param readCount - Number of recent reads
+     * @param writeCount - Number of recent writes
+     */
+    updateWorkloadBalance(readCount: number, writeCount: number): void;
+}
+/**
+ * Factory for creating operational modes
+ */
+export declare class OperationalModeFactory {
+    /**
+     * Create operational mode based on role
+     * @param role - The instance role
+     * @returns The appropriate operational mode
+     */
+    static createMode(role: InstanceRole): BaseOperationalMode;
+    /**
+     * Create mode with custom cache strategy
+     * @param role - The instance role
+     * @param customStrategy - Custom cache strategy overrides
+     * @returns The operational mode with custom strategy
+     */
+    static createModeWithStrategy(role: InstanceRole, customStrategy: Partial<CacheStrategy>): BaseOperationalMode;
+}