@soulcraft/brainy 0.36.0 → 0.38.0

package/dist/hnsw/optimizedHNSWIndex.d.ts

@@ -0,0 +1,97 @@
+/**
+ * Optimized HNSW Index for Large-Scale Vector Search
+ * Implements dynamic parameter tuning and performance optimizations
+ */
+import { DistanceFunction, HNSWConfig, Vector, VectorDocument } from '../coreTypes.js';
+import { HNSWIndex } from './hnswIndex.js';
+export interface OptimizedHNSWConfig extends HNSWConfig {
+    dynamicParameterTuning?: boolean;
+    targetSearchLatency?: number;
+    targetRecall?: number;
+    maxNodes?: number;
+    memoryBudget?: number;
+    diskCacheEnabled?: boolean;
+    compressionEnabled?: boolean;
+    performanceTracking?: boolean;
+    adaptiveEfSearch?: boolean;
+    levelMultiplier?: number;
+    seedConnections?: number;
+    pruningStrategy?: 'simple' | 'diverse' | 'hybrid';
+}
+interface PerformanceMetrics {
+    averageSearchTime: number;
+    averageRecall: number;
+    memoryUsage: number;
+    indexSize: number;
+    apiCalls: number;
+    cacheHitRate: number;
+}
+interface DynamicParameters {
+    efSearch: number;
+    efConstruction: number;
+    M: number;
+    ml: number;
+}
+/**
+ * Optimized HNSW Index with dynamic parameter tuning for large datasets
+ */
+export declare class OptimizedHNSWIndex extends HNSWIndex {
+    private optimizedConfig;
+    private performanceMetrics;
+    private dynamicParams;
+    private searchHistory;
+    private parameterTuningInterval?;
+    constructor(config?: Partial<OptimizedHNSWConfig>, distanceFunction?: DistanceFunction);
+    /**
+     * Optimized search with dynamic parameter adjustment
+     */
+    search(queryVector: Vector, k?: number): Promise<Array<[string, number]>>;
+    /**
+     * Dynamically adjust efSearch based on performance requirements
+     */
+    private adjustEfSearch;
+    /**
+     * Record search performance metrics
+     */
+    private recordSearchMetrics;
+    /**
+     * Check memory usage and trigger optimizations
+     */
+    private checkMemoryUsage;
+    /**
+     * Compress index to reduce memory usage (placeholder)
+     */
+    private compressIndex;
+    /**
+     * Start automatic parameter tuning
+     */
+    private startParameterTuning;
+    /**
+     * Automatic parameter tuning based on performance metrics
+     */
+    private tuneParameters;
+    /**
+     * Get optimized configuration recommendations for current dataset size
+     */
+    getOptimizedConfig(): OptimizedHNSWConfig;
+    /**
+     * Get current performance metrics
+     */
+    getPerformanceMetrics(): PerformanceMetrics & {
+        currentParams: DynamicParameters;
+        searchHistorySize: number;
+    };
+    /**
+     * Apply optimized bulk insertion strategy
+     */
+    bulkInsert(items: VectorDocument[]): Promise<string[]>;
+    /**
+     * Optimize insertion order to improve index quality
+     */
+    private optimizeInsertionOrder;
+    /**
+     * Cleanup resources
+     */
+    destroy(): void;
+}
+export {};

package/dist/hnsw/optimizedHNSWIndex.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"optimizedHNSWIndex.d.ts","sourceRoot":"","sources":["../../src/hnsw/optimizedHNSWIndex.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EACL,gBAAgB,EAChB,UAAU,EAEV,MAAM,EACN,cAAc,EACf,MAAM,iBAAiB,CAAA;AACxB,OAAO,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAA;AAG1C,MAAM,WAAW,mBAAoB,SAAQ,UAAU;IAErD,sBAAsB,CAAC,EAAE,OAAO,CAAA;IAChC,mBAAmB,CAAC,EAAE,MAAM,CAAA;IAC5B,YAAY,CAAC,EAAE,MAAM,CAAA;IAGrB,QAAQ,CAAC,EAAE,MAAM,CAAA;IACjB,YAAY,CAAC,EAAE,MAAM,CAAA;IACrB,gBAAgB,CAAC,EAAE,OAAO,CAAA;IAC1B,kBAAkB,CAAC,EAAE,OAAO,CAAA;IAG5B,mBAAmB,CAAC,EAAE,OAAO,CAAA;IAC7B,gBAAgB,CAAC,EAAE,OAAO,CAAA;IAG1B,eAAe,CAAC,EAAE,MAAM,CAAA;IACxB,eAAe,CAAC,EAAE,MAAM,CAAA;IACxB,eAAe,CAAC,EAAE,QAAQ,GAAG,SAAS,GAAG,QAAQ,CAAA;CAClD;AAED,UAAU,kBAAkB;IAC1B,iBAAiB,EAAE,MAAM,CAAA;IACzB,aAAa,EAAE,MAAM,CAAA;IACrB,WAAW,EAAE,MAAM,CAAA;IACnB,SAAS,EAAE,MAAM,CAAA;IACjB,QAAQ,EAAE,MAAM,CAAA;IAChB,YAAY,EAAE,MAAM,CAAA;CACrB;AAED,UAAU,iBAAiB;IACzB,QAAQ,EAAE,MAAM,CAAA;IAChB,cAAc,EAAE,MAAM,CAAA;IACtB,CAAC,EAAE,MAAM,CAAA;IACT,EAAE,EAAE,MAAM,CAAA;CACX;AAED;;GAEG;AACH,qBAAa,kBAAmB,SAAQ,SAAS;IAC/C,OAAO,CAAC,eAAe,CAA+B;IACtD,OAAO,CAAC,kBAAkB,CAAoB;IAC9C,OAAO,CAAC,aAAa,CAAmB;IACxC,OAAO,CAAC,aAAa,CAA+D;IACpF,OAAO,CAAC,uBAAuB,CAAC,CAAgB;gBAG9C,MAAM,GAAE,OAAO,CAAC,mBAAmB,CAAM,EACzC,gBAAgB,GAAE,gBAAoC;IA+DxD;;OAEG;IACU,MAAM,CACjB,WAAW,EAAE,MAAM,EACnB,CAAC,GAAE,MAAW,GACb,OAAO,CAAC,KAAK,CAAC,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC;IAyCnC;;OAEG;IACH,OAAO,CAAC,cAAc;IA+BtB;;OAEG;IACH,OAAO,CAAC,mBAAmB;IA0B3B;;OAEG;IACH,OAAO,CAAC,gBAAgB;IAexB;;OAEG;IACH,OAAO,CAAC,aAAa;IAKrB;;OAEG;IACH,OAAO,CAAC,oBAAoB;IAM5B;;OAEG;IACH,OAAO,CAAC,cAAc;IA0BtB;;OAEG;IACI,kBAAkB,IAAI,mBAAmB;IA6ChD;;OAEG;IACI,qBAAqB,IAAI,kBAAkB,GAAG;QACnD,aAAa,EAAE,iBAAiB,CAAA;QAChC,iBAAiB,EAAE,MAAM,CAAA;KAC1B;IAQD;;OAEG;IACU,UAAU,CAAC,KAAK,EAAE,cAAc,EAAE,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;IAwCnE;;OAEG;IACH,OAAO,CAAC,sBAAsB;IAU9B;;OAEG;IACI,OAAO,IAAI,IAAI;CAKvB"}

package/dist/hnsw/partitionedHNSWIndex.d.ts

@@ -0,0 +1,101 @@
+/**
+ * Partitioned HNSW Index for Large-Scale Vector Search
+ * Implements sharding strategies to handle millions of vectors efficiently
+ */
+import { DistanceFunction, HNSWConfig, Vector, VectorDocument } from '../coreTypes.js';
+export interface PartitionConfig {
+    maxNodesPerPartition: number;
+    partitionStrategy: 'semantic' | 'hash';
+    semanticClusters?: number;
+    autoTuneSemanticClusters?: boolean;
+}
+export interface PartitionMetadata {
+    id: string;
+    nodeCount: number;
+    bounds?: {
+        centroid: Vector;
+        radius: number;
+    };
+    strategy: string;
+    created: Date;
+}
+/**
+ * Partitioned HNSW Index that splits large datasets across multiple smaller indices
+ * This enables efficient search across millions of vectors by reducing memory usage
+ * and parallelizing search operations
+ */
+export declare class PartitionedHNSWIndex {
+    private partitions;
+    private partitionMetadata;
+    private config;
+    private hnswConfig;
+    private distanceFunction;
+    private dimension;
+    private nextPartitionId;
+    constructor(partitionConfig?: Partial<PartitionConfig>, hnswConfig?: Partial<HNSWConfig>, distanceFunction?: DistanceFunction);
+    /**
+     * Add a vector to the partitioned index
+     */
+    addItem(item: VectorDocument): Promise<string>;
+    /**
+     * Search across all partitions for nearest neighbors
+     */
+    search(queryVector: Vector, k?: number, searchScope?: {
+        partitionIds?: string[];
+        maxPartitions?: number;
+    }): Promise<Array<[string, number]>>;
+    /**
+     * Select the appropriate partition for a new item
+     * Automatically chooses semantic partitioning when beneficial, falls back to hash
+     */
+    private selectPartition;
+    /**
+     * Hash-based partitioning for even distribution
+     */
+    private hashPartition;
+    /**
+     * Semantic clustering partitioning
+     */
+    private semanticPartition;
+    /**
+     * Auto-tune semantic clusters based on dataset size and performance
+     */
+    private autoTuneSemanticClusters;
+    /**
+     * Select which partitions to search based on query
+     */
+    private selectSearchPartitions;
+    /**
+     * Update partition bounds for semantic clustering
+     */
+    private updatePartitionBounds;
+    /**
+     * Split an overgrown partition into smaller partitions
+     */
+    private splitPartition;
+    /**
+     * Simple hash function for consistent partitioning
+     */
+    private simpleHash;
+    /**
+     * Get partition statistics
+     */
+    getPartitionStats(): {
+        totalPartitions: number;
+        totalNodes: number;
+        averageNodesPerPartition: number;
+        partitionDetails: PartitionMetadata[];
+    };
+    /**
+     * Remove an item from the index
+     */
+    removeItem(id: string): Promise<boolean>;
+    /**
+     * Clear all partitions
+     */
+    clear(): void;
+    /**
+     * Get total size across all partitions
+     */
+    size(): number;
+}

package/dist/hnsw/partitionedHNSWIndex.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"partitionedHNSWIndex.d.ts","sourceRoot":"","sources":["../../src/hnsw/partitionedHNSWIndex.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EACL,gBAAgB,EAChB,UAAU,EAEV,MAAM,EACN,cAAc,EACf,MAAM,iBAAiB,CAAA;AAIxB,MAAM,WAAW,eAAe;IAC9B,oBAAoB,EAAE,MAAM,CAAA;IAC5B,iBAAiB,EAAE,UAAU,GAAG,MAAM,CAAA;IACtC,gBAAgB,CAAC,EAAE,MAAM,CAAA;IACzB,wBAAwB,CAAC,EAAE,OAAO,CAAA;CACnC;AAED,MAAM,WAAW,iBAAiB;IAChC,EAAE,EAAE,MAAM,CAAA;IACV,SAAS,EAAE,MAAM,CAAA;IACjB,MAAM,CAAC,EAAE;QACP,QAAQ,EAAE,MAAM,CAAA;QAChB,MAAM,EAAE,MAAM,CAAA;KACf,CAAA;IACD,QAAQ,EAAE,MAAM,CAAA;IAChB,OAAO,EAAE,IAAI,CAAA;CACd;AAED;;;;GAIG;AACH,qBAAa,oBAAoB;IAC/B,OAAO,CAAC,UAAU,CAAoC;IACtD,OAAO,CAAC,iBAAiB,CAA4C;IACrE,OAAO,CAAC,MAAM,CAAiB;IAC/B,OAAO,CAAC,UAAU,CAAY;IAC9B,OAAO,CAAC,gBAAgB,CAAkB;IAC1C,OAAO,CAAC,SAAS,CAAsB;IACvC,OAAO,CAAC,eAAe,CAAI;gBAGzB,eAAe,GAAE,OAAO,CAAC,eAAe,CAAM,EAC9C,UAAU,GAAE,OAAO,CAAC,UAAU,CAAM,EACpC,gBAAgB,GAAE,gBAAoC;IAsBxD;;OAEG;IACU,OAAO,CAAC,IAAI,EAAE,cAAc,GAAG,OAAO,CAAC,MAAM,CAAC;IA+C3D;;OAEG;IACU,MAAM,CACjB,WAAW,EAAE,MAAM,EACnB,CAAC,GAAE,MAAW,EACd,WAAW,CAAC,EAAE;QACZ,YAAY,CAAC,EAAE,MAAM,EAAE,CAAA;QACvB,aAAa,CAAC,EAAE,MAAM,CAAA;KACvB,GACA,OAAO,CAAC,KAAK,CAAC,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC;IA+BnC;;;OAGG;YACW,eAAe;IAgB7B;;OAEG;IACH,OAAO,CAAC,aAAa;IAgBrB;;OAEG;YACW,iBAAiB;IAwB/B;;OAEG;IACH,OAAO,CAAC,wBAAwB;IA0BhC;;OAEG;YACW,sBAAsB;IAwCpC;;OAEG;IACH,OAAO,CAAC,qBAAqB;IAwB7B;;OAEG;YACW,cAAc;IAc5B;;OAEG;IACH,OAAO,CAAC,UAAU;IAUlB;;OAEG;IACI,iBAAiB,IAAI;QAC1B,eAAe,EAAE,MAAM,CAAA;QACvB,UAAU,EAAE,MAAM,CAAA;QAClB,wBAAwB,EAAE,MAAM,CAAA;QAChC,gBAAgB,EAAE,iBAAiB,EAAE,CAAA;KACtC;IAYD;;OAEG;IACU,UAAU,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC;IAarD;;OAEG;IACI,KAAK,IAAI,IAAI;IASpB;;OAEG;IACI,IAAI,IAAI,MAAM;CAGtB"}

package/dist/hnsw/scaledHNSWSystem.d.ts

@@ -0,0 +1,142 @@
+/**
+ * Scaled HNSW System - Integration of All Optimization Strategies
+ * Production-ready system for handling millions of vectors with sub-second search
+ */
+import { Vector, VectorDocument } from '../coreTypes.js';
+import { PartitionConfig } from './partitionedHNSWIndex.js';
+import { OptimizedHNSWConfig } from './optimizedHNSWIndex.js';
+import { SearchStrategy } from './distributedSearch.js';
+export interface ScaledHNSWConfig {
+    expectedDatasetSize?: number;
+    maxMemoryUsage?: number;
+    targetSearchLatency?: number;
+    s3Config?: {
+        bucketName: string;
+        region: string;
+        endpoint?: string;
+        accessKeyId?: string;
+        secretAccessKey?: string;
+    };
+    autoConfigureEnvironment?: boolean;
+    learningEnabled?: boolean;
+    enablePartitioning?: boolean;
+    enableCompression?: boolean;
+    enableDistributedSearch?: boolean;
+    enablePredictiveCaching?: boolean;
+    partitionConfig?: Partial<PartitionConfig>;
+    hnswConfig?: Partial<OptimizedHNSWConfig>;
+    readOnlyMode?: boolean;
+}
+/**
+ * High-performance HNSW system with all optimizations integrated
+ * Handles datasets from thousands to millions of vectors
+ */
+export declare class ScaledHNSWSystem {
+    private config;
+    private autoConfig;
+    private partitionedIndex?;
+    private distributedSearch?;
+    private cacheManager?;
+    private batchOperations?;
+    private readOnlyOptimizations?;
+    private performanceMetrics;
+    constructor(config?: ScaledHNSWConfig);
+    /**
+     * Initialize the optimized system based on configuration
+     */
+    private initializeOptimizedSystem;
+    /**
+     * Calculate optimal configuration based on dataset size and constraints
+     */
+    private calculateOptimalConfiguration;
+    /**
+     * Add vector to the scaled system
+     */
+    addVector(item: VectorDocument): Promise<string>;
+    /**
+     * Bulk insert vectors with optimizations
+     */
+    bulkInsert(items: VectorDocument[]): Promise<string[]>;
+    /**
+     * High-performance vector search with all optimizations
+     */
+    search(queryVector: Vector, k?: number, options?: {
+        strategy?: SearchStrategy;
+        useCache?: boolean;
+        maxPartitions?: number;
+    }): Promise<Array<[string, number]>>;
+    /**
+     * Get system performance metrics
+     */
+    getPerformanceMetrics(): typeof this.performanceMetrics & {
+        partitionStats?: any;
+        cacheStats?: any;
+        compressionStats?: any;
+        distributedSearchStats?: any;
+    };
+    /**
+     * Optimize insertion order for better index quality
+     */
+    private optimizeInsertionOrder;
+    /**
+     * Calculate optimal batch size based on system resources
+     */
+    private calculateOptimalBatchSize;
+    /**
+     * Update search performance metrics
+     */
+    private updateSearchMetrics;
+    /**
+     * Estimate current memory usage
+     */
+    private estimateMemoryUsage;
+    /**
+     * Generate performance report
+     */
+    generatePerformanceReport(): string;
+    /**
+     * Get overall system status
+     */
+    private getSystemStatus;
+    /**
+     * Check if adaptive learning should be triggered
+     */
+    private shouldTriggerLearning;
+    /**
+     * Adaptively learn from performance and adjust configuration
+     */
+    private adaptivelyLearnFromPerformance;
+    /**
+     * Update dataset analysis for better auto-configuration
+     */
+    updateDatasetAnalysis(vectorCount: number, vectorDimension?: number): Promise<void>;
+    /**
+     * Infer access patterns from current metrics
+     */
+    private inferAccessPatterns;
+    /**
+     * Cleanup system resources
+     */
+    cleanup(): void;
+}
+/**
+ * Create a fully auto-configured Brainy system - minimal setup required!
+ * Just provide S3 config if you want persistence beyond the current session
+ */
+export declare function createAutoBrainy(s3Config?: {
+    bucketName: string;
+    region?: string;
+    accessKeyId?: string;
+    secretAccessKey?: string;
+}): ScaledHNSWSystem;
+/**
+ * Create a Brainy system optimized for specific scenarios
+ */
+export declare function createQuickBrainy(scenario: 'small' | 'medium' | 'large' | 'enterprise', s3Config?: {
+    bucketName: string;
+    region?: string;
+}): Promise<ScaledHNSWSystem>;
+/**
+ * Legacy factory function - still works but consider using createAutoBrainy() instead
+ */
+export declare function createScaledHNSWSystem(config?: ScaledHNSWConfig): ScaledHNSWSystem;

package/dist/hnsw/scaledHNSWSystem.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"scaledHNSWSystem.d.ts","sourceRoot":"","sources":["../../src/hnsw/scaledHNSWSystem.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAE,MAAM,EAAE,cAAc,EAAc,MAAM,iBAAiB,CAAA;AACpE,OAAO,EAAwB,eAAe,EAAE,MAAM,2BAA2B,CAAA;AACjF,OAAO,EAAsB,mBAAmB,EAAE,MAAM,yBAAyB,CAAA;AACjF,OAAO,EAA2B,cAAc,EAAE,MAAM,wBAAwB,CAAA;AAOhF,MAAM,WAAW,gBAAgB;IAE/B,mBAAmB,CAAC,EAAE,MAAM,CAAA;IAC5B,cAAc,CAAC,EAAE,MAAM,CAAA;IACvB,mBAAmB,CAAC,EAAE,MAAM,CAAA;IAG5B,QAAQ,CAAC,EAAE;QACT,UAAU,EAAE,MAAM,CAAA;QAClB,MAAM,EAAE,MAAM,CAAA;QACd,QAAQ,CAAC,EAAE,MAAM,CAAA;QACjB,WAAW,CAAC,EAAE,MAAM,CAAA;QACpB,eAAe,CAAC,EAAE,MAAM,CAAA;KACzB,CAAA;IAGD,wBAAwB,CAAC,EAAE,OAAO,CAAA;IAClC,eAAe,CAAC,EAAE,OAAO,CAAA;IAGzB,kBAAkB,CAAC,EAAE,OAAO,CAAA;IAC5B,iBAAiB,CAAC,EAAE,OAAO,CAAA;IAC3B,uBAAuB,CAAC,EAAE,OAAO,CAAA;IACjC,uBAAuB,CAAC,EAAE,OAAO,CAAA;IAGjC,eAAe,CAAC,EAAE,OAAO,CAAC,eAAe,CAAC,CAAA;IAC1C,UAAU,CAAC,EAAE,OAAO,CAAC,mBAAmB,CAAC,CAAA;IACzC,YAAY,CAAC,EAAE,OAAO,CAAA;CACvB;AAED;;;GAGG;AACH,qBAAa,gBAAgB;IAC3B,OAAO,CAAC,MAAM,CAWb;IACD,OAAO,CAAC,UAAU,CAAmB;IACrC,OAAO,CAAC,gBAAgB,CAAC,CAAsB;IAC/C,OAAO,CAAC,iBAAiB,CAAC,CAAyB;IACnD,OAAO,CAAC,YAAY,CAAC,CAA2B;IAChD,OAAO,CAAC,eAAe,CAAC,CAAmB;IAC3C,OAAO,CAAC,qBAAqB,CAAC,CAAuB;IAGrD,OAAO,CAAC,kBAAkB,CAQzB;gBAEW,MAAM,GAAE,gBAAqB;IAqBzC;;OAEG;YACW,yBAAyB;IAwGvC;;OAEG;IACH,OAAO,CAAC,6BAA6B;IAwGrC;;OAEG;IACU,SAAS,CAAC,IAAI,EAAE,cAAc,GAAG,OAAO,CAAC,MAAM,CAAC;IAc7D;;OAEG;IACU,UAAU,CAAC,KAAK,EAAE,cAAc,EAAE,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;IAoCnE;;OAEG;IACU,MAAM,CACjB,WAAW,EAAE,MAAM,EACnB,CAAC,GAAE,MAAW,EACd,OAAO,GAAE;QACP,QAAQ,CAAC,EAAE,cAAc,CAAA;QACzB,QAAQ,CAAC,EAAE,OAAO,CAAA;QAClB,aAAa,CAAC,EAAE,MAAM,CAAA;KAClB,GACL,OAAO,CAAC,KAAK,CAAC,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC;IA0CnC;;OAEG;IACI,qBAAqB,IAAI,OAAO,IAAI,CAAC,kBAAkB,GAAG;QAC/D,cAAc,CAAC,EAAE,GAAG,CAAA;QACpB,UAAU,CAAC,EAAE,GAAG,CAAA;QAChB,gBAAgB,CAAC,EAAE,GAAG,CAAA;QACtB,sBAAsB,CAAC,EAAE,GAAG,CAAA;KAC7B;IAuBD;;OAEG;IACH,OAAO,CAAC,sBAAsB;IAU9B;;OAEG;IACH,OAAO,CAAC,yBAAyB;IAUjC;;OAEG;IACH,OAAO,CAAC,mBAAmB;IAwB3B;;OAEG;IACH,OAAO,CAAC,mBAAmB;IAgB3B;;OAEG;IACI,yBAAyB,IAAI,MAAM;IAuB1C;;OAEG;IACH,OAAO,CAAC,eAAe;IAYvB;;OAEG;IACH,OAAO,CAAC,qBAAqB;IAU7B;;OAEG;YACW,8BAA8B;IAuD5C;;OAEG;IACU,qBAAqB,CAAC,WAAW,EAAE,MAAM,EAAE,eAAe,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAahG;;OAEG;IACH,OAAO,CAAC,mBAAmB;IAQ3B;;OAEG;IACI,OAAO,IAAI,IAAI;CASvB;AAID;;;GAGG;AACH,wBAAgB,gBAAgB,CAAC,QAAQ,CAAC,EAAE;IAC1C,UAAU,EAAE,MAAM,CAAA;IAClB,MAAM,CAAC,EAAE,MAAM,CAAA;IACf,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB,eAAe,CAAC,EAAE,MAAM,CAAA;CACzB,GAAG,gBAAgB,CAWnB;AAED;;GAEG;AACH,wBAAsB,iBAAiB,CACrC,QAAQ,EAAE,OAAO,GAAG,QAAQ,GAAG,OAAO,GAAG,YAAY,EACrD,QAAQ,CAAC,EAAE;IAAE,UAAU,EAAE,MAAM,CAAC;IAAC,MAAM,CAAC,EAAE,MAAM,CAAA;CAAE,GACjD,OAAO,CAAC,gBAAgB,CAAC,CAe3B;AAED;;GAEG;AACH,wBAAgB,sBAAsB,CAAC,MAAM,GAAE,gBAAqB,GAAG,gBAAgB,CAEtF"}

package/dist/storage/adapters/batchS3Operations.d.ts

@@ -0,0 +1,71 @@
+/**
+ * Enhanced Batch S3 Operations for High-Performance Vector Retrieval
+ * Implements optimized batch operations to reduce S3 API calls and latency
+ */
+import { HNSWNoun } from '../../coreTypes.js';
+type S3Client = any;
+export interface BatchRetrievalOptions {
+    maxConcurrency?: number;
+    prefetchSize?: number;
+    useS3Select?: boolean;
+    compressionEnabled?: boolean;
+}
+export interface BatchResult<T> {
+    items: Map<string, T>;
+    errors: Map<string, Error>;
+    statistics: {
+        totalRequested: number;
+        totalRetrieved: number;
+        totalErrors: number;
+        duration: number;
+        apiCalls: number;
+    };
+}
+/**
+ * High-performance batch operations for S3-compatible storage
+ * Optimizes retrieval patterns for HNSW search operations
+ */
+export declare class BatchS3Operations {
+    private s3Client;
+    private bucketName;
+    private options;
+    constructor(s3Client: S3Client, bucketName: string, options?: BatchRetrievalOptions);
+    /**
+     * Batch retrieve HNSW nodes with intelligent prefetching
+     */
+    batchGetNodes(nodeIds: string[], prefix?: string): Promise<BatchResult<HNSWNoun>>;
+    /**
+     * Parallel GetObject operations for small batches
+     */
+    private parallelGetObjects;
+    /**
+     * Chunked parallel retrieval with intelligent batching
+     */
+    private chunkedParallelGet;
+    /**
+     * List-based batch retrieval for large datasets
+     * Uses S3 ListObjects to reduce API calls
+     */
+    private listBasedBatchGet;
+    /**
+     * Intelligent prefetch based on HNSW graph connectivity
+     */
+    prefetchConnectedNodes(currentNodeIds: string[], connectionMap: Map<string, Set<string>>, prefix?: string): Promise<BatchResult<HNSWNoun>>;
+    /**
+     * S3 Select-based retrieval for filtered queries
+     */
+    selectiveRetrieve(prefix: string, filter: {
+        vectorDimension?: number;
+        metadataKey?: string;
+        metadataValue?: any;
+    }): Promise<BatchResult<HNSWNoun>>;
+    /**
+     * Parse stored object from JSON string
+     */
+    private parseStoredObject;
+    /**
+     * Utility function to chunk arrays
+     */
+    private chunkArray;
+}
+export {};

package/dist/storage/adapters/batchS3Operations.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"batchS3Operations.d.ts","sourceRoot":"","sources":["../../../src/storage/adapters/batchS3Operations.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAE,QAAQ,EAAY,MAAM,oBAAoB,CAAA;AAGvD,KAAK,QAAQ,GAAG,GAAG,CAAA;AAInB,MAAM,WAAW,qBAAqB;IACpC,cAAc,CAAC,EAAE,MAAM,CAAA;IACvB,YAAY,CAAC,EAAE,MAAM,CAAA;IACrB,WAAW,CAAC,EAAE,OAAO,CAAA;IACrB,kBAAkB,CAAC,EAAE,OAAO,CAAA;CAC7B;AAED,MAAM,WAAW,WAAW,CAAC,CAAC;IAC5B,KAAK,EAAE,GAAG,CAAC,MAAM,EAAE,CAAC,CAAC,CAAA;IACrB,MAAM,EAAE,GAAG,CAAC,MAAM,EAAE,KAAK,CAAC,CAAA;IAC1B,UAAU,EAAE;QACV,cAAc,EAAE,MAAM,CAAA;QACtB,cAAc,EAAE,MAAM,CAAA;QACtB,WAAW,EAAE,MAAM,CAAA;QACnB,QAAQ,EAAE,MAAM,CAAA;QAChB,QAAQ,EAAE,MAAM,CAAA;KACjB,CAAA;CACF;AAED;;;GAGG;AACH,qBAAa,iBAAiB;IAC5B,OAAO,CAAC,QAAQ,CAAU;IAC1B,OAAO,CAAC,UAAU,CAAQ;IAC1B,OAAO,CAAC,OAAO,CAAuB;gBAGpC,QAAQ,EAAE,QAAQ,EAClB,UAAU,EAAE,MAAM,EAClB,OAAO,GAAE,qBAA0B;IAarC;;OAEG;IACU,aAAa,CACxB,OAAO,EAAE,MAAM,EAAE,EACjB,MAAM,GAAE,MAAiB,GACxB,OAAO,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC;IAmCjC;;OAEG;YACW,kBAAkB;IAwChC;;OAEG;YACW,kBAAkB;IAuBhC;;;OAGG;YACW,iBAAiB;IA4E/B;;OAEG;IACU,sBAAsB,CACjC,cAAc,EAAE,MAAM,EAAE,EACxB,aAAa,EAAE,GAAG,CAAC,MAAM,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC,EACvC,MAAM,GAAE,MAAiB,GACxB,OAAO,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC;IAmCjC;;OAEG;IACU,iBAAiB,CAC5B,MAAM,EAAE,MAAM,EACd,MAAM,EAAE;QACN,eAAe,CAAC,EAAE,MAAM,CAAA;QACxB,WAAW,CAAC,EAAE,MAAM,CAAA;QACpB,aAAa,CAAC,EAAE,GAAG,CAAA;KACpB,GACA,OAAO,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC;IAyBjC;;OAEG;IACH,OAAO,CAAC,iBAAiB;IAoBzB;;OAEG;IACH,OAAO,CAAC,UAAU;CAOnB"}

package/dist/storage/enhancedCacheManager.d.ts

@@ -0,0 +1,141 @@
+/**
+ * Enhanced Multi-Level Cache Manager with Predictive Prefetching
+ * Optimized for HNSW search patterns and large-scale vector operations
+ */
+import { HNSWNoun, HNSWVerb } from '../coreTypes.js';
+import { BatchS3Operations } from './adapters/batchS3Operations.js';
+declare enum PrefetchStrategy {
+    GRAPH_CONNECTIVITY = "connectivity",
+    VECTOR_SIMILARITY = "similarity",
+    ACCESS_PATTERN = "pattern",
+    HYBRID = "hybrid"
+}
+interface EnhancedCacheConfig {
+    hotCacheMaxSize?: number;
+    hotCacheEvictionThreshold?: number;
+    warmCacheMaxSize?: number;
+    warmCacheTTL?: number;
+    prefetchEnabled?: boolean;
+    prefetchStrategy?: PrefetchStrategy;
+    prefetchBatchSize?: number;
+    predictionLookahead?: number;
+    similarityThreshold?: number;
+    maxSimilarityDistance?: number;
+    backgroundOptimization?: boolean;
+    statisticsCollection?: boolean;
+}
+/**
+ * Enhanced cache manager with intelligent prefetching for HNSW operations
+ * Provides multi-level caching optimized for vector search workloads
+ */
+export declare class EnhancedCacheManager<T extends HNSWNoun | HNSWVerb> {
+    private hotCache;
+    private warmCache;
+    private prefetchQueue;
+    private accessPatterns;
+    private vectorIndex;
+    private config;
+    private batchOperations?;
+    private storageAdapter?;
+    private prefetchInProgress;
+    private stats;
+    constructor(config?: EnhancedCacheConfig);
+    /**
+     * Set storage adapters for warm/cold storage operations
+     */
+    setStorageAdapters(storageAdapter: any, batchOperations?: BatchS3Operations): void;
+    /**
+     * Get item with intelligent prefetching
+     */
+    get(id: string): Promise<T | null>;
+    /**
+     * Get multiple items efficiently with batch operations
+     */
+    getMany(ids: string[]): Promise<Map<string, T>>;
+    /**
+     * Set item in cache with metadata
+     */
+    set(id: string, item: T): Promise<void>;
+    /**
+     * Intelligent prefetch based on access patterns and graph structure
+     */
+    private schedulePrefetch;
+    /**
+     * Predict next nodes based on graph connectivity
+     */
+    private predictByConnectivity;
+    /**
+     * Predict next nodes based on vector similarity
+     */
+    private predictBySimilarity;
+    /**
+     * Predict based on historical access patterns
+     */
+    private predictByAccessPattern;
+    /**
+     * Hybrid prediction combining multiple strategies
+     */
+    private hybridPrediction;
+    /**
+     * Execute prefetch operation in background
+     */
+    private executePrefetch;
+    /**
+     * Load item from storage adapter
+     */
+    private loadFromStorage;
+    /**
+     * Promote frequently accessed item to hot cache
+     */
+    private promoteToHotCache;
+    /**
+     * Evict least recently used items from hot cache
+     */
+    private evictFromHotCache;
+    /**
+     * Evict expired items from warm cache
+     */
+    private evictFromWarmCache;
+    /**
+     * Record access pattern for prediction
+     */
+    private recordAccess;
+    /**
+     * Extract connected node IDs from HNSW item
+     */
+    private extractConnectedNodes;
+    /**
+     * Check if cache entry is expired
+     */
+    private isExpired;
+    /**
+     * Calculate cosine similarity between vectors
+     */
+    private cosineSimilarity;
+    /**
+     * Calculate pattern similarity between access patterns
+     */
+    private patternSimilarity;
+    /**
+     * Start background optimization process
+     */
+    private startBackgroundOptimization;
+    /**
+     * Run background optimization tasks
+     */
+    private runBackgroundOptimization;
+    /**
+     * Get cache statistics
+     */
+    getStats(): typeof this.stats & {
+        hotCacheSize: number;
+        warmCacheSize: number;
+        prefetchQueueSize: number;
+        accessPatternsTracked: number;
+    };
+    /**
+     * Clear all caches
+     */
+    clear(): void;
+}
+export {};

package/dist/storage/enhancedCacheManager.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"enhancedCacheManager.d.ts","sourceRoot":"","sources":["../../src/storage/enhancedCacheManager.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAU,MAAM,iBAAiB,CAAA;AAC5D,OAAO,EAAE,iBAAiB,EAAe,MAAM,iCAAiC,CAAA;AAchF,aAAK,gBAAgB;IACnB,kBAAkB,iBAAiB;IACnC,iBAAiB,eAAe;IAChC,cAAc,YAAY;IAC1B,MAAM,WAAW;CAClB;AAGD,UAAU,mBAAmB;IAE3B,eAAe,CAAC,EAAE,MAAM,CAAA;IACxB,yBAAyB,CAAC,EAAE,MAAM,CAAA;IAGlC,gBAAgB,CAAC,EAAE,MAAM,CAAA;IACzB,YAAY,CAAC,EAAE,MAAM,CAAA;IAGrB,eAAe,CAAC,EAAE,OAAO,CAAA;IACzB,gBAAgB,CAAC,EAAE,gBAAgB,CAAA;IACnC,iBAAiB,CAAC,EAAE,MAAM,CAAA;IAC1B,mBAAmB,CAAC,EAAE,MAAM,CAAA;IAG5B,mBAAmB,CAAC,EAAE,MAAM,CAAA;IAC5B,qBAAqB,CAAC,EAAE,MAAM,CAAA;IAG9B,sBAAsB,CAAC,EAAE,OAAO,CAAA;IAChC,oBAAoB,CAAC,EAAE,OAAO,CAAA;CAC/B;AAED;;;GAGG;AACH,qBAAa,oBAAoB,CAAC,CAAC,SAAS,QAAQ,GAAG,QAAQ;IAC7D,OAAO,CAAC,QAAQ,CAA2C;IAC3D,OAAO,CAAC,SAAS,CAA2C;IAC5D,OAAO,CAAC,aAAa,CAAoB;IACzC,OAAO,CAAC,cAAc,CAA8B;IACpD,OAAO,CAAC,WAAW,CAA4B;IAE/C,OAAO,CAAC,MAAM,CAA+B;IAC7C,OAAO,CAAC,eAAe,CAAC,CAAmB;IAC3C,OAAO,CAAC,cAAc,CAAC,CAAK;IAC5B,OAAO,CAAC,kBAAkB,CAAQ;IAGlC,OAAO,CAAC,KAAK,CAUZ;gBAEW,MAAM,GAAE,mBAAwB;IAuB5C;;OAEG;IACI,kBAAkB,CACvB,cAAc,EAAE,GAAG,EACnB,eAAe,CAAC,EAAE,iBAAiB,GAClC,IAAI;IAKP;;OAEG;IACU,GAAG,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,CAAC,GAAG,IAAI,CAAC;IAqD/C;;OAEG;IACU,OAAO,CAAC,GAAG,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;IA4B5D;;OAEG;IACU,GAAG,CAAC,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC;IA6BpD;;OAEG;YACW,gBAAgB;IAoC9B;;OAEG;IACH,OAAO,CAAC,qBAAqB;IAwB7B;;OAEG;YACW,mBAAmB;IAuBjC;;OAEG;IACH,OAAO,CAAC,sBAAsB;IAsB9B;;OAEG;YACW,gBAAgB;IAkC9B;;OAEG;YACW,eAAe;IAiC7B;;OAEG;YACW,eAAe;IAa7B;;OAEG;IACH,OAAO,CAAC,iBAAiB;IAazB;;OAEG;IACH,OAAO,CAAC,iBAAiB;IAsBzB;;OAEG;IACH,OAAO,CAAC,kBAAkB;IA8B1B;;OAEG;IACH,OAAO,CAAC,YAAY;IAmBpB;;OAEG;IACH,OAAO,CAAC,qBAAqB;IAa7B;;OAEG;IACH,OAAO,CAAC,SAAS;IAIjB;;OAEG;IACH,OAAO,CAAC,gBAAgB;IAiBxB;;OAEG;IACH,OAAO,CAAC,iBAAiB;IAqBzB;;OAEG;IACH,OAAO,CAAC,2BAA2B;IAMnC;;OAEG;IACH,OAAO,CAAC,yBAAyB;IAmBjC;;OAEG;IACI,QAAQ,IAAI,OAAO,IAAI,CAAC,KAAK,GAAG;QACrC,YAAY,EAAE,MAAM,CAAA;QACpB,aAAa,EAAE,MAAM,CAAA;QACrB,iBAAiB,EAAE,MAAM,CAAA;QACzB,qBAAqB,EAAE,MAAM,CAAA;KAC9B;IAUD;;OAEG;IACI,KAAK,IAAI,IAAI;CAOrB"}