@soulcraft/brainy 0.36.0 → 0.38.0

package/dist/storage/readOnlyOptimizations.d.ts

@@ -0,0 +1,133 @@
+/**
+ * Read-Only Storage Optimizations for Production Deployments
+ * Implements compression, memory-mapping, and pre-built index segments
+ */
+import { HNSWNoun, Vector } from '../coreTypes.js';
+declare enum CompressionType {
+    NONE = "none",
+    GZIP = "gzip",
+    BROTLI = "brotli",
+    QUANTIZATION = "quantization",
+    HYBRID = "hybrid"
+}
+declare enum QuantizationType {
+    SCALAR = "scalar",// 8-bit scalar quantization
+    PRODUCT = "product",// Product quantization
+    BINARY = "binary"
+}
+interface CompressionConfig {
+    vectorCompression: CompressionType;
+    metadataCompression: CompressionType;
+    quantizationType?: QuantizationType;
+    quantizationBits?: number;
+    compressionLevel?: number;
+}
+interface ReadOnlyConfig {
+    prebuiltIndexPath?: string;
+    memoryMapped?: boolean;
+    compression: CompressionConfig;
+    segmentSize?: number;
+    prefetchSegments?: number;
+    cacheIndexInMemory?: boolean;
+}
+interface IndexSegment {
+    id: string;
+    nodeCount: number;
+    vectorDimension: number;
+    compression: CompressionType;
+    s3Key?: string;
+    localPath?: string;
+    loadedInMemory: boolean;
+    lastAccessed: number;
+}
+/**
+ * Read-only storage optimizations for high-performance production deployments
+ */
+export declare class ReadOnlyOptimizations {
+    private config;
+    private segments;
+    private compressionStats;
+    private quantizationCodebooks;
+    private memoryMappedBuffers;
+    constructor(config?: Partial<ReadOnlyConfig>);
+    /**
+     * Compress vector data using specified compression method
+     */
+    compressVector(vector: Vector, segmentId: string): Promise<ArrayBuffer>;
+    /**
+     * Decompress vector data
+     */
+    decompressVector(compressedData: ArrayBuffer, segmentId: string, originalDimension: number): Promise<Vector>;
+    /**
+     * Scalar quantization of vectors to 8-bit integers
+     */
+    private quantizeVector;
+    /**
+     * Dequantize 8-bit vectors back to float32
+     */
+    private dequantizeVector;
+    /**
+     * GZIP compression using browser/Node.js APIs
+     */
+    private gzipCompress;
+    /**
+     * GZIP decompression
+     */
+    private gzipDecompress;
+    /**
+     * Brotli compression (placeholder - similar to GZIP)
+     */
+    private brotliCompress;
+    /**
+     * Brotli decompression (placeholder)
+     */
+    private brotliDecompress;
+    /**
+     * Create prebuilt index segments for faster loading
+     */
+    createPrebuiltSegments(nodes: HNSWNoun[], outputPath: string): Promise<IndexSegment[]>;
+    /**
+     * Compress an entire segment of nodes
+     */
+    private compressSegment;
+    /**
+     * Load a segment from storage with caching
+     */
+    loadSegment(segmentId: string): Promise<HNSWNoun[]>;
+    /**
+     * Load segment data from storage
+     */
+    private loadSegmentFromStorage;
+    /**
+     * Deserialize and decompress segment data
+     */
+    private deserializeSegment;
+    /**
+     * Serialize connections Map for storage
+     */
+    private serializeConnections;
+    /**
+     * Deserialize connections from storage format
+     */
+    private deserializeConnections;
+    /**
+     * Prefetch segments based on access patterns
+     */
+    prefetchSegments(currentSegmentId: string): Promise<void>;
+    /**
+     * Update compression statistics
+     */
+    private updateCompressionRatio;
+    /**
+     * Get compression statistics
+     */
+    getCompressionStats(): typeof this.compressionStats & {
+        segmentCount: number;
+        memoryUsage: number;
+    };
+    /**
+     * Cleanup memory-mapped buffers
+     */
+    cleanup(): void;
+}
+export {};

package/dist/storage/readOnlyOptimizations.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"readOnlyOptimizations.d.ts","sourceRoot":"","sources":["../../src/storage/readOnlyOptimizations.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAE,QAAQ,EAAY,MAAM,EAAE,MAAM,iBAAiB,CAAA;AAG5D,aAAK,eAAe;IAClB,IAAI,SAAS;IACb,IAAI,SAAS;IACb,MAAM,WAAW;IACjB,YAAY,iBAAiB;IAC7B,MAAM,WAAW;CAClB;AAGD,aAAK,gBAAgB;IACnB,MAAM,WAAW,CAAO,4BAA4B;IACpD,OAAO,YAAY,CAAK,uBAAuB;IAC/C,MAAM,WAAW;CAClB;AAED,UAAU,iBAAiB;IACzB,iBAAiB,EAAE,eAAe,CAAA;IAClC,mBAAmB,EAAE,eAAe,CAAA;IACpC,gBAAgB,CAAC,EAAE,gBAAgB,CAAA;IACnC,gBAAgB,CAAC,EAAE,MAAM,CAAA;IACzB,gBAAgB,CAAC,EAAE,MAAM,CAAA;CAC1B;AAED,UAAU,cAAc;IACtB,iBAAiB,CAAC,EAAE,MAAM,CAAA;IAC1B,YAAY,CAAC,EAAE,OAAO,CAAA;IACtB,WAAW,EAAE,iBAAiB,CAAA;IAC9B,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB,gBAAgB,CAAC,EAAE,MAAM,CAAA;IACzB,kBAAkB,CAAC,EAAE,OAAO,CAAA;CAC7B;AAED,UAAU,YAAY;IACpB,EAAE,EAAE,MAAM,CAAA;IACV,SAAS,EAAE,MAAM,CAAA;IACjB,eAAe,EAAE,MAAM,CAAA;IACvB,WAAW,EAAE,eAAe,CAAA;IAC5B,KAAK,CAAC,EAAE,MAAM,CAAA;IACd,SAAS,CAAC,EAAE,MAAM,CAAA;IAClB,cAAc,EAAE,OAAO,CAAA;IACvB,YAAY,EAAE,MAAM,CAAA;CACrB;AAED;;GAEG;AACH,qBAAa,qBAAqB;IAChC,OAAO,CAAC,MAAM,CAA0B;IACxC,OAAO,CAAC,QAAQ,CAAuC;IACvD,OAAO,CAAC,gBAAgB,CAKvB;IAGD,OAAO,CAAC,qBAAqB,CAAuC;IAGpE,OAAO,CAAC,mBAAmB,CAAsC;gBAErD,MAAM,GAAE,OAAO,CAAC,cAAc,CAAM;IAsBhD;;OAEG;IACU,cAAc,CAAC,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,WAAW,CAAC;IA0CpF;;OAEG;IACU,gBAAgB,CAC3B,cAAc,EAAE,WAAW,EAC3B,SAAS,EAAE,MAAM,EACjB,iBAAiB,EAAE,MAAM,GACxB,OAAO,CAAC,MAAM,CAAC;IAsBlB;;OAEG;YACW,cAAc;IA+B5B;;OAEG;IACH,OAAO,CAAC,gBAAgB;IAwBxB;;OAEG;YACW,YAAY;IAoC1B;;OAEG;YACW,cAAc;IAmC5B;;OAEG;YACW,cAAc;IAM5B;;OAEG;YACW,gBAAgB;IAK9B;;OAEG;IACU,sBAAsB,CACjC,KAAK,EAAE,QAAQ,EAAE,EACjB,UAAU,EAAE,MAAM,GACjB,OAAO,CAAC,YAAY,EAAE,CAAC;IAiC1B;;OAEG;YACW,eAAe;IAqB7B;;OAEG;IACU,WAAW,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,QAAQ,EAAE,CAAC;IAyBhE;;OAEG;YACW,sBAAsB;IAOpC;;OAEG;YACW,kBAAkB;IA6BhC;;OAEG;IACH,OAAO,CAAC,oBAAoB;IAQ5B;;OAEG;IACH,OAAO,CAAC,sBAAsB;IAQ9B;;OAEG;IACU,gBAAgB,CAAC,gBAAgB,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IA4BtE;;OAEG;IACH,OAAO,CAAC,sBAAsB;IAO9B;;OAEG;IACI,mBAAmB,IAAI,OAAO,IAAI,CAAC,gBAAgB,GAAG;QAC3D,YAAY,EAAE,MAAM,CAAA;QACpB,WAAW,EAAE,MAAM,CAAA;KACpB;IAWD;;OAEG;IACI,OAAO,IAAI,IAAI;CASvB"}

package/dist/types/distributedTypes.d.ts

@@ -0,0 +1,197 @@
+/**
+ * Distributed types for Brainy
+ * Defines types for distributed operations across multiple instances
+ */
+export type InstanceRole = 'reader' | 'writer' | 'hybrid';
+export type PartitionStrategy = 'hash' | 'semantic' | 'manual';
+export interface DistributedConfig {
+    /**
+     * Enable distributed mode
+     * Can be boolean for auto-detection or specific configuration
+     */
+    enabled?: boolean | 'auto';
+    /**
+     * Role of this instance in the distributed system
+     * - reader: Read-only access, optimized for queries
+     * - writer: Write-focused, handles data ingestion
+     * - hybrid: Can both read and write (requires coordination)
+     */
+    role?: InstanceRole;
+    /**
+     * Unique identifier for this instance
+     * Auto-generated if not provided
+     */
+    instanceId?: string;
+    /**
+     * Path to shared configuration file in S3
+     * Default: '_brainy/config.json'
+     */
+    configPath?: string;
+    /**
+     * Heartbeat interval in milliseconds
+     * Default: 30000 (30 seconds)
+     */
+    heartbeatInterval?: number;
+    /**
+     * Config check interval in milliseconds
+     * Default: 10000 (10 seconds)
+     */
+    configCheckInterval?: number;
+    /**
+     * Instance timeout in milliseconds
+     * Instances not seen for this duration are considered dead
+     * Default: 60000 (60 seconds)
+     */
+    instanceTimeout?: number;
+}
+export interface SharedConfig {
+    /**
+     * Configuration version for compatibility checking
+     */
+    version: number;
+    /**
+     * Last update timestamp
+     */
+    updated: string;
+    /**
+     * Global settings that must be consistent across all instances
+     */
+    settings: {
+        /**
+         * Partitioning strategy
+         * - hash: Deterministic hash-based partitioning (recommended for multi-writer)
+         * - semantic: Group similar vectors (single writer only)
+         * - manual: Explicit partition assignment
+         */
+        partitionStrategy: PartitionStrategy;
+        /**
+         * Number of partitions (for hash strategy)
+         */
+        partitionCount: number;
+        /**
+         * Embedding model name (must be consistent)
+         */
+        embeddingModel: string;
+        /**
+         * Vector dimensions
+         */
+        dimensions: number;
+        /**
+         * Distance metric
+         */
+        distanceMetric: 'cosine' | 'euclidean' | 'manhattan';
+        /**
+         * HNSW parameters (must be consistent for index compatibility)
+         */
+        hnswParams?: {
+            M: number;
+            efConstruction: number;
+            maxElements?: number;
+        };
+    };
+    /**
+     * Active instances in the distributed system
+     */
+    instances: {
+        [instanceId: string]: InstanceInfo;
+    };
+    /**
+     * Partition assignments (for manual strategy)
+     */
+    partitionAssignments?: {
+        [instanceId: string]: string[];
+    };
+}
+export interface InstanceInfo {
+    /**
+     * Instance role
+     */
+    role: InstanceRole;
+    /**
+     * Instance status
+     */
+    status: 'active' | 'inactive' | 'unhealthy';
+    /**
+     * Last heartbeat timestamp
+     */
+    lastHeartbeat: string;
+    /**
+     * Optional endpoint for health checks
+     */
+    endpoint?: string;
+    /**
+     * Instance metrics
+     */
+    metrics?: {
+        vectorCount?: number;
+        cacheHitRate?: number;
+        memoryUsage?: number;
+        cpuUsage?: number;
+    };
+    /**
+     * Assigned partitions (for manual assignment)
+     */
+    assignedPartitions?: string[];
+    /**
+     * Preferred partitions (for affinity)
+     */
+    preferredPartitions?: number[];
+}
+export interface DomainMetadata {
+    /**
+     * Domain identifier for logical data separation
+     */
+    domain?: string;
+    /**
+     * Additional domain-specific metadata
+     */
+    domainMetadata?: Record<string, any>;
+}
+export interface CacheStrategy {
+    /**
+     * Percentage of memory allocated to hot cache (0-1)
+     */
+    hotCacheRatio: number;
+    /**
+     * Enable aggressive prefetching
+     */
+    prefetchAggressive?: boolean;
+    /**
+     * Cache time-to-live in milliseconds
+     */
+    ttl?: number;
+    /**
+     * Enable compression to trade CPU for memory
+     */
+    compressionEnabled?: boolean;
+    /**
+     * Write buffer size for batching
+     */
+    writeBufferSize?: number;
+    /**
+     * Enable write batching
+     */
+    batchWrites?: boolean;
+    /**
+     * Adaptive caching based on workload
+     */
+    adaptive?: boolean;
+}
+export interface OperationalMode {
+    /**
+     * Whether this mode can read
+     */
+    canRead: boolean;
+    /**
+     * Whether this mode can write
+     */
+    canWrite: boolean;
+    /**
+     * Whether this mode can delete
+     */
+    canDelete: boolean;
+    /**
+     * Cache strategy for this mode
+     */
+    cacheStrategy: CacheStrategy;
+}

package/dist/types/distributedTypes.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"distributedTypes.d.ts","sourceRoot":"","sources":["../../src/types/distributedTypes.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,MAAM,MAAM,YAAY,GAAG,QAAQ,GAAG,QAAQ,GAAG,QAAQ,CAAA;AAEzD,MAAM,MAAM,iBAAiB,GAAG,MAAM,GAAG,UAAU,GAAG,QAAQ,CAAA;AAE9D,MAAM,WAAW,iBAAiB;IAChC;;;OAGG;IACH,OAAO,CAAC,EAAE,OAAO,GAAG,MAAM,CAAA;IAE1B;;;;;OAKG;IACH,IAAI,CAAC,EAAE,YAAY,CAAA;IAEnB;;;OAGG;IACH,UAAU,CAAC,EAAE,MAAM,CAAA;IAEnB;;;OAGG;IACH,UAAU,CAAC,EAAE,MAAM,CAAA;IAEnB;;;OAGG;IACH,iBAAiB,CAAC,EAAE,MAAM,CAAA;IAE1B;;;OAGG;IACH,mBAAmB,CAAC,EAAE,MAAM,CAAA;IAE5B;;;;OAIG;IACH,eAAe,CAAC,EAAE,MAAM,CAAA;CACzB;AAED,MAAM,WAAW,YAAY;IAC3B;;OAEG;IACH,OAAO,EAAE,MAAM,CAAA;IAEf;;OAEG;IACH,OAAO,EAAE,MAAM,CAAA;IAEf;;OAEG;IACH,QAAQ,EAAE;QACR;;;;;WAKG;QACH,iBAAiB,EAAE,iBAAiB,CAAA;QAEpC;;WAEG;QACH,cAAc,EAAE,MAAM,CAAA;QAEtB;;WAEG;QACH,cAAc,EAAE,MAAM,CAAA;QAEtB;;WAEG;QACH,UAAU,EAAE,MAAM,CAAA;QAElB;;WAEG;QACH,cAAc,EAAE,QAAQ,GAAG,WAAW,GAAG,WAAW,CAAA;QAEpD;;WAEG;QACH,UAAU,CAAC,EAAE;YACX,CAAC,EAAE,MAAM,CAAA;YACT,cAAc,EAAE,MAAM,CAAA;YACtB,WAAW,CAAC,EAAE,MAAM,CAAA;SACrB,CAAA;KACF,CAAA;IAED;;OAEG;IACH,SAAS,EAAE;QACT,CAAC,UAAU,EAAE,MAAM,GAAG,YAAY,CAAA;KACnC,CAAA;IAED;;OAEG;IACH,oBAAoB,CAAC,EAAE;QACrB,CAAC,UAAU,EAAE,MAAM,GAAG,MAAM,EAAE,CAAA;KAC/B,CAAA;CACF;AAED,MAAM,WAAW,YAAY;IAC3B;;OAEG;IACH,IAAI,EAAE,YAAY,CAAA;IAElB;;OAEG;IACH,MAAM,EAAE,QAAQ,GAAG,UAAU,GAAG,WAAW,CAAA;IAE3C;;OAEG;IACH,aAAa,EAAE,MAAM,CAAA;IAErB;;OAEG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAA;IAEjB;;OAEG;IACH,OAAO,CAAC,EAAE;QACR,WAAW,CAAC,EAAE,MAAM,CAAA;QACpB,YAAY,CAAC,EAAE,MAAM,CAAA;QACrB,WAAW,CAAC,EAAE,MAAM,CAAA;QACpB,QAAQ,CAAC,EAAE,MAAM,CAAA;KAClB,CAAA;IAED;;OAEG;IACH,kBAAkB,CAAC,EAAE,MAAM,EAAE,CAAA;IAE7B;;OAEG;IACH,mBAAmB,CAAC,EAAE,MAAM,EAAE,CAAA;CAC/B;AAED,MAAM,WAAW,cAAc;IAC7B;;OAEG;IACH,MAAM,CAAC,EAAE,MAAM,CAAA;IAEf;;OAEG;IACH,cAAc,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAA;CACrC;AAED,MAAM,WAAW,aAAa;IAC5B;;OAEG;IACH,aAAa,EAAE,MAAM,CAAA;IAErB;;OAEG;IACH,kBAAkB,CAAC,EAAE,OAAO,CAAA;IAE5B;;OAEG;IACH,GAAG,CAAC,EAAE,MAAM,CAAA;IAEZ;;OAEG;IACH,kBAAkB,CAAC,EAAE,OAAO,CAAA;IAE5B;;OAEG;IACH,eAAe,CAAC,EAAE,MAAM,CAAA;IAExB;;OAEG;IACH,WAAW,CAAC,EAAE,OAAO,CAAA;IAErB;;OAEG;IACH,QAAQ,CAAC,EAAE,OAAO,CAAA;CACnB;AAED,MAAM,WAAW,eAAe;IAC9B;;OAEG;IACH,OAAO,EAAE,OAAO,CAAA;IAEhB;;OAEG;IACH,QAAQ,EAAE,OAAO,CAAA;IAEjB;;OAEG;IACH,SAAS,EAAE,OAAO,CAAA;IAElB;;OAEG;IACH,aAAa,EAAE,aAAa,CAAA;CAC7B"}