@soulcraft/brainy 2.12.0 → 2.14.0

package/dist/embeddings/EmbeddingManager.js

@@ -0,0 +1,296 @@
+/**
+ * Unified Embedding Manager
+ *
+ * THE single source of truth for all embedding operations in Brainy.
+ * Combines model management, precision configuration, and embedding generation
+ * into one clean, maintainable class.
+ *
+ * Features:
+ * - Singleton pattern ensures ONE model instance
+ * - Automatic Q8 (default) or FP32 precision
+ * - Model downloading and caching
+ * - Thread-safe initialization
+ * - Memory monitoring
+ *
+ * This replaces: SingletonModelManager, TransformerEmbedding, ModelPrecisionManager,
+ * hybridModelManager, universalMemoryManager, and more.
+ */
+import { pipeline, env } from '@huggingface/transformers';
+import { existsSync } from 'fs';
+import { join } from 'path';
+// Global state for true singleton across entire process
+let globalInstance = null;
+let globalInitPromise = null;
+/**
+ * Unified Embedding Manager - Clean, simple, reliable
+ */
+export class EmbeddingManager {
+    constructor() {
+        this.model = null;
+        this.modelName = 'Xenova/all-MiniLM-L6-v2';
+        this.initialized = false;
+        this.initTime = null;
+        this.embedCount = 0;
+        this.locked = false;
+        // Determine precision - Q8 by default
+        this.precision = this.determinePrecision();
+        console.log(`🎯 EmbeddingManager: Using ${this.precision.toUpperCase()} precision`);
+    }
+    /**
+     * Get the singleton instance
+     */
+    static getInstance() {
+        if (!globalInstance) {
+            globalInstance = new EmbeddingManager();
+        }
+        return globalInstance;
+    }
+    /**
+     * Initialize the model (happens once)
+     */
+    async init() {
+        // In unit test mode, skip real model initialization
+        if (process.env.BRAINY_UNIT_TEST === 'true' || globalThis.__BRAINY_UNIT_TEST__) {
+            if (!this.initialized) {
+                this.initialized = true;
+                this.initTime = 1; // Mock init time
+                console.log('🧪 EmbeddingManager: Using mocked embeddings for unit tests');
+            }
+            return;
+        }
+        // Already initialized
+        if (this.initialized && this.model) {
+            return;
+        }
+        // Initialization in progress
+        if (globalInitPromise) {
+            await globalInitPromise;
+            return;
+        }
+        // Start initialization
+        globalInitPromise = this.performInit();
+        try {
+            await globalInitPromise;
+        }
+        finally {
+            globalInitPromise = null;
+        }
+    }
+    /**
+     * Perform actual initialization
+     */
+    async performInit() {
+        const startTime = Date.now();
+        console.log(`🚀 Initializing embedding model (${this.precision.toUpperCase()})...`);
+        try {
+            // Configure transformers.js environment
+            const modelsPath = this.getModelsPath();
+            env.cacheDir = modelsPath;
+            env.allowLocalModels = true;
+            env.useFSCache = true;
+            // Check if models exist locally
+            const modelPath = join(modelsPath, ...this.modelName.split('/'));
+            const hasLocalModels = existsSync(modelPath);
+            if (hasLocalModels) {
+                console.log('✅ Using cached models from:', modelPath);
+            }
+            // Configure pipeline options for the selected precision
+            const pipelineOptions = {
+                cache_dir: modelsPath,
+                local_files_only: false,
+                // Specify precision
+                dtype: this.precision,
+                quantized: this.precision === 'q8',
+                // Memory optimizations
+                session_options: {
+                    enableCpuMemArena: false,
+                    enableMemPattern: false,
+                    interOpNumThreads: 1,
+                    intraOpNumThreads: 1,
+                    graphOptimizationLevel: 'disabled'
+                }
+            };
+            // Load the model
+            this.model = await pipeline('feature-extraction', this.modelName, pipelineOptions);
+            // Lock precision after successful initialization
+            this.locked = true;
+            this.initialized = true;
+            this.initTime = Date.now() - startTime;
+            // Log success
+            const memoryMB = this.getMemoryUsage();
+            console.log(`✅ Model loaded in ${this.initTime}ms`);
+            console.log(`📊 Precision: ${this.precision.toUpperCase()} | Memory: ${memoryMB}MB`);
+            console.log(`🔒 Configuration locked`);
+        }
+        catch (error) {
+            this.initialized = false;
+            this.model = null;
+            throw new Error(`Failed to initialize embedding model: ${error instanceof Error ? error.message : String(error)}`);
+        }
+    }
+    /**
+     * Generate embeddings
+     */
+    async embed(text) {
+        // Check for unit test environment - use mocks to prevent ONNX conflicts
+        if (process.env.BRAINY_UNIT_TEST === 'true' || globalThis.__BRAINY_UNIT_TEST__) {
+            return this.getMockEmbedding(text);
+        }
+        // Ensure initialized
+        await this.init();
+        if (!this.model) {
+            throw new Error('Model not initialized');
+        }
+        // Handle array input
+        const input = Array.isArray(text) ? text.join(' ') : text;
+        // Generate embedding
+        const output = await this.model(input, {
+            pooling: 'mean',
+            normalize: true
+        });
+        // Extract embedding vector
+        const embedding = Array.from(output.data);
+        // Validate dimensions
+        if (embedding.length !== 384) {
+            console.warn(`Unexpected embedding dimension: ${embedding.length}`);
+            // Pad or truncate
+            if (embedding.length < 384) {
+                return [...embedding, ...new Array(384 - embedding.length).fill(0)];
+            }
+            else {
+                return embedding.slice(0, 384);
+            }
+        }
+        this.embedCount++;
+        return embedding;
+    }
+    /**
+     * Generate mock embeddings for unit tests
+     */
+    getMockEmbedding(text) {
+        // Use the same mock logic as setup-unit.ts for consistency
+        const input = Array.isArray(text) ? text.join(' ') : text;
+        const str = typeof input === 'string' ? input : JSON.stringify(input);
+        const vector = new Array(384).fill(0);
+        // Create semi-realistic embeddings based on text content
+        for (let i = 0; i < Math.min(str.length, 384); i++) {
+            vector[i] = (str.charCodeAt(i % str.length) % 256) / 256;
+        }
+        // Add position-based variation
+        for (let i = 0; i < 384; i++) {
+            vector[i] += Math.sin(i * 0.1 + str.length) * 0.1;
+        }
+        // Track mock embedding count
+        this.embedCount++;
+        return vector;
+    }
+    /**
+     * Get embedding function for compatibility
+     */
+    getEmbeddingFunction() {
+        return async (data) => {
+            return await this.embed(data);
+        };
+    }
+    /**
+     * Determine model precision
+     */
+    determinePrecision() {
+        // Check environment variable overrides
+        if (process.env.BRAINY_MODEL_PRECISION === 'fp32') {
+            return 'fp32';
+        }
+        if (process.env.BRAINY_MODEL_PRECISION === 'q8') {
+            return 'q8';
+        }
+        if (process.env.BRAINY_FORCE_FP32 === 'true') {
+            return 'fp32';
+        }
+        // Default to Q8 - optimal for most use cases
+        return 'q8';
+    }
+    /**
+     * Get models directory path
+     */
+    getModelsPath() {
+        // Check various possible locations
+        const paths = [
+            process.env.BRAINY_MODELS_PATH,
+            './models',
+            join(process.cwd(), 'models'),
+            join(process.env.HOME || '', '.brainy', 'models')
+        ];
+        for (const path of paths) {
+            if (path && existsSync(path)) {
+                return path;
+            }
+        }
+        // Default
+        return join(process.cwd(), 'models');
+    }
+    /**
+     * Get memory usage in MB
+     */
+    getMemoryUsage() {
+        if (typeof process !== 'undefined' && process.memoryUsage) {
+            const usage = process.memoryUsage();
+            return Math.round(usage.heapUsed / 1024 / 1024);
+        }
+        return null;
+    }
+    /**
+     * Get current statistics
+     */
+    getStats() {
+        return {
+            initialized: this.initialized,
+            precision: this.precision,
+            modelName: this.modelName,
+            embedCount: this.embedCount,
+            initTime: this.initTime,
+            memoryMB: this.getMemoryUsage()
+        };
+    }
+    /**
+     * Check if initialized
+     */
+    isInitialized() {
+        return this.initialized;
+    }
+    /**
+     * Get current precision
+     */
+    getPrecision() {
+        return this.precision;
+    }
+    /**
+     * Validate precision matches expected
+     */
+    validatePrecision(expected) {
+        if (this.locked && expected !== this.precision) {
+            throw new Error(`Precision mismatch! System using ${this.precision.toUpperCase()} ` +
+                `but ${expected.toUpperCase()} was requested. Cannot mix precisions.`);
+        }
+    }
+}
+// Export singleton instance and convenience functions
+export const embeddingManager = EmbeddingManager.getInstance();
+/**
+ * Direct embed function
+ */
+export async function embed(text) {
+    return await embeddingManager.embed(text);
+}
+/**
+ * Get embedding function for compatibility
+ */
+export function getEmbeddingFunction() {
+    return embeddingManager.getEmbeddingFunction();
+}
+/**
+ * Get statistics
+ */
+export function getEmbeddingStats() {
+    return embeddingManager.getStats();
+}
+//# sourceMappingURL=EmbeddingManager.js.map

package/dist/embeddings/SingletonModelManager.d.ts

@@ -0,0 +1,95 @@
+/**
+ * Singleton Model Manager - THE ONLY SOURCE OF EMBEDDING MODELS
+ *
+ * This is the SINGLE, UNIFIED model initialization system that ensures:
+ * - Only ONE model instance exists across the entire system
+ * - Precision is configured once and locked
+ * - All components share the same model
+ * - No possibility of mixed precisions
+ *
+ * CRITICAL: This manager is used by EVERYTHING:
+ * - Storage operations (add, update)
+ * - Search operations (search, find)
+ * - Public API (embed, cluster)
+ * - Neural API (all neural.* methods)
+ * - Internal operations (deduplication, indexing)
+ */
+import { TransformerEmbedding } from '../utils/embedding.js';
+import { EmbeddingFunction, Vector } from '../coreTypes.js';
+/**
+ * Statistics for monitoring
+ */
+interface ModelStats {
+    initialized: boolean;
+    precision: string;
+    initCount: number;
+    embedCount: number;
+    lastUsed: Date | null;
+    memoryFootprint?: number;
+}
+/**
+ * The ONE TRUE model manager
+ */
+export declare class SingletonModelManager {
+    private static instance;
+    private stats;
+    private constructor();
+    /**
+     * Get the singleton instance
+     */
+    static getInstance(): SingletonModelManager;
+    /**
+     * Get the model instance - creates if needed, reuses if exists
+     * This is THE ONLY way to get a model in the entire system
+     */
+    getModel(): Promise<TransformerEmbedding>;
+    /**
+     * Initialize the model - happens exactly once
+     */
+    private initializeModel;
+    /**
+     * Get embedding function that uses the singleton model
+     */
+    getEmbeddingFunction(): Promise<EmbeddingFunction>;
+    /**
+     * Direct embed method for convenience
+     */
+    embed(data: string | string[]): Promise<Vector>;
+    /**
+     * Check if model is initialized
+     */
+    isInitialized(): boolean;
+    /**
+     * Get current statistics
+     */
+    getStats(): ModelStats;
+    /**
+     * Validate precision consistency
+     * Throws error if attempting to use different precision
+     */
+    validatePrecision(requestedPrecision?: string): void;
+    /**
+     * Force cleanup (for testing only)
+     * WARNING: This will break consistency - use only in tests
+     */
+    _testOnlyCleanup(): Promise<void>;
+}
+export declare const singletonModelManager: SingletonModelManager;
+/**
+ * THE ONLY embedding function that should be used anywhere
+ * This ensures all operations use the same model instance
+ */
+export declare function getUnifiedEmbeddingFunction(): Promise<EmbeddingFunction>;
+/**
+ * Direct embed function for convenience
+ */
+export declare function unifiedEmbed(data: string | string[]): Promise<Vector>;
+/**
+ * Check if model is ready
+ */
+export declare function isModelReady(): boolean;
+/**
+ * Get model statistics
+ */
+export declare function getModelStats(): ModelStats;
+export {};

package/dist/embeddings/SingletonModelManager.js

@@ -0,0 +1,220 @@
+/**
+ * Singleton Model Manager - THE ONLY SOURCE OF EMBEDDING MODELS
+ *
+ * This is the SINGLE, UNIFIED model initialization system that ensures:
+ * - Only ONE model instance exists across the entire system
+ * - Precision is configured once and locked
+ * - All components share the same model
+ * - No possibility of mixed precisions
+ *
+ * CRITICAL: This manager is used by EVERYTHING:
+ * - Storage operations (add, update)
+ * - Search operations (search, find)
+ * - Public API (embed, cluster)
+ * - Neural API (all neural.* methods)
+ * - Internal operations (deduplication, indexing)
+ */
+import { TransformerEmbedding } from '../utils/embedding.js';
+import { getModelPrecision, lockModelPrecision } from '../config/modelPrecisionManager.js';
+// Global state - ensures true singleton across entire process
+let globalModelInstance = null;
+let globalInitPromise = null;
+let globalInitialized = false;
+/**
+ * The ONE TRUE model manager
+ */
+export class SingletonModelManager {
+    constructor() {
+        this.stats = {
+            initialized: false,
+            precision: 'unknown',
+            initCount: 0,
+            embedCount: 0,
+            lastUsed: null
+        };
+        // Private constructor enforces singleton
+        this.stats.precision = getModelPrecision();
+        console.log(`🔐 SingletonModelManager initialized with ${this.stats.precision.toUpperCase()} precision`);
+    }
+    /**
+     * Get the singleton instance
+     */
+    static getInstance() {
+        if (!SingletonModelManager.instance) {
+            SingletonModelManager.instance = new SingletonModelManager();
+        }
+        return SingletonModelManager.instance;
+    }
+    /**
+     * Get the model instance - creates if needed, reuses if exists
+     * This is THE ONLY way to get a model in the entire system
+     */
+    async getModel() {
+        // If already initialized, return immediately
+        if (globalModelInstance && globalInitialized) {
+            this.stats.lastUsed = new Date();
+            return globalModelInstance;
+        }
+        // If initialization is in progress, wait for it
+        if (globalInitPromise) {
+            console.log('⏳ Model initialization already in progress, waiting...');
+            return await globalInitPromise;
+        }
+        // Start initialization (only happens once ever)
+        globalInitPromise = this.initializeModel();
+        try {
+            const model = await globalInitPromise;
+            globalInitialized = true;
+            return model;
+        }
+        catch (error) {
+            // Reset on error to allow retry
+            globalInitPromise = null;
+            throw error;
+        }
+    }
+    /**
+     * Initialize the model - happens exactly once
+     */
+    async initializeModel() {
+        console.log('🚀 Initializing singleton model instance...');
+        // Get precision from central manager
+        const precision = getModelPrecision();
+        console.log(`📊 Using ${precision.toUpperCase()} precision (${precision === 'q8' ? '23MB, 99% accuracy' : '90MB, 100% accuracy'})`);
+        // Detect environment for optimal settings
+        const isNode = typeof process !== 'undefined' && process.versions?.node;
+        const isBrowser = typeof window !== 'undefined' && typeof document !== 'undefined';
+        const isServerless = typeof process !== 'undefined' && (process.env.VERCEL ||
+            process.env.NETLIFY ||
+            process.env.AWS_LAMBDA_FUNCTION_NAME ||
+            process.env.FUNCTIONS_WORKER_RUNTIME);
+        const isTest = globalThis.__BRAINY_TEST_ENV__ || process.env.NODE_ENV === 'test';
+        // Create optimized options based on environment
+        const options = {
+            precision: precision,
+            verbose: !isTest && !isServerless && !isBrowser,
+            device: 'cpu', // CPU is most compatible
+            localFilesOnly: process.env.BRAINY_ALLOW_REMOTE_MODELS === 'false',
+            model: 'Xenova/all-MiniLM-L6-v2'
+        };
+        try {
+            // Create the ONE model instance
+            globalModelInstance = new TransformerEmbedding(options);
+            // Initialize it
+            await globalModelInstance.init();
+            // CRITICAL: Lock the precision after successful initialization
+            // This prevents any future changes to precision
+            lockModelPrecision();
+            console.log('🔒 Model precision locked at:', precision.toUpperCase());
+            // Update stats
+            this.stats.initialized = true;
+            this.stats.initCount++;
+            this.stats.lastUsed = new Date();
+            // Log memory usage if available
+            if (isNode && process.memoryUsage) {
+                const usage = process.memoryUsage();
+                this.stats.memoryFootprint = Math.round(usage.heapUsed / 1024 / 1024);
+                console.log(`💾 Model loaded, memory usage: ${this.stats.memoryFootprint}MB`);
+            }
+            console.log('✅ Singleton model initialized successfully');
+            return globalModelInstance;
+        }
+        catch (error) {
+            console.error('❌ Failed to initialize singleton model:', error);
+            globalModelInstance = null;
+            throw new Error(`Singleton model initialization failed: ${error instanceof Error ? error.message : String(error)}`);
+        }
+    }
+    /**
+     * Get embedding function that uses the singleton model
+     */
+    async getEmbeddingFunction() {
+        const model = await this.getModel();
+        return async (data) => {
+            this.stats.embedCount++;
+            this.stats.lastUsed = new Date();
+            return await model.embed(data);
+        };
+    }
+    /**
+     * Direct embed method for convenience
+     */
+    async embed(data) {
+        const model = await this.getModel();
+        this.stats.embedCount++;
+        this.stats.lastUsed = new Date();
+        return await model.embed(data);
+    }
+    /**
+     * Check if model is initialized
+     */
+    isInitialized() {
+        return globalInitialized && globalModelInstance !== null;
+    }
+    /**
+     * Get current statistics
+     */
+    getStats() {
+        return {
+            ...this.stats,
+            precision: getModelPrecision()
+        };
+    }
+    /**
+     * Validate precision consistency
+     * Throws error if attempting to use different precision
+     */
+    validatePrecision(requestedPrecision) {
+        const currentPrecision = getModelPrecision();
+        if (requestedPrecision && requestedPrecision !== currentPrecision) {
+            throw new Error(`❌ Precision mismatch! System is using ${currentPrecision.toUpperCase()} ` +
+                `but ${requestedPrecision.toUpperCase()} was requested. ` +
+                `All operations must use the same precision.`);
+        }
+    }
+    /**
+     * Force cleanup (for testing only)
+     * WARNING: This will break consistency - use only in tests
+     */
+    async _testOnlyCleanup() {
+        if (process.env.NODE_ENV !== 'test') {
+            throw new Error('Cleanup only allowed in test environment');
+        }
+        if (globalModelInstance && 'dispose' in globalModelInstance) {
+            await globalModelInstance.dispose();
+        }
+        globalModelInstance = null;
+        globalInitPromise = null;
+        globalInitialized = false;
+        this.stats.initialized = false;
+        console.log('🧹 Singleton model cleaned up (test only)');
+    }
+}
+// Export the singleton instance getter
+export const singletonModelManager = SingletonModelManager.getInstance();
+/**
+ * THE ONLY embedding function that should be used anywhere
+ * This ensures all operations use the same model instance
+ */
+export async function getUnifiedEmbeddingFunction() {
+    return await singletonModelManager.getEmbeddingFunction();
+}
+/**
+ * Direct embed function for convenience
+ */
+export async function unifiedEmbed(data) {
+    return await singletonModelManager.embed(data);
+}
+/**
+ * Check if model is ready
+ */
+export function isModelReady() {
+    return singletonModelManager.isInitialized();
+}
+/**
+ * Get model statistics
+ */
+export function getModelStats() {
+    return singletonModelManager.getStats();
+}
+//# sourceMappingURL=SingletonModelManager.js.map

package/dist/embeddings/index.d.ts

@@ -0,0 +1,12 @@
+/**
+ * Embeddings Module - Clean, Unified Architecture
+ *
+ * This module provides all embedding functionality for Brainy.
+ *
+ * Main Components:
+ * - EmbeddingManager: Core embedding generation with Q8/FP32 support
+ * - CachedEmbeddings: Performance optimization layer with pre-computed embeddings
+ */
+export { EmbeddingManager, embeddingManager, embed, getEmbeddingFunction, getEmbeddingStats, type ModelPrecision } from './EmbeddingManager.js';
+export { CachedEmbeddings, cachedEmbeddings } from './CachedEmbeddings.js';
+export { embeddingManager as default } from './EmbeddingManager.js';

package/dist/embeddings/index.js

@@ -0,0 +1,16 @@
+/**
+ * Embeddings Module - Clean, Unified Architecture
+ *
+ * This module provides all embedding functionality for Brainy.
+ *
+ * Main Components:
+ * - EmbeddingManager: Core embedding generation with Q8/FP32 support
+ * - CachedEmbeddings: Performance optimization layer with pre-computed embeddings
+ */
+// Core embedding functionality
+export { EmbeddingManager, embeddingManager, embed, getEmbeddingFunction, getEmbeddingStats } from './EmbeddingManager.js';
+// Cached embeddings for performance
+export { CachedEmbeddings, cachedEmbeddings } from './CachedEmbeddings.js';
+// Default export is the singleton manager
+export { embeddingManager as default } from './EmbeddingManager.js';
+//# sourceMappingURL=index.js.map

package/dist/embeddings/lightweight-embedder.d.ts

@@ -8,7 +8,6 @@
  */
 import { Vector } from '../coreTypes.js';
 export declare class LightweightEmbedder {
-    private onnxEmbedder;
     private stats;
     embed(text: string | string[]): Promise<Vector | Vector[]>;
     private embedSingle;

package/dist/embeddings/lightweight-embedder.js

@@ -6,6 +6,7 @@
  *
  * This reduces memory usage by 90% for typical queries
  */
+import { singletonModelManager } from './SingletonModelManager.js';
 // Pre-computed embeddings for top 10,000 common terms
 // In production, this would be loaded from a file
 const PRECOMPUTED_EMBEDDINGS = {
@@ -59,7 +60,6 @@ function computeSimpleEmbedding(text) {
 }
 export class LightweightEmbedder {
     constructor() {
-        this.onnxEmbedder = null;
         this.stats = {
             precomputedHits: 0,
             simpleComputes: 0,
@@ -92,18 +92,10 @@ export class LightweightEmbedder {
             this.stats.simpleComputes++;
             return computeSimpleEmbedding(normalized);
         }
-        // 4. Last resort: Load ONNX model (only if really needed)
-        if (!this.onnxEmbedder) {
-            console.log('⚠️ Loading ONNX model for complex text...');
-            const { TransformerEmbedding } = await import('../utils/embedding.js');
-            this.onnxEmbedder = new TransformerEmbedding({
-                precision: 'fp32',
-                verbose: false
-            });
-            await this.onnxEmbedder.init();
-        }
+        // 4. Last resort: Use SingletonModelManager for complex text
+        console.log('⚠️ Using singleton model for complex text...');
         this.stats.onnxComputes++;
-        return await this.onnxEmbedder.embed(text);
+        return await singletonModelManager.embed(text);
     }
     getStats() {
         return {