@soulcraft/brainy 1.2.0 → 1.4.0

package/dist/utils/embedding.d.ts

@@ -78,7 +78,8 @@ export declare const UniversalSentenceEncoder: typeof TransformerEmbedding;
  */
 export declare function createEmbeddingModel(options?: TransformerEmbeddingOptions): EmbeddingModel;
 /**
- * Default embedding function using the lightweight transformer model
+ * Default embedding function using the hybrid model manager (BEST OF BOTH WORLDS)
+ * Prevents multiple model loads while supporting multi-source downloading
  */
 export declare const defaultEmbeddingFunction: EmbeddingFunction;
 /**

package/dist/utils/embedding.js

@@ -3,6 +3,7 @@
  * Complete rewrite to eliminate TensorFlow.js and use ONNX-based models
  */
 import { isBrowser } from './environment.js';
+import { ModelManager } from '../embeddings/model-manager.js';
 // @ts-ignore - Transformers.js is now the primary embedding library
 import { pipeline, env } from '@huggingface/transformers';
 /**
@@ -192,6 +193,9 @@ export class TransformerEmbedding {
         }
         // Always use real implementation - no mocking
         try {
+            // Ensure models are available (downloads if needed)
+            const modelManager = ModelManager.getInstance();
+            await modelManager.ensureModels(this.options.model);
             // Resolve device configuration and cache directory
             const device = await resolveDevice(this.options.device);
             const cacheDir = this.options.cacheDir === './models'
@@ -361,11 +365,13 @@ export function createEmbeddingModel(options) {
     return new TransformerEmbedding(options);
 }
 /**
- * Default embedding function using the lightweight transformer model
+ * Default embedding function using the hybrid model manager (BEST OF BOTH WORLDS)
+ * Prevents multiple model loads while supporting multi-source downloading
  */
 export const defaultEmbeddingFunction = async (data) => {
-    const embedder = new TransformerEmbedding({ verbose: false });
-    return await embedder.embed(data);
+    const { getHybridEmbeddingFunction } = await import('./hybridModelManager.js');
+    const embeddingFn = await getHybridEmbeddingFunction();
+    return await embeddingFn(data);
 };
 /**
  * Create an embedding function with custom options

package/dist/utils/hybridModelManager.d.ts

@@ -0,0 +1,73 @@
+/**
+ * Hybrid Model Manager - BEST OF BOTH WORLDS
+ *
+ * Combines:
+ * 1. Multi-source downloading strategy (GitHub → CDN → Hugging Face)
+ * 2. Singleton pattern preventing multiple ONNX model loads
+ * 3. Environment-specific optimizations
+ * 4. Graceful fallbacks and error handling
+ */
+import { TransformerEmbedding } from './embedding.js';
+import { EmbeddingFunction } from '../coreTypes.js';
+/**
+ * Global singleton model manager - PREVENTS MULTIPLE MODEL LOADS
+ */
+declare class HybridModelManager {
+    private static instance;
+    private primaryModel;
+    private modelPromise;
+    private isInitialized;
+    private modelsPath;
+    private constructor();
+    static getInstance(): HybridModelManager;
+    /**
+     * Get the primary embedding model - LOADS ONCE, REUSES FOREVER
+     */
+    getPrimaryModel(): Promise<TransformerEmbedding>;
+    /**
+     * Smart model path detection
+     */
+    private getModelsPath;
+    /**
+     * Initialize with BEST OF BOTH: Multi-source + Singleton
+     */
+    private initializePrimaryModel;
+    /**
+     * Create model with multi-source fallback strategy
+     */
+    private createModelWithFallbacks;
+    /**
+     * Get embedding function that reuses the singleton model
+     */
+    getEmbeddingFunction(): Promise<EmbeddingFunction>;
+    /**
+     * Check if model is ready (loaded and initialized)
+     */
+    isModelReady(): boolean;
+    /**
+     * Force model reload (for testing or recovery)
+     */
+    reloadModel(): Promise<void>;
+    /**
+     * Get model status for debugging
+     */
+    getModelStatus(): {
+        loaded: boolean;
+        ready: boolean;
+        modelType: string;
+    };
+}
+export declare const hybridModelManager: HybridModelManager;
+/**
+ * Get the hybrid singleton embedding function - USE THIS EVERYWHERE!
+ */
+export declare function getHybridEmbeddingFunction(): Promise<EmbeddingFunction>;
+/**
+ * Optimized hybrid embedding function that uses multi-source + singleton
+ */
+export declare const hybridEmbeddingFunction: EmbeddingFunction;
+/**
+ * Preload model for tests or production - CALL THIS ONCE AT START
+ */
+export declare function preloadHybridModel(): Promise<void>;
+export {};

package/dist/utils/hybridModelManager.js

@@ -0,0 +1,254 @@
+/**
+ * Hybrid Model Manager - BEST OF BOTH WORLDS
+ *
+ * Combines:
+ * 1. Multi-source downloading strategy (GitHub → CDN → Hugging Face)
+ * 2. Singleton pattern preventing multiple ONNX model loads
+ * 3. Environment-specific optimizations
+ * 4. Graceful fallbacks and error handling
+ */
+import { TransformerEmbedding } from './embedding.js';
+import { existsSync } from 'fs';
+import { join } from 'path';
+/**
+ * Global singleton model manager - PREVENTS MULTIPLE MODEL LOADS
+ */
+class HybridModelManager {
+    constructor() {
+        this.primaryModel = null;
+        this.modelPromise = null;
+        this.isInitialized = false;
+        // Smart model path detection
+        this.modelsPath = this.getModelsPath();
+    }
+    static getInstance() {
+        if (!HybridModelManager.instance) {
+            HybridModelManager.instance = new HybridModelManager();
+        }
+        return HybridModelManager.instance;
+    }
+    /**
+     * Get the primary embedding model - LOADS ONCE, REUSES FOREVER
+     */
+    async getPrimaryModel() {
+        // If already initialized, return immediately
+        if (this.primaryModel && this.isInitialized) {
+            return this.primaryModel;
+        }
+        // If initialization is in progress, wait for it
+        if (this.modelPromise) {
+            return await this.modelPromise;
+        }
+        // Start initialization with multi-source strategy
+        this.modelPromise = this.initializePrimaryModel();
+        return await this.modelPromise;
+    }
+    /**
+     * Smart model path detection
+     */
+    getModelsPath() {
+        const paths = [
+            process.env.BRAINY_MODELS_PATH,
+            './models',
+            './node_modules/@soulcraft/brainy/models',
+            join(process.cwd(), 'models')
+        ];
+        // Find first existing path or use default
+        for (const path of paths) {
+            if (path && existsSync(path)) {
+                return path;
+            }
+        }
+        return join(process.cwd(), 'models');
+    }
+    /**
+     * Initialize with BEST OF BOTH: Multi-source + Singleton
+     */
+    async initializePrimaryModel() {
+        try {
+            // Environment detection for optimal configuration
+            const isTest = globalThis.__BRAINY_TEST_ENV__ || process.env.NODE_ENV === 'test';
+            const isBrowser = typeof window !== 'undefined' && typeof document !== 'undefined';
+            const isServerless = typeof process !== 'undefined' && (process.env.VERCEL ||
+                process.env.NETLIFY ||
+                process.env.AWS_LAMBDA_FUNCTION_NAME ||
+                process.env.FUNCTIONS_WORKER_RUNTIME);
+            const isDocker = typeof process !== 'undefined' && (process.env.DOCKER_CONTAINER ||
+                process.env.KUBERNETES_SERVICE_HOST);
+            // Smart configuration based on environment
+            let options = {
+                verbose: !isTest && !isServerless,
+                dtype: 'q8',
+                device: 'cpu'
+            };
+            // Environment-specific optimizations
+            if (isBrowser) {
+                options = {
+                    ...options,
+                    localFilesOnly: false,
+                    dtype: 'q8',
+                    device: 'cpu',
+                    verbose: false
+                };
+            }
+            else if (isServerless) {
+                options = {
+                    ...options,
+                    localFilesOnly: true,
+                    dtype: 'q8',
+                    device: 'cpu',
+                    verbose: false
+                };
+            }
+            else if (isDocker) {
+                options = {
+                    ...options,
+                    localFilesOnly: true,
+                    dtype: 'fp32',
+                    device: 'auto',
+                    verbose: false
+                };
+            }
+            else if (isTest) {
+                // CRITICAL FOR TESTS: Allow remote downloads but be smart about it
+                options = {
+                    ...options,
+                    localFilesOnly: false,
+                    dtype: 'q8',
+                    device: 'cpu',
+                    verbose: false
+                };
+            }
+            else {
+                options = {
+                    ...options,
+                    localFilesOnly: false,
+                    dtype: 'q8',
+                    device: 'auto',
+                    verbose: true
+                };
+            }
+            const environmentName = isBrowser ? 'browser' :
+                isServerless ? 'serverless' :
+                    isDocker ? 'container' :
+                        isTest ? 'test' : 'node';
+            if (options.verbose) {
+                console.log(`🧠 Initializing hybrid model manager (${environmentName} mode)...`);
+            }
+            // MULTI-SOURCE STRATEGY: Try local first, then remote fallbacks
+            this.primaryModel = await this.createModelWithFallbacks(options, environmentName);
+            this.isInitialized = true;
+            this.modelPromise = null; // Clear the promise
+            if (options.verbose) {
+                console.log(`✅ Hybrid model manager initialized successfully`);
+            }
+            return this.primaryModel;
+        }
+        catch (error) {
+            this.modelPromise = null; // Clear failed promise
+            const errorMessage = error instanceof Error ? error.message : String(error);
+            const environmentInfo = typeof window !== 'undefined' ? 'browser' :
+                typeof process !== 'undefined' ? `node (${process.version})` : 'unknown';
+            throw new Error(`Failed to initialize hybrid model manager in ${environmentInfo} environment: ${errorMessage}. ` +
+                `This is critical for all Brainy operations.`);
+        }
+    }
+    /**
+     * Create model with multi-source fallback strategy
+     */
+    async createModelWithFallbacks(options, environmentName) {
+        const attempts = [
+            // 1. Try with current configuration (may use local cache)
+            { ...options, localFilesOnly: false, source: 'primary' },
+            // 2. If that fails, explicitly allow remote with verbose logging
+            { ...options, localFilesOnly: false, verbose: true, source: 'fallback-verbose' },
+            // 3. Last resort: basic configuration
+            { verbose: false, dtype: 'q8', device: 'cpu', localFilesOnly: false, source: 'last-resort' }
+        ];
+        let lastError = null;
+        for (const attemptOptions of attempts) {
+            try {
+                const { source, ...modelOptions } = attemptOptions;
+                if (attemptOptions.verbose) {
+                    console.log(`🔄 Attempting model load (${source})...`);
+                }
+                const model = new TransformerEmbedding(modelOptions);
+                await model.init();
+                if (attemptOptions.verbose) {
+                    console.log(`✅ Model loaded successfully with ${source} strategy`);
+                }
+                return model;
+            }
+            catch (error) {
+                lastError = error instanceof Error ? error : new Error(String(error));
+                if (attemptOptions.verbose) {
+                    console.log(`❌ Failed ${attemptOptions.source} strategy:`, lastError.message);
+                }
+                // Continue to next attempt
+            }
+        }
+        // All attempts failed
+        throw new Error(`All model loading strategies failed in ${environmentName} environment. ` +
+            `Last error: ${lastError?.message}. ` +
+            `Check network connectivity or ensure models are available locally.`);
+    }
+    /**
+     * Get embedding function that reuses the singleton model
+     */
+    async getEmbeddingFunction() {
+        const model = await this.getPrimaryModel();
+        return async (data) => {
+            return await model.embed(data);
+        };
+    }
+    /**
+     * Check if model is ready (loaded and initialized)
+     */
+    isModelReady() {
+        return this.isInitialized && this.primaryModel !== null;
+    }
+    /**
+     * Force model reload (for testing or recovery)
+     */
+    async reloadModel() {
+        this.primaryModel = null;
+        this.isInitialized = false;
+        this.modelPromise = null;
+        await this.getPrimaryModel();
+    }
+    /**
+     * Get model status for debugging
+     */
+    getModelStatus() {
+        return {
+            loaded: this.primaryModel !== null,
+            ready: this.isInitialized,
+            modelType: 'HybridModelManager (Multi-source + Singleton)'
+        };
+    }
+}
+HybridModelManager.instance = null;
+// Export singleton instance
+export const hybridModelManager = HybridModelManager.getInstance();
+/**
+ * Get the hybrid singleton embedding function - USE THIS EVERYWHERE!
+ */
+export async function getHybridEmbeddingFunction() {
+    return await hybridModelManager.getEmbeddingFunction();
+}
+/**
+ * Optimized hybrid embedding function that uses multi-source + singleton
+ */
+export const hybridEmbeddingFunction = async (data) => {
+    const embeddingFn = await getHybridEmbeddingFunction();
+    return await embeddingFn(data);
+};
+/**
+ * Preload model for tests or production - CALL THIS ONCE AT START
+ */
+export async function preloadHybridModel() {
+    console.log('🚀 Preloading hybrid model...');
+    await hybridModelManager.getPrimaryModel();
+    console.log('✅ Hybrid model preloaded and ready!');
+}
+//# sourceMappingURL=hybridModelManager.js.map

package/dist/utils/modelLoader.d.ts

@@ -0,0 +1,32 @@
+/**
+ * Critical Model Loader - Ensures models are ALWAYS available
+ *
+ * The all-MiniLM-L6-v2 model (384 dimensions) is CRITICAL.
+ * Without this exact model, users cannot access their data.
+ *
+ * 3-Tier Loading Strategy:
+ * 1. Local pre-downloaded models (for production/Docker)
+ * 2. models.soulcraft.com CDN (our controlled source)
+ * 3. GitHub Releases (immutable backup)
+ * 4. Hugging Face (final fallback)
+ */
+/**
+ * Enhanced pipeline function with 3-tier model loading
+ * This wraps the transformers.js pipeline to ensure models are available
+ */
+export declare function criticalPipeline(task: any, // Task type from transformers.js
+model?: string, options?: any): Promise<any>;
+/**
+ * Pre-download models for production deployment
+ * This ensures models are available without runtime downloads
+ */
+export declare function preDownloadModels(cacheDir?: string): Promise<boolean>;
+/**
+ * Get model status and availability
+ */
+export declare function getModelStatus(cacheDir?: string): {
+    isAvailable: boolean;
+    source: string;
+    verified: boolean;
+    dimensions: number;
+};

package/dist/utils/modelLoader.js

@@ -0,0 +1,219 @@
+/**
+ * Critical Model Loader - Ensures models are ALWAYS available
+ *
+ * The all-MiniLM-L6-v2 model (384 dimensions) is CRITICAL.
+ * Without this exact model, users cannot access their data.
+ *
+ * 3-Tier Loading Strategy:
+ * 1. Local pre-downloaded models (for production/Docker)
+ * 2. models.soulcraft.com CDN (our controlled source)
+ * 3. GitHub Releases (immutable backup)
+ * 4. Hugging Face (final fallback)
+ */
+import { createHash } from 'crypto';
+import { existsSync, readFileSync, writeFileSync, mkdirSync } from 'fs';
+import { join, dirname } from 'path';
+// @ts-ignore - Transformers.js types
+import { pipeline as pipelineImport } from '@huggingface/transformers';
+// Critical model configuration
+const CRITICAL_MODEL = {
+    name: 'Xenova/all-MiniLM-L6-v2',
+    dimensions: 384,
+    // SHA256 of the ONNX model file for verification
+    modelSHA256: '759c3cd2b7fe7e93933ad23c4c9181b7396442a2ed746ec7c1d46192c469c46e',
+    tarballSHA256: '2692ac9f3493b5dd78327c36380f3aac395096aed918ccfd4faadb152cd410e0'
+};
+// Model sources in priority order
+const MODEL_SOURCES = [
+    // 1. models.soulcraft.com - Our CDN (fastest, we control)
+    {
+        name: 'models.soulcraft.com',
+        url: 'https://models.soulcraft.com/models/Xenova/all-MiniLM-L6-v2/',
+        files: {
+            'onnx/model.onnx': 'onnx/model.onnx',
+            'tokenizer.json': 'tokenizer.json',
+            'tokenizer_config.json': 'tokenizer_config.json',
+            'config.json': 'config.json'
+        }
+    },
+    // 2. GitHub Releases - Immutable backup
+    {
+        name: 'GitHub Releases',
+        url: 'https://github.com/soulcraftlabs/brainy/releases/download/models-v1.0.0/',
+        tarball: 'all-MiniLM-L6-v2.tar.gz'
+    },
+    // 3. Hugging Face will be handled by transformers.js as final fallback
+];
+/**
+ * Verify file integrity with SHA256
+ */
+function verifyFileSHA256(filePath, expectedHash) {
+    try {
+        const fileBuffer = readFileSync(filePath);
+        const hash = createHash('sha256').update(fileBuffer).digest('hex');
+        return hash === expectedHash;
+    }
+    catch (error) {
+        return false;
+    }
+}
+/**
+ * Download file from URL
+ */
+async function downloadFile(url, destPath) {
+    try {
+        const response = await fetch(url);
+        if (!response.ok)
+            return false;
+        const buffer = await response.arrayBuffer();
+        const dir = dirname(destPath);
+        if (!existsSync(dir)) {
+            mkdirSync(dir, { recursive: true });
+        }
+        writeFileSync(destPath, Buffer.from(buffer));
+        return true;
+    }
+    catch (error) {
+        console.warn(`Failed to download from ${url}:`, error);
+        return false;
+    }
+}
+/**
+ * Check if models are pre-downloaded locally
+ */
+function checkLocalModels(cacheDir) {
+    const modelPath = join(cacheDir, CRITICAL_MODEL.name, 'onnx', 'model.onnx');
+    const tokenizerPath = join(cacheDir, CRITICAL_MODEL.name, 'tokenizer.json');
+    // Check if critical files exist
+    if (!existsSync(modelPath) || !existsSync(tokenizerPath)) {
+        return false;
+    }
+    // Verify model integrity if possible
+    const isValid = verifyFileSHA256(modelPath, CRITICAL_MODEL.modelSHA256);
+    if (!isValid) {
+        console.warn('Local model found but SHA256 verification failed - will re-download');
+        return false;
+    }
+    return true;
+}
+/**
+ * Download models from our CDN sources before falling back to Hugging Face
+ */
+async function downloadModelsFromCDN(cacheDir) {
+    const modelDir = join(cacheDir, CRITICAL_MODEL.name);
+    // Try each source in order
+    for (const source of MODEL_SOURCES) {
+        console.log(`Attempting to download models from ${source.name}...`);
+        if (source.files) {
+            // Download individual files
+            let success = true;
+            for (const [file, remotePath] of Object.entries(source.files)) {
+                const url = source.url + remotePath;
+                const destPath = join(modelDir, file);
+                if (!await downloadFile(url, destPath)) {
+                    success = false;
+                    break;
+                }
+            }
+            if (success) {
+                console.log(`✅ Successfully downloaded models from ${source.name}`);
+                // Verify the critical model file
+                const modelPath = join(modelDir, 'onnx', 'model.onnx');
+                if (verifyFileSHA256(modelPath, CRITICAL_MODEL.modelSHA256)) {
+                    console.log('✅ Model integrity verified with SHA256');
+                    return true;
+                }
+                else {
+                    console.warn('⚠️ Model downloaded but SHA256 verification failed');
+                }
+            }
+        }
+        else if (source.tarball) {
+            // Download and extract tarball (implementation would go here)
+            // For now, skip tarball sources as they need extraction
+            console.log(`Skipping ${source.name} (tarball extraction not yet implemented)`);
+        }
+    }
+    return false;
+}
+/**
+ * Enhanced pipeline function with 3-tier model loading
+ * This wraps the transformers.js pipeline to ensure models are available
+ */
+export async function criticalPipeline(task, // Task type from transformers.js
+model = CRITICAL_MODEL.name, options = {}) {
+    // Determine cache directory
+    const cacheDir = options.cache_dir || './models';
+    // Step 1: Check if models are pre-downloaded locally
+    if (checkLocalModels(cacheDir)) {
+        console.log('✅ Using pre-downloaded models (production mode)');
+        // Models exist locally, transformers.js will use them
+        return pipelineImport(task, model, {
+            ...options,
+            local_files_only: true // Force local usage
+        });
+    }
+    // Step 2: Try to download from our CDN sources
+    console.log('📥 Models not found locally, downloading from CDN...');
+    const cdnSuccess = await downloadModelsFromCDN(cacheDir);
+    if (cdnSuccess) {
+        // Models downloaded from CDN, use them locally
+        return pipelineImport(task, model, {
+            ...options,
+            local_files_only: true // Force local usage
+        });
+    }
+    // Step 3: Fall back to Hugging Face (transformers.js default)
+    console.log('⚠️ CDN sources unavailable, falling back to Hugging Face...');
+    return pipelineImport(task, model, {
+        ...options,
+        local_files_only: false // Allow remote download from Hugging Face
+    });
+}
+/**
+ * Pre-download models for production deployment
+ * This ensures models are available without runtime downloads
+ */
+export async function preDownloadModels(cacheDir = './models') {
+    console.log('🚀 Pre-downloading critical models for production...');
+    console.log(`Model: ${CRITICAL_MODEL.name} (${CRITICAL_MODEL.dimensions} dimensions)`);
+    // Check if already downloaded
+    if (checkLocalModels(cacheDir)) {
+        console.log('✅ Models already downloaded and verified');
+        return true;
+    }
+    // Download from CDN
+    const success = await downloadModelsFromCDN(cacheDir);
+    if (success) {
+        console.log('✅ Models pre-downloaded successfully!');
+        console.log(`Location: ${join(cacheDir, CRITICAL_MODEL.name)}`);
+        return true;
+    }
+    else {
+        console.error('❌ Failed to pre-download models from CDN');
+        console.log('Falling back to Hugging Face download on first use');
+        return false;
+    }
+}
+/**
+ * Get model status and availability
+ */
+export function getModelStatus(cacheDir = './models') {
+    if (checkLocalModels(cacheDir)) {
+        const modelPath = join(cacheDir, CRITICAL_MODEL.name, 'onnx', 'model.onnx');
+        const verified = verifyFileSHA256(modelPath, CRITICAL_MODEL.modelSHA256);
+        return {
+            isAvailable: true,
+            source: 'local',
+            verified,
+            dimensions: CRITICAL_MODEL.dimensions
+        };
+    }
+    return {
+        isAvailable: false,
+        source: 'none',
+        verified: false,
+        dimensions: CRITICAL_MODEL.dimensions
+    };
+}
+//# sourceMappingURL=modelLoader.js.map