@soulcraft/brainy 1.4.0 → 2.0.0

package/dist/utils/modelLoader.d.ts

@@ -1,32 +0,0 @@
-/**
- * Critical Model Loader - Ensures models are ALWAYS available
- *
- * The all-MiniLM-L6-v2 model (384 dimensions) is CRITICAL.
- * Without this exact model, users cannot access their data.
- *
- * 3-Tier Loading Strategy:
- * 1. Local pre-downloaded models (for production/Docker)
- * 2. models.soulcraft.com CDN (our controlled source)
- * 3. GitHub Releases (immutable backup)
- * 4. Hugging Face (final fallback)
- */
-/**
- * Enhanced pipeline function with 3-tier model loading
- * This wraps the transformers.js pipeline to ensure models are available
- */
-export declare function criticalPipeline(task: any, // Task type from transformers.js
-model?: string, options?: any): Promise<any>;
-/**
- * Pre-download models for production deployment
- * This ensures models are available without runtime downloads
- */
-export declare function preDownloadModels(cacheDir?: string): Promise<boolean>;
-/**
- * Get model status and availability
- */
-export declare function getModelStatus(cacheDir?: string): {
-    isAvailable: boolean;
-    source: string;
-    verified: boolean;
-    dimensions: number;
-};

package/dist/utils/modelLoader.js

@@ -1,219 +0,0 @@
-/**
- * Critical Model Loader - Ensures models are ALWAYS available
- *
- * The all-MiniLM-L6-v2 model (384 dimensions) is CRITICAL.
- * Without this exact model, users cannot access their data.
- *
- * 3-Tier Loading Strategy:
- * 1. Local pre-downloaded models (for production/Docker)
- * 2. models.soulcraft.com CDN (our controlled source)
- * 3. GitHub Releases (immutable backup)
- * 4. Hugging Face (final fallback)
- */
-import { createHash } from 'crypto';
-import { existsSync, readFileSync, writeFileSync, mkdirSync } from 'fs';
-import { join, dirname } from 'path';
-// @ts-ignore - Transformers.js types
-import { pipeline as pipelineImport } from '@huggingface/transformers';
-// Critical model configuration
-const CRITICAL_MODEL = {
-    name: 'Xenova/all-MiniLM-L6-v2',
-    dimensions: 384,
-    // SHA256 of the ONNX model file for verification
-    modelSHA256: '759c3cd2b7fe7e93933ad23c4c9181b7396442a2ed746ec7c1d46192c469c46e',
-    tarballSHA256: '2692ac9f3493b5dd78327c36380f3aac395096aed918ccfd4faadb152cd410e0'
-};
-// Model sources in priority order
-const MODEL_SOURCES = [
-    // 1. models.soulcraft.com - Our CDN (fastest, we control)
-    {
-        name: 'models.soulcraft.com',
-        url: 'https://models.soulcraft.com/models/Xenova/all-MiniLM-L6-v2/',
-        files: {
-            'onnx/model.onnx': 'onnx/model.onnx',
-            'tokenizer.json': 'tokenizer.json',
-            'tokenizer_config.json': 'tokenizer_config.json',
-            'config.json': 'config.json'
-        }
-    },
-    // 2. GitHub Releases - Immutable backup
-    {
-        name: 'GitHub Releases',
-        url: 'https://github.com/soulcraftlabs/brainy/releases/download/models-v1.0.0/',
-        tarball: 'all-MiniLM-L6-v2.tar.gz'
-    },
-    // 3. Hugging Face will be handled by transformers.js as final fallback
-];
-/**
- * Verify file integrity with SHA256
- */
-function verifyFileSHA256(filePath, expectedHash) {
-    try {
-        const fileBuffer = readFileSync(filePath);
-        const hash = createHash('sha256').update(fileBuffer).digest('hex');
-        return hash === expectedHash;
-    }
-    catch (error) {
-        return false;
-    }
-}
-/**
- * Download file from URL
- */
-async function downloadFile(url, destPath) {
-    try {
-        const response = await fetch(url);
-        if (!response.ok)
-            return false;
-        const buffer = await response.arrayBuffer();
-        const dir = dirname(destPath);
-        if (!existsSync(dir)) {
-            mkdirSync(dir, { recursive: true });
-        }
-        writeFileSync(destPath, Buffer.from(buffer));
-        return true;
-    }
-    catch (error) {
-        console.warn(`Failed to download from ${url}:`, error);
-        return false;
-    }
-}
-/**
- * Check if models are pre-downloaded locally
- */
-function checkLocalModels(cacheDir) {
-    const modelPath = join(cacheDir, CRITICAL_MODEL.name, 'onnx', 'model.onnx');
-    const tokenizerPath = join(cacheDir, CRITICAL_MODEL.name, 'tokenizer.json');
-    // Check if critical files exist
-    if (!existsSync(modelPath) || !existsSync(tokenizerPath)) {
-        return false;
-    }
-    // Verify model integrity if possible
-    const isValid = verifyFileSHA256(modelPath, CRITICAL_MODEL.modelSHA256);
-    if (!isValid) {
-        console.warn('Local model found but SHA256 verification failed - will re-download');
-        return false;
-    }
-    return true;
-}
-/**
- * Download models from our CDN sources before falling back to Hugging Face
- */
-async function downloadModelsFromCDN(cacheDir) {
-    const modelDir = join(cacheDir, CRITICAL_MODEL.name);
-    // Try each source in order
-    for (const source of MODEL_SOURCES) {
-        console.log(`Attempting to download models from ${source.name}...`);
-        if (source.files) {
-            // Download individual files
-            let success = true;
-            for (const [file, remotePath] of Object.entries(source.files)) {
-                const url = source.url + remotePath;
-                const destPath = join(modelDir, file);
-                if (!await downloadFile(url, destPath)) {
-                    success = false;
-                    break;
-                }
-            }
-            if (success) {
-                console.log(`✅ Successfully downloaded models from ${source.name}`);
-                // Verify the critical model file
-                const modelPath = join(modelDir, 'onnx', 'model.onnx');
-                if (verifyFileSHA256(modelPath, CRITICAL_MODEL.modelSHA256)) {
-                    console.log('✅ Model integrity verified with SHA256');
-                    return true;
-                }
-                else {
-                    console.warn('⚠️ Model downloaded but SHA256 verification failed');
-                }
-            }
-        }
-        else if (source.tarball) {
-            // Download and extract tarball (implementation would go here)
-            // For now, skip tarball sources as they need extraction
-            console.log(`Skipping ${source.name} (tarball extraction not yet implemented)`);
-        }
-    }
-    return false;
-}
-/**
- * Enhanced pipeline function with 3-tier model loading
- * This wraps the transformers.js pipeline to ensure models are available
- */
-export async function criticalPipeline(task, // Task type from transformers.js
-model = CRITICAL_MODEL.name, options = {}) {
-    // Determine cache directory
-    const cacheDir = options.cache_dir || './models';
-    // Step 1: Check if models are pre-downloaded locally
-    if (checkLocalModels(cacheDir)) {
-        console.log('✅ Using pre-downloaded models (production mode)');
-        // Models exist locally, transformers.js will use them
-        return pipelineImport(task, model, {
-            ...options,
-            local_files_only: true // Force local usage
-        });
-    }
-    // Step 2: Try to download from our CDN sources
-    console.log('📥 Models not found locally, downloading from CDN...');
-    const cdnSuccess = await downloadModelsFromCDN(cacheDir);
-    if (cdnSuccess) {
-        // Models downloaded from CDN, use them locally
-        return pipelineImport(task, model, {
-            ...options,
-            local_files_only: true // Force local usage
-        });
-    }
-    // Step 3: Fall back to Hugging Face (transformers.js default)
-    console.log('⚠️ CDN sources unavailable, falling back to Hugging Face...');
-    return pipelineImport(task, model, {
-        ...options,
-        local_files_only: false // Allow remote download from Hugging Face
-    });
-}
-/**
- * Pre-download models for production deployment
- * This ensures models are available without runtime downloads
- */
-export async function preDownloadModels(cacheDir = './models') {
-    console.log('🚀 Pre-downloading critical models for production...');
-    console.log(`Model: ${CRITICAL_MODEL.name} (${CRITICAL_MODEL.dimensions} dimensions)`);
-    // Check if already downloaded
-    if (checkLocalModels(cacheDir)) {
-        console.log('✅ Models already downloaded and verified');
-        return true;
-    }
-    // Download from CDN
-    const success = await downloadModelsFromCDN(cacheDir);
-    if (success) {
-        console.log('✅ Models pre-downloaded successfully!');
-        console.log(`Location: ${join(cacheDir, CRITICAL_MODEL.name)}`);
-        return true;
-    }
-    else {
-        console.error('❌ Failed to pre-download models from CDN');
-        console.log('Falling back to Hugging Face download on first use');
-        return false;
-    }
-}
-/**
- * Get model status and availability
- */
-export function getModelStatus(cacheDir = './models') {
-    if (checkLocalModels(cacheDir)) {
-        const modelPath = join(cacheDir, CRITICAL_MODEL.name, 'onnx', 'model.onnx');
-        const verified = verifyFileSHA256(modelPath, CRITICAL_MODEL.modelSHA256);
-        return {
-            isAvailable: true,
-            source: 'local',
-            verified,
-            dimensions: CRITICAL_MODEL.dimensions
-        };
-    }
-    return {
-        isAvailable: false,
-        source: 'none',
-        verified: false,
-        dimensions: CRITICAL_MODEL.dimensions
-    };
-}
-//# sourceMappingURL=modelLoader.js.map

package/dist/utils/modelManager.d.ts

@@ -1,77 +0,0 @@
-/**
- * Model Manager - CRITICAL SINGLETON for preventing multiple model loads
- *
- * This ensures that ONNX/Transformer models are loaded ONCE and reused across:
- * - All BrainyData instances
- * - All test cases
- * - All embedding operations
- * - Neural intelligence operations
- */
-import { TransformerEmbedding } from './embedding.js';
-import { EmbeddingFunction } from '../coreTypes.js';
-/**
- * Global singleton model manager
- * CRITICAL: This prevents multiple model loads which cause:
- * - Test timeouts
- * - Memory bloat
- * - Performance degradation
- * - ONNX runtime conflicts
- */
-declare class ModelManager {
-    private static instance;
-    private primaryModel;
-    private modelPromise;
-    private isInitialized;
-    private constructor();
-    static getInstance(): ModelManager;
-    /**
-     * Get the primary embedding model - LOADS ONCE, REUSES FOREVER
-     */
-    getPrimaryModel(): Promise<TransformerEmbedding>;
-    /**
-     * Initialize the primary model ONCE - WORKS IN ALL ENVIRONMENTS
-     */
-    private initializePrimaryModel;
-    /**
-     * Get embedding function that reuses the singleton model
-     */
-    getEmbeddingFunction(): Promise<EmbeddingFunction>;
-    /**
-     * Check if model is ready (loaded and initialized)
-     */
-    isModelReady(): boolean;
-    /**
-     * Force model reload (for testing or recovery)
-     */
-    reloadModel(): Promise<void>;
-    /**
-     * Get model status for debugging
-     */
-    getModelStatus(): {
-        loaded: boolean;
-        ready: boolean;
-        modelType: string;
-    };
-    /**
-     * Cleanup method for testing
-     */
-    cleanup(): void;
-}
-export declare const modelManager: ModelManager;
-/**
- * Get the singleton embedding function - USE THIS EVERYWHERE!
- */
-export declare function getSingletonEmbeddingFunction(): Promise<EmbeddingFunction>;
-/**
- * Optimized default embedding function that reuses singleton model
- */
-export declare const optimizedDefaultEmbeddingFunction: EmbeddingFunction;
-/**
- * Preload model for tests or production - CALL THIS ONCE AT START
- */
-export declare function preloadModel(): Promise<void>;
-/**
- * Get model readiness status
- */
-export declare function isModelReady(): boolean;
-export {};

package/dist/utils/modelManager.js

@@ -1,219 +0,0 @@
-/**
- * Model Manager - CRITICAL SINGLETON for preventing multiple model loads
- *
- * This ensures that ONNX/Transformer models are loaded ONCE and reused across:
- * - All BrainyData instances
- * - All test cases
- * - All embedding operations
- * - Neural intelligence operations
- */
-import { TransformerEmbedding } from './embedding.js';
-/**
- * Global singleton model manager
- * CRITICAL: This prevents multiple model loads which cause:
- * - Test timeouts
- * - Memory bloat
- * - Performance degradation
- * - ONNX runtime conflicts
- */
-class ModelManager {
-    constructor() {
-        this.primaryModel = null;
-        this.modelPromise = null;
-        this.isInitialized = false;
-    }
-    static getInstance() {
-        if (!ModelManager.instance) {
-            ModelManager.instance = new ModelManager();
-        }
-        return ModelManager.instance;
-    }
-    /**
-     * Get the primary embedding model - LOADS ONCE, REUSES FOREVER
-     */
-    async getPrimaryModel() {
-        // If already initialized, return immediately
-        if (this.primaryModel && this.isInitialized) {
-            return this.primaryModel;
-        }
-        // If initialization is in progress, wait for it
-        if (this.modelPromise) {
-            return await this.modelPromise;
-        }
-        // Start initialization
-        this.modelPromise = this.initializePrimaryModel();
-        return await this.modelPromise;
-    }
-    /**
-     * Initialize the primary model ONCE - WORKS IN ALL ENVIRONMENTS
-     */
-    async initializePrimaryModel() {
-        try {
-            // Environment detection for optimal configuration
-            const isTest = globalThis.__BRAINY_TEST_ENV__ || process.env.NODE_ENV === 'test';
-            const isBrowser = typeof window !== 'undefined' && typeof document !== 'undefined';
-            const isServerless = typeof process !== 'undefined' && (process.env.VERCEL ||
-                process.env.NETLIFY ||
-                process.env.AWS_LAMBDA_FUNCTION_NAME ||
-                process.env.FUNCTIONS_WORKER_RUNTIME);
-            const isDocker = typeof process !== 'undefined' && (process.env.DOCKER_CONTAINER ||
-                process.env.KUBERNETES_SERVICE_HOST);
-            // Smart configuration based on environment
-            let options = {
-                verbose: !isTest && !isServerless, // Quiet in tests and serverless
-                dtype: 'q8', // Balanced performance/size
-                device: 'cpu' // Most compatible across all environments
-            };
-            // Environment-specific optimizations
-            if (isBrowser) {
-                // Browser: Prioritize compatibility and size
-                options = {
-                    ...options,
-                    localFilesOnly: false, // Allow CDN downloads
-                    dtype: 'q8', // Smaller models for bandwidth
-                    device: 'cpu', // WebGPU support varies widely
-                    verbose: false // Never verbose in browser
-                };
-            }
-            else if (isServerless) {
-                // Serverless: Prioritize cold start performance
-                options = {
-                    ...options,
-                    localFilesOnly: true, // Assume models are pre-deployed
-                    dtype: 'q8', // Faster loading
-                    device: 'cpu', // Consistent across platforms
-                    verbose: false // No console noise in serverless
-                };
-            }
-            else if (isDocker) {
-                // Docker: Assume models are in container
-                options = {
-                    ...options,
-                    localFilesOnly: true, // Models should be baked in
-                    dtype: 'fp32', // Use full precision if available
-                    device: 'auto', // Let it detect GPU in container
-                    verbose: false // Quiet for container logs
-                };
-            }
-            else if (isTest) {
-                // Test: Optimize for reliability and speed
-                options = {
-                    ...options,
-                    localFilesOnly: false, // Allow downloads in development
-                    dtype: 'q8', // Faster test execution
-                    device: 'cpu', // Consistent test results
-                    verbose: false // Clean test output
-                };
-            }
-            else {
-                // Development/Production Node.js: Full features
-                options = {
-                    ...options,
-                    localFilesOnly: false, // Allow downloads
-                    dtype: 'q8', // Good balance
-                    device: 'auto', // Use best available
-                    verbose: true // Helpful for development
-                };
-            }
-            // Graceful logging based on environment
-            const environmentName = isBrowser ? 'browser' :
-                isServerless ? 'serverless' :
-                    isDocker ? 'container' :
-                        isTest ? 'test' : 'node';
-            if (options.verbose) {
-                console.log(`🧠 Loading embedding model (${environmentName} mode)...`);
-            }
-            this.primaryModel = new TransformerEmbedding(options);
-            await this.primaryModel.init();
-            this.isInitialized = true;
-            this.modelPromise = null; // Clear the promise
-            if (options.verbose) {
-                console.log(`✅ Embedding model loaded successfully in ${environmentName} environment`);
-            }
-            return this.primaryModel;
-        }
-        catch (error) {
-            this.modelPromise = null; // Clear failed promise
-            // Enhanced error reporting with environment context
-            const errorMessage = error instanceof Error ? error.message : String(error);
-            const environmentInfo = typeof window !== 'undefined' ? 'browser' :
-                typeof process !== 'undefined' ? `node (${process.version})` : 'unknown';
-            throw new Error(`Failed to initialize embedding model in ${environmentInfo} environment: ${errorMessage}. ` +
-                `This is critical for all Brainy operations. Please check model availability and network connectivity.`);
-        }
-    }
-    /**
-     * Get embedding function that reuses the singleton model
-     */
-    async getEmbeddingFunction() {
-        const model = await this.getPrimaryModel();
-        return async (data) => {
-            return await model.embed(data);
-        };
-    }
-    /**
-     * Check if model is ready (loaded and initialized)
-     */
-    isModelReady() {
-        return this.isInitialized && this.primaryModel !== null;
-    }
-    /**
-     * Force model reload (for testing or recovery)
-     */
-    async reloadModel() {
-        this.primaryModel = null;
-        this.isInitialized = false;
-        this.modelPromise = null;
-        await this.getPrimaryModel();
-    }
-    /**
-     * Get model status for debugging
-     */
-    getModelStatus() {
-        return {
-            loaded: this.primaryModel !== null,
-            ready: this.isInitialized,
-            modelType: 'TransformerEmbedding (ONNX)'
-        };
-    }
-    /**
-     * Cleanup method for testing
-     */
-    cleanup() {
-        // Note: We don't actually destroy the model in cleanup
-        // because other parts of the system might still be using it
-        // This is mainly for status reset
-        console.log('🧹 Model manager cleanup requested (model remains loaded)');
-    }
-}
-ModelManager.instance = null;
-// Export singleton instance
-export const modelManager = ModelManager.getInstance();
-/**
- * Get the singleton embedding function - USE THIS EVERYWHERE!
- */
-export async function getSingletonEmbeddingFunction() {
-    return await modelManager.getEmbeddingFunction();
-}
-/**
- * Optimized default embedding function that reuses singleton model
- */
-export const optimizedDefaultEmbeddingFunction = async (data) => {
-    const embeddingFn = await getSingletonEmbeddingFunction();
-    return await embeddingFn(data);
-};
-/**
- * Preload model for tests or production - CALL THIS ONCE AT START
- */
-export async function preloadModel() {
-    console.log('🚀 Preloading embedding model...');
-    await modelManager.getPrimaryModel();
-    console.log('✅ Model preloaded and ready!');
-}
-/**
- * Get model readiness status
- */
-export function isModelReady() {
-    return modelManager.isModelReady();
-}
-//# sourceMappingURL=modelManager.js.map