@soulcraft/brainy 6.5.0 → 6.6.1

package/dist/critical/model-guardian.js

@@ -5,51 +5,24 @@
  * Without the exact model, users CANNOT access their data
  *
  * Requirements:
- * 1. Model MUST be Xenova/all-MiniLM-L6-v2 (never changes)
- * 2. Model MUST be available at runtime
+ * 1. Model MUST be all-MiniLM-L6-v2-q8 (bundled in package)
+ * 2. Model MUST be available at runtime (embedded in npm package)
  * 3. Model MUST produce consistent 384-dim embeddings
  * 4. System MUST fail fast if model unavailable in production
  */
-import { env } from '@huggingface/transformers';
+import { WASMEmbeddingEngine } from '../embeddings/wasm/index.js';
 // CRITICAL: These values MUST NEVER CHANGE
 const CRITICAL_MODEL_CONFIG = {
-    modelName: 'Xenova/all-MiniLM-L6-v2',
-    modelHash: {
-        // SHA256 of model.onnx - computed from actual model
-        'onnx/model.onnx': 'add_actual_hash_here',
-        'tokenizer.json': 'add_actual_hash_here'
-    },
-    modelSize: {
-        'onnx/model.onnx': 90387606, // Exact size in bytes (updated to match actual file)
-        'tokenizer.json': 711661
-    },
+    modelName: 'all-MiniLM-L6-v2-q8',
     embeddingDimensions: 384,
-    fallbackSources: [
-        // Primary: Our Google Cloud Storage CDN (we control this, fastest)
-        {
-            name: 'Soulcraft CDN (Primary)',
-            url: 'https://models.soulcraft.com/models/all-MiniLM-L6-v2.tar.gz',
-            type: 'tarball'
-        },
-        // Secondary: GitHub releases backup
-        {
-            name: 'GitHub Backup',
-            url: 'https://github.com/soulcraftlabs/brainy-models/releases/download/v1.0.0/all-MiniLM-L6-v2.tar.gz',
-            type: 'tarball'
-        },
-        // Tertiary: Hugging Face (original source)
-        {
-            name: 'Hugging Face',
-            url: 'huggingface',
-            type: 'transformers'
-        }
-    ]
+    // Model is bundled in package - no external downloads needed
+    bundled: true
 };
 export class ModelGuardian {
     constructor() {
         this.isVerified = false;
         this.lastVerification = null;
-        this.modelPath = this.detectModelPath();
+        // Model is bundled - no path detection needed
     }
     static getInstance() {
         if (!ModelGuardian.instance) {
@@ -62,200 +35,55 @@ export class ModelGuardian {
      * This MUST be called before any embedding operations
      */
     async ensureCriticalModel() {
-        console.log('DEBUG: ensureCriticalModel called');
-        console.log('🛡️ MODEL GUARDIAN: Verifying critical model availability...');
-        console.log(`🚀 Debug: Model path: ${this.modelPath}`);
-        console.log(`🚀 Debug: Already verified: ${this.isVerified}`);
         // Check if already verified in this session
         if (this.isVerified && this.lastVerification) {
             const hoursSinceVerification = (Date.now() - this.lastVerification.getTime()) / (1000 * 60 * 60);
             if (hoursSinceVerification < 24) {
-                console.log('✅ Model previously verified in this session');
                 return;
             }
         }
-        // Step 1: Check if model exists locally
-        console.log('🔍 Debug: Calling verifyLocalModel()');
-        const modelExists = await this.verifyLocalModel();
-        if (modelExists) {
-            console.log('✅ Critical model verified locally');
+        // Verify the bundled WASM model works
+        const modelWorks = await this.verifyBundledModel();
+        if (modelWorks) {
             this.isVerified = true;
             this.lastVerification = new Date();
-            this.configureTransformers();
             return;
         }
-        // Step 2: In production, FAIL FAST (Node.js only)
-        if (typeof window === 'undefined' && process.env.NODE_ENV === 'production' && !process.env.BRAINY_ALLOW_RUNTIME_DOWNLOAD) {
-            throw new Error('🚨 CRITICAL FAILURE: Transformer model not found in production!\n' +
-                'The model is REQUIRED for Brainy to function.\n' +
-                'Users CANNOT access their data without it.\n' +
-                'Solution: Run "npm run download-models" during build stage.');
-        }
-        // Step 3: Attempt to download from fallback sources
-        console.warn('⚠️ Model not found locally, attempting download...');
-        for (const source of CRITICAL_MODEL_CONFIG.fallbackSources) {
-            try {
-                console.log(`📥 Trying ${source.name}...`);
-                await this.downloadFromSource(source);
-                // Verify the download
-                if (await this.verifyLocalModel()) {
-                    console.log(`✅ Successfully downloaded from ${source.name}`);
-                    this.isVerified = true;
-                    this.lastVerification = new Date();
-                    this.configureTransformers();
-                    return;
-                }
-            }
-            catch (error) {
-                console.warn(`❌ ${source.name} failed:`, error.message);
-            }
-        }
-        // Step 4: CRITICAL FAILURE
-        throw new Error('🚨 CRITICAL FAILURE: Cannot obtain transformer model!\n' +
-            'Tried all fallback sources.\n' +
-            'Brainy CANNOT function without the model.\n' +
-            'Users CANNOT access their data.\n' +
-            'Please check network connectivity or pre-download models.');
+        // CRITICAL FAILURE
+        throw new Error('🚨 CRITICAL FAILURE: Bundled transformer model not working!\n' +
+            'The model is REQUIRED for Brainy to function.\n' +
+            'Users CANNOT access their data without it.\n' +
+            'This indicates a package installation issue.');
     }
     /**
-     * Verify the local model files exist and are correct
+     * Verify the bundled WASM model works correctly
      */
-    async verifyLocalModel() {
-        // Browser doesn't have local file access
-        if (typeof window !== 'undefined') {
-            console.log('⚠️ Model verification skipped in browser environment');
-            return false;
-        }
-        // Dynamically import Node.js modules
-        const fs = await import('node:fs');
-        const fsPromises = await import('node:fs/promises');
-        const path = await import('node:path');
-        const modelBasePath = path.join(this.modelPath, ...CRITICAL_MODEL_CONFIG.modelName.split('/'));
-        console.log(`🔍 Debug: Checking model at path: ${modelBasePath}`);
-        console.log(`🔍 Debug: Model path components: ${this.modelPath} + ${CRITICAL_MODEL_CONFIG.modelName.split('/')}`);
-        // Check critical files
-        const criticalFiles = [
-            'onnx/model.onnx',
-            'tokenizer.json',
-            'config.json'
-        ];
-        for (const file of criticalFiles) {
-            const filePath = path.join(modelBasePath, file);
-            console.log(`🔍 Debug: Checking file: ${filePath}`);
-            if (!fs.existsSync(filePath)) {
-                console.log(`❌ Missing critical file: ${file} at ${filePath}`);
+    async verifyBundledModel() {
+        try {
+            const engine = WASMEmbeddingEngine.getInstance();
+            // Initialize the engine (loads bundled model)
+            await engine.initialize();
+            // Test embedding generation
+            const testEmbedding = await engine.embed('test verification');
+            // Verify dimensions
+            if (testEmbedding.length !== CRITICAL_MODEL_CONFIG.embeddingDimensions) {
+                console.error(`❌ CRITICAL: Model dimension mismatch!\n` +
+                    `Expected: ${CRITICAL_MODEL_CONFIG.embeddingDimensions}\n` +
+                    `Got: ${testEmbedding.length}`);
                 return false;
             }
-            // Verify size for critical files
-            if (CRITICAL_MODEL_CONFIG.modelSize[file]) {
-                const stats = await fsPromises.stat(filePath);
-                const expectedSize = CRITICAL_MODEL_CONFIG.modelSize[file];
-                if (Math.abs(stats.size - expectedSize) > 1000) { // Allow 1KB variance
-                    console.error(`❌ CRITICAL: Model file size mismatch!\n` +
-                        `File: ${file}\n` +
-                        `Expected: ${expectedSize} bytes\n` +
-                        `Actual: ${stats.size} bytes\n` +
-                        `This indicates model corruption or version mismatch!`);
-                    return false;
-                }
-            }
-            // SHA256 verification for ultimate security
-            if (CRITICAL_MODEL_CONFIG.modelHash && CRITICAL_MODEL_CONFIG.modelHash[file]) {
-                const hash = await this.computeFileHash(filePath);
-                if (hash !== CRITICAL_MODEL_CONFIG.modelHash[file]) {
-                    console.error(`❌ CRITICAL: Model hash mismatch for ${file}!\n` +
-                        `Expected: ${CRITICAL_MODEL_CONFIG.modelHash[file]}\n` +
-                        `Got: ${hash}\n` +
-                        `This indicates model tampering or corruption!`);
-                    return false;
-                }
+            // Verify normalization (should be unit length)
+            const norm = Math.sqrt(testEmbedding.reduce((sum, v) => sum + v * v, 0));
+            if (Math.abs(norm - 1.0) > 0.01) {
+                console.error(`❌ CRITICAL: Embeddings not normalized! Norm: ${norm}`);
+                return false;
             }
-        }
-        return true;
-    }
-    /**
-     * Compute SHA256 hash of a file
-     */
-    async computeFileHash(filePath) {
-        try {
-            const { readFile } = await import('node:fs/promises');
-            const { createHash } = await import('node:crypto');
-            const fileBuffer = await readFile(filePath);
-            const hash = createHash('sha256').update(fileBuffer).digest('hex');
-            return hash;
+            return true;
         }
         catch (error) {
-            console.error(`Failed to compute hash for ${filePath}:`, error);
-            return '';
-        }
-    }
-    /**
-     * Download model from a fallback source
-     */
-    async downloadFromSource(source) {
-        if (source.type === 'transformers') {
-            // Use transformers.js native download
-            const { pipeline } = await import('@huggingface/transformers');
-            env.cacheDir = this.modelPath;
-            env.allowRemoteModels = true;
-            const extractor = await pipeline('feature-extraction', CRITICAL_MODEL_CONFIG.modelName);
-            // Test the model
-            const test = await extractor('test', { pooling: 'mean', normalize: true });
-            if (test.data.length !== CRITICAL_MODEL_CONFIG.embeddingDimensions) {
-                throw new Error(`CRITICAL: Model dimension mismatch! ` +
-                    `Expected ${CRITICAL_MODEL_CONFIG.embeddingDimensions}, ` +
-                    `got ${test.data.length}`);
-            }
-        }
-        else if (source.type === 'tarball') {
-            // Tarball extraction would require additional dependencies
-            // Skip this source and try next fallback
-            console.warn(`⚠️ Tarball extraction not available for ${source.name}. Trying next source...`);
-            return; // Will continue to next source in the loop
-        }
-    }
-    /**
-     * Configure transformers.js to use verified local model
-     */
-    configureTransformers() {
-        env.localModelPath = this.modelPath;
-        env.allowRemoteModels = false; // Force local only after verification
-        console.log('🔒 Transformers configured to use verified local model');
-    }
-    /**
-     * Detect where models should be stored
-     */
-    detectModelPath() {
-        // Browser always uses default path
-        if (typeof window !== 'undefined') {
-            return './models';
-        }
-        // Use require for synchronous access in Node.js
-        try {
-            const fs = require('node:fs');
-            const path = require('node:path');
-            const candidates = [
-                process.env.BRAINY_MODELS_PATH,
-                './models',
-                path.join(process.cwd(), 'models'),
-                path.join(process.env.HOME || '', '.brainy', 'models'),
-                '/opt/models', // Lambda/container path
-                env.cacheDir
-            ];
-            for (const candidatePath of candidates) {
-                if (candidatePath && fs.existsSync(candidatePath)) {
-                    const modelPath = path.join(candidatePath, ...CRITICAL_MODEL_CONFIG.modelName.split('/'));
-                    if (fs.existsSync(path.join(modelPath, 'onnx', 'model.onnx'))) {
-                        return candidatePath; // Return the models directory, not its parent
-                    }
-                }
-            }
-        }
-        catch (e) {
-            // If Node.js modules not available, return default
+            console.error('❌ Model verification failed:', error);
+            return false;
         }
-        // Default
-        return './models';
     }
     /**
      * Get model status for diagnostics
@@ -263,10 +91,10 @@ export class ModelGuardian {
     async getStatus() {
         return {
             verified: this.isVerified,
-            path: this.modelPath,
             lastVerification: this.lastVerification,
             modelName: CRITICAL_MODEL_CONFIG.modelName,
-            dimensions: CRITICAL_MODEL_CONFIG.embeddingDimensions
+            dimensions: CRITICAL_MODEL_CONFIG.embeddingDimensions,
+            bundled: CRITICAL_MODEL_CONFIG.bundled
         };
     }
     /**

package/dist/embeddings/EmbeddingManager.d.ts

@@ -2,18 +2,14 @@
  * Unified Embedding Manager
  *
  * THE single source of truth for all embedding operations in Brainy.
- * Combines model management, precision configuration, and embedding generation
- * into one clean, maintainable class.
+ * Uses direct ONNX WASM inference for universal compatibility.
  *
  * Features:
  * - Singleton pattern ensures ONE model instance
- * - Automatic Q8 (default) or FP32 precision
- * - Model downloading and caching
- * - Thread-safe initialization
+ * - Direct ONNX WASM (no transformers.js dependency)
+ * - Bundled model (no runtime downloads)
+ * - Works everywhere: Node.js, Bun, Bun --compile, browsers
  * - Memory monitoring
- *
- * This replaces: SingletonModelManager, TransformerEmbedding, ModelPrecisionManager,
- * hybridModelManager, universalMemoryManager, and more.
  */
 import { Vector, EmbeddingFunction } from '../coreTypes.js';
 export type ModelPrecision = 'q8' | 'fp32';
@@ -27,9 +23,11 @@ interface EmbeddingStats {
 }
 /**
  * Unified Embedding Manager - Clean, simple, reliable
+ *
+ * Now powered by direct ONNX WASM for universal compatibility.
  */
 export declare class EmbeddingManager {
-    private model;
+    private engine;
     private precision;
     private modelName;
     private initialized;
@@ -61,14 +59,6 @@ export declare class EmbeddingManager {
      * Get embedding function for compatibility
      */
     getEmbeddingFunction(): EmbeddingFunction;
-    /**
-     * Get models directory path
-     * Note: In browser environments, returns a simple default path
-     * In Node.js, checks multiple locations for the models directory
-     */
-    private getModelsPath;
-    private modelsPathCache;
-    private resolveModelsPathSync;
     /**
      * Get memory usage in MB
      */

package/dist/embeddings/EmbeddingManager.js

@@ -2,39 +2,34 @@
  * Unified Embedding Manager
  *
  * THE single source of truth for all embedding operations in Brainy.
- * Combines model management, precision configuration, and embedding generation
- * into one clean, maintainable class.
+ * Uses direct ONNX WASM inference for universal compatibility.
  *
  * Features:
  * - Singleton pattern ensures ONE model instance
- * - Automatic Q8 (default) or FP32 precision
- * - Model downloading and caching
- * - Thread-safe initialization
+ * - Direct ONNX WASM (no transformers.js dependency)
+ * - Bundled model (no runtime downloads)
+ * - Works everywhere: Node.js, Bun, Bun --compile, browsers
  * - Memory monitoring
- *
- * This replaces: SingletonModelManager, TransformerEmbedding, ModelPrecisionManager,
- * hybridModelManager, universalMemoryManager, and more.
  */
-import { pipeline, env } from '@huggingface/transformers';
-import { isNode } from '../utils/environment.js';
+import { WASMEmbeddingEngine } from './wasm/index.js';
 // Global state for true singleton across entire process
 let globalInstance = null;
 let globalInitPromise = null;
 /**
  * Unified Embedding Manager - Clean, simple, reliable
+ *
+ * Now powered by direct ONNX WASM for universal compatibility.
  */
 export class EmbeddingManager {
     constructor() {
-        this.model = null;
-        this.modelName = 'Xenova/all-MiniLM-L6-v2';
+        this.precision = 'q8';
+        this.modelName = 'all-MiniLM-L6-v2';
         this.initialized = false;
         this.initTime = null;
         this.embedCount = 0;
         this.locked = false;
-        this.modelsPathCache = null;
-        // Always use Q8 for optimal size/performance (99% accuracy, 75% smaller)
-        this.precision = 'q8';
-        console.log(`🎯 EmbeddingManager: Using Q8 precision`);
+        this.engine = WASMEmbeddingEngine.getInstance();
+        console.log('🎯 EmbeddingManager: Using Q8 precision (WASM)');
     }
     /**
      * Get the singleton instance
@@ -50,9 +45,10 @@ export class EmbeddingManager {
      */
     async init() {
         // In unit test mode, skip real model initialization
-        const isTestMode = process.env.BRAINY_UNIT_TEST === 'true' || globalThis.__BRAINY_UNIT_TEST__;
+        const isTestMode = process.env.BRAINY_UNIT_TEST === 'true' ||
+            globalThis.__BRAINY_UNIT_TEST__;
         if (isTestMode) {
-            // Production safeguard: Warn if mock mode is active but NODE_ENV is production
+            // Production safeguard
             if (process.env.NODE_ENV === 'production') {
                 throw new Error('CRITICAL: Mock embeddings detected in production environment! ' +
                     'BRAINY_UNIT_TEST or __BRAINY_UNIT_TEST__ is set while NODE_ENV=production. ' +
@@ -66,7 +62,7 @@ export class EmbeddingManager {
             return;
         }
         // Already initialized
-        if (this.initialized && this.model) {
+        if (this.initialized && this.engine.isInitialized()) {
             return;
         }
         // Initialization in progress
@@ -88,62 +84,20 @@ export class EmbeddingManager {
      */
     async performInit() {
         const startTime = Date.now();
-        console.log(`🚀 Initializing embedding model (${this.precision.toUpperCase()})...`);
         try {
-            // Configure transformers.js environment
-            const modelsPath = this.getModelsPath();
-            env.cacheDir = modelsPath;
-            env.allowLocalModels = true;
-            env.useFSCache = true;
-            // Check if models exist locally (only in Node.js)
-            if (isNode()) {
-                try {
-                    const nodeRequire = typeof require !== 'undefined' ? require : null;
-                    if (nodeRequire) {
-                        const path = nodeRequire('node:path');
-                        const fs = nodeRequire('node:fs');
-                        const modelPath = path.join(modelsPath, ...this.modelName.split('/'));
-                        const hasLocalModels = fs.existsSync(modelPath);
-                        if (hasLocalModels) {
-                            console.log('✅ Using cached models from:', modelPath);
-                        }
-                    }
-                }
-                catch {
-                    // Silently continue if require fails
-                }
-            }
-            // Configure pipeline options for the selected precision
-            const pipelineOptions = {
-                cache_dir: modelsPath,
-                local_files_only: false,
-                // Always use Q8 precision
-                dtype: 'q8',
-                quantized: true,
-                // Memory optimizations
-                session_options: {
-                    enableCpuMemArena: false,
-                    enableMemPattern: false,
-                    interOpNumThreads: 1,
-                    intraOpNumThreads: 1,
-                    graphOptimizationLevel: 'disabled'
-                }
-            };
-            // Load the model
-            this.model = await pipeline('feature-extraction', this.modelName, pipelineOptions);
+            // Initialize WASM engine (handles all model loading)
+            await this.engine.initialize();
             // Lock precision after successful initialization
             this.locked = true;
             this.initialized = true;
             this.initTime = Date.now() - startTime;
             // Log success
             const memoryMB = this.getMemoryUsage();
-            console.log(`✅ Model loaded in ${this.initTime}ms`);
             console.log(`📊 Precision: Q8 | Memory: ${memoryMB}MB`);
-            console.log(`🔒 Configuration locked`);
+            console.log('🔒 Configuration locked');
         }
         catch (error) {
             this.initialized = false;
-            this.model = null;
             throw new Error(`Failed to initialize embedding model: ${error instanceof Error ? error.message : String(error)}`);
         }
     }
@@ -151,10 +105,10 @@ export class EmbeddingManager {
      * Generate embeddings
      */
     async embed(text) {
-        // Check for unit test environment - use mocks to prevent ONNX conflicts
-        const isTestMode = process.env.BRAINY_UNIT_TEST === 'true' || globalThis.__BRAINY_UNIT_TEST__;
+        // Check for unit test environment
+        const isTestMode = process.env.BRAINY_UNIT_TEST === 'true' ||
+            globalThis.__BRAINY_UNIT_TEST__;
         if (isTestMode) {
-            // Production safeguard
             if (process.env.NODE_ENV === 'production') {
                 throw new Error('CRITICAL: Mock embeddings in production!');
             }
@@ -162,38 +116,26 @@ export class EmbeddingManager {
         }
         // Ensure initialized
         await this.init();
-        if (!this.model) {
-            throw new Error('Model not initialized');
-        }
-        // CRITICAL FIX: Ensure input is always a string
+        // Normalize input to string
         let input;
         if (Array.isArray(text)) {
-            // Join array elements, converting each to string first
-            input = text.map(t => typeof t === 'string' ? t : String(t)).join(' ');
+            input = text.map((t) => (typeof t === 'string' ? t : String(t))).join(' ');
         }
         else if (typeof text === 'string') {
             input = text;
         }
         else if (typeof text === 'object') {
-            // Convert object to string representation
             input = JSON.stringify(text);
         }
         else {
-            // This shouldn't happen but let's be defensive
             console.warn('EmbeddingManager.embed received unexpected input type:', typeof text);
             input = String(text);
         }
-        // Generate embedding
-        const output = await this.model(input, {
-            pooling: 'mean',
-            normalize: true
-        });
-        // Extract embedding vector
-        const embedding = Array.from(output.data);
+        // Generate embedding using WASM engine
+        const embedding = await this.engine.embed(input);
         // Validate dimensions
         if (embedding.length !== 384) {
             console.warn(`Unexpected embedding dimension: ${embedding.length}`);
-            // Pad or truncate
             if (embedding.length < 384) {
                 return [...embedding, ...new Array(384 - embedding.length).fill(0)];
             }
@@ -208,7 +150,6 @@ export class EmbeddingManager {
      * Generate mock embeddings for unit tests
      */
     getMockEmbedding(text) {
-        // Use the same mock logic as setup-unit.ts for consistency
         const input = Array.isArray(text) ? text.join(' ') : text;
         const str = typeof input === 'string' ? input : JSON.stringify(input);
         const vector = new Array(384).fill(0);
@@ -220,7 +161,6 @@ export class EmbeddingManager {
         for (let i = 0; i < 384; i++) {
             vector[i] += Math.sin(i * 0.1 + str.length) * 0.1;
         }
-        // Track mock embedding count
         this.embedCount++;
         return vector;
     }
@@ -232,55 +172,6 @@ export class EmbeddingManager {
             return await this.embed(data);
         };
     }
-    /**
-     * Get models directory path
-     * Note: In browser environments, returns a simple default path
-     * In Node.js, checks multiple locations for the models directory
-     */
-    getModelsPath() {
-        // In browser environments, use a default path
-        if (!isNode()) {
-            return './models';
-        }
-        // Node.js-specific model path resolution
-        // Cache the result for performance
-        if (!this.modelsPathCache) {
-            this.modelsPathCache = this.resolveModelsPathSync();
-        }
-        return this.modelsPathCache;
-    }
-    resolveModelsPathSync() {
-        // For Node.js environments, we can safely assume these modules exist
-        // TypeScript will handle the imports at build time
-        // At runtime, these will only be called if isNode() is true
-        // Default fallback path
-        const defaultPath = './models';
-        try {
-            // Create a conditional require function that only works in Node
-            const nodeRequire = typeof require !== 'undefined' ? require : null;
-            if (!nodeRequire)
-                return defaultPath;
-            const fs = nodeRequire('node:fs');
-            const path = nodeRequire('node:path');
-            const paths = [
-                process.env.BRAINY_MODELS_PATH,
-                './models',
-                path.join(process.cwd(), 'models'),
-                path.join(process.env.HOME || '', '.brainy', 'models')
-            ];
-            for (const p of paths) {
-                if (p && fs.existsSync(p)) {
-                    return p;
-                }
-            }
-            // Default Node.js path
-            return path.join(process.cwd(), 'models');
-        }
-        catch {
-            // Fallback if require fails
-            return defaultPath;
-        }
-    }
     /**
      * Get memory usage in MB
      */
@@ -295,13 +186,14 @@ export class EmbeddingManager {
      * Get current statistics
      */
     getStats() {
+        const engineStats = this.engine.getStats();
         return {
             initialized: this.initialized,
             precision: this.precision,
             modelName: this.modelName,
-            embedCount: this.embedCount,
+            embedCount: this.embedCount + engineStats.embedCount,
             initTime: this.initTime,
-            memoryMB: this.getMemoryUsage()
+            memoryMB: this.getMemoryUsage(),
         };
     }
     /**