@soulcraft/brainy 6.5.0 → 6.6.1

package/dist/utils/embedding.js

@@ -1,227 +1,23 @@
 /**
- * Embedding functions for converting data to vectors using Transformers.js
- * Complete rewrite to eliminate TensorFlow.js and use ONNX-based models
+ * Embedding functions for converting data to vectors
+ *
+ * Uses direct ONNX WASM for universal compatibility.
+ * No transformers.js dependency - clean, production-grade implementation.
  */
-import { isBrowser } from './environment.js';
-// @ts-ignore - Transformers.js is now the primary embedding library
-import { pipeline, env } from '@huggingface/transformers';
-// CRITICAL: Disable ONNX memory arena to prevent 4-8GB allocation
-// This is needed for BOTH production and testing - reduces memory by 50-75%
-if (typeof process !== 'undefined' && process.env) {
-    process.env.ORT_DISABLE_MEMORY_ARENA = '1';
-    process.env.ORT_DISABLE_MEMORY_PATTERN = '1';
-    // Force single-threaded operation for maximum stability (Node.js 22 LTS)
-    process.env.ORT_INTRA_OP_NUM_THREADS = '1'; // Single thread for operators
-    process.env.ORT_INTER_OP_NUM_THREADS = '1'; // Single thread for sessions
-    process.env.ORT_NUM_THREADS = '1'; // Additional safety override
-}
-/**
- * Detect the best available GPU device for the current environment
- */
-export async function detectBestDevice() {
-    // Browser environment - check for WebGPU support
-    if (isBrowser()) {
-        if (typeof navigator !== 'undefined' && 'gpu' in navigator) {
-            try {
-                const adapter = await navigator.gpu?.requestAdapter();
-                if (adapter) {
-                    return 'webgpu';
-                }
-            }
-            catch (error) {
-                // WebGPU not available or failed to initialize
-            }
-        }
-        return 'cpu';
-    }
-    // Node.js environment - check for CUDA support
-    try {
-        // Check if ONNX Runtime GPU packages are available
-        // This is a simple heuristic - in production you might want more sophisticated detection
-        const hasGpu = process.env.CUDA_VISIBLE_DEVICES !== undefined ||
-            process.env.ONNXRUNTIME_GPU_ENABLED === 'true';
-        return hasGpu ? 'cuda' : 'cpu';
-    }
-    catch (error) {
-        return 'cpu';
-    }
-}
+import { embeddingManager } from '../embeddings/EmbeddingManager.js';
 /**
- * Resolve device string to actual device configuration
+ * TransformerEmbedding - Sentence embeddings using WASM ONNX
+ *
+ * This class delegates all work to EmbeddingManager which uses
+ * the direct ONNX WASM engine. Kept for backward compatibility.
  */
-export async function resolveDevice(device = 'auto') {
-    if (device === 'auto') {
-        return await detectBestDevice();
-    }
-    // Map 'gpu' to appropriate GPU type for current environment
-    if (device === 'gpu') {
-        const detected = await detectBestDevice();
-        return detected === 'cpu' ? 'cpu' : detected;
-    }
-    return device;
-}
 export class TransformerEmbedding {
-    /**
-     * Create a new TransformerEmbedding instance
-     */
     constructor(options = {}) {
-        this.extractor = null;
         this.initialized = false;
-        this.verbose = true;
         this.verbose = options.verbose !== undefined ? options.verbose : true;
-        // PRODUCTION-READY MODEL CONFIGURATION
-        // Priority order: explicit option > environment variable > smart default
-        let localFilesOnly;
-        if (options.localFilesOnly !== undefined) {
-            // 1. Explicit option takes highest priority
-            localFilesOnly = options.localFilesOnly;
-        }
-        else if (process.env.BRAINY_ALLOW_REMOTE_MODELS === 'false') {
-            // 2. Environment variable explicitly disables remote models (legacy support)
-            localFilesOnly = true;
-        }
-        else if (process.env.NODE_ENV === 'development') {
-            // 3. Development mode allows remote models
-            localFilesOnly = false;
-        }
-        else if (isBrowser()) {
-            // 4. Browser defaults to allowing remote models
-            localFilesOnly = false;
-        }
-        else {
-            // 5. Node.js production: try local first, but allow remote as fallback
-            // This is the NEW production-friendly default
-            localFilesOnly = false;
-        }
-        this.options = {
-            model: options.model || 'Xenova/all-MiniLM-L6-v2',
-            verbose: this.verbose,
-            cacheDir: options.cacheDir || './models',
-            localFilesOnly: localFilesOnly,
-            precision: options.precision || 'fp32', // Clean and clear!
-            device: options.device || 'auto'
-        };
-        // ULTRA-CAREFUL: Runtime warnings for q8 usage
-        if (this.options.precision === 'q8') {
-            const confirmed = process.env.BRAINY_Q8_CONFIRMED === 'true';
-            if (!confirmed && this.verbose) {
-                console.warn('🚨 Q8 MODEL WARNING:');
-                console.warn('   • Q8 creates different embeddings than fp32');
-                console.warn('   • Q8 is incompatible with existing fp32 data');
-                console.warn('   • Only use q8 for new projects or when explicitly migrating');
-                console.warn('   • Set BRAINY_Q8_CONFIRMED=true to silence this warning');
-                console.warn('   • Q8 model is 75% smaller but may have slightly reduced accuracy');
-            }
-        }
         if (this.verbose) {
-            this.logger('log', `Embedding config: precision=${this.options.precision}, localFilesOnly=${localFilesOnly}, model=${this.options.model}`);
-        }
-        // Configure transformers.js environment
-        if (!isBrowser()) {
-            // Set cache directory for Node.js
-            env.cacheDir = this.options.cacheDir;
-            // Prioritize local models for offline operation
-            env.allowRemoteModels = !this.options.localFilesOnly;
-            env.allowLocalModels = true;
-        }
-        else {
-            // Browser configuration
-            // Allow both local and remote models, but prefer local if available
-            env.allowLocalModels = true;
-            env.allowRemoteModels = true;
-            // Force the configuration to ensure it's applied
-            if (this.verbose) {
-                this.logger('log', `Browser env config - allowLocalModels: ${env.allowLocalModels}, allowRemoteModels: ${env.allowRemoteModels}, localFilesOnly: ${this.options.localFilesOnly}`);
-            }
-        }
-    }
-    /**
-     * Get the default cache directory for models
-     */
-    async getDefaultCacheDir() {
-        if (isBrowser()) {
-            return './models'; // Browser default
-        }
-        // Check for bundled models in the package
-        const possiblePaths = [
-            // In the installed package
-            './node_modules/@soulcraft/brainy/models',
-            // In development/source
-            './models',
-            './dist/../models',
-            // Alternative locations
-            '../models',
-            '../../models'
-        ];
-        // Check if we're in Node.js and try to find the bundled models
-        if (typeof process !== 'undefined' && process.versions?.node) {
-            try {
-                // Use dynamic import instead of require for ES modules compatibility
-                const { createRequire } = await import('module');
-                const require = createRequire(import.meta.url);
-                const path = require('node:path');
-                const fs = require('node:fs');
-                // Try to resolve the package location
-                try {
-                    const brainyPackagePath = require.resolve('@soulcraft/brainy/package.json');
-                    const brainyPackageDir = path.dirname(brainyPackagePath);
-                    const bundledModelsPath = path.join(brainyPackageDir, 'models');
-                    if (fs.existsSync(bundledModelsPath)) {
-                        this.logger('log', `Using bundled models from package: ${bundledModelsPath}`);
-                        return bundledModelsPath;
-                    }
-                }
-                catch (e) {
-                    // Not installed as package, continue
-                }
-                // Try relative paths from current location
-                for (const relativePath of possiblePaths) {
-                    const fullPath = path.resolve(relativePath);
-                    if (fs.existsSync(fullPath)) {
-                        this.logger('log', `Using bundled models from: ${fullPath}`);
-                        return fullPath;
-                    }
-                }
-            }
-            catch (error) {
-                // Silently fall back to default path if module detection fails
-            }
-        }
-        // Fallback to default cache directory
-        return './models';
-    }
-    /**
-     * Check if we're running in a test environment
-     */
-    isTestEnvironment() {
-        // Always use real implementation - no more mocking
-        return false;
-    }
-    /**
-     * Log message only if verbose mode is enabled
-     */
-    logger(level, message, ...args) {
-        if (level === 'error' || this.verbose) {
-            console[level](`[TransformerEmbedding] ${message}`, ...args);
-        }
-    }
-    /**
-     * Generate mock embeddings for unit tests
-     */
-    getMockEmbedding(data) {
-        // Use the same mock logic as setup-unit.ts for consistency
-        const input = Array.isArray(data) ? data.join(' ') : data;
-        const str = typeof input === 'string' ? input : JSON.stringify(input);
-        const vector = new Array(384).fill(0);
-        // Create semi-realistic embeddings based on text content
-        for (let i = 0; i < Math.min(str.length, 384); i++) {
-            vector[i] = (str.charCodeAt(i % str.length) % 256) / 256;
+            console.log('[TransformerEmbedding] Using WASM ONNX backend (delegating to EmbeddingManager)');
         }
-        // Add position-based variation
-        for (let i = 0; i < 384; i++) {
-            vector[i] += Math.sin(i * 0.1 + str.length) * 0.1;
-        }
-        return vector;
     }
     /**
      * Initialize the embedding model
@@ -230,272 +26,97 @@ export class TransformerEmbedding {
         if (this.initialized) {
             return;
         }
-        // In unit test mode, skip real model initialization to prevent ONNX conflicts
-        if (process.env.BRAINY_UNIT_TEST === 'true' || globalThis.__BRAINY_UNIT_TEST__) {
-            this.initialized = true;
-            this.logger('log', '🧪 Using mocked embeddings for unit tests');
-            return;
-        }
         try {
-            // Resolve device configuration and cache directory
-            const device = await resolveDevice(this.options.device);
-            const cacheDir = this.options.cacheDir === './models'
-                ? await this.getDefaultCacheDir()
-                : this.options.cacheDir;
-            this.logger('log', `Loading Transformer model: ${this.options.model} on device: ${device}`);
-            const startTime = Date.now();
-            // Use the configured precision from EmbeddingManager
-            const { embeddingManager } = await import('../embeddings/EmbeddingManager.js');
-            let actualType = embeddingManager.getPrecision();
-            // CRITICAL: Control which model precision transformers.js uses
-            // Q8 models use quantized int8 weights for 75% size reduction
-            // Always use Q8 for optimal balance
-            actualType = 'q8'; // Always Q8
-            this.logger('log', '🎯 Using Q8 quantized model (75% smaller, 99% accuracy)');
-            // Load the feature extraction pipeline with memory optimizations
-            const pipelineOptions = {
-                cache_dir: cacheDir,
-                local_files_only: isBrowser() ? false : this.options.localFilesOnly,
-                // CRITICAL: Specify dtype for model precision
-                dtype: 'q8',
-                // CRITICAL: For Q8, explicitly use quantized model
-                quantized: true,
-                // CRITICAL: ONNX memory optimizations
-                session_options: {
-                    enableCpuMemArena: false, // Disable pre-allocated memory arena
-                    enableMemPattern: false, // Disable memory pattern optimization
-                    interOpNumThreads: 1, // Force single thread for V8 stability
-                    intraOpNumThreads: 1, // Force single thread for V8 stability
-                    graphOptimizationLevel: 'disabled' // Disable threading optimizations
-                }
-            };
-            // Add device configuration for GPU acceleration
-            if (device !== 'cpu') {
-                pipelineOptions.device = device;
-                this.logger('log', `🚀 GPU acceleration enabled: ${device}`);
-            }
+            await embeddingManager.init();
+            this.initialized = true;
             if (this.verbose) {
-                this.logger('log', `Pipeline options: ${JSON.stringify(pipelineOptions)}`);
+                console.log('[TransformerEmbedding] Initialized via EmbeddingManager (WASM)');
             }
-            try {
-                // For Q8 models, we need to explicitly specify the model file
-                if (actualType === 'q8' && !isBrowser()) {
-                    try {
-                        // Check if quantized model exists (Node.js only)
-                        const { join } = await import('node:path');
-                        const { existsSync } = await import('node:fs');
-                        const modelPath = join(cacheDir, this.options.model, 'onnx', 'model_quantized.onnx');
-                        if (existsSync(modelPath)) {
-                            this.logger('log', '✅ Q8 model found locally');
-                        }
-                        else {
-                            this.logger('warn', '⚠️ Q8 model not found');
-                            actualType = 'q8'; // Always Q8
-                        }
-                    }
-                    catch (error) {
-                        // Skip model path check in browser or if imports fail
-                        this.logger('log', '🌐 Skipping local model check in browser environment');
-                    }
-                }
-                this.extractor = await pipeline('feature-extraction', this.options.model, pipelineOptions);
-            }
-            catch (gpuError) {
-                // Fallback to CPU if GPU initialization fails
-                if (device !== 'cpu') {
-                    this.logger('warn', `GPU initialization failed, falling back to CPU: ${gpuError?.message || gpuError}`);
-                    const cpuOptions = { ...pipelineOptions };
-                    delete cpuOptions.device;
-                    this.extractor = await pipeline('feature-extraction', this.options.model, cpuOptions);
-                }
-                else {
-                    // PRODUCTION-READY ERROR HANDLING
-                    // If local_files_only is true and models are missing, try enabling remote downloads
-                    if (pipelineOptions.local_files_only && gpuError?.message?.includes('local_files_only')) {
-                        this.logger('warn', 'Local models not found, attempting remote download as fallback...');
-                        try {
-                            const remoteOptions = { ...pipelineOptions, local_files_only: false };
-                            this.extractor = await pipeline('feature-extraction', this.options.model, remoteOptions);
-                            this.logger('log', '✅ Successfully downloaded and loaded model from remote');
-                            // Update the configuration to reflect what actually worked
-                            this.options.localFilesOnly = false;
-                        }
-                        catch (remoteError) {
-                            // Both local and remote failed - throw comprehensive error
-                            const errorMsg = `Failed to load embedding model "${this.options.model}". ` +
-                                `Local models not found and remote download failed. ` +
-                                `To fix: 1) Run "npm run download-models", ` +
-                                `2) Check your internet connection, or ` +
-                                `3) Use a custom embedding function.`;
-                            throw new Error(errorMsg);
-                        }
-                    }
-                    else {
-                        throw gpuError;
-                    }
-                }
-            }
-            const loadTime = Date.now() - startTime;
-            this.logger('log', `✅ Model loaded successfully in ${loadTime}ms`);
-            this.initialized = true;
         }
         catch (error) {
-            this.logger('error', 'Failed to initialize Transformer embedding model:', error);
-            throw new Error(`Transformer embedding initialization failed: ${error}`);
+            console.error('[TransformerEmbedding] Failed to initialize:', error);
+            throw new Error(`TransformerEmbedding initialization failed: ${error}`);
         }
     }
     /**
      * Generate embeddings for text data
      */
     async embed(data) {
-        // In unit test mode, return mock embeddings
-        if (process.env.BRAINY_UNIT_TEST === 'true' || globalThis.__BRAINY_UNIT_TEST__) {
-            return this.getMockEmbedding(data);
-        }
         if (!this.initialized) {
             await this.init();
         }
-        try {
-            // Handle different input types
-            let textToEmbed;
-            if (typeof data === 'string') {
-                // Handle empty string case
-                if (data.trim() === '') {
-                    // Return a zero vector of 384 dimensions (all-MiniLM-L6-v2 standard)
-                    return new Array(384).fill(0);
-                }
-                textToEmbed = [data];
-            }
-            else if (Array.isArray(data) && data.every((item) => typeof item === 'string')) {
-                // Handle empty array or array with empty strings
-                if (data.length === 0 || data.every((item) => item.trim() === '')) {
-                    return new Array(384).fill(0);
-                }
-                // Filter out empty strings
-                textToEmbed = data.filter((item) => item.trim() !== '');
-                if (textToEmbed.length === 0) {
-                    return new Array(384).fill(0);
-                }
-            }
-            else {
-                throw new Error('TransformerEmbedding only supports string or string[] data');
-            }
-            // Ensure the extractor is available
-            if (!this.extractor) {
-                throw new Error('Transformer embedding model is not available');
-            }
-            // Generate embeddings with mean pooling and normalization
-            const result = await this.extractor(textToEmbed, {
-                pooling: 'mean',
-                normalize: true
-            });
-            // Extract the embedding data
-            let embedding;
-            if (textToEmbed.length === 1) {
-                // Single text input - return first embedding
-                embedding = Array.from(result.data.slice(0, 384));
-            }
-            else {
-                // Multiple texts - return first embedding (maintain compatibility)
-                embedding = Array.from(result.data.slice(0, 384));
-            }
-            // Validate embedding dimensions
-            if (embedding.length !== 384) {
-                this.logger('warn', `Unexpected embedding dimension: ${embedding.length}, expected 384`);
-                // Pad or truncate to 384 dimensions
-                if (embedding.length < 384) {
-                    embedding = [...embedding, ...new Array(384 - embedding.length).fill(0)];
-                }
-                else {
-                    embedding = embedding.slice(0, 384);
-                }
-            }
-            return embedding;
-        }
-        catch (error) {
-            this.logger('error', 'Error generating embeddings:', error);
-            throw new Error(`Failed to generate embeddings: ${error}`);
-        }
+        // Delegate to EmbeddingManager
+        return embeddingManager.embed(data);
     }
     /**
-     * Dispose of the model and free resources
+     * Get the embedding function
      */
-    async dispose() {
-        if (this.extractor && typeof this.extractor.dispose === 'function') {
-            await this.extractor.dispose();
-        }
-        this.extractor = null;
-        this.initialized = false;
+    getEmbeddingFunction() {
+        return async (data) => {
+            return this.embed(data);
+        };
     }
     /**
-     * Get the dimension of embeddings produced by this model
+     * Check if initialized
      */
-    getDimension() {
-        return 384;
+    isInitialized() {
+        return this.initialized;
     }
     /**
-     * Check if the model is initialized
+     * Dispose resources (no-op for WASM engine)
      */
-    isInitialized() {
-        return this.initialized;
+    async dispose() {
+        this.initialized = false;
     }
 }
-// Legacy alias for backward compatibility
-export const UniversalSentenceEncoder = TransformerEmbedding;
 /**
- * Create a new embedding model instance
+ * Create a simple embedding function using the default TransformerEmbedding
+ * This is the recommended way to create an embedding function for Brainy
  */
-export function createEmbeddingModel(options) {
+export function createEmbeddingFunction(options = {}) {
+    return embeddingManager.getEmbeddingFunction();
+}
+/**
+ * Create a TransformerEmbedding instance (backward compatibility)
+ */
+export function createTransformerEmbedding(options = {}) {
     return new TransformerEmbedding(options);
 }
 /**
- * Default embedding function using the unified EmbeddingManager
- * Simple, clean, reliable - no more layers of indirection
+ * Convenience function to detect best device (always returns 'wasm')
  */
-export const defaultEmbeddingFunction = async (data) => {
-    const { embed } = await import('../embeddings/EmbeddingManager.js');
-    return await embed(data);
-};
+export async function detectBestDevice() {
+    return 'wasm';
+}
 /**
- * Create an embedding function with custom options
- * NOTE: Options are validated but the singleton EmbeddingManager is always used
+ * Resolve device string (always returns 'wasm')
  */
-export function createEmbeddingFunction(options = {}) {
-    return async (data) => {
-        const { embeddingManager } = await import('../embeddings/EmbeddingManager.js');
-        // Validate precision if specified
-        // Precision is always Q8 now
-        return await embeddingManager.embed(data);
-    };
+export async function resolveDevice(_device = 'auto') {
+    return 'wasm';
 }
 /**
- * Batch embedding function for processing multiple texts efficiently
+ * Default embedding function (backward compatibility)
  */
-export async function batchEmbed(texts, options = {}) {
-    const embedder = new TransformerEmbedding(options);
-    await embedder.init();
-    const embeddings = [];
-    // Process in batches for memory efficiency
-    const batchSize = 32;
-    for (let i = 0; i < texts.length; i += batchSize) {
-        const batch = texts.slice(i, i + batchSize);
-        for (const text of batch) {
-            const embedding = await embedder.embed(text);
-            embeddings.push(embedding);
-        }
+export const defaultEmbeddingFunction = embeddingManager.getEmbeddingFunction();
+/**
+ * UniversalSentenceEncoder alias (backward compatibility)
+ */
+export const UniversalSentenceEncoder = TransformerEmbedding;
+/**
+ * Batch embed function (backward compatibility)
+ */
+export async function batchEmbed(texts) {
+    const results = [];
+    for (const text of texts) {
+        results.push(await embeddingManager.embed(text));
     }
-    await embedder.dispose();
-    return embeddings;
+    return results;
 }
 /**
- * Embedding functions for specific model types
+ * Embedding functions registry (backward compatibility)
  */
 export const embeddingFunctions = {
-    /** Default lightweight model (all-MiniLM-L6-v2, 384 dimensions) */
-    default: defaultEmbeddingFunction,
-    /** Create custom embedding function */
-    create: createEmbeddingFunction,
-    /** Batch processing */
-    batch: batchEmbed
+    transformer: createEmbeddingFunction,
+    default: createEmbeddingFunction,
 };
 //# sourceMappingURL=embedding.js.map

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@soulcraft/brainy",
-  "version": "6.5.0",
+  "version": "6.6.1",
   "description": "Universal Knowledge Protocol™ - World's first Triple Intelligence database unifying vector, graph, and document search in one API. Stage 3 CANONICAL: 42 nouns × 127 verbs covering 96-97% of all human knowledge.",
   "main": "dist/index.js",
   "module": "dist/index.js",
@@ -64,7 +64,8 @@
     "./dist/cortex/backupRestore.js": false
   },
   "engines": {
-    "node": "22.x"
+    "node": "22.x",
+    "bun": ">=1.0.0"
   },
   "scripts": {
     "build": "npm run build:types:if-needed && npm run build:patterns:if-needed && npm run build:keywords:if-needed && tsc && tsc -p tsconfig.cli.json",
@@ -90,6 +91,10 @@
     "test:ci-unit": "CI=true vitest run --config tests/configs/vitest.unit.config.ts",
     "test:ci-integration": "NODE_OPTIONS='--max-old-space-size=16384' CI=true vitest run --config tests/configs/vitest.integration.config.ts",
     "test:ci": "npm run test:ci-unit",
+    "test:bun": "bun tests/integration/bun-compile-test.ts",
+    "test:bun:compile": "bun build tests/integration/bun-compile-test.ts --compile --outfile /tmp/brainy-bun-test && /tmp/brainy-bun-test",
+    "test:wasm": "npx vitest run tests/integration/wasm-embeddings.test.ts",
+    "download-model": "node scripts/download-model.cjs",
     "download-models": "node scripts/download-models.cjs",
     "download-models:q8": "node scripts/download-models.cjs",
     "models:verify": "node scripts/ensure-models.js",
@@ -142,7 +147,9 @@
     "dist/**/*.js",
     "dist/**/*.d.ts",
     "bin/",
+    "assets/models/**/*",
     "scripts/download-models.cjs",
+    "scripts/download-model.cjs",
     "scripts/ensure-models.js",
     "scripts/prepare-models.js",
     "brainy.png",
@@ -180,7 +187,7 @@
     "@azure/identity": "^4.0.0",
     "@azure/storage-blob": "^12.17.0",
     "@google-cloud/storage": "^7.14.0",
-    "@huggingface/transformers": "^3.7.2",
+    "onnxruntime-web": "^1.22.0",
     "@msgpack/msgpack": "^3.1.2",
     "@types/js-yaml": "^4.0.9",
     "boxen": "^8.0.1",