@soulcraft/brainy 0.46.0 → 0.47.0

package/dist/utils/embedding.js

@@ -1,292 +1,144 @@
 /**
- * Embedding functions for converting data to vectors
+ * Embedding functions for converting data to vectors using Transformers.js
+ * Complete rewrite to eliminate TensorFlow.js and use ONNX-based models
  */
-import { executeInThread } from './workerUtils.js';
 import { isBrowser } from './environment.js';
-import { createRobustModelLoader, getUniversalSentenceEncoderFallbacks } from './robustModelLoader.js';
-export class UniversalSentenceEncoder {
+import { pipeline, env } from '@huggingface/transformers';
+export class TransformerEmbedding {
     /**
-     * Create a new UniversalSentenceEncoder instance
-     * @param options Configuration options including reliability settings
+     * Create a new TransformerEmbedding instance
      */
     constructor(options = {}) {
-        this.model = null;
+        this.extractor = null;
         this.initialized = false;
-        this.tf = null;
-        this.use = null;
-        this.backend = 'cpu'; // Default to CPU
-        this.verbose = true; // Whether to log non-essential messages
+        this.verbose = true;
         this.verbose = options.verbose !== undefined ? options.verbose : true;
-        // Create robust model loader with enhanced reliability features
-        this.robustLoader = createRobustModelLoader({
-            maxRetries: options.maxRetries ?? 3,
-            initialRetryDelay: options.initialRetryDelay ?? 1000,
-            maxRetryDelay: options.maxRetryDelay ?? 30000,
-            timeout: options.timeout ?? 60000,
-            useExponentialBackoff: options.useExponentialBackoff ?? true,
-            fallbackUrls: options.fallbackUrls ?? getUniversalSentenceEncoderFallbacks(),
+        this.options = {
+            model: options.model || 'Xenova/all-MiniLM-L6-v2',
             verbose: this.verbose,
-            preferLocalModel: options.preferLocalModel ?? true
-        });
+            cacheDir: options.cacheDir || this.getDefaultCacheDir(),
+            localFilesOnly: options.localFilesOnly !== undefined ? options.localFilesOnly : !isBrowser(),
+            dtype: options.dtype || 'fp32'
+        };
+        // Configure transformers.js environment
+        if (!isBrowser()) {
+            // Set cache directory for Node.js
+            env.cacheDir = this.options.cacheDir;
+            // Prioritize local models for offline operation
+            env.allowRemoteModels = !this.options.localFilesOnly;
+            env.allowLocalModels = true;
+        }
+        else {
+            // Browser configuration
+            // Allow both local and remote models, but prefer local if available
+            env.allowLocalModels = true;
+            env.allowRemoteModels = true;
+            // Force the configuration to ensure it's applied
+            if (this.verbose) {
+                this.logger('log', `Browser env config - allowLocalModels: ${env.allowLocalModels}, allowRemoteModels: ${env.allowRemoteModels}, localFilesOnly: ${this.options.localFilesOnly}`);
+            }
+        }
     }
     /**
-     * Add polyfills and patches for TensorFlow.js compatibility
-     * This addresses issues with TensorFlow.js across all server environments
-     * (Node.js, serverless, and other server environments)
-     *
-     * Note: The main TensorFlow.js patching is now centralized in textEncoding.ts
-     * and applied through setup.ts. This method only adds additional utility functions
-     * that might be needed by TensorFlow.js.
+     * Get the default cache directory for models
      */
-    addServerCompatibilityPolyfills() {
-        // Apply in all non-browser environments (Node.js, serverless, server environments)
+    getDefaultCacheDir() {
         if (isBrowser()) {
-            return; // Browser environments don't need these polyfills
+            return './models'; // Browser default
         }
-        // Get the appropriate global object for the current environment
-        const globalObj = (() => {
-            if (typeof globalThis !== 'undefined')
-                return globalThis;
-            if (typeof global !== 'undefined')
-                return global;
-            if (typeof self !== 'undefined')
-                return self;
-            return {}; // Fallback for unknown environments
-        })();
-        // Add polyfill for utility functions across all server environments
-        // This fixes issues like "Cannot read properties of undefined (reading 'isFloat32Array')"
-        try {
-            // Ensure the util object exists
-            if (!globalObj.util) {
-                globalObj.util = {};
-            }
-            // Add isFloat32Array method if it doesn't exist
-            if (!globalObj.util.isFloat32Array) {
-                globalObj.util.isFloat32Array = (obj) => {
-                    return !!(obj instanceof Float32Array ||
-                        (obj &&
-                            Object.prototype.toString.call(obj) === '[object Float32Array]'));
-                };
+        // Check for bundled models in the package
+        const possiblePaths = [
+            // In the installed package
+            './node_modules/@soulcraft/brainy/models',
+            // In development/source
+            './models',
+            './dist/../models',
+            // Alternative locations
+            '../models',
+            '../../models'
+        ];
+        // Check if we're in Node.js and try to find the bundled models
+        if (typeof process !== 'undefined' && process.versions?.node) {
+            try {
+                const path = require('path');
+                const fs = require('fs');
+                // Try to resolve the package location
+                try {
+                    const brainyPackagePath = require.resolve('@soulcraft/brainy/package.json');
+                    const brainyPackageDir = path.dirname(brainyPackagePath);
+                    const bundledModelsPath = path.join(brainyPackageDir, 'models');
+                    if (fs.existsSync(bundledModelsPath)) {
+                        this.logger('log', `Using bundled models from package: ${bundledModelsPath}`);
+                        return bundledModelsPath;
+                    }
+                }
+                catch (e) {
+                    // Not installed as package, continue
+                }
+                // Try relative paths from current location
+                for (const relativePath of possiblePaths) {
+                    const fullPath = path.resolve(relativePath);
+                    if (fs.existsSync(fullPath)) {
+                        this.logger('log', `Using bundled models from: ${fullPath}`);
+                        return fullPath;
+                    }
+                }
             }
-            // Add isTypedArray method if it doesn't exist
-            if (!globalObj.util.isTypedArray) {
-                globalObj.util.isTypedArray = (obj) => {
-                    return !!(ArrayBuffer.isView(obj) && !(obj instanceof DataView));
-                };
+            catch (error) {
+                this.logger('warn', 'Could not auto-detect bundled models directory:', error);
             }
         }
-        catch (error) {
-            console.warn('Failed to add utility polyfills:', error);
-        }
+        // Fallback to default cache directory
+        return './models';
     }
     /**
      * Check if we're running in a test environment
      */
     isTestEnvironment() {
-        // Safely check for Node.js environment first
-        if (typeof process === 'undefined') {
-            return false;
-        }
-        return (process.env.NODE_ENV === 'test' ||
-            process.env.VITEST === 'true' ||
-            (typeof global !== 'undefined' && global.__vitest__) ||
-            process.argv.some((arg) => arg.includes('vitest')));
+        // Always use real implementation - no more mocking
+        return false;
     }
     /**
-     * Log message only if verbose mode is enabled or if it's an error
-     * This helps suppress non-essential log messages
+     * Log message only if verbose mode is enabled
      */
     logger(level, message, ...args) {
-        // Always log errors, but only log other messages if verbose mode is enabled
         if (level === 'error' || this.verbose) {
-            console[level](message, ...args);
-        }
-    }
-    /**
-     * Load the Universal Sentence Encoder model with robust retry and fallback mechanisms
-     * @param loadFunction The function to load the model from TensorFlow Hub
-     */
-    async loadModelFromLocal(loadFunction) {
-        this.logger('log', 'Loading Universal Sentence Encoder model with robust loader...');
-        try {
-            // Use the robust model loader to handle all retry logic, timeouts, and fallbacks
-            const model = await this.robustLoader.loadModel(loadFunction, 'universal-sentence-encoder');
-            this.logger('log', 'Successfully loaded Universal Sentence Encoder model');
-            return model;
-        }
-        catch (error) {
-            const errorMessage = error instanceof Error ? error.message : String(error);
-            this.logger('error', `Failed to load Universal Sentence Encoder model: ${errorMessage}`);
-            // Log loading statistics for debugging
-            const stats = this.robustLoader.getLoadingStats();
-            if (Object.keys(stats).length > 0) {
-                this.logger('log', 'Loading attempt statistics:', stats);
-            }
-            throw error;
+            console[level](`[TransformerEmbedding] ${message}`, ...args);
         }
     }
     /**
      * Initialize the embedding model
      */
     async init() {
-        // Use a mock implementation in test environments
-        if (this.isTestEnvironment()) {
-            this.logger('log', 'Using mock Universal Sentence Encoder for tests');
-            // Create a mock model that returns fixed embeddings
-            this.model = {
-                embed: async (sentences) => {
-                    // Create a tensor-like object with a mock array method
-                    return {
-                        array: async () => {
-                            // Return fixed embeddings for each input sentence
-                            const inputArray = Array.isArray(sentences)
-                                ? sentences
-                                : [sentences];
-                            return inputArray.map(() => new Array(512).fill(0).map((_, i) => (i % 2 === 0 ? 0.1 : -0.1)));
-                        },
-                        dispose: () => { }
-                    };
-                }
-            };
-            this.initialized = true;
+        if (this.initialized) {
             return;
         }
+        // Always use real implementation - no mocking
         try {
-            // Save original console.warn
-            const originalWarn = console.warn;
-            // Override console.warn to suppress TensorFlow.js Node.js backend message
-            console.warn = function (message, ...optionalParams) {
-                if (message &&
-                    typeof message === 'string' &&
-                    message.includes('Hi, looks like you are running TensorFlow.js in Node.js')) {
-                    return; // Suppress the specific warning
-                }
-                originalWarn(message, ...optionalParams);
+            this.logger('log', `Loading Transformer model: ${this.options.model}`);
+            const startTime = Date.now();
+            // Load the feature extraction pipeline
+            // In browsers, never use local_files_only to avoid conflicts
+            const pipelineOptions = {
+                cache_dir: this.options.cacheDir,
+                local_files_only: isBrowser() ? false : this.options.localFilesOnly,
+                dtype: this.options.dtype
             };
-            // Add polyfills for TensorFlow.js compatibility
-            this.addServerCompatibilityPolyfills();
-            // CRITICAL: Ensure TextEncoder/TextDecoder are available before TensorFlow.js loads
-            try {
-                // Get the appropriate global object for the current environment
-                const globalObj = (() => {
-                    if (typeof globalThis !== 'undefined')
-                        return globalThis;
-                    if (typeof global !== 'undefined')
-                        return global;
-                    if (typeof self !== 'undefined')
-                        return self;
-                    return null;
-                })();
-                // Ensure TextEncoder/TextDecoder are globally available in server environments
-                if (globalObj) {
-                    // Try to use Node.js util module if available (Node.js environments)
-                    try {
-                        if (typeof process !== 'undefined' &&
-                            process.versions &&
-                            process.versions.node) {
-                            const util = await import('util');
-                            if (!globalObj.TextEncoder) {
-                                globalObj.TextEncoder = util.TextEncoder;
-                            }
-                            if (!globalObj.TextDecoder) {
-                                globalObj.TextDecoder =
-                                    util.TextDecoder;
-                            }
-                        }
-                    }
-                    catch (utilError) {
-                        // Fallback to standard TextEncoder/TextDecoder for non-Node.js server environments
-                        if (!globalObj.TextEncoder) {
-                            globalObj.TextEncoder = TextEncoder;
-                        }
-                        if (!globalObj.TextDecoder) {
-                            globalObj.TextDecoder = TextDecoder;
-                        }
-                    }
-                }
-                // Apply the TensorFlow.js patch
-                const { applyTensorFlowPatch } = await import('./textEncoding.js');
-                await applyTensorFlowPatch();
-                // Now load TensorFlow.js core module using dynamic imports
-                this.tf = await import('@tensorflow/tfjs-core');
-                // Import CPU backend (always needed as fallback)
-                await import('@tensorflow/tfjs-backend-cpu');
-                // Try to import WebGL backend for GPU acceleration in browser environments
-                try {
-                    if (isBrowser()) {
-                        await import('@tensorflow/tfjs-backend-webgl');
-                        // Check if WebGL is available
-                        try {
-                            if (this.tf.setBackend) {
-                                await this.tf.setBackend('webgl');
-                                this.backend = 'webgl';
-                                console.log('Using WebGL backend for TensorFlow.js');
-                            }
-                            else {
-                                console.warn('tf.setBackend is not available, falling back to CPU');
-                            }
-                        }
-                        catch (e) {
-                            console.warn('WebGL backend not available, falling back to CPU:', e);
-                            this.backend = 'cpu';
-                        }
-                    }
-                }
-                catch (error) {
-                    console.warn('WebGL backend not available, falling back to CPU:', error);
-                    this.backend = 'cpu';
-                }
-                // Note: @tensorflow-models/universal-sentence-encoder is no longer used
-                // Model loading is handled entirely by robustLoader
-            }
-            catch (error) {
-                this.logger('error', 'Failed to initialize TensorFlow.js:', error);
-                // No fallback allowed - throw error
-                throw new Error(`Universal Sentence Encoder initialization failed: ${error}`);
-            }
-            // Set the backend
-            if (this.tf && this.tf.setBackend) {
-                await this.tf.setBackend(this.backend);
-            }
-            // Load model using robustLoader which handles all loading strategies:
-            // 1. @soulcraft/brainy-models package if available (offline mode)
-            // 2. Direct TensorFlow.js URL loading as fallback
-            try {
-                this.model = await this.robustLoader.loadModelWithFallbacks();
-                this.initialized = true;
-                // If the model doesn't have an embed method but has embedToArrays, wrap it
-                if (!this.model.embed && this.model.embedToArrays) {
-                    const originalModel = this.model;
-                    this.model = {
-                        embed: async (sentences) => {
-                            const input = Array.isArray(sentences) ? sentences : [sentences];
-                            const embeddings = await originalModel.embedToArrays(input);
-                            // Return TensorFlow tensor-like object
-                            return {
-                                array: async () => embeddings,
-                                arraySync: () => embeddings
-                            };
-                        },
-                        dispose: () => originalModel.dispose ? originalModel.dispose() : undefined
-                    };
-                }
+            if (this.verbose) {
+                this.logger('log', `Pipeline options: ${JSON.stringify(pipelineOptions)}`);
             }
-            catch (modelError) {
-                this.logger('error', 'Failed to load Universal Sentence Encoder model:', modelError);
-                throw new Error(`Universal Sentence Encoder model loading failed: ${modelError}`);
-            }
-            // Restore original console.warn
-            console.warn = originalWarn;
+            this.extractor = await pipeline('feature-extraction', this.options.model, pipelineOptions);
+            const loadTime = Date.now() - startTime;
+            this.logger('log', `✅ Model loaded successfully in ${loadTime}ms`);
+            this.initialized = true;
         }
         catch (error) {
-            this.logger('error', 'Failed to initialize Universal Sentence Encoder:', error);
-            // No fallback allowed - throw error
-            throw new Error(`Universal Sentence Encoder initialization failed: ${error}`);
+            this.logger('error', 'Failed to initialize Transformer embedding model:', error);
+            throw new Error(`Transformer embedding initialization failed: ${error}`);
         }
     }
     /**
-     * Embed text into a vector using Universal Sentence Encoder
-     * @param data Text to embed
+     * Generate embeddings for text data
      */
     async embed(data) {
         if (!this.initialized) {
@@ -298,405 +150,137 @@ export class UniversalSentenceEncoder {
             if (typeof data === 'string') {
                 // Handle empty string case
                 if (data.trim() === '') {
-                    // Return a zero vector of 512 dimensions (standard for Universal Sentence Encoder)
-                    return new Array(512).fill(0);
+                    // Return a zero vector of 384 dimensions (all-MiniLM-L6-v2 standard)
+                    return new Array(384).fill(0);
                 }
                 textToEmbed = [data];
             }
-            else if (Array.isArray(data) &&
-                data.every((item) => typeof item === 'string')) {
+            else if (Array.isArray(data) && data.every((item) => typeof item === 'string')) {
                 // Handle empty array or array with empty strings
                 if (data.length === 0 || data.every((item) => item.trim() === '')) {
-                    return new Array(512).fill(0);
+                    return new Array(384).fill(0);
                 }
                 // Filter out empty strings
                 textToEmbed = data.filter((item) => item.trim() !== '');
                 if (textToEmbed.length === 0) {
-                    return new Array(512).fill(0);
+                    return new Array(384).fill(0);
                 }
             }
             else {
-                throw new Error('UniversalSentenceEncoder only supports string or string[] data');
+                throw new Error('TransformerEmbedding only supports string or string[] data');
             }
-            // Ensure the model is available
-            if (!this.model) {
-                throw new Error('Universal Sentence Encoder model is not available');
-            }
-            // Get embeddings
-            const embeddings = await this.model.embed(textToEmbed);
-            // Convert to array and return the first embedding
-            const embeddingArray = await embeddings.array();
-            // Dispose of the tensor to free memory
-            embeddings.dispose();
-            // Get the first embedding
-            let embedding = embeddingArray[0];
-            // Always ensure the embedding is exactly 512 dimensions
-            if (embedding.length !== 512) {
-                this.logger('warn', `Embedding dimension mismatch: expected 512, got ${embedding.length}. Standardizing to 512 dimensions.`);
-                // If the embedding is too short, pad with zeros
-                if (embedding.length < 512) {
-                    const paddedEmbedding = new Array(512).fill(0);
-                    for (let i = 0; i < embedding.length; i++) {
-                        paddedEmbedding[i] = embedding[i];
-                    }
-                    embedding = paddedEmbedding;
-                }
-                // If the embedding is too long, truncate
-                else if (embedding.length > 512) {
-                    embedding = embedding.slice(0, 512);
-                }
+            // Ensure the extractor is available
+            if (!this.extractor) {
+                throw new Error('Transformer embedding model is not available');
             }
-            return embedding;
-        }
-        catch (error) {
-            this.logger('error', 'Failed to embed text with Universal Sentence Encoder:', error);
-            throw new Error(`Universal Sentence Encoder embedding failed: ${error}`);
-        }
-    }
-    /**
-     * Embed multiple texts into vectors using Universal Sentence Encoder
-     * This is more efficient than calling embed() multiple times
-     * @param dataArray Array of texts to embed
-     * @returns Array of embedding vectors
-     */
-    async embedBatch(dataArray) {
-        if (!this.initialized) {
-            await this.init();
-        }
-        try {
-            // Handle empty array case
-            if (dataArray.length === 0) {
-                return [];
-            }
-            // Filter out empty strings and handle edge cases
-            const textToEmbed = dataArray.filter((text) => typeof text === 'string' && text.trim() !== '');
-            // If all strings were empty, return appropriate zero vectors
-            if (textToEmbed.length === 0) {
-                return dataArray.map(() => new Array(512).fill(0));
+            // Generate embeddings with mean pooling and normalization
+            const result = await this.extractor(textToEmbed, {
+                pooling: 'mean',
+                normalize: true
+            });
+            // Extract the embedding data
+            let embedding;
+            if (textToEmbed.length === 1) {
+                // Single text input - return first embedding
+                embedding = Array.from(result.data.slice(0, 384));
             }
-            // Ensure the model is available
-            if (!this.model) {
-                throw new Error('Universal Sentence Encoder model is not available');
+            else {
+                // Multiple texts - return first embedding (maintain compatibility)
+                embedding = Array.from(result.data.slice(0, 384));
             }
-            // Get embeddings for all texts in a single batch operation
-            const embeddings = await this.model.embed(textToEmbed);
-            // Convert to array
-            const embeddingArray = await embeddings.array();
-            // Dispose of the tensor to free memory
-            embeddings.dispose();
-            // Standardize embeddings to ensure they're all 512 dimensions
-            const standardizedEmbeddings = embeddingArray.map((embedding) => {
-                if (embedding.length !== 512) {
-                    this.logger('warn', `Batch embedding dimension mismatch: expected 512, got ${embedding.length}. Standardizing to 512 dimensions.`);
-                    // If the embedding is too short, pad with zeros
-                    if (embedding.length < 512) {
-                        const paddedEmbedding = new Array(512).fill(0);
-                        for (let i = 0; i < embedding.length; i++) {
-                            paddedEmbedding[i] = embedding[i];
-                        }
-                        return paddedEmbedding;
-                    }
-                    // If the embedding is too long, truncate
-                    else if (embedding.length > 512) {
-                        return embedding.slice(0, 512);
-                    }
-                }
-                return embedding;
-            });
-            // Map the results back to the original array order
-            const results = [];
-            let embeddingIndex = 0;
-            for (let i = 0; i < dataArray.length; i++) {
-                const text = dataArray[i];
-                if (typeof text === 'string' && text.trim() !== '') {
-                    // Use the standardized embedding for non-empty strings
-                    results.push(standardizedEmbeddings[embeddingIndex]);
-                    embeddingIndex++;
+            // Validate embedding dimensions
+            if (embedding.length !== 384) {
+                this.logger('warn', `Unexpected embedding dimension: ${embedding.length}, expected 384`);
+                // Pad or truncate to 384 dimensions
+                if (embedding.length < 384) {
+                    embedding = [...embedding, ...new Array(384 - embedding.length).fill(0)];
                 }
                 else {
-                    // Use a zero vector for empty strings
-                    results.push(new Array(512).fill(0));
+                    embedding = embedding.slice(0, 384);
                 }
             }
-            return results;
+            return embedding;
         }
         catch (error) {
-            this.logger('error', 'Failed to batch embed text with Universal Sentence Encoder:', error);
-            throw new Error(`Universal Sentence Encoder batch embedding failed: ${error}`);
+            this.logger('error', 'Error generating embeddings:', error);
+            throw new Error(`Failed to generate embeddings: ${error}`);
         }
     }
     /**
-     * Dispose of the model resources
+     * Dispose of the model and free resources
      */
     async dispose() {
-        if (this.model && this.tf) {
-            try {
-                // Dispose of the model and tensors
-                this.model.dispose();
-                this.tf.disposeVariables();
-                this.initialized = false;
-            }
-            catch (error) {
-                this.logger('error', 'Failed to dispose Universal Sentence Encoder:', error);
-            }
+        if (this.extractor && typeof this.extractor.dispose === 'function') {
+            await this.extractor.dispose();
         }
-        return Promise.resolve();
-    }
-}
-/**
- * Helper function - NO LONGER USED
- * Kept for compatibility but will be removed in next major version
- * @deprecated Since we removed @tensorflow-models/universal-sentence-encoder dependency
- */
-function findUSELoadFunction(sentenceEncoderModule) {
-    // Module structure available for debugging if needed
-    // Find the appropriate load function from the module
-    let loadFunction = null;
-    // Try sentenceEncoderModule.load first (direct export)
-    if (sentenceEncoderModule.load &&
-        typeof sentenceEncoderModule.load === 'function') {
-        loadFunction = sentenceEncoderModule.load;
-    }
-    // Then try sentenceEncoderModule.default.load (default export)
-    else if (sentenceEncoderModule.default &&
-        sentenceEncoderModule.default.load &&
-        typeof sentenceEncoderModule.default.load === 'function') {
-        loadFunction = sentenceEncoderModule.default.load;
-    }
-    // Try sentenceEncoderModule.default directly if it's a function
-    else if (sentenceEncoderModule.default &&
-        typeof sentenceEncoderModule.default === 'function') {
-        loadFunction = sentenceEncoderModule.default;
-    }
-    // Try sentenceEncoderModule directly if it's a function
-    else if (typeof sentenceEncoderModule === 'function') {
-        loadFunction = sentenceEncoderModule;
-    }
-    // Try additional common patterns
-    else if (sentenceEncoderModule.UniversalSentenceEncoder &&
-        typeof sentenceEncoderModule.UniversalSentenceEncoder.load === 'function') {
-        loadFunction = sentenceEncoderModule.UniversalSentenceEncoder.load;
-    }
-    else if (sentenceEncoderModule.default &&
-        sentenceEncoderModule.default.UniversalSentenceEncoder &&
-        typeof sentenceEncoderModule.default.UniversalSentenceEncoder.load ===
-            'function') {
-        loadFunction = sentenceEncoderModule.default.UniversalSentenceEncoder.load;
+        this.extractor = null;
+        this.initialized = false;
     }
-    // Try to find the load function in the module's properties
-    else {
-        // Look for any property that might be a load function
-        for (const key in sentenceEncoderModule) {
-            if (typeof sentenceEncoderModule[key] === 'function') {
-                // Check if the function name or key contains 'load'
-                const fnName = sentenceEncoderModule[key].name || key;
-                if (fnName.toLowerCase().includes('load')) {
-                    loadFunction = sentenceEncoderModule[key];
-                    break;
-                }
-            }
-            // Also check nested objects
-            else if (typeof sentenceEncoderModule[key] === 'object' &&
-                sentenceEncoderModule[key] !== null) {
-                for (const nestedKey in sentenceEncoderModule[key]) {
-                    if (typeof sentenceEncoderModule[key][nestedKey] === 'function') {
-                        const fnName = sentenceEncoderModule[key][nestedKey].name || nestedKey;
-                        if (fnName.toLowerCase().includes('load')) {
-                            loadFunction = sentenceEncoderModule[key][nestedKey];
-                            break;
-                        }
-                    }
-                }
-                if (loadFunction)
-                    break;
-            }
-        }
+    /**
+     * Get the dimension of embeddings produced by this model
+     */
+    getDimension() {
+        return 384;
     }
-    // Return a function that calls the load function without arguments
-    // This will use the bundled model from the package
-    if (loadFunction) {
-        return async () => await loadFunction();
+    /**
+     * Check if the model is initialized
+     */
+    isInitialized() {
+        return this.initialized;
     }
-    return null;
 }
+// Legacy alias for backward compatibility
+export const UniversalSentenceEncoder = TransformerEmbedding;
 /**
- * Check if we're running in a test environment (standalone version)
- * Uses the same logic as the class method to avoid duplication
+ * Create a new embedding model instance
  */
-function isTestEnvironment() {
-    // Use the same implementation as the class method
-    // Safely check for Node.js environment first
-    if (typeof process === 'undefined') {
-        return false;
-    }
-    return (process.env.NODE_ENV === 'test' ||
-        process.env.VITEST === 'true' ||
-        (typeof global !== 'undefined' && global.__vitest__) ||
-        process.argv.some((arg) => arg.includes('vitest')));
+export function createEmbeddingModel(options) {
+    return new TransformerEmbedding(options);
 }
 /**
- * Log message only if not in test environment and verbose mode is enabled (standalone version)
- * @param level Log level ('log', 'warn', 'error')
- * @param message Message to log
- * @param args Additional arguments to log
- * @param verbose Whether to log non-essential messages (default: true)
+ * Default embedding function using the lightweight transformer model
  */
-function logIfNotTest(level, message, args = [], verbose = true) {
-    // Always log errors, but only log other messages if verbose mode is enabled
-    if ((level === 'error' || verbose) && !isTestEnvironment()) {
-        console[level](message, ...args);
-    }
-}
+export const defaultEmbeddingFunction = async (data) => {
+    const embedder = new TransformerEmbedding({ verbose: false });
+    return await embedder.embed(data);
+};
 /**
- * Create an embedding function from an embedding model
- * @param model Embedding model to use (optional, defaults to UniversalSentenceEncoder)
+ * Create an embedding function with custom options
  */
-export function createEmbeddingFunction(model) {
-    // If no model is provided, use the default TensorFlow embedding function
-    if (!model) {
-        return createTensorFlowEmbeddingFunction();
-    }
+export function createEmbeddingFunction(options = {}) {
+    const embedder = new TransformerEmbedding(options);
     return async (data) => {
-        return await model.embed(data);
+        return await embedder.embed(data);
     };
 }
 /**
- * Creates a TensorFlow-based Universal Sentence Encoder embedding function
- * This is the required embedding function for all text embeddings
- * Uses a shared model instance for better performance across multiple calls
- * @param options Configuration options
- * @param options.verbose Whether to log non-essential messages (default: true)
+ * Batch embedding function for processing multiple texts efficiently
  */
-// Create a single shared instance of the model that persists across all embedding calls
-let sharedModel = null;
-let sharedModelInitialized = false;
-let sharedModelVerbose = true;
-export function createTensorFlowEmbeddingFunction(options = {}) {
-    // Update verbose setting if provided
-    if (options.verbose !== undefined) {
-        sharedModelVerbose = options.verbose;
-    }
-    // Create the shared model if it doesn't exist yet
-    if (!sharedModel) {
-        sharedModel = new UniversalSentenceEncoder({ verbose: sharedModelVerbose });
-    }
-    return async (data) => {
-        try {
-            // Initialize the model if it hasn't been initialized yet
-            if (!sharedModelInitialized) {
-                try {
-                    await sharedModel.init();
-                    sharedModelInitialized = true;
-                }
-                catch (initError) {
-                    // Reset the flag so we can retry initialization on the next call
-                    sharedModelInitialized = false;
-                    throw initError;
-                }
-            }
-            return await sharedModel.embed(data);
+export async function batchEmbed(texts, options = {}) {
+    const embedder = new TransformerEmbedding(options);
+    await embedder.init();
+    const embeddings = [];
+    // Process in batches for memory efficiency
+    const batchSize = 32;
+    for (let i = 0; i < texts.length; i += batchSize) {
+        const batch = texts.slice(i, i + batchSize);
+        for (const text of batch) {
+            const embedding = await embedder.embed(text);
+            embeddings.push(embedding);
         }
-        catch (error) {
-            logIfNotTest('error', 'Failed to use Universal Sentence Encoder:', [error], sharedModelVerbose);
-            // No fallback - Universal Sentence Encoder is required
-            throw new Error(`Universal Sentence Encoder is required and no fallbacks are allowed: ${error}`);
-        }
-    };
-}
-/**
- * Default embedding function
- * Uses UniversalSentenceEncoder for all text embeddings
- * TensorFlow.js is required for this to work
- * Uses CPU for compatibility
- * @param options Configuration options
- * @param options.verbose Whether to log non-essential messages (default: true)
- */
-export function getDefaultEmbeddingFunction(options = {}) {
-    return createTensorFlowEmbeddingFunction(options);
-}
-/**
- * Default embedding function with default options
- * Uses UniversalSentenceEncoder for all text embeddings
- * TensorFlow.js is required for this to work
- * Uses CPU for compatibility
- */
-export const defaultEmbeddingFunction = getDefaultEmbeddingFunction();
-/**
- * Creates a batch embedding function that uses UniversalSentenceEncoder
- * TensorFlow.js is required for this to work
- * Processes all items in a single batch operation
- * Uses a shared model instance for better performance across multiple calls
- * @param options Configuration options
- * @param options.verbose Whether to log non-essential messages (default: true)
- */
-// Create a single shared instance of the model that persists across function calls
-let sharedBatchModel = null;
-let sharedBatchModelInitialized = false;
-let sharedBatchModelVerbose = true;
-export function createBatchEmbeddingFunction(options = {}) {
-    // Update verbose setting if provided
-    if (options.verbose !== undefined) {
-        sharedBatchModelVerbose = options.verbose;
     }
-    // Create the shared model if it doesn't exist yet
-    if (!sharedBatchModel) {
-        sharedBatchModel = new UniversalSentenceEncoder({
-            verbose: sharedBatchModelVerbose
-        });
-    }
-    return async (dataArray) => {
-        try {
-            // Initialize the model if it hasn't been initialized yet
-            if (!sharedBatchModelInitialized) {
-                try {
-                    await sharedBatchModel.init();
-                    sharedBatchModelInitialized = true;
-                }
-                catch (initError) {
-                    // Reset the flag so we can retry initialization on the next call
-                    sharedBatchModelInitialized = false;
-                    throw initError;
-                }
-            }
-            return await sharedBatchModel.embedBatch(dataArray);
-        }
-        catch (error) {
-            logIfNotTest('error', 'Failed to use Universal Sentence Encoder batch embedding:', [error], sharedBatchModelVerbose);
-            // No fallback - Universal Sentence Encoder is required
-            throw new Error(`Universal Sentence Encoder is required for batch embedding and no fallbacks are allowed: ${error}`);
-        }
-    };
+    await embedder.dispose();
+    return embeddings;
 }
 /**
- * Get a batch embedding function with custom options
- * Uses UniversalSentenceEncoder for all text embeddings
- * TensorFlow.js is required for this to work
- * Processes all items in a single batch operation
- * @param options Configuration options
- * @param options.verbose Whether to log non-essential messages (default: true)
+ * Embedding functions for specific model types
  */
-export function getDefaultBatchEmbeddingFunction(options = {}) {
-    return createBatchEmbeddingFunction(options);
-}
-/**
- * Default batch embedding function with default options
- * Uses UniversalSentenceEncoder for all text embeddings
- * TensorFlow.js is required for this to work
- * Processes all items in a single batch operation
- */
-export const defaultBatchEmbeddingFunction = getDefaultBatchEmbeddingFunction();
-/**
- * Creates an embedding function that runs in a separate thread
- * This is a wrapper around createEmbeddingFunction that uses executeInThread
- * @param model Embedding model to use
- */
-export function createThreadedEmbeddingFunction(model) {
-    const embeddingFunction = createEmbeddingFunction(model);
-    return async (data) => {
-        // Convert the embedding function to a string
-        const fnString = embeddingFunction.toString();
-        // Execute the embedding function in a "thread" (main thread in this implementation)
-        return await executeInThread(fnString, data);
-    };
-}
+export const embeddingFunctions = {
+    /** Default lightweight model (all-MiniLM-L6-v2, 384 dimensions) */
+    default: defaultEmbeddingFunction,
+    /** Create custom embedding function */
+    create: createEmbeddingFunction,
+    /** Batch processing */
+    batch: batchEmbed
+};
 //# sourceMappingURL=embedding.js.map