doc2vec 1.3.0 → 2.2.0

package/dist/embedding-providers.js

@@ -1,157 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.TransformersEmbeddingProvider = exports.OpenAIEmbeddingProvider = void 0;
-exports.createEmbeddingProvider = createEmbeddingProvider;
-const openai_1 = require("openai");
-const transformers_1 = require("@huggingface/transformers");
-class OpenAIEmbeddingProvider {
-    constructor(params, logger) {
-        this.logger = logger.child('openai-embeddings');
-        this.model = params.model || 'text-embedding-3-large';
-        this.openai = new openai_1.OpenAI({
-            apiKey: params.api_key || process.env.OPENAI_API_KEY
-        });
-        this.logger.info(`OpenAI embedding provider initialized with model: ${this.model}`);
-    }
-    async createEmbeddings(texts) {
-        try {
-            this.logger.debug(`Creating embeddings for ${texts.length} texts using OpenAI`);
-            const response = await this.openai.embeddings.create({
-                model: this.model,
-                input: texts,
-            });
-            this.logger.debug(`Successfully created ${response.data.length} embeddings`);
-            return response.data.map(d => d.embedding);
-        }
-        catch (error) {
-            this.logger.error('Failed to create OpenAI embeddings:', error);
-            return [];
-        }
-    }
-    getDimensions() {
-        // Return dimensions based on model
-        if (this.model === 'text-embedding-3-large')
-            return 3072;
-        if (this.model === 'text-embedding-3-small')
-            return 1536;
-        if (this.model === 'text-embedding-ada-002')
-            return 1536;
-        return 1536; // Default fallback
-    }
-    getModelName() {
-        return this.model;
-    }
-}
-exports.OpenAIEmbeddingProvider = OpenAIEmbeddingProvider;
-class TransformersEmbeddingProvider {
-    constructor(params, logger) {
-        this.params = params;
-        this.dimensions = 384; // Default, will be updated after model loading
-        this.isInitialized = false;
-        this.logger = logger.child('transformers-embeddings');
-        this.modelName = params.model;
-        // Configure transformers.js environment
-        if (params.local_files_only) {
-            transformers_1.env.allowLocalModels = true;
-            transformers_1.env.allowRemoteModels = false;
-        }
-        if (params.cache_dir) {
-            transformers_1.env.cacheDir = params.cache_dir;
-        }
-        this.logger.info(`Transformers embedding provider initialized with model: ${this.modelName}`);
-    }
-    async initialize() {
-        if (this.isInitialized)
-            return;
-        try {
-            this.logger.info(`Loading transformers model: ${this.modelName}`);
-            // Load tokenizer and model
-            this.tokenizer = await transformers_1.AutoTokenizer.from_pretrained(this.modelName);
-            this.model = await transformers_1.AutoModel.from_pretrained(this.modelName);
-            // Try to determine dimensions from model config
-            if (this.model.config && this.model.config.hidden_size) {
-                this.dimensions = this.model.config.hidden_size;
-            }
-            this.isInitialized = true;
-            this.logger.info(`Transformers model loaded successfully. Dimensions: ${this.dimensions}`);
-        }
-        catch (error) {
-            this.logger.error(`Failed to load transformers model: ${this.modelName}`, error);
-            throw error;
-        }
-    }
-    async createEmbeddings(texts) {
-        await this.initialize();
-        try {
-            this.logger.debug(`Creating embeddings for ${texts.length} texts using transformers.js`);
-            const embeddings = [];
-            // Process texts in batches to avoid memory issues
-            const batchSize = Math.min(8, texts.length);
-            for (let i = 0; i < texts.length; i += batchSize) {
-                const batch = texts.slice(i, i + batchSize);
-                for (const text of batch) {
-                    // Tokenize the text
-                    const tokens = await this.tokenizer(text, {
-                        truncation: true,
-                        padding: true,
-                        max_length: 512,
-                        return_tensors: 'pt'
-                    });
-                    // Get model output
-                    const output = await this.model(tokens);
-                    // Mean pooling to get sentence embeddings
-                    let embedding;
-                    if (output.last_hidden_state) {
-                        // Mean pooling over sequence length
-                        const tensor = output.last_hidden_state;
-                        const meanPooled = tensor.mean(1); // Mean along sequence dimension
-                        embedding = Array.from(meanPooled.data);
-                    }
-                    else if (output.pooler_output) {
-                        // Use pooler output if available
-                        embedding = Array.from(output.pooler_output.data);
-                    }
-                    else {
-                        throw new Error('Unable to extract embeddings from model output');
-                    }
-                    embeddings.push(embedding);
-                }
-                // Log progress for large batches
-                if (texts.length > 10) {
-                    this.logger.debug(`Processed ${Math.min(i + batchSize, texts.length)}/${texts.length} texts`);
-                }
-            }
-            this.logger.debug(`Successfully created ${embeddings.length} embeddings`);
-            return embeddings;
-        }
-        catch (error) {
-            this.logger.error('Failed to create transformers embeddings:', error);
-            return [];
-        }
-    }
-    getDimensions() {
-        return this.dimensions;
-    }
-    getModelName() {
-        return this.modelName;
-    }
-    async cleanup() {
-        // Clean up model resources if needed
-        this.model = null;
-        this.tokenizer = null;
-        this.isInitialized = false;
-        this.logger.info('Transformers embedding provider cleaned up');
-    }
-}
-exports.TransformersEmbeddingProvider = TransformersEmbeddingProvider;
-function createEmbeddingProvider(config, logger) {
-    if (config.provider === 'openai') {
-        return new OpenAIEmbeddingProvider(config.params, logger);
-    }
-    else if (config.provider === 'transformers') {
-        return new TransformersEmbeddingProvider(config.params, logger);
-    }
-    else {
-        throw new Error(`Unsupported embedding provider: ${config.provider}`);
-    }
-}