@ninebix/nmt-system 1.0.0

package/dist/src/services/embedding-provider.js

@@ -0,0 +1,176 @@
+/**
+ * Embedding Provider - Xenova Transformers integration
+ *
+ * Provides semantic embeddings using Xenova/transformers.js
+ * Model: all-MiniLM-L6-v2 (384 dimensions)
+ *
+ * @module services/embedding-provider
+ */
+// Lazy-loaded pipeline
+let _pipeline = null;
+let _pipelinePromise = null;
+/**
+ * Initialize the embedding pipeline
+ */
+async function getPipeline() {
+    if (_pipeline)
+        return _pipeline;
+    if (_pipelinePromise)
+        return _pipelinePromise;
+    _pipelinePromise = (async () => {
+        const { pipeline } = await import('@xenova/transformers');
+        _pipeline = await pipeline('feature-extraction', 'Xenova/all-MiniLM-L6-v2', {
+            quantized: true,
+        });
+        return _pipeline;
+    })();
+    return _pipelinePromise;
+}
+/**
+ * Xenova Embedding Provider
+ *
+ * Uses Xenova/transformers.js for high-quality semantic embeddings
+ */
+export class XenovaEmbeddingProvider {
+    ready = false;
+    initPromise = null;
+    /**
+     * Initialize the provider (lazy)
+     */
+    async init() {
+        if (this.ready)
+            return;
+        if (this.initPromise)
+            return this.initPromise;
+        this.initPromise = (async () => {
+            await getPipeline();
+            this.ready = true;
+        })();
+        return this.initPromise;
+    }
+    /**
+     * Check if provider is ready
+     */
+    isReady() {
+        return this.ready;
+    }
+    /**
+     * Generate embedding for a single text
+     */
+    async embed(text) {
+        const pipe = await getPipeline();
+        const output = await pipe(text, { pooling: 'mean', normalize: true });
+        // Convert to Float32Array
+        const embedding = new Float32Array(384);
+        const data = output.data;
+        for (let i = 0; i < 384; i++) {
+            embedding[i] = data[i];
+        }
+        return embedding;
+    }
+    /**
+     * Generate embeddings for multiple texts (batched)
+     */
+    async embedBatch(texts) {
+        const pipe = await getPipeline();
+        // Process in batches of 32 for memory efficiency
+        const batchSize = 32;
+        const results = [];
+        for (let i = 0; i < texts.length; i += batchSize) {
+            const batch = texts.slice(i, i + batchSize);
+            const outputs = await pipe(batch, { pooling: 'mean', normalize: true });
+            for (let j = 0; j < batch.length; j++) {
+                const embedding = new Float32Array(384);
+                const startIdx = j * 384;
+                for (let k = 0; k < 384; k++) {
+                    embedding[k] = outputs.data[startIdx + k];
+                }
+                results.push(embedding);
+            }
+        }
+        return results;
+    }
+}
+/**
+ * Deterministic Embedding Provider (fallback)
+ *
+ * Creates pseudo-random but deterministic embeddings from text hash.
+ * Used when Xenova is not available or for testing.
+ */
+export class DeterministicEmbeddingProvider {
+    isReady() {
+        return true;
+    }
+    async embed(text) {
+        const embedding = new Float32Array(384);
+        // Generate deterministic hash-based embedding
+        let hash = 0;
+        for (let i = 0; i < text.length; i++) {
+            hash = ((hash << 5) - hash) + text.charCodeAt(i);
+            hash |= 0;
+        }
+        for (let i = 0; i < 384; i++) {
+            hash = ((hash << 5) - hash) + i;
+            hash |= 0;
+            embedding[i] = (hash % 1000) / 1000 - 0.5;
+        }
+        // Normalize to unit vector
+        let norm = 0;
+        for (let i = 0; i < 384; i++) {
+            norm += embedding[i] * embedding[i];
+        }
+        norm = Math.sqrt(norm);
+        for (let i = 0; i < 384; i++) {
+            embedding[i] /= norm;
+        }
+        return embedding;
+    }
+    async embedBatch(texts) {
+        return Promise.all(texts.map(t => this.embed(t)));
+    }
+}
+// Singleton instances
+let _xenovaProvider = null;
+let _deterministicProvider = null;
+/**
+ * Get the Xenova embedding provider (singleton)
+ */
+export function getXenovaProvider() {
+    if (!_xenovaProvider) {
+        _xenovaProvider = new XenovaEmbeddingProvider();
+    }
+    return _xenovaProvider;
+}
+/**
+ * Get the deterministic embedding provider (singleton)
+ */
+export function getDeterministicProvider() {
+    if (!_deterministicProvider) {
+        _deterministicProvider = new DeterministicEmbeddingProvider();
+    }
+    return _deterministicProvider;
+}
+/**
+ * Get the best available embedding provider
+ *
+ * Tries Xenova first, falls back to deterministic if unavailable
+ */
+export async function getEmbeddingProvider() {
+    try {
+        const xenova = getXenovaProvider();
+        await xenova.init();
+        return xenova;
+    }
+    catch {
+        console.warn('Xenova not available, using deterministic embeddings');
+        return getDeterministicProvider();
+    }
+}
+/**
+ * Quick embedding function using the best available provider
+ */
+export async function quickEmbed(text) {
+    const provider = await getEmbeddingProvider();
+    return provider.embed(text);
+}
+//# sourceMappingURL=embedding-provider.js.map

package/dist/src/services/embedding-provider.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"embedding-provider.js","sourceRoot":"","sources":["../../../src/services/embedding-provider.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAaH,uBAAuB;AACvB,IAAI,SAAS,GAAQ,IAAI,CAAC;AAC1B,IAAI,gBAAgB,GAAwB,IAAI,CAAC;AAEjD;;GAEG;AACH,KAAK,UAAU,WAAW;IACxB,IAAI,SAAS;QAAE,OAAO,SAAS,CAAC;IAEhC,IAAI,gBAAgB;QAAE,OAAO,gBAAgB,CAAC;IAE9C,gBAAgB,GAAG,CAAC,KAAK,IAAI,EAAE;QAC7B,MAAM,EAAE,QAAQ,EAAE,GAAG,MAAM,MAAM,CAAC,sBAAsB,CAAC,CAAC;QAC1D,SAAS,GAAG,MAAM,QAAQ,CAAC,oBAAoB,EAAE,yBAAyB,EAAE;YAC1E,SAAS,EAAE,IAAI;SAChB,CAAC,CAAC;QACH,OAAO,SAAS,CAAC;IACnB,CAAC,CAAC,EAAE,CAAC;IAEL,OAAO,gBAAgB,CAAC;AAC1B,CAAC;AAED;;;;GAIG;AACH,MAAM,OAAO,uBAAuB;IAC1B,KAAK,GAAY,KAAK,CAAC;IACvB,WAAW,GAAyB,IAAI,CAAC;IAEjD;;OAEG;IACH,KAAK,CAAC,IAAI;QACR,IAAI,IAAI,CAAC,KAAK;YAAE,OAAO;QACvB,IAAI,IAAI,CAAC,WAAW;YAAE,OAAO,IAAI,CAAC,WAAW,CAAC;QAE9C,IAAI,CAAC,WAAW,GAAG,CAAC,KAAK,IAAI,EAAE;YAC7B,MAAM,WAAW,EAAE,CAAC;YACpB,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC;QACpB,CAAC,CAAC,EAAE,CAAC;QAEL,OAAO,IAAI,CAAC,WAAW,CAAC;IAC1B,CAAC;IAED;;OAEG;IACH,OAAO;QACL,OAAO,IAAI,CAAC,KAAK,CAAC;IACpB,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,KAAK,CAAC,IAAY;QACtB,MAAM,IAAI,GAAG,MAAM,WAAW,EAAE,CAAC;QACjC,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,IAAI,EAAE,EAAE,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAEtE,0BAA0B;QAC1B,MAAM,SAAS,GAAG,IAAI,YAAY,CAAC,GAAG,CAAC,CAAC;QACxC,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC;QAEzB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC;YAC7B,SAAS,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC;QACzB,CAAC;QAED,OAAO,SAAS,CAAC;IACnB,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,UAAU,CAAC,KAAe;QAC9B,MAAM,IAAI,GAAG,MAAM,WAAW,EAAE,CAAC;QAEjC,iDAAiD;QACjD,MAAM,SAAS,GAAG,EAAE,CAAC;QACrB,MAAM,OAAO,GAAmB,EAAE,CAAC;QAEnC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,IAAI,SAAS,EAAE,CAAC;YACjD,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,SAAS,CAAC,CAAC;YAC5C,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,KAAK,EAAE,EAAE,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;YAExE,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;gBACtC,MAAM,SAAS,GAAG,IAAI,YAAY,CAAC,GAAG,CAAC,CAAC;gBACxC,MAAM,QAAQ,GAAG,CAAC,GAAG,GAAG,CAAC;gBAEzB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC;oBAC7B,SAAS,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,QAAQ,GAAG,CAAC,CAAC,CAAC;gBAC5C,CAAC;gBAED,OAAO,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;YAC1B,CAAC;QACH,CAAC;QAED,OAAO,OAAO,CAAC;IACjB,CAAC;CACF;AAED;;;;;GAKG;AACH,MAAM,OAAO,8BAA8B;IACzC,OAAO;QACL,OAAO,IAAI,CAAC;IACd,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,IAAY;QACtB,MAAM,SAAS,GAAG,IAAI,YAAY,CAAC,GAAG,CAAC,CAAC;QAExC,8CAA8C;QAC9C,IAAI,IAAI,GAAG,CAAC,CAAC;QACb,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACrC,IAAI,GAAG,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC,GAAG,IAAI,CAAC,GAAG,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC;YACjD,IAAI,IAAI,CAAC,CAAC;QACZ,CAAC;QAED,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC;YAC7B,IAAI,GAAG,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC;YAChC,IAAI,IAAI,CAAC,CAAC;YACV,SAAS,CAAC,CAAC,CAAC,GAAG,CAAC,IAAI,GAAG,IAAI,CAAC,GAAG,IAAI,GAAG,GAAG,CAAC;QAC5C,CAAC;QAED,2BAA2B;QAC3B,IAAI,IAAI,GAAG,CAAC,CAAC;QACb,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC;YAC7B,IAAI,IAAI,SAAS,CAAC,CAAC,CAAC,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC;QACtC,CAAC;QACD,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACvB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC;YAC7B,SAAS,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC;QACvB,CAAC;QAED,OAAO,SAAS,CAAC;IACnB,CAAC;IAED,KAAK,CAAC,UAAU,CAAC,KAAe;QAC9B,OAAO,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IACpD,CAAC;CACF;AAED,sBAAsB;AACtB,IAAI,eAAe,GAAmC,IAAI,CAAC;AAC3D,IAAI,sBAAsB,GAA0C,IAAI,CAAC;AAEzE;;GAEG;AACH,MAAM,UAAU,iBAAiB;IAC/B,IAAI,CAAC,eAAe,EAAE,CAAC;QACrB,eAAe,GAAG,IAAI,uBAAuB,EAAE,CAAC;IAClD,CAAC;IACD,OAAO,eAAe,CAAC;AACzB,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,wBAAwB;IACtC,IAAI,CAAC,sBAAsB,EAAE,CAAC;QAC5B,sBAAsB,GAAG,IAAI,8BAA8B,EAAE,CAAC;IAChE,CAAC;IACD,OAAO,sBAAsB,CAAC;AAChC,CAAC;AAED;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,oBAAoB;IACxC,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,iBAAiB,EAAE,CAAC;QACnC,MAAM,MAAM,CAAC,IAAI,EAAE,CAAC;QACpB,OAAO,MAAM,CAAC;IAChB,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,CAAC,IAAI,CAAC,sDAAsD,CAAC,CAAC;QACrE,OAAO,wBAAwB,EAAE,CAAC;IACpC,CAAC;AACH,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,UAAU,CAAC,IAAY;IAC3C,MAAM,QAAQ,GAAG,MAAM,oBAAoB,EAAE,CAAC;IAC9C,OAAO,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;AAC9B,CAAC"}