@soulcraft/brainy 6.5.0 → 6.6.1

package/dist/embeddings/wasm/WASMEmbeddingEngine.d.ts

@@ -0,0 +1,82 @@
+/**
+ * WASM Embedding Engine
+ *
+ * The main embedding engine that combines all components:
+ * - WordPieceTokenizer: Text → Token IDs
+ * - ONNXInferenceEngine: Token IDs → Hidden States
+ * - EmbeddingPostProcessor: Hidden States → Normalized Embedding
+ *
+ * This replaces transformers.js with a clean, production-grade implementation.
+ *
+ * Features:
+ * - Singleton pattern (one model instance)
+ * - Lazy initialization
+ * - Batch processing support
+ * - Zero runtime dependencies
+ */
+import { EmbeddingResult, EngineStats } from './types.js';
+/**
+ * WASM-based embedding engine
+ */
+export declare class WASMEmbeddingEngine {
+    private tokenizer;
+    private inference;
+    private postProcessor;
+    private initialized;
+    private embedCount;
+    private totalProcessingTimeMs;
+    private constructor();
+    /**
+     * Get the singleton instance
+     */
+    static getInstance(): WASMEmbeddingEngine;
+    /**
+     * Initialize all components
+     */
+    initialize(): Promise<void>;
+    /**
+     * Perform actual initialization
+     */
+    private performInit;
+    /**
+     * Generate embedding for text
+     */
+    embed(text: string): Promise<number[]>;
+    /**
+     * Generate embedding with metadata
+     */
+    embedWithMetadata(text: string): Promise<EmbeddingResult>;
+    /**
+     * Batch embed multiple texts
+     */
+    embedBatch(texts: string[]): Promise<number[][]>;
+    /**
+     * Check if initialized
+     */
+    isInitialized(): boolean;
+    /**
+     * Get engine statistics
+     */
+    getStats(): EngineStats;
+    /**
+     * Dispose and free resources
+     */
+    dispose(): Promise<void>;
+    /**
+     * Reset singleton (for testing)
+     */
+    static resetInstance(): void;
+}
+export declare const wasmEmbeddingEngine: WASMEmbeddingEngine;
+/**
+ * Convenience function to get embeddings
+ */
+export declare function embed(text: string): Promise<number[]>;
+/**
+ * Convenience function for batch embeddings
+ */
+export declare function embedBatch(texts: string[]): Promise<number[][]>;
+/**
+ * Get embedding stats
+ */
+export declare function getEmbeddingStats(): EngineStats;

package/dist/embeddings/wasm/WASMEmbeddingEngine.js

@@ -0,0 +1,231 @@
+/**
+ * WASM Embedding Engine
+ *
+ * The main embedding engine that combines all components:
+ * - WordPieceTokenizer: Text → Token IDs
+ * - ONNXInferenceEngine: Token IDs → Hidden States
+ * - EmbeddingPostProcessor: Hidden States → Normalized Embedding
+ *
+ * This replaces transformers.js with a clean, production-grade implementation.
+ *
+ * Features:
+ * - Singleton pattern (one model instance)
+ * - Lazy initialization
+ * - Batch processing support
+ * - Zero runtime dependencies
+ */
+import { WordPieceTokenizer } from './WordPieceTokenizer.js';
+import { ONNXInferenceEngine } from './ONNXInferenceEngine.js';
+import { EmbeddingPostProcessor } from './EmbeddingPostProcessor.js';
+import { getAssetLoader } from './AssetLoader.js';
+import { MODEL_CONSTANTS } from './types.js';
+// Global singleton instance
+let globalInstance = null;
+let globalInitPromise = null;
+/**
+ * WASM-based embedding engine
+ */
+export class WASMEmbeddingEngine {
+    constructor() {
+        this.tokenizer = null;
+        this.inference = null;
+        this.postProcessor = null;
+        this.initialized = false;
+        this.embedCount = 0;
+        this.totalProcessingTimeMs = 0;
+        // Private constructor for singleton
+    }
+    /**
+     * Get the singleton instance
+     */
+    static getInstance() {
+        if (!globalInstance) {
+            globalInstance = new WASMEmbeddingEngine();
+        }
+        return globalInstance;
+    }
+    /**
+     * Initialize all components
+     */
+    async initialize() {
+        // Already initialized
+        if (this.initialized) {
+            return;
+        }
+        // Initialization in progress
+        if (globalInitPromise) {
+            await globalInitPromise;
+            return;
+        }
+        // Start initialization
+        globalInitPromise = this.performInit();
+        try {
+            await globalInitPromise;
+        }
+        finally {
+            globalInitPromise = null;
+        }
+    }
+    /**
+     * Perform actual initialization
+     */
+    async performInit() {
+        const startTime = Date.now();
+        console.log('🚀 Initializing WASM Embedding Engine...');
+        try {
+            const assetLoader = getAssetLoader();
+            // Verify assets exist
+            const verification = await assetLoader.verifyAssets();
+            if (!verification.valid) {
+                throw new Error(`Missing model assets:\n${verification.errors.join('\n')}\n\n` +
+                    `Expected model at: ${verification.modelPath}\n` +
+                    `Expected vocab at: ${verification.vocabPath}\n\n` +
+                    `Run 'npm run download-model' to download the model files.`);
+            }
+            // Load vocabulary and create tokenizer
+            console.log('📖 Loading vocabulary...');
+            const vocab = await assetLoader.loadVocab();
+            this.tokenizer = new WordPieceTokenizer(vocab);
+            console.log(`✅ Vocabulary loaded: ${this.tokenizer.vocabSize} tokens`);
+            // Initialize ONNX inference engine
+            console.log('🧠 Loading ONNX model...');
+            const modelPath = await assetLoader.getModelPath();
+            this.inference = new ONNXInferenceEngine({ modelPath });
+            await this.inference.initialize(modelPath);
+            console.log('✅ ONNX model loaded');
+            // Create post-processor
+            this.postProcessor = new EmbeddingPostProcessor(MODEL_CONSTANTS.HIDDEN_SIZE);
+            this.initialized = true;
+            const initTime = Date.now() - startTime;
+            console.log(`✅ WASM Embedding Engine ready in ${initTime}ms`);
+        }
+        catch (error) {
+            this.initialized = false;
+            this.tokenizer = null;
+            this.inference = null;
+            this.postProcessor = null;
+            throw new Error(`Failed to initialize WASM Embedding Engine: ${error instanceof Error ? error.message : String(error)}`);
+        }
+    }
+    /**
+     * Generate embedding for text
+     */
+    async embed(text) {
+        const result = await this.embedWithMetadata(text);
+        return result.embedding;
+    }
+    /**
+     * Generate embedding with metadata
+     */
+    async embedWithMetadata(text) {
+        // Ensure initialized
+        if (!this.initialized) {
+            await this.initialize();
+        }
+        if (!this.tokenizer || !this.inference || !this.postProcessor) {
+            throw new Error('Engine not properly initialized');
+        }
+        const startTime = Date.now();
+        // 1. Tokenize
+        const tokenized = this.tokenizer.encode(text);
+        // 2. Run inference
+        const hiddenStates = await this.inference.inferSingle(tokenized.inputIds, tokenized.attentionMask, tokenized.tokenTypeIds);
+        // 3. Post-process (mean pool + normalize)
+        const embedding = this.postProcessor.process(hiddenStates, tokenized.attentionMask, tokenized.inputIds.length);
+        const processingTimeMs = Date.now() - startTime;
+        this.embedCount++;
+        this.totalProcessingTimeMs += processingTimeMs;
+        return {
+            embedding: Array.from(embedding),
+            tokenCount: tokenized.tokenCount,
+            processingTimeMs,
+        };
+    }
+    /**
+     * Batch embed multiple texts
+     */
+    async embedBatch(texts) {
+        // Ensure initialized
+        if (!this.initialized) {
+            await this.initialize();
+        }
+        if (!this.tokenizer || !this.inference || !this.postProcessor) {
+            throw new Error('Engine not properly initialized');
+        }
+        if (texts.length === 0) {
+            return [];
+        }
+        // Tokenize all texts
+        const batch = this.tokenizer.encodeBatch(texts);
+        const seqLen = batch.inputIds[0].length;
+        // Run batch inference
+        const hiddenStates = await this.inference.infer(batch.inputIds, batch.attentionMask, batch.tokenTypeIds);
+        // Post-process each result
+        const embeddings = this.postProcessor.processBatch(hiddenStates, batch.attentionMask, texts.length, seqLen);
+        this.embedCount += texts.length;
+        return embeddings.map(e => Array.from(e));
+    }
+    /**
+     * Check if initialized
+     */
+    isInitialized() {
+        return this.initialized;
+    }
+    /**
+     * Get engine statistics
+     */
+    getStats() {
+        return {
+            initialized: this.initialized,
+            embedCount: this.embedCount,
+            totalProcessingTimeMs: this.totalProcessingTimeMs,
+            avgProcessingTimeMs: this.embedCount > 0
+                ? this.totalProcessingTimeMs / this.embedCount
+                : 0,
+            modelName: MODEL_CONSTANTS.MODEL_NAME,
+        };
+    }
+    /**
+     * Dispose and free resources
+     */
+    async dispose() {
+        if (this.inference) {
+            await this.inference.dispose();
+            this.inference = null;
+        }
+        this.tokenizer = null;
+        this.postProcessor = null;
+        this.initialized = false;
+    }
+    /**
+     * Reset singleton (for testing)
+     */
+    static resetInstance() {
+        if (globalInstance) {
+            globalInstance.dispose();
+        }
+        globalInstance = null;
+        globalInitPromise = null;
+    }
+}
+// Export singleton access
+export const wasmEmbeddingEngine = WASMEmbeddingEngine.getInstance();
+/**
+ * Convenience function to get embeddings
+ */
+export async function embed(text) {
+    return wasmEmbeddingEngine.embed(text);
+}
+/**
+ * Convenience function for batch embeddings
+ */
+export async function embedBatch(texts) {
+    return wasmEmbeddingEngine.embedBatch(texts);
+}
+/**
+ * Get embedding stats
+ */
+export function getEmbeddingStats() {
+    return wasmEmbeddingEngine.getStats();
+}
+//# sourceMappingURL=WASMEmbeddingEngine.js.map

package/dist/embeddings/wasm/WordPieceTokenizer.d.ts

@@ -0,0 +1,71 @@
+/**
+ * WordPiece Tokenizer for BERT-based models
+ *
+ * Implements the WordPiece tokenization algorithm used by all-MiniLM-L6-v2.
+ * This is a clean, dependency-free implementation.
+ *
+ * Algorithm:
+ * 1. Normalize text (lowercase for uncased models)
+ * 2. Split on whitespace and punctuation
+ * 3. Apply WordPiece subword tokenization
+ * 4. Add special tokens ([CLS], [SEP])
+ * 5. Generate attention mask
+ */
+import { TokenizerConfig, TokenizedInput } from './types.js';
+/**
+ * WordPiece tokenizer for BERT-based sentence transformers
+ */
+export declare class WordPieceTokenizer {
+    private vocab;
+    private reverseVocab;
+    private config;
+    constructor(vocab: Map<string, number> | Record<string, number>, config?: Partial<TokenizerConfig>);
+    /**
+     * Tokenize text into token IDs
+     */
+    encode(text: string): TokenizedInput;
+    /**
+     * Encode with padding to fixed length
+     */
+    encodeWithPadding(text: string, targetLength?: number): TokenizedInput;
+    /**
+     * Batch encode multiple texts
+     */
+    encodeBatch(texts: string[]): {
+        inputIds: number[][];
+        attentionMask: number[][];
+        tokenTypeIds: number[][];
+    };
+    /**
+     * Basic tokenization: split on whitespace and punctuation
+     */
+    private basicTokenize;
+    /**
+     * WordPiece tokenization for a single word
+     */
+    private wordPieceTokenize;
+    /**
+     * Check if character is whitespace
+     */
+    private isWhitespace;
+    /**
+     * Check if character is punctuation
+     */
+    private isPunctuation;
+    /**
+     * Decode token IDs back to text (for debugging)
+     */
+    decode(tokenIds: number[]): string;
+    /**
+     * Get vocabulary size
+     */
+    get vocabSize(): number;
+    /**
+     * Get max sequence length
+     */
+    get maxLength(): number;
+}
+/**
+ * Create tokenizer from vocabulary JSON
+ */
+export declare function createTokenizer(vocabJson: Record<string, number>): WordPieceTokenizer;

package/dist/embeddings/wasm/WordPieceTokenizer.js

@@ -0,0 +1,264 @@
+/**
+ * WordPiece Tokenizer for BERT-based models
+ *
+ * Implements the WordPiece tokenization algorithm used by all-MiniLM-L6-v2.
+ * This is a clean, dependency-free implementation.
+ *
+ * Algorithm:
+ * 1. Normalize text (lowercase for uncased models)
+ * 2. Split on whitespace and punctuation
+ * 3. Apply WordPiece subword tokenization
+ * 4. Add special tokens ([CLS], [SEP])
+ * 5. Generate attention mask
+ */
+import { SPECIAL_TOKENS, MODEL_CONSTANTS, } from './types.js';
+/**
+ * WordPiece tokenizer for BERT-based sentence transformers
+ */
+export class WordPieceTokenizer {
+    constructor(vocab, config) {
+        // Convert Record to Map if needed
+        this.vocab = vocab instanceof Map ? vocab : new Map(Object.entries(vocab));
+        // Build reverse vocab for debugging
+        this.reverseVocab = new Map();
+        for (const [token, id] of this.vocab) {
+            this.reverseVocab.set(id, token);
+        }
+        // Default config for all-MiniLM-L6-v2
+        this.config = {
+            vocab: this.vocab,
+            unkTokenId: config?.unkTokenId ?? SPECIAL_TOKENS.UNK,
+            clsTokenId: config?.clsTokenId ?? SPECIAL_TOKENS.CLS,
+            sepTokenId: config?.sepTokenId ?? SPECIAL_TOKENS.SEP,
+            padTokenId: config?.padTokenId ?? SPECIAL_TOKENS.PAD,
+            maxLength: config?.maxLength ?? MODEL_CONSTANTS.MAX_SEQUENCE_LENGTH,
+            doLowerCase: config?.doLowerCase ?? true,
+        };
+    }
+    /**
+     * Tokenize text into token IDs
+     */
+    encode(text) {
+        // 1. Normalize
+        let normalizedText = text;
+        if (this.config.doLowerCase) {
+            normalizedText = text.toLowerCase();
+        }
+        // 2. Clean and split into words
+        const words = this.basicTokenize(normalizedText);
+        // 3. Apply WordPiece to each word
+        const tokens = [this.config.clsTokenId];
+        for (const word of words) {
+            const wordTokens = this.wordPieceTokenize(word);
+            // Check if adding these tokens would exceed max length (accounting for [SEP])
+            if (tokens.length + wordTokens.length + 1 > this.config.maxLength) {
+                break;
+            }
+            tokens.push(...wordTokens);
+        }
+        tokens.push(this.config.sepTokenId);
+        // 4. Generate attention mask and token type IDs
+        const attentionMask = new Array(tokens.length).fill(1);
+        const tokenTypeIds = new Array(tokens.length).fill(0);
+        return {
+            inputIds: tokens,
+            attentionMask,
+            tokenTypeIds,
+            tokenCount: tokens.length - 2, // Exclude [CLS] and [SEP]
+        };
+    }
+    /**
+     * Encode with padding to fixed length
+     */
+    encodeWithPadding(text, targetLength) {
+        const result = this.encode(text);
+        const padLength = targetLength ?? this.config.maxLength;
+        // Pad to target length
+        while (result.inputIds.length < padLength) {
+            result.inputIds.push(this.config.padTokenId);
+            result.attentionMask.push(0);
+            result.tokenTypeIds.push(0);
+        }
+        // Truncate if longer (shouldn't happen with proper encode())
+        if (result.inputIds.length > padLength) {
+            result.inputIds.length = padLength;
+            result.attentionMask.length = padLength;
+            result.tokenTypeIds.length = padLength;
+            // Ensure [SEP] is at the end
+            result.inputIds[padLength - 1] = this.config.sepTokenId;
+            result.attentionMask[padLength - 1] = 1;
+        }
+        return result;
+    }
+    /**
+     * Batch encode multiple texts
+     */
+    encodeBatch(texts) {
+        const results = texts.map((text) => this.encode(text));
+        // Find max length in batch
+        const maxLen = Math.max(...results.map((r) => r.inputIds.length));
+        // Pad all to same length
+        const inputIds = [];
+        const attentionMask = [];
+        const tokenTypeIds = [];
+        for (const result of results) {
+            const padded = this.encodeWithPadding('', // Not used since we're modifying result
+            maxLen);
+            // Copy original values
+            for (let i = 0; i < result.inputIds.length; i++) {
+                padded.inputIds[i] = result.inputIds[i];
+                padded.attentionMask[i] = result.attentionMask[i];
+                padded.tokenTypeIds[i] = result.tokenTypeIds[i];
+            }
+            // Pad the rest
+            for (let i = result.inputIds.length; i < maxLen; i++) {
+                padded.inputIds[i] = this.config.padTokenId;
+                padded.attentionMask[i] = 0;
+                padded.tokenTypeIds[i] = 0;
+            }
+            inputIds.push(padded.inputIds.slice(0, maxLen));
+            attentionMask.push(padded.attentionMask.slice(0, maxLen));
+            tokenTypeIds.push(padded.tokenTypeIds.slice(0, maxLen));
+        }
+        return { inputIds, attentionMask, tokenTypeIds };
+    }
+    /**
+     * Basic tokenization: split on whitespace and punctuation
+     */
+    basicTokenize(text) {
+        // Clean whitespace
+        text = text.trim().replace(/\s+/g, ' ');
+        if (!text) {
+            return [];
+        }
+        const words = [];
+        let currentWord = '';
+        for (const char of text) {
+            if (this.isWhitespace(char)) {
+                if (currentWord) {
+                    words.push(currentWord);
+                    currentWord = '';
+                }
+            }
+            else if (this.isPunctuation(char)) {
+                if (currentWord) {
+                    words.push(currentWord);
+                    currentWord = '';
+                }
+                words.push(char);
+            }
+            else {
+                currentWord += char;
+            }
+        }
+        if (currentWord) {
+            words.push(currentWord);
+        }
+        return words;
+    }
+    /**
+     * WordPiece tokenization for a single word
+     */
+    wordPieceTokenize(word) {
+        if (!word) {
+            return [];
+        }
+        // Check if whole word is in vocabulary
+        if (this.vocab.has(word)) {
+            return [this.vocab.get(word)];
+        }
+        const tokens = [];
+        let start = 0;
+        while (start < word.length) {
+            let end = word.length;
+            let foundToken = false;
+            while (start < end) {
+                let substr = word.slice(start, end);
+                // Add ## prefix for subwords (not at start of word)
+                if (start > 0) {
+                    substr = '##' + substr;
+                }
+                if (this.vocab.has(substr)) {
+                    tokens.push(this.vocab.get(substr));
+                    foundToken = true;
+                    break;
+                }
+                end--;
+            }
+            if (!foundToken) {
+                // Unknown character - use [UNK] for single character
+                tokens.push(this.config.unkTokenId);
+                start++;
+            }
+            else {
+                start = end;
+            }
+        }
+        return tokens;
+    }
+    /**
+     * Check if character is whitespace
+     */
+    isWhitespace(char) {
+        return /\s/.test(char);
+    }
+    /**
+     * Check if character is punctuation
+     */
+    isPunctuation(char) {
+        const code = char.charCodeAt(0);
+        // ASCII punctuation ranges
+        if ((code >= 33 && code <= 47) || // !"#$%&'()*+,-./
+            (code >= 58 && code <= 64) || // :;<=>?@
+            (code >= 91 && code <= 96) || // [\]^_`
+            (code >= 123 && code <= 126) // {|}~
+        ) {
+            return true;
+        }
+        // Unicode punctuation categories
+        return /[\u2000-\u206F\u2E00-\u2E7F\\'!"#$%&()*+,\-./:;<=>?@\[\]^_`{|}~]/.test(char);
+    }
+    /**
+     * Decode token IDs back to text (for debugging)
+     */
+    decode(tokenIds) {
+        const tokens = [];
+        for (const id of tokenIds) {
+            const token = this.reverseVocab.get(id);
+            if (token && !['[CLS]', '[SEP]', '[PAD]'].includes(token)) {
+                if (token.startsWith('##')) {
+                    // Subword - append without space
+                    if (tokens.length > 0) {
+                        tokens[tokens.length - 1] += token.slice(2);
+                    }
+                    else {
+                        tokens.push(token.slice(2));
+                    }
+                }
+                else {
+                    tokens.push(token);
+                }
+            }
+        }
+        return tokens.join(' ');
+    }
+    /**
+     * Get vocabulary size
+     */
+    get vocabSize() {
+        return this.vocab.size;
+    }
+    /**
+     * Get max sequence length
+     */
+    get maxLength() {
+        return this.config.maxLength;
+    }
+}
+/**
+ * Create tokenizer from vocabulary JSON
+ */
+export function createTokenizer(vocabJson) {
+    return new WordPieceTokenizer(vocabJson);
+}
+//# sourceMappingURL=WordPieceTokenizer.js.map

package/dist/embeddings/wasm/index.d.ts

@@ -0,0 +1,13 @@
+/**
+ * WASM Embedding Engine - Public Exports
+ *
+ * Clean, production-grade embedding engine using direct ONNX WASM.
+ * No transformers.js dependency, no runtime downloads, works everywhere.
+ */
+export { WASMEmbeddingEngine, wasmEmbeddingEngine, embed, embedBatch, getEmbeddingStats, } from './WASMEmbeddingEngine.js';
+export { WordPieceTokenizer, createTokenizer } from './WordPieceTokenizer.js';
+export { ONNXInferenceEngine, createInferenceEngine } from './ONNXInferenceEngine.js';
+export { EmbeddingPostProcessor, createPostProcessor } from './EmbeddingPostProcessor.js';
+export { AssetLoader, getAssetLoader, createAssetLoader } from './AssetLoader.js';
+export type { TokenizerConfig, TokenizedInput, InferenceConfig, EmbeddingResult, EngineStats, ModelConfig, } from './types.js';
+export { SPECIAL_TOKENS, MODEL_CONSTANTS } from './types.js';

package/dist/embeddings/wasm/index.js

@@ -0,0 +1,15 @@
+/**
+ * WASM Embedding Engine - Public Exports
+ *
+ * Clean, production-grade embedding engine using direct ONNX WASM.
+ * No transformers.js dependency, no runtime downloads, works everywhere.
+ */
+// Main engine
+export { WASMEmbeddingEngine, wasmEmbeddingEngine, embed, embedBatch, getEmbeddingStats, } from './WASMEmbeddingEngine.js';
+// Components (for advanced use)
+export { WordPieceTokenizer, createTokenizer } from './WordPieceTokenizer.js';
+export { ONNXInferenceEngine, createInferenceEngine } from './ONNXInferenceEngine.js';
+export { EmbeddingPostProcessor, createPostProcessor } from './EmbeddingPostProcessor.js';
+export { AssetLoader, getAssetLoader, createAssetLoader } from './AssetLoader.js';
+export { SPECIAL_TOKENS, MODEL_CONSTANTS } from './types.js';
+//# sourceMappingURL=index.js.map