@soulcraft/brainy 3.20.1 → 3.20.3

package/dist/embeddings/lightweight-embedder.js

@@ -1,128 +0,0 @@
-/**
- * Lightweight Embedding Alternative
- *
- * Uses pre-computed embeddings for common terms
- * Falls back to ONNX for unknown terms
- *
- * This reduces memory usage by 90% for typical queries
- */
-import { singletonModelManager } from './SingletonModelManager.js';
-// Pre-computed embeddings for top 10,000 common terms
-// In production, this would be loaded from a file
-const PRECOMPUTED_EMBEDDINGS = {
-    // Programming languages
-    'javascript': new Array(384).fill(0).map((_, i) => Math.sin(i * 0.1)),
-    'python': new Array(384).fill(0).map((_, i) => Math.cos(i * 0.1)),
-    'typescript': new Array(384).fill(0).map((_, i) => Math.sin(i * 0.15)),
-    'java': new Array(384).fill(0).map((_, i) => Math.cos(i * 0.15)),
-    'rust': new Array(384).fill(0).map((_, i) => Math.sin(i * 0.2)),
-    'go': new Array(384).fill(0).map((_, i) => Math.cos(i * 0.2)),
-    // Frameworks
-    'react': new Array(384).fill(0).map((_, i) => Math.sin(i * 0.25)),
-    'vue': new Array(384).fill(0).map((_, i) => Math.cos(i * 0.25)),
-    'angular': new Array(384).fill(0).map((_, i) => Math.sin(i * 0.3)),
-    'svelte': new Array(384).fill(0).map((_, i) => Math.cos(i * 0.3)),
-    // Databases
-    'postgresql': new Array(384).fill(0).map((_, i) => Math.sin(i * 0.35)),
-    'mysql': new Array(384).fill(0).map((_, i) => Math.cos(i * 0.35)),
-    'mongodb': new Array(384).fill(0).map((_, i) => Math.sin(i * 0.4)),
-    'redis': new Array(384).fill(0).map((_, i) => Math.cos(i * 0.4)),
-    // Common terms
-    'database': new Array(384).fill(0).map((_, i) => Math.sin(i * 0.45)),
-    'api': new Array(384).fill(0).map((_, i) => Math.cos(i * 0.45)),
-    'server': new Array(384).fill(0).map((_, i) => Math.sin(i * 0.5)),
-    'client': new Array(384).fill(0).map((_, i) => Math.cos(i * 0.5)),
-    'frontend': new Array(384).fill(0).map((_, i) => Math.sin(i * 0.55)),
-    'backend': new Array(384).fill(0).map((_, i) => Math.cos(i * 0.55)),
-    // Add more pre-computed embeddings here...
-};
-// Simple word similarity using character n-grams
-function computeSimpleEmbedding(text) {
-    const normalized = text.toLowerCase().trim();
-    const vector = new Array(384).fill(0);
-    // Character trigrams for simple semantic similarity
-    for (let i = 0; i < normalized.length - 2; i++) {
-        const trigram = normalized.slice(i, i + 3);
-        const hash = trigram.charCodeAt(0) * 31 +
-            trigram.charCodeAt(1) * 7 +
-            trigram.charCodeAt(2);
-        const index = Math.abs(hash) % 384;
-        vector[index] += 1 / (normalized.length - 2);
-    }
-    // Normalize vector
-    const magnitude = Math.sqrt(vector.reduce((sum, val) => sum + val * val, 0));
-    if (magnitude > 0) {
-        for (let i = 0; i < vector.length; i++) {
-            vector[i] /= magnitude;
-        }
-    }
-    return vector;
-}
-export class LightweightEmbedder {
-    constructor() {
-        this.stats = {
-            precomputedHits: 0,
-            simpleComputes: 0,
-            onnxComputes: 0
-        };
-    }
-    async embed(text) {
-        if (Array.isArray(text)) {
-            return Promise.all(text.map(t => this.embedSingle(t)));
-        }
-        return this.embedSingle(text);
-    }
-    async embedSingle(text) {
-        const normalized = text.toLowerCase().trim();
-        // 1. Check pre-computed embeddings (instant, zero memory)
-        if (PRECOMPUTED_EMBEDDINGS[normalized]) {
-            this.stats.precomputedHits++;
-            return PRECOMPUTED_EMBEDDINGS[normalized];
-        }
-        // 2. Check for close matches in pre-computed
-        for (const [term, embedding] of Object.entries(PRECOMPUTED_EMBEDDINGS)) {
-            if (normalized.includes(term) || term.includes(normalized)) {
-                this.stats.precomputedHits++;
-                // Return slightly modified version to maintain uniqueness
-                return embedding.map(v => v * 0.95);
-            }
-        }
-        // 3. For short text, use simple embedding (fast, low memory)
-        if (normalized.length < 50) {
-            this.stats.simpleComputes++;
-            return computeSimpleEmbedding(normalized);
-        }
-        // 4. Last resort: Use SingletonModelManager for complex text
-        console.log('⚠️ Using singleton model for complex text...');
-        this.stats.onnxComputes++;
-        return await singletonModelManager.embed(text);
-    }
-    getStats() {
-        return {
-            ...this.stats,
-            totalEmbeddings: this.stats.precomputedHits +
-                this.stats.simpleComputes +
-                this.stats.onnxComputes,
-            cacheHitRate: this.stats.precomputedHits /
-                (this.stats.precomputedHits +
-                    this.stats.simpleComputes +
-                    this.stats.onnxComputes)
-        };
-    }
-    // Pre-load common embeddings from file
-    async loadPrecomputed(filePath) {
-        if (!filePath)
-            return;
-        try {
-            const fs = await import('fs/promises');
-            const data = await fs.readFile(filePath, 'utf-8');
-            const embeddings = JSON.parse(data);
-            Object.assign(PRECOMPUTED_EMBEDDINGS, embeddings);
-            console.log(`✅ Loaded ${Object.keys(embeddings).length} pre-computed embeddings`);
-        }
-        catch (error) {
-            console.warn('Could not load pre-computed embeddings:', error);
-        }
-    }
-}
-//# sourceMappingURL=lightweight-embedder.js.map

package/dist/embeddings/model-manager.d.ts

@@ -1,39 +0,0 @@
-/**
- * Model Manager - Ensures transformer models are available at runtime
- *
- * Strategy (in order):
- * 1. Check local cache first (instant)
- * 2. Try Soulcraft CDN (fastest when available)
- * 3. Try GitHub release tar.gz with extraction (reliable backup)
- * 4. Fall back to Hugging Face (always works)
- *
- * NO USER CONFIGURATION REQUIRED - Everything is automatic!
- */
-export declare class ModelManager {
-    private static instance;
-    private modelsPath;
-    private isInitialized;
-    private constructor();
-    static getInstance(): ModelManager;
-    private getModelsPath;
-    ensureModels(modelName?: string): Promise<boolean>;
-    private verifyModelFiles;
-    /**
-     * Check which model variants are available locally
-     */
-    getAvailableModels(modelName?: string): {
-        fp32: boolean;
-        q8: boolean;
-    };
-    /**
-     * Get the best available model variant based on preference and availability
-     */
-    getBestAvailableModel(preferredType?: 'fp32' | 'q8', modelName?: string): 'fp32' | 'q8' | null;
-    private tryModelSource;
-    private downloadAndExtractFromGitHub;
-    /**
-     * Pre-download models for deployment
-     * This is what npm run download-models calls
-     */
-    static predownload(): Promise<void>;
-}

package/dist/embeddings/model-manager.js

@@ -1,245 +0,0 @@
-/**
- * Model Manager - Ensures transformer models are available at runtime
- *
- * Strategy (in order):
- * 1. Check local cache first (instant)
- * 2. Try Soulcraft CDN (fastest when available)
- * 3. Try GitHub release tar.gz with extraction (reliable backup)
- * 4. Fall back to Hugging Face (always works)
- *
- * NO USER CONFIGURATION REQUIRED - Everything is automatic!
- */
-import { existsSync } from 'fs';
-import { mkdir, writeFile } from 'fs/promises';
-import { join } from 'path';
-import { env } from '@huggingface/transformers';
-// Model sources in order of preference
-const MODEL_SOURCES = {
-    // CDN - Fastest when available (currently active)
-    cdn: {
-        host: 'https://models.soulcraft.com/models',
-        pathTemplate: '{model}/', // e.g., Xenova/all-MiniLM-L6-v2/
-        testFile: 'config.json' // File to test availability
-    },
-    // GitHub Release - tar.gz fallback (already exists and works)
-    githubRelease: {
-        tarUrl: 'https://github.com/soulcraftlabs/brainy/releases/download/models-v1/all-MiniLM-L6-v2.tar.gz'
-    },
-    // Original Hugging Face - final fallback (always works)
-    huggingface: {
-        host: 'https://huggingface.co',
-        pathTemplate: '{model}/resolve/{revision}/' // Default transformers.js pattern
-    }
-};
-// Model verification files - BOTH fp32 and q8 variants
-const REQUIRED_FILES = [
-    'config.json',
-    'tokenizer.json',
-    'tokenizer_config.json'
-];
-const MODEL_VARIANTS = {
-    fp32: 'onnx/model.onnx',
-    q8: 'onnx/model_quantized.onnx'
-};
-export class ModelManager {
-    constructor() {
-        this.isInitialized = false;
-        // Determine models path
-        this.modelsPath = this.getModelsPath();
-    }
-    static getInstance() {
-        if (!ModelManager.instance) {
-            ModelManager.instance = new ModelManager();
-        }
-        return ModelManager.instance;
-    }
-    getModelsPath() {
-        // Check various possible locations
-        const paths = [
-            process.env.BRAINY_MODELS_PATH,
-            './models',
-            join(process.cwd(), 'models'),
-            join(process.env.HOME || '', '.brainy', 'models'),
-            env.cacheDir
-        ];
-        // Find first existing path or use default
-        for (const path of paths) {
-            if (path && existsSync(path)) {
-                return path;
-            }
-        }
-        // Default to local models directory
-        return join(process.cwd(), 'models');
-    }
-    async ensureModels(modelName = 'Xenova/all-MiniLM-L6-v2') {
-        if (this.isInitialized) {
-            return true;
-        }
-        // Configure transformers.js environment
-        env.cacheDir = this.modelsPath;
-        env.allowLocalModels = true;
-        env.useFSCache = true;
-        // Check if model already exists locally
-        const modelPath = join(this.modelsPath, ...modelName.split('/'));
-        if (await this.verifyModelFiles(modelPath)) {
-            console.log('✅ Models found in cache:', modelPath);
-            env.allowRemoteModels = false; // Use local only
-            this.isInitialized = true;
-            return true;
-        }
-        // Try to download from our sources
-        console.log('📥 Downloading transformer models...');
-        // Try CDN first (fastest when available)
-        if (await this.tryModelSource('Soulcraft CDN', MODEL_SOURCES.cdn, modelName)) {
-            this.isInitialized = true;
-            return true;
-        }
-        // Try GitHub release with tar.gz extraction (reliable backup)
-        if (await this.downloadAndExtractFromGitHub(modelName)) {
-            this.isInitialized = true;
-            return true;
-        }
-        // Fall back to Hugging Face (always works)
-        console.log('⚠️ Using Hugging Face fallback for models');
-        env.remoteHost = MODEL_SOURCES.huggingface.host;
-        env.remotePathTemplate = MODEL_SOURCES.huggingface.pathTemplate;
-        env.allowRemoteModels = true;
-        this.isInitialized = true;
-        return true;
-    }
-    async verifyModelFiles(modelPath) {
-        // Check if essential files exist
-        for (const file of REQUIRED_FILES) {
-            const fullPath = join(modelPath, file);
-            if (!existsSync(fullPath)) {
-                return false;
-            }
-        }
-        // At least one model variant must exist (fp32 or q8)
-        const fp32Exists = existsSync(join(modelPath, MODEL_VARIANTS.fp32));
-        const q8Exists = existsSync(join(modelPath, MODEL_VARIANTS.q8));
-        return fp32Exists || q8Exists;
-    }
-    /**
-     * Check which model variants are available locally
-     */
-    getAvailableModels(modelName = 'Xenova/all-MiniLM-L6-v2') {
-        const modelPath = join(this.modelsPath, modelName);
-        return {
-            fp32: existsSync(join(modelPath, MODEL_VARIANTS.fp32)),
-            q8: existsSync(join(modelPath, MODEL_VARIANTS.q8))
-        };
-    }
-    /**
-     * Get the best available model variant based on preference and availability
-     */
-    getBestAvailableModel(preferredType = 'fp32', modelName = 'Xenova/all-MiniLM-L6-v2') {
-        const available = this.getAvailableModels(modelName);
-        // If preferred type is available, use it
-        if (available[preferredType]) {
-            return preferredType;
-        }
-        // Otherwise fall back to what's available
-        if (preferredType === 'q8' && available.fp32) {
-            console.warn('⚠️ Q8 model requested but not available, falling back to FP32');
-            return 'fp32';
-        }
-        if (preferredType === 'fp32' && available.q8) {
-            console.warn('⚠️ FP32 model requested but not available, falling back to Q8');
-            return 'q8';
-        }
-        return null;
-    }
-    async tryModelSource(name, source, modelName) {
-        try {
-            console.log(`📥 Trying ${name}...`);
-            // Test if the source is accessible by trying to fetch a test file
-            const testFile = source.testFile || 'config.json';
-            const modelPath = source.pathTemplate.replace('{model}', modelName).replace('{revision}', 'main');
-            const testUrl = `${source.host}/${modelPath}${testFile}`;
-            const response = await fetch(testUrl).catch(() => null);
-            if (response && response.ok) {
-                console.log(`✅ ${name} is available`);
-                // Configure transformers.js to use this source
-                env.remoteHost = source.host;
-                env.remotePathTemplate = source.pathTemplate;
-                env.allowRemoteModels = true;
-                // The model will be downloaded automatically by transformers.js when needed
-                return true;
-            }
-            else {
-                console.log(`⚠️ ${name} not available (${response?.status || 'unreachable'})`);
-                return false;
-            }
-        }
-        catch (error) {
-            console.log(`⚠️ ${name} check failed:`, error.message);
-            return false;
-        }
-    }
-    async downloadAndExtractFromGitHub(modelName) {
-        try {
-            console.log('📥 Trying GitHub Release (tar.gz)...');
-            // Download tar.gz file
-            const response = await fetch(MODEL_SOURCES.githubRelease.tarUrl);
-            if (!response.ok) {
-                console.log(`⚠️ GitHub Release not available (${response.status})`);
-                return false;
-            }
-            // Since we can't use tar-stream, we'll use Node's built-in child_process
-            // to extract using system tar command (available on all Unix systems)
-            const buffer = await response.arrayBuffer();
-            const modelPath = join(this.modelsPath, ...modelName.split('/'));
-            // Create model directory
-            await mkdir(modelPath, { recursive: true });
-            // Write tar.gz to temp file and extract
-            const tempFile = join(this.modelsPath, 'temp-model.tar.gz');
-            await writeFile(tempFile, Buffer.from(buffer));
-            // Extract using system tar command
-            const { exec } = await import('child_process');
-            const { promisify } = await import('util');
-            const execAsync = promisify(exec);
-            try {
-                // Extract and strip the first directory component
-                await execAsync(`tar -xzf ${tempFile} -C ${modelPath} --strip-components=1`, {
-                    cwd: this.modelsPath
-                });
-                // Clean up temp file
-                const { unlink } = await import('fs/promises');
-                await unlink(tempFile);
-                console.log('✅ GitHub Release models extracted and cached locally');
-                // Configure to use local models now
-                env.allowRemoteModels = false;
-                return true;
-            }
-            catch (extractError) {
-                console.log('⚠️ Tar extraction failed, trying alternative method');
-                return false;
-            }
-        }
-        catch (error) {
-            console.log('⚠️ GitHub Release download failed:', error.message);
-            return false;
-        }
-    }
-    /**
-     * Pre-download models for deployment
-     * This is what npm run download-models calls
-     */
-    static async predownload() {
-        const manager = ModelManager.getInstance();
-        const success = await manager.ensureModels();
-        if (!success) {
-            throw new Error('Failed to download models');
-        }
-        console.log('✅ Models downloaded successfully');
-    }
-}
-// Auto-initialize on import in production
-if (process.env.NODE_ENV === 'production' && process.env.SKIP_MODEL_CHECK !== 'true') {
-    ModelManager.getInstance().ensureModels().catch(error => {
-        console.error('⚠️ Model initialization failed:', error);
-        // Don't throw - allow app to start and try downloading on first use
-    });
-}
-//# sourceMappingURL=model-manager.js.map

package/dist/embeddings/universal-memory-manager.d.ts

@@ -1,38 +0,0 @@
-/**
- * Universal Memory Manager for Embeddings
- *
- * Works in ALL environments: Node.js, browsers, serverless, workers
- * Solves transformers.js memory leak with environment-specific strategies
- */
-import { Vector, EmbeddingFunction } from '../coreTypes.js';
-interface MemoryStats {
-    embeddings: number;
-    memoryUsage: string;
-    restarts: number;
-    strategy: string;
-}
-export declare class UniversalMemoryManager {
-    private embeddingFunction;
-    private embedCount;
-    private restartCount;
-    private lastRestart;
-    private strategy;
-    private maxEmbeddings;
-    constructor();
-    getEmbeddingFunction(): Promise<EmbeddingFunction>;
-    embed(data: string | string[]): Promise<Vector>;
-    private checkMemoryLimits;
-    private ensureEmbeddingFunction;
-    private initNodeDirect;
-    private initServerless;
-    private initBrowser;
-    private initFallback;
-    private initDirect;
-    private cleanup;
-    getMemoryStats(): MemoryStats;
-    dispose(): Promise<void>;
-}
-export declare const universalMemoryManager: UniversalMemoryManager;
-export declare function getUniversalEmbeddingFunction(): Promise<EmbeddingFunction>;
-export declare function getEmbeddingMemoryStats(): MemoryStats;
-export {};

package/dist/embeddings/universal-memory-manager.js

@@ -1,166 +0,0 @@
-/**
- * Universal Memory Manager for Embeddings
- *
- * Works in ALL environments: Node.js, browsers, serverless, workers
- * Solves transformers.js memory leak with environment-specific strategies
- */
-import { getModelPrecision } from '../config/modelPrecisionManager.js';
-// Environment detection
-const isNode = typeof process !== 'undefined' && process.versions?.node;
-const isBrowser = typeof window !== 'undefined' && typeof document !== 'undefined';
-const isServerless = typeof process !== 'undefined' && (process.env.VERCEL ||
-    process.env.NETLIFY ||
-    process.env.AWS_LAMBDA_FUNCTION_NAME ||
-    process.env.FUNCTIONS_WORKER_RUNTIME);
-export class UniversalMemoryManager {
-    constructor() {
-        // CRITICAL FIX: Never use worker threads with ONNX Runtime
-        // Worker threads cause HandleScope V8 API errors due to isolate issues
-        // Always use direct embedding on main thread for ONNX compatibility
-        this.embeddingFunction = null;
-        this.embedCount = 0;
-        this.restartCount = 0;
-        this.lastRestart = 0;
-        if (isServerless) {
-            this.strategy = 'serverless-restart';
-            this.maxEmbeddings = 50; // Restart frequently in serverless
-        }
-        else if (isNode && !isBrowser) {
-            // CHANGED: Use direct strategy instead of node-worker to avoid V8 isolate issues
-            this.strategy = 'node-direct';
-            this.maxEmbeddings = 200; // Main thread can handle more with single model instance
-        }
-        else if (isBrowser) {
-            this.strategy = 'browser-dispose';
-            this.maxEmbeddings = 25; // Browser memory is limited
-        }
-        else {
-            this.strategy = 'fallback-dispose';
-            this.maxEmbeddings = 75;
-        }
-        console.log(`🧠 Universal Memory Manager: Using ${this.strategy} strategy`);
-        console.log('✅ UNIVERSAL: Memory-safe embedding system initialized');
-    }
-    async getEmbeddingFunction() {
-        return async (data) => {
-            return this.embed(data);
-        };
-    }
-    async embed(data) {
-        // Check if we need to restart/cleanup
-        await this.checkMemoryLimits();
-        // Ensure embedding function is available
-        await this.ensureEmbeddingFunction();
-        // Perform embedding
-        const result = await this.embeddingFunction.embed(data);
-        this.embedCount++;
-        return result;
-    }
-    async checkMemoryLimits() {
-        if (this.embedCount >= this.maxEmbeddings) {
-            console.log(`🔄 Memory cleanup: ${this.embedCount} embeddings processed`);
-            await this.cleanup();
-        }
-    }
-    async ensureEmbeddingFunction() {
-        if (this.embeddingFunction) {
-            return;
-        }
-        switch (this.strategy) {
-            case 'node-direct':
-                await this.initNodeDirect();
-                break;
-            case 'serverless-restart':
-                await this.initServerless();
-                break;
-            case 'browser-dispose':
-                await this.initBrowser();
-                break;
-            default:
-                await this.initFallback();
-        }
-    }
-    async initNodeDirect() {
-        if (isNode) {
-            // CRITICAL: Use direct embedding to avoid worker thread V8 isolate issues
-            // This prevents HandleScope errors and ensures single model instance
-            console.log('✅ Using Node.js direct embedding (main thread - ONNX compatible)');
-            await this.initDirect();
-        }
-    }
-    async initServerless() {
-        // In serverless, use direct embedding but restart more aggressively
-        await this.initDirect();
-        console.log('✅ Using serverless strategy with aggressive cleanup');
-    }
-    async initBrowser() {
-        // In browser, use direct embedding with disposal
-        await this.initDirect();
-        console.log('✅ Using browser strategy with disposal');
-    }
-    async initFallback() {
-        await this.initDirect();
-        console.log('✅ Using fallback direct embedding strategy');
-    }
-    async initDirect() {
-        try {
-            // Dynamic import to handle different environments
-            const { TransformerEmbedding } = await import('../utils/embedding.js');
-            this.embeddingFunction = new TransformerEmbedding({
-                verbose: false,
-                precision: getModelPrecision(), // Use centrally managed precision
-                localFilesOnly: process.env.BRAINY_ALLOW_REMOTE_MODELS !== 'true'
-            });
-            await this.embeddingFunction.init();
-            console.log('✅ Direct embedding function initialized');
-        }
-        catch (error) {
-            throw new Error(`Failed to initialize embedding function: ${error instanceof Error ? error.message : String(error)}`);
-        }
-    }
-    async cleanup() {
-        const startTime = Date.now();
-        // SingletonModelManager persists - we just reset our counters
-        // The singleton model stays alive for consistency across all operations
-        // Reset counters
-        this.embedCount = 0;
-        this.restartCount++;
-        this.lastRestart = Date.now();
-        const cleanupTime = Date.now() - startTime;
-        console.log(`🧹 Memory counters reset in ${cleanupTime}ms (strategy: ${this.strategy})`);
-        console.log('ℹ️ Singleton model persists for consistency across all operations');
-    }
-    getMemoryStats() {
-        let memoryUsage = 'unknown';
-        // Get memory stats based on environment
-        if (isNode && typeof process !== 'undefined') {
-            const mem = process.memoryUsage();
-            memoryUsage = `${(mem.heapUsed / 1024 / 1024).toFixed(2)} MB`;
-        }
-        else if (isBrowser && performance.memory) {
-            const mem = performance.memory;
-            memoryUsage = `${(mem.usedJSHeapSize / 1024 / 1024).toFixed(2)} MB`;
-        }
-        return {
-            embeddings: this.embedCount,
-            memoryUsage,
-            restarts: this.restartCount,
-            strategy: this.strategy
-        };
-    }
-    async dispose() {
-        // SingletonModelManager persists - nothing to dispose
-        console.log('ℹ️ Universal Memory Manager: Singleton model persists');
-    }
-}
-// Export singleton instance
-export const universalMemoryManager = new UniversalMemoryManager();
-// Export convenience function
-export async function getUniversalEmbeddingFunction() {
-    return universalMemoryManager.getEmbeddingFunction();
-}
-// Export memory stats function
-export function getEmbeddingMemoryStats() {
-    return universalMemoryManager.getMemoryStats();
-}
-//# sourceMappingURL=universal-memory-manager.js.map

package/dist/embeddings/worker-embedding.d.ts

@@ -1,7 +0,0 @@
-/**
- * Worker process for embeddings - Workaround for transformers.js memory leak
- *
- * This worker can be killed and restarted to release memory completely.
- * Based on 2024 research: dispose() doesn't fully free memory in transformers.js
- */
-export {};