@soulcraft/brainy 2.12.0 → 2.14.1

package/dist/embeddings/universal-memory-manager.js

@@ -4,6 +4,7 @@
  * Works in ALL environments: Node.js, browsers, serverless, workers
  * Solves transformers.js memory leak with environment-specific strategies
  */
+import { getModelPrecision } from '../config/modelPrecisionManager.js';
 // Environment detection
 const isNode = typeof process !== 'undefined' && process.versions?.node;
 const isBrowser = typeof window !== 'undefined' && typeof document !== 'undefined';
@@ -107,7 +108,7 @@ export class UniversalMemoryManager {
             const { TransformerEmbedding } = await import('../utils/embedding.js');
             this.embeddingFunction = new TransformerEmbedding({
                 verbose: false,
-                precision: 'fp32',
+                precision: getModelPrecision(), // Use centrally managed precision
                 localFilesOnly: process.env.BRAINY_ALLOW_REMOTE_MODELS !== 'true'
             });
             await this.embeddingFunction.init();
@@ -119,49 +120,15 @@ export class UniversalMemoryManager {
     }
     async cleanup() {
         const startTime = Date.now();
-        try {
-            // Strategy-specific cleanup
-            switch (this.strategy) {
-                case 'node-worker':
-                    if (this.embeddingFunction?.forceRestart) {
-                        await this.embeddingFunction.forceRestart();
-                    }
-                    break;
-                case 'serverless-restart':
-                    // In serverless, create new instance
-                    if (this.embeddingFunction?.dispose) {
-                        this.embeddingFunction.dispose();
-                    }
-                    this.embeddingFunction = null;
-                    break;
-                case 'browser-dispose':
-                    // In browser, try disposal
-                    if (this.embeddingFunction?.dispose) {
-                        this.embeddingFunction.dispose();
-                    }
-                    // Force garbage collection if available
-                    if (typeof window !== 'undefined' && window.gc) {
-                        window.gc();
-                    }
-                    break;
-                default:
-                    // Fallback: dispose and recreate
-                    if (this.embeddingFunction?.dispose) {
-                        this.embeddingFunction.dispose();
-                    }
-                    this.embeddingFunction = null;
-            }
-            this.embedCount = 0;
-            this.restartCount++;
-            this.lastRestart = Date.now();
-            const cleanupTime = Date.now() - startTime;
-            console.log(`🧹 Memory cleanup completed in ${cleanupTime}ms (strategy: ${this.strategy})`);
-        }
-        catch (error) {
-            console.warn('⚠️ Cleanup failed:', error instanceof Error ? error.message : String(error));
-            // Force null assignment as last resort
-            this.embeddingFunction = null;
-        }
+        // SingletonModelManager persists - we just reset our counters
+        // The singleton model stays alive for consistency across all operations
+        // Reset counters
+        this.embedCount = 0;
+        this.restartCount++;
+        this.lastRestart = Date.now();
+        const cleanupTime = Date.now() - startTime;
+        console.log(`🧹 Memory counters reset in ${cleanupTime}ms (strategy: ${this.strategy})`);
+        console.log('ℹ️ Singleton model persists for consistency across all operations');
     }
     getMemoryStats() {
         let memoryUsage = 'unknown';
@@ -182,12 +149,8 @@ export class UniversalMemoryManager {
         };
     }
     async dispose() {
-        if (this.embeddingFunction) {
-            if (this.embeddingFunction.dispose) {
-                await this.embeddingFunction.dispose();
-            }
-            this.embeddingFunction = null;
-        }
+        // SingletonModelManager persists - nothing to dispose
+        console.log('ℹ️ Universal Memory Manager: Singleton model persists');
     }
 }
 // Export singleton instance

package/dist/embeddings/worker-embedding.js

@@ -6,6 +6,7 @@
  */
 import { TransformerEmbedding } from '../utils/embedding.js';
 import { parentPort } from 'worker_threads';
+import { getModelPrecision } from '../config/modelPrecisionManager.js';
 let model = null;
 let requestCount = 0;
 const MAX_REQUESTS = 100; // Restart worker after 100 requests to prevent memory leak
@@ -13,7 +14,7 @@ async function initModel() {
     if (!model) {
         model = new TransformerEmbedding({
             verbose: false,
-            precision: 'fp32',
+            precision: getModelPrecision(), // Use centrally managed precision
             localFilesOnly: process.env.BRAINY_ALLOW_REMOTE_MODELS !== 'true'
         });
         await model.init();
@@ -37,13 +38,8 @@ if (parentPort) {
                     }
                     break;
                 case 'dispose':
-                    if (model) {
-                        // This doesn't fully free memory (known issue), but try anyway
-                        if ('dispose' in model && typeof model.dispose === 'function') {
-                            model.dispose();
-                        }
-                        model = null;
-                    }
+                    // SingletonModelManager persists - just acknowledge
+                    console.log('ℹ️ Worker: Singleton model persists');
                     parentPort.postMessage({ id, success: true });
                     break;
                 case 'restart':

package/dist/storage/adapters/baseStorageAdapter.d.ts

@@ -75,6 +75,64 @@ export declare abstract class BaseStorageAdapter implements StorageAdapter {
         hasMore: boolean;
         nextCursor?: string;
     }>;
+    /**
+     * Get nouns with pagination (internal implementation)
+     * This method should be implemented by storage adapters to support efficient pagination
+     * @param options Pagination options
+     * @returns Promise that resolves to a paginated result of nouns
+     */
+    getNounsWithPagination?(options: {
+        limit?: number;
+        cursor?: string;
+        filter?: {
+            nounType?: string | string[];
+            service?: string | string[];
+            metadata?: Record<string, any>;
+        };
+    }): Promise<{
+        items: any[];
+        totalCount?: number;
+        hasMore: boolean;
+        nextCursor?: string;
+    }>;
+    /**
+     * Get verbs with pagination (internal implementation)
+     * This method should be implemented by storage adapters to support efficient pagination
+     * @param options Pagination options
+     * @returns Promise that resolves to a paginated result of verbs
+     */
+    getVerbsWithPagination?(options: {
+        limit?: number;
+        cursor?: string;
+        filter?: {
+            verbType?: string | string[];
+            sourceId?: string | string[];
+            targetId?: string | string[];
+            service?: string | string[];
+            metadata?: Record<string, any>;
+        };
+    }): Promise<{
+        items: any[];
+        totalCount?: number;
+        hasMore: boolean;
+        nextCursor?: string;
+    }>;
+    /**
+     * Count total number of nouns (optional)
+     * WARNING: Implementations should be efficient for large datasets.
+     * Consider caching counts or using database COUNT operations.
+     * @param filter Optional filter criteria
+     * @returns Promise that resolves to the count
+     */
+    countNouns?(filter?: any): Promise<number>;
+    /**
+     * Count total number of verbs (optional)
+     * WARNING: Implementations should be efficient for large datasets.
+     * Consider caching counts or using database COUNT operations.
+     * @param filter Optional filter criteria
+     * @returns Promise that resolves to the count
+     */
+    countVerbs?(filter?: any): Promise<number>;
     protected statisticsCache: StatisticsData | null;
     protected statisticsBatchUpdateTimerId: NodeJS.Timeout | null;
     protected statisticsModified: boolean;

package/dist/storage/adapters/fileSystemStorage.d.ts

@@ -189,6 +189,26 @@ export declare class FileSystemStorage extends BaseStorage {
      * Get verbs by type
      */
     protected getVerbsByType_internal(type: string): Promise<GraphVerb[]>;
+    /**
+     * Get verbs with pagination
+     * This method reads verb files from the filesystem and returns them with pagination
+     */
+    getVerbsWithPagination(options?: {
+        limit?: number;
+        cursor?: string;
+        filter?: {
+            verbType?: string | string[];
+            sourceId?: string | string[];
+            targetId?: string | string[];
+            service?: string | string[];
+            metadata?: Record<string, any>;
+        };
+    }): Promise<{
+        items: GraphVerb[];
+        totalCount?: number;
+        hasMore: boolean;
+        nextCursor?: string;
+    }>;
     /**
      * Delete a verb from storage
      */

package/dist/storage/adapters/fileSystemStorage.js

@@ -793,6 +793,103 @@ export class FileSystemStorage extends BaseStorage {
         console.warn('getVerbsByType_internal is deprecated and not efficiently supported in new storage pattern');
         return [];
     }
+    /**
+     * Get verbs with pagination
+     * This method reads verb files from the filesystem and returns them with pagination
+     */
+    async getVerbsWithPagination(options = {}) {
+        await this.ensureInitialized();
+        const limit = options.limit || 100;
+        const startIndex = options.cursor ? parseInt(options.cursor, 10) : 0;
+        try {
+            // List all verb files in the verbs directory
+            // Note: For very large directories (millions of files), this could be memory-intensive
+            // Future optimization: Use fs.opendir() for streaming directory reads
+            const files = await fs.promises.readdir(this.verbsDir);
+            const verbFiles = files.filter((f) => f.endsWith('.json'));
+            // Sort files for consistent ordering
+            verbFiles.sort();
+            // Calculate pagination
+            const totalCount = verbFiles.length;
+            const endIndex = Math.min(startIndex + limit, totalCount);
+            const hasMore = endIndex < totalCount;
+            // Safety check for large datasets
+            if (totalCount > 100000) {
+                console.warn(`Large verb dataset detected (${totalCount} verbs). Consider using a database for better performance.`);
+            }
+            // Load the requested page of verbs
+            const verbs = [];
+            for (let i = startIndex; i < endIndex; i++) {
+                const file = verbFiles[i];
+                const id = file.replace('.json', '');
+                try {
+                    // Read the verb data
+                    const filePath = path.join(this.verbsDir, file);
+                    const data = await fs.promises.readFile(filePath, 'utf-8');
+                    const edge = JSON.parse(data);
+                    // Also try to get metadata if it exists
+                    const metadata = await this.getVerbMetadata(id);
+                    // Convert to GraphVerb format
+                    const verb = {
+                        id: edge.id,
+                        source: metadata?.source || '',
+                        target: metadata?.target || '',
+                        type: metadata?.type || 'relationship',
+                        ...(metadata || {})
+                    };
+                    // Apply filters if provided
+                    if (options.filter) {
+                        const filter = options.filter;
+                        // Check verbType filter
+                        if (filter.verbType) {
+                            const types = Array.isArray(filter.verbType) ? filter.verbType : [filter.verbType];
+                            if (!types.includes(verb.type || ''))
+                                continue;
+                        }
+                        // Check sourceId filter
+                        if (filter.sourceId) {
+                            const sources = Array.isArray(filter.sourceId) ? filter.sourceId : [filter.sourceId];
+                            if (!sources.includes(verb.source || ''))
+                                continue;
+                        }
+                        // Check targetId filter
+                        if (filter.targetId) {
+                            const targets = Array.isArray(filter.targetId) ? filter.targetId : [filter.targetId];
+                            if (!targets.includes(verb.target || ''))
+                                continue;
+                        }
+                        // Check service filter
+                        if (filter.service && metadata?.service) {
+                            const services = Array.isArray(filter.service) ? filter.service : [filter.service];
+                            if (!services.includes(metadata.service))
+                                continue;
+                        }
+                    }
+                    verbs.push(verb);
+                }
+                catch (error) {
+                    console.warn(`Failed to read verb ${id}:`, error);
+                }
+            }
+            return {
+                items: verbs,
+                totalCount,
+                hasMore,
+                nextCursor: hasMore ? String(endIndex) : undefined
+            };
+        }
+        catch (error) {
+            if (error.code === 'ENOENT') {
+                // Verbs directory doesn't exist yet
+                return {
+                    items: [],
+                    totalCount: 0,
+                    hasMore: false
+                };
+            }
+            throw error;
+        }
+    }
     /**
      * Delete a verb from storage
      */

package/dist/utils/embedding.d.ts

@@ -51,6 +51,10 @@ export declare class TransformerEmbedding implements EmbeddingModel {
      * Log message only if verbose mode is enabled
      */
     private logger;
+    /**
+     * Generate mock embeddings for unit tests
+     */
+    private getMockEmbedding;
     /**
      * Initialize the embedding model
      */
@@ -78,12 +82,13 @@ export declare const UniversalSentenceEncoder: typeof TransformerEmbedding;
  */
 export declare function createEmbeddingModel(options?: TransformerEmbeddingOptions): EmbeddingModel;
 /**
- * Default embedding function using the hybrid model manager (BEST OF BOTH WORLDS)
- * Prevents multiple model loads while supporting multi-source downloading
+ * Default embedding function using the unified EmbeddingManager
+ * Simple, clean, reliable - no more layers of indirection
  */
 export declare const defaultEmbeddingFunction: EmbeddingFunction;
 /**
  * Create an embedding function with custom options
+ * NOTE: Options are validated but the singleton EmbeddingManager is always used
  */
 export declare function createEmbeddingFunction(options?: TransformerEmbeddingOptions): EmbeddingFunction;
 /**

package/dist/utils/embedding.js

@@ -3,7 +3,6 @@
  * Complete rewrite to eliminate TensorFlow.js and use ONNX-based models
  */
 import { isBrowser } from './environment.js';
-import { ModelManager } from '../embeddings/model-manager.js';
 import { join } from 'path';
 import { existsSync } from 'fs';
 // @ts-ignore - Transformers.js is now the primary embedding library
@@ -208,6 +207,24 @@ export class TransformerEmbedding {
             console[level](`[TransformerEmbedding] ${message}`, ...args);
         }
     }
+    /**
+     * Generate mock embeddings for unit tests
+     */
+    getMockEmbedding(data) {
+        // Use the same mock logic as setup-unit.ts for consistency
+        const input = Array.isArray(data) ? data.join(' ') : data;
+        const str = typeof input === 'string' ? input : JSON.stringify(input);
+        const vector = new Array(384).fill(0);
+        // Create semi-realistic embeddings based on text content
+        for (let i = 0; i < Math.min(str.length, 384); i++) {
+            vector[i] = (str.charCodeAt(i % str.length) % 256) / 256;
+        }
+        // Add position-based variation
+        for (let i = 0; i < 384; i++) {
+            vector[i] += Math.sin(i * 0.1 + str.length) * 0.1;
+        }
+        return vector;
+    }
     /**
      * Initialize the embedding model
      */
@@ -215,11 +232,13 @@ export class TransformerEmbedding {
         if (this.initialized) {
             return;
         }
-        // Always use real implementation - no mocking
+        // In unit test mode, skip real model initialization to prevent ONNX conflicts
+        if (process.env.BRAINY_UNIT_TEST === 'true' || globalThis.__BRAINY_UNIT_TEST__) {
+            this.initialized = true;
+            this.logger('log', '🧪 Using mocked embeddings for unit tests');
+            return;
+        }
         try {
-            // Ensure models are available (downloads if needed)
-            const modelManager = ModelManager.getInstance();
-            await modelManager.ensureModels(this.options.model);
             // Resolve device configuration and cache directory
             const device = await resolveDevice(this.options.device);
             const cacheDir = this.options.cacheDir === './models'
@@ -227,35 +246,26 @@ export class TransformerEmbedding {
                 : this.options.cacheDir;
             this.logger('log', `Loading Transformer model: ${this.options.model} on device: ${device}`);
             const startTime = Date.now();
-            // Check model availability and select appropriate variant
-            const available = modelManager.getAvailableModels(this.options.model);
-            let actualType = modelManager.getBestAvailableModel(this.options.precision, this.options.model);
-            if (!actualType) {
-                throw new Error(`No model variants available for ${this.options.model}. Run 'npm run download-models' to download models.`);
-            }
-            if (actualType !== this.options.precision) {
-                this.logger('log', `Using ${actualType} model (${this.options.precision} not available)`);
-            }
-            // CRITICAL FIX: Control which model file transformers.js loads
-            // When both model.onnx and model_quantized.onnx exist, transformers.js defaults to model.onnx
-            // We need to explicitly control this based on the precision setting
-            // Set environment to control model selection BEFORE creating pipeline
+            // Use the configured precision from EmbeddingManager
+            const { embeddingManager } = await import('../embeddings/EmbeddingManager.js');
+            let actualType = embeddingManager.getPrecision();
+            // CRITICAL: Control which model precision transformers.js uses
+            // Q8 models use quantized int8 weights for 75% size reduction
+            // FP32 models use full precision floating point
             if (actualType === 'q8') {
-                // For Q8, we want to use the quantized model
-                // transformers.js v3 doesn't have a direct flag, so we need to work around this
-                // HACK: Temporarily modify the model file preference
-                // This forces transformers.js to look for model_quantized.onnx first
-                const originalModelFileName = env.onnxModelFileName(env).onnxModelFileName = 'model_quantized';
-                this.logger('log', '🎯 Selecting Q8 quantized model (75% smaller)');
+                this.logger('log', '🎯 Selecting Q8 quantized model (75% smaller, 99% accuracy)');
             }
             else {
-                this.logger('log', '📦 Using FP32 model (full precision)');
+                this.logger('log', '📦 Using FP32 model (full precision, larger size)');
             }
             // Load the feature extraction pipeline with memory optimizations
             const pipelineOptions = {
                 cache_dir: cacheDir,
                 local_files_only: isBrowser() ? false : this.options.localFilesOnly,
-                // Remove the quantized flag - it doesn't work in transformers.js v3
+                // CRITICAL: Specify dtype for model precision
+                dtype: actualType === 'q8' ? 'q8' : 'fp32',
+                // CRITICAL: For Q8, explicitly use quantized model
+                quantized: actualType === 'q8',
                 // CRITICAL: ONNX memory optimizations
                 session_options: {
                     enableCpuMemArena: false, // Disable pre-allocated memory arena
@@ -336,6 +346,10 @@ export class TransformerEmbedding {
      * Generate embeddings for text data
      */
     async embed(data) {
+        // In unit test mode, return mock embeddings
+        if (process.env.BRAINY_UNIT_TEST === 'true' || globalThis.__BRAINY_UNIT_TEST__) {
+            return this.getMockEmbedding(data);
+        }
         if (!this.initialized) {
             await this.init();
         }
@@ -433,21 +447,25 @@ export function createEmbeddingModel(options) {
     return new TransformerEmbedding(options);
 }
 /**
- * Default embedding function using the hybrid model manager (BEST OF BOTH WORLDS)
- * Prevents multiple model loads while supporting multi-source downloading
+ * Default embedding function using the unified EmbeddingManager
+ * Simple, clean, reliable - no more layers of indirection
  */
 export const defaultEmbeddingFunction = async (data) => {
-    const { getHybridEmbeddingFunction } = await import('./hybridModelManager.js');
-    const embeddingFn = await getHybridEmbeddingFunction();
-    return await embeddingFn(data);
+    const { embed } = await import('../embeddings/EmbeddingManager.js');
+    return await embed(data);
 };
 /**
  * Create an embedding function with custom options
+ * NOTE: Options are validated but the singleton EmbeddingManager is always used
  */
 export function createEmbeddingFunction(options = {}) {
-    const embedder = new TransformerEmbedding(options);
     return async (data) => {
-        return await embedder.embed(data);
+        const { embeddingManager } = await import('../embeddings/EmbeddingManager.js');
+        // Validate precision if specified
+        if (options.precision) {
+            embeddingManager.validatePrecision(options.precision);
+        }
+        return await embeddingManager.embed(data);
     };
 }
 /**

package/dist/utils/hybridModelManager.d.ts

@@ -1,55 +1,44 @@
 /**
  * Hybrid Model Manager - BEST OF BOTH WORLDS
  *
- * Combines:
+ * NOW A WRAPPER AROUND SingletonModelManager
+ * Maintained for backward compatibility
+ *
+ * Previously combined:
  * 1. Multi-source downloading strategy (GitHub → CDN → Hugging Face)
  * 2. Singleton pattern preventing multiple ONNX model loads
  * 3. Environment-specific optimizations
  * 4. Graceful fallbacks and error handling
+ *
+ * Now delegates all operations to SingletonModelManager for true unification
  */
-import { TransformerEmbedding } from './embedding.js';
 import { EmbeddingFunction } from '../coreTypes.js';
 /**
- * Global singleton model manager - PREVENTS MULTIPLE MODEL LOADS
+ * HybridModelManager - Now a wrapper around SingletonModelManager
+ * Maintained for backward compatibility
  */
 declare class HybridModelManager {
     private static instance;
-    private primaryModel;
-    private modelPromise;
-    private isInitialized;
-    private modelsPath;
     private constructor();
     static getInstance(): HybridModelManager;
     /**
-     * Get the primary embedding model - LOADS ONCE, REUSES FOREVER
-     */
-    getPrimaryModel(): Promise<TransformerEmbedding>;
-    /**
-     * Smart model path detection
-     */
-    private getModelsPath;
-    /**
-     * Initialize with BEST OF BOTH: Multi-source + Singleton
-     */
-    private initializePrimaryModel;
-    /**
-     * Create model with multi-source fallback strategy
+     * Get the primary embedding model - delegates to SingletonModelManager
      */
-    private createModelWithFallbacks;
+    getPrimaryModel(): Promise<any>;
     /**
-     * Get embedding function that reuses the singleton model
+     * Get embedding function - delegates to SingletonModelManager
      */
     getEmbeddingFunction(): Promise<EmbeddingFunction>;
     /**
-     * Check if model is ready (loaded and initialized)
+     * Check if model is ready - delegates to SingletonModelManager
      */
     isModelReady(): boolean;
     /**
-     * Force model reload (for testing or recovery)
+     * Force model reload - not supported with SingletonModelManager
      */
     reloadModel(): Promise<void>;
     /**
-     * Get model status for debugging
+     * Get model status - delegates to SingletonModelManager
      */
     getModelStatus(): {
         loaded: boolean;
@@ -59,15 +48,17 @@ declare class HybridModelManager {
 }
 export declare const hybridModelManager: HybridModelManager;
 /**
- * Get the hybrid singleton embedding function - USE THIS EVERYWHERE!
+ * Get the hybrid singleton embedding function - Now delegates to SingletonModelManager
+ * Maintained for backward compatibility
  */
 export declare function getHybridEmbeddingFunction(): Promise<EmbeddingFunction>;
 /**
- * Optimized hybrid embedding function that uses multi-source + singleton
+ * Hybrid embedding function - Now delegates to SingletonModelManager
+ * Maintained for backward compatibility
  */
 export declare const hybridEmbeddingFunction: EmbeddingFunction;
 /**
- * Preload model for tests or production - CALL THIS ONCE AT START
+ * Preload model for tests or production - Now delegates to SingletonModelManager
  */
 export declare function preloadHybridModel(): Promise<void>;
 export {};