@soulcraft/brainy 2.11.0 → 2.14.0

package/CHANGELOG.md

@@ -2,6 +2,21 @@
 
 All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines.
 
+## [2.14.0](https://github.com/soulcraftlabs/brainy/compare/v2.13.0...v2.14.0) (2025-09-02)
+
+
+### Features
+
+* implement clean embedding architecture with Q8/FP32 precision control ([b55c454](https://github.com/soulcraftlabs/brainy/commit/b55c454))
+
+## [2.13.0](https://github.com/soulcraftlabs/brainy/compare/v2.12.0...v2.13.0) (2025-09-02)
+
+
+### Features
+
+* implement comprehensive neural clustering system ([7345e53](https://github.com/soulcraftlabs/brainy/commit/7345e53))
+* implement comprehensive type safety system with BrainyTypes API ([0f4ab52](https://github.com/soulcraftlabs/brainy/commit/0f4ab52))
+
 ## [2.10.0](https://github.com/soulcraftlabs/brainy/compare/v2.9.0...v2.10.0) (2025-08-29)
 
 ## [2.8.0](https://github.com/soulcraftlabs/brainy/compare/v2.7.4...v2.8.0) (2025-08-29)

package/dist/brainyData.d.ts

@@ -12,6 +12,7 @@ import { WebSocketConnection } from './types/augmentations.js';
 import { BrainyDataInterface } from './types/brainyDataInterface.js';
 import { DistributedConfig } from './types/distributedTypes.js';
 import { SearchCacheConfig } from './utils/searchCache.js';
+import { ImprovedNeuralAPI } from './neural/improvedNeuralAPI.js';
 import { TripleQuery, TripleResult } from './triple/TripleIntelligence.js';
 export interface BrainyDataConfig {
     /**
@@ -1687,23 +1688,19 @@ export declare class BrainyData<T = any> implements BrainyDataInterface<T> {
      * - brain.neural.clusterStream() - Progressive streaming
      * - brain.neural.getLOD() - Level-of-detail for scale
      */
-    get neural(): any;
+    get neural(): ImprovedNeuralAPI;
     /**
      * Simple similarity check (shorthand for neural.similar)
      */
-    similar(a: any, b: any): Promise<number>;
+    similar(a: any, b: any, options?: any): Promise<number>;
     /**
      * Get semantic clusters (shorthand for neural.clusters)
      */
-    clusters(options?: any): Promise<any[]>;
+    clusters(items?: any, options?: any): Promise<any[]>;
     /**
      * Get related items (shorthand for neural.neighbors)
      */
-    related(id: string, limit?: number): Promise<any[]>;
-    /**
-     * Get visualization data (shorthand for neural.visualize)
-     */
-    visualize(options?: any): Promise<any>;
+    related(id: string, options?: any): Promise<any[]>;
     /**
      * 🚀 TRIPLE INTELLIGENCE SEARCH - Natural Language & Complex Queries
      * The revolutionary search that combines vector, graph, and metadata intelligence!

package/dist/brainyData.js

@@ -29,7 +29,7 @@ import { EntityRegistryAugmentation, AutoRegisterEntitiesAugmentation } from './
 import { createDefaultAugmentations } from './augmentations/defaultAugmentations.js';
 // import { RealtimeStreamingAugmentation } from './augmentations/realtimeStreamingAugmentation.js'
 import { IntelligentVerbScoringAugmentation } from './augmentations/intelligentVerbScoringAugmentation.js';
-import { NeuralAPI } from './neural/neuralAPI.js';
+import { ImprovedNeuralAPI } from './neural/improvedNeuralAPI.js';
 import { TripleIntelligenceEngine } from './triple/TripleIntelligence.js';
 export class BrainyData {
     // REMOVED: HealthMonitor is now handled by MonitoringAugmentation
@@ -935,23 +935,10 @@ export class BrainyData {
                 // Continue with existing config
             }
         }
-        // CRITICAL: Initialize universal memory manager ONLY for default embedding function
-        // This preserves custom embedding functions (like test mocks)
-        if (typeof this.embeddingFunction === 'function' && this.embeddingFunction === defaultEmbeddingFunction) {
-            try {
-                const { universalMemoryManager } = await import('./embeddings/universal-memory-manager.js');
-                this.embeddingFunction = await universalMemoryManager.getEmbeddingFunction();
-                console.log('✅ UNIVERSAL: Memory-safe embedding system initialized');
-            }
-            catch (error) {
-                console.error('🚨 CRITICAL: Universal memory manager initialization failed!');
-                console.error('Falling back to standard embedding with potential memory issues.');
-                console.warn('Consider reducing usage or restarting process periodically.');
-                // Continue with default function - better than crashing
-            }
-        }
-        else if (this.embeddingFunction !== defaultEmbeddingFunction) {
-            console.log('✅ CUSTOM: Using custom embedding function (test or production override)');
+        // The embedding function is already set (either custom or default)
+        // EmbeddingManager handles all initialization internally
+        if (this.embeddingFunction !== defaultEmbeddingFunction) {
+            console.log('✅ Using custom embedding function');
         }
         try {
             // Pre-load the embedding model early to ensure it's always available
@@ -3011,9 +2998,16 @@ export class BrainyData {
      */
     async addVerbs(verbs) {
         const ids = [];
-        for (const verb of verbs) {
-            const id = await this.addVerb(verb.source, verb.target, verb.type, verb.metadata);
-            ids.push(id);
+        const chunkSize = 10; // Conservative chunk size for parallel processing
+        // Process verbs in parallel chunks to improve performance
+        for (let i = 0; i < verbs.length; i += chunkSize) {
+            const chunk = verbs.slice(i, i + chunkSize);
+            // Process chunk in parallel
+            const chunkPromises = chunk.map(verb => this.addVerb(verb.source, verb.target, verb.type, verb.metadata));
+            // Wait for all in chunk to complete
+            const chunkIds = await Promise.all(chunkPromises);
+            // Maintain order by adding chunk results
+            ids.push(...chunkIds);
         }
         return ids;
     }
@@ -3024,8 +3018,16 @@ export class BrainyData {
      */
     async deleteVerbs(ids) {
         const results = [];
-        for (const id of ids) {
-            results.push(await this.deleteVerb(id));
+        const chunkSize = 10; // Conservative chunk size for parallel processing
+        // Process deletions in parallel chunks to improve performance
+        for (let i = 0; i < ids.length; i += chunkSize) {
+            const chunk = ids.slice(i, i + chunkSize);
+            // Process chunk in parallel
+            const chunkPromises = chunk.map(id => this.deleteVerb(id));
+            // Wait for all in chunk to complete
+            const chunkResults = await Promise.all(chunkPromises);
+            // Maintain order by adding chunk results
+            results.push(...chunkResults);
         }
         return results;
     }
@@ -5749,8 +5751,16 @@ export class BrainyData {
      */
     async deleteNouns(ids) {
         const results = [];
-        for (const id of ids) {
-            results.push(await this.deleteNoun(id));
+        const chunkSize = 10; // Conservative chunk size for parallel processing
+        // Process deletions in parallel chunks to improve performance
+        for (let i = 0; i < ids.length; i += chunkSize) {
+            const chunk = ids.slice(i, i + chunkSize);
+            // Process chunk in parallel
+            const chunkPromises = chunk.map(id => this.deleteNoun(id));
+            // Wait for all in chunk to complete
+            const chunkResults = await Promise.all(chunkPromises);
+            // Maintain order by adding chunk results
+            results.push(...chunkResults);
         }
         return results;
     }
@@ -5931,34 +5941,41 @@ export class BrainyData {
     get neural() {
         if (!this._neural) {
             // Create the unified Neural API instance
-            this._neural = new NeuralAPI(this);
+            this._neural = new ImprovedNeuralAPI(this);
         }
         return this._neural;
     }
     /**
      * Simple similarity check (shorthand for neural.similar)
      */
-    async similar(a, b) {
-        return this.neural.similar(a, b);
+    async similar(a, b, options) {
+        const result = await this.neural.similar(a, b, options);
+        // Always return simple number for main class shortcut
+        return typeof result === 'object' ? result.score : result;
     }
     /**
      * Get semantic clusters (shorthand for neural.clusters)
      */
-    async clusters(options) {
-        return this.neural.clusters(options);
+    async clusters(items, options) {
+        // Support both (items, options) and (options) patterns
+        if (typeof items === 'object' && !Array.isArray(items) && options === undefined) {
+            // First argument is options object
+            return this.neural.clusters(items);
+        }
+        // Standard (items, options) pattern
+        if (options) {
+            return this.neural.clusters({ ...options, items });
+        }
+        return this.neural.clusters(items);
     }
     /**
      * Get related items (shorthand for neural.neighbors)
      */
-    async related(id, limit) {
-        const result = await this.neural.neighbors(id, { limit });
-        return result.neighbors;
-    }
-    /**
-     * Get visualization data (shorthand for neural.visualize)
-     */
-    async visualize(options) {
-        return this.neural.visualize(options);
+    async related(id, options) {
+        const limit = typeof options === 'number' ? options : options?.limit;
+        const fullOptions = typeof options === 'number' ? { limit } : options;
+        const result = await this.neural.neighbors(id, fullOptions);
+        return result.neighbors || [];
     }
     /**
      * 🚀 TRIPLE INTELLIGENCE SEARCH - Natural Language & Complex Queries

package/dist/config/index.d.ts

@@ -4,6 +4,7 @@
  */
 export { autoSelectModelPrecision, ModelPrecision as ModelPrecisionType, // Avoid conflict
 ModelPreset, shouldAutoDownloadModels, getModelPath, logModelConfig } from './modelAutoConfig.js';
+export { ModelPrecisionManager, getModelPrecision, setModelPrecision, lockModelPrecision, validateModelPrecision } from './modelPrecisionManager.js';
 export { autoDetectStorage, StorageType, StoragePreset, StorageConfigResult, logStorageConfig, type StorageTypeString, type StoragePresetString } from './storageAutoConfig.js';
 export { SharedConfig, SharedConfigManager } from './sharedConfigManager.js';
 export { BrainyZeroConfig, processZeroConfig, createEmbeddingFunctionWithPrecision } from './zeroConfig.js';

package/dist/config/index.js

@@ -4,6 +4,8 @@
  */
 // Model configuration
 export { autoSelectModelPrecision, shouldAutoDownloadModels, getModelPath, logModelConfig } from './modelAutoConfig.js';
+// Model precision manager
+export { ModelPrecisionManager, getModelPrecision, setModelPrecision, lockModelPrecision, validateModelPrecision } from './modelPrecisionManager.js';
 // Storage configuration
 export { autoDetectStorage, StorageType, StoragePreset, logStorageConfig } from './storageAutoConfig.js';
 // Shared configuration for multi-instance

package/dist/config/modelAutoConfig.d.ts

@@ -12,6 +12,7 @@ interface ModelConfigResult {
 }
 /**
  * Auto-select model precision based on environment and resources
+ * DEFAULT: Q8 for optimal size/performance balance
  * @param override - Manual override: 'fp32', 'q8', 'fast' (fp32), 'small' (q8), or 'auto'
  */
 export declare function autoSelectModelPrecision(override?: ModelPrecision | ModelPreset): ModelConfigResult;

package/dist/config/modelAutoConfig.js

@@ -4,13 +4,16 @@
  * while allowing manual override
  */
 import { isBrowser, isNode } from '../utils/environment.js';
+import { setModelPrecision } from './modelPrecisionManager.js';
 /**
  * Auto-select model precision based on environment and resources
+ * DEFAULT: Q8 for optimal size/performance balance
  * @param override - Manual override: 'fp32', 'q8', 'fast' (fp32), 'small' (q8), or 'auto'
  */
 export function autoSelectModelPrecision(override) {
     // Handle direct precision override
     if (override === 'fp32' || override === 'q8') {
+        setModelPrecision(override); // Update central config
         return {
             precision: override,
             reason: `Manually specified: ${override}`,
@@ -19,6 +22,7 @@ export function autoSelectModelPrecision(override) {
     }
     // Handle preset overrides
     if (override === 'fast') {
+        setModelPrecision('fp32'); // Update central config
         return {
             precision: 'fp32',
             reason: 'Preset: fast (fp32 for best quality)',
@@ -26,6 +30,7 @@ export function autoSelectModelPrecision(override) {
         };
     }
     if (override === 'small') {
+        setModelPrecision('q8'); // Update central config
         return {
             precision: 'q8',
             reason: 'Preset: small (q8 for reduced size)',
@@ -37,53 +42,53 @@ export function autoSelectModelPrecision(override) {
 }
 /**
  * Automatically detect the best model precision for the environment
+ * NEW DEFAULT: Q8 for optimal size/performance (75% smaller, 99% accuracy)
  */
 function autoDetectBestPrecision() {
+    // Check if user explicitly wants FP32 via environment variable
+    if (process.env.BRAINY_FORCE_FP32 === 'true') {
+        setModelPrecision('fp32');
+        return {
+            precision: 'fp32',
+            reason: 'FP32 forced via BRAINY_FORCE_FP32 environment variable',
+            autoSelected: false
+        };
+    }
     // Browser environment - use Q8 for smaller download/memory
     if (isBrowser()) {
+        setModelPrecision('q8');
         return {
             precision: 'q8',
-            reason: 'Browser environment detected - using Q8 for smaller size',
+            reason: 'Browser environment - using Q8 (23MB vs 90MB)',
             autoSelected: true
         };
     }
     // Serverless environments - use Q8 for faster cold starts
     if (isServerlessEnvironment()) {
+        setModelPrecision('q8');
         return {
             precision: 'q8',
-            reason: 'Serverless environment detected - using Q8 for faster cold starts',
+            reason: 'Serverless environment - using Q8 for 75% faster cold starts',
             autoSelected: true
         };
     }
     // Check available memory
     const memoryMB = getAvailableMemoryMB();
-    if (memoryMB < 512) {
-        return {
-            precision: 'q8',
-            reason: `Low memory detected (${memoryMB}MB) - using Q8`,
-            autoSelected: true
-        };
-    }
-    // Development environment - use FP32 for best quality
-    if (process.env.NODE_ENV === 'development') {
-        return {
-            precision: 'fp32',
-            reason: 'Development environment - using FP32 for best quality',
-            autoSelected: true
-        };
-    }
-    // Production with adequate memory - use FP32
-    if (memoryMB >= 2048) {
+    // Only use FP32 if explicitly high memory AND user opts in
+    if (memoryMB >= 4096 && process.env.BRAINY_PREFER_QUALITY === 'true') {
+        setModelPrecision('fp32');
         return {
             precision: 'fp32',
-            reason: `Adequate memory (${memoryMB}MB) - using FP32 for best quality`,
+            reason: `High memory (${memoryMB}MB) + quality preference - using FP32`,
             autoSelected: true
         };
     }
-    // Default to Q8 for moderate memory environments
+    // DEFAULT TO Q8 - Optimal for 99% of use cases
+    // Q8 provides 99% accuracy at 25% of the size
+    setModelPrecision('q8');
     return {
         precision: 'q8',
-        reason: `Moderate memory (${memoryMB}MB) - using Q8 for balance`,
+        reason: 'Default: Q8 model (23MB, 99% accuracy, 4x faster loads)',
         autoSelected: true
     };
 }

package/dist/config/modelPrecisionManager.d.ts

@@ -0,0 +1,42 @@
+/**
+ * Central Model Precision Manager
+ *
+ * Single source of truth for model precision configuration.
+ * Ensures consistent usage of Q8 or FP32 models throughout the system.
+ */
+import { ModelPrecision } from './modelAutoConfig.js';
+export declare class ModelPrecisionManager {
+    private static instance;
+    private precision;
+    private isLocked;
+    private constructor();
+    static getInstance(): ModelPrecisionManager;
+    /**
+     * Get the current model precision
+     */
+    getPrecision(): ModelPrecision;
+    /**
+     * Set the model precision (can only be done before first model load)
+     */
+    setPrecision(precision: ModelPrecision): void;
+    /**
+     * Lock the precision (called after first model load)
+     */
+    lock(): void;
+    /**
+     * Check if precision is locked
+     */
+    isConfigLocked(): boolean;
+    /**
+     * Get precision info for logging
+     */
+    getInfo(): string;
+    /**
+     * Validate that a given precision matches the configured one
+     */
+    validatePrecision(precision: ModelPrecision): boolean;
+}
+export declare const getModelPrecision: () => ModelPrecision;
+export declare const setModelPrecision: (precision: ModelPrecision) => void;
+export declare const lockModelPrecision: () => void;
+export declare const validateModelPrecision: (precision: ModelPrecision) => boolean;

package/dist/config/modelPrecisionManager.js

@@ -0,0 +1,98 @@
+/**
+ * Central Model Precision Manager
+ *
+ * Single source of truth for model precision configuration.
+ * Ensures consistent usage of Q8 or FP32 models throughout the system.
+ */
+export class ModelPrecisionManager {
+    constructor() {
+        this.precision = 'q8'; // DEFAULT TO Q8
+        this.isLocked = false;
+        // Check environment variable override
+        const envPrecision = process.env.BRAINY_MODEL_PRECISION;
+        if (envPrecision === 'fp32' || envPrecision === 'q8') {
+            this.precision = envPrecision;
+            console.log(`Model precision set from environment: ${envPrecision.toUpperCase()}`);
+        }
+        else {
+            console.log('Using default model precision: Q8 (75% smaller, 99% accuracy)');
+        }
+    }
+    static getInstance() {
+        if (!ModelPrecisionManager.instance) {
+            ModelPrecisionManager.instance = new ModelPrecisionManager();
+        }
+        return ModelPrecisionManager.instance;
+    }
+    /**
+     * Get the current model precision
+     */
+    getPrecision() {
+        return this.precision;
+    }
+    /**
+     * Set the model precision (can only be done before first model load)
+     */
+    setPrecision(precision) {
+        if (this.isLocked) {
+            console.warn(`⚠️ Cannot change precision after model initialization. Current: ${this.precision.toUpperCase()}`);
+            return;
+        }
+        if (precision !== this.precision) {
+            console.log(`Model precision changed: ${this.precision.toUpperCase()} → ${precision.toUpperCase()}`);
+            this.precision = precision;
+        }
+    }
+    /**
+     * Lock the precision (called after first model load)
+     */
+    lock() {
+        if (!this.isLocked) {
+            this.isLocked = true;
+            console.log(`Model precision locked: ${this.precision.toUpperCase()}`);
+        }
+    }
+    /**
+     * Check if precision is locked
+     */
+    isConfigLocked() {
+        return this.isLocked;
+    }
+    /**
+     * Get precision info for logging
+     */
+    getInfo() {
+        const info = this.precision === 'q8'
+            ? 'Q8 (quantized, 23MB, 99% accuracy)'
+            : 'FP32 (full precision, 90MB, 100% accuracy)';
+        return `${info}${this.isLocked ? ' [LOCKED]' : ''}`;
+    }
+    /**
+     * Validate that a given precision matches the configured one
+     */
+    validatePrecision(precision) {
+        if (precision !== this.precision) {
+            console.error(`❌ Precision mismatch! Expected: ${this.precision.toUpperCase()}, Got: ${precision.toUpperCase()}`);
+            console.error('This will cause incompatible embeddings!');
+            return false;
+        }
+        return true;
+    }
+}
+// Export singleton instance getter
+export const getModelPrecision = () => {
+    return ModelPrecisionManager.getInstance().getPrecision();
+};
+// Export setter (for configuration phase)
+export const setModelPrecision = (precision) => {
+    ModelPrecisionManager.getInstance().setPrecision(precision);
+};
+// Export lock function (for after model initialization)
+export const lockModelPrecision = () => {
+    ModelPrecisionManager.getInstance().lock();
+};
+// Export validation function
+export const validateModelPrecision = (precision) => {
+    return ModelPrecisionManager.getInstance().validatePrecision(precision);
+};
+//# sourceMappingURL=modelPrecisionManager.js.map

package/dist/config/zeroConfig.js

@@ -17,7 +17,7 @@ const PRESETS = {
     },
     development: {
         storage: 'memory',
-        model: 'fp32',
+        model: 'q8', // Q8 is now the default for all presets
         features: 'full',
         verbose: true
     },

package/dist/embeddings/CachedEmbeddings.d.ts

@@ -0,0 +1,40 @@
+/**
+ * Cached Embeddings - Performance Optimization Layer
+ *
+ * Provides pre-computed embeddings for common terms to avoid
+ * unnecessary model calls. Falls back to EmbeddingManager for
+ * unknown terms.
+ *
+ * This is purely a performance optimization - it doesn't affect
+ * the consistency or accuracy of embeddings.
+ */
+import { Vector } from '../coreTypes.js';
+/**
+ * Cached Embeddings with fallback to EmbeddingManager
+ */
+export declare class CachedEmbeddings {
+    private stats;
+    /**
+     * Generate embedding with caching
+     */
+    embed(text: string | string[]): Promise<Vector | Vector[]>;
+    /**
+     * Embed single text with cache lookup
+     */
+    private embedSingle;
+    /**
+     * Get cache statistics
+     */
+    getStats(): {
+        totalEmbeddings: number;
+        cacheHitRate: number;
+        cacheHits: number;
+        simpleComputes: number;
+        modelCalls: number;
+    };
+    /**
+     * Add custom pre-computed embeddings
+     */
+    addPrecomputed(term: string, embedding: Vector): void;
+}
+export declare const cachedEmbeddings: CachedEmbeddings;