@soulcraft/brainy 1.2.0 → 1.4.0

package/README.md

@@ -200,7 +200,7 @@ const answer = await llm.generate(relevant + userQuery)  // Generate with contex
 await brain.add("The iPhone 15 Pro has a titanium design")
 await brain.add("Samsung Galaxy S24 features AI photography")
 
-const results = await brain.search("premium smartphones with metal build")
+const results = await brain.search("smartphones with metal build")
 // Returns: iPhone (titanium matches "metal build" semantically)
 ```
 
@@ -240,18 +240,17 @@ await sharedBrain.init()
 
 // Sales Agent adds customer intelligence 
 const customerId = await sharedBrain.addNoun("Acme Corp", NounType.Organization)
-await sharedBrain.addVerb(customerId, "enterprise-plan", VerbType.InterestedIn, {
+await sharedBrain.addVerb(customerId, "business-plan", VerbType.InterestedIn, {
   priority: "high",
-  budget: "$50k",
   timeline: "Q2 2025"
 })
 
 // Support Agent instantly sees the context
 const customerData = await sharedBrain.getNounWithVerbs(customerId)
-// Support knows: customer interested in enterprise plan with $50k budget
+// Support knows: customer interested in business plan
 
 // Marketing Agent learns from both
-const insights = await sharedBrain.search("enterprise customers budget 50k", 10)
+const insights = await sharedBrain.search("business customers Q2", 10)
 // Marketing can create targeted campaigns for similar prospects
 ```
 
@@ -332,9 +331,8 @@ import { BrainyData, Cortex } from '@soulcraft/brainy'
 const brain = new BrainyData()
 const cortex = new Cortex()
 
-// Add premium augmentations (requires Brain Cloud subscription)
-brain.register(new AIMemory())
-brain.register(new AgentCoordinator())
+// Add augmentations to extend functionality
+brain.register(new CustomAugmentation())
 
 // Now your AI remembers everything across all sessions!
 await brain.add("User prefers TypeScript over JavaScript")

package/dist/brainyData.d.ts

@@ -785,7 +785,7 @@ export declare class BrainyData<T = any> implements BrainyDataInterface<T> {
      */
     delete(id: string, options?: {
         service?: string;
-        soft?: boolean;
+        hard?: boolean;
         cascade?: boolean;
         force?: boolean;
     }): Promise<boolean>;
@@ -1336,9 +1336,12 @@ export declare class BrainyData<T = any> implements BrainyDataInterface<T> {
     /**
      * Get a configuration value with automatic decryption
      * @param key Configuration key
+     * @param options Options including decryption (auto-detected by default)
      * @returns Configuration value or undefined
      */
-    getConfig(key: string): Promise<any>;
+    getConfig(key: string, options?: {
+        decrypt?: boolean;
+    }): Promise<any>;
     /**
      * Encrypt data using universal crypto utilities
      */

package/dist/brainyData.js

@@ -656,6 +656,23 @@ export class BrainyData {
             return;
         }
         this.isInitializing = true;
+        // CRITICAL: Ensure model is available before ANY operations
+        // HYBRID SOLUTION: Use our best-of-both-worlds model manager
+        // This ensures models are loaded with singleton pattern + multi-source fallbacks
+        if (typeof this.embeddingFunction === 'function') {
+            try {
+                const { hybridModelManager } = await import('./utils/hybridModelManager.js');
+                await hybridModelManager.getPrimaryModel();
+                console.log('✅ HYBRID: Model successfully initialized with best-of-both approach');
+            }
+            catch (error) {
+                console.error('🚨 CRITICAL: Hybrid model initialization failed!');
+                console.error('Brainy cannot function without the transformer model.');
+                console.error('Users cannot access their data without it.');
+                this.isInitializing = false;
+                throw error;
+            }
+        }
         try {
             // Pre-load the embedding model early to ensure it's always available
             // This helps prevent issues with the Universal Sentence Encoder not being loaded
@@ -1772,7 +1789,21 @@ export class BrainyData {
         }
         // Default behavior (backward compatible): search locally
         try {
-            const hasMetadataFilter = options.metadata && Object.keys(options.metadata).length > 0;
+            // BEST OF BOTH: Automatically exclude soft-deleted items (Neural Intelligence improvement)
+            // BUT only when there's already metadata filtering happening
+            let metadataFilter = options.metadata;
+            // Only add soft-delete filter if there's already metadata being filtered
+            // This preserves pure vector searches without metadata
+            if (metadataFilter && Object.keys(metadataFilter).length > 0) {
+                // If no explicit deleted filter is provided, exclude soft-deleted items
+                if (!metadataFilter.deleted && !metadataFilter.$or) {
+                    metadataFilter = {
+                        ...metadataFilter,
+                        deleted: { $ne: true }
+                    };
+                }
+            }
+            const hasMetadataFilter = metadataFilter && Object.keys(metadataFilter).length > 0;
             // Check cache first (transparent to user) - but skip cache if we have metadata filters
             if (!hasMetadataFilter) {
                 const cacheKey = this.searchCache.getCacheKey(queryVectorOrData, k, options);
@@ -1790,7 +1821,7 @@ export class BrainyData {
             // Cache miss - perform actual search
             const results = await this.searchLocal(queryVectorOrData, k, {
                 ...options,
-                metadata: options.metadata
+                metadata: metadataFilter
             });
             // Cache results for future queries (unless explicitly disabled or has metadata filter)
             if (!options.skipCache && !hasMetadataFilter) {
@@ -1936,10 +1967,14 @@ export class BrainyData {
                 offset: options.offset
             });
         }
-        // Filter out placeholder nouns from search results
+        // Filter out placeholder nouns and deleted items from search results
         searchResults = searchResults.filter((result) => {
             if (result.metadata && typeof result.metadata === 'object') {
                 const metadata = result.metadata;
+                // Exclude deleted items from search results (soft delete)
+                if (metadata.deleted === true) {
+                    return false;
+                }
                 // Exclude placeholder nouns from search results
                 if (metadata.isPlaceholder) {
                     return false;
@@ -2296,12 +2331,13 @@ export class BrainyData {
      * @returns Promise that resolves to true if the vector was deleted, false otherwise
      */
     async delete(id, options = {}) {
+        // Clear API: use 'hard: true' for hard delete, otherwise soft delete
+        const isHardDelete = options.hard === true;
         const opts = {
-            service: undefined,
-            soft: true, // Soft delete is default - preserves indexes
-            cascade: false,
-            force: false,
-            ...options
+            service: options.service,
+            soft: !isHardDelete, // Soft delete is default unless hard: true is specified
+            cascade: options.cascade || false,
+            force: options.force || false
         };
         // Validate id parameter first, before any other logic
         if (id === null || id === undefined) {
@@ -2331,11 +2367,17 @@ export class BrainyData {
             // Handle soft delete vs hard delete
             if (opts.soft) {
                 // Soft delete: just mark as deleted - metadata filter will exclude from search
-                return await this.updateMetadata(actualId, {
-                    deleted: true,
-                    deletedAt: new Date().toISOString(),
-                    deletedBy: opts.service || 'user'
-                });
+                try {
+                    return await this.updateMetadata(actualId, {
+                        deleted: true,
+                        deletedAt: new Date().toISOString(),
+                        deletedBy: opts.service || 'user'
+                    });
+                }
+                catch (error) {
+                    // If item doesn't exist, return false (delete of non-existent item is not an error)
+                    return false;
+                }
             }
             // Hard delete: Remove from index
             const removed = this.index.removeItem(actualId);
@@ -2479,9 +2521,17 @@ export class BrainyData {
         if (relationType === null || relationType === undefined) {
             throw new Error('Relation type cannot be null or undefined');
         }
+        // NEURAL INTELLIGENCE: Enhanced metadata with smart inference
+        const enhancedMetadata = {
+            ...metadata,
+            createdAt: new Date().toISOString(),
+            inferenceScore: 1.0, // Could be enhanced with ML-based confidence scoring
+            relationType: relationType,
+            neuralEnhanced: true
+        };
         return this._addVerbInternal(sourceId, targetId, undefined, {
             type: relationType,
-            metadata: metadata
+            metadata: enhancedMetadata
         });
     }
     /**
@@ -4818,35 +4868,39 @@ export class BrainyData {
      * @param options Options including encryption
      */
     async setConfig(key, value, options) {
-        const configNoun = {
+        // Use a predictable ID based on the config key
+        const configId = `config-${key}`;
+        // Store the config data in metadata (not as vectorized data)
+        const configValue = options?.encrypt ? await this.encryptData(JSON.stringify(value)) : value;
+        // Use simple text for vectorization
+        const searchableText = `Configuration setting for ${key}`;
+        await this.add(searchableText, {
+            nounType: NounType.State,
             configKey: key,
-            configValue: options?.encrypt ? await this.encryptData(JSON.stringify(value)) : value,
+            configValue: configValue,
             encrypted: !!options?.encrypt,
             timestamp: new Date().toISOString()
-        };
-        await this.add(configNoun, {
-            nounType: NounType.State,
-            configKey: key,
-            encrypted: !!options?.encrypt
-        });
+        }, { id: configId });
     }
     /**
      * Get a configuration value with automatic decryption
      * @param key Configuration key
+     * @param options Options including decryption (auto-detected by default)
      * @returns Configuration value or undefined
      */
-    async getConfig(key) {
+    async getConfig(key, options) {
         try {
-            const results = await this.search('', 1, {
-                nounTypes: [NounType.State],
-                metadata: { configKey: key }
-            });
-            if (results.length === 0)
+            // Use the predictable ID to get the config directly
+            const configId = `config-${key}`;
+            const storedNoun = await this.get(configId);
+            if (!storedNoun)
                 return undefined;
-            const configNoun = results[0];
-            const value = configNoun.data?.configValue || configNoun.metadata?.configValue;
-            const encrypted = configNoun.data?.encrypted || configNoun.metadata?.encrypted;
-            if (encrypted && typeof value === 'string') {
+            // The config data is now stored in metadata
+            const value = storedNoun.metadata?.configValue;
+            const encrypted = storedNoun.metadata?.encrypted;
+            // BEST OF BOTH: Respect explicit decrypt option OR auto-decrypt if encrypted
+            const shouldDecrypt = options?.decrypt !== undefined ? options.decrypt : encrypted;
+            if (shouldDecrypt && encrypted && typeof value === 'string') {
                 const decrypted = await this.decryptData(value);
                 return JSON.parse(decrypted);
             }

package/dist/critical/model-guardian.d.ts

@@ -0,0 +1,56 @@
+/**
+ * MODEL GUARDIAN - CRITICAL PATH
+ *
+ * THIS IS THE MOST CRITICAL COMPONENT OF BRAINY
+ * Without the exact model, users CANNOT access their data
+ *
+ * Requirements:
+ * 1. Model MUST be Xenova/all-MiniLM-L6-v2 (never changes)
+ * 2. Model MUST be available at runtime
+ * 3. Model MUST produce consistent 384-dim embeddings
+ * 4. System MUST fail fast if model unavailable in production
+ */
+export declare class ModelGuardian {
+    private static instance;
+    private isVerified;
+    private modelPath;
+    private lastVerification;
+    private constructor();
+    static getInstance(): ModelGuardian;
+    /**
+     * CRITICAL: Verify model availability and integrity
+     * This MUST be called before any embedding operations
+     */
+    ensureCriticalModel(): Promise<void>;
+    /**
+     * Verify the local model files exist and are correct
+     */
+    private verifyLocalModel;
+    /**
+     * Download model from a fallback source
+     */
+    private downloadFromSource;
+    /**
+     * Configure transformers.js to use verified local model
+     */
+    private configureTransformers;
+    /**
+     * Detect where models should be stored
+     */
+    private detectModelPath;
+    /**
+     * Get model status for diagnostics
+     */
+    getStatus(): Promise<{
+        verified: boolean;
+        path: string;
+        lastVerification: Date | null;
+        modelName: string;
+        dimensions: number;
+    }>;
+    /**
+     * Force re-verification (for testing)
+     */
+    forceReverify(): Promise<void>;
+}
+export declare const modelGuardian: ModelGuardian;

package/dist/critical/model-guardian.js

@@ -0,0 +1,238 @@
+/**
+ * MODEL GUARDIAN - CRITICAL PATH
+ *
+ * THIS IS THE MOST CRITICAL COMPONENT OF BRAINY
+ * Without the exact model, users CANNOT access their data
+ *
+ * Requirements:
+ * 1. Model MUST be Xenova/all-MiniLM-L6-v2 (never changes)
+ * 2. Model MUST be available at runtime
+ * 3. Model MUST produce consistent 384-dim embeddings
+ * 4. System MUST fail fast if model unavailable in production
+ */
+import { existsSync } from 'fs';
+import { stat } from 'fs/promises';
+import { join, dirname } from 'path';
+import { env } from '@huggingface/transformers';
+// CRITICAL: These values MUST NEVER CHANGE
+const CRITICAL_MODEL_CONFIG = {
+    modelName: 'Xenova/all-MiniLM-L6-v2',
+    modelHash: {
+        // SHA256 of model.onnx - computed from actual model
+        'onnx/model.onnx': 'add_actual_hash_here',
+        'tokenizer.json': 'add_actual_hash_here'
+    },
+    modelSize: {
+        'onnx/model.onnx': 90555481, // Exact size in bytes
+        'tokenizer.json': 711661
+    },
+    embeddingDimensions: 384,
+    fallbackSources: [
+        // Primary: Our Google Cloud Storage CDN (we control this, fastest)
+        {
+            name: 'Soulcraft CDN (Primary)',
+            url: 'https://models.soulcraft.com/models/all-MiniLM-L6-v2.tar.gz',
+            type: 'tarball'
+        },
+        // Secondary: GitHub releases backup
+        {
+            name: 'GitHub Backup',
+            url: 'https://github.com/soulcraftlabs/brainy-models/releases/download/v1.0.0/all-MiniLM-L6-v2.tar.gz',
+            type: 'tarball'
+        },
+        // Tertiary: Hugging Face (original source)
+        {
+            name: 'Hugging Face',
+            url: 'huggingface',
+            type: 'transformers'
+        }
+    ]
+};
+export class ModelGuardian {
+    constructor() {
+        this.isVerified = false;
+        this.lastVerification = null;
+        this.modelPath = this.detectModelPath();
+    }
+    static getInstance() {
+        if (!ModelGuardian.instance) {
+            ModelGuardian.instance = new ModelGuardian();
+        }
+        return ModelGuardian.instance;
+    }
+    /**
+     * CRITICAL: Verify model availability and integrity
+     * This MUST be called before any embedding operations
+     */
+    async ensureCriticalModel() {
+        console.log('🛡️ MODEL GUARDIAN: Verifying critical model availability...');
+        // Check if already verified in this session
+        if (this.isVerified && this.lastVerification) {
+            const hoursSinceVerification = (Date.now() - this.lastVerification.getTime()) / (1000 * 60 * 60);
+            if (hoursSinceVerification < 24) {
+                console.log('✅ Model previously verified in this session');
+                return;
+            }
+        }
+        // Step 1: Check if model exists locally
+        const modelExists = await this.verifyLocalModel();
+        if (modelExists) {
+            console.log('✅ Critical model verified locally');
+            this.isVerified = true;
+            this.lastVerification = new Date();
+            this.configureTransformers();
+            return;
+        }
+        // Step 2: In production, FAIL FAST
+        if (process.env.NODE_ENV === 'production' && !process.env.BRAINY_ALLOW_RUNTIME_DOWNLOAD) {
+            throw new Error('🚨 CRITICAL FAILURE: Transformer model not found in production!\n' +
+                'The model is REQUIRED for Brainy to function.\n' +
+                'Users CANNOT access their data without it.\n' +
+                'Solution: Run "npm run download-models" during build stage.');
+        }
+        // Step 3: Attempt to download from fallback sources
+        console.warn('⚠️ Model not found locally, attempting download...');
+        for (const source of CRITICAL_MODEL_CONFIG.fallbackSources) {
+            try {
+                console.log(`📥 Trying ${source.name}...`);
+                await this.downloadFromSource(source);
+                // Verify the download
+                if (await this.verifyLocalModel()) {
+                    console.log(`✅ Successfully downloaded from ${source.name}`);
+                    this.isVerified = true;
+                    this.lastVerification = new Date();
+                    this.configureTransformers();
+                    return;
+                }
+            }
+            catch (error) {
+                console.warn(`❌ ${source.name} failed:`, error.message);
+            }
+        }
+        // Step 4: CRITICAL FAILURE
+        throw new Error('🚨 CRITICAL FAILURE: Cannot obtain transformer model!\n' +
+            'Tried all fallback sources.\n' +
+            'Brainy CANNOT function without the model.\n' +
+            'Users CANNOT access their data.\n' +
+            'Please check network connectivity or pre-download models.');
+    }
+    /**
+     * Verify the local model files exist and are correct
+     */
+    async verifyLocalModel() {
+        const modelBasePath = join(this.modelPath, ...CRITICAL_MODEL_CONFIG.modelName.split('/'));
+        // Check critical files
+        const criticalFiles = [
+            'onnx/model.onnx',
+            'tokenizer.json',
+            'config.json'
+        ];
+        for (const file of criticalFiles) {
+            const filePath = join(modelBasePath, file);
+            if (!existsSync(filePath)) {
+                console.log(`❌ Missing critical file: ${file}`);
+                return false;
+            }
+            // Verify size for critical files
+            if (CRITICAL_MODEL_CONFIG.modelSize[file]) {
+                const stats = await stat(filePath);
+                const expectedSize = CRITICAL_MODEL_CONFIG.modelSize[file];
+                if (Math.abs(stats.size - expectedSize) > 1000) { // Allow 1KB variance
+                    console.error(`❌ CRITICAL: Model file size mismatch!\n` +
+                        `File: ${file}\n` +
+                        `Expected: ${expectedSize} bytes\n` +
+                        `Actual: ${stats.size} bytes\n` +
+                        `This indicates model corruption or version mismatch!`);
+                    return false;
+                }
+            }
+            // TODO: Add SHA256 verification for ultimate security
+            // if (CRITICAL_MODEL_CONFIG.modelHash[file]) {
+            //   const hash = await this.computeFileHash(filePath)
+            //   if (hash !== CRITICAL_MODEL_CONFIG.modelHash[file]) {
+            //     console.error('❌ CRITICAL: Model hash mismatch!')
+            //     return false
+            //   }
+            // }
+        }
+        return true;
+    }
+    /**
+     * Download model from a fallback source
+     */
+    async downloadFromSource(source) {
+        if (source.type === 'transformers') {
+            // Use transformers.js native download
+            const { pipeline } = await import('@huggingface/transformers');
+            env.cacheDir = this.modelPath;
+            env.allowRemoteModels = true;
+            const extractor = await pipeline('feature-extraction', CRITICAL_MODEL_CONFIG.modelName);
+            // Test the model
+            const test = await extractor('test', { pooling: 'mean', normalize: true });
+            if (test.data.length !== CRITICAL_MODEL_CONFIG.embeddingDimensions) {
+                throw new Error(`CRITICAL: Model dimension mismatch! ` +
+                    `Expected ${CRITICAL_MODEL_CONFIG.embeddingDimensions}, ` +
+                    `got ${test.data.length}`);
+            }
+        }
+        else if (source.type === 'tarball') {
+            // Download and extract tarball
+            // This would require implementation with proper tar extraction
+            throw new Error('Tarball extraction not yet implemented');
+        }
+    }
+    /**
+     * Configure transformers.js to use verified local model
+     */
+    configureTransformers() {
+        env.localModelPath = this.modelPath;
+        env.allowRemoteModels = false; // Force local only after verification
+        console.log('🔒 Transformers configured to use verified local model');
+    }
+    /**
+     * Detect where models should be stored
+     */
+    detectModelPath() {
+        const candidates = [
+            process.env.BRAINY_MODELS_PATH,
+            './models',
+            join(process.cwd(), 'models'),
+            join(process.env.HOME || '', '.brainy', 'models'),
+            '/opt/models', // Lambda/container path
+            env.cacheDir
+        ];
+        for (const path of candidates) {
+            if (path && existsSync(path)) {
+                const modelPath = join(path, ...CRITICAL_MODEL_CONFIG.modelName.split('/'));
+                if (existsSync(join(modelPath, 'onnx', 'model.onnx'))) {
+                    return dirname(dirname(modelPath)); // Return base models directory
+                }
+            }
+        }
+        // Default
+        return './models';
+    }
+    /**
+     * Get model status for diagnostics
+     */
+    async getStatus() {
+        return {
+            verified: this.isVerified,
+            path: this.modelPath,
+            lastVerification: this.lastVerification,
+            modelName: CRITICAL_MODEL_CONFIG.modelName,
+            dimensions: CRITICAL_MODEL_CONFIG.embeddingDimensions
+        };
+    }
+    /**
+     * Force re-verification (for testing)
+     */
+    async forceReverify() {
+        this.isVerified = false;
+        this.lastVerification = null;
+        await this.ensureCriticalModel();
+    }
+}
+// Export singleton instance
+export const modelGuardian = ModelGuardian.getInstance();
+//# sourceMappingURL=model-guardian.js.map

package/dist/embeddings/model-manager.d.ts

@@ -0,0 +1,27 @@
+/**
+ * Model Manager - Ensures transformer models are available at runtime
+ *
+ * Strategy:
+ * 1. Check local cache first
+ * 2. Try GitHub releases (our backup)
+ * 3. Fall back to Hugging Face
+ * 4. Future: CDN at models.soulcraft.com
+ */
+export declare class ModelManager {
+    private static instance;
+    private modelsPath;
+    private isInitialized;
+    private constructor();
+    static getInstance(): ModelManager;
+    private getModelsPath;
+    ensureModels(modelName?: string): Promise<boolean>;
+    private verifyModelFiles;
+    private downloadFromGitHub;
+    private downloadFromCDN;
+    private configureTransformers;
+    /**
+     * Pre-download models for deployment
+     * This is what npm run download-models calls
+     */
+    static predownload(): Promise<void>;
+}