@soulcraft/brainy 2.1.0 → 3.0.0

package/dist/brainyData.js

@@ -8,6 +8,8 @@ import { HNSWIndexOptimized } from './hnsw/hnswIndexOptimized.js';
 import { cosineDistance, defaultEmbeddingFunction, cleanupWorkerPools, batchEmbed } from './utils/index.js';
 import { getAugmentationVersion } from './utils/version.js';
 import { matchesMetadataFilter } from './utils/metadataFilter.js';
+import { createNamespacedMetadata, updateNamespacedMetadata, markDeleted, markRestored, isDeleted, getUserMetadata } from './utils/metadataNamespace.js';
+import { PeriodicCleanup } from './utils/periodicCleanup.js';
 import { NounType, VerbType } from './types/graphTypes.js';
 import { createServerSearchAugmentations } from './augmentations/serverSearchAugmentations.js';
 import { augmentationPipeline } from './augmentationPipeline.js';
@@ -94,6 +96,8 @@ export class BrainyData {
          * Handles WAL, connection pooling, batching, streaming, and intelligent scoring
          */
         this.augmentations = new AugmentationRegistry();
+        // Periodic cleanup for soft-deleted items
+        this.periodicCleanup = null;
         // Timeout and retry configuration
         this.timeoutConfig = {};
         this.retryConfig = {};
@@ -397,6 +401,41 @@ export class BrainyData {
         if (this.loggingConfig?.verbose) {
             console.log('🚀 New augmentation system initialized successfully');
         }
+        // Initialize periodic cleanup system
+        await this.initializePeriodicCleanup();
+    }
+    /**
+     * Initialize periodic cleanup system for old soft-deleted items
+     * SAFETY-CRITICAL: Coordinates with both HNSW and metadata indexes
+     */
+    async initializePeriodicCleanup() {
+        if (!this.storage) {
+            throw new Error('Cannot initialize periodic cleanup: storage not available');
+        }
+        // Skip cleanup if in read-only or frozen mode
+        if (this.readOnly || this.frozen) {
+            if (this.loggingConfig?.verbose) {
+                console.log('🧹 Periodic cleanup disabled: database is read-only or frozen');
+            }
+            return;
+        }
+        // Get cleanup config with safe defaults
+        const cleanupConfig = this.config.cleanup || {};
+        // Create cleanup system with all required dependencies
+        this.periodicCleanup = new PeriodicCleanup(this.storage, this.hnswIndex, this.metadataIndex, // Can be null, cleanup will handle gracefully
+        {
+            enabled: cleanupConfig.enabled !== false, // Enabled by default
+            maxAge: cleanupConfig.maxAge || 60 * 60 * 1000, // 1 hour default
+            batchSize: cleanupConfig.batchSize || 100, // 100 items per batch
+            cleanupInterval: cleanupConfig.cleanupInterval || 15 * 60 * 1000 // 15 minutes
+        });
+        // Start cleanup if enabled
+        if (this.periodicCleanup && cleanupConfig.enabled !== false) {
+            this.periodicCleanup.start();
+            if (this.loggingConfig?.verbose) {
+                console.log('🧹 Periodic cleanup system initialized and started');
+            }
+        }
     }
     checkReadOnly() {
         if (this.readOnly) {
@@ -1362,46 +1401,42 @@ export class BrainyData {
                         // Always update updatedAt
                         graphNoun.updatedAt = timestamp;
                     }
-                    // Create a copy of the metadata without modifying the original
-                    let metadataToSave = metadata;
-                    if (metadata && typeof metadata === 'object') {
-                        // Always make a copy without adding the ID
-                        metadataToSave = { ...metadata };
-                        // Add domain metadata if distributed mode is enabled
-                        if (this.domainDetector) {
-                            // First check if domain is already in metadata
-                            if (metadataToSave.domain) {
-                                // Domain already specified, keep it
-                                const domainInfo = this.domainDetector.detectDomain(metadataToSave);
+                    // Create properly namespaced metadata for new items
+                    let metadataToSave = createNamespacedMetadata(metadata);
+                    // Add domain metadata if distributed mode is enabled
+                    if (this.domainDetector) {
+                        // First check if domain is already in metadata
+                        if (metadataToSave.domain) {
+                            // Domain already specified, keep it
+                            const domainInfo = this.domainDetector.detectDomain(metadataToSave);
+                            if (domainInfo.domainMetadata) {
+                                ;
+                                metadataToSave.domainMetadata =
+                                    domainInfo.domainMetadata;
+                            }
+                        }
+                        else {
+                            // Try to detect domain from the data
+                            const dataToAnalyze = Array.isArray(vectorOrData)
+                                ? metadata
+                                : vectorOrData;
+                            const domainInfo = this.domainDetector.detectDomain(dataToAnalyze);
+                            if (domainInfo.domain) {
+                                ;
+                                metadataToSave.domain = domainInfo.domain;
                                 if (domainInfo.domainMetadata) {
                                     ;
                                     metadataToSave.domainMetadata =
                                         domainInfo.domainMetadata;
                                 }
                             }
-                            else {
-                                // Try to detect domain from the data
-                                const dataToAnalyze = Array.isArray(vectorOrData)
-                                    ? metadata
-                                    : vectorOrData;
-                                const domainInfo = this.domainDetector.detectDomain(dataToAnalyze);
-                                if (domainInfo.domain) {
-                                    ;
-                                    metadataToSave.domain = domainInfo.domain;
-                                    if (domainInfo.domainMetadata) {
-                                        ;
-                                        metadataToSave.domainMetadata =
-                                            domainInfo.domainMetadata;
-                                    }
-                                }
-                            }
-                        }
-                        // Add partition information if distributed mode is enabled
-                        if (this.partitioner) {
-                            const partition = this.partitioner.getPartition(id);
-                            metadataToSave.partition = partition;
                         }
                     }
+                    // Add partition information if distributed mode is enabled
+                    if (this.partitioner) {
+                        const partition = this.partitioner.getPartition(id);
+                        metadataToSave.partition = partition;
+                    }
                     await this.storage.saveMetadata(id, metadataToSave);
                     // Update metadata index (write-only mode should build indices!)
                     if (this.index && !this.frozen) {
@@ -2077,10 +2112,11 @@ export class BrainyData {
                 // This preserves pure vector searches without metadata
                 if (metadataFilter && Object.keys(metadataFilter).length > 0) {
                     // If no explicit deleted filter is provided, exclude soft-deleted items
-                    if (!metadataFilter.deleted && !metadataFilter.anyOf) {
+                    // Use namespaced field for O(1) performance
+                    if (!metadataFilter['_brainy.deleted'] && !metadataFilter.anyOf) {
                         metadataFilter = {
                             ...metadataFilter,
-                            deleted: { notEquals: true }
+                            ['_brainy.deleted']: false // O(1) positive match instead of notEquals
                         };
                     }
                 }
@@ -2264,7 +2300,8 @@ export class BrainyData {
             if (result.metadata && typeof result.metadata === 'object') {
                 const metadata = result.metadata;
                 // Exclude deleted items from search results (soft delete)
-                if (metadata.deleted === true) {
+                // Check namespaced field
+                if (metadata._brainy?.deleted === true) {
                     return false;
                 }
                 // Exclude placeholder nouns from search results
@@ -2877,9 +2914,9 @@ export class BrainyData {
                     finalWeight = options.weight;
                 }
             }
-            // Create complete verb metadata separately
-            // Merge original metadata with system metadata to preserve neural enhancements
-            const verbMetadata = {
+            // Create complete verb metadata with proper namespace
+            // First combine user metadata with verb-specific metadata
+            const userAndVerbMetadata = {
                 sourceId: sourceId,
                 targetId: targetId,
                 source: sourceId,
@@ -2899,6 +2936,8 @@ export class BrainyData {
                 ...(options.metadata || {}),
                 data: options.metadata // Also store in data field for backwards compatibility
             };
+            // Now wrap with namespace for internal fields
+            const verbMetadata = createNamespacedMetadata(userAndVerbMetadata);
             // Add to index
             await this.index.addItem({ id, vector: verbVector });
             // Get the noun from the index
@@ -2978,6 +3017,12 @@ export class BrainyData {
             const metadata = await this.storage.getVerbMetadata(id);
             if (!metadata) {
                 console.warn(`Verb ${id} found but no metadata - creating minimal GraphVerb`);
+            }
+            else if (isDeleted(metadata)) {
+                // Check if verb is soft-deleted
+                return null;
+            }
+            if (!metadata) {
                 // Return minimal GraphVerb if metadata is missing
                 return {
                     id: hnswVerb.id,
@@ -3226,50 +3271,65 @@ export class BrainyData {
         // Check if database is in read-only mode
         this.checkReadOnly();
         try {
-            // PERFORMANCE: Use soft delete by default for O(log n) filtering
-            // The MetadataIndex can efficiently filter out deleted items
-            if (!options.hard) {
-                // Soft delete: Just mark as deleted in metadata
-                try {
-                    await this.storage.saveVerbMetadata(id, {
-                        deleted: true,
-                        deletedAt: new Date().toISOString(),
-                        deletedBy: options.service || '2.0-api'
-                    });
-                    // Update MetadataIndex for O(log n) filtering
-                    if (this.metadataIndex) {
-                        await this.metadataIndex.updateIndex(id, { deleted: true });
-                    }
-                    return true;
-                }
-                catch (error) {
-                    // If verb doesn't exist, return false (not an error)
+            // CONSISTENT: Always use soft delete for graph integrity and recoverability
+            // The MetadataIndex efficiently filters out deleted items with O(1) performance
+            try {
+                const existing = await this.storage.getVerb(id);
+                if (!existing || !existing.metadata) {
+                    // Verb doesn't exist, return false (not an error)
                     return false;
                 }
+                const updatedMetadata = markDeleted(existing.metadata);
+                await this.storage.saveVerbMetadata(id, updatedMetadata);
+                // Update MetadataIndex for O(1) filtering
+                if (this.metadataIndex) {
+                    await this.metadataIndex.removeFromIndex(id, existing.metadata);
+                    await this.metadataIndex.addToIndex(id, updatedMetadata);
+                }
+                return true;
             }
-            // Hard delete path (explicit request only)
-            const existingMetadata = await this.storage.getVerbMetadata(id);
-            // Remove from index
-            const removed = this.index.removeItem(id);
-            if (!removed) {
+            catch (error) {
+                // If verb doesn't exist, return false (not an error)
                 return false;
             }
-            // Remove from metadata index
-            if (this.index && existingMetadata) {
-                await this.metadataIndex?.removeFromIndex?.(id, existingMetadata);
-            }
-            // Remove from storage
-            await this.storage.deleteVerb(id);
-            // Track deletion statistics
-            const service = this.getServiceName(options);
-            await this.storage.decrementStatistic('verb', service);
-            return true;
         }
         catch (error) {
             console.error(`Failed to delete verb ${id}:`, error);
             throw new Error(`Failed to delete verb ${id}: ${error}`);
         }
     }
+    /**
+     * Restore a soft-deleted verb (complement to consistent soft delete)
+     * @param id The verb ID to restore
+     * @param options Options for the restore operation
+     * @returns Promise<boolean> True if restored, false if not found or not deleted
+     */
+    async restoreVerb(id, options = {}) {
+        await this.ensureInitialized();
+        // Check if database is in read-only mode
+        this.checkReadOnly();
+        try {
+            const existing = await this.storage.getVerb(id);
+            if (!existing || !existing.metadata) {
+                return false; // Verb doesn't exist
+            }
+            if (!isDeleted(existing.metadata)) {
+                return false; // Verb not deleted, nothing to restore
+            }
+            const restoredMetadata = markRestored(existing.metadata);
+            await this.storage.saveVerbMetadata(id, restoredMetadata);
+            // Update MetadataIndex
+            if (this.metadataIndex) {
+                await this.metadataIndex.removeFromIndex(id, existing.metadata);
+                await this.metadataIndex.addToIndex(id, restoredMetadata);
+            }
+            return true;
+        }
+        catch (error) {
+            console.error(`Failed to restore verb ${id}:`, error);
+            throw new Error(`Failed to restore verb ${id}: ${error}`);
+        }
+    }
     /**
      * Get the number of vectors in the database
      */
@@ -5474,8 +5534,8 @@ export class BrainyData {
                 metadata = {};
             }
             else if (typeof metadata === 'object') {
-                // Check if this item is soft-deleted
-                if (metadata.deleted === true) {
+                // Check if this item is soft-deleted using namespace
+                if (isDeleted(metadata)) {
                     // Return null for soft-deleted items to match expected API behavior
                     return null;
                 }
@@ -5527,17 +5587,34 @@ export class BrainyData {
                 }
             }
             // For 2.0 API safety, we default to soft delete
-            // Soft delete: just mark as deleted - metadata filter will exclude from search
+            // Soft delete: mark as deleted using namespace for O(1) filtering
             try {
-                await this.updateNounMetadata(actualId, {
-                    deleted: true,
-                    deletedAt: new Date().toISOString(),
-                    deletedBy: '2.0-api'
-                });
+                const existing = await this.getNoun(actualId);
+                if (!existing) {
+                    // Item doesn't exist, return false (per API contract)
+                    return false;
+                }
+                if (existing.metadata) {
+                    // Directly save the metadata with deleted flag set
+                    const metadata = existing.metadata;
+                    const metadataWithNamespace = metadata._brainy
+                        ? metadata
+                        : createNamespacedMetadata(metadata);
+                    const updatedMetadata = markDeleted(metadataWithNamespace);
+                    // Save to storage
+                    await this.storage.saveMetadata(actualId, updatedMetadata);
+                    // CRITICAL: Update the metadata index for O(1) soft delete filtering
+                    if (this.metadataIndex) {
+                        // Remove old metadata from index
+                        await this.metadataIndex.removeFromIndex(actualId, metadataWithNamespace);
+                        // Add updated metadata with deleted flag
+                        await this.metadataIndex.addToIndex(actualId, updatedMetadata);
+                    }
+                }
                 return true;
             }
             catch (error) {
-                // If item doesn't exist, return false (delete of non-existent item is not an error)
+                // If an actual error occurs, return false
                 return false;
             }
         }
@@ -5546,6 +5623,62 @@ export class BrainyData {
             throw new Error(`Failed to delete vector ${id}: ${error}`);
         }
     }
+    /**
+     * Restore a soft-deleted noun (complement to consistent soft delete)
+     * @param id The noun ID to restore
+     * @returns Promise<boolean> True if restored, false if not found or not deleted
+     */
+    async restoreNoun(id) {
+        // Validate id parameter first, before any other logic
+        if (id === null || id === undefined) {
+            throw new Error('ID cannot be null or undefined');
+        }
+        await this.ensureInitialized();
+        // Check if database is in read-only mode
+        this.checkReadOnly();
+        try {
+            // Handle content text vs ID resolution (same as deleteNoun)
+            let actualId = id;
+            if (!this.index.getNouns().has(id)) {
+                // Try to find a noun with matching text content
+                for (const [nounId, noun] of this.index.getNouns().entries()) {
+                    if (noun.metadata?.text === id) {
+                        actualId = nounId;
+                        break;
+                    }
+                }
+            }
+            const existing = await this.getNoun(actualId);
+            if (!existing) {
+                return false; // Noun doesn't exist
+            }
+            if (!existing.metadata) {
+                return false; // No metadata
+            }
+            // Ensure metadata has namespace structure before checking if deleted
+            const metadata = existing.metadata;
+            const metadataWithNamespace = metadata._brainy
+                ? metadata
+                : createNamespacedMetadata(getUserMetadata(metadata));
+            if (!isDeleted(metadataWithNamespace)) {
+                return false; // Noun not deleted, nothing to restore
+            }
+            // Restore the noun using the namespace-aware metadata
+            const restoredMetadata = markRestored(metadataWithNamespace);
+            // Save to storage
+            await this.storage.saveMetadata(actualId, restoredMetadata);
+            // Update the metadata index
+            if (this.metadataIndex) {
+                await this.metadataIndex.removeFromIndex(actualId, metadataWithNamespace);
+                await this.metadataIndex.addToIndex(actualId, restoredMetadata);
+            }
+            return true;
+        }
+        catch (error) {
+            console.error(`Failed to restore noun ${id}:`, error);
+            throw new Error(`Failed to restore noun ${id}: ${error}`);
+        }
+    }
     /**
      * Delete multiple nouns by IDs
      * @param ids Array of noun IDs
@@ -5611,6 +5744,8 @@ export class BrainyData {
                 else if (!metadata && existingMetadata) {
                     finalMetadata = existingMetadata;
                 }
+                // Update metadata while preserving namespaces
+                finalMetadata = updateNamespacedMetadata(existingMetadata || {}, finalMetadata);
                 // Update the noun with new data and vector
                 const updatedNoun = {
                     ...existingNoun,
@@ -5668,8 +5803,17 @@ export class BrainyData {
             if (!noun) {
                 throw new Error(`Vector with ID ${id} does not exist`);
             }
+            // Get existing metadata to preserve namespaces
+            const existing = await this.storage.getMetadata(id) || {};
+            // Update metadata while preserving namespace structure
+            const metadataToSave = updateNamespacedMetadata(existing, metadata);
             // Save updated metadata to storage
-            await this.storage.saveMetadata(id, metadata);
+            await this.storage.saveMetadata(id, metadataToSave);
+            // Update metadata index for efficient filtering
+            if (this.metadataIndex) {
+                await this.metadataIndex.removeFromIndex(id, existing);
+                await this.metadataIndex.addToIndex(id, metadataToSave);
+            }
             // Invalidate search cache since metadata has changed
             this.cache?.invalidateOnDataChange('update');
         }
@@ -5803,12 +5947,14 @@ export class BrainyData {
         else if (offset > 0) {
             processedQuery.offset = offset;
         }
-        // Apply soft-delete filtering if needed
+        // Add soft-delete filter using POSITIVE match (O(1) hash lookup)
+        // We use _brainy.deleted to avoid conflicts with user metadata
         if (excludeDeleted) {
             if (!processedQuery.where) {
                 processedQuery.where = {};
             }
-            processedQuery.where.deleted = { notEquals: true };
+            // Use namespaced field for O(1) hash lookup in metadata index
+            processedQuery.where['_brainy.deleted'] = false; // or { equals: false }
         }
         // Apply mode-specific optimizations
         if (mode !== 'auto') {

package/dist/embeddings/model-manager.d.ts

@@ -1,11 +1,13 @@
 /**
  * Model Manager - Ensures transformer models are available at runtime
  *
- * Strategy:
- * 1. Check local cache first
- * 2. Try GitHub releases (our backup)
- * 3. Fall back to Hugging Face
- * 4. Future: CDN at models.soulcraft.com
+ * Strategy (in order):
+ * 1. Check local cache first (instant)
+ * 2. Try Soulcraft CDN (fastest when available)
+ * 3. Try GitHub release tar.gz with extraction (reliable backup)
+ * 4. Fall back to Hugging Face (always works)
+ *
+ * NO USER CONFIGURATION REQUIRED - Everything is automatic!
  */
 export declare class ModelManager {
     private static instance;
@@ -16,9 +18,8 @@ export declare class ModelManager {
     private getModelsPath;
     ensureModels(modelName?: string): Promise<boolean>;
     private verifyModelFiles;
-    private downloadFromGitHub;
-    private downloadFromCDN;
-    private configureTransformers;
+    private tryModelSource;
+    private downloadAndExtractFromGitHub;
     /**
      * Pre-download models for deployment
      * This is what npm run download-models calls