@soulcraft/brainy 2.1.0 → 3.0.0

package/dist/embeddings/model-manager.js

@@ -1,47 +1,43 @@
 /**
  * Model Manager - Ensures transformer models are available at runtime
  *
- * Strategy:
- * 1. Check local cache first
- * 2. Try GitHub releases (our backup)
- * 3. Fall back to Hugging Face
- * 4. Future: CDN at models.soulcraft.com
+ * Strategy (in order):
+ * 1. Check local cache first (instant)
+ * 2. Try Soulcraft CDN (fastest when available)
+ * 3. Try GitHub release tar.gz with extraction (reliable backup)
+ * 4. Fall back to Hugging Face (always works)
+ *
+ * NO USER CONFIGURATION REQUIRED - Everything is automatic!
  */
 import { existsSync } from 'fs';
-import { join, dirname } from 'path';
+import { mkdir, writeFile } from 'fs/promises';
+import { join } from 'path';
 import { env } from '@huggingface/transformers';
 // Model sources in order of preference
 const MODEL_SOURCES = {
-    // GitHub Release - our controlled backup
-    github: 'https://github.com/soulcraftlabs/brainy/releases/download/models-v1/all-MiniLM-L6-v2.tar.gz',
-    // Future CDN - fastest option when available
-    cdn: 'https://models.soulcraft.com/brainy/all-MiniLM-L6-v2.tar.gz',
-    // Original Hugging Face - fallback
-    huggingface: 'default' // Uses transformers.js default
-};
-// Expected model files and their hashes
-const MODEL_MANIFEST = {
-    'Xenova/all-MiniLM-L6-v2': {
-        files: {
-            'onnx/model.onnx': {
-                size: 90555481,
-                sha256: null // Will be computed from actual model
-            },
-            'tokenizer.json': {
-                size: 711661,
-                sha256: null
-            },
-            'config.json': {
-                size: 650,
-                sha256: null
-            },
-            'tokenizer_config.json': {
-                size: 366,
-                sha256: null
-            }
-        }
+    // CDN - Fastest when available (currently active)
+    cdn: {
+        host: 'https://models.soulcraft.com/models',
+        pathTemplate: '{model}/', // e.g., Xenova/all-MiniLM-L6-v2/
+        testFile: 'config.json' // File to test availability
+    },
+    // GitHub Release - tar.gz fallback (already exists and works)
+    githubRelease: {
+        tarUrl: 'https://github.com/soulcraftlabs/brainy/releases/download/models-v1/all-MiniLM-L6-v2.tar.gz'
+    },
+    // Original Hugging Face - final fallback (always works)
+    huggingface: {
+        host: 'https://huggingface.co',
+        pathTemplate: '{model}/resolve/{revision}/' // Default transformers.js pattern
     }
 };
+// Model verification files - minimal set needed for transformers.js
+const MODEL_FILES = [
+    'config.json',
+    'tokenizer.json',
+    'tokenizer_config.json',
+    'onnx/model.onnx'
+];
 export class ModelManager {
     constructor() {
         this.isInitialized = false;
@@ -76,96 +72,120 @@ export class ModelManager {
         if (this.isInitialized) {
             return true;
         }
-        const modelPath = join(this.modelsPath, ...modelName.split('/'));
+        // Configure transformers.js environment
+        env.cacheDir = this.modelsPath;
+        env.allowLocalModels = true;
+        env.useFSCache = true;
         // Check if model already exists locally
-        if (await this.verifyModelFiles(modelPath, modelName)) {
+        const modelPath = join(this.modelsPath, ...modelName.split('/'));
+        if (await this.verifyModelFiles(modelPath)) {
             console.log('✅ Models found in cache:', modelPath);
-            this.configureTransformers(modelPath);
+            env.allowRemoteModels = false; // Use local only
             this.isInitialized = true;
             return true;
         }
         // Try to download from our sources
         console.log('📥 Downloading transformer models...');
-        // Try GitHub first (our backup)
-        if (await this.downloadFromGitHub(modelName)) {
+        // Try CDN first (fastest when available)
+        if (await this.tryModelSource('Soulcraft CDN', MODEL_SOURCES.cdn, modelName)) {
             this.isInitialized = true;
             return true;
         }
-        // Try CDN (when available)
-        if (await this.downloadFromCDN(modelName)) {
+        // Try GitHub release with tar.gz extraction (reliable backup)
+        if (await this.downloadAndExtractFromGitHub(modelName)) {
             this.isInitialized = true;
             return true;
         }
-        // Fall back to Hugging Face (default transformers.js behavior)
+        // Fall back to Hugging Face (always works)
         console.log('⚠️ Using Hugging Face fallback for models');
+        env.remoteHost = MODEL_SOURCES.huggingface.host;
+        env.remotePathTemplate = MODEL_SOURCES.huggingface.pathTemplate;
         env.allowRemoteModels = true;
         this.isInitialized = true;
         return true;
     }
-    async verifyModelFiles(modelPath, modelName) {
-        const manifest = MODEL_MANIFEST[modelName];
-        if (!manifest)
-            return false;
-        for (const [filePath, info] of Object.entries(manifest.files)) {
-            const fullPath = join(modelPath, filePath);
+    async verifyModelFiles(modelPath) {
+        // Check if essential model files exist
+        for (const file of MODEL_FILES) {
+            const fullPath = join(modelPath, file);
             if (!existsSync(fullPath)) {
                 return false;
             }
-            // Optionally verify size
-            if (process.env.VERIFY_MODEL_SIZE === 'true') {
-                const stats = await import('fs').then(fs => fs.promises.stat(fullPath));
-                if (stats.size !== info.size) {
-                    console.warn(`⚠️ Model file size mismatch: ${filePath}`);
-                    return false;
-                }
-            }
         }
         return true;
     }
-    async downloadFromGitHub(modelName) {
+    async tryModelSource(name, source, modelName) {
         try {
-            const url = MODEL_SOURCES.github;
-            console.log('📥 Downloading from GitHub releases...');
-            // Download tar.gz file
-            const response = await fetch(url);
-            if (!response.ok) {
-                throw new Error(`GitHub download failed: ${response.status}`);
+            console.log(`📥 Trying ${name}...`);
+            // Test if the source is accessible by trying to fetch a test file
+            const testFile = source.testFile || 'config.json';
+            const modelPath = source.pathTemplate.replace('{model}', modelName).replace('{revision}', 'main');
+            const testUrl = `${source.host}/${modelPath}${testFile}`;
+            const response = await fetch(testUrl).catch(() => null);
+            if (response && response.ok) {
+                console.log(`✅ ${name} is available`);
+                // Configure transformers.js to use this source
+                env.remoteHost = source.host;
+                env.remotePathTemplate = source.pathTemplate;
+                env.allowRemoteModels = true;
+                // The model will be downloaded automatically by transformers.js when needed
+                return true;
+            }
+            else {
+                console.log(`⚠️ ${name} not available (${response?.status || 'unreachable'})`);
+                return false;
             }
-            const buffer = await response.arrayBuffer();
-            // Extract tar.gz (would need tar library in production)
-            // For now, return false to fall back to other methods
-            console.log('⚠️ GitHub model extraction not yet implemented');
-            return false;
         }
         catch (error) {
-            console.log('⚠️ GitHub download failed:', error.message);
+            console.log(`⚠️ ${name} check failed:`, error.message);
             return false;
         }
     }
-    async downloadFromCDN(modelName) {
+    async downloadAndExtractFromGitHub(modelName) {
         try {
-            const url = MODEL_SOURCES.cdn;
-            console.log('📥 Downloading from Soulcraft CDN...');
-            // Try to fetch from CDN
-            const response = await fetch(url);
+            console.log('📥 Trying GitHub Release (tar.gz)...');
+            // Download tar.gz file
+            const response = await fetch(MODEL_SOURCES.githubRelease.tarUrl);
             if (!response.ok) {
-                throw new Error(`CDN download failed: ${response.status}`);
+                console.log(`⚠️ GitHub Release not available (${response.status})`);
+                return false;
+            }
+            // Since we can't use tar-stream, we'll use Node's built-in child_process
+            // to extract using system tar command (available on all Unix systems)
+            const buffer = await response.arrayBuffer();
+            const modelPath = join(this.modelsPath, ...modelName.split('/'));
+            // Create model directory
+            await mkdir(modelPath, { recursive: true });
+            // Write tar.gz to temp file and extract
+            const tempFile = join(this.modelsPath, 'temp-model.tar.gz');
+            await writeFile(tempFile, Buffer.from(buffer));
+            // Extract using system tar command
+            const { exec } = await import('child_process');
+            const { promisify } = await import('util');
+            const execAsync = promisify(exec);
+            try {
+                // Extract and strip the first directory component
+                await execAsync(`tar -xzf ${tempFile} -C ${modelPath} --strip-components=1`, {
+                    cwd: this.modelsPath
+                });
+                // Clean up temp file
+                const { unlink } = await import('fs/promises');
+                await unlink(tempFile);
+                console.log('✅ GitHub Release models extracted and cached locally');
+                // Configure to use local models now
+                env.allowRemoteModels = false;
+                return true;
+            }
+            catch (extractError) {
+                console.log('⚠️ Tar extraction failed, trying alternative method');
+                return false;
             }
-            // Would extract files here
-            console.log('⚠️ CDN not yet available');
-            return false;
         }
         catch (error) {
-            console.log('⚠️ CDN download failed:', error.message);
+            console.log('⚠️ GitHub Release download failed:', error.message);
             return false;
         }
     }
-    configureTransformers(modelPath) {
-        // Configure transformers.js to use our local models
-        env.localModelPath = dirname(modelPath);
-        env.allowRemoteModels = false;
-        console.log('🔧 Configured transformers.js to use local models');
-    }
     /**
      * Pre-download models for deployment
      * This is what npm run download-models calls

package/dist/triple/TripleIntelligence.d.ts

@@ -83,6 +83,10 @@ export declare class TripleIntelligenceEngine {
      * Field-based filtering
      */
     private fieldFilter;
+    /**
+     * Fallback manual metadata filtering when index is not available
+     */
+    private manualMetadataFilter;
     /**
      * Fusion ranking combines all signals
      */

package/dist/triple/TripleIntelligence.js

@@ -171,25 +171,30 @@ export class TripleIntelligenceEngine {
                     }
                     break;
                 case 'vector':
-                    if (candidates.length === 0) {
-                        // Initial vector search
+                    // CRITICAL: If we have a previous step that returned 0 candidates, 
+                    // we must respect that and not do a fresh search
+                    if (candidates.length === 0 && plan.steps[0].type === 'vector') {
+                        // This is the first step - do initial vector search
                         const results = await this.vectorSearch(query.like || query.similar, query.limit);
                         candidates = results;
                     }
-                    else {
-                        // Vector search within candidates
+                    else if (candidates.length > 0) {
+                        // Vector search within existing candidates
                         candidates = await this.vectorSearchWithin(query.like || query.similar, candidates);
                     }
+                    // If candidates.length === 0 and this isn't the first step, keep empty candidates
                     break;
                 case 'graph':
-                    if (candidates.length === 0) {
-                        // Initial graph traversal
+                    // CRITICAL: Same logic as vector - respect empty candidates from previous steps
+                    if (candidates.length === 0 && plan.steps[0].type === 'graph') {
+                        // This is the first step - do initial graph traversal
                         candidates = await this.graphTraversal(query.connected);
                     }
-                    else {
-                        // Graph expansion from candidates
+                    else if (candidates.length > 0) {
+                        // Graph expansion from existing candidates
                         candidates = await this.graphExpand(candidates, query.connected);
                     }
+                    // If candidates.length === 0 and this isn't the first step, keep empty candidates
                     break;
                 case 'fusion':
                     // Final fusion ranking
@@ -248,7 +253,13 @@ export class TripleIntelligenceEngine {
         // Use the MetadataIndex directly for FAST field queries!
         // This uses B-tree indexes for O(log n) range queries
         // and hash indexes for O(1) exact matches
-        const matchingIds = await this.brain.metadataIndex?.getIdsForFilter(where) || [];
+        const metadataIndex = this.brain.metadataIndex;
+        // Check if metadata index is properly initialized
+        if (!metadataIndex || typeof metadataIndex.getIdsForFilter !== 'function') {
+            // Fallback to manual filtering - slower but works
+            return this.manualMetadataFilter(where);
+        }
+        const matchingIds = await metadataIndex.getIdsForFilter(where) || [];
         // Convert to result format with metadata
         const results = [];
         for (const id of matchingIds.slice(0, 1000)) {
@@ -263,6 +274,25 @@ export class TripleIntelligenceEngine {
         }
         return results;
     }
+    /**
+     * Fallback manual metadata filtering when index is not available
+     */
+    async manualMetadataFilter(where) {
+        const { matchesMetadataFilter } = await import('../utils/metadataFilter.js');
+        const results = [];
+        // Get all nouns and manually filter them
+        const allNouns = this.brain.index.getNouns();
+        for (const [id, noun] of Array.from(allNouns.entries()).slice(0, 1000)) {
+            if (noun && matchesMetadataFilter(noun.metadata || {}, where)) {
+                results.push({
+                    id,
+                    score: 1.0,
+                    metadata: noun.metadata || {}
+                });
+            }
+        }
+        return results;
+    }
     /**
      * Fusion ranking combines all signals
      */

package/dist/utils/deletedItemsIndex.d.ts

@@ -0,0 +1,59 @@
+/**
+ * Dedicated index for tracking soft-deleted items
+ * This is MUCH more efficient than checking every item in the database
+ *
+ * Performance characteristics:
+ * - Add deleted item: O(1)
+ * - Remove deleted item: O(1)
+ * - Check if deleted: O(1)
+ * - Get all deleted: O(d) where d = number of deleted items << total items
+ */
+export declare class DeletedItemsIndex {
+    private deletedIds;
+    private deletedCount;
+    /**
+     * Mark an item as deleted
+     */
+    markDeleted(id: string): void;
+    /**
+     * Mark an item as not deleted (restored)
+     */
+    markRestored(id: string): void;
+    /**
+     * Check if an item is deleted - O(1)
+     */
+    isDeleted(id: string): boolean;
+    /**
+     * Get all deleted item IDs - O(d)
+     */
+    getAllDeleted(): string[];
+    /**
+     * Filter out deleted items from results - O(k) where k = result count
+     */
+    filterDeleted<T extends {
+        id?: string;
+    }>(items: T[]): T[];
+    /**
+     * Get statistics
+     */
+    getStats(): {
+        deletedCount: number;
+        memoryUsage: number;
+    };
+    /**
+     * Clear all deleted items (for testing)
+     */
+    clear(): void;
+    /**
+     * Serialize for persistence
+     */
+    serialize(): string;
+    /**
+     * Deserialize from persistence
+     */
+    deserialize(data: string): void;
+}
+/**
+ * Global singleton for deleted items tracking
+ */
+export declare const deletedItemsIndex: DeletedItemsIndex;

package/dist/utils/deletedItemsIndex.js

@@ -0,0 +1,98 @@
+/**
+ * Dedicated index for tracking soft-deleted items
+ * This is MUCH more efficient than checking every item in the database
+ *
+ * Performance characteristics:
+ * - Add deleted item: O(1)
+ * - Remove deleted item: O(1)
+ * - Check if deleted: O(1)
+ * - Get all deleted: O(d) where d = number of deleted items << total items
+ */
+export class DeletedItemsIndex {
+    constructor() {
+        this.deletedIds = new Set();
+        this.deletedCount = 0;
+    }
+    /**
+     * Mark an item as deleted
+     */
+    markDeleted(id) {
+        if (!this.deletedIds.has(id)) {
+            this.deletedIds.add(id);
+            this.deletedCount++;
+        }
+    }
+    /**
+     * Mark an item as not deleted (restored)
+     */
+    markRestored(id) {
+        if (this.deletedIds.delete(id)) {
+            this.deletedCount--;
+        }
+    }
+    /**
+     * Check if an item is deleted - O(1)
+     */
+    isDeleted(id) {
+        return this.deletedIds.has(id);
+    }
+    /**
+     * Get all deleted item IDs - O(d)
+     */
+    getAllDeleted() {
+        return Array.from(this.deletedIds);
+    }
+    /**
+     * Filter out deleted items from results - O(k) where k = result count
+     */
+    filterDeleted(items) {
+        if (this.deletedCount === 0) {
+            // Fast path - no deleted items
+            return items;
+        }
+        return items.filter(item => {
+            const id = item.id;
+            return id ? !this.deletedIds.has(id) : true;
+        });
+    }
+    /**
+     * Get statistics
+     */
+    getStats() {
+        return {
+            deletedCount: this.deletedCount,
+            memoryUsage: this.deletedCount * 100 // Rough estimate: 100 bytes per ID
+        };
+    }
+    /**
+     * Clear all deleted items (for testing)
+     */
+    clear() {
+        this.deletedIds.clear();
+        this.deletedCount = 0;
+    }
+    /**
+     * Serialize for persistence
+     */
+    serialize() {
+        return JSON.stringify(Array.from(this.deletedIds));
+    }
+    /**
+     * Deserialize from persistence
+     */
+    deserialize(data) {
+        try {
+            const ids = JSON.parse(data);
+            this.deletedIds = new Set(ids);
+            this.deletedCount = this.deletedIds.size;
+        }
+        catch (e) {
+            console.warn('Failed to deserialize deleted items index');
+        }
+    }
+}
+/**
+ * Global singleton for deleted items tracking
+ */
+export const deletedItemsIndex = new DeletedItemsIndex();
+//# sourceMappingURL=deletedItemsIndex.js.map

package/dist/utils/ensureDeleted.d.ts

@@ -0,0 +1,38 @@
+/**
+ * Utility to ensure all metadata has the deleted field set properly
+ * This is CRITICAL for O(1) soft delete filtering performance
+ *
+ * Uses _brainy namespace to avoid conflicts with user metadata
+ */
+/**
+ * Ensure metadata has internal Brainy fields set
+ * @param metadata The metadata object (could be null/undefined)
+ * @param preserveExisting If true, preserve existing deleted value
+ * @returns Metadata with internal fields guaranteed
+ */
+export declare function ensureDeletedField(metadata: any, preserveExisting?: boolean): any;
+/**
+ * Mark an item as soft deleted
+ * @param metadata The metadata object
+ * @returns Metadata with _brainy.deleted=true
+ */
+export declare function markAsDeleted(metadata: any): any;
+/**
+ * Mark an item as restored (not deleted)
+ * @param metadata The metadata object
+ * @returns Metadata with _brainy.deleted=false
+ */
+export declare function markAsRestored(metadata: any): any;
+/**
+ * Check if an item is deleted
+ * @param metadata The metadata object
+ * @returns true if deleted, false otherwise (including if field missing)
+ */
+export declare function isDeleted(metadata: any): boolean;
+/**
+ * Check if an item is active (not deleted)
+ * @param metadata The metadata object
+ * @returns true if not deleted (default), false if deleted
+ */
+export declare function isActive(metadata: any): boolean;
+export declare const BRAINY_DELETED_FIELD = "_brainy.deleted";

package/dist/utils/ensureDeleted.js

@@ -0,0 +1,79 @@
+/**
+ * Utility to ensure all metadata has the deleted field set properly
+ * This is CRITICAL for O(1) soft delete filtering performance
+ *
+ * Uses _brainy namespace to avoid conflicts with user metadata
+ */
+const BRAINY_NAMESPACE = '_brainy';
+/**
+ * Ensure metadata has internal Brainy fields set
+ * @param metadata The metadata object (could be null/undefined)
+ * @param preserveExisting If true, preserve existing deleted value
+ * @returns Metadata with internal fields guaranteed
+ */
+export function ensureDeletedField(metadata, preserveExisting = true) {
+    // Handle null/undefined metadata
+    if (!metadata) {
+        return {
+            [BRAINY_NAMESPACE]: {
+                deleted: false,
+                version: 1
+            }
+        };
+    }
+    // Clone to avoid mutation
+    const result = { ...metadata };
+    // Ensure _brainy namespace exists
+    if (!result[BRAINY_NAMESPACE]) {
+        result[BRAINY_NAMESPACE] = {};
+    }
+    // Set deleted field if not present
+    if (!('deleted' in result[BRAINY_NAMESPACE])) {
+        result[BRAINY_NAMESPACE].deleted = false;
+    }
+    else if (!preserveExisting) {
+        // Force to false if not preserving
+        result[BRAINY_NAMESPACE].deleted = false;
+    }
+    return result;
+}
+/**
+ * Mark an item as soft deleted
+ * @param metadata The metadata object
+ * @returns Metadata with _brainy.deleted=true
+ */
+export function markAsDeleted(metadata) {
+    const result = ensureDeletedField(metadata);
+    result[BRAINY_NAMESPACE].deleted = true;
+    return result;
+}
+/**
+ * Mark an item as restored (not deleted)
+ * @param metadata The metadata object
+ * @returns Metadata with _brainy.deleted=false
+ */
+export function markAsRestored(metadata) {
+    const result = ensureDeletedField(metadata);
+    result[BRAINY_NAMESPACE].deleted = false;
+    return result;
+}
+/**
+ * Check if an item is deleted
+ * @param metadata The metadata object
+ * @returns true if deleted, false otherwise (including if field missing)
+ */
+export function isDeleted(metadata) {
+    return metadata?.[BRAINY_NAMESPACE]?.deleted === true;
+}
+/**
+ * Check if an item is active (not deleted)
+ * @param metadata The metadata object
+ * @returns true if not deleted (default), false if deleted
+ */
+export function isActive(metadata) {
+    // If no deleted field or deleted=false, item is active
+    return !isDeleted(metadata);
+}
+// Export the namespace constant for use in queries
+export const BRAINY_DELETED_FIELD = `${BRAINY_NAMESPACE}.deleted`;
+//# sourceMappingURL=ensureDeleted.js.map

package/dist/utils/metadataFilter.js

@@ -24,8 +24,13 @@ function matchesQuery(value, query) {
             case 'notEquals':
             case 'isNot':
             case 'ne':
+                // Special handling: if value is undefined and operand is not undefined,
+                // they are not equal (so the condition passes)
+                // This ensures items without a 'deleted' field match 'deleted !== true'
                 if (value === operand)
                     return false;
+                // If value is undefined and operand is not, they're not equal (pass)
+                // If both are undefined, they're equal (fail, handled above)
                 break;
             // Comparison operators
             case 'greaterThan':

package/dist/utils/metadataIndex.d.ts

@@ -107,6 +107,10 @@ export declare class MetadataIndexManager {
      * Remove item from metadata indexes
      */
     removeFromIndex(id: string, metadata?: any): Promise<void>;
+    /**
+     * Get all IDs in the index
+     */
+    getAllIds(): Promise<string[]>;
     /**
      * Get IDs for a specific field-value combination with caching
      */