@soulcraft/brainy 2.0.2 → 2.3.0

package/dist/importManager.js

@@ -0,0 +1,258 @@
+/**
+ * Import Manager - Comprehensive data import with intelligent type detection
+ *
+ * Handles multiple data sources:
+ * - Direct data (objects, arrays)
+ * - Files (JSON, CSV, text)
+ * - URLs (fetch and parse)
+ * - Streams (for large files)
+ *
+ * Uses NeuralImportAugmentation for intelligent processing
+ */
+import { VerbType } from './types/graphTypes.js';
+import { NeuralImportAugmentation } from './augmentations/neuralImport.js';
+import * as fs from './universal/fs.js';
+import * as path from './universal/path.js';
+import { prodLog } from './utils/logger.js';
+export class ImportManager {
+    constructor(brain) {
+        this.typeMatcher = null;
+        this.brain = brain;
+        this.neuralImport = new NeuralImportAugmentation();
+    }
+    /**
+     * Initialize the import manager
+     */
+    async init() {
+        // Initialize neural import with proper context
+        const context = {
+            brain: this.brain,
+            storage: this.brain.storage,
+            config: {},
+            log: (message, level) => {
+                if (level === 'error') {
+                    prodLog.error(message);
+                }
+                else if (level === 'warn') {
+                    prodLog.warn(message);
+                }
+                else {
+                    prodLog.info(message);
+                }
+            }
+        };
+        await this.neuralImport.initialize(context);
+        // Get type matcher
+        const { getTypeMatcher } = await import('./augmentations/typeMatching/intelligentTypeMatcher.js');
+        this.typeMatcher = await getTypeMatcher();
+    }
+    /**
+     * Main import method - handles all sources
+     */
+    async import(source, options = {}) {
+        const result = {
+            success: false,
+            nouns: [],
+            verbs: [],
+            errors: [],
+            stats: {
+                total: 0,
+                imported: 0,
+                failed: 0,
+                relationships: 0
+            }
+        };
+        try {
+            // Detect source type
+            const sourceType = await this.detectSourceType(source, options.source);
+            // Get data based on source type
+            let data;
+            let format = options.format || 'auto';
+            switch (sourceType) {
+                case 'url':
+                    data = await this.fetchFromUrl(source);
+                    break;
+                case 'file':
+                    const filePath = source;
+                    data = await this.readFile(filePath);
+                    if (format === 'auto') {
+                        format = this.detectFormatFromPath(filePath);
+                    }
+                    break;
+                case 'data':
+                default:
+                    data = source;
+                    break;
+            }
+            // Process data through neural import
+            let items;
+            let relationships = [];
+            if (Buffer.isBuffer(data) || typeof data === 'string') {
+                // Use neural import for parsing and analysis
+                const analysis = await this.neuralImport.getNeuralAnalysis(data, format);
+                // Extract items and relationships
+                items = analysis.detectedEntities.map(entity => ({
+                    data: entity.originalData,
+                    type: entity.nounType,
+                    confidence: entity.confidence,
+                    id: entity.suggestedId
+                }));
+                if (options.extractRelationships !== false) {
+                    relationships = analysis.detectedRelationships;
+                }
+                // Log insights
+                for (const insight of analysis.insights) {
+                    prodLog.info(`🧠 ${insight.description} (confidence: ${insight.confidence})`);
+                }
+            }
+            else if (Array.isArray(data)) {
+                items = data;
+            }
+            else {
+                items = [data];
+            }
+            result.stats.total = items.length;
+            // Import items in batches
+            const batchSize = options.batchSize || 50;
+            for (let i = 0; i < items.length; i += batchSize) {
+                const batch = items.slice(i, i + batchSize);
+                // Process batch in parallel if enabled
+                const promises = batch.map(async (item) => {
+                    try {
+                        // Detect type if needed
+                        let nounType = item.type || options.typeHint;
+                        if (!nounType && options.autoDetect !== false && this.typeMatcher) {
+                            const match = await this.typeMatcher.matchNounType(item.data || item);
+                            nounType = match.type;
+                        }
+                        // Prepare the data to import
+                        const dataToImport = item.data || item;
+                        // Create metadata combining original data with import metadata
+                        const metadata = {
+                            ...(typeof dataToImport === 'object' ? dataToImport : {}),
+                            ...(item.data?.metadata || {}),
+                            nounType,
+                            _importedAt: new Date().toISOString(),
+                            _confidence: item.confidence
+                        };
+                        // Add to brain - pass object once, it becomes both vector source and metadata
+                        const id = await this.brain.addNoun(metadata);
+                        result.nouns.push(id);
+                        result.stats.imported++;
+                        return id;
+                    }
+                    catch (error) {
+                        result.errors.push(`Failed to import item: ${error.message}`);
+                        result.stats.failed++;
+                        return null;
+                    }
+                });
+                if (options.parallel !== false) {
+                    await Promise.all(promises);
+                }
+                else {
+                    for (const promise of promises) {
+                        await promise;
+                    }
+                }
+            }
+            // Import relationships
+            for (const rel of relationships) {
+                try {
+                    // Match verb type if needed
+                    let verbType = rel.verbType;
+                    if (!Object.values(VerbType).includes(verbType) && this.typeMatcher) {
+                        const match = await this.typeMatcher.matchVerbType({ id: rel.sourceId }, { id: rel.targetId }, rel.verbType);
+                        verbType = match.type;
+                    }
+                    const verbId = await this.brain.addVerb(rel.sourceId, rel.targetId, verbType, rel.metadata, rel.weight);
+                    result.verbs.push(verbId);
+                    result.stats.relationships++;
+                }
+                catch (error) {
+                    result.errors.push(`Failed to create relationship: ${error.message}`);
+                }
+            }
+            result.success = result.stats.imported > 0;
+            prodLog.info(`✨ Import complete: ${result.stats.imported}/${result.stats.total} items, ${result.stats.relationships} relationships`);
+        }
+        catch (error) {
+            result.errors.push(`Import failed: ${error.message}`);
+            prodLog.error('Import failed:', error);
+        }
+        return result;
+    }
+    /**
+     * Import from file
+     */
+    async importFile(filePath, options = {}) {
+        return this.import(filePath, { ...options, source: 'file' });
+    }
+    /**
+     * Import from URL
+     */
+    async importUrl(url, options = {}) {
+        return this.import(url, { ...options, source: 'url' });
+    }
+    /**
+     * Detect source type
+     */
+    async detectSourceType(source, hint) {
+        if (hint && hint !== 'auto') {
+            return hint;
+        }
+        if (typeof source === 'string') {
+            // Check if URL
+            if (source.startsWith('http://') || source.startsWith('https://')) {
+                return 'url';
+            }
+            // Check if file path exists
+            try {
+                if (await fs.exists(source)) {
+                    return 'file';
+                }
+            }
+            catch { }
+        }
+        return 'data';
+    }
+    /**
+     * Detect format from file path
+     */
+    detectFormatFromPath(filePath) {
+        const ext = path.extname(filePath).toLowerCase();
+        switch (ext) {
+            case '.json': return 'json';
+            case '.csv': return 'csv';
+            case '.txt': return 'text';
+            case '.md': return 'text';
+            case '.yaml':
+            case '.yml': return 'yaml';
+            default: return 'auto';
+        }
+    }
+    /**
+     * Read file
+     */
+    async readFile(filePath) {
+        const content = await fs.readFile(filePath, 'utf8');
+        return Buffer.from(content, 'utf8');
+    }
+    /**
+     * Fetch from URL
+     */
+    async fetchFromUrl(url) {
+        const response = await fetch(url);
+        if (!response.ok) {
+            throw new Error(`Failed to fetch ${url}: ${response.statusText}`);
+        }
+        return response.text();
+    }
+}
+/**
+ * Create an import manager instance
+ */
+export function createImportManager(brain) {
+    return new ImportManager(brain);
+}
+//# sourceMappingURL=importManager.js.map

package/dist/neural/embeddedPatterns.d.ts

@@ -2,7 +2,7 @@
  * 🧠 BRAINY EMBEDDED PATTERNS
  *
  * AUTO-GENERATED - DO NOT EDIT
- * Generated: 2025-08-26T19:59:35.203Z
+ * Generated: 2025-08-27T16:58:35.801Z
  * Patterns: 220
  * Coverage: 94-98% of all queries
  *

package/dist/neural/embeddedPatterns.js

@@ -2,7 +2,7 @@
  * 🧠 BRAINY EMBEDDED PATTERNS
  *
  * AUTO-GENERATED - DO NOT EDIT
- * Generated: 2025-08-26T19:59:35.203Z
+ * Generated: 2025-08-27T16:58:35.801Z
  * Patterns: 220
  * Coverage: 94-98% of all queries
  *

package/dist/triple/TripleIntelligence.d.ts

@@ -83,6 +83,10 @@ export declare class TripleIntelligenceEngine {
      * Field-based filtering
      */
     private fieldFilter;
+    /**
+     * Fallback manual metadata filtering when index is not available
+     */
+    private manualMetadataFilter;
     /**
      * Fusion ranking combines all signals
      */

package/dist/triple/TripleIntelligence.js

@@ -171,25 +171,30 @@ export class TripleIntelligenceEngine {
                     }
                     break;
                 case 'vector':
-                    if (candidates.length === 0) {
-                        // Initial vector search
+                    // CRITICAL: If we have a previous step that returned 0 candidates, 
+                    // we must respect that and not do a fresh search
+                    if (candidates.length === 0 && plan.steps[0].type === 'vector') {
+                        // This is the first step - do initial vector search
                         const results = await this.vectorSearch(query.like || query.similar, query.limit);
                         candidates = results;
                     }
-                    else {
-                        // Vector search within candidates
+                    else if (candidates.length > 0) {
+                        // Vector search within existing candidates
                         candidates = await this.vectorSearchWithin(query.like || query.similar, candidates);
                     }
+                    // If candidates.length === 0 and this isn't the first step, keep empty candidates
                     break;
                 case 'graph':
-                    if (candidates.length === 0) {
-                        // Initial graph traversal
+                    // CRITICAL: Same logic as vector - respect empty candidates from previous steps
+                    if (candidates.length === 0 && plan.steps[0].type === 'graph') {
+                        // This is the first step - do initial graph traversal
                         candidates = await this.graphTraversal(query.connected);
                     }
-                    else {
-                        // Graph expansion from candidates
+                    else if (candidates.length > 0) {
+                        // Graph expansion from existing candidates
                         candidates = await this.graphExpand(candidates, query.connected);
                     }
+                    // If candidates.length === 0 and this isn't the first step, keep empty candidates
                     break;
                 case 'fusion':
                     // Final fusion ranking
@@ -248,7 +253,13 @@ export class TripleIntelligenceEngine {
         // Use the MetadataIndex directly for FAST field queries!
         // This uses B-tree indexes for O(log n) range queries
         // and hash indexes for O(1) exact matches
-        const matchingIds = await this.brain.metadataIndex?.getIdsForFilter(where) || [];
+        const metadataIndex = this.brain.metadataIndex;
+        // Check if metadata index is properly initialized
+        if (!metadataIndex || typeof metadataIndex.getIdsForFilter !== 'function') {
+            // Fallback to manual filtering - slower but works
+            return this.manualMetadataFilter(where);
+        }
+        const matchingIds = await metadataIndex.getIdsForFilter(where) || [];
         // Convert to result format with metadata
         const results = [];
         for (const id of matchingIds.slice(0, 1000)) {
@@ -263,6 +274,25 @@ export class TripleIntelligenceEngine {
         }
         return results;
     }
+    /**
+     * Fallback manual metadata filtering when index is not available
+     */
+    async manualMetadataFilter(where) {
+        const { matchesMetadataFilter } = await import('../utils/metadataFilter.js');
+        const results = [];
+        // Get all nouns and manually filter them
+        const allNouns = this.brain.index.getNouns();
+        for (const [id, noun] of Array.from(allNouns.entries()).slice(0, 1000)) {
+            if (noun && matchesMetadataFilter(noun.metadata || {}, where)) {
+                results.push({
+                    id,
+                    score: 1.0,
+                    metadata: noun.metadata || {}
+                });
+            }
+        }
+        return results;
+    }
     /**
      * Fusion ranking combines all signals
      */

package/dist/utils/deletedItemsIndex.d.ts

@@ -0,0 +1,59 @@
+/**
+ * Dedicated index for tracking soft-deleted items
+ * This is MUCH more efficient than checking every item in the database
+ *
+ * Performance characteristics:
+ * - Add deleted item: O(1)
+ * - Remove deleted item: O(1)
+ * - Check if deleted: O(1)
+ * - Get all deleted: O(d) where d = number of deleted items << total items
+ */
+export declare class DeletedItemsIndex {
+    private deletedIds;
+    private deletedCount;
+    /**
+     * Mark an item as deleted
+     */
+    markDeleted(id: string): void;
+    /**
+     * Mark an item as not deleted (restored)
+     */
+    markRestored(id: string): void;
+    /**
+     * Check if an item is deleted - O(1)
+     */
+    isDeleted(id: string): boolean;
+    /**
+     * Get all deleted item IDs - O(d)
+     */
+    getAllDeleted(): string[];
+    /**
+     * Filter out deleted items from results - O(k) where k = result count
+     */
+    filterDeleted<T extends {
+        id?: string;
+    }>(items: T[]): T[];
+    /**
+     * Get statistics
+     */
+    getStats(): {
+        deletedCount: number;
+        memoryUsage: number;
+    };
+    /**
+     * Clear all deleted items (for testing)
+     */
+    clear(): void;
+    /**
+     * Serialize for persistence
+     */
+    serialize(): string;
+    /**
+     * Deserialize from persistence
+     */
+    deserialize(data: string): void;
+}
+/**
+ * Global singleton for deleted items tracking
+ */
+export declare const deletedItemsIndex: DeletedItemsIndex;

package/dist/utils/deletedItemsIndex.js

@@ -0,0 +1,98 @@
+/**
+ * Dedicated index for tracking soft-deleted items
+ * This is MUCH more efficient than checking every item in the database
+ *
+ * Performance characteristics:
+ * - Add deleted item: O(1)
+ * - Remove deleted item: O(1)
+ * - Check if deleted: O(1)
+ * - Get all deleted: O(d) where d = number of deleted items << total items
+ */
+export class DeletedItemsIndex {
+    constructor() {
+        this.deletedIds = new Set();
+        this.deletedCount = 0;
+    }
+    /**
+     * Mark an item as deleted
+     */
+    markDeleted(id) {
+        if (!this.deletedIds.has(id)) {
+            this.deletedIds.add(id);
+            this.deletedCount++;
+        }
+    }
+    /**
+     * Mark an item as not deleted (restored)
+     */
+    markRestored(id) {
+        if (this.deletedIds.delete(id)) {
+            this.deletedCount--;
+        }
+    }
+    /**
+     * Check if an item is deleted - O(1)
+     */
+    isDeleted(id) {
+        return this.deletedIds.has(id);
+    }
+    /**
+     * Get all deleted item IDs - O(d)
+     */
+    getAllDeleted() {
+        return Array.from(this.deletedIds);
+    }
+    /**
+     * Filter out deleted items from results - O(k) where k = result count
+     */
+    filterDeleted(items) {
+        if (this.deletedCount === 0) {
+            // Fast path - no deleted items
+            return items;
+        }
+        return items.filter(item => {
+            const id = item.id;
+            return id ? !this.deletedIds.has(id) : true;
+        });
+    }
+    /**
+     * Get statistics
+     */
+    getStats() {
+        return {
+            deletedCount: this.deletedCount,
+            memoryUsage: this.deletedCount * 100 // Rough estimate: 100 bytes per ID
+        };
+    }
+    /**
+     * Clear all deleted items (for testing)
+     */
+    clear() {
+        this.deletedIds.clear();
+        this.deletedCount = 0;
+    }
+    /**
+     * Serialize for persistence
+     */
+    serialize() {
+        return JSON.stringify(Array.from(this.deletedIds));
+    }
+    /**
+     * Deserialize from persistence
+     */
+    deserialize(data) {
+        try {
+            const ids = JSON.parse(data);
+            this.deletedIds = new Set(ids);
+            this.deletedCount = this.deletedIds.size;
+        }
+        catch (e) {
+            console.warn('Failed to deserialize deleted items index');
+        }
+    }
+}
+/**
+ * Global singleton for deleted items tracking
+ */
+export const deletedItemsIndex = new DeletedItemsIndex();
+//# sourceMappingURL=deletedItemsIndex.js.map

package/dist/utils/ensureDeleted.d.ts

@@ -0,0 +1,38 @@
+/**
+ * Utility to ensure all metadata has the deleted field set properly
+ * This is CRITICAL for O(1) soft delete filtering performance
+ *
+ * Uses _brainy namespace to avoid conflicts with user metadata
+ */
+/**
+ * Ensure metadata has internal Brainy fields set
+ * @param metadata The metadata object (could be null/undefined)
+ * @param preserveExisting If true, preserve existing deleted value
+ * @returns Metadata with internal fields guaranteed
+ */
+export declare function ensureDeletedField(metadata: any, preserveExisting?: boolean): any;
+/**
+ * Mark an item as soft deleted
+ * @param metadata The metadata object
+ * @returns Metadata with _brainy.deleted=true
+ */
+export declare function markAsDeleted(metadata: any): any;
+/**
+ * Mark an item as restored (not deleted)
+ * @param metadata The metadata object
+ * @returns Metadata with _brainy.deleted=false
+ */
+export declare function markAsRestored(metadata: any): any;
+/**
+ * Check if an item is deleted
+ * @param metadata The metadata object
+ * @returns true if deleted, false otherwise (including if field missing)
+ */
+export declare function isDeleted(metadata: any): boolean;
+/**
+ * Check if an item is active (not deleted)
+ * @param metadata The metadata object
+ * @returns true if not deleted (default), false if deleted
+ */
+export declare function isActive(metadata: any): boolean;
+export declare const BRAINY_DELETED_FIELD = "_brainy.deleted";

package/dist/utils/ensureDeleted.js

@@ -0,0 +1,79 @@
+/**
+ * Utility to ensure all metadata has the deleted field set properly
+ * This is CRITICAL for O(1) soft delete filtering performance
+ *
+ * Uses _brainy namespace to avoid conflicts with user metadata
+ */
+const BRAINY_NAMESPACE = '_brainy';
+/**
+ * Ensure metadata has internal Brainy fields set
+ * @param metadata The metadata object (could be null/undefined)
+ * @param preserveExisting If true, preserve existing deleted value
+ * @returns Metadata with internal fields guaranteed
+ */
+export function ensureDeletedField(metadata, preserveExisting = true) {
+    // Handle null/undefined metadata
+    if (!metadata) {
+        return {
+            [BRAINY_NAMESPACE]: {
+                deleted: false,
+                version: 1
+            }
+        };
+    }
+    // Clone to avoid mutation
+    const result = { ...metadata };
+    // Ensure _brainy namespace exists
+    if (!result[BRAINY_NAMESPACE]) {
+        result[BRAINY_NAMESPACE] = {};
+    }
+    // Set deleted field if not present
+    if (!('deleted' in result[BRAINY_NAMESPACE])) {
+        result[BRAINY_NAMESPACE].deleted = false;
+    }
+    else if (!preserveExisting) {
+        // Force to false if not preserving
+        result[BRAINY_NAMESPACE].deleted = false;
+    }
+    return result;
+}
+/**
+ * Mark an item as soft deleted
+ * @param metadata The metadata object
+ * @returns Metadata with _brainy.deleted=true
+ */
+export function markAsDeleted(metadata) {
+    const result = ensureDeletedField(metadata);
+    result[BRAINY_NAMESPACE].deleted = true;
+    return result;
+}
+/**
+ * Mark an item as restored (not deleted)
+ * @param metadata The metadata object
+ * @returns Metadata with _brainy.deleted=false
+ */
+export function markAsRestored(metadata) {
+    const result = ensureDeletedField(metadata);
+    result[BRAINY_NAMESPACE].deleted = false;
+    return result;
+}
+/**
+ * Check if an item is deleted
+ * @param metadata The metadata object
+ * @returns true if deleted, false otherwise (including if field missing)
+ */
+export function isDeleted(metadata) {
+    return metadata?.[BRAINY_NAMESPACE]?.deleted === true;
+}
+/**
+ * Check if an item is active (not deleted)
+ * @param metadata The metadata object
+ * @returns true if not deleted (default), false if deleted
+ */
+export function isActive(metadata) {
+    // If no deleted field or deleted=false, item is active
+    return !isDeleted(metadata);
+}
+// Export the namespace constant for use in queries
+export const BRAINY_DELETED_FIELD = `${BRAINY_NAMESPACE}.deleted`;
+//# sourceMappingURL=ensureDeleted.js.map

package/dist/utils/metadataFilter.js

@@ -24,8 +24,13 @@ function matchesQuery(value, query) {
             case 'notEquals':
             case 'isNot':
             case 'ne':
+                // Special handling: if value is undefined and operand is not undefined,
+                // they are not equal (so the condition passes)
+                // This ensures items without a 'deleted' field match 'deleted !== true'
                 if (value === operand)
                     return false;
+                // If value is undefined and operand is not, they're not equal (pass)
+                // If both are undefined, they're equal (fail, handled above)
                 break;
             // Comparison operators
             case 'greaterThan':

package/dist/utils/metadataIndex.d.ts

@@ -107,6 +107,10 @@ export declare class MetadataIndexManager {
      * Remove item from metadata indexes
      */
     removeFromIndex(id: string, metadata?: any): Promise<void>;
+    /**
+     * Get all IDs in the index
+     */
+    getAllIds(): Promise<string[]>;
     /**
      * Get IDs for a specific field-value combination with caching
      */