@soulcraft/brainy 0.60.0 → 0.61.1

package/dist/brainyData.js

@@ -2146,26 +2146,8 @@ export class BrainyData {
         }
         return results;
     }
-    /**
-     * Get all nouns in the database
-     * @returns Array of vector documents
-     */
-    async getAllNouns() {
-        await this.ensureInitialized();
-        try {
-            // Use getNouns with no pagination to get all nouns
-            const result = await this.getNouns({
-                pagination: {
-                    limit: Number.MAX_SAFE_INTEGER // Request all nouns
-                }
-            });
-            return result.items;
-        }
-        catch (error) {
-            console.error('Failed to get all nouns:', error);
-            throw new Error(`Failed to get all nouns: ${error}`);
-        }
-    }
+    // getAllNouns() method removed - use getNouns() with pagination instead
+    // This method was dangerous and could cause expensive scans and memory issues
     /**
      * Get nouns with pagination and filtering
      * @param options Pagination and filtering options
@@ -2725,7 +2707,7 @@ export class BrainyData {
                     const scores = await this.intelligentVerbScoring.computeVerbScores(sourceId, targetId, verbType, options.weight, options.metadata);
                     finalWeight = scores.weight;
                     finalConfidence = scores.confidence;
-                    scoringReasoning = scores.reasoning;
+                    scoringReasoning = scores.reasoning || [];
                     if (this.loggingConfig?.verbose && scoringReasoning.length > 0) {
                         console.log(`Intelligent verb scoring for ${sourceId}-${verbType}-${targetId}:`, scoringReasoning);
                     }
@@ -2748,8 +2730,8 @@ export class BrainyData {
                 type: verbType, // Set the type property to match the verb type
                 weight: finalWeight,
                 confidence: finalConfidence, // Add confidence to metadata
-                intelligentScoring: scoringReasoning.length > 0 ? {
-                    reasoning: scoringReasoning,
+                intelligentScoring: this.intelligentVerbScoring?.enabled ? {
+                    reasoning: scoringReasoning.length > 0 ? scoringReasoning : [`Final weight ${finalWeight}`, `Base confidence ${finalConfidence || 0.5}`],
                     computedAt: new Date().toISOString()
                 } : undefined,
                 createdAt: timestamp,
@@ -2851,7 +2833,12 @@ export class BrainyData {
                 updatedAt: metadata.updatedAt,
                 createdBy: metadata.createdBy,
                 data: metadata.data,
-                metadata: metadata.data // Alias for backward compatibility
+                metadata: {
+                    ...metadata.data,
+                    weight: metadata.weight,
+                    confidence: metadata.confidence,
+                    ...(metadata.intelligentScoring && { intelligentScoring: metadata.intelligentScoring })
+                } // Complete metadata including intelligent scoring when available
             };
             return graphVerb;
         }
@@ -2861,47 +2848,94 @@ export class BrainyData {
         }
     }
     /**
-     * Get all verbs
-     * @returns Array of all verbs
+     * Internal performance optimization: intelligently load verbs when beneficial
+     * @internal - Used by search, indexing, and caching optimizations
      */
-    async getAllVerbs() {
-        await this.ensureInitialized();
-        try {
-            // Get all lightweight verbs from storage
-            const hnswVerbs = await this.storage.getAllVerbs();
-            // Convert each HNSWVerb to GraphVerb by loading metadata
-            const graphVerbs = [];
-            for (const hnswVerb of hnswVerbs) {
-                const metadata = await this.storage.getVerbMetadata(hnswVerb.id);
-                if (metadata) {
-                    const graphVerb = {
-                        id: hnswVerb.id,
-                        vector: hnswVerb.vector,
-                        sourceId: metadata.sourceId,
-                        targetId: metadata.targetId,
-                        source: metadata.source,
-                        target: metadata.target,
-                        verb: metadata.verb,
-                        type: metadata.type,
-                        weight: metadata.weight,
-                        createdAt: metadata.createdAt,
-                        updatedAt: metadata.updatedAt,
-                        createdBy: metadata.createdBy,
-                        data: metadata.data,
-                        metadata: metadata.data // Alias for backward compatibility
-                    };
-                    graphVerbs.push(graphVerb);
-                }
-                else {
-                    console.warn(`Verb ${hnswVerb.id} found but no metadata - skipping`);
-                }
+    async _optimizedLoadAllVerbs() {
+        // Only load all if it's safe and beneficial
+        if (await this._shouldPreloadAllData()) {
+            const result = await this.getVerbs({
+                pagination: { limit: Number.MAX_SAFE_INTEGER }
+            });
+            return result.items;
+        }
+        // Fall back to on-demand loading
+        return [];
+    }
+    /**
+     * Internal performance optimization: intelligently load nouns when beneficial
+     * @internal - Used by search, indexing, and caching optimizations
+     */
+    async _optimizedLoadAllNouns() {
+        // Only load all if it's safe and beneficial
+        if (await this._shouldPreloadAllData()) {
+            const result = await this.getNouns({
+                pagination: { limit: Number.MAX_SAFE_INTEGER }
+            });
+            return result.items;
+        }
+        // Fall back to on-demand loading
+        return [];
+    }
+    /**
+     * Intelligent decision making for when to preload all data
+     * @internal
+     */
+    async _shouldPreloadAllData() {
+        // Smart heuristics for performance optimization
+        // 1. Read-only mode is ideal for preloading
+        if (this.readOnly) {
+            return await this._isDatasetSizeReasonable();
+        }
+        // 2. Check available memory (Node.js)
+        if (typeof process !== 'undefined' && process.memoryUsage) {
+            const memUsage = process.memoryUsage();
+            const availableMemory = memUsage.heapTotal - memUsage.heapUsed;
+            const memoryMB = availableMemory / (1024 * 1024);
+            // Only preload if we have substantial free memory (>500MB)
+            if (memoryMB < 500) {
+                console.debug('Performance optimization: Skipping preload due to low memory');
+                return false;
             }
-            return graphVerbs;
         }
-        catch (error) {
-            console.error('Failed to get all verbs:', error);
-            throw new Error(`Failed to get all verbs: ${error}`);
+        // 3. Consider frozen/immutable mode  
+        if (this.frozen) {
+            return await this._isDatasetSizeReasonable();
         }
+        // 4. For frequent search operations, preloading can be beneficial
+        // TODO: Track search frequency and decide based on access patterns
+        return false; // Conservative default for write-heavy workloads
+    }
+    /**
+     * Estimate if dataset size is reasonable for in-memory loading
+     * @internal
+     */
+    async _isDatasetSizeReasonable() {
+        // Implement basic size estimation
+        // Check if we have recent statistics
+        const stats = await this.getStatistics();
+        if (stats) {
+            const totalEntities = Object.values(stats.nounCount || {}).reduce((a, b) => a + b, 0) +
+                Object.values(stats.verbCount || {}).reduce((a, b) => a + b, 0);
+            // Conservative thresholds
+            if (totalEntities > 100000) {
+                console.debug('Performance optimization: Dataset too large for preloading');
+                return false;
+            }
+            if (totalEntities < 10000) {
+                console.debug('Performance optimization: Small dataset - safe to preload');
+                return true;
+            }
+        }
+        // Medium datasets - check memory pressure
+        if (typeof process !== 'undefined' && process.memoryUsage) {
+            const memUsage = process.memoryUsage();
+            const heapUsedPercent = (memUsage.heapUsed / memUsage.heapTotal) * 100;
+            // Only preload if heap usage is low
+            return heapUsedPercent < 50;
+        }
+        // Default: conservative approach
+        return false;
     }
     /**
      * Get verbs with pagination and filtering
@@ -3679,19 +3713,40 @@ export class BrainyData {
             }
             // First use the HNSW index to find similar vectors efficiently
             const searchResults = await this.index.search(queryVector, k * 2);
-            // Get all verbs for filtering
-            const allVerbs = await this.getAllVerbs();
-            // Create a map of verb IDs for faster lookup
-            const verbMap = new Map();
-            for (const verb of allVerbs) {
-                verbMap.set(verb.id, verb);
+            // Intelligent verb loading: preload all if beneficial, otherwise on-demand
+            let verbMap = null;
+            let usePreloadedVerbs = false;
+            // Try to intelligently preload verbs for performance
+            const preloadedVerbs = await this._optimizedLoadAllVerbs();
+            if (preloadedVerbs.length > 0) {
+                verbMap = new Map();
+                for (const verb of preloadedVerbs) {
+                    verbMap.set(verb.id, verb);
+                }
+                usePreloadedVerbs = true;
+                console.debug(`Performance optimization: Preloaded ${preloadedVerbs.length} verbs for fast lookup`);
             }
+            // Fallback: on-demand verb loading function
+            const getVerbById = async (verbId) => {
+                if (usePreloadedVerbs && verbMap) {
+                    return verbMap.get(verbId) || null;
+                }
+                try {
+                    const verb = await this.getVerb(verbId);
+                    return verb;
+                }
+                catch (error) {
+                    console.warn(`Failed to load verb ${verbId}:`, error);
+                    return null;
+                }
+            };
             // Filter search results to only include verbs
             const verbResults = [];
+            // Process search results and load verbs on-demand
             for (const result of searchResults) {
                 // Search results are [id, distance] tuples
                 const [id, distance] = result;
-                const verb = verbMap.get(id);
+                const verb = await getVerbById(id);
                 if (verb) {
                     // If verb types are specified, check if this verb matches
                     if (options.verbTypes && options.verbTypes.length > 0) {
@@ -3721,8 +3776,11 @@ export class BrainyData {
                     }
                 }
                 else {
-                    // Use all verbs
-                    verbs = allVerbs;
+                    // Get all verbs with pagination
+                    const allVerbsResult = await this.getVerbs({
+                        pagination: { limit: 10000 }
+                    });
+                    verbs = allVerbsResult.items;
                 }
                 // Calculate similarity for each verb not already in results
                 const existingIds = new Set(verbResults.map((v) => v.id));
@@ -4248,10 +4306,18 @@ export class BrainyData {
     async backup() {
         await this.ensureInitialized();
         try {
-            // Get all nouns
-            const nouns = await this.getAllNouns();
-            // Get all verbs
-            const verbs = await this.getAllVerbs();
+            // Use intelligent loading for backup - this is a legitimate use case for full export
+            console.log('Creating backup - loading all data...');
+            // For backup, we legitimately need all data, so use large pagination
+            const nounsResult = await this.getNouns({
+                pagination: { limit: Number.MAX_SAFE_INTEGER }
+            });
+            const nouns = nounsResult.items;
+            const verbsResult = await this.getVerbs({
+                pagination: { limit: Number.MAX_SAFE_INTEGER }
+            });
+            const verbs = verbsResult.items;
+            console.log(`Backup: Loaded ${nouns.length} nouns and ${verbs.length} verbs`);
             // Get all noun types
             const nounTypes = Object.values(NounType);
             // Get all verb types

package/dist/coreTypes.d.ts

@@ -512,16 +512,4 @@ export interface StorageAdapter {
      * @returns Promise that resolves to an array of changes
      */
     getChangesSince?(timestamp: number, limit?: number): Promise<any[]>;
-    /**
-     * Get all nouns from storage
-     * @returns Promise that resolves to an array of all nouns
-     * @deprecated This method loads all data into memory and may cause performance issues. Use getNouns() with pagination instead.
-     */
-    getAllNouns(): Promise<HNSWNoun[]>;
-    /**
-     * Get all verbs from storage
-     * @returns Promise that resolves to an array of all HNSWVerbs
-     * @deprecated This method loads all data into memory and may cause performance issues. Use getVerbs() with pagination instead.
-     */
-    getAllVerbs(): Promise<HNSWVerb[]>;
 }

package/dist/cortex/cortex.js

@@ -2158,7 +2158,7 @@ export class Cortex {
         console.log(boxen(`${emojis.brain}☁️ ${colors.brain('BRAIN CLOUD PREMIUM FEATURES')}\n\n` +
             `Premium connectors and features have moved to Brain Cloud!\n\n` +
             `${colors.accent('◆')} ${colors.dim('Setup Brain Cloud:')} ${colors.highlight('brainy cloud')}\n` +
-            `${colors.accent('◆')} ${colors.dim('Learn more:')} ${colors.highlight('https://soulcraftlabs.com/brain-cloud')}\n\n` +
+            `${colors.accent('◆')} ${colors.dim('Learn more:')} ${colors.highlight('https://soulcraft.com/brain-cloud')}\n\n` +
             `${colors.retro('Available Tiers:')}\n` +
             `${colors.success('🫙')} Brain Jar (Free) - Local coordination\n` +
             `${colors.success('☁️')} Brain Cloud ($19/mo) - Sync everywhere\n` +
@@ -2174,7 +2174,7 @@ export class Cortex {
             `${colors.accent('◆')} ${colors.dim('No credit card required')}\n` +
             `${colors.accent('◆')} ${colors.dim('Cancel anytime')}\n\n` +
             `${colors.highlight('Run: brainy cloud')}\n\n` +
-            `Or visit: ${colors.accent('https://soulcraftlabs.com/brain-cloud')}`, { padding: 1, borderStyle: 'round', borderColor: '#FFD700' }));
+            `Or visit: ${colors.accent('https://soulcraft.com/brain-cloud')}`, { padding: 1, borderStyle: 'round', borderColor: '#FFD700' }));
     }
     async licenseValidate(featureId) {
         console.log(colors.info('Premium features available in Brain Cloud'));
@@ -2189,7 +2189,7 @@ export class Cortex {
             console.log(boxen(`${emojis.lock} ${colors.brain('BRAIN CLOUD FEATURE')} ${emojis.atom}\n\n` +
                 `This feature is available in Brain Cloud!\n\n` +
                 `${colors.highlight('Setup: brainy cloud')}\n` +
-                `${colors.dim('Learn more: https://soulcraftlabs.com/brain-cloud')}`, { padding: 1, borderStyle: 'round', borderColor: '#D67441' }));
+                `${colors.dim('Learn more: https://soulcraft.com/brain-cloud')}`, { padding: 1, borderStyle: 'round', borderColor: '#D67441' }));
         }
         return false;
     }
@@ -2389,7 +2389,7 @@ export class Cortex {
                 ]);
                 if (!accountResponse.hasAccount) {
                     console.log('\n' + boxen(`${emojis.sparkles} ${colors.brain('CREATE YOUR ACCOUNT')}\n\n` +
-                        `${colors.accent('◆')} Visit: ${colors.highlight('https://soulcraftlabs.com/brain-cloud')}\n` +
+                        `${colors.accent('◆')} Visit: ${colors.highlight('https://soulcraft.com/brain-cloud')}\n` +
                         `${colors.accent('◆')} Click "Start Free Trial"\n` +
                         `${colors.accent('◆')} Get your API key\n` +
                         `${colors.accent('◆')} Return here to continue setup`, { padding: 1, borderStyle: 'round', borderColor: '#FFD700' }));
@@ -2432,7 +2432,7 @@ export class Cortex {
         catch (error) {
             spinner.fail('Setup failed');
             console.error(colors.error('Error:'), error.message);
-            console.log('\n' + colors.dim('Need help? Visit https://soulcraftlabs.com/brain-cloud/support'));
+            console.log('\n' + colors.dim('Need help? Visit https://soulcraft.com/brain-cloud/support'));
         }
     }
     /**

package/dist/storage/adapters/baseStorageAdapter.d.ts

@@ -29,18 +29,6 @@ export declare abstract class BaseStorageAdapter implements StorageAdapter {
         quota: number | null;
         details?: Record<string, any>;
     }>;
-    /**
-     * Get all nouns from storage
-     * @returns Promise that resolves to an array of all nouns
-     * @deprecated This method loads all data into memory and may cause performance issues. Use getNouns() with pagination instead.
-     */
-    abstract getAllNouns(): Promise<any[]>;
-    /**
-     * Get all verbs from storage
-     * @returns Promise that resolves to an array of all HNSWVerbs
-     * @deprecated This method loads all data into memory and may cause performance issues. Use getVerbs() with pagination instead.
-     */
-    abstract getAllVerbs(): Promise<any[]>;
     /**
      * Get nouns with pagination and filtering
      * @param options Pagination and filtering options

package/dist/storage/adapters/opfsStorage.js

@@ -146,7 +146,7 @@ export class OPFSStorage extends BaseStorage {
                 connections: this.mapToObject(noun.connections, (set) => Array.from(set))
             };
             // Create or get the file for this noun
-            const fileHandle = await this.nounsDir.getFileHandle(noun.id, {
+            const fileHandle = await this.nounsDir.getFileHandle(`${noun.id}.json`, {
                 create: true
             });
             // Write the noun data to the file
@@ -166,7 +166,7 @@ export class OPFSStorage extends BaseStorage {
         await this.ensureInitialized();
         try {
             // Get the file handle for this noun
-            const fileHandle = await this.nounsDir.getFileHandle(id);
+            const fileHandle = await this.nounsDir.getFileHandle(`${id}.json`);
             // Read the noun data from the file
             const file = await fileHandle.getFile();
             const text = await file.text();
@@ -253,7 +253,7 @@ export class OPFSStorage extends BaseStorage {
     async deleteNode(id) {
         await this.ensureInitialized();
         try {
-            await this.nounsDir.removeEntry(id);
+            await this.nounsDir.removeEntry(`${id}.json`);
         }
         catch (error) {
             // Ignore NotFoundError, which means the file doesn't exist
@@ -281,7 +281,7 @@ export class OPFSStorage extends BaseStorage {
                 connections: this.mapToObject(edge.connections, (set) => Array.from(set))
             };
             // Create or get the file for this verb
-            const fileHandle = await this.verbsDir.getFileHandle(edge.id, {
+            const fileHandle = await this.verbsDir.getFileHandle(`${edge.id}.json`, {
                 create: true
             });
             // Write the verb data to the file
@@ -307,7 +307,7 @@ export class OPFSStorage extends BaseStorage {
         await this.ensureInitialized();
         try {
             // Get the file handle for this edge
-            const fileHandle = await this.verbsDir.getFileHandle(id);
+            const fileHandle = await this.verbsDir.getFileHandle(`${id}.json`);
             // Read the edge data from the file
             const file = await fileHandle.getFile();
             const text = await file.text();
@@ -389,10 +389,12 @@ export class OPFSStorage extends BaseStorage {
      * Get verbs by source (internal implementation)
      */
     async getVerbsBySource_internal(sourceId) {
-        // This method is deprecated and would require loading metadata for each edge
-        // For now, return empty array since this is not efficiently implementable with new storage pattern
-        console.warn('getVerbsBySource_internal is deprecated and not efficiently supported in new storage pattern');
-        return [];
+        // Use the paginated approach to properly handle HNSWVerb to GraphVerb conversion
+        const result = await this.getVerbsWithPagination({
+            filter: { sourceId: [sourceId] },
+            limit: Number.MAX_SAFE_INTEGER // Get all matching results
+        });
+        return result.items;
     }
     /**
      * Get edges by source
@@ -407,10 +409,12 @@ export class OPFSStorage extends BaseStorage {
      * Get verbs by target (internal implementation)
      */
     async getVerbsByTarget_internal(targetId) {
-        // This method is deprecated and would require loading metadata for each edge
-        // For now, return empty array since this is not efficiently implementable with new storage pattern
-        console.warn('getVerbsByTarget_internal is deprecated and not efficiently supported in new storage pattern');
-        return [];
+        // Use the paginated approach to properly handle HNSWVerb to GraphVerb conversion
+        const result = await this.getVerbsWithPagination({
+            filter: { targetId: [targetId] },
+            limit: Number.MAX_SAFE_INTEGER // Get all matching results
+        });
+        return result.items;
     }
     /**
      * Get edges by target
@@ -425,10 +429,12 @@ export class OPFSStorage extends BaseStorage {
      * Get verbs by type (internal implementation)
      */
     async getVerbsByType_internal(type) {
-        // This method is deprecated and would require loading metadata for each edge
-        // For now, return empty array since this is not efficiently implementable with new storage pattern
-        console.warn('getVerbsByType_internal is deprecated and not efficiently supported in new storage pattern');
-        return [];
+        // Use the paginated approach to properly handle HNSWVerb to GraphVerb conversion
+        const result = await this.getVerbsWithPagination({
+            filter: { verbType: [type] },
+            limit: Number.MAX_SAFE_INTEGER // Get all matching results
+        });
+        return result.items;
     }
     /**
      * Get edges by type
@@ -451,7 +457,7 @@ export class OPFSStorage extends BaseStorage {
     async deleteEdge(id) {
         await this.ensureInitialized();
         try {
-            await this.verbsDir.removeEntry(id);
+            await this.verbsDir.removeEntry(`${id}.json`);
         }
         catch (error) {
             // Ignore NotFoundError, which means the file doesn't exist
@@ -468,7 +474,7 @@ export class OPFSStorage extends BaseStorage {
         await this.ensureInitialized();
         try {
             // Create or get the file for this metadata
-            const fileHandle = await this.metadataDir.getFileHandle(id, {
+            const fileHandle = await this.metadataDir.getFileHandle(`${id}.json`, {
                 create: true
             });
             // Write the metadata to the file
@@ -488,7 +494,7 @@ export class OPFSStorage extends BaseStorage {
         await this.ensureInitialized();
         try {
             // Get the file handle for this metadata
-            const fileHandle = await this.metadataDir.getFileHandle(id);
+            const fileHandle = await this.metadataDir.getFileHandle(`${id}.json`);
             // Read the metadata from the file
             const file = await fileHandle.getFile();
             const text = await file.text();

package/dist/storage/adapters/s3CompatibleStorage.d.ts

@@ -305,26 +305,14 @@ export declare class S3CompatibleStorage extends BaseStorage {
      * Get verbs by source (internal implementation)
      */
     protected getVerbsBySource_internal(sourceId: string): Promise<GraphVerb[]>;
-    /**
-     * Get edges by source
-     */
-    protected getEdgesBySource(sourceId: string): Promise<GraphVerb[]>;
     /**
      * Get verbs by target (internal implementation)
      */
     protected getVerbsByTarget_internal(targetId: string): Promise<GraphVerb[]>;
-    /**
-     * Get edges by target
-     */
-    protected getEdgesByTarget(targetId: string): Promise<GraphVerb[]>;
     /**
      * Get verbs by type (internal implementation)
      */
     protected getVerbsByType_internal(type: string): Promise<GraphVerb[]>;
-    /**
-     * Get edges by type
-     */
-    protected getEdgesByType(type: string): Promise<GraphVerb[]>;
     /**
      * Delete a verb from storage (internal implementation)
      */