@soulcraft/brainy 0.60.0 → 0.61.1

package/README.md

@@ -7,9 +7,10 @@
 [![Node.js](https://img.shields.io/badge/node-%3E%3D24.4.1-brightgreen.svg)](https://nodejs.org/)
 [![TypeScript](https://img.shields.io/badge/TypeScript-5.4.5-blue.svg)](https://www.typescriptlang.org/)
 
-# BRAINY: The Brain in a Jar Database™
+# BRAINY: Multi-Dimensional AI Database™
 
-**The world's only Vector + Graph + AI database and realtime data platform**
+**The world's first Multi-Dimensional AI Database**  
+*Vector similarity • Graph relationships • Metadata facets • AI context*
 
 *Zero-to-Smart™ technology that thinks so you don't have to*
 
@@ -48,13 +49,13 @@ const results = await brainy.search("AI language models", 5, {
 Pinecone ($$$) + Neo4j ($$$) + Elasticsearch ($$$) + Sync Hell = 😱
 ```
 
-### ✅ The Brainy Way: One Smart Brain
+### ✅ The Brainy Way: One Multi-Dimensional Brain
 
 ```
-Vector Search + Graph Relations + Metadata Filtering + AI Intelligence = 🧠✨
+Multi-Dimensional AI Database = Vector + Graph + Facets + AI = 🧠✨
 ```
 
-**Your data gets a brain upgrade. No assembly required.**
+**Your data gets a multi-dimensional brain upgrade. No assembly required.**
 
 ## ⚡ QUICK & EASY: From Zero to Smart in 60 Seconds
 
@@ -458,6 +459,7 @@ brainy augment trial notion  # Start 14-day free trial
 
 ### Advanced Topics
 
+- [**🏗️ Storage & Retrieval Architecture**](docs/technical/STORAGE_AND_RETRIEVAL_ARCHITECTURE.md) - Multi-dimensional database internals
 - [**Brainy CLI**](docs/brainy-cli.md) - Command-line superpowers
 - [**Brainy Chat**](BRAINY-CHAT.md) - Conversational AI interface
 - [**Cortex AI**](CORTEX.md) - Intelligence augmentation

package/bin/brainy.js

@@ -13,6 +13,7 @@ import chalk from 'chalk'
 import { readFileSync } from 'fs'
 import { dirname, join } from 'path'
 import { fileURLToPath } from 'url'
+import { createInterface } from 'readline'
 
 // Use native fetch (available in Node.js 18+)
 
@@ -57,7 +58,7 @@ const wrapInteractive = (fn) => {
 
 program
   .name('brainy')
-  .description('🧠 Brainy - Vector + Graph Database with AI Coordination')
+  .description('🧠 Brainy - Multi-Dimensional AI Database')
   .version(packageJson.version)
 
 // ========================================
@@ -66,7 +67,7 @@ program
 
 program
   .command('init')
-  .description('🚀 Initialize Brainy in your project')
+  .description('Initialize Brainy in your project')
   .option('-s, --storage <type>', 'Storage type (filesystem, s3, r2, gcs, memory)')
   .option('-e, --encryption', 'Enable encryption for secrets')
   .action(wrapAction(async (options) => {
@@ -75,8 +76,8 @@ program
 
 program
   .command('add [data]')
-  .description('📊 Add data to Brainy')
-  .option('-m, --metadata <json>', 'Metadata as JSON')
+  .description('Add data across multiple dimensions (vector, graph, facets)')
+  .option('-m, --metadata <json>', 'Metadata facets as JSON')
   .option('-i, --id <id>', 'Custom ID')
   .action(wrapAction(async (data, options) => {
     let metadata = {}
@@ -96,11 +97,11 @@ program
 
 program
   .command('search <query>')
-  .description('🔍 Search your database')
+  .description('Multi-dimensional search across vector, graph, and facets')
   .option('-l, --limit <number>', 'Number of results', '10')
-  .option('-f, --filter <json>', 'MongoDB-style metadata filters')
-  .option('-v, --verbs <types>', 'Graph verb types to traverse (comma-separated)')
-  .option('-d, --depth <number>', 'Graph traversal depth', '1')
+  .option('-f, --filter <json>', 'Filter by metadata facets')
+  .option('-v, --verbs <types>', 'Include related data (comma-separated)')
+  .option('-d, --depth <number>', 'Relationship depth', '1')
   .action(wrapAction(async (query, options) => {
     const searchOptions = { limit: parseInt(options.limit) }
     
@@ -123,7 +124,7 @@ program
 
 program
   .command('chat [question]')
-  .description('💬 Chat with your data (interactive mode if no question)')
+  .description('AI-powered chat with multi-dimensional context')
   .option('-l, --llm <model>', 'LLM model to use')
   .action(wrapInteractive(async (question, options) => {
     await cortex.chat(question)
@@ -131,7 +132,7 @@ program
 
 program
   .command('stats')
-  .description('📊 Show database statistics')
+  .description('Show database statistics and insights')
   .option('-d, --detailed', 'Show detailed statistics')
   .action(wrapAction(async (options) => {
     await cortex.stats(options.detailed)
@@ -139,7 +140,7 @@ program
 
 program
   .command('health')
-  .description('🔋 Check system health')
+  .description('Check system health')
   .option('--auto-fix', 'Automatically apply safe repairs')
   .action(wrapAction(async (options) => {
     await cortex.health(options)
@@ -147,21 +148,21 @@ program
 
 program
   .command('find')
-  .description('🔍 Interactive advanced search')
+  .description('Advanced intelligent search (interactive)')
   .action(wrapInteractive(async () => {
     await cortex.advancedSearch()
   }))
 
 program
   .command('explore [nodeId]')
-  .description('🗺️ Interactively explore graph connections')
+  .description('Explore data relationships interactively')
   .action(wrapInteractive(async (nodeId) => {
     await cortex.explore(nodeId)
   }))
 
 program
   .command('backup')
-  .description('💾 Create database backup')
+  .description('Create database backup')
   .option('-c, --compress', 'Compress backup')
   .option('-o, --output <file>', 'Output file')
   .action(wrapAction(async (options) => {
@@ -170,7 +171,7 @@ program
 
 program
   .command('restore <file>')
-  .description('♻️ Restore from backup')
+  .description('Restore from backup')
   .action(wrapInteractive(async (file) => {
     await cortex.restore(file)
   }))
@@ -181,35 +182,30 @@ program
 
 program
   .command('connect')
-  .description('🔗 Connect me to your Brain Cloud so I remember everything')
+  .description('Connect to Brain Cloud for AI memory')
   .action(wrapInteractive(async () => {
-    console.log(chalk.cyan('\n🧠 Setting Up AI Memory...'))
-    console.log(chalk.gray('━'.repeat(50)))
+    console.log(chalk.cyan('\n🧠 Brain Cloud Setup'))
+    console.log(chalk.gray('━'.repeat(40)))
     
     try {
       // Detect customer ID
       const customerId = await detectCustomerId()
       
       if (customerId) {
-        console.log(chalk.green(`✅ Found your Brain Cloud: ${customerId}`))
-        console.log('\n🔧 I can set up AI memory so I remember our conversations:')
-        console.log(chalk.yellow('  • Update Claude configuration'))
+        console.log(chalk.green(`✅ Found Brain Cloud: ${customerId}`))
+        console.log('\n🔧 Setting up AI memory:')
+        console.log(chalk.yellow('  • Update configuration'))
         console.log(chalk.yellow('  • Add memory instructions'))
         console.log(chalk.yellow('  • Enable cross-session memory'))
         
-        // For now, auto-proceed (in a real CLI environment, user could be prompted)
-        console.log(chalk.cyan('\n🚀 Setting up AI memory...'))
-        const proceed = true
-        
-        if (proceed) {
-          await setupBrainCloudMemory(customerId)
-          console.log(chalk.green('\n🎉 AI Memory Connected!'))
-          console.log(chalk.cyan('Restart Claude Code and I\'ll remember everything!'))
-        }
+        console.log(chalk.cyan('\n🚀 Configuring...'))
+        await setupBrainCloudMemory(customerId)
+        console.log(chalk.green('\n✅ AI memory connected!'))
+        console.log(chalk.cyan('Restart Claude Code to activate memory.'))
       } else {
-        console.log(chalk.yellow('🤔 No Brain Cloud found. Let me help you set one up:'))
-        console.log('\n1. Visit: ' + chalk.cyan('https://app.soulcraftlabs.com'))
-        console.log('2. Sign up for Brain Cloud ($19/month)')
+        console.log(chalk.yellow('No Brain Cloud found. Setting up:'))
+        console.log('\n1. Visit: ' + chalk.cyan('https://soulcraft.com'))
+        console.log('2. Sign up for Brain Cloud')
         console.log('3. Run ' + chalk.green('brainy connect') + ' again')
       }
     } catch (error) {
@@ -219,7 +215,7 @@ program
 
 program
   .command('cloud [action]')
-  .description('☁️ Connect to Brain Cloud - AI memory that never forgets')
+  .description('Manage Brain Cloud connection')
   .option('--connect <id>', 'Connect to existing Brain Cloud instance')
   .option('--export <id>', 'Export all data from Brain Cloud instance')
   .option('--status <id>', 'Check status of Brain Cloud instance')
@@ -264,11 +260,11 @@ program
         } else {
           console.log(chalk.red('❌ Could not connect to Brain Cloud'))
           console.log(chalk.yellow('💡 Make sure you have an active instance'))
-          console.log('\nSign up at: ' + chalk.cyan('https://app.soulcraftlabs.com'))
+          console.log('\nSign up at: ' + chalk.cyan('https://app.soulcraft.com'))
         }
       } catch (error) {
         console.log(chalk.red('❌ Connection failed:'), error.message)
-        console.log('\nSign up at: ' + chalk.cyan('https://app.soulcraftlabs.com'))
+        console.log('\nSign up at: ' + chalk.cyan('https://app.soulcraft.com'))
       }
     } else if (options.export) {
       console.log(chalk.green(`📦 Exporting data from Brain Cloud instance: ${options.export}`))
@@ -326,7 +322,7 @@ program
     } else if (options.dashboard) {
       console.log(chalk.green(`🌐 Opening dashboard for Brain Cloud instance: ${options.dashboard}`))
       
-      const dashboardUrl = `https://app.soulcraftlabs.com/dashboard.html?customer_id=${options.dashboard}`
+      const dashboardUrl = `https://app.soulcraft.com/dashboard.html?customer_id=${options.dashboard}`
       console.log(chalk.cyan(`\n🔗 Dashboard URL: ${dashboardUrl}`))
       console.log(chalk.gray('Opening in your default browser...'))
       
@@ -346,7 +342,7 @@ program
       }
     } else {
       console.log(chalk.yellow('📡 Brain Cloud Setup'))
-      console.log('\n1. Sign up at: ' + chalk.cyan('https://app.soulcraftlabs.com'))
+      console.log('\n1. Sign up at: ' + chalk.cyan('https://app.soulcraft.com'))
       console.log('2. Get your customer ID')
       console.log('3. Connect with: ' + chalk.green('brainy cloud --connect YOUR_ID'))
       console.log('\nBenefits:')
@@ -363,7 +359,7 @@ program
 
 program
   .command('install <augmentation>')
-  .description('📦 Install augmentation')
+  .description('Install augmentation')
   .option('-m, --mode <type>', 'Installation mode (free|premium)', 'free')
   .option('-c, --config <json>', 'Configuration as JSON')
   .action(wrapAction(async (augmentation, options) => {
@@ -386,7 +382,7 @@ program
 
 program
   .command('run <augmentation>')
-  .description('⚡ Run augmentation')
+  .description('Run augmentation')
   .option('-c, --config <json>', 'Runtime configuration as JSON')
   .action(wrapAction(async (augmentation, options) => {
     if (augmentation === 'brain-jar') {
@@ -400,7 +396,7 @@ program
 
 program
   .command('status [augmentation]')
-  .description('📊 Show augmentation status')
+  .description('Show augmentation status')
   .action(wrapAction(async (augmentation) => {
     if (augmentation === 'brain-jar') {
       await cortex.brainJarStatus()
@@ -415,7 +411,7 @@ program
 
 program
   .command('stop [augmentation]')
-  .description('⏹️ Stop augmentation')
+  .description('Stop augmentation')
   .action(wrapAction(async (augmentation) => {
     if (augmentation === 'brain-jar') {
       await cortex.brainJarStop()
@@ -426,11 +422,11 @@ program
 
 program
   .command('list')
-  .description('📋 List installed augmentations')
+  .description('List installed augmentations')
   .option('-a, --available', 'Show available augmentations')
   .action(wrapAction(async (options) => {
     if (options.available) {
-      console.log(chalk.cyan('🧩 Available Augmentations:'))
+      console.log(chalk.cyan('Available Augmentations:'))
       console.log('  • brain-jar - AI coordination and collaboration')
       console.log('  • encryption - Data encryption and security')
       console.log('  • neural-import - AI-powered data analysis')
@@ -442,30 +438,17 @@ program
     }
   }))
 
-// ========================================
-// BRAIN CLOUD SUPER COMMAND (New!)
-// ========================================
-
-program
-  .command('cloud')
-  .description('☁️ Setup Brain Cloud - AI coordination across all devices')
-  .option('-m, --mode <type>', 'Setup mode (free|premium)', 'interactive')
-  .option('-k, --key <key>', 'License key for premium features')
-  .option('-s, --skip-install', 'Skip Brain Jar installation')
-  .action(wrapInteractive(async (options) => {
-    await cortex.setupBrainCloud(options)
-  }))
 
 // ========================================
 // BRAIN JAR SPECIFIC COMMANDS (Rich UX)
 // ========================================
 
 const brainJar = program.command('brain-jar')
-  .description('🧠🫙 AI coordination and collaboration')
+  .description('AI coordination and collaboration')
 
 brainJar
   .command('install')
-  .description('📦 Install Brain Jar coordination')
+  .description('Install Brain Jar coordination')
   .option('-m, --mode <type>', 'Installation mode (free|premium)', 'free')
   .action(wrapAction(async (options) => {
     await cortex.brainJarInstall(options.mode)
@@ -473,7 +456,7 @@ brainJar
 
 brainJar
   .command('start')
-  .description('🚀 Start Brain Jar coordination')
+  .description('Start Brain Jar coordination')
   .option('-s, --server <url>', 'Custom server URL')
   .option('-n, --name <name>', 'Agent name')
   .option('-r, --role <role>', 'Agent role')
@@ -483,7 +466,7 @@ brainJar
 
 brainJar
   .command('dashboard')
-  .description('📊 Open Brain Jar dashboard')
+  .description('Open Brain Jar dashboard')
   .option('-o, --open', 'Auto-open in browser', true)
   .action(wrapAction(async (options) => {
     await cortex.brainJarDashboard(options.open)
@@ -491,28 +474,28 @@ brainJar
 
 brainJar
   .command('status')
-  .description('🔍 Show Brain Jar status')
+  .description('Show Brain Jar status')
   .action(wrapAction(async () => {
     await cortex.brainJarStatus()
   }))
 
 brainJar
   .command('agents')
-  .description('👥 List connected agents')
+  .description('List connected agents')
   .action(wrapAction(async () => {
     await cortex.brainJarAgents()
   }))
 
 brainJar
   .command('message <text>')
-  .description('📨 Send message to coordination channel')
+  .description('Send message to coordination channel')
   .action(wrapAction(async (text) => {
     await cortex.brainJarMessage(text)
   }))
 
 brainJar
   .command('search <query>')
-  .description('🔍 Search coordination history')
+  .description('Search coordination history')
   .option('-l, --limit <number>', 'Number of results', '10')
   .action(wrapAction(async (query, options) => {
     await cortex.brainJarSearch(query, parseInt(options.limit))
@@ -523,7 +506,7 @@ brainJar
 // ========================================
 
 const config = program.command('config')
-  .description('⚙️ Manage configuration')
+  .description('Manage configuration')
 
 config
   .command('set <key> <value>')
@@ -557,11 +540,11 @@ config
 // ========================================
 
 const cortexCmd = program.command('cortex')
-  .description('🔧 Legacy Cortex commands (deprecated - use direct commands)')
+  .description('Legacy Cortex commands (deprecated - use direct commands)')
 
 cortexCmd
   .command('chat [question]')
-  .description('💬 Chat with your data')
+  .description('Chat with your data')
   .action(wrapInteractive(async (question) => {
     console.log(chalk.yellow('⚠️  Deprecated: Use "brainy chat" instead'))
     await cortex.chat(question)
@@ -569,7 +552,7 @@ cortexCmd
 
 cortexCmd
   .command('add [data]')
-  .description('📊 Add data')
+  .description('Add data')
   .action(wrapAction(async (data) => {
     console.log(chalk.yellow('⚠️  Deprecated: Use "brainy add" instead'))
     await cortex.add(data, {})
@@ -581,7 +564,7 @@ cortexCmd
 
 program
   .command('shell')
-  .description('🐚 Interactive Brainy shell')
+  .description('Interactive Brainy shell')
   .action(wrapInteractive(async () => {
     console.log(chalk.cyan('🧠 Brainy Interactive Shell'))
     console.log(chalk.dim('Type "help" for commands, "exit" to quit\n'))
@@ -596,21 +579,24 @@ program.parse(process.argv)
 
 // Show help if no command
 if (!process.argv.slice(2).length) {
-  console.log(chalk.cyan('🧠☁️ Brainy - AI Coordination Service'))
-  console.log('')
-  console.log(chalk.bold('One-Command Setup:'))
-  console.log(chalk.green('  brainy cloud                   # Setup Brain Cloud (recommended!)'))
-  console.log('')
+  console.log(chalk.cyan('🧠 Brainy - Multi-Dimensional AI Database'))
+  console.log(chalk.gray('Vector similarity, graph relationships, metadata facets, and AI context.\n'))
+  
   console.log(chalk.bold('Quick Start:'))
   console.log('  brainy init                    # Initialize project')
-  console.log('  brainy add "some data"         # Add data')
-  console.log('  brainy search "query"          # Search data')
-  console.log('  brainy chat                    # Chat with data')
+  console.log('  brainy add "some data"         # Add multi-dimensional data')
+  console.log('  brainy search "query"          # Search across all dimensions')
+  console.log('  brainy chat                    # AI chat with full context')
+  console.log('')
+  console.log(chalk.bold('AI Memory:'))
+  console.log(chalk.green('  brainy connect                 # Connect to Brain Cloud'))
+  console.log('  brainy cloud --status <id>     # Check cloud status')
   console.log('')
   console.log(chalk.bold('AI Coordination:'))
-  console.log('  brainy install brain-jar       # Install AI coordination')
+  console.log('  brainy install brain-jar       # Install coordination')
   console.log('  brainy brain-jar start         # Start coordination')
-  console.log('  brainy brain-jar dashboard     # View dashboard')
+  console.log('')
+  console.log(chalk.dim('Learn more: https://soulcraft.com'))
   console.log('')
   program.outputHelp()
 }

package/dist/augmentations/memoryAugmentations.js

@@ -129,31 +129,41 @@ class BaseMemoryAugmentation {
                     error: 'Query must be a vector (array of numbers) for vector search'
                 };
             }
-            // Get all nodes from storage
-            const nodes = await this.storage.getAllNouns();
-            // Calculate distances and prepare results
-            const results = [];
-            for (const node of nodes) {
-                // Skip nodes that don't have a vector
-                if (!node.vector || !Array.isArray(node.vector)) {
-                    continue;
-                }
-                // Get metadata for the node
-                const metadata = await this.storage.getMetadata(node.id);
-                // Calculate distance between query vector and node vector
-                const distance = cosineDistance(queryVector, node.vector);
-                // Convert distance to similarity score (1 - distance for cosine)
-                // This way higher scores are better (more similar)
-                const score = 1 - distance;
-                results.push({
-                    id: node.id,
-                    score,
-                    data: metadata
+            // Process nodes in batches to avoid loading everything into memory
+            const allResults = [];
+            let hasMore = true;
+            let cursor;
+            while (hasMore) {
+                // Get a batch of nodes
+                const batchResult = await this.storage.getNouns({
+                    pagination: { limit: 100, cursor }
                 });
+                // Process this batch
+                for (const noun of batchResult.items) {
+                    // Skip nodes that don't have a vector
+                    if (!noun.vector || !Array.isArray(noun.vector)) {
+                        continue;
+                    }
+                    // Get metadata for the node
+                    const metadata = await this.storage.getMetadata(noun.id);
+                    // Calculate distance between query vector and node vector
+                    const distance = cosineDistance(queryVector, noun.vector);
+                    // Convert distance to similarity score (1 - distance for cosine)
+                    // This way higher scores are better (more similar)
+                    const score = 1 - distance;
+                    allResults.push({
+                        id: noun.id,
+                        score,
+                        data: metadata
+                    });
+                }
+                // Update pagination state
+                hasMore = batchResult.hasMore;
+                cursor = batchResult.nextCursor;
             }
             // Sort results by score (descending) and take top k
-            results.sort((a, b) => b.score - a.score);
-            const topResults = results.slice(0, k);
+            allResults.sort((a, b) => b.score - a.score);
+            const topResults = allResults.slice(0, k);
             return {
                 success: true,
                 data: topResults

package/dist/brainyData.d.ts

@@ -736,11 +736,6 @@ export declare class BrainyData<T = any> implements BrainyDataInterface<T> {
      * @returns Promise<Array<VectorDocument<T> | null>> Array of documents (null for missing IDs)
      */
     getBatch(ids: string[]): Promise<Array<VectorDocument<T> | null>>;
-    /**
-     * Get all nouns in the database
-     * @returns Array of vector documents
-     */
-    getAllNouns(): Promise<VectorDocument<T>[]>;
     /**
      * Get nouns with pagination and filtering
      * @param options Pagination and filtering options
@@ -830,10 +825,25 @@ export declare class BrainyData<T = any> implements BrainyDataInterface<T> {
      */
     getVerb(id: string): Promise<GraphVerb | null>;
     /**
-     * Get all verbs
-     * @returns Array of all verbs
+     * Internal performance optimization: intelligently load verbs when beneficial
+     * @internal - Used by search, indexing, and caching optimizations
+     */
+    private _optimizedLoadAllVerbs;
+    /**
+     * Internal performance optimization: intelligently load nouns when beneficial
+     * @internal - Used by search, indexing, and caching optimizations
+     */
+    private _optimizedLoadAllNouns;
+    /**
+     * Intelligent decision making for when to preload all data
+     * @internal
+     */
+    private _shouldPreloadAllData;
+    /**
+     * Estimate if dataset size is reasonable for in-memory loading
+     * @internal
      */
-    getAllVerbs(): Promise<GraphVerb[]>;
+    private _isDatasetSizeReasonable;
     /**
      * Get verbs with pagination and filtering
      * @param options Pagination and filtering options