@soulcraft/brainy 5.6.2 → 5.7.0

package/dist/storage/baseStorage.js

@@ -10,6 +10,7 @@ import { getShardIdFromUuid } from './sharding.js';
 import { RefManager } from './cow/RefManager.js';
 import { BlobStorage } from './cow/BlobStorage.js';
 import { CommitLog } from './cow/CommitLog.js';
+import { prodLog } from '../utils/logger.js';
 // Clean directory structure (v4.7.2+)
 // All storage adapters use this consistent structure
 export const NOUNS_METADATA_DIR = 'entities/nouns/metadata';
@@ -118,7 +119,7 @@ export class BaseStorage extends BaseStorageAdapter {
         // UUID validation for entity keys
         const uuidRegex = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
         if (!uuidRegex.test(id)) {
-            console.warn(`[Storage] Unknown key format: ${id} - treating as system resource`);
+            prodLog.warn(`[Storage] Unknown key format: ${id} - treating as system resource`);
             return {
                 original: id,
                 isEntity: false,
@@ -472,7 +473,7 @@ export class BaseStorage extends BaseStorageAdapter {
         // Load metadata
         const metadata = await this.getNounMetadata(id);
         if (!metadata) {
-            console.warn(`[Storage] Noun ${id} has vector but no metadata - this should not happen in v4.0.0`);
+            prodLog.warn(`[Storage] Noun ${id} has vector but no metadata - this should not happen in v4.0.0`);
             return null;
         }
         // Combine into HNSWNounWithMetadata - v4.8.0: Extract standard fields to top-level
@@ -541,7 +542,7 @@ export class BaseStorage extends BaseStorageAdapter {
         }
         catch (error) {
             // Ignore if metadata file doesn't exist
-            console.debug(`No metadata file to delete for noun ${id}`);
+            prodLog.debug(`No metadata file to delete for noun ${id}`);
         }
     }
     /**
@@ -572,7 +573,7 @@ export class BaseStorage extends BaseStorageAdapter {
         // Load metadata
         const metadata = await this.getVerbMetadata(id);
         if (!metadata) {
-            console.warn(`[Storage] Verb ${id} has vector but no metadata - this should not happen in v4.0.0`);
+            prodLog.warn(`[Storage] Verb ${id} has vector but no metadata - this should not happen in v4.0.0`);
             return null;
         }
         // Combine into HNSWVerbWithMetadata - v4.8.0: Extract standard fields to top-level
@@ -650,7 +651,7 @@ export class BaseStorage extends BaseStorageAdapter {
             };
         }
         catch (error) {
-            console.error(`Failed to convert HNSWVerb to GraphVerb for ${hnswVerb.id}:`, error);
+            prodLog.error(`Failed to convert HNSWVerb to GraphVerb for ${hnswVerb.id}:`, error);
             return null;
         }
     }
@@ -778,7 +779,7 @@ export class BaseStorage extends BaseStorageAdapter {
             }
             catch (countError) {
                 // Ignore errors from count method, it's optional
-                console.warn('Error getting noun count:', countError);
+                prodLog.warn('Error getting noun count:', countError);
             }
             // Check if the adapter has a paginated method for getting nouns
             if (typeof this.getNounsWithPagination === 'function') {
@@ -799,7 +800,7 @@ export class BaseStorage extends BaseStorageAdapter {
                 // If adapter forgets to return totalCount, log warning and use pre-calculated count
                 let finalTotalCount = result.totalCount || totalCount;
                 if (result.totalCount === undefined && this.totalNounCount > 0) {
-                    console.warn(`⚠️  Storage adapter missing totalCount in getNounsWithPagination result! ` +
+                    prodLog.warn(`⚠️  Storage adapter missing totalCount in getNounsWithPagination result! ` +
                         `Using pre-calculated count (${this.totalNounCount}) as fallback. ` +
                         `Please ensure your storage adapter returns totalCount: this.totalNounCount`);
                     finalTotalCount = this.totalNounCount;
@@ -812,7 +813,7 @@ export class BaseStorage extends BaseStorageAdapter {
                 };
             }
             // Storage adapter does not support pagination
-            console.error('Storage adapter does not support pagination. The deprecated getAllNouns_internal() method has been removed. Please implement getNounsWithPagination() in your storage adapter.');
+            prodLog.error('Storage adapter does not support pagination. The deprecated getAllNouns_internal() method has been removed. Please implement getNounsWithPagination() in your storage adapter.');
             return {
                 items: [],
                 totalCount: 0,
@@ -820,7 +821,7 @@ export class BaseStorage extends BaseStorageAdapter {
             };
         }
         catch (error) {
-            console.error('Error getting nouns with pagination:', error);
+            prodLog.error('Error getting nouns with pagination:', error);
             return {
                 items: [],
                 totalCount: 0,
@@ -1158,7 +1159,7 @@ export class BaseStorage extends BaseStorageAdapter {
             }
             catch (countError) {
                 // Ignore errors from count method, it's optional
-                console.warn('Error getting verb count:', countError);
+                prodLog.warn('Error getting verb count:', countError);
             }
             // Check if the adapter has a paginated method for getting verbs
             if (typeof this.getVerbsWithPagination === 'function') {
@@ -1180,7 +1181,7 @@ export class BaseStorage extends BaseStorageAdapter {
                 // If adapter forgets to return totalCount, log warning and use pre-calculated count
                 let finalTotalCount = result.totalCount || totalCount;
                 if (result.totalCount === undefined && this.totalVerbCount > 0) {
-                    console.warn(`⚠️  Storage adapter missing totalCount in getVerbsWithPagination result! ` +
+                    prodLog.warn(`⚠️  Storage adapter missing totalCount in getVerbsWithPagination result! ` +
                         `Using pre-calculated count (${this.totalVerbCount}) as fallback. ` +
                         `Please ensure your storage adapter returns totalCount: this.totalVerbCount`);
                     finalTotalCount = this.totalVerbCount;
@@ -1194,7 +1195,7 @@ export class BaseStorage extends BaseStorageAdapter {
             }
             // UNIVERSAL FALLBACK: Iterate through verb types with early termination (billion-scale safe)
             // This approach works for ALL storage adapters without requiring adapter-specific pagination
-            console.warn('Using universal type-iteration strategy for getVerbs(). ' +
+            prodLog.warn('Using universal type-iteration strategy for getVerbs(). ' +
                 'This works for all adapters but may be slower than native pagination. ' +
                 'For optimal performance at scale, storage adapters can implement getVerbsWithPagination().');
             const collectedVerbs = [];
@@ -1273,7 +1274,7 @@ export class BaseStorage extends BaseStorageAdapter {
             };
         }
         catch (error) {
-            console.error('Error getting verbs with pagination:', error);
+            prodLog.error('Error getting verbs with pagination:', error);
             return {
                 items: [],
                 totalCount: 0,
@@ -1294,22 +1295,45 @@ export class BaseStorage extends BaseStorageAdapter {
         }
         catch (error) {
             // Ignore if metadata file doesn't exist
-            console.debug(`No metadata file to delete for verb ${id}`);
+            prodLog.debug(`No metadata file to delete for verb ${id}`);
         }
     }
     /**
-     * Get graph index (lazy initialization)
+     * Get graph index (lazy initialization with concurrent access protection)
+     * v5.7.1: Fixed race condition where concurrent calls could trigger multiple rebuilds
      */
     async getGraphIndex() {
-        if (!this.graphIndex) {
-            console.log('Initializing GraphAdjacencyIndex...');
-            this.graphIndex = new GraphAdjacencyIndex(this);
-            // Check if we need to rebuild from existing data
-            const sampleVerbs = await this.getVerbs({ pagination: { limit: 1 } });
-            if (sampleVerbs.items.length > 0) {
-                console.log('Found existing verbs, rebuilding graph index...');
-                await this.graphIndex.rebuild();
-            }
+        // If already initialized, return immediately
+        if (this.graphIndex) {
+            return this.graphIndex;
+        }
+        // If initialization in progress, wait for it
+        if (this.graphIndexPromise) {
+            return this.graphIndexPromise;
+        }
+        // Start initialization (only first caller reaches here)
+        this.graphIndexPromise = this._initializeGraphIndex();
+        try {
+            const index = await this.graphIndexPromise;
+            return index;
+        }
+        finally {
+            // Clear promise after completion (success or failure)
+            this.graphIndexPromise = undefined;
+        }
+    }
+    /**
+     * Internal method to initialize graph index (called once by getGraphIndex)
+     * @private
+     */
+    async _initializeGraphIndex() {
+        prodLog.info('Initializing GraphAdjacencyIndex...');
+        this.graphIndex = new GraphAdjacencyIndex(this);
+        // Check if we need to rebuild from existing data
+        const sampleVerbs = await this.getVerbs({ pagination: { limit: 1 } });
+        if (sampleVerbs.items.length > 0) {
+            prodLog.info('Found existing verbs, rebuilding graph index...');
+            await this.graphIndex.rebuild();
         }
         return this.graphIndex;
     }
@@ -1592,7 +1616,7 @@ export class BaseStorage extends BaseStorageAdapter {
      * Ensures verbCountsByType is always accurate for reliable pagination
      */
     async rebuildTypeCounts() {
-        console.log('[BaseStorage] Rebuilding type counts from storage...');
+        prodLog.info('[BaseStorage] Rebuilding type counts from storage...');
         // Rebuild verb counts by checking each type directory
         for (let i = 0; i < VERB_TYPE_COUNT; i++) {
             const type = TypeUtils.getVerbFromIndex(i);
@@ -1623,7 +1647,7 @@ export class BaseStorage extends BaseStorageAdapter {
         await this.saveTypeStatistics();
         const totalVerbs = this.verbCountsByType.reduce((sum, count) => sum + count, 0);
         const totalNouns = this.nounCountsByType.reduce((sum, count) => sum + count, 0);
-        console.log(`[BaseStorage] Rebuilt counts: ${totalNouns} nouns, ${totalVerbs} verbs`);
+        prodLog.info(`[BaseStorage] Rebuilt counts: ${totalNouns} nouns, ${totalVerbs} verbs`);
     }
     /**
      * Get noun type from cache or metadata
@@ -1637,7 +1661,7 @@ export class BaseStorage extends BaseStorageAdapter {
         }
         // Default to 'thing' if unknown
         // This should only happen if saveNoun_internal is called before saveNounMetadata
-        console.warn(`[BaseStorage] Unknown noun type for ${noun.id}, defaulting to 'thing'`);
+        prodLog.warn(`[BaseStorage] Unknown noun type for ${noun.id}, defaulting to 'thing'`);
         return 'thing';
     }
     /**
@@ -1654,7 +1678,7 @@ export class BaseStorage extends BaseStorageAdapter {
             return verb.type;
         }
         // This should never happen with current data
-        console.warn(`[BaseStorage] Verb missing type field for ${verb.id}, defaulting to 'relatedTo'`);
+        prodLog.warn(`[BaseStorage] Verb missing type field for ${verb.id}, defaulting to 'relatedTo'`);
         return 'relatedTo';
     }
     // ============================================================================
@@ -1729,7 +1753,7 @@ export class BaseStorage extends BaseStorageAdapter {
                 }
             }
             catch (error) {
-                console.warn(`[BaseStorage] Failed to load noun from ${path}:`, error);
+                prodLog.warn(`[BaseStorage] Failed to load noun from ${path}:`, error);
             }
         }
         return nouns;
@@ -1784,6 +1808,25 @@ export class BaseStorage extends BaseStorageAdapter {
         this.verbTypeCache.set(verb.id, type);
         // COW-aware write (v5.0.1): Use COW helper for branch isolation
         await this.writeObjectToBranch(path, verb);
+        // v5.7.0: Update GraphAdjacencyIndex incrementally for billion-scale optimization
+        // CRITICAL: Only update if index already initialized to avoid circular dependency
+        // Index is lazy-loaded on first query, then maintained incrementally
+        if (this.graphIndex && this.graphIndex.isInitialized) {
+            // Fast incremental update - no rebuild needed
+            await this.graphIndex.addVerb({
+                id: verb.id,
+                sourceId: verb.sourceId,
+                targetId: verb.targetId,
+                vector: verb.vector,
+                source: verb.sourceId,
+                target: verb.targetId,
+                verb: verb.verb,
+                type: verb.verb,
+                createdAt: { seconds: Math.floor(Date.now() / 1000), nanoseconds: 0 },
+                updatedAt: { seconds: Math.floor(Date.now() / 1000), nanoseconds: 0 },
+                createdBy: { augmentation: 'storage', version: '5.7.0' }
+            });
+        }
         // Periodically save statistics
         if (this.verbCountsByType[typeIndex] % 100 === 0) {
             await this.saveTypeStatistics();
@@ -1825,109 +1868,71 @@ export class BaseStorage extends BaseStorageAdapter {
      * v5.4.0: Fixed to directly list verb files instead of directories
      */
     async getVerbsBySource_internal(sourceId) {
-        // v5.4.0: Type-first implementation - scan across all verb types
-        // COW-aware: uses readWithInheritance for each verb
+        // v5.7.0: BILLION-SCALE OPTIMIZATION - Use GraphAdjacencyIndex for O(log n) lookup
+        // Previous: O(total_verbs) - scanned all 127 verb types
+        // Now: O(log n) LSM-tree lookup + O(verbs_for_source) load
         await this.ensureInitialized();
+        const startTime = performance.now();
+        // Get GraphAdjacencyIndex (lazy-initialized)
+        const graphIndex = await this.getGraphIndex();
+        // O(log n) lookup with bloom filter optimization
+        const verbIds = await graphIndex.getVerbIdsBySource(sourceId);
+        // Load each verb by ID (uses existing optimized getVerb())
         const results = [];
-        // Iterate through all verb types
-        for (let i = 0; i < VERB_TYPE_COUNT; i++) {
-            const type = TypeUtils.getVerbFromIndex(i);
-            const typeDir = `entities/verbs/${type}/vectors`;
+        for (const verbId of verbIds) {
             try {
-                // v5.4.0 FIX: List all verb files directly (not shard directories)
-                // listObjectsInBranch returns full paths to .json files, not directories
-                const verbFiles = await this.listObjectsInBranch(typeDir);
-                for (const verbPath of verbFiles) {
-                    // Skip if not a .json file
-                    if (!verbPath.endsWith('.json'))
-                        continue;
-                    try {
-                        const verb = await this.readWithInheritance(verbPath);
-                        if (verb && verb.sourceId === sourceId) {
-                            // v5.4.0: Use proper path helper instead of string replacement
-                            const metadataPath = getVerbMetadataPath(type, verb.id);
-                            const metadata = await this.readWithInheritance(metadataPath);
-                            // v5.4.0: Extract standard fields from metadata to top-level (like nouns)
-                            results.push({
-                                ...verb,
-                                weight: metadata?.weight,
-                                confidence: metadata?.confidence,
-                                createdAt: metadata?.createdAt
-                                    ? (typeof metadata.createdAt === 'number' ? metadata.createdAt : metadata.createdAt.seconds * 1000)
-                                    : Date.now(),
-                                updatedAt: metadata?.updatedAt
-                                    ? (typeof metadata.updatedAt === 'number' ? metadata.updatedAt : metadata.updatedAt.seconds * 1000)
-                                    : Date.now(),
-                                service: metadata?.service,
-                                createdBy: metadata?.createdBy,
-                                metadata: metadata || {}
-                            });
-                        }
-                    }
-                    catch (error) {
-                        // Skip verbs that fail to load
-                    }
+                const verb = await this.getVerb(verbId);
+                if (verb) {
+                    results.push(verb);
                 }
             }
             catch (error) {
-                // Skip types that have no data
+                // Skip verbs that fail to load (handles deleted/corrupted verbs gracefully)
             }
         }
+        const elapsed = performance.now() - startTime;
+        // Performance monitoring - should be 100-10,000x faster than old O(n) scan
+        if (elapsed > 50.0) {
+            prodLog.warn(`getVerbsBySource_internal: Slow query for ${sourceId} ` +
+                `(${verbIds.length} verbs, ${elapsed.toFixed(2)}ms). ` +
+                `Expected <50ms with index optimization.`);
+        }
         return results;
     }
     /**
      * Get verbs by target (COW-aware implementation)
-     * v5.4.0: Fixed to directly list verb files instead of directories
+     * v5.7.0: BILLION-SCALE OPTIMIZATION - Use GraphAdjacencyIndex for O(log n) lookup
      */
     async getVerbsByTarget_internal(targetId) {
-        // v5.4.0: Type-first implementation - scan across all verb types
-        // COW-aware: uses readWithInheritance for each verb
+        // v5.7.0: BILLION-SCALE OPTIMIZATION - Use GraphAdjacencyIndex for O(log n) lookup
+        // Previous: O(total_verbs) - scanned all 127 verb types
+        // Now: O(log n) LSM-tree lookup + O(verbs_for_target) load
         await this.ensureInitialized();
+        const startTime = performance.now();
+        // Get GraphAdjacencyIndex (lazy-initialized)
+        const graphIndex = await this.getGraphIndex();
+        // O(log n) lookup with bloom filter optimization
+        const verbIds = await graphIndex.getVerbIdsByTarget(targetId);
+        // Load each verb by ID (uses existing optimized getVerb())
         const results = [];
-        // Iterate through all verb types
-        for (let i = 0; i < VERB_TYPE_COUNT; i++) {
-            const type = TypeUtils.getVerbFromIndex(i);
-            const typeDir = `entities/verbs/${type}/vectors`;
+        for (const verbId of verbIds) {
             try {
-                // v5.4.0 FIX: List all verb files directly (not shard directories)
-                // listObjectsInBranch returns full paths to .json files, not directories
-                const verbFiles = await this.listObjectsInBranch(typeDir);
-                for (const verbPath of verbFiles) {
-                    // Skip if not a .json file
-                    if (!verbPath.endsWith('.json'))
-                        continue;
-                    try {
-                        const verb = await this.readWithInheritance(verbPath);
-                        if (verb && verb.targetId === targetId) {
-                            // v5.4.0: Use proper path helper instead of string replacement
-                            const metadataPath = getVerbMetadataPath(type, verb.id);
-                            const metadata = await this.readWithInheritance(metadataPath);
-                            // v5.4.0: Extract standard fields from metadata to top-level (like nouns)
-                            results.push({
-                                ...verb,
-                                weight: metadata?.weight,
-                                confidence: metadata?.confidence,
-                                createdAt: metadata?.createdAt
-                                    ? (typeof metadata.createdAt === 'number' ? metadata.createdAt : metadata.createdAt.seconds * 1000)
-                                    : Date.now(),
-                                updatedAt: metadata?.updatedAt
-                                    ? (typeof metadata.updatedAt === 'number' ? metadata.updatedAt : metadata.updatedAt.seconds * 1000)
-                                    : Date.now(),
-                                service: metadata?.service,
-                                createdBy: metadata?.createdBy,
-                                metadata: metadata || {}
-                            });
-                        }
-                    }
-                    catch (error) {
-                        // Skip verbs that fail to load
-                    }
+                const verb = await this.getVerb(verbId);
+                if (verb) {
+                    results.push(verb);
                 }
             }
             catch (error) {
-                // Skip types that have no data
+                // Skip verbs that fail to load (handles deleted/corrupted verbs gracefully)
             }
         }
+        const elapsed = performance.now() - startTime;
+        // Performance monitoring - should be 100-10,000x faster than old O(n) scan
+        if (elapsed > 50.0) {
+            prodLog.warn(`getVerbsByTarget_internal: Slow query for ${targetId} ` +
+                `(${verbIds.length} verbs, ${elapsed.toFixed(2)}ms). ` +
+                `Expected <50ms with index optimization.`);
+        }
         return results;
     }
     /**
@@ -1980,7 +1985,7 @@ export class BaseStorage extends BaseStorageAdapter {
                 verbs.push(verbWithMetadata);
             }
             catch (error) {
-                console.warn(`[BaseStorage] Failed to load verb from ${path}:`, error);
+                prodLog.warn(`[BaseStorage] Failed to load verb from ${path}:`, error);
             }
         }
         return verbs;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@soulcraft/brainy",
-  "version": "5.6.2",
+  "version": "5.7.0",
   "description": "Universal Knowledge Protocol™ - World's first Triple Intelligence database unifying vector, graph, and document search in one API. Stage 3 CANONICAL: 42 nouns × 127 verbs covering 96-97% of all human knowledge.",
   "main": "dist/index.js",
   "module": "dist/index.js",