@soulcraft/brainy 3.8.3 → 3.9.1

package/dist/neural/improvedNeuralAPI.js

@@ -80,8 +80,8 @@ export class ImprovedNeuralAPI {
         catch (error) {
             const errorMessage = error instanceof Error ? error.message : String(error);
             throw new SimilarityError(`Failed to calculate similarity: ${errorMessage}`, {
-                inputA: typeof a === 'object' ? 'vector' : String(a).substring(0, 50),
-                inputB: typeof b === 'object' ? 'vector' : String(b).substring(0, 50),
+                inputA: Array.isArray(a) ? 'vector' : typeof a === 'string' ? a.substring(0, 50) : 'unknown',
+                inputB: Array.isArray(b) ? 'vector' : typeof b === 'string' ? b.substring(0, 50) : 'unknown',
                 options
             });
         }
@@ -1172,8 +1172,8 @@ export class ImprovedNeuralAPI {
     // Utility methods for internal operations
     _isId(value) {
         return typeof value === 'string' &&
-            (value.length === 36 && value.includes('-')) || // UUID-like
-            (value.length > 10 && !value.includes(' ')); // ID-like string
+            ((value.length === 36 && value.includes('-')) || // UUID-like
+                (value.length > 10 && !value.includes(' '))); // ID-like string
     }
     _isVector(value) {
         return Array.isArray(value) &&
@@ -1441,28 +1441,67 @@ export class ImprovedNeuralAPI {
         }
         return groups;
     }
-    // Placeholder implementations for complex operations
-    async _getAllItemIds() {
-        // Get all noun IDs from the brain
-        // Get total item count using find with empty query
-        const allItems = await this.brain.find({ query: '', limit: Number.MAX_SAFE_INTEGER });
-        const stats = { totalNouns: allItems.length || 0 };
-        if (!stats.totalNouns || stats.totalNouns === 0) {
-            return [];
+    // Iterator-based implementations for scalability
+    /**
+     * Iterate through all items without loading them all at once
+     * This scales to millions of items without memory issues
+     */
+    async *_iterateAllItems(options) {
+        const batchSize = options?.batchSize || 1000;
+        let cursor;
+        let hasMore = true;
+        while (hasMore) {
+            const result = await this.brain.find({
+                query: '',
+                limit: batchSize,
+                cursor
+            });
+            for (const item of result.items || result) {
+                yield item;
+            }
+            hasMore = result.hasMore || false;
+            cursor = result.nextCursor;
+            // Safety check to prevent infinite loops
+            if (!result.items || result.items.length === 0) {
+                break;
+            }
         }
-        // Get nouns with pagination (limit to 10000 for performance)
-        const limit = Math.min(stats.totalNouns, 10000);
+    }
+    /**
+     * Get a sample of item IDs for operations that don't need all items
+     * This is O(1) for small samples
+     */
+    async _getSampleItemIds(sampleSize = 1000) {
         const result = await this.brain.find({
             query: '',
-            limit
+            limit: Math.min(sampleSize, 10000) // Cap at 10k for safety
         });
-        return result.map((item) => item.id).filter((id) => id);
+        const items = result.items || result;
+        return items.map((item) => item.entity?.id || item.id).filter((id) => id);
     }
+    /**
+     * Get total count using the brain's O(1) counting API
+     */
     async _getTotalItemCount() {
-        // Get total item count using find with empty query
-        const allItems = await this.brain.find({ query: '', limit: Number.MAX_SAFE_INTEGER });
-        const stats = { totalNouns: allItems.length || 0 };
-        return stats.totalNouns || 0;
+        // Use the brain's O(1) counting API if available
+        if (this.brain.counts && typeof this.brain.counts.entities === 'function') {
+            return await this.brain.counts.entities();
+        }
+        // Fallback: Get from storage statistics
+        const storage = this.brain.storage;
+        if (storage && typeof storage.getStatistics === 'function') {
+            const stats = await storage.getStatistics();
+            return stats?.totalNodes || 0;
+        }
+        // Last resort: Sample and estimate
+        const sample = await this.brain.find({ query: '', limit: 1 });
+        return sample.totalCount || 0;
+    }
+    // Deprecated: Remove methods that load everything
+    // These are kept for backward compatibility but should not be used
+    async _getAllItemIds() {
+        console.warn('⚠️ _getAllItemIds() is deprecated and will fail with large datasets. Use _iterateAllItems() or _getSampleItemIds() instead.');
+        return this._getSampleItemIds(10000); // Return sample only
     }
     // ===== GRAPH ALGORITHM SUPPORTING METHODS =====
     _calculateTotalWeight(edges) {

package/dist/neural/naturalLanguageProcessorStatic.d.ts

@@ -47,6 +47,7 @@ export declare class NaturalLanguageProcessor {
     private buildFieldConstraints;
     /**
      * Find similar queries from history (without using Brainy)
+     * NOTE: Currently unused - reserved for future query caching optimization
      */
     private findSimilarQueries;
     /**

package/dist/neural/naturalLanguageProcessorStatic.js

@@ -119,10 +119,11 @@ export class NaturalLanguageProcessor {
     }
     /**
      * Find similar queries from history (without using Brainy)
+     * NOTE: Currently unused - reserved for future query caching optimization
      */
     findSimilarQueries(embedding) {
-        // Simple similarity check against recent history
-        // This is just a placeholder - real implementation would use cosine similarity
+        // Not implemented - not required for core functionality
+        // Would implement cosine similarity against queryHistory if needed
         return [];
     }
     /**

package/dist/storage/adapters/baseStorageAdapter.d.ts

@@ -253,4 +253,63 @@ export declare abstract class BaseStorageAdapter implements StorageAdapter {
      * Include throttling metrics in statistics
      */
     getStatisticsWithThrottling(): Promise<StatisticsData | null>;
+    protected totalNounCount: number;
+    protected totalVerbCount: number;
+    protected entityCounts: Map<string, number>;
+    protected verbCounts: Map<string, number>;
+    protected countCache: Map<string, {
+        count: number;
+        timestamp: number;
+    }>;
+    protected readonly COUNT_CACHE_TTL = 60000;
+    /**
+     * Get total noun count - O(1) operation
+     * @returns Promise that resolves to the total number of nouns
+     */
+    getNounCount(): Promise<number>;
+    /**
+     * Get total verb count - O(1) operation
+     * @returns Promise that resolves to the total number of verbs
+     */
+    getVerbCount(): Promise<number>;
+    /**
+     * Increment count for entity type - O(1) operation
+     * Protected by storage-specific mechanisms (mutex, distributed consensus, etc.)
+     * @param type The entity type
+     */
+    protected incrementEntityCount(type: string): void;
+    /**
+     * Thread-safe increment for concurrent scenarios
+     * Uses mutex for single-node, distributed consensus for multi-node
+     */
+    protected incrementEntityCountSafe(type: string): Promise<void>;
+    /**
+     * Decrement count for entity type - O(1) operation
+     * @param type The entity type
+     */
+    protected decrementEntityCount(type: string): void;
+    /**
+     * Thread-safe decrement for concurrent scenarios
+     */
+    protected decrementEntityCountSafe(type: string): Promise<void>;
+    /**
+     * Increment verb count - O(1) operation with mutex protection
+     * @param type The verb type
+     */
+    protected incrementVerbCount(type: string): Promise<void>;
+    /**
+     * Decrement verb count - O(1) operation with mutex protection
+     * @param type The verb type
+     */
+    protected decrementVerbCount(type: string): Promise<void>;
+    /**
+     * Initialize counts from storage - must be implemented by each adapter
+     * @protected
+     */
+    protected abstract initializeCounts(): Promise<void>;
+    /**
+     * Persist counts to storage - must be implemented by each adapter
+     * @protected
+     */
+    protected abstract persistCounts(): Promise<void>;
 }

package/dist/storage/adapters/baseStorageAdapter.js

@@ -3,6 +3,7 @@
  * Provides common functionality for all storage adapters, including statistics tracking
  */
 import { extractFieldNamesFromJson, mapToStandardField } from '../../utils/fieldNameTracking.js';
+import { getGlobalMutex } from '../../utils/mutex.js';
 /**
  * Base class for storage adapters that implements statistics tracking
  */
@@ -37,6 +38,16 @@ export class BaseStorageAdapter {
         this.totalDelayMs = 0;
         // Service-level throttling
         this.serviceThrottling = new Map();
+        // =============================================
+        // Universal O(1) Count Management
+        // =============================================
+        // Universal count tracking - O(1) operations
+        this.totalNounCount = 0;
+        this.totalVerbCount = 0;
+        this.entityCounts = new Map(); // type -> count
+        this.verbCounts = new Map(); // verb type -> count
+        this.countCache = new Map();
+        this.COUNT_CACHE_TTL = 60000; // 1 minute cache TTL
     }
     /**
      * Save statistics data
@@ -609,5 +620,131 @@ export class BaseStorageAdapter {
         }
         return stats;
     }
+    /**
+     * Get total noun count - O(1) operation
+     * @returns Promise that resolves to the total number of nouns
+     */
+    async getNounCount() {
+        return this.totalNounCount;
+    }
+    /**
+     * Get total verb count - O(1) operation
+     * @returns Promise that resolves to the total number of verbs
+     */
+    async getVerbCount() {
+        return this.totalVerbCount;
+    }
+    /**
+     * Increment count for entity type - O(1) operation
+     * Protected by storage-specific mechanisms (mutex, distributed consensus, etc.)
+     * @param type The entity type
+     */
+    incrementEntityCount(type) {
+        // For distributed scenarios, this is aggregated across shards
+        // For single-node, this is protected by storage-specific locking
+        this.entityCounts.set(type, (this.entityCounts.get(type) || 0) + 1);
+        this.totalNounCount++;
+        // Update cache
+        this.countCache.set('nouns_count', {
+            count: this.totalNounCount,
+            timestamp: Date.now()
+        });
+    }
+    /**
+     * Thread-safe increment for concurrent scenarios
+     * Uses mutex for single-node, distributed consensus for multi-node
+     */
+    async incrementEntityCountSafe(type) {
+        // Single-node mutex protection (distributed mode handled by coordinator)
+        const mutex = getGlobalMutex();
+        await mutex.runExclusive(`count-entity-${type}`, async () => {
+            this.incrementEntityCount(type);
+            // Persist counts periodically
+            if (this.totalNounCount % 10 === 0) {
+                await this.persistCounts();
+            }
+        });
+    }
+    /**
+     * Decrement count for entity type - O(1) operation
+     * @param type The entity type
+     */
+    decrementEntityCount(type) {
+        const current = this.entityCounts.get(type) || 0;
+        if (current > 1) {
+            this.entityCounts.set(type, current - 1);
+        }
+        else {
+            this.entityCounts.delete(type);
+        }
+        if (this.totalNounCount > 0) {
+            this.totalNounCount--;
+        }
+        // Update cache
+        this.countCache.set('nouns_count', {
+            count: this.totalNounCount,
+            timestamp: Date.now()
+        });
+    }
+    /**
+     * Thread-safe decrement for concurrent scenarios
+     */
+    async decrementEntityCountSafe(type) {
+        const mutex = getGlobalMutex();
+        await mutex.runExclusive(`count-entity-${type}`, async () => {
+            this.decrementEntityCount(type);
+            if (this.totalNounCount % 10 === 0) {
+                await this.persistCounts();
+            }
+        });
+    }
+    /**
+     * Increment verb count - O(1) operation with mutex protection
+     * @param type The verb type
+     */
+    async incrementVerbCount(type) {
+        const mutex = getGlobalMutex();
+        await mutex.runExclusive(`count-verb-${type}`, async () => {
+            this.verbCounts.set(type, (this.verbCounts.get(type) || 0) + 1);
+            this.totalVerbCount++;
+            // Update cache
+            this.countCache.set('verbs_count', {
+                count: this.totalVerbCount,
+                timestamp: Date.now()
+            });
+            // Persist counts immediately for consistency
+            if (this.totalVerbCount % 10 === 0) {
+                await this.persistCounts();
+            }
+        });
+    }
+    /**
+     * Decrement verb count - O(1) operation with mutex protection
+     * @param type The verb type
+     */
+    async decrementVerbCount(type) {
+        const mutex = getGlobalMutex();
+        await mutex.runExclusive(`count-verb-${type}`, async () => {
+            const current = this.verbCounts.get(type) || 0;
+            if (current > 1) {
+                this.verbCounts.set(type, current - 1);
+            }
+            else {
+                this.verbCounts.delete(type);
+            }
+            if (this.totalVerbCount > 0) {
+                this.totalVerbCount--;
+            }
+            // Update cache
+            this.countCache.set('verbs_count', {
+                count: this.totalVerbCount,
+                timestamp: Date.now()
+            });
+            // Persist counts immediately for consistency
+            if (this.totalVerbCount % 10 === 0) {
+                await this.persistCounts();
+            }
+        });
+    }
 }
 //# sourceMappingURL=baseStorageAdapter.js.map

package/dist/storage/adapters/fileSystemStorage.d.ts

@@ -11,6 +11,10 @@ type Edge = HNSWVerb;
  * Uses the file system to store data in the specified directory structure
  */
 export declare class FileSystemStorage extends BaseStorage {
+    private countsFilePath?;
+    private readonly shardingDepth;
+    private readonly SHARDING_THRESHOLD;
+    private cachedShardingDepth?;
     private rootDir;
     private nounsDir;
     private verbsDir;
@@ -22,6 +26,8 @@ export declare class FileSystemStorage extends BaseStorage {
     private lockDir;
     private useDualWrite;
     private activeLocks;
+    private lockTimers;
+    private allTimers;
     /**
      * Initialize the storage adapter
      * @param rootDirectory The root directory for storage
@@ -251,5 +257,40 @@ export declare class FileSystemStorage extends BaseStorage {
      * Merge statistics from multiple sources
      */
     private mergeStatistics;
+    /**
+     * Initialize counts from filesystem storage
+     */
+    protected initializeCounts(): Promise<void>;
+    /**
+     * Initialize counts by scanning disk (only done once)
+     */
+    private initializeCountsFromDisk;
+    /**
+     * Persist counts to filesystem storage
+     */
+    protected persistCounts(): Promise<void>;
+    /**
+     * Determine optimal sharding depth based on dataset size
+     * This is called once during initialization for consistent behavior
+     */
+    private getOptimalShardingDepth;
+    /**
+     * Get the path for a node with consistent sharding strategy
+     * Clean, predictable path generation
+     */
+    private getNodePath;
+    /**
+     * Get the path for a verb with consistent sharding strategy
+     */
+    private getVerbPath;
+    /**
+     * Universal sharded path generator
+     * Consistent across all entity types
+     */
+    private getShardedPath;
+    /**
+     * Check if a file exists (handles both sharded and non-sharded)
+     */
+    private fileExists;
 }
 export {};