@soulcraft/brainy 3.8.3 → 3.9.0

package/dist/storage/adapters/baseStorageAdapter.js

@@ -3,6 +3,7 @@
  * Provides common functionality for all storage adapters, including statistics tracking
  */
 import { extractFieldNamesFromJson, mapToStandardField } from '../../utils/fieldNameTracking.js';
+import { getGlobalMutex } from '../../utils/mutex.js';
 /**
  * Base class for storage adapters that implements statistics tracking
  */
@@ -37,6 +38,16 @@ export class BaseStorageAdapter {
         this.totalDelayMs = 0;
         // Service-level throttling
         this.serviceThrottling = new Map();
+        // =============================================
+        // Universal O(1) Count Management
+        // =============================================
+        // Universal count tracking - O(1) operations
+        this.totalNounCount = 0;
+        this.totalVerbCount = 0;
+        this.entityCounts = new Map(); // type -> count
+        this.verbCounts = new Map(); // verb type -> count
+        this.countCache = new Map();
+        this.COUNT_CACHE_TTL = 60000; // 1 minute cache TTL
     }
     /**
      * Save statistics data
@@ -609,5 +620,131 @@ export class BaseStorageAdapter {
         }
         return stats;
     }
+    /**
+     * Get total noun count - O(1) operation
+     * @returns Promise that resolves to the total number of nouns
+     */
+    async getNounCount() {
+        return this.totalNounCount;
+    }
+    /**
+     * Get total verb count - O(1) operation
+     * @returns Promise that resolves to the total number of verbs
+     */
+    async getVerbCount() {
+        return this.totalVerbCount;
+    }
+    /**
+     * Increment count for entity type - O(1) operation
+     * Protected by storage-specific mechanisms (mutex, distributed consensus, etc.)
+     * @param type The entity type
+     */
+    incrementEntityCount(type) {
+        // For distributed scenarios, this is aggregated across shards
+        // For single-node, this is protected by storage-specific locking
+        this.entityCounts.set(type, (this.entityCounts.get(type) || 0) + 1);
+        this.totalNounCount++;
+        // Update cache
+        this.countCache.set('nouns_count', {
+            count: this.totalNounCount,
+            timestamp: Date.now()
+        });
+    }
+    /**
+     * Thread-safe increment for concurrent scenarios
+     * Uses mutex for single-node, distributed consensus for multi-node
+     */
+    async incrementEntityCountSafe(type) {
+        // Single-node mutex protection (distributed mode handled by coordinator)
+        const mutex = getGlobalMutex();
+        await mutex.runExclusive(`count-entity-${type}`, async () => {
+            this.incrementEntityCount(type);
+            // Persist counts periodically
+            if (this.totalNounCount % 10 === 0) {
+                await this.persistCounts();
+            }
+        });
+    }
+    /**
+     * Decrement count for entity type - O(1) operation
+     * @param type The entity type
+     */
+    decrementEntityCount(type) {
+        const current = this.entityCounts.get(type) || 0;
+        if (current > 1) {
+            this.entityCounts.set(type, current - 1);
+        }
+        else {
+            this.entityCounts.delete(type);
+        }
+        if (this.totalNounCount > 0) {
+            this.totalNounCount--;
+        }
+        // Update cache
+        this.countCache.set('nouns_count', {
+            count: this.totalNounCount,
+            timestamp: Date.now()
+        });
+    }
+    /**
+     * Thread-safe decrement for concurrent scenarios
+     */
+    async decrementEntityCountSafe(type) {
+        const mutex = getGlobalMutex();
+        await mutex.runExclusive(`count-entity-${type}`, async () => {
+            this.decrementEntityCount(type);
+            if (this.totalNounCount % 10 === 0) {
+                await this.persistCounts();
+            }
+        });
+    }
+    /**
+     * Increment verb count - O(1) operation with mutex protection
+     * @param type The verb type
+     */
+    async incrementVerbCount(type) {
+        const mutex = getGlobalMutex();
+        await mutex.runExclusive(`count-verb-${type}`, async () => {
+            this.verbCounts.set(type, (this.verbCounts.get(type) || 0) + 1);
+            this.totalVerbCount++;
+            // Update cache
+            this.countCache.set('verbs_count', {
+                count: this.totalVerbCount,
+                timestamp: Date.now()
+            });
+            // Persist counts immediately for consistency
+            if (this.totalVerbCount % 10 === 0) {
+                await this.persistCounts();
+            }
+        });
+    }
+    /**
+     * Decrement verb count - O(1) operation with mutex protection
+     * @param type The verb type
+     */
+    async decrementVerbCount(type) {
+        const mutex = getGlobalMutex();
+        await mutex.runExclusive(`count-verb-${type}`, async () => {
+            const current = this.verbCounts.get(type) || 0;
+            if (current > 1) {
+                this.verbCounts.set(type, current - 1);
+            }
+            else {
+                this.verbCounts.delete(type);
+            }
+            if (this.totalVerbCount > 0) {
+                this.totalVerbCount--;
+            }
+            // Update cache
+            this.countCache.set('verbs_count', {
+                count: this.totalVerbCount,
+                timestamp: Date.now()
+            });
+            // Persist counts immediately for consistency
+            if (this.totalVerbCount % 10 === 0) {
+                await this.persistCounts();
+            }
+        });
+    }
 }
 //# sourceMappingURL=baseStorageAdapter.js.map

package/dist/storage/adapters/fileSystemStorage.d.ts

@@ -11,6 +11,10 @@ type Edge = HNSWVerb;
  * Uses the file system to store data in the specified directory structure
  */
 export declare class FileSystemStorage extends BaseStorage {
+    private countsFilePath?;
+    private readonly shardingDepth;
+    private readonly SHARDING_THRESHOLD;
+    private cachedShardingDepth?;
     private rootDir;
     private nounsDir;
     private verbsDir;
@@ -22,6 +26,8 @@ export declare class FileSystemStorage extends BaseStorage {
     private lockDir;
     private useDualWrite;
     private activeLocks;
+    private lockTimers;
+    private allTimers;
     /**
      * Initialize the storage adapter
      * @param rootDirectory The root directory for storage
@@ -251,5 +257,40 @@ export declare class FileSystemStorage extends BaseStorage {
      * Merge statistics from multiple sources
      */
     private mergeStatistics;
+    /**
+     * Initialize counts from filesystem storage
+     */
+    protected initializeCounts(): Promise<void>;
+    /**
+     * Initialize counts by scanning disk (only done once)
+     */
+    private initializeCountsFromDisk;
+    /**
+     * Persist counts to filesystem storage
+     */
+    protected persistCounts(): Promise<void>;
+    /**
+     * Determine optimal sharding depth based on dataset size
+     * This is called once during initialization for consistent behavior
+     */
+    private getOptimalShardingDepth;
+    /**
+     * Get the path for a node with consistent sharding strategy
+     * Clean, predictable path generation
+     */
+    private getNodePath;
+    /**
+     * Get the path for a verb with consistent sharding strategy
+     */
+    private getVerbPath;
+    /**
+     * Universal sharded path generator
+     * Consistent across all entity types
+     */
+    private getShardedPath;
+    /**
+     * Check if a file exists (handles both sharded and non-sharded)
+     */
+    private fileExists;
 }
 export {};

package/dist/storage/adapters/fileSystemStorage.js

@@ -37,8 +37,13 @@ export class FileSystemStorage extends BaseStorage {
      */
     constructor(rootDirectory) {
         super();
+        // Intelligent sharding configuration
+        this.shardingDepth = 2; // 0=flat, 1=ab/, 2=ab/cd/
+        this.SHARDING_THRESHOLD = 1000; // Enable deep sharding at 1k files
         this.useDualWrite = true; // Write to both locations during migration
         this.activeLocks = new Set();
+        this.lockTimers = new Map(); // Track timers for cleanup
+        this.allTimers = new Set(); // Track all timers for cleanup
         this.rootDir = rootDirectory;
         // Defer path operations until init() when path module is guaranteed to be loaded
     }
@@ -92,6 +97,14 @@ export class FileSystemStorage extends BaseStorage {
             }
             // Create the locks directory if it doesn't exist
             await this.ensureDirectoryExists(this.lockDir);
+            // Initialize count management
+            this.countsFilePath = path.join(this.systemDir, 'counts.json');
+            await this.initializeCounts();
+            // Cache sharding depth for consistency during this session
+            this.cachedShardingDepth = this.getOptimalShardingDepth();
+            // Log sharding strategy for transparency
+            const strategy = this.cachedShardingDepth === 0 ? 'flat' : this.cachedShardingDepth === 1 ? 'single-level' : 'deep';
+            console.log(`📁 Using ${strategy} sharding for optimal performance (${this.totalNounCount} items)`);
             this.isInitialized = true;
         }
         catch (error) {
@@ -130,20 +143,33 @@ export class FileSystemStorage extends BaseStorage {
      */
     async saveNode(node) {
         await this.ensureInitialized();
+        // Check if this is a new node to update counts
+        const isNew = !(await this.fileExists(this.getNodePath(node.id)));
         // Convert connections Map to a serializable format
         const serializableNode = {
             ...node,
             connections: this.mapToObject(node.connections, (set) => Array.from(set))
         };
-        const filePath = path.join(this.nounsDir, `${node.id}.json`);
+        const filePath = this.getNodePath(node.id);
+        await this.ensureDirectoryExists(path.dirname(filePath));
         await fs.promises.writeFile(filePath, JSON.stringify(serializableNode, null, 2));
+        // Update counts for new nodes (intelligent type detection)
+        if (isNew) {
+            const type = node.metadata?.type || node.metadata?.nounType || 'default';
+            this.incrementEntityCount(type);
+            // Persist counts periodically (every 10 operations for efficiency)
+            if (this.totalNounCount % 10 === 0) {
+                await this.persistCounts();
+            }
+        }
     }
     /**
      * Get a node from storage
      */
     async getNode(id) {
         await this.ensureInitialized();
-        const filePath = path.join(this.nounsDir, `${id}.json`);
+        // Clean, predictable path - no backward compatibility needed
+        const filePath = this.getNodePath(id);
         try {
             const data = await fs.promises.readFile(filePath, 'utf-8');
             const parsedNode = JSON.parse(data);
@@ -246,9 +272,24 @@ export class FileSystemStorage extends BaseStorage {
      */
     async deleteNode(id) {
         await this.ensureInitialized();
-        const filePath = path.join(this.nounsDir, `${id}.json`);
+        const filePath = this.getNodePath(id);
+        // Load node to get type for count update
+        try {
+            const node = await this.getNode(id);
+            if (node) {
+                const type = node.metadata?.type || node.metadata?.nounType || 'default';
+                this.decrementEntityCount(type);
+            }
+        }
+        catch {
+            // Node might not exist, that's ok
+        }
         try {
             await fs.promises.unlink(filePath);
+            // Persist counts periodically
+            if (this.totalNounCount % 10 === 0) {
+                await this.persistCounts();
+            }
         }
         catch (error) {
             if (error.code !== 'ENOENT') {
@@ -267,7 +308,8 @@ export class FileSystemStorage extends BaseStorage {
             ...edge,
             connections: this.mapToObject(edge.connections, (set) => Array.from(set))
         };
-        const filePath = path.join(this.verbsDir, `${edge.id}.json`);
+        const filePath = this.getVerbPath(edge.id);
+        await this.ensureDirectoryExists(path.dirname(filePath));
         await fs.promises.writeFile(filePath, JSON.stringify(serializableEdge, null, 2));
     }
     /**
@@ -275,7 +317,7 @@ export class FileSystemStorage extends BaseStorage {
      */
     async getEdge(id) {
         await this.ensureInitialized();
-        const filePath = path.join(this.verbsDir, `${id}.json`);
+        const filePath = this.getVerbPath(id);
         try {
             const data = await fs.promises.readFile(filePath, 'utf-8');
             const parsedEdge = JSON.parse(data);
@@ -494,9 +536,15 @@ export class FileSystemStorage extends BaseStorage {
             const nounFiles = files.filter((f) => f.endsWith('.json'));
             // Sort for consistent pagination
             nounFiles.sort();
-            // Find starting position
+            // Find starting position - prioritize offset for O(1) operation
             let startIndex = 0;
-            if (cursor) {
+            const offset = options.offset; // Cast to any since offset might not be in type
+            if (offset !== undefined) {
+                // Direct offset - O(1) operation
+                startIndex = offset;
+            }
+            else if (cursor) {
+                // Cursor-based pagination
                 startIndex = nounFiles.findIndex((f) => f.replace('.json', '') > cursor);
                 if (startIndex === -1)
                     startIndex = nounFiles.length;
@@ -507,18 +555,10 @@ export class FileSystemStorage extends BaseStorage {
             const items = [];
             let successfullyLoaded = 0;
             let totalValidFiles = 0;
-            // First pass: count total valid files (for accurate totalCount)
-            // This is necessary to fix the pagination bug
-            for (const file of nounFiles) {
-                try {
-                    // Just check if file exists and is readable
-                    await fs.promises.access(path.join(this.nounsDir, file), fs.constants.R_OK);
-                    totalValidFiles++;
-                }
-                catch {
-                    // File not readable, skip
-                }
-            }
+            // Use persisted counts - O(1) operation!
+            totalValidFiles = this.totalNounCount;
+            // No need to count files anymore - we maintain accurate counts
+            // This eliminates the O(n) operation completely
             // Second pass: load the current page
             for (const file of pageFiles) {
                 try {
@@ -1253,5 +1293,173 @@ export class FileSystemStorage extends BaseStorage {
             lastUpdated: new Date().toISOString()
         };
     }
+    // =============================================
+    // Count Management for O(1) Scalability
+    // =============================================
+    /**
+     * Initialize counts from filesystem storage
+     */
+    async initializeCounts() {
+        if (!this.countsFilePath)
+            return;
+        try {
+            if (await this.fileExists(this.countsFilePath)) {
+                const data = await fs.promises.readFile(this.countsFilePath, 'utf-8');
+                const counts = JSON.parse(data);
+                // Restore entity counts
+                this.entityCounts = new Map(Object.entries(counts.entityCounts || {}));
+                this.verbCounts = new Map(Object.entries(counts.verbCounts || {}));
+                this.totalNounCount = counts.totalNounCount || 0;
+                this.totalVerbCount = counts.totalVerbCount || 0;
+                // Also populate the cache for backward compatibility
+                this.countCache.set('nouns_count', {
+                    count: this.totalNounCount,
+                    timestamp: Date.now()
+                });
+                this.countCache.set('verbs_count', {
+                    count: this.totalVerbCount,
+                    timestamp: Date.now()
+                });
+            }
+            else {
+                // If no counts file exists, do one initial count
+                await this.initializeCountsFromDisk();
+            }
+        }
+        catch (error) {
+            console.warn('Could not load persisted counts, will initialize from disk:', error);
+            await this.initializeCountsFromDisk();
+        }
+    }
+    /**
+     * Initialize counts by scanning disk (only done once)
+     */
+    async initializeCountsFromDisk() {
+        try {
+            // Count nouns
+            const nounFiles = await fs.promises.readdir(this.nounsDir);
+            const validNounFiles = nounFiles.filter((f) => f.endsWith('.json'));
+            this.totalNounCount = validNounFiles.length;
+            // Count verbs
+            const verbFiles = await fs.promises.readdir(this.verbsDir);
+            const validVerbFiles = verbFiles.filter((f) => f.endsWith('.json'));
+            this.totalVerbCount = validVerbFiles.length;
+            // Sample some files to get type distribution (don't read all)
+            const sampleSize = Math.min(100, validNounFiles.length);
+            for (let i = 0; i < sampleSize; i++) {
+                try {
+                    const file = validNounFiles[i];
+                    const data = await fs.promises.readFile(path.join(this.nounsDir, file), 'utf-8');
+                    const noun = JSON.parse(data);
+                    const type = noun.metadata?.type || noun.metadata?.nounType || 'default';
+                    this.entityCounts.set(type, (this.entityCounts.get(type) || 0) + 1);
+                }
+                catch {
+                    // Skip invalid files
+                }
+            }
+            // Extrapolate counts if we sampled
+            if (sampleSize < this.totalNounCount && sampleSize > 0) {
+                const multiplier = this.totalNounCount / sampleSize;
+                for (const [type, count] of this.entityCounts.entries()) {
+                    this.entityCounts.set(type, Math.round(count * multiplier));
+                }
+            }
+            await this.persistCounts();
+        }
+        catch (error) {
+            console.error('Error initializing counts from disk:', error);
+        }
+    }
+    /**
+     * Persist counts to filesystem storage
+     */
+    async persistCounts() {
+        if (!this.countsFilePath)
+            return;
+        try {
+            const counts = {
+                entityCounts: Object.fromEntries(this.entityCounts),
+                verbCounts: Object.fromEntries(this.verbCounts),
+                totalNounCount: this.totalNounCount,
+                totalVerbCount: this.totalVerbCount,
+                lastUpdated: new Date().toISOString()
+            };
+            await fs.promises.writeFile(this.countsFilePath, JSON.stringify(counts, null, 2));
+        }
+        catch (error) {
+            console.error('Error persisting counts:', error);
+        }
+    }
+    // =============================================
+    // Intelligent Directory Sharding
+    // =============================================
+    /**
+     * Determine optimal sharding depth based on dataset size
+     * This is called once during initialization for consistent behavior
+     */
+    getOptimalShardingDepth() {
+        // For new installations, use intelligent defaults
+        if (this.totalNounCount === 0 && this.totalVerbCount === 0) {
+            return 1; // Default to single-level sharding for new installs
+        }
+        const maxCount = Math.max(this.totalNounCount, this.totalVerbCount);
+        if (maxCount >= this.SHARDING_THRESHOLD) {
+            return 2; // Deep sharding for large datasets
+        }
+        else if (maxCount >= 100) {
+            return 1; // Single-level sharding for medium datasets
+        }
+        else {
+            return 1; // Always use at least single-level sharding for consistency
+        }
+    }
+    /**
+     * Get the path for a node with consistent sharding strategy
+     * Clean, predictable path generation
+     */
+    getNodePath(id) {
+        return this.getShardedPath(this.nounsDir, id);
+    }
+    /**
+     * Get the path for a verb with consistent sharding strategy
+     */
+    getVerbPath(id) {
+        return this.getShardedPath(this.verbsDir, id);
+    }
+    /**
+     * Universal sharded path generator
+     * Consistent across all entity types
+     */
+    getShardedPath(baseDir, id) {
+        const depth = this.cachedShardingDepth ?? this.getOptimalShardingDepth();
+        switch (depth) {
+            case 0:
+                // Flat structure: /nouns/uuid.json
+                return path.join(baseDir, `${id}.json`);
+            case 1:
+                // Single-level sharding: /nouns/ab/uuid.json
+                const shard1 = id.substring(0, 2);
+                return path.join(baseDir, shard1, `${id}.json`);
+            case 2:
+            default:
+                // Deep sharding: /nouns/ab/cd/uuid.json
+                const shard1Deep = id.substring(0, 2);
+                const shard2Deep = id.substring(2, 4);
+                return path.join(baseDir, shard1Deep, shard2Deep, `${id}.json`);
+        }
+    }
+    /**
+     * Check if a file exists (handles both sharded and non-sharded)
+     */
+    async fileExists(filePath) {
+        try {
+            await fs.promises.access(filePath, fs.constants.F_OK);
+            return true;
+        }
+        catch {
+            return false;
+        }
+    }
 }
 //# sourceMappingURL=fileSystemStorage.js.map

package/dist/storage/adapters/memoryStorage.d.ts

@@ -169,4 +169,12 @@ export declare class MemoryStorage extends BaseStorage {
      * @returns Promise that resolves to the statistics data or null if not found
      */
     protected getStatisticsData(): Promise<StatisticsData | null>;
+    /**
+     * Initialize counts from in-memory storage - O(1) operation
+     */
+    protected initializeCounts(): Promise<void>;
+    /**
+     * Persist counts to storage - no-op for memory storage
+     */
+    protected persistCounts(): Promise<void>;
 }

package/dist/storage/adapters/memoryStorage.js

@@ -30,6 +30,7 @@ export class MemoryStorage extends BaseStorage {
      * Save a noun to storage
      */
     async saveNoun_internal(noun) {
+        const isNew = !this.nouns.has(noun.id);
         // Create a deep copy to avoid reference issues
         const nounCopy = {
             id: noun.id,
@@ -44,6 +45,11 @@ export class MemoryStorage extends BaseStorage {
         }
         // Save the noun directly in the nouns map
         this.nouns.set(noun.id, nounCopy);
+        // Update counts for new entities
+        if (isNew) {
+            const type = noun.metadata?.type || noun.metadata?.nounType || 'default';
+            this.incrementEntityCount(type);
+        }
     }
     /**
      * Get a noun from storage
@@ -190,12 +196,18 @@ export class MemoryStorage extends BaseStorage {
      * Delete a noun from storage
      */
     async deleteNoun_internal(id) {
+        const noun = this.nouns.get(id);
+        if (noun) {
+            const type = noun.metadata?.type || noun.metadata?.nounType || 'default';
+            this.decrementEntityCount(type);
+        }
         this.nouns.delete(id);
     }
     /**
      * Save a verb to storage
      */
     async saveVerb_internal(verb) {
+        const isNew = !this.verbs.has(verb.id);
         // Create a deep copy to avoid reference issues
         const verbCopy = {
             id: verb.id,
@@ -208,6 +220,8 @@ export class MemoryStorage extends BaseStorage {
         }
         // Save the verb directly in the verbs map
         this.verbs.set(verb.id, verbCopy);
+        // Count tracking will be handled in saveVerbMetadata_internal
+        // since HNSWVerb doesn't contain type information
     }
     /**
      * Get a verb from storage
@@ -393,7 +407,8 @@ export class MemoryStorage extends BaseStorage {
      * Delete a verb from storage
      */
     async deleteVerb_internal(id) {
-        // Delete the verb directly from the verbs map
+        // Count tracking will be handled when verb metadata is deleted
+        // since HNSWVerb doesn't contain type information
         this.verbs.delete(id);
     }
     /**
@@ -448,7 +463,13 @@ export class MemoryStorage extends BaseStorage {
      * Save verb metadata to storage (internal implementation)
      */
     async saveVerbMetadata_internal(id, metadata) {
+        const isNew = !this.verbMetadata.has(id);
         this.verbMetadata.set(id, JSON.parse(JSON.stringify(metadata)));
+        // Update counts for new verbs
+        if (isNew) {
+            const type = metadata?.verb || metadata?.type || 'default';
+            this.incrementVerbCount(type);
+        }
     }
     /**
      * Get verb metadata from storage
@@ -549,5 +570,31 @@ export class MemoryStorage extends BaseStorage {
         // Since this is in-memory, there's no need for fallback mechanisms
         // to check multiple storage locations
     }
+    /**
+     * Initialize counts from in-memory storage - O(1) operation
+     */
+    async initializeCounts() {
+        // For memory storage, initialize counts from current in-memory state
+        this.totalNounCount = this.nouns.size;
+        this.totalVerbCount = this.verbMetadata.size;
+        // Initialize type-based counts by scanning current data
+        this.entityCounts.clear();
+        this.verbCounts.clear();
+        for (const noun of this.nouns.values()) {
+            const type = noun.metadata?.type || noun.metadata?.nounType || 'default';
+            this.entityCounts.set(type, (this.entityCounts.get(type) || 0) + 1);
+        }
+        for (const verbMetadata of this.verbMetadata.values()) {
+            const type = verbMetadata?.verb || verbMetadata?.type || 'default';
+            this.verbCounts.set(type, (this.verbCounts.get(type) || 0) + 1);
+        }
+    }
+    /**
+     * Persist counts to storage - no-op for memory storage
+     */
+    async persistCounts() {
+        // No persistence needed for in-memory storage
+        // Counts are always accurate from the live data structures
+    }
 }
 //# sourceMappingURL=memoryStorage.js.map

package/dist/storage/adapters/opfsStorage.d.ts

@@ -254,5 +254,17 @@ export declare class OPFSStorage extends BaseStorage {
         hasMore: boolean;
         nextCursor?: string;
     }>;
+    /**
+     * Initialize counts from OPFS storage
+     */
+    protected initializeCounts(): Promise<void>;
+    /**
+     * Initialize counts by scanning OPFS (fallback for missing counts file)
+     */
+    private initializeCountsFromScan;
+    /**
+     * Persist counts to OPFS storage
+     */
+    protected persistCounts(): Promise<void>;
 }
 export {};