@soulcraft/brainy 3.23.1 → 3.24.0

package/CHANGELOG.md

@@ -2,6 +2,11 @@
 
 All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines.
 
+### [3.24.0](https://github.com/soulcraftlabs/brainy/compare/v3.23.1...v3.24.0) (2025-10-07)
+
+- feat: simplify sharding to fixed depth-1 for reliability and performance (87515b9)
+
+
 ### [3.23.0](https://github.com/soulcraftlabs/brainy/compare/v3.22.0...v3.23.0) (2025-10-04)
 
 - refactor: streamline core API surface

package/dist/storage/adapters/fileSystemStorage.d.ts

@@ -12,9 +12,9 @@ type Edge = HNSWVerb;
  */
 export declare class FileSystemStorage extends BaseStorage {
     private countsFilePath?;
-    private readonly shardingDepth;
-    private readonly SHARDING_THRESHOLD;
-    private cachedShardingDepth?;
+    private readonly SHARDING_DEPTH;
+    private readonly MAX_SHARDS;
+    private cachedShardingDepth;
     private rootDir;
     private nounsDir;
     private verbsDir;
@@ -270,8 +270,55 @@ export declare class FileSystemStorage extends BaseStorage {
      */
     protected persistCounts(): Promise<void>;
     /**
-     * Determine optimal sharding depth based on dataset size
-     * This is called once during initialization for consistent behavior
+     * Migrate files from one sharding depth to another
+     * Handles: 0→1 (flat to single-level), 2→1 (deep to single-level)
+     * Uses atomic file operations and comprehensive error handling
+     *
+     * @param fromDepth - Source sharding depth
+     * @param toDepth - Target sharding depth (must be 1)
+     */
+    private migrateShardingStructure;
+    /**
+     * Discover all files that need to be migrated
+     * Constructs correct oldPath based on source depth
+     */
+    private discoverFilesForMigration;
+    /**
+     * Get all files at a specific depth
+     */
+    private getAllFilesAtDepth;
+    /**
+     * Create all 256 shard directories (00-ff)
+     */
+    private createAllShardDirectories;
+    /**
+     * Migrate a single file atomically
+     */
+    private migrateFile;
+    /**
+     * Clean up empty directories after migration
+     */
+    private cleanupEmptyDirectories;
+    /**
+     * Count files in the current structure
+     */
+    private countFilesInStructure;
+    /**
+     * Detect the actual sharding depth used by existing files
+     * Examines directory structure to determine current sharding strategy
+     * Returns null if no files exist yet (new installation)
+     */
+    private detectExistingShardingDepth;
+    /**
+     * Get sharding depth
+     * Always returns 1 (single-level sharding) for optimal balance of
+     * simplicity, performance, and reliability across all dataset sizes
+     *
+     * Single-level sharding (depth=1):
+     * - 256 shard directories (00-ff)
+     * - Handles 2.5M+ entities with excellent performance
+     * - No dynamic depth changes = no path mismatch bugs
+     * - Industry standard approach (Git uses similar)
      */
     private getOptimalShardingDepth;
     /**
@@ -285,12 +332,17 @@ export declare class FileSystemStorage extends BaseStorage {
     private getVerbPath;
     /**
      * Universal sharded path generator
-     * Consistent across all entity types
+     * Always uses depth=1 (single-level sharding) for consistency
+     *
+     * Format: baseDir/ab/uuid.json
+     * Where 'ab' = first 2 hex characters of UUID (lowercase)
+     *
+     * Validates UUID format and throws descriptive errors
      */
     private getShardedPath;
     /**
-     * Get all JSON files from a sharded directory structure
-     * Properly traverses sharded subdirectories based on current sharding depth
+     * Get all JSON files from the single-level sharded directory structure
+     * Traverses all shard subdirectories (00-ff)
      */
     private getAllShardedFiles;
     /**
@@ -302,6 +354,11 @@ export declare class FileSystemStorage extends BaseStorage {
      * Stream through sharded files without loading all names into memory
      * Production-scale implementation for millions of files
      */
+    /**
+     * Stream through files in single-level sharded structure
+     * Calls processor for each file until processor returns false
+     * Returns true if more files exist (processor stopped early), false if all processed
+     */
     private streamShardedFiles;
     /**
      * Check if a file exists (handles both sharded and non-sharded)

package/dist/storage/adapters/fileSystemStorage.js

@@ -37,9 +37,15 @@ export class FileSystemStorage extends BaseStorage {
      */
     constructor(rootDirectory) {
         super();
-        // Intelligent sharding configuration
-        this.shardingDepth = 2; // 0=flat, 1=ab/, 2=ab/cd/
-        this.SHARDING_THRESHOLD = 100; // Enable deep sharding at 100 files for optimal performance
+        // Fixed sharding configuration for optimal balance of simplicity and performance
+        // Single-level sharding (depth=1) provides excellent performance for 1-2.5M entities
+        // Structure: nouns/ab/uuid.json where 'ab' = first 2 hex chars of UUID
+        // - 256 shard directories (00-ff)
+        // - Handles 2.5M+ entities with < 10K files per shard
+        // - Eliminates dynamic depth changes that cause path mismatch bugs
+        this.SHARDING_DEPTH = 1;
+        this.MAX_SHARDS = 256; // Hex range: 00-ff
+        this.cachedShardingDepth = this.SHARDING_DEPTH; // Always use fixed depth
         this.useDualWrite = true; // Write to both locations during migration
         this.activeLocks = new Set();
         this.lockTimers = new Map(); // Track timers for cleanup
@@ -100,11 +106,27 @@ export class FileSystemStorage extends BaseStorage {
             // Initialize count management
             this.countsFilePath = path.join(this.systemDir, 'counts.json');
             await this.initializeCounts();
-            // Cache sharding depth for consistency during this session
-            this.cachedShardingDepth = this.getOptimalShardingDepth();
-            // Log sharding strategy for transparency
-            const strategy = this.cachedShardingDepth === 0 ? 'flat' : this.cachedShardingDepth === 1 ? 'single-level' : 'deep';
-            console.log(`📁 Using ${strategy} sharding for optimal performance (${this.totalNounCount} items)`);
+            // Detect existing sharding structure and migrate if needed
+            const detectedDepth = await this.detectExistingShardingDepth();
+            if (detectedDepth !== null && detectedDepth !== this.SHARDING_DEPTH) {
+                // Migration needed: existing structure doesn't match our fixed depth
+                console.log(`📦 Brainy Storage Migration`);
+                console.log(`   Current structure: depth ${detectedDepth}`);
+                console.log(`   Target structure: depth ${this.SHARDING_DEPTH}`);
+                console.log(`   Entities to migrate: ${this.totalNounCount}`);
+                await this.migrateShardingStructure(detectedDepth, this.SHARDING_DEPTH);
+                console.log(`✅ Migration complete - now using depth ${this.SHARDING_DEPTH} sharding`);
+            }
+            else if (detectedDepth === null) {
+                // New installation
+                console.log(`📁 New installation: using depth ${this.SHARDING_DEPTH} sharding (optimal for 1-2.5M entities)`);
+            }
+            else {
+                // Already using correct depth
+                console.log(`📁 Using depth ${this.SHARDING_DEPTH} sharding (${this.totalNounCount} entities)`);
+            }
+            // Always use fixed depth after migration/detection
+            this.cachedShardingDepth = this.SHARDING_DEPTH;
             this.isInitialized = true;
         }
         catch (error) {
@@ -1372,11 +1394,15 @@ export class FileSystemStorage extends BaseStorage {
      */
     async initializeCountsFromDisk() {
         try {
-            // Count nouns (handles sharding properly)
-            const validNounFiles = await this.getAllShardedFiles(this.nounsDir);
+            // CRITICAL: Detect existing depth before counting
+            // Can't use getAllShardedFiles() which assumes depth=1
+            const existingDepth = await this.detectExistingShardingDepth();
+            const depthToUse = existingDepth !== null ? existingDepth : this.SHARDING_DEPTH;
+            // Count nouns using detected depth
+            const validNounFiles = await this.getAllFilesAtDepth(this.nounsDir, depthToUse);
             this.totalNounCount = validNounFiles.length;
-            // Count verbs (handles sharding properly)
-            const validVerbFiles = await this.getAllShardedFiles(this.verbsDir);
+            // Count verbs using detected depth
+            const validVerbFiles = await this.getAllFilesAtDepth(this.verbsDir, depthToUse);
             this.totalVerbCount = validVerbFiles.length;
             // Sample some files to get type distribution (don't read all)
             const sampleSize = Math.min(100, validNounFiles.length);
@@ -1384,7 +1410,22 @@ export class FileSystemStorage extends BaseStorage {
                 try {
                     const file = validNounFiles[i];
                     const id = file.replace('.json', '');
-                    const data = await fs.promises.readFile(this.getNodePath(id), 'utf-8');
+                    // Construct path using detected depth (not cached depth which may be wrong)
+                    let filePath;
+                    switch (depthToUse) {
+                        case 0:
+                            filePath = path.join(this.nounsDir, `${id}.json`);
+                            break;
+                        case 1:
+                            filePath = path.join(this.nounsDir, id.substring(0, 2), `${id}.json`);
+                            break;
+                        case 2:
+                            filePath = path.join(this.nounsDir, id.substring(0, 2), id.substring(2, 4), `${id}.json`);
+                            break;
+                        default:
+                            throw new Error(`Unsupported depth: ${depthToUse}`);
+                    }
+                    const data = await fs.promises.readFile(filePath, 'utf-8');
                     const noun = JSON.parse(data);
                     const type = noun.metadata?.type || noun.metadata?.nounType || 'default';
                     this.entityCounts.set(type, (this.entityCounts.get(type) || 0) + 1);
@@ -1430,154 +1471,457 @@ export class FileSystemStorage extends BaseStorage {
     // Intelligent Directory Sharding
     // =============================================
     /**
-     * Determine optimal sharding depth based on dataset size
-     * This is called once during initialization for consistent behavior
+     * Migrate files from one sharding depth to another
+     * Handles: 0→1 (flat to single-level), 2→1 (deep to single-level)
+     * Uses atomic file operations and comprehensive error handling
+     *
+     * @param fromDepth - Source sharding depth
+     * @param toDepth - Target sharding depth (must be 1)
      */
-    getOptimalShardingDepth() {
-        // For new installations, use intelligent defaults
-        if (this.totalNounCount === 0 && this.totalVerbCount === 0) {
-            return 1; // Default to single-level sharding for new installs
-        }
-        const maxCount = Math.max(this.totalNounCount, this.totalVerbCount);
-        if (maxCount >= this.SHARDING_THRESHOLD) {
-            return 2; // Deep sharding for large datasets
+    async migrateShardingStructure(fromDepth, toDepth) {
+        // Validation
+        if (fromDepth === toDepth) {
+            throw new Error(`Migration not needed: already at depth ${toDepth}`);
+        }
+        if (toDepth !== 1) {
+            throw new Error(`Migration only supports target depth 1 (got ${toDepth})`);
+        }
+        if (fromDepth !== 0 && fromDepth !== 2) {
+            throw new Error(`Migration only supports source depth 0 or 2 (got ${fromDepth})`);
+        }
+        // Create migration lock to prevent concurrent migrations
+        const lockFile = path.join(this.systemDir, '.migration-lock');
+        const lockExists = await this.fileExists(lockFile);
+        if (lockExists) {
+            // Check if lock is stale (> 1 hour old)
+            try {
+                const stats = await fs.promises.stat(lockFile);
+                const lockAge = Date.now() - stats.mtimeMs;
+                const ONE_HOUR = 60 * 60 * 1000;
+                if (lockAge < ONE_HOUR) {
+                    throw new Error('Migration already in progress. If this is incorrect, delete .migration-lock file.');
+                }
+                // Lock is stale, remove it
+                console.log('⚠️  Removing stale migration lock (> 1 hour old)');
+                await fs.promises.unlink(lockFile);
+            }
+            catch (error) {
+                if (error.code !== 'ENOENT') {
+                    throw error;
+                }
+            }
         }
-        else if (maxCount >= 100) {
-            return 1; // Single-level sharding for medium datasets
+        try {
+            // Create lock file
+            await fs.promises.writeFile(lockFile, JSON.stringify({
+                startedAt: new Date().toISOString(),
+                fromDepth,
+                toDepth,
+                pid: process.pid
+            }));
+            // Discover all files to migrate
+            console.log('📊 Discovering files to migrate...');
+            const filesToMigrate = await this.discoverFilesForMigration(fromDepth);
+            if (filesToMigrate.length === 0) {
+                console.log('ℹ️  No files to migrate');
+                return;
+            }
+            console.log(`📦 Migrating ${filesToMigrate.length} files...`);
+            // Create all target shard directories upfront
+            await this.createAllShardDirectories(this.nounsDir);
+            await this.createAllShardDirectories(this.verbsDir);
+            // Migrate files with progress tracking
+            let migratedCount = 0;
+            let skippedCount = 0;
+            const errors = [];
+            for (const fileInfo of filesToMigrate) {
+                try {
+                    await this.migrateFile(fileInfo, fromDepth, toDepth);
+                    migratedCount++;
+                    // Progress update every 1000 files
+                    if (migratedCount % 1000 === 0) {
+                        const percent = ((migratedCount / filesToMigrate.length) * 100).toFixed(1);
+                        console.log(`   📊 Progress: ${migratedCount}/${filesToMigrate.length} (${percent}%)`);
+                    }
+                    // Yield to event loop every 100 files to prevent blocking
+                    if (migratedCount % 100 === 0) {
+                        await new Promise(resolve => setImmediate(resolve));
+                    }
+                }
+                catch (error) {
+                    skippedCount++;
+                    errors.push({
+                        file: fileInfo.oldPath,
+                        error: error.message
+                    });
+                    // Log first few errors
+                    if (errors.length <= 5) {
+                        console.warn(`⚠️  Skipped ${fileInfo.oldPath}: ${error.message}`);
+                    }
+                }
+            }
+            // Final summary
+            console.log(`\n✅ Migration Results:`);
+            console.log(`   Migrated: ${migratedCount} files`);
+            console.log(`   Skipped: ${skippedCount} files`);
+            if (errors.length > 0) {
+                console.warn(`\n⚠️  ${errors.length} files could not be migrated`);
+                if (errors.length > 5) {
+                    console.warn(`   (First 5 errors shown above, ${errors.length - 5} more occurred)`);
+                }
+            }
+            // Cleanup: Remove empty old directories
+            if (fromDepth === 0) {
+                // No subdirectories to clean for flat structure
+            }
+            else if (fromDepth === 2) {
+                await this.cleanupEmptyDirectories(this.nounsDir, fromDepth);
+                await this.cleanupEmptyDirectories(this.verbsDir, fromDepth);
+            }
+            // Verification: Count files in new structure
+            const verifyCount = await this.countFilesInStructure(toDepth);
+            console.log(`\n🔍 Verification: ${verifyCount} files in new structure`);
+            if (verifyCount < migratedCount) {
+                console.warn(`⚠️  Warning: Verification count (${verifyCount}) < migrated count (${migratedCount})`);
+            }
         }
-        else {
-            return 1; // Always use at least single-level sharding for consistency
+        finally {
+            // Always remove lock file
+            try {
+                await fs.promises.unlink(lockFile);
+            }
+            catch (error) {
+                // Ignore error if lock file doesn't exist
+            }
         }
     }
     /**
-     * Get the path for a node with consistent sharding strategy
-     * Clean, predictable path generation
+     * Discover all files that need to be migrated
+     * Constructs correct oldPath based on source depth
      */
-    getNodePath(id) {
-        return this.getShardedPath(this.nounsDir, id);
-    }
-    /**
-     * Get the path for a verb with consistent sharding strategy
-     */
-    getVerbPath(id) {
-        return this.getShardedPath(this.verbsDir, id);
+    async discoverFilesForMigration(fromDepth) {
+        const files = [];
+        // Discover noun files
+        const nounFiles = await this.getAllFilesAtDepth(this.nounsDir, fromDepth);
+        for (const filename of nounFiles) {
+            const id = filename.replace('.json', '');
+            // Construct correct oldPath based on fromDepth
+            let oldPath;
+            switch (fromDepth) {
+                case 0:
+                    // Flat: nouns/uuid.json
+                    oldPath = path.join(this.nounsDir, `${id}.json`);
+                    break;
+                case 1:
+                    // Single-level: nouns/ab/uuid.json
+                    oldPath = path.join(this.nounsDir, id.substring(0, 2), `${id}.json`);
+                    break;
+                case 2:
+                    // Deep: nouns/ab/cd/uuid.json
+                    oldPath = path.join(this.nounsDir, id.substring(0, 2), id.substring(2, 4), `${id}.json`);
+                    break;
+                default:
+                    throw new Error(`Unsupported fromDepth: ${fromDepth}`);
+            }
+            files.push({ oldPath, id, type: 'noun' });
+        }
+        // Discover verb files
+        const verbFiles = await this.getAllFilesAtDepth(this.verbsDir, fromDepth);
+        for (const filename of verbFiles) {
+            const id = filename.replace('.json', '');
+            // Construct correct oldPath based on fromDepth
+            let oldPath;
+            switch (fromDepth) {
+                case 0:
+                    // Flat: verbs/uuid.json
+                    oldPath = path.join(this.verbsDir, `${id}.json`);
+                    break;
+                case 1:
+                    // Single-level: verbs/ab/uuid.json
+                    oldPath = path.join(this.verbsDir, id.substring(0, 2), `${id}.json`);
+                    break;
+                case 2:
+                    // Deep: verbs/ab/cd/uuid.json
+                    oldPath = path.join(this.verbsDir, id.substring(0, 2), id.substring(2, 4), `${id}.json`);
+                    break;
+                default:
+                    throw new Error(`Unsupported fromDepth: ${fromDepth}`);
+            }
+            files.push({ oldPath, id, type: 'verb' });
+        }
+        return files;
     }
     /**
-     * Universal sharded path generator
-     * Consistent across all entity types
-     */
-    getShardedPath(baseDir, id) {
-        const depth = this.cachedShardingDepth ?? this.getOptimalShardingDepth();
-        switch (depth) {
-            case 0:
-                // Flat structure: /nouns/uuid.json
-                return path.join(baseDir, `${id}.json`);
-            case 1:
-                // Single-level sharding: /nouns/ab/uuid.json
-                const shard1 = id.substring(0, 2);
-                return path.join(baseDir, shard1, `${id}.json`);
-            case 2:
-            default:
-                // Deep sharding: /nouns/ab/cd/uuid.json
-                const shard1Deep = id.substring(0, 2);
-                const shard2Deep = id.substring(2, 4);
-                return path.join(baseDir, shard1Deep, shard2Deep, `${id}.json`);
-        }
-    }
-    /**
-     * Get all JSON files from a sharded directory structure
-     * Properly traverses sharded subdirectories based on current sharding depth
+     * Get all files at a specific depth
      */
-    async getAllShardedFiles(baseDir) {
+    async getAllFilesAtDepth(baseDir, depth) {
         const allFiles = [];
-        const depth = this.cachedShardingDepth ?? this.getOptimalShardingDepth();
         try {
+            const dirExists = await this.directoryExists(baseDir);
+            if (!dirExists) {
+                return [];
+            }
             switch (depth) {
                 case 0:
-                    // Flat structure: read directly from baseDir
-                    const flatFiles = await fs.promises.readdir(baseDir);
-                    for (const file of flatFiles) {
-                        if (file.endsWith('.json')) {
-                            allFiles.push(file);
+                    // Flat: files directly in baseDir
+                    const entries = await fs.promises.readdir(baseDir);
+                    for (const entry of entries) {
+                        if (entry.endsWith('.json')) {
+                            allFiles.push(entry);
                         }
                     }
                     break;
                 case 1:
-                    // Single-level sharding: baseDir/ab/
-                    try {
-                        const shardDirs = await fs.promises.readdir(baseDir);
-                        for (const shardDir of shardDirs) {
-                            const shardPath = path.join(baseDir, shardDir);
-                            try {
-                                const stat = await fs.promises.stat(shardPath);
-                                if (stat.isDirectory()) {
-                                    const shardFiles = await fs.promises.readdir(shardPath);
-                                    for (const file of shardFiles) {
-                                        if (file.endsWith('.json')) {
-                                            allFiles.push(file);
-                                        }
+                    // Single-level: baseDir/ab/uuid.json
+                    const shardDirs = await fs.promises.readdir(baseDir);
+                    for (const shard of shardDirs) {
+                        const shardPath = path.join(baseDir, shard);
+                        try {
+                            const stat = await fs.promises.stat(shardPath);
+                            if (stat.isDirectory()) {
+                                const shardFiles = await fs.promises.readdir(shardPath);
+                                for (const file of shardFiles) {
+                                    if (file.endsWith('.json')) {
+                                        allFiles.push(file);
                                     }
                                 }
                             }
-                            catch (shardError) {
-                                // Skip inaccessible shard directories
-                                continue;
-                            }
                         }
-                    }
-                    catch (baseError) {
-                        // If baseDir doesn't exist, return empty array
-                        if (baseError.code === 'ENOENT') {
-                            return [];
+                        catch (error) {
+                            // Skip inaccessible directories
                         }
-                        throw baseError;
                     }
                     break;
                 case 2:
-                default:
-                    // Deep sharding: baseDir/ab/cd/
-                    try {
-                        const level1Dirs = await fs.promises.readdir(baseDir);
-                        for (const level1Dir of level1Dirs) {
-                            const level1Path = path.join(baseDir, level1Dir);
-                            try {
-                                const level1Stat = await fs.promises.stat(level1Path);
-                                if (level1Stat.isDirectory()) {
-                                    const level2Dirs = await fs.promises.readdir(level1Path);
-                                    for (const level2Dir of level2Dirs) {
-                                        const level2Path = path.join(level1Path, level2Dir);
-                                        try {
-                                            const level2Stat = await fs.promises.stat(level2Path);
-                                            if (level2Stat.isDirectory()) {
-                                                const shardFiles = await fs.promises.readdir(level2Path);
-                                                for (const file of shardFiles) {
-                                                    if (file.endsWith('.json')) {
-                                                        allFiles.push(file);
-                                                    }
+                    // Deep: baseDir/ab/cd/uuid.json
+                    const level1Dirs = await fs.promises.readdir(baseDir);
+                    for (const level1 of level1Dirs) {
+                        const level1Path = path.join(baseDir, level1);
+                        try {
+                            const level1Stat = await fs.promises.stat(level1Path);
+                            if (level1Stat.isDirectory()) {
+                                const level2Dirs = await fs.promises.readdir(level1Path);
+                                for (const level2 of level2Dirs) {
+                                    const level2Path = path.join(level1Path, level2);
+                                    try {
+                                        const level2Stat = await fs.promises.stat(level2Path);
+                                        if (level2Stat.isDirectory()) {
+                                            const files = await fs.promises.readdir(level2Path);
+                                            for (const file of files) {
+                                                if (file.endsWith('.json')) {
+                                                    allFiles.push(file);
                                                 }
                                             }
                                         }
-                                        catch (level2Error) {
-                                            // Skip inaccessible level2 directories
-                                            continue;
-                                        }
+                                    }
+                                    catch (error) {
+                                        // Skip inaccessible directories
                                     }
                                 }
                             }
-                            catch (level1Error) {
-                                // Skip inaccessible level1 directories
-                                continue;
+                        }
+                        catch (error) {
+                            // Skip inaccessible directories
+                        }
+                    }
+                    break;
+            }
+        }
+        catch (error) {
+            // Directory doesn't exist or not accessible
+        }
+        return allFiles;
+    }
+    /**
+     * Create all 256 shard directories (00-ff)
+     */
+    async createAllShardDirectories(baseDir) {
+        for (let i = 0; i < this.MAX_SHARDS; i++) {
+            const shard = i.toString(16).padStart(2, '0');
+            const shardDir = path.join(baseDir, shard);
+            await this.ensureDirectoryExists(shardDir);
+        }
+    }
+    /**
+     * Migrate a single file atomically
+     */
+    async migrateFile(fileInfo, fromDepth, toDepth) {
+        const baseDir = fileInfo.type === 'noun' ? this.nounsDir : this.verbsDir;
+        // Calculate old path (already known)
+        const oldPath = fileInfo.oldPath;
+        // Calculate new path using target depth
+        const shard = fileInfo.id.substring(0, 2).toLowerCase();
+        const newPath = path.join(baseDir, shard, `${fileInfo.id}.json`);
+        // Check if file already exists at new location
+        if (await this.fileExists(newPath)) {
+            // File already migrated or duplicate - skip
+            return;
+        }
+        // Atomic rename/move
+        await fs.promises.rename(oldPath, newPath);
+    }
+    /**
+     * Clean up empty directories after migration
+     */
+    async cleanupEmptyDirectories(baseDir, depth) {
+        try {
+            if (depth === 2) {
+                // Clean up level2 and level1 directories
+                const level1Dirs = await fs.promises.readdir(baseDir);
+                for (const level1 of level1Dirs) {
+                    const level1Path = path.join(baseDir, level1);
+                    try {
+                        const level1Stat = await fs.promises.stat(level1Path);
+                        if (level1Stat.isDirectory()) {
+                            const level2Dirs = await fs.promises.readdir(level1Path);
+                            for (const level2 of level2Dirs) {
+                                const level2Path = path.join(level1Path, level2);
+                                try {
+                                    // Try to remove level2 directory (will fail if not empty)
+                                    await fs.promises.rmdir(level2Path);
+                                }
+                                catch (error) {
+                                    // Directory not empty or other error - ignore
+                                }
                             }
+                            // Try to remove level1 directory
+                            await fs.promises.rmdir(level1Path);
                         }
                     }
-                    catch (baseError) {
-                        // If baseDir doesn't exist, return empty array
-                        if (baseError.code === 'ENOENT') {
-                            return [];
+                    catch (error) {
+                        // Directory not empty or other error - ignore
+                    }
+                }
+            }
+        }
+        catch (error) {
+            // Cleanup is best-effort, don't throw
+        }
+    }
+    /**
+     * Count files in the current structure
+     */
+    async countFilesInStructure(depth) {
+        let count = 0;
+        count += (await this.getAllFilesAtDepth(this.nounsDir, depth)).length;
+        count += (await this.getAllFilesAtDepth(this.verbsDir, depth)).length;
+        return count;
+    }
+    /**
+     * Detect the actual sharding depth used by existing files
+     * Examines directory structure to determine current sharding strategy
+     * Returns null if no files exist yet (new installation)
+     */
+    async detectExistingShardingDepth() {
+        try {
+            // Check if nouns directory exists and has content
+            const dirExists = await this.directoryExists(this.nounsDir);
+            if (!dirExists) {
+                return null; // New installation
+            }
+            const entries = await fs.promises.readdir(this.nounsDir, { withFileTypes: true });
+            // Check if there are any .json files directly in nounsDir (flat structure)
+            const hasDirectJsonFiles = entries.some((e) => e.isFile() && e.name.endsWith('.json'));
+            if (hasDirectJsonFiles) {
+                return 0; // Flat structure: nouns/uuid.json
+            }
+            // Check for subdirectories with hex names (sharding directories)
+            const subdirs = entries.filter((e) => e.isDirectory() && /^[0-9a-f]{2}$/i.test(e.name));
+            if (subdirs.length === 0) {
+                return null; // No files yet
+            }
+            // Check first subdir to see if it has files or more subdirs
+            const firstSubdir = subdirs[0].name;
+            const subdirPath = path.join(this.nounsDir, firstSubdir);
+            const subdirEntries = await fs.promises.readdir(subdirPath, { withFileTypes: true });
+            const hasJsonFiles = subdirEntries.some((e) => e.isFile() && e.name.endsWith('.json'));
+            if (hasJsonFiles) {
+                return 1; // Single-level sharding: nouns/ab/uuid.json
+            }
+            const hasSubSubdirs = subdirEntries.some((e) => e.isDirectory() && /^[0-9a-f]{2}$/i.test(e.name));
+            if (hasSubSubdirs) {
+                return 2; // Deep sharding: nouns/ab/cd/uuid.json
+            }
+            return 1; // Default to single-level if structure is unclear
+        }
+        catch (error) {
+            // If we can't read the directory, assume new installation
+            return null;
+        }
+    }
+    /**
+     * Get sharding depth
+     * Always returns 1 (single-level sharding) for optimal balance of
+     * simplicity, performance, and reliability across all dataset sizes
+     *
+     * Single-level sharding (depth=1):
+     * - 256 shard directories (00-ff)
+     * - Handles 2.5M+ entities with excellent performance
+     * - No dynamic depth changes = no path mismatch bugs
+     * - Industry standard approach (Git uses similar)
+     */
+    getOptimalShardingDepth() {
+        return this.SHARDING_DEPTH;
+    }
+    /**
+     * Get the path for a node with consistent sharding strategy
+     * Clean, predictable path generation
+     */
+    getNodePath(id) {
+        return this.getShardedPath(this.nounsDir, id);
+    }
+    /**
+     * Get the path for a verb with consistent sharding strategy
+     */
+    getVerbPath(id) {
+        return this.getShardedPath(this.verbsDir, id);
+    }
+    /**
+     * Universal sharded path generator
+     * Always uses depth=1 (single-level sharding) for consistency
+     *
+     * Format: baseDir/ab/uuid.json
+     * Where 'ab' = first 2 hex characters of UUID (lowercase)
+     *
+     * Validates UUID format and throws descriptive errors
+     */
+    getShardedPath(baseDir, id) {
+        // Extract first 2 characters for shard directory
+        const shard = id.substring(0, 2).toLowerCase();
+        // Validate shard is valid hex (00-ff)
+        if (!/^[0-9a-f]{2}$/.test(shard)) {
+            throw new Error(`Invalid entity ID format: ${id}. ` +
+                `Expected UUID starting with 2 hex characters, got '${shard}'. ` +
+                `IDs must be UUIDs or hex strings.`);
+        }
+        // Single-level sharding: baseDir/ab/uuid.json
+        return path.join(baseDir, shard, `${id}.json`);
+    }
+    /**
+     * Get all JSON files from the single-level sharded directory structure
+     * Traverses all shard subdirectories (00-ff)
+     */
+    async getAllShardedFiles(baseDir) {
+        const allFiles = [];
+        try {
+            const shardDirs = await fs.promises.readdir(baseDir);
+            for (const shardDir of shardDirs) {
+                const shardPath = path.join(baseDir, shardDir);
+                try {
+                    const stat = await fs.promises.stat(shardPath);
+                    if (stat.isDirectory()) {
+                        const shardFiles = await fs.promises.readdir(shardPath);
+                        for (const file of shardFiles) {
+                            if (file.endsWith('.json')) {
+                                allFiles.push(file);
+                            }
                         }
-                        throw baseError;
                     }
-                    break;
+                }
+                catch (shardError) {
+                    // Skip inaccessible shard directories
+                    continue;
+                }
             }
             // Sort for consistent ordering
             allFiles.sort();
@@ -1702,109 +2046,45 @@ export class FileSystemStorage extends BaseStorage {
      * Stream through sharded files without loading all names into memory
      * Production-scale implementation for millions of files
      */
+    /**
+     * Stream through files in single-level sharded structure
+     * Calls processor for each file until processor returns false
+     * Returns true if more files exist (processor stopped early), false if all processed
+     */
     async streamShardedFiles(baseDir, depth, processor) {
         let hasMore = true;
-        switch (depth) {
-            case 0:
-                // Flat structure
-                try {
-                    const files = await fs.promises.readdir(baseDir);
-                    const sortedFiles = files.filter((f) => f.endsWith('.json')).sort();
-                    for (const file of sortedFiles) {
-                        const shouldContinue = await processor(file, path.join(baseDir, file));
-                        if (!shouldContinue) {
-                            hasMore = false;
-                            break;
-                        }
-                    }
-                }
-                catch (error) {
-                    if (error.code === 'ENOENT')
-                        hasMore = false;
-                }
-                break;
-            case 1:
-                // Single-level sharding: ab/
-                try {
-                    const shardDirs = await fs.promises.readdir(baseDir);
-                    const sortedShardDirs = shardDirs.sort();
-                    for (const shardDir of sortedShardDirs) {
-                        const shardPath = path.join(baseDir, shardDir);
-                        try {
-                            const stat = await fs.promises.stat(shardPath);
-                            if (stat.isDirectory()) {
-                                const files = await fs.promises.readdir(shardPath);
-                                const sortedFiles = files.filter((f) => f.endsWith('.json')).sort();
-                                for (const file of sortedFiles) {
-                                    const shouldContinue = await processor(file, path.join(shardPath, file));
-                                    if (!shouldContinue) {
-                                        hasMore = false;
-                                        break;
-                                    }
-                                }
-                                if (!hasMore)
-                                    break;
-                            }
-                        }
-                        catch (shardError) {
-                            continue; // Skip inaccessible shard directories
-                        }
-                    }
-                }
-                catch (error) {
-                    if (error.code === 'ENOENT')
-                        hasMore = false;
-                }
-                break;
-            case 2:
-            default:
-                // Deep sharding: ab/cd/
+        // Single-level sharding (depth=1): baseDir/ab/uuid.json
+        try {
+            const shardDirs = await fs.promises.readdir(baseDir);
+            const sortedShardDirs = shardDirs.sort();
+            for (const shardDir of sortedShardDirs) {
+                const shardPath = path.join(baseDir, shardDir);
                 try {
-                    const level1Dirs = await fs.promises.readdir(baseDir);
-                    const sortedLevel1Dirs = level1Dirs.sort();
-                    for (const level1Dir of sortedLevel1Dirs) {
-                        const level1Path = path.join(baseDir, level1Dir);
-                        try {
-                            const level1Stat = await fs.promises.stat(level1Path);
-                            if (level1Stat.isDirectory()) {
-                                const level2Dirs = await fs.promises.readdir(level1Path);
-                                const sortedLevel2Dirs = level2Dirs.sort();
-                                for (const level2Dir of sortedLevel2Dirs) {
-                                    const level2Path = path.join(level1Path, level2Dir);
-                                    try {
-                                        const level2Stat = await fs.promises.stat(level2Path);
-                                        if (level2Stat.isDirectory()) {
-                                            const files = await fs.promises.readdir(level2Path);
-                                            const sortedFiles = files.filter((f) => f.endsWith('.json')).sort();
-                                            for (const file of sortedFiles) {
-                                                const shouldContinue = await processor(file, path.join(level2Path, file));
-                                                if (!shouldContinue) {
-                                                    hasMore = false;
-                                                    break;
-                                                }
-                                            }
-                                            if (!hasMore)
-                                                break;
-                                        }
-                                    }
-                                    catch (level2Error) {
-                                        continue; // Skip inaccessible level2 directories
-                                    }
-                                }
-                                if (!hasMore)
-                                    break;
+                    const stat = await fs.promises.stat(shardPath);
+                    if (stat.isDirectory()) {
+                        const files = await fs.promises.readdir(shardPath);
+                        const sortedFiles = files.filter((f) => f.endsWith('.json')).sort();
+                        for (const file of sortedFiles) {
+                            const shouldContinue = await processor(file, path.join(shardPath, file));
+                            if (!shouldContinue) {
+                                hasMore = false;
+                                break;
                             }
                         }
-                        catch (level1Error) {
-                            continue; // Skip inaccessible level1 directories
-                        }
+                        if (!hasMore)
+                            break;
                     }
                 }
-                catch (error) {
-                    if (error.code === 'ENOENT')
-                        hasMore = false;
+                catch (shardError) {
+                    // Skip inaccessible shard directories
+                    continue;
                 }
-                break;
+            }
+        }
+        catch (error) {
+            if (error.code === 'ENOENT') {
+                hasMore = false;
+            }
         }
         return hasMore;
     }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@soulcraft/brainy",
-  "version": "3.23.1",
+  "version": "3.24.0",
   "description": "Universal Knowledge Protocol™ - World's first Triple Intelligence database unifying vector, graph, and document search in one API. 31 nouns × 40 verbs for infinite expressiveness.",
   "main": "dist/index.js",
   "module": "dist/index.js",