@soulcraft/brainy 3.25.2 → 3.26.0

package/CHANGELOG.md

@@ -2,6 +2,18 @@
 
 All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines.
 
+## [3.26.0](https://github.com/soulcraftlabs/brainy/compare/v3.25.2...v3.26.0) (2025-10-08)
+
+
+### ⚠ BREAKING CHANGES
+
+* Requires data migration for existing S3/GCS/R2/OpFS deployments.
+See .strategy/UNIFIED-UUID-SHARDING.md for migration guidance.
+
+### 🐛 Bug Fixes
+
+* implement unified UUID-based sharding for metadata across all storage adapters ([2f33571](https://github.com/soulcraftlabs/brainy/commit/2f3357132d06c70cd74532d22cbfbf6abb92903a))
+
 ### [3.25.2](https://github.com/soulcraftlabs/brainy/compare/v3.25.1...v3.25.2) (2025-10-08)
 
 

package/dist/storage/adapters/fileSystemStorage.js

@@ -510,7 +510,11 @@ export class FileSystemStorage extends BaseStorage {
      */
     async saveNounMetadata_internal(id, metadata) {
         await this.ensureInitialized();
-        const filePath = path.join(this.nounMetadataDir, `${id}.json`);
+        // Use UUID-based sharding for metadata (consistent with noun vectors)
+        const filePath = this.getShardedPath(this.nounMetadataDir, id);
+        // Ensure shard directory exists
+        const shardDir = path.dirname(filePath);
+        await fs.promises.mkdir(shardDir, { recursive: true });
         await fs.promises.writeFile(filePath, JSON.stringify(metadata, null, 2));
     }
     /**
@@ -518,7 +522,8 @@ export class FileSystemStorage extends BaseStorage {
      */
     async getNounMetadata(id) {
         await this.ensureInitialized();
-        const filePath = path.join(this.nounMetadataDir, `${id}.json`);
+        // Use UUID-based sharding for metadata (consistent with noun vectors)
+        const filePath = this.getShardedPath(this.nounMetadataDir, id);
         try {
             const data = await fs.promises.readFile(filePath, 'utf-8');
             return JSON.parse(data);

package/dist/storage/adapters/opfsStorage.js

@@ -3,6 +3,7 @@
  * Provides persistent storage for the vector database using the Origin Private File System API
  */
 import { BaseStorage, NOUNS_DIR, VERBS_DIR, METADATA_DIR, NOUN_METADATA_DIR, VERB_METADATA_DIR, INDEX_DIR } from '../baseStorage.js';
+import { getShardIdFromUuid } from '../sharding.js';
 import '../../types/fileSystemTypes.js';
 /**
  * Helper function to safely get a file from a FileSystemHandle
@@ -145,8 +146,14 @@ export class OPFSStorage extends BaseStorage {
                 ...noun,
                 connections: this.mapToObject(noun.connections, (set) => Array.from(set))
             };
-            // Create or get the file for this noun
-            const fileHandle = await this.nounsDir.getFileHandle(`${noun.id}.json`, {
+            // Use UUID-based sharding for nouns
+            const shardId = getShardIdFromUuid(noun.id);
+            // Get or create the shard directory
+            const shardDir = await this.nounsDir.getDirectoryHandle(shardId, {
+                create: true
+            });
+            // Create or get the file in the shard directory
+            const fileHandle = await shardDir.getFileHandle(`${noun.id}.json`, {
                 create: true
             });
             // Write the noun data to the file
@@ -165,8 +172,12 @@ export class OPFSStorage extends BaseStorage {
     async getNoun_internal(id) {
         await this.ensureInitialized();
         try {
-            // Get the file handle for this noun
-            const fileHandle = await this.nounsDir.getFileHandle(`${id}.json`);
+            // Use UUID-based sharding for nouns
+            const shardId = getShardIdFromUuid(id);
+            // Get the shard directory
+            const shardDir = await this.nounsDir.getDirectoryHandle(shardId);
+            // Get the file handle from the shard directory
+            const fileHandle = await shardDir.getFileHandle(`${id}.json`);
             // Read the noun data from the file
             const file = await fileHandle.getFile();
             const text = await file.text();
@@ -205,34 +216,40 @@ export class OPFSStorage extends BaseStorage {
         await this.ensureInitialized();
         const nodes = [];
         try {
-            // Iterate through all files in the nouns directory
-            for await (const [name, handle] of this.nounsDir.entries()) {
-                if (handle.kind === 'file') {
-                    try {
-                        // Read the node data from the file
-                        const file = await safeGetFile(handle);
-                        const text = await file.text();
-                        const data = JSON.parse(text);
-                        // Get the metadata to check the noun type
-                        const metadata = await this.getMetadata(data.id);
-                        // Include the node if its noun type matches the requested type
-                        if (metadata && metadata.noun === nounType) {
-                            // Convert serialized connections back to Map<number, Set<string>>
-                            const connections = new Map();
-                            for (const [level, nodeIds] of Object.entries(data.connections)) {
-                                connections.set(Number(level), new Set(nodeIds));
+            // Iterate through all shard directories
+            for await (const [shardName, shardHandle] of this.nounsDir.entries()) {
+                if (shardHandle.kind === 'directory') {
+                    const shardDir = shardHandle;
+                    // Iterate through all files in this shard
+                    for await (const [fileName, fileHandle] of shardDir.entries()) {
+                        if (fileHandle.kind === 'file') {
+                            try {
+                                // Read the node data from the file
+                                const file = await safeGetFile(fileHandle);
+                                const text = await file.text();
+                                const data = JSON.parse(text);
+                                // Get the metadata to check the noun type
+                                const metadata = await this.getMetadata(data.id);
+                                // Include the node if its noun type matches the requested type
+                                if (metadata && metadata.noun === nounType) {
+                                    // Convert serialized connections back to Map<number, Set<string>>
+                                    const connections = new Map();
+                                    for (const [level, nodeIds] of Object.entries(data.connections)) {
+                                        connections.set(Number(level), new Set(nodeIds));
+                                    }
+                                    nodes.push({
+                                        id: data.id,
+                                        vector: data.vector,
+                                        connections,
+                                        level: data.level || 0
+                                    });
+                                }
+                            }
+                            catch (error) {
+                                console.error(`Error reading node file ${shardName}/${fileName}:`, error);
                             }
-                            nodes.push({
-                                id: data.id,
-                                vector: data.vector,
-                                connections,
-                                level: data.level || 0
-                            });
                         }
                     }
-                    catch (error) {
-                        console.error(`Error reading node file ${name}:`, error);
-                    }
                 }
             }
         }
@@ -253,7 +270,12 @@ export class OPFSStorage extends BaseStorage {
     async deleteNode(id) {
         await this.ensureInitialized();
         try {
-            await this.nounsDir.removeEntry(`${id}.json`);
+            // Use UUID-based sharding for nouns
+            const shardId = getShardIdFromUuid(id);
+            // Get the shard directory
+            const shardDir = await this.nounsDir.getDirectoryHandle(shardId);
+            // Delete the file from the shard directory
+            await shardDir.removeEntry(`${id}.json`);
         }
         catch (error) {
             // Ignore NotFoundError, which means the file doesn't exist
@@ -280,8 +302,14 @@ export class OPFSStorage extends BaseStorage {
                 ...edge,
                 connections: this.mapToObject(edge.connections, (set) => Array.from(set))
             };
-            // Create or get the file for this verb
-            const fileHandle = await this.verbsDir.getFileHandle(`${edge.id}.json`, {
+            // Use UUID-based sharding for verbs
+            const shardId = getShardIdFromUuid(edge.id);
+            // Get or create the shard directory
+            const shardDir = await this.verbsDir.getDirectoryHandle(shardId, {
+                create: true
+            });
+            // Create or get the file in the shard directory
+            const fileHandle = await shardDir.getFileHandle(`${edge.id}.json`, {
                 create: true
             });
             // Write the verb data to the file
@@ -306,8 +334,12 @@ export class OPFSStorage extends BaseStorage {
     async getEdge(id) {
         await this.ensureInitialized();
         try {
-            // Get the file handle for this edge
-            const fileHandle = await this.verbsDir.getFileHandle(`${id}.json`);
+            // Use UUID-based sharding for verbs
+            const shardId = getShardIdFromUuid(id);
+            // Get the shard directory
+            const shardDir = await this.verbsDir.getDirectoryHandle(shardId);
+            // Get the file handle from the shard directory
+            const fileHandle = await shardDir.getFileHandle(`${id}.json`);
             // Read the edge data from the file
             const file = await fileHandle.getFile();
             const text = await file.text();
@@ -345,37 +377,43 @@ export class OPFSStorage extends BaseStorage {
         await this.ensureInitialized();
         const allEdges = [];
         try {
-            // Iterate through all files in the verbs directory
-            for await (const [name, handle] of this.verbsDir.entries()) {
-                if (handle.kind === 'file') {
-                    try {
-                        // Read the edge data from the file
-                        const file = await safeGetFile(handle);
-                        const text = await file.text();
-                        const data = JSON.parse(text);
-                        // Convert serialized connections back to Map<number, Set<string>>
-                        const connections = new Map();
-                        for (const [level, nodeIds] of Object.entries(data.connections)) {
-                            connections.set(Number(level), new Set(nodeIds));
+            // Iterate through all shard directories
+            for await (const [shardName, shardHandle] of this.verbsDir.entries()) {
+                if (shardHandle.kind === 'directory') {
+                    const shardDir = shardHandle;
+                    // Iterate through all files in this shard
+                    for await (const [fileName, fileHandle] of shardDir.entries()) {
+                        if (fileHandle.kind === 'file') {
+                            try {
+                                // Read the edge data from the file
+                                const file = await safeGetFile(fileHandle);
+                                const text = await file.text();
+                                const data = JSON.parse(text);
+                                // Convert serialized connections back to Map<number, Set<string>>
+                                const connections = new Map();
+                                for (const [level, nodeIds] of Object.entries(data.connections)) {
+                                    connections.set(Number(level), new Set(nodeIds));
+                                }
+                                // Create default timestamp if not present
+                                const defaultTimestamp = {
+                                    seconds: Math.floor(Date.now() / 1000),
+                                    nanoseconds: (Date.now() % 1000) * 1000000
+                                };
+                                // Create default createdBy if not present
+                                const defaultCreatedBy = {
+                                    augmentation: 'unknown',
+                                    version: '1.0'
+                                };
+                                allEdges.push({
+                                    id: data.id,
+                                    vector: data.vector,
+                                    connections
+                                });
+                            }
+                            catch (error) {
+                                console.error(`Error reading edge file ${shardName}/${fileName}:`, error);
+                            }
                         }
-                        // Create default timestamp if not present
-                        const defaultTimestamp = {
-                            seconds: Math.floor(Date.now() / 1000),
-                            nanoseconds: (Date.now() % 1000) * 1000000
-                        };
-                        // Create default createdBy if not present
-                        const defaultCreatedBy = {
-                            augmentation: 'unknown',
-                            version: '1.0'
-                        };
-                        allEdges.push({
-                            id: data.id,
-                            vector: data.vector,
-                            connections
-                        });
-                    }
-                    catch (error) {
-                        console.error(`Error reading edge file ${name}:`, error);
                     }
                 }
             }
@@ -457,7 +495,12 @@ export class OPFSStorage extends BaseStorage {
     async deleteEdge(id) {
         await this.ensureInitialized();
         try {
-            await this.verbsDir.removeEntry(`${id}.json`);
+            // Use UUID-based sharding for verbs
+            const shardId = getShardIdFromUuid(id);
+            // Get the shard directory
+            const shardDir = await this.verbsDir.getDirectoryHandle(shardId);
+            // Delete the file from the shard directory
+            await shardDir.removeEntry(`${id}.json`);
         }
         catch (error) {
             // Ignore NotFoundError, which means the file doesn't exist
@@ -542,8 +585,13 @@ export class OPFSStorage extends BaseStorage {
      */
     async saveVerbMetadata_internal(id, metadata) {
         await this.ensureInitialized();
+        // Use UUID-based sharding for metadata (consistent with verb vectors)
+        const shardId = getShardIdFromUuid(id);
+        // Get or create the shard directory
+        const shardDir = await this.verbMetadataDir.getDirectoryHandle(shardId, { create: true });
+        // Create or get the file in the shard directory
         const fileName = `${id}.json`;
-        const fileHandle = await this.verbMetadataDir.getFileHandle(fileName, { create: true });
+        const fileHandle = await shardDir.getFileHandle(fileName, { create: true });
         const writable = await fileHandle.createWritable();
         await writable.write(JSON.stringify(metadata, null, 2));
         await writable.close();
@@ -553,9 +601,14 @@ export class OPFSStorage extends BaseStorage {
      */
     async getVerbMetadata(id) {
         await this.ensureInitialized();
+        // Use UUID-based sharding for metadata (consistent with verb vectors)
+        const shardId = getShardIdFromUuid(id);
         const fileName = `${id}.json`;
         try {
-            const fileHandle = await this.verbMetadataDir.getFileHandle(fileName);
+            // Get the shard directory
+            const shardDir = await this.verbMetadataDir.getDirectoryHandle(shardId);
+            // Get the file from the shard directory
+            const fileHandle = await shardDir.getFileHandle(fileName);
             const file = await safeGetFile(fileHandle);
             const text = await file.text();
             return JSON.parse(text);
@@ -572,8 +625,13 @@ export class OPFSStorage extends BaseStorage {
      */
     async saveNounMetadata_internal(id, metadata) {
         await this.ensureInitialized();
+        // Use UUID-based sharding for metadata (consistent with noun vectors)
+        const shardId = getShardIdFromUuid(id);
+        // Get or create the shard directory
+        const shardDir = await this.nounMetadataDir.getDirectoryHandle(shardId, { create: true });
+        // Create or get the file in the shard directory
         const fileName = `${id}.json`;
-        const fileHandle = await this.nounMetadataDir.getFileHandle(fileName, { create: true });
+        const fileHandle = await shardDir.getFileHandle(fileName, { create: true });
         const writable = await fileHandle.createWritable();
         await writable.write(JSON.stringify(metadata, null, 2));
         await writable.close();
@@ -583,9 +641,14 @@ export class OPFSStorage extends BaseStorage {
      */
     async getNounMetadata(id) {
         await this.ensureInitialized();
+        // Use UUID-based sharding for metadata (consistent with noun vectors)
+        const shardId = getShardIdFromUuid(id);
         const fileName = `${id}.json`;
         try {
-            const fileHandle = await this.nounMetadataDir.getFileHandle(fileName);
+            // Get the shard directory
+            const shardDir = await this.nounMetadataDir.getDirectoryHandle(shardId);
+            // Get the file from the shard directory
+            const fileHandle = await shardDir.getFileHandle(fileName);
             const file = await safeGetFile(fileHandle);
             const text = await file.text();
             return JSON.parse(text);
@@ -1117,12 +1180,19 @@ export class OPFSStorage extends BaseStorage {
         await this.ensureInitialized();
         const limit = options.limit || 100;
         const cursor = options.cursor;
-        // Get all noun files
+        // Get all noun files from all shards
         const nounFiles = [];
         if (this.nounsDir) {
-            for await (const [name, handle] of this.nounsDir.entries()) {
-                if (handle.kind === 'file' && name.endsWith('.json')) {
-                    nounFiles.push(name);
+            // Iterate through all shard directories
+            for await (const [shardName, shardHandle] of this.nounsDir.entries()) {
+                if (shardHandle.kind === 'directory') {
+                    // Iterate through files in this shard
+                    const shardDir = shardHandle;
+                    for await (const [fileName, fileHandle] of shardDir.entries()) {
+                        if (fileHandle.kind === 'file' && fileName.endsWith('.json')) {
+                            nounFiles.push(`${shardName}/${fileName}`);
+                        }
+                    }
                 }
             }
         }
@@ -1141,7 +1211,8 @@ export class OPFSStorage extends BaseStorage {
         // Load nouns from files
         const items = [];
         for (const fileName of pageFiles) {
-            const id = fileName.replace('.json', '');
+            // fileName is in format "shard/uuid.json", extract just the UUID
+            const id = fileName.split('/')[1].replace('.json', '');
             const noun = await this.getNoun_internal(id);
             if (noun) {
                 // Apply filters if provided
@@ -1205,12 +1276,19 @@ export class OPFSStorage extends BaseStorage {
         await this.ensureInitialized();
         const limit = options.limit || 100;
         const cursor = options.cursor;
-        // Get all verb files
+        // Get all verb files from all shards
         const verbFiles = [];
         if (this.verbsDir) {
-            for await (const [name, handle] of this.verbsDir.entries()) {
-                if (handle.kind === 'file' && name.endsWith('.json')) {
-                    verbFiles.push(name);
+            // Iterate through all shard directories
+            for await (const [shardName, shardHandle] of this.verbsDir.entries()) {
+                if (shardHandle.kind === 'directory') {
+                    // Iterate through files in this shard
+                    const shardDir = shardHandle;
+                    for await (const [fileName, fileHandle] of shardDir.entries()) {
+                        if (fileHandle.kind === 'file' && fileName.endsWith('.json')) {
+                            verbFiles.push(`${shardName}/${fileName}`);
+                        }
+                    }
                 }
             }
         }
@@ -1229,7 +1307,8 @@ export class OPFSStorage extends BaseStorage {
         // Load verbs from files and convert to GraphVerb
         const items = [];
         for (const fileName of pageFiles) {
-            const id = fileName.replace('.json', '');
+            // fileName is in format "shard/uuid.json", extract just the UUID
+            const id = fileName.split('/')[1].replace('.json', '');
             const hnswVerb = await this.getVerb_internal(id);
             if (hnswVerb) {
                 // Convert HNSWVerb to GraphVerb
@@ -1330,16 +1409,26 @@ export class OPFSStorage extends BaseStorage {
      */
     async initializeCountsFromScan() {
         try {
-            // Count nouns
+            // Count nouns across all shards
             let nounCount = 0;
-            for await (const [,] of this.nounsDir.entries()) {
-                nounCount++;
+            for await (const [shardName, shardHandle] of this.nounsDir.entries()) {
+                if (shardHandle.kind === 'directory') {
+                    const shardDir = shardHandle;
+                    for await (const [,] of shardDir.entries()) {
+                        nounCount++;
+                    }
+                }
             }
             this.totalNounCount = nounCount;
-            // Count verbs
+            // Count verbs across all shards
             let verbCount = 0;
-            for await (const [,] of this.verbsDir.entries()) {
-                verbCount++;
+            for await (const [shardName, shardHandle] of this.verbsDir.entries()) {
+                if (shardHandle.kind === 'directory') {
+                    const shardDir = shardHandle;
+                    for await (const [,] of shardDir.entries()) {
+                        verbCount++;
+                    }
+                }
             }
             this.totalVerbCount = verbCount;
             // Save initial counts

package/dist/storage/adapters/s3CompatibleStorage.d.ts

@@ -73,7 +73,6 @@ export declare class S3CompatibleStorage extends BaseStorage {
     private nounWriteBuffer;
     private verbWriteBuffer;
     private coordinator?;
-    private shardManager?;
     private cacheSync?;
     private readWriteSeparation?;
     private requestCoalescer;
@@ -112,7 +111,9 @@ export declare class S3CompatibleStorage extends BaseStorage {
     init(): Promise<void>;
     /**
      * Set distributed components for multi-node coordination
-     * Zero-config: Automatically optimizes based on components provided
+     *
+     * Note: Sharding is always enabled via UUID-based prefixes (00-ff).
+     * ShardManager is no longer required - sharding is deterministic based on UUID.
      */
     setDistributedComponents(components: {
         coordinator?: any;
@@ -121,11 +122,25 @@ export declare class S3CompatibleStorage extends BaseStorage {
         readWriteSeparation?: any;
     }): void;
     /**
-     * Get the S3 key for a noun, using sharding if available
+     * Get the S3 key for a noun using UUID-based sharding
+     *
+     * Uses first 2 hex characters of UUID for consistent sharding.
+     * Path format: entities/nouns/vectors/{shardId}/{uuid}.json
+     *
+     * @example
+     * getNounKey('ab123456-1234-5678-9abc-def012345678')
+     * // returns 'entities/nouns/vectors/ab/ab123456-1234-5678-9abc-def012345678.json'
      */
     private getNounKey;
     /**
-     * Get the S3 key for a verb, using sharding if available
+     * Get the S3 key for a verb using UUID-based sharding
+     *
+     * Uses first 2 hex characters of UUID for consistent sharding.
+     * Path format: verbs/{shardId}/{uuid}.json
+     *
+     * @example
+     * getVerbKey('cd987654-4321-8765-cba9-fed543210987')
+     * // returns 'verbs/cd/cd987654-4321-8765-cba9-fed543210987.json'
      */
     private getVerbKey;
     /**
@@ -221,9 +236,23 @@ export declare class S3CompatibleStorage extends BaseStorage {
      */
     protected getAllNodes(): Promise<HNSWNode[]>;
     /**
-     * Get nodes with pagination
+     * Get nodes with pagination using UUID-based sharding
+     *
+     * Iterates through 256 UUID-based shards (00-ff) to retrieve nodes.
+     * Cursor format: "shardIndex:s3ContinuationToken" to support pagination across shards.
+     *
      * @param options Pagination options
      * @returns Promise that resolves to a paginated result of nodes
+     *
+     * @example
+     * // First page
+     * const page1 = await getNodesWithPagination({ limit: 100 })
+     * // page1.nodes contains up to 100 nodes
+     * // page1.nextCursor might be "5:some-s3-token" (currently in shard 05)
+     *
+     * // Next page
+     * const page2 = await getNodesWithPagination({ limit: 100, cursor: page1.nextCursor })
+     * // Continues from where page1 left off
      */
     protected getNodesWithPagination(options?: {
         limit?: number;
@@ -234,6 +263,10 @@ export declare class S3CompatibleStorage extends BaseStorage {
         hasMore: boolean;
         nextCursor?: string;
     }>;
+    /**
+     * Load nodes by IDs efficiently using cache or direct fetch
+     */
+    private loadNodesByIds;
     /**
      * Get nouns by noun type (internal implementation)
      * @param nounType The noun type to filter by
@@ -517,6 +550,11 @@ export declare class S3CompatibleStorage extends BaseStorage {
         hasMore: boolean;
         nextCursor?: string;
     }>;
+    /**
+     * Estimate total noun count by listing objects across all shards
+     * This is more efficient than loading all nouns
+     */
+    private estimateTotalNounCount;
     /**
      * Initialize counts from S3 storage
      */

package/dist/storage/adapters/s3CompatibleStorage.js

@@ -13,6 +13,7 @@ import { getGlobalSocketManager } from '../../utils/adaptiveSocketManager.js';
 import { getGlobalBackpressure } from '../../utils/adaptiveBackpressure.js';
 import { getWriteBuffer } from '../../utils/writeBuffer.js';
 import { getCoalescer } from '../../utils/requestCoalescer.js';
+import { getShardIdFromUuid, getShardIdByIndex, TOTAL_SHARDS } from '../sharding.js';
 // Export R2Storage as an alias for S3CompatibleStorage
 export { S3CompatibleStorage as R2Storage };
 /**
@@ -69,6 +70,8 @@ export class S3CompatibleStorage extends BaseStorage {
         // Write buffers for bulk operations
         this.nounWriteBuffer = null;
         this.verbWriteBuffer = null;
+        // Note: Sharding is always enabled via UUID-based prefixes (00-ff)
+        // ShardManager is no longer used - sharding is deterministic
         // Request coalescer for deduplication
         this.requestCoalescer = null;
         // High-volume mode detection - MUCH more aggressive
@@ -242,17 +245,16 @@ export class S3CompatibleStorage extends BaseStorage {
     }
     /**
      * Set distributed components for multi-node coordination
-     * Zero-config: Automatically optimizes based on components provided
+     *
+     * Note: Sharding is always enabled via UUID-based prefixes (00-ff).
+     * ShardManager is no longer required - sharding is deterministic based on UUID.
      */
     setDistributedComponents(components) {
         this.coordinator = components.coordinator;
-        this.shardManager = components.shardManager;
         this.cacheSync = components.cacheSync;
         this.readWriteSeparation = components.readWriteSeparation;
-        // Auto-configure based on what's available
-        if (this.shardManager) {
-            console.log(`🎯 S3 Storage: Sharding enabled with ${this.shardManager.config?.shardCount || 64} shards`);
-        }
+        // Note: UUID-based sharding is always active (256 shards: 00-ff)
+        console.log(`🎯 S3 Storage: UUID-based sharding active (256 shards: 00-ff)`);
         if (this.coordinator) {
             console.log(`🤝 S3 Storage: Distributed coordination active (node: ${this.coordinator.nodeId})`);
         }
@@ -264,24 +266,32 @@ export class S3CompatibleStorage extends BaseStorage {
         }
     }
     /**
-     * Get the S3 key for a noun, using sharding if available
+     * Get the S3 key for a noun using UUID-based sharding
+     *
+     * Uses first 2 hex characters of UUID for consistent sharding.
+     * Path format: entities/nouns/vectors/{shardId}/{uuid}.json
+     *
+     * @example
+     * getNounKey('ab123456-1234-5678-9abc-def012345678')
+     * // returns 'entities/nouns/vectors/ab/ab123456-1234-5678-9abc-def012345678.json'
      */
     getNounKey(id) {
-        if (this.shardManager) {
-            const shardId = this.shardManager.getShardForKey(id);
-            return `shards/${shardId}/${this.nounPrefix}${id}.json`;
-        }
-        return `${this.nounPrefix}${id}.json`;
+        const shardId = getShardIdFromUuid(id);
+        return `${this.nounPrefix}${shardId}/${id}.json`;
     }
     /**
-     * Get the S3 key for a verb, using sharding if available
+     * Get the S3 key for a verb using UUID-based sharding
+     *
+     * Uses first 2 hex characters of UUID for consistent sharding.
+     * Path format: verbs/{shardId}/{uuid}.json
+     *
+     * @example
+     * getVerbKey('cd987654-4321-8765-cba9-fed543210987')
+     * // returns 'verbs/cd/cd987654-4321-8765-cba9-fed543210987.json'
      */
     getVerbKey(id) {
-        if (this.shardManager) {
-            const shardId = this.shardManager.getShardForKey(id);
-            return `shards/${shardId}/${this.verbPrefix}${id}.json`;
-        }
-        return `${this.verbPrefix}${id}.json`;
+        const shardId = getShardIdFromUuid(id);
+        return `${this.verbPrefix}${shardId}/${id}.json`;
     }
     /**
      * Override base class method to detect S3-specific throttling errors
@@ -775,7 +785,8 @@ export class S3CompatibleStorage extends BaseStorage {
         try {
             // Import the GetObjectCommand only when needed
             const { GetObjectCommand } = await import('@aws-sdk/client-s3');
-            const key = `${this.nounPrefix}${id}.json`;
+            // Use getNounKey() to properly handle sharding
+            const key = this.getNounKey(id);
             this.logger.trace(`Getting node ${id} from key: ${key}`);
             // Try to get the node from the nouns directory
             const response = await this.s3Client.send(new GetObjectCommand({
@@ -853,86 +864,97 @@ export class S3CompatibleStorage extends BaseStorage {
         }
     }
     /**
-     * Get nodes with pagination
+     * Get nodes with pagination using UUID-based sharding
+     *
+     * Iterates through 256 UUID-based shards (00-ff) to retrieve nodes.
+     * Cursor format: "shardIndex:s3ContinuationToken" to support pagination across shards.
+     *
      * @param options Pagination options
      * @returns Promise that resolves to a paginated result of nodes
+     *
+     * @example
+     * // First page
+     * const page1 = await getNodesWithPagination({ limit: 100 })
+     * // page1.nodes contains up to 100 nodes
+     * // page1.nextCursor might be "5:some-s3-token" (currently in shard 05)
+     *
+     * // Next page
+     * const page2 = await getNodesWithPagination({ limit: 100, cursor: page1.nextCursor })
+     * // Continues from where page1 left off
      */
     async getNodesWithPagination(options = {}) {
         await this.ensureInitialized();
         const limit = options.limit || 100;
         const useCache = options.useCache !== false;
         try {
-            // Import the ListObjectsV2Command and GetObjectCommand only when needed
             const { ListObjectsV2Command } = await import('@aws-sdk/client-s3');
-            // List objects with pagination
-            const listResponse = await this.s3Client.send(new ListObjectsV2Command({
-                Bucket: this.bucketName,
-                Prefix: this.nounPrefix,
-                MaxKeys: limit,
-                ContinuationToken: options.cursor
-            }));
-            // If listResponse is null/undefined or there are no objects, return an empty result
-            if (!listResponse ||
-                !listResponse.Contents ||
-                listResponse.Contents.length === 0) {
-                return {
-                    nodes: [],
-                    hasMore: false
-                };
-            }
-            // Extract node IDs from the keys
-            const nodeIds = listResponse.Contents
-                .filter((object) => object && object.Key)
-                .map((object) => object.Key.replace(this.nounPrefix, '').replace('.json', ''));
-            // Use the cache manager to get nodes efficiently
             const nodes = [];
-            if (useCache) {
-                // Get nodes from cache manager
-                const cachedNodes = await this.nounCacheManager.getMany(nodeIds);
-                // Add nodes to result in the same order as nodeIds
-                for (const id of nodeIds) {
-                    const node = cachedNodes.get(id);
-                    if (node) {
-                        nodes.push(node);
-                    }
-                }
-            }
-            else {
-                // Get nodes directly from S3 without using cache
-                // Process in smaller batches to reduce memory usage
-                const batchSize = 50;
-                const batches = [];
-                // Split into batches
-                for (let i = 0; i < nodeIds.length; i += batchSize) {
-                    const batch = nodeIds.slice(i, i + batchSize);
-                    batches.push(batch);
-                }
-                // Process each batch sequentially
-                for (const batch of batches) {
-                    const batchNodes = await Promise.all(batch.map(async (id) => {
-                        try {
-                            return await this.getNoun_internal(id);
-                        }
-                        catch (error) {
-                            return null;
+            // Parse cursor (format: "shardIndex:s3ContinuationToken")
+            let startShardIndex = 0;
+            let s3ContinuationToken;
+            if (options.cursor) {
+                const parts = options.cursor.split(':', 2);
+                startShardIndex = parseInt(parts[0]) || 0;
+                s3ContinuationToken = parts[1] || undefined;
+            }
+            // Iterate through shards starting from cursor position
+            for (let shardIndex = startShardIndex; shardIndex < TOTAL_SHARDS; shardIndex++) {
+                const shardId = getShardIdByIndex(shardIndex);
+                const shardPrefix = `${this.nounPrefix}${shardId}/`;
+                // List objects in this shard
+                const listResponse = await this.s3Client.send(new ListObjectsV2Command({
+                    Bucket: this.bucketName,
+                    Prefix: shardPrefix,
+                    MaxKeys: limit - nodes.length,
+                    ContinuationToken: shardIndex === startShardIndex ? s3ContinuationToken : undefined
+                }));
+                // Extract node IDs from keys
+                if (listResponse.Contents && listResponse.Contents.length > 0) {
+                    const nodeIds = listResponse.Contents
+                        .filter((obj) => obj && obj.Key)
+                        .map((obj) => {
+                        // Extract UUID from: entities/nouns/vectors/ab/ab123456-uuid.json
+                        let key = obj.Key;
+                        if (key.startsWith(shardPrefix)) {
+                            key = key.substring(shardPrefix.length);
                         }
-                    }));
-                    // Add non-null nodes to result
-                    for (const node of batchNodes) {
-                        if (node) {
-                            nodes.push(node);
+                        if (key.endsWith('.json')) {
+                            key = key.substring(0, key.length - 5);
                         }
-                    }
+                        return key;
+                    });
+                    // Load nodes for this shard (use direct loading for pagination scans)
+                    const shardNodes = await this.loadNodesByIds(nodeIds, false);
+                    nodes.push(...shardNodes);
+                }
+                // Check if we've reached the limit
+                if (nodes.length >= limit) {
+                    const hasMore = !!listResponse.IsTruncated || shardIndex < TOTAL_SHARDS - 1;
+                    const nextCursor = listResponse.IsTruncated
+                        ? `${shardIndex}:${listResponse.NextContinuationToken}`
+                        : shardIndex < TOTAL_SHARDS - 1
+                            ? `${shardIndex + 1}:`
+                            : undefined;
+                    return {
+                        nodes: nodes.slice(0, limit),
+                        hasMore,
+                        nextCursor
+                    };
+                }
+                // If this shard has more data but we haven't hit limit, continue to next shard
+                if (listResponse.IsTruncated) {
+                    return {
+                        nodes,
+                        hasMore: true,
+                        nextCursor: `${shardIndex}:${listResponse.NextContinuationToken}`
+                    };
                 }
             }
-            // Determine if there are more nodes
-            const hasMore = !!listResponse.IsTruncated;
-            // Set next cursor if there are more nodes
-            const nextCursor = listResponse.NextContinuationToken;
+            // All shards exhausted
             return {
                 nodes,
-                hasMore,
-                nextCursor
+                hasMore: false,
+                nextCursor: undefined
             };
         }
         catch (error) {
@@ -943,6 +965,43 @@ export class S3CompatibleStorage extends BaseStorage {
             };
         }
     }
+    /**
+     * Load nodes by IDs efficiently using cache or direct fetch
+     */
+    async loadNodesByIds(nodeIds, useCache) {
+        const nodes = [];
+        if (useCache) {
+            const cachedNodes = await this.nounCacheManager.getMany(nodeIds);
+            for (const id of nodeIds) {
+                const node = cachedNodes.get(id);
+                if (node) {
+                    nodes.push(node);
+                }
+            }
+        }
+        else {
+            // Load directly in batches
+            const batchSize = 50;
+            for (let i = 0; i < nodeIds.length; i += batchSize) {
+                const batch = nodeIds.slice(i, i + batchSize);
+                const batchNodes = await Promise.all(batch.map(async (id) => {
+                    try {
+                        return await this.getNoun_internal(id);
+                    }
+                    catch (error) {
+                        this.logger.warn(`Failed to load node ${id}:`, error);
+                        return null;
+                    }
+                }));
+                for (const node of batchNodes) {
+                    if (node) {
+                        nodes.push(node);
+                    }
+                }
+            }
+        }
+        return nodes;
+    }
     /**
      * Get nouns by noun type (internal implementation)
      * @param nounType The noun type to filter by
@@ -1098,7 +1157,7 @@ export class S3CompatibleStorage extends BaseStorage {
         try {
             // Import the GetObjectCommand only when needed
             const { GetObjectCommand } = await import('@aws-sdk/client-s3');
-            const key = `${this.verbPrefix}${id}.json`;
+            const key = this.getVerbKey(id);
             this.logger.trace(`Getting edge ${id} from key: ${key}`);
             // Try to get the edge from the verbs directory
             const response = await this.s3Client.send(new GetObjectCommand({
@@ -1572,7 +1631,9 @@ export class S3CompatibleStorage extends BaseStorage {
         try {
             // Import the PutObjectCommand only when needed
             const { PutObjectCommand } = await import('@aws-sdk/client-s3');
-            const key = `${this.metadataPrefix}${id}.json`;
+            // Use UUID-based sharding for metadata (consistent with noun vectors)
+            const shardId = getShardIdFromUuid(id);
+            const key = `${this.metadataPrefix}${shardId}/${id}.json`;
             const body = JSON.stringify(metadata, null, 2);
             this.logger.trace(`Saving noun metadata for ${id} to key: ${key}`);
             // Save the noun metadata to S3-compatible storage
@@ -1701,7 +1762,9 @@ export class S3CompatibleStorage extends BaseStorage {
         try {
             // Import the GetObjectCommand only when needed
             const { GetObjectCommand } = await import('@aws-sdk/client-s3');
-            const key = `${this.metadataPrefix}${id}.json`;
+            // Use UUID-based sharding for metadata (consistent with noun vectors)
+            const shardId = getShardIdFromUuid(id);
+            const key = `${this.metadataPrefix}${shardId}/${id}.json`;
             this.logger.trace(`Getting noun metadata for ${id} from key: ${key}`);
             // Try to get the noun metadata
             const response = await this.s3Client.send(new GetObjectCommand({
@@ -2698,12 +2761,54 @@ export class S3CompatibleStorage extends BaseStorage {
                 filteredNodes = filteredByMetadata;
             }
         }
+        // Calculate total count efficiently
+        // For the first page (no cursor), we can estimate total count
+        let totalCount;
+        if (!cursor) {
+            try {
+                totalCount = await this.estimateTotalNounCount();
+            }
+            catch (error) {
+                this.logger.warn('Failed to estimate total noun count:', error);
+                // totalCount remains undefined
+            }
+        }
         return {
             items: filteredNodes,
+            totalCount,
             hasMore: result.hasMore,
             nextCursor: result.nextCursor
         };
     }
+    /**
+     * Estimate total noun count by listing objects across all shards
+     * This is more efficient than loading all nouns
+     */
+    async estimateTotalNounCount() {
+        const { ListObjectsV2Command } = await import('@aws-sdk/client-s3');
+        let totalCount = 0;
+        // Count across all UUID-based shards (00-ff)
+        for (let shardIndex = 0; shardIndex < TOTAL_SHARDS; shardIndex++) {
+            const shardId = getShardIdByIndex(shardIndex);
+            const shardPrefix = `${this.nounPrefix}${shardId}/`;
+            let shardCursor;
+            let hasMore = true;
+            while (hasMore) {
+                const listResponse = await this.s3Client.send(new ListObjectsV2Command({
+                    Bucket: this.bucketName,
+                    Prefix: shardPrefix,
+                    MaxKeys: 1000,
+                    ContinuationToken: shardCursor
+                }));
+                if (listResponse.Contents) {
+                    totalCount += listResponse.Contents.length;
+                }
+                hasMore = !!listResponse.IsTruncated;
+                shardCursor = listResponse.NextContinuationToken;
+            }
+        }
+        return totalCount;
+    }
     /**
      * Initialize counts from S3 storage
      */

package/dist/storage/sharding.d.ts

@@ -0,0 +1,103 @@
+/**
+ * Unified UUID-based sharding for all storage adapters
+ *
+ * Uses first 2 hex characters of UUID for consistent, predictable sharding
+ * that scales from hundreds to millions of entities without configuration.
+ *
+ * Sharding characteristics:
+ * - 256 buckets (00-ff)
+ * - Deterministic (same UUID always maps to same shard)
+ * - No configuration required
+ * - Works across all storage types (filesystem, S3, GCS, memory)
+ * - Efficient for list operations and pagination
+ */
+/**
+ * Extract shard ID from UUID
+ *
+ * Uses first 2 hex characters of the UUID as the shard ID.
+ * This provides 256 evenly-distributed buckets (00-ff).
+ *
+ * @param uuid - UUID string (with or without hyphens)
+ * @returns 2-character hex shard ID (00-ff)
+ *
+ * @example
+ * ```typescript
+ * getShardIdFromUuid('ab123456-1234-5678-9abc-def012345678') // returns 'ab'
+ * getShardIdFromUuid('cd987654-4321-8765-cba9-fed543210987') // returns 'cd'
+ * getShardIdFromUuid('00000000-0000-0000-0000-000000000000') // returns '00'
+ * ```
+ */
+export declare function getShardIdFromUuid(uuid: string): string;
+/**
+ * Get all possible shard IDs (00-ff)
+ *
+ * Returns array of 256 shard IDs in ascending order.
+ * Useful for iterating through all shards during pagination.
+ *
+ * @returns Array of 256 shard IDs
+ *
+ * @example
+ * ```typescript
+ * const shards = getAllShardIds()
+ * // ['00', '01', '02', ..., 'fd', 'fe', 'ff']
+ *
+ * for (const shardId of shards) {
+ *   const prefix = `entities/nouns/vectors/${shardId}/`
+ *   // List objects with this prefix
+ * }
+ * ```
+ */
+export declare function getAllShardIds(): string[];
+/**
+ * Get shard ID for a given index (0-255)
+ *
+ * @param index - Shard index (0-255)
+ * @returns 2-character hex shard ID
+ *
+ * @example
+ * ```typescript
+ * getShardIdByIndex(0)   // '00'
+ * getShardIdByIndex(15)  // '0f'
+ * getShardIdByIndex(255) // 'ff'
+ * ```
+ */
+export declare function getShardIdByIndex(index: number): string;
+/**
+ * Get shard index from shard ID (0-255)
+ *
+ * @param shardId - 2-character hex shard ID
+ * @returns Shard index (0-255)
+ *
+ * @example
+ * ```typescript
+ * getShardIndexFromId('00')  // 0
+ * getShardIndexFromId('0f')  // 15
+ * getShardIndexFromId('ff')  // 255
+ * ```
+ */
+export declare function getShardIndexFromId(shardId: string): number;
+/**
+ * Total number of shards in the system
+ */
+export declare const TOTAL_SHARDS = 256;
+/**
+ * Shard configuration (read-only)
+ */
+export declare const SHARD_CONFIG: {
+    /**
+     * Total number of shards (256)
+     */
+    readonly count: 256;
+    /**
+     * Number of hex characters used for sharding (2)
+     */
+    readonly prefixLength: 2;
+    /**
+     * Sharding method description
+     */
+    readonly method: "uuid-prefix";
+    /**
+     * Whether sharding is always enabled
+     */
+    readonly alwaysEnabled: true;
+};

package/dist/storage/sharding.js

@@ -0,0 +1,137 @@
+/**
+ * Unified UUID-based sharding for all storage adapters
+ *
+ * Uses first 2 hex characters of UUID for consistent, predictable sharding
+ * that scales from hundreds to millions of entities without configuration.
+ *
+ * Sharding characteristics:
+ * - 256 buckets (00-ff)
+ * - Deterministic (same UUID always maps to same shard)
+ * - No configuration required
+ * - Works across all storage types (filesystem, S3, GCS, memory)
+ * - Efficient for list operations and pagination
+ */
+/**
+ * Extract shard ID from UUID
+ *
+ * Uses first 2 hex characters of the UUID as the shard ID.
+ * This provides 256 evenly-distributed buckets (00-ff).
+ *
+ * @param uuid - UUID string (with or without hyphens)
+ * @returns 2-character hex shard ID (00-ff)
+ *
+ * @example
+ * ```typescript
+ * getShardIdFromUuid('ab123456-1234-5678-9abc-def012345678') // returns 'ab'
+ * getShardIdFromUuid('cd987654-4321-8765-cba9-fed543210987') // returns 'cd'
+ * getShardIdFromUuid('00000000-0000-0000-0000-000000000000') // returns '00'
+ * ```
+ */
+export function getShardIdFromUuid(uuid) {
+    if (!uuid) {
+        throw new Error('UUID is required for sharding');
+    }
+    // Remove hyphens and convert to lowercase
+    const normalized = uuid.toLowerCase().replace(/-/g, '');
+    // Validate UUID format (32 hex characters)
+    if (normalized.length !== 32) {
+        throw new Error(`Invalid UUID format: ${uuid} (expected 32 hex chars, got ${normalized.length})`);
+    }
+    // Extract first 2 characters
+    const shardId = normalized.substring(0, 2);
+    // Validate hex format
+    if (!/^[0-9a-f]{2}$/.test(shardId)) {
+        throw new Error(`Invalid UUID prefix: ${shardId} (expected 2 hex chars)`);
+    }
+    return shardId;
+}
+/**
+ * Get all possible shard IDs (00-ff)
+ *
+ * Returns array of 256 shard IDs in ascending order.
+ * Useful for iterating through all shards during pagination.
+ *
+ * @returns Array of 256 shard IDs
+ *
+ * @example
+ * ```typescript
+ * const shards = getAllShardIds()
+ * // ['00', '01', '02', ..., 'fd', 'fe', 'ff']
+ *
+ * for (const shardId of shards) {
+ *   const prefix = `entities/nouns/vectors/${shardId}/`
+ *   // List objects with this prefix
+ * }
+ * ```
+ */
+export function getAllShardIds() {
+    const shards = [];
+    for (let i = 0; i < 256; i++) {
+        shards.push(i.toString(16).padStart(2, '0'));
+    }
+    return shards;
+}
+/**
+ * Get shard ID for a given index (0-255)
+ *
+ * @param index - Shard index (0-255)
+ * @returns 2-character hex shard ID
+ *
+ * @example
+ * ```typescript
+ * getShardIdByIndex(0)   // '00'
+ * getShardIdByIndex(15)  // '0f'
+ * getShardIdByIndex(255) // 'ff'
+ * ```
+ */
+export function getShardIdByIndex(index) {
+    if (index < 0 || index > 255) {
+        throw new Error(`Shard index out of range: ${index} (expected 0-255)`);
+    }
+    return index.toString(16).padStart(2, '0');
+}
+/**
+ * Get shard index from shard ID (0-255)
+ *
+ * @param shardId - 2-character hex shard ID
+ * @returns Shard index (0-255)
+ *
+ * @example
+ * ```typescript
+ * getShardIndexFromId('00')  // 0
+ * getShardIndexFromId('0f')  // 15
+ * getShardIndexFromId('ff')  // 255
+ * ```
+ */
+export function getShardIndexFromId(shardId) {
+    if (!/^[0-9a-f]{2}$/.test(shardId)) {
+        throw new Error(`Invalid shard ID: ${shardId} (expected 2 hex chars)`);
+    }
+    return parseInt(shardId, 16);
+}
+/**
+ * Total number of shards in the system
+ */
+export const TOTAL_SHARDS = 256;
+/**
+ * Shard configuration (read-only)
+ */
+export const SHARD_CONFIG = {
+    /**
+     * Total number of shards (256)
+     */
+    count: TOTAL_SHARDS,
+    /**
+     * Number of hex characters used for sharding (2)
+     */
+    prefixLength: 2,
+    /**
+     * Sharding method description
+     */
+    method: 'uuid-prefix',
+    /**
+     * Whether sharding is always enabled
+     */
+    alwaysEnabled: true
+};
+//# sourceMappingURL=sharding.js.map

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@soulcraft/brainy",
-  "version": "3.25.2",
+  "version": "3.26.0",
   "description": "Universal Knowledge Protocol™ - World's first Triple Intelligence database unifying vector, graph, and document search in one API. 31 nouns × 40 verbs for infinite expressiveness.",
   "main": "dist/index.js",
   "module": "dist/index.js",