@soulcraft/brainy 3.25.2 → 3.27.0

package/dist/storage/adapters/opfsStorage.js

@@ -3,6 +3,7 @@
  * Provides persistent storage for the vector database using the Origin Private File System API
  */
 import { BaseStorage, NOUNS_DIR, VERBS_DIR, METADATA_DIR, NOUN_METADATA_DIR, VERB_METADATA_DIR, INDEX_DIR } from '../baseStorage.js';
+import { getShardIdFromUuid } from '../sharding.js';
 import '../../types/fileSystemTypes.js';
 /**
  * Helper function to safely get a file from a FileSystemHandle
@@ -145,8 +146,14 @@ export class OPFSStorage extends BaseStorage {
                 ...noun,
                 connections: this.mapToObject(noun.connections, (set) => Array.from(set))
             };
-            // Create or get the file for this noun
-            const fileHandle = await this.nounsDir.getFileHandle(`${noun.id}.json`, {
+            // Use UUID-based sharding for nouns
+            const shardId = getShardIdFromUuid(noun.id);
+            // Get or create the shard directory
+            const shardDir = await this.nounsDir.getDirectoryHandle(shardId, {
+                create: true
+            });
+            // Create or get the file in the shard directory
+            const fileHandle = await shardDir.getFileHandle(`${noun.id}.json`, {
                 create: true
             });
             // Write the noun data to the file
@@ -165,8 +172,12 @@ export class OPFSStorage extends BaseStorage {
     async getNoun_internal(id) {
         await this.ensureInitialized();
         try {
-            // Get the file handle for this noun
-            const fileHandle = await this.nounsDir.getFileHandle(`${id}.json`);
+            // Use UUID-based sharding for nouns
+            const shardId = getShardIdFromUuid(id);
+            // Get the shard directory
+            const shardDir = await this.nounsDir.getDirectoryHandle(shardId);
+            // Get the file handle from the shard directory
+            const fileHandle = await shardDir.getFileHandle(`${id}.json`);
             // Read the noun data from the file
             const file = await fileHandle.getFile();
             const text = await file.text();
@@ -205,34 +216,40 @@ export class OPFSStorage extends BaseStorage {
         await this.ensureInitialized();
         const nodes = [];
         try {
-            // Iterate through all files in the nouns directory
-            for await (const [name, handle] of this.nounsDir.entries()) {
-                if (handle.kind === 'file') {
-                    try {
-                        // Read the node data from the file
-                        const file = await safeGetFile(handle);
-                        const text = await file.text();
-                        const data = JSON.parse(text);
-                        // Get the metadata to check the noun type
-                        const metadata = await this.getMetadata(data.id);
-                        // Include the node if its noun type matches the requested type
-                        if (metadata && metadata.noun === nounType) {
-                            // Convert serialized connections back to Map<number, Set<string>>
-                            const connections = new Map();
-                            for (const [level, nodeIds] of Object.entries(data.connections)) {
-                                connections.set(Number(level), new Set(nodeIds));
+            // Iterate through all shard directories
+            for await (const [shardName, shardHandle] of this.nounsDir.entries()) {
+                if (shardHandle.kind === 'directory') {
+                    const shardDir = shardHandle;
+                    // Iterate through all files in this shard
+                    for await (const [fileName, fileHandle] of shardDir.entries()) {
+                        if (fileHandle.kind === 'file') {
+                            try {
+                                // Read the node data from the file
+                                const file = await safeGetFile(fileHandle);
+                                const text = await file.text();
+                                const data = JSON.parse(text);
+                                // Get the metadata to check the noun type
+                                const metadata = await this.getMetadata(data.id);
+                                // Include the node if its noun type matches the requested type
+                                if (metadata && metadata.noun === nounType) {
+                                    // Convert serialized connections back to Map<number, Set<string>>
+                                    const connections = new Map();
+                                    for (const [level, nodeIds] of Object.entries(data.connections)) {
+                                        connections.set(Number(level), new Set(nodeIds));
+                                    }
+                                    nodes.push({
+                                        id: data.id,
+                                        vector: data.vector,
+                                        connections,
+                                        level: data.level || 0
+                                    });
+                                }
+                            }
+                            catch (error) {
+                                console.error(`Error reading node file ${shardName}/${fileName}:`, error);
                             }
-                            nodes.push({
-                                id: data.id,
-                                vector: data.vector,
-                                connections,
-                                level: data.level || 0
-                            });
                         }
                     }
-                    catch (error) {
-                        console.error(`Error reading node file ${name}:`, error);
-                    }
                 }
             }
         }
@@ -253,7 +270,12 @@ export class OPFSStorage extends BaseStorage {
     async deleteNode(id) {
         await this.ensureInitialized();
         try {
-            await this.nounsDir.removeEntry(`${id}.json`);
+            // Use UUID-based sharding for nouns
+            const shardId = getShardIdFromUuid(id);
+            // Get the shard directory
+            const shardDir = await this.nounsDir.getDirectoryHandle(shardId);
+            // Delete the file from the shard directory
+            await shardDir.removeEntry(`${id}.json`);
         }
         catch (error) {
             // Ignore NotFoundError, which means the file doesn't exist
@@ -280,8 +302,14 @@ export class OPFSStorage extends BaseStorage {
                 ...edge,
                 connections: this.mapToObject(edge.connections, (set) => Array.from(set))
             };
-            // Create or get the file for this verb
-            const fileHandle = await this.verbsDir.getFileHandle(`${edge.id}.json`, {
+            // Use UUID-based sharding for verbs
+            const shardId = getShardIdFromUuid(edge.id);
+            // Get or create the shard directory
+            const shardDir = await this.verbsDir.getDirectoryHandle(shardId, {
+                create: true
+            });
+            // Create or get the file in the shard directory
+            const fileHandle = await shardDir.getFileHandle(`${edge.id}.json`, {
                 create: true
             });
             // Write the verb data to the file
@@ -306,8 +334,12 @@ export class OPFSStorage extends BaseStorage {
     async getEdge(id) {
         await this.ensureInitialized();
         try {
-            // Get the file handle for this edge
-            const fileHandle = await this.verbsDir.getFileHandle(`${id}.json`);
+            // Use UUID-based sharding for verbs
+            const shardId = getShardIdFromUuid(id);
+            // Get the shard directory
+            const shardDir = await this.verbsDir.getDirectoryHandle(shardId);
+            // Get the file handle from the shard directory
+            const fileHandle = await shardDir.getFileHandle(`${id}.json`);
             // Read the edge data from the file
             const file = await fileHandle.getFile();
             const text = await file.text();
@@ -345,37 +377,43 @@ export class OPFSStorage extends BaseStorage {
         await this.ensureInitialized();
         const allEdges = [];
         try {
-            // Iterate through all files in the verbs directory
-            for await (const [name, handle] of this.verbsDir.entries()) {
-                if (handle.kind === 'file') {
-                    try {
-                        // Read the edge data from the file
-                        const file = await safeGetFile(handle);
-                        const text = await file.text();
-                        const data = JSON.parse(text);
-                        // Convert serialized connections back to Map<number, Set<string>>
-                        const connections = new Map();
-                        for (const [level, nodeIds] of Object.entries(data.connections)) {
-                            connections.set(Number(level), new Set(nodeIds));
+            // Iterate through all shard directories
+            for await (const [shardName, shardHandle] of this.verbsDir.entries()) {
+                if (shardHandle.kind === 'directory') {
+                    const shardDir = shardHandle;
+                    // Iterate through all files in this shard
+                    for await (const [fileName, fileHandle] of shardDir.entries()) {
+                        if (fileHandle.kind === 'file') {
+                            try {
+                                // Read the edge data from the file
+                                const file = await safeGetFile(fileHandle);
+                                const text = await file.text();
+                                const data = JSON.parse(text);
+                                // Convert serialized connections back to Map<number, Set<string>>
+                                const connections = new Map();
+                                for (const [level, nodeIds] of Object.entries(data.connections)) {
+                                    connections.set(Number(level), new Set(nodeIds));
+                                }
+                                // Create default timestamp if not present
+                                const defaultTimestamp = {
+                                    seconds: Math.floor(Date.now() / 1000),
+                                    nanoseconds: (Date.now() % 1000) * 1000000
+                                };
+                                // Create default createdBy if not present
+                                const defaultCreatedBy = {
+                                    augmentation: 'unknown',
+                                    version: '1.0'
+                                };
+                                allEdges.push({
+                                    id: data.id,
+                                    vector: data.vector,
+                                    connections
+                                });
+                            }
+                            catch (error) {
+                                console.error(`Error reading edge file ${shardName}/${fileName}:`, error);
+                            }
                         }
-                        // Create default timestamp if not present
-                        const defaultTimestamp = {
-                            seconds: Math.floor(Date.now() / 1000),
-                            nanoseconds: (Date.now() % 1000) * 1000000
-                        };
-                        // Create default createdBy if not present
-                        const defaultCreatedBy = {
-                            augmentation: 'unknown',
-                            version: '1.0'
-                        };
-                        allEdges.push({
-                            id: data.id,
-                            vector: data.vector,
-                            connections
-                        });
-                    }
-                    catch (error) {
-                        console.error(`Error reading edge file ${name}:`, error);
                     }
                 }
             }
@@ -457,7 +495,12 @@ export class OPFSStorage extends BaseStorage {
     async deleteEdge(id) {
         await this.ensureInitialized();
         try {
-            await this.verbsDir.removeEntry(`${id}.json`);
+            // Use UUID-based sharding for verbs
+            const shardId = getShardIdFromUuid(id);
+            // Get the shard directory
+            const shardDir = await this.verbsDir.getDirectoryHandle(shardId);
+            // Delete the file from the shard directory
+            await shardDir.removeEntry(`${id}.json`);
         }
         catch (error) {
             // Ignore NotFoundError, which means the file doesn't exist
@@ -542,8 +585,13 @@ export class OPFSStorage extends BaseStorage {
      */
     async saveVerbMetadata_internal(id, metadata) {
         await this.ensureInitialized();
+        // Use UUID-based sharding for metadata (consistent with verb vectors)
+        const shardId = getShardIdFromUuid(id);
+        // Get or create the shard directory
+        const shardDir = await this.verbMetadataDir.getDirectoryHandle(shardId, { create: true });
+        // Create or get the file in the shard directory
         const fileName = `${id}.json`;
-        const fileHandle = await this.verbMetadataDir.getFileHandle(fileName, { create: true });
+        const fileHandle = await shardDir.getFileHandle(fileName, { create: true });
         const writable = await fileHandle.createWritable();
         await writable.write(JSON.stringify(metadata, null, 2));
         await writable.close();
@@ -553,9 +601,14 @@ export class OPFSStorage extends BaseStorage {
      */
     async getVerbMetadata(id) {
         await this.ensureInitialized();
+        // Use UUID-based sharding for metadata (consistent with verb vectors)
+        const shardId = getShardIdFromUuid(id);
         const fileName = `${id}.json`;
         try {
-            const fileHandle = await this.verbMetadataDir.getFileHandle(fileName);
+            // Get the shard directory
+            const shardDir = await this.verbMetadataDir.getDirectoryHandle(shardId);
+            // Get the file from the shard directory
+            const fileHandle = await shardDir.getFileHandle(fileName);
             const file = await safeGetFile(fileHandle);
             const text = await file.text();
             return JSON.parse(text);
@@ -572,8 +625,13 @@ export class OPFSStorage extends BaseStorage {
      */
     async saveNounMetadata_internal(id, metadata) {
         await this.ensureInitialized();
+        // Use UUID-based sharding for metadata (consistent with noun vectors)
+        const shardId = getShardIdFromUuid(id);
+        // Get or create the shard directory
+        const shardDir = await this.nounMetadataDir.getDirectoryHandle(shardId, { create: true });
+        // Create or get the file in the shard directory
         const fileName = `${id}.json`;
-        const fileHandle = await this.nounMetadataDir.getFileHandle(fileName, { create: true });
+        const fileHandle = await shardDir.getFileHandle(fileName, { create: true });
         const writable = await fileHandle.createWritable();
         await writable.write(JSON.stringify(metadata, null, 2));
         await writable.close();
@@ -583,9 +641,14 @@ export class OPFSStorage extends BaseStorage {
      */
     async getNounMetadata(id) {
         await this.ensureInitialized();
+        // Use UUID-based sharding for metadata (consistent with noun vectors)
+        const shardId = getShardIdFromUuid(id);
         const fileName = `${id}.json`;
         try {
-            const fileHandle = await this.nounMetadataDir.getFileHandle(fileName);
+            // Get the shard directory
+            const shardDir = await this.nounMetadataDir.getDirectoryHandle(shardId);
+            // Get the file from the shard directory
+            const fileHandle = await shardDir.getFileHandle(fileName);
             const file = await safeGetFile(fileHandle);
             const text = await file.text();
             return JSON.parse(text);
@@ -1117,12 +1180,19 @@ export class OPFSStorage extends BaseStorage {
         await this.ensureInitialized();
         const limit = options.limit || 100;
         const cursor = options.cursor;
-        // Get all noun files
+        // Get all noun files from all shards
         const nounFiles = [];
         if (this.nounsDir) {
-            for await (const [name, handle] of this.nounsDir.entries()) {
-                if (handle.kind === 'file' && name.endsWith('.json')) {
-                    nounFiles.push(name);
+            // Iterate through all shard directories
+            for await (const [shardName, shardHandle] of this.nounsDir.entries()) {
+                if (shardHandle.kind === 'directory') {
+                    // Iterate through files in this shard
+                    const shardDir = shardHandle;
+                    for await (const [fileName, fileHandle] of shardDir.entries()) {
+                        if (fileHandle.kind === 'file' && fileName.endsWith('.json')) {
+                            nounFiles.push(`${shardName}/${fileName}`);
+                        }
+                    }
                 }
             }
         }
@@ -1141,7 +1211,8 @@ export class OPFSStorage extends BaseStorage {
         // Load nouns from files
         const items = [];
         for (const fileName of pageFiles) {
-            const id = fileName.replace('.json', '');
+            // fileName is in format "shard/uuid.json", extract just the UUID
+            const id = fileName.split('/')[1].replace('.json', '');
             const noun = await this.getNoun_internal(id);
             if (noun) {
                 // Apply filters if provided
@@ -1205,12 +1276,19 @@ export class OPFSStorage extends BaseStorage {
         await this.ensureInitialized();
         const limit = options.limit || 100;
         const cursor = options.cursor;
-        // Get all verb files
+        // Get all verb files from all shards
         const verbFiles = [];
         if (this.verbsDir) {
-            for await (const [name, handle] of this.verbsDir.entries()) {
-                if (handle.kind === 'file' && name.endsWith('.json')) {
-                    verbFiles.push(name);
+            // Iterate through all shard directories
+            for await (const [shardName, shardHandle] of this.verbsDir.entries()) {
+                if (shardHandle.kind === 'directory') {
+                    // Iterate through files in this shard
+                    const shardDir = shardHandle;
+                    for await (const [fileName, fileHandle] of shardDir.entries()) {
+                        if (fileHandle.kind === 'file' && fileName.endsWith('.json')) {
+                            verbFiles.push(`${shardName}/${fileName}`);
+                        }
+                    }
                 }
             }
         }
@@ -1229,7 +1307,8 @@ export class OPFSStorage extends BaseStorage {
         // Load verbs from files and convert to GraphVerb
         const items = [];
         for (const fileName of pageFiles) {
-            const id = fileName.replace('.json', '');
+            // fileName is in format "shard/uuid.json", extract just the UUID
+            const id = fileName.split('/')[1].replace('.json', '');
             const hnswVerb = await this.getVerb_internal(id);
             if (hnswVerb) {
                 // Convert HNSWVerb to GraphVerb
@@ -1330,16 +1409,26 @@ export class OPFSStorage extends BaseStorage {
      */
     async initializeCountsFromScan() {
         try {
-            // Count nouns
+            // Count nouns across all shards
             let nounCount = 0;
-            for await (const [,] of this.nounsDir.entries()) {
-                nounCount++;
+            for await (const [shardName, shardHandle] of this.nounsDir.entries()) {
+                if (shardHandle.kind === 'directory') {
+                    const shardDir = shardHandle;
+                    for await (const [,] of shardDir.entries()) {
+                        nounCount++;
+                    }
+                }
             }
             this.totalNounCount = nounCount;
-            // Count verbs
+            // Count verbs across all shards
             let verbCount = 0;
-            for await (const [,] of this.verbsDir.entries()) {
-                verbCount++;
+            for await (const [shardName, shardHandle] of this.verbsDir.entries()) {
+                if (shardHandle.kind === 'directory') {
+                    const shardDir = shardHandle;
+                    for await (const [,] of shardDir.entries()) {
+                        verbCount++;
+                    }
+                }
             }
             this.totalVerbCount = verbCount;
             // Save initial counts

package/dist/storage/adapters/s3CompatibleStorage.d.ts

@@ -73,7 +73,6 @@ export declare class S3CompatibleStorage extends BaseStorage {
     private nounWriteBuffer;
     private verbWriteBuffer;
     private coordinator?;
-    private shardManager?;
     private cacheSync?;
     private readWriteSeparation?;
     private requestCoalescer;
@@ -112,7 +111,9 @@ export declare class S3CompatibleStorage extends BaseStorage {
     init(): Promise<void>;
     /**
      * Set distributed components for multi-node coordination
-     * Zero-config: Automatically optimizes based on components provided
+     *
+     * Note: Sharding is always enabled via UUID-based prefixes (00-ff).
+     * ShardManager is no longer required - sharding is deterministic based on UUID.
      */
     setDistributedComponents(components: {
         coordinator?: any;
@@ -121,11 +122,25 @@ export declare class S3CompatibleStorage extends BaseStorage {
         readWriteSeparation?: any;
     }): void;
     /**
-     * Get the S3 key for a noun, using sharding if available
+     * Get the S3 key for a noun using UUID-based sharding
+     *
+     * Uses first 2 hex characters of UUID for consistent sharding.
+     * Path format: entities/nouns/vectors/{shardId}/{uuid}.json
+     *
+     * @example
+     * getNounKey('ab123456-1234-5678-9abc-def012345678')
+     * // returns 'entities/nouns/vectors/ab/ab123456-1234-5678-9abc-def012345678.json'
      */
     private getNounKey;
     /**
-     * Get the S3 key for a verb, using sharding if available
+     * Get the S3 key for a verb using UUID-based sharding
+     *
+     * Uses first 2 hex characters of UUID for consistent sharding.
+     * Path format: verbs/{shardId}/{uuid}.json
+     *
+     * @example
+     * getVerbKey('cd987654-4321-8765-cba9-fed543210987')
+     * // returns 'verbs/cd/cd987654-4321-8765-cba9-fed543210987.json'
      */
     private getVerbKey;
     /**
@@ -221,9 +236,23 @@ export declare class S3CompatibleStorage extends BaseStorage {
      */
     protected getAllNodes(): Promise<HNSWNode[]>;
     /**
-     * Get nodes with pagination
+     * Get nodes with pagination using UUID-based sharding
+     *
+     * Iterates through 256 UUID-based shards (00-ff) to retrieve nodes.
+     * Cursor format: "shardIndex:s3ContinuationToken" to support pagination across shards.
+     *
      * @param options Pagination options
      * @returns Promise that resolves to a paginated result of nodes
+     *
+     * @example
+     * // First page
+     * const page1 = await getNodesWithPagination({ limit: 100 })
+     * // page1.nodes contains up to 100 nodes
+     * // page1.nextCursor might be "5:some-s3-token" (currently in shard 05)
+     *
+     * // Next page
+     * const page2 = await getNodesWithPagination({ limit: 100, cursor: page1.nextCursor })
+     * // Continues from where page1 left off
      */
     protected getNodesWithPagination(options?: {
         limit?: number;
@@ -234,6 +263,10 @@ export declare class S3CompatibleStorage extends BaseStorage {
         hasMore: boolean;
         nextCursor?: string;
     }>;
+    /**
+     * Load nodes by IDs efficiently using cache or direct fetch
+     */
+    private loadNodesByIds;
     /**
      * Get nouns by noun type (internal implementation)
      * @param nounType The noun type to filter by
@@ -517,6 +550,11 @@ export declare class S3CompatibleStorage extends BaseStorage {
         hasMore: boolean;
         nextCursor?: string;
     }>;
+    /**
+     * Estimate total noun count by listing objects across all shards
+     * This is more efficient than loading all nouns
+     */
+    private estimateTotalNounCount;
     /**
      * Initialize counts from S3 storage
      */