@soulcraft/brainy 3.36.1 → 3.37.0

package/CHANGELOG.md

@@ -2,6 +2,11 @@
 
 All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines.
 
+### [3.37.0](https://github.com/soulcraftlabs/brainy/compare/v3.36.1...v3.37.0) (2025-10-10)
+
+- fix: implement 2-file storage architecture for GCS scalability (59da5f6)
+
+
 ### [3.36.1](https://github.com/soulcraftlabs/brainy/compare/v3.36.0...v3.36.1) (2025-10-10)
 
 - fix: resolve critical GCS storage bugs preventing production use (3cd0b9a)

package/dist/storage/adapters/fileSystemStorage.js

@@ -168,9 +168,14 @@ export class FileSystemStorage extends BaseStorage {
         // Check if this is a new node to update counts
         const isNew = !(await this.fileExists(this.getNodePath(node.id)));
         // Convert connections Map to a serializable format
+        // CRITICAL: Only save lightweight vector data (no metadata)
+        // Metadata is saved separately via saveNounMetadata() (2-file system)
         const serializableNode = {
-            ...node,
-            connections: this.mapToObject(node.connections, (set) => Array.from(set))
+            id: node.id,
+            vector: node.vector,
+            connections: this.mapToObject(node.connections, (set) => Array.from(set)),
+            level: node.level || 0
+            // NO metadata field - saved separately for scalability
         };
         const filePath = this.getNodePath(node.id);
         await this.ensureDirectoryExists(path.dirname(filePath));
@@ -200,12 +205,14 @@ export class FileSystemStorage extends BaseStorage {
             for (const [level, nodeIds] of Object.entries(parsedNode.connections)) {
                 connections.set(Number(level), new Set(nodeIds));
             }
+            // CRITICAL: Only return lightweight vector data (no metadata)
+            // Metadata is retrieved separately via getNounMetadata() (2-file system)
             return {
                 id: parsedNode.id,
                 vector: parsedNode.vector,
                 connections,
-                level: parsedNode.level || 0,
-                metadata: parsedNode.metadata
+                level: parsedNode.level || 0
+                // NO metadata field - retrieved separately for scalability
             };
         }
         catch (error) {
@@ -329,9 +336,13 @@ export class FileSystemStorage extends BaseStorage {
         // Check if this is a new edge to update counts
         const isNew = !(await this.fileExists(this.getVerbPath(edge.id)));
         // Convert connections Map to a serializable format
+        // CRITICAL: Only save lightweight vector data (no metadata)
+        // Metadata is saved separately via saveVerbMetadata() (2-file system)
         const serializableEdge = {
-            ...edge,
+            id: edge.id,
+            vector: edge.vector,
             connections: this.mapToObject(edge.connections, (set) => Array.from(set))
+            // NO metadata field - saved separately for scalability
         };
         const filePath = this.getVerbPath(edge.id);
         await this.ensureDirectoryExists(path.dirname(filePath));
@@ -558,7 +569,9 @@ export class FileSystemStorage extends BaseStorage {
             const batch = ids.slice(i, i + batchSize);
             const batchPromises = batch.map(async (id) => {
                 try {
-                    const metadata = await this.getMetadata(id);
+                    // CRITICAL: Use getNounMetadata() instead of deprecated getMetadata()
+                    // This ensures we fetch from the correct noun metadata store (2-file system)
+                    const metadata = await this.getNounMetadata(id);
                     return { id, metadata };
                 }
                 catch (error) {

package/dist/storage/adapters/gcsStorage.d.ts

@@ -306,6 +306,13 @@ export declare class GcsStorage extends BaseStorage {
         hasMore: boolean;
         nextCursor?: string;
     }>;
+    /**
+     * Batch fetch metadata for multiple noun IDs (efficient for large queries)
+     * Uses smaller batches to prevent GCS socket exhaustion
+     * @param ids Array of noun IDs to fetch metadata for
+     * @returns Map of ID to metadata
+     */
+    getMetadataBatch(ids: string[]): Promise<Map<string, any>>;
     /**
      * Clear all data from storage
      */

package/dist/storage/adapters/gcsStorage.js

@@ -316,12 +316,17 @@ export class GcsStorage extends BaseStorage {
         try {
             this.logger.trace(`Saving node ${node.id}`);
             // Convert connections Map to a serializable format
+            // CRITICAL: Only save lightweight vector data (no metadata)
+            // Metadata is saved separately via saveNounMetadata() (2-file system)
             const serializableNode = {
-                ...node,
+                id: node.id,
+                vector: node.vector,
                 connections: Object.fromEntries(Array.from(node.connections.entries()).map(([level, nounIds]) => [
                     level,
                     Array.from(nounIds)
-                ]))
+                ])),
+                level: node.level || 0
+                // NO metadata field - saved separately for scalability
             };
             // Get the GCS key with UUID-based sharding
             const key = this.getNounKey(node.id);
@@ -385,12 +390,14 @@ export class GcsStorage extends BaseStorage {
             for (const [level, nounIds] of Object.entries(data.connections || {})) {
                 connections.set(Number(level), new Set(nounIds));
             }
+            // CRITICAL: Only return lightweight vector data (no metadata)
+            // Metadata is retrieved separately via getNounMetadata() (2-file system)
             const node = {
                 id: data.id,
                 vector: data.vector,
                 connections,
-                level: data.level || 0,
-                metadata: data.metadata // CRITICAL: Include metadata for entity reconstruction
+                level: data.level || 0
+                // NO metadata field - retrieved separately for scalability
             };
             // Update cache
             this.nounCacheManager.set(id, node);
@@ -571,12 +578,16 @@ export class GcsStorage extends BaseStorage {
         try {
             this.logger.trace(`Saving edge ${edge.id}`);
             // Convert connections Map to serializable format
+            // CRITICAL: Only save lightweight vector data (no metadata)
+            // Metadata is saved separately via saveVerbMetadata() (2-file system)
             const serializableEdge = {
-                ...edge,
+                id: edge.id,
+                vector: edge.vector,
                 connections: Object.fromEntries(Array.from(edge.connections.entries()).map(([level, verbIds]) => [
                     level,
                     Array.from(verbIds)
                 ]))
+                // NO metadata field - saved separately for scalability
             };
             // Get the GCS key with UUID-based sharding
             const key = this.getVerbKey(edge.id);
@@ -1001,6 +1012,46 @@ export class GcsStorage extends BaseStorage {
             filter: options?.filter
         });
     }
+    /**
+     * Batch fetch metadata for multiple noun IDs (efficient for large queries)
+     * Uses smaller batches to prevent GCS socket exhaustion
+     * @param ids Array of noun IDs to fetch metadata for
+     * @returns Map of ID to metadata
+     */
+    async getMetadataBatch(ids) {
+        await this.ensureInitialized();
+        const results = new Map();
+        const batchSize = 10; // Smaller batches for metadata to prevent socket exhaustion
+        // Process in smaller batches
+        for (let i = 0; i < ids.length; i += batchSize) {
+            const batch = ids.slice(i, i + batchSize);
+            const batchPromises = batch.map(async (id) => {
+                try {
+                    // CRITICAL: Use getNounMetadata() instead of deprecated getMetadata()
+                    // This ensures we fetch from the correct noun metadata store (2-file system)
+                    const metadata = await this.getNounMetadata(id);
+                    return { id, metadata };
+                }
+                catch (error) {
+                    // Handle GCS-specific errors
+                    if (this.isThrottlingError(error)) {
+                        await this.handleThrottling(error);
+                    }
+                    this.logger.debug(`Failed to read metadata for ${id}:`, error);
+                    return { id, metadata: null };
+                }
+            });
+            const batchResults = await Promise.all(batchPromises);
+            for (const { id, metadata } of batchResults) {
+                if (metadata !== null) {
+                    results.set(id, metadata);
+                }
+            }
+            // Small yield between batches to prevent overwhelming GCS
+            await new Promise(resolve => setImmediate(resolve));
+        }
+        return results;
+    }
     /**
      * Clear all data from storage
      */

package/dist/storage/adapters/memoryStorage.js

@@ -41,12 +41,14 @@ export class MemoryStorage extends BaseStorage {
     async saveNoun_internal(noun) {
         const isNew = !this.nouns.has(noun.id);
         // Create a deep copy to avoid reference issues
+        // CRITICAL: Only save lightweight vector data (no metadata)
+        // Metadata is saved separately via saveNounMetadata() (2-file system)
         const nounCopy = {
             id: noun.id,
             vector: [...noun.vector],
             connections: new Map(),
-            level: noun.level || 0,
-            metadata: noun.metadata
+            level: noun.level || 0
+            // NO metadata field - saved separately for scalability
         };
         // Copy connections
         for (const [level, connections] of noun.connections.entries()) {
@@ -71,12 +73,14 @@ export class MemoryStorage extends BaseStorage {
             return null;
         }
         // Return a deep copy to avoid reference issues
+        // CRITICAL: Only return lightweight vector data (no metadata)
+        // Metadata is retrieved separately via getNounMetadata() (2-file system)
         const nounCopy = {
             id: noun.id,
             vector: [...noun.vector],
             connections: new Map(),
-            level: noun.level || 0,
-            metadata: noun.metadata
+            level: noun.level || 0
+            // NO metadata field - retrieved separately for scalability
         };
         // Copy connections
         for (const [level, connections] of noun.connections.entries()) {
@@ -475,7 +479,9 @@ export class MemoryStorage extends BaseStorage {
         const results = new Map();
         // Memory storage can handle all IDs at once since it's in-memory
         for (const id of ids) {
-            const metadata = await this.getMetadata(id);
+            // CRITICAL: Use getNounMetadata() instead of deprecated getMetadata()
+            // This ensures we fetch from the correct noun metadata store (2-file system)
+            const metadata = await this.getNounMetadata(id);
             if (metadata) {
                 results.set(id, metadata);
             }

package/dist/storage/adapters/opfsStorage.js

@@ -143,10 +143,14 @@ export class OPFSStorage extends BaseStorage {
     async saveNoun_internal(noun) {
         await this.ensureInitialized();
         try {
-            // Convert connections Map to a serializable format
+            // CRITICAL: Only save lightweight vector data (no metadata)
+            // Metadata is saved separately via saveNounMetadata() (2-file system)
             const serializableNoun = {
-                ...noun,
-                connections: this.mapToObject(noun.connections, (set) => Array.from(set))
+                id: noun.id,
+                vector: noun.vector,
+                connections: this.mapToObject(noun.connections, (set) => Array.from(set)),
+                level: noun.level || 0
+                // NO metadata field - saved separately for scalability
             };
             // Use UUID-based sharding for nouns
             const shardId = getShardIdFromUuid(noun.id);
@@ -299,10 +303,13 @@ export class OPFSStorage extends BaseStorage {
     async saveEdge(edge) {
         await this.ensureInitialized();
         try {
-            // Convert connections Map to a serializable format
+            // CRITICAL: Only save lightweight vector data (no metadata)
+            // Metadata is saved separately via saveVerbMetadata() (2-file system)
             const serializableEdge = {
-                ...edge,
+                id: edge.id,
+                vector: edge.vector,
                 connections: this.mapToObject(edge.connections, (set) => Array.from(set))
+                // NO metadata field - saved separately for scalability
             };
             // Use UUID-based sharding for verbs
             const shardId = getShardIdFromUuid(edge.id);

package/dist/storage/adapters/s3CompatibleStorage.js

@@ -717,9 +717,14 @@ export class S3CompatibleStorage extends BaseStorage {
         try {
             this.logger.trace(`Saving node ${node.id}`);
             // Convert connections Map to a serializable format
+            // CRITICAL: Only save lightweight vector data (no metadata)
+            // Metadata is saved separately via saveNounMetadata() (2-file system)
             const serializableNode = {
-                ...node,
-                connections: this.mapToObject(node.connections, (set) => Array.from(set))
+                id: node.id,
+                vector: node.vector,
+                connections: this.mapToObject(node.connections, (set) => Array.from(set)),
+                level: node.level || 0
+                // NO metadata field - saved separately for scalability
             };
             // Import the PutObjectCommand only when needed
             const { PutObjectCommand } = await import('@aws-sdk/client-s3');
@@ -763,6 +768,13 @@ export class S3CompatibleStorage extends BaseStorage {
             catch (verifyError) {
                 this.logger.warn(`Failed to verify node ${node.id} was saved correctly:`, verifyError);
             }
+            // Increment noun count - always increment total, and increment by type if metadata exists
+            this.totalNounCount++;
+            const metadata = await this.getNounMetadata(node.id);
+            if (metadata && metadata.type) {
+                const currentCount = this.entityCounts.get(metadata.type) || 0;
+                this.entityCounts.set(metadata.type, currentCount + 1);
+            }
             // Release backpressure on success
             this.releaseBackpressure(true, requestId);
         }
@@ -1112,9 +1124,13 @@ export class S3CompatibleStorage extends BaseStorage {
         const requestId = await this.applyBackpressure();
         try {
             // Convert connections Map to a serializable format
+            // CRITICAL: Only save lightweight vector data (no metadata)
+            // Metadata is saved separately via saveVerbMetadata() (2-file system)
             const serializableEdge = {
-                ...edge,
+                id: edge.id,
+                vector: edge.vector,
                 connections: this.mapToObject(edge.connections, (set) => Array.from(set))
+                // NO metadata field - saved separately for scalability
             };
             // Import the PutObjectCommand only when needed
             const { PutObjectCommand } = await import('@aws-sdk/client-s3');
@@ -1135,6 +1151,13 @@ export class S3CompatibleStorage extends BaseStorage {
                     vector: edge.vector
                 }
             });
+            // Increment verb count - always increment total, and increment by type if metadata exists
+            this.totalVerbCount++;
+            const metadata = await this.getVerbMetadata(edge.id);
+            if (metadata && metadata.type) {
+                const currentCount = this.verbCounts.get(metadata.type) || 0;
+                this.verbCounts.set(metadata.type, currentCount + 1);
+            }
             // Release backpressure on success
             this.releaseBackpressure(true, requestId);
         }
@@ -1643,8 +1666,10 @@ export class S3CompatibleStorage extends BaseStorage {
             const batchPromises = batch.map(async (id) => {
                 try {
                     // Add timeout wrapper for individual metadata reads
+                    // CRITICAL: Use getNounMetadata() instead of deprecated getMetadata()
+                    // This ensures we fetch from the correct noun metadata store (2-file system)
                     const metadata = await Promise.race([
-                        this.getMetadata(id),
+                        this.getNounMetadata(id),
                         new Promise((_, reject) => setTimeout(() => reject(new Error('Metadata read timeout')), 5000) // 5 second timeout
                         )
                     ]);

package/dist/storage/baseStorage.d.ts

@@ -229,6 +229,11 @@ export declare abstract class BaseStorage extends BaseStorageAdapter {
      * Uses routing logic to handle both UUIDs (sharded) and system keys (unsharded)
      */
     getNounMetadata(id: string): Promise<any | null>;
+    /**
+     * Delete noun metadata from storage
+     * Uses routing logic to handle both UUIDs (sharded) and system keys (unsharded)
+     */
+    deleteNounMetadata(id: string): Promise<void>;
     /**
      * Save verb metadata to storage
      * Routes to correct sharded location based on UUID

package/dist/storage/baseStorage.js

@@ -141,11 +141,33 @@ export class BaseStorage extends BaseStorageAdapter {
     async saveNoun(noun) {
         await this.ensureInitialized();
         // Validate noun type before saving - storage boundary protection
-        const metadata = await this.getNounMetadata(noun.id);
-        if (metadata?.noun) {
-            validateNounType(metadata.noun);
+        if (noun.metadata?.noun) {
+            validateNounType(noun.metadata.noun);
+        }
+        // Save both the HNSWNoun vector data and metadata separately (2-file system)
+        try {
+            // Save the lightweight HNSWNoun vector file first
+            await this.saveNoun_internal(noun);
+            // Then save the metadata to separate file (if present)
+            if (noun.metadata) {
+                await this.saveNounMetadata(noun.id, noun.metadata);
+            }
+        }
+        catch (error) {
+            console.error(`[ERROR] Failed to save noun ${noun.id}:`, error);
+            // Attempt cleanup - remove noun file if metadata failed
+            try {
+                const nounExists = await this.getNoun_internal(noun.id);
+                if (nounExists) {
+                    console.log(`[CLEANUP] Attempting to remove orphaned noun file ${noun.id}`);
+                    await this.deleteNoun_internal(noun.id);
+                }
+            }
+            catch (cleanupError) {
+                console.error(`[ERROR] Failed to cleanup orphaned noun ${noun.id}:`, cleanupError);
+            }
+            throw new Error(`Failed to save noun ${noun.id}: ${error instanceof Error ? error.message : String(error)}`);
         }
-        return this.saveNoun_internal(noun);
     }
     /**
      * Get a noun from storage
@@ -168,7 +190,16 @@ export class BaseStorage extends BaseStorageAdapter {
      */
     async deleteNoun(id) {
         await this.ensureInitialized();
-        return this.deleteNoun_internal(id);
+        // Delete both the vector file and metadata file (2-file system)
+        await this.deleteNoun_internal(id);
+        // Delete metadata file (if it exists)
+        try {
+            await this.deleteNounMetadata(id);
+        }
+        catch (error) {
+            // Ignore if metadata file doesn't exist
+            console.debug(`No metadata file to delete for noun ${id}`);
+        }
     }
     /**
      * Save a verb to storage
@@ -618,7 +649,16 @@ export class BaseStorage extends BaseStorageAdapter {
      */
     async deleteVerb(id) {
         await this.ensureInitialized();
-        return this.deleteVerb_internal(id);
+        // Delete both the vector file and metadata file (2-file system)
+        await this.deleteVerb_internal(id);
+        // Delete metadata file (if it exists)
+        try {
+            await this.deleteVerbMetadata(id);
+        }
+        catch (error) {
+            // Ignore if metadata file doesn't exist
+            console.debug(`No metadata file to delete for verb ${id}`);
+        }
     }
     /**
      * Get graph index (lazy initialization)
@@ -684,6 +724,15 @@ export class BaseStorage extends BaseStorageAdapter {
         const keyInfo = this.analyzeKey(id, 'noun-metadata');
         return this.readObjectFromPath(keyInfo.fullPath);
     }
+    /**
+     * Delete noun metadata from storage
+     * Uses routing logic to handle both UUIDs (sharded) and system keys (unsharded)
+     */
+    async deleteNounMetadata(id) {
+        await this.ensureInitialized();
+        const keyInfo = this.analyzeKey(id, 'noun-metadata');
+        return this.deleteObjectFromPath(keyInfo.fullPath);
+    }
     /**
      * Save verb metadata to storage
      * Routes to correct sharded location based on UUID

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@soulcraft/brainy",
-  "version": "3.36.1",
+  "version": "3.37.0",
   "description": "Universal Knowledge Protocol™ - World's first Triple Intelligence database unifying vector, graph, and document search in one API. 31 nouns × 40 verbs for infinite expressiveness.",
   "main": "dist/index.js",
   "module": "dist/index.js",