@soulcraft/brainy 3.30.0 → 3.30.2

package/CHANGELOG.md

@@ -2,6 +2,58 @@
 
 All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines.
 
+### [3.30.2](https://github.com/soulcraftlabs/brainy/compare/v3.30.1...v3.30.2) (2025-10-09)
+
+- chore: update dependencies to latest safe versions (053f292)
+
+
+### [3.30.1](https://github.com/soulcraftlabs/brainy/compare/v3.30.0...v3.30.1) (2025-10-09)
+
+- fix: move metadata routing to base class, fix GCS/S3 system key crashes (1966c39)
+
+
+### [3.30.1] - Critical Storage Architecture Fix (2025-10-09)
+
+#### 🐛 Critical Bug Fixes
+
+**Fixed: GCS/S3 Storage Crash on System Metadata Keys**
+- GCS and S3 native adapters were crashing with "Invalid UUID format" errors when saving metadata index keys
+- Root cause: Storage adapters incorrectly assumed ALL metadata keys are UUIDs
+- System keys like `__metadata_field_index__status` and `statistics_` are NOT UUIDs and should not be sharded
+
+**Architecture Improvement: Base Class Enforcement Pattern**
+- Moved sharding/routing logic from individual adapters to BaseStorage class
+- All adapters now implement 4 primitive operations instead of metadata-specific methods:
+  - `writeObjectToPath(path, data)` - Write any object to storage
+  - `readObjectFromPath(path)` - Read any object from storage
+  - `deleteObjectFromPath(path)` - Delete object from storage
+  - `listObjectsUnderPath(prefix)` - List objects under path prefix
+- BaseStorage.analyzeKey() now routes ALL metadata operations through primitive layer
+- System keys automatically routed to `_system/` directory (no sharding)
+- Entity UUIDs automatically sharded to `entities/{type}/metadata/{shard}/` directories
+
+**Benefits:**
+- Impossible for future adapters to make the same mistake
+- Cleaner separation of concerns (routing vs. storage primitives)
+- Zero breaking changes for users
+- No data migration required
+- Full backward compatibility maintained
+
+**Updated Adapters:**
+- GcsStorage: Implements primitive operations using GCS bucket.file() API
+- S3CompatibleStorage: Implements primitive operations using AWS SDK
+- OPFSStorage: Implements primitive operations using browser FileSystem API
+- FileSystemStorage: Implements primitive operations using Node.js fs.promises
+- MemoryStorage: Implements primitive operations using Map data structures
+
+**Documentation:**
+- Added comprehensive storage architecture documentation: `docs/architecture/data-storage-architecture.md`
+- Linked from README for easy discovery
+
+**Impact:** CRITICAL FIX - GCS/S3 native storage now fully functional for metadata indexing
+
+---
+
 ### [3.30.0](https://github.com/soulcraftlabs/brainy/compare/v3.29.1...v3.30.0) (2025-10-09)
 
 - feat: remove legacy ImportManager, standardize getStats() API (58daf09)

package/README.md

@@ -888,6 +888,7 @@ We welcome contributions! See [CONTRIBUTING.md](CONTRIBUTING.md) for guidelines.
 - [Getting Started Guide](docs/guides/getting-started.md)
 - [API Reference](docs/api/README.md)
 - [Architecture Overview](docs/architecture/overview.md)
+- [Data Storage Architecture](docs/architecture/data-storage-architecture.md) - Deep dive into storage, indexing, and sharding
 - [Natural Language Guide](docs/guides/natural-language.md)
 - [Triple Intelligence](docs/architecture/triple-intelligence.md)
 - [Noun-Verb Taxonomy](docs/architecture/noun-verb-taxonomy.md)

package/dist/storage/adapters/fileSystemStorage.d.ts

@@ -96,34 +96,30 @@ export declare class FileSystemStorage extends BaseStorage {
      */
     protected deleteEdge(id: string): Promise<void>;
     /**
-     * Save metadata to storage
+     * Primitive operation: Write object to path
+     * All metadata operations use this internally via base class routing
      */
-    saveMetadata(id: string, metadata: any): Promise<void>;
+    protected writeObjectToPath(pathStr: string, data: any): Promise<void>;
     /**
-     * Get metadata from storage
+     * Primitive operation: Read object from path
+     * All metadata operations use this internally via base class routing
      */
-    getMetadata(id: string): Promise<any | null>;
+    protected readObjectFromPath(pathStr: string): Promise<any | null>;
     /**
-     * Get multiple metadata objects in batches (CRITICAL: Prevents socket exhaustion)
-     * FileSystem implementation uses controlled concurrency to prevent too many file reads
-     */
-    getMetadataBatch(ids: string[]): Promise<Map<string, any>>;
-    /**
-     * Save noun metadata to storage
+     * Primitive operation: Delete object from path
+     * All metadata operations use this internally via base class routing
      */
-    protected saveNounMetadata_internal(id: string, metadata: any): Promise<void>;
+    protected deleteObjectFromPath(pathStr: string): Promise<void>;
     /**
-     * Get noun metadata from storage
+     * Primitive operation: List objects under path prefix
+     * All metadata operations use this internally via base class routing
      */
-    getNounMetadata(id: string): Promise<any | null>;
+    protected listObjectsUnderPath(prefix: string): Promise<string[]>;
     /**
-     * Save verb metadata to storage
-     */
-    protected saveVerbMetadata_internal(id: string, metadata: any): Promise<void>;
-    /**
-     * Get verb metadata from storage
+     * Get multiple metadata objects in batches (CRITICAL: Prevents socket exhaustion)
+     * FileSystem implementation uses controlled concurrency to prevent too many file reads
      */
-    getVerbMetadata(id: string): Promise<any | null>;
+    getMetadataBatch(ids: string[]): Promise<Map<string, any>>;
     /**
      * Get nouns with pagination support
      * @param options Pagination options

package/dist/storage/adapters/fileSystemStorage.js

@@ -449,30 +449,79 @@ export class FileSystemStorage extends BaseStorage {
         }
     }
     /**
-     * Save metadata to storage
+     * Primitive operation: Write object to path
+     * All metadata operations use this internally via base class routing
      */
-    async saveMetadata(id, metadata) {
+    async writeObjectToPath(pathStr, data) {
         await this.ensureInitialized();
-        const filePath = path.join(this.metadataDir, `${id}.json`);
-        await fs.promises.writeFile(filePath, JSON.stringify(metadata, null, 2));
+        const fullPath = path.join(this.rootDir, pathStr);
+        await this.ensureDirectoryExists(path.dirname(fullPath));
+        await fs.promises.writeFile(fullPath, JSON.stringify(data, null, 2));
     }
     /**
-     * Get metadata from storage
+     * Primitive operation: Read object from path
+     * All metadata operations use this internally via base class routing
      */
-    async getMetadata(id) {
+    async readObjectFromPath(pathStr) {
         await this.ensureInitialized();
-        const filePath = path.join(this.metadataDir, `${id}.json`);
+        const fullPath = path.join(this.rootDir, pathStr);
         try {
-            const data = await fs.promises.readFile(filePath, 'utf-8');
+            const data = await fs.promises.readFile(fullPath, 'utf-8');
             return JSON.parse(data);
         }
         catch (error) {
-            if (error.code !== 'ENOENT') {
-                console.error(`Error reading metadata ${id}:`, error);
+            if (error.code === 'ENOENT') {
+                return null;
             }
+            console.error(`Error reading object from ${pathStr}:`, error);
             return null;
         }
     }
+    /**
+     * Primitive operation: Delete object from path
+     * All metadata operations use this internally via base class routing
+     */
+    async deleteObjectFromPath(pathStr) {
+        await this.ensureInitialized();
+        const fullPath = path.join(this.rootDir, pathStr);
+        try {
+            await fs.promises.unlink(fullPath);
+        }
+        catch (error) {
+            if (error.code !== 'ENOENT') {
+                console.error(`Error deleting object from ${pathStr}:`, error);
+                throw error;
+            }
+        }
+    }
+    /**
+     * Primitive operation: List objects under path prefix
+     * All metadata operations use this internally via base class routing
+     */
+    async listObjectsUnderPath(prefix) {
+        await this.ensureInitialized();
+        const fullPath = path.join(this.rootDir, prefix);
+        const paths = [];
+        try {
+            const entries = await fs.promises.readdir(fullPath, { withFileTypes: true });
+            for (const entry of entries) {
+                if (entry.isFile() && entry.name.endsWith('.json')) {
+                    paths.push(path.join(prefix, entry.name));
+                }
+                else if (entry.isDirectory()) {
+                    const subdirPaths = await this.listObjectsUnderPath(path.join(prefix, entry.name));
+                    paths.push(...subdirPaths);
+                }
+            }
+            return paths.sort();
+        }
+        catch (error) {
+            if (error.code === 'ENOENT') {
+                return [];
+            }
+            throw error;
+        }
+    }
     /**
      * Get multiple metadata objects in batches (CRITICAL: Prevents socket exhaustion)
      * FileSystem implementation uses controlled concurrency to prevent too many file reads
@@ -486,7 +535,7 @@ export class FileSystemStorage extends BaseStorage {
             const batch = ids.slice(i, i + batchSize);
             const batchPromises = batch.map(async (id) => {
                 try {
-                    const metadata = await this.getNounMetadata(id);
+                    const metadata = await this.getMetadata(id);
                     return { id, metadata };
                 }
                 catch (error) {
@@ -505,75 +554,6 @@ export class FileSystemStorage extends BaseStorage {
         }
         return results;
     }
-    /**
-     * Save noun metadata to storage
-     */
-    async saveNounMetadata_internal(id, metadata) {
-        await this.ensureInitialized();
-        // Use UUID-based sharding for metadata (consistent with noun vectors)
-        const filePath = this.getShardedPath(this.nounMetadataDir, id);
-        // Ensure shard directory exists
-        const shardDir = path.dirname(filePath);
-        await fs.promises.mkdir(shardDir, { recursive: true });
-        await fs.promises.writeFile(filePath, JSON.stringify(metadata, null, 2));
-    }
-    /**
-     * Get noun metadata from storage
-     */
-    async getNounMetadata(id) {
-        await this.ensureInitialized();
-        // Use UUID-based sharding for metadata (consistent with noun vectors)
-        const filePath = this.getShardedPath(this.nounMetadataDir, id);
-        try {
-            const data = await fs.promises.readFile(filePath, 'utf-8');
-            return JSON.parse(data);
-        }
-        catch (error) {
-            if (error.code !== 'ENOENT') {
-                console.error(`Error reading noun metadata ${id}:`, error);
-            }
-            return null;
-        }
-    }
-    /**
-     * Save verb metadata to storage
-     */
-    async saveVerbMetadata_internal(id, metadata) {
-        await this.ensureInitialized();
-        console.log(`[DEBUG] Saving verb metadata for ${id} to: ${this.verbMetadataDir}`);
-        const filePath = path.join(this.verbMetadataDir, `${id}.json`);
-        console.log(`[DEBUG] Full file path: ${filePath}`);
-        try {
-            await this.ensureDirectoryExists(path.dirname(filePath));
-            console.log(`[DEBUG] Directory ensured: ${path.dirname(filePath)}`);
-            await fs.promises.writeFile(filePath, JSON.stringify(metadata, null, 2));
-            console.log(`[DEBUG] File written successfully: ${filePath}`);
-            // Verify the file was actually written
-            const exists = await fs.promises.access(filePath).then(() => true).catch(() => false);
-            console.log(`[DEBUG] File exists after write: ${exists}`);
-        }
-        catch (error) {
-            console.error(`[DEBUG] Error saving verb metadata:`, error);
-            throw error;
-        }
-    }
-    /**
-     * Get verb metadata from storage
-     */
-    async getVerbMetadata(id) {
-        await this.ensureInitialized();
-        const filePath = path.join(this.verbMetadataDir, `${id}.json`);
-        try {
-            const data = await fs.promises.readFile(filePath, 'utf-8');
-            return JSON.parse(data);
-        }
-        catch (error) {
-            if (error.code !== 'ENOENT') {
-                console.error(`Error reading verb metadata ${id}:`, error);
-            }
-            return null;
-        }
-    }
     /**
      * Get nouns with pagination support
      * @param options Pagination options

package/dist/storage/adapters/gcsStorage.d.ts

@@ -150,29 +150,29 @@ export declare class GcsStorage extends BaseStorage {
      */
     protected deleteNoun_internal(id: string): Promise<void>;
     /**
-     * Save noun metadata to storage (internal implementation)
+     * Write an object to a specific path in GCS
+     * Primitive operation required by base class
+     * @protected
      */
-    protected saveNounMetadata_internal(id: string, metadata: any): Promise<void>;
+    protected writeObjectToPath(path: string, data: any): Promise<void>;
     /**
-     * Save metadata to storage (public API - delegates to saveNounMetadata_internal)
+     * Read an object from a specific path in GCS
+     * Primitive operation required by base class
+     * @protected
      */
-    saveMetadata(id: string, metadata: any): Promise<void>;
+    protected readObjectFromPath(path: string): Promise<any | null>;
     /**
-     * Get metadata from storage (public API - delegates to getNounMetadata)
+     * Delete an object from a specific path in GCS
+     * Primitive operation required by base class
+     * @protected
      */
-    getMetadata(id: string): Promise<any | null>;
+    protected deleteObjectFromPath(path: string): Promise<void>;
     /**
-     * Get noun metadata from storage
+     * List all objects under a specific prefix in GCS
+     * Primitive operation required by base class
+     * @protected
      */
-    getNounMetadata(id: string): Promise<any | null>;
-    /**
-     * Save verb metadata to storage (internal implementation)
-     */
-    protected saveVerbMetadata_internal(id: string, metadata: any): Promise<void>;
-    /**
-     * Get verb metadata from storage
-     */
-    getVerbMetadata(id: string): Promise<any | null>;
+    protected listObjectsUnderPath(prefix: string): Promise<string[]>;
     /**
      * Save a verb to storage (internal implementation)
      */

package/dist/storage/adapters/gcsStorage.js

@@ -438,113 +438,91 @@ export class GcsStorage extends BaseStorage {
         }
     }
     /**
-     * Save noun metadata to storage (internal implementation)
+     * Write an object to a specific path in GCS
+     * Primitive operation required by base class
+     * @protected
      */
-    async saveNounMetadata_internal(id, metadata) {
+    async writeObjectToPath(path, data) {
         await this.ensureInitialized();
         try {
-            // Use UUID-based sharding for metadata (consistent with noun vectors)
-            const shardId = getShardIdFromUuid(id);
-            const key = `${this.metadataPrefix}${shardId}/${id}.json`;
-            this.logger.trace(`Saving noun metadata for ${id} to key: ${key}`);
-            // Save to GCS
-            const file = this.bucket.file(key);
-            await file.save(JSON.stringify(metadata, null, 2), {
+            this.logger.trace(`Writing object to path: ${path}`);
+            const file = this.bucket.file(path);
+            await file.save(JSON.stringify(data, null, 2), {
                 contentType: 'application/json',
                 resumable: false
             });
-            this.logger.debug(`Noun metadata for ${id} saved successfully`);
+            this.logger.trace(`Object written successfully to ${path}`);
         }
         catch (error) {
-            this.logger.error(`Failed to save noun metadata for ${id}:`, error);
-            throw new Error(`Failed to save noun metadata for ${id}: ${error}`);
+            this.logger.error(`Failed to write object to ${path}:`, error);
+            throw new Error(`Failed to write object to ${path}: ${error}`);
         }
     }
     /**
-     * Save metadata to storage (public API - delegates to saveNounMetadata_internal)
-     */
-    async saveMetadata(id, metadata) {
-        return this.saveNounMetadata_internal(id, metadata);
-    }
-    /**
-     * Get metadata from storage (public API - delegates to getNounMetadata)
-     */
-    async getMetadata(id) {
-        return this.getNounMetadata(id);
-    }
-    /**
-     * Get noun metadata from storage
+     * Read an object from a specific path in GCS
+     * Primitive operation required by base class
+     * @protected
      */
-    async getNounMetadata(id) {
+    async readObjectFromPath(path) {
         await this.ensureInitialized();
         try {
-            // Use UUID-based sharding for metadata
-            const shardId = getShardIdFromUuid(id);
-            const key = `${this.metadataPrefix}${shardId}/${id}.json`;
-            this.logger.trace(`Getting noun metadata for ${id} from key: ${key}`);
-            // Download from GCS
-            const file = this.bucket.file(key);
+            this.logger.trace(`Reading object from path: ${path}`);
+            const file = this.bucket.file(path);
             const [contents] = await file.download();
-            // Parse JSON
-            const metadata = JSON.parse(contents.toString());
-            this.logger.trace(`Successfully retrieved noun metadata for ${id}`);
-            return metadata;
+            const data = JSON.parse(contents.toString());
+            this.logger.trace(`Object read successfully from ${path}`);
+            return data;
         }
         catch (error) {
             // Check if this is a "not found" error
             if (error.code === 404) {
-                this.logger.trace(`Noun metadata not found for ${id}`);
+                this.logger.trace(`Object not found at ${path}`);
                 return null;
             }
-            // For other types of errors, convert to BrainyError
-            throw BrainyError.fromError(error, `getNounMetadata(${id})`);
+            this.logger.error(`Failed to read object from ${path}:`, error);
+            throw BrainyError.fromError(error, `readObjectFromPath(${path})`);
         }
     }
     /**
-     * Save verb metadata to storage (internal implementation)
+     * Delete an object from a specific path in GCS
+     * Primitive operation required by base class
+     * @protected
      */
-    async saveVerbMetadata_internal(id, metadata) {
+    async deleteObjectFromPath(path) {
         await this.ensureInitialized();
         try {
-            const key = `${this.verbMetadataPrefix}${id}.json`;
-            this.logger.trace(`Saving verb metadata for ${id} to key: ${key}`);
-            // Save to GCS
-            const file = this.bucket.file(key);
-            await file.save(JSON.stringify(metadata, null, 2), {
-                contentType: 'application/json',
-                resumable: false
-            });
-            this.logger.debug(`Verb metadata for ${id} saved successfully`);
+            this.logger.trace(`Deleting object at path: ${path}`);
+            const file = this.bucket.file(path);
+            await file.delete();
+            this.logger.trace(`Object deleted successfully from ${path}`);
         }
         catch (error) {
-            this.logger.error(`Failed to save verb metadata for ${id}:`, error);
-            throw new Error(`Failed to save verb metadata for ${id}: ${error}`);
+            // If already deleted (404), treat as success
+            if (error.code === 404) {
+                this.logger.trace(`Object at ${path} not found (already deleted)`);
+                return;
+            }
+            this.logger.error(`Failed to delete object from ${path}:`, error);
+            throw new Error(`Failed to delete object from ${path}: ${error}`);
         }
     }
     /**
-     * Get verb metadata from storage
+     * List all objects under a specific prefix in GCS
+     * Primitive operation required by base class
+     * @protected
      */
-    async getVerbMetadata(id) {
+    async listObjectsUnderPath(prefix) {
         await this.ensureInitialized();
         try {
-            const key = `${this.verbMetadataPrefix}${id}.json`;
-            this.logger.trace(`Getting verb metadata for ${id} from key: ${key}`);
-            // Download from GCS
-            const file = this.bucket.file(key);
-            const [contents] = await file.download();
-            // Parse JSON
-            const metadata = JSON.parse(contents.toString());
-            this.logger.trace(`Successfully retrieved verb metadata for ${id}`);
-            return metadata;
+            this.logger.trace(`Listing objects under prefix: ${prefix}`);
+            const [files] = await this.bucket.getFiles({ prefix });
+            const paths = files.map((file) => file.name).filter((name) => name && name.length > 0);
+            this.logger.trace(`Found ${paths.length} objects under ${prefix}`);
+            return paths;
         }
         catch (error) {
-            // Check if this is a "not found" error
-            if (error.code === 404) {
-                this.logger.trace(`Verb metadata not found for ${id}`);
-                return null;
-            }
-            // For other types of errors, convert to BrainyError
-            throw BrainyError.fromError(error, `getVerbMetadata(${id})`);
+            this.logger.error(`Failed to list objects under ${prefix}:`, error);
+            throw new Error(`Failed to list objects under ${prefix}: ${error}`);
         }
     }
     /**

package/dist/storage/adapters/memoryStorage.d.ts

@@ -12,10 +12,11 @@ import { PaginatedResult } from '../../types/paginationTypes.js';
 export declare class MemoryStorage extends BaseStorage {
     private nouns;
     private verbs;
-    private metadata;
-    private nounMetadata;
-    private verbMetadata;
     private statistics;
+    private objectStore;
+    private get metadata();
+    private get nounMetadata();
+    private get verbMetadata();
     constructor();
     /**
      * Initialize the storage adapter
@@ -118,34 +119,30 @@ export declare class MemoryStorage extends BaseStorage {
      */
     protected deleteVerb_internal(id: string): Promise<void>;
     /**
-     * Save metadata to storage
+     * Primitive operation: Write object to path
+     * All metadata operations use this internally via base class routing
      */
-    saveMetadata(id: string, metadata: any): Promise<void>;
+    protected writeObjectToPath(path: string, data: any): Promise<void>;
     /**
-     * Get metadata from storage
+     * Primitive operation: Read object from path
+     * All metadata operations use this internally via base class routing
      */
-    getMetadata(id: string): Promise<any | null>;
+    protected readObjectFromPath(path: string): Promise<any | null>;
     /**
-     * Get multiple metadata objects in batches (CRITICAL: Prevents socket exhaustion)
-     * Memory storage implementation is simple since all data is already in memory
-     */
-    getMetadataBatch(ids: string[]): Promise<Map<string, any>>;
-    /**
-     * Save noun metadata to storage (internal implementation)
+     * Primitive operation: Delete object from path
+     * All metadata operations use this internally via base class routing
      */
-    protected saveNounMetadata_internal(id: string, metadata: any): Promise<void>;
+    protected deleteObjectFromPath(path: string): Promise<void>;
     /**
-     * Get noun metadata from storage
+     * Primitive operation: List objects under path prefix
+     * All metadata operations use this internally via base class routing
      */
-    getNounMetadata(id: string): Promise<any | null>;
+    protected listObjectsUnderPath(prefix: string): Promise<string[]>;
     /**
-     * Save verb metadata to storage (internal implementation)
-     */
-    protected saveVerbMetadata_internal(id: string, metadata: any): Promise<void>;
-    /**
-     * Get verb metadata from storage
+     * Get multiple metadata objects in batches (CRITICAL: Prevents socket exhaustion)
+     * Memory storage implementation is simple since all data is already in memory
      */
-    getVerbMetadata(id: string): Promise<any | null>;
+    getMetadataBatch(ids: string[]): Promise<Map<string, any>>;
     /**
      * Clear all data from storage
      */