@soulcraft/brainy 3.30.0 → 3.30.2

package/dist/storage/adapters/s3CompatibleStorage.js

@@ -1495,159 +1495,134 @@ export class S3CompatibleStorage extends BaseStorage {
         }
     }
     /**
-     * Save metadata to storage
+     * Primitive operation: Write object to path
+     * All metadata operations use this internally via base class routing
      */
-    async saveMetadata(id, metadata) {
+    async writeObjectToPath(path, data) {
         await this.ensureInitialized();
         // Apply backpressure before starting operation
         const requestId = await this.applyBackpressure();
         try {
-            // Import the PutObjectCommand only when needed
             const { PutObjectCommand } = await import('@aws-sdk/client-s3');
-            const key = `${this.metadataPrefix}${id}.json`;
-            const body = JSON.stringify(metadata, null, 2);
-            this.logger.trace(`Saving metadata for ${id} to key: ${key}`);
-            // Save the metadata to S3-compatible storage
-            const result = await this.s3Client.send(new PutObjectCommand({
+            const body = JSON.stringify(data, null, 2);
+            this.logger.trace(`Writing object to path: ${path}`);
+            await this.s3Client.send(new PutObjectCommand({
                 Bucket: this.bucketName,
-                Key: key,
+                Key: path,
                 Body: body,
                 ContentType: 'application/json'
             }));
-            this.logger.debug(`Metadata for ${id} saved successfully`);
+            this.logger.debug(`Object written successfully to ${path}`);
             // Log the change for efficient synchronization
             await this.appendToChangeLog({
                 timestamp: Date.now(),
-                operation: 'add', // Could be 'update' if we track existing metadata
+                operation: 'add',
                 entityType: 'metadata',
-                entityId: id,
-                data: metadata
+                entityId: path,
+                data: data
             });
-            // Verify the metadata was saved by trying to retrieve it
-            const { GetObjectCommand } = await import('@aws-sdk/client-s3');
-            try {
-                const verifyResponse = await this.s3Client.send(new GetObjectCommand({
-                    Bucket: this.bucketName,
-                    Key: key
-                }));
-                if (verifyResponse && verifyResponse.Body) {
-                    this.logger.trace(`Verified metadata for ${id} was saved correctly`);
-                }
-                else {
-                    this.logger.warn(`Failed to verify metadata for ${id} was saved correctly: no response or body`);
-                }
-            }
-            catch (verifyError) {
-                this.logger.warn(`Failed to verify metadata for ${id} was saved correctly:`, verifyError);
-            }
             // Release backpressure on success
             this.releaseBackpressure(true, requestId);
         }
         catch (error) {
             // Release backpressure on error
             this.releaseBackpressure(false, requestId);
-            this.logger.error(`Failed to save metadata for ${id}:`, error);
-            throw new Error(`Failed to save metadata for ${id}: ${error}`);
+            this.logger.error(`Failed to write object to ${path}:`, error);
+            throw new Error(`Failed to write object to ${path}: ${error}`);
         }
     }
     /**
-     * Save verb metadata to storage
+     * Primitive operation: Read object from path
+     * All metadata operations use this internally via base class routing
      */
-    async saveVerbMetadata_internal(id, metadata) {
+    async readObjectFromPath(path) {
         await this.ensureInitialized();
-        try {
-            // Import the PutObjectCommand only when needed
-            const { PutObjectCommand } = await import('@aws-sdk/client-s3');
-            const key = `${this.verbMetadataPrefix}${id}.json`;
-            const body = JSON.stringify(metadata, null, 2);
-            this.logger.trace(`Saving verb metadata for ${id} to key: ${key}`);
-            // Save the verb metadata to S3-compatible storage
-            const result = await this.s3Client.send(new PutObjectCommand({
-                Bucket: this.bucketName,
-                Key: key,
-                Body: body,
-                ContentType: 'application/json'
-            }));
-            this.logger.debug(`Verb metadata for ${id} saved successfully`);
-        }
-        catch (error) {
-            this.logger.error(`Failed to save verb metadata for ${id}:`, error);
-            throw new Error(`Failed to save verb metadata for ${id}: ${error}`);
-        }
+        return this.operationExecutors.executeGet(async () => {
+            try {
+                const { GetObjectCommand } = await import('@aws-sdk/client-s3');
+                this.logger.trace(`Reading object from path: ${path}`);
+                const response = await this.s3Client.send(new GetObjectCommand({
+                    Bucket: this.bucketName,
+                    Key: path
+                }));
+                if (!response || !response.Body) {
+                    this.logger.trace(`Object not found at ${path}`);
+                    return null;
+                }
+                const bodyContents = await response.Body.transformToString();
+                const data = JSON.parse(bodyContents);
+                this.logger.trace(`Object read successfully from ${path}`);
+                return data;
+            }
+            catch (error) {
+                // 404 errors return null (object doesn't exist)
+                if (error.name === 'NoSuchKey' ||
+                    (error.message &&
+                        (error.message.includes('NoSuchKey') ||
+                            error.message.includes('not found') ||
+                            error.message.includes('does not exist')))) {
+                    this.logger.trace(`Object not found at ${path}`);
+                    return null;
+                }
+                throw BrainyError.fromError(error, `readObjectFromPath(${path})`);
+            }
+        }, `readObjectFromPath(${path})`);
     }
     /**
-     * Get verb metadata from storage
+     * Primitive operation: Delete object from path
+     * All metadata operations use this internally via base class routing
      */
-    async getVerbMetadata(id) {
+    async deleteObjectFromPath(path) {
         await this.ensureInitialized();
         try {
-            // Import the GetObjectCommand only when needed
-            const { GetObjectCommand } = await import('@aws-sdk/client-s3');
-            const key = `${this.verbMetadataPrefix}${id}.json`;
-            this.logger.trace(`Getting verb metadata for ${id} from key: ${key}`);
-            // Try to get the verb metadata
-            const response = await this.s3Client.send(new GetObjectCommand({
+            const { DeleteObjectCommand } = await import('@aws-sdk/client-s3');
+            this.logger.trace(`Deleting object at path: ${path}`);
+            await this.s3Client.send(new DeleteObjectCommand({
                 Bucket: this.bucketName,
-                Key: key
+                Key: path
             }));
-            // Check if response is null or undefined
-            if (!response || !response.Body) {
-                this.logger.trace(`No verb metadata found for ${id}`);
-                return null;
-            }
-            // Convert the response body to a string
-            const bodyContents = await response.Body.transformToString();
-            this.logger.trace(`Retrieved verb metadata body for ${id}`);
-            // Parse the JSON string
-            try {
-                const parsedMetadata = JSON.parse(bodyContents);
-                this.logger.trace(`Successfully retrieved verb metadata for ${id}`);
-                return parsedMetadata;
-            }
-            catch (parseError) {
-                this.logger.error(`Failed to parse verb metadata for ${id}:`, parseError);
-                return null;
-            }
+            this.logger.trace(`Object deleted successfully from ${path}`);
         }
         catch (error) {
-            // Check if this is a "NoSuchKey" error (object doesn't exist)
+            // 404 errors are ok (already deleted)
             if (error.name === 'NoSuchKey' ||
                 (error.message &&
                     (error.message.includes('NoSuchKey') ||
                         error.message.includes('not found') ||
                         error.message.includes('does not exist')))) {
-                this.logger.trace(`Verb metadata not found for ${id}`);
-                return null;
+                this.logger.trace(`Object at ${path} not found (already deleted)`);
+                return;
             }
-            // For other types of errors, convert to BrainyError for better classification
-            throw BrainyError.fromError(error, `getVerbMetadata(${id})`);
+            this.logger.error(`Failed to delete object from ${path}:`, error);
+            throw new Error(`Failed to delete object from ${path}: ${error}`);
         }
     }
     /**
-     * Save noun metadata to storage
+     * Primitive operation: List objects under path prefix
+     * All metadata operations use this internally via base class routing
      */
-    async saveNounMetadata_internal(id, metadata) {
+    async listObjectsUnderPath(prefix) {
         await this.ensureInitialized();
         try {
-            // Import the PutObjectCommand only when needed
-            const { PutObjectCommand } = await import('@aws-sdk/client-s3');
-            // Use UUID-based sharding for metadata (consistent with noun vectors)
-            const shardId = getShardIdFromUuid(id);
-            const key = `${this.metadataPrefix}${shardId}/${id}.json`;
-            const body = JSON.stringify(metadata, null, 2);
-            this.logger.trace(`Saving noun metadata for ${id} to key: ${key}`);
-            // Save the noun metadata to S3-compatible storage
-            const result = await this.s3Client.send(new PutObjectCommand({
+            const { ListObjectsV2Command } = await import('@aws-sdk/client-s3');
+            this.logger.trace(`Listing objects under prefix: ${prefix}`);
+            const response = await this.s3Client.send(new ListObjectsV2Command({
                 Bucket: this.bucketName,
-                Key: key,
-                Body: body,
-                ContentType: 'application/json'
+                Prefix: prefix
             }));
-            this.logger.debug(`Noun metadata for ${id} saved successfully`);
+            if (!response || !response.Contents || response.Contents.length === 0) {
+                this.logger.trace(`No objects found under ${prefix}`);
+                return [];
+            }
+            const paths = response.Contents
+                .map((object) => object.Key)
+                .filter((key) => key && key.length > 0);
+            this.logger.trace(`Found ${paths.length} objects under ${prefix}`);
+            return paths;
         }
         catch (error) {
-            this.logger.error(`Failed to save noun metadata for ${id}:`, error);
-            throw new Error(`Failed to save noun metadata for ${id}: ${error}`);
+            this.logger.error(`Failed to list objects under ${prefix}:`, error);
+            throw new Error(`Failed to list objects under ${prefix}: ${error}`);
         }
     }
     /**
@@ -1754,109 +1729,6 @@ export class S3CompatibleStorage extends BaseStorage {
         }
         return results;
     }
-    /**
-     * Get noun metadata from storage
-     */
-    async getNounMetadata(id) {
-        await this.ensureInitialized();
-        try {
-            // Import the GetObjectCommand only when needed
-            const { GetObjectCommand } = await import('@aws-sdk/client-s3');
-            // Use UUID-based sharding for metadata (consistent with noun vectors)
-            const shardId = getShardIdFromUuid(id);
-            const key = `${this.metadataPrefix}${shardId}/${id}.json`;
-            this.logger.trace(`Getting noun metadata for ${id} from key: ${key}`);
-            // Try to get the noun metadata
-            const response = await this.s3Client.send(new GetObjectCommand({
-                Bucket: this.bucketName,
-                Key: key
-            }));
-            // Check if response is null or undefined
-            if (!response || !response.Body) {
-                this.logger.trace(`No noun metadata found for ${id}`);
-                return null;
-            }
-            // Convert the response body to a string
-            const bodyContents = await response.Body.transformToString();
-            this.logger.trace(`Retrieved noun metadata body for ${id}`);
-            // Parse the JSON string
-            try {
-                const parsedMetadata = JSON.parse(bodyContents);
-                this.logger.trace(`Successfully retrieved noun metadata for ${id}`);
-                return parsedMetadata;
-            }
-            catch (parseError) {
-                this.logger.error(`Failed to parse noun metadata for ${id}:`, parseError);
-                return null;
-            }
-        }
-        catch (error) {
-            // Check if this is a "NoSuchKey" error (object doesn't exist)
-            if (error.name === 'NoSuchKey' ||
-                (error.message &&
-                    (error.message.includes('NoSuchKey') ||
-                        error.message.includes('not found') ||
-                        error.message.includes('does not exist')))) {
-                this.logger.trace(`Noun metadata not found for ${id}`);
-                return null;
-            }
-            // For other types of errors, convert to BrainyError for better classification
-            throw BrainyError.fromError(error, `getNounMetadata(${id})`);
-        }
-    }
-    /**
-     * Get metadata from storage
-     */
-    async getMetadata(id) {
-        await this.ensureInitialized();
-        return this.operationExecutors.executeGet(async () => {
-            try {
-                // Import the GetObjectCommand only when needed
-                const { GetObjectCommand } = await import('@aws-sdk/client-s3');
-                prodLog.debug(`Getting metadata for ${id} from bucket ${this.bucketName}`);
-                const key = `${this.metadataPrefix}${id}.json`;
-                prodLog.debug(`Looking for metadata at key: ${key}`);
-                // Try to get the metadata from the metadata directory
-                const response = await this.s3Client.send(new GetObjectCommand({
-                    Bucket: this.bucketName,
-                    Key: key
-                }));
-                // Check if response is null or undefined (can happen in mock implementations)
-                if (!response || !response.Body) {
-                    prodLog.debug(`No metadata found for ${id}`);
-                    return null;
-                }
-                // Convert the response body to a string
-                const bodyContents = await response.Body.transformToString();
-                prodLog.debug(`Retrieved metadata body: ${bodyContents}`);
-                // Parse the JSON string
-                try {
-                    const parsedMetadata = JSON.parse(bodyContents);
-                    prodLog.debug(`Successfully retrieved metadata for ${id}:`, parsedMetadata);
-                    return parsedMetadata;
-                }
-                catch (parseError) {
-                    prodLog.error(`Failed to parse metadata for ${id}:`, parseError);
-                    return null;
-                }
-            }
-            catch (error) {
-                // Check if this is a "NoSuchKey" error (object doesn't exist)
-                // In AWS SDK, this would be error.name === 'NoSuchKey'
-                // In our mock, we might get different error types
-                if (error.name === 'NoSuchKey' ||
-                    (error.message &&
-                        (error.message.includes('NoSuchKey') ||
-                            error.message.includes('not found') ||
-                            error.message.includes('does not exist')))) {
-                    prodLog.debug(`Metadata not found for ${id}`);
-                    return null;
-                }
-                // For other types of errors, convert to BrainyError for better classification
-                throw BrainyError.fromError(error, `getMetadata(${id})`);
-            }
-        }, `getMetadata(${id})`);
-    }
     /**
      * Clear all data from storage
      */

package/dist/storage/baseStorage.d.ts

@@ -34,6 +34,14 @@ export declare abstract class BaseStorage extends BaseStorageAdapter {
     protected isInitialized: boolean;
     protected graphIndex?: GraphAdjacencyIndex;
     protected readOnly: boolean;
+    /**
+     * Analyze a storage key to determine its routing and path
+     * @param id - The key to analyze (UUID or system key)
+     * @param context - The context for the key (noun-metadata, verb-metadata, or system)
+     * @returns Storage key information including path and shard ID
+     * @private
+     */
+    private analyzeKey;
     /**
      * Initialize the storage adapter
      * This method should be implemented by each specific adapter
@@ -164,46 +172,79 @@ export declare abstract class BaseStorage extends BaseStorageAdapter {
         quota: number | null;
         details?: Record<string, any>;
     }>;
+    /**
+     * Write a JSON object to a specific path in storage
+     * This is a primitive operation that all adapters must implement
+     * @param path - Full path including filename (e.g., "_system/statistics.json" or "entities/nouns/metadata/3f/3fa85f64-....json")
+     * @param data - Data to write (will be JSON.stringify'd)
+     * @protected
+     */
+    protected abstract writeObjectToPath(path: string, data: any): Promise<void>;
+    /**
+     * Read a JSON object from a specific path in storage
+     * This is a primitive operation that all adapters must implement
+     * @param path - Full path including filename
+     * @returns The parsed JSON object, or null if not found
+     * @protected
+     */
+    protected abstract readObjectFromPath(path: string): Promise<any | null>;
+    /**
+     * Delete an object from a specific path in storage
+     * This is a primitive operation that all adapters must implement
+     * @param path - Full path including filename
+     * @protected
+     */
+    protected abstract deleteObjectFromPath(path: string): Promise<void>;
+    /**
+     * List all object paths under a given prefix
+     * This is a primitive operation that all adapters must implement
+     * @param prefix - Directory prefix to list (e.g., "entities/nouns/metadata/3f/")
+     * @returns Array of full paths
+     * @protected
+     */
+    protected abstract listObjectsUnderPath(prefix: string): Promise<string[]>;
     /**
      * Save metadata to storage
-     * This method should be implemented by each specific adapter
+     * Routes to correct location (system or entity) based on key format
      */
-    abstract saveMetadata(id: string, metadata: any): Promise<void>;
+    saveMetadata(id: string, metadata: any): Promise<void>;
     /**
      * Get metadata from storage
-     * This method should be implemented by each specific adapter
+     * Routes to correct location (system or entity) based on key format
      */
-    abstract getMetadata(id: string): Promise<any | null>;
+    getMetadata(id: string): Promise<any | null>;
     /**
      * Save noun metadata to storage
-     * This method should be implemented by each specific adapter
+     * Routes to correct sharded location based on UUID
      */
     saveNounMetadata(id: string, metadata: any): Promise<void>;
     /**
      * Internal method for saving noun metadata
-     * This method should be implemented by each specific adapter
+     * Uses routing logic to handle both UUIDs (sharded) and system keys (unsharded)
+     * @protected
      */
-    protected abstract saveNounMetadata_internal(id: string, metadata: any): Promise<void>;
+    protected saveNounMetadata_internal(id: string, metadata: any): Promise<void>;
     /**
      * Get noun metadata from storage
-     * This method should be implemented by each specific adapter
+     * Uses routing logic to handle both UUIDs (sharded) and system keys (unsharded)
      */
-    abstract getNounMetadata(id: string): Promise<any | null>;
+    getNounMetadata(id: string): Promise<any | null>;
     /**
      * Save verb metadata to storage
-     * This method should be implemented by each specific adapter
+     * Routes to correct sharded location based on UUID
      */
     saveVerbMetadata(id: string, metadata: any): Promise<void>;
     /**
      * Internal method for saving verb metadata
-     * This method should be implemented by each specific adapter
+     * Uses routing logic to handle both UUIDs (sharded) and system keys (unsharded)
+     * @protected
      */
-    protected abstract saveVerbMetadata_internal(id: string, metadata: any): Promise<void>;
+    protected saveVerbMetadata_internal(id: string, metadata: any): Promise<void>;
     /**
      * Get verb metadata from storage
-     * This method should be implemented by each specific adapter
+     * Uses routing logic to handle both UUIDs (sharded) and system keys (unsharded)
      */
-    abstract getVerbMetadata(id: string): Promise<any | null>;
+    getVerbMetadata(id: string): Promise<any | null>;
     /**
      * Save a noun to storage
      * This method should be implemented by each specific adapter

package/dist/storage/baseStorage.js

@@ -5,6 +5,7 @@
 import { GraphAdjacencyIndex } from '../graph/graphAdjacencyIndex.js';
 import { BaseStorageAdapter } from './adapters/baseStorageAdapter.js';
 import { validateNounType, validateVerbType } from '../utils/typeValidation.js';
+import { getShardIdFromUuid } from './sharding.js';
 // Common directory/prefix names
 // Option A: Entity-Based Directory Structure
 export const ENTITIES_DIR = 'entities';
@@ -60,6 +61,72 @@ export class BaseStorage extends BaseStorageAdapter {
         this.isInitialized = false;
         this.readOnly = false;
     }
+    /**
+     * Analyze a storage key to determine its routing and path
+     * @param id - The key to analyze (UUID or system key)
+     * @param context - The context for the key (noun-metadata, verb-metadata, or system)
+     * @returns Storage key information including path and shard ID
+     * @private
+     */
+    analyzeKey(id, context) {
+        // System resource detection
+        const isSystemKey = id.startsWith('__metadata_') ||
+            id.startsWith('__index_') ||
+            id.startsWith('__system_') ||
+            id.startsWith('statistics_') ||
+            id === 'statistics';
+        if (isSystemKey) {
+            return {
+                original: id,
+                isEntity: false,
+                shardId: null,
+                directory: SYSTEM_DIR,
+                fullPath: `${SYSTEM_DIR}/${id}.json`
+            };
+        }
+        // UUID validation for entity keys
+        const uuidRegex = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
+        if (!uuidRegex.test(id)) {
+            console.warn(`[Storage] Unknown key format: ${id} - treating as system resource`);
+            return {
+                original: id,
+                isEntity: false,
+                shardId: null,
+                directory: SYSTEM_DIR,
+                fullPath: `${SYSTEM_DIR}/${id}.json`
+            };
+        }
+        // Valid entity UUID - apply sharding
+        const shardId = getShardIdFromUuid(id);
+        if (context === 'noun-metadata') {
+            return {
+                original: id,
+                isEntity: true,
+                shardId,
+                directory: `${NOUNS_METADATA_DIR}/${shardId}`,
+                fullPath: `${NOUNS_METADATA_DIR}/${shardId}/${id}.json`
+            };
+        }
+        else if (context === 'verb-metadata') {
+            return {
+                original: id,
+                isEntity: true,
+                shardId,
+                directory: `${VERBS_METADATA_DIR}/${shardId}`,
+                fullPath: `${VERBS_METADATA_DIR}/${shardId}/${id}.json`
+            };
+        }
+        else {
+            // system context - but UUID format
+            return {
+                original: id,
+                isEntity: false,
+                shardId: null,
+                directory: SYSTEM_DIR,
+                fullPath: `${SYSTEM_DIR}/${id}.json`
+            };
+        }
+    }
     /**
      * Ensure the storage adapter is initialized
      */
@@ -551,9 +618,27 @@ export class BaseStorage extends BaseStorageAdapter {
         }
         return this.graphIndex;
     }
+    /**
+     * Save metadata to storage
+     * Routes to correct location (system or entity) based on key format
+     */
+    async saveMetadata(id, metadata) {
+        await this.ensureInitialized();
+        const keyInfo = this.analyzeKey(id, 'system');
+        return this.writeObjectToPath(keyInfo.fullPath, metadata);
+    }
+    /**
+     * Get metadata from storage
+     * Routes to correct location (system or entity) based on key format
+     */
+    async getMetadata(id) {
+        await this.ensureInitialized();
+        const keyInfo = this.analyzeKey(id, 'system');
+        return this.readObjectFromPath(keyInfo.fullPath);
+    }
     /**
      * Save noun metadata to storage
-     * This method should be implemented by each specific adapter
+     * Routes to correct sharded location based on UUID
      */
     async saveNounMetadata(id, metadata) {
         // Validate noun type in metadata - storage boundary protection
@@ -562,9 +647,28 @@ export class BaseStorage extends BaseStorageAdapter {
         }
         return this.saveNounMetadata_internal(id, metadata);
     }
+    /**
+     * Internal method for saving noun metadata
+     * Uses routing logic to handle both UUIDs (sharded) and system keys (unsharded)
+     * @protected
+     */
+    async saveNounMetadata_internal(id, metadata) {
+        await this.ensureInitialized();
+        const keyInfo = this.analyzeKey(id, 'noun-metadata');
+        return this.writeObjectToPath(keyInfo.fullPath, metadata);
+    }
+    /**
+     * Get noun metadata from storage
+     * Uses routing logic to handle both UUIDs (sharded) and system keys (unsharded)
+     */
+    async getNounMetadata(id) {
+        await this.ensureInitialized();
+        const keyInfo = this.analyzeKey(id, 'noun-metadata');
+        return this.readObjectFromPath(keyInfo.fullPath);
+    }
     /**
      * Save verb metadata to storage
-     * This method should be implemented by each specific adapter
+     * Routes to correct sharded location based on UUID
      */
     async saveVerbMetadata(id, metadata) {
         // Validate verb type in metadata - storage boundary protection
@@ -573,6 +677,25 @@ export class BaseStorage extends BaseStorageAdapter {
         }
         return this.saveVerbMetadata_internal(id, metadata);
     }
+    /**
+     * Internal method for saving verb metadata
+     * Uses routing logic to handle both UUIDs (sharded) and system keys (unsharded)
+     * @protected
+     */
+    async saveVerbMetadata_internal(id, metadata) {
+        await this.ensureInitialized();
+        const keyInfo = this.analyzeKey(id, 'verb-metadata');
+        return this.writeObjectToPath(keyInfo.fullPath, metadata);
+    }
+    /**
+     * Get verb metadata from storage
+     * Uses routing logic to handle both UUIDs (sharded) and system keys (unsharded)
+     */
+    async getVerbMetadata(id) {
+        await this.ensureInitialized();
+        const keyInfo = this.analyzeKey(id, 'verb-metadata');
+        return this.readObjectFromPath(keyInfo.fullPath);
+    }
     /**
      * Helper method to convert a Map to a plain object for serialization
      */

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@soulcraft/brainy",
-  "version": "3.30.0",
+  "version": "3.30.2",
   "description": "Universal Knowledge Protocol™ - World's first Triple Intelligence database unifying vector, graph, and document search in one API. 31 nouns × 40 verbs for infinite expressiveness.",
   "main": "dist/index.js",
   "module": "dist/index.js",