@soulcraft/brainy 3.25.2 → 3.27.0

package/dist/storage/adapters/s3CompatibleStorage.js

@@ -13,6 +13,7 @@ import { getGlobalSocketManager } from '../../utils/adaptiveSocketManager.js';
 import { getGlobalBackpressure } from '../../utils/adaptiveBackpressure.js';
 import { getWriteBuffer } from '../../utils/writeBuffer.js';
 import { getCoalescer } from '../../utils/requestCoalescer.js';
+import { getShardIdFromUuid, getShardIdByIndex, TOTAL_SHARDS } from '../sharding.js';
 // Export R2Storage as an alias for S3CompatibleStorage
 export { S3CompatibleStorage as R2Storage };
 /**
@@ -69,6 +70,8 @@ export class S3CompatibleStorage extends BaseStorage {
         // Write buffers for bulk operations
         this.nounWriteBuffer = null;
         this.verbWriteBuffer = null;
+        // Note: Sharding is always enabled via UUID-based prefixes (00-ff)
+        // ShardManager is no longer used - sharding is deterministic
         // Request coalescer for deduplication
         this.requestCoalescer = null;
         // High-volume mode detection - MUCH more aggressive
@@ -242,17 +245,16 @@ export class S3CompatibleStorage extends BaseStorage {
     }
     /**
      * Set distributed components for multi-node coordination
-     * Zero-config: Automatically optimizes based on components provided
+     *
+     * Note: Sharding is always enabled via UUID-based prefixes (00-ff).
+     * ShardManager is no longer required - sharding is deterministic based on UUID.
      */
     setDistributedComponents(components) {
         this.coordinator = components.coordinator;
-        this.shardManager = components.shardManager;
         this.cacheSync = components.cacheSync;
         this.readWriteSeparation = components.readWriteSeparation;
-        // Auto-configure based on what's available
-        if (this.shardManager) {
-            console.log(`🎯 S3 Storage: Sharding enabled with ${this.shardManager.config?.shardCount || 64} shards`);
-        }
+        // Note: UUID-based sharding is always active (256 shards: 00-ff)
+        console.log(`🎯 S3 Storage: UUID-based sharding active (256 shards: 00-ff)`);
         if (this.coordinator) {
             console.log(`🤝 S3 Storage: Distributed coordination active (node: ${this.coordinator.nodeId})`);
         }
@@ -264,24 +266,32 @@ export class S3CompatibleStorage extends BaseStorage {
         }
     }
     /**
-     * Get the S3 key for a noun, using sharding if available
+     * Get the S3 key for a noun using UUID-based sharding
+     *
+     * Uses first 2 hex characters of UUID for consistent sharding.
+     * Path format: entities/nouns/vectors/{shardId}/{uuid}.json
+     *
+     * @example
+     * getNounKey('ab123456-1234-5678-9abc-def012345678')
+     * // returns 'entities/nouns/vectors/ab/ab123456-1234-5678-9abc-def012345678.json'
      */
     getNounKey(id) {
-        if (this.shardManager) {
-            const shardId = this.shardManager.getShardForKey(id);
-            return `shards/${shardId}/${this.nounPrefix}${id}.json`;
-        }
-        return `${this.nounPrefix}${id}.json`;
+        const shardId = getShardIdFromUuid(id);
+        return `${this.nounPrefix}${shardId}/${id}.json`;
     }
     /**
-     * Get the S3 key for a verb, using sharding if available
+     * Get the S3 key for a verb using UUID-based sharding
+     *
+     * Uses first 2 hex characters of UUID for consistent sharding.
+     * Path format: verbs/{shardId}/{uuid}.json
+     *
+     * @example
+     * getVerbKey('cd987654-4321-8765-cba9-fed543210987')
+     * // returns 'verbs/cd/cd987654-4321-8765-cba9-fed543210987.json'
      */
     getVerbKey(id) {
-        if (this.shardManager) {
-            const shardId = this.shardManager.getShardForKey(id);
-            return `shards/${shardId}/${this.verbPrefix}${id}.json`;
-        }
-        return `${this.verbPrefix}${id}.json`;
+        const shardId = getShardIdFromUuid(id);
+        return `${this.verbPrefix}${shardId}/${id}.json`;
     }
     /**
      * Override base class method to detect S3-specific throttling errors
@@ -775,7 +785,8 @@ export class S3CompatibleStorage extends BaseStorage {
         try {
             // Import the GetObjectCommand only when needed
             const { GetObjectCommand } = await import('@aws-sdk/client-s3');
-            const key = `${this.nounPrefix}${id}.json`;
+            // Use getNounKey() to properly handle sharding
+            const key = this.getNounKey(id);
             this.logger.trace(`Getting node ${id} from key: ${key}`);
             // Try to get the node from the nouns directory
             const response = await this.s3Client.send(new GetObjectCommand({
@@ -853,86 +864,97 @@ export class S3CompatibleStorage extends BaseStorage {
         }
     }
     /**
-     * Get nodes with pagination
+     * Get nodes with pagination using UUID-based sharding
+     *
+     * Iterates through 256 UUID-based shards (00-ff) to retrieve nodes.
+     * Cursor format: "shardIndex:s3ContinuationToken" to support pagination across shards.
+     *
      * @param options Pagination options
      * @returns Promise that resolves to a paginated result of nodes
+     *
+     * @example
+     * // First page
+     * const page1 = await getNodesWithPagination({ limit: 100 })
+     * // page1.nodes contains up to 100 nodes
+     * // page1.nextCursor might be "5:some-s3-token" (currently in shard 05)
+     *
+     * // Next page
+     * const page2 = await getNodesWithPagination({ limit: 100, cursor: page1.nextCursor })
+     * // Continues from where page1 left off
      */
     async getNodesWithPagination(options = {}) {
         await this.ensureInitialized();
         const limit = options.limit || 100;
         const useCache = options.useCache !== false;
         try {
-            // Import the ListObjectsV2Command and GetObjectCommand only when needed
             const { ListObjectsV2Command } = await import('@aws-sdk/client-s3');
-            // List objects with pagination
-            const listResponse = await this.s3Client.send(new ListObjectsV2Command({
-                Bucket: this.bucketName,
-                Prefix: this.nounPrefix,
-                MaxKeys: limit,
-                ContinuationToken: options.cursor
-            }));
-            // If listResponse is null/undefined or there are no objects, return an empty result
-            if (!listResponse ||
-                !listResponse.Contents ||
-                listResponse.Contents.length === 0) {
-                return {
-                    nodes: [],
-                    hasMore: false
-                };
-            }
-            // Extract node IDs from the keys
-            const nodeIds = listResponse.Contents
-                .filter((object) => object && object.Key)
-                .map((object) => object.Key.replace(this.nounPrefix, '').replace('.json', ''));
-            // Use the cache manager to get nodes efficiently
             const nodes = [];
-            if (useCache) {
-                // Get nodes from cache manager
-                const cachedNodes = await this.nounCacheManager.getMany(nodeIds);
-                // Add nodes to result in the same order as nodeIds
-                for (const id of nodeIds) {
-                    const node = cachedNodes.get(id);
-                    if (node) {
-                        nodes.push(node);
-                    }
-                }
-            }
-            else {
-                // Get nodes directly from S3 without using cache
-                // Process in smaller batches to reduce memory usage
-                const batchSize = 50;
-                const batches = [];
-                // Split into batches
-                for (let i = 0; i < nodeIds.length; i += batchSize) {
-                    const batch = nodeIds.slice(i, i + batchSize);
-                    batches.push(batch);
-                }
-                // Process each batch sequentially
-                for (const batch of batches) {
-                    const batchNodes = await Promise.all(batch.map(async (id) => {
-                        try {
-                            return await this.getNoun_internal(id);
-                        }
-                        catch (error) {
-                            return null;
+            // Parse cursor (format: "shardIndex:s3ContinuationToken")
+            let startShardIndex = 0;
+            let s3ContinuationToken;
+            if (options.cursor) {
+                const parts = options.cursor.split(':', 2);
+                startShardIndex = parseInt(parts[0]) || 0;
+                s3ContinuationToken = parts[1] || undefined;
+            }
+            // Iterate through shards starting from cursor position
+            for (let shardIndex = startShardIndex; shardIndex < TOTAL_SHARDS; shardIndex++) {
+                const shardId = getShardIdByIndex(shardIndex);
+                const shardPrefix = `${this.nounPrefix}${shardId}/`;
+                // List objects in this shard
+                const listResponse = await this.s3Client.send(new ListObjectsV2Command({
+                    Bucket: this.bucketName,
+                    Prefix: shardPrefix,
+                    MaxKeys: limit - nodes.length,
+                    ContinuationToken: shardIndex === startShardIndex ? s3ContinuationToken : undefined
+                }));
+                // Extract node IDs from keys
+                if (listResponse.Contents && listResponse.Contents.length > 0) {
+                    const nodeIds = listResponse.Contents
+                        .filter((obj) => obj && obj.Key)
+                        .map((obj) => {
+                        // Extract UUID from: entities/nouns/vectors/ab/ab123456-uuid.json
+                        let key = obj.Key;
+                        if (key.startsWith(shardPrefix)) {
+                            key = key.substring(shardPrefix.length);
                         }
-                    }));
-                    // Add non-null nodes to result
-                    for (const node of batchNodes) {
-                        if (node) {
-                            nodes.push(node);
+                        if (key.endsWith('.json')) {
+                            key = key.substring(0, key.length - 5);
                         }
-                    }
+                        return key;
+                    });
+                    // Load nodes for this shard (use direct loading for pagination scans)
+                    const shardNodes = await this.loadNodesByIds(nodeIds, false);
+                    nodes.push(...shardNodes);
+                }
+                // Check if we've reached the limit
+                if (nodes.length >= limit) {
+                    const hasMore = !!listResponse.IsTruncated || shardIndex < TOTAL_SHARDS - 1;
+                    const nextCursor = listResponse.IsTruncated
+                        ? `${shardIndex}:${listResponse.NextContinuationToken}`
+                        : shardIndex < TOTAL_SHARDS - 1
+                            ? `${shardIndex + 1}:`
+                            : undefined;
+                    return {
+                        nodes: nodes.slice(0, limit),
+                        hasMore,
+                        nextCursor
+                    };
+                }
+                // If this shard has more data but we haven't hit limit, continue to next shard
+                if (listResponse.IsTruncated) {
+                    return {
+                        nodes,
+                        hasMore: true,
+                        nextCursor: `${shardIndex}:${listResponse.NextContinuationToken}`
+                    };
                 }
             }
-            // Determine if there are more nodes
-            const hasMore = !!listResponse.IsTruncated;
-            // Set next cursor if there are more nodes
-            const nextCursor = listResponse.NextContinuationToken;
+            // All shards exhausted
             return {
                 nodes,
-                hasMore,
-                nextCursor
+                hasMore: false,
+                nextCursor: undefined
             };
         }
         catch (error) {
@@ -943,6 +965,43 @@ export class S3CompatibleStorage extends BaseStorage {
             };
         }
     }
+    /**
+     * Load nodes by IDs efficiently using cache or direct fetch
+     */
+    async loadNodesByIds(nodeIds, useCache) {
+        const nodes = [];
+        if (useCache) {
+            const cachedNodes = await this.nounCacheManager.getMany(nodeIds);
+            for (const id of nodeIds) {
+                const node = cachedNodes.get(id);
+                if (node) {
+                    nodes.push(node);
+                }
+            }
+        }
+        else {
+            // Load directly in batches
+            const batchSize = 50;
+            for (let i = 0; i < nodeIds.length; i += batchSize) {
+                const batch = nodeIds.slice(i, i + batchSize);
+                const batchNodes = await Promise.all(batch.map(async (id) => {
+                    try {
+                        return await this.getNoun_internal(id);
+                    }
+                    catch (error) {
+                        this.logger.warn(`Failed to load node ${id}:`, error);
+                        return null;
+                    }
+                }));
+                for (const node of batchNodes) {
+                    if (node) {
+                        nodes.push(node);
+                    }
+                }
+            }
+        }
+        return nodes;
+    }
     /**
      * Get nouns by noun type (internal implementation)
      * @param nounType The noun type to filter by
@@ -1098,7 +1157,7 @@ export class S3CompatibleStorage extends BaseStorage {
         try {
             // Import the GetObjectCommand only when needed
             const { GetObjectCommand } = await import('@aws-sdk/client-s3');
-            const key = `${this.verbPrefix}${id}.json`;
+            const key = this.getVerbKey(id);
             this.logger.trace(`Getting edge ${id} from key: ${key}`);
             // Try to get the edge from the verbs directory
             const response = await this.s3Client.send(new GetObjectCommand({
@@ -1572,7 +1631,9 @@ export class S3CompatibleStorage extends BaseStorage {
         try {
             // Import the PutObjectCommand only when needed
             const { PutObjectCommand } = await import('@aws-sdk/client-s3');
-            const key = `${this.metadataPrefix}${id}.json`;
+            // Use UUID-based sharding for metadata (consistent with noun vectors)
+            const shardId = getShardIdFromUuid(id);
+            const key = `${this.metadataPrefix}${shardId}/${id}.json`;
             const body = JSON.stringify(metadata, null, 2);
             this.logger.trace(`Saving noun metadata for ${id} to key: ${key}`);
             // Save the noun metadata to S3-compatible storage
@@ -1701,7 +1762,9 @@ export class S3CompatibleStorage extends BaseStorage {
         try {
             // Import the GetObjectCommand only when needed
             const { GetObjectCommand } = await import('@aws-sdk/client-s3');
-            const key = `${this.metadataPrefix}${id}.json`;
+            // Use UUID-based sharding for metadata (consistent with noun vectors)
+            const shardId = getShardIdFromUuid(id);
+            const key = `${this.metadataPrefix}${shardId}/${id}.json`;
             this.logger.trace(`Getting noun metadata for ${id} from key: ${key}`);
             // Try to get the noun metadata
             const response = await this.s3Client.send(new GetObjectCommand({
@@ -2698,12 +2761,54 @@ export class S3CompatibleStorage extends BaseStorage {
                 filteredNodes = filteredByMetadata;
             }
         }
+        // Calculate total count efficiently
+        // For the first page (no cursor), we can estimate total count
+        let totalCount;
+        if (!cursor) {
+            try {
+                totalCount = await this.estimateTotalNounCount();
+            }
+            catch (error) {
+                this.logger.warn('Failed to estimate total noun count:', error);
+                // totalCount remains undefined
+            }
+        }
         return {
             items: filteredNodes,
+            totalCount,
             hasMore: result.hasMore,
             nextCursor: result.nextCursor
         };
     }
+    /**
+     * Estimate total noun count by listing objects across all shards
+     * This is more efficient than loading all nouns
+     */
+    async estimateTotalNounCount() {
+        const { ListObjectsV2Command } = await import('@aws-sdk/client-s3');
+        let totalCount = 0;
+        // Count across all UUID-based shards (00-ff)
+        for (let shardIndex = 0; shardIndex < TOTAL_SHARDS; shardIndex++) {
+            const shardId = getShardIdByIndex(shardIndex);
+            const shardPrefix = `${this.nounPrefix}${shardId}/`;
+            let shardCursor;
+            let hasMore = true;
+            while (hasMore) {
+                const listResponse = await this.s3Client.send(new ListObjectsV2Command({
+                    Bucket: this.bucketName,
+                    Prefix: shardPrefix,
+                    MaxKeys: 1000,
+                    ContinuationToken: shardCursor
+                }));
+                if (listResponse.Contents) {
+                    totalCount += listResponse.Contents.length;
+                }
+                hasMore = !!listResponse.IsTruncated;
+                shardCursor = listResponse.NextContinuationToken;
+            }
+        }
+        return totalCount;
+    }
     /**
      * Initialize counts from S3 storage
      */

package/dist/storage/sharding.d.ts

@@ -0,0 +1,103 @@
+/**
+ * Unified UUID-based sharding for all storage adapters
+ *
+ * Uses first 2 hex characters of UUID for consistent, predictable sharding
+ * that scales from hundreds to millions of entities without configuration.
+ *
+ * Sharding characteristics:
+ * - 256 buckets (00-ff)
+ * - Deterministic (same UUID always maps to same shard)
+ * - No configuration required
+ * - Works across all storage types (filesystem, S3, GCS, memory)
+ * - Efficient for list operations and pagination
+ */
+/**
+ * Extract shard ID from UUID
+ *
+ * Uses first 2 hex characters of the UUID as the shard ID.
+ * This provides 256 evenly-distributed buckets (00-ff).
+ *
+ * @param uuid - UUID string (with or without hyphens)
+ * @returns 2-character hex shard ID (00-ff)
+ *
+ * @example
+ * ```typescript
+ * getShardIdFromUuid('ab123456-1234-5678-9abc-def012345678') // returns 'ab'
+ * getShardIdFromUuid('cd987654-4321-8765-cba9-fed543210987') // returns 'cd'
+ * getShardIdFromUuid('00000000-0000-0000-0000-000000000000') // returns '00'
+ * ```
+ */
+export declare function getShardIdFromUuid(uuid: string): string;
+/**
+ * Get all possible shard IDs (00-ff)
+ *
+ * Returns array of 256 shard IDs in ascending order.
+ * Useful for iterating through all shards during pagination.
+ *
+ * @returns Array of 256 shard IDs
+ *
+ * @example
+ * ```typescript
+ * const shards = getAllShardIds()
+ * // ['00', '01', '02', ..., 'fd', 'fe', 'ff']
+ *
+ * for (const shardId of shards) {
+ *   const prefix = `entities/nouns/vectors/${shardId}/`
+ *   // List objects with this prefix
+ * }
+ * ```
+ */
+export declare function getAllShardIds(): string[];
+/**
+ * Get shard ID for a given index (0-255)
+ *
+ * @param index - Shard index (0-255)
+ * @returns 2-character hex shard ID
+ *
+ * @example
+ * ```typescript
+ * getShardIdByIndex(0)   // '00'
+ * getShardIdByIndex(15)  // '0f'
+ * getShardIdByIndex(255) // 'ff'
+ * ```
+ */
+export declare function getShardIdByIndex(index: number): string;
+/**
+ * Get shard index from shard ID (0-255)
+ *
+ * @param shardId - 2-character hex shard ID
+ * @returns Shard index (0-255)
+ *
+ * @example
+ * ```typescript
+ * getShardIndexFromId('00')  // 0
+ * getShardIndexFromId('0f')  // 15
+ * getShardIndexFromId('ff')  // 255
+ * ```
+ */
+export declare function getShardIndexFromId(shardId: string): number;
+/**
+ * Total number of shards in the system
+ */
+export declare const TOTAL_SHARDS = 256;
+/**
+ * Shard configuration (read-only)
+ */
+export declare const SHARD_CONFIG: {
+    /**
+     * Total number of shards (256)
+     */
+    readonly count: 256;
+    /**
+     * Number of hex characters used for sharding (2)
+     */
+    readonly prefixLength: 2;
+    /**
+     * Sharding method description
+     */
+    readonly method: "uuid-prefix";
+    /**
+     * Whether sharding is always enabled
+     */
+    readonly alwaysEnabled: true;
+};

package/dist/storage/sharding.js

@@ -0,0 +1,137 @@
+/**
+ * Unified UUID-based sharding for all storage adapters
+ *
+ * Uses first 2 hex characters of UUID for consistent, predictable sharding
+ * that scales from hundreds to millions of entities without configuration.
+ *
+ * Sharding characteristics:
+ * - 256 buckets (00-ff)
+ * - Deterministic (same UUID always maps to same shard)
+ * - No configuration required
+ * - Works across all storage types (filesystem, S3, GCS, memory)
+ * - Efficient for list operations and pagination
+ */
+/**
+ * Extract shard ID from UUID
+ *
+ * Uses first 2 hex characters of the UUID as the shard ID.
+ * This provides 256 evenly-distributed buckets (00-ff).
+ *
+ * @param uuid - UUID string (with or without hyphens)
+ * @returns 2-character hex shard ID (00-ff)
+ *
+ * @example
+ * ```typescript
+ * getShardIdFromUuid('ab123456-1234-5678-9abc-def012345678') // returns 'ab'
+ * getShardIdFromUuid('cd987654-4321-8765-cba9-fed543210987') // returns 'cd'
+ * getShardIdFromUuid('00000000-0000-0000-0000-000000000000') // returns '00'
+ * ```
+ */
+export function getShardIdFromUuid(uuid) {
+    if (!uuid) {
+        throw new Error('UUID is required for sharding');
+    }
+    // Remove hyphens and convert to lowercase
+    const normalized = uuid.toLowerCase().replace(/-/g, '');
+    // Validate UUID format (32 hex characters)
+    if (normalized.length !== 32) {
+        throw new Error(`Invalid UUID format: ${uuid} (expected 32 hex chars, got ${normalized.length})`);
+    }
+    // Extract first 2 characters
+    const shardId = normalized.substring(0, 2);
+    // Validate hex format
+    if (!/^[0-9a-f]{2}$/.test(shardId)) {
+        throw new Error(`Invalid UUID prefix: ${shardId} (expected 2 hex chars)`);
+    }
+    return shardId;
+}
+/**
+ * Get all possible shard IDs (00-ff)
+ *
+ * Returns array of 256 shard IDs in ascending order.
+ * Useful for iterating through all shards during pagination.
+ *
+ * @returns Array of 256 shard IDs
+ *
+ * @example
+ * ```typescript
+ * const shards = getAllShardIds()
+ * // ['00', '01', '02', ..., 'fd', 'fe', 'ff']
+ *
+ * for (const shardId of shards) {
+ *   const prefix = `entities/nouns/vectors/${shardId}/`
+ *   // List objects with this prefix
+ * }
+ * ```
+ */
+export function getAllShardIds() {
+    const shards = [];
+    for (let i = 0; i < 256; i++) {
+        shards.push(i.toString(16).padStart(2, '0'));
+    }
+    return shards;
+}
+/**
+ * Get shard ID for a given index (0-255)
+ *
+ * @param index - Shard index (0-255)
+ * @returns 2-character hex shard ID
+ *
+ * @example
+ * ```typescript
+ * getShardIdByIndex(0)   // '00'
+ * getShardIdByIndex(15)  // '0f'
+ * getShardIdByIndex(255) // 'ff'
+ * ```
+ */
+export function getShardIdByIndex(index) {
+    if (index < 0 || index > 255) {
+        throw new Error(`Shard index out of range: ${index} (expected 0-255)`);
+    }
+    return index.toString(16).padStart(2, '0');
+}
+/**
+ * Get shard index from shard ID (0-255)
+ *
+ * @param shardId - 2-character hex shard ID
+ * @returns Shard index (0-255)
+ *
+ * @example
+ * ```typescript
+ * getShardIndexFromId('00')  // 0
+ * getShardIndexFromId('0f')  // 15
+ * getShardIndexFromId('ff')  // 255
+ * ```
+ */
+export function getShardIndexFromId(shardId) {
+    if (!/^[0-9a-f]{2}$/.test(shardId)) {
+        throw new Error(`Invalid shard ID: ${shardId} (expected 2 hex chars)`);
+    }
+    return parseInt(shardId, 16);
+}
+/**
+ * Total number of shards in the system
+ */
+export const TOTAL_SHARDS = 256;
+/**
+ * Shard configuration (read-only)
+ */
+export const SHARD_CONFIG = {
+    /**
+     * Total number of shards (256)
+     */
+    count: TOTAL_SHARDS,
+    /**
+     * Number of hex characters used for sharding (2)
+     */
+    prefixLength: 2,
+    /**
+     * Sharding method description
+     */
+    method: 'uuid-prefix',
+    /**
+     * Whether sharding is always enabled
+     */
+    alwaysEnabled: true
+};
+//# sourceMappingURL=sharding.js.map