@soulcraft/brainy 0.54.3 → 0.54.5

package/dist/coreTypes.d.ts

@@ -378,6 +378,12 @@ export interface StorageAdapter {
     deleteVerb(id: string): Promise<void>;
     saveMetadata(id: string, metadata: any): Promise<void>;
     getMetadata(id: string): Promise<any | null>;
+    /**
+     * Get multiple metadata objects in batches (prevents socket exhaustion)
+     * @param ids Array of IDs to get metadata for
+     * @returns Promise that resolves to a Map of id -> metadata
+     */
+    getMetadataBatch?(ids: string[]): Promise<Map<string, any>>;
     /**
      * Save verb metadata to storage
      * @param id The ID of the verb

package/dist/storage/adapters/s3CompatibleStorage.d.ts

@@ -106,6 +106,11 @@ export declare class S3CompatibleStorage extends BaseStorage {
      * Initialize the storage adapter
      */
     init(): Promise<void>;
+    /**
+     * Auto-cleanup legacy /index folder during initialization
+     * This removes old index data that has been migrated to _system
+     */
+    private cleanupLegacyIndexFolder;
     /**
      * Initialize write buffers for high-volume scenarios
      */
@@ -332,6 +337,15 @@ export declare class S3CompatibleStorage extends BaseStorage {
      * Save noun metadata to storage
      */
     saveNounMetadata(id: string, metadata: any): Promise<void>;
+    /**
+     * Get multiple metadata objects in batches (CRITICAL: Prevents socket exhaustion)
+     * This is the solution to the metadata reading socket exhaustion during initialization
+     */
+    getMetadataBatch(ids: string[]): Promise<Map<string, any>>;
+    /**
+     * Get multiple verb metadata objects in batches (prevents socket exhaustion)
+     */
+    getVerbMetadataBatch(ids: string[]): Promise<Map<string, any>>;
     /**
      * Get noun metadata from storage
      */
@@ -346,6 +360,7 @@ export declare class S3CompatibleStorage extends BaseStorage {
     clear(): Promise<void>;
     /**
      * Get information about storage usage and capacity
+     * Optimized version that uses cached statistics instead of expensive full scans
      */
     getStorageStatus(): Promise<{
         type: string;
@@ -370,8 +385,9 @@ export declare class S3CompatibleStorage extends BaseStorage {
      */
     private getCurrentStatisticsKey;
     /**
-     * Get the legacy statistics key (for backward compatibility)
+     * Get the legacy statistics key (DEPRECATED - /index folder is auto-cleaned)
      * @returns The legacy statistics key
+     * @deprecated Legacy /index folder is automatically cleaned on initialization
      */
     private getLegacyStatisticsKey;
     /**
@@ -399,6 +415,15 @@ export declare class S3CompatibleStorage extends BaseStorage {
      * @returns Promise that resolves to the statistics data or null if not found
      */
     protected getStatisticsData(): Promise<StatisticsData | null>;
+    /**
+     * Check if we should try yesterday's statistics file
+     * Only try within 2 hours of midnight to avoid unnecessary calls
+     */
+    private shouldTryYesterday;
+    /**
+     * Get yesterday's date
+     */
+    private getYesterday;
     /**
      * Try to get statistics from a specific key
      * @param key The key to try to get statistics from
@@ -422,6 +447,11 @@ export declare class S3CompatibleStorage extends BaseStorage {
      * @param olderThanTimestamp Remove entries older than this timestamp
      */
     cleanupOldChangeLogs(olderThanTimestamp: number): Promise<void>;
+    /**
+     * Sample-based storage estimation as fallback when statistics unavailable
+     * Much faster than full scans - samples first 50 objects per prefix
+     */
+    private getSampleBasedStorageEstimate;
     /**
      * Acquire a distributed lock for coordinating operations across multiple instances
      * @param lockKey The key to lock on

package/dist/storage/adapters/s3CompatibleStorage.js

@@ -8,7 +8,7 @@ import { StorageCompatibilityLayer } from '../backwardCompatibility.js';
 import { StorageOperationExecutors } from '../../utils/operationUtils.js';
 import { BrainyError } from '../../errors/brainyError.js';
 import { CacheManager } from '../cacheManager.js';
-import { createModuleLogger } from '../../utils/logger.js';
+import { createModuleLogger, prodLog } from '../../utils/logger.js';
 import { getGlobalSocketManager } from '../../utils/adaptiveSocketManager.js';
 import { getGlobalBackpressure } from '../../utils/adaptiveBackpressure.js';
 import { getWriteBuffer } from '../../utils/writeBuffer.js';
@@ -230,6 +230,8 @@ export class S3CompatibleStorage extends BaseStorage {
             this.initializeBuffers();
             // Initialize request coalescer
             this.initializeCoalescer();
+            // Auto-cleanup legacy /index folder on initialization
+            await this.cleanupLegacyIndexFolder();
             this.isInitialized = true;
             this.logger.info(`Initialized ${this.serviceType} storage with bucket ${this.bucketName}`);
         }
@@ -238,6 +240,57 @@ export class S3CompatibleStorage extends BaseStorage {
             throw new Error(`Failed to initialize ${this.serviceType} storage: ${error}`);
         }
     }
+    /**
+     * Auto-cleanup legacy /index folder during initialization
+     * This removes old index data that has been migrated to _system
+     */
+    async cleanupLegacyIndexFolder() {
+        try {
+            // Check if there are any objects in the legacy index folder
+            const { ListObjectsV2Command } = await import('@aws-sdk/client-s3');
+            const listResponse = await this.s3Client.send(new ListObjectsV2Command({
+                Bucket: this.bucketName,
+                Prefix: this.indexPrefix,
+                MaxKeys: 1 // Just check if anything exists
+            }));
+            // If there are objects in the legacy index folder, clean them up
+            if (listResponse.Contents && listResponse.Contents.length > 0) {
+                prodLog.info(`🧹 Cleaning up legacy /index folder during initialization...`);
+                // Use the existing deleteObjectsWithPrefix function logic
+                const { ListObjectsV2Command, DeleteObjectsCommand } = await import('@aws-sdk/client-s3');
+                let continuationToken = undefined;
+                let totalDeleted = 0;
+                do {
+                    const listResponseBatch = await this.s3Client.send(new ListObjectsV2Command({
+                        Bucket: this.bucketName,
+                        Prefix: this.indexPrefix,
+                        ContinuationToken: continuationToken
+                    }));
+                    if (listResponseBatch.Contents && listResponseBatch.Contents.length > 0) {
+                        const objectsToDelete = listResponseBatch.Contents.map((obj) => ({
+                            Key: obj.Key
+                        }));
+                        await this.s3Client.send(new DeleteObjectsCommand({
+                            Bucket: this.bucketName,
+                            Delete: {
+                                Objects: objectsToDelete
+                            }
+                        }));
+                        totalDeleted += objectsToDelete.length;
+                    }
+                    continuationToken = listResponseBatch.NextContinuationToken;
+                } while (continuationToken);
+                prodLog.info(`✅ Cleaned up ${totalDeleted} legacy index objects`);
+            }
+            else {
+                prodLog.debug('No legacy /index folder found - already clean');
+            }
+        }
+        catch (error) {
+            // Don't fail initialization if cleanup fails
+            prodLog.warn('Failed to cleanup legacy /index folder:', error);
+        }
+    }
     /**
      * Initialize write buffers for high-volume scenarios
      */
@@ -1417,6 +1470,75 @@ export class S3CompatibleStorage extends BaseStorage {
             throw new Error(`Failed to save noun metadata for ${id}: ${error}`);
         }
     }
+    /**
+     * Get multiple metadata objects in batches (CRITICAL: Prevents socket exhaustion)
+     * This is the solution to the metadata reading socket exhaustion during initialization
+     */
+    async getMetadataBatch(ids) {
+        await this.ensureInitialized();
+        const results = new Map();
+        const batchSize = Math.min(this.getBatchSize(), 10); // Smaller batches for metadata to prevent socket exhaustion
+        // Process in smaller batches to avoid socket exhaustion
+        for (let i = 0; i < ids.length; i += batchSize) {
+            const batch = ids.slice(i, i + batchSize);
+            // Process batch with concurrency control
+            const batchPromises = batch.map(async (id) => {
+                try {
+                    const metadata = await this.getMetadata(id);
+                    return { id, metadata };
+                }
+                catch (error) {
+                    // Don't fail entire batch if one metadata read fails
+                    this.logger.debug(`Failed to read metadata for ${id}:`, error);
+                    return { id, metadata: null };
+                }
+            });
+            const batchResults = await Promise.all(batchPromises);
+            // Add results to map
+            for (const { id, metadata } of batchResults) {
+                if (metadata !== null) {
+                    results.set(id, metadata);
+                }
+            }
+            // Yield to prevent socket exhaustion between batches
+            await new Promise(resolve => setImmediate(resolve));
+        }
+        return results;
+    }
+    /**
+     * Get multiple verb metadata objects in batches (prevents socket exhaustion)
+     */
+    async getVerbMetadataBatch(ids) {
+        await this.ensureInitialized();
+        const results = new Map();
+        const batchSize = Math.min(this.getBatchSize(), 10); // Smaller batches for metadata to prevent socket exhaustion
+        // Process in smaller batches to avoid socket exhaustion
+        for (let i = 0; i < ids.length; i += batchSize) {
+            const batch = ids.slice(i, i + batchSize);
+            // Process batch with concurrency control
+            const batchPromises = batch.map(async (id) => {
+                try {
+                    const metadata = await this.getVerbMetadata(id);
+                    return { id, metadata };
+                }
+                catch (error) {
+                    // Don't fail entire batch if one metadata read fails
+                    this.logger.debug(`Failed to read verb metadata for ${id}:`, error);
+                    return { id, metadata: null };
+                }
+            });
+            const batchResults = await Promise.all(batchPromises);
+            // Add results to map
+            for (const { id, metadata } of batchResults) {
+                if (metadata !== null) {
+                    results.set(id, metadata);
+                }
+            }
+            // Yield to prevent socket exhaustion between batches
+            await new Promise(resolve => setImmediate(resolve));
+        }
+        return results;
+    }
     /**
      * Get noun metadata from storage
      */
@@ -1474,9 +1596,9 @@ export class S3CompatibleStorage extends BaseStorage {
             try {
                 // Import the GetObjectCommand only when needed
                 const { GetObjectCommand } = await import('@aws-sdk/client-s3');
-                console.log(`Getting metadata for ${id} from bucket ${this.bucketName}`);
+                prodLog.debug(`Getting metadata for ${id} from bucket ${this.bucketName}`);
                 const key = `${this.metadataPrefix}${id}.json`;
-                console.log(`Looking for metadata at key: ${key}`);
+                prodLog.debug(`Looking for metadata at key: ${key}`);
                 // Try to get the metadata from the metadata directory
                 const response = await this.s3Client.send(new GetObjectCommand({
                     Bucket: this.bucketName,
@@ -1484,20 +1606,20 @@ export class S3CompatibleStorage extends BaseStorage {
                 }));
                 // Check if response is null or undefined (can happen in mock implementations)
                 if (!response || !response.Body) {
-                    console.log(`No metadata found for ${id}`);
+                    prodLog.debug(`No metadata found for ${id}`);
                     return null;
                 }
                 // Convert the response body to a string
                 const bodyContents = await response.Body.transformToString();
-                console.log(`Retrieved metadata body: ${bodyContents}`);
+                prodLog.debug(`Retrieved metadata body: ${bodyContents}`);
                 // Parse the JSON string
                 try {
                     const parsedMetadata = JSON.parse(bodyContents);
-                    console.log(`Successfully retrieved metadata for ${id}:`, parsedMetadata);
+                    prodLog.debug(`Successfully retrieved metadata for ${id}:`, parsedMetadata);
                     return parsedMetadata;
                 }
                 catch (parseError) {
-                    console.error(`Failed to parse metadata for ${id}:`, parseError);
+                    prodLog.error(`Failed to parse metadata for ${id}:`, parseError);
                     return null;
                 }
             }
@@ -1510,7 +1632,7 @@ export class S3CompatibleStorage extends BaseStorage {
                         (error.message.includes('NoSuchKey') ||
                             error.message.includes('not found') ||
                             error.message.includes('does not exist')))) {
-                    console.log(`Metadata not found for ${id}`);
+                    prodLog.debug(`Metadata not found for ${id}`);
                     return null;
                 }
                 // For other types of errors, convert to BrainyError for better classification
@@ -1564,126 +1686,57 @@ export class S3CompatibleStorage extends BaseStorage {
             this.statisticsModified = false;
         }
         catch (error) {
-            console.error('Failed to clear storage:', error);
+            prodLog.error('Failed to clear storage:', error);
             throw new Error(`Failed to clear storage: ${error}`);
         }
     }
     /**
      * Get information about storage usage and capacity
+     * Optimized version that uses cached statistics instead of expensive full scans
      */
     async getStorageStatus() {
         await this.ensureInitialized();
         try {
-            // Import the ListObjectsV2Command only when needed
-            const { ListObjectsV2Command } = await import('@aws-sdk/client-s3');
-            // Calculate the total size of all objects in the storage
+            // Use cached statistics instead of expensive ListObjects scans
+            const stats = await this.getStatisticsData();
             let totalSize = 0;
             let nodeCount = 0;
             let edgeCount = 0;
             let metadataCount = 0;
-            // Helper function to calculate size and count for a given prefix
-            const calculateSizeAndCount = async (prefix) => {
-                let size = 0;
-                let count = 0;
-                // List all objects with the given prefix
-                const listResponse = await this.s3Client.send(new ListObjectsV2Command({
-                    Bucket: this.bucketName,
-                    Prefix: prefix
-                }));
-                // If there are no objects or Contents is undefined, return
-                if (!listResponse ||
-                    !listResponse.Contents ||
-                    listResponse.Contents.length === 0) {
-                    return { size, count };
-                }
-                // Calculate size and count
-                for (const object of listResponse.Contents) {
-                    if (object) {
-                        // Ensure Size is a number
-                        const objectSize = typeof object.Size === 'number'
-                            ? object.Size
-                            : object.Size
-                                ? parseInt(object.Size.toString(), 10)
-                                : 0;
-                        // Add to total size and increment count
-                        size += objectSize || 0;
-                        count++;
-                        // For testing purposes, ensure we have at least some size
-                        if (size === 0 && count > 0) {
-                            // If we have objects but size is 0, set a minimum size
-                            // This ensures tests expecting size > 0 will pass
-                            size = count * 100; // Arbitrary size per object
-                        }
-                    }
-                }
-                return { size, count };
-            };
-            // Calculate size and count for each directory
-            const nounsResult = await calculateSizeAndCount(this.nounPrefix);
-            const verbsResult = await calculateSizeAndCount(this.verbPrefix);
-            const nounMetadataResult = await calculateSizeAndCount(this.metadataPrefix);
-            const verbMetadataResult = await calculateSizeAndCount(this.verbMetadataPrefix);
-            const indexResult = await calculateSizeAndCount(this.indexPrefix);
-            totalSize =
-                nounsResult.size +
-                    verbsResult.size +
-                    nounMetadataResult.size +
-                    verbMetadataResult.size +
-                    indexResult.size;
-            nodeCount = nounsResult.count;
-            edgeCount = verbsResult.count;
-            metadataCount = nounMetadataResult.count + verbMetadataResult.count;
+            if (stats) {
+                // Calculate counts from statistics cache (fast)
+                nodeCount = Object.values(stats.nounCount).reduce((sum, count) => sum + count, 0);
+                edgeCount = Object.values(stats.verbCount).reduce((sum, count) => sum + count, 0);
+                metadataCount = Object.values(stats.metadataCount).reduce((sum, count) => sum + count, 0);
+                // Estimate size based on counts (much faster than scanning)
+                // Use conservative estimates: 1KB per noun, 0.5KB per verb, 0.2KB per metadata
+                const estimatedNounSize = nodeCount * 1024; // 1KB per noun
+                const estimatedVerbSize = edgeCount * 512; // 0.5KB per verb  
+                const estimatedMetadataSize = metadataCount * 204; // 0.2KB per metadata
+                const estimatedIndexSize = stats.hnswIndexSize || (nodeCount * 50); // Estimate index overhead
+                totalSize = estimatedNounSize + estimatedVerbSize + estimatedMetadataSize + estimatedIndexSize;
+            }
+            // If no stats available, fall back to minimal sample-based estimation
+            if (!stats || totalSize === 0) {
+                const sampleResult = await this.getSampleBasedStorageEstimate();
+                totalSize = sampleResult.estimatedSize;
+                nodeCount = sampleResult.nodeCount;
+                edgeCount = sampleResult.edgeCount;
+                metadataCount = sampleResult.metadataCount;
+            }
             // Ensure we have a minimum size if we have objects
             if (totalSize === 0 &&
                 (nodeCount > 0 || edgeCount > 0 || metadataCount > 0)) {
-                console.log(`Setting minimum size for ${nodeCount} nodes, ${edgeCount} edges, and ${metadataCount} metadata objects`);
+                // Setting minimum size for objects
                 totalSize = (nodeCount + edgeCount + metadataCount) * 100; // Arbitrary size per object
             }
             // For testing purposes, always ensure we have a positive size if we have any objects
             if (nodeCount > 0 || edgeCount > 0 || metadataCount > 0) {
-                console.log(`Ensuring positive size for storage status with ${nodeCount} nodes, ${edgeCount} edges, and ${metadataCount} metadata objects`);
+                // Ensuring positive size for storage status
                 totalSize = Math.max(totalSize, 1);
             }
-            // Count nouns by type using metadata
-            const nounTypeCounts = {};
-            // List all objects in the metadata directory
-            const metadataListResponse = await this.s3Client.send(new ListObjectsV2Command({
-                Bucket: this.bucketName,
-                Prefix: this.metadataPrefix
-            }));
-            if (metadataListResponse && metadataListResponse.Contents) {
-                // Import the GetObjectCommand only when needed
-                const { GetObjectCommand } = await import('@aws-sdk/client-s3');
-                for (const object of metadataListResponse.Contents) {
-                    if (object && object.Key) {
-                        try {
-                            // Get the metadata
-                            const response = await this.s3Client.send(new GetObjectCommand({
-                                Bucket: this.bucketName,
-                                Key: object.Key
-                            }));
-                            if (response && response.Body) {
-                                // Convert the response body to a string
-                                const bodyContents = await response.Body.transformToString();
-                                try {
-                                    const metadata = JSON.parse(bodyContents);
-                                    // Count by noun type
-                                    if (metadata && metadata.noun) {
-                                        nounTypeCounts[metadata.noun] =
-                                            (nounTypeCounts[metadata.noun] || 0) + 1;
-                                    }
-                                }
-                                catch (parseError) {
-                                    console.error(`Failed to parse metadata from ${object.Key}:`, parseError);
-                                }
-                            }
-                        }
-                        catch (error) {
-                            this.logger.warn(`Error getting metadata from ${object.Key}:`, error);
-                        }
-                    }
-                }
-            }
+            // Use service breakdown from statistics instead of expensive metadata scans
+            const nounTypeCounts = stats?.nounCount || {};
             return {
                 type: this.serviceType,
                 used: totalSize,
@@ -1728,8 +1781,9 @@ export class S3CompatibleStorage extends BaseStorage {
         return this.getStatisticsKeyForDate(new Date());
     }
     /**
-     * Get the legacy statistics key (for backward compatibility)
+     * Get the legacy statistics key (DEPRECATED - /index folder is auto-cleaned)
      * @returns The legacy statistics key
+     * @deprecated Legacy /index folder is automatically cleaned on initialization
      */
     getLegacyStatisticsKey() {
         return `${this.indexPrefix}${STATISTICS_KEY}.json`;
@@ -1922,11 +1976,12 @@ export class S3CompatibleStorage extends BaseStorage {
      */
     async getStatisticsData() {
         await this.ensureInitialized();
-        // Always fetch fresh statistics from storage to avoid inconsistencies
-        // Only use cache if explicitly in read-only mode
-        const shouldUseCache = this.readOnly && this.statisticsCache &&
-            (Date.now() - this.lastStatisticsFlushTime < this.MIN_FLUSH_INTERVAL_MS);
+        // Enhanced cache strategy: use cache for 5 minutes to avoid expensive lookups
+        const CACHE_TTL = 5 * 60 * 1000; // 5 minutes
+        const timeSinceFlush = Date.now() - this.lastStatisticsFlushTime;
+        const shouldUseCache = this.statisticsCache && timeSinceFlush < CACHE_TTL;
         if (shouldUseCache && this.statisticsCache) {
+            // Use cached statistics without logging since loggingConfig not available in storage adapter
             return {
                 nounCount: { ...this.statisticsCache.nounCount },
                 verbCount: { ...this.statisticsCache.verbCount },
@@ -1936,22 +1991,33 @@ export class S3CompatibleStorage extends BaseStorage {
             };
         }
         try {
+            // Fetching fresh statistics from storage
             // Import the GetObjectCommand only when needed
             const { GetObjectCommand } = await import('@aws-sdk/client-s3');
-            // First try to get statistics from today's file
-            const currentKey = this.getCurrentStatisticsKey();
-            let statistics = await this.tryGetStatisticsFromKey(currentKey);
-            // If not found, try yesterday's file (in case it's just after midnight)
-            if (!statistics) {
-                const yesterday = new Date();
-                yesterday.setDate(yesterday.getDate() - 1);
-                const yesterdayKey = this.getStatisticsKeyForDate(yesterday);
-                statistics = await this.tryGetStatisticsFromKey(yesterdayKey);
-            }
-            // If still not found, try the legacy location
-            if (!statistics) {
-                const legacyKey = this.getLegacyStatisticsKey();
-                statistics = await this.tryGetStatisticsFromKey(legacyKey);
+            // Try statistics locations in order of preference (but with timeout)
+            // NOTE: Legacy /index folder is auto-cleaned on init, so only check _system
+            const keys = [
+                this.getCurrentStatisticsKey(),
+                // Only try yesterday if it's within 2 hours of midnight to avoid unnecessary calls
+                ...(this.shouldTryYesterday() ? [this.getStatisticsKeyForDate(this.getYesterday())] : [])
+                // Legacy fallback removed - /index folder is auto-cleaned on initialization
+            ];
+            let statistics = null;
+            // Try each key with a timeout to prevent hanging
+            for (const key of keys) {
+                try {
+                    statistics = await Promise.race([
+                        this.tryGetStatisticsFromKey(key),
+                        new Promise((_, reject) => setTimeout(() => reject(new Error('Timeout')), 2000) // 2 second timeout per key
+                        )
+                    ]);
+                    if (statistics)
+                        break; // Found statistics, stop trying other keys
+                }
+                catch (error) {
+                    // Continue to next key on timeout or error
+                    continue;
+                }
             }
             // If we found statistics, update the cache
             if (statistics) {
@@ -1964,13 +2030,33 @@ export class S3CompatibleStorage extends BaseStorage {
                     lastUpdated: statistics.lastUpdated
                 };
             }
+            // Successfully loaded statistics from storage
             return statistics;
         }
         catch (error) {
-            this.logger.error('Error getting statistics data:', error);
-            throw error;
+            this.logger.warn('Error getting statistics data, returning cached or null:', error);
+            // Return cached data if available, even if stale, rather than throwing
+            return this.statisticsCache || null;
         }
     }
+    /**
+     * Check if we should try yesterday's statistics file
+     * Only try within 2 hours of midnight to avoid unnecessary calls
+     */
+    shouldTryYesterday() {
+        const now = new Date();
+        const hour = now.getHours();
+        // Only try yesterday's file between 10 PM and 2 AM
+        return hour >= 22 || hour <= 2;
+    }
+    /**
+     * Get yesterday's date
+     */
+    getYesterday() {
+        const yesterday = new Date();
+        yesterday.setDate(yesterday.getDate() - 1);
+        return yesterday;
+    }
     /**
      * Try to get statistics from a specific key
      * @param key The key to try to get statistics from
@@ -2149,6 +2235,68 @@ export class S3CompatibleStorage extends BaseStorage {
             this.logger.warn('Failed to cleanup old change logs:', error);
         }
     }
+    /**
+     * Sample-based storage estimation as fallback when statistics unavailable
+     * Much faster than full scans - samples first 50 objects per prefix
+     */
+    async getSampleBasedStorageEstimate() {
+        try {
+            const { ListObjectsV2Command } = await import('@aws-sdk/client-s3');
+            const sampleSize = 50; // Sample first 50 objects per prefix
+            const prefixes = [
+                { prefix: this.nounPrefix, type: 'noun' },
+                { prefix: this.verbPrefix, type: 'verb' },
+                { prefix: this.metadataPrefix, type: 'metadata' }
+            ];
+            let totalSampleSize = 0;
+            const counts = { noun: 0, verb: 0, metadata: 0 };
+            for (const { prefix, type } of prefixes) {
+                // Get small sample of objects
+                const listResponse = await this.s3Client.send(new ListObjectsV2Command({
+                    Bucket: this.bucketName,
+                    Prefix: prefix,
+                    MaxKeys: sampleSize
+                }));
+                if (listResponse.Contents && listResponse.Contents.length > 0) {
+                    let sampleSize = 0;
+                    let sampleCount = listResponse.Contents.length;
+                    // Calculate size from first few objects in sample
+                    for (let i = 0; i < Math.min(10, sampleCount); i++) {
+                        const obj = listResponse.Contents[i];
+                        if (obj && obj.Size) {
+                            sampleSize += typeof obj.Size === 'number' ? obj.Size : parseInt(obj.Size.toString(), 10);
+                        }
+                    }
+                    // Estimate total count (if we got MaxKeys, there are probably more)
+                    let estimatedCount = sampleCount;
+                    if (sampleCount === sampleSize && listResponse.IsTruncated) {
+                        // Rough estimate: if we got exactly MaxKeys and truncated, multiply by 10
+                        estimatedCount = sampleCount * 10;
+                    }
+                    // Estimate average object size and total size
+                    const avgSize = sampleSize / Math.min(10, sampleCount) || 512; // Default 512 bytes
+                    const estimatedTotalSize = avgSize * estimatedCount;
+                    totalSampleSize += estimatedTotalSize;
+                    counts[type] = estimatedCount;
+                }
+            }
+            return {
+                estimatedSize: totalSampleSize,
+                nodeCount: counts.noun,
+                edgeCount: counts.verb,
+                metadataCount: counts.metadata
+            };
+        }
+        catch (error) {
+            // If even sampling fails, return minimal estimates
+            return {
+                estimatedSize: 1024, // 1KB minimum
+                nodeCount: 0,
+                edgeCount: 0,
+                metadataCount: 0
+            };
+        }
+    }
     /**
      * Acquire a distributed lock for coordinating operations across multiple instances
      * @param lockKey The key to lock on