@soulcraft/brainy 3.20.3 → 3.20.4

package/dist/storage/adapters/fileSystemStorage.js

@@ -897,26 +897,32 @@ export class FileSystemStorage extends BaseStorage {
         const limit = options.limit || 100;
         const startIndex = options.cursor ? parseInt(options.cursor, 10) : 0;
         try {
-            // Production-scale optimization: Use persisted count for total instead of scanning
-            const totalCount = this.totalVerbCount || 0;
+            // Get actual verb files first (critical for accuracy)
+            const verbFiles = await this.getAllShardedFiles(this.verbsDir);
+            verbFiles.sort(); // Consistent ordering for pagination
+            // Use actual file count - don't trust cached totalVerbCount
+            // This prevents accessing undefined array elements
+            const actualFileCount = verbFiles.length;
             // For large datasets, warn about performance
-            if (totalCount > 1000000) {
-                console.warn(`Very large verb dataset detected (${totalCount} verbs). Performance may be degraded. Consider database storage for optimal performance.`);
+            if (actualFileCount > 1000000) {
+                console.warn(`Very large verb dataset detected (${actualFileCount} verbs). Performance may be degraded. Consider database storage for optimal performance.`);
             }
-            // Calculate pagination bounds
-            const endIndex = Math.min(startIndex + limit, totalCount);
-            const hasMore = endIndex < totalCount;
             // For production-scale datasets, use streaming approach
-            if (totalCount > 50000) {
+            if (actualFileCount > 50000) {
                 return await this.getVerbsWithPaginationStreaming(options, startIndex, limit);
             }
-            // For smaller datasets, use the current approach (with optimizations)
-            const verbFiles = await this.getAllShardedFiles(this.verbsDir);
-            verbFiles.sort(); // This is still acceptable for <50k files
+            // Calculate pagination bounds using ACTUAL file count
+            const endIndex = Math.min(startIndex + limit, actualFileCount);
             // Load the requested page of verbs
             const verbs = [];
+            let successfullyLoaded = 0;
             for (let i = startIndex; i < endIndex; i++) {
                 const file = verbFiles[i];
+                // CRITICAL: Null-safety check for undefined array elements
+                if (!file) {
+                    console.warn(`Unexpected undefined file at index ${i}, skipping`);
+                    continue;
+                }
                 const id = file.replace('.json', '');
                 try {
                     // Read the verb data (HNSWVerb stored as edge) - use sharded path
@@ -1003,14 +1009,18 @@ export class FileSystemStorage extends BaseStorage {
                         }
                     }
                     verbs.push(verb);
+                    successfullyLoaded++;
                 }
                 catch (error) {
                     console.warn(`Failed to read verb ${id}:`, error);
                 }
             }
+            // CRITICAL FIX: hasMore based on actual file count, not cached totalVerbCount
+            // Also verify we successfully loaded items (prevents infinite loops on corrupted storage)
+            const hasMore = (endIndex < actualFileCount) && (successfullyLoaded > 0 || startIndex === 0);
             return {
                 items: verbs,
-                totalCount,
+                totalCount: actualFileCount, // Return actual count, not cached value
                 hasMore,
                 nextCursor: hasMore ? String(endIndex) : undefined
             };
@@ -1587,14 +1597,14 @@ export class FileSystemStorage extends BaseStorage {
      */
     async getVerbsWithPaginationStreaming(options, startIndex, limit) {
         const verbs = [];
-        const totalCount = this.totalVerbCount || 0;
         let processedCount = 0;
         let skippedCount = 0;
         let resultCount = 0;
         const depth = this.cachedShardingDepth ?? this.getOptimalShardingDepth();
         try {
             // Stream through sharded directories efficiently
-            const hasMore = await this.streamShardedFiles(this.verbsDir, depth, async (filename, filePath) => {
+            // hasMore=false means we reached the end of files, hasMore=true means streaming stopped early
+            const streamingHasMore = await this.streamShardedFiles(this.verbsDir, depth, async (filename, filePath) => {
                 // Skip files until we reach start index
                 if (skippedCount < startIndex) {
                     skippedCount++;
@@ -1602,7 +1612,7 @@ export class FileSystemStorage extends BaseStorage {
                 }
                 // Stop if we have enough results
                 if (resultCount >= limit) {
-                    return false; // stop streaming
+                    return false; // stop streaming - more files exist
                 }
                 try {
                     const id = filename.replace('.json', '');
@@ -1666,10 +1676,13 @@ export class FileSystemStorage extends BaseStorage {
                     return true; // continue
                 }
             });
-            const finalHasMore = (startIndex + resultCount) < totalCount;
+            // CRITICAL FIX: Use streaming result for hasMore, not cached totalVerbCount
+            // streamingHasMore=false means we exhausted all files
+            // Also verify we loaded items to prevent infinite loops
+            const finalHasMore = streamingHasMore && (resultCount > 0 || startIndex === 0);
             return {
                 items: verbs,
-                totalCount,
+                totalCount: this.totalVerbCount || undefined, // Return cached count as hint only
                 hasMore: finalHasMore,
                 nextCursor: finalHasMore ? String(startIndex + resultCount) : undefined
             };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@soulcraft/brainy",
-  "version": "3.20.3",
+  "version": "3.20.4",
   "description": "Universal Knowledge Protocol™ - World's first Triple Intelligence database unifying vector, graph, and document search in one API. 31 nouns × 40 verbs for infinite expressiveness.",
   "main": "dist/index.js",
   "module": "dist/index.js",