@soulcraft/brainy 5.7.8 → 5.7.10

package/CHANGELOG.md

@@ -2,6 +2,11 @@
 
 All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines.
 
+### [5.7.9](https://github.com/soulcraftlabs/brainy/compare/v5.7.8...v5.7.9) (2025-11-13)
+
+- fix: implement exists: false and missing operators in MetadataIndexManager (b0f72ef)
+
+
 ### [5.7.8](https://github.com/soulcraftlabs/brainy/compare/v5.7.7...v5.7.8) (2025-11-13)
 
 - fix: reconstruct Map from JSON for HNSW connections (v5.7.8 hotfix) (f6f2717)

package/dist/storage/baseStorage.d.ts

@@ -458,6 +458,30 @@ export declare abstract class BaseStorage extends BaseStorageAdapter {
      * Verb type is a required field in HNSWVerb
      */
     protected getVerbType(verb: HNSWVerb | GraphVerb): VerbType;
+    /**
+     * Deserialize HNSW connections from JSON storage format
+     *
+     * Converts plain object { "0": ["id1"], "1": ["id2"] }
+     * into Map<number, Set<string>>
+     *
+     * v5.7.10: Central helper to fix serialization bug across all code paths
+     * Root cause: JSON.stringify(Map) = {} (empty object), must reconstruct on read
+     */
+    protected deserializeConnections(connections: any): Map<number, Set<string>>;
+    /**
+     * Deserialize HNSWNoun from JSON storage format
+     *
+     * v5.7.10: Ensures connections are properly reconstructed from Map → object → Map
+     * Fixes: "TypeError: noun.connections.entries is not a function"
+     */
+    protected deserializeNoun(data: any): HNSWNoun;
+    /**
+     * Deserialize HNSWVerb from JSON storage format
+     *
+     * v5.7.10: Ensures connections are properly reconstructed from Map → object → Map
+     * Fixes same serialization bug for verbs
+     */
+    protected deserializeVerb(data: any): HNSWVerb;
     /**
      * Save a noun to storage (type-first path)
      */

package/dist/storage/baseStorage.js

@@ -912,8 +912,11 @@ export class BaseStorage extends BaseStorageAdapter {
                     if (!nounPath.endsWith('.json'))
                         continue;
                     try {
-                        const noun = await this.readWithInheritance(nounPath);
-                        if (noun) {
+                        const rawNoun = await this.readWithInheritance(nounPath);
+                        if (rawNoun) {
+                            // v5.7.10: Deserialize connections Map from JSON storage format
+                            // Replaces v5.7.8 manual deserialization (removed 13 lines at 1156-1168)
+                            const noun = this.deserializeNoun(rawNoun);
                             // Load metadata
                             const metadataPath = getNounMetadataPath(type, noun.id);
                             const metadata = await this.readWithInheritance(metadataPath);
@@ -925,24 +928,10 @@ export class BaseStorage extends BaseStorageAdapter {
                                         continue;
                                     }
                                 }
-                                // v5.7.8: Convert connections from plain object to Map (JSON deserialization fix)
-                                // When loaded from JSON, Map becomes plain object - must reconstruct
-                                const connections = new Map();
-                                if (noun.connections && typeof noun.connections === 'object') {
-                                    for (const [levelStr, ids] of Object.entries(noun.connections)) {
-                                        if (Array.isArray(ids)) {
-                                            connections.set(parseInt(levelStr, 10), new Set(ids));
-                                        }
-                                        else if (ids && typeof ids === 'object') {
-                                            // Handle if it's already an array-like or Set-like object
-                                            connections.set(parseInt(levelStr, 10), new Set(Object.values(ids)));
-                                        }
-                                    }
-                                }
                                 // Combine noun + metadata (v5.4.0: Extract standard fields to top-level)
                                 collectedNouns.push({
                                     ...noun,
-                                    connections, // Use reconstructed Map instead of plain object
+                                    // v5.7.10: connections already deserialized by deserializeNoun()
                                     type: metadata.noun || type, // Required: Extract type from metadata
                                     confidence: metadata.confidence,
                                     weight: metadata.weight,
@@ -1731,6 +1720,64 @@ export class BaseStorage extends BaseStorageAdapter {
         return 'relatedTo';
     }
     // ============================================================================
+    // DESERIALIZATION HELPERS (v5.7.10)
+    // Centralized Map/Set reconstruction from JSON storage format
+    // ============================================================================
+    /**
+     * Deserialize HNSW connections from JSON storage format
+     *
+     * Converts plain object { "0": ["id1"], "1": ["id2"] }
+     * into Map<number, Set<string>>
+     *
+     * v5.7.10: Central helper to fix serialization bug across all code paths
+     * Root cause: JSON.stringify(Map) = {} (empty object), must reconstruct on read
+     */
+    deserializeConnections(connections) {
+        const result = new Map();
+        if (!connections || typeof connections !== 'object') {
+            return result;
+        }
+        // Already a Map (in-memory, not from JSON)
+        if (connections instanceof Map) {
+            return connections;
+        }
+        // Deserialize from plain object
+        for (const [levelStr, ids] of Object.entries(connections)) {
+            if (Array.isArray(ids)) {
+                result.set(parseInt(levelStr, 10), new Set(ids));
+            }
+            else if (ids && typeof ids === 'object') {
+                // Handle Set-like or array-like objects
+                result.set(parseInt(levelStr, 10), new Set(Object.values(ids)));
+            }
+        }
+        return result;
+    }
+    /**
+     * Deserialize HNSWNoun from JSON storage format
+     *
+     * v5.7.10: Ensures connections are properly reconstructed from Map → object → Map
+     * Fixes: "TypeError: noun.connections.entries is not a function"
+     */
+    deserializeNoun(data) {
+        return {
+            ...data,
+            connections: this.deserializeConnections(data.connections)
+        };
+    }
+    /**
+     * Deserialize HNSWVerb from JSON storage format
+     *
+     * v5.7.10: Ensures connections are properly reconstructed from Map → object → Map
+     * Fixes same serialization bug for verbs
+     */
+    deserializeVerb(data) {
+        return {
+            ...data,
+            connections: this.deserializeConnections(data.connections)
+        };
+    }
+    // ============================================================================
     // ABSTRACT METHOD IMPLEMENTATIONS (v5.4.0)
     // Converted from abstract to concrete - all adapters now have built-in type-aware
     // ============================================================================
@@ -1760,7 +1807,9 @@ export class BaseStorage extends BaseStorageAdapter {
         if (cachedType) {
             const path = getNounVectorPath(cachedType, id);
             // COW-aware read (v5.0.1): Use COW helper for branch isolation
-            return await this.readWithInheritance(path);
+            const data = await this.readWithInheritance(path);
+            // v5.7.10: Deserialize connections Map from JSON storage format
+            return data ? this.deserializeNoun(data) : null;
         }
         // Need to search across all types (expensive, but cached after first access)
         for (let i = 0; i < NOUN_TYPE_COUNT; i++) {
@@ -1772,7 +1821,8 @@ export class BaseStorage extends BaseStorageAdapter {
                 if (noun) {
                     // Cache the type for next time
                     this.nounTypeCache.set(id, type);
-                    return noun;
+                    // v5.7.10: Deserialize connections Map from JSON storage format
+                    return this.deserializeNoun(noun);
                 }
             }
             catch (error) {
@@ -1796,7 +1846,8 @@ export class BaseStorage extends BaseStorageAdapter {
                 // COW-aware read (v5.0.1): Use COW helper for branch isolation
                 const noun = await this.readWithInheritance(path);
                 if (noun) {
-                    nouns.push(noun);
+                    // v5.7.10: Deserialize connections Map from JSON storage format
+                    nouns.push(this.deserializeNoun(noun));
                     // Cache the type
                     this.nounTypeCache.set(noun.id, type);
                 }
@@ -1891,7 +1942,8 @@ export class BaseStorage extends BaseStorageAdapter {
             const path = getVerbVectorPath(cachedType, id);
             // COW-aware read (v5.0.1): Use COW helper for branch isolation
             const verb = await this.readWithInheritance(path);
-            return verb;
+            // v5.7.10: Deserialize connections Map from JSON storage format
+            return verb ? this.deserializeVerb(verb) : null;
         }
         // Search across all types (only on first access)
         for (let i = 0; i < VERB_TYPE_COUNT; i++) {
@@ -1903,7 +1955,8 @@ export class BaseStorage extends BaseStorageAdapter {
                 if (verb) {
                     // Cache the type for next time (read from verb.verb field)
                     this.verbTypeCache.set(id, verb.verb);
-                    return verb;
+                    // v5.7.10: Deserialize connections Map from JSON storage format
+                    return this.deserializeVerb(verb);
                 }
             }
             catch (error) {
@@ -2040,29 +2093,24 @@ export class BaseStorage extends BaseStorageAdapter {
         for (const path of paths) {
             try {
                 // COW-aware read (v5.0.1): Use COW helper for branch isolation
-                const hnswVerb = await this.readWithInheritance(path);
-                if (!hnswVerb)
+                const rawVerb = await this.readWithInheritance(path);
+                if (!rawVerb)
                     continue;
+                // v5.7.10: Deserialize connections Map from JSON storage format
+                // Replaces v5.7.8 manual deserialization (lines 2599-2605)
+                const hnswVerb = this.deserializeVerb(rawVerb);
                 // Cache type from HNSWVerb for future O(1) retrievals
                 this.verbTypeCache.set(hnswVerb.id, hnswVerb.verb);
                 // Load metadata separately (optional in v4.0.0!)
                 // FIX: Don't skip verbs without metadata - metadata is optional!
                 const metadata = await this.getVerbMetadata(hnswVerb.id);
-                // Create HNSWVerbWithMetadata (verbs don't have level field)
-                // Convert connections from plain object to Map<number, Set<string>>
-                const connectionsMap = new Map();
-                if (hnswVerb.connections && typeof hnswVerb.connections === 'object') {
-                    for (const [level, ids] of Object.entries(hnswVerb.connections)) {
-                        connectionsMap.set(Number(level), new Set(ids));
-                    }
-                }
                 // v4.8.0: Extract standard fields from metadata to top-level
                 const metadataObj = (metadata || {});
                 const { createdAt, updatedAt, confidence, weight, service, data, createdBy, ...customMetadata } = metadataObj;
                 const verbWithMetadata = {
                     id: hnswVerb.id,
                     vector: [...hnswVerb.vector],
-                    connections: connectionsMap,
+                    connections: hnswVerb.connections, // v5.7.10: Already deserialized
                     verb: hnswVerb.verb,
                     sourceId: hnswVerb.sourceId,
                     targetId: hnswVerb.targetId,

package/dist/utils/metadataIndex.js

@@ -1322,7 +1322,8 @@ export class MetadataIndexManager {
                         // exists: boolean - check if field exists (any value)
                         case 'exists':
                             if (operand) {
-                                // Get all IDs that have this field (any value) from chunked sparse index with roaring bitmaps (v3.43.0)
+                                // exists: true - Get all IDs that have this field (any value)
+                                // v3.43.0: From chunked sparse index with roaring bitmaps
                                 // v3.44.1: Now fully lazy-loaded via UnifiedCache (no local sparseIndices Map)
                                 const allIntIds = new Set();
                                 // Load sparse index via UnifiedCache (lazy loading)
@@ -1344,6 +1345,76 @@ export class MetadataIndexManager {
                                 // Convert integer IDs back to UUIDs
                                 fieldResults = this.idMapper.intsIterableToUuids(allIntIds);
                             }
+                            else {
+                                // exists: false - Get all IDs that DON'T have this field
+                                // v5.7.9: Fixed excludeVFS bug (was returning empty array)
+                                const allItemIds = await this.getAllIds();
+                                const existsIntIds = new Set();
+                                // Get IDs that HAVE this field
+                                const sparseIndex = await this.loadSparseIndex(field);
+                                if (sparseIndex) {
+                                    for (const chunkId of sparseIndex.getAllChunkIds()) {
+                                        const chunk = await this.chunkManager.loadChunk(field, chunkId);
+                                        if (chunk) {
+                                            for (const bitmap of chunk.entries.values()) {
+                                                for (const intId of bitmap) {
+                                                    existsIntIds.add(intId);
+                                                }
+                                            }
+                                        }
+                                    }
+                                }
+                                // Convert to UUIDs and subtract from all IDs
+                                const existsUuids = this.idMapper.intsIterableToUuids(existsIntIds);
+                                const existsSet = new Set(existsUuids);
+                                fieldResults = allItemIds.filter(id => !existsSet.has(id));
+                            }
+                            break;
+                        // ===== MISSING OPERATOR =====
+                        // missing: boolean - equivalent to exists: !boolean (for compatibility with metadataFilter.ts)
+                        case 'missing':
+                            // missing: true is equivalent to exists: false
+                            // missing: false is equivalent to exists: true
+                            // v5.7.9: Added for API consistency with in-memory metadataFilter
+                            if (operand) {
+                                // missing: true - field does NOT exist (same as exists: false)
+                                const allItemIds = await this.getAllIds();
+                                const existsIntIds = new Set();
+                                const sparseIndex = await this.loadSparseIndex(field);
+                                if (sparseIndex) {
+                                    for (const chunkId of sparseIndex.getAllChunkIds()) {
+                                        const chunk = await this.chunkManager.loadChunk(field, chunkId);
+                                        if (chunk) {
+                                            for (const bitmap of chunk.entries.values()) {
+                                                for (const intId of bitmap) {
+                                                    existsIntIds.add(intId);
+                                                }
+                                            }
+                                        }
+                                    }
+                                }
+                                const existsUuids = this.idMapper.intsIterableToUuids(existsIntIds);
+                                const existsSet = new Set(existsUuids);
+                                fieldResults = allItemIds.filter(id => !existsSet.has(id));
+                            }
+                            else {
+                                // missing: false - field DOES exist (same as exists: true)
+                                const allIntIds = new Set();
+                                const sparseIndex = await this.loadSparseIndex(field);
+                                if (sparseIndex) {
+                                    for (const chunkId of sparseIndex.getAllChunkIds()) {
+                                        const chunk = await this.chunkManager.loadChunk(field, chunkId);
+                                        if (chunk) {
+                                            for (const bitmap of chunk.entries.values()) {
+                                                for (const intId of bitmap) {
+                                                    allIntIds.add(intId);
+                                                }
+                                            }
+                                        }
+                                    }
+                                }
+                                fieldResults = this.idMapper.intsIterableToUuids(allIntIds);
+                            }
                             break;
                     }
                 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@soulcraft/brainy",
-  "version": "5.7.8",
+  "version": "5.7.10",
   "description": "Universal Knowledge Protocol™ - World's first Triple Intelligence database unifying vector, graph, and document search in one API. Stage 3 CANONICAL: 42 nouns × 127 verbs covering 96-97% of all human knowledge.",
   "main": "dist/index.js",
   "module": "dist/index.js",