@soulcraft/brainy 4.7.3 → 4.8.0

package/dist/storage/adapters/typeAwareStorageAdapter.js

@@ -354,19 +354,36 @@ export class TypeAwareStorageAdapter extends BaseStorage {
                     // Check sourceId from HNSWVerb (v4.0.0: core fields are in HNSWVerb)
                     if (hnswVerb.sourceId !== sourceId)
                         continue;
-                    // Load metadata separately
+                    // Load metadata separately (optional in v4.0.0!)
+                    // FIX: Don't skip verbs without metadata - metadata is optional!
+                    // VFS relationships often have NO metadata (just verb/source/target)
                     const metadata = await this.getVerbMetadata(id);
-                    if (!metadata)
-                        continue;
                     // Create HNSWVerbWithMetadata (verbs don't have level field)
+                    // Convert connections from plain object to Map<number, Set<string>>
+                    const connectionsMap = new Map();
+                    if (hnswVerb.connections && typeof hnswVerb.connections === 'object') {
+                        for (const [level, ids] of Object.entries(hnswVerb.connections)) {
+                            connectionsMap.set(Number(level), new Set(ids));
+                        }
+                    }
+                    // v4.8.0: Extract standard fields from metadata to top-level
+                    const metadataObj = (metadata || {});
+                    const { createdAt, updatedAt, confidence, weight, service, data, createdBy, ...customMetadata } = metadataObj;
                     const verbWithMetadata = {
                         id: hnswVerb.id,
                         vector: [...hnswVerb.vector],
-                        connections: new Map(hnswVerb.connections),
+                        connections: connectionsMap,
                         verb: hnswVerb.verb,
                         sourceId: hnswVerb.sourceId,
                         targetId: hnswVerb.targetId,
-                        metadata: metadata
+                        createdAt: createdAt || Date.now(),
+                        updatedAt: updatedAt || Date.now(),
+                        confidence: confidence,
+                        weight: weight,
+                        service: service,
+                        data: data,
+                        createdBy,
+                        metadata: customMetadata
                     };
                     verbs.push(verbWithMetadata);
                 }
@@ -399,19 +416,35 @@ export class TypeAwareStorageAdapter extends BaseStorage {
                     // Check targetId from HNSWVerb (v4.0.0: core fields are in HNSWVerb)
                     if (hnswVerb.targetId !== targetId)
                         continue;
-                    // Load metadata separately
+                    // Load metadata separately (optional in v4.0.0!)
+                    // FIX: Don't skip verbs without metadata - metadata is optional!
                     const metadata = await this.getVerbMetadata(id);
-                    if (!metadata)
-                        continue;
                     // Create HNSWVerbWithMetadata (verbs don't have level field)
+                    // Convert connections from plain object to Map<number, Set<string>>
+                    const connectionsMap = new Map();
+                    if (hnswVerb.connections && typeof hnswVerb.connections === 'object') {
+                        for (const [level, ids] of Object.entries(hnswVerb.connections)) {
+                            connectionsMap.set(Number(level), new Set(ids));
+                        }
+                    }
+                    // v4.8.0: Extract standard fields from metadata to top-level
+                    const metadataObj = (metadata || {});
+                    const { createdAt, updatedAt, confidence, weight, service, data, createdBy, ...customMetadata } = metadataObj;
                     const verbWithMetadata = {
                         id: hnswVerb.id,
                         vector: [...hnswVerb.vector],
-                        connections: new Map(hnswVerb.connections),
+                        connections: connectionsMap,
                         verb: hnswVerb.verb,
                         sourceId: hnswVerb.sourceId,
                         targetId: hnswVerb.targetId,
-                        metadata: metadata
+                        createdAt: createdAt || Date.now(),
+                        updatedAt: updatedAt || Date.now(),
+                        confidence: confidence,
+                        weight: weight,
+                        service: service,
+                        data: data,
+                        createdBy,
+                        metadata: customMetadata
                     };
                     verbs.push(verbWithMetadata);
                 }
@@ -439,19 +472,35 @@ export class TypeAwareStorageAdapter extends BaseStorage {
                     continue;
                 // Cache type from HNSWVerb for future O(1) retrievals
                 this.verbTypeCache.set(hnswVerb.id, hnswVerb.verb);
-                // Load metadata separately
+                // Load metadata separately (optional in v4.0.0!)
+                // FIX: Don't skip verbs without metadata - metadata is optional!
                 const metadata = await this.getVerbMetadata(hnswVerb.id);
-                if (!metadata)
-                    continue;
                 // Create HNSWVerbWithMetadata (verbs don't have level field)
+                // Convert connections from plain object to Map<number, Set<string>>
+                const connectionsMap = new Map();
+                if (hnswVerb.connections && typeof hnswVerb.connections === 'object') {
+                    for (const [level, ids] of Object.entries(hnswVerb.connections)) {
+                        connectionsMap.set(Number(level), new Set(ids));
+                    }
+                }
+                // v4.8.0: Extract standard fields from metadata to top-level
+                const metadataObj = (metadata || {});
+                const { createdAt, updatedAt, confidence, weight, service, data, createdBy, ...customMetadata } = metadataObj;
                 const verbWithMetadata = {
                     id: hnswVerb.id,
                     vector: [...hnswVerb.vector],
-                    connections: new Map(hnswVerb.connections),
+                    connections: connectionsMap,
                     verb: hnswVerb.verb,
                     sourceId: hnswVerb.sourceId,
                     targetId: hnswVerb.targetId,
-                    metadata: metadata
+                    createdAt: createdAt || Date.now(),
+                    updatedAt: updatedAt || Date.now(),
+                    confidence: confidence,
+                    weight: weight,
+                    service: service,
+                    data: data,
+                    createdBy,
+                    metadata: customMetadata
                 };
                 verbs.push(verbWithMetadata);
             }

package/dist/storage/baseStorage.js

@@ -5,6 +5,7 @@
 import { GraphAdjacencyIndex } from '../graph/graphAdjacencyIndex.js';
 import { BaseStorageAdapter } from './adapters/baseStorageAdapter.js';
 import { validateNounType, validateVerbType } from '../utils/typeValidation.js';
+import { NounType } from '../types/graphTypes.js';
 import { getShardIdFromUuid } from './sharding.js';
 // Clean directory structure (v4.7.2+)
 // All storage adapters use this consistent structure
@@ -46,6 +47,10 @@ export class BaseStorage extends BaseStorageAdapter {
      * @private
      */
     analyzeKey(id, context) {
+        // v4.8.0: Guard against undefined/null IDs
+        if (!id || typeof id !== 'string') {
+            throw new Error(`Invalid storage key: ${id} (must be a non-empty string)`);
+        }
         // System resource detection
         const isSystemKey = id.startsWith('__metadata_') ||
             id.startsWith('__index_') ||
@@ -142,13 +147,24 @@ export class BaseStorage extends BaseStorageAdapter {
             console.warn(`[Storage] Noun ${id} has vector but no metadata - this should not happen in v4.0.0`);
             return null;
         }
-        // Combine into HNSWNounWithMetadata
+        // Combine into HNSWNounWithMetadata - v4.8.0: Extract standard fields to top-level
+        const { noun, createdAt, updatedAt, confidence, weight, service, data, createdBy, ...customMetadata } = metadata;
         return {
             id: vector.id,
             vector: vector.vector,
             connections: vector.connections,
             level: vector.level,
-            metadata
+            // v4.8.0: Standard fields at top-level
+            type: noun || NounType.Thing,
+            createdAt: createdAt || Date.now(),
+            updatedAt: updatedAt || Date.now(),
+            confidence: confidence,
+            weight: weight,
+            service: service,
+            data: data,
+            createdBy,
+            // Only custom user fields remain in metadata
+            metadata: customMetadata
         };
     }
     /**
@@ -160,14 +176,25 @@ export class BaseStorage extends BaseStorageAdapter {
         await this.ensureInitialized();
         // Internal method returns HNSWNoun[], need to combine with metadata
         const nouns = await this.getNounsByNounType_internal(nounType);
-        // Combine each noun with its metadata
+        // Combine each noun with its metadata - v4.8.0: Extract standard fields to top-level
         const nounsWithMetadata = [];
         for (const noun of nouns) {
             const metadata = await this.getNounMetadata(noun.id);
             if (metadata) {
+                const { noun: nounType, createdAt, updatedAt, confidence, weight, service, data, createdBy, ...customMetadata } = metadata;
                 nounsWithMetadata.push({
                     ...noun,
-                    metadata
+                    // v4.8.0: Standard fields at top-level
+                    type: nounType || NounType.Thing,
+                    createdAt: createdAt || Date.now(),
+                    updatedAt: updatedAt || Date.now(),
+                    confidence: confidence,
+                    weight: weight,
+                    service: service,
+                    data: data,
+                    createdBy,
+                    // Only custom user fields in metadata
+                    metadata: customMetadata
                 });
             }
         }
@@ -220,7 +247,8 @@ export class BaseStorage extends BaseStorageAdapter {
             console.warn(`[Storage] Verb ${id} has vector but no metadata - this should not happen in v4.0.0`);
             return null;
         }
-        // Combine into HNSWVerbWithMetadata
+        // Combine into HNSWVerbWithMetadata - v4.8.0: Extract standard fields to top-level
+        const { createdAt, updatedAt, confidence, weight, service, data, createdBy, ...customMetadata } = metadata;
         return {
             id: verb.id,
             vector: verb.vector,
@@ -228,7 +256,16 @@ export class BaseStorage extends BaseStorageAdapter {
             verb: verb.verb,
             sourceId: verb.sourceId,
             targetId: verb.targetId,
-            metadata
+            // v4.8.0: Standard fields at top-level
+            createdAt: createdAt || Date.now(),
+            updatedAt: updatedAt || Date.now(),
+            confidence: confidence,
+            weight: weight,
+            service: service,
+            data: data,
+            createdBy,
+            // Only custom user fields remain in metadata
+            metadata: customMetadata
         };
     }
     /**

package/dist/types/brainy.types.d.ts

@@ -89,6 +89,10 @@ export interface AddParams<T = any> {
     service?: string;
     confidence?: number;
     weight?: number;
+    createdBy?: {
+        augmentation: string;
+        version: string;
+    };
 }
 /**
  * Parameters for updating entities

package/dist/types/graphTypes.d.ts

@@ -206,6 +206,7 @@ export interface GraphVerb {
     createdAt: Timestamp;
     updatedAt: Timestamp;
     createdBy: CreatorMetadata;
+    service?: string;
     data?: Record<string, any>;
     embedding?: number[];
     confidence?: number;

package/dist/utils/entityIdMapper.js

@@ -33,8 +33,9 @@ export class EntityIdMapper {
     async init() {
         try {
             const metadata = await this.storage.getMetadata(this.storageKey);
-            if (metadata && metadata.data) {
-                const data = metadata.data;
+            // v4.8.0: metadata IS the data (no nested 'data' property)
+            if (metadata && metadata.nextId !== undefined) {
+                const data = metadata;
                 this.nextId = data.nextId;
                 // Rebuild maps from serialized data
                 this.uuidToInt = new Map(Object.entries(data.uuidToInt).map(([k, v]) => [k, Number(v)]));

package/dist/utils/metadataIndex.d.ts

@@ -228,7 +228,13 @@ export declare class MetadataIndexManager {
      */
     private shouldIndexField;
     /**
-     * Extract indexable field-value pairs from metadata
+     * Extract indexable field-value pairs from entity or metadata
+     *
+     * v4.8.0: Now handles BOTH entity structure (with top-level fields) AND plain metadata
+     * - Extracts from top-level fields (confidence, weight, timestamps, type, service, etc.)
+     * - Also extracts from nested metadata field (custom user fields)
+     * - Skips HNSW-specific fields (vector, connections, level, id)
+     * - Maps 'type' → 'noun' for backward compatibility with existing indexes
      *
      * BUG FIX (v3.50.1): Exclude vector embeddings and large arrays from indexing
      * BUG FIX (v3.50.2): Also exclude purely numeric field names (array indices)
@@ -238,14 +244,29 @@ export declare class MetadataIndexManager {
     private extractIndexableFields;
     /**
      * Add item to metadata indexes
+     *
+     * v4.8.0: Now accepts either entity structure or plain metadata
+     * - Entity structure: { id, type, confidence, weight, createdAt, metadata: {...} }
+     * - Plain metadata: { noun, confidence, weight, createdAt, ... }
+     *
+     * @param id - Entity ID
+     * @param entityOrMetadata - Either full entity structure (v4.8.0+) or plain metadata (backward compat)
+     * @param skipFlush - Skip automatic flush (used during batch operations)
      */
-    addToIndex(id: string, metadata: any, skipFlush?: boolean): Promise<void>;
+    addToIndex(id: string, entityOrMetadata: any, skipFlush?: boolean): Promise<void>;
     /**
      * Update field index with value count
      */
     private updateFieldIndex;
     /**
      * Remove item from metadata indexes
+     *
+     * v4.8.0: Now accepts either entity structure or plain metadata (same as addToIndex)
+     * - Entity structure: { id, type, confidence, weight, createdAt, metadata: {...} }
+     * - Plain metadata: { noun, confidence, weight, createdAt, ... }
+     *
+     * @param id - Entity ID to remove
+     * @param metadata - Optional entity or metadata structure (if not provided, requires scanning all fields - slow!)
      */
     removeFromIndex(id: string, metadata?: any): Promise<void>;
     /**

package/dist/utils/metadataIndex.js

@@ -856,22 +856,28 @@ export class MetadataIndexManager {
         return true;
     }
     /**
-     * Extract indexable field-value pairs from metadata
+     * Extract indexable field-value pairs from entity or metadata
+     *
+     * v4.8.0: Now handles BOTH entity structure (with top-level fields) AND plain metadata
+     * - Extracts from top-level fields (confidence, weight, timestamps, type, service, etc.)
+     * - Also extracts from nested metadata field (custom user fields)
+     * - Skips HNSW-specific fields (vector, connections, level, id)
+     * - Maps 'type' → 'noun' for backward compatibility with existing indexes
      *
      * BUG FIX (v3.50.1): Exclude vector embeddings and large arrays from indexing
      * BUG FIX (v3.50.2): Also exclude purely numeric field names (array indices)
      * - Vector fields (384+ dimensions) were creating 825K chunk files for 1,144 entities
      * - Arrays converted to objects with numeric keys were still being indexed
      */
-    extractIndexableFields(metadata) {
+    extractIndexableFields(data) {
         const fields = [];
-        // Fields that should NEVER be indexed (vectors, embeddings, large arrays)
-        const NEVER_INDEX = new Set(['vector', 'embedding', 'embeddings', 'connections']);
+        // Fields that should NEVER be indexed (vectors, embeddings, large arrays, HNSW internals)
+        const NEVER_INDEX = new Set(['vector', 'embedding', 'embeddings', 'connections', 'level', 'id']);
         const extract = (obj, prefix = '') => {
             for (const [key, value] of Object.entries(obj)) {
                 const fullKey = prefix ? `${prefix}.${key}` : key;
-                // Skip fields in never-index list (CRITICAL: prevents vector indexing bug)
-                if (NEVER_INDEX.has(key))
+                // Skip fields in never-index list (CRITICAL: prevents vector indexing bug + HNSW fields)
+                if (!prefix && NEVER_INDEX.has(key))
                     continue;
                 // Skip purely numeric field names (array indices converted to object keys)
                 // Legitimate field names should never be purely numeric
@@ -881,6 +887,14 @@ export class MetadataIndexManager {
                 // Skip fields based on user configuration
                 if (!this.shouldIndexField(fullKey))
                     continue;
+                // Special handling for metadata field at top level
+                // v4.8.0: Flatten metadata fields to top-level (no prefix) for cleaner queries
+                // Standard fields are already at top-level, custom fields go in metadata
+                // By flattening here, queries can use { category: 'B' } instead of { 'metadata.category': 'B' }
+                if (key === 'metadata' && !prefix && typeof value === 'object' && !Array.isArray(value)) {
+                    extract(value, ''); // Flatten to top-level, no prefix
+                    continue;
+                }
                 // Skip large arrays (> 10 elements) - likely vectors or bulk data
                 if (Array.isArray(value) && value.length > 10)
                     continue;
@@ -900,20 +914,30 @@ export class MetadataIndexManager {
                 }
                 else {
                     // Primitive value: index it
-                    fields.push({ field: fullKey, value });
+                    // v4.8.0: Map 'type' → 'noun' for backward compatibility
+                    const indexField = (!prefix && key === 'type') ? 'noun' : fullKey;
+                    fields.push({ field: indexField, value });
                 }
             }
         };
-        if (metadata && typeof metadata === 'object') {
-            extract(metadata);
+        if (data && typeof data === 'object') {
+            extract(data);
         }
         return fields;
     }
     /**
      * Add item to metadata indexes
+     *
+     * v4.8.0: Now accepts either entity structure or plain metadata
+     * - Entity structure: { id, type, confidence, weight, createdAt, metadata: {...} }
+     * - Plain metadata: { noun, confidence, weight, createdAt, ... }
+     *
+     * @param id - Entity ID
+     * @param entityOrMetadata - Either full entity structure (v4.8.0+) or plain metadata (backward compat)
+     * @param skipFlush - Skip automatic flush (used during batch operations)
      */
-    async addToIndex(id, metadata, skipFlush = false) {
-        const fields = this.extractIndexableFields(metadata);
+    async addToIndex(id, entityOrMetadata, skipFlush = false) {
+        const fields = this.extractIndexableFields(entityOrMetadata);
         // Sort fields to process 'noun' field first for type-field affinity tracking
         fields.sort((a, b) => {
             if (a.field === 'noun')
@@ -930,7 +954,7 @@ export class MetadataIndexManager {
             await this.addToChunkedIndex(field, value, id);
             // Update statistics and tracking
             this.updateCardinalityStats(field, value, 'add');
-            this.updateTypeFieldAffinity(id, field, value, 'add', metadata);
+            this.updateTypeFieldAffinity(id, field, value, 'add', entityOrMetadata);
             await this.updateFieldIndex(field, value, 1);
             // Yield to event loop every 5 fields to prevent blocking
             if (i % 5 === 4) {
@@ -988,6 +1012,13 @@ export class MetadataIndexManager {
     }
     /**
      * Remove item from metadata indexes
+     *
+     * v4.8.0: Now accepts either entity structure or plain metadata (same as addToIndex)
+     * - Entity structure: { id, type, confidence, weight, createdAt, metadata: {...} }
+     * - Plain metadata: { noun, confidence, weight, createdAt, ... }
+     *
+     * @param id - Entity ID to remove
+     * @param metadata - Optional entity or metadata structure (if not provided, requires scanning all fields - slow!)
      */
     async removeFromIndex(id, metadata) {
         if (metadata) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@soulcraft/brainy",
-  "version": "4.7.3",
+  "version": "4.8.0",
   "description": "Universal Knowledge Protocol™ - World's first Triple Intelligence database unifying vector, graph, and document search in one API. 31 nouns × 40 verbs for infinite expressiveness.",
   "main": "dist/index.js",
   "module": "dist/index.js",