@soulcraft/brainy 3.50.2 → 4.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (57) hide show
  1. package/CHANGELOG.md +201 -0
  2. package/README.md +358 -658
  3. package/dist/api/ConfigAPI.js +56 -19
  4. package/dist/api/DataAPI.js +24 -18
  5. package/dist/augmentations/storageAugmentations.d.ts +24 -0
  6. package/dist/augmentations/storageAugmentations.js +22 -0
  7. package/dist/brainy.js +32 -9
  8. package/dist/cli/commands/core.d.ts +20 -10
  9. package/dist/cli/commands/core.js +384 -82
  10. package/dist/cli/commands/import.d.ts +41 -0
  11. package/dist/cli/commands/import.js +456 -0
  12. package/dist/cli/commands/insights.d.ts +34 -0
  13. package/dist/cli/commands/insights.js +300 -0
  14. package/dist/cli/commands/neural.d.ts +6 -12
  15. package/dist/cli/commands/neural.js +113 -10
  16. package/dist/cli/commands/nlp.d.ts +28 -0
  17. package/dist/cli/commands/nlp.js +246 -0
  18. package/dist/cli/commands/storage.d.ts +64 -0
  19. package/dist/cli/commands/storage.js +730 -0
  20. package/dist/cli/index.js +210 -24
  21. package/dist/coreTypes.d.ts +206 -34
  22. package/dist/distributed/configManager.js +8 -6
  23. package/dist/distributed/shardMigration.js +2 -0
  24. package/dist/distributed/storageDiscovery.js +6 -4
  25. package/dist/embeddings/EmbeddingManager.d.ts +2 -2
  26. package/dist/embeddings/EmbeddingManager.js +5 -1
  27. package/dist/graph/lsm/LSMTree.js +32 -20
  28. package/dist/hnsw/typeAwareHNSWIndex.js +6 -2
  29. package/dist/storage/adapters/azureBlobStorage.d.ts +545 -0
  30. package/dist/storage/adapters/azureBlobStorage.js +1809 -0
  31. package/dist/storage/adapters/baseStorageAdapter.d.ts +16 -13
  32. package/dist/storage/adapters/fileSystemStorage.d.ts +21 -9
  33. package/dist/storage/adapters/fileSystemStorage.js +204 -127
  34. package/dist/storage/adapters/gcsStorage.d.ts +119 -9
  35. package/dist/storage/adapters/gcsStorage.js +317 -62
  36. package/dist/storage/adapters/memoryStorage.d.ts +30 -18
  37. package/dist/storage/adapters/memoryStorage.js +99 -94
  38. package/dist/storage/adapters/opfsStorage.d.ts +48 -10
  39. package/dist/storage/adapters/opfsStorage.js +201 -80
  40. package/dist/storage/adapters/r2Storage.d.ts +12 -5
  41. package/dist/storage/adapters/r2Storage.js +63 -15
  42. package/dist/storage/adapters/s3CompatibleStorage.d.ts +164 -17
  43. package/dist/storage/adapters/s3CompatibleStorage.js +472 -80
  44. package/dist/storage/adapters/typeAwareStorageAdapter.d.ts +38 -6
  45. package/dist/storage/adapters/typeAwareStorageAdapter.js +218 -39
  46. package/dist/storage/baseStorage.d.ts +41 -38
  47. package/dist/storage/baseStorage.js +110 -134
  48. package/dist/storage/storageFactory.d.ts +29 -2
  49. package/dist/storage/storageFactory.js +30 -1
  50. package/dist/utils/entityIdMapper.js +5 -2
  51. package/dist/utils/fieldTypeInference.js +8 -1
  52. package/dist/utils/metadataFilter.d.ts +3 -2
  53. package/dist/utils/metadataFilter.js +1 -0
  54. package/dist/utils/metadataIndex.js +2 -0
  55. package/dist/utils/metadataIndexChunking.js +9 -4
  56. package/dist/utils/periodicCleanup.js +1 -0
  57. package/package.json +3 -1
@@ -46,8 +46,9 @@ export interface PaginatedSearchResult<T = any> {
46
46
  export type DistanceFunction = (a: Vector, b: Vector) => number;
47
47
  /**
48
48
  * Embedding function for converting data to vectors
49
+ * v4.0.0: Now properly typed - accepts string, string array (batch), or object, no `any`
49
50
  */
50
- export type EmbeddingFunction = (data: any) => Promise<Vector>;
51
+ export type EmbeddingFunction = (data: string | string[] | Record<string, unknown>) => Promise<Vector>;
51
52
  /**
52
53
  * Embedding model interface
53
54
  */
@@ -58,38 +59,49 @@ export interface EmbeddingModel {
58
59
  init(): Promise<void>;
59
60
  /**
60
61
  * Embed data into a vector
62
+ * v4.0.0: Now properly typed - accepts string, string array (batch), or object, no `any`
61
63
  */
62
- embed(data: any): Promise<Vector>;
64
+ embed(data: string | string[] | Record<string, unknown>): Promise<Vector>;
63
65
  /**
64
66
  * Dispose of the model resources
65
67
  */
66
68
  dispose(): Promise<void>;
67
69
  }
68
70
  /**
69
- * HNSW graph noun
71
+ * HNSW graph noun - Pure vector structure (v4.0.0)
72
+ *
73
+ * v4.0.0 BREAKING CHANGE: metadata field removed
74
+ * - Stores ONLY vector data for optimal memory usage
75
+ * - Metadata stored separately and combined on retrieval
76
+ * - 25% memory reduction @ 1B scale (no in-memory metadata)
77
+ * - Prevents metadata explosion bugs at compile-time
70
78
  */
71
79
  export interface HNSWNoun {
72
80
  id: string;
73
81
  vector: Vector;
74
82
  connections: Map<number, Set<string>>;
75
83
  level: number;
76
- metadata?: any;
77
84
  }
78
85
  /**
79
- * Lightweight verb for HNSW index storage
80
- * Contains essential data including core relational fields
86
+ * Lightweight verb for HNSW index storage - Core relational structure (v4.0.0)
81
87
  *
82
- * ARCHITECTURAL FIX (v3.50.1): verb/sourceId/targetId are now first-class fields
88
+ * Core fields (v3.50.1+): verb/sourceId/targetId are first-class fields
83
89
  * These are NOT metadata - they're the essence of what a verb IS:
84
90
  * - verb: The relationship type (creates, contains, etc.) - needed for routing & display
85
91
  * - sourceId: What entity this verb connects FROM - needed for graph traversal
86
92
  * - targetId: What entity this verb connects TO - needed for graph traversal
87
93
  *
94
+ * v4.0.0 BREAKING CHANGE: metadata field removed
95
+ * - Stores ONLY vector + core relational data
96
+ * - User metadata (weight, custom fields) stored separately
97
+ * - 10x faster metadata-only updates (skip HNSW rebuild)
98
+ * - Prevents metadata explosion bugs at compile-time
99
+ *
88
100
  * Benefits:
89
- * - ONE file read instead of two for 90% of operations
101
+ * - ONE file read for graph operations (core fields always available)
90
102
  * - No type caching needed (type is always available)
91
103
  * - Faster graph traversal (source/target immediately available)
92
- * - Aligns with actual usage patterns
104
+ * - Optimal memory usage (no user metadata in HNSW)
93
105
  */
94
106
  export interface HNSWVerb {
95
107
  id: string;
@@ -98,11 +110,88 @@ export interface HNSWVerb {
98
110
  verb: VerbType;
99
111
  sourceId: string;
100
112
  targetId: string;
101
- metadata?: any;
113
+ }
114
+ /**
115
+ * Noun metadata structure (v4.0.0)
116
+ *
117
+ * Stores all metadata separately from vector data.
118
+ * Combines with HNSWNoun to form complete entity.
119
+ */
120
+ export interface NounMetadata {
121
+ noun: string;
122
+ data?: unknown;
123
+ createdAt?: {
124
+ seconds: number;
125
+ nanoseconds: number;
126
+ } | number;
127
+ updatedAt?: {
128
+ seconds: number;
129
+ nanoseconds: number;
130
+ } | number;
131
+ createdBy?: {
132
+ augmentation: string;
133
+ version: string;
134
+ };
135
+ service?: string;
136
+ [key: string]: unknown;
137
+ }
138
+ /**
139
+ * Verb metadata structure (v4.0.0)
140
+ *
141
+ * Stores all metadata separately from vector + core relational data.
142
+ * Core fields (verb, sourceId, targetId) remain in HNSWVerb.
143
+ */
144
+ export interface VerbMetadata {
145
+ weight?: number;
146
+ data?: unknown;
147
+ createdAt?: {
148
+ seconds: number;
149
+ nanoseconds: number;
150
+ } | number;
151
+ updatedAt?: {
152
+ seconds: number;
153
+ nanoseconds: number;
154
+ } | number;
155
+ createdBy?: {
156
+ augmentation: string;
157
+ version: string;
158
+ };
159
+ service?: string;
160
+ [key: string]: unknown;
161
+ }
162
+ /**
163
+ * Combined noun structure for transport/API boundaries (v4.0.0)
164
+ *
165
+ * Combines pure HNSWNoun vector + separate NounMetadata.
166
+ * Used for API responses and storage retrieval.
167
+ */
168
+ export interface HNSWNounWithMetadata {
169
+ id: string;
170
+ vector: Vector;
171
+ connections: Map<number, Set<string>>;
172
+ level: number;
173
+ metadata: NounMetadata;
174
+ }
175
+ /**
176
+ * Combined verb structure for transport/API boundaries (v4.0.0)
177
+ *
178
+ * Combines pure HNSWVerb (vector + core fields) + separate VerbMetadata.
179
+ * Used for API responses and storage retrieval.
180
+ */
181
+ export interface HNSWVerbWithMetadata {
182
+ id: string;
183
+ vector: Vector;
184
+ connections: Map<number, Set<string>>;
185
+ verb: VerbType;
186
+ sourceId: string;
187
+ targetId: string;
188
+ metadata: VerbMetadata;
102
189
  }
103
190
  /**
104
191
  * Verb representing a relationship between nouns
105
192
  * Stored separately from HNSW index for lightweight performance
193
+ *
194
+ * @deprecated Will be replaced by HNSWVerbWithMetadata in future versions
106
195
  */
107
196
  export interface GraphVerb {
108
197
  id: string;
@@ -346,10 +435,41 @@ export interface StatisticsData {
346
435
  */
347
436
  distributedConfig?: import('./types/distributedTypes.js').SharedConfig;
348
437
  }
438
+ /**
439
+ * Change record for getChangesSince (v4.0.0)
440
+ * Replaces `any[]` with properly typed structure
441
+ */
442
+ export interface Change {
443
+ id: string;
444
+ type: 'noun' | 'verb';
445
+ operation: 'create' | 'update' | 'delete';
446
+ timestamp: number;
447
+ data?: HNSWNounWithMetadata | HNSWVerbWithMetadata;
448
+ }
349
449
  export interface StorageAdapter {
350
450
  init(): Promise<void>;
451
+ /**
452
+ * Save noun - Pure HNSW vector data only (v4.0.0)
453
+ * @param noun Pure HNSW vector data (no metadata)
454
+ * Note: Use saveNounMetadata() to save metadata separately
455
+ */
351
456
  saveNoun(noun: HNSWNoun): Promise<void>;
352
- getNoun(id: string): Promise<HNSWNoun | null>;
457
+ /**
458
+ * Save noun metadata separately (v4.0.0)
459
+ * @param id Noun ID
460
+ * @param metadata Noun metadata
461
+ */
462
+ saveNounMetadata(id: string, metadata: NounMetadata): Promise<void>;
463
+ /**
464
+ * Delete noun metadata (v4.0.0)
465
+ * @param id Noun ID
466
+ */
467
+ deleteNounMetadata(id: string): Promise<void>;
468
+ /**
469
+ * Get noun with metadata combined (v4.0.0)
470
+ * @returns Combined HNSWNounWithMetadata or null
471
+ */
472
+ getNoun(id: string): Promise<HNSWNounWithMetadata | null>;
353
473
  /**
354
474
  * Get nouns with pagination and filtering
355
475
  * @param options Pagination and filtering options
@@ -367,7 +487,7 @@ export interface StorageAdapter {
367
487
  metadata?: Record<string, any>;
368
488
  };
369
489
  }): Promise<{
370
- items: HNSWNoun[];
490
+ items: HNSWNounWithMetadata[];
371
491
  totalCount?: number;
372
492
  hasMore: boolean;
373
493
  nextCursor?: string;
@@ -378,10 +498,19 @@ export interface StorageAdapter {
378
498
  * @returns Promise that resolves to an array of nouns of the specified noun type
379
499
  * @deprecated Use getNouns() with filter.nounType instead
380
500
  */
381
- getNounsByNounType(nounType: string): Promise<HNSWNoun[]>;
501
+ getNounsByNounType(nounType: string): Promise<HNSWNounWithMetadata[]>;
382
502
  deleteNoun(id: string): Promise<void>;
383
- saveVerb(verb: GraphVerb): Promise<void>;
384
- getVerb(id: string): Promise<GraphVerb | null>;
503
+ /**
504
+ * Save verb - Pure HNSW verb with core fields only (v4.0.0)
505
+ * @param verb Pure HNSW verb data (vector + core fields, no user metadata)
506
+ * Note: Use saveVerbMetadata() to save metadata separately
507
+ */
508
+ saveVerb(verb: HNSWVerb): Promise<void>;
509
+ /**
510
+ * Get verb with metadata combined (v4.0.0)
511
+ * @returns Combined HNSWVerbWithMetadata or null
512
+ */
513
+ getVerb(id: string): Promise<HNSWVerbWithMetadata | null>;
385
514
  /**
386
515
  * Get verbs with pagination and filtering
387
516
  * @param options Pagination and filtering options
@@ -401,7 +530,7 @@ export interface StorageAdapter {
401
530
  metadata?: Record<string, any>;
402
531
  };
403
532
  }): Promise<{
404
- items: GraphVerb[];
533
+ items: HNSWVerbWithMetadata[];
405
534
  totalCount?: number;
406
535
  hasMore: boolean;
407
536
  nextCursor?: string;
@@ -412,50 +541,93 @@ export interface StorageAdapter {
412
541
  * @returns Promise that resolves to an array of verbs with the specified source ID
413
542
  * @deprecated Use getVerbs() with filter.sourceId instead
414
543
  */
415
- getVerbsBySource(sourceId: string): Promise<GraphVerb[]>;
544
+ getVerbsBySource(sourceId: string): Promise<HNSWVerbWithMetadata[]>;
416
545
  /**
417
546
  * Get verbs by target
418
547
  * @param targetId The target ID to filter by
419
548
  * @returns Promise that resolves to an array of verbs with the specified target ID
420
549
  * @deprecated Use getVerbs() with filter.targetId instead
421
550
  */
422
- getVerbsByTarget(targetId: string): Promise<GraphVerb[]>;
551
+ getVerbsByTarget(targetId: string): Promise<HNSWVerbWithMetadata[]>;
423
552
  /**
424
553
  * Get verbs by type
425
554
  * @param type The verb type to filter by
426
555
  * @returns Promise that resolves to an array of verbs with the specified type
427
556
  * @deprecated Use getVerbs() with filter.verbType instead
428
557
  */
429
- getVerbsByType(type: string): Promise<GraphVerb[]>;
558
+ getVerbsByType(type: string): Promise<HNSWVerbWithMetadata[]>;
430
559
  deleteVerb(id: string): Promise<void>;
431
- saveMetadata(id: string, metadata: any): Promise<void>;
432
- getMetadata(id: string): Promise<any | null>;
560
+ /**
561
+ * Save metadata (v4.0.0: now typed)
562
+ * @param id Entity ID
563
+ * @param metadata Typed noun metadata
564
+ */
565
+ saveMetadata(id: string, metadata: NounMetadata): Promise<void>;
566
+ /**
567
+ * Get metadata (v4.0.0: now typed)
568
+ * @param id Entity ID
569
+ * @returns Typed noun metadata or null
570
+ */
571
+ getMetadata(id: string): Promise<NounMetadata | null>;
433
572
  /**
434
573
  * Get multiple metadata objects in batches (prevents socket exhaustion)
435
574
  * @param ids Array of IDs to get metadata for
436
- * @returns Promise that resolves to a Map of id -> metadata
575
+ * @returns Promise that resolves to a Map of id -> metadata (v4.0.0: typed)
437
576
  */
438
- getMetadataBatch?(ids: string[]): Promise<Map<string, any>>;
577
+ getMetadataBatch?(ids: string[]): Promise<Map<string, NounMetadata>>;
439
578
  /**
440
- * Get noun metadata from storage
579
+ * Get noun metadata from storage (v4.0.0: now typed)
441
580
  * @param id The ID of the noun
442
581
  * @returns Promise that resolves to the metadata or null if not found
443
582
  */
444
- getNounMetadata(id: string): Promise<any | null>;
583
+ getNounMetadata(id: string): Promise<NounMetadata | null>;
445
584
  /**
446
- * Save verb metadata to storage
585
+ * Save verb metadata to storage (v4.0.0: now typed)
447
586
  * @param id The ID of the verb
448
587
  * @param metadata The metadata to save
449
588
  * @returns Promise that resolves when the metadata is saved
450
589
  */
451
- saveVerbMetadata(id: string, metadata: any): Promise<void>;
590
+ saveVerbMetadata(id: string, metadata: VerbMetadata): Promise<void>;
452
591
  /**
453
- * Get verb metadata from storage
592
+ * Get verb metadata from storage (v4.0.0: now typed)
454
593
  * @param id The ID of the verb
455
594
  * @returns Promise that resolves to the metadata or null if not found
456
595
  */
457
- getVerbMetadata(id: string): Promise<any | null>;
596
+ getVerbMetadata(id: string): Promise<VerbMetadata | null>;
458
597
  clear(): Promise<void>;
598
+ /**
599
+ * Batch delete multiple objects from storage (v4.0.0)
600
+ * Efficient deletion of large numbers of entities using cloud provider batch APIs.
601
+ * Significantly faster and cheaper than individual deletes (up to 1000x speedup).
602
+ *
603
+ * @param keys - Array of object keys (paths) to delete
604
+ * @param options - Optional configuration for batch deletion
605
+ * @param options.maxRetries - Maximum number of retry attempts per batch (default: 3)
606
+ * @param options.retryDelayMs - Base delay between retries in milliseconds (default: 1000)
607
+ * @param options.continueOnError - Continue processing remaining batches if one fails (default: true)
608
+ * @returns Promise with deletion statistics
609
+ *
610
+ * @example
611
+ * const result = await storage.batchDelete(
612
+ * ['path1', 'path2', 'path3'],
613
+ * { continueOnError: true }
614
+ * )
615
+ * console.log(`Deleted: ${result.successfulDeletes}/${result.totalRequested}`)
616
+ * console.log(`Failed: ${result.failedDeletes}`)
617
+ */
618
+ batchDelete?(keys: string[], options?: {
619
+ maxRetries?: number;
620
+ retryDelayMs?: number;
621
+ continueOnError?: boolean;
622
+ }): Promise<{
623
+ totalRequested: number;
624
+ successfulDeletes: number;
625
+ failedDeletes: number;
626
+ errors: Array<{
627
+ key: string;
628
+ error: string;
629
+ }>;
630
+ }>;
459
631
  /**
460
632
  * Get information about storage usage and capacity
461
633
  * @returns Promise that resolves to an object containing storage status information
@@ -513,11 +685,11 @@ export interface StorageAdapter {
513
685
  */
514
686
  flushStatisticsToStorage(): Promise<void>;
515
687
  /**
516
- * Track field names from a JSON document
688
+ * Track field names from a JSON document (v4.0.0: now typed)
517
689
  * @param jsonDocument The JSON document to extract field names from
518
690
  * @param service The service that inserted the data
519
691
  */
520
- trackFieldNames(jsonDocument: any, service: string): Promise<void>;
692
+ trackFieldNames(jsonDocument: Record<string, unknown>, service: string): Promise<void>;
521
693
  /**
522
694
  * Get available field names by service
523
695
  * @returns Record of field names by service
@@ -529,12 +701,12 @@ export interface StorageAdapter {
529
701
  */
530
702
  getStandardFieldMappings(): Promise<Record<string, Record<string, string[]>>>;
531
703
  /**
532
- * Get changes since a specific timestamp
704
+ * Get changes since a specific timestamp (v4.0.0: now typed)
533
705
  * @param timestamp The timestamp to get changes since
534
706
  * @param limit Optional limit on the number of changes to return
535
- * @returns Promise that resolves to an array of changes
707
+ * @returns Promise that resolves to an array of properly typed changes
536
708
  */
537
- getChangesSince?(timestamp: number, limit?: number): Promise<any[]>;
709
+ getChangesSince?(timestamp: number, limit?: number): Promise<Change[]>;
538
710
  /**
539
711
  * Get total count of nouns in storage - O(1) operation
540
712
  * @returns Promise that resolves to the total number of nouns
@@ -77,9 +77,10 @@ export class DistributedConfigManager {
77
77
  const configData = await this.storage.getMetadata(LEGACY_CONFIG_KEY);
78
78
  if (configData) {
79
79
  // Migrate to new location
80
- await this.migrateConfig(configData);
81
- this.lastConfigVersion = configData.version;
82
- return configData;
80
+ const config = configData;
81
+ await this.migrateConfig(config);
82
+ this.lastConfigVersion = config.version;
83
+ return config;
83
84
  }
84
85
  }
85
86
  catch (error) {
@@ -170,13 +171,14 @@ export class DistributedConfigManager {
170
171
  const legacyConfig = await this.storage.getMetadata(LEGACY_CONFIG_KEY);
171
172
  if (legacyConfig) {
172
173
  console.log('Migrating distributed config from legacy location to index folder...');
174
+ const config = legacyConfig;
173
175
  // Save to new location
174
- await this.migrateConfig(legacyConfig);
176
+ await this.migrateConfig(config);
175
177
  // Delete from old location (optional - we can keep it for rollback)
176
178
  // await this.storage.deleteMetadata(LEGACY_CONFIG_KEY)
177
179
  this.hasMigrated = true;
178
- this.lastConfigVersion = legacyConfig.version;
179
- return legacyConfig;
180
+ this.lastConfigVersion = config.version;
181
+ return config;
180
182
  }
181
183
  }
182
184
  catch (error) {
@@ -217,6 +217,7 @@ export class ShardMigrationManager extends EventEmitter {
217
217
  // Don't delete immediately in case of rollback
218
218
  const cleanupKey = `cleanup:${shardId}:${Date.now()}`;
219
219
  await this.storage.saveMetadata(cleanupKey, {
220
+ noun: 'Document',
220
221
  shardId,
221
222
  scheduledFor: Date.now() + 3600000 // Delete after 1 hour
222
223
  });
@@ -236,6 +237,7 @@ export class ShardMigrationManager extends EventEmitter {
236
237
  }
237
238
  // Track progress
238
239
  const progress = {
240
+ noun: 'Document',
239
241
  migrationId: data.migrationId,
240
242
  shardId: data.shardId,
241
243
  received: data.offset + data.items.length,
@@ -163,7 +163,7 @@ export class StorageDiscovery extends EventEmitter {
163
163
  // Remove ourselves from node registry
164
164
  try {
165
165
  // Mark as deleted rather than actually deleting
166
- const deadNode = { ...this.nodeInfo, lastSeen: 0, status: 'inactive' };
166
+ const deadNode = { noun: 'Document', ...this.nodeInfo, lastSeen: 0, status: 'inactive' };
167
167
  await this.storage.saveMetadata(`${this.CLUSTER_PATH}/nodes/${this.nodeId}.json`, deadNode);
168
168
  }
169
169
  catch (err) {
@@ -181,7 +181,7 @@ export class StorageDiscovery extends EventEmitter {
181
181
  */
182
182
  async registerNode() {
183
183
  const path = `${this.CLUSTER_PATH}/nodes/${this.nodeId}.json`;
184
- await this.storage.saveMetadata(path, this.nodeInfo);
184
+ await this.storage.saveMetadata(path, { noun: 'Document', ...this.nodeInfo });
185
185
  // Also update registry
186
186
  await this.updateNodeRegistry(this.nodeId);
187
187
  }
@@ -235,7 +235,8 @@ export class StorageDiscovery extends EventEmitter {
235
235
  if (nodeId === this.nodeId)
236
236
  continue;
237
237
  try {
238
- const nodeInfo = await this.storage.getMetadata(`${this.CLUSTER_PATH}/nodes/${nodeId}.json`);
238
+ const nodeInfoData = await this.storage.getMetadata(`${this.CLUSTER_PATH}/nodes/${nodeId}.json`);
239
+ const nodeInfo = nodeInfoData;
239
240
  // Check if node is alive
240
241
  if (now - nodeInfo.lastSeen < this.NODE_TIMEOUT) {
241
242
  if (!this.clusterConfig.nodes[nodeId]) {
@@ -291,6 +292,7 @@ export class StorageDiscovery extends EventEmitter {
291
292
  registry = registry.filter(id => id !== remove);
292
293
  }
293
294
  await this.storage.saveMetadata(`${this.CLUSTER_PATH}/registry.json`, {
295
+ noun: 'Document',
294
296
  nodes: registry,
295
297
  updated: Date.now()
296
298
  });
@@ -348,7 +350,7 @@ export class StorageDiscovery extends EventEmitter {
348
350
  async saveClusterConfig() {
349
351
  if (!this.clusterConfig)
350
352
  return;
351
- await this.storage.saveMetadata(`${this.CLUSTER_PATH}/config.json`, this.clusterConfig);
353
+ await this.storage.saveMetadata(`${this.CLUSTER_PATH}/config.json`, { noun: 'Document', ...this.clusterConfig });
352
354
  }
353
355
  /**
354
356
  * Trigger leader election (simplified - not full Raft)
@@ -52,7 +52,7 @@ export declare class EmbeddingManager {
52
52
  /**
53
53
  * Generate embeddings
54
54
  */
55
- embed(text: string | string[]): Promise<Vector>;
55
+ embed(text: string | string[] | Record<string, unknown>): Promise<Vector>;
56
56
  /**
57
57
  * Generate mock embeddings for unit tests
58
58
  */
@@ -94,7 +94,7 @@ export declare const embeddingManager: EmbeddingManager;
94
94
  /**
95
95
  * Direct embed function
96
96
  */
97
- export declare function embed(text: string | string[]): Promise<Vector>;
97
+ export declare function embed(text: string | string[] | Record<string, unknown>): Promise<Vector>;
98
98
  /**
99
99
  * Get embedding function for compatibility
100
100
  */
@@ -174,9 +174,13 @@ export class EmbeddingManager {
174
174
  else if (typeof text === 'string') {
175
175
  input = text;
176
176
  }
177
+ else if (typeof text === 'object') {
178
+ // Convert object to string representation
179
+ input = JSON.stringify(text);
180
+ }
177
181
  else {
178
182
  // This shouldn't happen but let's be defensive
179
- console.warn('EmbeddingManager.embed received non-string input:', typeof text);
183
+ console.warn('EmbeddingManager.embed received unexpected input type:', typeof text);
180
184
  input = String(text);
181
185
  }
182
186
  // Generate embedding
@@ -218,8 +218,11 @@ export class LSMTree {
218
218
  const data = sstable.serialize();
219
219
  const storageKey = `${this.config.storagePrefix}-${sstable.metadata.id}`;
220
220
  await this.storage.saveMetadata(storageKey, {
221
- type: 'lsm-sstable',
222
- data: Array.from(data) // Convert Uint8Array to number[] for JSON storage
221
+ noun: 'thing', // Required for NounMetadata
222
+ data: {
223
+ type: 'lsm-sstable',
224
+ data: Array.from(data) // Convert Uint8Array to number[] for JSON storage
225
+ }
223
226
  });
224
227
  // Add to L0 SSTables
225
228
  if (!this.sstablesByLevel.has(0)) {
@@ -269,8 +272,11 @@ export class LSMTree {
269
272
  const data = merged.serialize();
270
273
  const storageKey = `${this.config.storagePrefix}-${merged.metadata.id}`;
271
274
  await this.storage.saveMetadata(storageKey, {
272
- type: 'lsm-sstable',
273
- data: Array.from(data)
275
+ noun: 'thing', // Required for NounMetadata
276
+ data: {
277
+ type: 'lsm-sstable',
278
+ data: Array.from(data)
279
+ }
274
280
  });
275
281
  // Delete old SSTables from storage
276
282
  for (const sstable of sstables) {
@@ -339,8 +345,9 @@ export class LSMTree {
339
345
  */
340
346
  async loadManifest() {
341
347
  try {
342
- const data = await this.storage.getMetadata(`${this.config.storagePrefix}-manifest`);
343
- if (data) {
348
+ const metadata = await this.storage.getMetadata(`${this.config.storagePrefix}-manifest`);
349
+ if (metadata && metadata.data) {
350
+ const data = metadata.data;
344
351
  this.manifest.sstables = new Map(Object.entries(data.sstables || {}));
345
352
  this.manifest.lastCompaction = data.lastCompaction || Date.now();
346
353
  this.manifest.totalRelationships = data.totalRelationships || 0;
@@ -361,15 +368,18 @@ export class LSMTree {
361
368
  const loadPromise = (async () => {
362
369
  try {
363
370
  const storageKey = `${this.config.storagePrefix}-${sstableId}`;
364
- const data = await this.storage.getMetadata(storageKey);
365
- if (data && data.type === 'lsm-sstable') {
366
- // Convert number[] back to Uint8Array
367
- const uint8Data = new Uint8Array(data.data);
368
- const sstable = SSTable.deserialize(uint8Data);
369
- if (!this.sstablesByLevel.has(level)) {
370
- this.sstablesByLevel.set(level, []);
371
+ const metadata = await this.storage.getMetadata(storageKey);
372
+ if (metadata && metadata.data) {
373
+ const data = metadata.data;
374
+ if (data.type === 'lsm-sstable') {
375
+ // Convert number[] back to Uint8Array
376
+ const uint8Data = new Uint8Array(data.data);
377
+ const sstable = SSTable.deserialize(uint8Data);
378
+ if (!this.sstablesByLevel.has(level)) {
379
+ this.sstablesByLevel.set(level, []);
380
+ }
381
+ this.sstablesByLevel.get(level).push(sstable);
371
382
  }
372
- this.sstablesByLevel.get(level).push(sstable);
373
383
  }
374
384
  }
375
385
  catch (error) {
@@ -386,12 +396,14 @@ export class LSMTree {
386
396
  */
387
397
  async saveManifest() {
388
398
  try {
389
- const manifestData = {
390
- sstables: Object.fromEntries(this.manifest.sstables),
391
- lastCompaction: this.manifest.lastCompaction,
392
- totalRelationships: this.manifest.totalRelationships
393
- };
394
- await this.storage.saveMetadata(`${this.config.storagePrefix}-manifest`, manifestData);
399
+ await this.storage.saveMetadata(`${this.config.storagePrefix}-manifest`, {
400
+ noun: 'thing', // Required for NounMetadata
401
+ data: {
402
+ sstables: Object.fromEntries(this.manifest.sstables),
403
+ lastCompaction: this.manifest.lastCompaction,
404
+ totalRelationships: this.manifest.totalRelationships
405
+ }
406
+ });
395
407
  }
396
408
  catch (error) {
397
409
  prodLog.error('LSMTree: Failed to save manifest', error);
@@ -332,8 +332,12 @@ export class TypeAwareHNSWIndex {
332
332
  // Route each noun to its type index
333
333
  for (const nounData of result.items) {
334
334
  try {
335
- // Determine noun type from multiple possible sources
336
- const nounType = nounData.nounType || nounData.metadata?.noun || nounData.metadata?.type;
335
+ // v4.0.0: Load metadata separately to get noun type
336
+ let nounType = nounData.nounType;
337
+ if (!nounType) {
338
+ const metadata = await this.storage.getNounMetadata(nounData.id);
339
+ nounType = (metadata?.noun || metadata?.type);
340
+ }
337
341
  // Skip if type not in rebuild list
338
342
  if (!nounType || !typesToRebuild.includes(nounType)) {
339
343
  continue;