@soulcraft/brainy 1.5.0 → 2.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (141) hide show
  1. package/CHANGELOG.md +188 -0
  2. package/LICENSE +2 -2
  3. package/README.md +200 -595
  4. package/bin/brainy-interactive.js +564 -0
  5. package/bin/brainy-ts.js +18 -0
  6. package/bin/brainy.js +672 -81
  7. package/dist/augmentationPipeline.d.ts +48 -220
  8. package/dist/augmentationPipeline.js +60 -508
  9. package/dist/augmentationRegistry.d.ts +22 -31
  10. package/dist/augmentationRegistry.js +28 -79
  11. package/dist/augmentations/apiServerAugmentation.d.ts +108 -0
  12. package/dist/augmentations/apiServerAugmentation.js +502 -0
  13. package/dist/augmentations/batchProcessingAugmentation.d.ts +95 -0
  14. package/dist/augmentations/batchProcessingAugmentation.js +567 -0
  15. package/dist/augmentations/brainyAugmentation.d.ts +153 -0
  16. package/dist/augmentations/brainyAugmentation.js +145 -0
  17. package/dist/augmentations/cacheAugmentation.d.ts +105 -0
  18. package/dist/augmentations/cacheAugmentation.js +238 -0
  19. package/dist/augmentations/conduitAugmentations.d.ts +54 -156
  20. package/dist/augmentations/conduitAugmentations.js +156 -1082
  21. package/dist/augmentations/connectionPoolAugmentation.d.ts +62 -0
  22. package/dist/augmentations/connectionPoolAugmentation.js +316 -0
  23. package/dist/augmentations/defaultAugmentations.d.ts +53 -0
  24. package/dist/augmentations/defaultAugmentations.js +88 -0
  25. package/dist/augmentations/entityRegistryAugmentation.d.ts +126 -0
  26. package/dist/augmentations/entityRegistryAugmentation.js +386 -0
  27. package/dist/augmentations/indexAugmentation.d.ts +117 -0
  28. package/dist/augmentations/indexAugmentation.js +284 -0
  29. package/dist/augmentations/intelligentVerbScoringAugmentation.d.ts +152 -0
  30. package/dist/augmentations/intelligentVerbScoringAugmentation.js +554 -0
  31. package/dist/augmentations/metricsAugmentation.d.ts +202 -0
  32. package/dist/augmentations/metricsAugmentation.js +291 -0
  33. package/dist/augmentations/monitoringAugmentation.d.ts +94 -0
  34. package/dist/augmentations/monitoringAugmentation.js +227 -0
  35. package/dist/augmentations/neuralImport.d.ts +50 -117
  36. package/dist/augmentations/neuralImport.js +255 -629
  37. package/dist/augmentations/requestDeduplicatorAugmentation.d.ts +52 -0
  38. package/dist/augmentations/requestDeduplicatorAugmentation.js +162 -0
  39. package/dist/augmentations/serverSearchAugmentations.d.ts +43 -22
  40. package/dist/augmentations/serverSearchAugmentations.js +125 -72
  41. package/dist/augmentations/storageAugmentation.d.ts +54 -0
  42. package/dist/augmentations/storageAugmentation.js +93 -0
  43. package/dist/augmentations/storageAugmentations.d.ts +96 -0
  44. package/dist/augmentations/storageAugmentations.js +182 -0
  45. package/dist/augmentations/synapseAugmentation.d.ts +156 -0
  46. package/dist/augmentations/synapseAugmentation.js +312 -0
  47. package/dist/augmentations/walAugmentation.d.ts +108 -0
  48. package/dist/augmentations/walAugmentation.js +515 -0
  49. package/dist/brainyData.d.ts +404 -130
  50. package/dist/brainyData.js +1331 -853
  51. package/dist/chat/BrainyChat.d.ts +16 -8
  52. package/dist/chat/BrainyChat.js +60 -32
  53. package/dist/chat/ChatCLI.d.ts +1 -1
  54. package/dist/chat/ChatCLI.js +6 -6
  55. package/dist/cli/catalog.d.ts +3 -3
  56. package/dist/cli/catalog.js +116 -70
  57. package/dist/cli/commands/core.d.ts +61 -0
  58. package/dist/cli/commands/core.js +348 -0
  59. package/dist/cli/commands/neural.d.ts +25 -0
  60. package/dist/cli/commands/neural.js +508 -0
  61. package/dist/cli/commands/utility.d.ts +37 -0
  62. package/dist/cli/commands/utility.js +276 -0
  63. package/dist/cli/index.d.ts +7 -0
  64. package/dist/cli/index.js +167 -0
  65. package/dist/cli/interactive.d.ts +164 -0
  66. package/dist/cli/interactive.js +542 -0
  67. package/dist/cortex/neuralImport.js +5 -5
  68. package/dist/critical/model-guardian.js +11 -4
  69. package/dist/embeddings/lightweight-embedder.d.ts +23 -0
  70. package/dist/embeddings/lightweight-embedder.js +136 -0
  71. package/dist/embeddings/universal-memory-manager.d.ts +38 -0
  72. package/dist/embeddings/universal-memory-manager.js +206 -0
  73. package/dist/embeddings/worker-embedding.d.ts +7 -0
  74. package/dist/embeddings/worker-embedding.js +77 -0
  75. package/dist/embeddings/worker-manager.d.ts +28 -0
  76. package/dist/embeddings/worker-manager.js +162 -0
  77. package/dist/examples/basicUsage.js +7 -7
  78. package/dist/graph/pathfinding.d.ts +78 -0
  79. package/dist/graph/pathfinding.js +393 -0
  80. package/dist/hnsw/hnswIndex.d.ts +13 -0
  81. package/dist/hnsw/hnswIndex.js +35 -0
  82. package/dist/hnsw/hnswIndexOptimized.d.ts +1 -0
  83. package/dist/hnsw/hnswIndexOptimized.js +3 -0
  84. package/dist/index.d.ts +9 -11
  85. package/dist/index.js +21 -11
  86. package/dist/indices/fieldIndex.d.ts +76 -0
  87. package/dist/indices/fieldIndex.js +357 -0
  88. package/dist/mcp/brainyMCPAdapter.js +3 -2
  89. package/dist/mcp/mcpAugmentationToolset.js +11 -17
  90. package/dist/neural/embeddedPatterns.d.ts +41 -0
  91. package/dist/neural/embeddedPatterns.js +4044 -0
  92. package/dist/neural/naturalLanguageProcessor.d.ts +94 -0
  93. package/dist/neural/naturalLanguageProcessor.js +317 -0
  94. package/dist/neural/naturalLanguageProcessorStatic.d.ts +64 -0
  95. package/dist/neural/naturalLanguageProcessorStatic.js +151 -0
  96. package/dist/neural/neuralAPI.d.ts +255 -0
  97. package/dist/neural/neuralAPI.js +612 -0
  98. package/dist/neural/patternLibrary.d.ts +101 -0
  99. package/dist/neural/patternLibrary.js +313 -0
  100. package/dist/neural/patterns.d.ts +27 -0
  101. package/dist/neural/patterns.js +68 -0
  102. package/dist/neural/staticPatternMatcher.d.ts +35 -0
  103. package/dist/neural/staticPatternMatcher.js +153 -0
  104. package/dist/scripts/precomputePatternEmbeddings.d.ts +19 -0
  105. package/dist/scripts/precomputePatternEmbeddings.js +100 -0
  106. package/dist/storage/adapters/fileSystemStorage.d.ts +5 -0
  107. package/dist/storage/adapters/fileSystemStorage.js +20 -0
  108. package/dist/storage/adapters/s3CompatibleStorage.d.ts +5 -0
  109. package/dist/storage/adapters/s3CompatibleStorage.js +16 -0
  110. package/dist/storage/enhancedClearOperations.d.ts +83 -0
  111. package/dist/storage/enhancedClearOperations.js +345 -0
  112. package/dist/storage/storageFactory.js +31 -27
  113. package/dist/triple/TripleIntelligence.d.ts +134 -0
  114. package/dist/triple/TripleIntelligence.js +548 -0
  115. package/dist/types/augmentations.d.ts +45 -344
  116. package/dist/types/augmentations.js +5 -2
  117. package/dist/types/brainyDataInterface.d.ts +20 -10
  118. package/dist/types/graphTypes.d.ts +46 -0
  119. package/dist/types/graphTypes.js +16 -2
  120. package/dist/utils/BoundedRegistry.d.ts +29 -0
  121. package/dist/utils/BoundedRegistry.js +54 -0
  122. package/dist/utils/embedding.js +20 -3
  123. package/dist/utils/hybridModelManager.js +10 -5
  124. package/dist/utils/metadataFilter.d.ts +33 -19
  125. package/dist/utils/metadataFilter.js +58 -23
  126. package/dist/utils/metadataIndex.d.ts +37 -6
  127. package/dist/utils/metadataIndex.js +427 -64
  128. package/dist/utils/requestDeduplicator.d.ts +10 -0
  129. package/dist/utils/requestDeduplicator.js +24 -0
  130. package/dist/utils/unifiedCache.d.ts +103 -0
  131. package/dist/utils/unifiedCache.js +311 -0
  132. package/package.json +40 -125
  133. package/scripts/ensure-models.js +108 -0
  134. package/scripts/prepare-models.js +387 -0
  135. package/OFFLINE_MODELS.md +0 -56
  136. package/dist/intelligence/neuralEngine.d.ts +0 -207
  137. package/dist/intelligence/neuralEngine.js +0 -706
  138. package/dist/utils/modelLoader.d.ts +0 -32
  139. package/dist/utils/modelLoader.js +0 -219
  140. package/dist/utils/modelManager.d.ts +0 -77
  141. package/dist/utils/modelManager.js +0 -219
@@ -4,25 +4,52 @@
4
4
  */
5
5
  import { v4 as uuidv4 } from './universal/uuid.js';
6
6
  import { HNSWIndex } from './hnsw/hnswIndex.js';
7
- import { ExecutionMode } from './augmentationPipeline.js';
8
7
  import { HNSWIndexOptimized } from './hnsw/hnswIndexOptimized.js';
9
- import { createStorage } from './storage/storageFactory.js';
10
8
  import { cosineDistance, defaultEmbeddingFunction, cleanupWorkerPools, batchEmbed } from './utils/index.js';
11
9
  import { getAugmentationVersion } from './utils/version.js';
12
10
  import { matchesMetadataFilter } from './utils/metadataFilter.js';
13
- import { MetadataIndexManager } from './utils/metadataIndex.js';
14
11
  import { NounType, VerbType } from './types/graphTypes.js';
15
12
  import { createServerSearchAugmentations } from './augmentations/serverSearchAugmentations.js';
16
- import { IntelligentVerbScoring } from './augmentations/intelligentVerbScoring.js';
17
13
  import { augmentationPipeline } from './augmentationPipeline.js';
18
14
  import { prodLog } from './utils/logger.js';
19
15
  import { prepareJsonForVectorization, extractFieldFromJson } from './utils/jsonProcessing.js';
20
- import { DistributedConfigManager, HashPartitioner, OperationalModeFactory, DomainDetector, HealthMonitor } from './distributed/index.js';
21
- import { SearchCache } from './utils/searchCache.js';
16
+ import { DistributedConfigManager, HashPartitioner, OperationalModeFactory, DomainDetector } from './distributed/index.js';
22
17
  import { CacheAutoConfigurator } from './utils/cacheAutoConfig.js';
23
- import { StatisticsCollector } from './utils/statisticsCollector.js';
24
- import { AugmentationManager } from './augmentationManager.js';
18
+ import { RequestDeduplicator } from './utils/requestDeduplicator.js';
19
+ import { AugmentationRegistry } from './augmentations/brainyAugmentation.js';
20
+ import { WALAugmentation } from './augmentations/walAugmentation.js';
21
+ import { RequestDeduplicatorAugmentation } from './augmentations/requestDeduplicatorAugmentation.js';
22
+ import { ConnectionPoolAugmentation } from './augmentations/connectionPoolAugmentation.js';
23
+ import { BatchProcessingAugmentation } from './augmentations/batchProcessingAugmentation.js';
24
+ import { EntityRegistryAugmentation, AutoRegisterEntitiesAugmentation } from './augmentations/entityRegistryAugmentation.js';
25
+ import { createDefaultAugmentations } from './augmentations/defaultAugmentations.js';
26
+ // import { RealtimeStreamingAugmentation } from './augmentations/realtimeStreamingAugmentation.js'
27
+ import { IntelligentVerbScoringAugmentation } from './augmentations/intelligentVerbScoringAugmentation.js';
28
+ import { NeuralAPI } from './neural/neuralAPI.js';
29
+ import { TripleIntelligenceEngine } from './triple/TripleIntelligence.js';
25
30
  export class BrainyData {
31
+ // REMOVED: HealthMonitor is now handled by MonitoringAugmentation
32
+ // Statistics collector
33
+ // REMOVED: StatisticsCollector is now handled by MetricsAugmentation
34
+ // Clean augmentation accessors for internal use
35
+ get cache() {
36
+ return this.augmentations.get('cache');
37
+ }
38
+ // IMPORTANT: this.index returns the HNSW vector index, NOT the metadata index!
39
+ // The metadata index is available through this.metadataIndex
40
+ get index() {
41
+ return this.hnswIndex;
42
+ }
43
+ // Metadata index for field-based queries (from IndexAugmentation)
44
+ get metadataIndex() {
45
+ return this.augmentations.get('index');
46
+ }
47
+ get metrics() {
48
+ return this.augmentations.get('metrics');
49
+ }
50
+ get monitoring() {
51
+ return this.augmentations.get('monitoring');
52
+ }
26
53
  /**
27
54
  * Get the vector dimensions
28
55
  */
@@ -43,18 +70,30 @@ export class BrainyData {
43
70
  const config = this.index.getConfig();
44
71
  return config.efConstruction || 200;
45
72
  }
73
+ /**
74
+ * Check if BrainyData has been initialized
75
+ */
76
+ get initialized() {
77
+ return this.isInitialized;
78
+ }
46
79
  /**
47
80
  * Create a new vector database
48
81
  */
49
82
  constructor(config = {}) {
50
83
  this.storage = null;
51
- this.metadataIndex = null;
84
+ // REMOVED: MetadataIndex is now handled by IndexAugmentation
52
85
  this.isInitialized = false;
53
86
  this.isInitializing = false;
54
87
  this.storageConfig = {};
55
88
  this.useOptimizedIndex = false;
56
89
  this.loggingConfig = { verbose: true };
57
90
  this.defaultService = 'default';
91
+ // REMOVED: SearchCache is now handled by CacheAugmentation
92
+ /**
93
+ * Enterprise augmentation system
94
+ * Handles WAL, connection pooling, batching, streaming, and intelligent scoring
95
+ */
96
+ this.augmentations = new AugmentationRegistry();
58
97
  // Timeout and retry configuration
59
98
  this.timeoutConfig = {};
60
99
  this.retryConfig = {};
@@ -69,10 +108,10 @@ export class BrainyData {
69
108
  this.maintenanceIntervals = [];
70
109
  this.lastUpdateTime = 0;
71
110
  this.lastKnownNounCount = 0;
72
- // Remote server properties
111
+ // Remote server properties - TODO: Implement in post-2.0.0 release
73
112
  this.remoteServerConfig = null;
74
- this.serverSearchConduit = null;
75
- this.serverConnection = null;
113
+ // private serverSearchConduit: ServerSearchConduitAugmentation | null = null
114
+ // private serverConnection: WebSocketConnection | null = null
76
115
  this.intelligentVerbScoring = null;
77
116
  // Distributed mode properties
78
117
  this.distributedConfig = null;
@@ -80,9 +119,6 @@ export class BrainyData {
80
119
  this.partitioner = null;
81
120
  this.operationalMode = null;
82
121
  this.domainDetector = null;
83
- this.healthMonitor = null;
84
- // Statistics collector
85
- this.statisticsCollector = new StatisticsCollector();
86
122
  // Store config
87
123
  this.config = config;
88
124
  // Set dimensions to fixed value of 384 (all-MiniLM-L6-v2 dimension)
@@ -96,7 +132,7 @@ export class BrainyData {
96
132
  hnswConfig.useDiskBasedIndex = true;
97
133
  }
98
134
  // Temporarily use base HNSW index for metadata filtering
99
- this.index = new HNSWIndex(hnswConfig, this.distanceFunction);
135
+ this.hnswIndex = new HNSWIndex(hnswConfig, this.distanceFunction);
100
136
  this.useOptimizedIndex = false;
101
137
  // Set storage if provided, otherwise it will be initialized in init()
102
138
  this.storage = config.storageAdapter || null;
@@ -203,20 +239,165 @@ export class BrainyData {
203
239
  prodLog.info(this.cacheAutoConfigurator.getConfigExplanation(autoConfig));
204
240
  }
205
241
  }
206
- // Initialize search cache with final configuration
207
- this.searchCache = new SearchCache(finalSearchCacheConfig);
208
- // Initialize augmentation manager
209
- this.augmentations = new AugmentationManager();
210
- // Initialize intelligent verb scoring if enabled
211
- if (config.intelligentVerbScoring?.enabled) {
212
- this.intelligentVerbScoring = new IntelligentVerbScoring(config.intelligentVerbScoring);
213
- this.intelligentVerbScoring.enabled = true;
214
- }
242
+ // Search cache is now handled by CacheAugmentation
243
+ // this.searchCache = new SearchCache<T>(finalSearchCacheConfig)
244
+ // Keep reference for compatibility (will be set by augmentation)
245
+ // Augmentation system will be initialized in init() method
246
+ // Legacy systems completely replaced by augmentation architecture
247
+ // All intelligent systems now handled by augmentations
215
248
  }
216
249
  /**
217
250
  * Check if the database is in read-only mode and throw an error if it is
218
251
  * @throws Error if the database is in read-only mode
219
252
  */
253
+ /**
254
+ * Register default augmentations without initializing them
255
+ * Phase 1 of two-phase initialization
256
+ */
257
+ registerDefaultAugmentations() {
258
+ // Register enterprise-grade augmentations in priority order
259
+ // Note: These are registered but NOT initialized yet (no context)
260
+ // Register core feature augmentations (previously hardcoded)
261
+ // These replace SearchCache, MetadataIndex, StatisticsCollector, HealthMonitor
262
+ const defaultAugs = createDefaultAugmentations({
263
+ cache: this.config.searchCache !== undefined ? this.config.searchCache : true,
264
+ index: this.config.metadataIndex !== undefined ? this.config.metadataIndex : true,
265
+ metrics: this.config.statistics !== false,
266
+ monitoring: Boolean(this.config.health || this.distributedConfig?.enabled)
267
+ });
268
+ for (const aug of defaultAugs) {
269
+ this.augmentations.register(aug);
270
+ }
271
+ // Priority 100: Critical system operations
272
+ // Disable WAL in test environments to avoid directory creation issues
273
+ const isTestEnvironment = process.env.NODE_ENV === 'test' || process.env.VITEST === 'true';
274
+ this.augmentations.register(new WALAugmentation({ enabled: !isTestEnvironment }));
275
+ this.augmentations.register(new ConnectionPoolAugmentation());
276
+ // Priority 95: Entity registry for fast external-ID to UUID mapping
277
+ this.augmentations.register(new EntityRegistryAugmentation({
278
+ maxCacheSize: this.config.entityCacheSize || 100000,
279
+ cacheTTL: this.config.entityCacheTTL || 300000,
280
+ persistence: 'hybrid',
281
+ indexedFields: ['did', 'handle', 'uri', 'external_id', 'id']
282
+ }));
283
+ // Priority 85: Auto-register entities after they're added
284
+ this.augmentations.register(new AutoRegisterEntitiesAugmentation());
285
+ // Priority 80: High-throughput batch processing
286
+ this.augmentations.register(new BatchProcessingAugmentation({
287
+ maxBatchSize: this.config.batchSize || 1000,
288
+ maxWaitTime: this.config.batchWaitTime || 100
289
+ }));
290
+ // Priority 50: Performance optimizations
291
+ this.augmentations.register(new RequestDeduplicatorAugmentation({
292
+ ttl: 5000,
293
+ maxSize: 1000
294
+ }));
295
+ // Priority 10: Enhancement features
296
+ const intelligentVerbAugmentation = new IntelligentVerbScoringAugmentation(this.config.intelligentVerbScoring || { enabled: false });
297
+ this.augmentations.register(intelligentVerbAugmentation);
298
+ // Store reference if intelligent verb scoring is enabled
299
+ if (this.config.intelligentVerbScoring?.enabled) {
300
+ this.intelligentVerbScoring = intelligentVerbAugmentation.getScoring();
301
+ }
302
+ }
303
+ /**
304
+ * Resolve storage from augmentation or config
305
+ * Phase 2 of two-phase initialization
306
+ */
307
+ async resolveStorage() {
308
+ // Check if storage augmentation is registered
309
+ const storageAug = this.augmentations.findByOperation('storage');
310
+ if (storageAug && 'provideStorage' in storageAug) {
311
+ // Get storage from augmentation
312
+ this.storage = await storageAug.provideStorage();
313
+ if (this.loggingConfig?.verbose) {
314
+ console.log('Using storage from augmentation:', storageAug.name);
315
+ }
316
+ }
317
+ else if (!this.storage) {
318
+ // No storage augmentation and no provided adapter
319
+ // Use zero-config approach
320
+ // Import storage augmentation helpers
321
+ const { DynamicStorageAugmentation, createStorageAugmentationFromConfig } = await import('./augmentations/storageAugmentation.js');
322
+ const { createAutoStorageAugmentation } = await import('./augmentations/storageAugmentations.js');
323
+ // Build storage options from config
324
+ let storageOptions = {
325
+ ...this.storageConfig,
326
+ requestPersistentStorage: this.requestPersistentStorage
327
+ };
328
+ // Add cache configuration if provided
329
+ if (this.cacheConfig) {
330
+ storageOptions.cacheConfig = {
331
+ ...this.cacheConfig,
332
+ readOnly: this.readOnly
333
+ };
334
+ }
335
+ // Ensure s3Storage has all required fields if it's provided
336
+ if (storageOptions.s3Storage) {
337
+ if (storageOptions.s3Storage.bucketName &&
338
+ storageOptions.s3Storage.accessKeyId &&
339
+ storageOptions.s3Storage.secretAccessKey) {
340
+ // All required fields are present
341
+ }
342
+ else {
343
+ // Missing required fields, remove s3Storage
344
+ const { s3Storage, ...rest } = storageOptions;
345
+ storageOptions = rest;
346
+ console.warn('Ignoring s3Storage configuration due to missing required fields');
347
+ }
348
+ }
349
+ // Check if specific storage is configured
350
+ if (storageOptions.s3Storage || storageOptions.r2Storage ||
351
+ storageOptions.gcsStorage || storageOptions.forceMemoryStorage ||
352
+ storageOptions.forceFileSystemStorage) {
353
+ // Create storage from config
354
+ const { createStorage } = await import('./storage/storageFactory.js');
355
+ this.storage = await createStorage(storageOptions);
356
+ // Wrap in augmentation for consistency
357
+ const wrapper = new DynamicStorageAugmentation(this.storage);
358
+ this.augmentations.register(wrapper);
359
+ }
360
+ else {
361
+ // Zero-config: auto-select based on environment
362
+ const autoAug = await createAutoStorageAugmentation({
363
+ rootDirectory: storageOptions.rootDirectory,
364
+ requestPersistentStorage: storageOptions.requestPersistentStorage
365
+ });
366
+ this.augmentations.register(autoAug);
367
+ this.storage = await autoAug.provideStorage();
368
+ }
369
+ }
370
+ // Initialize storage
371
+ if (this.storage) {
372
+ await this.storage.init();
373
+ }
374
+ else {
375
+ throw new Error('Failed to resolve storage');
376
+ }
377
+ }
378
+ /**
379
+ * Initialize the augmentation system with full context
380
+ * Phase 3 of two-phase initialization
381
+ */
382
+ async initializeAugmentations() {
383
+ // Create augmentation context
384
+ const context = {
385
+ brain: this,
386
+ storage: this.storage,
387
+ config: this.config,
388
+ log: (message, level = 'info') => {
389
+ if (this.loggingConfig?.verbose || level !== 'info') {
390
+ const prefix = level === 'error' ? '❌' : level === 'warn' ? '⚠️' : '✅';
391
+ console.log(`${prefix} ${message}`);
392
+ }
393
+ }
394
+ };
395
+ // Initialize all augmentations (already registered in registerDefaultAugmentations)
396
+ await this.augmentations.initialize(context);
397
+ if (this.loggingConfig?.verbose) {
398
+ console.log('🚀 New augmentation system initialized successfully');
399
+ }
400
+ }
220
401
  checkReadOnly() {
221
402
  if (this.readOnly) {
222
403
  throw new Error('Cannot perform write operation: database is in read-only mode');
@@ -329,12 +510,13 @@ export class BrainyData {
329
510
  * Start metadata index maintenance
330
511
  */
331
512
  startMetadataIndexMaintenance() {
332
- if (!this.metadataIndex)
513
+ const metaIndex = this.metadataIndex;
514
+ if (!metaIndex)
333
515
  return;
334
516
  // Flush index periodically to persist changes
335
517
  const flushInterval = setInterval(async () => {
336
518
  try {
337
- await this.metadataIndex.flush();
519
+ await metaIndex.flush();
338
520
  }
339
521
  catch (error) {
340
522
  prodLog.warn('Error flushing metadata index:', error);
@@ -397,7 +579,7 @@ export class BrainyData {
397
579
  }
398
580
  }
399
581
  // Cleanup expired cache entries (defensive mechanism for distributed scenarios)
400
- const expiredCount = this.searchCache.cleanupExpiredEntries();
582
+ const expiredCount = this.cache?.cleanupExpiredEntries() || 0;
401
583
  if (expiredCount > 0 && this.loggingConfig?.verbose) {
402
584
  prodLog.debug(`Cleaned up ${expiredCount} expired cache entries`);
403
585
  }
@@ -484,7 +666,7 @@ export class BrainyData {
484
666
  }
485
667
  // Invalidate search cache if any external changes were detected
486
668
  if (addedCount > 0 || updatedCount > 0 || deletedCount > 0) {
487
- this.searchCache.invalidateOnDataChange('update');
669
+ this.cache?.invalidateOnDataChange('update');
488
670
  if (this.loggingConfig?.verbose) {
489
671
  console.log('Search cache invalidated due to external data changes');
490
672
  }
@@ -545,7 +727,7 @@ export class BrainyData {
545
727
  this.lastKnownNounCount = currentCount;
546
728
  // Invalidate search cache if new nouns were detected
547
729
  if (totalNewNouns > 0) {
548
- this.searchCache.invalidateOnDataChange('add');
730
+ this.cache?.invalidateOnDataChange('add');
549
731
  if (this.loggingConfig?.verbose) {
550
732
  console.log('Search cache invalidated due to external data changes');
551
733
  }
@@ -573,7 +755,8 @@ export class BrainyData {
573
755
  */
574
756
  async provideFeedbackForVerbScoring(sourceId, targetId, verbType, feedbackWeight, feedbackConfidence, feedbackType = 'correction') {
575
757
  if (this.intelligentVerbScoring?.enabled) {
576
- await this.intelligentVerbScoring.provideFeedback(sourceId, targetId, verbType, feedbackWeight, feedbackConfidence, feedbackType);
758
+ // The augmentation doesn't use feedbackConfidence separately
759
+ await this.intelligentVerbScoring.provideFeedback(sourceId, targetId, verbType, feedbackWeight, feedbackType);
577
760
  }
578
761
  }
579
762
  /**
@@ -614,9 +797,9 @@ export class BrainyData {
614
797
  // Check each type of augmentation
615
798
  for (const type of augmentationTypes) {
616
799
  const augmentations = augmentationPipeline.getAugmentationsByType(type);
617
- // Find the first enabled augmentation
800
+ // Find the first augmentation (all registered augmentations are considered enabled)
618
801
  for (const augmentation of augmentations) {
619
- if (augmentation.enabled) {
802
+ if (augmentation) {
620
803
  return augmentation.name;
621
804
  }
622
805
  }
@@ -656,23 +839,24 @@ export class BrainyData {
656
839
  return;
657
840
  }
658
841
  this.isInitializing = true;
659
- // CRITICAL: Ensure model is available before ANY operations
660
- // HYBRID SOLUTION: Use our best-of-both-worlds model manager
661
- // This ensures models are loaded with singleton pattern + multi-source fallbacks
662
- if (typeof this.embeddingFunction === 'function') {
842
+ // CRITICAL: Initialize universal memory manager ONLY for default embedding function
843
+ // This preserves custom embedding functions (like test mocks)
844
+ if (typeof this.embeddingFunction === 'function' && this.embeddingFunction === defaultEmbeddingFunction) {
663
845
  try {
664
- const { hybridModelManager } = await import('./utils/hybridModelManager.js');
665
- await hybridModelManager.getPrimaryModel();
666
- console.log('✅ HYBRID: Model successfully initialized with best-of-both approach');
846
+ const { universalMemoryManager } = await import('./embeddings/universal-memory-manager.js');
847
+ this.embeddingFunction = await universalMemoryManager.getEmbeddingFunction();
848
+ console.log('✅ UNIVERSAL: Memory-safe embedding system initialized');
667
849
  }
668
850
  catch (error) {
669
- console.error('🚨 CRITICAL: Hybrid model initialization failed!');
670
- console.error('Brainy cannot function without the transformer model.');
671
- console.error('Users cannot access their data without it.');
672
- this.isInitializing = false;
673
- throw error;
851
+ console.error('🚨 CRITICAL: Universal memory manager initialization failed!');
852
+ console.error('Falling back to standard embedding with potential memory issues.');
853
+ console.warn('Consider reducing usage or restarting process periodically.');
854
+ // Continue with default function - better than crashing
674
855
  }
675
856
  }
857
+ else if (this.embeddingFunction !== defaultEmbeddingFunction) {
858
+ console.log('✅ CUSTOM: Using custom embedding function (test or production override)');
859
+ }
676
860
  try {
677
861
  // Pre-load the embedding model early to ensure it's always available
678
862
  // This helps prevent issues with the Universal Sentence Encoder not being loaded
@@ -705,48 +889,19 @@ export class BrainyData {
705
889
  // The application will need to handle missing embedding functionality
706
890
  }
707
891
  }
708
- // Initialize storage if not provided in constructor
709
- if (!this.storage) {
710
- // Combine storage config with requestPersistentStorage for backward compatibility
711
- let storageOptions = {
712
- ...this.storageConfig,
713
- requestPersistentStorage: this.requestPersistentStorage
714
- };
715
- // Add cache configuration if provided
716
- if (this.cacheConfig) {
717
- storageOptions.cacheConfig = {
718
- ...this.cacheConfig,
719
- // Pass read-only flag to optimize cache behavior
720
- readOnly: this.readOnly
721
- };
722
- }
723
- // Ensure s3Storage has all required fields if it's provided
724
- if (storageOptions.s3Storage) {
725
- // Only include s3Storage if all required fields are present
726
- if (storageOptions.s3Storage.bucketName &&
727
- storageOptions.s3Storage.accessKeyId &&
728
- storageOptions.s3Storage.secretAccessKey) {
729
- // All required fields are present, keep s3Storage as is
730
- }
731
- else {
732
- // Missing required fields, remove s3Storage to avoid type errors
733
- const { s3Storage, ...rest } = storageOptions;
734
- storageOptions = rest;
735
- console.warn('Ignoring s3Storage configuration due to missing required fields');
736
- }
737
- }
738
- // Use type assertion to tell TypeScript that storageOptions conforms to StorageOptions
739
- this.storage = await createStorage(storageOptions);
740
- }
741
- // Initialize storage
742
- await this.storage.init();
892
+ // Phase 1: Register default augmentations (without initialization)
893
+ this.registerDefaultAugmentations();
894
+ // Phase 2: Resolve storage (either from augmentation or config)
895
+ await this.resolveStorage();
896
+ // Phase 3: Initialize all augmentations with full context
897
+ await this.initializeAugmentations();
743
898
  // Initialize distributed mode if configured
744
899
  if (this.distributedConfig) {
745
900
  await this.initializeDistributedMode();
746
901
  }
747
902
  // If using optimized index, set the storage adapter
748
- if (this.useOptimizedIndex && this.index instanceof HNSWIndexOptimized) {
749
- this.index.setStorage(this.storage);
903
+ if (this.useOptimizedIndex && this.hnswIndex instanceof HNSWIndexOptimized) {
904
+ this.hnswIndex.setStorage(this.storage);
750
905
  }
751
906
  // In write-only mode, skip loading the index into memory
752
907
  if (this.writeOnly) {
@@ -760,11 +915,11 @@ export class BrainyData {
760
915
  console.log('Database is in read-only mode with lazy loading enabled, skipping initial full load');
761
916
  }
762
917
  // Just initialize an empty index
763
- this.index.clear();
918
+ this.hnswIndex.clear();
764
919
  }
765
920
  else {
766
921
  // Clear the index and load nouns using pagination
767
- this.index.clear();
922
+ this.hnswIndex.clear();
768
923
  let offset = 0;
769
924
  const limit = 100;
770
925
  let hasMore = true;
@@ -804,21 +959,25 @@ export class BrainyData {
804
959
  try {
805
960
  const existingStats = await this.storage.getStatistics();
806
961
  if (existingStats) {
807
- this.statisticsCollector.mergeFromStorage(existingStats);
962
+ this.metrics.mergeFromStorage(existingStats);
808
963
  }
809
964
  }
810
965
  catch (e) {
811
966
  // Ignore errors loading existing statistics
812
967
  }
813
968
  // Initialize metadata index unless in read-only mode
969
+ // Metadata index is now handled by IndexAugmentation
814
970
  // Write-only mode NEEDS metadata indexing for search capability!
815
971
  if (!this.readOnly) {
816
- this.metadataIndex = new MetadataIndexManager(this.storage, this.config.metadataIndex);
972
+ // this.index = new MetadataIndexManager(
973
+ // this.storage!,
974
+ // this.config.metadataIndex
975
+ // )
817
976
  // Check if we need to rebuild the index (for existing data)
818
977
  // Skip rebuild for memory storage (starts empty) or when in read-only mode
819
978
  // Also skip if index already has entries
820
979
  const isMemoryStorage = this.storage?.constructor?.name === 'MemoryStorage';
821
- const stats = await this.metadataIndex.getStats();
980
+ const stats = await this.metadataIndex?.getStats?.() || { totalEntries: 0 };
822
981
  if (!isMemoryStorage && !this.readOnly && stats.totalEntries === 0) {
823
982
  // Check if we have existing data that needs indexing
824
983
  // Use a simple check to avoid expensive operations
@@ -831,9 +990,9 @@ export class BrainyData {
831
990
  if (this.loggingConfig?.verbose) {
832
991
  console.log('🔄 Rebuilding metadata index for existing data...');
833
992
  }
834
- await this.metadataIndex.rebuild();
993
+ await this.metadataIndex?.rebuild?.();
835
994
  if (this.loggingConfig?.verbose) {
836
- const newStats = await this.metadataIndex.getStats();
995
+ const newStats = await this.metadataIndex?.getStats?.() || { totalEntries: 0 };
837
996
  console.log(`✅ Metadata index rebuilt: ${newStats.totalEntries} entries, ${newStats.fieldsIndexed.length} fields`);
838
997
  }
839
998
  }
@@ -853,13 +1012,7 @@ export class BrainyData {
853
1012
  }
854
1013
  }
855
1014
  }
856
- // Initialize intelligent verb scoring augmentation if enabled
857
- if (this.intelligentVerbScoring) {
858
- await this.intelligentVerbScoring.initialize();
859
- this.intelligentVerbScoring.setBrainyInstance(this);
860
- // Register with augmentation pipeline
861
- augmentationPipeline.register(this.intelligentVerbScoring);
862
- }
1015
+ // Intelligent verb scoring is now initialized through the augmentation system
863
1016
  // Initialize default augmentations (Neural Import, etc.)
864
1017
  // TODO: Fix TypeScript issues in v0.57.0
865
1018
  // try {
@@ -877,7 +1030,7 @@ export class BrainyData {
877
1030
  // Start real-time updates if enabled
878
1031
  this.startRealtimeUpdates();
879
1032
  // Start metadata index maintenance
880
- if (this.metadataIndex) {
1033
+ if (this.index) {
881
1034
  this.startMetadataIndexMaintenance();
882
1035
  }
883
1036
  }
@@ -945,9 +1098,9 @@ export class BrainyData {
945
1098
  }
946
1099
  // Initialize domain detector
947
1100
  this.domainDetector = new DomainDetector();
948
- // Initialize health monitor
949
- this.healthMonitor = new HealthMonitor(this.configManager);
950
- this.healthMonitor.start();
1101
+ // Health monitor is now handled by MonitoringAugmentation
1102
+ // this.monitoring = new HealthMonitor(this.configManager)
1103
+ // this.monitoring.start()
951
1104
  // Set up config update listener
952
1105
  this.configManager.setOnConfigUpdate((config) => {
953
1106
  this.handleDistributedConfigUpdate(config);
@@ -974,10 +1127,7 @@ export class BrainyData {
974
1127
  * @returns Health status if distributed mode is enabled
975
1128
  */
976
1129
  getHealthStatus() {
977
- if (this.healthMonitor) {
978
- return this.healthMonitor.getHealthEndpointData();
979
- }
980
- return null;
1130
+ return this.monitoring?.getHealthStatus() || null;
981
1131
  }
982
1132
  /**
983
1133
  * Connect to a remote Brainy server for search operations
@@ -993,9 +1143,9 @@ export class BrainyData {
993
1143
  protocols,
994
1144
  localDb: this
995
1145
  });
996
- // Store the conduit and connection
997
- this.serverSearchConduit = conduit;
998
- this.serverConnection = connection;
1146
+ // TODO: Store conduit and connection (post-2.0.0 feature)
1147
+ // this.serverSearchConduit = conduit
1148
+ // this.serverConnection = connection
999
1149
  return connection;
1000
1150
  }
1001
1151
  catch (error) {
@@ -1066,6 +1216,15 @@ export class BrainyData {
1066
1216
  ]
1067
1217
  });
1068
1218
  vector = await this.embeddingFunction(preparedText);
1219
+ // IMPORTANT: When an object is passed as data and no metadata is provided,
1220
+ // use the object AS the metadata too. This is expected behavior for the API.
1221
+ // Users can pass either:
1222
+ // 1. addNoun(string, metadata) - vectorize string, store metadata
1223
+ // 2. addNoun(object) - vectorize object text, store object as metadata
1224
+ // 3. addNoun(object, metadata) - vectorize object text, store provided metadata
1225
+ if (!metadata) {
1226
+ metadata = vectorOrData;
1227
+ }
1069
1228
  // Track field names for this JSON document
1070
1229
  const service = this.getServiceName(options);
1071
1230
  if (this.storage) {
@@ -1148,20 +1307,21 @@ export class BrainyData {
1148
1307
  };
1149
1308
  }
1150
1309
  else {
1151
- // Normal mode: Add to index first
1152
- await this.index.addItem({ id, vector });
1153
- // Get the noun from the index
1154
- const indexNoun = this.index.getNouns().get(id);
1310
+ // Normal mode: Add to HNSW index first
1311
+ await this.hnswIndex.addItem({ id, vector, metadata });
1312
+ // Get the noun from the HNSW index
1313
+ const indexNoun = this.hnswIndex.getNouns().get(id);
1155
1314
  if (!indexNoun) {
1156
1315
  throw new Error(`Failed to retrieve newly created noun with ID ${id}`);
1157
1316
  }
1158
1317
  noun = indexNoun;
1159
1318
  }
1160
- // Save noun to storage
1161
- await this.storage.saveNoun(noun);
1162
- // Track noun statistics
1163
- const service = this.getServiceName(options);
1164
- await this.storage.incrementStatistic('noun', service);
1319
+ // Save noun to storage using augmentation system
1320
+ await this.augmentations.execute('saveNoun', { noun, options }, async () => {
1321
+ await this.storage.saveNoun(noun);
1322
+ const service = this.getServiceName(options);
1323
+ await this.storage.incrementStatistic('noun', service);
1324
+ });
1165
1325
  // Save metadata if provided and not empty
1166
1326
  if (metadata !== undefined) {
1167
1327
  // Skip saving if metadata is an empty object
@@ -1244,8 +1404,8 @@ export class BrainyData {
1244
1404
  }
1245
1405
  await this.storage.saveMetadata(id, metadataToSave);
1246
1406
  // Update metadata index (write-only mode should build indices!)
1247
- if (this.metadataIndex && !this.frozen) {
1248
- await this.metadataIndex.addToIndex(id, metadataToSave);
1407
+ if (this.index && !this.frozen) {
1408
+ await this.metadataIndex?.addToIndex?.(id, metadataToSave);
1249
1409
  }
1250
1410
  // Track metadata statistics
1251
1411
  const metadataService = this.getServiceName(options);
@@ -1254,19 +1414,18 @@ export class BrainyData {
1254
1414
  if (metadataToSave &&
1255
1415
  typeof metadataToSave === 'object' &&
1256
1416
  'noun' in metadataToSave) {
1257
- this.statisticsCollector.trackContentType(metadataToSave.noun);
1417
+ this.metrics.trackContentType(metadataToSave.noun);
1258
1418
  }
1259
- // Track update timestamp
1260
- this.statisticsCollector.trackUpdate();
1419
+ // Track update timestamp (handled by metrics augmentation)
1261
1420
  }
1262
1421
  }
1263
1422
  // Update HNSW index size with actual index size
1264
1423
  const indexSize = this.index.size();
1265
1424
  await this.storage.updateHnswIndexSize(indexSize);
1266
1425
  // Update health metrics if in distributed mode
1267
- if (this.healthMonitor) {
1426
+ if (this.monitoring) {
1268
1427
  const vectorCount = await this.getNounCount();
1269
- this.healthMonitor.updateVectorCount(vectorCount);
1428
+ this.monitoring.updateVectorCount(vectorCount);
1270
1429
  }
1271
1430
  // If addToRemote is true and we're connected to a remote server, add to remote as well
1272
1431
  if (options.addToRemote && this.isConnectedToRemoteServer()) {
@@ -1278,7 +1437,7 @@ export class BrainyData {
1278
1437
  }
1279
1438
  }
1280
1439
  // Invalidate search cache since data has changed
1281
- this.searchCache.invalidateOnDataChange('add');
1440
+ this.cache?.invalidateOnDataChange('add');
1282
1441
  // Determine processing mode
1283
1442
  const processingMode = options.process || 'auto';
1284
1443
  let shouldProcessNeurally = false;
@@ -1293,8 +1452,9 @@ export class BrainyData {
1293
1452
  // 🧠 AI Processing (Neural Import) - Based on processing mode
1294
1453
  if (shouldProcessNeurally) {
1295
1454
  try {
1296
- // Execute SENSE pipeline (includes Neural Import and other AI augmentations)
1297
- await augmentationPipeline.executeSensePipeline('processRawData', [vectorOrData, typeof vectorOrData === 'string' ? 'text' : 'data'], { mode: ExecutionMode.SEQUENTIAL });
1455
+ // Execute augmentation pipeline for data processing
1456
+ // Note: Augmentations will be called via this.augmentations.execute during the actual add operation
1457
+ // This replaces the legacy SENSE pipeline
1298
1458
  if (this.loggingConfig?.verbose) {
1299
1459
  console.log(`🧠 AI processing completed for data: ${id}`);
1300
1460
  }
@@ -1309,39 +1469,14 @@ export class BrainyData {
1309
1469
  catch (error) {
1310
1470
  console.error('Failed to add vector:', error);
1311
1471
  // Track error in health monitor
1312
- if (this.healthMonitor) {
1313
- this.healthMonitor.recordRequest(0, true);
1472
+ if (this.monitoring) {
1473
+ this.monitoring.recordRequest(0, true);
1314
1474
  }
1315
1475
  throw new Error(`Failed to add vector: ${error}`);
1316
1476
  }
1317
1477
  }
1318
- /**
1319
- * Add a text item to the database with automatic embedding
1320
- * This is a convenience method for adding text data with metadata
1321
- * @param text Text data to add
1322
- * @param metadata Metadata to associate with the text
1323
- * @param options Additional options
1324
- * @returns The ID of the added item
1325
- */
1326
- async addItem(text, metadata, options = {}) {
1327
- // Use the existing add method with forceEmbed to ensure text is embedded
1328
- return this.add(text, metadata, { ...options, forceEmbed: true });
1329
- }
1330
- /**
1331
- * Add data to both local and remote Brainy instances
1332
- * @param vectorOrData Vector or data to add
1333
- * @param metadata Optional metadata to associate with the vector
1334
- * @param options Additional options
1335
- * @returns The ID of the added vector
1336
- */
1337
- async addToBoth(vectorOrData, metadata, options = {}) {
1338
- // Check if connected to a remote server
1339
- if (!this.isConnectedToRemoteServer()) {
1340
- throw new Error('Not connected to a remote server. Call connectToRemoteServer() first.');
1341
- }
1342
- // Add to local with addToRemote option
1343
- return this.add(vectorOrData, metadata, { ...options, addToRemote: true });
1344
- }
1478
+ // REMOVED: addItem() - Use addNoun() instead (cleaner 2.0 API)
1479
+ // REMOVED: addToBoth() - Remote server functionality moved to post-2.0.0
1345
1480
  /**
1346
1481
  * Add a vector to the remote server
1347
1482
  * @param id ID of the vector to add
@@ -1355,14 +1490,23 @@ export class BrainyData {
1355
1490
  return false;
1356
1491
  }
1357
1492
  try {
1358
- if (!this.serverSearchConduit || !this.serverConnection) {
1359
- throw new Error('Server search conduit or connection is not initialized');
1360
- }
1361
- // Add to remote server
1362
- const addResult = await this.serverSearchConduit.addToBoth(this.serverConnection.connectionId, vector, metadata);
1363
- if (!addResult.success) {
1364
- throw new Error(`Remote add failed: ${addResult.error}`);
1365
- }
1493
+ // TODO: Remote server operations (post-2.0.0 feature)
1494
+ // if (!this.serverSearchConduit || !this.serverConnection) {
1495
+ // throw new Error(
1496
+ // 'Server search conduit or connection is not initialized'
1497
+ // )
1498
+ // }
1499
+ // TODO: Add to remote server
1500
+ // const addResult = await this.serverSearchConduit.addToBoth(
1501
+ // this.serverConnection.connectionId,
1502
+ // vector,
1503
+ // metadata
1504
+ // )
1505
+ throw new Error('Remote server functionality not yet implemented in Brainy 2.0.0');
1506
+ // TODO: Handle remote add result (post-2.0.0 feature)
1507
+ // if (!addResult.success) {
1508
+ // throw new Error(`Remote add failed: ${addResult.error}`)
1509
+ // }
1366
1510
  return true;
1367
1511
  }
1368
1512
  catch (error) {
@@ -1376,7 +1520,13 @@ export class BrainyData {
1376
1520
  * @param options Additional options
1377
1521
  * @returns Array of IDs for the added items
1378
1522
  */
1379
- async addBatch(items, options = {}) {
1523
+ /**
1524
+ * Add multiple nouns in batch
1525
+ * @param items Array of nouns to add
1526
+ * @param options Batch processing options
1527
+ * @returns Array of generated IDs
1528
+ */
1529
+ async addNouns(items, options = {}) {
1380
1530
  await this.ensureInitialized();
1381
1531
  // Check if database is in read-only mode
1382
1532
  this.checkReadOnly();
@@ -1426,7 +1576,7 @@ export class BrainyData {
1426
1576
  }
1427
1577
  });
1428
1578
  // Process vector items (already embedded)
1429
- const vectorPromises = vectorItems.map((item) => this.add(item.vectorOrData, item.metadata, options));
1579
+ const vectorPromises = vectorItems.map((item) => this.addNoun(item.vectorOrData, item.metadata));
1430
1580
  // Process text items in a single batch embedding operation
1431
1581
  let textPromises = [];
1432
1582
  if (textItems.length > 0) {
@@ -1435,10 +1585,7 @@ export class BrainyData {
1435
1585
  // Perform batch embedding
1436
1586
  const embeddings = await batchEmbed(texts);
1437
1587
  // Add each item with its embedding
1438
- textPromises = textItems.map((item, i) => this.add(embeddings[i], item.metadata, {
1439
- ...options,
1440
- forceEmbed: false
1441
- }));
1588
+ textPromises = textItems.map((item, i) => this.addNoun(embeddings[i], item.metadata));
1442
1589
  }
1443
1590
  // Combine all promises
1444
1591
  const batchResults = await Promise.all([
@@ -1467,7 +1614,7 @@ export class BrainyData {
1467
1614
  throw new Error('Not connected to a remote server. Call connectToRemoteServer() first.');
1468
1615
  }
1469
1616
  // Add to local with addToRemote option
1470
- return this.addBatch(items, { ...options, addToRemote: true });
1617
+ return this.addNouns(items, { ...options, addToRemote: true });
1471
1618
  }
1472
1619
  /**
1473
1620
  * Filter search results by service
@@ -1498,6 +1645,14 @@ export class BrainyData {
1498
1645
  * @param options Additional options
1499
1646
  * @returns Array of search results
1500
1647
  */
1648
+ /**
1649
+ * @deprecated Use search() with nounTypes option instead
1650
+ * @example
1651
+ * // Old way (deprecated)
1652
+ * await brain.searchByNounTypes(query, 10, ['type1', 'type2'])
1653
+ * // New way
1654
+ * await brain.search(query, { limit: 10, nounTypes: ['type1', 'type2'] })
1655
+ */
1501
1656
  async searchByNounTypes(queryVectorOrData, k = 10, nounTypes = null, options = {}) {
1502
1657
  // Helper function to filter results by service
1503
1658
  const filterByService = (metadata) => {
@@ -1585,9 +1740,9 @@ export class BrainyData {
1585
1740
  if (hasMetadataFilter && this.metadataIndex) {
1586
1741
  try {
1587
1742
  // Ensure metadata index is up to date
1588
- await this.metadataIndex.flush();
1743
+ await this.metadataIndex?.flush?.();
1589
1744
  // Get candidate IDs from metadata index
1590
- const candidateIds = await this.metadataIndex.getIdsForFilter(options.metadata);
1745
+ const candidateIds = await this.metadataIndex?.getIdsForFilter?.(options.metadata) || [];
1591
1746
  if (candidateIds.length > 0) {
1592
1747
  preFilteredIds = new Set(candidateIds);
1593
1748
  // Create a simple filter function that just checks the pre-filtered set
@@ -1663,13 +1818,11 @@ export class BrainyData {
1663
1818
  if (metadata === null) {
1664
1819
  metadata = {};
1665
1820
  }
1666
- // Ensure metadata has the id field
1667
- if (metadata && typeof metadata === 'object') {
1668
- metadata = { ...metadata, id };
1669
- }
1821
+ // Preserve original metadata without overwriting user's custom fields
1822
+ // The search result already has Brainy's UUID in the main 'id' field
1670
1823
  searchResults.push({
1671
1824
  id,
1672
- score,
1825
+ score: 1 - score, // Convert distance to similarity (higher = more similar)
1673
1826
  vector: noun.vector,
1674
1827
  metadata: metadata
1675
1828
  });
@@ -1708,13 +1861,11 @@ export class BrainyData {
1708
1861
  if (metadata === null) {
1709
1862
  metadata = {};
1710
1863
  }
1711
- // Ensure metadata has the id field
1712
- if (metadata && typeof metadata === 'object') {
1713
- metadata = { ...metadata, id };
1714
- }
1864
+ // Preserve original metadata without overwriting user's custom fields
1865
+ // The search result already has Brainy's UUID in the main 'id' field
1715
1866
  searchResults.push({
1716
1867
  id,
1717
- score,
1868
+ score: 1 - score, // Convert distance to similarity (higher = more similar)
1718
1869
  vector: noun.vector,
1719
1870
  metadata: metadata
1720
1871
  });
@@ -1735,7 +1886,133 @@ export class BrainyData {
1735
1886
  * @param options Additional options
1736
1887
  * @returns Array of search results
1737
1888
  */
1738
- async search(queryVectorOrData, k = 10, options = {}) {
1889
+ /**
1890
+ * 🔍 SIMPLE VECTOR SEARCH - Clean wrapper around find() for pure vector search
1891
+ *
1892
+ * @param queryVectorOrData Vector or text to search for
1893
+ * @param k Number of results to return
1894
+ * @param options Simple search options (metadata filters only)
1895
+ * @returns Vector search results
1896
+ */
1897
+ /**
1898
+ * 🔍 Simple Vector Similarity Search - Clean wrapper around find()
1899
+ *
1900
+ * search(query) = find({like: query}) - Pure vector similarity search
1901
+ *
1902
+ * @param queryVectorOrData - Query string, vector, or object to search with
1903
+ * @param options - Search options for filtering and pagination
1904
+ * @returns Array of search results with scores and metadata
1905
+ *
1906
+ * @example
1907
+ * // Simple vector search
1908
+ * await brain.search('machine learning')
1909
+ *
1910
+ * // With filters and pagination
1911
+ * await brain.search('AI', {
1912
+ * limit: 20,
1913
+ * metadata: { type: 'article' },
1914
+ * nounTypes: ['document']
1915
+ * })
1916
+ */
1917
+ async search(queryVectorOrData, options = {}) {
1918
+ // Build metadata filter from options
1919
+ const metadataFilter = { ...options.metadata };
1920
+ // Add noun type filtering
1921
+ if (options.nounTypes && options.nounTypes.length > 0) {
1922
+ metadataFilter.nounType = { in: options.nounTypes };
1923
+ }
1924
+ // Add item ID filtering
1925
+ if (options.itemIds && options.itemIds.length > 0) {
1926
+ metadataFilter.id = { in: options.itemIds };
1927
+ }
1928
+ // Build simple TripleQuery for vector similarity
1929
+ const tripleQuery = {
1930
+ like: queryVectorOrData
1931
+ };
1932
+ // Add metadata filter if we have conditions
1933
+ if (Object.keys(metadataFilter).length > 0) {
1934
+ tripleQuery.where = metadataFilter;
1935
+ }
1936
+ // Extract find() options
1937
+ const findOptions = {
1938
+ limit: options.limit,
1939
+ offset: options.offset,
1940
+ cursor: options.cursor,
1941
+ excludeDeleted: options.excludeDeleted,
1942
+ timeout: options.timeout
1943
+ };
1944
+ // Call find() with structured query - this is the key simplification!
1945
+ let results = await this.find(tripleQuery, findOptions);
1946
+ // Apply threshold filtering if specified
1947
+ if (options.threshold !== undefined) {
1948
+ results = results.filter(r => (r.fusionScore || r.score || 0) >= options.threshold);
1949
+ }
1950
+ // Convert to SearchResult format
1951
+ return results.map(r => ({
1952
+ ...r,
1953
+ score: r.fusionScore || r.score || 0
1954
+ }));
1955
+ return results;
1956
+ }
1957
+ /**
1958
+ * Helper method to encode cursor for pagination
1959
+ * @internal
1960
+ */
1961
+ encodeCursor(data) {
1962
+ return Buffer.from(JSON.stringify(data)).toString('base64');
1963
+ }
1964
+ /**
1965
+ * Helper method to decode cursor for pagination
1966
+ * @internal
1967
+ */
1968
+ decodeCursor(cursor) {
1969
+ try {
1970
+ return JSON.parse(Buffer.from(cursor, 'base64').toString());
1971
+ }
1972
+ catch {
1973
+ return { offset: 0, timestamp: 0 };
1974
+ }
1975
+ }
1976
+ /**
1977
+ * Internal method for direct HNSW vector search
1978
+ * Used by TripleIntelligence to avoid circular dependencies
1979
+ * Note: For pure metadata filtering, use metadataIndex.getIdsForFilter() directly - it's O(log n)!
1980
+ * This method is for vector similarity search with optional metadata filtering during search
1981
+ * @internal
1982
+ */
1983
+ async _internalVectorSearch(queryVectorOrData, k = 10, options = {}) {
1984
+ // Generate query vector
1985
+ const queryVector = Array.isArray(queryVectorOrData) &&
1986
+ typeof queryVectorOrData[0] === 'number' ?
1987
+ queryVectorOrData :
1988
+ await this.embed(queryVectorOrData);
1989
+ // Apply metadata filter if provided
1990
+ let filterFunction;
1991
+ if (options.metadata) {
1992
+ const matchingIdsArray = await this.metadataIndex?.getIdsForFilter(options.metadata) || [];
1993
+ const matchingIds = new Set(matchingIdsArray);
1994
+ filterFunction = async (id) => matchingIds.has(id);
1995
+ }
1996
+ // Direct HNSW search
1997
+ const results = await this.index.search(queryVector, k, filterFunction);
1998
+ // Get metadata for results
1999
+ const searchResults = [];
2000
+ for (const [id, similarity] of results) {
2001
+ const metadata = await this.getNoun(id);
2002
+ searchResults.push({
2003
+ id,
2004
+ score: similarity,
2005
+ vector: [],
2006
+ metadata: metadata?.metadata || {}
2007
+ });
2008
+ }
2009
+ return searchResults;
2010
+ }
2011
+ /**
2012
+ * 🎯 LEGACY: Original search implementation (kept for complex cases)
2013
+ * This is the original search method, now used as fallback for edge cases
2014
+ */
2015
+ async _legacySearch(queryVectorOrData, k = 10, options = {}) {
1739
2016
  const startTime = Date.now();
1740
2017
  // Validate input is not null or undefined
1741
2018
  if (queryVectorOrData === null || queryVectorOrData === undefined) {
@@ -1787,63 +2064,68 @@ export class BrainyData {
1787
2064
  else if (options.searchMode === 'combined') {
1788
2065
  return this.searchCombined(queryVectorOrData, k, options);
1789
2066
  }
1790
- // Default behavior (backward compatible): search locally
1791
- try {
1792
- // BEST OF BOTH: Automatically exclude soft-deleted items (Neural Intelligence improvement)
1793
- // BUT only when there's already metadata filtering happening
1794
- let metadataFilter = options.metadata;
1795
- // Only add soft-delete filter if there's already metadata being filtered
1796
- // This preserves pure vector searches without metadata
1797
- if (metadataFilter && Object.keys(metadataFilter).length > 0) {
1798
- // If no explicit deleted filter is provided, exclude soft-deleted items
1799
- if (!metadataFilter.deleted && !metadataFilter.$or) {
1800
- metadataFilter = {
1801
- ...metadataFilter,
1802
- deleted: { $ne: true }
1803
- };
2067
+ // Generate deduplication key for concurrent request handling
2068
+ const dedupeKey = RequestDeduplicator.getSearchKey(typeof queryVectorOrData === 'string' ? queryVectorOrData : JSON.stringify(queryVectorOrData), k, options);
2069
+ // Use augmentation system for search (includes deduplication, batching, and caching)
2070
+ return this.augmentations.execute('search', { query: queryVectorOrData, k, options, dedupeKey }, async () => {
2071
+ // Default behavior (backward compatible): search locally
2072
+ try {
2073
+ // BEST OF BOTH: Automatically exclude soft-deleted items (Neural Intelligence improvement)
2074
+ // BUT only when there's already metadata filtering happening
2075
+ let metadataFilter = options.metadata;
2076
+ // Only add soft-delete filter if there's already metadata being filtered
2077
+ // This preserves pure vector searches without metadata
2078
+ if (metadataFilter && Object.keys(metadataFilter).length > 0) {
2079
+ // If no explicit deleted filter is provided, exclude soft-deleted items
2080
+ if (!metadataFilter.deleted && !metadataFilter.anyOf) {
2081
+ metadataFilter = {
2082
+ ...metadataFilter,
2083
+ deleted: { notEquals: true }
2084
+ };
2085
+ }
1804
2086
  }
1805
- }
1806
- const hasMetadataFilter = metadataFilter && Object.keys(metadataFilter).length > 0;
1807
- // Check cache first (transparent to user) - but skip cache if we have metadata filters
1808
- if (!hasMetadataFilter) {
1809
- const cacheKey = this.searchCache.getCacheKey(queryVectorOrData, k, options);
1810
- const cachedResults = this.searchCache.get(cacheKey);
1811
- if (cachedResults) {
1812
- // Track cache hit in health monitor
1813
- if (this.healthMonitor) {
1814
- const latency = Date.now() - startTime;
1815
- this.healthMonitor.recordRequest(latency, false);
1816
- this.healthMonitor.recordCacheAccess(true);
2087
+ const hasMetadataFilter = metadataFilter && Object.keys(metadataFilter).length > 0;
2088
+ // Check cache first (transparent to user) - but skip cache if we have metadata filters
2089
+ if (!hasMetadataFilter) {
2090
+ const cacheKey = this.cache?.getCacheKey(queryVectorOrData, k, options);
2091
+ const cachedResults = this.cache?.get(cacheKey);
2092
+ if (cachedResults) {
2093
+ // Track cache hit in health monitor
2094
+ if (this.monitoring) {
2095
+ const latency = Date.now() - startTime;
2096
+ this.monitoring.recordRequest(latency, false);
2097
+ this.monitoring.recordCacheAccess(true);
2098
+ }
2099
+ return cachedResults;
1817
2100
  }
1818
- return cachedResults;
1819
2101
  }
2102
+ // Cache miss - perform actual search
2103
+ const results = await this.searchLocal(queryVectorOrData, k, {
2104
+ ...options,
2105
+ metadata: metadataFilter
2106
+ });
2107
+ // Cache results for future queries (unless explicitly disabled or has metadata filter)
2108
+ if (!options.skipCache && !hasMetadataFilter) {
2109
+ const cacheKey = this.cache?.getCacheKey(queryVectorOrData, k, options);
2110
+ this.cache?.set(cacheKey, results);
2111
+ }
2112
+ // Track successful search in health monitor
2113
+ if (this.monitoring) {
2114
+ const latency = Date.now() - startTime;
2115
+ this.monitoring.recordRequest(latency, false);
2116
+ this.monitoring.recordCacheAccess(false);
2117
+ }
2118
+ return results;
1820
2119
  }
1821
- // Cache miss - perform actual search
1822
- const results = await this.searchLocal(queryVectorOrData, k, {
1823
- ...options,
1824
- metadata: metadataFilter
1825
- });
1826
- // Cache results for future queries (unless explicitly disabled or has metadata filter)
1827
- if (!options.skipCache && !hasMetadataFilter) {
1828
- const cacheKey = this.searchCache.getCacheKey(queryVectorOrData, k, options);
1829
- this.searchCache.set(cacheKey, results);
1830
- }
1831
- // Track successful search in health monitor
1832
- if (this.healthMonitor) {
1833
- const latency = Date.now() - startTime;
1834
- this.healthMonitor.recordRequest(latency, false);
1835
- this.healthMonitor.recordCacheAccess(false);
1836
- }
1837
- return results;
1838
- }
1839
- catch (error) {
1840
- // Track error in health monitor
1841
- if (this.healthMonitor) {
1842
- const latency = Date.now() - startTime;
1843
- this.healthMonitor.recordRequest(latency, true);
2120
+ catch (error) {
2121
+ // Track error in health monitor
2122
+ if (this.monitoring) {
2123
+ const latency = Date.now() - startTime;
2124
+ this.monitoring.recordRequest(latency, true);
2125
+ }
2126
+ throw error;
1844
2127
  }
1845
- throw error;
1846
- }
2128
+ });
1847
2129
  }
1848
2130
  /**
1849
2131
  * Search with cursor-based pagination for better performance on large datasets
@@ -1852,13 +2134,23 @@ export class BrainyData {
1852
2134
  * @param options Additional options including cursor for pagination
1853
2135
  * @returns Paginated search results with cursor for next page
1854
2136
  */
2137
+ /**
2138
+ * @deprecated Use search() with cursor option instead
2139
+ * @example
2140
+ * // Old way (deprecated)
2141
+ * await brain.searchWithCursor(query, 10, { cursor: 'abc123' })
2142
+ * // New way
2143
+ * await brain.search(query, { limit: 10, cursor: 'abc123' })
2144
+ */
1855
2145
  async searchWithCursor(queryVectorOrData, k = 10, options = {}) {
1856
2146
  // For cursor-based search, we need to fetch more results and filter
1857
2147
  const searchK = options.cursor ? k + 20 : k; // Get extra results for filtering
1858
2148
  // Perform regular search
1859
- const allResults = await this.search(queryVectorOrData, searchK, {
1860
- ...options,
1861
- skipCache: options.skipCache
2149
+ const { cursor, ...searchOptions } = options;
2150
+ const allResults = await this.search(queryVectorOrData, {
2151
+ limit: searchK,
2152
+ nounTypes: searchOptions.nounTypes,
2153
+ metadata: searchOptions.filter
1862
2154
  });
1863
2155
  let results = allResults;
1864
2156
  let startIndex = 0;
@@ -2022,7 +2314,7 @@ export class BrainyData {
2022
2314
  async findSimilar(id, options = {}) {
2023
2315
  await this.ensureInitialized();
2024
2316
  // Get the entity by ID
2025
- const entity = await this.get(id);
2317
+ const entity = await this.getNoun(id);
2026
2318
  if (!entity) {
2027
2319
  throw new Error(`Entity with ID ${id} not found`);
2028
2320
  }
@@ -2040,7 +2332,7 @@ export class BrainyData {
2040
2332
  // Skip undefined targetIds
2041
2333
  if (typeof targetId !== 'string')
2042
2334
  continue;
2043
- const targetEntity = await this.get(targetId);
2335
+ const targetEntity = await this.getNoun(targetId);
2044
2336
  if (targetEntity) {
2045
2337
  results.push({
2046
2338
  id: targetId,
@@ -2055,11 +2347,10 @@ export class BrainyData {
2055
2347
  }
2056
2348
  // If no relationType is specified, use the original vector similarity search
2057
2349
  const k = (options.limit || 10) + 1; // Add 1 to account for the original entity
2058
- const searchResults = await this.search(entity.vector, k, {
2059
- forceEmbed: false,
2060
- nounTypes: options.nounTypes,
2061
- includeVerbs: options.includeVerbs,
2062
- searchMode: options.searchMode
2350
+ const searchResults = await this.search(entity.vector, {
2351
+ limit: k,
2352
+ excludeDeleted: false,
2353
+ nounTypes: options.nounTypes
2063
2354
  });
2064
2355
  // Filter out the original entity and limit to the requested number
2065
2356
  return searchResults
@@ -2069,69 +2360,7 @@ export class BrainyData {
2069
2360
  /**
2070
2361
  * Get a vector by ID
2071
2362
  */
2072
- async get(id) {
2073
- // Validate id parameter first, before any other logic
2074
- if (id === null || id === undefined) {
2075
- throw new Error('ID cannot be null or undefined');
2076
- }
2077
- await this.ensureInitialized();
2078
- try {
2079
- let noun;
2080
- // In write-only mode, query storage directly since index is not loaded
2081
- if (this.writeOnly) {
2082
- try {
2083
- noun = (await this.storage.getNoun(id)) ?? undefined;
2084
- }
2085
- catch (storageError) {
2086
- // If storage lookup fails, return null (noun doesn't exist)
2087
- return null;
2088
- }
2089
- }
2090
- else {
2091
- // Normal mode: Get noun from index first
2092
- noun = this.index.getNouns().get(id);
2093
- // If not found in index, fallback to storage (for race conditions)
2094
- if (!noun && this.storage) {
2095
- try {
2096
- noun = (await this.storage.getNoun(id)) ?? undefined;
2097
- }
2098
- catch (storageError) {
2099
- // Storage lookup failed, noun doesn't exist
2100
- return null;
2101
- }
2102
- }
2103
- }
2104
- if (!noun) {
2105
- return null;
2106
- }
2107
- // Get metadata
2108
- let metadata = await this.storage.getMetadata(id);
2109
- // Handle special cases for metadata
2110
- if (metadata === null) {
2111
- metadata = {};
2112
- }
2113
- else if (typeof metadata === 'object') {
2114
- // For empty metadata test: if metadata only has an ID, return empty object
2115
- if (Object.keys(metadata).length === 1 && 'id' in metadata) {
2116
- metadata = {};
2117
- }
2118
- // Always remove the ID from metadata if present
2119
- else if ('id' in metadata) {
2120
- const { id: _, ...rest } = metadata;
2121
- metadata = rest;
2122
- }
2123
- }
2124
- return {
2125
- id,
2126
- vector: noun.vector,
2127
- metadata: metadata
2128
- };
2129
- }
2130
- catch (error) {
2131
- console.error(`Failed to get vector ${id}:`, error);
2132
- throw new Error(`Failed to get vector ${id}: ${error}`);
2133
- }
2134
- }
2363
+ // Legacy get() method removed - use getNoun() instead
2135
2364
  /**
2136
2365
  * Check if a document with the given ID exists
2137
2366
  * This is a direct storage operation that works in write-only mode when allowDirectReads is enabled
@@ -2163,8 +2392,13 @@ export class BrainyData {
2163
2392
  * @param id The ID to check for existence
2164
2393
  * @returns Promise<boolean> True if the document exists, false otherwise
2165
2394
  */
2166
- async exists(id) {
2167
- return this.has(id);
2395
+ /**
2396
+ * Check if a noun exists
2397
+ * @param id The noun ID
2398
+ * @returns True if exists
2399
+ */
2400
+ async hasNoun(id) {
2401
+ return this.hasNoun(id);
2168
2402
  }
2169
2403
  /**
2170
2404
  * Get metadata for a document by ID
@@ -2172,31 +2406,53 @@ export class BrainyData {
2172
2406
  * @param id The ID of the document
2173
2407
  * @returns Promise<T | null> The metadata object or null if not found
2174
2408
  */
2175
- async getMetadata(id) {
2176
- if (id === null || id === undefined) {
2177
- throw new Error('ID cannot be null or undefined');
2178
- }
2179
- await this.ensureInitialized();
2180
- // This is a direct storage operation - check if allowed in write-only mode
2181
- if (this.writeOnly && !this.allowDirectReads) {
2182
- throw new Error('Cannot perform getMetadata() operation: database is in write-only mode. Enable allowDirectReads for direct storage operations.');
2183
- }
2184
- try {
2185
- const metadata = await this.storage.getMetadata(id);
2186
- return metadata;
2187
- }
2188
- catch (error) {
2189
- console.error(`Failed to get metadata for ${id}:`, error);
2190
- return null;
2191
- }
2192
- }
2409
+ // Legacy getMetadata() method removed - use getNounMetadata() instead
2193
2410
  /**
2194
2411
  * Get multiple documents by their IDs
2195
2412
  * This is a direct storage operation that works in write-only mode when allowDirectReads is enabled
2196
2413
  * @param ids Array of IDs to retrieve
2197
2414
  * @returns Promise<Array<VectorDocument<T> | null>> Array of documents (null for missing IDs)
2198
2415
  */
2199
- async getBatch(ids) {
2416
+ /**
2417
+ * Get multiple nouns - by IDs, filters, or pagination
2418
+ * @param idsOrOptions Array of IDs or query options
2419
+ * @returns Array of noun documents
2420
+ *
2421
+ * @example
2422
+ * // Get by IDs
2423
+ * await brain.getNouns(['id1', 'id2'])
2424
+ *
2425
+ * // Get with filters
2426
+ * await brain.getNouns({
2427
+ * filter: { type: 'article' },
2428
+ * limit: 10
2429
+ * })
2430
+ *
2431
+ * // Get with pagination
2432
+ * await brain.getNouns({
2433
+ * offset: 20,
2434
+ * limit: 10
2435
+ * })
2436
+ */
2437
+ async getNouns(idsOrOptions) {
2438
+ // Handle array of IDs
2439
+ if (Array.isArray(idsOrOptions)) {
2440
+ return this.getNounsByIds(idsOrOptions);
2441
+ }
2442
+ // Handle options object
2443
+ const options = idsOrOptions || {};
2444
+ // If ids are provided in options, get by IDs
2445
+ if (options.ids) {
2446
+ return this.getNounsByIds(options.ids);
2447
+ }
2448
+ // Otherwise, do a filtered/paginated query and extract items
2449
+ const result = await this.queryNounsByFilter(options);
2450
+ return result.items;
2451
+ }
2452
+ /**
2453
+ * Internal: Get nouns by IDs
2454
+ */
2455
+ async getNounsByIds(ids) {
2200
2456
  if (!Array.isArray(ids)) {
2201
2457
  throw new Error('IDs must be provided as an array');
2202
2458
  }
@@ -2212,7 +2468,7 @@ export class BrainyData {
2212
2468
  continue;
2213
2469
  }
2214
2470
  try {
2215
- const result = await this.get(id);
2471
+ const result = await this.getNoun(id);
2216
2472
  results.push(result);
2217
2473
  }
2218
2474
  catch (error) {
@@ -2229,7 +2485,10 @@ export class BrainyData {
2229
2485
  * @param options Pagination and filtering options
2230
2486
  * @returns Paginated result of vector documents
2231
2487
  */
2232
- async getNouns(options = {}) {
2488
+ /**
2489
+ * Internal: Query nouns with filtering and pagination
2490
+ */
2491
+ async queryNounsByFilter(options = {}) {
2233
2492
  await this.ensureInitialized();
2234
2493
  try {
2235
2494
  // First try to use the storage adapter's paginated method
@@ -2324,223 +2583,11 @@ export class BrainyData {
2324
2583
  throw new Error(`Failed to get nouns with pagination: ${error}`);
2325
2584
  }
2326
2585
  }
2327
- /**
2328
- * Delete a vector by ID
2329
- * @param id The ID of the vector to delete
2330
- * @param options Additional options
2331
- * @returns Promise that resolves to true if the vector was deleted, false otherwise
2332
- */
2333
- async delete(id, options = {}) {
2334
- // Clear API: use 'hard: true' for hard delete, otherwise soft delete
2335
- const isHardDelete = options.hard === true;
2336
- const opts = {
2337
- service: options.service,
2338
- soft: !isHardDelete, // Soft delete is default unless hard: true is specified
2339
- cascade: options.cascade || false,
2340
- force: options.force || false
2341
- };
2342
- // Validate id parameter first, before any other logic
2343
- if (id === null || id === undefined) {
2344
- throw new Error('ID cannot be null or undefined');
2345
- }
2346
- await this.ensureInitialized();
2347
- // Check if database is in read-only mode
2348
- this.checkReadOnly();
2349
- try {
2350
- // Check if the id is actually content text rather than an ID
2351
- // This handles cases where tests or users pass content text instead of IDs
2352
- let actualId = id;
2353
- console.log(`Delete called with ID: ${id}`);
2354
- console.log(`Index has ID directly: ${this.index.getNouns().has(id)}`);
2355
- if (!this.index.getNouns().has(id)) {
2356
- console.log(`Looking for noun with text content: ${id}`);
2357
- // Try to find a noun with matching text content
2358
- for (const [nounId, noun] of this.index.getNouns().entries()) {
2359
- console.log(`Checking noun ${nounId}: text=${noun.metadata?.text || 'undefined'}`);
2360
- if (noun.metadata?.text === id) {
2361
- actualId = nounId;
2362
- console.log(`Found matching noun with ID: ${actualId}`);
2363
- break;
2364
- }
2365
- }
2366
- }
2367
- // Handle soft delete vs hard delete
2368
- if (opts.soft) {
2369
- // Soft delete: just mark as deleted - metadata filter will exclude from search
2370
- try {
2371
- return await this.updateMetadata(actualId, {
2372
- deleted: true,
2373
- deletedAt: new Date().toISOString(),
2374
- deletedBy: opts.service || 'user'
2375
- });
2376
- }
2377
- catch (error) {
2378
- // If item doesn't exist, return false (delete of non-existent item is not an error)
2379
- return false;
2380
- }
2381
- }
2382
- // Hard delete: Remove from index
2383
- const removed = this.index.removeItem(actualId);
2384
- if (!removed) {
2385
- return false;
2386
- }
2387
- // Remove from storage
2388
- await this.storage.deleteNoun(actualId);
2389
- // Track deletion statistics
2390
- const service = this.getServiceName({ service: opts.service });
2391
- await this.storage.decrementStatistic('noun', service);
2392
- // Try to remove metadata (ignore errors)
2393
- try {
2394
- // Get metadata before removing for index cleanup
2395
- const existingMetadata = await this.storage.getMetadata(actualId);
2396
- // Remove from metadata index (write-only mode should update indices!)
2397
- if (this.metadataIndex && existingMetadata && !this.frozen) {
2398
- await this.metadataIndex.removeFromIndex(actualId, existingMetadata);
2399
- }
2400
- await this.storage.saveMetadata(actualId, null);
2401
- await this.storage.decrementStatistic('metadata', service);
2402
- }
2403
- catch (error) {
2404
- // Ignore
2405
- }
2406
- // Invalidate search cache since data has changed
2407
- this.searchCache.invalidateOnDataChange('delete');
2408
- return true;
2409
- }
2410
- catch (error) {
2411
- console.error(`Failed to delete vector ${id}:`, error);
2412
- throw new Error(`Failed to delete vector ${id}: ${error}`);
2413
- }
2414
- }
2415
- /**
2416
- * Update metadata for a vector
2417
- * @param id The ID of the vector to update metadata for
2418
- * @param metadata The new metadata
2419
- * @param options Additional options
2420
- * @returns Promise that resolves to true if the metadata was updated, false otherwise
2421
- */
2422
- async updateMetadata(id, metadata, options = {}) {
2423
- // Validate id parameter first, before any other logic
2424
- if (id === null || id === undefined) {
2425
- throw new Error('ID cannot be null or undefined');
2426
- }
2427
- // Validate that metadata is not null or undefined
2428
- if (metadata === null || metadata === undefined) {
2429
- throw new Error(`Metadata cannot be null or undefined`);
2430
- }
2431
- await this.ensureInitialized();
2432
- // Check if database is in read-only mode
2433
- this.checkReadOnly();
2434
- try {
2435
- // Check if a vector exists
2436
- const noun = this.index.getNouns().get(id);
2437
- if (!noun) {
2438
- throw new Error(`Vector with ID ${id} does not exist`);
2439
- }
2440
- // Validate noun type if metadata is for a GraphNoun
2441
- if (metadata && typeof metadata === 'object' && 'noun' in metadata) {
2442
- const nounType = metadata.noun;
2443
- // Check if the noun type is valid
2444
- const isValidNounType = Object.values(NounType).includes(nounType);
2445
- if (!isValidNounType) {
2446
- console.warn(`Invalid noun type: ${nounType}. Falling back to GraphNoun.`);
2447
- metadata.noun = NounType.Concept;
2448
- }
2449
- // Get the service that's updating the metadata
2450
- const service = this.getServiceName(options);
2451
- const graphNoun = metadata;
2452
- // Preserve existing createdBy and createdAt if they exist
2453
- const existingMetadata = (await this.storage.getMetadata(id));
2454
- if (existingMetadata &&
2455
- typeof existingMetadata === 'object' &&
2456
- 'createdBy' in existingMetadata) {
2457
- // Preserve the original creator information
2458
- graphNoun.createdBy = existingMetadata.createdBy;
2459
- // Also preserve creation timestamp if it exists
2460
- if ('createdAt' in existingMetadata) {
2461
- graphNoun.createdAt = existingMetadata.createdAt;
2462
- }
2463
- }
2464
- else if (!graphNoun.createdBy) {
2465
- // If no existing createdBy and none in the update, set it
2466
- graphNoun.createdBy = getAugmentationVersion(service);
2467
- // Set createdAt if it doesn't exist
2468
- if (!graphNoun.createdAt) {
2469
- const now = new Date();
2470
- graphNoun.createdAt = {
2471
- seconds: Math.floor(now.getTime() / 1000),
2472
- nanoseconds: (now.getTime() % 1000) * 1000000
2473
- };
2474
- }
2475
- }
2476
- // Always update the updatedAt timestamp
2477
- const now = new Date();
2478
- graphNoun.updatedAt = {
2479
- seconds: Math.floor(now.getTime() / 1000),
2480
- nanoseconds: (now.getTime() % 1000) * 1000000
2481
- };
2482
- }
2483
- // Update metadata
2484
- await this.storage.saveMetadata(id, metadata);
2485
- // Update metadata index (write-only mode should build indices!)
2486
- if (this.metadataIndex && !this.frozen) {
2487
- // Remove old metadata from index if it exists
2488
- const oldMetadata = await this.storage.getMetadata(id);
2489
- if (oldMetadata) {
2490
- await this.metadataIndex.removeFromIndex(id, oldMetadata);
2491
- }
2492
- // Add new metadata to index
2493
- if (metadata) {
2494
- await this.metadataIndex.addToIndex(id, metadata);
2495
- }
2496
- }
2497
- // Track metadata statistics
2498
- const service = this.getServiceName(options);
2499
- await this.storage.incrementStatistic('metadata', service);
2500
- // Invalidate search cache since metadata has changed
2501
- this.searchCache.invalidateOnDataChange('update');
2502
- return true;
2503
- }
2504
- catch (error) {
2505
- console.error(`Failed to update metadata for vector ${id}:`, error);
2506
- throw new Error(`Failed to update metadata for vector ${id}: ${error}`);
2507
- }
2508
- }
2509
- /**
2510
- * Create a relationship between two entities
2511
- * This is a convenience wrapper around addVerb
2512
- */
2513
- async relate(sourceId, targetId, relationType, metadata) {
2514
- // Validate inputs are not null or undefined
2515
- if (sourceId === null || sourceId === undefined) {
2516
- throw new Error('Source ID cannot be null or undefined');
2517
- }
2518
- if (targetId === null || targetId === undefined) {
2519
- throw new Error('Target ID cannot be null or undefined');
2520
- }
2521
- if (relationType === null || relationType === undefined) {
2522
- throw new Error('Relation type cannot be null or undefined');
2523
- }
2524
- // NEURAL INTELLIGENCE: Enhanced metadata with smart inference
2525
- const enhancedMetadata = {
2526
- ...metadata,
2527
- createdAt: new Date().toISOString(),
2528
- inferenceScore: 1.0, // Could be enhanced with ML-based confidence scoring
2529
- relationType: relationType,
2530
- neuralEnhanced: true
2531
- };
2532
- return this._addVerbInternal(sourceId, targetId, undefined, {
2533
- type: relationType,
2534
- metadata: enhancedMetadata
2535
- });
2536
- }
2537
- /**
2538
- * Create a connection between two entities
2539
- * This is an alias for relate() for backward compatibility
2540
- */
2541
- async connect(sourceId, targetId, relationType, metadata) {
2542
- return this.relate(sourceId, targetId, relationType, metadata);
2543
- }
2586
+ // Legacy private methods removed - use public 2.0 API methods instead:
2587
+ // - delete() removed - use deleteNoun() instead
2588
+ // - updateMetadata() removed - use updateNoun() or updateNounMetadata() instead
2589
+ // REMOVED: relate() - Use addVerb() instead (cleaner 2.0 API)
2590
+ // REMOVED: connect() - Use addVerb() instead (cleaner 2.0 API)
2544
2591
  /**
2545
2592
  * Add a verb between two nouns
2546
2593
  * If metadata is provided and vector is not, the metadata will be vectorized using the embedding function
@@ -2691,8 +2738,8 @@ export class BrainyData {
2691
2738
  noun: NounType.Concept,
2692
2739
  createdBy: getAugmentationVersion(service)
2693
2740
  };
2694
- // Add the missing noun
2695
- await this.add(placeholderVector, metadata, { id: sourceId });
2741
+ // Add the missing noun (custom ID not supported in 2.0 addNoun yet)
2742
+ await this.addNoun(placeholderVector, metadata);
2696
2743
  // Get the newly created noun
2697
2744
  sourceNoun = this.index.getNouns().get(sourceId);
2698
2745
  console.warn(`Auto-created missing source noun with ID ${sourceId}`);
@@ -2720,8 +2767,8 @@ export class BrainyData {
2720
2767
  noun: NounType.Concept,
2721
2768
  createdBy: getAugmentationVersion(service)
2722
2769
  };
2723
- // Add the missing noun
2724
- await this.add(placeholderVector, metadata, { id: targetId });
2770
+ // Add the missing noun (custom ID not supported in 2.0 addNoun yet)
2771
+ await this.addNoun(placeholderVector, metadata);
2725
2772
  // Get the newly created noun
2726
2773
  targetNoun = this.index.getNouns().get(targetId);
2727
2774
  console.warn(`Auto-created missing target noun with ID ${targetId}`);
@@ -2811,7 +2858,10 @@ export class BrainyData {
2811
2858
  let scoringReasoning = [];
2812
2859
  if (this.intelligentVerbScoring?.enabled && (!options.weight || options.weight === 0.5)) {
2813
2860
  try {
2814
- const scores = await this.intelligentVerbScoring.computeVerbScores(sourceId, targetId, verbType, options.weight, options.metadata);
2861
+ // Get the source and target nouns for semantic scoring
2862
+ const sourceNoun = await this.storage?.getNoun(sourceId);
2863
+ const targetNoun = await this.storage?.getNoun(targetId);
2864
+ const scores = await this.intelligentVerbScoring.computeVerbScores(sourceNoun, targetNoun, verbType);
2815
2865
  finalWeight = scores.weight;
2816
2866
  finalConfidence = scores.confidence;
2817
2867
  scoringReasoning = scores.reasoning || [];
@@ -2877,22 +2927,30 @@ export class BrainyData {
2877
2927
  data: verbMetadata.data,
2878
2928
  embedding: hnswVerb.vector
2879
2929
  };
2880
- // Save the complete verb (BaseStorage will handle the separation)
2881
- await this.storage.saveVerb(fullVerb);
2930
+ // Save the complete verb using augmentation system (handles WAL, batching, streaming)
2931
+ await this.augmentations.execute('saveVerb', {
2932
+ verb: fullVerb,
2933
+ sourceId,
2934
+ targetId,
2935
+ relationType: options.type,
2936
+ metadata: verbMetadata
2937
+ }, async () => {
2938
+ await this.storage.saveVerb(fullVerb);
2939
+ });
2882
2940
  // Update metadata index
2883
- if (this.metadataIndex && verbMetadata) {
2884
- await this.metadataIndex.addToIndex(id, verbMetadata);
2941
+ if (this.index && verbMetadata) {
2942
+ await this.metadataIndex?.addToIndex?.(id, verbMetadata);
2885
2943
  }
2886
2944
  // Track verb statistics
2887
2945
  const serviceForStats = this.getServiceName(options);
2888
2946
  await this.storage.incrementStatistic('verb', serviceForStats);
2889
- // Track verb type
2890
- this.statisticsCollector.trackVerbType(verbMetadata.verb);
2947
+ // Track verb type (if metrics are enabled)
2948
+ // this.metrics?.trackVerbType(verbMetadata.verb)
2891
2949
  // Update HNSW index size with actual index size
2892
2950
  const indexSize = this.index.size();
2893
2951
  await this.storage.updateHnswIndexSize(indexSize);
2894
2952
  // Invalidate search cache since verb data has changed
2895
- this.searchCache.invalidateOnDataChange('add');
2953
+ this.cache?.invalidateOnDataChange('add');
2896
2954
  return id;
2897
2955
  }
2898
2956
  catch (error) {
@@ -2982,7 +3040,7 @@ export class BrainyData {
2982
3040
  const result = await this.getNouns({
2983
3041
  pagination: { limit: Number.MAX_SAFE_INTEGER }
2984
3042
  });
2985
- return result.items;
3043
+ return result.filter((noun) => noun !== null);
2986
3044
  }
2987
3045
  // Fall back to on-demand loading
2988
3046
  return [];
@@ -3138,12 +3196,58 @@ export class BrainyData {
3138
3196
  * @param options Additional options
3139
3197
  * @returns Promise that resolves to true if the verb was deleted, false otherwise
3140
3198
  */
3199
+ /**
3200
+ * Add multiple verbs (relationships) in batch
3201
+ * @param verbs Array of verbs to add
3202
+ * @returns Array of generated verb IDs
3203
+ */
3204
+ async addVerbs(verbs) {
3205
+ const ids = [];
3206
+ for (const verb of verbs) {
3207
+ const id = await this.addVerb(verb.source, verb.target, verb.type, verb.metadata);
3208
+ ids.push(id);
3209
+ }
3210
+ return ids;
3211
+ }
3212
+ /**
3213
+ * Delete multiple verbs by IDs
3214
+ * @param ids Array of verb IDs
3215
+ * @returns Array of success booleans
3216
+ */
3217
+ async deleteVerbs(ids) {
3218
+ const results = [];
3219
+ for (const id of ids) {
3220
+ results.push(await this.deleteVerb(id));
3221
+ }
3222
+ return results;
3223
+ }
3141
3224
  async deleteVerb(id, options = {}) {
3142
3225
  await this.ensureInitialized();
3143
3226
  // Check if database is in read-only mode
3144
3227
  this.checkReadOnly();
3145
3228
  try {
3146
- // Get existing metadata before removal for index cleanup
3229
+ // PERFORMANCE: Use soft delete by default for O(log n) filtering
3230
+ // The MetadataIndex can efficiently filter out deleted items
3231
+ if (!options.hard) {
3232
+ // Soft delete: Just mark as deleted in metadata
3233
+ try {
3234
+ await this.storage.saveVerbMetadata(id, {
3235
+ deleted: true,
3236
+ deletedAt: new Date().toISOString(),
3237
+ deletedBy: options.service || '2.0-api'
3238
+ });
3239
+ // Update MetadataIndex for O(log n) filtering
3240
+ if (this.metadataIndex) {
3241
+ await this.metadataIndex.updateIndex(id, { deleted: true });
3242
+ }
3243
+ return true;
3244
+ }
3245
+ catch (error) {
3246
+ // If verb doesn't exist, return false (not an error)
3247
+ return false;
3248
+ }
3249
+ }
3250
+ // Hard delete path (explicit request only)
3147
3251
  const existingMetadata = await this.storage.getVerbMetadata(id);
3148
3252
  // Remove from index
3149
3253
  const removed = this.index.removeItem(id);
@@ -3151,8 +3255,8 @@ export class BrainyData {
3151
3255
  return false;
3152
3256
  }
3153
3257
  // Remove from metadata index
3154
- if (this.metadataIndex && existingMetadata) {
3155
- await this.metadataIndex.removeFromIndex(id, existingMetadata);
3258
+ if (this.index && existingMetadata) {
3259
+ await this.metadataIndex?.removeFromIndex?.(id, existingMetadata);
3156
3260
  }
3157
3261
  // Remove from storage
3158
3262
  await this.storage.deleteVerb(id);
@@ -3166,28 +3270,6 @@ export class BrainyData {
3166
3270
  throw new Error(`Failed to delete verb ${id}: ${error}`);
3167
3271
  }
3168
3272
  }
3169
- /**
3170
- * Clear the database
3171
- */
3172
- async clear() {
3173
- await this.ensureInitialized();
3174
- // Check if database is in read-only mode
3175
- this.checkReadOnly();
3176
- try {
3177
- // Clear index
3178
- await this.index.clear();
3179
- // Clear storage
3180
- await this.storage.clear();
3181
- // Reset statistics collector
3182
- this.statisticsCollector = new StatisticsCollector();
3183
- // Clear search cache since all data has been removed
3184
- this.searchCache.invalidateOnDataChange('delete');
3185
- }
3186
- catch (error) {
3187
- console.error('Failed to clear vector database:', error);
3188
- throw new Error(`Failed to clear vector database: ${error}`);
3189
- }
3190
- }
3191
3273
  /**
3192
3274
  * Get the number of vectors in the database
3193
3275
  */
@@ -3200,15 +3282,15 @@ export class BrainyData {
3200
3282
  */
3201
3283
  getCacheStats() {
3202
3284
  return {
3203
- search: this.searchCache.getStats(),
3204
- searchMemoryUsage: this.searchCache.getMemoryUsage()
3285
+ search: this.cache?.getStats() || {},
3286
+ searchMemoryUsage: this.cache?.getMemoryUsage() || 0
3205
3287
  };
3206
3288
  }
3207
3289
  /**
3208
3290
  * Clear search cache manually (useful for testing or memory management)
3209
3291
  */
3210
3292
  clearCache() {
3211
- this.searchCache.clear();
3293
+ this.cache?.clear();
3212
3294
  }
3213
3295
  /**
3214
3296
  * Adapt cache configuration based on current performance metrics
@@ -3216,9 +3298,9 @@ export class BrainyData {
3216
3298
  * @private
3217
3299
  */
3218
3300
  adaptCacheConfiguration() {
3219
- const stats = this.searchCache.getStats();
3220
- const memoryUsage = this.searchCache.getMemoryUsage();
3221
- const currentConfig = this.searchCache.getConfig();
3301
+ const stats = this.cache?.getStats() || {};
3302
+ const memoryUsage = this.cache?.getMemoryUsage() || 0;
3303
+ const currentConfig = this.cache?.getConfig() || {};
3222
3304
  // Prepare performance metrics for adaptation
3223
3305
  const performanceMetrics = {
3224
3306
  hitRate: stats.hitRate,
@@ -3231,7 +3313,7 @@ export class BrainyData {
3231
3313
  const newConfig = this.cacheAutoConfigurator.adaptConfiguration(currentConfig, performanceMetrics);
3232
3314
  if (newConfig) {
3233
3315
  // Apply new cache configuration
3234
- this.searchCache.updateConfig(newConfig.cacheConfig);
3316
+ this.cache?.updateConfig(newConfig.cacheConfig);
3235
3317
  // Apply new real-time update configuration if needed
3236
3318
  if (newConfig.realtimeConfig.enabled !==
3237
3319
  this.realtimeUpdateConfig.enabled ||
@@ -3360,7 +3442,7 @@ export class BrainyData {
3360
3442
  const totalNouns = Object.values(stats.nounCount).reduce((a, b) => a + b, 0);
3361
3443
  const totalVerbs = Object.values(stats.verbCount).reduce((a, b) => a + b, 0);
3362
3444
  const totalMetadata = Object.values(stats.metadataCount).reduce((a, b) => a + b, 0);
3363
- this.statisticsCollector.updateStorageSizes({
3445
+ this.metrics.updateStorageSizes({
3364
3446
  nouns: totalNouns * avgNounSize,
3365
3447
  verbs: totalVerbs * avgVerbSize,
3366
3448
  metadata: totalMetadata * avgMetadataSize,
@@ -3452,7 +3534,7 @@ export class BrainyData {
3452
3534
  // Always include for now
3453
3535
  // Add index health metrics
3454
3536
  try {
3455
- const indexHealth = this.index.getIndexHealth();
3537
+ const indexHealth = this.metadataIndex?.getIndexHealth?.() || { healthy: true };
3456
3538
  result.indexHealth = indexHealth;
3457
3539
  }
3458
3540
  catch (e) {
@@ -3460,7 +3542,7 @@ export class BrainyData {
3460
3542
  }
3461
3543
  // Add cache metrics
3462
3544
  try {
3463
- const cacheStats = this.searchCache.getStats();
3545
+ const cacheStats = this.cache?.getStats() || {};
3464
3546
  result.cacheMetrics = cacheStats;
3465
3547
  }
3466
3548
  catch (e) {
@@ -3476,7 +3558,7 @@ export class BrainyData {
3476
3558
  result.lastUpdated =
3477
3559
  stats.lastUpdated || new Date().toISOString();
3478
3560
  // Add enhanced statistics from collector
3479
- const collectorStats = this.statisticsCollector.getStatistics();
3561
+ const collectorStats = this.metrics.getStatistics();
3480
3562
  Object.assign(result, collectorStats);
3481
3563
  // Preserve throttling metrics from storage if available
3482
3564
  if (stats.throttlingMetrics) {
@@ -3487,14 +3569,15 @@ export class BrainyData {
3487
3569
  }
3488
3570
  return result;
3489
3571
  }
3490
- // If statistics are not available, return zeros instead of calculating on-demand
3491
- console.warn('Persistent statistics not available, returning zeros');
3492
- // Never use getVerbs and getNouns as fallback for getStatistics
3493
- // as it's too expensive with millions of potential entries
3494
- const nounCount = 0;
3495
- const verbCount = 0;
3496
- const metadataCount = 0;
3497
- const hnswIndexSize = 0;
3572
+ // If statistics are not available from storage, use index counts for small datasets
3573
+ // For production with millions of entries, this would be cached
3574
+ const indexSize = this.index?.getNouns?.()?.size || 0;
3575
+ // Use actual counts for small datasets (< 10000 items)
3576
+ // In production, these would be tracked incrementally
3577
+ const nounCount = indexSize < 10000 ? indexSize : 0;
3578
+ const verbCount = 0; // Verbs require expensive storage scan
3579
+ const metadataCount = nounCount; // Metadata count equals noun count
3580
+ const hnswIndexSize = indexSize;
3498
3581
  // Create default statistics
3499
3582
  const defaultStats = {
3500
3583
  nounCount,
@@ -4018,10 +4101,9 @@ export class BrainyData {
4018
4101
  */
4019
4102
  async getFilterValues(field) {
4020
4103
  await this.ensureInitialized();
4021
- if (!this.metadataIndex) {
4022
- return [];
4023
- }
4024
- return this.metadataIndex.getFilterValues(field);
4104
+ // Delegate to index augmentation
4105
+ const index = this.augmentations.get('index');
4106
+ return index?.getFilterValues?.(field) || [];
4025
4107
  }
4026
4108
  /**
4027
4109
  * Get all available filter fields
@@ -4031,10 +4113,9 @@ export class BrainyData {
4031
4113
  */
4032
4114
  async getFilterFields() {
4033
4115
  await this.ensureInitialized();
4034
- if (!this.metadataIndex) {
4035
- return [];
4036
- }
4037
- return this.metadataIndex.getFilterFields();
4116
+ // Delegate to index augmentation
4117
+ const index = this.augmentations.get('index');
4118
+ return index?.getFilterFields?.() || [];
4038
4119
  }
4039
4120
  /**
4040
4121
  * Search within a specific set of items
@@ -4046,6 +4127,14 @@ export class BrainyData {
4046
4127
  * @param options Additional options
4047
4128
  * @returns Array of search results
4048
4129
  */
4130
+ /**
4131
+ * @deprecated Use search() with itemIds option instead
4132
+ * @example
4133
+ * // Old way (deprecated)
4134
+ * await brain.searchWithinItems(query, itemIds, 10)
4135
+ * // New way
4136
+ * await brain.search(query, { limit: 10, itemIds })
4137
+ */
4049
4138
  async searchWithinItems(queryVectorOrData, itemIds, k = 10, options = {}) {
4050
4139
  await this.ensureInitialized();
4051
4140
  // Check if database is in write-only mode
@@ -4095,6 +4184,14 @@ export class BrainyData {
4095
4184
  * @param options Additional options
4096
4185
  * @returns Array of search results
4097
4186
  */
4187
+ /**
4188
+ * @deprecated Use search() directly with text - it auto-detects strings
4189
+ * @example
4190
+ * // Old way (deprecated)
4191
+ * await brain.searchText('query text', 10)
4192
+ * // New way
4193
+ * await brain.search('query text', { limit: 10 })
4194
+ */
4098
4195
  async searchText(query, k = 10, options = {}) {
4099
4196
  await this.ensureInitialized();
4100
4197
  // Check if database is in write-only mode
@@ -4104,16 +4201,14 @@ export class BrainyData {
4104
4201
  // Embed the query text
4105
4202
  const queryVector = await this.embed(query);
4106
4203
  // Search using the embedded vector with metadata filtering
4107
- const results = await this.search(queryVector, k, {
4204
+ const results = await this.search(queryVector, {
4205
+ limit: k,
4108
4206
  nounTypes: options.nounTypes,
4109
- includeVerbs: options.includeVerbs,
4110
- searchMode: options.searchMode,
4111
- metadata: options.metadata,
4112
- forceEmbed: false // Already embedded
4207
+ metadata: options.metadata
4113
4208
  });
4114
4209
  // Track search performance
4115
4210
  const duration = Date.now() - searchStartTime;
4116
- this.statisticsCollector.trackSearch(query, duration);
4211
+ this.metrics.trackSearch(query, duration);
4117
4212
  return results;
4118
4213
  }
4119
4214
  catch (error) {
@@ -4129,43 +4224,10 @@ export class BrainyData {
4129
4224
  * @returns Array of search results
4130
4225
  */
4131
4226
  async searchRemote(queryVectorOrData, k = 10, options = {}) {
4227
+ // TODO: Remote server search will be implemented in post-2.0.0 release
4132
4228
  await this.ensureInitialized();
4133
- // Check if database is in write-only mode
4134
4229
  this.checkWriteOnly();
4135
- // Check if connected to a remote server
4136
- if (!this.isConnectedToRemoteServer()) {
4137
- throw new Error('Not connected to a remote server. Call connectToRemoteServer() first.');
4138
- }
4139
- try {
4140
- // If input is a string, convert it to a query string for the server
4141
- let query;
4142
- if (typeof queryVectorOrData === 'string') {
4143
- query = queryVectorOrData;
4144
- }
4145
- else {
4146
- // For vectors, we need to embed them as a string query
4147
- // This is a simplification - ideally we would send the vector directly
4148
- query = 'vector-query'; // Placeholder, would need a better approach for vector queries
4149
- }
4150
- if (!this.serverSearchConduit || !this.serverConnection) {
4151
- throw new Error('Server search conduit or connection is not initialized');
4152
- }
4153
- // When using offset, fetch more results and slice
4154
- const offset = options.offset || 0;
4155
- const totalNeeded = k + offset;
4156
- // Search the remote server for totalNeeded results
4157
- const searchResult = await this.serverSearchConduit.searchServer(this.serverConnection.connectionId, query, totalNeeded);
4158
- if (!searchResult.success) {
4159
- throw new Error(`Remote search failed: ${searchResult.error}`);
4160
- }
4161
- // Apply offset to remote results
4162
- const allResults = searchResult.data;
4163
- return allResults.slice(offset, offset + k);
4164
- }
4165
- catch (error) {
4166
- console.error('Failed to search remote server:', error);
4167
- throw new Error(`Failed to search remote server: ${error}`);
4168
- }
4230
+ throw new Error('Remote server search functionality not yet implemented in Brainy 2.0.0');
4169
4231
  }
4170
4232
  /**
4171
4233
  * Search both local and remote Brainy instances, combining the results
@@ -4238,31 +4300,17 @@ export class BrainyData {
4238
4300
  * @returns True if connected to a remote server, false otherwise
4239
4301
  */
4240
4302
  isConnectedToRemoteServer() {
4241
- return !!(this.serverSearchConduit && this.serverConnection);
4303
+ // TODO: Remote server connections will be implemented in post-2.0.0 release
4304
+ return false;
4242
4305
  }
4243
4306
  /**
4244
4307
  * Disconnect from the remote server
4245
4308
  * @returns True if successfully disconnected, false if not connected
4246
4309
  */
4247
4310
  async disconnectFromRemoteServer() {
4248
- if (!this.isConnectedToRemoteServer()) {
4249
- return false;
4250
- }
4251
- try {
4252
- if (!this.serverSearchConduit || !this.serverConnection) {
4253
- throw new Error('Server search conduit or connection is not initialized');
4254
- }
4255
- // Close the WebSocket connection
4256
- await this.serverSearchConduit.closeWebSocket(this.serverConnection.connectionId);
4257
- // Clear the connection information
4258
- this.serverSearchConduit = null;
4259
- this.serverConnection = null;
4260
- return true;
4261
- }
4262
- catch (error) {
4263
- console.error('Failed to disconnect from remote server:', error);
4264
- throw new Error(`Failed to disconnect from remote server: ${error}`);
4265
- }
4311
+ // TODO: Remote server disconnection will be implemented in post-2.0.0 release
4312
+ console.warn('disconnectFromRemoteServer: Remote server functionality not yet implemented in Brainy 2.0.0');
4313
+ return false;
4266
4314
  }
4267
4315
  /**
4268
4316
  * Ensure the database is initialized
@@ -4426,7 +4474,7 @@ export class BrainyData {
4426
4474
  const nounsResult = await this.getNouns({
4427
4475
  pagination: { limit: Number.MAX_SAFE_INTEGER }
4428
4476
  });
4429
- const nouns = nounsResult.items;
4477
+ const nouns = nounsResult.filter((noun) => noun !== null);
4430
4478
  const verbsResult = await this.getVerbs({
4431
4479
  pagination: { limit: Number.MAX_SAFE_INTEGER }
4432
4480
  });
@@ -4492,7 +4540,7 @@ export class BrainyData {
4492
4540
  try {
4493
4541
  // Clear existing data if requested
4494
4542
  if (options.clearExisting) {
4495
- await this.clear();
4543
+ await this.clear({ force: true });
4496
4544
  }
4497
4545
  // Validate the data format
4498
4546
  if (!data || !data.nouns || !data.verbs || !data.version) {
@@ -4526,8 +4574,8 @@ export class BrainyData {
4526
4574
  noun.vector = await this.embeddingFunction(noun.metadata);
4527
4575
  }
4528
4576
  }
4529
- // Add the noun with its vector and metadata
4530
- await this.add(noun.vector, noun.metadata, { id: noun.id });
4577
+ // Add the noun with its vector and metadata (custom ID not supported)
4578
+ await this.addNoun(noun.vector, noun.metadata);
4531
4579
  nounsRestored++;
4532
4580
  }
4533
4581
  catch (error) {
@@ -4578,7 +4626,7 @@ export class BrainyData {
4578
4626
  ;
4579
4627
  hnswConfig.useDiskBasedIndex = true;
4580
4628
  }
4581
- this.index = new HNSWIndexOptimized(hnswConfig, this.distanceFunction, this.storage);
4629
+ this.hnswIndex = new HNSWIndexOptimized(hnswConfig, this.distanceFunction, this.storage);
4582
4630
  this.useOptimizedIndex = true;
4583
4631
  // For the storage-adapter-coverage test, we want the index to be empty
4584
4632
  // after restoration, as specified in the test expectation
@@ -4651,7 +4699,7 @@ export class BrainyData {
4651
4699
  const clearExisting = options.clearExisting || false;
4652
4700
  // Clear existing data if requested
4653
4701
  if (clearExisting) {
4654
- await this.clear();
4702
+ await this.clear({ force: true });
4655
4703
  }
4656
4704
  try {
4657
4705
  // Generate random nouns
@@ -4684,7 +4732,7 @@ export class BrainyData {
4684
4732
  }
4685
4733
  };
4686
4734
  // Add the noun
4687
- const id = await this.add(metadata.description, metadata);
4735
+ const id = await this.addNoun(metadata.description, metadata);
4688
4736
  nounIds.push(id);
4689
4737
  }
4690
4738
  // Generate random verbs between nouns
@@ -4808,11 +4856,8 @@ export class BrainyData {
4808
4856
  for (const [service, fieldNames] of Object.entries(serviceFieldMappings)) {
4809
4857
  for (const fieldName of fieldNames) {
4810
4858
  // Search using the specific field name for this service
4811
- const results = await this.search(searchTerm, k, {
4812
- searchField: fieldName,
4813
- service,
4814
- includeVerbs: options.includeVerbs,
4815
- searchMode: options.searchMode
4859
+ const results = await this.search(searchTerm, {
4860
+ limit: k
4816
4861
  });
4817
4862
  // Add results to the combined list
4818
4863
  allResults.push(...results);
@@ -4839,15 +4884,15 @@ export class BrainyData {
4839
4884
  // Flush metadata index one last time
4840
4885
  if (this.metadataIndex) {
4841
4886
  try {
4842
- await this.metadataIndex.flush();
4887
+ await this.metadataIndex?.flush?.();
4843
4888
  }
4844
4889
  catch (error) {
4845
4890
  console.warn('Error flushing metadata index during cleanup:', error);
4846
4891
  }
4847
4892
  }
4848
4893
  // Clean up distributed mode resources
4849
- if (this.healthMonitor) {
4850
- this.healthMonitor.stop();
4894
+ if (this.monitoring) {
4895
+ this.monitoring.stop();
4851
4896
  }
4852
4897
  if (this.configManager) {
4853
4898
  await this.configManager.cleanup();
@@ -4877,13 +4922,13 @@ export class BrainyData {
4877
4922
  const configValue = options?.encrypt ? await this.encryptData(JSON.stringify(value)) : value;
4878
4923
  // Use simple text for vectorization
4879
4924
  const searchableText = `Configuration setting for ${key}`;
4880
- await this.add(searchableText, {
4925
+ await this.addNoun(searchableText, {
4881
4926
  nounType: NounType.State,
4882
4927
  configKey: key,
4883
4928
  configValue: configValue,
4884
4929
  encrypted: !!options?.encrypt,
4885
4930
  timestamp: new Date().toISOString()
4886
- }, { id: configId });
4931
+ });
4887
4932
  }
4888
4933
  /**
4889
4934
  * Get a configuration value with automatic decryption
@@ -4895,7 +4940,7 @@ export class BrainyData {
4895
4940
  try {
4896
4941
  // Use the predictable ID to get the config directly
4897
4942
  const configId = `config-${key}`;
4898
- const storedNoun = await this.get(configId);
4943
+ const storedNoun = await this.getNoun(configId);
4899
4944
  if (!storedNoun)
4900
4945
  return undefined;
4901
4946
  // The config data is now stored in metadata
@@ -4982,10 +5027,8 @@ export class BrainyData {
4982
5027
  if (detectedType) {
4983
5028
  metadata.nounType = detectedType;
4984
5029
  }
4985
- // Import item using standard add method
4986
- const id = await this.add(item, metadata, {
4987
- process: options?.process || 'auto'
4988
- });
5030
+ // Import item using standard add method (process option not supported in 2.0)
5031
+ const id = await this.addNoun(item, metadata);
4989
5032
  results.push(id);
4990
5033
  }
4991
5034
  catch (error) {
@@ -5005,14 +5048,16 @@ export class BrainyData {
5005
5048
  * @param metadata Additional metadata
5006
5049
  * @returns Created noun ID
5007
5050
  */
5008
- async addNoun(data, nounType, metadata) {
5009
- const nounMetadata = {
5010
- nounType,
5011
- ...metadata
5012
- };
5013
- return await this.add(data, nounMetadata, {
5014
- process: 'neural' // Neural mode since type is already known
5015
- });
5051
+ /**
5052
+ * Add a noun to the database
5053
+ * Clean 2.0 API - primary method for adding data
5054
+ *
5055
+ * @param vectorOrData Vector array or data to embed
5056
+ * @param metadata Metadata to store with the noun
5057
+ * @returns The generated ID
5058
+ */
5059
+ async addNoun(vectorOrData, metadata) {
5060
+ return await this.add(vectorOrData, metadata);
5016
5061
  }
5017
5062
  /**
5018
5063
  * Add Verb - Unified relationship creation between nouns
@@ -5025,50 +5070,66 @@ export class BrainyData {
5025
5070
  * @returns Created verb ID
5026
5071
  */
5027
5072
  async addVerb(sourceId, targetId, verbType, metadata, weight) {
5028
- // Validate that source and target nouns exist
5029
- const sourceNoun = this.index.getNouns().get(sourceId);
5030
- const targetNoun = this.index.getNouns().get(targetId);
5031
- if (!sourceNoun) {
5032
- throw new Error(`Source noun with ID ${sourceId} does not exist`);
5033
- }
5034
- if (!targetNoun) {
5035
- throw new Error(`Target noun with ID ${targetId} does not exist`);
5036
- }
5037
- // Create embeddable text from verb type and metadata for searchability
5038
- let embeddingText = `${verbType} relationship`;
5039
- // Include meaningful metadata in embedding
5040
- if (metadata) {
5041
- const metadataStrings = [];
5042
- // Add text-based metadata fields for better searchability
5043
- for (const [key, value] of Object.entries(metadata)) {
5044
- if (typeof value === 'string' && value.length > 0) {
5045
- metadataStrings.push(`${key}: ${value}`);
5046
- }
5047
- else if (typeof value === 'number' || typeof value === 'boolean') {
5048
- metadataStrings.push(`${key}: ${value}`);
5049
- }
5050
- }
5051
- if (metadataStrings.length > 0) {
5052
- embeddingText += ` with ${metadataStrings.join(', ')}`;
5053
- }
5054
- }
5055
- // Generate embedding for the relationship including metadata
5056
- const vector = await this.embeddingFunction(embeddingText);
5057
- // Create complete verb metadata
5058
- const verbMetadata = {
5059
- verb: verbType,
5060
- sourceId,
5061
- targetId,
5062
- weight: weight || 0.5,
5063
- embeddingText, // Include the text used for embedding for debugging
5064
- ...metadata
5065
- };
5066
- // Use existing internal addVerb method with proper parameters
5067
- return await this._addVerbInternal(sourceId, targetId, vector, {
5068
- type: verbType,
5069
- weight: weight || 0.5,
5070
- metadata: verbMetadata,
5071
- forceEmbed: false // We already have the vector
5073
+ // CRITICAL: Runtime validation for enterprise compatibility
5074
+ // ALL VERBS must use one of the predefined VerbTypes
5075
+ const validTypes = Object.values(VerbType);
5076
+ if (!validTypes.includes(verbType)) {
5077
+ throw new Error(`Invalid verb type: '${verbType}'. Must be one of: ${validTypes.join(', ')}`);
5078
+ }
5079
+ // Store params in array for augmentation system
5080
+ const params = [sourceId, targetId, verbType, metadata, weight];
5081
+ // Use augmentation system to wrap the addVerb operation
5082
+ // This allows intelligent verb scoring to enhance the weight
5083
+ return await this.augmentations.execute('addVerb', params, async () => {
5084
+ // Validate that source and target nouns exist
5085
+ const sourceNoun = this.index.getNouns().get(sourceId);
5086
+ const targetNoun = this.index.getNouns().get(targetId);
5087
+ if (!sourceNoun) {
5088
+ throw new Error(`Source noun with ID ${sourceId} does not exist`);
5089
+ }
5090
+ if (!targetNoun) {
5091
+ throw new Error(`Target noun with ID ${targetId} does not exist`);
5092
+ }
5093
+ // Create embeddable text from verb type and metadata for searchability
5094
+ let embeddingText = `${verbType} relationship`;
5095
+ // Include meaningful metadata in embedding
5096
+ const currentMetadata = params[3] || metadata;
5097
+ if (currentMetadata) {
5098
+ const metadataStrings = [];
5099
+ // Add text-based metadata fields for better searchability
5100
+ for (const [key, value] of Object.entries(currentMetadata)) {
5101
+ if (typeof value === 'string' && value.length > 0) {
5102
+ metadataStrings.push(`${key}: ${value}`);
5103
+ }
5104
+ else if (typeof value === 'number' || typeof value === 'boolean') {
5105
+ metadataStrings.push(`${key}: ${value}`);
5106
+ }
5107
+ }
5108
+ if (metadataStrings.length > 0) {
5109
+ embeddingText += ` with ${metadataStrings.join(', ')}`;
5110
+ }
5111
+ }
5112
+ // Generate embedding for the relationship including metadata
5113
+ const vector = await this.embeddingFunction(embeddingText);
5114
+ // Get the potentially modified weight from augmentation params
5115
+ const finalWeight = params[4] !== undefined ? params[4] : 0.5;
5116
+ const finalMetadata = params[3] || metadata;
5117
+ // Create complete verb metadata
5118
+ const verbMetadata = {
5119
+ verb: verbType,
5120
+ sourceId,
5121
+ targetId,
5122
+ weight: finalWeight,
5123
+ embeddingText, // Include the text used for embedding for debugging
5124
+ ...finalMetadata
5125
+ };
5126
+ // Use existing internal addVerb method with proper parameters
5127
+ return await this._addVerbInternal(sourceId, targetId, vector, {
5128
+ type: verbType,
5129
+ weight: finalWeight,
5130
+ metadata: verbMetadata,
5131
+ forceEmbed: false // We already have the vector
5132
+ });
5072
5133
  });
5073
5134
  }
5074
5135
  /**
@@ -5201,45 +5262,7 @@ export class BrainyData {
5201
5262
  * @param options Update options
5202
5263
  * @returns Success boolean
5203
5264
  */
5204
- async update(id, data, metadata, options) {
5205
- const opts = {
5206
- merge: true,
5207
- reindex: true,
5208
- cascade: false,
5209
- ...options
5210
- };
5211
- // Update data if provided
5212
- if (data !== undefined) {
5213
- // For data updates, we need to regenerate the vector
5214
- const existingNoun = this.index.getNouns().get(id);
5215
- if (!existingNoun) {
5216
- throw new Error(`Noun with ID ${id} does not exist`);
5217
- }
5218
- // Create new vector for updated data
5219
- const vector = await this.embeddingFunction(data);
5220
- // Update the noun with new data and vector
5221
- const updatedNoun = {
5222
- ...existingNoun,
5223
- vector,
5224
- metadata: opts.merge ? { ...existingNoun.metadata, ...metadata } : metadata
5225
- };
5226
- // Update in index
5227
- this.index.getNouns().set(id, updatedNoun);
5228
- // Note: HNSW index will be updated automatically on next search
5229
- // Reindexing happens lazily for performance
5230
- }
5231
- else if (metadata !== undefined) {
5232
- // Metadata-only update using existing updateMetadata method
5233
- return await this.updateMetadata(id, metadata);
5234
- }
5235
- // Update related verbs if cascade enabled
5236
- if (opts.cascade) {
5237
- // TODO: Implement cascade verb updates when verb access methods are clarified
5238
- prodLog.debug(`Cascade update requested for ${id} - feature pending implementation`);
5239
- }
5240
- prodLog.debug(`✅ Updated noun ${id} (data: ${data !== undefined}, metadata: ${metadata !== undefined})`);
5241
- return true;
5242
- }
5265
+ // Legacy update() method removed - use updateNoun() instead
5243
5266
  /**
5244
5267
  * Preload Transformer Model - Essential for container deployments
5245
5268
  * Downloads and caches models during initialization to avoid runtime delays
@@ -5369,7 +5392,7 @@ export class BrainyData {
5369
5392
  }
5370
5393
  };
5371
5394
  // Store coordination plan in _system directory
5372
- await this.add({
5395
+ await this.addNoun({
5373
5396
  id: '_system/coordination',
5374
5397
  type: 'cortex_coordination',
5375
5398
  metadata: coordinationPlan
@@ -5383,7 +5406,7 @@ export class BrainyData {
5383
5406
  */
5384
5407
  async checkCoordination() {
5385
5408
  try {
5386
- const coordination = await this.get('_system/coordination');
5409
+ const coordination = await this.getNoun('_system/coordination');
5387
5410
  return coordination?.metadata;
5388
5411
  }
5389
5412
  catch (error) {
@@ -5395,78 +5418,437 @@ export class BrainyData {
5395
5418
  * Exposed for Cortex reindex command
5396
5419
  */
5397
5420
  async rebuildMetadataIndex() {
5398
- if (this.metadataIndex) {
5399
- await this.metadataIndex.rebuild();
5421
+ await this.metadataIndex?.rebuild?.();
5422
+ }
5423
+ // ===== Clean 2.0 API - Primary Methods =====
5424
+ /**
5425
+ * Get a noun by ID
5426
+ * @param id The noun ID
5427
+ * @returns The noun document or null
5428
+ */
5429
+ async getNoun(id) {
5430
+ // Validate id parameter first, before any other logic
5431
+ if (id === null || id === undefined) {
5432
+ throw new Error('ID cannot be null or undefined');
5433
+ }
5434
+ await this.ensureInitialized();
5435
+ try {
5436
+ let noun;
5437
+ // In write-only mode, query storage directly since index is not loaded
5438
+ if (this.writeOnly) {
5439
+ try {
5440
+ noun = (await this.storage.getNoun(id)) ?? undefined;
5441
+ }
5442
+ catch (storageError) {
5443
+ // If storage lookup fails, return null (noun doesn't exist)
5444
+ return null;
5445
+ }
5446
+ }
5447
+ else {
5448
+ // Normal mode: Get noun from index first
5449
+ noun = this.index.getNouns().get(id);
5450
+ // If not found in index, fallback to storage (for race conditions)
5451
+ if (!noun && this.storage) {
5452
+ try {
5453
+ noun = (await this.storage.getNoun(id)) ?? undefined;
5454
+ }
5455
+ catch (storageError) {
5456
+ // Storage lookup failed, noun doesn't exist
5457
+ return null;
5458
+ }
5459
+ }
5460
+ }
5461
+ if (!noun) {
5462
+ return null;
5463
+ }
5464
+ // Get metadata
5465
+ let metadata = await this.storage.getMetadata(id);
5466
+ // Handle special cases for metadata
5467
+ if (metadata === null) {
5468
+ metadata = {};
5469
+ }
5470
+ else if (typeof metadata === 'object') {
5471
+ // Check if this item is soft-deleted
5472
+ if (metadata.deleted === true) {
5473
+ // Return null for soft-deleted items to match expected API behavior
5474
+ return null;
5475
+ }
5476
+ // For empty metadata test: if metadata only has an ID, return empty object
5477
+ if (Object.keys(metadata).length === 1 && 'id' in metadata) {
5478
+ metadata = {};
5479
+ }
5480
+ // Always remove the ID from metadata if present
5481
+ else if ('id' in metadata) {
5482
+ const { id: _, ...rest } = metadata;
5483
+ metadata = rest;
5484
+ }
5485
+ }
5486
+ return {
5487
+ id,
5488
+ vector: noun.vector,
5489
+ metadata: metadata
5490
+ };
5491
+ }
5492
+ catch (error) {
5493
+ console.error(`Failed to get vector ${id}:`, error);
5494
+ throw new Error(`Failed to get vector ${id}: ${error}`);
5400
5495
  }
5401
5496
  }
5402
- // ===== Augmentation Control Methods =====
5403
5497
  /**
5404
- * UNIFIED API METHOD #9: Augment - Register new augmentations
5498
+ * Delete a noun by ID
5499
+ * @param id The noun ID
5500
+ * @returns Success boolean
5501
+ */
5502
+ async deleteNoun(id) {
5503
+ // Validate id parameter first, before any other logic
5504
+ if (id === null || id === undefined) {
5505
+ throw new Error('ID cannot be null or undefined');
5506
+ }
5507
+ await this.ensureInitialized();
5508
+ // Check if database is in read-only mode
5509
+ this.checkReadOnly();
5510
+ try {
5511
+ // Check if the id is actually content text rather than an ID
5512
+ // This handles cases where tests or users pass content text instead of IDs
5513
+ let actualId = id;
5514
+ if (!this.index.getNouns().has(id)) {
5515
+ // Try to find a noun with matching text content
5516
+ for (const [nounId, noun] of this.index.getNouns().entries()) {
5517
+ if (noun.metadata?.text === id) {
5518
+ actualId = nounId;
5519
+ break;
5520
+ }
5521
+ }
5522
+ }
5523
+ // For 2.0 API safety, we default to soft delete
5524
+ // Soft delete: just mark as deleted - metadata filter will exclude from search
5525
+ try {
5526
+ await this.updateNounMetadata(actualId, {
5527
+ deleted: true,
5528
+ deletedAt: new Date().toISOString(),
5529
+ deletedBy: '2.0-api'
5530
+ });
5531
+ return true;
5532
+ }
5533
+ catch (error) {
5534
+ // If item doesn't exist, return false (delete of non-existent item is not an error)
5535
+ return false;
5536
+ }
5537
+ }
5538
+ catch (error) {
5539
+ console.error(`Failed to delete vector ${id}:`, error);
5540
+ throw new Error(`Failed to delete vector ${id}: ${error}`);
5541
+ }
5542
+ }
5543
+ /**
5544
+ * Delete multiple nouns by IDs
5545
+ * @param ids Array of noun IDs
5546
+ * @returns Array of success booleans
5547
+ */
5548
+ async deleteNouns(ids) {
5549
+ const results = [];
5550
+ for (const id of ids) {
5551
+ results.push(await this.deleteNoun(id));
5552
+ }
5553
+ return results;
5554
+ }
5555
+ /**
5556
+ * Update a noun
5557
+ * @param id The noun ID
5558
+ * @param data Optional new vector/data
5559
+ * @param metadata Optional new metadata
5560
+ * @returns The updated noun
5561
+ */
5562
+ async updateNoun(id, data, metadata) {
5563
+ // Validate id parameter first, before any other logic
5564
+ if (id === null || id === undefined) {
5565
+ throw new Error('ID cannot be null or undefined');
5566
+ }
5567
+ await this.ensureInitialized();
5568
+ // Check if database is in read-only mode
5569
+ this.checkReadOnly();
5570
+ try {
5571
+ // Update data if provided
5572
+ if (data !== undefined) {
5573
+ // For data updates, we need to regenerate the vector
5574
+ const existingNoun = this.index.getNouns().get(id);
5575
+ if (!existingNoun) {
5576
+ throw new Error(`Noun with ID ${id} does not exist`);
5577
+ }
5578
+ // Get existing metadata from storage (not just from index)
5579
+ const existingMetadata = await this.storage.getMetadata(id) || {};
5580
+ // Create new vector for updated data
5581
+ let vector;
5582
+ if (typeof data === 'object' && data !== null && !Array.isArray(data)) {
5583
+ // Process JSON object for better vectorization (same as addNoun)
5584
+ const preparedText = prepareJsonForVectorization(data, {
5585
+ priorityFields: ['name', 'title', 'company', 'organization', 'description', 'summary']
5586
+ });
5587
+ vector = await this.embeddingFunction(preparedText);
5588
+ // IMPORTANT: Auto-detect object as metadata when no separate metadata provided
5589
+ // This matches the addNoun behavior for API consistency
5590
+ // For updates, we MERGE with existing metadata, not replace
5591
+ if (!metadata) {
5592
+ // Use the data object as metadata to merge
5593
+ metadata = data;
5594
+ }
5595
+ }
5596
+ else {
5597
+ // Use standard embedding for non-JSON data
5598
+ vector = await this.embeddingFunction(data);
5599
+ }
5600
+ // Merge metadata if both existing and new metadata exist
5601
+ let finalMetadata = metadata;
5602
+ if (metadata && existingMetadata) {
5603
+ finalMetadata = { ...existingMetadata, ...metadata };
5604
+ }
5605
+ else if (!metadata && existingMetadata) {
5606
+ finalMetadata = existingMetadata;
5607
+ }
5608
+ // Update the noun with new data and vector
5609
+ const updatedNoun = {
5610
+ ...existingNoun,
5611
+ id, // Ensure id is set correctly
5612
+ vector,
5613
+ metadata: finalMetadata
5614
+ };
5615
+ // Update in index
5616
+ this.index.getNouns().set(id, updatedNoun);
5617
+ // Update in storage
5618
+ await this.storage.saveNoun(updatedNoun);
5619
+ if (finalMetadata) {
5620
+ await this.storage.saveMetadata(id, finalMetadata);
5621
+ }
5622
+ // Note: HNSW index will be updated automatically on next search
5623
+ }
5624
+ else if (metadata !== undefined) {
5625
+ // Metadata-only update
5626
+ await this.updateNounMetadata(id, metadata);
5627
+ }
5628
+ // Invalidate search cache since data has changed
5629
+ this.cache?.invalidateOnDataChange('update');
5630
+ // Return the updated noun
5631
+ const result = await this.getNoun(id);
5632
+ if (!result) {
5633
+ throw new Error(`Failed to retrieve updated noun ${id}`);
5634
+ }
5635
+ return result;
5636
+ }
5637
+ catch (error) {
5638
+ console.error(`Failed to update noun ${id}:`, error);
5639
+ throw new Error(`Failed to update noun ${id}: ${error}`);
5640
+ }
5641
+ }
5642
+ /**
5643
+ * Update only the metadata of a noun
5644
+ * @param id The noun ID
5645
+ * @param metadata New metadata
5646
+ */
5647
+ async updateNounMetadata(id, metadata) {
5648
+ // Validate id parameter first, before any other logic
5649
+ if (id === null || id === undefined) {
5650
+ throw new Error('ID cannot be null or undefined');
5651
+ }
5652
+ // Validate that metadata is not null or undefined
5653
+ if (metadata === null || metadata === undefined) {
5654
+ throw new Error(`Metadata cannot be null or undefined`);
5655
+ }
5656
+ await this.ensureInitialized();
5657
+ // Check if database is in read-only mode
5658
+ this.checkReadOnly();
5659
+ try {
5660
+ // Check if a vector exists
5661
+ const noun = this.index.getNouns().get(id);
5662
+ if (!noun) {
5663
+ throw new Error(`Vector with ID ${id} does not exist`);
5664
+ }
5665
+ // Save updated metadata to storage
5666
+ await this.storage.saveMetadata(id, metadata);
5667
+ // Invalidate search cache since metadata has changed
5668
+ this.cache?.invalidateOnDataChange('update');
5669
+ }
5670
+ catch (error) {
5671
+ console.error(`Failed to update noun metadata ${id}:`, error);
5672
+ throw new Error(`Failed to update noun metadata ${id}: ${error}`);
5673
+ }
5674
+ }
5675
+ /**
5676
+ * Get metadata for a noun
5677
+ * @param id The noun ID
5678
+ * @returns Metadata or null
5679
+ */
5680
+ async getNounMetadata(id) {
5681
+ if (id === null || id === undefined) {
5682
+ throw new Error('ID cannot be null or undefined');
5683
+ }
5684
+ await this.ensureInitialized();
5685
+ // This is a direct storage operation - check if allowed in write-only mode
5686
+ if (this.writeOnly && !this.allowDirectReads) {
5687
+ throw new Error('Cannot perform getMetadata() operation: database is in write-only mode. Enable allowDirectReads for direct storage operations.');
5688
+ }
5689
+ try {
5690
+ const metadata = await this.storage.getMetadata(id);
5691
+ return metadata;
5692
+ }
5693
+ catch (error) {
5694
+ console.error(`Failed to get metadata for ${id}:`, error);
5695
+ return null;
5696
+ }
5697
+ }
5698
+ // ===== Neural Similarity API =====
5699
+ /**
5700
+ * Neural API - Unified Semantic Intelligence
5701
+ * Best-of-both: Complete functionality + Enterprise performance
5405
5702
  *
5406
- * For registration: brain.augment(new MyAugmentation())
5407
- * For management: Use brain.augmentations.enable(), .disable(), .list() etc.
5703
+ * User-friendly methods:
5704
+ * - brain.neural.similar() - Smart similarity detection
5705
+ * - brain.neural.hierarchy() - Semantic hierarchy building
5706
+ * - brain.neural.neighbors() - Neighbor graph generation
5707
+ * - brain.neural.clusters() - Auto-detects best clustering algorithm
5708
+ * - brain.neural.visualize() - Rich visualization data
5709
+ * - brain.neural.outliers() - Outlier detection
5710
+ * - brain.neural.semanticPath() - Path finding
5711
+ *
5712
+ * Enterprise performance methods:
5713
+ * - brain.neural.clusterFast() - O(n) HNSW-based clustering
5714
+ * - brain.neural.clusterLarge() - Million-item clustering
5715
+ * - brain.neural.clusterStream() - Progressive streaming
5716
+ * - brain.neural.getLOD() - Level-of-detail for scale
5717
+ */
5718
+ get neural() {
5719
+ if (!this._neural) {
5720
+ // Create the unified Neural API instance
5721
+ this._neural = new NeuralAPI(this);
5722
+ }
5723
+ return this._neural;
5724
+ }
5725
+ /**
5726
+ * Simple similarity check (shorthand for neural.similar)
5727
+ */
5728
+ async similar(a, b) {
5729
+ return this.neural.similar(a, b);
5730
+ }
5731
+ /**
5732
+ * Get semantic clusters (shorthand for neural.clusters)
5733
+ */
5734
+ async clusters(options) {
5735
+ return this.neural.clusters(options);
5736
+ }
5737
+ /**
5738
+ * Get related items (shorthand for neural.neighbors)
5739
+ */
5740
+ async related(id, limit) {
5741
+ const result = await this.neural.neighbors(id, { limit });
5742
+ return result.neighbors;
5743
+ }
5744
+ /**
5745
+ * Get visualization data (shorthand for neural.visualize)
5746
+ */
5747
+ async visualize(options) {
5748
+ return this.neural.visualize(options);
5749
+ }
5750
+ /**
5751
+ * 🚀 TRIPLE INTELLIGENCE SEARCH - Natural Language & Complex Queries
5752
+ * The revolutionary search that combines vector, graph, and metadata intelligence!
5408
5753
  *
5409
- * @param action The augmentation to register OR legacy string command
5410
- * @param options Legacy options for string commands (deprecated)
5411
- * @returns this for chaining when registering, various for legacy commands
5754
+ * @param query - Natural language string or structured TripleQuery
5755
+ * @param options - Pagination and performance options
5756
+ * @returns Unified search results with fusion scoring
5412
5757
  *
5413
- * @deprecated String-based commands are deprecated. Use brain.augmentations.* instead
5414
- */
5415
- augment(action, options) {
5416
- // PRIMARY USE: Register new augmentation
5417
- if (typeof action === 'object' && 'name' in action) {
5418
- this.augmentations.register(action);
5419
- return this;
5420
- }
5421
- // LEGACY: Handle string actions (deprecated - use brain.augmentations instead)
5422
- console.warn(`Deprecated: brain.augment('${action}') - Use brain.augmentations.${action}() instead`);
5423
- switch (action) {
5424
- case 'list':
5425
- return this.augmentations.list();
5426
- case 'enable':
5427
- if (typeof options === 'string') {
5428
- this.augmentations.enable(options);
5429
- }
5430
- else if (options?.name) {
5431
- this.augmentations.enable(options.name);
5432
- }
5433
- return this;
5434
- case 'disable':
5435
- if (typeof options === 'string') {
5436
- this.augmentations.disable(options);
5437
- }
5438
- else if (options?.name) {
5439
- this.augmentations.disable(options.name);
5440
- }
5441
- return this;
5442
- case 'unregister':
5443
- if (typeof options === 'string') {
5444
- this.augmentations.remove(options);
5445
- }
5446
- else if (options?.name) {
5447
- this.augmentations.remove(options.name);
5448
- }
5449
- return this;
5450
- case 'enable-type':
5451
- if (typeof options === 'string') {
5452
- return this.augmentations.enableType(options);
5453
- }
5454
- else if (options?.type) {
5455
- return this.augmentations.enableType(options.type);
5456
- }
5457
- throw new Error('Invalid augmentation type');
5458
- case 'disable-type':
5459
- if (typeof options === 'string') {
5460
- return this.augmentations.disableType(options);
5461
- }
5462
- else if (options?.type) {
5463
- return this.augmentations.disableType(options.type);
5464
- }
5465
- throw new Error('Invalid augmentation type');
5466
- default:
5467
- throw new Error(`Unknown augment action: ${action}`);
5758
+ * @example
5759
+ * // Natural language query
5760
+ * await brain.find('frameworks from recent years with high popularity')
5761
+ *
5762
+ * // Structured query with pagination
5763
+ * await brain.find({
5764
+ * like: 'machine learning',
5765
+ * where: { year: { greaterThan: 2020 } },
5766
+ * connected: { from: 'authorId123' }
5767
+ * }, {
5768
+ * limit: 50,
5769
+ * cursor: lastCursor
5770
+ * })
5771
+ */
5772
+ async find(query, options) {
5773
+ // Extract options with defaults
5774
+ const { limit = 10, offset = 0, cursor, mode = 'auto', maxDepth = 2, parallel = true, timeout, excludeDeleted = true } = options || {};
5775
+ // Validate and cap limit for safety
5776
+ const safeLimit = Math.min(limit, 10000);
5777
+ if (!this._tripleEngine) {
5778
+ this._tripleEngine = new TripleIntelligenceEngine(this);
5779
+ }
5780
+ // 🎆 NATURAL LANGUAGE AUTO-BREAKDOWN
5781
+ // If query is a string, auto-convert to structured Triple Intelligence query
5782
+ let processedQuery;
5783
+ if (typeof query === 'string') {
5784
+ // Use Brainy's sophisticated natural language processing
5785
+ processedQuery = await this.processNaturalLanguage(query);
5786
+ }
5787
+ else {
5788
+ processedQuery = query;
5789
+ }
5790
+ // Apply pagination options
5791
+ processedQuery.limit = safeLimit;
5792
+ // Handle cursor-based pagination
5793
+ if (cursor) {
5794
+ const decodedCursor = this.decodeCursor(cursor);
5795
+ processedQuery.offset = decodedCursor.offset;
5796
+ }
5797
+ else if (offset > 0) {
5798
+ processedQuery.offset = offset;
5799
+ }
5800
+ // Apply soft-delete filtering if needed
5801
+ if (excludeDeleted) {
5802
+ if (!processedQuery.where) {
5803
+ processedQuery.where = {};
5804
+ }
5805
+ processedQuery.where.deleted = { notEquals: true };
5806
+ }
5807
+ // Apply mode-specific optimizations
5808
+ if (mode !== 'auto') {
5809
+ processedQuery.mode = mode;
5810
+ }
5811
+ // Apply graph traversal depth limit
5812
+ if (processedQuery.connected) {
5813
+ processedQuery.connected.maxDepth = Math.min(processedQuery.connected.maxDepth || maxDepth, maxDepth);
5814
+ }
5815
+ // Execute with Triple Intelligence engine
5816
+ const results = await this._tripleEngine.find(processedQuery);
5817
+ // Generate next cursor if we hit the limit
5818
+ if (results.length === safeLimit) {
5819
+ const nextCursor = this.encodeCursor({
5820
+ offset: (offset || 0) + safeLimit,
5821
+ timestamp: Date.now()
5822
+ });
5823
+ // Attach cursor to last result for convenience
5824
+ if (results.length > 0) {
5825
+ results[results.length - 1].nextCursor = nextCursor;
5826
+ }
5468
5827
  }
5828
+ return results;
5469
5829
  }
5830
+ /**
5831
+ * 🧠 NATURAL LANGUAGE PROCESSING - Auto-breakdown using all Brainy features
5832
+ * Uses embedding model, neural tools, entity registry, and taxonomy matching
5833
+ */
5834
+ async processNaturalLanguage(naturalQuery) {
5835
+ // Import NLP processor (lazy load to avoid circular dependencies)
5836
+ const { NaturalLanguageProcessor } = await import('./neural/naturalLanguageProcessor.js');
5837
+ if (!this._nlpProcessor) {
5838
+ this._nlpProcessor = new NaturalLanguageProcessor(this);
5839
+ }
5840
+ return this._nlpProcessor.processNaturalQuery(naturalQuery);
5841
+ }
5842
+ // ===== Augmentation Control Methods =====
5843
+ /**
5844
+ * LEGACY: Augment method temporarily disabled during new augmentation system implementation
5845
+ */
5846
+ // augment(
5847
+ // action: IAugmentation | 'list' | 'enable' | 'disable' | 'unregister' | 'enable-type' | 'disable-type',
5848
+ // options?: string | { name?: string; type?: string }
5849
+ // ): this | any {
5850
+ // // Implementation temporarily disabled
5851
+ // }
5470
5852
  /**
5471
5853
  * UNIFIED API METHOD #9: Export - Extract your data in various formats
5472
5854
  * Export your brain's knowledge for backup, migration, or integration
@@ -5478,7 +5860,7 @@ export class BrainyData {
5478
5860
  const { format = 'json', includeVectors = false, includeMetadata = true, includeRelationships = true, filter = {}, limit } = options;
5479
5861
  // Get all data with optional filtering
5480
5862
  const nounsResult = await this.getNouns();
5481
- const allNouns = nounsResult.items || [];
5863
+ const allNouns = (nounsResult || []).filter((noun) => noun !== null);
5482
5864
  let exportData = [];
5483
5865
  // Apply filters and limits
5484
5866
  let nouns = allNouns;
@@ -5652,6 +6034,102 @@ export class BrainyData {
5652
6034
  disableAugmentationType(type) {
5653
6035
  return augmentationPipeline.disableAugmentationType(type);
5654
6036
  }
6037
+ // ===== Enhanced Clear Methods (2.0.0 API) =====
6038
+ /**
6039
+ * Clear only nouns from the database
6040
+ * @param options Clear options requiring force confirmation
6041
+ */
6042
+ /**
6043
+ * Clear all nouns from the database
6044
+ * @param options Options including force flag to skip confirmation
6045
+ */
6046
+ async clearNouns(options = {}) {
6047
+ if (!options.force) {
6048
+ throw new Error('clearNouns requires force: true option for safety');
6049
+ }
6050
+ await this.ensureInitialized();
6051
+ this.checkReadOnly();
6052
+ try {
6053
+ // Clear only nouns from storage and index
6054
+ if (this.storage) {
6055
+ // Use existing clear method for now - storage adapters don't have clearNouns
6056
+ await this.storage.clear();
6057
+ }
6058
+ // Clear HNSW index by creating a new one
6059
+ const { HNSWIndex } = await import('./hnsw/hnswIndex.js');
6060
+ this.hnswIndex = new HNSWIndex();
6061
+ // Clear search cache
6062
+ this.cache?.clear();
6063
+ }
6064
+ catch (error) {
6065
+ console.error('Failed to clear nouns:', error);
6066
+ throw new Error(`Failed to clear nouns: ${error}`);
6067
+ }
6068
+ }
6069
+ /**
6070
+ * Clear only verbs from the database
6071
+ * @param options Clear options requiring force confirmation
6072
+ */
6073
+ /**
6074
+ * Clear all verbs from the database
6075
+ * @param options Options including force flag to skip confirmation
6076
+ */
6077
+ async clearVerbs(options = {}) {
6078
+ if (!options.force) {
6079
+ throw new Error('clearVerbs requires force: true option for safety');
6080
+ }
6081
+ await this.ensureInitialized();
6082
+ this.checkReadOnly();
6083
+ try {
6084
+ // Clear only verbs from storage
6085
+ if (this.storage) {
6086
+ // Use existing clear method for now - storage adapters don't have clearVerbs
6087
+ // This would need custom implementation per storage adapter
6088
+ console.warn('clearVerbs not fully implemented - using full clear');
6089
+ await this.storage.clear();
6090
+ }
6091
+ }
6092
+ catch (error) {
6093
+ console.error('Failed to clear verbs:', error);
6094
+ throw new Error(`Failed to clear verbs: ${error}`);
6095
+ }
6096
+ }
6097
+ /**
6098
+ * Clear all data from the database (nouns and verbs)
6099
+ * @param options Clear options requiring force confirmation
6100
+ */
6101
+ /**
6102
+ * Clear all data from the database
6103
+ * @param options Options including force flag to skip confirmation
6104
+ */
6105
+ async clear(options = {}) {
6106
+ if (!options.force) {
6107
+ throw new Error('clearAll requires force: true option for safety');
6108
+ }
6109
+ await this.ensureInitialized();
6110
+ this.checkReadOnly();
6111
+ try {
6112
+ // Clear index
6113
+ await this.index.clear();
6114
+ // Clear storage
6115
+ await this.storage.clear();
6116
+ // Statistics collector is now handled by MetricsAugmentation
6117
+ // this.metrics = new StatisticsCollector()
6118
+ // Clear search cache since all data has been removed
6119
+ this.cache?.invalidateOnDataChange('delete');
6120
+ }
6121
+ catch (error) {
6122
+ console.error('Failed to clear all data:', error);
6123
+ throw new Error(`Failed to clear all data: ${error}`);
6124
+ }
6125
+ }
6126
+ /**
6127
+ * Clear all data from the database (alias for clear)
6128
+ * @param options Options including force flag to skip confirmation
6129
+ */
6130
+ async clearAll(options = {}) {
6131
+ return this.clear(options);
6132
+ }
5655
6133
  }
5656
6134
  // Export distance functions for convenience
5657
6135
  export { euclideanDistance, cosineDistance, manhattanDistance, dotProductDistance } from './utils/index.js';