@soulcraft/brainy 1.4.0 → 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +188 -0
- package/LICENSE +2 -2
- package/README.md +201 -596
- package/bin/brainy-interactive.js +564 -0
- package/bin/brainy-ts.js +18 -0
- package/bin/brainy.js +672 -81
- package/dist/augmentationPipeline.d.ts +48 -220
- package/dist/augmentationPipeline.js +60 -508
- package/dist/augmentationRegistry.d.ts +22 -31
- package/dist/augmentationRegistry.js +28 -79
- package/dist/augmentations/apiServerAugmentation.d.ts +108 -0
- package/dist/augmentations/apiServerAugmentation.js +502 -0
- package/dist/augmentations/batchProcessingAugmentation.d.ts +95 -0
- package/dist/augmentations/batchProcessingAugmentation.js +567 -0
- package/dist/augmentations/brainyAugmentation.d.ts +153 -0
- package/dist/augmentations/brainyAugmentation.js +145 -0
- package/dist/augmentations/cacheAugmentation.d.ts +105 -0
- package/dist/augmentations/cacheAugmentation.js +238 -0
- package/dist/augmentations/conduitAugmentations.d.ts +54 -156
- package/dist/augmentations/conduitAugmentations.js +156 -1082
- package/dist/augmentations/connectionPoolAugmentation.d.ts +62 -0
- package/dist/augmentations/connectionPoolAugmentation.js +316 -0
- package/dist/augmentations/defaultAugmentations.d.ts +53 -0
- package/dist/augmentations/defaultAugmentations.js +88 -0
- package/dist/augmentations/entityRegistryAugmentation.d.ts +126 -0
- package/dist/augmentations/entityRegistryAugmentation.js +386 -0
- package/dist/augmentations/indexAugmentation.d.ts +117 -0
- package/dist/augmentations/indexAugmentation.js +284 -0
- package/dist/augmentations/intelligentVerbScoringAugmentation.d.ts +152 -0
- package/dist/augmentations/intelligentVerbScoringAugmentation.js +554 -0
- package/dist/augmentations/metricsAugmentation.d.ts +202 -0
- package/dist/augmentations/metricsAugmentation.js +291 -0
- package/dist/augmentations/monitoringAugmentation.d.ts +94 -0
- package/dist/augmentations/monitoringAugmentation.js +227 -0
- package/dist/augmentations/neuralImport.d.ts +50 -117
- package/dist/augmentations/neuralImport.js +255 -629
- package/dist/augmentations/requestDeduplicatorAugmentation.d.ts +52 -0
- package/dist/augmentations/requestDeduplicatorAugmentation.js +162 -0
- package/dist/augmentations/serverSearchAugmentations.d.ts +43 -22
- package/dist/augmentations/serverSearchAugmentations.js +125 -72
- package/dist/augmentations/storageAugmentation.d.ts +54 -0
- package/dist/augmentations/storageAugmentation.js +93 -0
- package/dist/augmentations/storageAugmentations.d.ts +96 -0
- package/dist/augmentations/storageAugmentations.js +182 -0
- package/dist/augmentations/synapseAugmentation.d.ts +156 -0
- package/dist/augmentations/synapseAugmentation.js +312 -0
- package/dist/augmentations/walAugmentation.d.ts +108 -0
- package/dist/augmentations/walAugmentation.js +515 -0
- package/dist/brainyData.d.ts +404 -130
- package/dist/brainyData.js +1336 -855
- package/dist/chat/BrainyChat.d.ts +16 -8
- package/dist/chat/BrainyChat.js +60 -32
- package/dist/chat/ChatCLI.d.ts +1 -1
- package/dist/chat/ChatCLI.js +6 -6
- package/dist/cli/catalog.d.ts +3 -3
- package/dist/cli/catalog.js +116 -70
- package/dist/cli/commands/core.d.ts +61 -0
- package/dist/cli/commands/core.js +348 -0
- package/dist/cli/commands/neural.d.ts +25 -0
- package/dist/cli/commands/neural.js +508 -0
- package/dist/cli/commands/utility.d.ts +37 -0
- package/dist/cli/commands/utility.js +276 -0
- package/dist/cli/index.d.ts +7 -0
- package/dist/cli/index.js +167 -0
- package/dist/cli/interactive.d.ts +164 -0
- package/dist/cli/interactive.js +542 -0
- package/dist/cortex/neuralImport.js +5 -5
- package/dist/critical/model-guardian.js +11 -4
- package/dist/embeddings/lightweight-embedder.d.ts +23 -0
- package/dist/embeddings/lightweight-embedder.js +136 -0
- package/dist/embeddings/universal-memory-manager.d.ts +38 -0
- package/dist/embeddings/universal-memory-manager.js +206 -0
- package/dist/embeddings/worker-embedding.d.ts +7 -0
- package/dist/embeddings/worker-embedding.js +77 -0
- package/dist/embeddings/worker-manager.d.ts +28 -0
- package/dist/embeddings/worker-manager.js +162 -0
- package/dist/examples/basicUsage.js +7 -7
- package/dist/graph/pathfinding.d.ts +78 -0
- package/dist/graph/pathfinding.js +393 -0
- package/dist/hnsw/hnswIndex.d.ts +13 -0
- package/dist/hnsw/hnswIndex.js +35 -0
- package/dist/hnsw/hnswIndexOptimized.d.ts +1 -0
- package/dist/hnsw/hnswIndexOptimized.js +3 -0
- package/dist/index.d.ts +9 -11
- package/dist/index.js +21 -11
- package/dist/indices/fieldIndex.d.ts +76 -0
- package/dist/indices/fieldIndex.js +357 -0
- package/dist/mcp/brainyMCPAdapter.js +3 -2
- package/dist/mcp/mcpAugmentationToolset.js +11 -17
- package/dist/neural/embeddedPatterns.d.ts +41 -0
- package/dist/neural/embeddedPatterns.js +4044 -0
- package/dist/neural/naturalLanguageProcessor.d.ts +94 -0
- package/dist/neural/naturalLanguageProcessor.js +317 -0
- package/dist/neural/naturalLanguageProcessorStatic.d.ts +64 -0
- package/dist/neural/naturalLanguageProcessorStatic.js +151 -0
- package/dist/neural/neuralAPI.d.ts +255 -0
- package/dist/neural/neuralAPI.js +612 -0
- package/dist/neural/patternLibrary.d.ts +101 -0
- package/dist/neural/patternLibrary.js +313 -0
- package/dist/neural/patterns.d.ts +27 -0
- package/dist/neural/patterns.js +68 -0
- package/dist/neural/staticPatternMatcher.d.ts +35 -0
- package/dist/neural/staticPatternMatcher.js +153 -0
- package/dist/scripts/precomputePatternEmbeddings.d.ts +19 -0
- package/dist/scripts/precomputePatternEmbeddings.js +100 -0
- package/dist/storage/adapters/fileSystemStorage.d.ts +5 -0
- package/dist/storage/adapters/fileSystemStorage.js +20 -0
- package/dist/storage/adapters/s3CompatibleStorage.d.ts +5 -0
- package/dist/storage/adapters/s3CompatibleStorage.js +16 -0
- package/dist/storage/enhancedClearOperations.d.ts +83 -0
- package/dist/storage/enhancedClearOperations.js +345 -0
- package/dist/storage/storageFactory.js +31 -27
- package/dist/triple/TripleIntelligence.d.ts +134 -0
- package/dist/triple/TripleIntelligence.js +548 -0
- package/dist/types/augmentations.d.ts +45 -344
- package/dist/types/augmentations.js +5 -2
- package/dist/types/brainyDataInterface.d.ts +20 -10
- package/dist/types/graphTypes.d.ts +46 -0
- package/dist/types/graphTypes.js +16 -2
- package/dist/utils/BoundedRegistry.d.ts +29 -0
- package/dist/utils/BoundedRegistry.js +54 -0
- package/dist/utils/embedding.js +20 -3
- package/dist/utils/hybridModelManager.js +10 -5
- package/dist/utils/metadataFilter.d.ts +33 -19
- package/dist/utils/metadataFilter.js +58 -23
- package/dist/utils/metadataIndex.d.ts +37 -6
- package/dist/utils/metadataIndex.js +427 -64
- package/dist/utils/requestDeduplicator.d.ts +10 -0
- package/dist/utils/requestDeduplicator.js +24 -0
- package/dist/utils/unifiedCache.d.ts +103 -0
- package/dist/utils/unifiedCache.js +311 -0
- package/package.json +43 -128
- package/scripts/ensure-models.js +108 -0
- package/scripts/prepare-models.js +387 -0
- package/OFFLINE_MODELS.md +0 -56
- package/dist/intelligence/neuralEngine.d.ts +0 -207
- package/dist/intelligence/neuralEngine.js +0 -706
- package/dist/utils/modelLoader.d.ts +0 -32
- package/dist/utils/modelLoader.js +0 -219
- package/dist/utils/modelManager.d.ts +0 -77
- package/dist/utils/modelManager.js +0 -219
package/dist/brainyData.js
CHANGED
|
@@ -4,25 +4,52 @@
|
|
|
4
4
|
*/
|
|
5
5
|
import { v4 as uuidv4 } from './universal/uuid.js';
|
|
6
6
|
import { HNSWIndex } from './hnsw/hnswIndex.js';
|
|
7
|
-
import { ExecutionMode } from './augmentationPipeline.js';
|
|
8
7
|
import { HNSWIndexOptimized } from './hnsw/hnswIndexOptimized.js';
|
|
9
|
-
import { createStorage } from './storage/storageFactory.js';
|
|
10
8
|
import { cosineDistance, defaultEmbeddingFunction, cleanupWorkerPools, batchEmbed } from './utils/index.js';
|
|
11
9
|
import { getAugmentationVersion } from './utils/version.js';
|
|
12
10
|
import { matchesMetadataFilter } from './utils/metadataFilter.js';
|
|
13
|
-
import { MetadataIndexManager } from './utils/metadataIndex.js';
|
|
14
11
|
import { NounType, VerbType } from './types/graphTypes.js';
|
|
15
12
|
import { createServerSearchAugmentations } from './augmentations/serverSearchAugmentations.js';
|
|
16
|
-
import { IntelligentVerbScoring } from './augmentations/intelligentVerbScoring.js';
|
|
17
13
|
import { augmentationPipeline } from './augmentationPipeline.js';
|
|
18
14
|
import { prodLog } from './utils/logger.js';
|
|
19
15
|
import { prepareJsonForVectorization, extractFieldFromJson } from './utils/jsonProcessing.js';
|
|
20
|
-
import { DistributedConfigManager, HashPartitioner, OperationalModeFactory, DomainDetector
|
|
21
|
-
import { SearchCache } from './utils/searchCache.js';
|
|
16
|
+
import { DistributedConfigManager, HashPartitioner, OperationalModeFactory, DomainDetector } from './distributed/index.js';
|
|
22
17
|
import { CacheAutoConfigurator } from './utils/cacheAutoConfig.js';
|
|
23
|
-
import {
|
|
24
|
-
import {
|
|
18
|
+
import { RequestDeduplicator } from './utils/requestDeduplicator.js';
|
|
19
|
+
import { AugmentationRegistry } from './augmentations/brainyAugmentation.js';
|
|
20
|
+
import { WALAugmentation } from './augmentations/walAugmentation.js';
|
|
21
|
+
import { RequestDeduplicatorAugmentation } from './augmentations/requestDeduplicatorAugmentation.js';
|
|
22
|
+
import { ConnectionPoolAugmentation } from './augmentations/connectionPoolAugmentation.js';
|
|
23
|
+
import { BatchProcessingAugmentation } from './augmentations/batchProcessingAugmentation.js';
|
|
24
|
+
import { EntityRegistryAugmentation, AutoRegisterEntitiesAugmentation } from './augmentations/entityRegistryAugmentation.js';
|
|
25
|
+
import { createDefaultAugmentations } from './augmentations/defaultAugmentations.js';
|
|
26
|
+
// import { RealtimeStreamingAugmentation } from './augmentations/realtimeStreamingAugmentation.js'
|
|
27
|
+
import { IntelligentVerbScoringAugmentation } from './augmentations/intelligentVerbScoringAugmentation.js';
|
|
28
|
+
import { NeuralAPI } from './neural/neuralAPI.js';
|
|
29
|
+
import { TripleIntelligenceEngine } from './triple/TripleIntelligence.js';
|
|
25
30
|
export class BrainyData {
|
|
31
|
+
// REMOVED: HealthMonitor is now handled by MonitoringAugmentation
|
|
32
|
+
// Statistics collector
|
|
33
|
+
// REMOVED: StatisticsCollector is now handled by MetricsAugmentation
|
|
34
|
+
// Clean augmentation accessors for internal use
|
|
35
|
+
get cache() {
|
|
36
|
+
return this.augmentations.get('cache');
|
|
37
|
+
}
|
|
38
|
+
// IMPORTANT: this.index returns the HNSW vector index, NOT the metadata index!
|
|
39
|
+
// The metadata index is available through this.metadataIndex
|
|
40
|
+
get index() {
|
|
41
|
+
return this.hnswIndex;
|
|
42
|
+
}
|
|
43
|
+
// Metadata index for field-based queries (from IndexAugmentation)
|
|
44
|
+
get metadataIndex() {
|
|
45
|
+
return this.augmentations.get('index');
|
|
46
|
+
}
|
|
47
|
+
get metrics() {
|
|
48
|
+
return this.augmentations.get('metrics');
|
|
49
|
+
}
|
|
50
|
+
get monitoring() {
|
|
51
|
+
return this.augmentations.get('monitoring');
|
|
52
|
+
}
|
|
26
53
|
/**
|
|
27
54
|
* Get the vector dimensions
|
|
28
55
|
*/
|
|
@@ -43,18 +70,30 @@ export class BrainyData {
|
|
|
43
70
|
const config = this.index.getConfig();
|
|
44
71
|
return config.efConstruction || 200;
|
|
45
72
|
}
|
|
73
|
+
/**
|
|
74
|
+
* Check if BrainyData has been initialized
|
|
75
|
+
*/
|
|
76
|
+
get initialized() {
|
|
77
|
+
return this.isInitialized;
|
|
78
|
+
}
|
|
46
79
|
/**
|
|
47
80
|
* Create a new vector database
|
|
48
81
|
*/
|
|
49
82
|
constructor(config = {}) {
|
|
50
83
|
this.storage = null;
|
|
51
|
-
|
|
84
|
+
// REMOVED: MetadataIndex is now handled by IndexAugmentation
|
|
52
85
|
this.isInitialized = false;
|
|
53
86
|
this.isInitializing = false;
|
|
54
87
|
this.storageConfig = {};
|
|
55
88
|
this.useOptimizedIndex = false;
|
|
56
89
|
this.loggingConfig = { verbose: true };
|
|
57
90
|
this.defaultService = 'default';
|
|
91
|
+
// REMOVED: SearchCache is now handled by CacheAugmentation
|
|
92
|
+
/**
|
|
93
|
+
* Enterprise augmentation system
|
|
94
|
+
* Handles WAL, connection pooling, batching, streaming, and intelligent scoring
|
|
95
|
+
*/
|
|
96
|
+
this.augmentations = new AugmentationRegistry();
|
|
58
97
|
// Timeout and retry configuration
|
|
59
98
|
this.timeoutConfig = {};
|
|
60
99
|
this.retryConfig = {};
|
|
@@ -69,10 +108,10 @@ export class BrainyData {
|
|
|
69
108
|
this.maintenanceIntervals = [];
|
|
70
109
|
this.lastUpdateTime = 0;
|
|
71
110
|
this.lastKnownNounCount = 0;
|
|
72
|
-
// Remote server properties
|
|
111
|
+
// Remote server properties - TODO: Implement in post-2.0.0 release
|
|
73
112
|
this.remoteServerConfig = null;
|
|
74
|
-
|
|
75
|
-
|
|
113
|
+
// private serverSearchConduit: ServerSearchConduitAugmentation | null = null
|
|
114
|
+
// private serverConnection: WebSocketConnection | null = null
|
|
76
115
|
this.intelligentVerbScoring = null;
|
|
77
116
|
// Distributed mode properties
|
|
78
117
|
this.distributedConfig = null;
|
|
@@ -80,9 +119,6 @@ export class BrainyData {
|
|
|
80
119
|
this.partitioner = null;
|
|
81
120
|
this.operationalMode = null;
|
|
82
121
|
this.domainDetector = null;
|
|
83
|
-
this.healthMonitor = null;
|
|
84
|
-
// Statistics collector
|
|
85
|
-
this.statisticsCollector = new StatisticsCollector();
|
|
86
122
|
// Store config
|
|
87
123
|
this.config = config;
|
|
88
124
|
// Set dimensions to fixed value of 384 (all-MiniLM-L6-v2 dimension)
|
|
@@ -96,7 +132,7 @@ export class BrainyData {
|
|
|
96
132
|
hnswConfig.useDiskBasedIndex = true;
|
|
97
133
|
}
|
|
98
134
|
// Temporarily use base HNSW index for metadata filtering
|
|
99
|
-
this.
|
|
135
|
+
this.hnswIndex = new HNSWIndex(hnswConfig, this.distanceFunction);
|
|
100
136
|
this.useOptimizedIndex = false;
|
|
101
137
|
// Set storage if provided, otherwise it will be initialized in init()
|
|
102
138
|
this.storage = config.storageAdapter || null;
|
|
@@ -203,20 +239,165 @@ export class BrainyData {
|
|
|
203
239
|
prodLog.info(this.cacheAutoConfigurator.getConfigExplanation(autoConfig));
|
|
204
240
|
}
|
|
205
241
|
}
|
|
206
|
-
//
|
|
207
|
-
this.searchCache = new SearchCache(finalSearchCacheConfig)
|
|
208
|
-
//
|
|
209
|
-
|
|
210
|
-
//
|
|
211
|
-
|
|
212
|
-
this.intelligentVerbScoring = new IntelligentVerbScoring(config.intelligentVerbScoring);
|
|
213
|
-
this.intelligentVerbScoring.enabled = true;
|
|
214
|
-
}
|
|
242
|
+
// Search cache is now handled by CacheAugmentation
|
|
243
|
+
// this.searchCache = new SearchCache<T>(finalSearchCacheConfig)
|
|
244
|
+
// Keep reference for compatibility (will be set by augmentation)
|
|
245
|
+
// Augmentation system will be initialized in init() method
|
|
246
|
+
// Legacy systems completely replaced by augmentation architecture
|
|
247
|
+
// All intelligent systems now handled by augmentations
|
|
215
248
|
}
|
|
216
249
|
/**
|
|
217
250
|
* Check if the database is in read-only mode and throw an error if it is
|
|
218
251
|
* @throws Error if the database is in read-only mode
|
|
219
252
|
*/
|
|
253
|
+
/**
|
|
254
|
+
* Register default augmentations without initializing them
|
|
255
|
+
* Phase 1 of two-phase initialization
|
|
256
|
+
*/
|
|
257
|
+
registerDefaultAugmentations() {
|
|
258
|
+
// Register enterprise-grade augmentations in priority order
|
|
259
|
+
// Note: These are registered but NOT initialized yet (no context)
|
|
260
|
+
// Register core feature augmentations (previously hardcoded)
|
|
261
|
+
// These replace SearchCache, MetadataIndex, StatisticsCollector, HealthMonitor
|
|
262
|
+
const defaultAugs = createDefaultAugmentations({
|
|
263
|
+
cache: this.config.searchCache !== undefined ? this.config.searchCache : true,
|
|
264
|
+
index: this.config.metadataIndex !== undefined ? this.config.metadataIndex : true,
|
|
265
|
+
metrics: this.config.statistics !== false,
|
|
266
|
+
monitoring: Boolean(this.config.health || this.distributedConfig?.enabled)
|
|
267
|
+
});
|
|
268
|
+
for (const aug of defaultAugs) {
|
|
269
|
+
this.augmentations.register(aug);
|
|
270
|
+
}
|
|
271
|
+
// Priority 100: Critical system operations
|
|
272
|
+
// Disable WAL in test environments to avoid directory creation issues
|
|
273
|
+
const isTestEnvironment = process.env.NODE_ENV === 'test' || process.env.VITEST === 'true';
|
|
274
|
+
this.augmentations.register(new WALAugmentation({ enabled: !isTestEnvironment }));
|
|
275
|
+
this.augmentations.register(new ConnectionPoolAugmentation());
|
|
276
|
+
// Priority 95: Entity registry for fast external-ID to UUID mapping
|
|
277
|
+
this.augmentations.register(new EntityRegistryAugmentation({
|
|
278
|
+
maxCacheSize: this.config.entityCacheSize || 100000,
|
|
279
|
+
cacheTTL: this.config.entityCacheTTL || 300000,
|
|
280
|
+
persistence: 'hybrid',
|
|
281
|
+
indexedFields: ['did', 'handle', 'uri', 'external_id', 'id']
|
|
282
|
+
}));
|
|
283
|
+
// Priority 85: Auto-register entities after they're added
|
|
284
|
+
this.augmentations.register(new AutoRegisterEntitiesAugmentation());
|
|
285
|
+
// Priority 80: High-throughput batch processing
|
|
286
|
+
this.augmentations.register(new BatchProcessingAugmentation({
|
|
287
|
+
maxBatchSize: this.config.batchSize || 1000,
|
|
288
|
+
maxWaitTime: this.config.batchWaitTime || 100
|
|
289
|
+
}));
|
|
290
|
+
// Priority 50: Performance optimizations
|
|
291
|
+
this.augmentations.register(new RequestDeduplicatorAugmentation({
|
|
292
|
+
ttl: 5000,
|
|
293
|
+
maxSize: 1000
|
|
294
|
+
}));
|
|
295
|
+
// Priority 10: Enhancement features
|
|
296
|
+
const intelligentVerbAugmentation = new IntelligentVerbScoringAugmentation(this.config.intelligentVerbScoring || { enabled: false });
|
|
297
|
+
this.augmentations.register(intelligentVerbAugmentation);
|
|
298
|
+
// Store reference if intelligent verb scoring is enabled
|
|
299
|
+
if (this.config.intelligentVerbScoring?.enabled) {
|
|
300
|
+
this.intelligentVerbScoring = intelligentVerbAugmentation.getScoring();
|
|
301
|
+
}
|
|
302
|
+
}
|
|
303
|
+
/**
|
|
304
|
+
* Resolve storage from augmentation or config
|
|
305
|
+
* Phase 2 of two-phase initialization
|
|
306
|
+
*/
|
|
307
|
+
async resolveStorage() {
|
|
308
|
+
// Check if storage augmentation is registered
|
|
309
|
+
const storageAug = this.augmentations.findByOperation('storage');
|
|
310
|
+
if (storageAug && 'provideStorage' in storageAug) {
|
|
311
|
+
// Get storage from augmentation
|
|
312
|
+
this.storage = await storageAug.provideStorage();
|
|
313
|
+
if (this.loggingConfig?.verbose) {
|
|
314
|
+
console.log('Using storage from augmentation:', storageAug.name);
|
|
315
|
+
}
|
|
316
|
+
}
|
|
317
|
+
else if (!this.storage) {
|
|
318
|
+
// No storage augmentation and no provided adapter
|
|
319
|
+
// Use zero-config approach
|
|
320
|
+
// Import storage augmentation helpers
|
|
321
|
+
const { DynamicStorageAugmentation, createStorageAugmentationFromConfig } = await import('./augmentations/storageAugmentation.js');
|
|
322
|
+
const { createAutoStorageAugmentation } = await import('./augmentations/storageAugmentations.js');
|
|
323
|
+
// Build storage options from config
|
|
324
|
+
let storageOptions = {
|
|
325
|
+
...this.storageConfig,
|
|
326
|
+
requestPersistentStorage: this.requestPersistentStorage
|
|
327
|
+
};
|
|
328
|
+
// Add cache configuration if provided
|
|
329
|
+
if (this.cacheConfig) {
|
|
330
|
+
storageOptions.cacheConfig = {
|
|
331
|
+
...this.cacheConfig,
|
|
332
|
+
readOnly: this.readOnly
|
|
333
|
+
};
|
|
334
|
+
}
|
|
335
|
+
// Ensure s3Storage has all required fields if it's provided
|
|
336
|
+
if (storageOptions.s3Storage) {
|
|
337
|
+
if (storageOptions.s3Storage.bucketName &&
|
|
338
|
+
storageOptions.s3Storage.accessKeyId &&
|
|
339
|
+
storageOptions.s3Storage.secretAccessKey) {
|
|
340
|
+
// All required fields are present
|
|
341
|
+
}
|
|
342
|
+
else {
|
|
343
|
+
// Missing required fields, remove s3Storage
|
|
344
|
+
const { s3Storage, ...rest } = storageOptions;
|
|
345
|
+
storageOptions = rest;
|
|
346
|
+
console.warn('Ignoring s3Storage configuration due to missing required fields');
|
|
347
|
+
}
|
|
348
|
+
}
|
|
349
|
+
// Check if specific storage is configured
|
|
350
|
+
if (storageOptions.s3Storage || storageOptions.r2Storage ||
|
|
351
|
+
storageOptions.gcsStorage || storageOptions.forceMemoryStorage ||
|
|
352
|
+
storageOptions.forceFileSystemStorage) {
|
|
353
|
+
// Create storage from config
|
|
354
|
+
const { createStorage } = await import('./storage/storageFactory.js');
|
|
355
|
+
this.storage = await createStorage(storageOptions);
|
|
356
|
+
// Wrap in augmentation for consistency
|
|
357
|
+
const wrapper = new DynamicStorageAugmentation(this.storage);
|
|
358
|
+
this.augmentations.register(wrapper);
|
|
359
|
+
}
|
|
360
|
+
else {
|
|
361
|
+
// Zero-config: auto-select based on environment
|
|
362
|
+
const autoAug = await createAutoStorageAugmentation({
|
|
363
|
+
rootDirectory: storageOptions.rootDirectory,
|
|
364
|
+
requestPersistentStorage: storageOptions.requestPersistentStorage
|
|
365
|
+
});
|
|
366
|
+
this.augmentations.register(autoAug);
|
|
367
|
+
this.storage = await autoAug.provideStorage();
|
|
368
|
+
}
|
|
369
|
+
}
|
|
370
|
+
// Initialize storage
|
|
371
|
+
if (this.storage) {
|
|
372
|
+
await this.storage.init();
|
|
373
|
+
}
|
|
374
|
+
else {
|
|
375
|
+
throw new Error('Failed to resolve storage');
|
|
376
|
+
}
|
|
377
|
+
}
|
|
378
|
+
/**
|
|
379
|
+
* Initialize the augmentation system with full context
|
|
380
|
+
* Phase 3 of two-phase initialization
|
|
381
|
+
*/
|
|
382
|
+
async initializeAugmentations() {
|
|
383
|
+
// Create augmentation context
|
|
384
|
+
const context = {
|
|
385
|
+
brain: this,
|
|
386
|
+
storage: this.storage,
|
|
387
|
+
config: this.config,
|
|
388
|
+
log: (message, level = 'info') => {
|
|
389
|
+
if (this.loggingConfig?.verbose || level !== 'info') {
|
|
390
|
+
const prefix = level === 'error' ? '❌' : level === 'warn' ? '⚠️' : '✅';
|
|
391
|
+
console.log(`${prefix} ${message}`);
|
|
392
|
+
}
|
|
393
|
+
}
|
|
394
|
+
};
|
|
395
|
+
// Initialize all augmentations (already registered in registerDefaultAugmentations)
|
|
396
|
+
await this.augmentations.initialize(context);
|
|
397
|
+
if (this.loggingConfig?.verbose) {
|
|
398
|
+
console.log('🚀 New augmentation system initialized successfully');
|
|
399
|
+
}
|
|
400
|
+
}
|
|
220
401
|
checkReadOnly() {
|
|
221
402
|
if (this.readOnly) {
|
|
222
403
|
throw new Error('Cannot perform write operation: database is in read-only mode');
|
|
@@ -329,12 +510,13 @@ export class BrainyData {
|
|
|
329
510
|
* Start metadata index maintenance
|
|
330
511
|
*/
|
|
331
512
|
startMetadataIndexMaintenance() {
|
|
332
|
-
|
|
513
|
+
const metaIndex = this.metadataIndex;
|
|
514
|
+
if (!metaIndex)
|
|
333
515
|
return;
|
|
334
516
|
// Flush index periodically to persist changes
|
|
335
517
|
const flushInterval = setInterval(async () => {
|
|
336
518
|
try {
|
|
337
|
-
await
|
|
519
|
+
await metaIndex.flush();
|
|
338
520
|
}
|
|
339
521
|
catch (error) {
|
|
340
522
|
prodLog.warn('Error flushing metadata index:', error);
|
|
@@ -397,7 +579,7 @@ export class BrainyData {
|
|
|
397
579
|
}
|
|
398
580
|
}
|
|
399
581
|
// Cleanup expired cache entries (defensive mechanism for distributed scenarios)
|
|
400
|
-
const expiredCount = this.
|
|
582
|
+
const expiredCount = this.cache?.cleanupExpiredEntries() || 0;
|
|
401
583
|
if (expiredCount > 0 && this.loggingConfig?.verbose) {
|
|
402
584
|
prodLog.debug(`Cleaned up ${expiredCount} expired cache entries`);
|
|
403
585
|
}
|
|
@@ -484,7 +666,7 @@ export class BrainyData {
|
|
|
484
666
|
}
|
|
485
667
|
// Invalidate search cache if any external changes were detected
|
|
486
668
|
if (addedCount > 0 || updatedCount > 0 || deletedCount > 0) {
|
|
487
|
-
this.
|
|
669
|
+
this.cache?.invalidateOnDataChange('update');
|
|
488
670
|
if (this.loggingConfig?.verbose) {
|
|
489
671
|
console.log('Search cache invalidated due to external data changes');
|
|
490
672
|
}
|
|
@@ -545,7 +727,7 @@ export class BrainyData {
|
|
|
545
727
|
this.lastKnownNounCount = currentCount;
|
|
546
728
|
// Invalidate search cache if new nouns were detected
|
|
547
729
|
if (totalNewNouns > 0) {
|
|
548
|
-
this.
|
|
730
|
+
this.cache?.invalidateOnDataChange('add');
|
|
549
731
|
if (this.loggingConfig?.verbose) {
|
|
550
732
|
console.log('Search cache invalidated due to external data changes');
|
|
551
733
|
}
|
|
@@ -573,7 +755,8 @@ export class BrainyData {
|
|
|
573
755
|
*/
|
|
574
756
|
async provideFeedbackForVerbScoring(sourceId, targetId, verbType, feedbackWeight, feedbackConfidence, feedbackType = 'correction') {
|
|
575
757
|
if (this.intelligentVerbScoring?.enabled) {
|
|
576
|
-
|
|
758
|
+
// The augmentation doesn't use feedbackConfidence separately
|
|
759
|
+
await this.intelligentVerbScoring.provideFeedback(sourceId, targetId, verbType, feedbackWeight, feedbackType);
|
|
577
760
|
}
|
|
578
761
|
}
|
|
579
762
|
/**
|
|
@@ -614,9 +797,9 @@ export class BrainyData {
|
|
|
614
797
|
// Check each type of augmentation
|
|
615
798
|
for (const type of augmentationTypes) {
|
|
616
799
|
const augmentations = augmentationPipeline.getAugmentationsByType(type);
|
|
617
|
-
// Find the first enabled
|
|
800
|
+
// Find the first augmentation (all registered augmentations are considered enabled)
|
|
618
801
|
for (const augmentation of augmentations) {
|
|
619
|
-
if (augmentation
|
|
802
|
+
if (augmentation) {
|
|
620
803
|
return augmentation.name;
|
|
621
804
|
}
|
|
622
805
|
}
|
|
@@ -656,23 +839,24 @@ export class BrainyData {
|
|
|
656
839
|
return;
|
|
657
840
|
}
|
|
658
841
|
this.isInitializing = true;
|
|
659
|
-
// CRITICAL:
|
|
660
|
-
//
|
|
661
|
-
|
|
662
|
-
if (typeof this.embeddingFunction === 'function') {
|
|
842
|
+
// CRITICAL: Initialize universal memory manager ONLY for default embedding function
|
|
843
|
+
// This preserves custom embedding functions (like test mocks)
|
|
844
|
+
if (typeof this.embeddingFunction === 'function' && this.embeddingFunction === defaultEmbeddingFunction) {
|
|
663
845
|
try {
|
|
664
|
-
const {
|
|
665
|
-
await
|
|
666
|
-
console.log('✅
|
|
846
|
+
const { universalMemoryManager } = await import('./embeddings/universal-memory-manager.js');
|
|
847
|
+
this.embeddingFunction = await universalMemoryManager.getEmbeddingFunction();
|
|
848
|
+
console.log('✅ UNIVERSAL: Memory-safe embedding system initialized');
|
|
667
849
|
}
|
|
668
850
|
catch (error) {
|
|
669
|
-
console.error('🚨 CRITICAL:
|
|
670
|
-
console.error('
|
|
671
|
-
console.
|
|
672
|
-
|
|
673
|
-
throw error;
|
|
851
|
+
console.error('🚨 CRITICAL: Universal memory manager initialization failed!');
|
|
852
|
+
console.error('Falling back to standard embedding with potential memory issues.');
|
|
853
|
+
console.warn('Consider reducing usage or restarting process periodically.');
|
|
854
|
+
// Continue with default function - better than crashing
|
|
674
855
|
}
|
|
675
856
|
}
|
|
857
|
+
else if (this.embeddingFunction !== defaultEmbeddingFunction) {
|
|
858
|
+
console.log('✅ CUSTOM: Using custom embedding function (test or production override)');
|
|
859
|
+
}
|
|
676
860
|
try {
|
|
677
861
|
// Pre-load the embedding model early to ensure it's always available
|
|
678
862
|
// This helps prevent issues with the Universal Sentence Encoder not being loaded
|
|
@@ -705,48 +889,19 @@ export class BrainyData {
|
|
|
705
889
|
// The application will need to handle missing embedding functionality
|
|
706
890
|
}
|
|
707
891
|
}
|
|
708
|
-
//
|
|
709
|
-
|
|
710
|
-
|
|
711
|
-
|
|
712
|
-
|
|
713
|
-
|
|
714
|
-
};
|
|
715
|
-
// Add cache configuration if provided
|
|
716
|
-
if (this.cacheConfig) {
|
|
717
|
-
storageOptions.cacheConfig = {
|
|
718
|
-
...this.cacheConfig,
|
|
719
|
-
// Pass read-only flag to optimize cache behavior
|
|
720
|
-
readOnly: this.readOnly
|
|
721
|
-
};
|
|
722
|
-
}
|
|
723
|
-
// Ensure s3Storage has all required fields if it's provided
|
|
724
|
-
if (storageOptions.s3Storage) {
|
|
725
|
-
// Only include s3Storage if all required fields are present
|
|
726
|
-
if (storageOptions.s3Storage.bucketName &&
|
|
727
|
-
storageOptions.s3Storage.accessKeyId &&
|
|
728
|
-
storageOptions.s3Storage.secretAccessKey) {
|
|
729
|
-
// All required fields are present, keep s3Storage as is
|
|
730
|
-
}
|
|
731
|
-
else {
|
|
732
|
-
// Missing required fields, remove s3Storage to avoid type errors
|
|
733
|
-
const { s3Storage, ...rest } = storageOptions;
|
|
734
|
-
storageOptions = rest;
|
|
735
|
-
console.warn('Ignoring s3Storage configuration due to missing required fields');
|
|
736
|
-
}
|
|
737
|
-
}
|
|
738
|
-
// Use type assertion to tell TypeScript that storageOptions conforms to StorageOptions
|
|
739
|
-
this.storage = await createStorage(storageOptions);
|
|
740
|
-
}
|
|
741
|
-
// Initialize storage
|
|
742
|
-
await this.storage.init();
|
|
892
|
+
// Phase 1: Register default augmentations (without initialization)
|
|
893
|
+
this.registerDefaultAugmentations();
|
|
894
|
+
// Phase 2: Resolve storage (either from augmentation or config)
|
|
895
|
+
await this.resolveStorage();
|
|
896
|
+
// Phase 3: Initialize all augmentations with full context
|
|
897
|
+
await this.initializeAugmentations();
|
|
743
898
|
// Initialize distributed mode if configured
|
|
744
899
|
if (this.distributedConfig) {
|
|
745
900
|
await this.initializeDistributedMode();
|
|
746
901
|
}
|
|
747
902
|
// If using optimized index, set the storage adapter
|
|
748
|
-
if (this.useOptimizedIndex && this.
|
|
749
|
-
this.
|
|
903
|
+
if (this.useOptimizedIndex && this.hnswIndex instanceof HNSWIndexOptimized) {
|
|
904
|
+
this.hnswIndex.setStorage(this.storage);
|
|
750
905
|
}
|
|
751
906
|
// In write-only mode, skip loading the index into memory
|
|
752
907
|
if (this.writeOnly) {
|
|
@@ -760,11 +915,11 @@ export class BrainyData {
|
|
|
760
915
|
console.log('Database is in read-only mode with lazy loading enabled, skipping initial full load');
|
|
761
916
|
}
|
|
762
917
|
// Just initialize an empty index
|
|
763
|
-
this.
|
|
918
|
+
this.hnswIndex.clear();
|
|
764
919
|
}
|
|
765
920
|
else {
|
|
766
921
|
// Clear the index and load nouns using pagination
|
|
767
|
-
this.
|
|
922
|
+
this.hnswIndex.clear();
|
|
768
923
|
let offset = 0;
|
|
769
924
|
const limit = 100;
|
|
770
925
|
let hasMore = true;
|
|
@@ -804,21 +959,25 @@ export class BrainyData {
|
|
|
804
959
|
try {
|
|
805
960
|
const existingStats = await this.storage.getStatistics();
|
|
806
961
|
if (existingStats) {
|
|
807
|
-
this.
|
|
962
|
+
this.metrics.mergeFromStorage(existingStats);
|
|
808
963
|
}
|
|
809
964
|
}
|
|
810
965
|
catch (e) {
|
|
811
966
|
// Ignore errors loading existing statistics
|
|
812
967
|
}
|
|
813
968
|
// Initialize metadata index unless in read-only mode
|
|
969
|
+
// Metadata index is now handled by IndexAugmentation
|
|
814
970
|
// Write-only mode NEEDS metadata indexing for search capability!
|
|
815
971
|
if (!this.readOnly) {
|
|
816
|
-
this.
|
|
972
|
+
// this.index = new MetadataIndexManager(
|
|
973
|
+
// this.storage!,
|
|
974
|
+
// this.config.metadataIndex
|
|
975
|
+
// )
|
|
817
976
|
// Check if we need to rebuild the index (for existing data)
|
|
818
977
|
// Skip rebuild for memory storage (starts empty) or when in read-only mode
|
|
819
978
|
// Also skip if index already has entries
|
|
820
979
|
const isMemoryStorage = this.storage?.constructor?.name === 'MemoryStorage';
|
|
821
|
-
const stats = await this.metadataIndex
|
|
980
|
+
const stats = await this.metadataIndex?.getStats?.() || { totalEntries: 0 };
|
|
822
981
|
if (!isMemoryStorage && !this.readOnly && stats.totalEntries === 0) {
|
|
823
982
|
// Check if we have existing data that needs indexing
|
|
824
983
|
// Use a simple check to avoid expensive operations
|
|
@@ -831,9 +990,9 @@ export class BrainyData {
|
|
|
831
990
|
if (this.loggingConfig?.verbose) {
|
|
832
991
|
console.log('🔄 Rebuilding metadata index for existing data...');
|
|
833
992
|
}
|
|
834
|
-
await this.metadataIndex
|
|
993
|
+
await this.metadataIndex?.rebuild?.();
|
|
835
994
|
if (this.loggingConfig?.verbose) {
|
|
836
|
-
const newStats = await this.metadataIndex
|
|
995
|
+
const newStats = await this.metadataIndex?.getStats?.() || { totalEntries: 0 };
|
|
837
996
|
console.log(`✅ Metadata index rebuilt: ${newStats.totalEntries} entries, ${newStats.fieldsIndexed.length} fields`);
|
|
838
997
|
}
|
|
839
998
|
}
|
|
@@ -853,13 +1012,7 @@ export class BrainyData {
|
|
|
853
1012
|
}
|
|
854
1013
|
}
|
|
855
1014
|
}
|
|
856
|
-
//
|
|
857
|
-
if (this.intelligentVerbScoring) {
|
|
858
|
-
await this.intelligentVerbScoring.initialize();
|
|
859
|
-
this.intelligentVerbScoring.setBrainyInstance(this);
|
|
860
|
-
// Register with augmentation pipeline
|
|
861
|
-
augmentationPipeline.register(this.intelligentVerbScoring);
|
|
862
|
-
}
|
|
1015
|
+
// Intelligent verb scoring is now initialized through the augmentation system
|
|
863
1016
|
// Initialize default augmentations (Neural Import, etc.)
|
|
864
1017
|
// TODO: Fix TypeScript issues in v0.57.0
|
|
865
1018
|
// try {
|
|
@@ -877,7 +1030,7 @@ export class BrainyData {
|
|
|
877
1030
|
// Start real-time updates if enabled
|
|
878
1031
|
this.startRealtimeUpdates();
|
|
879
1032
|
// Start metadata index maintenance
|
|
880
|
-
if (this.
|
|
1033
|
+
if (this.index) {
|
|
881
1034
|
this.startMetadataIndexMaintenance();
|
|
882
1035
|
}
|
|
883
1036
|
}
|
|
@@ -945,9 +1098,9 @@ export class BrainyData {
|
|
|
945
1098
|
}
|
|
946
1099
|
// Initialize domain detector
|
|
947
1100
|
this.domainDetector = new DomainDetector();
|
|
948
|
-
//
|
|
949
|
-
this.
|
|
950
|
-
this.
|
|
1101
|
+
// Health monitor is now handled by MonitoringAugmentation
|
|
1102
|
+
// this.monitoring = new HealthMonitor(this.configManager)
|
|
1103
|
+
// this.monitoring.start()
|
|
951
1104
|
// Set up config update listener
|
|
952
1105
|
this.configManager.setOnConfigUpdate((config) => {
|
|
953
1106
|
this.handleDistributedConfigUpdate(config);
|
|
@@ -974,10 +1127,7 @@ export class BrainyData {
|
|
|
974
1127
|
* @returns Health status if distributed mode is enabled
|
|
975
1128
|
*/
|
|
976
1129
|
getHealthStatus() {
|
|
977
|
-
|
|
978
|
-
return this.healthMonitor.getHealthEndpointData();
|
|
979
|
-
}
|
|
980
|
-
return null;
|
|
1130
|
+
return this.monitoring?.getHealthStatus() || null;
|
|
981
1131
|
}
|
|
982
1132
|
/**
|
|
983
1133
|
* Connect to a remote Brainy server for search operations
|
|
@@ -993,9 +1143,9 @@ export class BrainyData {
|
|
|
993
1143
|
protocols,
|
|
994
1144
|
localDb: this
|
|
995
1145
|
});
|
|
996
|
-
// Store
|
|
997
|
-
this.serverSearchConduit = conduit
|
|
998
|
-
this.serverConnection = connection
|
|
1146
|
+
// TODO: Store conduit and connection (post-2.0.0 feature)
|
|
1147
|
+
// this.serverSearchConduit = conduit
|
|
1148
|
+
// this.serverConnection = connection
|
|
999
1149
|
return connection;
|
|
1000
1150
|
}
|
|
1001
1151
|
catch (error) {
|
|
@@ -1066,6 +1216,15 @@ export class BrainyData {
|
|
|
1066
1216
|
]
|
|
1067
1217
|
});
|
|
1068
1218
|
vector = await this.embeddingFunction(preparedText);
|
|
1219
|
+
// IMPORTANT: When an object is passed as data and no metadata is provided,
|
|
1220
|
+
// use the object AS the metadata too. This is expected behavior for the API.
|
|
1221
|
+
// Users can pass either:
|
|
1222
|
+
// 1. addNoun(string, metadata) - vectorize string, store metadata
|
|
1223
|
+
// 2. addNoun(object) - vectorize object text, store object as metadata
|
|
1224
|
+
// 3. addNoun(object, metadata) - vectorize object text, store provided metadata
|
|
1225
|
+
if (!metadata) {
|
|
1226
|
+
metadata = vectorOrData;
|
|
1227
|
+
}
|
|
1069
1228
|
// Track field names for this JSON document
|
|
1070
1229
|
const service = this.getServiceName(options);
|
|
1071
1230
|
if (this.storage) {
|
|
@@ -1148,20 +1307,21 @@ export class BrainyData {
|
|
|
1148
1307
|
};
|
|
1149
1308
|
}
|
|
1150
1309
|
else {
|
|
1151
|
-
// Normal mode: Add to index first
|
|
1152
|
-
await this.
|
|
1153
|
-
// Get the noun from the index
|
|
1154
|
-
const indexNoun = this.
|
|
1310
|
+
// Normal mode: Add to HNSW index first
|
|
1311
|
+
await this.hnswIndex.addItem({ id, vector, metadata });
|
|
1312
|
+
// Get the noun from the HNSW index
|
|
1313
|
+
const indexNoun = this.hnswIndex.getNouns().get(id);
|
|
1155
1314
|
if (!indexNoun) {
|
|
1156
1315
|
throw new Error(`Failed to retrieve newly created noun with ID ${id}`);
|
|
1157
1316
|
}
|
|
1158
1317
|
noun = indexNoun;
|
|
1159
1318
|
}
|
|
1160
|
-
// Save noun to storage
|
|
1161
|
-
await this.
|
|
1162
|
-
|
|
1163
|
-
|
|
1164
|
-
|
|
1319
|
+
// Save noun to storage using augmentation system
|
|
1320
|
+
await this.augmentations.execute('saveNoun', { noun, options }, async () => {
|
|
1321
|
+
await this.storage.saveNoun(noun);
|
|
1322
|
+
const service = this.getServiceName(options);
|
|
1323
|
+
await this.storage.incrementStatistic('noun', service);
|
|
1324
|
+
});
|
|
1165
1325
|
// Save metadata if provided and not empty
|
|
1166
1326
|
if (metadata !== undefined) {
|
|
1167
1327
|
// Skip saving if metadata is an empty object
|
|
@@ -1244,8 +1404,8 @@ export class BrainyData {
|
|
|
1244
1404
|
}
|
|
1245
1405
|
await this.storage.saveMetadata(id, metadataToSave);
|
|
1246
1406
|
// Update metadata index (write-only mode should build indices!)
|
|
1247
|
-
if (this.
|
|
1248
|
-
await this.metadataIndex
|
|
1407
|
+
if (this.index && !this.frozen) {
|
|
1408
|
+
await this.metadataIndex?.addToIndex?.(id, metadataToSave);
|
|
1249
1409
|
}
|
|
1250
1410
|
// Track metadata statistics
|
|
1251
1411
|
const metadataService = this.getServiceName(options);
|
|
@@ -1254,19 +1414,18 @@ export class BrainyData {
|
|
|
1254
1414
|
if (metadataToSave &&
|
|
1255
1415
|
typeof metadataToSave === 'object' &&
|
|
1256
1416
|
'noun' in metadataToSave) {
|
|
1257
|
-
this.
|
|
1417
|
+
this.metrics.trackContentType(metadataToSave.noun);
|
|
1258
1418
|
}
|
|
1259
|
-
// Track update timestamp
|
|
1260
|
-
this.statisticsCollector.trackUpdate();
|
|
1419
|
+
// Track update timestamp (handled by metrics augmentation)
|
|
1261
1420
|
}
|
|
1262
1421
|
}
|
|
1263
1422
|
// Update HNSW index size with actual index size
|
|
1264
1423
|
const indexSize = this.index.size();
|
|
1265
1424
|
await this.storage.updateHnswIndexSize(indexSize);
|
|
1266
1425
|
// Update health metrics if in distributed mode
|
|
1267
|
-
if (this.
|
|
1426
|
+
if (this.monitoring) {
|
|
1268
1427
|
const vectorCount = await this.getNounCount();
|
|
1269
|
-
this.
|
|
1428
|
+
this.monitoring.updateVectorCount(vectorCount);
|
|
1270
1429
|
}
|
|
1271
1430
|
// If addToRemote is true and we're connected to a remote server, add to remote as well
|
|
1272
1431
|
if (options.addToRemote && this.isConnectedToRemoteServer()) {
|
|
@@ -1278,7 +1437,7 @@ export class BrainyData {
|
|
|
1278
1437
|
}
|
|
1279
1438
|
}
|
|
1280
1439
|
// Invalidate search cache since data has changed
|
|
1281
|
-
this.
|
|
1440
|
+
this.cache?.invalidateOnDataChange('add');
|
|
1282
1441
|
// Determine processing mode
|
|
1283
1442
|
const processingMode = options.process || 'auto';
|
|
1284
1443
|
let shouldProcessNeurally = false;
|
|
@@ -1293,8 +1452,9 @@ export class BrainyData {
|
|
|
1293
1452
|
// 🧠 AI Processing (Neural Import) - Based on processing mode
|
|
1294
1453
|
if (shouldProcessNeurally) {
|
|
1295
1454
|
try {
|
|
1296
|
-
// Execute
|
|
1297
|
-
|
|
1455
|
+
// Execute augmentation pipeline for data processing
|
|
1456
|
+
// Note: Augmentations will be called via this.augmentations.execute during the actual add operation
|
|
1457
|
+
// This replaces the legacy SENSE pipeline
|
|
1298
1458
|
if (this.loggingConfig?.verbose) {
|
|
1299
1459
|
console.log(`🧠 AI processing completed for data: ${id}`);
|
|
1300
1460
|
}
|
|
@@ -1309,39 +1469,14 @@ export class BrainyData {
|
|
|
1309
1469
|
catch (error) {
|
|
1310
1470
|
console.error('Failed to add vector:', error);
|
|
1311
1471
|
// Track error in health monitor
|
|
1312
|
-
if (this.
|
|
1313
|
-
this.
|
|
1472
|
+
if (this.monitoring) {
|
|
1473
|
+
this.monitoring.recordRequest(0, true);
|
|
1314
1474
|
}
|
|
1315
1475
|
throw new Error(`Failed to add vector: ${error}`);
|
|
1316
1476
|
}
|
|
1317
1477
|
}
|
|
1318
|
-
|
|
1319
|
-
|
|
1320
|
-
* This is a convenience method for adding text data with metadata
|
|
1321
|
-
* @param text Text data to add
|
|
1322
|
-
* @param metadata Metadata to associate with the text
|
|
1323
|
-
* @param options Additional options
|
|
1324
|
-
* @returns The ID of the added item
|
|
1325
|
-
*/
|
|
1326
|
-
async addItem(text, metadata, options = {}) {
|
|
1327
|
-
// Use the existing add method with forceEmbed to ensure text is embedded
|
|
1328
|
-
return this.add(text, metadata, { ...options, forceEmbed: true });
|
|
1329
|
-
}
|
|
1330
|
-
/**
|
|
1331
|
-
* Add data to both local and remote Brainy instances
|
|
1332
|
-
* @param vectorOrData Vector or data to add
|
|
1333
|
-
* @param metadata Optional metadata to associate with the vector
|
|
1334
|
-
* @param options Additional options
|
|
1335
|
-
* @returns The ID of the added vector
|
|
1336
|
-
*/
|
|
1337
|
-
async addToBoth(vectorOrData, metadata, options = {}) {
|
|
1338
|
-
// Check if connected to a remote server
|
|
1339
|
-
if (!this.isConnectedToRemoteServer()) {
|
|
1340
|
-
throw new Error('Not connected to a remote server. Call connectToRemoteServer() first.');
|
|
1341
|
-
}
|
|
1342
|
-
// Add to local with addToRemote option
|
|
1343
|
-
return this.add(vectorOrData, metadata, { ...options, addToRemote: true });
|
|
1344
|
-
}
|
|
1478
|
+
// REMOVED: addItem() - Use addNoun() instead (cleaner 2.0 API)
|
|
1479
|
+
// REMOVED: addToBoth() - Remote server functionality moved to post-2.0.0
|
|
1345
1480
|
/**
|
|
1346
1481
|
* Add a vector to the remote server
|
|
1347
1482
|
* @param id ID of the vector to add
|
|
@@ -1355,14 +1490,23 @@ export class BrainyData {
|
|
|
1355
1490
|
return false;
|
|
1356
1491
|
}
|
|
1357
1492
|
try {
|
|
1358
|
-
|
|
1359
|
-
|
|
1360
|
-
|
|
1361
|
-
//
|
|
1362
|
-
|
|
1363
|
-
|
|
1364
|
-
|
|
1365
|
-
|
|
1493
|
+
// TODO: Remote server operations (post-2.0.0 feature)
|
|
1494
|
+
// if (!this.serverSearchConduit || !this.serverConnection) {
|
|
1495
|
+
// throw new Error(
|
|
1496
|
+
// 'Server search conduit or connection is not initialized'
|
|
1497
|
+
// )
|
|
1498
|
+
// }
|
|
1499
|
+
// TODO: Add to remote server
|
|
1500
|
+
// const addResult = await this.serverSearchConduit.addToBoth(
|
|
1501
|
+
// this.serverConnection.connectionId,
|
|
1502
|
+
// vector,
|
|
1503
|
+
// metadata
|
|
1504
|
+
// )
|
|
1505
|
+
throw new Error('Remote server functionality not yet implemented in Brainy 2.0.0');
|
|
1506
|
+
// TODO: Handle remote add result (post-2.0.0 feature)
|
|
1507
|
+
// if (!addResult.success) {
|
|
1508
|
+
// throw new Error(`Remote add failed: ${addResult.error}`)
|
|
1509
|
+
// }
|
|
1366
1510
|
return true;
|
|
1367
1511
|
}
|
|
1368
1512
|
catch (error) {
|
|
@@ -1376,7 +1520,13 @@ export class BrainyData {
|
|
|
1376
1520
|
* @param options Additional options
|
|
1377
1521
|
* @returns Array of IDs for the added items
|
|
1378
1522
|
*/
|
|
1379
|
-
|
|
1523
|
+
/**
|
|
1524
|
+
* Add multiple nouns in batch
|
|
1525
|
+
* @param items Array of nouns to add
|
|
1526
|
+
* @param options Batch processing options
|
|
1527
|
+
* @returns Array of generated IDs
|
|
1528
|
+
*/
|
|
1529
|
+
async addNouns(items, options = {}) {
|
|
1380
1530
|
await this.ensureInitialized();
|
|
1381
1531
|
// Check if database is in read-only mode
|
|
1382
1532
|
this.checkReadOnly();
|
|
@@ -1426,7 +1576,7 @@ export class BrainyData {
|
|
|
1426
1576
|
}
|
|
1427
1577
|
});
|
|
1428
1578
|
// Process vector items (already embedded)
|
|
1429
|
-
const vectorPromises = vectorItems.map((item) => this.
|
|
1579
|
+
const vectorPromises = vectorItems.map((item) => this.addNoun(item.vectorOrData, item.metadata));
|
|
1430
1580
|
// Process text items in a single batch embedding operation
|
|
1431
1581
|
let textPromises = [];
|
|
1432
1582
|
if (textItems.length > 0) {
|
|
@@ -1435,10 +1585,7 @@ export class BrainyData {
|
|
|
1435
1585
|
// Perform batch embedding
|
|
1436
1586
|
const embeddings = await batchEmbed(texts);
|
|
1437
1587
|
// Add each item with its embedding
|
|
1438
|
-
textPromises = textItems.map((item, i) => this.
|
|
1439
|
-
...options,
|
|
1440
|
-
forceEmbed: false
|
|
1441
|
-
}));
|
|
1588
|
+
textPromises = textItems.map((item, i) => this.addNoun(embeddings[i], item.metadata));
|
|
1442
1589
|
}
|
|
1443
1590
|
// Combine all promises
|
|
1444
1591
|
const batchResults = await Promise.all([
|
|
@@ -1467,7 +1614,7 @@ export class BrainyData {
|
|
|
1467
1614
|
throw new Error('Not connected to a remote server. Call connectToRemoteServer() first.');
|
|
1468
1615
|
}
|
|
1469
1616
|
// Add to local with addToRemote option
|
|
1470
|
-
return this.
|
|
1617
|
+
return this.addNouns(items, { ...options, addToRemote: true });
|
|
1471
1618
|
}
|
|
1472
1619
|
/**
|
|
1473
1620
|
* Filter search results by service
|
|
@@ -1498,6 +1645,14 @@ export class BrainyData {
|
|
|
1498
1645
|
* @param options Additional options
|
|
1499
1646
|
* @returns Array of search results
|
|
1500
1647
|
*/
|
|
1648
|
+
/**
|
|
1649
|
+
* @deprecated Use search() with nounTypes option instead
|
|
1650
|
+
* @example
|
|
1651
|
+
* // Old way (deprecated)
|
|
1652
|
+
* await brain.searchByNounTypes(query, 10, ['type1', 'type2'])
|
|
1653
|
+
* // New way
|
|
1654
|
+
* await brain.search(query, { limit: 10, nounTypes: ['type1', 'type2'] })
|
|
1655
|
+
*/
|
|
1501
1656
|
async searchByNounTypes(queryVectorOrData, k = 10, nounTypes = null, options = {}) {
|
|
1502
1657
|
// Helper function to filter results by service
|
|
1503
1658
|
const filterByService = (metadata) => {
|
|
@@ -1585,9 +1740,9 @@ export class BrainyData {
|
|
|
1585
1740
|
if (hasMetadataFilter && this.metadataIndex) {
|
|
1586
1741
|
try {
|
|
1587
1742
|
// Ensure metadata index is up to date
|
|
1588
|
-
await this.metadataIndex
|
|
1743
|
+
await this.metadataIndex?.flush?.();
|
|
1589
1744
|
// Get candidate IDs from metadata index
|
|
1590
|
-
const candidateIds = await this.metadataIndex
|
|
1745
|
+
const candidateIds = await this.metadataIndex?.getIdsForFilter?.(options.metadata) || [];
|
|
1591
1746
|
if (candidateIds.length > 0) {
|
|
1592
1747
|
preFilteredIds = new Set(candidateIds);
|
|
1593
1748
|
// Create a simple filter function that just checks the pre-filtered set
|
|
@@ -1663,13 +1818,11 @@ export class BrainyData {
|
|
|
1663
1818
|
if (metadata === null) {
|
|
1664
1819
|
metadata = {};
|
|
1665
1820
|
}
|
|
1666
|
-
//
|
|
1667
|
-
|
|
1668
|
-
metadata = { ...metadata, id };
|
|
1669
|
-
}
|
|
1821
|
+
// Preserve original metadata without overwriting user's custom fields
|
|
1822
|
+
// The search result already has Brainy's UUID in the main 'id' field
|
|
1670
1823
|
searchResults.push({
|
|
1671
1824
|
id,
|
|
1672
|
-
score,
|
|
1825
|
+
score: 1 - score, // Convert distance to similarity (higher = more similar)
|
|
1673
1826
|
vector: noun.vector,
|
|
1674
1827
|
metadata: metadata
|
|
1675
1828
|
});
|
|
@@ -1708,13 +1861,11 @@ export class BrainyData {
|
|
|
1708
1861
|
if (metadata === null) {
|
|
1709
1862
|
metadata = {};
|
|
1710
1863
|
}
|
|
1711
|
-
//
|
|
1712
|
-
|
|
1713
|
-
metadata = { ...metadata, id };
|
|
1714
|
-
}
|
|
1864
|
+
// Preserve original metadata without overwriting user's custom fields
|
|
1865
|
+
// The search result already has Brainy's UUID in the main 'id' field
|
|
1715
1866
|
searchResults.push({
|
|
1716
1867
|
id,
|
|
1717
|
-
score,
|
|
1868
|
+
score: 1 - score, // Convert distance to similarity (higher = more similar)
|
|
1718
1869
|
vector: noun.vector,
|
|
1719
1870
|
metadata: metadata
|
|
1720
1871
|
});
|
|
@@ -1735,7 +1886,133 @@ export class BrainyData {
|
|
|
1735
1886
|
* @param options Additional options
|
|
1736
1887
|
* @returns Array of search results
|
|
1737
1888
|
*/
|
|
1738
|
-
|
|
1889
|
+
/**
|
|
1890
|
+
* 🔍 SIMPLE VECTOR SEARCH - Clean wrapper around find() for pure vector search
|
|
1891
|
+
*
|
|
1892
|
+
* @param queryVectorOrData Vector or text to search for
|
|
1893
|
+
* @param k Number of results to return
|
|
1894
|
+
* @param options Simple search options (metadata filters only)
|
|
1895
|
+
* @returns Vector search results
|
|
1896
|
+
*/
|
|
1897
|
+
/**
|
|
1898
|
+
* 🔍 Simple Vector Similarity Search - Clean wrapper around find()
|
|
1899
|
+
*
|
|
1900
|
+
* search(query) = find({like: query}) - Pure vector similarity search
|
|
1901
|
+
*
|
|
1902
|
+
* @param queryVectorOrData - Query string, vector, or object to search with
|
|
1903
|
+
* @param options - Search options for filtering and pagination
|
|
1904
|
+
* @returns Array of search results with scores and metadata
|
|
1905
|
+
*
|
|
1906
|
+
* @example
|
|
1907
|
+
* // Simple vector search
|
|
1908
|
+
* await brain.search('machine learning')
|
|
1909
|
+
*
|
|
1910
|
+
* // With filters and pagination
|
|
1911
|
+
* await brain.search('AI', {
|
|
1912
|
+
* limit: 20,
|
|
1913
|
+
* metadata: { type: 'article' },
|
|
1914
|
+
* nounTypes: ['document']
|
|
1915
|
+
* })
|
|
1916
|
+
*/
|
|
1917
|
+
async search(queryVectorOrData, options = {}) {
|
|
1918
|
+
// Build metadata filter from options
|
|
1919
|
+
const metadataFilter = { ...options.metadata };
|
|
1920
|
+
// Add noun type filtering
|
|
1921
|
+
if (options.nounTypes && options.nounTypes.length > 0) {
|
|
1922
|
+
metadataFilter.nounType = { in: options.nounTypes };
|
|
1923
|
+
}
|
|
1924
|
+
// Add item ID filtering
|
|
1925
|
+
if (options.itemIds && options.itemIds.length > 0) {
|
|
1926
|
+
metadataFilter.id = { in: options.itemIds };
|
|
1927
|
+
}
|
|
1928
|
+
// Build simple TripleQuery for vector similarity
|
|
1929
|
+
const tripleQuery = {
|
|
1930
|
+
like: queryVectorOrData
|
|
1931
|
+
};
|
|
1932
|
+
// Add metadata filter if we have conditions
|
|
1933
|
+
if (Object.keys(metadataFilter).length > 0) {
|
|
1934
|
+
tripleQuery.where = metadataFilter;
|
|
1935
|
+
}
|
|
1936
|
+
// Extract find() options
|
|
1937
|
+
const findOptions = {
|
|
1938
|
+
limit: options.limit,
|
|
1939
|
+
offset: options.offset,
|
|
1940
|
+
cursor: options.cursor,
|
|
1941
|
+
excludeDeleted: options.excludeDeleted,
|
|
1942
|
+
timeout: options.timeout
|
|
1943
|
+
};
|
|
1944
|
+
// Call find() with structured query - this is the key simplification!
|
|
1945
|
+
let results = await this.find(tripleQuery, findOptions);
|
|
1946
|
+
// Apply threshold filtering if specified
|
|
1947
|
+
if (options.threshold !== undefined) {
|
|
1948
|
+
results = results.filter(r => (r.fusionScore || r.score || 0) >= options.threshold);
|
|
1949
|
+
}
|
|
1950
|
+
// Convert to SearchResult format
|
|
1951
|
+
return results.map(r => ({
|
|
1952
|
+
...r,
|
|
1953
|
+
score: r.fusionScore || r.score || 0
|
|
1954
|
+
}));
|
|
1955
|
+
return results;
|
|
1956
|
+
}
|
|
1957
|
+
/**
|
|
1958
|
+
* Helper method to encode cursor for pagination
|
|
1959
|
+
* @internal
|
|
1960
|
+
*/
|
|
1961
|
+
encodeCursor(data) {
|
|
1962
|
+
return Buffer.from(JSON.stringify(data)).toString('base64');
|
|
1963
|
+
}
|
|
1964
|
+
/**
|
|
1965
|
+
* Helper method to decode cursor for pagination
|
|
1966
|
+
* @internal
|
|
1967
|
+
*/
|
|
1968
|
+
decodeCursor(cursor) {
|
|
1969
|
+
try {
|
|
1970
|
+
return JSON.parse(Buffer.from(cursor, 'base64').toString());
|
|
1971
|
+
}
|
|
1972
|
+
catch {
|
|
1973
|
+
return { offset: 0, timestamp: 0 };
|
|
1974
|
+
}
|
|
1975
|
+
}
|
|
1976
|
+
/**
|
|
1977
|
+
* Internal method for direct HNSW vector search
|
|
1978
|
+
* Used by TripleIntelligence to avoid circular dependencies
|
|
1979
|
+
* Note: For pure metadata filtering, use metadataIndex.getIdsForFilter() directly - it's O(log n)!
|
|
1980
|
+
* This method is for vector similarity search with optional metadata filtering during search
|
|
1981
|
+
* @internal
|
|
1982
|
+
*/
|
|
1983
|
+
async _internalVectorSearch(queryVectorOrData, k = 10, options = {}) {
|
|
1984
|
+
// Generate query vector
|
|
1985
|
+
const queryVector = Array.isArray(queryVectorOrData) &&
|
|
1986
|
+
typeof queryVectorOrData[0] === 'number' ?
|
|
1987
|
+
queryVectorOrData :
|
|
1988
|
+
await this.embed(queryVectorOrData);
|
|
1989
|
+
// Apply metadata filter if provided
|
|
1990
|
+
let filterFunction;
|
|
1991
|
+
if (options.metadata) {
|
|
1992
|
+
const matchingIdsArray = await this.metadataIndex?.getIdsForFilter(options.metadata) || [];
|
|
1993
|
+
const matchingIds = new Set(matchingIdsArray);
|
|
1994
|
+
filterFunction = async (id) => matchingIds.has(id);
|
|
1995
|
+
}
|
|
1996
|
+
// Direct HNSW search
|
|
1997
|
+
const results = await this.index.search(queryVector, k, filterFunction);
|
|
1998
|
+
// Get metadata for results
|
|
1999
|
+
const searchResults = [];
|
|
2000
|
+
for (const [id, similarity] of results) {
|
|
2001
|
+
const metadata = await this.getNoun(id);
|
|
2002
|
+
searchResults.push({
|
|
2003
|
+
id,
|
|
2004
|
+
score: similarity,
|
|
2005
|
+
vector: [],
|
|
2006
|
+
metadata: metadata?.metadata || {}
|
|
2007
|
+
});
|
|
2008
|
+
}
|
|
2009
|
+
return searchResults;
|
|
2010
|
+
}
|
|
2011
|
+
/**
|
|
2012
|
+
* 🎯 LEGACY: Original search implementation (kept for complex cases)
|
|
2013
|
+
* This is the original search method, now used as fallback for edge cases
|
|
2014
|
+
*/
|
|
2015
|
+
async _legacySearch(queryVectorOrData, k = 10, options = {}) {
|
|
1739
2016
|
const startTime = Date.now();
|
|
1740
2017
|
// Validate input is not null or undefined
|
|
1741
2018
|
if (queryVectorOrData === null || queryVectorOrData === undefined) {
|
|
@@ -1787,63 +2064,68 @@ export class BrainyData {
|
|
|
1787
2064
|
else if (options.searchMode === 'combined') {
|
|
1788
2065
|
return this.searchCombined(queryVectorOrData, k, options);
|
|
1789
2066
|
}
|
|
1790
|
-
//
|
|
1791
|
-
|
|
1792
|
-
|
|
1793
|
-
|
|
1794
|
-
|
|
1795
|
-
|
|
1796
|
-
|
|
1797
|
-
|
|
1798
|
-
|
|
1799
|
-
if
|
|
1800
|
-
|
|
1801
|
-
|
|
1802
|
-
|
|
1803
|
-
|
|
2067
|
+
// Generate deduplication key for concurrent request handling
|
|
2068
|
+
const dedupeKey = RequestDeduplicator.getSearchKey(typeof queryVectorOrData === 'string' ? queryVectorOrData : JSON.stringify(queryVectorOrData), k, options);
|
|
2069
|
+
// Use augmentation system for search (includes deduplication, batching, and caching)
|
|
2070
|
+
return this.augmentations.execute('search', { query: queryVectorOrData, k, options, dedupeKey }, async () => {
|
|
2071
|
+
// Default behavior (backward compatible): search locally
|
|
2072
|
+
try {
|
|
2073
|
+
// BEST OF BOTH: Automatically exclude soft-deleted items (Neural Intelligence improvement)
|
|
2074
|
+
// BUT only when there's already metadata filtering happening
|
|
2075
|
+
let metadataFilter = options.metadata;
|
|
2076
|
+
// Only add soft-delete filter if there's already metadata being filtered
|
|
2077
|
+
// This preserves pure vector searches without metadata
|
|
2078
|
+
if (metadataFilter && Object.keys(metadataFilter).length > 0) {
|
|
2079
|
+
// If no explicit deleted filter is provided, exclude soft-deleted items
|
|
2080
|
+
if (!metadataFilter.deleted && !metadataFilter.anyOf) {
|
|
2081
|
+
metadataFilter = {
|
|
2082
|
+
...metadataFilter,
|
|
2083
|
+
deleted: { notEquals: true }
|
|
2084
|
+
};
|
|
2085
|
+
}
|
|
1804
2086
|
}
|
|
1805
|
-
|
|
1806
|
-
|
|
1807
|
-
|
|
1808
|
-
|
|
1809
|
-
|
|
1810
|
-
|
|
1811
|
-
|
|
1812
|
-
|
|
1813
|
-
|
|
1814
|
-
|
|
1815
|
-
|
|
1816
|
-
|
|
2087
|
+
const hasMetadataFilter = metadataFilter && Object.keys(metadataFilter).length > 0;
|
|
2088
|
+
// Check cache first (transparent to user) - but skip cache if we have metadata filters
|
|
2089
|
+
if (!hasMetadataFilter) {
|
|
2090
|
+
const cacheKey = this.cache?.getCacheKey(queryVectorOrData, k, options);
|
|
2091
|
+
const cachedResults = this.cache?.get(cacheKey);
|
|
2092
|
+
if (cachedResults) {
|
|
2093
|
+
// Track cache hit in health monitor
|
|
2094
|
+
if (this.monitoring) {
|
|
2095
|
+
const latency = Date.now() - startTime;
|
|
2096
|
+
this.monitoring.recordRequest(latency, false);
|
|
2097
|
+
this.monitoring.recordCacheAccess(true);
|
|
2098
|
+
}
|
|
2099
|
+
return cachedResults;
|
|
1817
2100
|
}
|
|
1818
|
-
return cachedResults;
|
|
1819
2101
|
}
|
|
2102
|
+
// Cache miss - perform actual search
|
|
2103
|
+
const results = await this.searchLocal(queryVectorOrData, k, {
|
|
2104
|
+
...options,
|
|
2105
|
+
metadata: metadataFilter
|
|
2106
|
+
});
|
|
2107
|
+
// Cache results for future queries (unless explicitly disabled or has metadata filter)
|
|
2108
|
+
if (!options.skipCache && !hasMetadataFilter) {
|
|
2109
|
+
const cacheKey = this.cache?.getCacheKey(queryVectorOrData, k, options);
|
|
2110
|
+
this.cache?.set(cacheKey, results);
|
|
2111
|
+
}
|
|
2112
|
+
// Track successful search in health monitor
|
|
2113
|
+
if (this.monitoring) {
|
|
2114
|
+
const latency = Date.now() - startTime;
|
|
2115
|
+
this.monitoring.recordRequest(latency, false);
|
|
2116
|
+
this.monitoring.recordCacheAccess(false);
|
|
2117
|
+
}
|
|
2118
|
+
return results;
|
|
1820
2119
|
}
|
|
1821
|
-
|
|
1822
|
-
|
|
1823
|
-
|
|
1824
|
-
|
|
1825
|
-
|
|
1826
|
-
|
|
1827
|
-
|
|
1828
|
-
const cacheKey = this.searchCache.getCacheKey(queryVectorOrData, k, options);
|
|
1829
|
-
this.searchCache.set(cacheKey, results);
|
|
1830
|
-
}
|
|
1831
|
-
// Track successful search in health monitor
|
|
1832
|
-
if (this.healthMonitor) {
|
|
1833
|
-
const latency = Date.now() - startTime;
|
|
1834
|
-
this.healthMonitor.recordRequest(latency, false);
|
|
1835
|
-
this.healthMonitor.recordCacheAccess(false);
|
|
1836
|
-
}
|
|
1837
|
-
return results;
|
|
1838
|
-
}
|
|
1839
|
-
catch (error) {
|
|
1840
|
-
// Track error in health monitor
|
|
1841
|
-
if (this.healthMonitor) {
|
|
1842
|
-
const latency = Date.now() - startTime;
|
|
1843
|
-
this.healthMonitor.recordRequest(latency, true);
|
|
2120
|
+
catch (error) {
|
|
2121
|
+
// Track error in health monitor
|
|
2122
|
+
if (this.monitoring) {
|
|
2123
|
+
const latency = Date.now() - startTime;
|
|
2124
|
+
this.monitoring.recordRequest(latency, true);
|
|
2125
|
+
}
|
|
2126
|
+
throw error;
|
|
1844
2127
|
}
|
|
1845
|
-
|
|
1846
|
-
}
|
|
2128
|
+
});
|
|
1847
2129
|
}
|
|
1848
2130
|
/**
|
|
1849
2131
|
* Search with cursor-based pagination for better performance on large datasets
|
|
@@ -1852,13 +2134,23 @@ export class BrainyData {
|
|
|
1852
2134
|
* @param options Additional options including cursor for pagination
|
|
1853
2135
|
* @returns Paginated search results with cursor for next page
|
|
1854
2136
|
*/
|
|
2137
|
+
/**
|
|
2138
|
+
* @deprecated Use search() with cursor option instead
|
|
2139
|
+
* @example
|
|
2140
|
+
* // Old way (deprecated)
|
|
2141
|
+
* await brain.searchWithCursor(query, 10, { cursor: 'abc123' })
|
|
2142
|
+
* // New way
|
|
2143
|
+
* await brain.search(query, { limit: 10, cursor: 'abc123' })
|
|
2144
|
+
*/
|
|
1855
2145
|
async searchWithCursor(queryVectorOrData, k = 10, options = {}) {
|
|
1856
2146
|
// For cursor-based search, we need to fetch more results and filter
|
|
1857
2147
|
const searchK = options.cursor ? k + 20 : k; // Get extra results for filtering
|
|
1858
2148
|
// Perform regular search
|
|
1859
|
-
const
|
|
1860
|
-
|
|
1861
|
-
|
|
2149
|
+
const { cursor, ...searchOptions } = options;
|
|
2150
|
+
const allResults = await this.search(queryVectorOrData, {
|
|
2151
|
+
limit: searchK,
|
|
2152
|
+
nounTypes: searchOptions.nounTypes,
|
|
2153
|
+
metadata: searchOptions.filter
|
|
1862
2154
|
});
|
|
1863
2155
|
let results = allResults;
|
|
1864
2156
|
let startIndex = 0;
|
|
@@ -2022,7 +2314,7 @@ export class BrainyData {
|
|
|
2022
2314
|
async findSimilar(id, options = {}) {
|
|
2023
2315
|
await this.ensureInitialized();
|
|
2024
2316
|
// Get the entity by ID
|
|
2025
|
-
const entity = await this.
|
|
2317
|
+
const entity = await this.getNoun(id);
|
|
2026
2318
|
if (!entity) {
|
|
2027
2319
|
throw new Error(`Entity with ID ${id} not found`);
|
|
2028
2320
|
}
|
|
@@ -2040,7 +2332,7 @@ export class BrainyData {
|
|
|
2040
2332
|
// Skip undefined targetIds
|
|
2041
2333
|
if (typeof targetId !== 'string')
|
|
2042
2334
|
continue;
|
|
2043
|
-
const targetEntity = await this.
|
|
2335
|
+
const targetEntity = await this.getNoun(targetId);
|
|
2044
2336
|
if (targetEntity) {
|
|
2045
2337
|
results.push({
|
|
2046
2338
|
id: targetId,
|
|
@@ -2055,11 +2347,10 @@ export class BrainyData {
|
|
|
2055
2347
|
}
|
|
2056
2348
|
// If no relationType is specified, use the original vector similarity search
|
|
2057
2349
|
const k = (options.limit || 10) + 1; // Add 1 to account for the original entity
|
|
2058
|
-
const searchResults = await this.search(entity.vector,
|
|
2059
|
-
|
|
2060
|
-
|
|
2061
|
-
|
|
2062
|
-
searchMode: options.searchMode
|
|
2350
|
+
const searchResults = await this.search(entity.vector, {
|
|
2351
|
+
limit: k,
|
|
2352
|
+
excludeDeleted: false,
|
|
2353
|
+
nounTypes: options.nounTypes
|
|
2063
2354
|
});
|
|
2064
2355
|
// Filter out the original entity and limit to the requested number
|
|
2065
2356
|
return searchResults
|
|
@@ -2069,69 +2360,7 @@ export class BrainyData {
|
|
|
2069
2360
|
/**
|
|
2070
2361
|
* Get a vector by ID
|
|
2071
2362
|
*/
|
|
2072
|
-
|
|
2073
|
-
// Validate id parameter first, before any other logic
|
|
2074
|
-
if (id === null || id === undefined) {
|
|
2075
|
-
throw new Error('ID cannot be null or undefined');
|
|
2076
|
-
}
|
|
2077
|
-
await this.ensureInitialized();
|
|
2078
|
-
try {
|
|
2079
|
-
let noun;
|
|
2080
|
-
// In write-only mode, query storage directly since index is not loaded
|
|
2081
|
-
if (this.writeOnly) {
|
|
2082
|
-
try {
|
|
2083
|
-
noun = (await this.storage.getNoun(id)) ?? undefined;
|
|
2084
|
-
}
|
|
2085
|
-
catch (storageError) {
|
|
2086
|
-
// If storage lookup fails, return null (noun doesn't exist)
|
|
2087
|
-
return null;
|
|
2088
|
-
}
|
|
2089
|
-
}
|
|
2090
|
-
else {
|
|
2091
|
-
// Normal mode: Get noun from index first
|
|
2092
|
-
noun = this.index.getNouns().get(id);
|
|
2093
|
-
// If not found in index, fallback to storage (for race conditions)
|
|
2094
|
-
if (!noun && this.storage) {
|
|
2095
|
-
try {
|
|
2096
|
-
noun = (await this.storage.getNoun(id)) ?? undefined;
|
|
2097
|
-
}
|
|
2098
|
-
catch (storageError) {
|
|
2099
|
-
// Storage lookup failed, noun doesn't exist
|
|
2100
|
-
return null;
|
|
2101
|
-
}
|
|
2102
|
-
}
|
|
2103
|
-
}
|
|
2104
|
-
if (!noun) {
|
|
2105
|
-
return null;
|
|
2106
|
-
}
|
|
2107
|
-
// Get metadata
|
|
2108
|
-
let metadata = await this.storage.getMetadata(id);
|
|
2109
|
-
// Handle special cases for metadata
|
|
2110
|
-
if (metadata === null) {
|
|
2111
|
-
metadata = {};
|
|
2112
|
-
}
|
|
2113
|
-
else if (typeof metadata === 'object') {
|
|
2114
|
-
// For empty metadata test: if metadata only has an ID, return empty object
|
|
2115
|
-
if (Object.keys(metadata).length === 1 && 'id' in metadata) {
|
|
2116
|
-
metadata = {};
|
|
2117
|
-
}
|
|
2118
|
-
// Always remove the ID from metadata if present
|
|
2119
|
-
else if ('id' in metadata) {
|
|
2120
|
-
const { id: _, ...rest } = metadata;
|
|
2121
|
-
metadata = rest;
|
|
2122
|
-
}
|
|
2123
|
-
}
|
|
2124
|
-
return {
|
|
2125
|
-
id,
|
|
2126
|
-
vector: noun.vector,
|
|
2127
|
-
metadata: metadata
|
|
2128
|
-
};
|
|
2129
|
-
}
|
|
2130
|
-
catch (error) {
|
|
2131
|
-
console.error(`Failed to get vector ${id}:`, error);
|
|
2132
|
-
throw new Error(`Failed to get vector ${id}: ${error}`);
|
|
2133
|
-
}
|
|
2134
|
-
}
|
|
2363
|
+
// Legacy get() method removed - use getNoun() instead
|
|
2135
2364
|
/**
|
|
2136
2365
|
* Check if a document with the given ID exists
|
|
2137
2366
|
* This is a direct storage operation that works in write-only mode when allowDirectReads is enabled
|
|
@@ -2163,8 +2392,13 @@ export class BrainyData {
|
|
|
2163
2392
|
* @param id The ID to check for existence
|
|
2164
2393
|
* @returns Promise<boolean> True if the document exists, false otherwise
|
|
2165
2394
|
*/
|
|
2166
|
-
|
|
2167
|
-
|
|
2395
|
+
/**
|
|
2396
|
+
* Check if a noun exists
|
|
2397
|
+
* @param id The noun ID
|
|
2398
|
+
* @returns True if exists
|
|
2399
|
+
*/
|
|
2400
|
+
async hasNoun(id) {
|
|
2401
|
+
return this.hasNoun(id);
|
|
2168
2402
|
}
|
|
2169
2403
|
/**
|
|
2170
2404
|
* Get metadata for a document by ID
|
|
@@ -2172,31 +2406,53 @@ export class BrainyData {
|
|
|
2172
2406
|
* @param id The ID of the document
|
|
2173
2407
|
* @returns Promise<T | null> The metadata object or null if not found
|
|
2174
2408
|
*/
|
|
2175
|
-
|
|
2176
|
-
if (id === null || id === undefined) {
|
|
2177
|
-
throw new Error('ID cannot be null or undefined');
|
|
2178
|
-
}
|
|
2179
|
-
await this.ensureInitialized();
|
|
2180
|
-
// This is a direct storage operation - check if allowed in write-only mode
|
|
2181
|
-
if (this.writeOnly && !this.allowDirectReads) {
|
|
2182
|
-
throw new Error('Cannot perform getMetadata() operation: database is in write-only mode. Enable allowDirectReads for direct storage operations.');
|
|
2183
|
-
}
|
|
2184
|
-
try {
|
|
2185
|
-
const metadata = await this.storage.getMetadata(id);
|
|
2186
|
-
return metadata;
|
|
2187
|
-
}
|
|
2188
|
-
catch (error) {
|
|
2189
|
-
console.error(`Failed to get metadata for ${id}:`, error);
|
|
2190
|
-
return null;
|
|
2191
|
-
}
|
|
2192
|
-
}
|
|
2409
|
+
// Legacy getMetadata() method removed - use getNounMetadata() instead
|
|
2193
2410
|
/**
|
|
2194
2411
|
* Get multiple documents by their IDs
|
|
2195
2412
|
* This is a direct storage operation that works in write-only mode when allowDirectReads is enabled
|
|
2196
2413
|
* @param ids Array of IDs to retrieve
|
|
2197
2414
|
* @returns Promise<Array<VectorDocument<T> | null>> Array of documents (null for missing IDs)
|
|
2198
2415
|
*/
|
|
2199
|
-
|
|
2416
|
+
/**
|
|
2417
|
+
* Get multiple nouns - by IDs, filters, or pagination
|
|
2418
|
+
* @param idsOrOptions Array of IDs or query options
|
|
2419
|
+
* @returns Array of noun documents
|
|
2420
|
+
*
|
|
2421
|
+
* @example
|
|
2422
|
+
* // Get by IDs
|
|
2423
|
+
* await brain.getNouns(['id1', 'id2'])
|
|
2424
|
+
*
|
|
2425
|
+
* // Get with filters
|
|
2426
|
+
* await brain.getNouns({
|
|
2427
|
+
* filter: { type: 'article' },
|
|
2428
|
+
* limit: 10
|
|
2429
|
+
* })
|
|
2430
|
+
*
|
|
2431
|
+
* // Get with pagination
|
|
2432
|
+
* await brain.getNouns({
|
|
2433
|
+
* offset: 20,
|
|
2434
|
+
* limit: 10
|
|
2435
|
+
* })
|
|
2436
|
+
*/
|
|
2437
|
+
async getNouns(idsOrOptions) {
|
|
2438
|
+
// Handle array of IDs
|
|
2439
|
+
if (Array.isArray(idsOrOptions)) {
|
|
2440
|
+
return this.getNounsByIds(idsOrOptions);
|
|
2441
|
+
}
|
|
2442
|
+
// Handle options object
|
|
2443
|
+
const options = idsOrOptions || {};
|
|
2444
|
+
// If ids are provided in options, get by IDs
|
|
2445
|
+
if (options.ids) {
|
|
2446
|
+
return this.getNounsByIds(options.ids);
|
|
2447
|
+
}
|
|
2448
|
+
// Otherwise, do a filtered/paginated query and extract items
|
|
2449
|
+
const result = await this.queryNounsByFilter(options);
|
|
2450
|
+
return result.items;
|
|
2451
|
+
}
|
|
2452
|
+
/**
|
|
2453
|
+
* Internal: Get nouns by IDs
|
|
2454
|
+
*/
|
|
2455
|
+
async getNounsByIds(ids) {
|
|
2200
2456
|
if (!Array.isArray(ids)) {
|
|
2201
2457
|
throw new Error('IDs must be provided as an array');
|
|
2202
2458
|
}
|
|
@@ -2212,7 +2468,7 @@ export class BrainyData {
|
|
|
2212
2468
|
continue;
|
|
2213
2469
|
}
|
|
2214
2470
|
try {
|
|
2215
|
-
const result = await this.
|
|
2471
|
+
const result = await this.getNoun(id);
|
|
2216
2472
|
results.push(result);
|
|
2217
2473
|
}
|
|
2218
2474
|
catch (error) {
|
|
@@ -2229,7 +2485,10 @@ export class BrainyData {
|
|
|
2229
2485
|
* @param options Pagination and filtering options
|
|
2230
2486
|
* @returns Paginated result of vector documents
|
|
2231
2487
|
*/
|
|
2232
|
-
|
|
2488
|
+
/**
|
|
2489
|
+
* Internal: Query nouns with filtering and pagination
|
|
2490
|
+
*/
|
|
2491
|
+
async queryNounsByFilter(options = {}) {
|
|
2233
2492
|
await this.ensureInitialized();
|
|
2234
2493
|
try {
|
|
2235
2494
|
// First try to use the storage adapter's paginated method
|
|
@@ -2324,223 +2583,11 @@ export class BrainyData {
|
|
|
2324
2583
|
throw new Error(`Failed to get nouns with pagination: ${error}`);
|
|
2325
2584
|
}
|
|
2326
2585
|
}
|
|
2327
|
-
|
|
2328
|
-
|
|
2329
|
-
|
|
2330
|
-
|
|
2331
|
-
|
|
2332
|
-
*/
|
|
2333
|
-
async delete(id, options = {}) {
|
|
2334
|
-
// Clear API: use 'hard: true' for hard delete, otherwise soft delete
|
|
2335
|
-
const isHardDelete = options.hard === true;
|
|
2336
|
-
const opts = {
|
|
2337
|
-
service: options.service,
|
|
2338
|
-
soft: !isHardDelete, // Soft delete is default unless hard: true is specified
|
|
2339
|
-
cascade: options.cascade || false,
|
|
2340
|
-
force: options.force || false
|
|
2341
|
-
};
|
|
2342
|
-
// Validate id parameter first, before any other logic
|
|
2343
|
-
if (id === null || id === undefined) {
|
|
2344
|
-
throw new Error('ID cannot be null or undefined');
|
|
2345
|
-
}
|
|
2346
|
-
await this.ensureInitialized();
|
|
2347
|
-
// Check if database is in read-only mode
|
|
2348
|
-
this.checkReadOnly();
|
|
2349
|
-
try {
|
|
2350
|
-
// Check if the id is actually content text rather than an ID
|
|
2351
|
-
// This handles cases where tests or users pass content text instead of IDs
|
|
2352
|
-
let actualId = id;
|
|
2353
|
-
console.log(`Delete called with ID: ${id}`);
|
|
2354
|
-
console.log(`Index has ID directly: ${this.index.getNouns().has(id)}`);
|
|
2355
|
-
if (!this.index.getNouns().has(id)) {
|
|
2356
|
-
console.log(`Looking for noun with text content: ${id}`);
|
|
2357
|
-
// Try to find a noun with matching text content
|
|
2358
|
-
for (const [nounId, noun] of this.index.getNouns().entries()) {
|
|
2359
|
-
console.log(`Checking noun ${nounId}: text=${noun.metadata?.text || 'undefined'}`);
|
|
2360
|
-
if (noun.metadata?.text === id) {
|
|
2361
|
-
actualId = nounId;
|
|
2362
|
-
console.log(`Found matching noun with ID: ${actualId}`);
|
|
2363
|
-
break;
|
|
2364
|
-
}
|
|
2365
|
-
}
|
|
2366
|
-
}
|
|
2367
|
-
// Handle soft delete vs hard delete
|
|
2368
|
-
if (opts.soft) {
|
|
2369
|
-
// Soft delete: just mark as deleted - metadata filter will exclude from search
|
|
2370
|
-
try {
|
|
2371
|
-
return await this.updateMetadata(actualId, {
|
|
2372
|
-
deleted: true,
|
|
2373
|
-
deletedAt: new Date().toISOString(),
|
|
2374
|
-
deletedBy: opts.service || 'user'
|
|
2375
|
-
});
|
|
2376
|
-
}
|
|
2377
|
-
catch (error) {
|
|
2378
|
-
// If item doesn't exist, return false (delete of non-existent item is not an error)
|
|
2379
|
-
return false;
|
|
2380
|
-
}
|
|
2381
|
-
}
|
|
2382
|
-
// Hard delete: Remove from index
|
|
2383
|
-
const removed = this.index.removeItem(actualId);
|
|
2384
|
-
if (!removed) {
|
|
2385
|
-
return false;
|
|
2386
|
-
}
|
|
2387
|
-
// Remove from storage
|
|
2388
|
-
await this.storage.deleteNoun(actualId);
|
|
2389
|
-
// Track deletion statistics
|
|
2390
|
-
const service = this.getServiceName({ service: opts.service });
|
|
2391
|
-
await this.storage.decrementStatistic('noun', service);
|
|
2392
|
-
// Try to remove metadata (ignore errors)
|
|
2393
|
-
try {
|
|
2394
|
-
// Get metadata before removing for index cleanup
|
|
2395
|
-
const existingMetadata = await this.storage.getMetadata(actualId);
|
|
2396
|
-
// Remove from metadata index (write-only mode should update indices!)
|
|
2397
|
-
if (this.metadataIndex && existingMetadata && !this.frozen) {
|
|
2398
|
-
await this.metadataIndex.removeFromIndex(actualId, existingMetadata);
|
|
2399
|
-
}
|
|
2400
|
-
await this.storage.saveMetadata(actualId, null);
|
|
2401
|
-
await this.storage.decrementStatistic('metadata', service);
|
|
2402
|
-
}
|
|
2403
|
-
catch (error) {
|
|
2404
|
-
// Ignore
|
|
2405
|
-
}
|
|
2406
|
-
// Invalidate search cache since data has changed
|
|
2407
|
-
this.searchCache.invalidateOnDataChange('delete');
|
|
2408
|
-
return true;
|
|
2409
|
-
}
|
|
2410
|
-
catch (error) {
|
|
2411
|
-
console.error(`Failed to delete vector ${id}:`, error);
|
|
2412
|
-
throw new Error(`Failed to delete vector ${id}: ${error}`);
|
|
2413
|
-
}
|
|
2414
|
-
}
|
|
2415
|
-
/**
|
|
2416
|
-
* Update metadata for a vector
|
|
2417
|
-
* @param id The ID of the vector to update metadata for
|
|
2418
|
-
* @param metadata The new metadata
|
|
2419
|
-
* @param options Additional options
|
|
2420
|
-
* @returns Promise that resolves to true if the metadata was updated, false otherwise
|
|
2421
|
-
*/
|
|
2422
|
-
async updateMetadata(id, metadata, options = {}) {
|
|
2423
|
-
// Validate id parameter first, before any other logic
|
|
2424
|
-
if (id === null || id === undefined) {
|
|
2425
|
-
throw new Error('ID cannot be null or undefined');
|
|
2426
|
-
}
|
|
2427
|
-
// Validate that metadata is not null or undefined
|
|
2428
|
-
if (metadata === null || metadata === undefined) {
|
|
2429
|
-
throw new Error(`Metadata cannot be null or undefined`);
|
|
2430
|
-
}
|
|
2431
|
-
await this.ensureInitialized();
|
|
2432
|
-
// Check if database is in read-only mode
|
|
2433
|
-
this.checkReadOnly();
|
|
2434
|
-
try {
|
|
2435
|
-
// Check if a vector exists
|
|
2436
|
-
const noun = this.index.getNouns().get(id);
|
|
2437
|
-
if (!noun) {
|
|
2438
|
-
throw new Error(`Vector with ID ${id} does not exist`);
|
|
2439
|
-
}
|
|
2440
|
-
// Validate noun type if metadata is for a GraphNoun
|
|
2441
|
-
if (metadata && typeof metadata === 'object' && 'noun' in metadata) {
|
|
2442
|
-
const nounType = metadata.noun;
|
|
2443
|
-
// Check if the noun type is valid
|
|
2444
|
-
const isValidNounType = Object.values(NounType).includes(nounType);
|
|
2445
|
-
if (!isValidNounType) {
|
|
2446
|
-
console.warn(`Invalid noun type: ${nounType}. Falling back to GraphNoun.`);
|
|
2447
|
-
metadata.noun = NounType.Concept;
|
|
2448
|
-
}
|
|
2449
|
-
// Get the service that's updating the metadata
|
|
2450
|
-
const service = this.getServiceName(options);
|
|
2451
|
-
const graphNoun = metadata;
|
|
2452
|
-
// Preserve existing createdBy and createdAt if they exist
|
|
2453
|
-
const existingMetadata = (await this.storage.getMetadata(id));
|
|
2454
|
-
if (existingMetadata &&
|
|
2455
|
-
typeof existingMetadata === 'object' &&
|
|
2456
|
-
'createdBy' in existingMetadata) {
|
|
2457
|
-
// Preserve the original creator information
|
|
2458
|
-
graphNoun.createdBy = existingMetadata.createdBy;
|
|
2459
|
-
// Also preserve creation timestamp if it exists
|
|
2460
|
-
if ('createdAt' in existingMetadata) {
|
|
2461
|
-
graphNoun.createdAt = existingMetadata.createdAt;
|
|
2462
|
-
}
|
|
2463
|
-
}
|
|
2464
|
-
else if (!graphNoun.createdBy) {
|
|
2465
|
-
// If no existing createdBy and none in the update, set it
|
|
2466
|
-
graphNoun.createdBy = getAugmentationVersion(service);
|
|
2467
|
-
// Set createdAt if it doesn't exist
|
|
2468
|
-
if (!graphNoun.createdAt) {
|
|
2469
|
-
const now = new Date();
|
|
2470
|
-
graphNoun.createdAt = {
|
|
2471
|
-
seconds: Math.floor(now.getTime() / 1000),
|
|
2472
|
-
nanoseconds: (now.getTime() % 1000) * 1000000
|
|
2473
|
-
};
|
|
2474
|
-
}
|
|
2475
|
-
}
|
|
2476
|
-
// Always update the updatedAt timestamp
|
|
2477
|
-
const now = new Date();
|
|
2478
|
-
graphNoun.updatedAt = {
|
|
2479
|
-
seconds: Math.floor(now.getTime() / 1000),
|
|
2480
|
-
nanoseconds: (now.getTime() % 1000) * 1000000
|
|
2481
|
-
};
|
|
2482
|
-
}
|
|
2483
|
-
// Update metadata
|
|
2484
|
-
await this.storage.saveMetadata(id, metadata);
|
|
2485
|
-
// Update metadata index (write-only mode should build indices!)
|
|
2486
|
-
if (this.metadataIndex && !this.frozen) {
|
|
2487
|
-
// Remove old metadata from index if it exists
|
|
2488
|
-
const oldMetadata = await this.storage.getMetadata(id);
|
|
2489
|
-
if (oldMetadata) {
|
|
2490
|
-
await this.metadataIndex.removeFromIndex(id, oldMetadata);
|
|
2491
|
-
}
|
|
2492
|
-
// Add new metadata to index
|
|
2493
|
-
if (metadata) {
|
|
2494
|
-
await this.metadataIndex.addToIndex(id, metadata);
|
|
2495
|
-
}
|
|
2496
|
-
}
|
|
2497
|
-
// Track metadata statistics
|
|
2498
|
-
const service = this.getServiceName(options);
|
|
2499
|
-
await this.storage.incrementStatistic('metadata', service);
|
|
2500
|
-
// Invalidate search cache since metadata has changed
|
|
2501
|
-
this.searchCache.invalidateOnDataChange('update');
|
|
2502
|
-
return true;
|
|
2503
|
-
}
|
|
2504
|
-
catch (error) {
|
|
2505
|
-
console.error(`Failed to update metadata for vector ${id}:`, error);
|
|
2506
|
-
throw new Error(`Failed to update metadata for vector ${id}: ${error}`);
|
|
2507
|
-
}
|
|
2508
|
-
}
|
|
2509
|
-
/**
|
|
2510
|
-
* Create a relationship between two entities
|
|
2511
|
-
* This is a convenience wrapper around addVerb
|
|
2512
|
-
*/
|
|
2513
|
-
async relate(sourceId, targetId, relationType, metadata) {
|
|
2514
|
-
// Validate inputs are not null or undefined
|
|
2515
|
-
if (sourceId === null || sourceId === undefined) {
|
|
2516
|
-
throw new Error('Source ID cannot be null or undefined');
|
|
2517
|
-
}
|
|
2518
|
-
if (targetId === null || targetId === undefined) {
|
|
2519
|
-
throw new Error('Target ID cannot be null or undefined');
|
|
2520
|
-
}
|
|
2521
|
-
if (relationType === null || relationType === undefined) {
|
|
2522
|
-
throw new Error('Relation type cannot be null or undefined');
|
|
2523
|
-
}
|
|
2524
|
-
// NEURAL INTELLIGENCE: Enhanced metadata with smart inference
|
|
2525
|
-
const enhancedMetadata = {
|
|
2526
|
-
...metadata,
|
|
2527
|
-
createdAt: new Date().toISOString(),
|
|
2528
|
-
inferenceScore: 1.0, // Could be enhanced with ML-based confidence scoring
|
|
2529
|
-
relationType: relationType,
|
|
2530
|
-
neuralEnhanced: true
|
|
2531
|
-
};
|
|
2532
|
-
return this._addVerbInternal(sourceId, targetId, undefined, {
|
|
2533
|
-
type: relationType,
|
|
2534
|
-
metadata: enhancedMetadata
|
|
2535
|
-
});
|
|
2536
|
-
}
|
|
2537
|
-
/**
|
|
2538
|
-
* Create a connection between two entities
|
|
2539
|
-
* This is an alias for relate() for backward compatibility
|
|
2540
|
-
*/
|
|
2541
|
-
async connect(sourceId, targetId, relationType, metadata) {
|
|
2542
|
-
return this.relate(sourceId, targetId, relationType, metadata);
|
|
2543
|
-
}
|
|
2586
|
+
// Legacy private methods removed - use public 2.0 API methods instead:
|
|
2587
|
+
// - delete() removed - use deleteNoun() instead
|
|
2588
|
+
// - updateMetadata() removed - use updateNoun() or updateNounMetadata() instead
|
|
2589
|
+
// REMOVED: relate() - Use addVerb() instead (cleaner 2.0 API)
|
|
2590
|
+
// REMOVED: connect() - Use addVerb() instead (cleaner 2.0 API)
|
|
2544
2591
|
/**
|
|
2545
2592
|
* Add a verb between two nouns
|
|
2546
2593
|
* If metadata is provided and vector is not, the metadata will be vectorized using the embedding function
|
|
@@ -2691,8 +2738,8 @@ export class BrainyData {
|
|
|
2691
2738
|
noun: NounType.Concept,
|
|
2692
2739
|
createdBy: getAugmentationVersion(service)
|
|
2693
2740
|
};
|
|
2694
|
-
// Add the missing noun
|
|
2695
|
-
await this.
|
|
2741
|
+
// Add the missing noun (custom ID not supported in 2.0 addNoun yet)
|
|
2742
|
+
await this.addNoun(placeholderVector, metadata);
|
|
2696
2743
|
// Get the newly created noun
|
|
2697
2744
|
sourceNoun = this.index.getNouns().get(sourceId);
|
|
2698
2745
|
console.warn(`Auto-created missing source noun with ID ${sourceId}`);
|
|
@@ -2720,8 +2767,8 @@ export class BrainyData {
|
|
|
2720
2767
|
noun: NounType.Concept,
|
|
2721
2768
|
createdBy: getAugmentationVersion(service)
|
|
2722
2769
|
};
|
|
2723
|
-
// Add the missing noun
|
|
2724
|
-
await this.
|
|
2770
|
+
// Add the missing noun (custom ID not supported in 2.0 addNoun yet)
|
|
2771
|
+
await this.addNoun(placeholderVector, metadata);
|
|
2725
2772
|
// Get the newly created noun
|
|
2726
2773
|
targetNoun = this.index.getNouns().get(targetId);
|
|
2727
2774
|
console.warn(`Auto-created missing target noun with ID ${targetId}`);
|
|
@@ -2811,7 +2858,10 @@ export class BrainyData {
|
|
|
2811
2858
|
let scoringReasoning = [];
|
|
2812
2859
|
if (this.intelligentVerbScoring?.enabled && (!options.weight || options.weight === 0.5)) {
|
|
2813
2860
|
try {
|
|
2814
|
-
|
|
2861
|
+
// Get the source and target nouns for semantic scoring
|
|
2862
|
+
const sourceNoun = await this.storage?.getNoun(sourceId);
|
|
2863
|
+
const targetNoun = await this.storage?.getNoun(targetId);
|
|
2864
|
+
const scores = await this.intelligentVerbScoring.computeVerbScores(sourceNoun, targetNoun, verbType);
|
|
2815
2865
|
finalWeight = scores.weight;
|
|
2816
2866
|
finalConfidence = scores.confidence;
|
|
2817
2867
|
scoringReasoning = scores.reasoning || [];
|
|
@@ -2828,6 +2878,7 @@ export class BrainyData {
|
|
|
2828
2878
|
}
|
|
2829
2879
|
}
|
|
2830
2880
|
// Create complete verb metadata separately
|
|
2881
|
+
// Merge original metadata with system metadata to preserve neural enhancements
|
|
2831
2882
|
const verbMetadata = {
|
|
2832
2883
|
sourceId: sourceId,
|
|
2833
2884
|
targetId: targetId,
|
|
@@ -2844,7 +2895,9 @@ export class BrainyData {
|
|
|
2844
2895
|
createdAt: timestamp,
|
|
2845
2896
|
updatedAt: timestamp,
|
|
2846
2897
|
createdBy: getAugmentationVersion(service),
|
|
2847
|
-
|
|
2898
|
+
// Merge original metadata to preserve neural enhancements from relate()
|
|
2899
|
+
...(options.metadata || {}),
|
|
2900
|
+
data: options.metadata // Also store in data field for backwards compatibility
|
|
2848
2901
|
};
|
|
2849
2902
|
// Add to index
|
|
2850
2903
|
await this.index.addItem({ id, vector: verbVector });
|
|
@@ -2870,26 +2923,34 @@ export class BrainyData {
|
|
|
2870
2923
|
createdAt: verbMetadata.createdAt,
|
|
2871
2924
|
updatedAt: verbMetadata.updatedAt,
|
|
2872
2925
|
createdBy: verbMetadata.createdBy,
|
|
2873
|
-
metadata: verbMetadata
|
|
2926
|
+
metadata: verbMetadata, // Use full metadata with neural enhancements
|
|
2874
2927
|
data: verbMetadata.data,
|
|
2875
2928
|
embedding: hnswVerb.vector
|
|
2876
2929
|
};
|
|
2877
|
-
// Save the complete verb (
|
|
2878
|
-
await this.
|
|
2930
|
+
// Save the complete verb using augmentation system (handles WAL, batching, streaming)
|
|
2931
|
+
await this.augmentations.execute('saveVerb', {
|
|
2932
|
+
verb: fullVerb,
|
|
2933
|
+
sourceId,
|
|
2934
|
+
targetId,
|
|
2935
|
+
relationType: options.type,
|
|
2936
|
+
metadata: verbMetadata
|
|
2937
|
+
}, async () => {
|
|
2938
|
+
await this.storage.saveVerb(fullVerb);
|
|
2939
|
+
});
|
|
2879
2940
|
// Update metadata index
|
|
2880
|
-
if (this.
|
|
2881
|
-
await this.metadataIndex
|
|
2941
|
+
if (this.index && verbMetadata) {
|
|
2942
|
+
await this.metadataIndex?.addToIndex?.(id, verbMetadata);
|
|
2882
2943
|
}
|
|
2883
2944
|
// Track verb statistics
|
|
2884
2945
|
const serviceForStats = this.getServiceName(options);
|
|
2885
2946
|
await this.storage.incrementStatistic('verb', serviceForStats);
|
|
2886
|
-
// Track verb type
|
|
2887
|
-
this.
|
|
2947
|
+
// Track verb type (if metrics are enabled)
|
|
2948
|
+
// this.metrics?.trackVerbType(verbMetadata.verb)
|
|
2888
2949
|
// Update HNSW index size with actual index size
|
|
2889
2950
|
const indexSize = this.index.size();
|
|
2890
2951
|
await this.storage.updateHnswIndexSize(indexSize);
|
|
2891
2952
|
// Invalidate search cache since verb data has changed
|
|
2892
|
-
this.
|
|
2953
|
+
this.cache?.invalidateOnDataChange('add');
|
|
2893
2954
|
return id;
|
|
2894
2955
|
}
|
|
2895
2956
|
catch (error) {
|
|
@@ -2979,7 +3040,7 @@ export class BrainyData {
|
|
|
2979
3040
|
const result = await this.getNouns({
|
|
2980
3041
|
pagination: { limit: Number.MAX_SAFE_INTEGER }
|
|
2981
3042
|
});
|
|
2982
|
-
return result.
|
|
3043
|
+
return result.filter((noun) => noun !== null);
|
|
2983
3044
|
}
|
|
2984
3045
|
// Fall back to on-demand loading
|
|
2985
3046
|
return [];
|
|
@@ -3135,12 +3196,58 @@ export class BrainyData {
|
|
|
3135
3196
|
* @param options Additional options
|
|
3136
3197
|
* @returns Promise that resolves to true if the verb was deleted, false otherwise
|
|
3137
3198
|
*/
|
|
3199
|
+
/**
|
|
3200
|
+
* Add multiple verbs (relationships) in batch
|
|
3201
|
+
* @param verbs Array of verbs to add
|
|
3202
|
+
* @returns Array of generated verb IDs
|
|
3203
|
+
*/
|
|
3204
|
+
async addVerbs(verbs) {
|
|
3205
|
+
const ids = [];
|
|
3206
|
+
for (const verb of verbs) {
|
|
3207
|
+
const id = await this.addVerb(verb.source, verb.target, verb.type, verb.metadata);
|
|
3208
|
+
ids.push(id);
|
|
3209
|
+
}
|
|
3210
|
+
return ids;
|
|
3211
|
+
}
|
|
3212
|
+
/**
|
|
3213
|
+
* Delete multiple verbs by IDs
|
|
3214
|
+
* @param ids Array of verb IDs
|
|
3215
|
+
* @returns Array of success booleans
|
|
3216
|
+
*/
|
|
3217
|
+
async deleteVerbs(ids) {
|
|
3218
|
+
const results = [];
|
|
3219
|
+
for (const id of ids) {
|
|
3220
|
+
results.push(await this.deleteVerb(id));
|
|
3221
|
+
}
|
|
3222
|
+
return results;
|
|
3223
|
+
}
|
|
3138
3224
|
async deleteVerb(id, options = {}) {
|
|
3139
3225
|
await this.ensureInitialized();
|
|
3140
3226
|
// Check if database is in read-only mode
|
|
3141
3227
|
this.checkReadOnly();
|
|
3142
3228
|
try {
|
|
3143
|
-
//
|
|
3229
|
+
// PERFORMANCE: Use soft delete by default for O(log n) filtering
|
|
3230
|
+
// The MetadataIndex can efficiently filter out deleted items
|
|
3231
|
+
if (!options.hard) {
|
|
3232
|
+
// Soft delete: Just mark as deleted in metadata
|
|
3233
|
+
try {
|
|
3234
|
+
await this.storage.saveVerbMetadata(id, {
|
|
3235
|
+
deleted: true,
|
|
3236
|
+
deletedAt: new Date().toISOString(),
|
|
3237
|
+
deletedBy: options.service || '2.0-api'
|
|
3238
|
+
});
|
|
3239
|
+
// Update MetadataIndex for O(log n) filtering
|
|
3240
|
+
if (this.metadataIndex) {
|
|
3241
|
+
await this.metadataIndex.updateIndex(id, { deleted: true });
|
|
3242
|
+
}
|
|
3243
|
+
return true;
|
|
3244
|
+
}
|
|
3245
|
+
catch (error) {
|
|
3246
|
+
// If verb doesn't exist, return false (not an error)
|
|
3247
|
+
return false;
|
|
3248
|
+
}
|
|
3249
|
+
}
|
|
3250
|
+
// Hard delete path (explicit request only)
|
|
3144
3251
|
const existingMetadata = await this.storage.getVerbMetadata(id);
|
|
3145
3252
|
// Remove from index
|
|
3146
3253
|
const removed = this.index.removeItem(id);
|
|
@@ -3148,8 +3255,8 @@ export class BrainyData {
|
|
|
3148
3255
|
return false;
|
|
3149
3256
|
}
|
|
3150
3257
|
// Remove from metadata index
|
|
3151
|
-
if (this.
|
|
3152
|
-
await this.metadataIndex
|
|
3258
|
+
if (this.index && existingMetadata) {
|
|
3259
|
+
await this.metadataIndex?.removeFromIndex?.(id, existingMetadata);
|
|
3153
3260
|
}
|
|
3154
3261
|
// Remove from storage
|
|
3155
3262
|
await this.storage.deleteVerb(id);
|
|
@@ -3163,28 +3270,6 @@ export class BrainyData {
|
|
|
3163
3270
|
throw new Error(`Failed to delete verb ${id}: ${error}`);
|
|
3164
3271
|
}
|
|
3165
3272
|
}
|
|
3166
|
-
/**
|
|
3167
|
-
* Clear the database
|
|
3168
|
-
*/
|
|
3169
|
-
async clear() {
|
|
3170
|
-
await this.ensureInitialized();
|
|
3171
|
-
// Check if database is in read-only mode
|
|
3172
|
-
this.checkReadOnly();
|
|
3173
|
-
try {
|
|
3174
|
-
// Clear index
|
|
3175
|
-
await this.index.clear();
|
|
3176
|
-
// Clear storage
|
|
3177
|
-
await this.storage.clear();
|
|
3178
|
-
// Reset statistics collector
|
|
3179
|
-
this.statisticsCollector = new StatisticsCollector();
|
|
3180
|
-
// Clear search cache since all data has been removed
|
|
3181
|
-
this.searchCache.invalidateOnDataChange('delete');
|
|
3182
|
-
}
|
|
3183
|
-
catch (error) {
|
|
3184
|
-
console.error('Failed to clear vector database:', error);
|
|
3185
|
-
throw new Error(`Failed to clear vector database: ${error}`);
|
|
3186
|
-
}
|
|
3187
|
-
}
|
|
3188
3273
|
/**
|
|
3189
3274
|
* Get the number of vectors in the database
|
|
3190
3275
|
*/
|
|
@@ -3197,15 +3282,15 @@ export class BrainyData {
|
|
|
3197
3282
|
*/
|
|
3198
3283
|
getCacheStats() {
|
|
3199
3284
|
return {
|
|
3200
|
-
search: this.
|
|
3201
|
-
searchMemoryUsage: this.
|
|
3285
|
+
search: this.cache?.getStats() || {},
|
|
3286
|
+
searchMemoryUsage: this.cache?.getMemoryUsage() || 0
|
|
3202
3287
|
};
|
|
3203
3288
|
}
|
|
3204
3289
|
/**
|
|
3205
3290
|
* Clear search cache manually (useful for testing or memory management)
|
|
3206
3291
|
*/
|
|
3207
3292
|
clearCache() {
|
|
3208
|
-
this.
|
|
3293
|
+
this.cache?.clear();
|
|
3209
3294
|
}
|
|
3210
3295
|
/**
|
|
3211
3296
|
* Adapt cache configuration based on current performance metrics
|
|
@@ -3213,9 +3298,9 @@ export class BrainyData {
|
|
|
3213
3298
|
* @private
|
|
3214
3299
|
*/
|
|
3215
3300
|
adaptCacheConfiguration() {
|
|
3216
|
-
const stats = this.
|
|
3217
|
-
const memoryUsage = this.
|
|
3218
|
-
const currentConfig = this.
|
|
3301
|
+
const stats = this.cache?.getStats() || {};
|
|
3302
|
+
const memoryUsage = this.cache?.getMemoryUsage() || 0;
|
|
3303
|
+
const currentConfig = this.cache?.getConfig() || {};
|
|
3219
3304
|
// Prepare performance metrics for adaptation
|
|
3220
3305
|
const performanceMetrics = {
|
|
3221
3306
|
hitRate: stats.hitRate,
|
|
@@ -3228,7 +3313,7 @@ export class BrainyData {
|
|
|
3228
3313
|
const newConfig = this.cacheAutoConfigurator.adaptConfiguration(currentConfig, performanceMetrics);
|
|
3229
3314
|
if (newConfig) {
|
|
3230
3315
|
// Apply new cache configuration
|
|
3231
|
-
this.
|
|
3316
|
+
this.cache?.updateConfig(newConfig.cacheConfig);
|
|
3232
3317
|
// Apply new real-time update configuration if needed
|
|
3233
3318
|
if (newConfig.realtimeConfig.enabled !==
|
|
3234
3319
|
this.realtimeUpdateConfig.enabled ||
|
|
@@ -3357,7 +3442,7 @@ export class BrainyData {
|
|
|
3357
3442
|
const totalNouns = Object.values(stats.nounCount).reduce((a, b) => a + b, 0);
|
|
3358
3443
|
const totalVerbs = Object.values(stats.verbCount).reduce((a, b) => a + b, 0);
|
|
3359
3444
|
const totalMetadata = Object.values(stats.metadataCount).reduce((a, b) => a + b, 0);
|
|
3360
|
-
this.
|
|
3445
|
+
this.metrics.updateStorageSizes({
|
|
3361
3446
|
nouns: totalNouns * avgNounSize,
|
|
3362
3447
|
verbs: totalVerbs * avgVerbSize,
|
|
3363
3448
|
metadata: totalMetadata * avgMetadataSize,
|
|
@@ -3449,7 +3534,7 @@ export class BrainyData {
|
|
|
3449
3534
|
// Always include for now
|
|
3450
3535
|
// Add index health metrics
|
|
3451
3536
|
try {
|
|
3452
|
-
const indexHealth = this.
|
|
3537
|
+
const indexHealth = this.metadataIndex?.getIndexHealth?.() || { healthy: true };
|
|
3453
3538
|
result.indexHealth = indexHealth;
|
|
3454
3539
|
}
|
|
3455
3540
|
catch (e) {
|
|
@@ -3457,7 +3542,7 @@ export class BrainyData {
|
|
|
3457
3542
|
}
|
|
3458
3543
|
// Add cache metrics
|
|
3459
3544
|
try {
|
|
3460
|
-
const cacheStats = this.
|
|
3545
|
+
const cacheStats = this.cache?.getStats() || {};
|
|
3461
3546
|
result.cacheMetrics = cacheStats;
|
|
3462
3547
|
}
|
|
3463
3548
|
catch (e) {
|
|
@@ -3473,7 +3558,7 @@ export class BrainyData {
|
|
|
3473
3558
|
result.lastUpdated =
|
|
3474
3559
|
stats.lastUpdated || new Date().toISOString();
|
|
3475
3560
|
// Add enhanced statistics from collector
|
|
3476
|
-
const collectorStats = this.
|
|
3561
|
+
const collectorStats = this.metrics.getStatistics();
|
|
3477
3562
|
Object.assign(result, collectorStats);
|
|
3478
3563
|
// Preserve throttling metrics from storage if available
|
|
3479
3564
|
if (stats.throttlingMetrics) {
|
|
@@ -3484,14 +3569,15 @@ export class BrainyData {
|
|
|
3484
3569
|
}
|
|
3485
3570
|
return result;
|
|
3486
3571
|
}
|
|
3487
|
-
// If statistics are not available,
|
|
3488
|
-
|
|
3489
|
-
|
|
3490
|
-
//
|
|
3491
|
-
|
|
3492
|
-
const
|
|
3493
|
-
const
|
|
3494
|
-
const
|
|
3572
|
+
// If statistics are not available from storage, use index counts for small datasets
|
|
3573
|
+
// For production with millions of entries, this would be cached
|
|
3574
|
+
const indexSize = this.index?.getNouns?.()?.size || 0;
|
|
3575
|
+
// Use actual counts for small datasets (< 10000 items)
|
|
3576
|
+
// In production, these would be tracked incrementally
|
|
3577
|
+
const nounCount = indexSize < 10000 ? indexSize : 0;
|
|
3578
|
+
const verbCount = 0; // Verbs require expensive storage scan
|
|
3579
|
+
const metadataCount = nounCount; // Metadata count equals noun count
|
|
3580
|
+
const hnswIndexSize = indexSize;
|
|
3495
3581
|
// Create default statistics
|
|
3496
3582
|
const defaultStats = {
|
|
3497
3583
|
nounCount,
|
|
@@ -4015,10 +4101,9 @@ export class BrainyData {
|
|
|
4015
4101
|
*/
|
|
4016
4102
|
async getFilterValues(field) {
|
|
4017
4103
|
await this.ensureInitialized();
|
|
4018
|
-
|
|
4019
|
-
|
|
4020
|
-
|
|
4021
|
-
return this.metadataIndex.getFilterValues(field);
|
|
4104
|
+
// Delegate to index augmentation
|
|
4105
|
+
const index = this.augmentations.get('index');
|
|
4106
|
+
return index?.getFilterValues?.(field) || [];
|
|
4022
4107
|
}
|
|
4023
4108
|
/**
|
|
4024
4109
|
* Get all available filter fields
|
|
@@ -4028,10 +4113,9 @@ export class BrainyData {
|
|
|
4028
4113
|
*/
|
|
4029
4114
|
async getFilterFields() {
|
|
4030
4115
|
await this.ensureInitialized();
|
|
4031
|
-
|
|
4032
|
-
|
|
4033
|
-
|
|
4034
|
-
return this.metadataIndex.getFilterFields();
|
|
4116
|
+
// Delegate to index augmentation
|
|
4117
|
+
const index = this.augmentations.get('index');
|
|
4118
|
+
return index?.getFilterFields?.() || [];
|
|
4035
4119
|
}
|
|
4036
4120
|
/**
|
|
4037
4121
|
* Search within a specific set of items
|
|
@@ -4043,6 +4127,14 @@ export class BrainyData {
|
|
|
4043
4127
|
* @param options Additional options
|
|
4044
4128
|
* @returns Array of search results
|
|
4045
4129
|
*/
|
|
4130
|
+
/**
|
|
4131
|
+
* @deprecated Use search() with itemIds option instead
|
|
4132
|
+
* @example
|
|
4133
|
+
* // Old way (deprecated)
|
|
4134
|
+
* await brain.searchWithinItems(query, itemIds, 10)
|
|
4135
|
+
* // New way
|
|
4136
|
+
* await brain.search(query, { limit: 10, itemIds })
|
|
4137
|
+
*/
|
|
4046
4138
|
async searchWithinItems(queryVectorOrData, itemIds, k = 10, options = {}) {
|
|
4047
4139
|
await this.ensureInitialized();
|
|
4048
4140
|
// Check if database is in write-only mode
|
|
@@ -4092,6 +4184,14 @@ export class BrainyData {
|
|
|
4092
4184
|
* @param options Additional options
|
|
4093
4185
|
* @returns Array of search results
|
|
4094
4186
|
*/
|
|
4187
|
+
/**
|
|
4188
|
+
* @deprecated Use search() directly with text - it auto-detects strings
|
|
4189
|
+
* @example
|
|
4190
|
+
* // Old way (deprecated)
|
|
4191
|
+
* await brain.searchText('query text', 10)
|
|
4192
|
+
* // New way
|
|
4193
|
+
* await brain.search('query text', { limit: 10 })
|
|
4194
|
+
*/
|
|
4095
4195
|
async searchText(query, k = 10, options = {}) {
|
|
4096
4196
|
await this.ensureInitialized();
|
|
4097
4197
|
// Check if database is in write-only mode
|
|
@@ -4101,16 +4201,14 @@ export class BrainyData {
|
|
|
4101
4201
|
// Embed the query text
|
|
4102
4202
|
const queryVector = await this.embed(query);
|
|
4103
4203
|
// Search using the embedded vector with metadata filtering
|
|
4104
|
-
const results = await this.search(queryVector,
|
|
4204
|
+
const results = await this.search(queryVector, {
|
|
4205
|
+
limit: k,
|
|
4105
4206
|
nounTypes: options.nounTypes,
|
|
4106
|
-
|
|
4107
|
-
searchMode: options.searchMode,
|
|
4108
|
-
metadata: options.metadata,
|
|
4109
|
-
forceEmbed: false // Already embedded
|
|
4207
|
+
metadata: options.metadata
|
|
4110
4208
|
});
|
|
4111
4209
|
// Track search performance
|
|
4112
4210
|
const duration = Date.now() - searchStartTime;
|
|
4113
|
-
this.
|
|
4211
|
+
this.metrics.trackSearch(query, duration);
|
|
4114
4212
|
return results;
|
|
4115
4213
|
}
|
|
4116
4214
|
catch (error) {
|
|
@@ -4126,43 +4224,10 @@ export class BrainyData {
|
|
|
4126
4224
|
* @returns Array of search results
|
|
4127
4225
|
*/
|
|
4128
4226
|
async searchRemote(queryVectorOrData, k = 10, options = {}) {
|
|
4227
|
+
// TODO: Remote server search will be implemented in post-2.0.0 release
|
|
4129
4228
|
await this.ensureInitialized();
|
|
4130
|
-
// Check if database is in write-only mode
|
|
4131
4229
|
this.checkWriteOnly();
|
|
4132
|
-
|
|
4133
|
-
if (!this.isConnectedToRemoteServer()) {
|
|
4134
|
-
throw new Error('Not connected to a remote server. Call connectToRemoteServer() first.');
|
|
4135
|
-
}
|
|
4136
|
-
try {
|
|
4137
|
-
// If input is a string, convert it to a query string for the server
|
|
4138
|
-
let query;
|
|
4139
|
-
if (typeof queryVectorOrData === 'string') {
|
|
4140
|
-
query = queryVectorOrData;
|
|
4141
|
-
}
|
|
4142
|
-
else {
|
|
4143
|
-
// For vectors, we need to embed them as a string query
|
|
4144
|
-
// This is a simplification - ideally we would send the vector directly
|
|
4145
|
-
query = 'vector-query'; // Placeholder, would need a better approach for vector queries
|
|
4146
|
-
}
|
|
4147
|
-
if (!this.serverSearchConduit || !this.serverConnection) {
|
|
4148
|
-
throw new Error('Server search conduit or connection is not initialized');
|
|
4149
|
-
}
|
|
4150
|
-
// When using offset, fetch more results and slice
|
|
4151
|
-
const offset = options.offset || 0;
|
|
4152
|
-
const totalNeeded = k + offset;
|
|
4153
|
-
// Search the remote server for totalNeeded results
|
|
4154
|
-
const searchResult = await this.serverSearchConduit.searchServer(this.serverConnection.connectionId, query, totalNeeded);
|
|
4155
|
-
if (!searchResult.success) {
|
|
4156
|
-
throw new Error(`Remote search failed: ${searchResult.error}`);
|
|
4157
|
-
}
|
|
4158
|
-
// Apply offset to remote results
|
|
4159
|
-
const allResults = searchResult.data;
|
|
4160
|
-
return allResults.slice(offset, offset + k);
|
|
4161
|
-
}
|
|
4162
|
-
catch (error) {
|
|
4163
|
-
console.error('Failed to search remote server:', error);
|
|
4164
|
-
throw new Error(`Failed to search remote server: ${error}`);
|
|
4165
|
-
}
|
|
4230
|
+
throw new Error('Remote server search functionality not yet implemented in Brainy 2.0.0');
|
|
4166
4231
|
}
|
|
4167
4232
|
/**
|
|
4168
4233
|
* Search both local and remote Brainy instances, combining the results
|
|
@@ -4235,31 +4300,17 @@ export class BrainyData {
|
|
|
4235
4300
|
* @returns True if connected to a remote server, false otherwise
|
|
4236
4301
|
*/
|
|
4237
4302
|
isConnectedToRemoteServer() {
|
|
4238
|
-
|
|
4303
|
+
// TODO: Remote server connections will be implemented in post-2.0.0 release
|
|
4304
|
+
return false;
|
|
4239
4305
|
}
|
|
4240
4306
|
/**
|
|
4241
4307
|
* Disconnect from the remote server
|
|
4242
4308
|
* @returns True if successfully disconnected, false if not connected
|
|
4243
4309
|
*/
|
|
4244
4310
|
async disconnectFromRemoteServer() {
|
|
4245
|
-
|
|
4246
|
-
|
|
4247
|
-
|
|
4248
|
-
try {
|
|
4249
|
-
if (!this.serverSearchConduit || !this.serverConnection) {
|
|
4250
|
-
throw new Error('Server search conduit or connection is not initialized');
|
|
4251
|
-
}
|
|
4252
|
-
// Close the WebSocket connection
|
|
4253
|
-
await this.serverSearchConduit.closeWebSocket(this.serverConnection.connectionId);
|
|
4254
|
-
// Clear the connection information
|
|
4255
|
-
this.serverSearchConduit = null;
|
|
4256
|
-
this.serverConnection = null;
|
|
4257
|
-
return true;
|
|
4258
|
-
}
|
|
4259
|
-
catch (error) {
|
|
4260
|
-
console.error('Failed to disconnect from remote server:', error);
|
|
4261
|
-
throw new Error(`Failed to disconnect from remote server: ${error}`);
|
|
4262
|
-
}
|
|
4311
|
+
// TODO: Remote server disconnection will be implemented in post-2.0.0 release
|
|
4312
|
+
console.warn('disconnectFromRemoteServer: Remote server functionality not yet implemented in Brainy 2.0.0');
|
|
4313
|
+
return false;
|
|
4263
4314
|
}
|
|
4264
4315
|
/**
|
|
4265
4316
|
* Ensure the database is initialized
|
|
@@ -4423,7 +4474,7 @@ export class BrainyData {
|
|
|
4423
4474
|
const nounsResult = await this.getNouns({
|
|
4424
4475
|
pagination: { limit: Number.MAX_SAFE_INTEGER }
|
|
4425
4476
|
});
|
|
4426
|
-
const nouns = nounsResult.
|
|
4477
|
+
const nouns = nounsResult.filter((noun) => noun !== null);
|
|
4427
4478
|
const verbsResult = await this.getVerbs({
|
|
4428
4479
|
pagination: { limit: Number.MAX_SAFE_INTEGER }
|
|
4429
4480
|
});
|
|
@@ -4489,7 +4540,7 @@ export class BrainyData {
|
|
|
4489
4540
|
try {
|
|
4490
4541
|
// Clear existing data if requested
|
|
4491
4542
|
if (options.clearExisting) {
|
|
4492
|
-
await this.clear();
|
|
4543
|
+
await this.clear({ force: true });
|
|
4493
4544
|
}
|
|
4494
4545
|
// Validate the data format
|
|
4495
4546
|
if (!data || !data.nouns || !data.verbs || !data.version) {
|
|
@@ -4523,8 +4574,8 @@ export class BrainyData {
|
|
|
4523
4574
|
noun.vector = await this.embeddingFunction(noun.metadata);
|
|
4524
4575
|
}
|
|
4525
4576
|
}
|
|
4526
|
-
// Add the noun with its vector and metadata
|
|
4527
|
-
await this.
|
|
4577
|
+
// Add the noun with its vector and metadata (custom ID not supported)
|
|
4578
|
+
await this.addNoun(noun.vector, noun.metadata);
|
|
4528
4579
|
nounsRestored++;
|
|
4529
4580
|
}
|
|
4530
4581
|
catch (error) {
|
|
@@ -4575,7 +4626,7 @@ export class BrainyData {
|
|
|
4575
4626
|
;
|
|
4576
4627
|
hnswConfig.useDiskBasedIndex = true;
|
|
4577
4628
|
}
|
|
4578
|
-
this.
|
|
4629
|
+
this.hnswIndex = new HNSWIndexOptimized(hnswConfig, this.distanceFunction, this.storage);
|
|
4579
4630
|
this.useOptimizedIndex = true;
|
|
4580
4631
|
// For the storage-adapter-coverage test, we want the index to be empty
|
|
4581
4632
|
// after restoration, as specified in the test expectation
|
|
@@ -4648,7 +4699,7 @@ export class BrainyData {
|
|
|
4648
4699
|
const clearExisting = options.clearExisting || false;
|
|
4649
4700
|
// Clear existing data if requested
|
|
4650
4701
|
if (clearExisting) {
|
|
4651
|
-
await this.clear();
|
|
4702
|
+
await this.clear({ force: true });
|
|
4652
4703
|
}
|
|
4653
4704
|
try {
|
|
4654
4705
|
// Generate random nouns
|
|
@@ -4681,7 +4732,7 @@ export class BrainyData {
|
|
|
4681
4732
|
}
|
|
4682
4733
|
};
|
|
4683
4734
|
// Add the noun
|
|
4684
|
-
const id = await this.
|
|
4735
|
+
const id = await this.addNoun(metadata.description, metadata);
|
|
4685
4736
|
nounIds.push(id);
|
|
4686
4737
|
}
|
|
4687
4738
|
// Generate random verbs between nouns
|
|
@@ -4805,11 +4856,8 @@ export class BrainyData {
|
|
|
4805
4856
|
for (const [service, fieldNames] of Object.entries(serviceFieldMappings)) {
|
|
4806
4857
|
for (const fieldName of fieldNames) {
|
|
4807
4858
|
// Search using the specific field name for this service
|
|
4808
|
-
const results = await this.search(searchTerm,
|
|
4809
|
-
|
|
4810
|
-
service,
|
|
4811
|
-
includeVerbs: options.includeVerbs,
|
|
4812
|
-
searchMode: options.searchMode
|
|
4859
|
+
const results = await this.search(searchTerm, {
|
|
4860
|
+
limit: k
|
|
4813
4861
|
});
|
|
4814
4862
|
// Add results to the combined list
|
|
4815
4863
|
allResults.push(...results);
|
|
@@ -4836,15 +4884,15 @@ export class BrainyData {
|
|
|
4836
4884
|
// Flush metadata index one last time
|
|
4837
4885
|
if (this.metadataIndex) {
|
|
4838
4886
|
try {
|
|
4839
|
-
await this.metadataIndex
|
|
4887
|
+
await this.metadataIndex?.flush?.();
|
|
4840
4888
|
}
|
|
4841
4889
|
catch (error) {
|
|
4842
4890
|
console.warn('Error flushing metadata index during cleanup:', error);
|
|
4843
4891
|
}
|
|
4844
4892
|
}
|
|
4845
4893
|
// Clean up distributed mode resources
|
|
4846
|
-
if (this.
|
|
4847
|
-
this.
|
|
4894
|
+
if (this.monitoring) {
|
|
4895
|
+
this.monitoring.stop();
|
|
4848
4896
|
}
|
|
4849
4897
|
if (this.configManager) {
|
|
4850
4898
|
await this.configManager.cleanup();
|
|
@@ -4874,13 +4922,13 @@ export class BrainyData {
|
|
|
4874
4922
|
const configValue = options?.encrypt ? await this.encryptData(JSON.stringify(value)) : value;
|
|
4875
4923
|
// Use simple text for vectorization
|
|
4876
4924
|
const searchableText = `Configuration setting for ${key}`;
|
|
4877
|
-
await this.
|
|
4925
|
+
await this.addNoun(searchableText, {
|
|
4878
4926
|
nounType: NounType.State,
|
|
4879
4927
|
configKey: key,
|
|
4880
4928
|
configValue: configValue,
|
|
4881
4929
|
encrypted: !!options?.encrypt,
|
|
4882
4930
|
timestamp: new Date().toISOString()
|
|
4883
|
-
}
|
|
4931
|
+
});
|
|
4884
4932
|
}
|
|
4885
4933
|
/**
|
|
4886
4934
|
* Get a configuration value with automatic decryption
|
|
@@ -4892,7 +4940,7 @@ export class BrainyData {
|
|
|
4892
4940
|
try {
|
|
4893
4941
|
// Use the predictable ID to get the config directly
|
|
4894
4942
|
const configId = `config-${key}`;
|
|
4895
|
-
const storedNoun = await this.
|
|
4943
|
+
const storedNoun = await this.getNoun(configId);
|
|
4896
4944
|
if (!storedNoun)
|
|
4897
4945
|
return undefined;
|
|
4898
4946
|
// The config data is now stored in metadata
|
|
@@ -4979,10 +5027,8 @@ export class BrainyData {
|
|
|
4979
5027
|
if (detectedType) {
|
|
4980
5028
|
metadata.nounType = detectedType;
|
|
4981
5029
|
}
|
|
4982
|
-
// Import item using standard add method
|
|
4983
|
-
const id = await this.
|
|
4984
|
-
process: options?.process || 'auto'
|
|
4985
|
-
});
|
|
5030
|
+
// Import item using standard add method (process option not supported in 2.0)
|
|
5031
|
+
const id = await this.addNoun(item, metadata);
|
|
4986
5032
|
results.push(id);
|
|
4987
5033
|
}
|
|
4988
5034
|
catch (error) {
|
|
@@ -5002,14 +5048,16 @@ export class BrainyData {
|
|
|
5002
5048
|
* @param metadata Additional metadata
|
|
5003
5049
|
* @returns Created noun ID
|
|
5004
5050
|
*/
|
|
5005
|
-
|
|
5006
|
-
|
|
5007
|
-
|
|
5008
|
-
|
|
5009
|
-
|
|
5010
|
-
|
|
5011
|
-
|
|
5012
|
-
|
|
5051
|
+
/**
|
|
5052
|
+
* Add a noun to the database
|
|
5053
|
+
* Clean 2.0 API - primary method for adding data
|
|
5054
|
+
*
|
|
5055
|
+
* @param vectorOrData Vector array or data to embed
|
|
5056
|
+
* @param metadata Metadata to store with the noun
|
|
5057
|
+
* @returns The generated ID
|
|
5058
|
+
*/
|
|
5059
|
+
async addNoun(vectorOrData, metadata) {
|
|
5060
|
+
return await this.add(vectorOrData, metadata);
|
|
5013
5061
|
}
|
|
5014
5062
|
/**
|
|
5015
5063
|
* Add Verb - Unified relationship creation between nouns
|
|
@@ -5022,50 +5070,66 @@ export class BrainyData {
|
|
|
5022
5070
|
* @returns Created verb ID
|
|
5023
5071
|
*/
|
|
5024
5072
|
async addVerb(sourceId, targetId, verbType, metadata, weight) {
|
|
5025
|
-
//
|
|
5026
|
-
|
|
5027
|
-
const
|
|
5028
|
-
if (!
|
|
5029
|
-
throw new Error(`
|
|
5030
|
-
}
|
|
5031
|
-
|
|
5032
|
-
|
|
5033
|
-
|
|
5034
|
-
//
|
|
5035
|
-
|
|
5036
|
-
|
|
5037
|
-
|
|
5038
|
-
const
|
|
5039
|
-
|
|
5040
|
-
|
|
5041
|
-
|
|
5042
|
-
|
|
5043
|
-
}
|
|
5044
|
-
|
|
5045
|
-
|
|
5046
|
-
|
|
5047
|
-
|
|
5048
|
-
|
|
5049
|
-
|
|
5050
|
-
|
|
5051
|
-
|
|
5052
|
-
|
|
5053
|
-
|
|
5054
|
-
|
|
5055
|
-
|
|
5056
|
-
|
|
5057
|
-
|
|
5058
|
-
|
|
5059
|
-
|
|
5060
|
-
|
|
5061
|
-
|
|
5062
|
-
|
|
5063
|
-
|
|
5064
|
-
|
|
5065
|
-
|
|
5066
|
-
|
|
5067
|
-
|
|
5068
|
-
|
|
5073
|
+
// CRITICAL: Runtime validation for enterprise compatibility
|
|
5074
|
+
// ALL VERBS must use one of the predefined VerbTypes
|
|
5075
|
+
const validTypes = Object.values(VerbType);
|
|
5076
|
+
if (!validTypes.includes(verbType)) {
|
|
5077
|
+
throw new Error(`Invalid verb type: '${verbType}'. Must be one of: ${validTypes.join(', ')}`);
|
|
5078
|
+
}
|
|
5079
|
+
// Store params in array for augmentation system
|
|
5080
|
+
const params = [sourceId, targetId, verbType, metadata, weight];
|
|
5081
|
+
// Use augmentation system to wrap the addVerb operation
|
|
5082
|
+
// This allows intelligent verb scoring to enhance the weight
|
|
5083
|
+
return await this.augmentations.execute('addVerb', params, async () => {
|
|
5084
|
+
// Validate that source and target nouns exist
|
|
5085
|
+
const sourceNoun = this.index.getNouns().get(sourceId);
|
|
5086
|
+
const targetNoun = this.index.getNouns().get(targetId);
|
|
5087
|
+
if (!sourceNoun) {
|
|
5088
|
+
throw new Error(`Source noun with ID ${sourceId} does not exist`);
|
|
5089
|
+
}
|
|
5090
|
+
if (!targetNoun) {
|
|
5091
|
+
throw new Error(`Target noun with ID ${targetId} does not exist`);
|
|
5092
|
+
}
|
|
5093
|
+
// Create embeddable text from verb type and metadata for searchability
|
|
5094
|
+
let embeddingText = `${verbType} relationship`;
|
|
5095
|
+
// Include meaningful metadata in embedding
|
|
5096
|
+
const currentMetadata = params[3] || metadata;
|
|
5097
|
+
if (currentMetadata) {
|
|
5098
|
+
const metadataStrings = [];
|
|
5099
|
+
// Add text-based metadata fields for better searchability
|
|
5100
|
+
for (const [key, value] of Object.entries(currentMetadata)) {
|
|
5101
|
+
if (typeof value === 'string' && value.length > 0) {
|
|
5102
|
+
metadataStrings.push(`${key}: ${value}`);
|
|
5103
|
+
}
|
|
5104
|
+
else if (typeof value === 'number' || typeof value === 'boolean') {
|
|
5105
|
+
metadataStrings.push(`${key}: ${value}`);
|
|
5106
|
+
}
|
|
5107
|
+
}
|
|
5108
|
+
if (metadataStrings.length > 0) {
|
|
5109
|
+
embeddingText += ` with ${metadataStrings.join(', ')}`;
|
|
5110
|
+
}
|
|
5111
|
+
}
|
|
5112
|
+
// Generate embedding for the relationship including metadata
|
|
5113
|
+
const vector = await this.embeddingFunction(embeddingText);
|
|
5114
|
+
// Get the potentially modified weight from augmentation params
|
|
5115
|
+
const finalWeight = params[4] !== undefined ? params[4] : 0.5;
|
|
5116
|
+
const finalMetadata = params[3] || metadata;
|
|
5117
|
+
// Create complete verb metadata
|
|
5118
|
+
const verbMetadata = {
|
|
5119
|
+
verb: verbType,
|
|
5120
|
+
sourceId,
|
|
5121
|
+
targetId,
|
|
5122
|
+
weight: finalWeight,
|
|
5123
|
+
embeddingText, // Include the text used for embedding for debugging
|
|
5124
|
+
...finalMetadata
|
|
5125
|
+
};
|
|
5126
|
+
// Use existing internal addVerb method with proper parameters
|
|
5127
|
+
return await this._addVerbInternal(sourceId, targetId, vector, {
|
|
5128
|
+
type: verbType,
|
|
5129
|
+
weight: finalWeight,
|
|
5130
|
+
metadata: verbMetadata,
|
|
5131
|
+
forceEmbed: false // We already have the vector
|
|
5132
|
+
});
|
|
5069
5133
|
});
|
|
5070
5134
|
}
|
|
5071
5135
|
/**
|
|
@@ -5198,45 +5262,7 @@ export class BrainyData {
|
|
|
5198
5262
|
* @param options Update options
|
|
5199
5263
|
* @returns Success boolean
|
|
5200
5264
|
*/
|
|
5201
|
-
|
|
5202
|
-
const opts = {
|
|
5203
|
-
merge: true,
|
|
5204
|
-
reindex: true,
|
|
5205
|
-
cascade: false,
|
|
5206
|
-
...options
|
|
5207
|
-
};
|
|
5208
|
-
// Update data if provided
|
|
5209
|
-
if (data !== undefined) {
|
|
5210
|
-
// For data updates, we need to regenerate the vector
|
|
5211
|
-
const existingNoun = this.index.getNouns().get(id);
|
|
5212
|
-
if (!existingNoun) {
|
|
5213
|
-
throw new Error(`Noun with ID ${id} does not exist`);
|
|
5214
|
-
}
|
|
5215
|
-
// Create new vector for updated data
|
|
5216
|
-
const vector = await this.embeddingFunction(data);
|
|
5217
|
-
// Update the noun with new data and vector
|
|
5218
|
-
const updatedNoun = {
|
|
5219
|
-
...existingNoun,
|
|
5220
|
-
vector,
|
|
5221
|
-
metadata: opts.merge ? { ...existingNoun.metadata, ...metadata } : metadata
|
|
5222
|
-
};
|
|
5223
|
-
// Update in index
|
|
5224
|
-
this.index.getNouns().set(id, updatedNoun);
|
|
5225
|
-
// Note: HNSW index will be updated automatically on next search
|
|
5226
|
-
// Reindexing happens lazily for performance
|
|
5227
|
-
}
|
|
5228
|
-
else if (metadata !== undefined) {
|
|
5229
|
-
// Metadata-only update using existing updateMetadata method
|
|
5230
|
-
return await this.updateMetadata(id, metadata);
|
|
5231
|
-
}
|
|
5232
|
-
// Update related verbs if cascade enabled
|
|
5233
|
-
if (opts.cascade) {
|
|
5234
|
-
// TODO: Implement cascade verb updates when verb access methods are clarified
|
|
5235
|
-
prodLog.debug(`Cascade update requested for ${id} - feature pending implementation`);
|
|
5236
|
-
}
|
|
5237
|
-
prodLog.debug(`✅ Updated noun ${id} (data: ${data !== undefined}, metadata: ${metadata !== undefined})`);
|
|
5238
|
-
return true;
|
|
5239
|
-
}
|
|
5265
|
+
// Legacy update() method removed - use updateNoun() instead
|
|
5240
5266
|
/**
|
|
5241
5267
|
* Preload Transformer Model - Essential for container deployments
|
|
5242
5268
|
* Downloads and caches models during initialization to avoid runtime delays
|
|
@@ -5366,7 +5392,7 @@ export class BrainyData {
|
|
|
5366
5392
|
}
|
|
5367
5393
|
};
|
|
5368
5394
|
// Store coordination plan in _system directory
|
|
5369
|
-
await this.
|
|
5395
|
+
await this.addNoun({
|
|
5370
5396
|
id: '_system/coordination',
|
|
5371
5397
|
type: 'cortex_coordination',
|
|
5372
5398
|
metadata: coordinationPlan
|
|
@@ -5380,7 +5406,7 @@ export class BrainyData {
|
|
|
5380
5406
|
*/
|
|
5381
5407
|
async checkCoordination() {
|
|
5382
5408
|
try {
|
|
5383
|
-
const coordination = await this.
|
|
5409
|
+
const coordination = await this.getNoun('_system/coordination');
|
|
5384
5410
|
return coordination?.metadata;
|
|
5385
5411
|
}
|
|
5386
5412
|
catch (error) {
|
|
@@ -5392,78 +5418,437 @@ export class BrainyData {
|
|
|
5392
5418
|
* Exposed for Cortex reindex command
|
|
5393
5419
|
*/
|
|
5394
5420
|
async rebuildMetadataIndex() {
|
|
5395
|
-
|
|
5396
|
-
|
|
5421
|
+
await this.metadataIndex?.rebuild?.();
|
|
5422
|
+
}
|
|
5423
|
+
// ===== Clean 2.0 API - Primary Methods =====
|
|
5424
|
+
/**
|
|
5425
|
+
* Get a noun by ID
|
|
5426
|
+
* @param id The noun ID
|
|
5427
|
+
* @returns The noun document or null
|
|
5428
|
+
*/
|
|
5429
|
+
async getNoun(id) {
|
|
5430
|
+
// Validate id parameter first, before any other logic
|
|
5431
|
+
if (id === null || id === undefined) {
|
|
5432
|
+
throw new Error('ID cannot be null or undefined');
|
|
5433
|
+
}
|
|
5434
|
+
await this.ensureInitialized();
|
|
5435
|
+
try {
|
|
5436
|
+
let noun;
|
|
5437
|
+
// In write-only mode, query storage directly since index is not loaded
|
|
5438
|
+
if (this.writeOnly) {
|
|
5439
|
+
try {
|
|
5440
|
+
noun = (await this.storage.getNoun(id)) ?? undefined;
|
|
5441
|
+
}
|
|
5442
|
+
catch (storageError) {
|
|
5443
|
+
// If storage lookup fails, return null (noun doesn't exist)
|
|
5444
|
+
return null;
|
|
5445
|
+
}
|
|
5446
|
+
}
|
|
5447
|
+
else {
|
|
5448
|
+
// Normal mode: Get noun from index first
|
|
5449
|
+
noun = this.index.getNouns().get(id);
|
|
5450
|
+
// If not found in index, fallback to storage (for race conditions)
|
|
5451
|
+
if (!noun && this.storage) {
|
|
5452
|
+
try {
|
|
5453
|
+
noun = (await this.storage.getNoun(id)) ?? undefined;
|
|
5454
|
+
}
|
|
5455
|
+
catch (storageError) {
|
|
5456
|
+
// Storage lookup failed, noun doesn't exist
|
|
5457
|
+
return null;
|
|
5458
|
+
}
|
|
5459
|
+
}
|
|
5460
|
+
}
|
|
5461
|
+
if (!noun) {
|
|
5462
|
+
return null;
|
|
5463
|
+
}
|
|
5464
|
+
// Get metadata
|
|
5465
|
+
let metadata = await this.storage.getMetadata(id);
|
|
5466
|
+
// Handle special cases for metadata
|
|
5467
|
+
if (metadata === null) {
|
|
5468
|
+
metadata = {};
|
|
5469
|
+
}
|
|
5470
|
+
else if (typeof metadata === 'object') {
|
|
5471
|
+
// Check if this item is soft-deleted
|
|
5472
|
+
if (metadata.deleted === true) {
|
|
5473
|
+
// Return null for soft-deleted items to match expected API behavior
|
|
5474
|
+
return null;
|
|
5475
|
+
}
|
|
5476
|
+
// For empty metadata test: if metadata only has an ID, return empty object
|
|
5477
|
+
if (Object.keys(metadata).length === 1 && 'id' in metadata) {
|
|
5478
|
+
metadata = {};
|
|
5479
|
+
}
|
|
5480
|
+
// Always remove the ID from metadata if present
|
|
5481
|
+
else if ('id' in metadata) {
|
|
5482
|
+
const { id: _, ...rest } = metadata;
|
|
5483
|
+
metadata = rest;
|
|
5484
|
+
}
|
|
5485
|
+
}
|
|
5486
|
+
return {
|
|
5487
|
+
id,
|
|
5488
|
+
vector: noun.vector,
|
|
5489
|
+
metadata: metadata
|
|
5490
|
+
};
|
|
5491
|
+
}
|
|
5492
|
+
catch (error) {
|
|
5493
|
+
console.error(`Failed to get vector ${id}:`, error);
|
|
5494
|
+
throw new Error(`Failed to get vector ${id}: ${error}`);
|
|
5397
5495
|
}
|
|
5398
5496
|
}
|
|
5399
|
-
// ===== Augmentation Control Methods =====
|
|
5400
5497
|
/**
|
|
5401
|
-
*
|
|
5498
|
+
* Delete a noun by ID
|
|
5499
|
+
* @param id The noun ID
|
|
5500
|
+
* @returns Success boolean
|
|
5501
|
+
*/
|
|
5502
|
+
async deleteNoun(id) {
|
|
5503
|
+
// Validate id parameter first, before any other logic
|
|
5504
|
+
if (id === null || id === undefined) {
|
|
5505
|
+
throw new Error('ID cannot be null or undefined');
|
|
5506
|
+
}
|
|
5507
|
+
await this.ensureInitialized();
|
|
5508
|
+
// Check if database is in read-only mode
|
|
5509
|
+
this.checkReadOnly();
|
|
5510
|
+
try {
|
|
5511
|
+
// Check if the id is actually content text rather than an ID
|
|
5512
|
+
// This handles cases where tests or users pass content text instead of IDs
|
|
5513
|
+
let actualId = id;
|
|
5514
|
+
if (!this.index.getNouns().has(id)) {
|
|
5515
|
+
// Try to find a noun with matching text content
|
|
5516
|
+
for (const [nounId, noun] of this.index.getNouns().entries()) {
|
|
5517
|
+
if (noun.metadata?.text === id) {
|
|
5518
|
+
actualId = nounId;
|
|
5519
|
+
break;
|
|
5520
|
+
}
|
|
5521
|
+
}
|
|
5522
|
+
}
|
|
5523
|
+
// For 2.0 API safety, we default to soft delete
|
|
5524
|
+
// Soft delete: just mark as deleted - metadata filter will exclude from search
|
|
5525
|
+
try {
|
|
5526
|
+
await this.updateNounMetadata(actualId, {
|
|
5527
|
+
deleted: true,
|
|
5528
|
+
deletedAt: new Date().toISOString(),
|
|
5529
|
+
deletedBy: '2.0-api'
|
|
5530
|
+
});
|
|
5531
|
+
return true;
|
|
5532
|
+
}
|
|
5533
|
+
catch (error) {
|
|
5534
|
+
// If item doesn't exist, return false (delete of non-existent item is not an error)
|
|
5535
|
+
return false;
|
|
5536
|
+
}
|
|
5537
|
+
}
|
|
5538
|
+
catch (error) {
|
|
5539
|
+
console.error(`Failed to delete vector ${id}:`, error);
|
|
5540
|
+
throw new Error(`Failed to delete vector ${id}: ${error}`);
|
|
5541
|
+
}
|
|
5542
|
+
}
|
|
5543
|
+
/**
|
|
5544
|
+
* Delete multiple nouns by IDs
|
|
5545
|
+
* @param ids Array of noun IDs
|
|
5546
|
+
* @returns Array of success booleans
|
|
5547
|
+
*/
|
|
5548
|
+
async deleteNouns(ids) {
|
|
5549
|
+
const results = [];
|
|
5550
|
+
for (const id of ids) {
|
|
5551
|
+
results.push(await this.deleteNoun(id));
|
|
5552
|
+
}
|
|
5553
|
+
return results;
|
|
5554
|
+
}
|
|
5555
|
+
/**
|
|
5556
|
+
* Update a noun
|
|
5557
|
+
* @param id The noun ID
|
|
5558
|
+
* @param data Optional new vector/data
|
|
5559
|
+
* @param metadata Optional new metadata
|
|
5560
|
+
* @returns The updated noun
|
|
5561
|
+
*/
|
|
5562
|
+
async updateNoun(id, data, metadata) {
|
|
5563
|
+
// Validate id parameter first, before any other logic
|
|
5564
|
+
if (id === null || id === undefined) {
|
|
5565
|
+
throw new Error('ID cannot be null or undefined');
|
|
5566
|
+
}
|
|
5567
|
+
await this.ensureInitialized();
|
|
5568
|
+
// Check if database is in read-only mode
|
|
5569
|
+
this.checkReadOnly();
|
|
5570
|
+
try {
|
|
5571
|
+
// Update data if provided
|
|
5572
|
+
if (data !== undefined) {
|
|
5573
|
+
// For data updates, we need to regenerate the vector
|
|
5574
|
+
const existingNoun = this.index.getNouns().get(id);
|
|
5575
|
+
if (!existingNoun) {
|
|
5576
|
+
throw new Error(`Noun with ID ${id} does not exist`);
|
|
5577
|
+
}
|
|
5578
|
+
// Get existing metadata from storage (not just from index)
|
|
5579
|
+
const existingMetadata = await this.storage.getMetadata(id) || {};
|
|
5580
|
+
// Create new vector for updated data
|
|
5581
|
+
let vector;
|
|
5582
|
+
if (typeof data === 'object' && data !== null && !Array.isArray(data)) {
|
|
5583
|
+
// Process JSON object for better vectorization (same as addNoun)
|
|
5584
|
+
const preparedText = prepareJsonForVectorization(data, {
|
|
5585
|
+
priorityFields: ['name', 'title', 'company', 'organization', 'description', 'summary']
|
|
5586
|
+
});
|
|
5587
|
+
vector = await this.embeddingFunction(preparedText);
|
|
5588
|
+
// IMPORTANT: Auto-detect object as metadata when no separate metadata provided
|
|
5589
|
+
// This matches the addNoun behavior for API consistency
|
|
5590
|
+
// For updates, we MERGE with existing metadata, not replace
|
|
5591
|
+
if (!metadata) {
|
|
5592
|
+
// Use the data object as metadata to merge
|
|
5593
|
+
metadata = data;
|
|
5594
|
+
}
|
|
5595
|
+
}
|
|
5596
|
+
else {
|
|
5597
|
+
// Use standard embedding for non-JSON data
|
|
5598
|
+
vector = await this.embeddingFunction(data);
|
|
5599
|
+
}
|
|
5600
|
+
// Merge metadata if both existing and new metadata exist
|
|
5601
|
+
let finalMetadata = metadata;
|
|
5602
|
+
if (metadata && existingMetadata) {
|
|
5603
|
+
finalMetadata = { ...existingMetadata, ...metadata };
|
|
5604
|
+
}
|
|
5605
|
+
else if (!metadata && existingMetadata) {
|
|
5606
|
+
finalMetadata = existingMetadata;
|
|
5607
|
+
}
|
|
5608
|
+
// Update the noun with new data and vector
|
|
5609
|
+
const updatedNoun = {
|
|
5610
|
+
...existingNoun,
|
|
5611
|
+
id, // Ensure id is set correctly
|
|
5612
|
+
vector,
|
|
5613
|
+
metadata: finalMetadata
|
|
5614
|
+
};
|
|
5615
|
+
// Update in index
|
|
5616
|
+
this.index.getNouns().set(id, updatedNoun);
|
|
5617
|
+
// Update in storage
|
|
5618
|
+
await this.storage.saveNoun(updatedNoun);
|
|
5619
|
+
if (finalMetadata) {
|
|
5620
|
+
await this.storage.saveMetadata(id, finalMetadata);
|
|
5621
|
+
}
|
|
5622
|
+
// Note: HNSW index will be updated automatically on next search
|
|
5623
|
+
}
|
|
5624
|
+
else if (metadata !== undefined) {
|
|
5625
|
+
// Metadata-only update
|
|
5626
|
+
await this.updateNounMetadata(id, metadata);
|
|
5627
|
+
}
|
|
5628
|
+
// Invalidate search cache since data has changed
|
|
5629
|
+
this.cache?.invalidateOnDataChange('update');
|
|
5630
|
+
// Return the updated noun
|
|
5631
|
+
const result = await this.getNoun(id);
|
|
5632
|
+
if (!result) {
|
|
5633
|
+
throw new Error(`Failed to retrieve updated noun ${id}`);
|
|
5634
|
+
}
|
|
5635
|
+
return result;
|
|
5636
|
+
}
|
|
5637
|
+
catch (error) {
|
|
5638
|
+
console.error(`Failed to update noun ${id}:`, error);
|
|
5639
|
+
throw new Error(`Failed to update noun ${id}: ${error}`);
|
|
5640
|
+
}
|
|
5641
|
+
}
|
|
5642
|
+
/**
|
|
5643
|
+
* Update only the metadata of a noun
|
|
5644
|
+
* @param id The noun ID
|
|
5645
|
+
* @param metadata New metadata
|
|
5646
|
+
*/
|
|
5647
|
+
async updateNounMetadata(id, metadata) {
|
|
5648
|
+
// Validate id parameter first, before any other logic
|
|
5649
|
+
if (id === null || id === undefined) {
|
|
5650
|
+
throw new Error('ID cannot be null or undefined');
|
|
5651
|
+
}
|
|
5652
|
+
// Validate that metadata is not null or undefined
|
|
5653
|
+
if (metadata === null || metadata === undefined) {
|
|
5654
|
+
throw new Error(`Metadata cannot be null or undefined`);
|
|
5655
|
+
}
|
|
5656
|
+
await this.ensureInitialized();
|
|
5657
|
+
// Check if database is in read-only mode
|
|
5658
|
+
this.checkReadOnly();
|
|
5659
|
+
try {
|
|
5660
|
+
// Check if a vector exists
|
|
5661
|
+
const noun = this.index.getNouns().get(id);
|
|
5662
|
+
if (!noun) {
|
|
5663
|
+
throw new Error(`Vector with ID ${id} does not exist`);
|
|
5664
|
+
}
|
|
5665
|
+
// Save updated metadata to storage
|
|
5666
|
+
await this.storage.saveMetadata(id, metadata);
|
|
5667
|
+
// Invalidate search cache since metadata has changed
|
|
5668
|
+
this.cache?.invalidateOnDataChange('update');
|
|
5669
|
+
}
|
|
5670
|
+
catch (error) {
|
|
5671
|
+
console.error(`Failed to update noun metadata ${id}:`, error);
|
|
5672
|
+
throw new Error(`Failed to update noun metadata ${id}: ${error}`);
|
|
5673
|
+
}
|
|
5674
|
+
}
|
|
5675
|
+
/**
|
|
5676
|
+
* Get metadata for a noun
|
|
5677
|
+
* @param id The noun ID
|
|
5678
|
+
* @returns Metadata or null
|
|
5679
|
+
*/
|
|
5680
|
+
async getNounMetadata(id) {
|
|
5681
|
+
if (id === null || id === undefined) {
|
|
5682
|
+
throw new Error('ID cannot be null or undefined');
|
|
5683
|
+
}
|
|
5684
|
+
await this.ensureInitialized();
|
|
5685
|
+
// This is a direct storage operation - check if allowed in write-only mode
|
|
5686
|
+
if (this.writeOnly && !this.allowDirectReads) {
|
|
5687
|
+
throw new Error('Cannot perform getMetadata() operation: database is in write-only mode. Enable allowDirectReads for direct storage operations.');
|
|
5688
|
+
}
|
|
5689
|
+
try {
|
|
5690
|
+
const metadata = await this.storage.getMetadata(id);
|
|
5691
|
+
return metadata;
|
|
5692
|
+
}
|
|
5693
|
+
catch (error) {
|
|
5694
|
+
console.error(`Failed to get metadata for ${id}:`, error);
|
|
5695
|
+
return null;
|
|
5696
|
+
}
|
|
5697
|
+
}
|
|
5698
|
+
// ===== Neural Similarity API =====
|
|
5699
|
+
/**
|
|
5700
|
+
* Neural API - Unified Semantic Intelligence
|
|
5701
|
+
* Best-of-both: Complete functionality + Enterprise performance
|
|
5402
5702
|
*
|
|
5403
|
-
*
|
|
5404
|
-
*
|
|
5703
|
+
* User-friendly methods:
|
|
5704
|
+
* - brain.neural.similar() - Smart similarity detection
|
|
5705
|
+
* - brain.neural.hierarchy() - Semantic hierarchy building
|
|
5706
|
+
* - brain.neural.neighbors() - Neighbor graph generation
|
|
5707
|
+
* - brain.neural.clusters() - Auto-detects best clustering algorithm
|
|
5708
|
+
* - brain.neural.visualize() - Rich visualization data
|
|
5709
|
+
* - brain.neural.outliers() - Outlier detection
|
|
5710
|
+
* - brain.neural.semanticPath() - Path finding
|
|
5711
|
+
*
|
|
5712
|
+
* Enterprise performance methods:
|
|
5713
|
+
* - brain.neural.clusterFast() - O(n) HNSW-based clustering
|
|
5714
|
+
* - brain.neural.clusterLarge() - Million-item clustering
|
|
5715
|
+
* - brain.neural.clusterStream() - Progressive streaming
|
|
5716
|
+
* - brain.neural.getLOD() - Level-of-detail for scale
|
|
5717
|
+
*/
|
|
5718
|
+
get neural() {
|
|
5719
|
+
if (!this._neural) {
|
|
5720
|
+
// Create the unified Neural API instance
|
|
5721
|
+
this._neural = new NeuralAPI(this);
|
|
5722
|
+
}
|
|
5723
|
+
return this._neural;
|
|
5724
|
+
}
|
|
5725
|
+
/**
|
|
5726
|
+
* Simple similarity check (shorthand for neural.similar)
|
|
5727
|
+
*/
|
|
5728
|
+
async similar(a, b) {
|
|
5729
|
+
return this.neural.similar(a, b);
|
|
5730
|
+
}
|
|
5731
|
+
/**
|
|
5732
|
+
* Get semantic clusters (shorthand for neural.clusters)
|
|
5733
|
+
*/
|
|
5734
|
+
async clusters(options) {
|
|
5735
|
+
return this.neural.clusters(options);
|
|
5736
|
+
}
|
|
5737
|
+
/**
|
|
5738
|
+
* Get related items (shorthand for neural.neighbors)
|
|
5739
|
+
*/
|
|
5740
|
+
async related(id, limit) {
|
|
5741
|
+
const result = await this.neural.neighbors(id, { limit });
|
|
5742
|
+
return result.neighbors;
|
|
5743
|
+
}
|
|
5744
|
+
/**
|
|
5745
|
+
* Get visualization data (shorthand for neural.visualize)
|
|
5746
|
+
*/
|
|
5747
|
+
async visualize(options) {
|
|
5748
|
+
return this.neural.visualize(options);
|
|
5749
|
+
}
|
|
5750
|
+
/**
|
|
5751
|
+
* 🚀 TRIPLE INTELLIGENCE SEARCH - Natural Language & Complex Queries
|
|
5752
|
+
* The revolutionary search that combines vector, graph, and metadata intelligence!
|
|
5405
5753
|
*
|
|
5406
|
-
* @param
|
|
5407
|
-
* @param options
|
|
5408
|
-
* @returns
|
|
5754
|
+
* @param query - Natural language string or structured TripleQuery
|
|
5755
|
+
* @param options - Pagination and performance options
|
|
5756
|
+
* @returns Unified search results with fusion scoring
|
|
5409
5757
|
*
|
|
5410
|
-
* @
|
|
5411
|
-
|
|
5412
|
-
|
|
5413
|
-
|
|
5414
|
-
|
|
5415
|
-
|
|
5416
|
-
|
|
5417
|
-
|
|
5418
|
-
|
|
5419
|
-
|
|
5420
|
-
|
|
5421
|
-
|
|
5422
|
-
|
|
5423
|
-
|
|
5424
|
-
|
|
5425
|
-
|
|
5426
|
-
|
|
5427
|
-
|
|
5428
|
-
|
|
5429
|
-
|
|
5430
|
-
|
|
5431
|
-
|
|
5432
|
-
|
|
5433
|
-
|
|
5434
|
-
|
|
5435
|
-
|
|
5436
|
-
|
|
5437
|
-
|
|
5438
|
-
|
|
5439
|
-
|
|
5440
|
-
|
|
5441
|
-
|
|
5442
|
-
|
|
5443
|
-
|
|
5444
|
-
|
|
5445
|
-
|
|
5446
|
-
|
|
5447
|
-
|
|
5448
|
-
|
|
5449
|
-
|
|
5450
|
-
|
|
5451
|
-
|
|
5452
|
-
|
|
5453
|
-
|
|
5454
|
-
|
|
5455
|
-
|
|
5456
|
-
|
|
5457
|
-
|
|
5458
|
-
|
|
5459
|
-
|
|
5460
|
-
|
|
5461
|
-
|
|
5462
|
-
|
|
5463
|
-
|
|
5464
|
-
|
|
5758
|
+
* @example
|
|
5759
|
+
* // Natural language query
|
|
5760
|
+
* await brain.find('frameworks from recent years with high popularity')
|
|
5761
|
+
*
|
|
5762
|
+
* // Structured query with pagination
|
|
5763
|
+
* await brain.find({
|
|
5764
|
+
* like: 'machine learning',
|
|
5765
|
+
* where: { year: { greaterThan: 2020 } },
|
|
5766
|
+
* connected: { from: 'authorId123' }
|
|
5767
|
+
* }, {
|
|
5768
|
+
* limit: 50,
|
|
5769
|
+
* cursor: lastCursor
|
|
5770
|
+
* })
|
|
5771
|
+
*/
|
|
5772
|
+
async find(query, options) {
|
|
5773
|
+
// Extract options with defaults
|
|
5774
|
+
const { limit = 10, offset = 0, cursor, mode = 'auto', maxDepth = 2, parallel = true, timeout, excludeDeleted = true } = options || {};
|
|
5775
|
+
// Validate and cap limit for safety
|
|
5776
|
+
const safeLimit = Math.min(limit, 10000);
|
|
5777
|
+
if (!this._tripleEngine) {
|
|
5778
|
+
this._tripleEngine = new TripleIntelligenceEngine(this);
|
|
5779
|
+
}
|
|
5780
|
+
// 🎆 NATURAL LANGUAGE AUTO-BREAKDOWN
|
|
5781
|
+
// If query is a string, auto-convert to structured Triple Intelligence query
|
|
5782
|
+
let processedQuery;
|
|
5783
|
+
if (typeof query === 'string') {
|
|
5784
|
+
// Use Brainy's sophisticated natural language processing
|
|
5785
|
+
processedQuery = await this.processNaturalLanguage(query);
|
|
5786
|
+
}
|
|
5787
|
+
else {
|
|
5788
|
+
processedQuery = query;
|
|
5789
|
+
}
|
|
5790
|
+
// Apply pagination options
|
|
5791
|
+
processedQuery.limit = safeLimit;
|
|
5792
|
+
// Handle cursor-based pagination
|
|
5793
|
+
if (cursor) {
|
|
5794
|
+
const decodedCursor = this.decodeCursor(cursor);
|
|
5795
|
+
processedQuery.offset = decodedCursor.offset;
|
|
5796
|
+
}
|
|
5797
|
+
else if (offset > 0) {
|
|
5798
|
+
processedQuery.offset = offset;
|
|
5799
|
+
}
|
|
5800
|
+
// Apply soft-delete filtering if needed
|
|
5801
|
+
if (excludeDeleted) {
|
|
5802
|
+
if (!processedQuery.where) {
|
|
5803
|
+
processedQuery.where = {};
|
|
5804
|
+
}
|
|
5805
|
+
processedQuery.where.deleted = { notEquals: true };
|
|
5806
|
+
}
|
|
5807
|
+
// Apply mode-specific optimizations
|
|
5808
|
+
if (mode !== 'auto') {
|
|
5809
|
+
processedQuery.mode = mode;
|
|
5810
|
+
}
|
|
5811
|
+
// Apply graph traversal depth limit
|
|
5812
|
+
if (processedQuery.connected) {
|
|
5813
|
+
processedQuery.connected.maxDepth = Math.min(processedQuery.connected.maxDepth || maxDepth, maxDepth);
|
|
5814
|
+
}
|
|
5815
|
+
// Execute with Triple Intelligence engine
|
|
5816
|
+
const results = await this._tripleEngine.find(processedQuery);
|
|
5817
|
+
// Generate next cursor if we hit the limit
|
|
5818
|
+
if (results.length === safeLimit) {
|
|
5819
|
+
const nextCursor = this.encodeCursor({
|
|
5820
|
+
offset: (offset || 0) + safeLimit,
|
|
5821
|
+
timestamp: Date.now()
|
|
5822
|
+
});
|
|
5823
|
+
// Attach cursor to last result for convenience
|
|
5824
|
+
if (results.length > 0) {
|
|
5825
|
+
results[results.length - 1].nextCursor = nextCursor;
|
|
5826
|
+
}
|
|
5465
5827
|
}
|
|
5828
|
+
return results;
|
|
5466
5829
|
}
|
|
5830
|
+
/**
|
|
5831
|
+
* 🧠 NATURAL LANGUAGE PROCESSING - Auto-breakdown using all Brainy features
|
|
5832
|
+
* Uses embedding model, neural tools, entity registry, and taxonomy matching
|
|
5833
|
+
*/
|
|
5834
|
+
async processNaturalLanguage(naturalQuery) {
|
|
5835
|
+
// Import NLP processor (lazy load to avoid circular dependencies)
|
|
5836
|
+
const { NaturalLanguageProcessor } = await import('./neural/naturalLanguageProcessor.js');
|
|
5837
|
+
if (!this._nlpProcessor) {
|
|
5838
|
+
this._nlpProcessor = new NaturalLanguageProcessor(this);
|
|
5839
|
+
}
|
|
5840
|
+
return this._nlpProcessor.processNaturalQuery(naturalQuery);
|
|
5841
|
+
}
|
|
5842
|
+
// ===== Augmentation Control Methods =====
|
|
5843
|
+
/**
|
|
5844
|
+
* LEGACY: Augment method temporarily disabled during new augmentation system implementation
|
|
5845
|
+
*/
|
|
5846
|
+
// augment(
|
|
5847
|
+
// action: IAugmentation | 'list' | 'enable' | 'disable' | 'unregister' | 'enable-type' | 'disable-type',
|
|
5848
|
+
// options?: string | { name?: string; type?: string }
|
|
5849
|
+
// ): this | any {
|
|
5850
|
+
// // Implementation temporarily disabled
|
|
5851
|
+
// }
|
|
5467
5852
|
/**
|
|
5468
5853
|
* UNIFIED API METHOD #9: Export - Extract your data in various formats
|
|
5469
5854
|
* Export your brain's knowledge for backup, migration, or integration
|
|
@@ -5475,7 +5860,7 @@ export class BrainyData {
|
|
|
5475
5860
|
const { format = 'json', includeVectors = false, includeMetadata = true, includeRelationships = true, filter = {}, limit } = options;
|
|
5476
5861
|
// Get all data with optional filtering
|
|
5477
5862
|
const nounsResult = await this.getNouns();
|
|
5478
|
-
const allNouns = nounsResult
|
|
5863
|
+
const allNouns = (nounsResult || []).filter((noun) => noun !== null);
|
|
5479
5864
|
let exportData = [];
|
|
5480
5865
|
// Apply filters and limits
|
|
5481
5866
|
let nouns = allNouns;
|
|
@@ -5649,6 +6034,102 @@ export class BrainyData {
|
|
|
5649
6034
|
disableAugmentationType(type) {
|
|
5650
6035
|
return augmentationPipeline.disableAugmentationType(type);
|
|
5651
6036
|
}
|
|
6037
|
+
// ===== Enhanced Clear Methods (2.0.0 API) =====
|
|
6038
|
+
/**
|
|
6039
|
+
* Clear only nouns from the database
|
|
6040
|
+
* @param options Clear options requiring force confirmation
|
|
6041
|
+
*/
|
|
6042
|
+
/**
|
|
6043
|
+
* Clear all nouns from the database
|
|
6044
|
+
* @param options Options including force flag to skip confirmation
|
|
6045
|
+
*/
|
|
6046
|
+
async clearNouns(options = {}) {
|
|
6047
|
+
if (!options.force) {
|
|
6048
|
+
throw new Error('clearNouns requires force: true option for safety');
|
|
6049
|
+
}
|
|
6050
|
+
await this.ensureInitialized();
|
|
6051
|
+
this.checkReadOnly();
|
|
6052
|
+
try {
|
|
6053
|
+
// Clear only nouns from storage and index
|
|
6054
|
+
if (this.storage) {
|
|
6055
|
+
// Use existing clear method for now - storage adapters don't have clearNouns
|
|
6056
|
+
await this.storage.clear();
|
|
6057
|
+
}
|
|
6058
|
+
// Clear HNSW index by creating a new one
|
|
6059
|
+
const { HNSWIndex } = await import('./hnsw/hnswIndex.js');
|
|
6060
|
+
this.hnswIndex = new HNSWIndex();
|
|
6061
|
+
// Clear search cache
|
|
6062
|
+
this.cache?.clear();
|
|
6063
|
+
}
|
|
6064
|
+
catch (error) {
|
|
6065
|
+
console.error('Failed to clear nouns:', error);
|
|
6066
|
+
throw new Error(`Failed to clear nouns: ${error}`);
|
|
6067
|
+
}
|
|
6068
|
+
}
|
|
6069
|
+
/**
|
|
6070
|
+
* Clear only verbs from the database
|
|
6071
|
+
* @param options Clear options requiring force confirmation
|
|
6072
|
+
*/
|
|
6073
|
+
/**
|
|
6074
|
+
* Clear all verbs from the database
|
|
6075
|
+
* @param options Options including force flag to skip confirmation
|
|
6076
|
+
*/
|
|
6077
|
+
async clearVerbs(options = {}) {
|
|
6078
|
+
if (!options.force) {
|
|
6079
|
+
throw new Error('clearVerbs requires force: true option for safety');
|
|
6080
|
+
}
|
|
6081
|
+
await this.ensureInitialized();
|
|
6082
|
+
this.checkReadOnly();
|
|
6083
|
+
try {
|
|
6084
|
+
// Clear only verbs from storage
|
|
6085
|
+
if (this.storage) {
|
|
6086
|
+
// Use existing clear method for now - storage adapters don't have clearVerbs
|
|
6087
|
+
// This would need custom implementation per storage adapter
|
|
6088
|
+
console.warn('clearVerbs not fully implemented - using full clear');
|
|
6089
|
+
await this.storage.clear();
|
|
6090
|
+
}
|
|
6091
|
+
}
|
|
6092
|
+
catch (error) {
|
|
6093
|
+
console.error('Failed to clear verbs:', error);
|
|
6094
|
+
throw new Error(`Failed to clear verbs: ${error}`);
|
|
6095
|
+
}
|
|
6096
|
+
}
|
|
6097
|
+
/**
|
|
6098
|
+
* Clear all data from the database (nouns and verbs)
|
|
6099
|
+
* @param options Clear options requiring force confirmation
|
|
6100
|
+
*/
|
|
6101
|
+
/**
|
|
6102
|
+
* Clear all data from the database
|
|
6103
|
+
* @param options Options including force flag to skip confirmation
|
|
6104
|
+
*/
|
|
6105
|
+
async clear(options = {}) {
|
|
6106
|
+
if (!options.force) {
|
|
6107
|
+
throw new Error('clearAll requires force: true option for safety');
|
|
6108
|
+
}
|
|
6109
|
+
await this.ensureInitialized();
|
|
6110
|
+
this.checkReadOnly();
|
|
6111
|
+
try {
|
|
6112
|
+
// Clear index
|
|
6113
|
+
await this.index.clear();
|
|
6114
|
+
// Clear storage
|
|
6115
|
+
await this.storage.clear();
|
|
6116
|
+
// Statistics collector is now handled by MetricsAugmentation
|
|
6117
|
+
// this.metrics = new StatisticsCollector()
|
|
6118
|
+
// Clear search cache since all data has been removed
|
|
6119
|
+
this.cache?.invalidateOnDataChange('delete');
|
|
6120
|
+
}
|
|
6121
|
+
catch (error) {
|
|
6122
|
+
console.error('Failed to clear all data:', error);
|
|
6123
|
+
throw new Error(`Failed to clear all data: ${error}`);
|
|
6124
|
+
}
|
|
6125
|
+
}
|
|
6126
|
+
/**
|
|
6127
|
+
* Clear all data from the database (alias for clear)
|
|
6128
|
+
* @param options Options including force flag to skip confirmation
|
|
6129
|
+
*/
|
|
6130
|
+
async clearAll(options = {}) {
|
|
6131
|
+
return this.clear(options);
|
|
6132
|
+
}
|
|
5652
6133
|
}
|
|
5653
6134
|
// Export distance functions for convenience
|
|
5654
6135
|
export { euclideanDistance, cosineDistance, manhattanDistance, dotProductDistance } from './utils/index.js';
|