@soulcraft/brainy 1.5.0 → 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +188 -0
- package/LICENSE +2 -2
- package/README.md +201 -596
- package/bin/brainy-interactive.js +564 -0
- package/bin/brainy-ts.js +18 -0
- package/bin/brainy.js +672 -81
- package/dist/augmentationPipeline.d.ts +48 -220
- package/dist/augmentationPipeline.js +60 -508
- package/dist/augmentationRegistry.d.ts +22 -31
- package/dist/augmentationRegistry.js +28 -79
- package/dist/augmentations/apiServerAugmentation.d.ts +108 -0
- package/dist/augmentations/apiServerAugmentation.js +502 -0
- package/dist/augmentations/batchProcessingAugmentation.d.ts +95 -0
- package/dist/augmentations/batchProcessingAugmentation.js +567 -0
- package/dist/augmentations/brainyAugmentation.d.ts +153 -0
- package/dist/augmentations/brainyAugmentation.js +145 -0
- package/dist/augmentations/cacheAugmentation.d.ts +105 -0
- package/dist/augmentations/cacheAugmentation.js +238 -0
- package/dist/augmentations/conduitAugmentations.d.ts +54 -156
- package/dist/augmentations/conduitAugmentations.js +156 -1082
- package/dist/augmentations/connectionPoolAugmentation.d.ts +62 -0
- package/dist/augmentations/connectionPoolAugmentation.js +316 -0
- package/dist/augmentations/defaultAugmentations.d.ts +53 -0
- package/dist/augmentations/defaultAugmentations.js +88 -0
- package/dist/augmentations/entityRegistryAugmentation.d.ts +126 -0
- package/dist/augmentations/entityRegistryAugmentation.js +386 -0
- package/dist/augmentations/indexAugmentation.d.ts +117 -0
- package/dist/augmentations/indexAugmentation.js +284 -0
- package/dist/augmentations/intelligentVerbScoringAugmentation.d.ts +152 -0
- package/dist/augmentations/intelligentVerbScoringAugmentation.js +554 -0
- package/dist/augmentations/metricsAugmentation.d.ts +202 -0
- package/dist/augmentations/metricsAugmentation.js +291 -0
- package/dist/augmentations/monitoringAugmentation.d.ts +94 -0
- package/dist/augmentations/monitoringAugmentation.js +227 -0
- package/dist/augmentations/neuralImport.d.ts +50 -117
- package/dist/augmentations/neuralImport.js +255 -629
- package/dist/augmentations/requestDeduplicatorAugmentation.d.ts +52 -0
- package/dist/augmentations/requestDeduplicatorAugmentation.js +162 -0
- package/dist/augmentations/serverSearchAugmentations.d.ts +43 -22
- package/dist/augmentations/serverSearchAugmentations.js +125 -72
- package/dist/augmentations/storageAugmentation.d.ts +54 -0
- package/dist/augmentations/storageAugmentation.js +93 -0
- package/dist/augmentations/storageAugmentations.d.ts +96 -0
- package/dist/augmentations/storageAugmentations.js +182 -0
- package/dist/augmentations/synapseAugmentation.d.ts +156 -0
- package/dist/augmentations/synapseAugmentation.js +312 -0
- package/dist/augmentations/walAugmentation.d.ts +108 -0
- package/dist/augmentations/walAugmentation.js +515 -0
- package/dist/brainyData.d.ts +404 -130
- package/dist/brainyData.js +1331 -853
- package/dist/chat/BrainyChat.d.ts +16 -8
- package/dist/chat/BrainyChat.js +60 -32
- package/dist/chat/ChatCLI.d.ts +1 -1
- package/dist/chat/ChatCLI.js +6 -6
- package/dist/cli/catalog.d.ts +3 -3
- package/dist/cli/catalog.js +116 -70
- package/dist/cli/commands/core.d.ts +61 -0
- package/dist/cli/commands/core.js +348 -0
- package/dist/cli/commands/neural.d.ts +25 -0
- package/dist/cli/commands/neural.js +508 -0
- package/dist/cli/commands/utility.d.ts +37 -0
- package/dist/cli/commands/utility.js +276 -0
- package/dist/cli/index.d.ts +7 -0
- package/dist/cli/index.js +167 -0
- package/dist/cli/interactive.d.ts +164 -0
- package/dist/cli/interactive.js +542 -0
- package/dist/cortex/neuralImport.js +5 -5
- package/dist/critical/model-guardian.js +11 -4
- package/dist/embeddings/lightweight-embedder.d.ts +23 -0
- package/dist/embeddings/lightweight-embedder.js +136 -0
- package/dist/embeddings/universal-memory-manager.d.ts +38 -0
- package/dist/embeddings/universal-memory-manager.js +206 -0
- package/dist/embeddings/worker-embedding.d.ts +7 -0
- package/dist/embeddings/worker-embedding.js +77 -0
- package/dist/embeddings/worker-manager.d.ts +28 -0
- package/dist/embeddings/worker-manager.js +162 -0
- package/dist/examples/basicUsage.js +7 -7
- package/dist/graph/pathfinding.d.ts +78 -0
- package/dist/graph/pathfinding.js +393 -0
- package/dist/hnsw/hnswIndex.d.ts +13 -0
- package/dist/hnsw/hnswIndex.js +35 -0
- package/dist/hnsw/hnswIndexOptimized.d.ts +1 -0
- package/dist/hnsw/hnswIndexOptimized.js +3 -0
- package/dist/index.d.ts +9 -11
- package/dist/index.js +21 -11
- package/dist/indices/fieldIndex.d.ts +76 -0
- package/dist/indices/fieldIndex.js +357 -0
- package/dist/mcp/brainyMCPAdapter.js +3 -2
- package/dist/mcp/mcpAugmentationToolset.js +11 -17
- package/dist/neural/embeddedPatterns.d.ts +41 -0
- package/dist/neural/embeddedPatterns.js +4044 -0
- package/dist/neural/naturalLanguageProcessor.d.ts +94 -0
- package/dist/neural/naturalLanguageProcessor.js +317 -0
- package/dist/neural/naturalLanguageProcessorStatic.d.ts +64 -0
- package/dist/neural/naturalLanguageProcessorStatic.js +151 -0
- package/dist/neural/neuralAPI.d.ts +255 -0
- package/dist/neural/neuralAPI.js +612 -0
- package/dist/neural/patternLibrary.d.ts +101 -0
- package/dist/neural/patternLibrary.js +313 -0
- package/dist/neural/patterns.d.ts +27 -0
- package/dist/neural/patterns.js +68 -0
- package/dist/neural/staticPatternMatcher.d.ts +35 -0
- package/dist/neural/staticPatternMatcher.js +153 -0
- package/dist/scripts/precomputePatternEmbeddings.d.ts +19 -0
- package/dist/scripts/precomputePatternEmbeddings.js +100 -0
- package/dist/storage/adapters/fileSystemStorage.d.ts +5 -0
- package/dist/storage/adapters/fileSystemStorage.js +20 -0
- package/dist/storage/adapters/s3CompatibleStorage.d.ts +5 -0
- package/dist/storage/adapters/s3CompatibleStorage.js +16 -0
- package/dist/storage/enhancedClearOperations.d.ts +83 -0
- package/dist/storage/enhancedClearOperations.js +345 -0
- package/dist/storage/storageFactory.js +31 -27
- package/dist/triple/TripleIntelligence.d.ts +134 -0
- package/dist/triple/TripleIntelligence.js +548 -0
- package/dist/types/augmentations.d.ts +45 -344
- package/dist/types/augmentations.js +5 -2
- package/dist/types/brainyDataInterface.d.ts +20 -10
- package/dist/types/graphTypes.d.ts +46 -0
- package/dist/types/graphTypes.js +16 -2
- package/dist/utils/BoundedRegistry.d.ts +29 -0
- package/dist/utils/BoundedRegistry.js +54 -0
- package/dist/utils/embedding.js +20 -3
- package/dist/utils/hybridModelManager.js +10 -5
- package/dist/utils/metadataFilter.d.ts +33 -19
- package/dist/utils/metadataFilter.js +58 -23
- package/dist/utils/metadataIndex.d.ts +37 -6
- package/dist/utils/metadataIndex.js +427 -64
- package/dist/utils/requestDeduplicator.d.ts +10 -0
- package/dist/utils/requestDeduplicator.js +24 -0
- package/dist/utils/unifiedCache.d.ts +103 -0
- package/dist/utils/unifiedCache.js +311 -0
- package/package.json +43 -128
- package/scripts/ensure-models.js +108 -0
- package/scripts/prepare-models.js +387 -0
- package/OFFLINE_MODELS.md +0 -56
- package/dist/intelligence/neuralEngine.d.ts +0 -207
- package/dist/intelligence/neuralEngine.js +0 -706
- package/dist/utils/modelLoader.d.ts +0 -32
- package/dist/utils/modelLoader.js +0 -219
- package/dist/utils/modelManager.d.ts +0 -77
- package/dist/utils/modelManager.js +0 -219
package/dist/brainyData.js
CHANGED
|
@@ -4,25 +4,52 @@
|
|
|
4
4
|
*/
|
|
5
5
|
import { v4 as uuidv4 } from './universal/uuid.js';
|
|
6
6
|
import { HNSWIndex } from './hnsw/hnswIndex.js';
|
|
7
|
-
import { ExecutionMode } from './augmentationPipeline.js';
|
|
8
7
|
import { HNSWIndexOptimized } from './hnsw/hnswIndexOptimized.js';
|
|
9
|
-
import { createStorage } from './storage/storageFactory.js';
|
|
10
8
|
import { cosineDistance, defaultEmbeddingFunction, cleanupWorkerPools, batchEmbed } from './utils/index.js';
|
|
11
9
|
import { getAugmentationVersion } from './utils/version.js';
|
|
12
10
|
import { matchesMetadataFilter } from './utils/metadataFilter.js';
|
|
13
|
-
import { MetadataIndexManager } from './utils/metadataIndex.js';
|
|
14
11
|
import { NounType, VerbType } from './types/graphTypes.js';
|
|
15
12
|
import { createServerSearchAugmentations } from './augmentations/serverSearchAugmentations.js';
|
|
16
|
-
import { IntelligentVerbScoring } from './augmentations/intelligentVerbScoring.js';
|
|
17
13
|
import { augmentationPipeline } from './augmentationPipeline.js';
|
|
18
14
|
import { prodLog } from './utils/logger.js';
|
|
19
15
|
import { prepareJsonForVectorization, extractFieldFromJson } from './utils/jsonProcessing.js';
|
|
20
|
-
import { DistributedConfigManager, HashPartitioner, OperationalModeFactory, DomainDetector
|
|
21
|
-
import { SearchCache } from './utils/searchCache.js';
|
|
16
|
+
import { DistributedConfigManager, HashPartitioner, OperationalModeFactory, DomainDetector } from './distributed/index.js';
|
|
22
17
|
import { CacheAutoConfigurator } from './utils/cacheAutoConfig.js';
|
|
23
|
-
import {
|
|
24
|
-
import {
|
|
18
|
+
import { RequestDeduplicator } from './utils/requestDeduplicator.js';
|
|
19
|
+
import { AugmentationRegistry } from './augmentations/brainyAugmentation.js';
|
|
20
|
+
import { WALAugmentation } from './augmentations/walAugmentation.js';
|
|
21
|
+
import { RequestDeduplicatorAugmentation } from './augmentations/requestDeduplicatorAugmentation.js';
|
|
22
|
+
import { ConnectionPoolAugmentation } from './augmentations/connectionPoolAugmentation.js';
|
|
23
|
+
import { BatchProcessingAugmentation } from './augmentations/batchProcessingAugmentation.js';
|
|
24
|
+
import { EntityRegistryAugmentation, AutoRegisterEntitiesAugmentation } from './augmentations/entityRegistryAugmentation.js';
|
|
25
|
+
import { createDefaultAugmentations } from './augmentations/defaultAugmentations.js';
|
|
26
|
+
// import { RealtimeStreamingAugmentation } from './augmentations/realtimeStreamingAugmentation.js'
|
|
27
|
+
import { IntelligentVerbScoringAugmentation } from './augmentations/intelligentVerbScoringAugmentation.js';
|
|
28
|
+
import { NeuralAPI } from './neural/neuralAPI.js';
|
|
29
|
+
import { TripleIntelligenceEngine } from './triple/TripleIntelligence.js';
|
|
25
30
|
export class BrainyData {
|
|
31
|
+
// REMOVED: HealthMonitor is now handled by MonitoringAugmentation
|
|
32
|
+
// Statistics collector
|
|
33
|
+
// REMOVED: StatisticsCollector is now handled by MetricsAugmentation
|
|
34
|
+
// Clean augmentation accessors for internal use
|
|
35
|
+
get cache() {
|
|
36
|
+
return this.augmentations.get('cache');
|
|
37
|
+
}
|
|
38
|
+
// IMPORTANT: this.index returns the HNSW vector index, NOT the metadata index!
|
|
39
|
+
// The metadata index is available through this.metadataIndex
|
|
40
|
+
get index() {
|
|
41
|
+
return this.hnswIndex;
|
|
42
|
+
}
|
|
43
|
+
// Metadata index for field-based queries (from IndexAugmentation)
|
|
44
|
+
get metadataIndex() {
|
|
45
|
+
return this.augmentations.get('index');
|
|
46
|
+
}
|
|
47
|
+
get metrics() {
|
|
48
|
+
return this.augmentations.get('metrics');
|
|
49
|
+
}
|
|
50
|
+
get monitoring() {
|
|
51
|
+
return this.augmentations.get('monitoring');
|
|
52
|
+
}
|
|
26
53
|
/**
|
|
27
54
|
* Get the vector dimensions
|
|
28
55
|
*/
|
|
@@ -43,18 +70,30 @@ export class BrainyData {
|
|
|
43
70
|
const config = this.index.getConfig();
|
|
44
71
|
return config.efConstruction || 200;
|
|
45
72
|
}
|
|
73
|
+
/**
|
|
74
|
+
* Check if BrainyData has been initialized
|
|
75
|
+
*/
|
|
76
|
+
get initialized() {
|
|
77
|
+
return this.isInitialized;
|
|
78
|
+
}
|
|
46
79
|
/**
|
|
47
80
|
* Create a new vector database
|
|
48
81
|
*/
|
|
49
82
|
constructor(config = {}) {
|
|
50
83
|
this.storage = null;
|
|
51
|
-
|
|
84
|
+
// REMOVED: MetadataIndex is now handled by IndexAugmentation
|
|
52
85
|
this.isInitialized = false;
|
|
53
86
|
this.isInitializing = false;
|
|
54
87
|
this.storageConfig = {};
|
|
55
88
|
this.useOptimizedIndex = false;
|
|
56
89
|
this.loggingConfig = { verbose: true };
|
|
57
90
|
this.defaultService = 'default';
|
|
91
|
+
// REMOVED: SearchCache is now handled by CacheAugmentation
|
|
92
|
+
/**
|
|
93
|
+
* Enterprise augmentation system
|
|
94
|
+
* Handles WAL, connection pooling, batching, streaming, and intelligent scoring
|
|
95
|
+
*/
|
|
96
|
+
this.augmentations = new AugmentationRegistry();
|
|
58
97
|
// Timeout and retry configuration
|
|
59
98
|
this.timeoutConfig = {};
|
|
60
99
|
this.retryConfig = {};
|
|
@@ -69,10 +108,10 @@ export class BrainyData {
|
|
|
69
108
|
this.maintenanceIntervals = [];
|
|
70
109
|
this.lastUpdateTime = 0;
|
|
71
110
|
this.lastKnownNounCount = 0;
|
|
72
|
-
// Remote server properties
|
|
111
|
+
// Remote server properties - TODO: Implement in post-2.0.0 release
|
|
73
112
|
this.remoteServerConfig = null;
|
|
74
|
-
|
|
75
|
-
|
|
113
|
+
// private serverSearchConduit: ServerSearchConduitAugmentation | null = null
|
|
114
|
+
// private serverConnection: WebSocketConnection | null = null
|
|
76
115
|
this.intelligentVerbScoring = null;
|
|
77
116
|
// Distributed mode properties
|
|
78
117
|
this.distributedConfig = null;
|
|
@@ -80,9 +119,6 @@ export class BrainyData {
|
|
|
80
119
|
this.partitioner = null;
|
|
81
120
|
this.operationalMode = null;
|
|
82
121
|
this.domainDetector = null;
|
|
83
|
-
this.healthMonitor = null;
|
|
84
|
-
// Statistics collector
|
|
85
|
-
this.statisticsCollector = new StatisticsCollector();
|
|
86
122
|
// Store config
|
|
87
123
|
this.config = config;
|
|
88
124
|
// Set dimensions to fixed value of 384 (all-MiniLM-L6-v2 dimension)
|
|
@@ -96,7 +132,7 @@ export class BrainyData {
|
|
|
96
132
|
hnswConfig.useDiskBasedIndex = true;
|
|
97
133
|
}
|
|
98
134
|
// Temporarily use base HNSW index for metadata filtering
|
|
99
|
-
this.
|
|
135
|
+
this.hnswIndex = new HNSWIndex(hnswConfig, this.distanceFunction);
|
|
100
136
|
this.useOptimizedIndex = false;
|
|
101
137
|
// Set storage if provided, otherwise it will be initialized in init()
|
|
102
138
|
this.storage = config.storageAdapter || null;
|
|
@@ -203,20 +239,165 @@ export class BrainyData {
|
|
|
203
239
|
prodLog.info(this.cacheAutoConfigurator.getConfigExplanation(autoConfig));
|
|
204
240
|
}
|
|
205
241
|
}
|
|
206
|
-
//
|
|
207
|
-
this.searchCache = new SearchCache(finalSearchCacheConfig)
|
|
208
|
-
//
|
|
209
|
-
|
|
210
|
-
//
|
|
211
|
-
|
|
212
|
-
this.intelligentVerbScoring = new IntelligentVerbScoring(config.intelligentVerbScoring);
|
|
213
|
-
this.intelligentVerbScoring.enabled = true;
|
|
214
|
-
}
|
|
242
|
+
// Search cache is now handled by CacheAugmentation
|
|
243
|
+
// this.searchCache = new SearchCache<T>(finalSearchCacheConfig)
|
|
244
|
+
// Keep reference for compatibility (will be set by augmentation)
|
|
245
|
+
// Augmentation system will be initialized in init() method
|
|
246
|
+
// Legacy systems completely replaced by augmentation architecture
|
|
247
|
+
// All intelligent systems now handled by augmentations
|
|
215
248
|
}
|
|
216
249
|
/**
|
|
217
250
|
* Check if the database is in read-only mode and throw an error if it is
|
|
218
251
|
* @throws Error if the database is in read-only mode
|
|
219
252
|
*/
|
|
253
|
+
/**
|
|
254
|
+
* Register default augmentations without initializing them
|
|
255
|
+
* Phase 1 of two-phase initialization
|
|
256
|
+
*/
|
|
257
|
+
registerDefaultAugmentations() {
|
|
258
|
+
// Register enterprise-grade augmentations in priority order
|
|
259
|
+
// Note: These are registered but NOT initialized yet (no context)
|
|
260
|
+
// Register core feature augmentations (previously hardcoded)
|
|
261
|
+
// These replace SearchCache, MetadataIndex, StatisticsCollector, HealthMonitor
|
|
262
|
+
const defaultAugs = createDefaultAugmentations({
|
|
263
|
+
cache: this.config.searchCache !== undefined ? this.config.searchCache : true,
|
|
264
|
+
index: this.config.metadataIndex !== undefined ? this.config.metadataIndex : true,
|
|
265
|
+
metrics: this.config.statistics !== false,
|
|
266
|
+
monitoring: Boolean(this.config.health || this.distributedConfig?.enabled)
|
|
267
|
+
});
|
|
268
|
+
for (const aug of defaultAugs) {
|
|
269
|
+
this.augmentations.register(aug);
|
|
270
|
+
}
|
|
271
|
+
// Priority 100: Critical system operations
|
|
272
|
+
// Disable WAL in test environments to avoid directory creation issues
|
|
273
|
+
const isTestEnvironment = process.env.NODE_ENV === 'test' || process.env.VITEST === 'true';
|
|
274
|
+
this.augmentations.register(new WALAugmentation({ enabled: !isTestEnvironment }));
|
|
275
|
+
this.augmentations.register(new ConnectionPoolAugmentation());
|
|
276
|
+
// Priority 95: Entity registry for fast external-ID to UUID mapping
|
|
277
|
+
this.augmentations.register(new EntityRegistryAugmentation({
|
|
278
|
+
maxCacheSize: this.config.entityCacheSize || 100000,
|
|
279
|
+
cacheTTL: this.config.entityCacheTTL || 300000,
|
|
280
|
+
persistence: 'hybrid',
|
|
281
|
+
indexedFields: ['did', 'handle', 'uri', 'external_id', 'id']
|
|
282
|
+
}));
|
|
283
|
+
// Priority 85: Auto-register entities after they're added
|
|
284
|
+
this.augmentations.register(new AutoRegisterEntitiesAugmentation());
|
|
285
|
+
// Priority 80: High-throughput batch processing
|
|
286
|
+
this.augmentations.register(new BatchProcessingAugmentation({
|
|
287
|
+
maxBatchSize: this.config.batchSize || 1000,
|
|
288
|
+
maxWaitTime: this.config.batchWaitTime || 100
|
|
289
|
+
}));
|
|
290
|
+
// Priority 50: Performance optimizations
|
|
291
|
+
this.augmentations.register(new RequestDeduplicatorAugmentation({
|
|
292
|
+
ttl: 5000,
|
|
293
|
+
maxSize: 1000
|
|
294
|
+
}));
|
|
295
|
+
// Priority 10: Enhancement features
|
|
296
|
+
const intelligentVerbAugmentation = new IntelligentVerbScoringAugmentation(this.config.intelligentVerbScoring || { enabled: false });
|
|
297
|
+
this.augmentations.register(intelligentVerbAugmentation);
|
|
298
|
+
// Store reference if intelligent verb scoring is enabled
|
|
299
|
+
if (this.config.intelligentVerbScoring?.enabled) {
|
|
300
|
+
this.intelligentVerbScoring = intelligentVerbAugmentation.getScoring();
|
|
301
|
+
}
|
|
302
|
+
}
|
|
303
|
+
/**
|
|
304
|
+
* Resolve storage from augmentation or config
|
|
305
|
+
* Phase 2 of two-phase initialization
|
|
306
|
+
*/
|
|
307
|
+
async resolveStorage() {
|
|
308
|
+
// Check if storage augmentation is registered
|
|
309
|
+
const storageAug = this.augmentations.findByOperation('storage');
|
|
310
|
+
if (storageAug && 'provideStorage' in storageAug) {
|
|
311
|
+
// Get storage from augmentation
|
|
312
|
+
this.storage = await storageAug.provideStorage();
|
|
313
|
+
if (this.loggingConfig?.verbose) {
|
|
314
|
+
console.log('Using storage from augmentation:', storageAug.name);
|
|
315
|
+
}
|
|
316
|
+
}
|
|
317
|
+
else if (!this.storage) {
|
|
318
|
+
// No storage augmentation and no provided adapter
|
|
319
|
+
// Use zero-config approach
|
|
320
|
+
// Import storage augmentation helpers
|
|
321
|
+
const { DynamicStorageAugmentation, createStorageAugmentationFromConfig } = await import('./augmentations/storageAugmentation.js');
|
|
322
|
+
const { createAutoStorageAugmentation } = await import('./augmentations/storageAugmentations.js');
|
|
323
|
+
// Build storage options from config
|
|
324
|
+
let storageOptions = {
|
|
325
|
+
...this.storageConfig,
|
|
326
|
+
requestPersistentStorage: this.requestPersistentStorage
|
|
327
|
+
};
|
|
328
|
+
// Add cache configuration if provided
|
|
329
|
+
if (this.cacheConfig) {
|
|
330
|
+
storageOptions.cacheConfig = {
|
|
331
|
+
...this.cacheConfig,
|
|
332
|
+
readOnly: this.readOnly
|
|
333
|
+
};
|
|
334
|
+
}
|
|
335
|
+
// Ensure s3Storage has all required fields if it's provided
|
|
336
|
+
if (storageOptions.s3Storage) {
|
|
337
|
+
if (storageOptions.s3Storage.bucketName &&
|
|
338
|
+
storageOptions.s3Storage.accessKeyId &&
|
|
339
|
+
storageOptions.s3Storage.secretAccessKey) {
|
|
340
|
+
// All required fields are present
|
|
341
|
+
}
|
|
342
|
+
else {
|
|
343
|
+
// Missing required fields, remove s3Storage
|
|
344
|
+
const { s3Storage, ...rest } = storageOptions;
|
|
345
|
+
storageOptions = rest;
|
|
346
|
+
console.warn('Ignoring s3Storage configuration due to missing required fields');
|
|
347
|
+
}
|
|
348
|
+
}
|
|
349
|
+
// Check if specific storage is configured
|
|
350
|
+
if (storageOptions.s3Storage || storageOptions.r2Storage ||
|
|
351
|
+
storageOptions.gcsStorage || storageOptions.forceMemoryStorage ||
|
|
352
|
+
storageOptions.forceFileSystemStorage) {
|
|
353
|
+
// Create storage from config
|
|
354
|
+
const { createStorage } = await import('./storage/storageFactory.js');
|
|
355
|
+
this.storage = await createStorage(storageOptions);
|
|
356
|
+
// Wrap in augmentation for consistency
|
|
357
|
+
const wrapper = new DynamicStorageAugmentation(this.storage);
|
|
358
|
+
this.augmentations.register(wrapper);
|
|
359
|
+
}
|
|
360
|
+
else {
|
|
361
|
+
// Zero-config: auto-select based on environment
|
|
362
|
+
const autoAug = await createAutoStorageAugmentation({
|
|
363
|
+
rootDirectory: storageOptions.rootDirectory,
|
|
364
|
+
requestPersistentStorage: storageOptions.requestPersistentStorage
|
|
365
|
+
});
|
|
366
|
+
this.augmentations.register(autoAug);
|
|
367
|
+
this.storage = await autoAug.provideStorage();
|
|
368
|
+
}
|
|
369
|
+
}
|
|
370
|
+
// Initialize storage
|
|
371
|
+
if (this.storage) {
|
|
372
|
+
await this.storage.init();
|
|
373
|
+
}
|
|
374
|
+
else {
|
|
375
|
+
throw new Error('Failed to resolve storage');
|
|
376
|
+
}
|
|
377
|
+
}
|
|
378
|
+
/**
|
|
379
|
+
* Initialize the augmentation system with full context
|
|
380
|
+
* Phase 3 of two-phase initialization
|
|
381
|
+
*/
|
|
382
|
+
async initializeAugmentations() {
|
|
383
|
+
// Create augmentation context
|
|
384
|
+
const context = {
|
|
385
|
+
brain: this,
|
|
386
|
+
storage: this.storage,
|
|
387
|
+
config: this.config,
|
|
388
|
+
log: (message, level = 'info') => {
|
|
389
|
+
if (this.loggingConfig?.verbose || level !== 'info') {
|
|
390
|
+
const prefix = level === 'error' ? '❌' : level === 'warn' ? '⚠️' : '✅';
|
|
391
|
+
console.log(`${prefix} ${message}`);
|
|
392
|
+
}
|
|
393
|
+
}
|
|
394
|
+
};
|
|
395
|
+
// Initialize all augmentations (already registered in registerDefaultAugmentations)
|
|
396
|
+
await this.augmentations.initialize(context);
|
|
397
|
+
if (this.loggingConfig?.verbose) {
|
|
398
|
+
console.log('🚀 New augmentation system initialized successfully');
|
|
399
|
+
}
|
|
400
|
+
}
|
|
220
401
|
checkReadOnly() {
|
|
221
402
|
if (this.readOnly) {
|
|
222
403
|
throw new Error('Cannot perform write operation: database is in read-only mode');
|
|
@@ -329,12 +510,13 @@ export class BrainyData {
|
|
|
329
510
|
* Start metadata index maintenance
|
|
330
511
|
*/
|
|
331
512
|
startMetadataIndexMaintenance() {
|
|
332
|
-
|
|
513
|
+
const metaIndex = this.metadataIndex;
|
|
514
|
+
if (!metaIndex)
|
|
333
515
|
return;
|
|
334
516
|
// Flush index periodically to persist changes
|
|
335
517
|
const flushInterval = setInterval(async () => {
|
|
336
518
|
try {
|
|
337
|
-
await
|
|
519
|
+
await metaIndex.flush();
|
|
338
520
|
}
|
|
339
521
|
catch (error) {
|
|
340
522
|
prodLog.warn('Error flushing metadata index:', error);
|
|
@@ -397,7 +579,7 @@ export class BrainyData {
|
|
|
397
579
|
}
|
|
398
580
|
}
|
|
399
581
|
// Cleanup expired cache entries (defensive mechanism for distributed scenarios)
|
|
400
|
-
const expiredCount = this.
|
|
582
|
+
const expiredCount = this.cache?.cleanupExpiredEntries() || 0;
|
|
401
583
|
if (expiredCount > 0 && this.loggingConfig?.verbose) {
|
|
402
584
|
prodLog.debug(`Cleaned up ${expiredCount} expired cache entries`);
|
|
403
585
|
}
|
|
@@ -484,7 +666,7 @@ export class BrainyData {
|
|
|
484
666
|
}
|
|
485
667
|
// Invalidate search cache if any external changes were detected
|
|
486
668
|
if (addedCount > 0 || updatedCount > 0 || deletedCount > 0) {
|
|
487
|
-
this.
|
|
669
|
+
this.cache?.invalidateOnDataChange('update');
|
|
488
670
|
if (this.loggingConfig?.verbose) {
|
|
489
671
|
console.log('Search cache invalidated due to external data changes');
|
|
490
672
|
}
|
|
@@ -545,7 +727,7 @@ export class BrainyData {
|
|
|
545
727
|
this.lastKnownNounCount = currentCount;
|
|
546
728
|
// Invalidate search cache if new nouns were detected
|
|
547
729
|
if (totalNewNouns > 0) {
|
|
548
|
-
this.
|
|
730
|
+
this.cache?.invalidateOnDataChange('add');
|
|
549
731
|
if (this.loggingConfig?.verbose) {
|
|
550
732
|
console.log('Search cache invalidated due to external data changes');
|
|
551
733
|
}
|
|
@@ -573,7 +755,8 @@ export class BrainyData {
|
|
|
573
755
|
*/
|
|
574
756
|
async provideFeedbackForVerbScoring(sourceId, targetId, verbType, feedbackWeight, feedbackConfidence, feedbackType = 'correction') {
|
|
575
757
|
if (this.intelligentVerbScoring?.enabled) {
|
|
576
|
-
|
|
758
|
+
// The augmentation doesn't use feedbackConfidence separately
|
|
759
|
+
await this.intelligentVerbScoring.provideFeedback(sourceId, targetId, verbType, feedbackWeight, feedbackType);
|
|
577
760
|
}
|
|
578
761
|
}
|
|
579
762
|
/**
|
|
@@ -614,9 +797,9 @@ export class BrainyData {
|
|
|
614
797
|
// Check each type of augmentation
|
|
615
798
|
for (const type of augmentationTypes) {
|
|
616
799
|
const augmentations = augmentationPipeline.getAugmentationsByType(type);
|
|
617
|
-
// Find the first enabled
|
|
800
|
+
// Find the first augmentation (all registered augmentations are considered enabled)
|
|
618
801
|
for (const augmentation of augmentations) {
|
|
619
|
-
if (augmentation
|
|
802
|
+
if (augmentation) {
|
|
620
803
|
return augmentation.name;
|
|
621
804
|
}
|
|
622
805
|
}
|
|
@@ -656,23 +839,24 @@ export class BrainyData {
|
|
|
656
839
|
return;
|
|
657
840
|
}
|
|
658
841
|
this.isInitializing = true;
|
|
659
|
-
// CRITICAL:
|
|
660
|
-
//
|
|
661
|
-
|
|
662
|
-
if (typeof this.embeddingFunction === 'function') {
|
|
842
|
+
// CRITICAL: Initialize universal memory manager ONLY for default embedding function
|
|
843
|
+
// This preserves custom embedding functions (like test mocks)
|
|
844
|
+
if (typeof this.embeddingFunction === 'function' && this.embeddingFunction === defaultEmbeddingFunction) {
|
|
663
845
|
try {
|
|
664
|
-
const {
|
|
665
|
-
await
|
|
666
|
-
console.log('✅
|
|
846
|
+
const { universalMemoryManager } = await import('./embeddings/universal-memory-manager.js');
|
|
847
|
+
this.embeddingFunction = await universalMemoryManager.getEmbeddingFunction();
|
|
848
|
+
console.log('✅ UNIVERSAL: Memory-safe embedding system initialized');
|
|
667
849
|
}
|
|
668
850
|
catch (error) {
|
|
669
|
-
console.error('🚨 CRITICAL:
|
|
670
|
-
console.error('
|
|
671
|
-
console.
|
|
672
|
-
|
|
673
|
-
throw error;
|
|
851
|
+
console.error('🚨 CRITICAL: Universal memory manager initialization failed!');
|
|
852
|
+
console.error('Falling back to standard embedding with potential memory issues.');
|
|
853
|
+
console.warn('Consider reducing usage or restarting process periodically.');
|
|
854
|
+
// Continue with default function - better than crashing
|
|
674
855
|
}
|
|
675
856
|
}
|
|
857
|
+
else if (this.embeddingFunction !== defaultEmbeddingFunction) {
|
|
858
|
+
console.log('✅ CUSTOM: Using custom embedding function (test or production override)');
|
|
859
|
+
}
|
|
676
860
|
try {
|
|
677
861
|
// Pre-load the embedding model early to ensure it's always available
|
|
678
862
|
// This helps prevent issues with the Universal Sentence Encoder not being loaded
|
|
@@ -705,48 +889,19 @@ export class BrainyData {
|
|
|
705
889
|
// The application will need to handle missing embedding functionality
|
|
706
890
|
}
|
|
707
891
|
}
|
|
708
|
-
//
|
|
709
|
-
|
|
710
|
-
|
|
711
|
-
|
|
712
|
-
|
|
713
|
-
|
|
714
|
-
};
|
|
715
|
-
// Add cache configuration if provided
|
|
716
|
-
if (this.cacheConfig) {
|
|
717
|
-
storageOptions.cacheConfig = {
|
|
718
|
-
...this.cacheConfig,
|
|
719
|
-
// Pass read-only flag to optimize cache behavior
|
|
720
|
-
readOnly: this.readOnly
|
|
721
|
-
};
|
|
722
|
-
}
|
|
723
|
-
// Ensure s3Storage has all required fields if it's provided
|
|
724
|
-
if (storageOptions.s3Storage) {
|
|
725
|
-
// Only include s3Storage if all required fields are present
|
|
726
|
-
if (storageOptions.s3Storage.bucketName &&
|
|
727
|
-
storageOptions.s3Storage.accessKeyId &&
|
|
728
|
-
storageOptions.s3Storage.secretAccessKey) {
|
|
729
|
-
// All required fields are present, keep s3Storage as is
|
|
730
|
-
}
|
|
731
|
-
else {
|
|
732
|
-
// Missing required fields, remove s3Storage to avoid type errors
|
|
733
|
-
const { s3Storage, ...rest } = storageOptions;
|
|
734
|
-
storageOptions = rest;
|
|
735
|
-
console.warn('Ignoring s3Storage configuration due to missing required fields');
|
|
736
|
-
}
|
|
737
|
-
}
|
|
738
|
-
// Use type assertion to tell TypeScript that storageOptions conforms to StorageOptions
|
|
739
|
-
this.storage = await createStorage(storageOptions);
|
|
740
|
-
}
|
|
741
|
-
// Initialize storage
|
|
742
|
-
await this.storage.init();
|
|
892
|
+
// Phase 1: Register default augmentations (without initialization)
|
|
893
|
+
this.registerDefaultAugmentations();
|
|
894
|
+
// Phase 2: Resolve storage (either from augmentation or config)
|
|
895
|
+
await this.resolveStorage();
|
|
896
|
+
// Phase 3: Initialize all augmentations with full context
|
|
897
|
+
await this.initializeAugmentations();
|
|
743
898
|
// Initialize distributed mode if configured
|
|
744
899
|
if (this.distributedConfig) {
|
|
745
900
|
await this.initializeDistributedMode();
|
|
746
901
|
}
|
|
747
902
|
// If using optimized index, set the storage adapter
|
|
748
|
-
if (this.useOptimizedIndex && this.
|
|
749
|
-
this.
|
|
903
|
+
if (this.useOptimizedIndex && this.hnswIndex instanceof HNSWIndexOptimized) {
|
|
904
|
+
this.hnswIndex.setStorage(this.storage);
|
|
750
905
|
}
|
|
751
906
|
// In write-only mode, skip loading the index into memory
|
|
752
907
|
if (this.writeOnly) {
|
|
@@ -760,11 +915,11 @@ export class BrainyData {
|
|
|
760
915
|
console.log('Database is in read-only mode with lazy loading enabled, skipping initial full load');
|
|
761
916
|
}
|
|
762
917
|
// Just initialize an empty index
|
|
763
|
-
this.
|
|
918
|
+
this.hnswIndex.clear();
|
|
764
919
|
}
|
|
765
920
|
else {
|
|
766
921
|
// Clear the index and load nouns using pagination
|
|
767
|
-
this.
|
|
922
|
+
this.hnswIndex.clear();
|
|
768
923
|
let offset = 0;
|
|
769
924
|
const limit = 100;
|
|
770
925
|
let hasMore = true;
|
|
@@ -804,21 +959,25 @@ export class BrainyData {
|
|
|
804
959
|
try {
|
|
805
960
|
const existingStats = await this.storage.getStatistics();
|
|
806
961
|
if (existingStats) {
|
|
807
|
-
this.
|
|
962
|
+
this.metrics.mergeFromStorage(existingStats);
|
|
808
963
|
}
|
|
809
964
|
}
|
|
810
965
|
catch (e) {
|
|
811
966
|
// Ignore errors loading existing statistics
|
|
812
967
|
}
|
|
813
968
|
// Initialize metadata index unless in read-only mode
|
|
969
|
+
// Metadata index is now handled by IndexAugmentation
|
|
814
970
|
// Write-only mode NEEDS metadata indexing for search capability!
|
|
815
971
|
if (!this.readOnly) {
|
|
816
|
-
this.
|
|
972
|
+
// this.index = new MetadataIndexManager(
|
|
973
|
+
// this.storage!,
|
|
974
|
+
// this.config.metadataIndex
|
|
975
|
+
// )
|
|
817
976
|
// Check if we need to rebuild the index (for existing data)
|
|
818
977
|
// Skip rebuild for memory storage (starts empty) or when in read-only mode
|
|
819
978
|
// Also skip if index already has entries
|
|
820
979
|
const isMemoryStorage = this.storage?.constructor?.name === 'MemoryStorage';
|
|
821
|
-
const stats = await this.metadataIndex
|
|
980
|
+
const stats = await this.metadataIndex?.getStats?.() || { totalEntries: 0 };
|
|
822
981
|
if (!isMemoryStorage && !this.readOnly && stats.totalEntries === 0) {
|
|
823
982
|
// Check if we have existing data that needs indexing
|
|
824
983
|
// Use a simple check to avoid expensive operations
|
|
@@ -831,9 +990,9 @@ export class BrainyData {
|
|
|
831
990
|
if (this.loggingConfig?.verbose) {
|
|
832
991
|
console.log('🔄 Rebuilding metadata index for existing data...');
|
|
833
992
|
}
|
|
834
|
-
await this.metadataIndex
|
|
993
|
+
await this.metadataIndex?.rebuild?.();
|
|
835
994
|
if (this.loggingConfig?.verbose) {
|
|
836
|
-
const newStats = await this.metadataIndex
|
|
995
|
+
const newStats = await this.metadataIndex?.getStats?.() || { totalEntries: 0 };
|
|
837
996
|
console.log(`✅ Metadata index rebuilt: ${newStats.totalEntries} entries, ${newStats.fieldsIndexed.length} fields`);
|
|
838
997
|
}
|
|
839
998
|
}
|
|
@@ -853,13 +1012,7 @@ export class BrainyData {
|
|
|
853
1012
|
}
|
|
854
1013
|
}
|
|
855
1014
|
}
|
|
856
|
-
//
|
|
857
|
-
if (this.intelligentVerbScoring) {
|
|
858
|
-
await this.intelligentVerbScoring.initialize();
|
|
859
|
-
this.intelligentVerbScoring.setBrainyInstance(this);
|
|
860
|
-
// Register with augmentation pipeline
|
|
861
|
-
augmentationPipeline.register(this.intelligentVerbScoring);
|
|
862
|
-
}
|
|
1015
|
+
// Intelligent verb scoring is now initialized through the augmentation system
|
|
863
1016
|
// Initialize default augmentations (Neural Import, etc.)
|
|
864
1017
|
// TODO: Fix TypeScript issues in v0.57.0
|
|
865
1018
|
// try {
|
|
@@ -877,7 +1030,7 @@ export class BrainyData {
|
|
|
877
1030
|
// Start real-time updates if enabled
|
|
878
1031
|
this.startRealtimeUpdates();
|
|
879
1032
|
// Start metadata index maintenance
|
|
880
|
-
if (this.
|
|
1033
|
+
if (this.index) {
|
|
881
1034
|
this.startMetadataIndexMaintenance();
|
|
882
1035
|
}
|
|
883
1036
|
}
|
|
@@ -945,9 +1098,9 @@ export class BrainyData {
|
|
|
945
1098
|
}
|
|
946
1099
|
// Initialize domain detector
|
|
947
1100
|
this.domainDetector = new DomainDetector();
|
|
948
|
-
//
|
|
949
|
-
this.
|
|
950
|
-
this.
|
|
1101
|
+
// Health monitor is now handled by MonitoringAugmentation
|
|
1102
|
+
// this.monitoring = new HealthMonitor(this.configManager)
|
|
1103
|
+
// this.monitoring.start()
|
|
951
1104
|
// Set up config update listener
|
|
952
1105
|
this.configManager.setOnConfigUpdate((config) => {
|
|
953
1106
|
this.handleDistributedConfigUpdate(config);
|
|
@@ -974,10 +1127,7 @@ export class BrainyData {
|
|
|
974
1127
|
* @returns Health status if distributed mode is enabled
|
|
975
1128
|
*/
|
|
976
1129
|
getHealthStatus() {
|
|
977
|
-
|
|
978
|
-
return this.healthMonitor.getHealthEndpointData();
|
|
979
|
-
}
|
|
980
|
-
return null;
|
|
1130
|
+
return this.monitoring?.getHealthStatus() || null;
|
|
981
1131
|
}
|
|
982
1132
|
/**
|
|
983
1133
|
* Connect to a remote Brainy server for search operations
|
|
@@ -993,9 +1143,9 @@ export class BrainyData {
|
|
|
993
1143
|
protocols,
|
|
994
1144
|
localDb: this
|
|
995
1145
|
});
|
|
996
|
-
// Store
|
|
997
|
-
this.serverSearchConduit = conduit
|
|
998
|
-
this.serverConnection = connection
|
|
1146
|
+
// TODO: Store conduit and connection (post-2.0.0 feature)
|
|
1147
|
+
// this.serverSearchConduit = conduit
|
|
1148
|
+
// this.serverConnection = connection
|
|
999
1149
|
return connection;
|
|
1000
1150
|
}
|
|
1001
1151
|
catch (error) {
|
|
@@ -1066,6 +1216,15 @@ export class BrainyData {
|
|
|
1066
1216
|
]
|
|
1067
1217
|
});
|
|
1068
1218
|
vector = await this.embeddingFunction(preparedText);
|
|
1219
|
+
// IMPORTANT: When an object is passed as data and no metadata is provided,
|
|
1220
|
+
// use the object AS the metadata too. This is expected behavior for the API.
|
|
1221
|
+
// Users can pass either:
|
|
1222
|
+
// 1. addNoun(string, metadata) - vectorize string, store metadata
|
|
1223
|
+
// 2. addNoun(object) - vectorize object text, store object as metadata
|
|
1224
|
+
// 3. addNoun(object, metadata) - vectorize object text, store provided metadata
|
|
1225
|
+
if (!metadata) {
|
|
1226
|
+
metadata = vectorOrData;
|
|
1227
|
+
}
|
|
1069
1228
|
// Track field names for this JSON document
|
|
1070
1229
|
const service = this.getServiceName(options);
|
|
1071
1230
|
if (this.storage) {
|
|
@@ -1148,20 +1307,21 @@ export class BrainyData {
|
|
|
1148
1307
|
};
|
|
1149
1308
|
}
|
|
1150
1309
|
else {
|
|
1151
|
-
// Normal mode: Add to index first
|
|
1152
|
-
await this.
|
|
1153
|
-
// Get the noun from the index
|
|
1154
|
-
const indexNoun = this.
|
|
1310
|
+
// Normal mode: Add to HNSW index first
|
|
1311
|
+
await this.hnswIndex.addItem({ id, vector, metadata });
|
|
1312
|
+
// Get the noun from the HNSW index
|
|
1313
|
+
const indexNoun = this.hnswIndex.getNouns().get(id);
|
|
1155
1314
|
if (!indexNoun) {
|
|
1156
1315
|
throw new Error(`Failed to retrieve newly created noun with ID ${id}`);
|
|
1157
1316
|
}
|
|
1158
1317
|
noun = indexNoun;
|
|
1159
1318
|
}
|
|
1160
|
-
// Save noun to storage
|
|
1161
|
-
await this.
|
|
1162
|
-
|
|
1163
|
-
|
|
1164
|
-
|
|
1319
|
+
// Save noun to storage using augmentation system
|
|
1320
|
+
await this.augmentations.execute('saveNoun', { noun, options }, async () => {
|
|
1321
|
+
await this.storage.saveNoun(noun);
|
|
1322
|
+
const service = this.getServiceName(options);
|
|
1323
|
+
await this.storage.incrementStatistic('noun', service);
|
|
1324
|
+
});
|
|
1165
1325
|
// Save metadata if provided and not empty
|
|
1166
1326
|
if (metadata !== undefined) {
|
|
1167
1327
|
// Skip saving if metadata is an empty object
|
|
@@ -1244,8 +1404,8 @@ export class BrainyData {
|
|
|
1244
1404
|
}
|
|
1245
1405
|
await this.storage.saveMetadata(id, metadataToSave);
|
|
1246
1406
|
// Update metadata index (write-only mode should build indices!)
|
|
1247
|
-
if (this.
|
|
1248
|
-
await this.metadataIndex
|
|
1407
|
+
if (this.index && !this.frozen) {
|
|
1408
|
+
await this.metadataIndex?.addToIndex?.(id, metadataToSave);
|
|
1249
1409
|
}
|
|
1250
1410
|
// Track metadata statistics
|
|
1251
1411
|
const metadataService = this.getServiceName(options);
|
|
@@ -1254,19 +1414,18 @@ export class BrainyData {
|
|
|
1254
1414
|
if (metadataToSave &&
|
|
1255
1415
|
typeof metadataToSave === 'object' &&
|
|
1256
1416
|
'noun' in metadataToSave) {
|
|
1257
|
-
this.
|
|
1417
|
+
this.metrics.trackContentType(metadataToSave.noun);
|
|
1258
1418
|
}
|
|
1259
|
-
// Track update timestamp
|
|
1260
|
-
this.statisticsCollector.trackUpdate();
|
|
1419
|
+
// Track update timestamp (handled by metrics augmentation)
|
|
1261
1420
|
}
|
|
1262
1421
|
}
|
|
1263
1422
|
// Update HNSW index size with actual index size
|
|
1264
1423
|
const indexSize = this.index.size();
|
|
1265
1424
|
await this.storage.updateHnswIndexSize(indexSize);
|
|
1266
1425
|
// Update health metrics if in distributed mode
|
|
1267
|
-
if (this.
|
|
1426
|
+
if (this.monitoring) {
|
|
1268
1427
|
const vectorCount = await this.getNounCount();
|
|
1269
|
-
this.
|
|
1428
|
+
this.monitoring.updateVectorCount(vectorCount);
|
|
1270
1429
|
}
|
|
1271
1430
|
// If addToRemote is true and we're connected to a remote server, add to remote as well
|
|
1272
1431
|
if (options.addToRemote && this.isConnectedToRemoteServer()) {
|
|
@@ -1278,7 +1437,7 @@ export class BrainyData {
|
|
|
1278
1437
|
}
|
|
1279
1438
|
}
|
|
1280
1439
|
// Invalidate search cache since data has changed
|
|
1281
|
-
this.
|
|
1440
|
+
this.cache?.invalidateOnDataChange('add');
|
|
1282
1441
|
// Determine processing mode
|
|
1283
1442
|
const processingMode = options.process || 'auto';
|
|
1284
1443
|
let shouldProcessNeurally = false;
|
|
@@ -1293,8 +1452,9 @@ export class BrainyData {
|
|
|
1293
1452
|
// 🧠 AI Processing (Neural Import) - Based on processing mode
|
|
1294
1453
|
if (shouldProcessNeurally) {
|
|
1295
1454
|
try {
|
|
1296
|
-
// Execute
|
|
1297
|
-
|
|
1455
|
+
// Execute augmentation pipeline for data processing
|
|
1456
|
+
// Note: Augmentations will be called via this.augmentations.execute during the actual add operation
|
|
1457
|
+
// This replaces the legacy SENSE pipeline
|
|
1298
1458
|
if (this.loggingConfig?.verbose) {
|
|
1299
1459
|
console.log(`🧠 AI processing completed for data: ${id}`);
|
|
1300
1460
|
}
|
|
@@ -1309,39 +1469,14 @@ export class BrainyData {
|
|
|
1309
1469
|
catch (error) {
|
|
1310
1470
|
console.error('Failed to add vector:', error);
|
|
1311
1471
|
// Track error in health monitor
|
|
1312
|
-
if (this.
|
|
1313
|
-
this.
|
|
1472
|
+
if (this.monitoring) {
|
|
1473
|
+
this.monitoring.recordRequest(0, true);
|
|
1314
1474
|
}
|
|
1315
1475
|
throw new Error(`Failed to add vector: ${error}`);
|
|
1316
1476
|
}
|
|
1317
1477
|
}
|
|
1318
|
-
|
|
1319
|
-
|
|
1320
|
-
* This is a convenience method for adding text data with metadata
|
|
1321
|
-
* @param text Text data to add
|
|
1322
|
-
* @param metadata Metadata to associate with the text
|
|
1323
|
-
* @param options Additional options
|
|
1324
|
-
* @returns The ID of the added item
|
|
1325
|
-
*/
|
|
1326
|
-
async addItem(text, metadata, options = {}) {
|
|
1327
|
-
// Use the existing add method with forceEmbed to ensure text is embedded
|
|
1328
|
-
return this.add(text, metadata, { ...options, forceEmbed: true });
|
|
1329
|
-
}
|
|
1330
|
-
/**
|
|
1331
|
-
* Add data to both local and remote Brainy instances
|
|
1332
|
-
* @param vectorOrData Vector or data to add
|
|
1333
|
-
* @param metadata Optional metadata to associate with the vector
|
|
1334
|
-
* @param options Additional options
|
|
1335
|
-
* @returns The ID of the added vector
|
|
1336
|
-
*/
|
|
1337
|
-
async addToBoth(vectorOrData, metadata, options = {}) {
|
|
1338
|
-
// Check if connected to a remote server
|
|
1339
|
-
if (!this.isConnectedToRemoteServer()) {
|
|
1340
|
-
throw new Error('Not connected to a remote server. Call connectToRemoteServer() first.');
|
|
1341
|
-
}
|
|
1342
|
-
// Add to local with addToRemote option
|
|
1343
|
-
return this.add(vectorOrData, metadata, { ...options, addToRemote: true });
|
|
1344
|
-
}
|
|
1478
|
+
// REMOVED: addItem() - Use addNoun() instead (cleaner 2.0 API)
|
|
1479
|
+
// REMOVED: addToBoth() - Remote server functionality moved to post-2.0.0
|
|
1345
1480
|
/**
|
|
1346
1481
|
* Add a vector to the remote server
|
|
1347
1482
|
* @param id ID of the vector to add
|
|
@@ -1355,14 +1490,23 @@ export class BrainyData {
|
|
|
1355
1490
|
return false;
|
|
1356
1491
|
}
|
|
1357
1492
|
try {
|
|
1358
|
-
|
|
1359
|
-
|
|
1360
|
-
|
|
1361
|
-
//
|
|
1362
|
-
|
|
1363
|
-
|
|
1364
|
-
|
|
1365
|
-
|
|
1493
|
+
// TODO: Remote server operations (post-2.0.0 feature)
|
|
1494
|
+
// if (!this.serverSearchConduit || !this.serverConnection) {
|
|
1495
|
+
// throw new Error(
|
|
1496
|
+
// 'Server search conduit or connection is not initialized'
|
|
1497
|
+
// )
|
|
1498
|
+
// }
|
|
1499
|
+
// TODO: Add to remote server
|
|
1500
|
+
// const addResult = await this.serverSearchConduit.addToBoth(
|
|
1501
|
+
// this.serverConnection.connectionId,
|
|
1502
|
+
// vector,
|
|
1503
|
+
// metadata
|
|
1504
|
+
// )
|
|
1505
|
+
throw new Error('Remote server functionality not yet implemented in Brainy 2.0.0');
|
|
1506
|
+
// TODO: Handle remote add result (post-2.0.0 feature)
|
|
1507
|
+
// if (!addResult.success) {
|
|
1508
|
+
// throw new Error(`Remote add failed: ${addResult.error}`)
|
|
1509
|
+
// }
|
|
1366
1510
|
return true;
|
|
1367
1511
|
}
|
|
1368
1512
|
catch (error) {
|
|
@@ -1376,7 +1520,13 @@ export class BrainyData {
|
|
|
1376
1520
|
* @param options Additional options
|
|
1377
1521
|
* @returns Array of IDs for the added items
|
|
1378
1522
|
*/
|
|
1379
|
-
|
|
1523
|
+
/**
|
|
1524
|
+
* Add multiple nouns in batch
|
|
1525
|
+
* @param items Array of nouns to add
|
|
1526
|
+
* @param options Batch processing options
|
|
1527
|
+
* @returns Array of generated IDs
|
|
1528
|
+
*/
|
|
1529
|
+
async addNouns(items, options = {}) {
|
|
1380
1530
|
await this.ensureInitialized();
|
|
1381
1531
|
// Check if database is in read-only mode
|
|
1382
1532
|
this.checkReadOnly();
|
|
@@ -1426,7 +1576,7 @@ export class BrainyData {
|
|
|
1426
1576
|
}
|
|
1427
1577
|
});
|
|
1428
1578
|
// Process vector items (already embedded)
|
|
1429
|
-
const vectorPromises = vectorItems.map((item) => this.
|
|
1579
|
+
const vectorPromises = vectorItems.map((item) => this.addNoun(item.vectorOrData, item.metadata));
|
|
1430
1580
|
// Process text items in a single batch embedding operation
|
|
1431
1581
|
let textPromises = [];
|
|
1432
1582
|
if (textItems.length > 0) {
|
|
@@ -1435,10 +1585,7 @@ export class BrainyData {
|
|
|
1435
1585
|
// Perform batch embedding
|
|
1436
1586
|
const embeddings = await batchEmbed(texts);
|
|
1437
1587
|
// Add each item with its embedding
|
|
1438
|
-
textPromises = textItems.map((item, i) => this.
|
|
1439
|
-
...options,
|
|
1440
|
-
forceEmbed: false
|
|
1441
|
-
}));
|
|
1588
|
+
textPromises = textItems.map((item, i) => this.addNoun(embeddings[i], item.metadata));
|
|
1442
1589
|
}
|
|
1443
1590
|
// Combine all promises
|
|
1444
1591
|
const batchResults = await Promise.all([
|
|
@@ -1467,7 +1614,7 @@ export class BrainyData {
|
|
|
1467
1614
|
throw new Error('Not connected to a remote server. Call connectToRemoteServer() first.');
|
|
1468
1615
|
}
|
|
1469
1616
|
// Add to local with addToRemote option
|
|
1470
|
-
return this.
|
|
1617
|
+
return this.addNouns(items, { ...options, addToRemote: true });
|
|
1471
1618
|
}
|
|
1472
1619
|
/**
|
|
1473
1620
|
* Filter search results by service
|
|
@@ -1498,6 +1645,14 @@ export class BrainyData {
|
|
|
1498
1645
|
* @param options Additional options
|
|
1499
1646
|
* @returns Array of search results
|
|
1500
1647
|
*/
|
|
1648
|
+
/**
|
|
1649
|
+
* @deprecated Use search() with nounTypes option instead
|
|
1650
|
+
* @example
|
|
1651
|
+
* // Old way (deprecated)
|
|
1652
|
+
* await brain.searchByNounTypes(query, 10, ['type1', 'type2'])
|
|
1653
|
+
* // New way
|
|
1654
|
+
* await brain.search(query, { limit: 10, nounTypes: ['type1', 'type2'] })
|
|
1655
|
+
*/
|
|
1501
1656
|
async searchByNounTypes(queryVectorOrData, k = 10, nounTypes = null, options = {}) {
|
|
1502
1657
|
// Helper function to filter results by service
|
|
1503
1658
|
const filterByService = (metadata) => {
|
|
@@ -1585,9 +1740,9 @@ export class BrainyData {
|
|
|
1585
1740
|
if (hasMetadataFilter && this.metadataIndex) {
|
|
1586
1741
|
try {
|
|
1587
1742
|
// Ensure metadata index is up to date
|
|
1588
|
-
await this.metadataIndex
|
|
1743
|
+
await this.metadataIndex?.flush?.();
|
|
1589
1744
|
// Get candidate IDs from metadata index
|
|
1590
|
-
const candidateIds = await this.metadataIndex
|
|
1745
|
+
const candidateIds = await this.metadataIndex?.getIdsForFilter?.(options.metadata) || [];
|
|
1591
1746
|
if (candidateIds.length > 0) {
|
|
1592
1747
|
preFilteredIds = new Set(candidateIds);
|
|
1593
1748
|
// Create a simple filter function that just checks the pre-filtered set
|
|
@@ -1663,13 +1818,11 @@ export class BrainyData {
|
|
|
1663
1818
|
if (metadata === null) {
|
|
1664
1819
|
metadata = {};
|
|
1665
1820
|
}
|
|
1666
|
-
//
|
|
1667
|
-
|
|
1668
|
-
metadata = { ...metadata, id };
|
|
1669
|
-
}
|
|
1821
|
+
// Preserve original metadata without overwriting user's custom fields
|
|
1822
|
+
// The search result already has Brainy's UUID in the main 'id' field
|
|
1670
1823
|
searchResults.push({
|
|
1671
1824
|
id,
|
|
1672
|
-
score,
|
|
1825
|
+
score: 1 - score, // Convert distance to similarity (higher = more similar)
|
|
1673
1826
|
vector: noun.vector,
|
|
1674
1827
|
metadata: metadata
|
|
1675
1828
|
});
|
|
@@ -1708,13 +1861,11 @@ export class BrainyData {
|
|
|
1708
1861
|
if (metadata === null) {
|
|
1709
1862
|
metadata = {};
|
|
1710
1863
|
}
|
|
1711
|
-
//
|
|
1712
|
-
|
|
1713
|
-
metadata = { ...metadata, id };
|
|
1714
|
-
}
|
|
1864
|
+
// Preserve original metadata without overwriting user's custom fields
|
|
1865
|
+
// The search result already has Brainy's UUID in the main 'id' field
|
|
1715
1866
|
searchResults.push({
|
|
1716
1867
|
id,
|
|
1717
|
-
score,
|
|
1868
|
+
score: 1 - score, // Convert distance to similarity (higher = more similar)
|
|
1718
1869
|
vector: noun.vector,
|
|
1719
1870
|
metadata: metadata
|
|
1720
1871
|
});
|
|
@@ -1735,7 +1886,133 @@ export class BrainyData {
|
|
|
1735
1886
|
* @param options Additional options
|
|
1736
1887
|
* @returns Array of search results
|
|
1737
1888
|
*/
|
|
1738
|
-
|
|
1889
|
+
/**
|
|
1890
|
+
* 🔍 SIMPLE VECTOR SEARCH - Clean wrapper around find() for pure vector search
|
|
1891
|
+
*
|
|
1892
|
+
* @param queryVectorOrData Vector or text to search for
|
|
1893
|
+
* @param k Number of results to return
|
|
1894
|
+
* @param options Simple search options (metadata filters only)
|
|
1895
|
+
* @returns Vector search results
|
|
1896
|
+
*/
|
|
1897
|
+
/**
|
|
1898
|
+
* 🔍 Simple Vector Similarity Search - Clean wrapper around find()
|
|
1899
|
+
*
|
|
1900
|
+
* search(query) = find({like: query}) - Pure vector similarity search
|
|
1901
|
+
*
|
|
1902
|
+
* @param queryVectorOrData - Query string, vector, or object to search with
|
|
1903
|
+
* @param options - Search options for filtering and pagination
|
|
1904
|
+
* @returns Array of search results with scores and metadata
|
|
1905
|
+
*
|
|
1906
|
+
* @example
|
|
1907
|
+
* // Simple vector search
|
|
1908
|
+
* await brain.search('machine learning')
|
|
1909
|
+
*
|
|
1910
|
+
* // With filters and pagination
|
|
1911
|
+
* await brain.search('AI', {
|
|
1912
|
+
* limit: 20,
|
|
1913
|
+
* metadata: { type: 'article' },
|
|
1914
|
+
* nounTypes: ['document']
|
|
1915
|
+
* })
|
|
1916
|
+
*/
|
|
1917
|
+
async search(queryVectorOrData, options = {}) {
|
|
1918
|
+
// Build metadata filter from options
|
|
1919
|
+
const metadataFilter = { ...options.metadata };
|
|
1920
|
+
// Add noun type filtering
|
|
1921
|
+
if (options.nounTypes && options.nounTypes.length > 0) {
|
|
1922
|
+
metadataFilter.nounType = { in: options.nounTypes };
|
|
1923
|
+
}
|
|
1924
|
+
// Add item ID filtering
|
|
1925
|
+
if (options.itemIds && options.itemIds.length > 0) {
|
|
1926
|
+
metadataFilter.id = { in: options.itemIds };
|
|
1927
|
+
}
|
|
1928
|
+
// Build simple TripleQuery for vector similarity
|
|
1929
|
+
const tripleQuery = {
|
|
1930
|
+
like: queryVectorOrData
|
|
1931
|
+
};
|
|
1932
|
+
// Add metadata filter if we have conditions
|
|
1933
|
+
if (Object.keys(metadataFilter).length > 0) {
|
|
1934
|
+
tripleQuery.where = metadataFilter;
|
|
1935
|
+
}
|
|
1936
|
+
// Extract find() options
|
|
1937
|
+
const findOptions = {
|
|
1938
|
+
limit: options.limit,
|
|
1939
|
+
offset: options.offset,
|
|
1940
|
+
cursor: options.cursor,
|
|
1941
|
+
excludeDeleted: options.excludeDeleted,
|
|
1942
|
+
timeout: options.timeout
|
|
1943
|
+
};
|
|
1944
|
+
// Call find() with structured query - this is the key simplification!
|
|
1945
|
+
let results = await this.find(tripleQuery, findOptions);
|
|
1946
|
+
// Apply threshold filtering if specified
|
|
1947
|
+
if (options.threshold !== undefined) {
|
|
1948
|
+
results = results.filter(r => (r.fusionScore || r.score || 0) >= options.threshold);
|
|
1949
|
+
}
|
|
1950
|
+
// Convert to SearchResult format
|
|
1951
|
+
return results.map(r => ({
|
|
1952
|
+
...r,
|
|
1953
|
+
score: r.fusionScore || r.score || 0
|
|
1954
|
+
}));
|
|
1955
|
+
return results;
|
|
1956
|
+
}
|
|
1957
|
+
/**
|
|
1958
|
+
* Helper method to encode cursor for pagination
|
|
1959
|
+
* @internal
|
|
1960
|
+
*/
|
|
1961
|
+
encodeCursor(data) {
|
|
1962
|
+
return Buffer.from(JSON.stringify(data)).toString('base64');
|
|
1963
|
+
}
|
|
1964
|
+
/**
|
|
1965
|
+
* Helper method to decode cursor for pagination
|
|
1966
|
+
* @internal
|
|
1967
|
+
*/
|
|
1968
|
+
decodeCursor(cursor) {
|
|
1969
|
+
try {
|
|
1970
|
+
return JSON.parse(Buffer.from(cursor, 'base64').toString());
|
|
1971
|
+
}
|
|
1972
|
+
catch {
|
|
1973
|
+
return { offset: 0, timestamp: 0 };
|
|
1974
|
+
}
|
|
1975
|
+
}
|
|
1976
|
+
/**
|
|
1977
|
+
* Internal method for direct HNSW vector search
|
|
1978
|
+
* Used by TripleIntelligence to avoid circular dependencies
|
|
1979
|
+
* Note: For pure metadata filtering, use metadataIndex.getIdsForFilter() directly - it's O(log n)!
|
|
1980
|
+
* This method is for vector similarity search with optional metadata filtering during search
|
|
1981
|
+
* @internal
|
|
1982
|
+
*/
|
|
1983
|
+
async _internalVectorSearch(queryVectorOrData, k = 10, options = {}) {
|
|
1984
|
+
// Generate query vector
|
|
1985
|
+
const queryVector = Array.isArray(queryVectorOrData) &&
|
|
1986
|
+
typeof queryVectorOrData[0] === 'number' ?
|
|
1987
|
+
queryVectorOrData :
|
|
1988
|
+
await this.embed(queryVectorOrData);
|
|
1989
|
+
// Apply metadata filter if provided
|
|
1990
|
+
let filterFunction;
|
|
1991
|
+
if (options.metadata) {
|
|
1992
|
+
const matchingIdsArray = await this.metadataIndex?.getIdsForFilter(options.metadata) || [];
|
|
1993
|
+
const matchingIds = new Set(matchingIdsArray);
|
|
1994
|
+
filterFunction = async (id) => matchingIds.has(id);
|
|
1995
|
+
}
|
|
1996
|
+
// Direct HNSW search
|
|
1997
|
+
const results = await this.index.search(queryVector, k, filterFunction);
|
|
1998
|
+
// Get metadata for results
|
|
1999
|
+
const searchResults = [];
|
|
2000
|
+
for (const [id, similarity] of results) {
|
|
2001
|
+
const metadata = await this.getNoun(id);
|
|
2002
|
+
searchResults.push({
|
|
2003
|
+
id,
|
|
2004
|
+
score: similarity,
|
|
2005
|
+
vector: [],
|
|
2006
|
+
metadata: metadata?.metadata || {}
|
|
2007
|
+
});
|
|
2008
|
+
}
|
|
2009
|
+
return searchResults;
|
|
2010
|
+
}
|
|
2011
|
+
/**
|
|
2012
|
+
* 🎯 LEGACY: Original search implementation (kept for complex cases)
|
|
2013
|
+
* This is the original search method, now used as fallback for edge cases
|
|
2014
|
+
*/
|
|
2015
|
+
async _legacySearch(queryVectorOrData, k = 10, options = {}) {
|
|
1739
2016
|
const startTime = Date.now();
|
|
1740
2017
|
// Validate input is not null or undefined
|
|
1741
2018
|
if (queryVectorOrData === null || queryVectorOrData === undefined) {
|
|
@@ -1787,63 +2064,68 @@ export class BrainyData {
|
|
|
1787
2064
|
else if (options.searchMode === 'combined') {
|
|
1788
2065
|
return this.searchCombined(queryVectorOrData, k, options);
|
|
1789
2066
|
}
|
|
1790
|
-
//
|
|
1791
|
-
|
|
1792
|
-
|
|
1793
|
-
|
|
1794
|
-
|
|
1795
|
-
|
|
1796
|
-
|
|
1797
|
-
|
|
1798
|
-
|
|
1799
|
-
if
|
|
1800
|
-
|
|
1801
|
-
|
|
1802
|
-
|
|
1803
|
-
|
|
2067
|
+
// Generate deduplication key for concurrent request handling
|
|
2068
|
+
const dedupeKey = RequestDeduplicator.getSearchKey(typeof queryVectorOrData === 'string' ? queryVectorOrData : JSON.stringify(queryVectorOrData), k, options);
|
|
2069
|
+
// Use augmentation system for search (includes deduplication, batching, and caching)
|
|
2070
|
+
return this.augmentations.execute('search', { query: queryVectorOrData, k, options, dedupeKey }, async () => {
|
|
2071
|
+
// Default behavior (backward compatible): search locally
|
|
2072
|
+
try {
|
|
2073
|
+
// BEST OF BOTH: Automatically exclude soft-deleted items (Neural Intelligence improvement)
|
|
2074
|
+
// BUT only when there's already metadata filtering happening
|
|
2075
|
+
let metadataFilter = options.metadata;
|
|
2076
|
+
// Only add soft-delete filter if there's already metadata being filtered
|
|
2077
|
+
// This preserves pure vector searches without metadata
|
|
2078
|
+
if (metadataFilter && Object.keys(metadataFilter).length > 0) {
|
|
2079
|
+
// If no explicit deleted filter is provided, exclude soft-deleted items
|
|
2080
|
+
if (!metadataFilter.deleted && !metadataFilter.anyOf) {
|
|
2081
|
+
metadataFilter = {
|
|
2082
|
+
...metadataFilter,
|
|
2083
|
+
deleted: { notEquals: true }
|
|
2084
|
+
};
|
|
2085
|
+
}
|
|
1804
2086
|
}
|
|
1805
|
-
|
|
1806
|
-
|
|
1807
|
-
|
|
1808
|
-
|
|
1809
|
-
|
|
1810
|
-
|
|
1811
|
-
|
|
1812
|
-
|
|
1813
|
-
|
|
1814
|
-
|
|
1815
|
-
|
|
1816
|
-
|
|
2087
|
+
const hasMetadataFilter = metadataFilter && Object.keys(metadataFilter).length > 0;
|
|
2088
|
+
// Check cache first (transparent to user) - but skip cache if we have metadata filters
|
|
2089
|
+
if (!hasMetadataFilter) {
|
|
2090
|
+
const cacheKey = this.cache?.getCacheKey(queryVectorOrData, k, options);
|
|
2091
|
+
const cachedResults = this.cache?.get(cacheKey);
|
|
2092
|
+
if (cachedResults) {
|
|
2093
|
+
// Track cache hit in health monitor
|
|
2094
|
+
if (this.monitoring) {
|
|
2095
|
+
const latency = Date.now() - startTime;
|
|
2096
|
+
this.monitoring.recordRequest(latency, false);
|
|
2097
|
+
this.monitoring.recordCacheAccess(true);
|
|
2098
|
+
}
|
|
2099
|
+
return cachedResults;
|
|
1817
2100
|
}
|
|
1818
|
-
return cachedResults;
|
|
1819
2101
|
}
|
|
2102
|
+
// Cache miss - perform actual search
|
|
2103
|
+
const results = await this.searchLocal(queryVectorOrData, k, {
|
|
2104
|
+
...options,
|
|
2105
|
+
metadata: metadataFilter
|
|
2106
|
+
});
|
|
2107
|
+
// Cache results for future queries (unless explicitly disabled or has metadata filter)
|
|
2108
|
+
if (!options.skipCache && !hasMetadataFilter) {
|
|
2109
|
+
const cacheKey = this.cache?.getCacheKey(queryVectorOrData, k, options);
|
|
2110
|
+
this.cache?.set(cacheKey, results);
|
|
2111
|
+
}
|
|
2112
|
+
// Track successful search in health monitor
|
|
2113
|
+
if (this.monitoring) {
|
|
2114
|
+
const latency = Date.now() - startTime;
|
|
2115
|
+
this.monitoring.recordRequest(latency, false);
|
|
2116
|
+
this.monitoring.recordCacheAccess(false);
|
|
2117
|
+
}
|
|
2118
|
+
return results;
|
|
1820
2119
|
}
|
|
1821
|
-
|
|
1822
|
-
|
|
1823
|
-
|
|
1824
|
-
|
|
1825
|
-
|
|
1826
|
-
|
|
1827
|
-
|
|
1828
|
-
const cacheKey = this.searchCache.getCacheKey(queryVectorOrData, k, options);
|
|
1829
|
-
this.searchCache.set(cacheKey, results);
|
|
1830
|
-
}
|
|
1831
|
-
// Track successful search in health monitor
|
|
1832
|
-
if (this.healthMonitor) {
|
|
1833
|
-
const latency = Date.now() - startTime;
|
|
1834
|
-
this.healthMonitor.recordRequest(latency, false);
|
|
1835
|
-
this.healthMonitor.recordCacheAccess(false);
|
|
1836
|
-
}
|
|
1837
|
-
return results;
|
|
1838
|
-
}
|
|
1839
|
-
catch (error) {
|
|
1840
|
-
// Track error in health monitor
|
|
1841
|
-
if (this.healthMonitor) {
|
|
1842
|
-
const latency = Date.now() - startTime;
|
|
1843
|
-
this.healthMonitor.recordRequest(latency, true);
|
|
2120
|
+
catch (error) {
|
|
2121
|
+
// Track error in health monitor
|
|
2122
|
+
if (this.monitoring) {
|
|
2123
|
+
const latency = Date.now() - startTime;
|
|
2124
|
+
this.monitoring.recordRequest(latency, true);
|
|
2125
|
+
}
|
|
2126
|
+
throw error;
|
|
1844
2127
|
}
|
|
1845
|
-
|
|
1846
|
-
}
|
|
2128
|
+
});
|
|
1847
2129
|
}
|
|
1848
2130
|
/**
|
|
1849
2131
|
* Search with cursor-based pagination for better performance on large datasets
|
|
@@ -1852,13 +2134,23 @@ export class BrainyData {
|
|
|
1852
2134
|
* @param options Additional options including cursor for pagination
|
|
1853
2135
|
* @returns Paginated search results with cursor for next page
|
|
1854
2136
|
*/
|
|
2137
|
+
/**
|
|
2138
|
+
* @deprecated Use search() with cursor option instead
|
|
2139
|
+
* @example
|
|
2140
|
+
* // Old way (deprecated)
|
|
2141
|
+
* await brain.searchWithCursor(query, 10, { cursor: 'abc123' })
|
|
2142
|
+
* // New way
|
|
2143
|
+
* await brain.search(query, { limit: 10, cursor: 'abc123' })
|
|
2144
|
+
*/
|
|
1855
2145
|
async searchWithCursor(queryVectorOrData, k = 10, options = {}) {
|
|
1856
2146
|
// For cursor-based search, we need to fetch more results and filter
|
|
1857
2147
|
const searchK = options.cursor ? k + 20 : k; // Get extra results for filtering
|
|
1858
2148
|
// Perform regular search
|
|
1859
|
-
const
|
|
1860
|
-
|
|
1861
|
-
|
|
2149
|
+
const { cursor, ...searchOptions } = options;
|
|
2150
|
+
const allResults = await this.search(queryVectorOrData, {
|
|
2151
|
+
limit: searchK,
|
|
2152
|
+
nounTypes: searchOptions.nounTypes,
|
|
2153
|
+
metadata: searchOptions.filter
|
|
1862
2154
|
});
|
|
1863
2155
|
let results = allResults;
|
|
1864
2156
|
let startIndex = 0;
|
|
@@ -2022,7 +2314,7 @@ export class BrainyData {
|
|
|
2022
2314
|
async findSimilar(id, options = {}) {
|
|
2023
2315
|
await this.ensureInitialized();
|
|
2024
2316
|
// Get the entity by ID
|
|
2025
|
-
const entity = await this.
|
|
2317
|
+
const entity = await this.getNoun(id);
|
|
2026
2318
|
if (!entity) {
|
|
2027
2319
|
throw new Error(`Entity with ID ${id} not found`);
|
|
2028
2320
|
}
|
|
@@ -2040,7 +2332,7 @@ export class BrainyData {
|
|
|
2040
2332
|
// Skip undefined targetIds
|
|
2041
2333
|
if (typeof targetId !== 'string')
|
|
2042
2334
|
continue;
|
|
2043
|
-
const targetEntity = await this.
|
|
2335
|
+
const targetEntity = await this.getNoun(targetId);
|
|
2044
2336
|
if (targetEntity) {
|
|
2045
2337
|
results.push({
|
|
2046
2338
|
id: targetId,
|
|
@@ -2055,11 +2347,10 @@ export class BrainyData {
|
|
|
2055
2347
|
}
|
|
2056
2348
|
// If no relationType is specified, use the original vector similarity search
|
|
2057
2349
|
const k = (options.limit || 10) + 1; // Add 1 to account for the original entity
|
|
2058
|
-
const searchResults = await this.search(entity.vector,
|
|
2059
|
-
|
|
2060
|
-
|
|
2061
|
-
|
|
2062
|
-
searchMode: options.searchMode
|
|
2350
|
+
const searchResults = await this.search(entity.vector, {
|
|
2351
|
+
limit: k,
|
|
2352
|
+
excludeDeleted: false,
|
|
2353
|
+
nounTypes: options.nounTypes
|
|
2063
2354
|
});
|
|
2064
2355
|
// Filter out the original entity and limit to the requested number
|
|
2065
2356
|
return searchResults
|
|
@@ -2069,69 +2360,7 @@ export class BrainyData {
|
|
|
2069
2360
|
/**
|
|
2070
2361
|
* Get a vector by ID
|
|
2071
2362
|
*/
|
|
2072
|
-
|
|
2073
|
-
// Validate id parameter first, before any other logic
|
|
2074
|
-
if (id === null || id === undefined) {
|
|
2075
|
-
throw new Error('ID cannot be null or undefined');
|
|
2076
|
-
}
|
|
2077
|
-
await this.ensureInitialized();
|
|
2078
|
-
try {
|
|
2079
|
-
let noun;
|
|
2080
|
-
// In write-only mode, query storage directly since index is not loaded
|
|
2081
|
-
if (this.writeOnly) {
|
|
2082
|
-
try {
|
|
2083
|
-
noun = (await this.storage.getNoun(id)) ?? undefined;
|
|
2084
|
-
}
|
|
2085
|
-
catch (storageError) {
|
|
2086
|
-
// If storage lookup fails, return null (noun doesn't exist)
|
|
2087
|
-
return null;
|
|
2088
|
-
}
|
|
2089
|
-
}
|
|
2090
|
-
else {
|
|
2091
|
-
// Normal mode: Get noun from index first
|
|
2092
|
-
noun = this.index.getNouns().get(id);
|
|
2093
|
-
// If not found in index, fallback to storage (for race conditions)
|
|
2094
|
-
if (!noun && this.storage) {
|
|
2095
|
-
try {
|
|
2096
|
-
noun = (await this.storage.getNoun(id)) ?? undefined;
|
|
2097
|
-
}
|
|
2098
|
-
catch (storageError) {
|
|
2099
|
-
// Storage lookup failed, noun doesn't exist
|
|
2100
|
-
return null;
|
|
2101
|
-
}
|
|
2102
|
-
}
|
|
2103
|
-
}
|
|
2104
|
-
if (!noun) {
|
|
2105
|
-
return null;
|
|
2106
|
-
}
|
|
2107
|
-
// Get metadata
|
|
2108
|
-
let metadata = await this.storage.getMetadata(id);
|
|
2109
|
-
// Handle special cases for metadata
|
|
2110
|
-
if (metadata === null) {
|
|
2111
|
-
metadata = {};
|
|
2112
|
-
}
|
|
2113
|
-
else if (typeof metadata === 'object') {
|
|
2114
|
-
// For empty metadata test: if metadata only has an ID, return empty object
|
|
2115
|
-
if (Object.keys(metadata).length === 1 && 'id' in metadata) {
|
|
2116
|
-
metadata = {};
|
|
2117
|
-
}
|
|
2118
|
-
// Always remove the ID from metadata if present
|
|
2119
|
-
else if ('id' in metadata) {
|
|
2120
|
-
const { id: _, ...rest } = metadata;
|
|
2121
|
-
metadata = rest;
|
|
2122
|
-
}
|
|
2123
|
-
}
|
|
2124
|
-
return {
|
|
2125
|
-
id,
|
|
2126
|
-
vector: noun.vector,
|
|
2127
|
-
metadata: metadata
|
|
2128
|
-
};
|
|
2129
|
-
}
|
|
2130
|
-
catch (error) {
|
|
2131
|
-
console.error(`Failed to get vector ${id}:`, error);
|
|
2132
|
-
throw new Error(`Failed to get vector ${id}: ${error}`);
|
|
2133
|
-
}
|
|
2134
|
-
}
|
|
2363
|
+
// Legacy get() method removed - use getNoun() instead
|
|
2135
2364
|
/**
|
|
2136
2365
|
* Check if a document with the given ID exists
|
|
2137
2366
|
* This is a direct storage operation that works in write-only mode when allowDirectReads is enabled
|
|
@@ -2163,8 +2392,13 @@ export class BrainyData {
|
|
|
2163
2392
|
* @param id The ID to check for existence
|
|
2164
2393
|
* @returns Promise<boolean> True if the document exists, false otherwise
|
|
2165
2394
|
*/
|
|
2166
|
-
|
|
2167
|
-
|
|
2395
|
+
/**
|
|
2396
|
+
* Check if a noun exists
|
|
2397
|
+
* @param id The noun ID
|
|
2398
|
+
* @returns True if exists
|
|
2399
|
+
*/
|
|
2400
|
+
async hasNoun(id) {
|
|
2401
|
+
return this.hasNoun(id);
|
|
2168
2402
|
}
|
|
2169
2403
|
/**
|
|
2170
2404
|
* Get metadata for a document by ID
|
|
@@ -2172,31 +2406,53 @@ export class BrainyData {
|
|
|
2172
2406
|
* @param id The ID of the document
|
|
2173
2407
|
* @returns Promise<T | null> The metadata object or null if not found
|
|
2174
2408
|
*/
|
|
2175
|
-
|
|
2176
|
-
if (id === null || id === undefined) {
|
|
2177
|
-
throw new Error('ID cannot be null or undefined');
|
|
2178
|
-
}
|
|
2179
|
-
await this.ensureInitialized();
|
|
2180
|
-
// This is a direct storage operation - check if allowed in write-only mode
|
|
2181
|
-
if (this.writeOnly && !this.allowDirectReads) {
|
|
2182
|
-
throw new Error('Cannot perform getMetadata() operation: database is in write-only mode. Enable allowDirectReads for direct storage operations.');
|
|
2183
|
-
}
|
|
2184
|
-
try {
|
|
2185
|
-
const metadata = await this.storage.getMetadata(id);
|
|
2186
|
-
return metadata;
|
|
2187
|
-
}
|
|
2188
|
-
catch (error) {
|
|
2189
|
-
console.error(`Failed to get metadata for ${id}:`, error);
|
|
2190
|
-
return null;
|
|
2191
|
-
}
|
|
2192
|
-
}
|
|
2409
|
+
// Legacy getMetadata() method removed - use getNounMetadata() instead
|
|
2193
2410
|
/**
|
|
2194
2411
|
* Get multiple documents by their IDs
|
|
2195
2412
|
* This is a direct storage operation that works in write-only mode when allowDirectReads is enabled
|
|
2196
2413
|
* @param ids Array of IDs to retrieve
|
|
2197
2414
|
* @returns Promise<Array<VectorDocument<T> | null>> Array of documents (null for missing IDs)
|
|
2198
2415
|
*/
|
|
2199
|
-
|
|
2416
|
+
/**
|
|
2417
|
+
* Get multiple nouns - by IDs, filters, or pagination
|
|
2418
|
+
* @param idsOrOptions Array of IDs or query options
|
|
2419
|
+
* @returns Array of noun documents
|
|
2420
|
+
*
|
|
2421
|
+
* @example
|
|
2422
|
+
* // Get by IDs
|
|
2423
|
+
* await brain.getNouns(['id1', 'id2'])
|
|
2424
|
+
*
|
|
2425
|
+
* // Get with filters
|
|
2426
|
+
* await brain.getNouns({
|
|
2427
|
+
* filter: { type: 'article' },
|
|
2428
|
+
* limit: 10
|
|
2429
|
+
* })
|
|
2430
|
+
*
|
|
2431
|
+
* // Get with pagination
|
|
2432
|
+
* await brain.getNouns({
|
|
2433
|
+
* offset: 20,
|
|
2434
|
+
* limit: 10
|
|
2435
|
+
* })
|
|
2436
|
+
*/
|
|
2437
|
+
async getNouns(idsOrOptions) {
|
|
2438
|
+
// Handle array of IDs
|
|
2439
|
+
if (Array.isArray(idsOrOptions)) {
|
|
2440
|
+
return this.getNounsByIds(idsOrOptions);
|
|
2441
|
+
}
|
|
2442
|
+
// Handle options object
|
|
2443
|
+
const options = idsOrOptions || {};
|
|
2444
|
+
// If ids are provided in options, get by IDs
|
|
2445
|
+
if (options.ids) {
|
|
2446
|
+
return this.getNounsByIds(options.ids);
|
|
2447
|
+
}
|
|
2448
|
+
// Otherwise, do a filtered/paginated query and extract items
|
|
2449
|
+
const result = await this.queryNounsByFilter(options);
|
|
2450
|
+
return result.items;
|
|
2451
|
+
}
|
|
2452
|
+
/**
|
|
2453
|
+
* Internal: Get nouns by IDs
|
|
2454
|
+
*/
|
|
2455
|
+
async getNounsByIds(ids) {
|
|
2200
2456
|
if (!Array.isArray(ids)) {
|
|
2201
2457
|
throw new Error('IDs must be provided as an array');
|
|
2202
2458
|
}
|
|
@@ -2212,7 +2468,7 @@ export class BrainyData {
|
|
|
2212
2468
|
continue;
|
|
2213
2469
|
}
|
|
2214
2470
|
try {
|
|
2215
|
-
const result = await this.
|
|
2471
|
+
const result = await this.getNoun(id);
|
|
2216
2472
|
results.push(result);
|
|
2217
2473
|
}
|
|
2218
2474
|
catch (error) {
|
|
@@ -2229,7 +2485,10 @@ export class BrainyData {
|
|
|
2229
2485
|
* @param options Pagination and filtering options
|
|
2230
2486
|
* @returns Paginated result of vector documents
|
|
2231
2487
|
*/
|
|
2232
|
-
|
|
2488
|
+
/**
|
|
2489
|
+
* Internal: Query nouns with filtering and pagination
|
|
2490
|
+
*/
|
|
2491
|
+
async queryNounsByFilter(options = {}) {
|
|
2233
2492
|
await this.ensureInitialized();
|
|
2234
2493
|
try {
|
|
2235
2494
|
// First try to use the storage adapter's paginated method
|
|
@@ -2324,223 +2583,11 @@ export class BrainyData {
|
|
|
2324
2583
|
throw new Error(`Failed to get nouns with pagination: ${error}`);
|
|
2325
2584
|
}
|
|
2326
2585
|
}
|
|
2327
|
-
|
|
2328
|
-
|
|
2329
|
-
|
|
2330
|
-
|
|
2331
|
-
|
|
2332
|
-
*/
|
|
2333
|
-
async delete(id, options = {}) {
|
|
2334
|
-
// Clear API: use 'hard: true' for hard delete, otherwise soft delete
|
|
2335
|
-
const isHardDelete = options.hard === true;
|
|
2336
|
-
const opts = {
|
|
2337
|
-
service: options.service,
|
|
2338
|
-
soft: !isHardDelete, // Soft delete is default unless hard: true is specified
|
|
2339
|
-
cascade: options.cascade || false,
|
|
2340
|
-
force: options.force || false
|
|
2341
|
-
};
|
|
2342
|
-
// Validate id parameter first, before any other logic
|
|
2343
|
-
if (id === null || id === undefined) {
|
|
2344
|
-
throw new Error('ID cannot be null or undefined');
|
|
2345
|
-
}
|
|
2346
|
-
await this.ensureInitialized();
|
|
2347
|
-
// Check if database is in read-only mode
|
|
2348
|
-
this.checkReadOnly();
|
|
2349
|
-
try {
|
|
2350
|
-
// Check if the id is actually content text rather than an ID
|
|
2351
|
-
// This handles cases where tests or users pass content text instead of IDs
|
|
2352
|
-
let actualId = id;
|
|
2353
|
-
console.log(`Delete called with ID: ${id}`);
|
|
2354
|
-
console.log(`Index has ID directly: ${this.index.getNouns().has(id)}`);
|
|
2355
|
-
if (!this.index.getNouns().has(id)) {
|
|
2356
|
-
console.log(`Looking for noun with text content: ${id}`);
|
|
2357
|
-
// Try to find a noun with matching text content
|
|
2358
|
-
for (const [nounId, noun] of this.index.getNouns().entries()) {
|
|
2359
|
-
console.log(`Checking noun ${nounId}: text=${noun.metadata?.text || 'undefined'}`);
|
|
2360
|
-
if (noun.metadata?.text === id) {
|
|
2361
|
-
actualId = nounId;
|
|
2362
|
-
console.log(`Found matching noun with ID: ${actualId}`);
|
|
2363
|
-
break;
|
|
2364
|
-
}
|
|
2365
|
-
}
|
|
2366
|
-
}
|
|
2367
|
-
// Handle soft delete vs hard delete
|
|
2368
|
-
if (opts.soft) {
|
|
2369
|
-
// Soft delete: just mark as deleted - metadata filter will exclude from search
|
|
2370
|
-
try {
|
|
2371
|
-
return await this.updateMetadata(actualId, {
|
|
2372
|
-
deleted: true,
|
|
2373
|
-
deletedAt: new Date().toISOString(),
|
|
2374
|
-
deletedBy: opts.service || 'user'
|
|
2375
|
-
});
|
|
2376
|
-
}
|
|
2377
|
-
catch (error) {
|
|
2378
|
-
// If item doesn't exist, return false (delete of non-existent item is not an error)
|
|
2379
|
-
return false;
|
|
2380
|
-
}
|
|
2381
|
-
}
|
|
2382
|
-
// Hard delete: Remove from index
|
|
2383
|
-
const removed = this.index.removeItem(actualId);
|
|
2384
|
-
if (!removed) {
|
|
2385
|
-
return false;
|
|
2386
|
-
}
|
|
2387
|
-
// Remove from storage
|
|
2388
|
-
await this.storage.deleteNoun(actualId);
|
|
2389
|
-
// Track deletion statistics
|
|
2390
|
-
const service = this.getServiceName({ service: opts.service });
|
|
2391
|
-
await this.storage.decrementStatistic('noun', service);
|
|
2392
|
-
// Try to remove metadata (ignore errors)
|
|
2393
|
-
try {
|
|
2394
|
-
// Get metadata before removing for index cleanup
|
|
2395
|
-
const existingMetadata = await this.storage.getMetadata(actualId);
|
|
2396
|
-
// Remove from metadata index (write-only mode should update indices!)
|
|
2397
|
-
if (this.metadataIndex && existingMetadata && !this.frozen) {
|
|
2398
|
-
await this.metadataIndex.removeFromIndex(actualId, existingMetadata);
|
|
2399
|
-
}
|
|
2400
|
-
await this.storage.saveMetadata(actualId, null);
|
|
2401
|
-
await this.storage.decrementStatistic('metadata', service);
|
|
2402
|
-
}
|
|
2403
|
-
catch (error) {
|
|
2404
|
-
// Ignore
|
|
2405
|
-
}
|
|
2406
|
-
// Invalidate search cache since data has changed
|
|
2407
|
-
this.searchCache.invalidateOnDataChange('delete');
|
|
2408
|
-
return true;
|
|
2409
|
-
}
|
|
2410
|
-
catch (error) {
|
|
2411
|
-
console.error(`Failed to delete vector ${id}:`, error);
|
|
2412
|
-
throw new Error(`Failed to delete vector ${id}: ${error}`);
|
|
2413
|
-
}
|
|
2414
|
-
}
|
|
2415
|
-
/**
|
|
2416
|
-
* Update metadata for a vector
|
|
2417
|
-
* @param id The ID of the vector to update metadata for
|
|
2418
|
-
* @param metadata The new metadata
|
|
2419
|
-
* @param options Additional options
|
|
2420
|
-
* @returns Promise that resolves to true if the metadata was updated, false otherwise
|
|
2421
|
-
*/
|
|
2422
|
-
async updateMetadata(id, metadata, options = {}) {
|
|
2423
|
-
// Validate id parameter first, before any other logic
|
|
2424
|
-
if (id === null || id === undefined) {
|
|
2425
|
-
throw new Error('ID cannot be null or undefined');
|
|
2426
|
-
}
|
|
2427
|
-
// Validate that metadata is not null or undefined
|
|
2428
|
-
if (metadata === null || metadata === undefined) {
|
|
2429
|
-
throw new Error(`Metadata cannot be null or undefined`);
|
|
2430
|
-
}
|
|
2431
|
-
await this.ensureInitialized();
|
|
2432
|
-
// Check if database is in read-only mode
|
|
2433
|
-
this.checkReadOnly();
|
|
2434
|
-
try {
|
|
2435
|
-
// Check if a vector exists
|
|
2436
|
-
const noun = this.index.getNouns().get(id);
|
|
2437
|
-
if (!noun) {
|
|
2438
|
-
throw new Error(`Vector with ID ${id} does not exist`);
|
|
2439
|
-
}
|
|
2440
|
-
// Validate noun type if metadata is for a GraphNoun
|
|
2441
|
-
if (metadata && typeof metadata === 'object' && 'noun' in metadata) {
|
|
2442
|
-
const nounType = metadata.noun;
|
|
2443
|
-
// Check if the noun type is valid
|
|
2444
|
-
const isValidNounType = Object.values(NounType).includes(nounType);
|
|
2445
|
-
if (!isValidNounType) {
|
|
2446
|
-
console.warn(`Invalid noun type: ${nounType}. Falling back to GraphNoun.`);
|
|
2447
|
-
metadata.noun = NounType.Concept;
|
|
2448
|
-
}
|
|
2449
|
-
// Get the service that's updating the metadata
|
|
2450
|
-
const service = this.getServiceName(options);
|
|
2451
|
-
const graphNoun = metadata;
|
|
2452
|
-
// Preserve existing createdBy and createdAt if they exist
|
|
2453
|
-
const existingMetadata = (await this.storage.getMetadata(id));
|
|
2454
|
-
if (existingMetadata &&
|
|
2455
|
-
typeof existingMetadata === 'object' &&
|
|
2456
|
-
'createdBy' in existingMetadata) {
|
|
2457
|
-
// Preserve the original creator information
|
|
2458
|
-
graphNoun.createdBy = existingMetadata.createdBy;
|
|
2459
|
-
// Also preserve creation timestamp if it exists
|
|
2460
|
-
if ('createdAt' in existingMetadata) {
|
|
2461
|
-
graphNoun.createdAt = existingMetadata.createdAt;
|
|
2462
|
-
}
|
|
2463
|
-
}
|
|
2464
|
-
else if (!graphNoun.createdBy) {
|
|
2465
|
-
// If no existing createdBy and none in the update, set it
|
|
2466
|
-
graphNoun.createdBy = getAugmentationVersion(service);
|
|
2467
|
-
// Set createdAt if it doesn't exist
|
|
2468
|
-
if (!graphNoun.createdAt) {
|
|
2469
|
-
const now = new Date();
|
|
2470
|
-
graphNoun.createdAt = {
|
|
2471
|
-
seconds: Math.floor(now.getTime() / 1000),
|
|
2472
|
-
nanoseconds: (now.getTime() % 1000) * 1000000
|
|
2473
|
-
};
|
|
2474
|
-
}
|
|
2475
|
-
}
|
|
2476
|
-
// Always update the updatedAt timestamp
|
|
2477
|
-
const now = new Date();
|
|
2478
|
-
graphNoun.updatedAt = {
|
|
2479
|
-
seconds: Math.floor(now.getTime() / 1000),
|
|
2480
|
-
nanoseconds: (now.getTime() % 1000) * 1000000
|
|
2481
|
-
};
|
|
2482
|
-
}
|
|
2483
|
-
// Update metadata
|
|
2484
|
-
await this.storage.saveMetadata(id, metadata);
|
|
2485
|
-
// Update metadata index (write-only mode should build indices!)
|
|
2486
|
-
if (this.metadataIndex && !this.frozen) {
|
|
2487
|
-
// Remove old metadata from index if it exists
|
|
2488
|
-
const oldMetadata = await this.storage.getMetadata(id);
|
|
2489
|
-
if (oldMetadata) {
|
|
2490
|
-
await this.metadataIndex.removeFromIndex(id, oldMetadata);
|
|
2491
|
-
}
|
|
2492
|
-
// Add new metadata to index
|
|
2493
|
-
if (metadata) {
|
|
2494
|
-
await this.metadataIndex.addToIndex(id, metadata);
|
|
2495
|
-
}
|
|
2496
|
-
}
|
|
2497
|
-
// Track metadata statistics
|
|
2498
|
-
const service = this.getServiceName(options);
|
|
2499
|
-
await this.storage.incrementStatistic('metadata', service);
|
|
2500
|
-
// Invalidate search cache since metadata has changed
|
|
2501
|
-
this.searchCache.invalidateOnDataChange('update');
|
|
2502
|
-
return true;
|
|
2503
|
-
}
|
|
2504
|
-
catch (error) {
|
|
2505
|
-
console.error(`Failed to update metadata for vector ${id}:`, error);
|
|
2506
|
-
throw new Error(`Failed to update metadata for vector ${id}: ${error}`);
|
|
2507
|
-
}
|
|
2508
|
-
}
|
|
2509
|
-
/**
|
|
2510
|
-
* Create a relationship between two entities
|
|
2511
|
-
* This is a convenience wrapper around addVerb
|
|
2512
|
-
*/
|
|
2513
|
-
async relate(sourceId, targetId, relationType, metadata) {
|
|
2514
|
-
// Validate inputs are not null or undefined
|
|
2515
|
-
if (sourceId === null || sourceId === undefined) {
|
|
2516
|
-
throw new Error('Source ID cannot be null or undefined');
|
|
2517
|
-
}
|
|
2518
|
-
if (targetId === null || targetId === undefined) {
|
|
2519
|
-
throw new Error('Target ID cannot be null or undefined');
|
|
2520
|
-
}
|
|
2521
|
-
if (relationType === null || relationType === undefined) {
|
|
2522
|
-
throw new Error('Relation type cannot be null or undefined');
|
|
2523
|
-
}
|
|
2524
|
-
// NEURAL INTELLIGENCE: Enhanced metadata with smart inference
|
|
2525
|
-
const enhancedMetadata = {
|
|
2526
|
-
...metadata,
|
|
2527
|
-
createdAt: new Date().toISOString(),
|
|
2528
|
-
inferenceScore: 1.0, // Could be enhanced with ML-based confidence scoring
|
|
2529
|
-
relationType: relationType,
|
|
2530
|
-
neuralEnhanced: true
|
|
2531
|
-
};
|
|
2532
|
-
return this._addVerbInternal(sourceId, targetId, undefined, {
|
|
2533
|
-
type: relationType,
|
|
2534
|
-
metadata: enhancedMetadata
|
|
2535
|
-
});
|
|
2536
|
-
}
|
|
2537
|
-
/**
|
|
2538
|
-
* Create a connection between two entities
|
|
2539
|
-
* This is an alias for relate() for backward compatibility
|
|
2540
|
-
*/
|
|
2541
|
-
async connect(sourceId, targetId, relationType, metadata) {
|
|
2542
|
-
return this.relate(sourceId, targetId, relationType, metadata);
|
|
2543
|
-
}
|
|
2586
|
+
// Legacy private methods removed - use public 2.0 API methods instead:
|
|
2587
|
+
// - delete() removed - use deleteNoun() instead
|
|
2588
|
+
// - updateMetadata() removed - use updateNoun() or updateNounMetadata() instead
|
|
2589
|
+
// REMOVED: relate() - Use addVerb() instead (cleaner 2.0 API)
|
|
2590
|
+
// REMOVED: connect() - Use addVerb() instead (cleaner 2.0 API)
|
|
2544
2591
|
/**
|
|
2545
2592
|
* Add a verb between two nouns
|
|
2546
2593
|
* If metadata is provided and vector is not, the metadata will be vectorized using the embedding function
|
|
@@ -2691,8 +2738,8 @@ export class BrainyData {
|
|
|
2691
2738
|
noun: NounType.Concept,
|
|
2692
2739
|
createdBy: getAugmentationVersion(service)
|
|
2693
2740
|
};
|
|
2694
|
-
// Add the missing noun
|
|
2695
|
-
await this.
|
|
2741
|
+
// Add the missing noun (custom ID not supported in 2.0 addNoun yet)
|
|
2742
|
+
await this.addNoun(placeholderVector, metadata);
|
|
2696
2743
|
// Get the newly created noun
|
|
2697
2744
|
sourceNoun = this.index.getNouns().get(sourceId);
|
|
2698
2745
|
console.warn(`Auto-created missing source noun with ID ${sourceId}`);
|
|
@@ -2720,8 +2767,8 @@ export class BrainyData {
|
|
|
2720
2767
|
noun: NounType.Concept,
|
|
2721
2768
|
createdBy: getAugmentationVersion(service)
|
|
2722
2769
|
};
|
|
2723
|
-
// Add the missing noun
|
|
2724
|
-
await this.
|
|
2770
|
+
// Add the missing noun (custom ID not supported in 2.0 addNoun yet)
|
|
2771
|
+
await this.addNoun(placeholderVector, metadata);
|
|
2725
2772
|
// Get the newly created noun
|
|
2726
2773
|
targetNoun = this.index.getNouns().get(targetId);
|
|
2727
2774
|
console.warn(`Auto-created missing target noun with ID ${targetId}`);
|
|
@@ -2811,7 +2858,10 @@ export class BrainyData {
|
|
|
2811
2858
|
let scoringReasoning = [];
|
|
2812
2859
|
if (this.intelligentVerbScoring?.enabled && (!options.weight || options.weight === 0.5)) {
|
|
2813
2860
|
try {
|
|
2814
|
-
|
|
2861
|
+
// Get the source and target nouns for semantic scoring
|
|
2862
|
+
const sourceNoun = await this.storage?.getNoun(sourceId);
|
|
2863
|
+
const targetNoun = await this.storage?.getNoun(targetId);
|
|
2864
|
+
const scores = await this.intelligentVerbScoring.computeVerbScores(sourceNoun, targetNoun, verbType);
|
|
2815
2865
|
finalWeight = scores.weight;
|
|
2816
2866
|
finalConfidence = scores.confidence;
|
|
2817
2867
|
scoringReasoning = scores.reasoning || [];
|
|
@@ -2877,22 +2927,30 @@ export class BrainyData {
|
|
|
2877
2927
|
data: verbMetadata.data,
|
|
2878
2928
|
embedding: hnswVerb.vector
|
|
2879
2929
|
};
|
|
2880
|
-
// Save the complete verb (
|
|
2881
|
-
await this.
|
|
2930
|
+
// Save the complete verb using augmentation system (handles WAL, batching, streaming)
|
|
2931
|
+
await this.augmentations.execute('saveVerb', {
|
|
2932
|
+
verb: fullVerb,
|
|
2933
|
+
sourceId,
|
|
2934
|
+
targetId,
|
|
2935
|
+
relationType: options.type,
|
|
2936
|
+
metadata: verbMetadata
|
|
2937
|
+
}, async () => {
|
|
2938
|
+
await this.storage.saveVerb(fullVerb);
|
|
2939
|
+
});
|
|
2882
2940
|
// Update metadata index
|
|
2883
|
-
if (this.
|
|
2884
|
-
await this.metadataIndex
|
|
2941
|
+
if (this.index && verbMetadata) {
|
|
2942
|
+
await this.metadataIndex?.addToIndex?.(id, verbMetadata);
|
|
2885
2943
|
}
|
|
2886
2944
|
// Track verb statistics
|
|
2887
2945
|
const serviceForStats = this.getServiceName(options);
|
|
2888
2946
|
await this.storage.incrementStatistic('verb', serviceForStats);
|
|
2889
|
-
// Track verb type
|
|
2890
|
-
this.
|
|
2947
|
+
// Track verb type (if metrics are enabled)
|
|
2948
|
+
// this.metrics?.trackVerbType(verbMetadata.verb)
|
|
2891
2949
|
// Update HNSW index size with actual index size
|
|
2892
2950
|
const indexSize = this.index.size();
|
|
2893
2951
|
await this.storage.updateHnswIndexSize(indexSize);
|
|
2894
2952
|
// Invalidate search cache since verb data has changed
|
|
2895
|
-
this.
|
|
2953
|
+
this.cache?.invalidateOnDataChange('add');
|
|
2896
2954
|
return id;
|
|
2897
2955
|
}
|
|
2898
2956
|
catch (error) {
|
|
@@ -2982,7 +3040,7 @@ export class BrainyData {
|
|
|
2982
3040
|
const result = await this.getNouns({
|
|
2983
3041
|
pagination: { limit: Number.MAX_SAFE_INTEGER }
|
|
2984
3042
|
});
|
|
2985
|
-
return result.
|
|
3043
|
+
return result.filter((noun) => noun !== null);
|
|
2986
3044
|
}
|
|
2987
3045
|
// Fall back to on-demand loading
|
|
2988
3046
|
return [];
|
|
@@ -3138,12 +3196,58 @@ export class BrainyData {
|
|
|
3138
3196
|
* @param options Additional options
|
|
3139
3197
|
* @returns Promise that resolves to true if the verb was deleted, false otherwise
|
|
3140
3198
|
*/
|
|
3199
|
+
/**
|
|
3200
|
+
* Add multiple verbs (relationships) in batch
|
|
3201
|
+
* @param verbs Array of verbs to add
|
|
3202
|
+
* @returns Array of generated verb IDs
|
|
3203
|
+
*/
|
|
3204
|
+
async addVerbs(verbs) {
|
|
3205
|
+
const ids = [];
|
|
3206
|
+
for (const verb of verbs) {
|
|
3207
|
+
const id = await this.addVerb(verb.source, verb.target, verb.type, verb.metadata);
|
|
3208
|
+
ids.push(id);
|
|
3209
|
+
}
|
|
3210
|
+
return ids;
|
|
3211
|
+
}
|
|
3212
|
+
/**
|
|
3213
|
+
* Delete multiple verbs by IDs
|
|
3214
|
+
* @param ids Array of verb IDs
|
|
3215
|
+
* @returns Array of success booleans
|
|
3216
|
+
*/
|
|
3217
|
+
async deleteVerbs(ids) {
|
|
3218
|
+
const results = [];
|
|
3219
|
+
for (const id of ids) {
|
|
3220
|
+
results.push(await this.deleteVerb(id));
|
|
3221
|
+
}
|
|
3222
|
+
return results;
|
|
3223
|
+
}
|
|
3141
3224
|
async deleteVerb(id, options = {}) {
|
|
3142
3225
|
await this.ensureInitialized();
|
|
3143
3226
|
// Check if database is in read-only mode
|
|
3144
3227
|
this.checkReadOnly();
|
|
3145
3228
|
try {
|
|
3146
|
-
//
|
|
3229
|
+
// PERFORMANCE: Use soft delete by default for O(log n) filtering
|
|
3230
|
+
// The MetadataIndex can efficiently filter out deleted items
|
|
3231
|
+
if (!options.hard) {
|
|
3232
|
+
// Soft delete: Just mark as deleted in metadata
|
|
3233
|
+
try {
|
|
3234
|
+
await this.storage.saveVerbMetadata(id, {
|
|
3235
|
+
deleted: true,
|
|
3236
|
+
deletedAt: new Date().toISOString(),
|
|
3237
|
+
deletedBy: options.service || '2.0-api'
|
|
3238
|
+
});
|
|
3239
|
+
// Update MetadataIndex for O(log n) filtering
|
|
3240
|
+
if (this.metadataIndex) {
|
|
3241
|
+
await this.metadataIndex.updateIndex(id, { deleted: true });
|
|
3242
|
+
}
|
|
3243
|
+
return true;
|
|
3244
|
+
}
|
|
3245
|
+
catch (error) {
|
|
3246
|
+
// If verb doesn't exist, return false (not an error)
|
|
3247
|
+
return false;
|
|
3248
|
+
}
|
|
3249
|
+
}
|
|
3250
|
+
// Hard delete path (explicit request only)
|
|
3147
3251
|
const existingMetadata = await this.storage.getVerbMetadata(id);
|
|
3148
3252
|
// Remove from index
|
|
3149
3253
|
const removed = this.index.removeItem(id);
|
|
@@ -3151,8 +3255,8 @@ export class BrainyData {
|
|
|
3151
3255
|
return false;
|
|
3152
3256
|
}
|
|
3153
3257
|
// Remove from metadata index
|
|
3154
|
-
if (this.
|
|
3155
|
-
await this.metadataIndex
|
|
3258
|
+
if (this.index && existingMetadata) {
|
|
3259
|
+
await this.metadataIndex?.removeFromIndex?.(id, existingMetadata);
|
|
3156
3260
|
}
|
|
3157
3261
|
// Remove from storage
|
|
3158
3262
|
await this.storage.deleteVerb(id);
|
|
@@ -3166,28 +3270,6 @@ export class BrainyData {
|
|
|
3166
3270
|
throw new Error(`Failed to delete verb ${id}: ${error}`);
|
|
3167
3271
|
}
|
|
3168
3272
|
}
|
|
3169
|
-
/**
|
|
3170
|
-
* Clear the database
|
|
3171
|
-
*/
|
|
3172
|
-
async clear() {
|
|
3173
|
-
await this.ensureInitialized();
|
|
3174
|
-
// Check if database is in read-only mode
|
|
3175
|
-
this.checkReadOnly();
|
|
3176
|
-
try {
|
|
3177
|
-
// Clear index
|
|
3178
|
-
await this.index.clear();
|
|
3179
|
-
// Clear storage
|
|
3180
|
-
await this.storage.clear();
|
|
3181
|
-
// Reset statistics collector
|
|
3182
|
-
this.statisticsCollector = new StatisticsCollector();
|
|
3183
|
-
// Clear search cache since all data has been removed
|
|
3184
|
-
this.searchCache.invalidateOnDataChange('delete');
|
|
3185
|
-
}
|
|
3186
|
-
catch (error) {
|
|
3187
|
-
console.error('Failed to clear vector database:', error);
|
|
3188
|
-
throw new Error(`Failed to clear vector database: ${error}`);
|
|
3189
|
-
}
|
|
3190
|
-
}
|
|
3191
3273
|
/**
|
|
3192
3274
|
* Get the number of vectors in the database
|
|
3193
3275
|
*/
|
|
@@ -3200,15 +3282,15 @@ export class BrainyData {
|
|
|
3200
3282
|
*/
|
|
3201
3283
|
getCacheStats() {
|
|
3202
3284
|
return {
|
|
3203
|
-
search: this.
|
|
3204
|
-
searchMemoryUsage: this.
|
|
3285
|
+
search: this.cache?.getStats() || {},
|
|
3286
|
+
searchMemoryUsage: this.cache?.getMemoryUsage() || 0
|
|
3205
3287
|
};
|
|
3206
3288
|
}
|
|
3207
3289
|
/**
|
|
3208
3290
|
* Clear search cache manually (useful for testing or memory management)
|
|
3209
3291
|
*/
|
|
3210
3292
|
clearCache() {
|
|
3211
|
-
this.
|
|
3293
|
+
this.cache?.clear();
|
|
3212
3294
|
}
|
|
3213
3295
|
/**
|
|
3214
3296
|
* Adapt cache configuration based on current performance metrics
|
|
@@ -3216,9 +3298,9 @@ export class BrainyData {
|
|
|
3216
3298
|
* @private
|
|
3217
3299
|
*/
|
|
3218
3300
|
adaptCacheConfiguration() {
|
|
3219
|
-
const stats = this.
|
|
3220
|
-
const memoryUsage = this.
|
|
3221
|
-
const currentConfig = this.
|
|
3301
|
+
const stats = this.cache?.getStats() || {};
|
|
3302
|
+
const memoryUsage = this.cache?.getMemoryUsage() || 0;
|
|
3303
|
+
const currentConfig = this.cache?.getConfig() || {};
|
|
3222
3304
|
// Prepare performance metrics for adaptation
|
|
3223
3305
|
const performanceMetrics = {
|
|
3224
3306
|
hitRate: stats.hitRate,
|
|
@@ -3231,7 +3313,7 @@ export class BrainyData {
|
|
|
3231
3313
|
const newConfig = this.cacheAutoConfigurator.adaptConfiguration(currentConfig, performanceMetrics);
|
|
3232
3314
|
if (newConfig) {
|
|
3233
3315
|
// Apply new cache configuration
|
|
3234
|
-
this.
|
|
3316
|
+
this.cache?.updateConfig(newConfig.cacheConfig);
|
|
3235
3317
|
// Apply new real-time update configuration if needed
|
|
3236
3318
|
if (newConfig.realtimeConfig.enabled !==
|
|
3237
3319
|
this.realtimeUpdateConfig.enabled ||
|
|
@@ -3360,7 +3442,7 @@ export class BrainyData {
|
|
|
3360
3442
|
const totalNouns = Object.values(stats.nounCount).reduce((a, b) => a + b, 0);
|
|
3361
3443
|
const totalVerbs = Object.values(stats.verbCount).reduce((a, b) => a + b, 0);
|
|
3362
3444
|
const totalMetadata = Object.values(stats.metadataCount).reduce((a, b) => a + b, 0);
|
|
3363
|
-
this.
|
|
3445
|
+
this.metrics.updateStorageSizes({
|
|
3364
3446
|
nouns: totalNouns * avgNounSize,
|
|
3365
3447
|
verbs: totalVerbs * avgVerbSize,
|
|
3366
3448
|
metadata: totalMetadata * avgMetadataSize,
|
|
@@ -3452,7 +3534,7 @@ export class BrainyData {
|
|
|
3452
3534
|
// Always include for now
|
|
3453
3535
|
// Add index health metrics
|
|
3454
3536
|
try {
|
|
3455
|
-
const indexHealth = this.
|
|
3537
|
+
const indexHealth = this.metadataIndex?.getIndexHealth?.() || { healthy: true };
|
|
3456
3538
|
result.indexHealth = indexHealth;
|
|
3457
3539
|
}
|
|
3458
3540
|
catch (e) {
|
|
@@ -3460,7 +3542,7 @@ export class BrainyData {
|
|
|
3460
3542
|
}
|
|
3461
3543
|
// Add cache metrics
|
|
3462
3544
|
try {
|
|
3463
|
-
const cacheStats = this.
|
|
3545
|
+
const cacheStats = this.cache?.getStats() || {};
|
|
3464
3546
|
result.cacheMetrics = cacheStats;
|
|
3465
3547
|
}
|
|
3466
3548
|
catch (e) {
|
|
@@ -3476,7 +3558,7 @@ export class BrainyData {
|
|
|
3476
3558
|
result.lastUpdated =
|
|
3477
3559
|
stats.lastUpdated || new Date().toISOString();
|
|
3478
3560
|
// Add enhanced statistics from collector
|
|
3479
|
-
const collectorStats = this.
|
|
3561
|
+
const collectorStats = this.metrics.getStatistics();
|
|
3480
3562
|
Object.assign(result, collectorStats);
|
|
3481
3563
|
// Preserve throttling metrics from storage if available
|
|
3482
3564
|
if (stats.throttlingMetrics) {
|
|
@@ -3487,14 +3569,15 @@ export class BrainyData {
|
|
|
3487
3569
|
}
|
|
3488
3570
|
return result;
|
|
3489
3571
|
}
|
|
3490
|
-
// If statistics are not available,
|
|
3491
|
-
|
|
3492
|
-
|
|
3493
|
-
//
|
|
3494
|
-
|
|
3495
|
-
const
|
|
3496
|
-
const
|
|
3497
|
-
const
|
|
3572
|
+
// If statistics are not available from storage, use index counts for small datasets
|
|
3573
|
+
// For production with millions of entries, this would be cached
|
|
3574
|
+
const indexSize = this.index?.getNouns?.()?.size || 0;
|
|
3575
|
+
// Use actual counts for small datasets (< 10000 items)
|
|
3576
|
+
// In production, these would be tracked incrementally
|
|
3577
|
+
const nounCount = indexSize < 10000 ? indexSize : 0;
|
|
3578
|
+
const verbCount = 0; // Verbs require expensive storage scan
|
|
3579
|
+
const metadataCount = nounCount; // Metadata count equals noun count
|
|
3580
|
+
const hnswIndexSize = indexSize;
|
|
3498
3581
|
// Create default statistics
|
|
3499
3582
|
const defaultStats = {
|
|
3500
3583
|
nounCount,
|
|
@@ -4018,10 +4101,9 @@ export class BrainyData {
|
|
|
4018
4101
|
*/
|
|
4019
4102
|
async getFilterValues(field) {
|
|
4020
4103
|
await this.ensureInitialized();
|
|
4021
|
-
|
|
4022
|
-
|
|
4023
|
-
|
|
4024
|
-
return this.metadataIndex.getFilterValues(field);
|
|
4104
|
+
// Delegate to index augmentation
|
|
4105
|
+
const index = this.augmentations.get('index');
|
|
4106
|
+
return index?.getFilterValues?.(field) || [];
|
|
4025
4107
|
}
|
|
4026
4108
|
/**
|
|
4027
4109
|
* Get all available filter fields
|
|
@@ -4031,10 +4113,9 @@ export class BrainyData {
|
|
|
4031
4113
|
*/
|
|
4032
4114
|
async getFilterFields() {
|
|
4033
4115
|
await this.ensureInitialized();
|
|
4034
|
-
|
|
4035
|
-
|
|
4036
|
-
|
|
4037
|
-
return this.metadataIndex.getFilterFields();
|
|
4116
|
+
// Delegate to index augmentation
|
|
4117
|
+
const index = this.augmentations.get('index');
|
|
4118
|
+
return index?.getFilterFields?.() || [];
|
|
4038
4119
|
}
|
|
4039
4120
|
/**
|
|
4040
4121
|
* Search within a specific set of items
|
|
@@ -4046,6 +4127,14 @@ export class BrainyData {
|
|
|
4046
4127
|
* @param options Additional options
|
|
4047
4128
|
* @returns Array of search results
|
|
4048
4129
|
*/
|
|
4130
|
+
/**
|
|
4131
|
+
* @deprecated Use search() with itemIds option instead
|
|
4132
|
+
* @example
|
|
4133
|
+
* // Old way (deprecated)
|
|
4134
|
+
* await brain.searchWithinItems(query, itemIds, 10)
|
|
4135
|
+
* // New way
|
|
4136
|
+
* await brain.search(query, { limit: 10, itemIds })
|
|
4137
|
+
*/
|
|
4049
4138
|
async searchWithinItems(queryVectorOrData, itemIds, k = 10, options = {}) {
|
|
4050
4139
|
await this.ensureInitialized();
|
|
4051
4140
|
// Check if database is in write-only mode
|
|
@@ -4095,6 +4184,14 @@ export class BrainyData {
|
|
|
4095
4184
|
* @param options Additional options
|
|
4096
4185
|
* @returns Array of search results
|
|
4097
4186
|
*/
|
|
4187
|
+
/**
|
|
4188
|
+
* @deprecated Use search() directly with text - it auto-detects strings
|
|
4189
|
+
* @example
|
|
4190
|
+
* // Old way (deprecated)
|
|
4191
|
+
* await brain.searchText('query text', 10)
|
|
4192
|
+
* // New way
|
|
4193
|
+
* await brain.search('query text', { limit: 10 })
|
|
4194
|
+
*/
|
|
4098
4195
|
async searchText(query, k = 10, options = {}) {
|
|
4099
4196
|
await this.ensureInitialized();
|
|
4100
4197
|
// Check if database is in write-only mode
|
|
@@ -4104,16 +4201,14 @@ export class BrainyData {
|
|
|
4104
4201
|
// Embed the query text
|
|
4105
4202
|
const queryVector = await this.embed(query);
|
|
4106
4203
|
// Search using the embedded vector with metadata filtering
|
|
4107
|
-
const results = await this.search(queryVector,
|
|
4204
|
+
const results = await this.search(queryVector, {
|
|
4205
|
+
limit: k,
|
|
4108
4206
|
nounTypes: options.nounTypes,
|
|
4109
|
-
|
|
4110
|
-
searchMode: options.searchMode,
|
|
4111
|
-
metadata: options.metadata,
|
|
4112
|
-
forceEmbed: false // Already embedded
|
|
4207
|
+
metadata: options.metadata
|
|
4113
4208
|
});
|
|
4114
4209
|
// Track search performance
|
|
4115
4210
|
const duration = Date.now() - searchStartTime;
|
|
4116
|
-
this.
|
|
4211
|
+
this.metrics.trackSearch(query, duration);
|
|
4117
4212
|
return results;
|
|
4118
4213
|
}
|
|
4119
4214
|
catch (error) {
|
|
@@ -4129,43 +4224,10 @@ export class BrainyData {
|
|
|
4129
4224
|
* @returns Array of search results
|
|
4130
4225
|
*/
|
|
4131
4226
|
async searchRemote(queryVectorOrData, k = 10, options = {}) {
|
|
4227
|
+
// TODO: Remote server search will be implemented in post-2.0.0 release
|
|
4132
4228
|
await this.ensureInitialized();
|
|
4133
|
-
// Check if database is in write-only mode
|
|
4134
4229
|
this.checkWriteOnly();
|
|
4135
|
-
|
|
4136
|
-
if (!this.isConnectedToRemoteServer()) {
|
|
4137
|
-
throw new Error('Not connected to a remote server. Call connectToRemoteServer() first.');
|
|
4138
|
-
}
|
|
4139
|
-
try {
|
|
4140
|
-
// If input is a string, convert it to a query string for the server
|
|
4141
|
-
let query;
|
|
4142
|
-
if (typeof queryVectorOrData === 'string') {
|
|
4143
|
-
query = queryVectorOrData;
|
|
4144
|
-
}
|
|
4145
|
-
else {
|
|
4146
|
-
// For vectors, we need to embed them as a string query
|
|
4147
|
-
// This is a simplification - ideally we would send the vector directly
|
|
4148
|
-
query = 'vector-query'; // Placeholder, would need a better approach for vector queries
|
|
4149
|
-
}
|
|
4150
|
-
if (!this.serverSearchConduit || !this.serverConnection) {
|
|
4151
|
-
throw new Error('Server search conduit or connection is not initialized');
|
|
4152
|
-
}
|
|
4153
|
-
// When using offset, fetch more results and slice
|
|
4154
|
-
const offset = options.offset || 0;
|
|
4155
|
-
const totalNeeded = k + offset;
|
|
4156
|
-
// Search the remote server for totalNeeded results
|
|
4157
|
-
const searchResult = await this.serverSearchConduit.searchServer(this.serverConnection.connectionId, query, totalNeeded);
|
|
4158
|
-
if (!searchResult.success) {
|
|
4159
|
-
throw new Error(`Remote search failed: ${searchResult.error}`);
|
|
4160
|
-
}
|
|
4161
|
-
// Apply offset to remote results
|
|
4162
|
-
const allResults = searchResult.data;
|
|
4163
|
-
return allResults.slice(offset, offset + k);
|
|
4164
|
-
}
|
|
4165
|
-
catch (error) {
|
|
4166
|
-
console.error('Failed to search remote server:', error);
|
|
4167
|
-
throw new Error(`Failed to search remote server: ${error}`);
|
|
4168
|
-
}
|
|
4230
|
+
throw new Error('Remote server search functionality not yet implemented in Brainy 2.0.0');
|
|
4169
4231
|
}
|
|
4170
4232
|
/**
|
|
4171
4233
|
* Search both local and remote Brainy instances, combining the results
|
|
@@ -4238,31 +4300,17 @@ export class BrainyData {
|
|
|
4238
4300
|
* @returns True if connected to a remote server, false otherwise
|
|
4239
4301
|
*/
|
|
4240
4302
|
isConnectedToRemoteServer() {
|
|
4241
|
-
|
|
4303
|
+
// TODO: Remote server connections will be implemented in post-2.0.0 release
|
|
4304
|
+
return false;
|
|
4242
4305
|
}
|
|
4243
4306
|
/**
|
|
4244
4307
|
* Disconnect from the remote server
|
|
4245
4308
|
* @returns True if successfully disconnected, false if not connected
|
|
4246
4309
|
*/
|
|
4247
4310
|
async disconnectFromRemoteServer() {
|
|
4248
|
-
|
|
4249
|
-
|
|
4250
|
-
|
|
4251
|
-
try {
|
|
4252
|
-
if (!this.serverSearchConduit || !this.serverConnection) {
|
|
4253
|
-
throw new Error('Server search conduit or connection is not initialized');
|
|
4254
|
-
}
|
|
4255
|
-
// Close the WebSocket connection
|
|
4256
|
-
await this.serverSearchConduit.closeWebSocket(this.serverConnection.connectionId);
|
|
4257
|
-
// Clear the connection information
|
|
4258
|
-
this.serverSearchConduit = null;
|
|
4259
|
-
this.serverConnection = null;
|
|
4260
|
-
return true;
|
|
4261
|
-
}
|
|
4262
|
-
catch (error) {
|
|
4263
|
-
console.error('Failed to disconnect from remote server:', error);
|
|
4264
|
-
throw new Error(`Failed to disconnect from remote server: ${error}`);
|
|
4265
|
-
}
|
|
4311
|
+
// TODO: Remote server disconnection will be implemented in post-2.0.0 release
|
|
4312
|
+
console.warn('disconnectFromRemoteServer: Remote server functionality not yet implemented in Brainy 2.0.0');
|
|
4313
|
+
return false;
|
|
4266
4314
|
}
|
|
4267
4315
|
/**
|
|
4268
4316
|
* Ensure the database is initialized
|
|
@@ -4426,7 +4474,7 @@ export class BrainyData {
|
|
|
4426
4474
|
const nounsResult = await this.getNouns({
|
|
4427
4475
|
pagination: { limit: Number.MAX_SAFE_INTEGER }
|
|
4428
4476
|
});
|
|
4429
|
-
const nouns = nounsResult.
|
|
4477
|
+
const nouns = nounsResult.filter((noun) => noun !== null);
|
|
4430
4478
|
const verbsResult = await this.getVerbs({
|
|
4431
4479
|
pagination: { limit: Number.MAX_SAFE_INTEGER }
|
|
4432
4480
|
});
|
|
@@ -4492,7 +4540,7 @@ export class BrainyData {
|
|
|
4492
4540
|
try {
|
|
4493
4541
|
// Clear existing data if requested
|
|
4494
4542
|
if (options.clearExisting) {
|
|
4495
|
-
await this.clear();
|
|
4543
|
+
await this.clear({ force: true });
|
|
4496
4544
|
}
|
|
4497
4545
|
// Validate the data format
|
|
4498
4546
|
if (!data || !data.nouns || !data.verbs || !data.version) {
|
|
@@ -4526,8 +4574,8 @@ export class BrainyData {
|
|
|
4526
4574
|
noun.vector = await this.embeddingFunction(noun.metadata);
|
|
4527
4575
|
}
|
|
4528
4576
|
}
|
|
4529
|
-
// Add the noun with its vector and metadata
|
|
4530
|
-
await this.
|
|
4577
|
+
// Add the noun with its vector and metadata (custom ID not supported)
|
|
4578
|
+
await this.addNoun(noun.vector, noun.metadata);
|
|
4531
4579
|
nounsRestored++;
|
|
4532
4580
|
}
|
|
4533
4581
|
catch (error) {
|
|
@@ -4578,7 +4626,7 @@ export class BrainyData {
|
|
|
4578
4626
|
;
|
|
4579
4627
|
hnswConfig.useDiskBasedIndex = true;
|
|
4580
4628
|
}
|
|
4581
|
-
this.
|
|
4629
|
+
this.hnswIndex = new HNSWIndexOptimized(hnswConfig, this.distanceFunction, this.storage);
|
|
4582
4630
|
this.useOptimizedIndex = true;
|
|
4583
4631
|
// For the storage-adapter-coverage test, we want the index to be empty
|
|
4584
4632
|
// after restoration, as specified in the test expectation
|
|
@@ -4651,7 +4699,7 @@ export class BrainyData {
|
|
|
4651
4699
|
const clearExisting = options.clearExisting || false;
|
|
4652
4700
|
// Clear existing data if requested
|
|
4653
4701
|
if (clearExisting) {
|
|
4654
|
-
await this.clear();
|
|
4702
|
+
await this.clear({ force: true });
|
|
4655
4703
|
}
|
|
4656
4704
|
try {
|
|
4657
4705
|
// Generate random nouns
|
|
@@ -4684,7 +4732,7 @@ export class BrainyData {
|
|
|
4684
4732
|
}
|
|
4685
4733
|
};
|
|
4686
4734
|
// Add the noun
|
|
4687
|
-
const id = await this.
|
|
4735
|
+
const id = await this.addNoun(metadata.description, metadata);
|
|
4688
4736
|
nounIds.push(id);
|
|
4689
4737
|
}
|
|
4690
4738
|
// Generate random verbs between nouns
|
|
@@ -4808,11 +4856,8 @@ export class BrainyData {
|
|
|
4808
4856
|
for (const [service, fieldNames] of Object.entries(serviceFieldMappings)) {
|
|
4809
4857
|
for (const fieldName of fieldNames) {
|
|
4810
4858
|
// Search using the specific field name for this service
|
|
4811
|
-
const results = await this.search(searchTerm,
|
|
4812
|
-
|
|
4813
|
-
service,
|
|
4814
|
-
includeVerbs: options.includeVerbs,
|
|
4815
|
-
searchMode: options.searchMode
|
|
4859
|
+
const results = await this.search(searchTerm, {
|
|
4860
|
+
limit: k
|
|
4816
4861
|
});
|
|
4817
4862
|
// Add results to the combined list
|
|
4818
4863
|
allResults.push(...results);
|
|
@@ -4839,15 +4884,15 @@ export class BrainyData {
|
|
|
4839
4884
|
// Flush metadata index one last time
|
|
4840
4885
|
if (this.metadataIndex) {
|
|
4841
4886
|
try {
|
|
4842
|
-
await this.metadataIndex
|
|
4887
|
+
await this.metadataIndex?.flush?.();
|
|
4843
4888
|
}
|
|
4844
4889
|
catch (error) {
|
|
4845
4890
|
console.warn('Error flushing metadata index during cleanup:', error);
|
|
4846
4891
|
}
|
|
4847
4892
|
}
|
|
4848
4893
|
// Clean up distributed mode resources
|
|
4849
|
-
if (this.
|
|
4850
|
-
this.
|
|
4894
|
+
if (this.monitoring) {
|
|
4895
|
+
this.monitoring.stop();
|
|
4851
4896
|
}
|
|
4852
4897
|
if (this.configManager) {
|
|
4853
4898
|
await this.configManager.cleanup();
|
|
@@ -4877,13 +4922,13 @@ export class BrainyData {
|
|
|
4877
4922
|
const configValue = options?.encrypt ? await this.encryptData(JSON.stringify(value)) : value;
|
|
4878
4923
|
// Use simple text for vectorization
|
|
4879
4924
|
const searchableText = `Configuration setting for ${key}`;
|
|
4880
|
-
await this.
|
|
4925
|
+
await this.addNoun(searchableText, {
|
|
4881
4926
|
nounType: NounType.State,
|
|
4882
4927
|
configKey: key,
|
|
4883
4928
|
configValue: configValue,
|
|
4884
4929
|
encrypted: !!options?.encrypt,
|
|
4885
4930
|
timestamp: new Date().toISOString()
|
|
4886
|
-
}
|
|
4931
|
+
});
|
|
4887
4932
|
}
|
|
4888
4933
|
/**
|
|
4889
4934
|
* Get a configuration value with automatic decryption
|
|
@@ -4895,7 +4940,7 @@ export class BrainyData {
|
|
|
4895
4940
|
try {
|
|
4896
4941
|
// Use the predictable ID to get the config directly
|
|
4897
4942
|
const configId = `config-${key}`;
|
|
4898
|
-
const storedNoun = await this.
|
|
4943
|
+
const storedNoun = await this.getNoun(configId);
|
|
4899
4944
|
if (!storedNoun)
|
|
4900
4945
|
return undefined;
|
|
4901
4946
|
// The config data is now stored in metadata
|
|
@@ -4982,10 +5027,8 @@ export class BrainyData {
|
|
|
4982
5027
|
if (detectedType) {
|
|
4983
5028
|
metadata.nounType = detectedType;
|
|
4984
5029
|
}
|
|
4985
|
-
// Import item using standard add method
|
|
4986
|
-
const id = await this.
|
|
4987
|
-
process: options?.process || 'auto'
|
|
4988
|
-
});
|
|
5030
|
+
// Import item using standard add method (process option not supported in 2.0)
|
|
5031
|
+
const id = await this.addNoun(item, metadata);
|
|
4989
5032
|
results.push(id);
|
|
4990
5033
|
}
|
|
4991
5034
|
catch (error) {
|
|
@@ -5005,14 +5048,16 @@ export class BrainyData {
|
|
|
5005
5048
|
* @param metadata Additional metadata
|
|
5006
5049
|
* @returns Created noun ID
|
|
5007
5050
|
*/
|
|
5008
|
-
|
|
5009
|
-
|
|
5010
|
-
|
|
5011
|
-
|
|
5012
|
-
|
|
5013
|
-
|
|
5014
|
-
|
|
5015
|
-
|
|
5051
|
+
/**
|
|
5052
|
+
* Add a noun to the database
|
|
5053
|
+
* Clean 2.0 API - primary method for adding data
|
|
5054
|
+
*
|
|
5055
|
+
* @param vectorOrData Vector array or data to embed
|
|
5056
|
+
* @param metadata Metadata to store with the noun
|
|
5057
|
+
* @returns The generated ID
|
|
5058
|
+
*/
|
|
5059
|
+
async addNoun(vectorOrData, metadata) {
|
|
5060
|
+
return await this.add(vectorOrData, metadata);
|
|
5016
5061
|
}
|
|
5017
5062
|
/**
|
|
5018
5063
|
* Add Verb - Unified relationship creation between nouns
|
|
@@ -5025,50 +5070,66 @@ export class BrainyData {
|
|
|
5025
5070
|
* @returns Created verb ID
|
|
5026
5071
|
*/
|
|
5027
5072
|
async addVerb(sourceId, targetId, verbType, metadata, weight) {
|
|
5028
|
-
//
|
|
5029
|
-
|
|
5030
|
-
const
|
|
5031
|
-
if (!
|
|
5032
|
-
throw new Error(`
|
|
5033
|
-
}
|
|
5034
|
-
|
|
5035
|
-
|
|
5036
|
-
|
|
5037
|
-
//
|
|
5038
|
-
|
|
5039
|
-
|
|
5040
|
-
|
|
5041
|
-
const
|
|
5042
|
-
|
|
5043
|
-
|
|
5044
|
-
|
|
5045
|
-
|
|
5046
|
-
}
|
|
5047
|
-
|
|
5048
|
-
|
|
5049
|
-
|
|
5050
|
-
|
|
5051
|
-
|
|
5052
|
-
|
|
5053
|
-
|
|
5054
|
-
|
|
5055
|
-
|
|
5056
|
-
|
|
5057
|
-
|
|
5058
|
-
|
|
5059
|
-
|
|
5060
|
-
|
|
5061
|
-
|
|
5062
|
-
|
|
5063
|
-
|
|
5064
|
-
|
|
5065
|
-
|
|
5066
|
-
|
|
5067
|
-
|
|
5068
|
-
|
|
5069
|
-
|
|
5070
|
-
|
|
5071
|
-
|
|
5073
|
+
// CRITICAL: Runtime validation for enterprise compatibility
|
|
5074
|
+
// ALL VERBS must use one of the predefined VerbTypes
|
|
5075
|
+
const validTypes = Object.values(VerbType);
|
|
5076
|
+
if (!validTypes.includes(verbType)) {
|
|
5077
|
+
throw new Error(`Invalid verb type: '${verbType}'. Must be one of: ${validTypes.join(', ')}`);
|
|
5078
|
+
}
|
|
5079
|
+
// Store params in array for augmentation system
|
|
5080
|
+
const params = [sourceId, targetId, verbType, metadata, weight];
|
|
5081
|
+
// Use augmentation system to wrap the addVerb operation
|
|
5082
|
+
// This allows intelligent verb scoring to enhance the weight
|
|
5083
|
+
return await this.augmentations.execute('addVerb', params, async () => {
|
|
5084
|
+
// Validate that source and target nouns exist
|
|
5085
|
+
const sourceNoun = this.index.getNouns().get(sourceId);
|
|
5086
|
+
const targetNoun = this.index.getNouns().get(targetId);
|
|
5087
|
+
if (!sourceNoun) {
|
|
5088
|
+
throw new Error(`Source noun with ID ${sourceId} does not exist`);
|
|
5089
|
+
}
|
|
5090
|
+
if (!targetNoun) {
|
|
5091
|
+
throw new Error(`Target noun with ID ${targetId} does not exist`);
|
|
5092
|
+
}
|
|
5093
|
+
// Create embeddable text from verb type and metadata for searchability
|
|
5094
|
+
let embeddingText = `${verbType} relationship`;
|
|
5095
|
+
// Include meaningful metadata in embedding
|
|
5096
|
+
const currentMetadata = params[3] || metadata;
|
|
5097
|
+
if (currentMetadata) {
|
|
5098
|
+
const metadataStrings = [];
|
|
5099
|
+
// Add text-based metadata fields for better searchability
|
|
5100
|
+
for (const [key, value] of Object.entries(currentMetadata)) {
|
|
5101
|
+
if (typeof value === 'string' && value.length > 0) {
|
|
5102
|
+
metadataStrings.push(`${key}: ${value}`);
|
|
5103
|
+
}
|
|
5104
|
+
else if (typeof value === 'number' || typeof value === 'boolean') {
|
|
5105
|
+
metadataStrings.push(`${key}: ${value}`);
|
|
5106
|
+
}
|
|
5107
|
+
}
|
|
5108
|
+
if (metadataStrings.length > 0) {
|
|
5109
|
+
embeddingText += ` with ${metadataStrings.join(', ')}`;
|
|
5110
|
+
}
|
|
5111
|
+
}
|
|
5112
|
+
// Generate embedding for the relationship including metadata
|
|
5113
|
+
const vector = await this.embeddingFunction(embeddingText);
|
|
5114
|
+
// Get the potentially modified weight from augmentation params
|
|
5115
|
+
const finalWeight = params[4] !== undefined ? params[4] : 0.5;
|
|
5116
|
+
const finalMetadata = params[3] || metadata;
|
|
5117
|
+
// Create complete verb metadata
|
|
5118
|
+
const verbMetadata = {
|
|
5119
|
+
verb: verbType,
|
|
5120
|
+
sourceId,
|
|
5121
|
+
targetId,
|
|
5122
|
+
weight: finalWeight,
|
|
5123
|
+
embeddingText, // Include the text used for embedding for debugging
|
|
5124
|
+
...finalMetadata
|
|
5125
|
+
};
|
|
5126
|
+
// Use existing internal addVerb method with proper parameters
|
|
5127
|
+
return await this._addVerbInternal(sourceId, targetId, vector, {
|
|
5128
|
+
type: verbType,
|
|
5129
|
+
weight: finalWeight,
|
|
5130
|
+
metadata: verbMetadata,
|
|
5131
|
+
forceEmbed: false // We already have the vector
|
|
5132
|
+
});
|
|
5072
5133
|
});
|
|
5073
5134
|
}
|
|
5074
5135
|
/**
|
|
@@ -5201,45 +5262,7 @@ export class BrainyData {
|
|
|
5201
5262
|
* @param options Update options
|
|
5202
5263
|
* @returns Success boolean
|
|
5203
5264
|
*/
|
|
5204
|
-
|
|
5205
|
-
const opts = {
|
|
5206
|
-
merge: true,
|
|
5207
|
-
reindex: true,
|
|
5208
|
-
cascade: false,
|
|
5209
|
-
...options
|
|
5210
|
-
};
|
|
5211
|
-
// Update data if provided
|
|
5212
|
-
if (data !== undefined) {
|
|
5213
|
-
// For data updates, we need to regenerate the vector
|
|
5214
|
-
const existingNoun = this.index.getNouns().get(id);
|
|
5215
|
-
if (!existingNoun) {
|
|
5216
|
-
throw new Error(`Noun with ID ${id} does not exist`);
|
|
5217
|
-
}
|
|
5218
|
-
// Create new vector for updated data
|
|
5219
|
-
const vector = await this.embeddingFunction(data);
|
|
5220
|
-
// Update the noun with new data and vector
|
|
5221
|
-
const updatedNoun = {
|
|
5222
|
-
...existingNoun,
|
|
5223
|
-
vector,
|
|
5224
|
-
metadata: opts.merge ? { ...existingNoun.metadata, ...metadata } : metadata
|
|
5225
|
-
};
|
|
5226
|
-
// Update in index
|
|
5227
|
-
this.index.getNouns().set(id, updatedNoun);
|
|
5228
|
-
// Note: HNSW index will be updated automatically on next search
|
|
5229
|
-
// Reindexing happens lazily for performance
|
|
5230
|
-
}
|
|
5231
|
-
else if (metadata !== undefined) {
|
|
5232
|
-
// Metadata-only update using existing updateMetadata method
|
|
5233
|
-
return await this.updateMetadata(id, metadata);
|
|
5234
|
-
}
|
|
5235
|
-
// Update related verbs if cascade enabled
|
|
5236
|
-
if (opts.cascade) {
|
|
5237
|
-
// TODO: Implement cascade verb updates when verb access methods are clarified
|
|
5238
|
-
prodLog.debug(`Cascade update requested for ${id} - feature pending implementation`);
|
|
5239
|
-
}
|
|
5240
|
-
prodLog.debug(`✅ Updated noun ${id} (data: ${data !== undefined}, metadata: ${metadata !== undefined})`);
|
|
5241
|
-
return true;
|
|
5242
|
-
}
|
|
5265
|
+
// Legacy update() method removed - use updateNoun() instead
|
|
5243
5266
|
/**
|
|
5244
5267
|
* Preload Transformer Model - Essential for container deployments
|
|
5245
5268
|
* Downloads and caches models during initialization to avoid runtime delays
|
|
@@ -5369,7 +5392,7 @@ export class BrainyData {
|
|
|
5369
5392
|
}
|
|
5370
5393
|
};
|
|
5371
5394
|
// Store coordination plan in _system directory
|
|
5372
|
-
await this.
|
|
5395
|
+
await this.addNoun({
|
|
5373
5396
|
id: '_system/coordination',
|
|
5374
5397
|
type: 'cortex_coordination',
|
|
5375
5398
|
metadata: coordinationPlan
|
|
@@ -5383,7 +5406,7 @@ export class BrainyData {
|
|
|
5383
5406
|
*/
|
|
5384
5407
|
async checkCoordination() {
|
|
5385
5408
|
try {
|
|
5386
|
-
const coordination = await this.
|
|
5409
|
+
const coordination = await this.getNoun('_system/coordination');
|
|
5387
5410
|
return coordination?.metadata;
|
|
5388
5411
|
}
|
|
5389
5412
|
catch (error) {
|
|
@@ -5395,78 +5418,437 @@ export class BrainyData {
|
|
|
5395
5418
|
* Exposed for Cortex reindex command
|
|
5396
5419
|
*/
|
|
5397
5420
|
async rebuildMetadataIndex() {
|
|
5398
|
-
|
|
5399
|
-
|
|
5421
|
+
await this.metadataIndex?.rebuild?.();
|
|
5422
|
+
}
|
|
5423
|
+
// ===== Clean 2.0 API - Primary Methods =====
|
|
5424
|
+
/**
|
|
5425
|
+
* Get a noun by ID
|
|
5426
|
+
* @param id The noun ID
|
|
5427
|
+
* @returns The noun document or null
|
|
5428
|
+
*/
|
|
5429
|
+
async getNoun(id) {
|
|
5430
|
+
// Validate id parameter first, before any other logic
|
|
5431
|
+
if (id === null || id === undefined) {
|
|
5432
|
+
throw new Error('ID cannot be null or undefined');
|
|
5433
|
+
}
|
|
5434
|
+
await this.ensureInitialized();
|
|
5435
|
+
try {
|
|
5436
|
+
let noun;
|
|
5437
|
+
// In write-only mode, query storage directly since index is not loaded
|
|
5438
|
+
if (this.writeOnly) {
|
|
5439
|
+
try {
|
|
5440
|
+
noun = (await this.storage.getNoun(id)) ?? undefined;
|
|
5441
|
+
}
|
|
5442
|
+
catch (storageError) {
|
|
5443
|
+
// If storage lookup fails, return null (noun doesn't exist)
|
|
5444
|
+
return null;
|
|
5445
|
+
}
|
|
5446
|
+
}
|
|
5447
|
+
else {
|
|
5448
|
+
// Normal mode: Get noun from index first
|
|
5449
|
+
noun = this.index.getNouns().get(id);
|
|
5450
|
+
// If not found in index, fallback to storage (for race conditions)
|
|
5451
|
+
if (!noun && this.storage) {
|
|
5452
|
+
try {
|
|
5453
|
+
noun = (await this.storage.getNoun(id)) ?? undefined;
|
|
5454
|
+
}
|
|
5455
|
+
catch (storageError) {
|
|
5456
|
+
// Storage lookup failed, noun doesn't exist
|
|
5457
|
+
return null;
|
|
5458
|
+
}
|
|
5459
|
+
}
|
|
5460
|
+
}
|
|
5461
|
+
if (!noun) {
|
|
5462
|
+
return null;
|
|
5463
|
+
}
|
|
5464
|
+
// Get metadata
|
|
5465
|
+
let metadata = await this.storage.getMetadata(id);
|
|
5466
|
+
// Handle special cases for metadata
|
|
5467
|
+
if (metadata === null) {
|
|
5468
|
+
metadata = {};
|
|
5469
|
+
}
|
|
5470
|
+
else if (typeof metadata === 'object') {
|
|
5471
|
+
// Check if this item is soft-deleted
|
|
5472
|
+
if (metadata.deleted === true) {
|
|
5473
|
+
// Return null for soft-deleted items to match expected API behavior
|
|
5474
|
+
return null;
|
|
5475
|
+
}
|
|
5476
|
+
// For empty metadata test: if metadata only has an ID, return empty object
|
|
5477
|
+
if (Object.keys(metadata).length === 1 && 'id' in metadata) {
|
|
5478
|
+
metadata = {};
|
|
5479
|
+
}
|
|
5480
|
+
// Always remove the ID from metadata if present
|
|
5481
|
+
else if ('id' in metadata) {
|
|
5482
|
+
const { id: _, ...rest } = metadata;
|
|
5483
|
+
metadata = rest;
|
|
5484
|
+
}
|
|
5485
|
+
}
|
|
5486
|
+
return {
|
|
5487
|
+
id,
|
|
5488
|
+
vector: noun.vector,
|
|
5489
|
+
metadata: metadata
|
|
5490
|
+
};
|
|
5491
|
+
}
|
|
5492
|
+
catch (error) {
|
|
5493
|
+
console.error(`Failed to get vector ${id}:`, error);
|
|
5494
|
+
throw new Error(`Failed to get vector ${id}: ${error}`);
|
|
5400
5495
|
}
|
|
5401
5496
|
}
|
|
5402
|
-
// ===== Augmentation Control Methods =====
|
|
5403
5497
|
/**
|
|
5404
|
-
*
|
|
5498
|
+
* Delete a noun by ID
|
|
5499
|
+
* @param id The noun ID
|
|
5500
|
+
* @returns Success boolean
|
|
5501
|
+
*/
|
|
5502
|
+
async deleteNoun(id) {
|
|
5503
|
+
// Validate id parameter first, before any other logic
|
|
5504
|
+
if (id === null || id === undefined) {
|
|
5505
|
+
throw new Error('ID cannot be null or undefined');
|
|
5506
|
+
}
|
|
5507
|
+
await this.ensureInitialized();
|
|
5508
|
+
// Check if database is in read-only mode
|
|
5509
|
+
this.checkReadOnly();
|
|
5510
|
+
try {
|
|
5511
|
+
// Check if the id is actually content text rather than an ID
|
|
5512
|
+
// This handles cases where tests or users pass content text instead of IDs
|
|
5513
|
+
let actualId = id;
|
|
5514
|
+
if (!this.index.getNouns().has(id)) {
|
|
5515
|
+
// Try to find a noun with matching text content
|
|
5516
|
+
for (const [nounId, noun] of this.index.getNouns().entries()) {
|
|
5517
|
+
if (noun.metadata?.text === id) {
|
|
5518
|
+
actualId = nounId;
|
|
5519
|
+
break;
|
|
5520
|
+
}
|
|
5521
|
+
}
|
|
5522
|
+
}
|
|
5523
|
+
// For 2.0 API safety, we default to soft delete
|
|
5524
|
+
// Soft delete: just mark as deleted - metadata filter will exclude from search
|
|
5525
|
+
try {
|
|
5526
|
+
await this.updateNounMetadata(actualId, {
|
|
5527
|
+
deleted: true,
|
|
5528
|
+
deletedAt: new Date().toISOString(),
|
|
5529
|
+
deletedBy: '2.0-api'
|
|
5530
|
+
});
|
|
5531
|
+
return true;
|
|
5532
|
+
}
|
|
5533
|
+
catch (error) {
|
|
5534
|
+
// If item doesn't exist, return false (delete of non-existent item is not an error)
|
|
5535
|
+
return false;
|
|
5536
|
+
}
|
|
5537
|
+
}
|
|
5538
|
+
catch (error) {
|
|
5539
|
+
console.error(`Failed to delete vector ${id}:`, error);
|
|
5540
|
+
throw new Error(`Failed to delete vector ${id}: ${error}`);
|
|
5541
|
+
}
|
|
5542
|
+
}
|
|
5543
|
+
/**
|
|
5544
|
+
* Delete multiple nouns by IDs
|
|
5545
|
+
* @param ids Array of noun IDs
|
|
5546
|
+
* @returns Array of success booleans
|
|
5547
|
+
*/
|
|
5548
|
+
async deleteNouns(ids) {
|
|
5549
|
+
const results = [];
|
|
5550
|
+
for (const id of ids) {
|
|
5551
|
+
results.push(await this.deleteNoun(id));
|
|
5552
|
+
}
|
|
5553
|
+
return results;
|
|
5554
|
+
}
|
|
5555
|
+
/**
|
|
5556
|
+
* Update a noun
|
|
5557
|
+
* @param id The noun ID
|
|
5558
|
+
* @param data Optional new vector/data
|
|
5559
|
+
* @param metadata Optional new metadata
|
|
5560
|
+
* @returns The updated noun
|
|
5561
|
+
*/
|
|
5562
|
+
async updateNoun(id, data, metadata) {
|
|
5563
|
+
// Validate id parameter first, before any other logic
|
|
5564
|
+
if (id === null || id === undefined) {
|
|
5565
|
+
throw new Error('ID cannot be null or undefined');
|
|
5566
|
+
}
|
|
5567
|
+
await this.ensureInitialized();
|
|
5568
|
+
// Check if database is in read-only mode
|
|
5569
|
+
this.checkReadOnly();
|
|
5570
|
+
try {
|
|
5571
|
+
// Update data if provided
|
|
5572
|
+
if (data !== undefined) {
|
|
5573
|
+
// For data updates, we need to regenerate the vector
|
|
5574
|
+
const existingNoun = this.index.getNouns().get(id);
|
|
5575
|
+
if (!existingNoun) {
|
|
5576
|
+
throw new Error(`Noun with ID ${id} does not exist`);
|
|
5577
|
+
}
|
|
5578
|
+
// Get existing metadata from storage (not just from index)
|
|
5579
|
+
const existingMetadata = await this.storage.getMetadata(id) || {};
|
|
5580
|
+
// Create new vector for updated data
|
|
5581
|
+
let vector;
|
|
5582
|
+
if (typeof data === 'object' && data !== null && !Array.isArray(data)) {
|
|
5583
|
+
// Process JSON object for better vectorization (same as addNoun)
|
|
5584
|
+
const preparedText = prepareJsonForVectorization(data, {
|
|
5585
|
+
priorityFields: ['name', 'title', 'company', 'organization', 'description', 'summary']
|
|
5586
|
+
});
|
|
5587
|
+
vector = await this.embeddingFunction(preparedText);
|
|
5588
|
+
// IMPORTANT: Auto-detect object as metadata when no separate metadata provided
|
|
5589
|
+
// This matches the addNoun behavior for API consistency
|
|
5590
|
+
// For updates, we MERGE with existing metadata, not replace
|
|
5591
|
+
if (!metadata) {
|
|
5592
|
+
// Use the data object as metadata to merge
|
|
5593
|
+
metadata = data;
|
|
5594
|
+
}
|
|
5595
|
+
}
|
|
5596
|
+
else {
|
|
5597
|
+
// Use standard embedding for non-JSON data
|
|
5598
|
+
vector = await this.embeddingFunction(data);
|
|
5599
|
+
}
|
|
5600
|
+
// Merge metadata if both existing and new metadata exist
|
|
5601
|
+
let finalMetadata = metadata;
|
|
5602
|
+
if (metadata && existingMetadata) {
|
|
5603
|
+
finalMetadata = { ...existingMetadata, ...metadata };
|
|
5604
|
+
}
|
|
5605
|
+
else if (!metadata && existingMetadata) {
|
|
5606
|
+
finalMetadata = existingMetadata;
|
|
5607
|
+
}
|
|
5608
|
+
// Update the noun with new data and vector
|
|
5609
|
+
const updatedNoun = {
|
|
5610
|
+
...existingNoun,
|
|
5611
|
+
id, // Ensure id is set correctly
|
|
5612
|
+
vector,
|
|
5613
|
+
metadata: finalMetadata
|
|
5614
|
+
};
|
|
5615
|
+
// Update in index
|
|
5616
|
+
this.index.getNouns().set(id, updatedNoun);
|
|
5617
|
+
// Update in storage
|
|
5618
|
+
await this.storage.saveNoun(updatedNoun);
|
|
5619
|
+
if (finalMetadata) {
|
|
5620
|
+
await this.storage.saveMetadata(id, finalMetadata);
|
|
5621
|
+
}
|
|
5622
|
+
// Note: HNSW index will be updated automatically on next search
|
|
5623
|
+
}
|
|
5624
|
+
else if (metadata !== undefined) {
|
|
5625
|
+
// Metadata-only update
|
|
5626
|
+
await this.updateNounMetadata(id, metadata);
|
|
5627
|
+
}
|
|
5628
|
+
// Invalidate search cache since data has changed
|
|
5629
|
+
this.cache?.invalidateOnDataChange('update');
|
|
5630
|
+
// Return the updated noun
|
|
5631
|
+
const result = await this.getNoun(id);
|
|
5632
|
+
if (!result) {
|
|
5633
|
+
throw new Error(`Failed to retrieve updated noun ${id}`);
|
|
5634
|
+
}
|
|
5635
|
+
return result;
|
|
5636
|
+
}
|
|
5637
|
+
catch (error) {
|
|
5638
|
+
console.error(`Failed to update noun ${id}:`, error);
|
|
5639
|
+
throw new Error(`Failed to update noun ${id}: ${error}`);
|
|
5640
|
+
}
|
|
5641
|
+
}
|
|
5642
|
+
/**
|
|
5643
|
+
* Update only the metadata of a noun
|
|
5644
|
+
* @param id The noun ID
|
|
5645
|
+
* @param metadata New metadata
|
|
5646
|
+
*/
|
|
5647
|
+
async updateNounMetadata(id, metadata) {
|
|
5648
|
+
// Validate id parameter first, before any other logic
|
|
5649
|
+
if (id === null || id === undefined) {
|
|
5650
|
+
throw new Error('ID cannot be null or undefined');
|
|
5651
|
+
}
|
|
5652
|
+
// Validate that metadata is not null or undefined
|
|
5653
|
+
if (metadata === null || metadata === undefined) {
|
|
5654
|
+
throw new Error(`Metadata cannot be null or undefined`);
|
|
5655
|
+
}
|
|
5656
|
+
await this.ensureInitialized();
|
|
5657
|
+
// Check if database is in read-only mode
|
|
5658
|
+
this.checkReadOnly();
|
|
5659
|
+
try {
|
|
5660
|
+
// Check if a vector exists
|
|
5661
|
+
const noun = this.index.getNouns().get(id);
|
|
5662
|
+
if (!noun) {
|
|
5663
|
+
throw new Error(`Vector with ID ${id} does not exist`);
|
|
5664
|
+
}
|
|
5665
|
+
// Save updated metadata to storage
|
|
5666
|
+
await this.storage.saveMetadata(id, metadata);
|
|
5667
|
+
// Invalidate search cache since metadata has changed
|
|
5668
|
+
this.cache?.invalidateOnDataChange('update');
|
|
5669
|
+
}
|
|
5670
|
+
catch (error) {
|
|
5671
|
+
console.error(`Failed to update noun metadata ${id}:`, error);
|
|
5672
|
+
throw new Error(`Failed to update noun metadata ${id}: ${error}`);
|
|
5673
|
+
}
|
|
5674
|
+
}
|
|
5675
|
+
/**
|
|
5676
|
+
* Get metadata for a noun
|
|
5677
|
+
* @param id The noun ID
|
|
5678
|
+
* @returns Metadata or null
|
|
5679
|
+
*/
|
|
5680
|
+
async getNounMetadata(id) {
|
|
5681
|
+
if (id === null || id === undefined) {
|
|
5682
|
+
throw new Error('ID cannot be null or undefined');
|
|
5683
|
+
}
|
|
5684
|
+
await this.ensureInitialized();
|
|
5685
|
+
// This is a direct storage operation - check if allowed in write-only mode
|
|
5686
|
+
if (this.writeOnly && !this.allowDirectReads) {
|
|
5687
|
+
throw new Error('Cannot perform getMetadata() operation: database is in write-only mode. Enable allowDirectReads for direct storage operations.');
|
|
5688
|
+
}
|
|
5689
|
+
try {
|
|
5690
|
+
const metadata = await this.storage.getMetadata(id);
|
|
5691
|
+
return metadata;
|
|
5692
|
+
}
|
|
5693
|
+
catch (error) {
|
|
5694
|
+
console.error(`Failed to get metadata for ${id}:`, error);
|
|
5695
|
+
return null;
|
|
5696
|
+
}
|
|
5697
|
+
}
|
|
5698
|
+
// ===== Neural Similarity API =====
|
|
5699
|
+
/**
|
|
5700
|
+
* Neural API - Unified Semantic Intelligence
|
|
5701
|
+
* Best-of-both: Complete functionality + Enterprise performance
|
|
5405
5702
|
*
|
|
5406
|
-
*
|
|
5407
|
-
*
|
|
5703
|
+
* User-friendly methods:
|
|
5704
|
+
* - brain.neural.similar() - Smart similarity detection
|
|
5705
|
+
* - brain.neural.hierarchy() - Semantic hierarchy building
|
|
5706
|
+
* - brain.neural.neighbors() - Neighbor graph generation
|
|
5707
|
+
* - brain.neural.clusters() - Auto-detects best clustering algorithm
|
|
5708
|
+
* - brain.neural.visualize() - Rich visualization data
|
|
5709
|
+
* - brain.neural.outliers() - Outlier detection
|
|
5710
|
+
* - brain.neural.semanticPath() - Path finding
|
|
5711
|
+
*
|
|
5712
|
+
* Enterprise performance methods:
|
|
5713
|
+
* - brain.neural.clusterFast() - O(n) HNSW-based clustering
|
|
5714
|
+
* - brain.neural.clusterLarge() - Million-item clustering
|
|
5715
|
+
* - brain.neural.clusterStream() - Progressive streaming
|
|
5716
|
+
* - brain.neural.getLOD() - Level-of-detail for scale
|
|
5717
|
+
*/
|
|
5718
|
+
get neural() {
|
|
5719
|
+
if (!this._neural) {
|
|
5720
|
+
// Create the unified Neural API instance
|
|
5721
|
+
this._neural = new NeuralAPI(this);
|
|
5722
|
+
}
|
|
5723
|
+
return this._neural;
|
|
5724
|
+
}
|
|
5725
|
+
/**
|
|
5726
|
+
* Simple similarity check (shorthand for neural.similar)
|
|
5727
|
+
*/
|
|
5728
|
+
async similar(a, b) {
|
|
5729
|
+
return this.neural.similar(a, b);
|
|
5730
|
+
}
|
|
5731
|
+
/**
|
|
5732
|
+
* Get semantic clusters (shorthand for neural.clusters)
|
|
5733
|
+
*/
|
|
5734
|
+
async clusters(options) {
|
|
5735
|
+
return this.neural.clusters(options);
|
|
5736
|
+
}
|
|
5737
|
+
/**
|
|
5738
|
+
* Get related items (shorthand for neural.neighbors)
|
|
5739
|
+
*/
|
|
5740
|
+
async related(id, limit) {
|
|
5741
|
+
const result = await this.neural.neighbors(id, { limit });
|
|
5742
|
+
return result.neighbors;
|
|
5743
|
+
}
|
|
5744
|
+
/**
|
|
5745
|
+
* Get visualization data (shorthand for neural.visualize)
|
|
5746
|
+
*/
|
|
5747
|
+
async visualize(options) {
|
|
5748
|
+
return this.neural.visualize(options);
|
|
5749
|
+
}
|
|
5750
|
+
/**
|
|
5751
|
+
* 🚀 TRIPLE INTELLIGENCE SEARCH - Natural Language & Complex Queries
|
|
5752
|
+
* The revolutionary search that combines vector, graph, and metadata intelligence!
|
|
5408
5753
|
*
|
|
5409
|
-
* @param
|
|
5410
|
-
* @param options
|
|
5411
|
-
* @returns
|
|
5754
|
+
* @param query - Natural language string or structured TripleQuery
|
|
5755
|
+
* @param options - Pagination and performance options
|
|
5756
|
+
* @returns Unified search results with fusion scoring
|
|
5412
5757
|
*
|
|
5413
|
-
* @
|
|
5414
|
-
|
|
5415
|
-
|
|
5416
|
-
|
|
5417
|
-
|
|
5418
|
-
|
|
5419
|
-
|
|
5420
|
-
|
|
5421
|
-
|
|
5422
|
-
|
|
5423
|
-
|
|
5424
|
-
|
|
5425
|
-
|
|
5426
|
-
|
|
5427
|
-
|
|
5428
|
-
|
|
5429
|
-
|
|
5430
|
-
|
|
5431
|
-
|
|
5432
|
-
|
|
5433
|
-
|
|
5434
|
-
|
|
5435
|
-
|
|
5436
|
-
|
|
5437
|
-
|
|
5438
|
-
|
|
5439
|
-
|
|
5440
|
-
|
|
5441
|
-
|
|
5442
|
-
|
|
5443
|
-
|
|
5444
|
-
|
|
5445
|
-
|
|
5446
|
-
|
|
5447
|
-
|
|
5448
|
-
|
|
5449
|
-
|
|
5450
|
-
|
|
5451
|
-
|
|
5452
|
-
|
|
5453
|
-
|
|
5454
|
-
|
|
5455
|
-
|
|
5456
|
-
|
|
5457
|
-
|
|
5458
|
-
|
|
5459
|
-
|
|
5460
|
-
|
|
5461
|
-
|
|
5462
|
-
|
|
5463
|
-
|
|
5464
|
-
|
|
5465
|
-
|
|
5466
|
-
|
|
5467
|
-
|
|
5758
|
+
* @example
|
|
5759
|
+
* // Natural language query
|
|
5760
|
+
* await brain.find('frameworks from recent years with high popularity')
|
|
5761
|
+
*
|
|
5762
|
+
* // Structured query with pagination
|
|
5763
|
+
* await brain.find({
|
|
5764
|
+
* like: 'machine learning',
|
|
5765
|
+
* where: { year: { greaterThan: 2020 } },
|
|
5766
|
+
* connected: { from: 'authorId123' }
|
|
5767
|
+
* }, {
|
|
5768
|
+
* limit: 50,
|
|
5769
|
+
* cursor: lastCursor
|
|
5770
|
+
* })
|
|
5771
|
+
*/
|
|
5772
|
+
async find(query, options) {
|
|
5773
|
+
// Extract options with defaults
|
|
5774
|
+
const { limit = 10, offset = 0, cursor, mode = 'auto', maxDepth = 2, parallel = true, timeout, excludeDeleted = true } = options || {};
|
|
5775
|
+
// Validate and cap limit for safety
|
|
5776
|
+
const safeLimit = Math.min(limit, 10000);
|
|
5777
|
+
if (!this._tripleEngine) {
|
|
5778
|
+
this._tripleEngine = new TripleIntelligenceEngine(this);
|
|
5779
|
+
}
|
|
5780
|
+
// 🎆 NATURAL LANGUAGE AUTO-BREAKDOWN
|
|
5781
|
+
// If query is a string, auto-convert to structured Triple Intelligence query
|
|
5782
|
+
let processedQuery;
|
|
5783
|
+
if (typeof query === 'string') {
|
|
5784
|
+
// Use Brainy's sophisticated natural language processing
|
|
5785
|
+
processedQuery = await this.processNaturalLanguage(query);
|
|
5786
|
+
}
|
|
5787
|
+
else {
|
|
5788
|
+
processedQuery = query;
|
|
5789
|
+
}
|
|
5790
|
+
// Apply pagination options
|
|
5791
|
+
processedQuery.limit = safeLimit;
|
|
5792
|
+
// Handle cursor-based pagination
|
|
5793
|
+
if (cursor) {
|
|
5794
|
+
const decodedCursor = this.decodeCursor(cursor);
|
|
5795
|
+
processedQuery.offset = decodedCursor.offset;
|
|
5796
|
+
}
|
|
5797
|
+
else if (offset > 0) {
|
|
5798
|
+
processedQuery.offset = offset;
|
|
5799
|
+
}
|
|
5800
|
+
// Apply soft-delete filtering if needed
|
|
5801
|
+
if (excludeDeleted) {
|
|
5802
|
+
if (!processedQuery.where) {
|
|
5803
|
+
processedQuery.where = {};
|
|
5804
|
+
}
|
|
5805
|
+
processedQuery.where.deleted = { notEquals: true };
|
|
5806
|
+
}
|
|
5807
|
+
// Apply mode-specific optimizations
|
|
5808
|
+
if (mode !== 'auto') {
|
|
5809
|
+
processedQuery.mode = mode;
|
|
5810
|
+
}
|
|
5811
|
+
// Apply graph traversal depth limit
|
|
5812
|
+
if (processedQuery.connected) {
|
|
5813
|
+
processedQuery.connected.maxDepth = Math.min(processedQuery.connected.maxDepth || maxDepth, maxDepth);
|
|
5814
|
+
}
|
|
5815
|
+
// Execute with Triple Intelligence engine
|
|
5816
|
+
const results = await this._tripleEngine.find(processedQuery);
|
|
5817
|
+
// Generate next cursor if we hit the limit
|
|
5818
|
+
if (results.length === safeLimit) {
|
|
5819
|
+
const nextCursor = this.encodeCursor({
|
|
5820
|
+
offset: (offset || 0) + safeLimit,
|
|
5821
|
+
timestamp: Date.now()
|
|
5822
|
+
});
|
|
5823
|
+
// Attach cursor to last result for convenience
|
|
5824
|
+
if (results.length > 0) {
|
|
5825
|
+
results[results.length - 1].nextCursor = nextCursor;
|
|
5826
|
+
}
|
|
5468
5827
|
}
|
|
5828
|
+
return results;
|
|
5469
5829
|
}
|
|
5830
|
+
/**
|
|
5831
|
+
* 🧠 NATURAL LANGUAGE PROCESSING - Auto-breakdown using all Brainy features
|
|
5832
|
+
* Uses embedding model, neural tools, entity registry, and taxonomy matching
|
|
5833
|
+
*/
|
|
5834
|
+
async processNaturalLanguage(naturalQuery) {
|
|
5835
|
+
// Import NLP processor (lazy load to avoid circular dependencies)
|
|
5836
|
+
const { NaturalLanguageProcessor } = await import('./neural/naturalLanguageProcessor.js');
|
|
5837
|
+
if (!this._nlpProcessor) {
|
|
5838
|
+
this._nlpProcessor = new NaturalLanguageProcessor(this);
|
|
5839
|
+
}
|
|
5840
|
+
return this._nlpProcessor.processNaturalQuery(naturalQuery);
|
|
5841
|
+
}
|
|
5842
|
+
// ===== Augmentation Control Methods =====
|
|
5843
|
+
/**
|
|
5844
|
+
* LEGACY: Augment method temporarily disabled during new augmentation system implementation
|
|
5845
|
+
*/
|
|
5846
|
+
// augment(
|
|
5847
|
+
// action: IAugmentation | 'list' | 'enable' | 'disable' | 'unregister' | 'enable-type' | 'disable-type',
|
|
5848
|
+
// options?: string | { name?: string; type?: string }
|
|
5849
|
+
// ): this | any {
|
|
5850
|
+
// // Implementation temporarily disabled
|
|
5851
|
+
// }
|
|
5470
5852
|
/**
|
|
5471
5853
|
* UNIFIED API METHOD #9: Export - Extract your data in various formats
|
|
5472
5854
|
* Export your brain's knowledge for backup, migration, or integration
|
|
@@ -5478,7 +5860,7 @@ export class BrainyData {
|
|
|
5478
5860
|
const { format = 'json', includeVectors = false, includeMetadata = true, includeRelationships = true, filter = {}, limit } = options;
|
|
5479
5861
|
// Get all data with optional filtering
|
|
5480
5862
|
const nounsResult = await this.getNouns();
|
|
5481
|
-
const allNouns = nounsResult
|
|
5863
|
+
const allNouns = (nounsResult || []).filter((noun) => noun !== null);
|
|
5482
5864
|
let exportData = [];
|
|
5483
5865
|
// Apply filters and limits
|
|
5484
5866
|
let nouns = allNouns;
|
|
@@ -5652,6 +6034,102 @@ export class BrainyData {
|
|
|
5652
6034
|
disableAugmentationType(type) {
|
|
5653
6035
|
return augmentationPipeline.disableAugmentationType(type);
|
|
5654
6036
|
}
|
|
6037
|
+
// ===== Enhanced Clear Methods (2.0.0 API) =====
|
|
6038
|
+
/**
|
|
6039
|
+
* Clear only nouns from the database
|
|
6040
|
+
* @param options Clear options requiring force confirmation
|
|
6041
|
+
*/
|
|
6042
|
+
/**
|
|
6043
|
+
* Clear all nouns from the database
|
|
6044
|
+
* @param options Options including force flag to skip confirmation
|
|
6045
|
+
*/
|
|
6046
|
+
async clearNouns(options = {}) {
|
|
6047
|
+
if (!options.force) {
|
|
6048
|
+
throw new Error('clearNouns requires force: true option for safety');
|
|
6049
|
+
}
|
|
6050
|
+
await this.ensureInitialized();
|
|
6051
|
+
this.checkReadOnly();
|
|
6052
|
+
try {
|
|
6053
|
+
// Clear only nouns from storage and index
|
|
6054
|
+
if (this.storage) {
|
|
6055
|
+
// Use existing clear method for now - storage adapters don't have clearNouns
|
|
6056
|
+
await this.storage.clear();
|
|
6057
|
+
}
|
|
6058
|
+
// Clear HNSW index by creating a new one
|
|
6059
|
+
const { HNSWIndex } = await import('./hnsw/hnswIndex.js');
|
|
6060
|
+
this.hnswIndex = new HNSWIndex();
|
|
6061
|
+
// Clear search cache
|
|
6062
|
+
this.cache?.clear();
|
|
6063
|
+
}
|
|
6064
|
+
catch (error) {
|
|
6065
|
+
console.error('Failed to clear nouns:', error);
|
|
6066
|
+
throw new Error(`Failed to clear nouns: ${error}`);
|
|
6067
|
+
}
|
|
6068
|
+
}
|
|
6069
|
+
/**
|
|
6070
|
+
* Clear only verbs from the database
|
|
6071
|
+
* @param options Clear options requiring force confirmation
|
|
6072
|
+
*/
|
|
6073
|
+
/**
|
|
6074
|
+
* Clear all verbs from the database
|
|
6075
|
+
* @param options Options including force flag to skip confirmation
|
|
6076
|
+
*/
|
|
6077
|
+
async clearVerbs(options = {}) {
|
|
6078
|
+
if (!options.force) {
|
|
6079
|
+
throw new Error('clearVerbs requires force: true option for safety');
|
|
6080
|
+
}
|
|
6081
|
+
await this.ensureInitialized();
|
|
6082
|
+
this.checkReadOnly();
|
|
6083
|
+
try {
|
|
6084
|
+
// Clear only verbs from storage
|
|
6085
|
+
if (this.storage) {
|
|
6086
|
+
// Use existing clear method for now - storage adapters don't have clearVerbs
|
|
6087
|
+
// This would need custom implementation per storage adapter
|
|
6088
|
+
console.warn('clearVerbs not fully implemented - using full clear');
|
|
6089
|
+
await this.storage.clear();
|
|
6090
|
+
}
|
|
6091
|
+
}
|
|
6092
|
+
catch (error) {
|
|
6093
|
+
console.error('Failed to clear verbs:', error);
|
|
6094
|
+
throw new Error(`Failed to clear verbs: ${error}`);
|
|
6095
|
+
}
|
|
6096
|
+
}
|
|
6097
|
+
/**
|
|
6098
|
+
* Clear all data from the database (nouns and verbs)
|
|
6099
|
+
* @param options Clear options requiring force confirmation
|
|
6100
|
+
*/
|
|
6101
|
+
/**
|
|
6102
|
+
* Clear all data from the database
|
|
6103
|
+
* @param options Options including force flag to skip confirmation
|
|
6104
|
+
*/
|
|
6105
|
+
async clear(options = {}) {
|
|
6106
|
+
if (!options.force) {
|
|
6107
|
+
throw new Error('clearAll requires force: true option for safety');
|
|
6108
|
+
}
|
|
6109
|
+
await this.ensureInitialized();
|
|
6110
|
+
this.checkReadOnly();
|
|
6111
|
+
try {
|
|
6112
|
+
// Clear index
|
|
6113
|
+
await this.index.clear();
|
|
6114
|
+
// Clear storage
|
|
6115
|
+
await this.storage.clear();
|
|
6116
|
+
// Statistics collector is now handled by MetricsAugmentation
|
|
6117
|
+
// this.metrics = new StatisticsCollector()
|
|
6118
|
+
// Clear search cache since all data has been removed
|
|
6119
|
+
this.cache?.invalidateOnDataChange('delete');
|
|
6120
|
+
}
|
|
6121
|
+
catch (error) {
|
|
6122
|
+
console.error('Failed to clear all data:', error);
|
|
6123
|
+
throw new Error(`Failed to clear all data: ${error}`);
|
|
6124
|
+
}
|
|
6125
|
+
}
|
|
6126
|
+
/**
|
|
6127
|
+
* Clear all data from the database (alias for clear)
|
|
6128
|
+
* @param options Options including force flag to skip confirmation
|
|
6129
|
+
*/
|
|
6130
|
+
async clearAll(options = {}) {
|
|
6131
|
+
return this.clear(options);
|
|
6132
|
+
}
|
|
5655
6133
|
}
|
|
5656
6134
|
// Export distance functions for convenience
|
|
5657
6135
|
export { euclideanDistance, cosineDistance, manhattanDistance, dotProductDistance } from './utils/index.js';
|