@soulcraft/brainy 3.0.0 → 3.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +53 -3
- package/README.md +353 -110
- package/bin/brainy.js +340 -62
- package/dist/api/ConfigAPI.d.ts +67 -0
- package/dist/api/ConfigAPI.js +166 -0
- package/dist/api/DataAPI.d.ts +123 -0
- package/dist/api/DataAPI.js +391 -0
- package/dist/api/SecurityAPI.d.ts +50 -0
- package/dist/api/SecurityAPI.js +139 -0
- package/dist/api/UniversalImportAPI.d.ts +134 -0
- package/dist/api/UniversalImportAPI.js +615 -0
- package/dist/augmentationManager.js +12 -7
- package/dist/augmentationPipeline.d.ts +0 -61
- package/dist/augmentationPipeline.js +0 -87
- package/dist/augmentationRegistry.d.ts +1 -1
- package/dist/augmentationRegistry.js +1 -1
- package/dist/augmentations/apiServerAugmentation.d.ts +27 -1
- package/dist/augmentations/apiServerAugmentation.js +290 -9
- package/dist/augmentations/auditLogAugmentation.d.ts +109 -0
- package/dist/augmentations/auditLogAugmentation.js +358 -0
- package/dist/augmentations/batchProcessingAugmentation.d.ts +3 -2
- package/dist/augmentations/batchProcessingAugmentation.js +123 -22
- package/dist/augmentations/brainyAugmentation.d.ts +142 -8
- package/dist/augmentations/brainyAugmentation.js +179 -2
- package/dist/augmentations/cacheAugmentation.d.ts +8 -5
- package/dist/augmentations/cacheAugmentation.js +116 -17
- package/dist/augmentations/conduitAugmentations.d.ts +2 -2
- package/dist/augmentations/conduitAugmentations.js +2 -2
- package/dist/augmentations/configResolver.d.ts +122 -0
- package/dist/augmentations/configResolver.js +440 -0
- package/dist/augmentations/connectionPoolAugmentation.d.ts +3 -1
- package/dist/augmentations/connectionPoolAugmentation.js +37 -12
- package/dist/augmentations/defaultAugmentations.d.ts +14 -10
- package/dist/augmentations/defaultAugmentations.js +16 -11
- package/dist/augmentations/discovery/catalogDiscovery.d.ts +142 -0
- package/dist/augmentations/discovery/catalogDiscovery.js +249 -0
- package/dist/augmentations/discovery/localDiscovery.d.ts +84 -0
- package/dist/augmentations/discovery/localDiscovery.js +246 -0
- package/dist/augmentations/discovery/runtimeLoader.d.ts +97 -0
- package/dist/augmentations/discovery/runtimeLoader.js +337 -0
- package/dist/augmentations/discovery.d.ts +152 -0
- package/dist/augmentations/discovery.js +441 -0
- package/dist/augmentations/display/cache.d.ts +130 -0
- package/dist/augmentations/display/cache.js +319 -0
- package/dist/augmentations/display/fieldPatterns.d.ts +52 -0
- package/dist/augmentations/display/fieldPatterns.js +393 -0
- package/dist/augmentations/display/iconMappings.d.ts +57 -0
- package/dist/augmentations/display/iconMappings.js +68 -0
- package/dist/augmentations/display/intelligentComputation.d.ts +109 -0
- package/dist/augmentations/display/intelligentComputation.js +462 -0
- package/dist/augmentations/display/types.d.ts +203 -0
- package/dist/augmentations/display/types.js +7 -0
- package/dist/augmentations/entityRegistryAugmentation.d.ts +3 -1
- package/dist/augmentations/entityRegistryAugmentation.js +5 -1
- package/dist/augmentations/indexAugmentation.d.ts +5 -3
- package/dist/augmentations/indexAugmentation.js +5 -2
- package/dist/augmentations/intelligentVerbScoringAugmentation.d.ts +24 -7
- package/dist/augmentations/intelligentVerbScoringAugmentation.js +111 -27
- package/dist/augmentations/manifest.d.ts +176 -0
- package/dist/augmentations/manifest.js +8 -0
- package/dist/augmentations/marketplace/AugmentationMarketplace.d.ts +168 -0
- package/dist/augmentations/marketplace/AugmentationMarketplace.js +329 -0
- package/dist/augmentations/marketplace/cli.d.ts +47 -0
- package/dist/augmentations/marketplace/cli.js +265 -0
- package/dist/augmentations/metricsAugmentation.d.ts +3 -3
- package/dist/augmentations/metricsAugmentation.js +2 -2
- package/dist/augmentations/monitoringAugmentation.d.ts +3 -3
- package/dist/augmentations/monitoringAugmentation.js +2 -2
- package/dist/augmentations/neuralImport.d.ts +1 -1
- package/dist/augmentations/neuralImport.js +4 -4
- package/dist/augmentations/rateLimitAugmentation.d.ts +82 -0
- package/dist/augmentations/rateLimitAugmentation.js +321 -0
- package/dist/augmentations/requestDeduplicatorAugmentation.d.ts +2 -2
- package/dist/augmentations/requestDeduplicatorAugmentation.js +1 -1
- package/dist/augmentations/storageAugmentation.d.ts +1 -1
- package/dist/augmentations/storageAugmentation.js +2 -2
- package/dist/augmentations/storageAugmentations.d.ts +37 -8
- package/dist/augmentations/storageAugmentations.js +204 -15
- package/dist/augmentations/synapseAugmentation.d.ts +1 -1
- package/dist/augmentations/synapseAugmentation.js +35 -16
- package/dist/augmentations/typeMatching/brainyTypes.d.ts +83 -0
- package/dist/augmentations/typeMatching/brainyTypes.js +425 -0
- package/dist/augmentations/typeMatching/intelligentTypeMatcher.d.ts +39 -59
- package/dist/augmentations/typeMatching/intelligentTypeMatcher.js +103 -389
- package/dist/augmentations/universalDisplayAugmentation.d.ts +191 -0
- package/dist/augmentations/universalDisplayAugmentation.js +371 -0
- package/dist/brainy-unified.d.ts +106 -0
- package/dist/brainy-unified.js +327 -0
- package/dist/brainy.d.ts +273 -0
- package/dist/brainy.js +1181 -0
- package/dist/brainyData.d.ts +56 -111
- package/dist/brainyData.js +912 -756
- package/dist/brainyDataV3.d.ts +186 -0
- package/dist/brainyDataV3.js +337 -0
- package/dist/browserFramework.d.ts +6 -6
- package/dist/browserFramework.js +11 -8
- package/dist/browserFramework.minimal.d.ts +5 -5
- package/dist/browserFramework.minimal.js +11 -8
- package/dist/config/distributedPresets-new.d.ts +118 -0
- package/dist/config/distributedPresets-new.js +318 -0
- package/dist/config/distributedPresets.d.ts +118 -0
- package/dist/config/distributedPresets.js +318 -0
- package/dist/config/extensibleConfig.d.ts +99 -0
- package/dist/config/extensibleConfig.js +268 -0
- package/dist/config/index.d.ts +17 -0
- package/dist/config/index.js +35 -0
- package/dist/config/modelAutoConfig.d.ts +32 -0
- package/dist/config/modelAutoConfig.js +139 -0
- package/dist/config/modelPrecisionManager.d.ts +42 -0
- package/dist/config/modelPrecisionManager.js +98 -0
- package/dist/config/sharedConfigManager.d.ts +67 -0
- package/dist/config/sharedConfigManager.js +215 -0
- package/dist/config/storageAutoConfig.d.ts +41 -0
- package/dist/config/storageAutoConfig.js +328 -0
- package/dist/config/zeroConfig.d.ts +68 -0
- package/dist/config/zeroConfig.js +301 -0
- package/dist/cortex/backupRestore.d.ts +2 -2
- package/dist/cortex/backupRestore.js +85 -27
- package/dist/cortex/healthCheck.d.ts +2 -2
- package/dist/cortex/neuralImport.d.ts +2 -2
- package/dist/cortex/neuralImport.js +18 -13
- package/dist/cortex/performanceMonitor.d.ts +2 -2
- package/dist/critical/model-guardian.d.ts +4 -0
- package/dist/critical/model-guardian.js +31 -11
- package/dist/demo.d.ts +4 -4
- package/dist/demo.js +7 -7
- package/dist/distributed/cacheSync.d.ts +112 -0
- package/dist/distributed/cacheSync.js +265 -0
- package/dist/distributed/coordinator.d.ts +193 -0
- package/dist/distributed/coordinator.js +548 -0
- package/dist/distributed/httpTransport.d.ts +120 -0
- package/dist/distributed/httpTransport.js +446 -0
- package/dist/distributed/index.d.ts +8 -0
- package/dist/distributed/index.js +5 -0
- package/dist/distributed/networkTransport.d.ts +132 -0
- package/dist/distributed/networkTransport.js +633 -0
- package/dist/distributed/queryPlanner.d.ts +104 -0
- package/dist/distributed/queryPlanner.js +327 -0
- package/dist/distributed/readWriteSeparation.d.ts +134 -0
- package/dist/distributed/readWriteSeparation.js +350 -0
- package/dist/distributed/shardManager.d.ts +114 -0
- package/dist/distributed/shardManager.js +357 -0
- package/dist/distributed/shardMigration.d.ts +110 -0
- package/dist/distributed/shardMigration.js +289 -0
- package/dist/distributed/storageDiscovery.d.ts +160 -0
- package/dist/distributed/storageDiscovery.js +551 -0
- package/dist/embeddings/CachedEmbeddings.d.ts +40 -0
- package/dist/embeddings/CachedEmbeddings.js +146 -0
- package/dist/embeddings/EmbeddingManager.d.ts +102 -0
- package/dist/embeddings/EmbeddingManager.js +291 -0
- package/dist/embeddings/SingletonModelManager.d.ts +95 -0
- package/dist/embeddings/SingletonModelManager.js +220 -0
- package/dist/embeddings/index.d.ts +12 -0
- package/dist/embeddings/index.js +16 -0
- package/dist/embeddings/lightweight-embedder.d.ts +0 -1
- package/dist/embeddings/lightweight-embedder.js +4 -12
- package/dist/embeddings/model-manager.d.ts +11 -0
- package/dist/embeddings/model-manager.js +43 -7
- package/dist/embeddings/universal-memory-manager.d.ts +1 -1
- package/dist/embeddings/universal-memory-manager.js +27 -67
- package/dist/embeddings/worker-embedding.js +4 -8
- package/dist/errors/brainyError.d.ts +5 -1
- package/dist/errors/brainyError.js +12 -0
- package/dist/examples/basicUsage.js +7 -4
- package/dist/graph/graphAdjacencyIndex.d.ts +96 -0
- package/dist/graph/graphAdjacencyIndex.js +288 -0
- package/dist/graph/pathfinding.js +4 -2
- package/dist/hnsw/scaledHNSWSystem.js +11 -2
- package/dist/importManager.js +8 -5
- package/dist/index.d.ts +17 -22
- package/dist/index.js +37 -23
- package/dist/mcp/brainyMCPAdapter.d.ts +4 -4
- package/dist/mcp/brainyMCPAdapter.js +5 -5
- package/dist/mcp/brainyMCPService.d.ts +3 -3
- package/dist/mcp/brainyMCPService.js +3 -11
- package/dist/mcp/mcpAugmentationToolset.js +20 -30
- package/dist/neural/embeddedPatterns.d.ts +1 -1
- package/dist/neural/embeddedPatterns.js +2 -2
- package/dist/neural/entityExtractor.d.ts +65 -0
- package/dist/neural/entityExtractor.js +316 -0
- package/dist/neural/improvedNeuralAPI.d.ts +357 -0
- package/dist/neural/improvedNeuralAPI.js +2628 -0
- package/dist/neural/naturalLanguageProcessor.d.ts +155 -10
- package/dist/neural/naturalLanguageProcessor.js +941 -66
- package/dist/neural/naturalLanguageProcessorStatic.d.ts +2 -2
- package/dist/neural/naturalLanguageProcessorStatic.js +3 -3
- package/dist/neural/neuralAPI.js +8 -2
- package/dist/neural/patternLibrary.d.ts +57 -3
- package/dist/neural/patternLibrary.js +348 -13
- package/dist/neural/staticPatternMatcher.d.ts +2 -2
- package/dist/neural/staticPatternMatcher.js +2 -2
- package/dist/neural/types.d.ts +287 -0
- package/dist/neural/types.js +24 -0
- package/dist/shared/default-augmentations.d.ts +3 -3
- package/dist/shared/default-augmentations.js +5 -5
- package/dist/storage/adapters/baseStorageAdapter.d.ts +42 -0
- package/dist/storage/adapters/fileSystemStorage.d.ts +26 -2
- package/dist/storage/adapters/fileSystemStorage.js +218 -15
- package/dist/storage/adapters/memoryStorage.d.ts +4 -4
- package/dist/storage/adapters/memoryStorage.js +17 -12
- package/dist/storage/adapters/opfsStorage.d.ts +2 -2
- package/dist/storage/adapters/opfsStorage.js +2 -2
- package/dist/storage/adapters/s3CompatibleStorage.d.ts +2 -2
- package/dist/storage/adapters/s3CompatibleStorage.js +2 -2
- package/dist/storage/backwardCompatibility.d.ts +10 -78
- package/dist/storage/backwardCompatibility.js +17 -132
- package/dist/storage/baseStorage.d.ts +18 -2
- package/dist/storage/baseStorage.js +74 -3
- package/dist/storage/cacheManager.js +2 -2
- package/dist/storage/readOnlyOptimizations.js +8 -3
- package/dist/streaming/pipeline.d.ts +154 -0
- package/dist/streaming/pipeline.js +551 -0
- package/dist/triple/TripleIntelligence.d.ts +25 -110
- package/dist/triple/TripleIntelligence.js +4 -574
- package/dist/triple/TripleIntelligenceSystem.d.ts +159 -0
- package/dist/triple/TripleIntelligenceSystem.js +519 -0
- package/dist/types/apiTypes.d.ts +278 -0
- package/dist/types/apiTypes.js +33 -0
- package/dist/types/brainy.types.d.ts +308 -0
- package/dist/types/brainy.types.js +8 -0
- package/dist/types/brainyDataInterface.d.ts +5 -8
- package/dist/types/brainyDataInterface.js +2 -2
- package/dist/types/graphTypes.js +2 -2
- package/dist/utils/brainyTypes.d.ts +217 -0
- package/dist/utils/brainyTypes.js +261 -0
- package/dist/utils/cacheAutoConfig.d.ts +3 -3
- package/dist/utils/embedding.d.ts +9 -4
- package/dist/utils/embedding.js +89 -26
- package/dist/utils/enhancedLogger.d.ts +104 -0
- package/dist/utils/enhancedLogger.js +232 -0
- package/dist/utils/hybridModelManager.d.ts +19 -28
- package/dist/utils/hybridModelManager.js +36 -200
- package/dist/utils/index.d.ts +1 -1
- package/dist/utils/index.js +1 -1
- package/dist/utils/intelligentTypeMapper.d.ts +60 -0
- package/dist/utils/intelligentTypeMapper.js +349 -0
- package/dist/utils/metadataIndex.d.ts +118 -1
- package/dist/utils/metadataIndex.js +539 -16
- package/dist/utils/nodeVersionCheck.d.ts +24 -0
- package/dist/utils/nodeVersionCheck.js +65 -0
- package/dist/utils/paramValidation.d.ts +39 -0
- package/dist/utils/paramValidation.js +192 -0
- package/dist/utils/rateLimiter.d.ts +160 -0
- package/dist/utils/rateLimiter.js +271 -0
- package/dist/utils/statistics.d.ts +4 -4
- package/dist/utils/statistics.js +3 -3
- package/dist/utils/structuredLogger.d.ts +146 -0
- package/dist/utils/structuredLogger.js +394 -0
- package/dist/utils/textEncoding.js +2 -1
- package/dist/utils/typeValidation.d.ts +59 -0
- package/dist/utils/typeValidation.js +374 -0
- package/dist/utils/version.js +19 -3
- package/package.json +15 -4
- package/scripts/download-models.cjs +94 -20
- package/dist/augmentations/walAugmentation.d.ts +0 -109
- package/dist/augmentations/walAugmentation.js +0 -516
- package/dist/chat/BrainyChat.d.ts +0 -121
- package/dist/chat/BrainyChat.js +0 -396
- package/dist/chat/ChatCLI.d.ts +0 -61
- package/dist/chat/ChatCLI.js +0 -351
|
@@ -0,0 +1,220 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Singleton Model Manager - THE ONLY SOURCE OF EMBEDDING MODELS
|
|
3
|
+
*
|
|
4
|
+
* This is the SINGLE, UNIFIED model initialization system that ensures:
|
|
5
|
+
* - Only ONE model instance exists across the entire system
|
|
6
|
+
* - Precision is configured once and locked
|
|
7
|
+
* - All components share the same model
|
|
8
|
+
* - No possibility of mixed precisions
|
|
9
|
+
*
|
|
10
|
+
* CRITICAL: This manager is used by EVERYTHING:
|
|
11
|
+
* - Storage operations (add, update)
|
|
12
|
+
* - Search operations (search, find)
|
|
13
|
+
* - Public API (embed, cluster)
|
|
14
|
+
* - Neural API (all neural.* methods)
|
|
15
|
+
* - Internal operations (deduplication, indexing)
|
|
16
|
+
*/
|
|
17
|
+
import { TransformerEmbedding } from '../utils/embedding.js';
|
|
18
|
+
import { getModelPrecision, lockModelPrecision } from '../config/modelPrecisionManager.js';
|
|
19
|
+
// Global state - ensures true singleton across entire process
|
|
20
|
+
let globalModelInstance = null;
|
|
21
|
+
let globalInitPromise = null;
|
|
22
|
+
let globalInitialized = false;
|
|
23
|
+
/**
|
|
24
|
+
* The ONE TRUE model manager
|
|
25
|
+
*/
|
|
26
|
+
export class SingletonModelManager {
|
|
27
|
+
constructor() {
|
|
28
|
+
this.stats = {
|
|
29
|
+
initialized: false,
|
|
30
|
+
precision: 'unknown',
|
|
31
|
+
initCount: 0,
|
|
32
|
+
embedCount: 0,
|
|
33
|
+
lastUsed: null
|
|
34
|
+
};
|
|
35
|
+
// Private constructor enforces singleton
|
|
36
|
+
this.stats.precision = getModelPrecision();
|
|
37
|
+
console.log(`🔐 SingletonModelManager initialized with ${this.stats.precision.toUpperCase()} precision`);
|
|
38
|
+
}
|
|
39
|
+
/**
|
|
40
|
+
* Get the singleton instance
|
|
41
|
+
*/
|
|
42
|
+
static getInstance() {
|
|
43
|
+
if (!SingletonModelManager.instance) {
|
|
44
|
+
SingletonModelManager.instance = new SingletonModelManager();
|
|
45
|
+
}
|
|
46
|
+
return SingletonModelManager.instance;
|
|
47
|
+
}
|
|
48
|
+
/**
|
|
49
|
+
* Get the model instance - creates if needed, reuses if exists
|
|
50
|
+
* This is THE ONLY way to get a model in the entire system
|
|
51
|
+
*/
|
|
52
|
+
async getModel() {
|
|
53
|
+
// If already initialized, return immediately
|
|
54
|
+
if (globalModelInstance && globalInitialized) {
|
|
55
|
+
this.stats.lastUsed = new Date();
|
|
56
|
+
return globalModelInstance;
|
|
57
|
+
}
|
|
58
|
+
// If initialization is in progress, wait for it
|
|
59
|
+
if (globalInitPromise) {
|
|
60
|
+
console.log('⏳ Model initialization already in progress, waiting...');
|
|
61
|
+
return await globalInitPromise;
|
|
62
|
+
}
|
|
63
|
+
// Start initialization (only happens once ever)
|
|
64
|
+
globalInitPromise = this.initializeModel();
|
|
65
|
+
try {
|
|
66
|
+
const model = await globalInitPromise;
|
|
67
|
+
globalInitialized = true;
|
|
68
|
+
return model;
|
|
69
|
+
}
|
|
70
|
+
catch (error) {
|
|
71
|
+
// Reset on error to allow retry
|
|
72
|
+
globalInitPromise = null;
|
|
73
|
+
throw error;
|
|
74
|
+
}
|
|
75
|
+
}
|
|
76
|
+
/**
|
|
77
|
+
* Initialize the model - happens exactly once
|
|
78
|
+
*/
|
|
79
|
+
async initializeModel() {
|
|
80
|
+
console.log('🚀 Initializing singleton model instance...');
|
|
81
|
+
// Get precision from central manager
|
|
82
|
+
const precision = getModelPrecision();
|
|
83
|
+
console.log(`📊 Using ${precision.toUpperCase()} precision (${precision === 'q8' ? '23MB, 99% accuracy' : '90MB, 100% accuracy'})`);
|
|
84
|
+
// Detect environment for optimal settings
|
|
85
|
+
const isNode = typeof process !== 'undefined' && process.versions?.node;
|
|
86
|
+
const isBrowser = typeof window !== 'undefined' && typeof document !== 'undefined';
|
|
87
|
+
const isServerless = typeof process !== 'undefined' && (process.env.VERCEL ||
|
|
88
|
+
process.env.NETLIFY ||
|
|
89
|
+
process.env.AWS_LAMBDA_FUNCTION_NAME ||
|
|
90
|
+
process.env.FUNCTIONS_WORKER_RUNTIME);
|
|
91
|
+
const isTest = globalThis.__BRAINY_TEST_ENV__ || process.env.NODE_ENV === 'test';
|
|
92
|
+
// Create optimized options based on environment
|
|
93
|
+
const options = {
|
|
94
|
+
precision: precision,
|
|
95
|
+
verbose: !isTest && !isServerless && !isBrowser,
|
|
96
|
+
device: 'cpu', // CPU is most compatible
|
|
97
|
+
localFilesOnly: process.env.BRAINY_ALLOW_REMOTE_MODELS === 'false',
|
|
98
|
+
model: 'Xenova/all-MiniLM-L6-v2'
|
|
99
|
+
};
|
|
100
|
+
try {
|
|
101
|
+
// Create the ONE model instance
|
|
102
|
+
globalModelInstance = new TransformerEmbedding(options);
|
|
103
|
+
// Initialize it
|
|
104
|
+
await globalModelInstance.init();
|
|
105
|
+
// CRITICAL: Lock the precision after successful initialization
|
|
106
|
+
// This prevents any future changes to precision
|
|
107
|
+
lockModelPrecision();
|
|
108
|
+
console.log('🔒 Model precision locked at:', precision.toUpperCase());
|
|
109
|
+
// Update stats
|
|
110
|
+
this.stats.initialized = true;
|
|
111
|
+
this.stats.initCount++;
|
|
112
|
+
this.stats.lastUsed = new Date();
|
|
113
|
+
// Log memory usage if available
|
|
114
|
+
if (isNode && process.memoryUsage) {
|
|
115
|
+
const usage = process.memoryUsage();
|
|
116
|
+
this.stats.memoryFootprint = Math.round(usage.heapUsed / 1024 / 1024);
|
|
117
|
+
console.log(`💾 Model loaded, memory usage: ${this.stats.memoryFootprint}MB`);
|
|
118
|
+
}
|
|
119
|
+
console.log('✅ Singleton model initialized successfully');
|
|
120
|
+
return globalModelInstance;
|
|
121
|
+
}
|
|
122
|
+
catch (error) {
|
|
123
|
+
console.error('❌ Failed to initialize singleton model:', error);
|
|
124
|
+
globalModelInstance = null;
|
|
125
|
+
throw new Error(`Singleton model initialization failed: ${error instanceof Error ? error.message : String(error)}`);
|
|
126
|
+
}
|
|
127
|
+
}
|
|
128
|
+
/**
|
|
129
|
+
* Get embedding function that uses the singleton model
|
|
130
|
+
*/
|
|
131
|
+
async getEmbeddingFunction() {
|
|
132
|
+
const model = await this.getModel();
|
|
133
|
+
return async (data) => {
|
|
134
|
+
this.stats.embedCount++;
|
|
135
|
+
this.stats.lastUsed = new Date();
|
|
136
|
+
return await model.embed(data);
|
|
137
|
+
};
|
|
138
|
+
}
|
|
139
|
+
/**
|
|
140
|
+
* Direct embed method for convenience
|
|
141
|
+
*/
|
|
142
|
+
async embed(data) {
|
|
143
|
+
const model = await this.getModel();
|
|
144
|
+
this.stats.embedCount++;
|
|
145
|
+
this.stats.lastUsed = new Date();
|
|
146
|
+
return await model.embed(data);
|
|
147
|
+
}
|
|
148
|
+
/**
|
|
149
|
+
* Check if model is initialized
|
|
150
|
+
*/
|
|
151
|
+
isInitialized() {
|
|
152
|
+
return globalInitialized && globalModelInstance !== null;
|
|
153
|
+
}
|
|
154
|
+
/**
|
|
155
|
+
* Get current statistics
|
|
156
|
+
*/
|
|
157
|
+
getStats() {
|
|
158
|
+
return {
|
|
159
|
+
...this.stats,
|
|
160
|
+
precision: getModelPrecision()
|
|
161
|
+
};
|
|
162
|
+
}
|
|
163
|
+
/**
|
|
164
|
+
* Validate precision consistency
|
|
165
|
+
* Throws error if attempting to use different precision
|
|
166
|
+
*/
|
|
167
|
+
validatePrecision(requestedPrecision) {
|
|
168
|
+
const currentPrecision = getModelPrecision();
|
|
169
|
+
if (requestedPrecision && requestedPrecision !== currentPrecision) {
|
|
170
|
+
throw new Error(`❌ Precision mismatch! System is using ${currentPrecision.toUpperCase()} ` +
|
|
171
|
+
`but ${requestedPrecision.toUpperCase()} was requested. ` +
|
|
172
|
+
`All operations must use the same precision.`);
|
|
173
|
+
}
|
|
174
|
+
}
|
|
175
|
+
/**
|
|
176
|
+
* Force cleanup (for testing only)
|
|
177
|
+
* WARNING: This will break consistency - use only in tests
|
|
178
|
+
*/
|
|
179
|
+
async _testOnlyCleanup() {
|
|
180
|
+
if (process.env.NODE_ENV !== 'test') {
|
|
181
|
+
throw new Error('Cleanup only allowed in test environment');
|
|
182
|
+
}
|
|
183
|
+
if (globalModelInstance && 'dispose' in globalModelInstance) {
|
|
184
|
+
await globalModelInstance.dispose();
|
|
185
|
+
}
|
|
186
|
+
globalModelInstance = null;
|
|
187
|
+
globalInitPromise = null;
|
|
188
|
+
globalInitialized = false;
|
|
189
|
+
this.stats.initialized = false;
|
|
190
|
+
console.log('🧹 Singleton model cleaned up (test only)');
|
|
191
|
+
}
|
|
192
|
+
}
|
|
193
|
+
// Export the singleton instance getter
|
|
194
|
+
export const singletonModelManager = SingletonModelManager.getInstance();
|
|
195
|
+
/**
|
|
196
|
+
* THE ONLY embedding function that should be used anywhere
|
|
197
|
+
* This ensures all operations use the same model instance
|
|
198
|
+
*/
|
|
199
|
+
export async function getUnifiedEmbeddingFunction() {
|
|
200
|
+
return await singletonModelManager.getEmbeddingFunction();
|
|
201
|
+
}
|
|
202
|
+
/**
|
|
203
|
+
* Direct embed function for convenience
|
|
204
|
+
*/
|
|
205
|
+
export async function unifiedEmbed(data) {
|
|
206
|
+
return await singletonModelManager.embed(data);
|
|
207
|
+
}
|
|
208
|
+
/**
|
|
209
|
+
* Check if model is ready
|
|
210
|
+
*/
|
|
211
|
+
export function isModelReady() {
|
|
212
|
+
return singletonModelManager.isInitialized();
|
|
213
|
+
}
|
|
214
|
+
/**
|
|
215
|
+
* Get model statistics
|
|
216
|
+
*/
|
|
217
|
+
export function getModelStats() {
|
|
218
|
+
return singletonModelManager.getStats();
|
|
219
|
+
}
|
|
220
|
+
//# sourceMappingURL=SingletonModelManager.js.map
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Embeddings Module - Clean, Unified Architecture
|
|
3
|
+
*
|
|
4
|
+
* This module provides all embedding functionality for Brainy.
|
|
5
|
+
*
|
|
6
|
+
* Main Components:
|
|
7
|
+
* - EmbeddingManager: Core embedding generation with Q8/FP32 support
|
|
8
|
+
* - CachedEmbeddings: Performance optimization layer with pre-computed embeddings
|
|
9
|
+
*/
|
|
10
|
+
export { EmbeddingManager, embeddingManager, embed, getEmbeddingFunction, getEmbeddingStats, type ModelPrecision } from './EmbeddingManager.js';
|
|
11
|
+
export { CachedEmbeddings, cachedEmbeddings } from './CachedEmbeddings.js';
|
|
12
|
+
export { embeddingManager as default } from './EmbeddingManager.js';
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Embeddings Module - Clean, Unified Architecture
|
|
3
|
+
*
|
|
4
|
+
* This module provides all embedding functionality for Brainy.
|
|
5
|
+
*
|
|
6
|
+
* Main Components:
|
|
7
|
+
* - EmbeddingManager: Core embedding generation with Q8/FP32 support
|
|
8
|
+
* - CachedEmbeddings: Performance optimization layer with pre-computed embeddings
|
|
9
|
+
*/
|
|
10
|
+
// Core embedding functionality
|
|
11
|
+
export { EmbeddingManager, embeddingManager, embed, getEmbeddingFunction, getEmbeddingStats } from './EmbeddingManager.js';
|
|
12
|
+
// Cached embeddings for performance
|
|
13
|
+
export { CachedEmbeddings, cachedEmbeddings } from './CachedEmbeddings.js';
|
|
14
|
+
// Default export is the singleton manager
|
|
15
|
+
export { embeddingManager as default } from './EmbeddingManager.js';
|
|
16
|
+
//# sourceMappingURL=index.js.map
|
|
@@ -6,6 +6,7 @@
|
|
|
6
6
|
*
|
|
7
7
|
* This reduces memory usage by 90% for typical queries
|
|
8
8
|
*/
|
|
9
|
+
import { singletonModelManager } from './SingletonModelManager.js';
|
|
9
10
|
// Pre-computed embeddings for top 10,000 common terms
|
|
10
11
|
// In production, this would be loaded from a file
|
|
11
12
|
const PRECOMPUTED_EMBEDDINGS = {
|
|
@@ -59,7 +60,6 @@ function computeSimpleEmbedding(text) {
|
|
|
59
60
|
}
|
|
60
61
|
export class LightweightEmbedder {
|
|
61
62
|
constructor() {
|
|
62
|
-
this.onnxEmbedder = null;
|
|
63
63
|
this.stats = {
|
|
64
64
|
precomputedHits: 0,
|
|
65
65
|
simpleComputes: 0,
|
|
@@ -92,18 +92,10 @@ export class LightweightEmbedder {
|
|
|
92
92
|
this.stats.simpleComputes++;
|
|
93
93
|
return computeSimpleEmbedding(normalized);
|
|
94
94
|
}
|
|
95
|
-
// 4. Last resort:
|
|
96
|
-
|
|
97
|
-
console.log('⚠️ Loading ONNX model for complex text...');
|
|
98
|
-
const { TransformerEmbedding } = await import('../utils/embedding.js');
|
|
99
|
-
this.onnxEmbedder = new TransformerEmbedding({
|
|
100
|
-
dtype: 'q8',
|
|
101
|
-
verbose: false
|
|
102
|
-
});
|
|
103
|
-
await this.onnxEmbedder.init();
|
|
104
|
-
}
|
|
95
|
+
// 4. Last resort: Use SingletonModelManager for complex text
|
|
96
|
+
console.log('⚠️ Using singleton model for complex text...');
|
|
105
97
|
this.stats.onnxComputes++;
|
|
106
|
-
return await
|
|
98
|
+
return await singletonModelManager.embed(text);
|
|
107
99
|
}
|
|
108
100
|
getStats() {
|
|
109
101
|
return {
|
|
@@ -18,6 +18,17 @@ export declare class ModelManager {
|
|
|
18
18
|
private getModelsPath;
|
|
19
19
|
ensureModels(modelName?: string): Promise<boolean>;
|
|
20
20
|
private verifyModelFiles;
|
|
21
|
+
/**
|
|
22
|
+
* Check which model variants are available locally
|
|
23
|
+
*/
|
|
24
|
+
getAvailableModels(modelName?: string): {
|
|
25
|
+
fp32: boolean;
|
|
26
|
+
q8: boolean;
|
|
27
|
+
};
|
|
28
|
+
/**
|
|
29
|
+
* Get the best available model variant based on preference and availability
|
|
30
|
+
*/
|
|
31
|
+
getBestAvailableModel(preferredType?: 'fp32' | 'q8', modelName?: string): 'fp32' | 'q8' | null;
|
|
21
32
|
private tryModelSource;
|
|
22
33
|
private downloadAndExtractFromGitHub;
|
|
23
34
|
/**
|
|
@@ -31,13 +31,16 @@ const MODEL_SOURCES = {
|
|
|
31
31
|
pathTemplate: '{model}/resolve/{revision}/' // Default transformers.js pattern
|
|
32
32
|
}
|
|
33
33
|
};
|
|
34
|
-
// Model verification files -
|
|
35
|
-
const
|
|
34
|
+
// Model verification files - BOTH fp32 and q8 variants
|
|
35
|
+
const REQUIRED_FILES = [
|
|
36
36
|
'config.json',
|
|
37
37
|
'tokenizer.json',
|
|
38
|
-
'tokenizer_config.json'
|
|
39
|
-
'onnx/model.onnx'
|
|
38
|
+
'tokenizer_config.json'
|
|
40
39
|
];
|
|
40
|
+
const MODEL_VARIANTS = {
|
|
41
|
+
fp32: 'onnx/model.onnx',
|
|
42
|
+
q8: 'onnx/model_quantized.onnx'
|
|
43
|
+
};
|
|
41
44
|
export class ModelManager {
|
|
42
45
|
constructor() {
|
|
43
46
|
this.isInitialized = false;
|
|
@@ -105,14 +108,47 @@ export class ModelManager {
|
|
|
105
108
|
return true;
|
|
106
109
|
}
|
|
107
110
|
async verifyModelFiles(modelPath) {
|
|
108
|
-
// Check if essential
|
|
109
|
-
for (const file of
|
|
111
|
+
// Check if essential files exist
|
|
112
|
+
for (const file of REQUIRED_FILES) {
|
|
110
113
|
const fullPath = join(modelPath, file);
|
|
111
114
|
if (!existsSync(fullPath)) {
|
|
112
115
|
return false;
|
|
113
116
|
}
|
|
114
117
|
}
|
|
115
|
-
|
|
118
|
+
// At least one model variant must exist (fp32 or q8)
|
|
119
|
+
const fp32Exists = existsSync(join(modelPath, MODEL_VARIANTS.fp32));
|
|
120
|
+
const q8Exists = existsSync(join(modelPath, MODEL_VARIANTS.q8));
|
|
121
|
+
return fp32Exists || q8Exists;
|
|
122
|
+
}
|
|
123
|
+
/**
|
|
124
|
+
* Check which model variants are available locally
|
|
125
|
+
*/
|
|
126
|
+
getAvailableModels(modelName = 'Xenova/all-MiniLM-L6-v2') {
|
|
127
|
+
const modelPath = join(this.modelsPath, modelName);
|
|
128
|
+
return {
|
|
129
|
+
fp32: existsSync(join(modelPath, MODEL_VARIANTS.fp32)),
|
|
130
|
+
q8: existsSync(join(modelPath, MODEL_VARIANTS.q8))
|
|
131
|
+
};
|
|
132
|
+
}
|
|
133
|
+
/**
|
|
134
|
+
* Get the best available model variant based on preference and availability
|
|
135
|
+
*/
|
|
136
|
+
getBestAvailableModel(preferredType = 'fp32', modelName = 'Xenova/all-MiniLM-L6-v2') {
|
|
137
|
+
const available = this.getAvailableModels(modelName);
|
|
138
|
+
// If preferred type is available, use it
|
|
139
|
+
if (available[preferredType]) {
|
|
140
|
+
return preferredType;
|
|
141
|
+
}
|
|
142
|
+
// Otherwise fall back to what's available
|
|
143
|
+
if (preferredType === 'q8' && available.fp32) {
|
|
144
|
+
console.warn('⚠️ Q8 model requested but not available, falling back to FP32');
|
|
145
|
+
return 'fp32';
|
|
146
|
+
}
|
|
147
|
+
if (preferredType === 'fp32' && available.q8) {
|
|
148
|
+
console.warn('⚠️ FP32 model requested but not available, falling back to Q8');
|
|
149
|
+
return 'q8';
|
|
150
|
+
}
|
|
151
|
+
return null;
|
|
116
152
|
}
|
|
117
153
|
async tryModelSource(name, source, modelName) {
|
|
118
154
|
try {
|
|
@@ -23,7 +23,7 @@ export declare class UniversalMemoryManager {
|
|
|
23
23
|
embed(data: string | string[]): Promise<Vector>;
|
|
24
24
|
private checkMemoryLimits;
|
|
25
25
|
private ensureEmbeddingFunction;
|
|
26
|
-
private
|
|
26
|
+
private initNodeDirect;
|
|
27
27
|
private initServerless;
|
|
28
28
|
private initBrowser;
|
|
29
29
|
private initFallback;
|
|
@@ -4,6 +4,7 @@
|
|
|
4
4
|
* Works in ALL environments: Node.js, browsers, serverless, workers
|
|
5
5
|
* Solves transformers.js memory leak with environment-specific strategies
|
|
6
6
|
*/
|
|
7
|
+
import { getModelPrecision } from '../config/modelPrecisionManager.js';
|
|
7
8
|
// Environment detection
|
|
8
9
|
const isNode = typeof process !== 'undefined' && process.versions?.node;
|
|
9
10
|
const isBrowser = typeof window !== 'undefined' && typeof document !== 'undefined';
|
|
@@ -13,18 +14,21 @@ const isServerless = typeof process !== 'undefined' && (process.env.VERCEL ||
|
|
|
13
14
|
process.env.FUNCTIONS_WORKER_RUNTIME);
|
|
14
15
|
export class UniversalMemoryManager {
|
|
15
16
|
constructor() {
|
|
17
|
+
// CRITICAL FIX: Never use worker threads with ONNX Runtime
|
|
18
|
+
// Worker threads cause HandleScope V8 API errors due to isolate issues
|
|
19
|
+
// Always use direct embedding on main thread for ONNX compatibility
|
|
16
20
|
this.embeddingFunction = null;
|
|
17
21
|
this.embedCount = 0;
|
|
18
22
|
this.restartCount = 0;
|
|
19
23
|
this.lastRestart = 0;
|
|
20
|
-
// Choose strategy based on environment
|
|
21
24
|
if (isServerless) {
|
|
22
25
|
this.strategy = 'serverless-restart';
|
|
23
26
|
this.maxEmbeddings = 50; // Restart frequently in serverless
|
|
24
27
|
}
|
|
25
28
|
else if (isNode && !isBrowser) {
|
|
26
|
-
|
|
27
|
-
this.
|
|
29
|
+
// CHANGED: Use direct strategy instead of node-worker to avoid V8 isolate issues
|
|
30
|
+
this.strategy = 'node-direct';
|
|
31
|
+
this.maxEmbeddings = 200; // Main thread can handle more with single model instance
|
|
28
32
|
}
|
|
29
33
|
else if (isBrowser) {
|
|
30
34
|
this.strategy = 'browser-dispose';
|
|
@@ -35,6 +39,7 @@ export class UniversalMemoryManager {
|
|
|
35
39
|
this.maxEmbeddings = 75;
|
|
36
40
|
}
|
|
37
41
|
console.log(`🧠 Universal Memory Manager: Using ${this.strategy} strategy`);
|
|
42
|
+
console.log('✅ UNIVERSAL: Memory-safe embedding system initialized');
|
|
38
43
|
}
|
|
39
44
|
async getEmbeddingFunction() {
|
|
40
45
|
return async (data) => {
|
|
@@ -62,8 +67,8 @@ export class UniversalMemoryManager {
|
|
|
62
67
|
return;
|
|
63
68
|
}
|
|
64
69
|
switch (this.strategy) {
|
|
65
|
-
case 'node-
|
|
66
|
-
await this.
|
|
70
|
+
case 'node-direct':
|
|
71
|
+
await this.initNodeDirect();
|
|
67
72
|
break;
|
|
68
73
|
case 'serverless-restart':
|
|
69
74
|
await this.initServerless();
|
|
@@ -75,19 +80,12 @@ export class UniversalMemoryManager {
|
|
|
75
80
|
await this.initFallback();
|
|
76
81
|
}
|
|
77
82
|
}
|
|
78
|
-
async
|
|
83
|
+
async initNodeDirect() {
|
|
79
84
|
if (isNode) {
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
console.log('✅ Using Node.js worker threads for embeddings');
|
|
85
|
-
}
|
|
86
|
-
catch (error) {
|
|
87
|
-
console.warn('⚠️ Worker threads not available, falling back to direct embedding');
|
|
88
|
-
console.warn('Error:', error instanceof Error ? error.message : String(error));
|
|
89
|
-
await this.initDirect();
|
|
90
|
-
}
|
|
85
|
+
// CRITICAL: Use direct embedding to avoid worker thread V8 isolate issues
|
|
86
|
+
// This prevents HandleScope errors and ensures single model instance
|
|
87
|
+
console.log('✅ Using Node.js direct embedding (main thread - ONNX compatible)');
|
|
88
|
+
await this.initDirect();
|
|
91
89
|
}
|
|
92
90
|
}
|
|
93
91
|
async initServerless() {
|
|
@@ -110,7 +108,7 @@ export class UniversalMemoryManager {
|
|
|
110
108
|
const { TransformerEmbedding } = await import('../utils/embedding.js');
|
|
111
109
|
this.embeddingFunction = new TransformerEmbedding({
|
|
112
110
|
verbose: false,
|
|
113
|
-
|
|
111
|
+
precision: getModelPrecision(), // Use centrally managed precision
|
|
114
112
|
localFilesOnly: process.env.BRAINY_ALLOW_REMOTE_MODELS !== 'true'
|
|
115
113
|
});
|
|
116
114
|
await this.embeddingFunction.init();
|
|
@@ -122,49 +120,15 @@ export class UniversalMemoryManager {
|
|
|
122
120
|
}
|
|
123
121
|
async cleanup() {
|
|
124
122
|
const startTime = Date.now();
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
// In serverless, create new instance
|
|
135
|
-
if (this.embeddingFunction?.dispose) {
|
|
136
|
-
this.embeddingFunction.dispose();
|
|
137
|
-
}
|
|
138
|
-
this.embeddingFunction = null;
|
|
139
|
-
break;
|
|
140
|
-
case 'browser-dispose':
|
|
141
|
-
// In browser, try disposal
|
|
142
|
-
if (this.embeddingFunction?.dispose) {
|
|
143
|
-
this.embeddingFunction.dispose();
|
|
144
|
-
}
|
|
145
|
-
// Force garbage collection if available
|
|
146
|
-
if (typeof window !== 'undefined' && window.gc) {
|
|
147
|
-
window.gc();
|
|
148
|
-
}
|
|
149
|
-
break;
|
|
150
|
-
default:
|
|
151
|
-
// Fallback: dispose and recreate
|
|
152
|
-
if (this.embeddingFunction?.dispose) {
|
|
153
|
-
this.embeddingFunction.dispose();
|
|
154
|
-
}
|
|
155
|
-
this.embeddingFunction = null;
|
|
156
|
-
}
|
|
157
|
-
this.embedCount = 0;
|
|
158
|
-
this.restartCount++;
|
|
159
|
-
this.lastRestart = Date.now();
|
|
160
|
-
const cleanupTime = Date.now() - startTime;
|
|
161
|
-
console.log(`🧹 Memory cleanup completed in ${cleanupTime}ms (strategy: ${this.strategy})`);
|
|
162
|
-
}
|
|
163
|
-
catch (error) {
|
|
164
|
-
console.warn('⚠️ Cleanup failed:', error instanceof Error ? error.message : String(error));
|
|
165
|
-
// Force null assignment as last resort
|
|
166
|
-
this.embeddingFunction = null;
|
|
167
|
-
}
|
|
123
|
+
// SingletonModelManager persists - we just reset our counters
|
|
124
|
+
// The singleton model stays alive for consistency across all operations
|
|
125
|
+
// Reset counters
|
|
126
|
+
this.embedCount = 0;
|
|
127
|
+
this.restartCount++;
|
|
128
|
+
this.lastRestart = Date.now();
|
|
129
|
+
const cleanupTime = Date.now() - startTime;
|
|
130
|
+
console.log(`🧹 Memory counters reset in ${cleanupTime}ms (strategy: ${this.strategy})`);
|
|
131
|
+
console.log('ℹ️ Singleton model persists for consistency across all operations');
|
|
168
132
|
}
|
|
169
133
|
getMemoryStats() {
|
|
170
134
|
let memoryUsage = 'unknown';
|
|
@@ -185,12 +149,8 @@ export class UniversalMemoryManager {
|
|
|
185
149
|
};
|
|
186
150
|
}
|
|
187
151
|
async dispose() {
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
await this.embeddingFunction.dispose();
|
|
191
|
-
}
|
|
192
|
-
this.embeddingFunction = null;
|
|
193
|
-
}
|
|
152
|
+
// SingletonModelManager persists - nothing to dispose
|
|
153
|
+
console.log('ℹ️ Universal Memory Manager: Singleton model persists');
|
|
194
154
|
}
|
|
195
155
|
}
|
|
196
156
|
// Export singleton instance
|
|
@@ -6,6 +6,7 @@
|
|
|
6
6
|
*/
|
|
7
7
|
import { TransformerEmbedding } from '../utils/embedding.js';
|
|
8
8
|
import { parentPort } from 'worker_threads';
|
|
9
|
+
import { getModelPrecision } from '../config/modelPrecisionManager.js';
|
|
9
10
|
let model = null;
|
|
10
11
|
let requestCount = 0;
|
|
11
12
|
const MAX_REQUESTS = 100; // Restart worker after 100 requests to prevent memory leak
|
|
@@ -13,7 +14,7 @@ async function initModel() {
|
|
|
13
14
|
if (!model) {
|
|
14
15
|
model = new TransformerEmbedding({
|
|
15
16
|
verbose: false,
|
|
16
|
-
|
|
17
|
+
precision: getModelPrecision(), // Use centrally managed precision
|
|
17
18
|
localFilesOnly: process.env.BRAINY_ALLOW_REMOTE_MODELS !== 'true'
|
|
18
19
|
});
|
|
19
20
|
await model.init();
|
|
@@ -37,13 +38,8 @@ if (parentPort) {
|
|
|
37
38
|
}
|
|
38
39
|
break;
|
|
39
40
|
case 'dispose':
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
if ('dispose' in model && typeof model.dispose === 'function') {
|
|
43
|
-
model.dispose();
|
|
44
|
-
}
|
|
45
|
-
model = null;
|
|
46
|
-
}
|
|
41
|
+
// SingletonModelManager persists - just acknowledge
|
|
42
|
+
console.log('ℹ️ Worker: Singleton model persists');
|
|
47
43
|
parentPort.postMessage({ id, success: true });
|
|
48
44
|
break;
|
|
49
45
|
case 'restart':
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
* Custom error types for Brainy operations
|
|
3
3
|
* Provides better error classification and handling
|
|
4
4
|
*/
|
|
5
|
-
export type BrainyErrorType = 'TIMEOUT' | 'NETWORK' | 'STORAGE' | 'NOT_FOUND' | 'RETRY_EXHAUSTED';
|
|
5
|
+
export type BrainyErrorType = 'TIMEOUT' | 'NETWORK' | 'STORAGE' | 'NOT_FOUND' | 'RETRY_EXHAUSTED' | 'VALIDATION';
|
|
6
6
|
/**
|
|
7
7
|
* Custom error class for Brainy operations
|
|
8
8
|
* Provides error type classification and retry information
|
|
@@ -34,6 +34,10 @@ export declare class BrainyError extends Error {
|
|
|
34
34
|
* Create a retry exhausted error
|
|
35
35
|
*/
|
|
36
36
|
static retryExhausted(operation: string, maxRetries: number, lastError?: Error): BrainyError;
|
|
37
|
+
/**
|
|
38
|
+
* Create a validation error
|
|
39
|
+
*/
|
|
40
|
+
static validation(parameter: string, constraint: string, value?: any): BrainyError;
|
|
37
41
|
/**
|
|
38
42
|
* Check if an error is retryable
|
|
39
43
|
*/
|
|
@@ -50,6 +50,12 @@ export class BrainyError extends Error {
|
|
|
50
50
|
static retryExhausted(operation, maxRetries, lastError) {
|
|
51
51
|
return new BrainyError(`Operation '${operation}' failed after ${maxRetries} retry attempts`, 'RETRY_EXHAUSTED', false, lastError, maxRetries, maxRetries);
|
|
52
52
|
}
|
|
53
|
+
/**
|
|
54
|
+
* Create a validation error
|
|
55
|
+
*/
|
|
56
|
+
static validation(parameter, constraint, value) {
|
|
57
|
+
return new BrainyError(`Invalid ${parameter}: ${constraint}`, 'VALIDATION', false);
|
|
58
|
+
}
|
|
53
59
|
/**
|
|
54
60
|
* Check if an error is retryable
|
|
55
61
|
*/
|
|
@@ -106,6 +112,12 @@ export class BrainyError extends Error {
|
|
|
106
112
|
message.includes('does not exist')) {
|
|
107
113
|
return BrainyError.notFound(operation || 'resource');
|
|
108
114
|
}
|
|
115
|
+
if (message.includes('invalid') ||
|
|
116
|
+
message.includes('validation') ||
|
|
117
|
+
message.includes('cannot be null') ||
|
|
118
|
+
message.includes('must be')) {
|
|
119
|
+
return new BrainyError(error.message, 'VALIDATION', false, error);
|
|
120
|
+
}
|
|
109
121
|
// Default to storage error for unclassified errors
|
|
110
122
|
return BrainyError.storage(error.message, error);
|
|
111
123
|
}
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Basic usage example for the Soulcraft Brainy database
|
|
3
3
|
*/
|
|
4
|
-
import {
|
|
4
|
+
import { Brainy } from '../brainy.js';
|
|
5
5
|
// Example data - word embeddings
|
|
6
6
|
const wordEmbeddings = {
|
|
7
7
|
cat: [0.2, 0.3, 0.4, 0.1],
|
|
@@ -30,13 +30,16 @@ const metadata = {
|
|
|
30
30
|
async function runExample() {
|
|
31
31
|
console.log('Initializing vector database...');
|
|
32
32
|
// Create a new vector database
|
|
33
|
-
const db = new
|
|
33
|
+
const db = new Brainy();
|
|
34
34
|
await db.init();
|
|
35
35
|
console.log('Adding vectors to the database...');
|
|
36
36
|
// Add vectors to the database
|
|
37
37
|
const ids = {};
|
|
38
38
|
for (const [word, vector] of Object.entries(wordEmbeddings)) {
|
|
39
|
-
|
|
39
|
+
// Determine noun type based on the metadata
|
|
40
|
+
const meta = metadata[word];
|
|
41
|
+
const nounType = meta.type === 'mammal' || meta.type === 'bird' || meta.type === 'fish' ? 'Thing' : 'Content';
|
|
42
|
+
ids[word] = await db.addNoun(vector, nounType, meta);
|
|
40
43
|
console.log(`Added "${word}" with ID: ${ids[word]}`);
|
|
41
44
|
}
|
|
42
45
|
console.log('\nDatabase size:', db.size());
|
|
@@ -84,7 +87,7 @@ if (typeof window !== 'undefined') {
|
|
|
84
87
|
// Browser environment
|
|
85
88
|
document.addEventListener('DOMContentLoaded', () => {
|
|
86
89
|
const button = document.createElement('button');
|
|
87
|
-
button.textContent = 'Run
|
|
90
|
+
button.textContent = 'Run Brainy Example';
|
|
88
91
|
button.addEventListener('click', async () => {
|
|
89
92
|
const output = document.createElement('pre');
|
|
90
93
|
document.body.appendChild(output);
|