@soulcraft/brainy 3.0.0 → 3.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (270) hide show
  1. package/CHANGELOG.md +53 -3
  2. package/README.md +427 -111
  3. package/bin/brainy.js +340 -62
  4. package/dist/api/ConfigAPI.d.ts +67 -0
  5. package/dist/api/ConfigAPI.js +166 -0
  6. package/dist/api/DataAPI.d.ts +123 -0
  7. package/dist/api/DataAPI.js +391 -0
  8. package/dist/api/SecurityAPI.d.ts +50 -0
  9. package/dist/api/SecurityAPI.js +139 -0
  10. package/dist/api/UniversalImportAPI.d.ts +134 -0
  11. package/dist/api/UniversalImportAPI.js +615 -0
  12. package/dist/augmentationManager.js +12 -7
  13. package/dist/augmentationPipeline.d.ts +0 -61
  14. package/dist/augmentationPipeline.js +0 -87
  15. package/dist/augmentationRegistry.d.ts +1 -1
  16. package/dist/augmentationRegistry.js +1 -1
  17. package/dist/augmentations/apiServerAugmentation.d.ts +27 -1
  18. package/dist/augmentations/apiServerAugmentation.js +290 -9
  19. package/dist/augmentations/auditLogAugmentation.d.ts +109 -0
  20. package/dist/augmentations/auditLogAugmentation.js +358 -0
  21. package/dist/augmentations/batchProcessingAugmentation.d.ts +3 -2
  22. package/dist/augmentations/batchProcessingAugmentation.js +123 -22
  23. package/dist/augmentations/brainyAugmentation.d.ts +142 -8
  24. package/dist/augmentations/brainyAugmentation.js +179 -2
  25. package/dist/augmentations/cacheAugmentation.d.ts +8 -5
  26. package/dist/augmentations/cacheAugmentation.js +116 -17
  27. package/dist/augmentations/conduitAugmentations.d.ts +2 -2
  28. package/dist/augmentations/conduitAugmentations.js +2 -2
  29. package/dist/augmentations/configResolver.d.ts +122 -0
  30. package/dist/augmentations/configResolver.js +440 -0
  31. package/dist/augmentations/connectionPoolAugmentation.d.ts +3 -1
  32. package/dist/augmentations/connectionPoolAugmentation.js +37 -12
  33. package/dist/augmentations/defaultAugmentations.d.ts +14 -10
  34. package/dist/augmentations/defaultAugmentations.js +16 -11
  35. package/dist/augmentations/discovery/catalogDiscovery.d.ts +142 -0
  36. package/dist/augmentations/discovery/catalogDiscovery.js +249 -0
  37. package/dist/augmentations/discovery/localDiscovery.d.ts +84 -0
  38. package/dist/augmentations/discovery/localDiscovery.js +246 -0
  39. package/dist/augmentations/discovery/runtimeLoader.d.ts +97 -0
  40. package/dist/augmentations/discovery/runtimeLoader.js +337 -0
  41. package/dist/augmentations/discovery.d.ts +152 -0
  42. package/dist/augmentations/discovery.js +441 -0
  43. package/dist/augmentations/display/cache.d.ts +130 -0
  44. package/dist/augmentations/display/cache.js +319 -0
  45. package/dist/augmentations/display/fieldPatterns.d.ts +52 -0
  46. package/dist/augmentations/display/fieldPatterns.js +393 -0
  47. package/dist/augmentations/display/iconMappings.d.ts +57 -0
  48. package/dist/augmentations/display/iconMappings.js +68 -0
  49. package/dist/augmentations/display/intelligentComputation.d.ts +109 -0
  50. package/dist/augmentations/display/intelligentComputation.js +462 -0
  51. package/dist/augmentations/display/types.d.ts +203 -0
  52. package/dist/augmentations/display/types.js +7 -0
  53. package/dist/augmentations/entityRegistryAugmentation.d.ts +3 -1
  54. package/dist/augmentations/entityRegistryAugmentation.js +5 -1
  55. package/dist/augmentations/indexAugmentation.d.ts +5 -3
  56. package/dist/augmentations/indexAugmentation.js +5 -2
  57. package/dist/augmentations/intelligentVerbScoringAugmentation.d.ts +24 -7
  58. package/dist/augmentations/intelligentVerbScoringAugmentation.js +111 -27
  59. package/dist/augmentations/manifest.d.ts +176 -0
  60. package/dist/augmentations/manifest.js +8 -0
  61. package/dist/augmentations/marketplace/AugmentationMarketplace.d.ts +168 -0
  62. package/dist/augmentations/marketplace/AugmentationMarketplace.js +329 -0
  63. package/dist/augmentations/marketplace/cli.d.ts +47 -0
  64. package/dist/augmentations/marketplace/cli.js +265 -0
  65. package/dist/augmentations/metricsAugmentation.d.ts +3 -3
  66. package/dist/augmentations/metricsAugmentation.js +2 -2
  67. package/dist/augmentations/monitoringAugmentation.d.ts +3 -3
  68. package/dist/augmentations/monitoringAugmentation.js +2 -2
  69. package/dist/augmentations/neuralImport.d.ts +1 -1
  70. package/dist/augmentations/neuralImport.js +4 -4
  71. package/dist/augmentations/rateLimitAugmentation.d.ts +82 -0
  72. package/dist/augmentations/rateLimitAugmentation.js +321 -0
  73. package/dist/augmentations/requestDeduplicatorAugmentation.d.ts +2 -2
  74. package/dist/augmentations/requestDeduplicatorAugmentation.js +1 -1
  75. package/dist/augmentations/storageAugmentation.d.ts +1 -1
  76. package/dist/augmentations/storageAugmentation.js +2 -2
  77. package/dist/augmentations/storageAugmentations.d.ts +37 -8
  78. package/dist/augmentations/storageAugmentations.js +204 -15
  79. package/dist/augmentations/synapseAugmentation.d.ts +1 -1
  80. package/dist/augmentations/synapseAugmentation.js +35 -16
  81. package/dist/augmentations/typeMatching/brainyTypes.d.ts +83 -0
  82. package/dist/augmentations/typeMatching/brainyTypes.js +425 -0
  83. package/dist/augmentations/typeMatching/intelligentTypeMatcher.d.ts +39 -59
  84. package/dist/augmentations/typeMatching/intelligentTypeMatcher.js +103 -389
  85. package/dist/augmentations/universalDisplayAugmentation.d.ts +191 -0
  86. package/dist/augmentations/universalDisplayAugmentation.js +371 -0
  87. package/dist/brainy-unified.d.ts +106 -0
  88. package/dist/brainy-unified.js +327 -0
  89. package/dist/brainy.d.ts +277 -0
  90. package/dist/brainy.js +1241 -0
  91. package/dist/brainyData.d.ts +56 -111
  92. package/dist/brainyData.js +912 -756
  93. package/dist/brainyDataV3.d.ts +186 -0
  94. package/dist/brainyDataV3.js +337 -0
  95. package/dist/config/distributedPresets-new.d.ts +118 -0
  96. package/dist/config/distributedPresets-new.js +318 -0
  97. package/dist/config/distributedPresets.d.ts +118 -0
  98. package/dist/config/distributedPresets.js +318 -0
  99. package/dist/config/extensibleConfig.d.ts +99 -0
  100. package/dist/config/extensibleConfig.js +268 -0
  101. package/dist/config/index.d.ts +17 -0
  102. package/dist/config/index.js +35 -0
  103. package/dist/config/modelAutoConfig.d.ts +32 -0
  104. package/dist/config/modelAutoConfig.js +139 -0
  105. package/dist/config/modelPrecisionManager.d.ts +42 -0
  106. package/dist/config/modelPrecisionManager.js +98 -0
  107. package/dist/config/sharedConfigManager.d.ts +67 -0
  108. package/dist/config/sharedConfigManager.js +215 -0
  109. package/dist/config/storageAutoConfig.d.ts +41 -0
  110. package/dist/config/storageAutoConfig.js +328 -0
  111. package/dist/config/zeroConfig.d.ts +68 -0
  112. package/dist/config/zeroConfig.js +301 -0
  113. package/dist/cortex/backupRestore.d.ts +2 -2
  114. package/dist/cortex/backupRestore.js +85 -27
  115. package/dist/cortex/healthCheck.d.ts +2 -2
  116. package/dist/cortex/neuralImport.d.ts +2 -2
  117. package/dist/cortex/neuralImport.js +18 -13
  118. package/dist/cortex/performanceMonitor.d.ts +2 -2
  119. package/dist/critical/model-guardian.d.ts +4 -0
  120. package/dist/critical/model-guardian.js +31 -11
  121. package/dist/demo.d.ts +4 -4
  122. package/dist/demo.js +7 -7
  123. package/dist/distributed/cacheSync.d.ts +112 -0
  124. package/dist/distributed/cacheSync.js +265 -0
  125. package/dist/distributed/coordinator.d.ts +193 -0
  126. package/dist/distributed/coordinator.js +548 -0
  127. package/dist/distributed/httpTransport.d.ts +120 -0
  128. package/dist/distributed/httpTransport.js +446 -0
  129. package/dist/distributed/index.d.ts +8 -0
  130. package/dist/distributed/index.js +5 -0
  131. package/dist/distributed/networkTransport.d.ts +132 -0
  132. package/dist/distributed/networkTransport.js +633 -0
  133. package/dist/distributed/queryPlanner.d.ts +104 -0
  134. package/dist/distributed/queryPlanner.js +327 -0
  135. package/dist/distributed/readWriteSeparation.d.ts +134 -0
  136. package/dist/distributed/readWriteSeparation.js +350 -0
  137. package/dist/distributed/shardManager.d.ts +114 -0
  138. package/dist/distributed/shardManager.js +357 -0
  139. package/dist/distributed/shardMigration.d.ts +110 -0
  140. package/dist/distributed/shardMigration.js +289 -0
  141. package/dist/distributed/storageDiscovery.d.ts +160 -0
  142. package/dist/distributed/storageDiscovery.js +551 -0
  143. package/dist/embeddings/CachedEmbeddings.d.ts +40 -0
  144. package/dist/embeddings/CachedEmbeddings.js +146 -0
  145. package/dist/embeddings/EmbeddingManager.d.ts +102 -0
  146. package/dist/embeddings/EmbeddingManager.js +291 -0
  147. package/dist/embeddings/SingletonModelManager.d.ts +95 -0
  148. package/dist/embeddings/SingletonModelManager.js +220 -0
  149. package/dist/embeddings/index.d.ts +12 -0
  150. package/dist/embeddings/index.js +16 -0
  151. package/dist/embeddings/lightweight-embedder.d.ts +0 -1
  152. package/dist/embeddings/lightweight-embedder.js +4 -12
  153. package/dist/embeddings/model-manager.d.ts +11 -0
  154. package/dist/embeddings/model-manager.js +43 -7
  155. package/dist/embeddings/universal-memory-manager.d.ts +1 -1
  156. package/dist/embeddings/universal-memory-manager.js +27 -67
  157. package/dist/embeddings/worker-embedding.js +4 -8
  158. package/dist/errors/brainyError.d.ts +5 -1
  159. package/dist/errors/brainyError.js +12 -0
  160. package/dist/examples/basicUsage.js +7 -4
  161. package/dist/graph/graphAdjacencyIndex.d.ts +96 -0
  162. package/dist/graph/graphAdjacencyIndex.js +288 -0
  163. package/dist/graph/pathfinding.js +4 -2
  164. package/dist/hnsw/scaledHNSWSystem.js +11 -2
  165. package/dist/importManager.js +8 -5
  166. package/dist/index.d.ts +17 -22
  167. package/dist/index.js +37 -23
  168. package/dist/mcp/brainyMCPAdapter.d.ts +4 -4
  169. package/dist/mcp/brainyMCPAdapter.js +5 -5
  170. package/dist/mcp/brainyMCPService.d.ts +3 -3
  171. package/dist/mcp/brainyMCPService.js +3 -11
  172. package/dist/mcp/mcpAugmentationToolset.js +20 -30
  173. package/dist/neural/embeddedPatterns.d.ts +1 -1
  174. package/dist/neural/embeddedPatterns.js +2 -2
  175. package/dist/neural/entityExtractor.d.ts +65 -0
  176. package/dist/neural/entityExtractor.js +316 -0
  177. package/dist/neural/improvedNeuralAPI.d.ts +357 -0
  178. package/dist/neural/improvedNeuralAPI.js +2628 -0
  179. package/dist/neural/naturalLanguageProcessor.d.ts +155 -10
  180. package/dist/neural/naturalLanguageProcessor.js +941 -66
  181. package/dist/neural/naturalLanguageProcessorStatic.d.ts +2 -2
  182. package/dist/neural/naturalLanguageProcessorStatic.js +3 -3
  183. package/dist/neural/neuralAPI.js +8 -2
  184. package/dist/neural/patternLibrary.d.ts +57 -3
  185. package/dist/neural/patternLibrary.js +348 -13
  186. package/dist/neural/staticPatternMatcher.d.ts +2 -2
  187. package/dist/neural/staticPatternMatcher.js +2 -2
  188. package/dist/neural/types.d.ts +287 -0
  189. package/dist/neural/types.js +24 -0
  190. package/dist/shared/default-augmentations.d.ts +3 -3
  191. package/dist/shared/default-augmentations.js +5 -5
  192. package/dist/storage/adapters/baseStorageAdapter.d.ts +42 -0
  193. package/dist/storage/adapters/fileSystemStorage.d.ts +26 -2
  194. package/dist/storage/adapters/fileSystemStorage.js +218 -15
  195. package/dist/storage/adapters/memoryStorage.d.ts +4 -4
  196. package/dist/storage/adapters/memoryStorage.js +17 -12
  197. package/dist/storage/adapters/opfsStorage.d.ts +2 -2
  198. package/dist/storage/adapters/opfsStorage.js +2 -2
  199. package/dist/storage/adapters/s3CompatibleStorage.d.ts +2 -2
  200. package/dist/storage/adapters/s3CompatibleStorage.js +2 -2
  201. package/dist/storage/backwardCompatibility.d.ts +10 -78
  202. package/dist/storage/backwardCompatibility.js +17 -132
  203. package/dist/storage/baseStorage.d.ts +18 -2
  204. package/dist/storage/baseStorage.js +74 -3
  205. package/dist/storage/cacheManager.js +2 -2
  206. package/dist/storage/readOnlyOptimizations.js +8 -3
  207. package/dist/streaming/pipeline.d.ts +154 -0
  208. package/dist/streaming/pipeline.js +551 -0
  209. package/dist/triple/TripleIntelligence.d.ts +25 -110
  210. package/dist/triple/TripleIntelligence.js +4 -574
  211. package/dist/triple/TripleIntelligenceSystem.d.ts +159 -0
  212. package/dist/triple/TripleIntelligenceSystem.js +519 -0
  213. package/dist/types/apiTypes.d.ts +278 -0
  214. package/dist/types/apiTypes.js +33 -0
  215. package/dist/types/brainy.types.d.ts +308 -0
  216. package/dist/types/brainy.types.js +8 -0
  217. package/dist/types/brainyDataInterface.d.ts +5 -8
  218. package/dist/types/brainyDataInterface.js +2 -2
  219. package/dist/types/graphTypes.js +2 -2
  220. package/dist/universal/crypto.d.ts +11 -1
  221. package/dist/universal/crypto.js +24 -93
  222. package/dist/universal/events.d.ts +3 -2
  223. package/dist/universal/events.js +6 -75
  224. package/dist/universal/fs.d.ts +2 -3
  225. package/dist/universal/fs.js +5 -211
  226. package/dist/universal/path.d.ts +3 -2
  227. package/dist/universal/path.js +22 -78
  228. package/dist/universal/uuid.d.ts +1 -1
  229. package/dist/universal/uuid.js +1 -1
  230. package/dist/utils/brainyTypes.d.ts +217 -0
  231. package/dist/utils/brainyTypes.js +261 -0
  232. package/dist/utils/cacheAutoConfig.d.ts +3 -3
  233. package/dist/utils/embedding.d.ts +9 -4
  234. package/dist/utils/embedding.js +89 -26
  235. package/dist/utils/enhancedLogger.d.ts +104 -0
  236. package/dist/utils/enhancedLogger.js +232 -0
  237. package/dist/utils/hybridModelManager.d.ts +19 -28
  238. package/dist/utils/hybridModelManager.js +36 -200
  239. package/dist/utils/index.d.ts +1 -1
  240. package/dist/utils/index.js +1 -1
  241. package/dist/utils/intelligentTypeMapper.d.ts +60 -0
  242. package/dist/utils/intelligentTypeMapper.js +349 -0
  243. package/dist/utils/metadataIndex.d.ts +118 -1
  244. package/dist/utils/metadataIndex.js +539 -16
  245. package/dist/utils/nodeVersionCheck.d.ts +24 -0
  246. package/dist/utils/nodeVersionCheck.js +65 -0
  247. package/dist/utils/paramValidation.d.ts +39 -0
  248. package/dist/utils/paramValidation.js +192 -0
  249. package/dist/utils/rateLimiter.d.ts +160 -0
  250. package/dist/utils/rateLimiter.js +271 -0
  251. package/dist/utils/statistics.d.ts +4 -4
  252. package/dist/utils/statistics.js +3 -3
  253. package/dist/utils/structuredLogger.d.ts +146 -0
  254. package/dist/utils/structuredLogger.js +394 -0
  255. package/dist/utils/textEncoding.js +2 -1
  256. package/dist/utils/typeValidation.d.ts +59 -0
  257. package/dist/utils/typeValidation.js +374 -0
  258. package/dist/utils/version.js +19 -3
  259. package/package.json +15 -17
  260. package/scripts/download-models.cjs +94 -20
  261. package/dist/augmentations/walAugmentation.d.ts +0 -109
  262. package/dist/augmentations/walAugmentation.js +0 -516
  263. package/dist/browserFramework.d.ts +0 -15
  264. package/dist/browserFramework.js +0 -31
  265. package/dist/browserFramework.minimal.d.ts +0 -14
  266. package/dist/browserFramework.minimal.js +0 -31
  267. package/dist/chat/BrainyChat.d.ts +0 -121
  268. package/dist/chat/BrainyChat.js +0 -396
  269. package/dist/chat/ChatCLI.d.ts +0 -61
  270. package/dist/chat/ChatCLI.js +0 -351
@@ -0,0 +1,220 @@
1
+ /**
2
+ * Singleton Model Manager - THE ONLY SOURCE OF EMBEDDING MODELS
3
+ *
4
+ * This is the SINGLE, UNIFIED model initialization system that ensures:
5
+ * - Only ONE model instance exists across the entire system
6
+ * - Precision is configured once and locked
7
+ * - All components share the same model
8
+ * - No possibility of mixed precisions
9
+ *
10
+ * CRITICAL: This manager is used by EVERYTHING:
11
+ * - Storage operations (add, update)
12
+ * - Search operations (search, find)
13
+ * - Public API (embed, cluster)
14
+ * - Neural API (all neural.* methods)
15
+ * - Internal operations (deduplication, indexing)
16
+ */
17
+ import { TransformerEmbedding } from '../utils/embedding.js';
18
+ import { getModelPrecision, lockModelPrecision } from '../config/modelPrecisionManager.js';
19
+ // Global state - ensures true singleton across entire process
20
+ let globalModelInstance = null;
21
+ let globalInitPromise = null;
22
+ let globalInitialized = false;
23
+ /**
24
+ * The ONE TRUE model manager
25
+ */
26
+ export class SingletonModelManager {
27
+ constructor() {
28
+ this.stats = {
29
+ initialized: false,
30
+ precision: 'unknown',
31
+ initCount: 0,
32
+ embedCount: 0,
33
+ lastUsed: null
34
+ };
35
+ // Private constructor enforces singleton
36
+ this.stats.precision = getModelPrecision();
37
+ console.log(`🔐 SingletonModelManager initialized with ${this.stats.precision.toUpperCase()} precision`);
38
+ }
39
+ /**
40
+ * Get the singleton instance
41
+ */
42
+ static getInstance() {
43
+ if (!SingletonModelManager.instance) {
44
+ SingletonModelManager.instance = new SingletonModelManager();
45
+ }
46
+ return SingletonModelManager.instance;
47
+ }
48
+ /**
49
+ * Get the model instance - creates if needed, reuses if exists
50
+ * This is THE ONLY way to get a model in the entire system
51
+ */
52
+ async getModel() {
53
+ // If already initialized, return immediately
54
+ if (globalModelInstance && globalInitialized) {
55
+ this.stats.lastUsed = new Date();
56
+ return globalModelInstance;
57
+ }
58
+ // If initialization is in progress, wait for it
59
+ if (globalInitPromise) {
60
+ console.log('⏳ Model initialization already in progress, waiting...');
61
+ return await globalInitPromise;
62
+ }
63
+ // Start initialization (only happens once ever)
64
+ globalInitPromise = this.initializeModel();
65
+ try {
66
+ const model = await globalInitPromise;
67
+ globalInitialized = true;
68
+ return model;
69
+ }
70
+ catch (error) {
71
+ // Reset on error to allow retry
72
+ globalInitPromise = null;
73
+ throw error;
74
+ }
75
+ }
76
+ /**
77
+ * Initialize the model - happens exactly once
78
+ */
79
+ async initializeModel() {
80
+ console.log('🚀 Initializing singleton model instance...');
81
+ // Get precision from central manager
82
+ const precision = getModelPrecision();
83
+ console.log(`📊 Using ${precision.toUpperCase()} precision (${precision === 'q8' ? '23MB, 99% accuracy' : '90MB, 100% accuracy'})`);
84
+ // Detect environment for optimal settings
85
+ const isNode = typeof process !== 'undefined' && process.versions?.node;
86
+ const isBrowser = typeof window !== 'undefined' && typeof document !== 'undefined';
87
+ const isServerless = typeof process !== 'undefined' && (process.env.VERCEL ||
88
+ process.env.NETLIFY ||
89
+ process.env.AWS_LAMBDA_FUNCTION_NAME ||
90
+ process.env.FUNCTIONS_WORKER_RUNTIME);
91
+ const isTest = globalThis.__BRAINY_TEST_ENV__ || process.env.NODE_ENV === 'test';
92
+ // Create optimized options based on environment
93
+ const options = {
94
+ precision: precision,
95
+ verbose: !isTest && !isServerless && !isBrowser,
96
+ device: 'cpu', // CPU is most compatible
97
+ localFilesOnly: process.env.BRAINY_ALLOW_REMOTE_MODELS === 'false',
98
+ model: 'Xenova/all-MiniLM-L6-v2'
99
+ };
100
+ try {
101
+ // Create the ONE model instance
102
+ globalModelInstance = new TransformerEmbedding(options);
103
+ // Initialize it
104
+ await globalModelInstance.init();
105
+ // CRITICAL: Lock the precision after successful initialization
106
+ // This prevents any future changes to precision
107
+ lockModelPrecision();
108
+ console.log('🔒 Model precision locked at:', precision.toUpperCase());
109
+ // Update stats
110
+ this.stats.initialized = true;
111
+ this.stats.initCount++;
112
+ this.stats.lastUsed = new Date();
113
+ // Log memory usage if available
114
+ if (isNode && process.memoryUsage) {
115
+ const usage = process.memoryUsage();
116
+ this.stats.memoryFootprint = Math.round(usage.heapUsed / 1024 / 1024);
117
+ console.log(`💾 Model loaded, memory usage: ${this.stats.memoryFootprint}MB`);
118
+ }
119
+ console.log('✅ Singleton model initialized successfully');
120
+ return globalModelInstance;
121
+ }
122
+ catch (error) {
123
+ console.error('❌ Failed to initialize singleton model:', error);
124
+ globalModelInstance = null;
125
+ throw new Error(`Singleton model initialization failed: ${error instanceof Error ? error.message : String(error)}`);
126
+ }
127
+ }
128
+ /**
129
+ * Get embedding function that uses the singleton model
130
+ */
131
+ async getEmbeddingFunction() {
132
+ const model = await this.getModel();
133
+ return async (data) => {
134
+ this.stats.embedCount++;
135
+ this.stats.lastUsed = new Date();
136
+ return await model.embed(data);
137
+ };
138
+ }
139
+ /**
140
+ * Direct embed method for convenience
141
+ */
142
+ async embed(data) {
143
+ const model = await this.getModel();
144
+ this.stats.embedCount++;
145
+ this.stats.lastUsed = new Date();
146
+ return await model.embed(data);
147
+ }
148
+ /**
149
+ * Check if model is initialized
150
+ */
151
+ isInitialized() {
152
+ return globalInitialized && globalModelInstance !== null;
153
+ }
154
+ /**
155
+ * Get current statistics
156
+ */
157
+ getStats() {
158
+ return {
159
+ ...this.stats,
160
+ precision: getModelPrecision()
161
+ };
162
+ }
163
+ /**
164
+ * Validate precision consistency
165
+ * Throws error if attempting to use different precision
166
+ */
167
+ validatePrecision(requestedPrecision) {
168
+ const currentPrecision = getModelPrecision();
169
+ if (requestedPrecision && requestedPrecision !== currentPrecision) {
170
+ throw new Error(`❌ Precision mismatch! System is using ${currentPrecision.toUpperCase()} ` +
171
+ `but ${requestedPrecision.toUpperCase()} was requested. ` +
172
+ `All operations must use the same precision.`);
173
+ }
174
+ }
175
+ /**
176
+ * Force cleanup (for testing only)
177
+ * WARNING: This will break consistency - use only in tests
178
+ */
179
+ async _testOnlyCleanup() {
180
+ if (process.env.NODE_ENV !== 'test') {
181
+ throw new Error('Cleanup only allowed in test environment');
182
+ }
183
+ if (globalModelInstance && 'dispose' in globalModelInstance) {
184
+ await globalModelInstance.dispose();
185
+ }
186
+ globalModelInstance = null;
187
+ globalInitPromise = null;
188
+ globalInitialized = false;
189
+ this.stats.initialized = false;
190
+ console.log('🧹 Singleton model cleaned up (test only)');
191
+ }
192
+ }
193
+ // Export the singleton instance getter
194
+ export const singletonModelManager = SingletonModelManager.getInstance();
195
+ /**
196
+ * THE ONLY embedding function that should be used anywhere
197
+ * This ensures all operations use the same model instance
198
+ */
199
+ export async function getUnifiedEmbeddingFunction() {
200
+ return await singletonModelManager.getEmbeddingFunction();
201
+ }
202
+ /**
203
+ * Direct embed function for convenience
204
+ */
205
+ export async function unifiedEmbed(data) {
206
+ return await singletonModelManager.embed(data);
207
+ }
208
+ /**
209
+ * Check if model is ready
210
+ */
211
+ export function isModelReady() {
212
+ return singletonModelManager.isInitialized();
213
+ }
214
+ /**
215
+ * Get model statistics
216
+ */
217
+ export function getModelStats() {
218
+ return singletonModelManager.getStats();
219
+ }
220
+ //# sourceMappingURL=SingletonModelManager.js.map
@@ -0,0 +1,12 @@
1
+ /**
2
+ * Embeddings Module - Clean, Unified Architecture
3
+ *
4
+ * This module provides all embedding functionality for Brainy.
5
+ *
6
+ * Main Components:
7
+ * - EmbeddingManager: Core embedding generation with Q8/FP32 support
8
+ * - CachedEmbeddings: Performance optimization layer with pre-computed embeddings
9
+ */
10
+ export { EmbeddingManager, embeddingManager, embed, getEmbeddingFunction, getEmbeddingStats, type ModelPrecision } from './EmbeddingManager.js';
11
+ export { CachedEmbeddings, cachedEmbeddings } from './CachedEmbeddings.js';
12
+ export { embeddingManager as default } from './EmbeddingManager.js';
@@ -0,0 +1,16 @@
1
+ /**
2
+ * Embeddings Module - Clean, Unified Architecture
3
+ *
4
+ * This module provides all embedding functionality for Brainy.
5
+ *
6
+ * Main Components:
7
+ * - EmbeddingManager: Core embedding generation with Q8/FP32 support
8
+ * - CachedEmbeddings: Performance optimization layer with pre-computed embeddings
9
+ */
10
+ // Core embedding functionality
11
+ export { EmbeddingManager, embeddingManager, embed, getEmbeddingFunction, getEmbeddingStats } from './EmbeddingManager.js';
12
+ // Cached embeddings for performance
13
+ export { CachedEmbeddings, cachedEmbeddings } from './CachedEmbeddings.js';
14
+ // Default export is the singleton manager
15
+ export { embeddingManager as default } from './EmbeddingManager.js';
16
+ //# sourceMappingURL=index.js.map
@@ -8,7 +8,6 @@
8
8
  */
9
9
  import { Vector } from '../coreTypes.js';
10
10
  export declare class LightweightEmbedder {
11
- private onnxEmbedder;
12
11
  private stats;
13
12
  embed(text: string | string[]): Promise<Vector | Vector[]>;
14
13
  private embedSingle;
@@ -6,6 +6,7 @@
6
6
  *
7
7
  * This reduces memory usage by 90% for typical queries
8
8
  */
9
+ import { singletonModelManager } from './SingletonModelManager.js';
9
10
  // Pre-computed embeddings for top 10,000 common terms
10
11
  // In production, this would be loaded from a file
11
12
  const PRECOMPUTED_EMBEDDINGS = {
@@ -59,7 +60,6 @@ function computeSimpleEmbedding(text) {
59
60
  }
60
61
  export class LightweightEmbedder {
61
62
  constructor() {
62
- this.onnxEmbedder = null;
63
63
  this.stats = {
64
64
  precomputedHits: 0,
65
65
  simpleComputes: 0,
@@ -92,18 +92,10 @@ export class LightweightEmbedder {
92
92
  this.stats.simpleComputes++;
93
93
  return computeSimpleEmbedding(normalized);
94
94
  }
95
- // 4. Last resort: Load ONNX model (only if really needed)
96
- if (!this.onnxEmbedder) {
97
- console.log('⚠️ Loading ONNX model for complex text...');
98
- const { TransformerEmbedding } = await import('../utils/embedding.js');
99
- this.onnxEmbedder = new TransformerEmbedding({
100
- dtype: 'q8',
101
- verbose: false
102
- });
103
- await this.onnxEmbedder.init();
104
- }
95
+ // 4. Last resort: Use SingletonModelManager for complex text
96
+ console.log('⚠️ Using singleton model for complex text...');
105
97
  this.stats.onnxComputes++;
106
- return await this.onnxEmbedder.embed(text);
98
+ return await singletonModelManager.embed(text);
107
99
  }
108
100
  getStats() {
109
101
  return {
@@ -18,6 +18,17 @@ export declare class ModelManager {
18
18
  private getModelsPath;
19
19
  ensureModels(modelName?: string): Promise<boolean>;
20
20
  private verifyModelFiles;
21
+ /**
22
+ * Check which model variants are available locally
23
+ */
24
+ getAvailableModels(modelName?: string): {
25
+ fp32: boolean;
26
+ q8: boolean;
27
+ };
28
+ /**
29
+ * Get the best available model variant based on preference and availability
30
+ */
31
+ getBestAvailableModel(preferredType?: 'fp32' | 'q8', modelName?: string): 'fp32' | 'q8' | null;
21
32
  private tryModelSource;
22
33
  private downloadAndExtractFromGitHub;
23
34
  /**
@@ -31,13 +31,16 @@ const MODEL_SOURCES = {
31
31
  pathTemplate: '{model}/resolve/{revision}/' // Default transformers.js pattern
32
32
  }
33
33
  };
34
- // Model verification files - minimal set needed for transformers.js
35
- const MODEL_FILES = [
34
+ // Model verification files - BOTH fp32 and q8 variants
35
+ const REQUIRED_FILES = [
36
36
  'config.json',
37
37
  'tokenizer.json',
38
- 'tokenizer_config.json',
39
- 'onnx/model.onnx'
38
+ 'tokenizer_config.json'
40
39
  ];
40
+ const MODEL_VARIANTS = {
41
+ fp32: 'onnx/model.onnx',
42
+ q8: 'onnx/model_quantized.onnx'
43
+ };
41
44
  export class ModelManager {
42
45
  constructor() {
43
46
  this.isInitialized = false;
@@ -105,14 +108,47 @@ export class ModelManager {
105
108
  return true;
106
109
  }
107
110
  async verifyModelFiles(modelPath) {
108
- // Check if essential model files exist
109
- for (const file of MODEL_FILES) {
111
+ // Check if essential files exist
112
+ for (const file of REQUIRED_FILES) {
110
113
  const fullPath = join(modelPath, file);
111
114
  if (!existsSync(fullPath)) {
112
115
  return false;
113
116
  }
114
117
  }
115
- return true;
118
+ // At least one model variant must exist (fp32 or q8)
119
+ const fp32Exists = existsSync(join(modelPath, MODEL_VARIANTS.fp32));
120
+ const q8Exists = existsSync(join(modelPath, MODEL_VARIANTS.q8));
121
+ return fp32Exists || q8Exists;
122
+ }
123
+ /**
124
+ * Check which model variants are available locally
125
+ */
126
+ getAvailableModels(modelName = 'Xenova/all-MiniLM-L6-v2') {
127
+ const modelPath = join(this.modelsPath, modelName);
128
+ return {
129
+ fp32: existsSync(join(modelPath, MODEL_VARIANTS.fp32)),
130
+ q8: existsSync(join(modelPath, MODEL_VARIANTS.q8))
131
+ };
132
+ }
133
+ /**
134
+ * Get the best available model variant based on preference and availability
135
+ */
136
+ getBestAvailableModel(preferredType = 'fp32', modelName = 'Xenova/all-MiniLM-L6-v2') {
137
+ const available = this.getAvailableModels(modelName);
138
+ // If preferred type is available, use it
139
+ if (available[preferredType]) {
140
+ return preferredType;
141
+ }
142
+ // Otherwise fall back to what's available
143
+ if (preferredType === 'q8' && available.fp32) {
144
+ console.warn('⚠️ Q8 model requested but not available, falling back to FP32');
145
+ return 'fp32';
146
+ }
147
+ if (preferredType === 'fp32' && available.q8) {
148
+ console.warn('⚠️ FP32 model requested but not available, falling back to Q8');
149
+ return 'q8';
150
+ }
151
+ return null;
116
152
  }
117
153
  async tryModelSource(name, source, modelName) {
118
154
  try {
@@ -23,7 +23,7 @@ export declare class UniversalMemoryManager {
23
23
  embed(data: string | string[]): Promise<Vector>;
24
24
  private checkMemoryLimits;
25
25
  private ensureEmbeddingFunction;
26
- private initNodeWorker;
26
+ private initNodeDirect;
27
27
  private initServerless;
28
28
  private initBrowser;
29
29
  private initFallback;
@@ -4,6 +4,7 @@
4
4
  * Works in ALL environments: Node.js, browsers, serverless, workers
5
5
  * Solves transformers.js memory leak with environment-specific strategies
6
6
  */
7
+ import { getModelPrecision } from '../config/modelPrecisionManager.js';
7
8
  // Environment detection
8
9
  const isNode = typeof process !== 'undefined' && process.versions?.node;
9
10
  const isBrowser = typeof window !== 'undefined' && typeof document !== 'undefined';
@@ -13,18 +14,21 @@ const isServerless = typeof process !== 'undefined' && (process.env.VERCEL ||
13
14
  process.env.FUNCTIONS_WORKER_RUNTIME);
14
15
  export class UniversalMemoryManager {
15
16
  constructor() {
17
+ // CRITICAL FIX: Never use worker threads with ONNX Runtime
18
+ // Worker threads cause HandleScope V8 API errors due to isolate issues
19
+ // Always use direct embedding on main thread for ONNX compatibility
16
20
  this.embeddingFunction = null;
17
21
  this.embedCount = 0;
18
22
  this.restartCount = 0;
19
23
  this.lastRestart = 0;
20
- // Choose strategy based on environment
21
24
  if (isServerless) {
22
25
  this.strategy = 'serverless-restart';
23
26
  this.maxEmbeddings = 50; // Restart frequently in serverless
24
27
  }
25
28
  else if (isNode && !isBrowser) {
26
- this.strategy = 'node-worker';
27
- this.maxEmbeddings = 100; // Worker can handle more
29
+ // CHANGED: Use direct strategy instead of node-worker to avoid V8 isolate issues
30
+ this.strategy = 'node-direct';
31
+ this.maxEmbeddings = 200; // Main thread can handle more with single model instance
28
32
  }
29
33
  else if (isBrowser) {
30
34
  this.strategy = 'browser-dispose';
@@ -35,6 +39,7 @@ export class UniversalMemoryManager {
35
39
  this.maxEmbeddings = 75;
36
40
  }
37
41
  console.log(`🧠 Universal Memory Manager: Using ${this.strategy} strategy`);
42
+ console.log('✅ UNIVERSAL: Memory-safe embedding system initialized');
38
43
  }
39
44
  async getEmbeddingFunction() {
40
45
  return async (data) => {
@@ -62,8 +67,8 @@ export class UniversalMemoryManager {
62
67
  return;
63
68
  }
64
69
  switch (this.strategy) {
65
- case 'node-worker':
66
- await this.initNodeWorker();
70
+ case 'node-direct':
71
+ await this.initNodeDirect();
67
72
  break;
68
73
  case 'serverless-restart':
69
74
  await this.initServerless();
@@ -75,19 +80,12 @@ export class UniversalMemoryManager {
75
80
  await this.initFallback();
76
81
  }
77
82
  }
78
- async initNodeWorker() {
83
+ async initNodeDirect() {
79
84
  if (isNode) {
80
- try {
81
- // Try to use worker threads if available
82
- const { workerEmbeddingManager } = await import('./worker-manager.js');
83
- this.embeddingFunction = workerEmbeddingManager;
84
- console.log('✅ Using Node.js worker threads for embeddings');
85
- }
86
- catch (error) {
87
- console.warn('⚠️ Worker threads not available, falling back to direct embedding');
88
- console.warn('Error:', error instanceof Error ? error.message : String(error));
89
- await this.initDirect();
90
- }
85
+ // CRITICAL: Use direct embedding to avoid worker thread V8 isolate issues
86
+ // This prevents HandleScope errors and ensures single model instance
87
+ console.log('✅ Using Node.js direct embedding (main thread - ONNX compatible)');
88
+ await this.initDirect();
91
89
  }
92
90
  }
93
91
  async initServerless() {
@@ -110,7 +108,7 @@ export class UniversalMemoryManager {
110
108
  const { TransformerEmbedding } = await import('../utils/embedding.js');
111
109
  this.embeddingFunction = new TransformerEmbedding({
112
110
  verbose: false,
113
- dtype: 'q8',
111
+ precision: getModelPrecision(), // Use centrally managed precision
114
112
  localFilesOnly: process.env.BRAINY_ALLOW_REMOTE_MODELS !== 'true'
115
113
  });
116
114
  await this.embeddingFunction.init();
@@ -122,49 +120,15 @@ export class UniversalMemoryManager {
122
120
  }
123
121
  async cleanup() {
124
122
  const startTime = Date.now();
125
- try {
126
- // Strategy-specific cleanup
127
- switch (this.strategy) {
128
- case 'node-worker':
129
- if (this.embeddingFunction?.forceRestart) {
130
- await this.embeddingFunction.forceRestart();
131
- }
132
- break;
133
- case 'serverless-restart':
134
- // In serverless, create new instance
135
- if (this.embeddingFunction?.dispose) {
136
- this.embeddingFunction.dispose();
137
- }
138
- this.embeddingFunction = null;
139
- break;
140
- case 'browser-dispose':
141
- // In browser, try disposal
142
- if (this.embeddingFunction?.dispose) {
143
- this.embeddingFunction.dispose();
144
- }
145
- // Force garbage collection if available
146
- if (typeof window !== 'undefined' && window.gc) {
147
- window.gc();
148
- }
149
- break;
150
- default:
151
- // Fallback: dispose and recreate
152
- if (this.embeddingFunction?.dispose) {
153
- this.embeddingFunction.dispose();
154
- }
155
- this.embeddingFunction = null;
156
- }
157
- this.embedCount = 0;
158
- this.restartCount++;
159
- this.lastRestart = Date.now();
160
- const cleanupTime = Date.now() - startTime;
161
- console.log(`🧹 Memory cleanup completed in ${cleanupTime}ms (strategy: ${this.strategy})`);
162
- }
163
- catch (error) {
164
- console.warn('⚠️ Cleanup failed:', error instanceof Error ? error.message : String(error));
165
- // Force null assignment as last resort
166
- this.embeddingFunction = null;
167
- }
123
+ // SingletonModelManager persists - we just reset our counters
124
+ // The singleton model stays alive for consistency across all operations
125
+ // Reset counters
126
+ this.embedCount = 0;
127
+ this.restartCount++;
128
+ this.lastRestart = Date.now();
129
+ const cleanupTime = Date.now() - startTime;
130
+ console.log(`🧹 Memory counters reset in ${cleanupTime}ms (strategy: ${this.strategy})`);
131
+ console.log('ℹ️ Singleton model persists for consistency across all operations');
168
132
  }
169
133
  getMemoryStats() {
170
134
  let memoryUsage = 'unknown';
@@ -185,12 +149,8 @@ export class UniversalMemoryManager {
185
149
  };
186
150
  }
187
151
  async dispose() {
188
- if (this.embeddingFunction) {
189
- if (this.embeddingFunction.dispose) {
190
- await this.embeddingFunction.dispose();
191
- }
192
- this.embeddingFunction = null;
193
- }
152
+ // SingletonModelManager persists - nothing to dispose
153
+ console.log('ℹ️ Universal Memory Manager: Singleton model persists');
194
154
  }
195
155
  }
196
156
  // Export singleton instance
@@ -6,6 +6,7 @@
6
6
  */
7
7
  import { TransformerEmbedding } from '../utils/embedding.js';
8
8
  import { parentPort } from 'worker_threads';
9
+ import { getModelPrecision } from '../config/modelPrecisionManager.js';
9
10
  let model = null;
10
11
  let requestCount = 0;
11
12
  const MAX_REQUESTS = 100; // Restart worker after 100 requests to prevent memory leak
@@ -13,7 +14,7 @@ async function initModel() {
13
14
  if (!model) {
14
15
  model = new TransformerEmbedding({
15
16
  verbose: false,
16
- dtype: 'q8',
17
+ precision: getModelPrecision(), // Use centrally managed precision
17
18
  localFilesOnly: process.env.BRAINY_ALLOW_REMOTE_MODELS !== 'true'
18
19
  });
19
20
  await model.init();
@@ -37,13 +38,8 @@ if (parentPort) {
37
38
  }
38
39
  break;
39
40
  case 'dispose':
40
- if (model) {
41
- // This doesn't fully free memory (known issue), but try anyway
42
- if ('dispose' in model && typeof model.dispose === 'function') {
43
- model.dispose();
44
- }
45
- model = null;
46
- }
41
+ // SingletonModelManager persists - just acknowledge
42
+ console.log('ℹ️ Worker: Singleton model persists');
47
43
  parentPort.postMessage({ id, success: true });
48
44
  break;
49
45
  case 'restart':
@@ -2,7 +2,7 @@
2
2
  * Custom error types for Brainy operations
3
3
  * Provides better error classification and handling
4
4
  */
5
- export type BrainyErrorType = 'TIMEOUT' | 'NETWORK' | 'STORAGE' | 'NOT_FOUND' | 'RETRY_EXHAUSTED';
5
+ export type BrainyErrorType = 'TIMEOUT' | 'NETWORK' | 'STORAGE' | 'NOT_FOUND' | 'RETRY_EXHAUSTED' | 'VALIDATION';
6
6
  /**
7
7
  * Custom error class for Brainy operations
8
8
  * Provides error type classification and retry information
@@ -34,6 +34,10 @@ export declare class BrainyError extends Error {
34
34
  * Create a retry exhausted error
35
35
  */
36
36
  static retryExhausted(operation: string, maxRetries: number, lastError?: Error): BrainyError;
37
+ /**
38
+ * Create a validation error
39
+ */
40
+ static validation(parameter: string, constraint: string, value?: any): BrainyError;
37
41
  /**
38
42
  * Check if an error is retryable
39
43
  */
@@ -50,6 +50,12 @@ export class BrainyError extends Error {
50
50
  static retryExhausted(operation, maxRetries, lastError) {
51
51
  return new BrainyError(`Operation '${operation}' failed after ${maxRetries} retry attempts`, 'RETRY_EXHAUSTED', false, lastError, maxRetries, maxRetries);
52
52
  }
53
+ /**
54
+ * Create a validation error
55
+ */
56
+ static validation(parameter, constraint, value) {
57
+ return new BrainyError(`Invalid ${parameter}: ${constraint}`, 'VALIDATION', false);
58
+ }
53
59
  /**
54
60
  * Check if an error is retryable
55
61
  */
@@ -106,6 +112,12 @@ export class BrainyError extends Error {
106
112
  message.includes('does not exist')) {
107
113
  return BrainyError.notFound(operation || 'resource');
108
114
  }
115
+ if (message.includes('invalid') ||
116
+ message.includes('validation') ||
117
+ message.includes('cannot be null') ||
118
+ message.includes('must be')) {
119
+ return new BrainyError(error.message, 'VALIDATION', false, error);
120
+ }
109
121
  // Default to storage error for unclassified errors
110
122
  return BrainyError.storage(error.message, error);
111
123
  }
@@ -1,7 +1,7 @@
1
1
  /**
2
2
  * Basic usage example for the Soulcraft Brainy database
3
3
  */
4
- import { BrainyData } from '../brainyData.js';
4
+ import { Brainy } from '../brainy.js';
5
5
  // Example data - word embeddings
6
6
  const wordEmbeddings = {
7
7
  cat: [0.2, 0.3, 0.4, 0.1],
@@ -30,13 +30,16 @@ const metadata = {
30
30
  async function runExample() {
31
31
  console.log('Initializing vector database...');
32
32
  // Create a new vector database
33
- const db = new BrainyData();
33
+ const db = new Brainy();
34
34
  await db.init();
35
35
  console.log('Adding vectors to the database...');
36
36
  // Add vectors to the database
37
37
  const ids = {};
38
38
  for (const [word, vector] of Object.entries(wordEmbeddings)) {
39
- ids[word] = await db.addNoun(vector, metadata[word]);
39
+ // Determine noun type based on the metadata
40
+ const meta = metadata[word];
41
+ const nounType = meta.type === 'mammal' || meta.type === 'bird' || meta.type === 'fish' ? 'Thing' : 'Content';
42
+ ids[word] = await db.addNoun(vector, nounType, meta);
40
43
  console.log(`Added "${word}" with ID: ${ids[word]}`);
41
44
  }
42
45
  console.log('\nDatabase size:', db.size());
@@ -84,7 +87,7 @@ if (typeof window !== 'undefined') {
84
87
  // Browser environment
85
88
  document.addEventListener('DOMContentLoaded', () => {
86
89
  const button = document.createElement('button');
87
- button.textContent = 'Run BrainyData Example';
90
+ button.textContent = 'Run Brainy Example';
88
91
  button.addEventListener('click', async () => {
89
92
  const output = document.createElement('pre');
90
93
  document.body.appendChild(output);