@soulcraft/brainy 0.9.11 → 0.9.13

package/dist/index.d.ts

@@ -8,7 +8,8 @@ export type { BrainyDataConfig };
 import { euclideanDistance, cosineDistance, manhattanDistance, dotProductDistance } from './utils/index.js';
 export { euclideanDistance, cosineDistance, manhattanDistance, dotProductDistance };
 import { UniversalSentenceEncoder, createEmbeddingFunction, createTensorFlowEmbeddingFunction, createThreadedEmbeddingFunction, defaultEmbeddingFunction } from './utils/embedding.js';
-export { UniversalSentenceEncoder, createEmbeddingFunction, createTensorFlowEmbeddingFunction, createThreadedEmbeddingFunction, defaultEmbeddingFunction };
+import { executeInThread, cleanupWorkerPools } from './utils/workerUtils.js';
+export { UniversalSentenceEncoder, createEmbeddingFunction, createTensorFlowEmbeddingFunction, createThreadedEmbeddingFunction, defaultEmbeddingFunction, executeInThread, cleanupWorkerPools };
 import { OPFSStorage, MemoryStorage, createStorage } from './storage/opfsStorage.js';
 import { FileSystemStorage } from './storage/fileSystemStorage.js';
 import { R2Storage, S3CompatibleStorage } from './storage/s3CompatibleStorage.js';

package/dist/unified.js

@@ -2670,33 +2670,213 @@ async function calculateDistancesBatch(queryVector, vectors, distanceFunction =
 }
 
 /**
- * Utility functions for executing functions (without Web Workers or Worker Threads)
- * This is a replacement for the original multithreaded implementation
+ * Utility functions for environment detection
+ */
+/**
+ * Check if code is running in a browser environment
+ */
+function isBrowser$1() {
+    return typeof window !== 'undefined' && typeof document !== 'undefined';
+}
+/**
+ * Check if code is running in a Node.js environment
+ */
+function isNode() {
+    return typeof process !== 'undefined' &&
+        process.versions != null &&
+        process.versions.node != null;
+}
+/**
+ * Check if Web Workers are available in the current environment
  */
+function areWebWorkersAvailable() {
+    return isBrowser$1() && typeof Worker !== 'undefined';
+}
+/**
+ * Check if Worker Threads are available in the current environment (Node.js)
+ */
+function areWorkerThreadsAvailable() {
+    if (!isNode())
+        return false;
+    try {
+        // Dynamic import to avoid errors in browser environments
+        require('worker_threads');
+        return true;
+    }
+    catch (e) {
+        return false;
+    }
+}
 /**
- * Execute a function directly in the main thread
+ * Determine if threading is available in the current environment
+ * Returns true if either Web Workers (browser) or Worker Threads (Node.js) are available
+ */
+function isThreadingAvailable() {
+    return areWebWorkersAvailable() || areWorkerThreadsAvailable();
+}
+
+/**
+ * Utility functions for executing functions in Worker Threads (Node.js) or Web Workers (Browser)
+ * This implementation leverages Node.js 24's improved Worker Threads API for better performance
+ */
+// Worker pool to reuse workers
+const workerPool = new Map();
+const MAX_POOL_SIZE = 4; // Adjust based on system capabilities
+/**
+ * Execute a function in a separate thread
  *
  * @param fnString The function to execute as a string
  * @param args The arguments to pass to the function
  * @returns A promise that resolves with the result of the function
  */
 function executeInThread(fnString, args) {
-    try {
-        // Parse the function from string and execute it
-        const fn = new Function('return ' + fnString)();
-        return Promise.resolve(fn(args));
+    if (isNode()) {
+        return executeInNodeWorker(fnString, args);
     }
-    catch (error) {
-        return Promise.reject(error);
+    else if (isBrowser$1() && typeof window !== 'undefined' && window.Worker) {
+        return executeInWebWorker(fnString, args);
+    }
+    else {
+        // Fallback to main thread execution
+        try {
+            const fn = new Function('return ' + fnString)();
+            return Promise.resolve(fn(args));
+        }
+        catch (error) {
+            return Promise.reject(error);
+        }
     }
 }
 /**
- * No-op function for backward compatibility
- * This function does nothing since there are no worker pools to clean up
+ * Execute a function in a Node.js Worker Thread
+ * Optimized for Node.js 24 with improved Worker Threads performance
+ */
+function executeInNodeWorker(fnString, args) {
+    return new Promise((resolve, reject) => {
+        try {
+            // Dynamically import worker_threads (Node.js only)
+            import('node:worker_threads').then(({ Worker, isMainThread, parentPort, workerData }) => {
+                if (!isMainThread && parentPort) {
+                    // We're inside a worker, execute the function
+                    const fn = new Function('return ' + workerData.fnString)();
+                    const result = fn(workerData.args);
+                    parentPort.postMessage({ result });
+                    return;
+                }
+                // Get a worker from the pool or create a new one
+                const workerId = `worker-${Math.random().toString(36).substring(2, 9)}`;
+                let worker;
+                if (workerPool.size < MAX_POOL_SIZE) {
+                    // Create a new worker
+                    worker = new Worker(`
+            import { parentPort, workerData } from 'node:worker_threads';
+            const fn = new Function('return ' + workerData.fnString)();
+            const result = fn(workerData.args);
+            parentPort.postMessage({ result });
+          `, {
+                        eval: true,
+                        workerData: { fnString, args }
+                    });
+                    workerPool.set(workerId, worker);
+                }
+                else {
+                    // Reuse an existing worker
+                    const poolKeys = Array.from(workerPool.keys());
+                    const randomKey = poolKeys[Math.floor(Math.random() * poolKeys.length)];
+                    worker = workerPool.get(randomKey);
+                    // Terminate and recreate if the worker is busy
+                    if (worker._busy) {
+                        worker.terminate();
+                        worker = new Worker(`
+              import { parentPort, workerData } from 'node:worker_threads';
+              const fn = new Function('return ' + workerData.fnString)();
+              const result = fn(workerData.args);
+              parentPort.postMessage({ result });
+            `, {
+                            eval: true,
+                            workerData: { fnString, args }
+                        });
+                        workerPool.set(randomKey, worker);
+                    }
+                    worker._busy = true;
+                }
+                worker.on('message', (message) => {
+                    worker._busy = false;
+                    resolve(message.result);
+                });
+                worker.on('error', (err) => {
+                    worker._busy = false;
+                    reject(err);
+                });
+                worker.on('exit', (code) => {
+                    if (code !== 0) {
+                        worker._busy = false;
+                        reject(new Error(`Worker stopped with exit code ${code}`));
+                    }
+                });
+            }).catch(reject);
+        }
+        catch (error) {
+            reject(error);
+        }
+    });
+}
+/**
+ * Execute a function in a Web Worker (Browser environment)
+ */
+function executeInWebWorker(fnString, args) {
+    return new Promise((resolve, reject) => {
+        try {
+            const workerCode = `
+        self.onmessage = function(e) {
+          try {
+            const fn = new Function('return ' + e.data.fnString)();
+            const result = fn(e.data.args);
+            self.postMessage({ result: result });
+          } catch (error) {
+            self.postMessage({ error: error.message });
+          }
+        };
+      `;
+            const blob = new Blob([workerCode], { type: 'application/javascript' });
+            const url = URL.createObjectURL(blob);
+            const worker = new Worker(url);
+            worker.onmessage = function (e) {
+                if (e.data.error) {
+                    reject(new Error(e.data.error));
+                }
+                else {
+                    resolve(e.data.result);
+                }
+                worker.terminate();
+                URL.revokeObjectURL(url);
+            };
+            worker.onerror = function (e) {
+                reject(new Error(`Worker error: ${e.message}`));
+                worker.terminate();
+                URL.revokeObjectURL(url);
+            };
+            worker.postMessage({ fnString, args });
+        }
+        catch (error) {
+            reject(error);
+        }
+    });
+}
+/**
+ * Clean up all worker pools
+ * This should be called when the application is shutting down
  */
 function cleanupWorkerPools() {
-    // No-op function
-    console.log('Worker pools cleanup called (no-op in non-threaded version)');
+    if (isNode()) {
+        import('node:worker_threads').then(({ Worker }) => {
+            for (const worker of workerPool.values()) {
+                worker.terminate();
+            }
+            workerPool.clear();
+            console.log('Worker pools cleaned up');
+        }).catch(console.error);
+    }
 }
 
 /**
@@ -2965,7 +3145,8 @@ function findUSELoadFunction(sentenceEncoderModule) {
                 }
             }
             // Also check nested objects
-            else if (typeof sentenceEncoderModule[key] === 'object' && sentenceEncoderModule[key] !== null) {
+            else if (typeof sentenceEncoderModule[key] === 'object' &&
+                sentenceEncoderModule[key] !== null) {
                 for (const nestedKey in sentenceEncoderModule[key]) {
                     if (typeof sentenceEncoderModule[key][nestedKey] === 'function') {
                         const fnName = sentenceEncoderModule[key][nestedKey].name || nestedKey;
@@ -7499,7 +7680,7 @@ class BrainyData {
                 console.log('Retrying Universal Sentence Encoder initialization...');
                 try {
                     // Wait a moment before retrying
-                    await new Promise(resolve => setTimeout(resolve, 1000));
+                    await new Promise((resolve) => setTimeout(resolve, 1000));
                     // Try again with a different approach - use the non-threaded version
                     // This is a fallback in case the threaded version fails
                     const { createTensorFlowEmbeddingFunction } = await Promise.resolve().then(function () { return embedding; });
@@ -7762,19 +7943,25 @@ class BrainyData {
                     }
                 });
                 // Process vector items (already embedded)
-                const vectorPromises = vectorItems.map(item => this.add(item.vectorOrData, item.metadata, options));
+                const vectorPromises = vectorItems.map((item) => this.add(item.vectorOrData, item.metadata, options));
                 // Process text items in a single batch embedding operation
                 let textPromises = [];
                 if (textItems.length > 0) {
                     // Extract just the text for batch embedding
-                    const texts = textItems.map(item => item.text);
+                    const texts = textItems.map((item) => item.text);
                     // Perform batch embedding
                     const embeddings = await defaultBatchEmbeddingFunction(texts);
                     // Add each item with its embedding
-                    textPromises = textItems.map((item, i) => this.add(embeddings[i], item.metadata, { ...options, forceEmbed: false }));
+                    textPromises = textItems.map((item, i) => this.add(embeddings[i], item.metadata, {
+                        ...options,
+                        forceEmbed: false
+                    }));
                 }
                 // Combine all promises
-                const batchResults = await Promise.all([...vectorPromises, ...textPromises]);
+                const batchResults = await Promise.all([
+                    ...vectorPromises,
+                    ...textPromises
+                ]);
                 // Add the results to our ids array
                 ids.push(...batchResults);
             }
@@ -8712,8 +8899,10 @@ class BrainyData {
             let attempts = 0;
             const maxAttempts = 100; // Prevent infinite loop
             const delay = 50; // ms
-            while (this.isInitializing && !this.isInitialized && attempts < maxAttempts) {
-                await new Promise(resolve => setTimeout(resolve, delay));
+            while (this.isInitializing &&
+                !this.isInitialized &&
+                attempts < maxAttempts) {
+                await new Promise((resolve) => setTimeout(resolve, delay));
                 attempts++;
             }
             if (!this.isInitialized) {
@@ -10845,31 +11034,6 @@ var s3CompatibleStorage = /*#__PURE__*/Object.freeze({
 	S3CompatibleStorage: S3CompatibleStorage
 });
 
-/**
- * Utility functions for environment detection
- */
-/**
- * Check if code is running in a browser environment
- */
-function isBrowser$1() {
-    return typeof window !== 'undefined' && typeof document !== 'undefined';
-}
-/**
- * Check if code is running in a Node.js environment
- */
-function isNode() {
-    return typeof process !== 'undefined' &&
-        process.versions != null &&
-        process.versions.node != null;
-}
-/**
- * Determine if threading is available in the current environment
- * Always returns false since multithreading has been removed
- */
-function isThreadingAvailable() {
-    return false;
-}
-
 /**
  * Augmentation Registry
  *
@@ -87025,5 +87189,5 @@ var _child_processShim = /*#__PURE__*/Object.freeze({
 	__proto__: null
 });
 
-export { AugmentationType, BrainyData, BrainyMCPAdapter, BrainyMCPService, ExecutionMode$1 as ExecutionMode, FileSystemStorage, FileSystemStorageAugmentation, HNSWIndex, HNSWIndexOptimized, MCPAugmentationToolset, MCPRequestType, MCP_VERSION, MemoryStorage, MemoryStorageAugmentation, NounType, OPFSStorage, OPFSStorageAugmentation, Pipeline, S3CompatibleStorage as R2Storage, S3CompatibleStorage, SequentialPipeline, ServerSearchActivationAugmentation, ServerSearchConduitAugmentation, StreamlinedExecutionMode, UniversalSentenceEncoder$1 as UniversalSentenceEncoder, VerbType, WebRTCConduitAugmentation, WebSocketConduitAugmentation, addWebSocketSupport, augmentationPipeline$1 as augmentationPipeline, availableAugmentations, cosineDistance$1 as cosineDistance, createAugmentationRegistryPlugin, createAugmentationRegistryRollupPlugin, createConduitAugmentation, createEmbeddingFunction, createMemoryAugmentation, createPipeline, createSenseAugmentation, createServerSearchAugmentations, createStorage, createStreamingPipeline, createTensorFlowEmbeddingFunction, createThreadedEmbeddingFunction, defaultEmbeddingFunction, dotProductDistance, environment, euclideanDistance, executeAugmentation, executeByType, executeSingle, executeStreamlined, getAugmentationsByType, initializeAugmentationPipeline, loadAugmentationModule, loadAugmentationsFromModules, manhattanDistance, pipeline, processStaticData, processStreamingData, registerAugmentation, sequentialPipeline, setAugmentationEnabled };
+export { AugmentationType, BrainyData, BrainyMCPAdapter, BrainyMCPService, ExecutionMode$1 as ExecutionMode, FileSystemStorage, FileSystemStorageAugmentation, HNSWIndex, HNSWIndexOptimized, MCPAugmentationToolset, MCPRequestType, MCP_VERSION, MemoryStorage, MemoryStorageAugmentation, NounType, OPFSStorage, OPFSStorageAugmentation, Pipeline, S3CompatibleStorage as R2Storage, S3CompatibleStorage, SequentialPipeline, ServerSearchActivationAugmentation, ServerSearchConduitAugmentation, StreamlinedExecutionMode, UniversalSentenceEncoder$1 as UniversalSentenceEncoder, VerbType, WebRTCConduitAugmentation, WebSocketConduitAugmentation, addWebSocketSupport, augmentationPipeline$1 as augmentationPipeline, availableAugmentations, cleanupWorkerPools, cosineDistance$1 as cosineDistance, createAugmentationRegistryPlugin, createAugmentationRegistryRollupPlugin, createConduitAugmentation, createEmbeddingFunction, createMemoryAugmentation, createPipeline, createSenseAugmentation, createServerSearchAugmentations, createStorage, createStreamingPipeline, createTensorFlowEmbeddingFunction, createThreadedEmbeddingFunction, defaultEmbeddingFunction, dotProductDistance, environment, euclideanDistance, executeAugmentation, executeByType, executeInThread, executeSingle, executeStreamlined, getAugmentationsByType, initializeAugmentationPipeline, loadAugmentationModule, loadAugmentationsFromModules, manhattanDistance, pipeline, processStaticData, processStreamingData, registerAugmentation, sequentialPipeline, setAugmentationEnabled };
 //# sourceMappingURL=unified.js.map