@soulcraft/brainy 0.45.0 → 0.47.0

package/dist/utils/textEncoding.js

@@ -1,267 +1,33 @@
 import { isNode } from './environment.js';
+// Simplified TextEncoder/TextDecoder utilities for Node.js compatibility
+// No longer needs complex TensorFlow.js patches - only basic TextEncoder/TextDecoder
 /**
  * Flag to track if the patch has been applied
  */
 let patchApplied = false;
 /**
- * Monkeypatch TensorFlow.js's PlatformNode class to fix TextEncoder/TextDecoder issues
- * CRITICAL: This runs immediately at the top level when this module is imported
+ * Apply TextEncoder/TextDecoder patches for Node.js compatibility
+ * Simplified version for Transformers.js/ONNX Runtime
  */
-if (typeof globalThis !== 'undefined' && isNode()) {
+export async function applyTensorFlowPatch() {
+    // Apply patches for all non-browser environments that might need TextEncoder/TextDecoder
+    const isBrowserEnv = typeof window !== 'undefined' && typeof document !== 'undefined';
+    if (isBrowserEnv || patchApplied) {
+        return; // Browser environments don't need these patches, and don't patch twice
+    }
+    if (!isNode()) {
+        return; // Only patch Node.js environments
+    }
     try {
-        // Ensure TextEncoder/TextDecoder are globally available
-        if (typeof globalThis.TextEncoder === 'undefined') {
-            globalThis.TextEncoder = TextEncoder;
-        }
-        if (typeof globalThis.TextDecoder === 'undefined') {
-            globalThis.TextDecoder = TextDecoder;
-        }
-        // Patch global objects to handle the TensorFlow.js constructor issue
-        // This is needed because TF accesses TextEncoder/TextDecoder as constructors via this.util
-        if (typeof global !== 'undefined') {
-            if (!global.TextEncoder) {
-                global.TextEncoder = TextEncoder;
-            }
-            if (!global.TextDecoder) {
-                global.TextDecoder = TextDecoder;
-            }
-            // Also set the special global constructors that TensorFlow can use safely
-            global.__TextEncoder__ = TextEncoder;
-            global.__TextDecoder__ = TextDecoder;
-        }
-        // CRITICAL FIX: Create a custom util object that TensorFlow.js can use
-        // We'll make this available globally so TensorFlow.js can find it
-        const customUtil = {
-            TextEncoder: TextEncoder,
-            TextDecoder: TextDecoder,
-            types: {
-                isFloat32Array: (arr) => arr instanceof Float32Array,
-                isInt32Array: (arr) => arr instanceof Int32Array,
-                isUint8Array: (arr) => arr instanceof Uint8Array,
-                isUint8ClampedArray: (arr) => arr instanceof Uint8ClampedArray
-            }
-        };
-        // Make the custom util available globally
-        if (typeof global !== 'undefined') {
-            global.__brainy_util__ = customUtil;
-        }
-        // Try to patch the global require cache if possible
-        if (typeof global !== 'undefined' &&
-            global.require &&
-            global.require.cache) {
-            // Find the util module in the cache and patch it
-            for (const key in global.require.cache) {
-                if (key.endsWith('/util.js') || key === 'util') {
-                    const utilModule = global.require.cache[key];
-                    if (utilModule && utilModule.exports) {
-                        Object.assign(utilModule.exports, customUtil);
-                    }
-                }
-            }
-        }
-        // CRITICAL: Patch the Node.js util module directly
-        try {
-            const util = require('util');
-            // Ensure TextEncoder and TextDecoder are available as constructors
-            util.TextEncoder = TextEncoder;
-            util.TextDecoder = TextDecoder;
-        }
-        catch (error) {
-            // Ignore if util module is not available
-        }
-        // CRITICAL: Patch Float32Array to handle buffer alignment issues
-        // This fixes the "byte length of Float32Array should be a multiple of 4" error
-        // Get the appropriate global object for the current environment
+        console.log('Brainy: Applying TextEncoder/TextDecoder patch for Node.js');
+        // Get the appropriate global object
         const globalObj = (() => {
             if (typeof globalThis !== 'undefined')
                 return globalThis;
             if (typeof global !== 'undefined')
                 return global;
-            if (typeof self !== 'undefined')
-                return self;
-            if (typeof window !== 'undefined')
-                return window;
-            return {}; // Fallback for unknown environments
+            return {};
         })();
-        if (globalObj && globalObj.Float32Array) {
-            const originalFloat32Array = globalObj.Float32Array;
-            // Create a patched Float32Array class that handles alignment issues
-            const PatchedFloat32Array = class extends originalFloat32Array {
-                constructor(arg, byteOffset, length) {
-                    if (arg instanceof ArrayBuffer) {
-                        // Ensure buffer is properly aligned for Float32Array (multiple of 4 bytes)
-                        const alignedByteOffset = byteOffset || 0;
-                        const alignedLength = length !== undefined
-                            ? length
-                            : (arg.byteLength - alignedByteOffset) / 4;
-                        // Check if the buffer slice is properly aligned
-                        if ((arg.byteLength - alignedByteOffset) % 4 !== 0 &&
-                            length === undefined) {
-                            try {
-                                // Create a new aligned buffer if the original isn't properly aligned
-                                const alignedByteLength = Math.floor((arg.byteLength - alignedByteOffset) / 4) * 4;
-                                const alignedBuffer = new ArrayBuffer(alignedByteLength);
-                                const sourceView = new Uint8Array(arg, alignedByteOffset, alignedByteLength);
-                                const targetView = new Uint8Array(alignedBuffer);
-                                targetView.set(sourceView);
-                                super(alignedBuffer);
-                            }
-                            catch (error) {
-                                // If alignment fails, try the original approach
-                                console.warn('Float32Array alignment failed, using original constructor:', error);
-                                super(arg, alignedByteOffset, alignedLength);
-                            }
-                        }
-                        else {
-                            super(arg, alignedByteOffset, alignedLength);
-                        }
-                    }
-                    else {
-                        super(arg, byteOffset, length);
-                    }
-                }
-            };
-            // Apply the patch to the global object
-            try {
-                // Preserve static methods and properties
-                Object.setPrototypeOf(PatchedFloat32Array, originalFloat32Array);
-                Object.defineProperty(PatchedFloat32Array, 'name', {
-                    value: 'Float32Array'
-                });
-                Object.defineProperty(PatchedFloat32Array, 'BYTES_PER_ELEMENT', {
-                    value: 4
-                });
-                // Replace the global Float32Array with our patched version
-                globalObj.Float32Array = PatchedFloat32Array;
-            }
-            catch (error) {
-                console.warn('Failed to patch Float32Array:', error);
-            }
-        }
-        // CRITICAL: Patch any empty util shims that bundlers might create
-        // This handles cases where bundlers provide empty shims for Node.js modules
-        if (typeof global !== 'undefined') {
-            // Look for common patterns of util shims in bundled code
-            const checkAndPatchUtilShim = (obj) => {
-                if (obj && typeof obj === 'object' && !obj.TextEncoder) {
-                    obj.TextEncoder = TextEncoder;
-                    obj.TextDecoder = TextDecoder;
-                    obj.types = obj.types || {
-                        isFloat32Array: (arr) => arr instanceof Float32Array,
-                        isInt32Array: (arr) => arr instanceof Int32Array,
-                        isUint8Array: (arr) => arr instanceof Uint8Array,
-                        isUint8ClampedArray: (arr) => arr instanceof Uint8ClampedArray
-                    };
-                }
-            };
-            // Patch any existing util-like objects in global scope
-            if (global._utilShim) {
-                checkAndPatchUtilShim(global._utilShim);
-            }
-            // CRITICAL: Patch the bundled util shim directly
-            // In bundled code, there's often a _utilShim object that needs patching
-            if (typeof globalThis !== 'undefined' &&
-                globalThis._utilShim) {
-                checkAndPatchUtilShim(globalThis._utilShim);
-            }
-            // CRITICAL: Create and patch a global _utilShim if it doesn't exist
-            // This ensures the bundled code will find the patched version
-            if (!global._utilShim) {
-                global._utilShim = {
-                    TextEncoder: TextEncoder,
-                    TextDecoder: TextDecoder,
-                    types: {
-                        isFloat32Array: (arr) => arr instanceof Float32Array,
-                        isInt32Array: (arr) => arr instanceof Int32Array,
-                        isUint8Array: (arr) => arr instanceof Uint8Array,
-                        isUint8ClampedArray: (arr) => arr instanceof Uint8ClampedArray
-                    }
-                };
-            }
-            else {
-                checkAndPatchUtilShim(global._utilShim);
-            }
-            // Also ensure it's available on globalThis
-            if (typeof globalThis !== 'undefined' &&
-                !globalThis._utilShim) {
-                ;
-                globalThis._utilShim = global._utilShim;
-            }
-            // Set up a property descriptor to catch util shim assignments
-            try {
-                Object.defineProperty(global, '_utilShim', {
-                    get() {
-                        return this.__utilShim || {};
-                    },
-                    set(value) {
-                        checkAndPatchUtilShim(value);
-                        this.__utilShim = value;
-                    },
-                    configurable: true
-                });
-            }
-            catch (e) {
-                // Ignore if property can't be defined
-            }
-            // Also set up property descriptor on globalThis
-            try {
-                Object.defineProperty(globalThis, '_utilShim', {
-                    get() {
-                        return this.__utilShim || {};
-                    },
-                    set(value) {
-                        checkAndPatchUtilShim(value);
-                        this.__utilShim = value;
-                    },
-                    configurable: true
-                });
-            }
-            catch (e) {
-                // Ignore if property can't be defined
-            }
-        }
-        console.log('Brainy: Successfully patched TensorFlow.js PlatformNode at module load time');
-        patchApplied = true;
-    }
-    catch (error) {
-        console.warn('Brainy: Failed to apply early TensorFlow.js platform patch:', error);
-    }
-}
-/**
- * Apply the TensorFlow.js platform patch if it hasn't been applied already
- * This is a safety measure in case the module-level patch didn't run
- * Now works across all environments: browser, Node.js, and serverless/server
- */
-export async function applyTensorFlowPatch() {
-    // Apply patches for all non-browser environments that might need TensorFlow.js compatibility
-    // This includes Node.js, serverless environments, and other server environments
-    const isBrowserEnv = typeof window !== 'undefined' && typeof document !== 'undefined';
-    if (isBrowserEnv) {
-        return; // Browser environments don't need these patches
-    }
-    // Get the appropriate global object for the current environment
-    const globalObj = (() => {
-        if (typeof globalThis !== 'undefined')
-            return globalThis;
-        if (typeof global !== 'undefined')
-            return global;
-        if (typeof self !== 'undefined')
-            return self;
-        return {}; // Fallback for unknown environments
-    })();
-    // Check if the critical globals exist, not just the flag
-    // This allows re-patching if globals have been deleted
-    const needsPatch = !patchApplied ||
-        typeof globalObj.__TextEncoder__ === 'undefined' ||
-        typeof globalObj.__TextDecoder__ === 'undefined';
-    if (!needsPatch) {
-        return;
-    }
-    try {
-        console.log('Brainy: Applying TensorFlow.js platform patch via function call');
-        // CRITICAL FIX: Patch the global environment to ensure TextEncoder/TextDecoder are available
-        // This approach works by ensuring the global constructors are available before TensorFlow.js loads
-        // Now works across all environments: Node.js, serverless, and other server environments
         // Make sure TextEncoder and TextDecoder are available globally
         if (!globalObj.TextEncoder) {
             globalObj.TextEncoder = TextEncoder;
@@ -269,31 +35,20 @@ export async function applyTensorFlowPatch() {
         if (!globalObj.TextDecoder) {
             globalObj.TextDecoder = TextDecoder;
         }
-        // Also set the special global constructors that TensorFlow can use safely
-        ;
-        globalObj.__TextEncoder__ = TextEncoder;
-        globalObj.__TextDecoder__ = TextDecoder;
-        // Also patch process.versions to ensure TensorFlow.js detects Node.js correctly
-        if (typeof process !== 'undefined' && process.versions) {
-            // Ensure TensorFlow.js sees this as a Node.js environment
-            if (!process.versions.node) {
-                process.versions.node = process.version;
+        // Also set them on the global object for older code
+        if (typeof global !== 'undefined') {
+            if (!global.TextEncoder) {
+                global.TextEncoder = TextEncoder;
+            }
+            if (!global.TextDecoder) {
+                global.TextDecoder = TextDecoder;
             }
-        }
-        // CRITICAL: Patch the Node.js util module directly
-        try {
-            const util = await import('util');
-            // Ensure TextEncoder and TextDecoder are available as constructors
-            util.TextEncoder = TextEncoder;
-            util.TextDecoder = TextDecoder;
-        }
-        catch (error) {
-            // Ignore if util module is not available
         }
         patchApplied = true;
+        console.log('Brainy: TextEncoder/TextDecoder patches applied successfully');
     }
     catch (error) {
-        console.warn('Brainy: Failed to apply TensorFlow.js platform patch:', error);
+        console.warn('Brainy: Failed to apply TextEncoder/TextDecoder patch:', error);
     }
 }
 export function getTextEncoder() {
@@ -302,8 +57,10 @@ export function getTextEncoder() {
 export function getTextDecoder() {
     return new TextDecoder();
 }
-// Apply patch immediately
-applyTensorFlowPatch().catch((error) => {
-    console.warn('Failed to apply TensorFlow patch at module load:', error);
-});
+// Apply patch immediately if in Node.js
+if (isNode()) {
+    applyTensorFlowPatch().catch((error) => {
+        console.warn('Failed to apply TextEncoder/TextDecoder patch at module load:', error);
+    });
+}
 //# sourceMappingURL=textEncoding.js.map

package/dist/utils/textEncoding.js.map

@@ -1 +1 @@
-{"version":3,"file":"textEncoding.js","sourceRoot":"","sources":["../../src/utils/textEncoding.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAA;AAczC;;GAEG;AACH,IAAI,YAAY,GAAG,KAAK,CAAA;AAExB;;;GAGG;AACH,IAAI,OAAO,UAAU,KAAK,WAAW,IAAI,MAAM,EAAE,EAAE,CAAC;IAClD,IAAI,CAAC;QACH,wDAAwD;QACxD,IAAI,OAAO,UAAU,CAAC,WAAW,KAAK,WAAW,EAAE,CAAC;YAClD,UAAU,CAAC,WAAW,GAAG,WAAW,CAAA;QACtC,CAAC;QACD,IAAI,OAAO,UAAU,CAAC,WAAW,KAAK,WAAW,EAAE,CAAC;YAClD,UAAU,CAAC,WAAW,GAAG,WAAW,CAAA;QACtC,CAAC;QAED,qEAAqE;QACrE,2FAA2F;QAC3F,IAAI,OAAO,MAAM,KAAK,WAAW,EAAE,CAAC;YAClC,IAAI,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC;gBACxB,MAAM,CAAC,WAAW,GAAG,WAAW,CAAA;YAClC,CAAC;YACD,IAAI,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC;gBACxB,MAAM,CAAC,WAAW,GAAG,WAAW,CAAA;YAClC,CAAC;YACD,0EAA0E;YAC1E,MAAM,CAAC,eAAe,GAAG,WAAW,CAAA;YACpC,MAAM,CAAC,eAAe,GAAG,WAAW,CAAA;QACtC,CAAC;QAED,uEAAuE;QACvE,kEAAkE;QAClE,MAAM,UAAU,GAAG;YACjB,WAAW,EAAE,WAAW;YACxB,WAAW,EAAE,WAAW;YACxB,KAAK,EAAE;gBACL,cAAc,EAAE,CAAC,GAAQ,EAAE,EAAE,CAAC,GAAG,YAAY,YAAY;gBACzD,YAAY,EAAE,CAAC,GAAQ,EAAE,EAAE,CAAC,GAAG,YAAY,UAAU;gBACrD,YAAY,EAAE,CAAC,GAAQ,EAAE,EAAE,CAAC,GAAG,YAAY,UAAU;gBACrD,mBAAmB,EAAE,CAAC,GAAQ,EAAE,EAAE,CAAC,GAAG,YAAY,iBAAiB;aACpE;SACF,CAAA;QAED,0CAA0C;QAC1C,IAAI,OAAO,MAAM,KAAK,WAAW,EAAE,CAAC;YAClC,MAAM,CAAC,eAAe,GAAG,UAAU,CAAA;QACrC,CAAC;QAED,oDAAoD;QACpD,IACE,OAAO,MAAM,KAAK,WAAW;YAC7B,MAAM,CAAC,OAAO;YACd,MAAM,CAAC,OAAO,CAAC,KAAK,EACpB,CAAC;YACD,iDAAiD;YACjD,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;gBACvC,IAAI,GAAG,CAAC,QAAQ,CAAC,UAAU,CAAC,IAAI,GAAG,KAAK,MAAM,EAAE,CAAC;oBAC/C,MAAM,UAAU,GAAG,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,CAAA;oBAC5C,IAAI,UAAU,IAAI,UAAU,CAAC,OAAO,EAAE,CAAC;wBACrC,MAAM,CAAC,MAAM,CAAC,UAAU,CAAC,OAAO,EAAE,UAAU,CAAC,CAAA;oBAC/C,CAAC;gBACH,CAAC;YACH,CAAC;QACH,CAAC;QAED,mDAAmD;QACnD,IAAI,CAAC;YACH,MAAM,IAAI,GAAG,OAAO,CAAC,MAAM,CAAC,CAAA;YAC5B,mEAAmE;YACnE,IAAI,CAAC,WAAW,GAAG,WAAsC,CAAA;YACzD,IAAI,CAAC,WAAW,GAAG,WAAsC,CAAA;QAC3D,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,yCAAyC;QAC3C,CAAC;QAED,iEAAiE;QACjE,+EAA+E;QAC/E,gEAAgE;QAChE,MAAM,SAAS,GAAG,CAAC,GAAG,EAAE;YACtB,IAAI,OAAO,UAAU,KAAK,WAAW;gBAAE,OAAO,UAAU,CAAA;YACxD,IAAI,OAAO,MAAM,KAAK,WAAW;gBAAE,OAAO,MAAM,CAAA;YAChD,IAAI,OAAO,IAAI,KAAK,WAAW;gBAAE,OAAO,IAAI,CAAA;YAC5C,IAAI,OAAO,MAAM,KAAK,WAAW;gBAAE,OAAO,MAAM,CAAA;YAChD,OAAO,EAAS,CAAA,CAAC,oCAAoC;QACvD,CAAC,CAAC,EAAE,CAAA;QAEJ,IAAI,SAAS,IAAI,SAAS,CAAC,YAAY,EAAE,CAAC;YACxC,MAAM,oBAAoB,GAAG,SAAS,CAAC,YAAY,CAAA;YAEnD,oEAAoE;YACpE,MAAM,mBAAmB,GAAG,KAAM,SAAQ,oBAAoB;gBAC5D,YAAY,GAAS,EAAE,UAAmB,EAAE,MAAe;oBACzD,IAAI,GAAG,YAAY,WAAW,EAAE,CAAC;wBAC/B,2EAA2E;wBAC3E,MAAM,iBAAiB,GAAG,UAAU,IAAI,CAAC,CAAA;wBACzC,MAAM,aAAa,GACjB,MAAM,KAAK,SAAS;4BAClB,CAAC,CAAC,MAAM;4BACR,CAAC,CAAC,CAAC,GAAG,CAAC,UAAU,GAAG,iBAAiB,CAAC,GAAG,CAAC,CAAA;wBAE9C,gDAAgD;wBAChD,IACE,CAAC,GAAG,CAAC,UAAU,GAAG,iBAAiB,CAAC,GAAG,CAAC,KAAK,CAAC;4BAC9C,MAAM,KAAK,SAAS,EACpB,CAAC;4BACD,IAAI,CAAC;gCACH,qEAAqE;gCACrE,MAAM,iBAAiB,GACrB,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,UAAU,GAAG,iBAAiB,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAA;gCAC1D,MAAM,aAAa,GAAG,IAAI,WAAW,CAAC,iBAAiB,CAAC,CAAA;gCACxD,MAAM,UAAU,GAAG,IAAI,UAAU,CAC/B,GAAG,EACH,iBAAiB,EACjB,iBAAiB,CAClB,CAAA;gCACD,MAAM,UAAU,GAAG,IAAI,UAAU,CAAC,aAAa,CAAC,CAAA;gCAChD,UAAU,CAAC,GAAG,CAAC,UAAU,CAAC,CAAA;gCAC1B,KAAK,CAAC,aAAa,CAAC,CAAA;4BACtB,CAAC;4BAAC,OAAO,KAAK,EAAE,CAAC;gCACf,gDAAgD;gCAChD,OAAO,CAAC,IAAI,CAAC,4DAA4D,EAAE,KAAK,CAAC,CAAA;gCACjF,KAAK,CAAC,GAAG,EAAE,iBAAiB,EAAE,aAAa,CAAC,CAAA;4BAC9C,CAAC;wBACH,CAAC;6BAAM,CAAC;4BACN,KAAK,CAAC,GAAG,EAAE,iBAAiB,EAAE,aAAa,CAAC,CAAA;wBAC9C,CAAC;oBACH,CAAC;yBAAM,CAAC;wBACN,KAAK,CAAC,GAAG,EAAE,UAAU,EAAE,MAAM,CAAC,CAAA;oBAChC,CAAC;gBACH,CAAC;aACK,CAAA;YAER,uCAAuC;YACvC,IAAI,CAAC;gBACH,yCAAyC;gBACzC,MAAM,CAAC,cAAc,CAAC,mBAAmB,EAAE,oBAAoB,CAAC,CAAA;gBAChE,MAAM,CAAC,cAAc,CAAC,mBAAmB,EAAE,MAAM,EAAE;oBACjD,KAAK,EAAE,cAAc;iBACtB,CAAC,CAAA;gBACF,MAAM,CAAC,cAAc,CAAC,mBAAmB,EAAE,mBAAmB,EAAE;oBAC9D,KAAK,EAAE,CAAC;iBACT,CAAC,CAAA;gBAEF,2DAA2D;gBAC3D,SAAS,CAAC,YAAY,GAAG,mBAAmB,CAAA;YAC9C,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,OAAO,CAAC,IAAI,CAAC,+BAA+B,EAAE,KAAK,CAAC,CAAA;YACtD,CAAC;QACH,CAAC;QAED,kEAAkE;QAClE,4EAA4E;QAC5E,IAAI,OAAO,MAAM,KAAK,WAAW,EAAE,CAAC;YAClC,yDAAyD;YACzD,MAAM,qBAAqB,GAAG,CAAC,GAAQ,EAAE,EAAE;gBACzC,IAAI,GAAG,IAAI,OAAO,GAAG,KAAK,QAAQ,IAAI,CAAC,GAAG,CAAC,WAAW,EAAE,CAAC;oBACvD,GAAG,CAAC,WAAW,GAAG,WAAW,CAAA;oBAC7B,GAAG,CAAC,WAAW,GAAG,WAAW,CAAA;oBAC7B,GAAG,CAAC,KAAK,GAAG,GAAG,CAAC,KAAK,IAAI;wBACvB,cAAc,EAAE,CAAC,GAAQ,EAAE,EAAE,CAAC,GAAG,YAAY,YAAY;wBACzD,YAAY,EAAE,CAAC,GAAQ,EAAE,EAAE,CAAC,GAAG,YAAY,UAAU;wBACrD,YAAY,EAAE,CAAC,GAAQ,EAAE,EAAE,CAAC,GAAG,YAAY,UAAU;wBACrD,mBAAmB,EAAE,CAAC,GAAQ,EAAE,EAAE,CAAC,GAAG,YAAY,iBAAiB;qBACpE,CAAA;gBACH,CAAC;YACH,CAAC,CAAA;YAED,uDAAuD;YACvD,IAAI,MAAM,CAAC,SAAS,EAAE,CAAC;gBACrB,qBAAqB,CAAC,MAAM,CAAC,SAAS,CAAC,CAAA;YACzC,CAAC;YAED,iDAAiD;YACjD,wEAAwE;YACxE,IACE,OAAO,UAAU,KAAK,WAAW;gBAChC,UAAyB,CAAC,SAAS,EACpC,CAAC;gBACD,qBAAqB,CAAE,UAAyB,CAAC,SAAS,CAAC,CAAA;YAC7D,CAAC;YAED,oEAAoE;YACpE,8DAA8D;YAC9D,IAAI,CAAC,MAAM,CAAC,SAAS,EAAE,CAAC;gBACtB,MAAM,CAAC,SAAS,GAAG;oBACjB,WAAW,EAAE,WAAW;oBACxB,WAAW,EAAE,WAAW;oBACxB,KAAK,EAAE;wBACL,cAAc,EAAE,CAAC,GAAQ,EAAE,EAAE,CAAC,GAAG,YAAY,YAAY;wBACzD,YAAY,EAAE,CAAC,GAAQ,EAAE,EAAE,CAAC,GAAG,YAAY,UAAU;wBACrD,YAAY,EAAE,CAAC,GAAQ,EAAE,EAAE,CAAC,GAAG,YAAY,UAAU;wBACrD,mBAAmB,EAAE,CAAC,GAAQ,EAAE,EAAE,CAAC,GAAG,YAAY,iBAAiB;qBACpE;iBACF,CAAA;YACH,CAAC;iBAAM,CAAC;gBACN,qBAAqB,CAAC,MAAM,CAAC,SAAS,CAAC,CAAA;YACzC,CAAC;YAED,2CAA2C;YAC3C,IACE,OAAO,UAAU,KAAK,WAAW;gBACjC,CAAE,UAAyB,CAAC,SAAS,EACrC,CAAC;gBACD,CAAC;gBAAC,UAAyB,CAAC,SAAS,GAAG,MAAM,CAAC,SAAS,CAAA;YAC1D,CAAC;YAED,8DAA8D;YAC9D,IAAI,CAAC;gBACH,MAAM,CAAC,cAAc,CAAC,MAAM,EAAE,WAAW,EAAE;oBACzC,GAAG;wBACD,OAAO,IAAI,CAAC,UAAU,IAAI,EAAE,CAAA;oBAC9B,CAAC;oBACD,GAAG,CAAC,KAAK;wBACP,qBAAqB,CAAC,KAAK,CAAC,CAAA;wBAC5B,IAAI,CAAC,UAAU,GAAG,KAAK,CAAA;oBACzB,CAAC;oBACD,YAAY,EAAE,IAAI;iBACnB,CAAC,CAAA;YACJ,CAAC;YAAC,OAAO,CAAC,EAAE,CAAC;gBACX,sCAAsC;YACxC,CAAC;YAED,gDAAgD;YAChD,IAAI,CAAC;gBACH,MAAM,CAAC,cAAc,CAAC,UAAU,EAAE,WAAW,EAAE;oBAC7C,GAAG;wBACD,OAAO,IAAI,CAAC,UAAU,IAAI,EAAE,CAAA;oBAC9B,CAAC;oBACD,GAAG,CAAC,KAAK;wBACP,qBAAqB,CAAC,KAAK,CAAC,CAAA;wBAC5B,IAAI,CAAC,UAAU,GAAG,KAAK,CAAA;oBACzB,CAAC;oBACD,YAAY,EAAE,IAAI;iBACnB,CAAC,CAAA;YACJ,CAAC;YAAC,OAAO,CAAC,EAAE,CAAC;gBACX,sCAAsC;YACxC,CAAC;QACH,CAAC;QAED,OAAO,CAAC,GAAG,CACT,6EAA6E,CAC9E,CAAA;QACD,YAAY,GAAG,IAAI,CAAA;IACrB,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,OAAO,CAAC,IAAI,CACV,6DAA6D,EAC7D,KAAK,CACN,CAAA;IACH,CAAC;AACH,CAAC;AAED;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,oBAAoB;IACxC,6FAA6F;IAC7F,gFAAgF;IAChF,MAAM,YAAY,GAAG,OAAO,MAAM,KAAK,WAAW,IAAI,OAAO,QAAQ,KAAK,WAAW,CAAA;IACrF,IAAI,YAAY,EAAE,CAAC;QACjB,OAAM,CAAC,gDAAgD;IACzD,CAAC;IAED,gEAAgE;IAChE,MAAM,SAAS,GAAG,CAAC,GAAG,EAAE;QACtB,IAAI,OAAO,UAAU,KAAK,WAAW;YAAE,OAAO,UAAU,CAAA;QACxD,IAAI,OAAO,MAAM,KAAK,WAAW;YAAE,OAAO,MAAM,CAAA;QAChD,IAAI,OAAO,IAAI,KAAK,WAAW;YAAE,OAAO,IAAI,CAAA;QAC5C,OAAO,EAAS,CAAA,CAAC,oCAAoC;IACvD,CAAC,CAAC,EAAE,CAAA;IAEJ,yDAAyD;IACzD,uDAAuD;IACvD,MAAM,UAAU,GAAG,CAAC,YAAY;QAC9B,OAAO,SAAS,CAAC,eAAe,KAAK,WAAW;QAChD,OAAO,SAAS,CAAC,eAAe,KAAK,WAAW,CAAA;IAElD,IAAI,CAAC,UAAU,EAAE,CAAC;QAChB,OAAM;IACR,CAAC;IAED,IAAI,CAAC;QACH,OAAO,CAAC,GAAG,CACT,iEAAiE,CAClE,CAAA;QAED,6FAA6F;QAC7F,mGAAmG;QACnG,wFAAwF;QAExF,+DAA+D;QAC/D,IAAI,CAAC,SAAS,CAAC,WAAW,EAAE,CAAC;YAC3B,SAAS,CAAC,WAAW,GAAG,WAAW,CAAA;QACrC,CAAC;QACD,IAAI,CAAC,SAAS,CAAC,WAAW,EAAE,CAAC;YAC3B,SAAS,CAAC,WAAW,GAAG,WAAW,CAAA;QACrC,CAAC;QAED,0EAA0E;QAC1E,CAAC;QAAC,SAAiB,CAAC,eAAe,GAAG,WAAW,CAChD;QAAC,SAAiB,CAAC,eAAe,GAAG,WAAW,CAAA;QAEjD,gFAAgF;QAChF,IAAI,OAAO,OAAO,KAAK,WAAW,IAAI,OAAO,CAAC,QAAQ,EAAE,CAAC;YACvD,0DAA0D;YAC1D,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC;gBAC3B,OAAO,CAAC,QAAQ,CAAC,IAAI,GAAG,OAAO,CAAC,OAAO,CAAA;YACzC,CAAC;QACH,CAAC;QAED,mDAAmD;QACnD,IAAI,CAAC;YACH,MAAM,IAAI,GAAG,MAAM,MAAM,CAAC,MAAM,CAAC,CAAA;YACjC,mEAAmE;YACnE,IAAI,CAAC,WAAW,GAAG,WAAsC,CAAA;YACzD,IAAI,CAAC,WAAW,GAAG,WAAsC,CAAA;QAC3D,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,yCAAyC;QAC3C,CAAC;QAED,YAAY,GAAG,IAAI,CAAA;IACrB,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,OAAO,CAAC,IAAI,CAAC,uDAAuD,EAAE,KAAK,CAAC,CAAA;IAC9E,CAAC;AACH,CAAC;AAED,MAAM,UAAU,cAAc;IAC5B,OAAO,IAAI,WAAW,EAAE,CAAA;AAC1B,CAAC;AAED,MAAM,UAAU,cAAc;IAC5B,OAAO,IAAI,WAAW,EAAE,CAAA;AAC1B,CAAC;AAED,0BAA0B;AAC1B,oBAAoB,EAAE,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE;IACrC,OAAO,CAAC,IAAI,CAAC,kDAAkD,EAAE,KAAK,CAAC,CAAA;AACzE,CAAC,CAAC,CAAA"}
+{"version":3,"file":"textEncoding.js","sourceRoot":"","sources":["../../src/utils/textEncoding.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAA;AAEzC,yEAAyE;AACzE,qFAAqF;AAErF;;GAEG;AACH,IAAI,YAAY,GAAG,KAAK,CAAA;AAExB;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,oBAAoB;IACxC,yFAAyF;IACzF,MAAM,YAAY,GAAG,OAAO,MAAM,KAAK,WAAW,IAAI,OAAO,QAAQ,KAAK,WAAW,CAAA;IACrF,IAAI,YAAY,IAAI,YAAY,EAAE,CAAC;QACjC,OAAM,CAAC,uEAAuE;IAChF,CAAC;IAED,IAAI,CAAC,MAAM,EAAE,EAAE,CAAC;QACd,OAAM,CAAC,kCAAkC;IAC3C,CAAC;IAED,IAAI,CAAC;QACH,OAAO,CAAC,GAAG,CAAC,4DAA4D,CAAC,CAAA;QAEzE,oCAAoC;QACpC,MAAM,SAAS,GAAG,CAAC,GAAG,EAAE;YACtB,IAAI,OAAO,UAAU,KAAK,WAAW;gBAAE,OAAO,UAAU,CAAA;YACxD,IAAI,OAAO,MAAM,KAAK,WAAW;gBAAE,OAAO,MAAM,CAAA;YAChD,OAAO,EAAS,CAAA;QAClB,CAAC,CAAC,EAAE,CAAA;QAEJ,+DAA+D;QAC/D,IAAI,CAAC,SAAS,CAAC,WAAW,EAAE,CAAC;YAC3B,SAAS,CAAC,WAAW,GAAG,WAAW,CAAA;QACrC,CAAC;QACD,IAAI,CAAC,SAAS,CAAC,WAAW,EAAE,CAAC;YAC3B,SAAS,CAAC,WAAW,GAAG,WAAW,CAAA;QACrC,CAAC;QAED,oDAAoD;QACpD,IAAI,OAAO,MAAM,KAAK,WAAW,EAAE,CAAC;YAClC,IAAI,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC;gBACxB,MAAM,CAAC,WAAW,GAAG,WAAW,CAAA;YAClC,CAAC;YACD,IAAI,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC;gBACxB,MAAM,CAAC,WAAW,GAAG,WAAW,CAAA;YAClC,CAAC;QACH,CAAC;QAED,YAAY,GAAG,IAAI,CAAA;QACnB,OAAO,CAAC,GAAG,CAAC,8DAA8D,CAAC,CAAA;IAC7E,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,OAAO,CAAC,IAAI,CAAC,wDAAwD,EAAE,KAAK,CAAC,CAAA;IAC/E,CAAC;AACH,CAAC;AAED,MAAM,UAAU,cAAc;IAC5B,OAAO,IAAI,WAAW,EAAE,CAAA;AAC1B,CAAC;AAED,MAAM,UAAU,cAAc;IAC5B,OAAO,IAAI,WAAW,EAAE,CAAA;AAC1B,CAAC;AAED,wCAAwC;AACxC,IAAI,MAAM,EAAE,EAAE,CAAC;IACb,oBAAoB,EAAE,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE;QACrC,OAAO,CAAC,IAAI,CAAC,+DAA+D,EAAE,KAAK,CAAC,CAAA;IACtF,CAAC,CAAC,CAAA;AACJ,CAAC"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@soulcraft/brainy",
-  "version": "0.45.0",
+  "version": "0.47.0",
   "description": "A vector graph database using HNSW indexing with Origin Private File System storage",
   "main": "dist/index.js",
   "module": "dist/index.js",
@@ -92,7 +92,8 @@
     "_workflow:minor": "node scripts/release-workflow.js minor",
     "_workflow:major": "node scripts/release-workflow.js major",
     "_workflow:dry-run": "npm run build && npm test && npm run _release:dry-run",
-    "_dry-run": "npm pack --dry-run"
+    "_dry-run": "npm pack --dry-run",
+    "download-models": "node scripts/download-models.cjs"
   },
   "keywords": [
     "vector-database",
@@ -128,7 +129,9 @@
     "!dist/framework.min.js.map",
     "LICENSE",
     "README.md",
-    "brainy.png"
+    "brainy.png",
+    "scripts/download-models.cjs",
+    "OFFLINE_MODELS.md"
   ],
   "devDependencies": {
     "@rollup/plugin-commonjs": "^25.0.7",
@@ -159,23 +162,11 @@
   },
   "dependencies": {
     "@aws-sdk/client-s3": "^3.540.0",
-    "@tensorflow/tfjs": "^4.22.0",
-    "@tensorflow/tfjs-backend-cpu": "^4.22.0",
-    "@tensorflow/tfjs-backend-webgl": "^4.22.0",
-    "@tensorflow/tfjs-converter": "^4.22.0",
-    "@tensorflow/tfjs-core": "^4.22.0",
+    "@huggingface/transformers": "^3.1.0",
     "buffer": "^6.0.3",
     "dotenv": "^16.4.5",
     "uuid": "^9.0.1"
   },
-  "peerDependencies": {
-    "@soulcraft/brainy-models": ">=0.7.0"
-  },
-  "peerDependenciesMeta": {
-    "@soulcraft/brainy-models": {
-      "optional": true
-    }
-  },
   "prettier": {
     "arrowParens": "always",
     "bracketSameLine": true,

package/scripts/download-models.cjs

@@ -0,0 +1,190 @@
+#!/usr/bin/env node
+/**
+ * Download and bundle models for offline usage
+ */
+
+const fs = require('fs').promises
+const path = require('path')
+
+const MODEL_NAME = 'Xenova/all-MiniLM-L6-v2'
+const OUTPUT_DIR = './models'
+
+async function downloadModels() {
+  // Use dynamic import for ES modules in CommonJS
+  const { pipeline, env } = await import('@huggingface/transformers')
+  
+  // Configure transformers.js to use local cache
+  env.cacheDir = './models-cache'
+  env.allowRemoteModels = true
+  try {
+    console.log('🔄 Downloading all-MiniLM-L6-v2 model for offline bundling...')
+    console.log(`   Model: ${MODEL_NAME}`)
+    console.log(`   Cache: ${env.cacheDir}`)
+    
+    // Create output directory
+    await fs.mkdir(OUTPUT_DIR, { recursive: true })
+    
+    // Load the model to force download
+    console.log('📥 Loading model pipeline...')
+    const extractor = await pipeline('feature-extraction', MODEL_NAME)
+    
+    // Test the model to make sure it works
+    console.log('🧪 Testing model...')
+    const testResult = await extractor(['Hello world!'], {
+      pooling: 'mean',
+      normalize: true
+    })
+    
+    console.log(`✅ Model test successful! Embedding dimensions: ${testResult.data.length}`)
+    
+    // Copy ALL model files from cache to our models directory
+    console.log('📋 Copying ALL model files to bundle directory...')
+    
+    const cacheDir = path.resolve(env.cacheDir)
+    const outputDir = path.resolve(OUTPUT_DIR)
+    
+    console.log(`   From: ${cacheDir}`)
+    console.log(`   To: ${outputDir}`)
+    
+    // Copy the entire cache directory structure to ensure we get ALL files
+    // including tokenizer.json, config.json, and all ONNX model files
+    const modelCacheDir = path.join(cacheDir, 'Xenova', 'all-MiniLM-L6-v2')
+    
+    if (await dirExists(modelCacheDir)) {
+      const targetModelDir = path.join(outputDir, 'Xenova', 'all-MiniLM-L6-v2')
+      console.log(`   Copying complete model: Xenova/all-MiniLM-L6-v2`)
+      await copyDirectory(modelCacheDir, targetModelDir)
+    } else {
+      throw new Error(`Model cache directory not found: ${modelCacheDir}`)
+    }
+    
+    console.log('✅ Model bundling complete!')
+    console.log(`   Total size: ${await calculateDirectorySize(outputDir)} MB`)
+    console.log(`   Location: ${outputDir}`)
+    
+    // Create a marker file
+    await fs.writeFile(
+      path.join(outputDir, '.brainy-models-bundled'),
+      JSON.stringify({
+        model: MODEL_NAME,
+        bundledAt: new Date().toISOString(),
+        version: '1.0.0'
+      }, null, 2)
+    )
+    
+  } catch (error) {
+    console.error('❌ Error downloading models:', error)
+    process.exit(1)
+  }
+}
+
+async function findModelDirectories(baseDir, modelName) {
+  const dirs = []
+  
+  try {
+    // Convert model name to expected directory structure
+    const modelPath = modelName.replace('/', '--')
+    
+    async function searchDirectory(currentDir) {
+      try {
+        const entries = await fs.readdir(currentDir, { withFileTypes: true })
+        
+        for (const entry of entries) {
+          if (entry.isDirectory()) {
+            const fullPath = path.join(currentDir, entry.name)
+            
+            // Check if this directory contains model files
+            if (entry.name.includes(modelPath) || entry.name === 'onnx') {
+              const hasModelFiles = await containsModelFiles(fullPath)
+              if (hasModelFiles) {
+                dirs.push(fullPath)
+              }
+            }
+            
+            // Recursively search subdirectories
+            await searchDirectory(fullPath)
+          }
+        }
+      } catch (error) {
+        // Ignore access errors
+      }
+    }
+    
+    await searchDirectory(baseDir)
+  } catch (error) {
+    console.warn('Warning: Error searching for model directories:', error)
+  }
+  
+  return dirs
+}
+
+async function containsModelFiles(dir) {
+  try {
+    const files = await fs.readdir(dir)
+    return files.some(file => 
+      file.endsWith('.onnx') || 
+      file.endsWith('.json') ||
+      file === 'config.json' ||
+      file === 'tokenizer.json'
+    )
+  } catch (error) {
+    return false
+  }
+}
+
+async function dirExists(dir) {
+  try {
+    const stats = await fs.stat(dir)
+    return stats.isDirectory()
+  } catch (error) {
+    return false
+  }
+}
+
+async function copyDirectory(src, dest) {
+  await fs.mkdir(dest, { recursive: true })
+  const entries = await fs.readdir(src, { withFileTypes: true })
+  
+  for (const entry of entries) {
+    const srcPath = path.join(src, entry.name)
+    const destPath = path.join(dest, entry.name)
+    
+    if (entry.isDirectory()) {
+      await copyDirectory(srcPath, destPath)
+    } else {
+      await fs.copyFile(srcPath, destPath)
+    }
+  }
+}
+
+async function calculateDirectorySize(dir) {
+  let size = 0
+  
+  async function calculateSize(currentDir) {
+    try {
+      const entries = await fs.readdir(currentDir, { withFileTypes: true })
+      
+      for (const entry of entries) {
+        const fullPath = path.join(currentDir, entry.name)
+        
+        if (entry.isDirectory()) {
+          await calculateSize(fullPath)
+        } else {
+          const stats = await fs.stat(fullPath)
+          size += stats.size
+        }
+      }
+    } catch (error) {
+      // Ignore access errors
+    }
+  }
+  
+  await calculateSize(dir)
+  return Math.round(size / (1024 * 1024))
+}
+
+// Run the download
+downloadModels().catch(error => {
+  console.error('Fatal error:', error)
+  process.exit(1)
+})