@soulcraft/brainy 0.10.0 → 0.12.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.demo.md +1 -1
- package/README.md +3 -48
- package/dist/brainy.js +1196 -923
- package/dist/brainy.min.js +765 -750
- package/dist/brainyData.d.ts +29 -0
- package/dist/index.d.ts +3 -2
- package/dist/patched-platform-node.d.ts +17 -0
- package/dist/setup.d.ts +17 -0
- package/dist/storage/fileSystemStorage.d.ts +27 -69
- package/dist/storage/fileSystemStorage.d.ts.map +1 -1
- package/dist/types/tensorflowTypes.d.ts +10 -2
- package/dist/types/tensorflowTypes.d.ts.map +1 -1
- package/dist/unified.d.ts +5 -3
- package/dist/unified.js +1195 -924
- package/dist/unified.min.js +765 -750
- package/dist/utils/distance.d.ts.map +1 -1
- package/dist/utils/embedding.d.ts +25 -4
- package/dist/utils/embedding.d.ts.map +1 -1
- package/dist/utils/environment.d.ts.map +1 -1
- package/dist/utils/tensorflowUtils.d.ts.map +1 -1
- package/dist/utils/textEncoding.d.ts +12 -64
- package/dist/utils/textEncoding.d.ts.map +1 -1
- package/dist/utils/version.d.ts +1 -1
- package/dist/utils/workerUtils.d.ts.map +1 -1
- package/package.json +42 -14
- package/CHANGELOG.md +0 -24
- package/dist/cli.d.ts +0 -7
- package/dist/utils/distance-js.d.ts +0 -38
- package/dist/utils/distance-js.d.ts.map +0 -1
- package/dist/utils/distance-wasm.d.ts +0 -36
- package/dist/utils/distance-wasm.d.ts.map +0 -1
- package/dist/utils/textEncoderPolyfill.d.ts +0 -6
- package/dist/utils/textEncoderPolyfill.d.ts.map +0 -1
package/dist/unified.js
CHANGED
|
@@ -2395,135 +2395,400 @@ ieee754.write = function (buffer, value, offset, isLE, mLen, nBytes) {
|
|
|
2395
2395
|
} (buffer$1));
|
|
2396
2396
|
|
|
2397
2397
|
/**
|
|
2398
|
-
*
|
|
2399
|
-
*
|
|
2400
|
-
* This module provides a consistent way to handle text encoding/decoding across all environments
|
|
2401
|
-
* without relying on TextEncoder/TextDecoder polyfills or patches.
|
|
2398
|
+
* Utility functions for environment detection
|
|
2402
2399
|
*/
|
|
2403
2400
|
/**
|
|
2404
|
-
*
|
|
2405
|
-
* This avoids the need for TextEncoder polyfills and patches
|
|
2401
|
+
* Check if code is running in a browser environment
|
|
2406
2402
|
*/
|
|
2407
|
-
|
|
2408
|
-
|
|
2409
|
-
* Encode a string to a Uint8Array
|
|
2410
|
-
* @param input - The string to encode
|
|
2411
|
-
* @returns A Uint8Array containing the encoded string
|
|
2412
|
-
*/
|
|
2413
|
-
encode(input) {
|
|
2414
|
-
// Simple UTF-8 encoding implementation that works everywhere
|
|
2415
|
-
return new Uint8Array([...input].map((c) => c.charCodeAt(0)));
|
|
2416
|
-
}
|
|
2403
|
+
function isBrowser$1() {
|
|
2404
|
+
return typeof window !== 'undefined' && typeof document !== 'undefined';
|
|
2417
2405
|
}
|
|
2418
2406
|
/**
|
|
2419
|
-
*
|
|
2420
|
-
* This avoids the need for TextDecoder polyfills and patches
|
|
2407
|
+
* Check if code is running in a Node.js environment
|
|
2421
2408
|
*/
|
|
2422
|
-
|
|
2423
|
-
|
|
2424
|
-
|
|
2425
|
-
|
|
2426
|
-
|
|
2427
|
-
*/
|
|
2428
|
-
decode(input) {
|
|
2429
|
-
// Simple UTF-8 decoding implementation that works everywhere
|
|
2430
|
-
return String.fromCharCode.apply(null, [...input]);
|
|
2409
|
+
function isNode() {
|
|
2410
|
+
// If browser environment is detected, prioritize it over Node.js
|
|
2411
|
+
// This handles cases like jsdom where both window and process exist
|
|
2412
|
+
if (isBrowser$1()) {
|
|
2413
|
+
return false;
|
|
2431
2414
|
}
|
|
2415
|
+
return (typeof process !== 'undefined' &&
|
|
2416
|
+
process.versions != null &&
|
|
2417
|
+
process.versions.node != null);
|
|
2432
2418
|
}
|
|
2433
2419
|
/**
|
|
2434
|
-
*
|
|
2420
|
+
* Check if code is running in a Web Worker environment
|
|
2435
2421
|
*/
|
|
2436
|
-
function
|
|
2437
|
-
|
|
2438
|
-
|
|
2439
|
-
|
|
2440
|
-
try {
|
|
2441
|
-
// Try to use the native TextEncoder if available
|
|
2442
|
-
const nativeEncoder = new TextEncoder();
|
|
2443
|
-
this.encode = nativeEncoder.encode.bind(nativeEncoder);
|
|
2444
|
-
}
|
|
2445
|
-
catch (e) {
|
|
2446
|
-
// Fall back to our simple implementation
|
|
2447
|
-
const simpleEncoder = new SimpleTextEncoder();
|
|
2448
|
-
this.encode = simpleEncoder.encode.bind(simpleEncoder);
|
|
2449
|
-
}
|
|
2422
|
+
function isWebWorker() {
|
|
2423
|
+
return (typeof self === 'object' &&
|
|
2424
|
+
self.constructor &&
|
|
2425
|
+
self.constructor.name === 'DedicatedWorkerGlobalScope');
|
|
2450
2426
|
}
|
|
2451
2427
|
/**
|
|
2452
|
-
*
|
|
2428
|
+
* Check if Web Workers are available in the current environment
|
|
2453
2429
|
*/
|
|
2454
|
-
function
|
|
2455
|
-
|
|
2456
|
-
|
|
2457
|
-
|
|
2430
|
+
function areWebWorkersAvailable() {
|
|
2431
|
+
return isBrowser$1() && typeof Worker !== 'undefined';
|
|
2432
|
+
}
|
|
2433
|
+
/**
|
|
2434
|
+
* Check if Worker Threads are available in the current environment (Node.js)
|
|
2435
|
+
*/
|
|
2436
|
+
async function areWorkerThreadsAvailable() {
|
|
2437
|
+
if (!isNode())
|
|
2438
|
+
return false;
|
|
2458
2439
|
try {
|
|
2459
|
-
//
|
|
2460
|
-
|
|
2461
|
-
|
|
2440
|
+
// Use dynamic import to avoid errors in browser environments
|
|
2441
|
+
await import('worker_threads');
|
|
2442
|
+
return true;
|
|
2462
2443
|
}
|
|
2463
2444
|
catch (e) {
|
|
2464
|
-
|
|
2465
|
-
const simpleDecoder = new SimpleTextDecoder();
|
|
2466
|
-
this.decode = simpleDecoder.decode.bind(simpleDecoder);
|
|
2445
|
+
return false;
|
|
2467
2446
|
}
|
|
2468
2447
|
}
|
|
2469
2448
|
/**
|
|
2470
|
-
*
|
|
2471
|
-
*
|
|
2449
|
+
* Synchronous version that doesn't actually try to load the module
|
|
2450
|
+
* This is safer in ES module environments
|
|
2472
2451
|
*/
|
|
2473
|
-
function
|
|
2474
|
-
|
|
2452
|
+
function areWorkerThreadsAvailableSync() {
|
|
2453
|
+
if (!isNode())
|
|
2454
|
+
return false;
|
|
2455
|
+
// In Node.js 24.4.0+, worker_threads is always available
|
|
2456
|
+
return parseInt(process.versions.node.split('.')[0]) >= 24;
|
|
2475
2457
|
}
|
|
2476
2458
|
/**
|
|
2477
|
-
*
|
|
2478
|
-
*
|
|
2459
|
+
* Determine if threading is available in the current environment
|
|
2460
|
+
* Returns true if either Web Workers (browser) or Worker Threads (Node.js) are available
|
|
2479
2461
|
*/
|
|
2480
|
-
function
|
|
2481
|
-
return
|
|
2462
|
+
function isThreadingAvailable() {
|
|
2463
|
+
return areWebWorkersAvailable() || areWorkerThreadsAvailableSync();
|
|
2482
2464
|
}
|
|
2483
2465
|
/**
|
|
2484
|
-
*
|
|
2485
|
-
|
|
2486
|
-
|
|
2466
|
+
* Async version of isThreadingAvailable
|
|
2467
|
+
*/
|
|
2468
|
+
async function isThreadingAvailableAsync() {
|
|
2469
|
+
return areWebWorkersAvailable() || (await areWorkerThreadsAvailable());
|
|
2470
|
+
}
|
|
2471
|
+
|
|
2472
|
+
/**
|
|
2473
|
+
* Flag to track if the patch has been applied
|
|
2487
2474
|
*/
|
|
2488
|
-
|
|
2489
|
-
|
|
2490
|
-
|
|
2491
|
-
|
|
2492
|
-
|
|
2493
|
-
|
|
2475
|
+
let patchApplied = false;
|
|
2476
|
+
/**
|
|
2477
|
+
* Monkeypatch TensorFlow.js's PlatformNode class to fix TextEncoder/TextDecoder issues
|
|
2478
|
+
* CRITICAL: This runs immediately at the top level when this module is imported
|
|
2479
|
+
*/
|
|
2480
|
+
if (typeof globalThis !== 'undefined' && isNode()) {
|
|
2481
|
+
try {
|
|
2482
|
+
// Ensure TextEncoder/TextDecoder are globally available
|
|
2483
|
+
if (typeof globalThis.TextEncoder === 'undefined') {
|
|
2484
|
+
globalThis.TextEncoder = TextEncoder;
|
|
2485
|
+
}
|
|
2486
|
+
if (typeof globalThis.TextDecoder === 'undefined') {
|
|
2487
|
+
globalThis.TextDecoder = TextDecoder;
|
|
2488
|
+
}
|
|
2489
|
+
// Patch global objects to handle the TensorFlow.js constructor issue
|
|
2490
|
+
// This is needed because TF accesses TextEncoder/TextDecoder as constructors via this.util
|
|
2491
|
+
if (typeof global !== 'undefined') {
|
|
2492
|
+
if (!global.TextEncoder) {
|
|
2493
|
+
global.TextEncoder = TextEncoder;
|
|
2494
|
+
}
|
|
2495
|
+
if (!global.TextDecoder) {
|
|
2496
|
+
global.TextDecoder = TextDecoder;
|
|
2497
|
+
}
|
|
2498
|
+
// Also set the special global constructors that TensorFlow can use safely
|
|
2499
|
+
global.__TextEncoder__ = TextEncoder;
|
|
2500
|
+
global.__TextDecoder__ = TextDecoder;
|
|
2501
|
+
}
|
|
2502
|
+
// CRITICAL FIX: Create a custom util object that TensorFlow.js can use
|
|
2503
|
+
// We'll make this available globally so TensorFlow.js can find it
|
|
2504
|
+
const customUtil = {
|
|
2505
|
+
TextEncoder: TextEncoder,
|
|
2506
|
+
TextDecoder: TextDecoder,
|
|
2507
|
+
types: {
|
|
2508
|
+
isFloat32Array: (arr) => arr instanceof Float32Array,
|
|
2509
|
+
isInt32Array: (arr) => arr instanceof Int32Array,
|
|
2510
|
+
isUint8Array: (arr) => arr instanceof Uint8Array,
|
|
2511
|
+
isUint8ClampedArray: (arr) => arr instanceof Uint8ClampedArray
|
|
2512
|
+
}
|
|
2513
|
+
};
|
|
2514
|
+
// Make the custom util available globally
|
|
2515
|
+
if (typeof global !== 'undefined') {
|
|
2516
|
+
global.__brainy_util__ = customUtil;
|
|
2517
|
+
}
|
|
2518
|
+
// Try to patch the global require cache if possible
|
|
2519
|
+
if (typeof global !== 'undefined' &&
|
|
2520
|
+
global.require &&
|
|
2521
|
+
global.require.cache) {
|
|
2522
|
+
// Find the util module in the cache and patch it
|
|
2523
|
+
for (const key in global.require.cache) {
|
|
2524
|
+
if (key.endsWith('/util.js') || key === 'util') {
|
|
2525
|
+
const utilModule = global.require.cache[key];
|
|
2526
|
+
if (utilModule && utilModule.exports) {
|
|
2527
|
+
Object.assign(utilModule.exports, customUtil);
|
|
2528
|
+
}
|
|
2529
|
+
}
|
|
2530
|
+
}
|
|
2531
|
+
}
|
|
2532
|
+
// CRITICAL: Patch the Node.js util module directly
|
|
2494
2533
|
try {
|
|
2495
|
-
|
|
2496
|
-
|
|
2497
|
-
|
|
2498
|
-
|
|
2499
|
-
|
|
2500
|
-
|
|
2501
|
-
|
|
2502
|
-
|
|
2503
|
-
|
|
2504
|
-
|
|
2505
|
-
|
|
2506
|
-
|
|
2507
|
-
|
|
2508
|
-
|
|
2509
|
-
|
|
2510
|
-
|
|
2534
|
+
const util = require('util');
|
|
2535
|
+
// Ensure TextEncoder and TextDecoder are available as constructors
|
|
2536
|
+
util.TextEncoder = TextEncoder;
|
|
2537
|
+
util.TextDecoder = TextDecoder;
|
|
2538
|
+
}
|
|
2539
|
+
catch (error) {
|
|
2540
|
+
// Ignore if util module is not available
|
|
2541
|
+
}
|
|
2542
|
+
// CRITICAL: Patch Float32Array to handle buffer alignment issues
|
|
2543
|
+
// This fixes the "byte length of Float32Array should be a multiple of 4" error
|
|
2544
|
+
if (typeof global !== 'undefined') {
|
|
2545
|
+
const originalFloat32Array = global.Float32Array;
|
|
2546
|
+
global.Float32Array = class extends originalFloat32Array {
|
|
2547
|
+
constructor(arg, byteOffset, length) {
|
|
2548
|
+
if (arg instanceof ArrayBuffer) {
|
|
2549
|
+
// Ensure buffer is properly aligned for Float32Array (multiple of 4 bytes)
|
|
2550
|
+
const alignedByteOffset = byteOffset || 0;
|
|
2551
|
+
const alignedLength = length !== undefined
|
|
2552
|
+
? length
|
|
2553
|
+
: (arg.byteLength - alignedByteOffset) / 4;
|
|
2554
|
+
// Check if the buffer slice is properly aligned
|
|
2555
|
+
if ((arg.byteLength - alignedByteOffset) % 4 !== 0 &&
|
|
2556
|
+
length === undefined) {
|
|
2557
|
+
// Create a new aligned buffer if the original isn't properly aligned
|
|
2558
|
+
const alignedByteLength = Math.floor((arg.byteLength - alignedByteOffset) / 4) * 4;
|
|
2559
|
+
const alignedBuffer = new ArrayBuffer(alignedByteLength);
|
|
2560
|
+
const sourceView = new Uint8Array(arg, alignedByteOffset, alignedByteLength);
|
|
2561
|
+
const targetView = new Uint8Array(alignedBuffer);
|
|
2562
|
+
targetView.set(sourceView);
|
|
2563
|
+
super(alignedBuffer);
|
|
2564
|
+
}
|
|
2565
|
+
else {
|
|
2566
|
+
super(arg, alignedByteOffset, alignedLength);
|
|
2567
|
+
}
|
|
2568
|
+
}
|
|
2569
|
+
else {
|
|
2570
|
+
super(arg, byteOffset, length);
|
|
2571
|
+
}
|
|
2572
|
+
}
|
|
2573
|
+
};
|
|
2574
|
+
// Preserve static methods and properties
|
|
2575
|
+
Object.setPrototypeOf(global.Float32Array, originalFloat32Array);
|
|
2576
|
+
Object.defineProperty(global.Float32Array, 'name', {
|
|
2577
|
+
value: 'Float32Array'
|
|
2578
|
+
});
|
|
2579
|
+
Object.defineProperty(global.Float32Array, 'BYTES_PER_ELEMENT', {
|
|
2580
|
+
value: 4
|
|
2581
|
+
});
|
|
2582
|
+
}
|
|
2583
|
+
// CRITICAL: Patch any empty util shims that bundlers might create
|
|
2584
|
+
// This handles cases where bundlers provide empty shims for Node.js modules
|
|
2585
|
+
if (typeof global !== 'undefined') {
|
|
2586
|
+
// Look for common patterns of util shims in bundled code
|
|
2587
|
+
const checkAndPatchUtilShim = (obj) => {
|
|
2588
|
+
if (obj && typeof obj === 'object' && !obj.TextEncoder) {
|
|
2589
|
+
obj.TextEncoder = TextEncoder;
|
|
2590
|
+
obj.TextDecoder = TextDecoder;
|
|
2591
|
+
obj.types = obj.types || {
|
|
2592
|
+
isFloat32Array: (arr) => arr instanceof Float32Array,
|
|
2593
|
+
isInt32Array: (arr) => arr instanceof Int32Array,
|
|
2594
|
+
isUint8Array: (arr) => arr instanceof Uint8Array,
|
|
2595
|
+
isUint8ClampedArray: (arr) => arr instanceof Uint8ClampedArray
|
|
2511
2596
|
};
|
|
2512
|
-
// Initialize using the constructors from util
|
|
2513
|
-
this.textEncoder = new TextEncoder();
|
|
2514
|
-
this.textDecoder = new TextDecoder();
|
|
2515
2597
|
}
|
|
2598
|
+
};
|
|
2599
|
+
// Patch any existing util-like objects in global scope
|
|
2600
|
+
if (global._utilShim) {
|
|
2601
|
+
checkAndPatchUtilShim(global._utilShim);
|
|
2602
|
+
}
|
|
2603
|
+
// CRITICAL: Patch the bundled util shim directly
|
|
2604
|
+
// In bundled code, there's often a _utilShim object that needs patching
|
|
2605
|
+
if (typeof globalThis !== 'undefined' &&
|
|
2606
|
+
globalThis._utilShim) {
|
|
2607
|
+
checkAndPatchUtilShim(globalThis._utilShim);
|
|
2608
|
+
}
|
|
2609
|
+
// CRITICAL: Create and patch a global _utilShim if it doesn't exist
|
|
2610
|
+
// This ensures the bundled code will find the patched version
|
|
2611
|
+
if (!global._utilShim) {
|
|
2612
|
+
global._utilShim = {
|
|
2613
|
+
TextEncoder: TextEncoder,
|
|
2614
|
+
TextDecoder: TextDecoder,
|
|
2615
|
+
types: {
|
|
2616
|
+
isFloat32Array: (arr) => arr instanceof Float32Array,
|
|
2617
|
+
isInt32Array: (arr) => arr instanceof Int32Array,
|
|
2618
|
+
isUint8Array: (arr) => arr instanceof Uint8Array,
|
|
2619
|
+
isUint8ClampedArray: (arr) => arr instanceof Uint8ClampedArray
|
|
2620
|
+
}
|
|
2621
|
+
};
|
|
2622
|
+
}
|
|
2623
|
+
else {
|
|
2624
|
+
checkAndPatchUtilShim(global._utilShim);
|
|
2625
|
+
}
|
|
2626
|
+
// Also ensure it's available on globalThis
|
|
2627
|
+
if (typeof globalThis !== 'undefined' &&
|
|
2628
|
+
!globalThis._utilShim) {
|
|
2629
|
+
;
|
|
2630
|
+
globalThis._utilShim = global._utilShim;
|
|
2631
|
+
}
|
|
2632
|
+
// Set up a property descriptor to catch util shim assignments
|
|
2633
|
+
try {
|
|
2634
|
+
Object.defineProperty(global, '_utilShim', {
|
|
2635
|
+
get() {
|
|
2636
|
+
return this.__utilShim || {};
|
|
2637
|
+
},
|
|
2638
|
+
set(value) {
|
|
2639
|
+
checkAndPatchUtilShim(value);
|
|
2640
|
+
this.__utilShim = value;
|
|
2641
|
+
},
|
|
2642
|
+
configurable: true
|
|
2643
|
+
});
|
|
2644
|
+
}
|
|
2645
|
+
catch (e) {
|
|
2646
|
+
// Ignore if property can't be defined
|
|
2647
|
+
}
|
|
2648
|
+
// Also set up property descriptor on globalThis
|
|
2649
|
+
try {
|
|
2650
|
+
Object.defineProperty(globalThis, '_utilShim', {
|
|
2651
|
+
get() {
|
|
2652
|
+
return this.__utilShim || {};
|
|
2653
|
+
},
|
|
2654
|
+
set(value) {
|
|
2655
|
+
checkAndPatchUtilShim(value);
|
|
2656
|
+
this.__utilShim = value;
|
|
2657
|
+
},
|
|
2658
|
+
configurable: true
|
|
2659
|
+
});
|
|
2660
|
+
}
|
|
2661
|
+
catch (e) {
|
|
2662
|
+
// Ignore if property can't be defined
|
|
2516
2663
|
}
|
|
2517
|
-
|
|
2518
|
-
|
|
2519
|
-
|
|
2520
|
-
|
|
2664
|
+
}
|
|
2665
|
+
console.log('Brainy: Successfully patched TensorFlow.js PlatformNode at module load time');
|
|
2666
|
+
patchApplied = true;
|
|
2667
|
+
}
|
|
2668
|
+
catch (error) {
|
|
2669
|
+
console.warn('Brainy: Failed to apply early TensorFlow.js platform patch:', error);
|
|
2670
|
+
}
|
|
2671
|
+
}
|
|
2672
|
+
/**
|
|
2673
|
+
* Apply the TensorFlow.js platform patch if it hasn't been applied already
|
|
2674
|
+
* This is a safety measure in case the module-level patch didn't run
|
|
2675
|
+
* Now works across all environments: browser, Node.js, and serverless/server
|
|
2676
|
+
*/
|
|
2677
|
+
async function applyTensorFlowPatch() {
|
|
2678
|
+
// Apply patches for all non-browser environments that might need TensorFlow.js compatibility
|
|
2679
|
+
// This includes Node.js, serverless environments, and other server environments
|
|
2680
|
+
const isBrowserEnv = typeof window !== 'undefined' && typeof document !== 'undefined';
|
|
2681
|
+
if (isBrowserEnv) {
|
|
2682
|
+
return; // Browser environments don't need these patches
|
|
2683
|
+
}
|
|
2684
|
+
// Get the appropriate global object for the current environment
|
|
2685
|
+
const globalObj = (() => {
|
|
2686
|
+
if (typeof globalThis !== 'undefined')
|
|
2687
|
+
return globalThis;
|
|
2688
|
+
if (typeof global !== 'undefined')
|
|
2689
|
+
return global;
|
|
2690
|
+
if (typeof self !== 'undefined')
|
|
2691
|
+
return self;
|
|
2692
|
+
return {}; // Fallback for unknown environments
|
|
2693
|
+
})();
|
|
2694
|
+
// Check if the critical globals exist, not just the flag
|
|
2695
|
+
// This allows re-patching if globals have been deleted
|
|
2696
|
+
const needsPatch = !patchApplied ||
|
|
2697
|
+
typeof globalObj.__TextEncoder__ === 'undefined' ||
|
|
2698
|
+
typeof globalObj.__TextDecoder__ === 'undefined';
|
|
2699
|
+
if (!needsPatch) {
|
|
2700
|
+
return;
|
|
2701
|
+
}
|
|
2702
|
+
try {
|
|
2703
|
+
console.log('Brainy: Applying TensorFlow.js platform patch via function call');
|
|
2704
|
+
// CRITICAL FIX: Patch the global environment to ensure TextEncoder/TextDecoder are available
|
|
2705
|
+
// This approach works by ensuring the global constructors are available before TensorFlow.js loads
|
|
2706
|
+
// Now works across all environments: Node.js, serverless, and other server environments
|
|
2707
|
+
// Make sure TextEncoder and TextDecoder are available globally
|
|
2708
|
+
if (!globalObj.TextEncoder) {
|
|
2709
|
+
globalObj.TextEncoder = TextEncoder;
|
|
2710
|
+
}
|
|
2711
|
+
if (!globalObj.TextDecoder) {
|
|
2712
|
+
globalObj.TextDecoder = TextDecoder;
|
|
2713
|
+
}
|
|
2714
|
+
// Also set the special global constructors that TensorFlow can use safely
|
|
2715
|
+
;
|
|
2716
|
+
globalObj.__TextEncoder__ = TextEncoder;
|
|
2717
|
+
globalObj.__TextDecoder__ = TextDecoder;
|
|
2718
|
+
// Also patch process.versions to ensure TensorFlow.js detects Node.js correctly
|
|
2719
|
+
if (typeof process !== 'undefined' && process.versions) {
|
|
2720
|
+
// Ensure TensorFlow.js sees this as a Node.js environment
|
|
2721
|
+
if (!process.versions.node) {
|
|
2722
|
+
process.versions.node = process.version;
|
|
2723
|
+
}
|
|
2724
|
+
}
|
|
2725
|
+
// CRITICAL: Patch the Node.js util module directly
|
|
2726
|
+
try {
|
|
2727
|
+
const util = await Promise.resolve().then(function () { return _utilShim$1; });
|
|
2728
|
+
// Ensure TextEncoder and TextDecoder are available as constructors
|
|
2729
|
+
util.TextEncoder = TextEncoder;
|
|
2730
|
+
util.TextDecoder = TextDecoder;
|
|
2521
2731
|
}
|
|
2522
2732
|
catch (error) {
|
|
2523
|
-
|
|
2733
|
+
// Ignore if util module is not available
|
|
2524
2734
|
}
|
|
2735
|
+
patchApplied = true;
|
|
2736
|
+
}
|
|
2737
|
+
catch (error) {
|
|
2738
|
+
console.warn('Brainy: Failed to apply TensorFlow.js platform patch:', error);
|
|
2739
|
+
}
|
|
2740
|
+
}
|
|
2741
|
+
// Apply patch immediately
|
|
2742
|
+
applyTensorFlowPatch().catch((error) => {
|
|
2743
|
+
console.warn('Failed to apply TensorFlow patch at module load:', error);
|
|
2744
|
+
});
|
|
2745
|
+
|
|
2746
|
+
var textEncoding = /*#__PURE__*/Object.freeze({
|
|
2747
|
+
__proto__: null,
|
|
2748
|
+
applyTensorFlowPatch: applyTensorFlowPatch
|
|
2749
|
+
});
|
|
2750
|
+
|
|
2751
|
+
/**
|
|
2752
|
+
* CRITICAL: This file is imported for its side effects to patch the environment
|
|
2753
|
+
* for TensorFlow.js before any other library code runs.
|
|
2754
|
+
*
|
|
2755
|
+
* It ensures that by the time TensorFlow.js is imported by any other
|
|
2756
|
+
* module, the necessary compatibility fixes for the current Node.js
|
|
2757
|
+
* environment are already in place.
|
|
2758
|
+
*
|
|
2759
|
+
* This file MUST be imported as the first import in unified.ts to prevent
|
|
2760
|
+
* race conditions with TensorFlow.js initialization. Failure to do so will
|
|
2761
|
+
* result in errors like "TextEncoder is not a constructor" when the package
|
|
2762
|
+
* is used in Node.js environments.
|
|
2763
|
+
*
|
|
2764
|
+
* The package.json file marks this file as having side effects to prevent
|
|
2765
|
+
* tree-shaking by bundlers, ensuring the patch is always applied.
|
|
2766
|
+
*/
|
|
2767
|
+
// Get the appropriate global object for the current environment
|
|
2768
|
+
const globalObj = (() => {
|
|
2769
|
+
if (typeof globalThis !== 'undefined')
|
|
2770
|
+
return globalThis;
|
|
2771
|
+
if (typeof global !== 'undefined')
|
|
2772
|
+
return global;
|
|
2773
|
+
if (typeof self !== 'undefined')
|
|
2774
|
+
return self;
|
|
2775
|
+
return null; // No global object available
|
|
2776
|
+
})();
|
|
2777
|
+
// Define TextEncoder and TextDecoder globally to make sure they're available
|
|
2778
|
+
// Now works across all environments: Node.js, serverless, and other server environments
|
|
2779
|
+
if (globalObj) {
|
|
2780
|
+
if (!globalObj.TextEncoder) {
|
|
2781
|
+
globalObj.TextEncoder = TextEncoder;
|
|
2525
2782
|
}
|
|
2783
|
+
if (!globalObj.TextDecoder) {
|
|
2784
|
+
globalObj.TextDecoder = TextDecoder;
|
|
2785
|
+
}
|
|
2786
|
+
globalObj.__TextEncoder__ = TextEncoder;
|
|
2787
|
+
globalObj.__TextDecoder__ = TextDecoder;
|
|
2526
2788
|
}
|
|
2789
|
+
// Apply the TensorFlow.js platform patch
|
|
2790
|
+
applyTensorFlowPatch();
|
|
2791
|
+
console.log('Applied TensorFlow.js patch via ES modules in setup.ts');
|
|
2527
2792
|
|
|
2528
2793
|
// Unique ID creation requires a high quality random # generator. In the browser we therefore
|
|
2529
2794
|
// require the crypto API and do not support built-in fallback to lower quality random number
|
|
@@ -2683,12 +2948,31 @@ async function calculateDistancesBatch(queryVector, vectors, distanceFunction =
|
|
|
2683
2948
|
tf = await self.importTensorFlow();
|
|
2684
2949
|
}
|
|
2685
2950
|
else {
|
|
2686
|
-
//
|
|
2687
|
-
|
|
2688
|
-
|
|
2689
|
-
|
|
2690
|
-
|
|
2691
|
-
|
|
2951
|
+
// CRITICAL: Ensure TextEncoder/TextDecoder are available before TensorFlow.js loads
|
|
2952
|
+
try {
|
|
2953
|
+
// Use dynamic imports for all environments to ensure TensorFlow loads after patch
|
|
2954
|
+
if (typeof process !== 'undefined' && process.versions && process.versions.node) {
|
|
2955
|
+
// Ensure TextEncoder/TextDecoder are globally available in Node.js
|
|
2956
|
+
const util = await Promise.resolve().then(function () { return _utilShim$1; });
|
|
2957
|
+
if (typeof global.TextEncoder === 'undefined') {
|
|
2958
|
+
global.TextEncoder = util.TextEncoder;
|
|
2959
|
+
}
|
|
2960
|
+
if (typeof global.TextDecoder === 'undefined') {
|
|
2961
|
+
global.TextDecoder = util.TextDecoder;
|
|
2962
|
+
}
|
|
2963
|
+
}
|
|
2964
|
+
// Apply the TensorFlow.js patch
|
|
2965
|
+
const { applyTensorFlowPatch } = await Promise.resolve().then(function () { return textEncoding; });
|
|
2966
|
+
await applyTensorFlowPatch();
|
|
2967
|
+
// Now load TensorFlow.js core module using dynamic imports
|
|
2968
|
+
tf = await Promise.resolve().then(function () { return index$2; });
|
|
2969
|
+
await Promise.resolve().then(function () { return index$1; });
|
|
2970
|
+
await tf.setBackend('cpu');
|
|
2971
|
+
}
|
|
2972
|
+
catch (error) {
|
|
2973
|
+
console.error('Failed to initialize TensorFlow.js:', error);
|
|
2974
|
+
throw error;
|
|
2975
|
+
}
|
|
2692
2976
|
}
|
|
2693
2977
|
// Convert vectors to tensors
|
|
2694
2978
|
const queryTensor = tf.tensor2d([queryVector]);
|
|
@@ -2800,76 +3084,6 @@ async function calculateDistancesBatch(queryVector, vectors, distanceFunction =
|
|
|
2800
3084
|
}
|
|
2801
3085
|
}
|
|
2802
3086
|
|
|
2803
|
-
/**
|
|
2804
|
-
* Utility functions for environment detection
|
|
2805
|
-
*/
|
|
2806
|
-
/**
|
|
2807
|
-
* Check if code is running in a browser environment
|
|
2808
|
-
*/
|
|
2809
|
-
function isBrowser$1() {
|
|
2810
|
-
return typeof window !== 'undefined' && typeof document !== 'undefined';
|
|
2811
|
-
}
|
|
2812
|
-
/**
|
|
2813
|
-
* Check if code is running in a Node.js environment
|
|
2814
|
-
*/
|
|
2815
|
-
function isNode() {
|
|
2816
|
-
return (typeof process !== 'undefined' &&
|
|
2817
|
-
process.versions != null &&
|
|
2818
|
-
process.versions.node != null);
|
|
2819
|
-
}
|
|
2820
|
-
/**
|
|
2821
|
-
* Check if code is running in a Web Worker environment
|
|
2822
|
-
*/
|
|
2823
|
-
function isWebWorker() {
|
|
2824
|
-
return (typeof self === 'object' &&
|
|
2825
|
-
self.constructor &&
|
|
2826
|
-
self.constructor.name === 'DedicatedWorkerGlobalScope');
|
|
2827
|
-
}
|
|
2828
|
-
/**
|
|
2829
|
-
* Check if Web Workers are available in the current environment
|
|
2830
|
-
*/
|
|
2831
|
-
function areWebWorkersAvailable() {
|
|
2832
|
-
return isBrowser$1() && typeof Worker !== 'undefined';
|
|
2833
|
-
}
|
|
2834
|
-
/**
|
|
2835
|
-
* Check if Worker Threads are available in the current environment (Node.js)
|
|
2836
|
-
*/
|
|
2837
|
-
async function areWorkerThreadsAvailable() {
|
|
2838
|
-
if (!isNode())
|
|
2839
|
-
return false;
|
|
2840
|
-
try {
|
|
2841
|
-
// Use dynamic import to avoid errors in browser environments
|
|
2842
|
-
await import('worker_threads');
|
|
2843
|
-
return true;
|
|
2844
|
-
}
|
|
2845
|
-
catch (e) {
|
|
2846
|
-
return false;
|
|
2847
|
-
}
|
|
2848
|
-
}
|
|
2849
|
-
/**
|
|
2850
|
-
* Synchronous version that doesn't actually try to load the module
|
|
2851
|
-
* This is safer in ES module environments
|
|
2852
|
-
*/
|
|
2853
|
-
function areWorkerThreadsAvailableSync() {
|
|
2854
|
-
if (!isNode())
|
|
2855
|
-
return false;
|
|
2856
|
-
// In Node.js 24.3.0+, worker_threads is always available
|
|
2857
|
-
return parseInt(process.versions.node.split('.')[0]) >= 24;
|
|
2858
|
-
}
|
|
2859
|
-
/**
|
|
2860
|
-
* Determine if threading is available in the current environment
|
|
2861
|
-
* Returns true if either Web Workers (browser) or Worker Threads (Node.js) are available
|
|
2862
|
-
*/
|
|
2863
|
-
function isThreadingAvailable() {
|
|
2864
|
-
return areWebWorkersAvailable() || areWorkerThreadsAvailableSync();
|
|
2865
|
-
}
|
|
2866
|
-
/**
|
|
2867
|
-
* Async version of isThreadingAvailable
|
|
2868
|
-
*/
|
|
2869
|
-
async function isThreadingAvailableAsync() {
|
|
2870
|
-
return areWebWorkersAvailable() || (await areWorkerThreadsAvailable());
|
|
2871
|
-
}
|
|
2872
|
-
|
|
2873
3087
|
/**
|
|
2874
3088
|
* Utility functions for executing functions in Worker Threads (Node.js) or Web Workers (Browser)
|
|
2875
3089
|
* This implementation leverages Node.js 24's improved Worker Threads API for better performance
|
|
@@ -2913,9 +3127,16 @@ function executeInThread(fnString, args) {
|
|
|
2913
3127
|
fn = new Function(fnString)();
|
|
2914
3128
|
}
|
|
2915
3129
|
catch (directError) {
|
|
2916
|
-
console.
|
|
2917
|
-
|
|
2918
|
-
|
|
3130
|
+
console.warn('Fallback: Direct approach failed, trying with function wrapper', directError);
|
|
3131
|
+
try {
|
|
3132
|
+
// Try wrapping in a function that returns the function expression
|
|
3133
|
+
fn = new Function('return function(args) { return (' + fnString + ')(args); }')();
|
|
3134
|
+
}
|
|
3135
|
+
catch (wrapperError) {
|
|
3136
|
+
console.error('Fallback: All approaches to create function failed', wrapperError);
|
|
3137
|
+
throw new Error('Failed to create function from string: ' +
|
|
3138
|
+
functionError.message);
|
|
3139
|
+
}
|
|
2919
3140
|
}
|
|
2920
3141
|
}
|
|
2921
3142
|
}
|
|
@@ -2950,6 +3171,82 @@ function executeInNodeWorker(fnString, args) {
|
|
|
2950
3171
|
// Create a new worker
|
|
2951
3172
|
worker = new Worker(`
|
|
2952
3173
|
import { parentPort, workerData } from 'node:worker_threads';
|
|
3174
|
+
|
|
3175
|
+
// Add TensorFlow.js platform patch for Node.js
|
|
3176
|
+
if (typeof global !== 'undefined') {
|
|
3177
|
+
try {
|
|
3178
|
+
// Define a custom PlatformNode class
|
|
3179
|
+
class PlatformNode {
|
|
3180
|
+
constructor() {
|
|
3181
|
+
// Create a util object with necessary methods
|
|
3182
|
+
this.util = {
|
|
3183
|
+
// Add isFloat32Array and isTypedArray directly to util
|
|
3184
|
+
isFloat32Array: (arr) => {
|
|
3185
|
+
return !!(
|
|
3186
|
+
arr instanceof Float32Array ||
|
|
3187
|
+
(arr &&
|
|
3188
|
+
Object.prototype.toString.call(arr) === '[object Float32Array]')
|
|
3189
|
+
);
|
|
3190
|
+
},
|
|
3191
|
+
isTypedArray: (arr) => {
|
|
3192
|
+
return !!(ArrayBuffer.isView(arr) && !(arr instanceof DataView));
|
|
3193
|
+
},
|
|
3194
|
+
// Use native TextEncoder and TextDecoder
|
|
3195
|
+
TextEncoder: TextEncoder,
|
|
3196
|
+
TextDecoder: TextDecoder
|
|
3197
|
+
};
|
|
3198
|
+
|
|
3199
|
+
// Initialize encoders using native constructors
|
|
3200
|
+
this.textEncoder = new TextEncoder();
|
|
3201
|
+
this.textDecoder = new TextDecoder();
|
|
3202
|
+
}
|
|
3203
|
+
|
|
3204
|
+
// Define isFloat32Array directly on the instance
|
|
3205
|
+
isFloat32Array(arr) {
|
|
3206
|
+
return !!(
|
|
3207
|
+
arr instanceof Float32Array ||
|
|
3208
|
+
(arr && Object.prototype.toString.call(arr) === '[object Float32Array]')
|
|
3209
|
+
);
|
|
3210
|
+
}
|
|
3211
|
+
|
|
3212
|
+
// Define isTypedArray directly on the instance
|
|
3213
|
+
isTypedArray(arr) {
|
|
3214
|
+
return !!(ArrayBuffer.isView(arr) && !(arr instanceof DataView));
|
|
3215
|
+
}
|
|
3216
|
+
}
|
|
3217
|
+
|
|
3218
|
+
// Assign the PlatformNode class to the global object
|
|
3219
|
+
global.PlatformNode = PlatformNode;
|
|
3220
|
+
|
|
3221
|
+
// Also create an instance and assign it to global.platformNode
|
|
3222
|
+
global.platformNode = new PlatformNode();
|
|
3223
|
+
|
|
3224
|
+
// Ensure global.util exists and has the necessary methods
|
|
3225
|
+
if (!global.util) {
|
|
3226
|
+
global.util = {};
|
|
3227
|
+
}
|
|
3228
|
+
|
|
3229
|
+
// Add isFloat32Array method if it doesn't exist
|
|
3230
|
+
if (!global.util.isFloat32Array) {
|
|
3231
|
+
global.util.isFloat32Array = (arr) => {
|
|
3232
|
+
return !!(
|
|
3233
|
+
arr instanceof Float32Array ||
|
|
3234
|
+
(arr && Object.prototype.toString.call(arr) === '[object Float32Array]')
|
|
3235
|
+
);
|
|
3236
|
+
};
|
|
3237
|
+
}
|
|
3238
|
+
|
|
3239
|
+
// Add isTypedArray method if it doesn't exist
|
|
3240
|
+
if (!global.util.isTypedArray) {
|
|
3241
|
+
global.util.isTypedArray = (arr) => {
|
|
3242
|
+
return !!(ArrayBuffer.isView(arr) && !(arr instanceof DataView));
|
|
3243
|
+
};
|
|
3244
|
+
}
|
|
3245
|
+
} catch (error) {
|
|
3246
|
+
console.warn('Failed to apply TensorFlow.js platform patch:', error);
|
|
3247
|
+
}
|
|
3248
|
+
}
|
|
3249
|
+
|
|
2953
3250
|
const fn = new Function('return ' + workerData.fnString)();
|
|
2954
3251
|
const result = fn(workerData.args);
|
|
2955
3252
|
parentPort.postMessage({ result });
|
|
@@ -2969,6 +3266,71 @@ function executeInNodeWorker(fnString, args) {
|
|
|
2969
3266
|
worker.terminate();
|
|
2970
3267
|
worker = new Worker(`
|
|
2971
3268
|
import { parentPort, workerData } from 'node:worker_threads';
|
|
3269
|
+
|
|
3270
|
+
// Add TensorFlow.js platform patch for Node.js
|
|
3271
|
+
if (typeof global !== 'undefined') {
|
|
3272
|
+
try {
|
|
3273
|
+
// Define a custom PlatformNode class
|
|
3274
|
+
class PlatformNode {
|
|
3275
|
+
constructor() {
|
|
3276
|
+
// Create a util object with necessary methods
|
|
3277
|
+
this.util = {
|
|
3278
|
+
// Use native TextEncoder and TextDecoder
|
|
3279
|
+
TextEncoder: TextEncoder,
|
|
3280
|
+
TextDecoder: TextDecoder
|
|
3281
|
+
};
|
|
3282
|
+
|
|
3283
|
+
// Initialize encoders using native constructors
|
|
3284
|
+
this.textEncoder = new TextEncoder();
|
|
3285
|
+
this.textDecoder = new TextDecoder();
|
|
3286
|
+
}
|
|
3287
|
+
|
|
3288
|
+
// Define isFloat32Array directly on the instance
|
|
3289
|
+
isFloat32Array(arr) {
|
|
3290
|
+
return !!(
|
|
3291
|
+
arr instanceof Float32Array ||
|
|
3292
|
+
(arr && Object.prototype.toString.call(arr) === '[object Float32Array]')
|
|
3293
|
+
);
|
|
3294
|
+
}
|
|
3295
|
+
|
|
3296
|
+
// Define isTypedArray directly on the instance
|
|
3297
|
+
isTypedArray(arr) {
|
|
3298
|
+
return !!(ArrayBuffer.isView(arr) && !(arr instanceof DataView));
|
|
3299
|
+
}
|
|
3300
|
+
}
|
|
3301
|
+
|
|
3302
|
+
// Assign the PlatformNode class to the global object
|
|
3303
|
+
global.PlatformNode = PlatformNode;
|
|
3304
|
+
|
|
3305
|
+
// Also create an instance and assign it to global.platformNode
|
|
3306
|
+
global.platformNode = new PlatformNode();
|
|
3307
|
+
|
|
3308
|
+
// Ensure global.util exists and has the necessary methods
|
|
3309
|
+
if (!global.util) {
|
|
3310
|
+
global.util = {};
|
|
3311
|
+
}
|
|
3312
|
+
|
|
3313
|
+
// Add isFloat32Array method if it doesn't exist
|
|
3314
|
+
if (!global.util.isFloat32Array) {
|
|
3315
|
+
global.util.isFloat32Array = (arr) => {
|
|
3316
|
+
return !!(
|
|
3317
|
+
arr instanceof Float32Array ||
|
|
3318
|
+
(arr && Object.prototype.toString.call(arr) === '[object Float32Array]')
|
|
3319
|
+
);
|
|
3320
|
+
};
|
|
3321
|
+
}
|
|
3322
|
+
|
|
3323
|
+
// Add isTypedArray method if it doesn't exist
|
|
3324
|
+
if (!global.util.isTypedArray) {
|
|
3325
|
+
global.util.isTypedArray = (arr) => {
|
|
3326
|
+
return !!(ArrayBuffer.isView(arr) && !(arr instanceof DataView));
|
|
3327
|
+
};
|
|
3328
|
+
}
|
|
3329
|
+
} catch (error) {
|
|
3330
|
+
console.warn('Failed to apply TensorFlow.js platform patch:', error);
|
|
3331
|
+
}
|
|
3332
|
+
}
|
|
3333
|
+
|
|
2972
3334
|
const fn = new Function('return ' + workerData.fnString)();
|
|
2973
3335
|
const result = fn(workerData.args);
|
|
2974
3336
|
parentPort.postMessage({ result });
|
|
@@ -3200,16 +3562,126 @@ let UniversalSentenceEncoder$1 = class UniversalSentenceEncoder {
|
|
|
3200
3562
|
}
|
|
3201
3563
|
/**
|
|
3202
3564
|
* Add polyfills and patches for TensorFlow.js compatibility
|
|
3203
|
-
* This addresses issues with TensorFlow.js
|
|
3565
|
+
* This addresses issues with TensorFlow.js across all server environments
|
|
3566
|
+
* (Node.js, serverless, and other server environments)
|
|
3567
|
+
*
|
|
3568
|
+
* Note: The main TensorFlow.js patching is now centralized in textEncoding.ts
|
|
3569
|
+
* and applied through setup.ts. This method only adds additional utility functions
|
|
3570
|
+
* that might be needed by TensorFlow.js.
|
|
3204
3571
|
*/
|
|
3205
|
-
|
|
3206
|
-
//
|
|
3207
|
-
|
|
3208
|
-
|
|
3209
|
-
|
|
3210
|
-
|
|
3572
|
+
addServerCompatibilityPolyfills() {
|
|
3573
|
+
// Apply in all non-browser environments (Node.js, serverless, server environments)
|
|
3574
|
+
const isBrowserEnv = typeof window !== 'undefined' && typeof document !== 'undefined';
|
|
3575
|
+
if (isBrowserEnv) {
|
|
3576
|
+
return; // Browser environments don't need these polyfills
|
|
3577
|
+
}
|
|
3578
|
+
// Get the appropriate global object for the current environment
|
|
3579
|
+
const globalObj = (() => {
|
|
3580
|
+
if (typeof globalThis !== 'undefined')
|
|
3581
|
+
return globalThis;
|
|
3582
|
+
if (typeof global !== 'undefined')
|
|
3583
|
+
return global;
|
|
3584
|
+
if (typeof self !== 'undefined')
|
|
3585
|
+
return self;
|
|
3586
|
+
return {}; // Fallback for unknown environments
|
|
3587
|
+
})();
|
|
3588
|
+
// Add polyfill for utility functions across all server environments
|
|
3589
|
+
// This fixes issues like "Cannot read properties of undefined (reading 'isFloat32Array')"
|
|
3590
|
+
try {
|
|
3591
|
+
// Ensure the util object exists
|
|
3592
|
+
if (!globalObj.util) {
|
|
3593
|
+
globalObj.util = {};
|
|
3594
|
+
}
|
|
3595
|
+
// Add isFloat32Array method if it doesn't exist
|
|
3596
|
+
if (!globalObj.util.isFloat32Array) {
|
|
3597
|
+
globalObj.util.isFloat32Array = (obj) => {
|
|
3598
|
+
return !!(obj instanceof Float32Array ||
|
|
3599
|
+
(obj &&
|
|
3600
|
+
Object.prototype.toString.call(obj) === '[object Float32Array]'));
|
|
3601
|
+
};
|
|
3602
|
+
}
|
|
3603
|
+
// Add isTypedArray method if it doesn't exist
|
|
3604
|
+
if (!globalObj.util.isTypedArray) {
|
|
3605
|
+
globalObj.util.isTypedArray = (obj) => {
|
|
3606
|
+
return !!(ArrayBuffer.isView(obj) && !(obj instanceof DataView));
|
|
3607
|
+
};
|
|
3608
|
+
}
|
|
3609
|
+
}
|
|
3610
|
+
catch (error) {
|
|
3611
|
+
console.warn('Failed to add utility polyfills:', error);
|
|
3612
|
+
}
|
|
3613
|
+
}
|
|
3614
|
+
/**
|
|
3615
|
+
* Check if we're running in a test environment
|
|
3616
|
+
*/
|
|
3617
|
+
isTestEnvironment() {
|
|
3618
|
+
// Safely check for Node.js environment first
|
|
3619
|
+
if (typeof process === 'undefined') {
|
|
3620
|
+
return false;
|
|
3621
|
+
}
|
|
3622
|
+
return (process.env.VITEST === 'true' ||
|
|
3623
|
+
(typeof global !== 'undefined' && global.__vitest__) ||
|
|
3624
|
+
process.argv.some((arg) => arg.includes('vitest')));
|
|
3625
|
+
}
|
|
3626
|
+
/**
|
|
3627
|
+
* Log message only if not in test environment
|
|
3628
|
+
*/
|
|
3629
|
+
logIfNotTest(level, message, ...args) {
|
|
3630
|
+
if (!this.isTestEnvironment()) {
|
|
3631
|
+
console[level](message, ...args);
|
|
3632
|
+
}
|
|
3633
|
+
}
|
|
3634
|
+
/**
|
|
3635
|
+
* Load the Universal Sentence Encoder model with retry logic
|
|
3636
|
+
* This helps handle network failures and JSON parsing errors from TensorFlow Hub
|
|
3637
|
+
* @param loadFunction The function to load the model
|
|
3638
|
+
* @param maxRetries Maximum number of retry attempts
|
|
3639
|
+
* @param baseDelay Base delay in milliseconds for exponential backoff
|
|
3640
|
+
*/
|
|
3641
|
+
async loadModelWithRetry(loadFunction, maxRetries = 3, baseDelay = 1000) {
|
|
3642
|
+
let lastError = null;
|
|
3643
|
+
for (let attempt = 0; attempt <= maxRetries; attempt++) {
|
|
3644
|
+
try {
|
|
3645
|
+
this.logIfNotTest('log', attempt === 0
|
|
3646
|
+
? 'Loading Universal Sentence Encoder model...'
|
|
3647
|
+
: `Retrying Universal Sentence Encoder model loading (attempt ${attempt + 1}/${maxRetries + 1})...`);
|
|
3648
|
+
const model = await loadFunction();
|
|
3649
|
+
if (attempt > 0) {
|
|
3650
|
+
this.logIfNotTest('log', 'Universal Sentence Encoder model loaded successfully after retry');
|
|
3651
|
+
}
|
|
3652
|
+
return model;
|
|
3653
|
+
}
|
|
3654
|
+
catch (error) {
|
|
3655
|
+
lastError = error;
|
|
3656
|
+
const errorMessage = lastError.message || String(lastError);
|
|
3657
|
+
// Check if this is a network-related error that might benefit from retry
|
|
3658
|
+
const isRetryableError = errorMessage.includes('Failed to parse model JSON') ||
|
|
3659
|
+
errorMessage.includes('Failed to fetch') ||
|
|
3660
|
+
errorMessage.includes('Network error') ||
|
|
3661
|
+
errorMessage.includes('ENOTFOUND') ||
|
|
3662
|
+
errorMessage.includes('ECONNRESET') ||
|
|
3663
|
+
errorMessage.includes('ETIMEDOUT') ||
|
|
3664
|
+
errorMessage.includes('JSON') ||
|
|
3665
|
+
errorMessage.includes('model.json');
|
|
3666
|
+
if (attempt < maxRetries && isRetryableError) {
|
|
3667
|
+
const delay = baseDelay * Math.pow(2, attempt); // Exponential backoff
|
|
3668
|
+
this.logIfNotTest('warn', `Universal Sentence Encoder model loading failed (attempt ${attempt + 1}): ${errorMessage}. Retrying in ${delay}ms...`);
|
|
3669
|
+
await new Promise((resolve) => setTimeout(resolve, delay));
|
|
3670
|
+
}
|
|
3671
|
+
else {
|
|
3672
|
+
// Either we've exhausted retries or this is not a retryable error
|
|
3673
|
+
if (attempt >= maxRetries) {
|
|
3674
|
+
this.logIfNotTest('error', `Universal Sentence Encoder model loading failed after ${maxRetries + 1} attempts. Last error: ${errorMessage}`);
|
|
3675
|
+
}
|
|
3676
|
+
else {
|
|
3677
|
+
this.logIfNotTest('error', `Universal Sentence Encoder model loading failed with non-retryable error: ${errorMessage}`);
|
|
3678
|
+
}
|
|
3679
|
+
throw lastError;
|
|
3680
|
+
}
|
|
3681
|
+
}
|
|
3211
3682
|
}
|
|
3212
|
-
//
|
|
3683
|
+
// This should never be reached, but just in case
|
|
3684
|
+
throw lastError || new Error('Unknown error during model loading');
|
|
3213
3685
|
}
|
|
3214
3686
|
/**
|
|
3215
3687
|
* Initialize the embedding model
|
|
@@ -3228,43 +3700,87 @@ let UniversalSentenceEncoder$1 = class UniversalSentenceEncoder {
|
|
|
3228
3700
|
originalWarn(message, ...optionalParams);
|
|
3229
3701
|
};
|
|
3230
3702
|
// Add polyfills for TensorFlow.js compatibility
|
|
3231
|
-
this.
|
|
3703
|
+
this.addServerCompatibilityPolyfills();
|
|
3232
3704
|
// TensorFlow.js will use its default EPSILON value
|
|
3233
|
-
//
|
|
3234
|
-
// Use type assertions to tell TypeScript these modules exist
|
|
3235
|
-
this.tf = await Promise.resolve().then(function () { return index$2; });
|
|
3236
|
-
// Import CPU backend (always needed as fallback)
|
|
3237
|
-
await Promise.resolve().then(function () { return index$1; });
|
|
3238
|
-
// Try to import WebGL backend for GPU acceleration in browser environments
|
|
3705
|
+
// CRITICAL: Ensure TextEncoder/TextDecoder are available before TensorFlow.js loads
|
|
3239
3706
|
try {
|
|
3240
|
-
|
|
3241
|
-
|
|
3242
|
-
|
|
3707
|
+
// Get the appropriate global object for the current environment
|
|
3708
|
+
const globalObj = (() => {
|
|
3709
|
+
if (typeof globalThis !== 'undefined')
|
|
3710
|
+
return globalThis;
|
|
3711
|
+
if (typeof global !== 'undefined')
|
|
3712
|
+
return global;
|
|
3713
|
+
if (typeof self !== 'undefined')
|
|
3714
|
+
return self;
|
|
3715
|
+
return null;
|
|
3716
|
+
})();
|
|
3717
|
+
// Ensure TextEncoder/TextDecoder are globally available in server environments
|
|
3718
|
+
if (globalObj) {
|
|
3719
|
+
// Try to use Node.js util module if available (Node.js environments)
|
|
3243
3720
|
try {
|
|
3244
|
-
if (
|
|
3245
|
-
await
|
|
3246
|
-
|
|
3247
|
-
|
|
3721
|
+
if (typeof process !== 'undefined' && process.versions && process.versions.node) {
|
|
3722
|
+
const util = await Promise.resolve().then(function () { return _utilShim$1; });
|
|
3723
|
+
if (!globalObj.TextEncoder) {
|
|
3724
|
+
globalObj.TextEncoder = util.TextEncoder;
|
|
3725
|
+
}
|
|
3726
|
+
if (!globalObj.TextDecoder) {
|
|
3727
|
+
globalObj.TextDecoder = util.TextDecoder;
|
|
3728
|
+
}
|
|
3248
3729
|
}
|
|
3249
|
-
|
|
3250
|
-
|
|
3730
|
+
}
|
|
3731
|
+
catch (utilError) {
|
|
3732
|
+
// Fallback to standard TextEncoder/TextDecoder for non-Node.js server environments
|
|
3733
|
+
if (!globalObj.TextEncoder) {
|
|
3734
|
+
globalObj.TextEncoder = TextEncoder;
|
|
3735
|
+
}
|
|
3736
|
+
if (!globalObj.TextDecoder) {
|
|
3737
|
+
globalObj.TextDecoder = TextDecoder;
|
|
3251
3738
|
}
|
|
3252
3739
|
}
|
|
3253
|
-
|
|
3254
|
-
|
|
3255
|
-
|
|
3740
|
+
}
|
|
3741
|
+
// Apply the TensorFlow.js patch
|
|
3742
|
+
const { applyTensorFlowPatch } = await Promise.resolve().then(function () { return textEncoding; });
|
|
3743
|
+
await applyTensorFlowPatch();
|
|
3744
|
+
// Now load TensorFlow.js core module using dynamic imports
|
|
3745
|
+
this.tf = await Promise.resolve().then(function () { return index$2; });
|
|
3746
|
+
// Import CPU backend (always needed as fallback)
|
|
3747
|
+
await Promise.resolve().then(function () { return index$1; });
|
|
3748
|
+
// Try to import WebGL backend for GPU acceleration in browser environments
|
|
3749
|
+
try {
|
|
3750
|
+
if (typeof window !== 'undefined') {
|
|
3751
|
+
await Promise.resolve().then(function () { return index; });
|
|
3752
|
+
// Check if WebGL is available
|
|
3753
|
+
try {
|
|
3754
|
+
if (this.tf.setBackend) {
|
|
3755
|
+
await this.tf.setBackend('webgl');
|
|
3756
|
+
this.backend = 'webgl';
|
|
3757
|
+
console.log('Using WebGL backend for TensorFlow.js');
|
|
3758
|
+
}
|
|
3759
|
+
else {
|
|
3760
|
+
console.warn('tf.setBackend is not available, falling back to CPU');
|
|
3761
|
+
}
|
|
3762
|
+
}
|
|
3763
|
+
catch (e) {
|
|
3764
|
+
console.warn('WebGL backend not available, falling back to CPU:', e);
|
|
3765
|
+
this.backend = 'cpu';
|
|
3766
|
+
}
|
|
3256
3767
|
}
|
|
3257
3768
|
}
|
|
3769
|
+
catch (error) {
|
|
3770
|
+
console.warn('WebGL backend not available, falling back to CPU:', error);
|
|
3771
|
+
this.backend = 'cpu';
|
|
3772
|
+
}
|
|
3773
|
+
// Load Universal Sentence Encoder using dynamic import
|
|
3774
|
+
this.use = await Promise.resolve().then(function () { return universalSentenceEncoder_esm; });
|
|
3258
3775
|
}
|
|
3259
3776
|
catch (error) {
|
|
3260
|
-
|
|
3261
|
-
|
|
3777
|
+
this.logIfNotTest('error', 'Failed to initialize TensorFlow.js:', error);
|
|
3778
|
+
throw error;
|
|
3262
3779
|
}
|
|
3263
3780
|
// Set the backend
|
|
3264
3781
|
if (this.tf.setBackend) {
|
|
3265
3782
|
await this.tf.setBackend(this.backend);
|
|
3266
3783
|
}
|
|
3267
|
-
this.use = await Promise.resolve().then(function () { return universalSentenceEncoder_esm; });
|
|
3268
3784
|
// Log the module structure to help with debugging
|
|
3269
3785
|
console.log('Universal Sentence Encoder module structure in main thread:', Object.keys(this.use), this.use.default ? Object.keys(this.use.default) : 'No default export');
|
|
3270
3786
|
// Try to find the load function in different possible module structures
|
|
@@ -3272,14 +3788,14 @@ let UniversalSentenceEncoder$1 = class UniversalSentenceEncoder {
|
|
|
3272
3788
|
if (!loadFunction) {
|
|
3273
3789
|
throw new Error('Could not find Universal Sentence Encoder load function');
|
|
3274
3790
|
}
|
|
3275
|
-
// Load the model
|
|
3276
|
-
this.model = await loadFunction
|
|
3791
|
+
// Load the model with retry logic for network failures
|
|
3792
|
+
this.model = await this.loadModelWithRetry(loadFunction);
|
|
3277
3793
|
this.initialized = true;
|
|
3278
3794
|
// Restore original console.warn
|
|
3279
3795
|
console.warn = originalWarn;
|
|
3280
3796
|
}
|
|
3281
3797
|
catch (error) {
|
|
3282
|
-
|
|
3798
|
+
this.logIfNotTest('error', 'Failed to initialize Universal Sentence Encoder:', error);
|
|
3283
3799
|
throw new Error(`Failed to initialize Universal Sentence Encoder: ${error}`);
|
|
3284
3800
|
}
|
|
3285
3801
|
}
|
|
@@ -3326,7 +3842,7 @@ let UniversalSentenceEncoder$1 = class UniversalSentenceEncoder {
|
|
|
3326
3842
|
return embeddingArray[0];
|
|
3327
3843
|
}
|
|
3328
3844
|
catch (error) {
|
|
3329
|
-
|
|
3845
|
+
this.logIfNotTest('error', 'Failed to embed text with Universal Sentence Encoder:', error);
|
|
3330
3846
|
throw new Error(`Failed to embed text with Universal Sentence Encoder: ${error}`);
|
|
3331
3847
|
}
|
|
3332
3848
|
}
|
|
@@ -3375,7 +3891,7 @@ let UniversalSentenceEncoder$1 = class UniversalSentenceEncoder {
|
|
|
3375
3891
|
return results;
|
|
3376
3892
|
}
|
|
3377
3893
|
catch (error) {
|
|
3378
|
-
|
|
3894
|
+
this.logIfNotTest('error', 'Failed to batch embed text with Universal Sentence Encoder:', error);
|
|
3379
3895
|
throw new Error(`Failed to batch embed text with Universal Sentence Encoder: ${error}`);
|
|
3380
3896
|
}
|
|
3381
3897
|
}
|
|
@@ -3391,7 +3907,7 @@ let UniversalSentenceEncoder$1 = class UniversalSentenceEncoder {
|
|
|
3391
3907
|
this.initialized = false;
|
|
3392
3908
|
}
|
|
3393
3909
|
catch (error) {
|
|
3394
|
-
|
|
3910
|
+
this.logIfNotTest('error', 'Failed to dispose Universal Sentence Encoder:', error);
|
|
3395
3911
|
}
|
|
3396
3912
|
}
|
|
3397
3913
|
return Promise.resolve();
|
|
@@ -3479,11 +3995,35 @@ function findUSELoadFunction(sentenceEncoderModule) {
|
|
|
3479
3995
|
}
|
|
3480
3996
|
return loadFunction;
|
|
3481
3997
|
}
|
|
3998
|
+
/**
|
|
3999
|
+
* Check if we're running in a test environment (standalone version)
|
|
4000
|
+
*/
|
|
4001
|
+
function isTestEnvironment() {
|
|
4002
|
+
// Safely check for Node.js environment first
|
|
4003
|
+
if (typeof process === 'undefined') {
|
|
4004
|
+
return false;
|
|
4005
|
+
}
|
|
4006
|
+
return (process.env.VITEST === 'true' ||
|
|
4007
|
+
(typeof global !== 'undefined' && global.__vitest__) ||
|
|
4008
|
+
process.argv.some((arg) => arg.includes('vitest')));
|
|
4009
|
+
}
|
|
4010
|
+
/**
|
|
4011
|
+
* Log message only if not in test environment (standalone version)
|
|
4012
|
+
*/
|
|
4013
|
+
function logIfNotTest(level, message, ...args) {
|
|
4014
|
+
if (!isTestEnvironment()) {
|
|
4015
|
+
console[level](message, ...args);
|
|
4016
|
+
}
|
|
4017
|
+
}
|
|
3482
4018
|
/**
|
|
3483
4019
|
* Create an embedding function from an embedding model
|
|
3484
|
-
* @param model Embedding model to use
|
|
4020
|
+
* @param model Embedding model to use (optional, defaults to UniversalSentenceEncoder)
|
|
3485
4021
|
*/
|
|
3486
4022
|
function createEmbeddingFunction(model) {
|
|
4023
|
+
// If no model is provided, use the default TensorFlow embedding function
|
|
4024
|
+
if (!model) {
|
|
4025
|
+
return createTensorFlowEmbeddingFunction();
|
|
4026
|
+
}
|
|
3487
4027
|
return async (data) => {
|
|
3488
4028
|
return await model.embed(data);
|
|
3489
4029
|
};
|
|
@@ -3501,13 +4041,20 @@ function createTensorFlowEmbeddingFunction() {
|
|
|
3501
4041
|
try {
|
|
3502
4042
|
// Initialize the model if it hasn't been initialized yet
|
|
3503
4043
|
if (!sharedModelInitialized) {
|
|
3504
|
-
|
|
3505
|
-
|
|
4044
|
+
try {
|
|
4045
|
+
await sharedModel.init();
|
|
4046
|
+
sharedModelInitialized = true;
|
|
4047
|
+
}
|
|
4048
|
+
catch (initError) {
|
|
4049
|
+
// Reset the flag so we can retry initialization on the next call
|
|
4050
|
+
sharedModelInitialized = false;
|
|
4051
|
+
throw initError;
|
|
4052
|
+
}
|
|
3506
4053
|
}
|
|
3507
4054
|
return await sharedModel.embed(data);
|
|
3508
4055
|
}
|
|
3509
4056
|
catch (error) {
|
|
3510
|
-
|
|
4057
|
+
logIfNotTest('error', 'Failed to use TensorFlow embedding:', error);
|
|
3511
4058
|
throw new Error(`Universal Sentence Encoder is required but failed: ${error}`);
|
|
3512
4059
|
}
|
|
3513
4060
|
};
|
|
@@ -3539,7 +4086,7 @@ const defaultBatchEmbeddingFunction = async (dataArray) => {
|
|
|
3539
4086
|
return await sharedBatchModel.embedBatch(dataArray);
|
|
3540
4087
|
}
|
|
3541
4088
|
catch (error) {
|
|
3542
|
-
|
|
4089
|
+
logIfNotTest('error', 'Failed to use TensorFlow batch embedding:', error);
|
|
3543
4090
|
throw new Error(`Universal Sentence Encoder batch embedding failed: ${error}`);
|
|
3544
4091
|
}
|
|
3545
4092
|
};
|
|
@@ -7916,6 +8463,26 @@ async function createServerSearchAugmentations(serverUrl, options = {}) {
|
|
|
7916
8463
|
* Main class that provides the vector database functionality
|
|
7917
8464
|
*/
|
|
7918
8465
|
class BrainyData {
|
|
8466
|
+
/**
|
|
8467
|
+
* Get the vector dimensions
|
|
8468
|
+
*/
|
|
8469
|
+
get dimensions() {
|
|
8470
|
+
return this._dimensions;
|
|
8471
|
+
}
|
|
8472
|
+
/**
|
|
8473
|
+
* Get the maximum connections parameter from HNSW configuration
|
|
8474
|
+
*/
|
|
8475
|
+
get maxConnections() {
|
|
8476
|
+
const config = this.index.getConfig();
|
|
8477
|
+
return config.M || 16;
|
|
8478
|
+
}
|
|
8479
|
+
/**
|
|
8480
|
+
* Get the efConstruction parameter from HNSW configuration
|
|
8481
|
+
*/
|
|
8482
|
+
get efConstruction() {
|
|
8483
|
+
const config = this.index.getConfig();
|
|
8484
|
+
return config.efConstruction || 200;
|
|
8485
|
+
}
|
|
7919
8486
|
/**
|
|
7920
8487
|
* Create a new vector database
|
|
7921
8488
|
*/
|
|
@@ -7929,6 +8496,12 @@ class BrainyData {
|
|
|
7929
8496
|
this.remoteServerConfig = null;
|
|
7930
8497
|
this.serverSearchConduit = null;
|
|
7931
8498
|
this.serverConnection = null;
|
|
8499
|
+
// Validate dimensions
|
|
8500
|
+
if (config.dimensions !== undefined && config.dimensions <= 0) {
|
|
8501
|
+
throw new Error('Dimensions must be a positive number');
|
|
8502
|
+
}
|
|
8503
|
+
// Set dimensions (default to 512 for embedding functions, or require explicit config)
|
|
8504
|
+
this._dimensions = config.dimensions || 512;
|
|
7932
8505
|
// Set distance function
|
|
7933
8506
|
this.distanceFunction = config.distanceFunction || cosineDistance$1;
|
|
7934
8507
|
// Check if optimized HNSW index configuration is provided
|
|
@@ -8032,6 +8605,13 @@ class BrainyData {
|
|
|
8032
8605
|
// Clear the index and add all nouns
|
|
8033
8606
|
this.index.clear();
|
|
8034
8607
|
for (const noun of nouns) {
|
|
8608
|
+
// Check if the vector dimensions match the expected dimensions
|
|
8609
|
+
if (noun.vector.length !== this._dimensions) {
|
|
8610
|
+
console.warn(`Skipping noun ${noun.id} due to dimension mismatch: expected ${this._dimensions}, got ${noun.vector.length}`);
|
|
8611
|
+
// Optionally, you could delete the mismatched noun from storage
|
|
8612
|
+
// await this.storage!.deleteNoun(noun.id)
|
|
8613
|
+
continue;
|
|
8614
|
+
}
|
|
8035
8615
|
// Add to index
|
|
8036
8616
|
await this.index.addItem({
|
|
8037
8617
|
id: noun.id,
|
|
@@ -8115,6 +8695,10 @@ class BrainyData {
|
|
|
8115
8695
|
if (!vector) {
|
|
8116
8696
|
throw new Error('Vector is undefined or null');
|
|
8117
8697
|
}
|
|
8698
|
+
// Validate vector dimensions
|
|
8699
|
+
if (vector.length !== this._dimensions) {
|
|
8700
|
+
throw new Error(`Vector dimension mismatch: expected ${this._dimensions}, got ${vector.length}`);
|
|
8701
|
+
}
|
|
8118
8702
|
// Use ID from options if it exists, otherwise from metadata, otherwise generate a new UUID
|
|
8119
8703
|
const id = options.id ||
|
|
8120
8704
|
(metadata && typeof metadata === 'object' && 'id' in metadata
|
|
@@ -8141,7 +8725,12 @@ class BrainyData {
|
|
|
8141
8725
|
metadata.noun = NounType.Concept;
|
|
8142
8726
|
}
|
|
8143
8727
|
}
|
|
8144
|
-
|
|
8728
|
+
// Ensure metadata has the correct id field
|
|
8729
|
+
let metadataToSave = metadata;
|
|
8730
|
+
if (metadata && typeof metadata === 'object') {
|
|
8731
|
+
metadataToSave = { ...metadata, id };
|
|
8732
|
+
}
|
|
8733
|
+
await this.storage.saveMetadata(id, metadataToSave);
|
|
8145
8734
|
}
|
|
8146
8735
|
// If addToRemote is true and we're connected to a remote server, add to remote as well
|
|
8147
8736
|
if (options.addToRemote && this.isConnectedToRemoteServer()) {
|
|
@@ -8159,6 +8748,18 @@ class BrainyData {
|
|
|
8159
8748
|
throw new Error(`Failed to add vector: ${error}`);
|
|
8160
8749
|
}
|
|
8161
8750
|
}
|
|
8751
|
+
/**
|
|
8752
|
+
* Add a text item to the database with automatic embedding
|
|
8753
|
+
* This is a convenience method for adding text data with metadata
|
|
8754
|
+
* @param text Text data to add
|
|
8755
|
+
* @param metadata Metadata to associate with the text
|
|
8756
|
+
* @param options Additional options
|
|
8757
|
+
* @returns The ID of the added item
|
|
8758
|
+
*/
|
|
8759
|
+
async addItem(text, metadata, options = {}) {
|
|
8760
|
+
// Use the existing add method with forceEmbed to ensure text is embedded
|
|
8761
|
+
return this.add(text, metadata, { ...options, forceEmbed: true });
|
|
8762
|
+
}
|
|
8162
8763
|
/**
|
|
8163
8764
|
* Add data to both local and remote Brainy instances
|
|
8164
8765
|
* @param vectorOrData Vector or data to add
|
|
@@ -8310,7 +8911,9 @@ class BrainyData {
|
|
|
8310
8911
|
* @returns Array of search results
|
|
8311
8912
|
*/
|
|
8312
8913
|
async searchByNounTypes(queryVectorOrData, k = 10, nounTypes = null, options = {}) {
|
|
8313
|
-
|
|
8914
|
+
if (!this.isInitialized) {
|
|
8915
|
+
throw new Error('BrainyData must be initialized before searching. Call init() first.');
|
|
8916
|
+
}
|
|
8314
8917
|
try {
|
|
8315
8918
|
let queryVector;
|
|
8316
8919
|
// Check if input is already a vector
|
|
@@ -8404,6 +9007,9 @@ class BrainyData {
|
|
|
8404
9007
|
* @returns Array of search results
|
|
8405
9008
|
*/
|
|
8406
9009
|
async search(queryVectorOrData, k = 10, options = {}) {
|
|
9010
|
+
if (!this.isInitialized) {
|
|
9011
|
+
throw new Error('BrainyData must be initialized before searching. Call init() first.');
|
|
9012
|
+
}
|
|
8407
9013
|
// If searching for verbs directly
|
|
8408
9014
|
if (options.searchVerbs) {
|
|
8409
9015
|
const verbResults = await this.searchVerbs(queryVectorOrData, k, {
|
|
@@ -8452,6 +9058,9 @@ class BrainyData {
|
|
|
8452
9058
|
* @returns Array of search results
|
|
8453
9059
|
*/
|
|
8454
9060
|
async searchLocal(queryVectorOrData, k = 10, options = {}) {
|
|
9061
|
+
if (!this.isInitialized) {
|
|
9062
|
+
throw new Error('BrainyData must be initialized before searching. Call init() first.');
|
|
9063
|
+
}
|
|
8455
9064
|
// If input is a string and not a vector, automatically vectorize it
|
|
8456
9065
|
let queryToUse = queryVectorOrData;
|
|
8457
9066
|
if (typeof queryVectorOrData === 'string' && !options.forceEmbed) {
|
|
@@ -9625,12 +10234,40 @@ class BrainyData {
|
|
|
9625
10234
|
}
|
|
9626
10235
|
}
|
|
9627
10236
|
|
|
9628
|
-
//
|
|
9629
|
-
//
|
|
10237
|
+
// Dynamically and asynchronously load Node.js modules at the top level.
|
|
10238
|
+
// This ensures they are available as soon as this module is imported,
|
|
10239
|
+
// preventing race conditions with dependencies like TensorFlow.js.
|
|
9630
10240
|
let fs;
|
|
9631
10241
|
let path;
|
|
9632
|
-
|
|
9633
|
-
//
|
|
10242
|
+
const nodeModulesPromise = (async () => {
|
|
10243
|
+
// A reliable check for a Node.js environment.
|
|
10244
|
+
const isNode = typeof process !== 'undefined' &&
|
|
10245
|
+
process.versions != null &&
|
|
10246
|
+
process.versions.node != null;
|
|
10247
|
+
if (!isNode) {
|
|
10248
|
+
return { fs: null, path: null };
|
|
10249
|
+
}
|
|
10250
|
+
try {
|
|
10251
|
+
// Use the 'node:' prefix for unambiguous importing of built-in modules.
|
|
10252
|
+
const fsModule = await import('node:fs');
|
|
10253
|
+
const pathModule = await import('node:path');
|
|
10254
|
+
// Return the modules, preferring the default export if it exists.
|
|
10255
|
+
return {
|
|
10256
|
+
fs: fsModule.default || fsModule,
|
|
10257
|
+
path: pathModule.default || pathModule
|
|
10258
|
+
};
|
|
10259
|
+
}
|
|
10260
|
+
catch (error) {
|
|
10261
|
+
console.error('FileSystemStorage: Failed to load Node.js modules. This adapter is not supported in this environment.', error);
|
|
10262
|
+
return { fs: null, path: null };
|
|
10263
|
+
}
|
|
10264
|
+
})();
|
|
10265
|
+
// Immediately assign the modules once the promise resolves.
|
|
10266
|
+
nodeModulesPromise.then((modules) => {
|
|
10267
|
+
fs = modules.fs;
|
|
10268
|
+
path = modules.path;
|
|
10269
|
+
});
|
|
10270
|
+
// --- End of Refactored Code ---
|
|
9634
10271
|
// Constants for directory and file names
|
|
9635
10272
|
const ROOT_DIR = 'brainy-data';
|
|
9636
10273
|
const NOUNS_DIR = 'nouns';
|
|
@@ -9670,58 +10307,13 @@ class FileSystemStorage {
|
|
|
9670
10307
|
if (this.isInitialized) {
|
|
9671
10308
|
return;
|
|
9672
10309
|
}
|
|
10310
|
+
// Wait for the top-level module loading to complete.
|
|
10311
|
+
await nodeModulesPromise;
|
|
10312
|
+
// Check if the modules were loaded successfully.
|
|
10313
|
+
if (!fs || !path) {
|
|
10314
|
+
throw new Error('FileSystemStorage requires a Node.js environment, but `fs` and `path` modules could not be loaded.');
|
|
10315
|
+
}
|
|
9673
10316
|
try {
|
|
9674
|
-
// Check if fs and path modules are available and have required methods
|
|
9675
|
-
if (!fs || !path || typeof path.resolve !== 'function') {
|
|
9676
|
-
console.log('Node.js modules not properly loaded, attempting to load them now');
|
|
9677
|
-
// Try multiple approaches to load the modules
|
|
9678
|
-
const loadAttempts = [
|
|
9679
|
-
// Attempt 1: Use dynamic import
|
|
9680
|
-
async () => {
|
|
9681
|
-
console.log('Attempting to load Node.js modules with dynamic import');
|
|
9682
|
-
const fsModule = await import('fs');
|
|
9683
|
-
const pathModule = await import('path');
|
|
9684
|
-
const fsResolved = fsModule.default || fsModule;
|
|
9685
|
-
const pathResolved = pathModule.default || pathModule;
|
|
9686
|
-
if (!pathResolved || typeof pathResolved.resolve !== 'function') {
|
|
9687
|
-
throw new Error('path.resolve is not a function after dynamic import');
|
|
9688
|
-
}
|
|
9689
|
-
return { fs: fsResolved, path: pathResolved };
|
|
9690
|
-
},
|
|
9691
|
-
// Attempt 2: Use dynamic import with node: prefix
|
|
9692
|
-
async () => {
|
|
9693
|
-
console.log('Attempting to load Node.js modules with dynamic import("node:...")');
|
|
9694
|
-
const fsModule = await import('node:fs');
|
|
9695
|
-
const pathModule = await import('node:path');
|
|
9696
|
-
const fsResolved = fsModule.default || fsModule;
|
|
9697
|
-
const pathResolved = pathModule.default || pathModule;
|
|
9698
|
-
if (!pathResolved || typeof pathResolved.resolve !== 'function') {
|
|
9699
|
-
throw new Error('path.resolve is not a function after dynamic import("node:path")');
|
|
9700
|
-
}
|
|
9701
|
-
return { fs: fsResolved, path: pathResolved };
|
|
9702
|
-
}
|
|
9703
|
-
];
|
|
9704
|
-
// Try each loading method until one succeeds
|
|
9705
|
-
let lastError = null;
|
|
9706
|
-
for (const loadAttempt of loadAttempts) {
|
|
9707
|
-
try {
|
|
9708
|
-
const modules = await loadAttempt();
|
|
9709
|
-
fs = modules.fs;
|
|
9710
|
-
path = modules.path;
|
|
9711
|
-
console.log('Successfully loaded Node.js modules');
|
|
9712
|
-
break;
|
|
9713
|
-
}
|
|
9714
|
-
catch (error) {
|
|
9715
|
-
lastError = error;
|
|
9716
|
-
console.warn(`Module loading attempt failed:`, error);
|
|
9717
|
-
// Continue to the next attempt
|
|
9718
|
-
}
|
|
9719
|
-
}
|
|
9720
|
-
// If all attempts failed, throw an error
|
|
9721
|
-
if (!fs || !path || typeof path.resolve !== 'function') {
|
|
9722
|
-
throw new Error(`Failed to import Node.js modules after multiple attempts: ${lastError}. This adapter requires a Node.js environment.`);
|
|
9723
|
-
}
|
|
9724
|
-
}
|
|
9725
10317
|
// Now set up the directory paths
|
|
9726
10318
|
const rootDir = this.rootDir || process.cwd();
|
|
9727
10319
|
this.rootDir = path.resolve(rootDir, ROOT_DIR);
|
|
@@ -9741,7 +10333,6 @@ class FileSystemStorage {
|
|
|
9741
10333
|
await this.ensureDirectoryExists(this.nounsDir);
|
|
9742
10334
|
await this.ensureDirectoryExists(this.verbsDir);
|
|
9743
10335
|
await this.ensureDirectoryExists(this.metadataDir);
|
|
9744
|
-
// Create noun type directories
|
|
9745
10336
|
await this.ensureDirectoryExists(this.personDir);
|
|
9746
10337
|
await this.ensureDirectoryExists(this.placeDir);
|
|
9747
10338
|
await this.ensureDirectoryExists(this.thingDir);
|
|
@@ -9752,736 +10343,403 @@ class FileSystemStorage {
|
|
|
9752
10343
|
this.isInitialized = true;
|
|
9753
10344
|
}
|
|
9754
10345
|
catch (error) {
|
|
9755
|
-
console.error('
|
|
9756
|
-
throw
|
|
10346
|
+
console.error('Error initializing FileSystemStorage:', error);
|
|
10347
|
+
throw error;
|
|
9757
10348
|
}
|
|
9758
10349
|
}
|
|
9759
|
-
|
|
9760
|
-
* Save a node to storage
|
|
9761
|
-
*/
|
|
9762
|
-
async saveNoun(noun) {
|
|
9763
|
-
await this.ensureInitialized();
|
|
10350
|
+
async ensureDirectoryExists(dirPath) {
|
|
9764
10351
|
try {
|
|
9765
|
-
|
|
9766
|
-
const serializableNode = {
|
|
9767
|
-
...noun,
|
|
9768
|
-
connections: this.mapToObject(noun.connections, (set) => Array.from(set))
|
|
9769
|
-
};
|
|
9770
|
-
// Get the appropriate directory based on the node's metadata
|
|
9771
|
-
const nodeDir = await this.getNodeDirectory(noun.id);
|
|
9772
|
-
const filePath = path.join(nodeDir, `${noun.id}.json`);
|
|
9773
|
-
await fs.promises.writeFile(filePath, JSON.stringify(serializableNode, null, 2), 'utf8');
|
|
10352
|
+
await fs.promises.mkdir(dirPath, { recursive: true });
|
|
9774
10353
|
}
|
|
9775
10354
|
catch (error) {
|
|
9776
|
-
|
|
9777
|
-
|
|
9778
|
-
|
|
9779
|
-
}
|
|
9780
|
-
/**
|
|
9781
|
-
* Get a node from storage
|
|
9782
|
-
*/
|
|
9783
|
-
async getNoun(id) {
|
|
9784
|
-
await this.ensureInitialized();
|
|
9785
|
-
try {
|
|
9786
|
-
// Get the appropriate directory based on the node's metadata
|
|
9787
|
-
const nodeDir = await this.getNodeDirectory(id);
|
|
9788
|
-
const filePath = path.join(nodeDir, `${id}.json`);
|
|
9789
|
-
// Check if a file exists
|
|
9790
|
-
try {
|
|
9791
|
-
await fs.promises.access(filePath);
|
|
9792
|
-
}
|
|
9793
|
-
catch {
|
|
9794
|
-
// If the file doesn't exist in the expected directory, try the default directory
|
|
9795
|
-
if (nodeDir !== this.defaultDir) {
|
|
9796
|
-
const defaultFilePath = path.join(this.defaultDir, `${id}.json`);
|
|
9797
|
-
try {
|
|
9798
|
-
await fs.promises.access(defaultFilePath);
|
|
9799
|
-
// If found in default directory, use that path
|
|
9800
|
-
const data = await fs.promises.readFile(defaultFilePath, 'utf8');
|
|
9801
|
-
const parsedNode = JSON.parse(data);
|
|
9802
|
-
// Convert serialized connections back to Map<number, Set<string>>
|
|
9803
|
-
const connections = new Map();
|
|
9804
|
-
for (const [level, nodeIds] of Object.entries(parsedNode.connections)) {
|
|
9805
|
-
connections.set(Number(level), new Set(nodeIds));
|
|
9806
|
-
}
|
|
9807
|
-
return {
|
|
9808
|
-
id: parsedNode.id,
|
|
9809
|
-
vector: parsedNode.vector,
|
|
9810
|
-
connections
|
|
9811
|
-
};
|
|
9812
|
-
}
|
|
9813
|
-
catch {
|
|
9814
|
-
// If not found in default directory either, try all noun type directories
|
|
9815
|
-
const directories = [
|
|
9816
|
-
this.personDir,
|
|
9817
|
-
this.placeDir,
|
|
9818
|
-
this.thingDir,
|
|
9819
|
-
this.eventDir,
|
|
9820
|
-
this.conceptDir,
|
|
9821
|
-
this.contentDir
|
|
9822
|
-
];
|
|
9823
|
-
for (const dir of directories) {
|
|
9824
|
-
if (dir === nodeDir)
|
|
9825
|
-
continue; // Skip the already checked directory
|
|
9826
|
-
const dirFilePath = path.join(dir, `${id}.json`);
|
|
9827
|
-
try {
|
|
9828
|
-
await fs.promises.access(dirFilePath);
|
|
9829
|
-
// If found in this directory, use that path
|
|
9830
|
-
const data = await fs.promises.readFile(dirFilePath, 'utf8');
|
|
9831
|
-
const parsedNode = JSON.parse(data);
|
|
9832
|
-
// Convert serialized connections back to Map<number, Set<string>>
|
|
9833
|
-
const connections = new Map();
|
|
9834
|
-
for (const [level, nodeIds] of Object.entries(parsedNode.connections)) {
|
|
9835
|
-
connections.set(Number(level), new Set(nodeIds));
|
|
9836
|
-
}
|
|
9837
|
-
return {
|
|
9838
|
-
id: parsedNode.id,
|
|
9839
|
-
vector: parsedNode.vector,
|
|
9840
|
-
connections
|
|
9841
|
-
};
|
|
9842
|
-
}
|
|
9843
|
-
catch {
|
|
9844
|
-
// Continue to the next directory
|
|
9845
|
-
}
|
|
9846
|
-
}
|
|
9847
|
-
return null; // File doesn't exist in any directory
|
|
9848
|
-
}
|
|
9849
|
-
}
|
|
9850
|
-
return null; // File doesn't exist
|
|
9851
|
-
}
|
|
9852
|
-
const data = await fs.promises.readFile(filePath, 'utf8');
|
|
9853
|
-
const parsedNode = JSON.parse(data);
|
|
9854
|
-
// Convert serialized connections back to Map<number, Set<string>>
|
|
9855
|
-
const connections = new Map();
|
|
9856
|
-
for (const [level, nodeIds] of Object.entries(parsedNode.connections)) {
|
|
9857
|
-
connections.set(Number(level), new Set(nodeIds));
|
|
10355
|
+
// Ignore EEXIST error, which means the directory already exists
|
|
10356
|
+
if (error.code !== 'EEXIST') {
|
|
10357
|
+
throw error;
|
|
9858
10358
|
}
|
|
9859
|
-
return {
|
|
9860
|
-
id: parsedNode.id,
|
|
9861
|
-
vector: parsedNode.vector,
|
|
9862
|
-
connections
|
|
9863
|
-
};
|
|
9864
|
-
}
|
|
9865
|
-
catch (error) {
|
|
9866
|
-
console.error(`Failed to get node ${id}:`, error);
|
|
9867
|
-
return null;
|
|
9868
10359
|
}
|
|
9869
10360
|
}
|
|
9870
|
-
|
|
9871
|
-
|
|
9872
|
-
|
|
9873
|
-
|
|
9874
|
-
*/
|
|
9875
|
-
async getNounsByNounType(nounType) {
|
|
9876
|
-
await this.ensureInitialized();
|
|
9877
|
-
try {
|
|
9878
|
-
// Determine the directory based on the noun type
|
|
9879
|
-
let dir;
|
|
9880
|
-
switch (nounType) {
|
|
10361
|
+
getNounPath(id, nounType) {
|
|
10362
|
+
let typeDir = this.defaultDir;
|
|
10363
|
+
if (nounType) {
|
|
10364
|
+
switch (nounType.toLowerCase()) {
|
|
9881
10365
|
case 'person':
|
|
9882
|
-
|
|
10366
|
+
typeDir = this.personDir;
|
|
9883
10367
|
break;
|
|
9884
10368
|
case 'place':
|
|
9885
|
-
|
|
10369
|
+
typeDir = this.placeDir;
|
|
9886
10370
|
break;
|
|
9887
10371
|
case 'thing':
|
|
9888
|
-
|
|
10372
|
+
typeDir = this.thingDir;
|
|
9889
10373
|
break;
|
|
9890
10374
|
case 'event':
|
|
9891
|
-
|
|
10375
|
+
typeDir = this.eventDir;
|
|
9892
10376
|
break;
|
|
9893
10377
|
case 'concept':
|
|
9894
|
-
|
|
10378
|
+
typeDir = this.conceptDir;
|
|
9895
10379
|
break;
|
|
9896
10380
|
case 'content':
|
|
9897
|
-
|
|
10381
|
+
typeDir = this.contentDir;
|
|
9898
10382
|
break;
|
|
9899
10383
|
default:
|
|
9900
|
-
|
|
9901
|
-
}
|
|
9902
|
-
const nodes = [];
|
|
9903
|
-
try {
|
|
9904
|
-
const files = await fs.promises.readdir(dir);
|
|
9905
|
-
const nodePromises = files
|
|
9906
|
-
.filter((file) => file.endsWith('.json'))
|
|
9907
|
-
.map((file) => {
|
|
9908
|
-
// Use the file path directly instead of getNode to avoid redundant searches
|
|
9909
|
-
return this.readNodeFromFile(path.join(dir, file));
|
|
9910
|
-
});
|
|
9911
|
-
const dirNodes = await Promise.all(nodePromises);
|
|
9912
|
-
nodes.push(...dirNodes.filter((node) => node !== null));
|
|
9913
|
-
}
|
|
9914
|
-
catch (dirError) {
|
|
9915
|
-
// If directory doesn't exist or can't be read, log a warning
|
|
9916
|
-
console.warn(`Could not read directory for noun type ${nounType}:`, dirError);
|
|
10384
|
+
typeDir = this.defaultDir;
|
|
9917
10385
|
}
|
|
9918
|
-
return nodes;
|
|
9919
|
-
}
|
|
9920
|
-
catch (error) {
|
|
9921
|
-
console.error(`Failed to get nodes for noun type ${nounType}:`, error);
|
|
9922
|
-
throw new Error(`Failed to get nodes for noun type ${nounType}: ${error}`);
|
|
9923
10386
|
}
|
|
10387
|
+
return path.join(typeDir, `${id}.json`);
|
|
9924
10388
|
}
|
|
9925
|
-
|
|
9926
|
-
|
|
9927
|
-
|
|
9928
|
-
|
|
9929
|
-
|
|
9930
|
-
|
|
9931
|
-
|
|
9932
|
-
const nounTypes = [
|
|
9933
|
-
'person',
|
|
9934
|
-
'place',
|
|
9935
|
-
'thing',
|
|
9936
|
-
'event',
|
|
9937
|
-
'concept',
|
|
9938
|
-
'content',
|
|
9939
|
-
'default'
|
|
9940
|
-
];
|
|
9941
|
-
// Run searches in parallel for all noun types
|
|
9942
|
-
const nodePromises = nounTypes.map((nounType) => this.getNounsByNounType(nounType));
|
|
9943
|
-
const nodeArrays = await Promise.all(nodePromises);
|
|
9944
|
-
// Combine all results
|
|
9945
|
-
const allNodes = [];
|
|
9946
|
-
for (const nodes of nodeArrays) {
|
|
9947
|
-
allNodes.push(...nodes);
|
|
9948
|
-
}
|
|
9949
|
-
return allNodes;
|
|
9950
|
-
}
|
|
9951
|
-
catch (error) {
|
|
9952
|
-
console.error('Failed to get all nodes:', error);
|
|
9953
|
-
throw new Error(`Failed to get all nodes: ${error}`);
|
|
9954
|
-
}
|
|
10389
|
+
async saveNoun(noun) {
|
|
10390
|
+
if (!this.isInitialized)
|
|
10391
|
+
await this.init();
|
|
10392
|
+
const nounType = noun.metadata?.noun;
|
|
10393
|
+
const filePath = this.getNounPath(noun.id, nounType);
|
|
10394
|
+
await this.ensureDirectoryExists(path.dirname(filePath));
|
|
10395
|
+
await fs.promises.writeFile(filePath, JSON.stringify(noun, null, 2));
|
|
9955
10396
|
}
|
|
9956
|
-
|
|
9957
|
-
|
|
9958
|
-
|
|
9959
|
-
|
|
9960
|
-
|
|
9961
|
-
|
|
9962
|
-
|
|
9963
|
-
|
|
9964
|
-
|
|
9965
|
-
|
|
9966
|
-
|
|
10397
|
+
async getNoun(id) {
|
|
10398
|
+
if (!this.isInitialized)
|
|
10399
|
+
await this.init();
|
|
10400
|
+
const nounDirs = [
|
|
10401
|
+
this.personDir,
|
|
10402
|
+
this.placeDir,
|
|
10403
|
+
this.thingDir,
|
|
10404
|
+
this.eventDir,
|
|
10405
|
+
this.conceptDir,
|
|
10406
|
+
this.contentDir,
|
|
10407
|
+
this.defaultDir
|
|
10408
|
+
];
|
|
10409
|
+
for (const dir of nounDirs) {
|
|
10410
|
+
const filePath = path.join(dir, `${id}.json`);
|
|
10411
|
+
try {
|
|
10412
|
+
const data = await fs.promises.readFile(filePath, 'utf-8');
|
|
10413
|
+
return JSON.parse(data);
|
|
10414
|
+
}
|
|
10415
|
+
catch (error) {
|
|
10416
|
+
if (error.code !== 'ENOENT') {
|
|
10417
|
+
console.error(`Error reading noun ${id}:`, error);
|
|
10418
|
+
}
|
|
9967
10419
|
}
|
|
9968
|
-
return {
|
|
9969
|
-
id: parsedNode.id,
|
|
9970
|
-
vector: parsedNode.vector,
|
|
9971
|
-
connections
|
|
9972
|
-
};
|
|
9973
|
-
}
|
|
9974
|
-
catch (error) {
|
|
9975
|
-
console.error(`Failed to read node from file ${filePath}:`, error);
|
|
9976
|
-
return null;
|
|
9977
10420
|
}
|
|
10421
|
+
return null;
|
|
9978
10422
|
}
|
|
9979
|
-
/**
|
|
9980
|
-
* Delete a node from storage
|
|
9981
|
-
*/
|
|
9982
10423
|
async deleteNoun(id) {
|
|
9983
|
-
|
|
9984
|
-
|
|
9985
|
-
|
|
9986
|
-
|
|
9987
|
-
const
|
|
9988
|
-
|
|
10424
|
+
if (!this.isInitialized)
|
|
10425
|
+
await this.init();
|
|
10426
|
+
const noun = await this.getNoun(id);
|
|
10427
|
+
if (noun) {
|
|
10428
|
+
const nounType = noun.metadata?.noun;
|
|
10429
|
+
const filePath = this.getNounPath(id, nounType);
|
|
9989
10430
|
try {
|
|
9990
|
-
await fs.promises.access(filePath);
|
|
9991
10431
|
await fs.promises.unlink(filePath);
|
|
9992
|
-
return; // File found and deleted
|
|
9993
10432
|
}
|
|
9994
|
-
catch {
|
|
9995
|
-
|
|
9996
|
-
|
|
9997
|
-
|
|
9998
|
-
try {
|
|
9999
|
-
await fs.promises.access(defaultFilePath);
|
|
10000
|
-
await fs.promises.unlink(defaultFilePath);
|
|
10001
|
-
return; // File found and deleted
|
|
10002
|
-
}
|
|
10003
|
-
catch {
|
|
10004
|
-
// If not found in default directory either, try all noun type directories
|
|
10005
|
-
const directories = [
|
|
10006
|
-
this.personDir,
|
|
10007
|
-
this.placeDir,
|
|
10008
|
-
this.thingDir,
|
|
10009
|
-
this.eventDir,
|
|
10010
|
-
this.conceptDir,
|
|
10011
|
-
this.contentDir
|
|
10012
|
-
];
|
|
10013
|
-
for (const dir of directories) {
|
|
10014
|
-
if (dir === nodeDir)
|
|
10015
|
-
continue; // Skip the already checked directory
|
|
10016
|
-
const dirFilePath = path.join(dir, `${id}.json`);
|
|
10017
|
-
try {
|
|
10018
|
-
await fs.promises.access(dirFilePath);
|
|
10019
|
-
await fs.promises.unlink(dirFilePath);
|
|
10020
|
-
return; // File found and deleted
|
|
10021
|
-
}
|
|
10022
|
-
catch {
|
|
10023
|
-
// Continue to the next directory
|
|
10024
|
-
}
|
|
10025
|
-
}
|
|
10026
|
-
return; // File doesn't exist in any directory, nothing to delete
|
|
10027
|
-
}
|
|
10433
|
+
catch (error) {
|
|
10434
|
+
if (error.code !== 'ENOENT') {
|
|
10435
|
+
console.error(`Error deleting noun file ${filePath}:`, error);
|
|
10436
|
+
throw error;
|
|
10028
10437
|
}
|
|
10029
|
-
return; // File doesn't exist, nothing to delete
|
|
10030
10438
|
}
|
|
10031
10439
|
}
|
|
10032
|
-
catch (error) {
|
|
10033
|
-
console.error(`Failed to delete node ${id}:`, error);
|
|
10034
|
-
throw new Error(`Failed to delete node ${id}: ${error}`);
|
|
10035
|
-
}
|
|
10036
|
-
}
|
|
10037
|
-
/**
|
|
10038
|
-
* Save an edge to storage
|
|
10039
|
-
*/
|
|
10040
|
-
async saveVerb(verb) {
|
|
10041
|
-
await this.ensureInitialized();
|
|
10042
|
-
try {
|
|
10043
|
-
// Convert connections Map to a serializable format
|
|
10044
|
-
const serializableEdge = {
|
|
10045
|
-
...verb,
|
|
10046
|
-
connections: this.mapToObject(verb.connections, (set) => Array.from(set))
|
|
10047
|
-
};
|
|
10048
|
-
const filePath = path.join(this.verbsDir, `${verb.id}.json`);
|
|
10049
|
-
await fs.promises.writeFile(filePath, JSON.stringify(serializableEdge, null, 2), 'utf8');
|
|
10050
|
-
}
|
|
10051
|
-
catch (error) {
|
|
10052
|
-
console.error(`Failed to save edge ${verb.id}:`, error);
|
|
10053
|
-
throw new Error(`Failed to save edge ${verb.id}: ${error}`);
|
|
10054
|
-
}
|
|
10055
10440
|
}
|
|
10056
|
-
|
|
10057
|
-
|
|
10058
|
-
|
|
10059
|
-
|
|
10060
|
-
|
|
10061
|
-
|
|
10062
|
-
|
|
10063
|
-
|
|
10441
|
+
async getAllNouns() {
|
|
10442
|
+
if (!this.isInitialized)
|
|
10443
|
+
await this.init();
|
|
10444
|
+
const allNouns = [];
|
|
10445
|
+
const nounDirs = [
|
|
10446
|
+
this.personDir,
|
|
10447
|
+
this.placeDir,
|
|
10448
|
+
this.thingDir,
|
|
10449
|
+
this.eventDir,
|
|
10450
|
+
this.conceptDir,
|
|
10451
|
+
this.contentDir,
|
|
10452
|
+
this.defaultDir
|
|
10453
|
+
];
|
|
10454
|
+
for (const dir of nounDirs) {
|
|
10064
10455
|
try {
|
|
10065
|
-
await fs.promises.
|
|
10066
|
-
|
|
10067
|
-
|
|
10068
|
-
|
|
10456
|
+
const files = await fs.promises.readdir(dir);
|
|
10457
|
+
for (const file of files) {
|
|
10458
|
+
if (file.endsWith('.json')) {
|
|
10459
|
+
const filePath = path.join(dir, file);
|
|
10460
|
+
const data = await fs.promises.readFile(filePath, 'utf-8');
|
|
10461
|
+
allNouns.push(JSON.parse(data));
|
|
10462
|
+
}
|
|
10463
|
+
}
|
|
10069
10464
|
}
|
|
10070
|
-
|
|
10071
|
-
|
|
10072
|
-
|
|
10073
|
-
|
|
10074
|
-
for (const [level, nodeIds] of Object.entries(parsedEdge.connections)) {
|
|
10075
|
-
connections.set(Number(level), new Set(nodeIds));
|
|
10465
|
+
catch (error) {
|
|
10466
|
+
if (error.code !== 'ENOENT') {
|
|
10467
|
+
console.error(`Error reading directory ${dir}:`, error);
|
|
10468
|
+
}
|
|
10076
10469
|
}
|
|
10077
|
-
return {
|
|
10078
|
-
id: parsedEdge.id,
|
|
10079
|
-
vector: parsedEdge.vector,
|
|
10080
|
-
connections,
|
|
10081
|
-
sourceId: parsedEdge.sourceId,
|
|
10082
|
-
targetId: parsedEdge.targetId,
|
|
10083
|
-
type: parsedEdge.type,
|
|
10084
|
-
weight: parsedEdge.weight,
|
|
10085
|
-
metadata: parsedEdge.metadata
|
|
10086
|
-
};
|
|
10087
|
-
}
|
|
10088
|
-
catch (error) {
|
|
10089
|
-
console.error(`Failed to get edge ${id}:`, error);
|
|
10090
|
-
return null;
|
|
10091
10470
|
}
|
|
10471
|
+
return allNouns;
|
|
10092
10472
|
}
|
|
10093
10473
|
/**
|
|
10094
|
-
* Get
|
|
10474
|
+
* Get nouns by noun type
|
|
10475
|
+
* @param nounType The noun type to filter by
|
|
10476
|
+
* @returns Promise that resolves to an array of nouns of the specified noun type
|
|
10095
10477
|
*/
|
|
10096
|
-
async
|
|
10097
|
-
|
|
10478
|
+
async getNounsByNounType(nounType) {
|
|
10479
|
+
if (!this.isInitialized)
|
|
10480
|
+
await this.init();
|
|
10481
|
+
let typeDir;
|
|
10482
|
+
switch (nounType.toLowerCase()) {
|
|
10483
|
+
case 'person':
|
|
10484
|
+
typeDir = this.personDir;
|
|
10485
|
+
break;
|
|
10486
|
+
case 'place':
|
|
10487
|
+
typeDir = this.placeDir;
|
|
10488
|
+
break;
|
|
10489
|
+
case 'thing':
|
|
10490
|
+
typeDir = this.thingDir;
|
|
10491
|
+
break;
|
|
10492
|
+
case 'event':
|
|
10493
|
+
typeDir = this.eventDir;
|
|
10494
|
+
break;
|
|
10495
|
+
case 'concept':
|
|
10496
|
+
typeDir = this.conceptDir;
|
|
10497
|
+
break;
|
|
10498
|
+
case 'content':
|
|
10499
|
+
typeDir = this.contentDir;
|
|
10500
|
+
break;
|
|
10501
|
+
default:
|
|
10502
|
+
typeDir = this.defaultDir;
|
|
10503
|
+
}
|
|
10504
|
+
const nouns = [];
|
|
10098
10505
|
try {
|
|
10099
|
-
const files = await fs.promises.readdir(
|
|
10100
|
-
const
|
|
10101
|
-
|
|
10102
|
-
|
|
10103
|
-
|
|
10104
|
-
|
|
10105
|
-
|
|
10106
|
-
|
|
10107
|
-
return edges.filter((edge) => edge !== null);
|
|
10506
|
+
const files = await fs.promises.readdir(typeDir);
|
|
10507
|
+
for (const file of files) {
|
|
10508
|
+
if (file.endsWith('.json')) {
|
|
10509
|
+
const filePath = path.join(typeDir, file);
|
|
10510
|
+
const data = await fs.promises.readFile(filePath, 'utf-8');
|
|
10511
|
+
nouns.push(JSON.parse(data));
|
|
10512
|
+
}
|
|
10513
|
+
}
|
|
10108
10514
|
}
|
|
10109
10515
|
catch (error) {
|
|
10110
|
-
|
|
10111
|
-
|
|
10516
|
+
if (error.code !== 'ENOENT') {
|
|
10517
|
+
console.error(`Error reading directory ${typeDir}:`, error);
|
|
10518
|
+
}
|
|
10112
10519
|
}
|
|
10520
|
+
return nouns;
|
|
10521
|
+
}
|
|
10522
|
+
async saveVerb(verb) {
|
|
10523
|
+
if (!this.isInitialized)
|
|
10524
|
+
await this.init();
|
|
10525
|
+
const filePath = path.join(this.verbsDir, `${verb.id}.json`);
|
|
10526
|
+
await fs.promises.writeFile(filePath, JSON.stringify(verb, null, 2));
|
|
10113
10527
|
}
|
|
10114
10528
|
/**
|
|
10115
|
-
* Get
|
|
10529
|
+
* Get a verb by its ID
|
|
10530
|
+
* @param id The ID of the verb to retrieve
|
|
10531
|
+
* @returns Promise that resolves to the verb or null if not found
|
|
10116
10532
|
*/
|
|
10117
|
-
async
|
|
10118
|
-
|
|
10533
|
+
async getVerb(id) {
|
|
10534
|
+
if (!this.isInitialized)
|
|
10535
|
+
await this.init();
|
|
10536
|
+
const filePath = path.join(this.verbsDir, `${id}.json`);
|
|
10119
10537
|
try {
|
|
10120
|
-
const
|
|
10121
|
-
return
|
|
10538
|
+
const data = await fs.promises.readFile(filePath, 'utf-8');
|
|
10539
|
+
return JSON.parse(data);
|
|
10122
10540
|
}
|
|
10123
10541
|
catch (error) {
|
|
10124
|
-
|
|
10125
|
-
|
|
10542
|
+
if (error.code !== 'ENOENT') {
|
|
10543
|
+
console.error(`Error reading verb ${id}:`, error);
|
|
10544
|
+
}
|
|
10545
|
+
return null;
|
|
10126
10546
|
}
|
|
10127
10547
|
}
|
|
10548
|
+
async getVerbsBySource(sourceId) {
|
|
10549
|
+
if (!this.isInitialized)
|
|
10550
|
+
await this.init();
|
|
10551
|
+
const allVerbs = await this.getAllVerbs();
|
|
10552
|
+
return allVerbs.filter((verb) => verb.sourceId === sourceId);
|
|
10553
|
+
}
|
|
10128
10554
|
/**
|
|
10129
|
-
* Get
|
|
10555
|
+
* Get verbs by target ID
|
|
10556
|
+
* @param targetId The target ID to filter by
|
|
10557
|
+
* @returns Promise that resolves to an array of verbs with the specified target ID
|
|
10130
10558
|
*/
|
|
10131
10559
|
async getVerbsByTarget(targetId) {
|
|
10132
|
-
|
|
10133
|
-
|
|
10134
|
-
|
|
10135
|
-
|
|
10136
|
-
}
|
|
10137
|
-
catch (error) {
|
|
10138
|
-
console.error(`Failed to get edges by target ${targetId}:`, error);
|
|
10139
|
-
throw new Error(`Failed to get edges by target ${targetId}: ${error}`);
|
|
10140
|
-
}
|
|
10560
|
+
if (!this.isInitialized)
|
|
10561
|
+
await this.init();
|
|
10562
|
+
const allVerbs = await this.getAllVerbs();
|
|
10563
|
+
return allVerbs.filter((verb) => verb.targetId === targetId);
|
|
10141
10564
|
}
|
|
10142
10565
|
/**
|
|
10143
|
-
* Get
|
|
10566
|
+
* Get verbs by type
|
|
10567
|
+
* @param type The verb type to filter by
|
|
10568
|
+
* @returns Promise that resolves to an array of verbs of the specified type
|
|
10144
10569
|
*/
|
|
10145
10570
|
async getVerbsByType(type) {
|
|
10146
|
-
|
|
10571
|
+
if (!this.isInitialized)
|
|
10572
|
+
await this.init();
|
|
10573
|
+
const allVerbs = await this.getAllVerbs();
|
|
10574
|
+
return allVerbs.filter((verb) => verb.type === type);
|
|
10575
|
+
}
|
|
10576
|
+
async getAllVerbs() {
|
|
10577
|
+
if (!this.isInitialized)
|
|
10578
|
+
await this.init();
|
|
10579
|
+
const allVerbs = [];
|
|
10147
10580
|
try {
|
|
10148
|
-
const
|
|
10149
|
-
|
|
10581
|
+
const files = await fs.promises.readdir(this.verbsDir);
|
|
10582
|
+
for (const file of files) {
|
|
10583
|
+
if (file.endsWith('.json')) {
|
|
10584
|
+
const filePath = path.join(this.verbsDir, file);
|
|
10585
|
+
const data = await fs.promises.readFile(filePath, 'utf-8');
|
|
10586
|
+
allVerbs.push(JSON.parse(data));
|
|
10587
|
+
}
|
|
10588
|
+
}
|
|
10150
10589
|
}
|
|
10151
10590
|
catch (error) {
|
|
10152
|
-
|
|
10153
|
-
|
|
10591
|
+
if (error.code !== 'ENOENT') {
|
|
10592
|
+
console.error(`Error reading verbs directory ${this.verbsDir}:`, error);
|
|
10593
|
+
}
|
|
10154
10594
|
}
|
|
10595
|
+
return allVerbs;
|
|
10155
10596
|
}
|
|
10156
|
-
/**
|
|
10157
|
-
* Delete an edge from storage
|
|
10158
|
-
*/
|
|
10159
10597
|
async deleteVerb(id) {
|
|
10160
|
-
|
|
10598
|
+
if (!this.isInitialized)
|
|
10599
|
+
await this.init();
|
|
10600
|
+
const filePath = path.join(this.verbsDir, `${id}.json`);
|
|
10161
10601
|
try {
|
|
10162
|
-
const filePath = path.join(this.verbsDir, `${id}.json`);
|
|
10163
|
-
// Check if a file exists before attempting to delete
|
|
10164
|
-
try {
|
|
10165
|
-
await fs.promises.access(filePath);
|
|
10166
|
-
}
|
|
10167
|
-
catch {
|
|
10168
|
-
return; // File doesn't exist, nothing to delete
|
|
10169
|
-
}
|
|
10170
10602
|
await fs.promises.unlink(filePath);
|
|
10171
10603
|
}
|
|
10172
10604
|
catch (error) {
|
|
10173
|
-
|
|
10174
|
-
|
|
10605
|
+
if (error.code !== 'ENOENT') {
|
|
10606
|
+
console.error(`Error deleting verb file ${filePath}:`, error);
|
|
10607
|
+
throw error;
|
|
10608
|
+
}
|
|
10175
10609
|
}
|
|
10176
10610
|
}
|
|
10177
10611
|
/**
|
|
10178
|
-
* Save metadata
|
|
10612
|
+
* Save metadata for an entity
|
|
10613
|
+
* @param id The ID of the entity
|
|
10614
|
+
* @param metadata The metadata to save
|
|
10179
10615
|
*/
|
|
10180
10616
|
async saveMetadata(id, metadata) {
|
|
10181
|
-
|
|
10182
|
-
|
|
10183
|
-
|
|
10184
|
-
|
|
10185
|
-
|
|
10186
|
-
catch (error) {
|
|
10187
|
-
console.error(`Failed to save metadata for ${id}:`, error);
|
|
10188
|
-
throw new Error(`Failed to save metadata for ${id}: ${error}`);
|
|
10189
|
-
}
|
|
10617
|
+
if (!this.isInitialized)
|
|
10618
|
+
await this.init();
|
|
10619
|
+
const filePath = path.join(this.metadataDir, `${id}.json`);
|
|
10620
|
+
await this.ensureDirectoryExists(path.dirname(filePath));
|
|
10621
|
+
await fs.promises.writeFile(filePath, JSON.stringify(metadata, null, 2));
|
|
10190
10622
|
}
|
|
10191
10623
|
/**
|
|
10192
|
-
* Get metadata
|
|
10624
|
+
* Get metadata for an entity
|
|
10625
|
+
* @param id The ID of the entity
|
|
10626
|
+
* @returns Promise that resolves to the metadata or null if not found
|
|
10193
10627
|
*/
|
|
10194
10628
|
async getMetadata(id) {
|
|
10195
|
-
|
|
10629
|
+
if (!this.isInitialized)
|
|
10630
|
+
await this.init();
|
|
10631
|
+
const filePath = path.join(this.metadataDir, `${id}.json`);
|
|
10196
10632
|
try {
|
|
10197
|
-
const
|
|
10198
|
-
// Check if a file exists
|
|
10199
|
-
try {
|
|
10200
|
-
await fs.promises.access(filePath);
|
|
10201
|
-
}
|
|
10202
|
-
catch {
|
|
10203
|
-
return null; // File doesn't exist
|
|
10204
|
-
}
|
|
10205
|
-
const data = await fs.promises.readFile(filePath, 'utf8');
|
|
10633
|
+
const data = await fs.promises.readFile(filePath, 'utf-8');
|
|
10206
10634
|
return JSON.parse(data);
|
|
10207
10635
|
}
|
|
10208
10636
|
catch (error) {
|
|
10209
|
-
|
|
10637
|
+
if (error.code !== 'ENOENT') {
|
|
10638
|
+
console.error(`Error reading metadata for ${id}:`, error);
|
|
10639
|
+
}
|
|
10210
10640
|
return null;
|
|
10211
10641
|
}
|
|
10212
10642
|
}
|
|
10213
|
-
/**
|
|
10214
|
-
* Clear all data from storage
|
|
10215
|
-
*/
|
|
10216
10643
|
async clear() {
|
|
10217
|
-
|
|
10218
|
-
try {
|
|
10219
|
-
// Delete and recreate the nodes, edges, and metadata directories
|
|
10220
|
-
await this.deleteDirectory(this.nounsDir);
|
|
10221
|
-
await this.deleteDirectory(this.verbsDir);
|
|
10222
|
-
await this.deleteDirectory(this.metadataDir);
|
|
10223
|
-
await this.ensureDirectoryExists(this.nounsDir);
|
|
10224
|
-
await this.ensureDirectoryExists(this.verbsDir);
|
|
10225
|
-
await this.ensureDirectoryExists(this.metadataDir);
|
|
10226
|
-
// Create noun type directories
|
|
10227
|
-
await this.ensureDirectoryExists(this.personDir);
|
|
10228
|
-
await this.ensureDirectoryExists(this.placeDir);
|
|
10229
|
-
await this.ensureDirectoryExists(this.thingDir);
|
|
10230
|
-
await this.ensureDirectoryExists(this.eventDir);
|
|
10231
|
-
await this.ensureDirectoryExists(this.conceptDir);
|
|
10232
|
-
await this.ensureDirectoryExists(this.contentDir);
|
|
10233
|
-
await this.ensureDirectoryExists(this.defaultDir);
|
|
10234
|
-
}
|
|
10235
|
-
catch (error) {
|
|
10236
|
-
console.error('Failed to clear storage:', error);
|
|
10237
|
-
throw new Error(`Failed to clear storage: ${error}`);
|
|
10238
|
-
}
|
|
10239
|
-
}
|
|
10240
|
-
/**
|
|
10241
|
-
* Ensure the storage adapter is initialized
|
|
10242
|
-
*/
|
|
10243
|
-
async ensureInitialized() {
|
|
10244
|
-
if (!this.isInitialized) {
|
|
10644
|
+
if (!this.isInitialized)
|
|
10245
10645
|
await this.init();
|
|
10246
|
-
|
|
10247
|
-
|
|
10248
|
-
|
|
10249
|
-
|
|
10250
|
-
|
|
10251
|
-
|
|
10252
|
-
|
|
10253
|
-
|
|
10254
|
-
|
|
10255
|
-
|
|
10256
|
-
|
|
10257
|
-
|
|
10258
|
-
|
|
10259
|
-
|
|
10260
|
-
|
|
10261
|
-
|
|
10262
|
-
|
|
10263
|
-
|
|
10264
|
-
|
|
10265
|
-
|
|
10266
|
-
for (const file of files) {
|
|
10267
|
-
const filePath = path.join(dirPath, file);
|
|
10268
|
-
const stats = await fs.promises.stat(filePath);
|
|
10269
|
-
if (stats.isDirectory()) {
|
|
10270
|
-
// Recursively delete subdirectories
|
|
10271
|
-
await this.deleteDirectory(filePath);
|
|
10272
|
-
}
|
|
10273
|
-
else {
|
|
10274
|
-
// Delete files
|
|
10275
|
-
await fs.promises.unlink(filePath);
|
|
10646
|
+
// Helper function to recursively remove directory contents
|
|
10647
|
+
const removeDirectoryContents = async (dirPath) => {
|
|
10648
|
+
try {
|
|
10649
|
+
const files = await fs.promises.readdir(dirPath, { withFileTypes: true });
|
|
10650
|
+
for (const file of files) {
|
|
10651
|
+
const fullPath = path.join(dirPath, file.name);
|
|
10652
|
+
if (file.isDirectory()) {
|
|
10653
|
+
await removeDirectoryContents(fullPath);
|
|
10654
|
+
// Use fs.promises.rm with recursive option instead of rmdir
|
|
10655
|
+
try {
|
|
10656
|
+
await fs.promises.rm(fullPath, { recursive: true, force: true });
|
|
10657
|
+
}
|
|
10658
|
+
catch (rmError) {
|
|
10659
|
+
// Fallback to rmdir if rm fails
|
|
10660
|
+
await fs.promises.rmdir(fullPath);
|
|
10661
|
+
}
|
|
10662
|
+
}
|
|
10663
|
+
else {
|
|
10664
|
+
await fs.promises.unlink(fullPath);
|
|
10665
|
+
}
|
|
10276
10666
|
}
|
|
10277
10667
|
}
|
|
10278
|
-
|
|
10279
|
-
|
|
10280
|
-
|
|
10281
|
-
|
|
10282
|
-
|
|
10283
|
-
if (error.code !== 'ENOENT') {
|
|
10284
|
-
throw error;
|
|
10668
|
+
catch (error) {
|
|
10669
|
+
if (error.code !== 'ENOENT') {
|
|
10670
|
+
console.error(`Error removing directory contents ${dirPath}:`, error);
|
|
10671
|
+
throw error;
|
|
10672
|
+
}
|
|
10285
10673
|
}
|
|
10286
|
-
}
|
|
10287
|
-
}
|
|
10288
|
-
/**
|
|
10289
|
-
* Count the number of JSON files in a directory
|
|
10290
|
-
*/
|
|
10291
|
-
async countFilesInDirectory(dirPath) {
|
|
10674
|
+
};
|
|
10292
10675
|
try {
|
|
10293
|
-
|
|
10294
|
-
|
|
10676
|
+
// First try the modern approach
|
|
10677
|
+
await fs.promises.rm(this.rootDir, { recursive: true, force: true });
|
|
10295
10678
|
}
|
|
10296
10679
|
catch (error) {
|
|
10297
|
-
|
|
10298
|
-
|
|
10299
|
-
|
|
10680
|
+
console.warn('Modern rm failed, falling back to manual cleanup:', error);
|
|
10681
|
+
// Fallback: manually remove contents then directory
|
|
10682
|
+
try {
|
|
10683
|
+
await removeDirectoryContents(this.rootDir);
|
|
10684
|
+
// Use fs.promises.rm with recursive option instead of rmdir
|
|
10685
|
+
try {
|
|
10686
|
+
await fs.promises.rm(this.rootDir, { recursive: true, force: true });
|
|
10687
|
+
}
|
|
10688
|
+
catch (rmError) {
|
|
10689
|
+
// Final fallback to rmdir if rm fails
|
|
10690
|
+
await fs.promises.rmdir(this.rootDir);
|
|
10691
|
+
}
|
|
10300
10692
|
}
|
|
10301
|
-
|
|
10302
|
-
|
|
10303
|
-
|
|
10304
|
-
|
|
10305
|
-
* Convert a Map to a plain object for serialization
|
|
10306
|
-
*/
|
|
10307
|
-
mapToObject(map, valueTransformer = (v) => v) {
|
|
10308
|
-
const obj = {};
|
|
10309
|
-
for (const [key, value] of map.entries()) {
|
|
10310
|
-
obj[key.toString()] = valueTransformer(value);
|
|
10311
|
-
}
|
|
10312
|
-
return obj;
|
|
10313
|
-
}
|
|
10314
|
-
/**
|
|
10315
|
-
* Get the appropriate directory for a node based on its metadata
|
|
10316
|
-
*/
|
|
10317
|
-
async getNodeDirectory(id) {
|
|
10318
|
-
try {
|
|
10319
|
-
// Try to get the metadata for the node
|
|
10320
|
-
const metadata = await this.getMetadata(id);
|
|
10321
|
-
// If metadata exists and has a noun field, use the corresponding directory
|
|
10322
|
-
if (metadata && metadata.noun) {
|
|
10323
|
-
switch (metadata.noun) {
|
|
10324
|
-
case 'person':
|
|
10325
|
-
return this.personDir;
|
|
10326
|
-
case 'place':
|
|
10327
|
-
return this.placeDir;
|
|
10328
|
-
case 'thing':
|
|
10329
|
-
return this.thingDir;
|
|
10330
|
-
case 'event':
|
|
10331
|
-
return this.eventDir;
|
|
10332
|
-
case 'concept':
|
|
10333
|
-
return this.conceptDir;
|
|
10334
|
-
case 'content':
|
|
10335
|
-
return this.contentDir;
|
|
10336
|
-
default:
|
|
10337
|
-
return this.defaultDir;
|
|
10693
|
+
catch (fallbackError) {
|
|
10694
|
+
if (fallbackError.code !== 'ENOENT') {
|
|
10695
|
+
console.error('Manual cleanup also failed:', fallbackError);
|
|
10696
|
+
throw fallbackError;
|
|
10338
10697
|
}
|
|
10339
10698
|
}
|
|
10340
|
-
// If no metadata or no noun field, use the default directory
|
|
10341
|
-
return this.defaultDir;
|
|
10342
|
-
}
|
|
10343
|
-
catch (error) {
|
|
10344
|
-
// If there's an error getting the metadata, use the default directory
|
|
10345
|
-
return this.defaultDir;
|
|
10346
10699
|
}
|
|
10700
|
+
this.isInitialized = false; // Reset state
|
|
10701
|
+
await this.init(); // Re-create directories
|
|
10347
10702
|
}
|
|
10348
|
-
/**
|
|
10349
|
-
* Get information about storage usage and capacity
|
|
10350
|
-
*/
|
|
10351
10703
|
async getStorageStatus() {
|
|
10352
|
-
|
|
10353
|
-
|
|
10354
|
-
|
|
10355
|
-
let
|
|
10356
|
-
|
|
10357
|
-
|
|
10358
|
-
|
|
10359
|
-
|
|
10360
|
-
|
|
10361
|
-
|
|
10362
|
-
|
|
10363
|
-
|
|
10364
|
-
if (stats.isDirectory()) {
|
|
10365
|
-
size += await calculateDirSize(filePath);
|
|
10366
|
-
}
|
|
10367
|
-
else {
|
|
10368
|
-
size += stats.size;
|
|
10369
|
-
}
|
|
10704
|
+
if (!this.isInitialized)
|
|
10705
|
+
await this.init();
|
|
10706
|
+
const calculateSize = async (dirPath) => {
|
|
10707
|
+
let size = 0;
|
|
10708
|
+
try {
|
|
10709
|
+
const files = await fs.promises.readdir(dirPath, {
|
|
10710
|
+
withFileTypes: true
|
|
10711
|
+
});
|
|
10712
|
+
for (const file of files) {
|
|
10713
|
+
const fullPath = path.join(dirPath, file.name);
|
|
10714
|
+
if (file.isDirectory()) {
|
|
10715
|
+
size += await calculateSize(fullPath);
|
|
10370
10716
|
}
|
|
10371
|
-
|
|
10372
|
-
|
|
10373
|
-
|
|
10374
|
-
}
|
|
10375
|
-
return size;
|
|
10376
|
-
};
|
|
10377
|
-
// Calculate size for each directory
|
|
10378
|
-
const nodesDirSize = await calculateDirSize(this.nounsDir);
|
|
10379
|
-
const edgesDirSize = await calculateDirSize(this.verbsDir);
|
|
10380
|
-
const metadataDirSize = await calculateDirSize(this.metadataDir);
|
|
10381
|
-
// Calculate sizes of noun type directories
|
|
10382
|
-
const personDirSize = await calculateDirSize(this.personDir);
|
|
10383
|
-
const placeDirSize = await calculateDirSize(this.placeDir);
|
|
10384
|
-
const thingDirSize = await calculateDirSize(this.thingDir);
|
|
10385
|
-
const eventDirSize = await calculateDirSize(this.eventDir);
|
|
10386
|
-
const conceptDirSize = await calculateDirSize(this.conceptDir);
|
|
10387
|
-
const contentDirSize = await calculateDirSize(this.contentDir);
|
|
10388
|
-
const defaultDirSize = await calculateDirSize(this.defaultDir);
|
|
10389
|
-
// Note: The noun type directories are subdirectories of the nodes directory,
|
|
10390
|
-
// so their sizes are already included in nodesDirSize.
|
|
10391
|
-
// We don't need to add them again to avoid double counting.
|
|
10392
|
-
totalSize = nodesDirSize + edgesDirSize + metadataDirSize;
|
|
10393
|
-
// Get filesystem information
|
|
10394
|
-
let quota = null;
|
|
10395
|
-
let details = {
|
|
10396
|
-
nounTypes: {
|
|
10397
|
-
person: {
|
|
10398
|
-
size: personDirSize,
|
|
10399
|
-
count: await this.countFilesInDirectory(this.personDir)
|
|
10400
|
-
},
|
|
10401
|
-
place: {
|
|
10402
|
-
size: placeDirSize,
|
|
10403
|
-
count: await this.countFilesInDirectory(this.placeDir)
|
|
10404
|
-
},
|
|
10405
|
-
thing: {
|
|
10406
|
-
size: thingDirSize,
|
|
10407
|
-
count: await this.countFilesInDirectory(this.thingDir)
|
|
10408
|
-
},
|
|
10409
|
-
event: {
|
|
10410
|
-
size: eventDirSize,
|
|
10411
|
-
count: await this.countFilesInDirectory(this.eventDir)
|
|
10412
|
-
},
|
|
10413
|
-
concept: {
|
|
10414
|
-
size: conceptDirSize,
|
|
10415
|
-
count: await this.countFilesInDirectory(this.conceptDir)
|
|
10416
|
-
},
|
|
10417
|
-
content: {
|
|
10418
|
-
size: contentDirSize,
|
|
10419
|
-
count: await this.countFilesInDirectory(this.contentDir)
|
|
10420
|
-
},
|
|
10421
|
-
default: {
|
|
10422
|
-
size: defaultDirSize,
|
|
10423
|
-
count: await this.countFilesInDirectory(this.defaultDir)
|
|
10717
|
+
else {
|
|
10718
|
+
const stats = await fs.promises.stat(fullPath);
|
|
10719
|
+
size += stats.size;
|
|
10424
10720
|
}
|
|
10425
10721
|
}
|
|
10426
|
-
};
|
|
10427
|
-
try {
|
|
10428
|
-
// Try to get disk space information
|
|
10429
|
-
const stats = await fs.promises.statfs(this.rootDir);
|
|
10430
|
-
if (stats) {
|
|
10431
|
-
const availableSpace = stats.bavail * stats.bsize;
|
|
10432
|
-
const totalSpace = stats.blocks * stats.bsize;
|
|
10433
|
-
quota = totalSpace;
|
|
10434
|
-
details = {
|
|
10435
|
-
availableSpace,
|
|
10436
|
-
totalSpace,
|
|
10437
|
-
freePercentage: (availableSpace / totalSpace) * 100
|
|
10438
|
-
};
|
|
10439
|
-
}
|
|
10440
10722
|
}
|
|
10441
10723
|
catch (error) {
|
|
10442
|
-
|
|
10443
|
-
|
|
10444
|
-
try {
|
|
10445
|
-
const { exec } = await Promise.resolve().then(function () { return _child_processShim; });
|
|
10446
|
-
const util = await Promise.resolve().then(function () { return _utilShim$1; });
|
|
10447
|
-
const execPromise = util.promisify(exec);
|
|
10448
|
-
const { stdout } = await execPromise(`df -k "${this.rootDir}"`);
|
|
10449
|
-
const lines = stdout.trim().split('\n');
|
|
10450
|
-
if (lines.length > 1) {
|
|
10451
|
-
const parts = lines[1].split(/\s+/);
|
|
10452
|
-
if (parts.length >= 4) {
|
|
10453
|
-
const totalKB = parseInt(parts[1], 10);
|
|
10454
|
-
const usedKB = parseInt(parts[2], 10);
|
|
10455
|
-
const availableKB = parseInt(parts[3], 10);
|
|
10456
|
-
quota = totalKB * 1024;
|
|
10457
|
-
details = {
|
|
10458
|
-
availableSpace: availableKB * 1024,
|
|
10459
|
-
totalSpace: totalKB * 1024,
|
|
10460
|
-
freePercentage: (availableKB / totalKB) * 100
|
|
10461
|
-
};
|
|
10462
|
-
}
|
|
10463
|
-
}
|
|
10464
|
-
}
|
|
10465
|
-
catch (dfError) {
|
|
10466
|
-
console.warn('Unable to get disk space using df command:', dfError);
|
|
10724
|
+
if (error.code !== 'ENOENT') {
|
|
10725
|
+
console.error(`Could not calculate size for ${dirPath}:`, error);
|
|
10467
10726
|
}
|
|
10468
10727
|
}
|
|
10469
|
-
return
|
|
10470
|
-
|
|
10471
|
-
|
|
10472
|
-
|
|
10473
|
-
|
|
10474
|
-
|
|
10475
|
-
|
|
10476
|
-
|
|
10477
|
-
|
|
10478
|
-
|
|
10479
|
-
|
|
10480
|
-
|
|
10481
|
-
|
|
10482
|
-
|
|
10483
|
-
|
|
10484
|
-
}
|
|
10728
|
+
return size;
|
|
10729
|
+
};
|
|
10730
|
+
const totalSize = await calculateSize(this.rootDir);
|
|
10731
|
+
const nouns = await this.getAllNouns();
|
|
10732
|
+
const verbs = await this.getAllVerbs();
|
|
10733
|
+
return {
|
|
10734
|
+
type: 'FileSystem',
|
|
10735
|
+
used: totalSize,
|
|
10736
|
+
quota: null, // File system quota is not easily available from Node.js
|
|
10737
|
+
details: {
|
|
10738
|
+
rootDir: this.rootDir,
|
|
10739
|
+
nounCount: nouns.length,
|
|
10740
|
+
verbCount: verbs.length
|
|
10741
|
+
}
|
|
10742
|
+
};
|
|
10485
10743
|
}
|
|
10486
10744
|
}
|
|
10487
10745
|
|
|
@@ -14193,28 +14451,21 @@ class BrainyMCPService {
|
|
|
14193
14451
|
}
|
|
14194
14452
|
}
|
|
14195
14453
|
|
|
14196
|
-
/**
|
|
14197
|
-
* OPFS BrainyData
|
|
14198
|
-
* A vector database using HNSW indexing with Origin Private File System storage
|
|
14199
|
-
*/
|
|
14200
|
-
// Import unified text encoding utilities first to ensure they're available
|
|
14201
|
-
// Apply the TensorFlow.js platform patch if needed
|
|
14202
|
-
applyTensorFlowPatch();
|
|
14203
|
-
|
|
14204
14454
|
// Make Buffer available globally
|
|
14205
14455
|
if (typeof window !== 'undefined' && typeof globalThis.Buffer === 'undefined') {
|
|
14206
14456
|
globalThis.Buffer = buffer$1.Buffer;
|
|
14207
14457
|
}
|
|
14208
|
-
//
|
|
14209
|
-
applyTensorFlowPatch();
|
|
14210
|
-
// Export environment information
|
|
14458
|
+
// Export environment information with lazy evaluation
|
|
14211
14459
|
const environment = {
|
|
14212
|
-
isBrowser
|
|
14213
|
-
|
|
14214
|
-
|
|
14215
|
-
|
|
14216
|
-
|
|
14217
|
-
|
|
14460
|
+
get isBrowser() {
|
|
14461
|
+
return isBrowser$1();
|
|
14462
|
+
},
|
|
14463
|
+
get isNode() {
|
|
14464
|
+
return isNode();
|
|
14465
|
+
},
|
|
14466
|
+
get isServerless() {
|
|
14467
|
+
return !isBrowser$1() && !isNode();
|
|
14468
|
+
}
|
|
14218
14469
|
};
|
|
14219
14470
|
// Make environment information available globally
|
|
14220
14471
|
if (typeof globalThis !== 'undefined') {
|
|
@@ -14227,6 +14478,38 @@ console.log(`Brainy running in ${environment.isBrowser
|
|
|
14227
14478
|
? 'Node.js'
|
|
14228
14479
|
: 'serverless/unknown'} environment`);
|
|
14229
14480
|
|
|
14481
|
+
// Util shim with TextEncoder/TextDecoder support
|
|
14482
|
+
const TextEncoder$1 = globalThis.TextEncoder || (typeof global !== 'undefined' && global.TextEncoder) || class TextEncoder {
|
|
14483
|
+
encode(input) {
|
|
14484
|
+
return new Uint8Array(Buffer.from(input, 'utf8'));
|
|
14485
|
+
}
|
|
14486
|
+
};
|
|
14487
|
+
|
|
14488
|
+
const TextDecoder$1 = globalThis.TextDecoder || (typeof global !== 'undefined' && global.TextDecoder) || class TextDecoder {
|
|
14489
|
+
decode(input) {
|
|
14490
|
+
return Buffer.from(input).toString('utf8');
|
|
14491
|
+
}
|
|
14492
|
+
};
|
|
14493
|
+
|
|
14494
|
+
const types = {
|
|
14495
|
+
isFloat32Array: (arr) => arr instanceof Float32Array,
|
|
14496
|
+
isInt32Array: (arr) => arr instanceof Int32Array,
|
|
14497
|
+
isUint8Array: (arr) => arr instanceof Uint8Array,
|
|
14498
|
+
isUint8ClampedArray: (arr) => arr instanceof Uint8ClampedArray
|
|
14499
|
+
};
|
|
14500
|
+
|
|
14501
|
+
var _utilShim = { TextEncoder: TextEncoder$1, TextDecoder: TextDecoder$1, types };
|
|
14502
|
+
const promises = {};
|
|
14503
|
+
|
|
14504
|
+
var _utilShim$1 = /*#__PURE__*/Object.freeze({
|
|
14505
|
+
__proto__: null,
|
|
14506
|
+
TextDecoder: TextDecoder$1,
|
|
14507
|
+
TextEncoder: TextEncoder$1,
|
|
14508
|
+
default: _utilShim,
|
|
14509
|
+
promises: promises,
|
|
14510
|
+
types: types
|
|
14511
|
+
});
|
|
14512
|
+
|
|
14230
14513
|
/**
|
|
14231
14514
|
* @license
|
|
14232
14515
|
* Copyright 2020 Google LLC. All Rights Reserved.
|
|
@@ -22274,14 +22557,6 @@ var _nodeResolve_empty$1 = /*#__PURE__*/Object.freeze({
|
|
|
22274
22557
|
|
|
22275
22558
|
var require$$0 = /*@__PURE__*/getAugmentedNamespace(_nodeResolve_empty$1);
|
|
22276
22559
|
|
|
22277
|
-
var _utilShim = {}; const promises = {};
|
|
22278
|
-
|
|
22279
|
-
var _utilShim$1 = /*#__PURE__*/Object.freeze({
|
|
22280
|
-
__proto__: null,
|
|
22281
|
-
default: _utilShim,
|
|
22282
|
-
promises: promises
|
|
22283
|
-
});
|
|
22284
|
-
|
|
22285
22560
|
var require$$1 = /*@__PURE__*/getAugmentedNamespace(_utilShim$1);
|
|
22286
22561
|
|
|
22287
22562
|
/**
|
|
@@ -22312,7 +22587,7 @@ class PlatformNode {
|
|
|
22312
22587
|
this.util = require$$1;
|
|
22313
22588
|
// According to the spec, the built-in encoder can do only UTF-8 encoding.
|
|
22314
22589
|
// https://developer.mozilla.org/en-US/docs/Web/API/TextEncoder/TextEncoder
|
|
22315
|
-
this.textEncoder = new TextEncoder();
|
|
22590
|
+
this.textEncoder = new this.util.TextEncoder();
|
|
22316
22591
|
}
|
|
22317
22592
|
fetch(path, requestInits) {
|
|
22318
22593
|
if (env().global.fetch != null) {
|
|
@@ -87531,9 +87806,5 @@ var universalSentenceEncoder_esm = /*#__PURE__*/Object.freeze({
|
|
|
87531
87806
|
version: version
|
|
87532
87807
|
});
|
|
87533
87808
|
|
|
87534
|
-
|
|
87535
|
-
__proto__: null
|
|
87536
|
-
});
|
|
87537
|
-
|
|
87538
|
-
export { AugmentationType, BrainyData, BrainyMCPAdapter, BrainyMCPService, ExecutionMode$1 as ExecutionMode, FileSystemStorage, FileSystemStorageAugmentation, HNSWIndex, HNSWIndexOptimized, MCPAugmentationToolset, MCPRequestType, MCP_VERSION, MemoryStorage, MemoryStorageAugmentation, NounType, OPFSStorage, OPFSStorageAugmentation, Pipeline, S3CompatibleStorage as R2Storage, S3CompatibleStorage, SequentialPipeline, ServerSearchActivationAugmentation, ServerSearchConduitAugmentation, StreamlinedExecutionMode, UniversalSentenceEncoder$1 as UniversalSentenceEncoder, VerbType, WebRTCConduitAugmentation, WebSocketConduitAugmentation, addWebSocketSupport, areWebWorkersAvailable, areWorkerThreadsAvailable, areWorkerThreadsAvailableSync, augmentationPipeline$1 as augmentationPipeline, availableAugmentations, cleanupWorkerPools, cosineDistance$1 as cosineDistance, createAugmentationRegistryPlugin, createAugmentationRegistryRollupPlugin, createConduitAugmentation, createEmbeddingFunction, createMemoryAugmentation, createPipeline, createSenseAugmentation, createServerSearchAugmentations, createStorage, createStreamingPipeline, createTensorFlowEmbeddingFunction, createThreadedEmbeddingFunction, defaultEmbeddingFunction, dotProductDistance, environment, euclideanDistance, executeAugmentation, executeByType, executeInThread, executeSingle, executeStreamlined, getAugmentationsByType, initializeAugmentationPipeline, isBrowser$1 as isBrowser, isNode, isThreadingAvailable, isThreadingAvailableAsync, isWebWorker, loadAugmentationModule, loadAugmentationsFromModules, manhattanDistance, pipeline, processStaticData, processStreamingData, registerAugmentation, sequentialPipeline, setAugmentationEnabled };
|
|
87809
|
+
export { AugmentationType, BrainyData, BrainyMCPAdapter, BrainyMCPService, ExecutionMode$1 as ExecutionMode, FileSystemStorage, FileSystemStorageAugmentation, HNSWIndex, HNSWIndexOptimized, MCPAugmentationToolset, MCPRequestType, MCP_VERSION, MemoryStorage, MemoryStorageAugmentation, NounType, OPFSStorage, OPFSStorageAugmentation, Pipeline, S3CompatibleStorage as R2Storage, S3CompatibleStorage, SequentialPipeline, ServerSearchActivationAugmentation, ServerSearchConduitAugmentation, StreamlinedExecutionMode, UniversalSentenceEncoder$1 as UniversalSentenceEncoder, VerbType, WebRTCConduitAugmentation, WebSocketConduitAugmentation, addWebSocketSupport, applyTensorFlowPatch, areWebWorkersAvailable, areWorkerThreadsAvailable, areWorkerThreadsAvailableSync, augmentationPipeline$1 as augmentationPipeline, availableAugmentations, cleanupWorkerPools, cosineDistance$1 as cosineDistance, createAugmentationRegistryPlugin, createAugmentationRegistryRollupPlugin, createConduitAugmentation, createEmbeddingFunction, createMemoryAugmentation, createPipeline, createSenseAugmentation, createServerSearchAugmentations, createStorage, createStreamingPipeline, createTensorFlowEmbeddingFunction, createThreadedEmbeddingFunction, defaultEmbeddingFunction, dotProductDistance, environment, euclideanDistance, executeAugmentation, executeByType, executeInThread, executeSingle, executeStreamlined, getAugmentationsByType, initializeAugmentationPipeline, isBrowser$1 as isBrowser, isNode, isThreadingAvailable, isThreadingAvailableAsync, isWebWorker, loadAugmentationModule, loadAugmentationsFromModules, manhattanDistance, pipeline, processStaticData, processStreamingData, registerAugmentation, sequentialPipeline, setAugmentationEnabled };
|
|
87539
87810
|
//# sourceMappingURL=unified.js.map
|