@soulcraft/brainy 4.1.0 β 4.1.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +21 -0
- package/dist/brainy.js +2 -0
- package/dist/storage/adapters/azureBlobStorage.js +2 -5
- package/dist/storage/adapters/baseStorageAdapter.d.ts +16 -4
- package/dist/storage/adapters/baseStorageAdapter.js +42 -26
- package/dist/storage/adapters/fileSystemStorage.js +4 -23
- package/dist/storage/adapters/gcsStorage.js +4 -10
- package/dist/storage/adapters/memoryStorage.js +2 -1
- package/dist/storage/adapters/r2Storage.js +2 -4
- package/dist/storage/baseStorage.d.ts +12 -0
- package/dist/storage/baseStorage.js +45 -2
- package/dist/utils/embedding.js +1 -1
- package/dist/utils/environment.js +2 -2
- package/dist/utils/rebuildCounts.d.ts +40 -0
- package/dist/utils/rebuildCounts.js +112 -0
- package/dist/utils/workerUtils.d.ts +1 -1
- package/dist/utils/workerUtils.js +2 -2
- package/package.json +1 -1
package/CHANGELOG.md
CHANGED
|
@@ -2,6 +2,27 @@
|
|
|
2
2
|
|
|
3
3
|
All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines.
|
|
4
4
|
|
|
5
|
+
### [4.1.2](https://github.com/soulcraftlabs/brainy/compare/v4.1.1...v4.1.2) (2025-10-21)
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
### π Bug Fixes
|
|
9
|
+
|
|
10
|
+
* **storage**: resolve count synchronization race condition across all storage adapters ([798a694](https://github.com/soulcraftlabs/brainy/commit/798a694))
|
|
11
|
+
- Fixed critical bug where entity and relationship counts were not tracked correctly during add(), relate(), and import()
|
|
12
|
+
- Root cause: Race condition where count increment tried to read metadata before it was saved
|
|
13
|
+
- Fixed in baseStorage for all storage adapters (FileSystem, GCS, R2, Azure, Memory, OPFS, S3, TypeAware)
|
|
14
|
+
- Added verb type to VerbMetadata for proper count tracking
|
|
15
|
+
- Refactored verb count methods to prevent mutex deadlocks
|
|
16
|
+
- Added rebuildCounts utility to repair corrupted counts from actual storage data
|
|
17
|
+
- Added comprehensive integration tests (11 tests covering all operations)
|
|
18
|
+
|
|
19
|
+
### [4.1.1](https://github.com/soulcraftlabs/brainy/compare/v4.1.0...v4.1.1) (2025-10-20)
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
### π Bug Fixes
|
|
23
|
+
|
|
24
|
+
* correct Node.js version references from 24 to 22 in comments and code ([22513ff](https://github.com/soulcraftlabs/brainy/commit/22513ffcb40cc6498898400ac5d1bae19c5d02ed))
|
|
25
|
+
|
|
5
26
|
## [4.1.0](https://github.com/soulcraftlabs/brainy/compare/v4.0.1...v4.1.0) (2025-10-20)
|
|
6
27
|
|
|
7
28
|
|
package/dist/brainy.js
CHANGED
|
@@ -648,7 +648,9 @@ export class Brainy {
|
|
|
648
648
|
const relationVector = fromEntity.vector.map((v, i) => (v + toEntity.vector[i]) / 2);
|
|
649
649
|
return this.augmentationRegistry.execute('relate', params, async () => {
|
|
650
650
|
// v4.0.0: Prepare verb metadata
|
|
651
|
+
// CRITICAL (v4.1.2): Include verb type in metadata for count tracking
|
|
651
652
|
const verbMetadata = {
|
|
653
|
+
verb: params.type, // Store verb type for count synchronization
|
|
652
654
|
weight: params.weight ?? 1.0,
|
|
653
655
|
...(params.metadata || {}),
|
|
654
656
|
createdAt: Date.now()
|
|
@@ -791,11 +791,8 @@ export class AzureBlobStorage extends BaseStorage {
|
|
|
791
791
|
});
|
|
792
792
|
// Update cache
|
|
793
793
|
this.verbCacheManager.set(edge.id, edge);
|
|
794
|
-
//
|
|
795
|
-
|
|
796
|
-
if (metadata && metadata.type) {
|
|
797
|
-
await this.incrementVerbCount(metadata.type);
|
|
798
|
-
}
|
|
794
|
+
// Count tracking happens in baseStorage.saveVerbMetadata_internal (v4.1.2)
|
|
795
|
+
// This fixes the race condition where metadata didn't exist yet
|
|
799
796
|
this.logger.trace(`Edge ${edge.id} saved successfully`);
|
|
800
797
|
this.releaseBackpressure(true, requestId);
|
|
801
798
|
}
|
|
@@ -320,15 +320,27 @@ export declare abstract class BaseStorageAdapter implements StorageAdapter {
|
|
|
320
320
|
*/
|
|
321
321
|
protected decrementEntityCountSafe(type: string): Promise<void>;
|
|
322
322
|
/**
|
|
323
|
-
* Increment verb count - O(1) operation
|
|
323
|
+
* Increment verb count - O(1) operation (v4.1.2: now synchronous)
|
|
324
|
+
* Protected by storage-specific mechanisms (mutex, distributed consensus, etc.)
|
|
325
|
+
* @param type The verb type
|
|
326
|
+
*/
|
|
327
|
+
protected incrementVerbCount(type: string): void;
|
|
328
|
+
/**
|
|
329
|
+
* Thread-safe increment for verb counts (v4.1.2)
|
|
330
|
+
* Uses mutex for single-node, distributed consensus for multi-node
|
|
331
|
+
* @param type The verb type
|
|
332
|
+
*/
|
|
333
|
+
protected incrementVerbCountSafe(type: string): Promise<void>;
|
|
334
|
+
/**
|
|
335
|
+
* Decrement verb count - O(1) operation (v4.1.2: now synchronous)
|
|
324
336
|
* @param type The verb type
|
|
325
337
|
*/
|
|
326
|
-
protected
|
|
338
|
+
protected decrementVerbCount(type: string): void;
|
|
327
339
|
/**
|
|
328
|
-
*
|
|
340
|
+
* Thread-safe decrement for verb counts (v4.1.2)
|
|
329
341
|
* @param type The verb type
|
|
330
342
|
*/
|
|
331
|
-
protected
|
|
343
|
+
protected decrementVerbCountSafe(type: string): Promise<void>;
|
|
332
344
|
/**
|
|
333
345
|
* Detect if this storage adapter uses cloud storage (network I/O)
|
|
334
346
|
* Cloud storage benefits from batching; local storage does not.
|
|
@@ -709,45 +709,61 @@ export class BaseStorageAdapter {
|
|
|
709
709
|
});
|
|
710
710
|
}
|
|
711
711
|
/**
|
|
712
|
-
* Increment verb count - O(1) operation
|
|
712
|
+
* Increment verb count - O(1) operation (v4.1.2: now synchronous)
|
|
713
|
+
* Protected by storage-specific mechanisms (mutex, distributed consensus, etc.)
|
|
714
|
+
* @param type The verb type
|
|
715
|
+
*/
|
|
716
|
+
incrementVerbCount(type) {
|
|
717
|
+
this.verbCounts.set(type, (this.verbCounts.get(type) || 0) + 1);
|
|
718
|
+
this.totalVerbCount++;
|
|
719
|
+
// Update cache
|
|
720
|
+
this.countCache.set('verbs_count', {
|
|
721
|
+
count: this.totalVerbCount,
|
|
722
|
+
timestamp: Date.now()
|
|
723
|
+
});
|
|
724
|
+
}
|
|
725
|
+
/**
|
|
726
|
+
* Thread-safe increment for verb counts (v4.1.2)
|
|
727
|
+
* Uses mutex for single-node, distributed consensus for multi-node
|
|
713
728
|
* @param type The verb type
|
|
714
729
|
*/
|
|
715
|
-
async
|
|
730
|
+
async incrementVerbCountSafe(type) {
|
|
716
731
|
const mutex = getGlobalMutex();
|
|
717
732
|
await mutex.runExclusive(`count-verb-${type}`, async () => {
|
|
718
|
-
this.
|
|
719
|
-
this.totalVerbCount++;
|
|
720
|
-
// Update cache
|
|
721
|
-
this.countCache.set('verbs_count', {
|
|
722
|
-
count: this.totalVerbCount,
|
|
723
|
-
timestamp: Date.now()
|
|
724
|
-
});
|
|
733
|
+
this.incrementVerbCount(type);
|
|
725
734
|
// Smart batching (v3.32.3+): Adapts to storage type
|
|
726
735
|
await this.scheduleCountPersist();
|
|
727
736
|
});
|
|
728
737
|
}
|
|
729
738
|
/**
|
|
730
|
-
* Decrement verb count - O(1) operation
|
|
739
|
+
* Decrement verb count - O(1) operation (v4.1.2: now synchronous)
|
|
740
|
+
* @param type The verb type
|
|
741
|
+
*/
|
|
742
|
+
decrementVerbCount(type) {
|
|
743
|
+
const current = this.verbCounts.get(type) || 0;
|
|
744
|
+
if (current > 1) {
|
|
745
|
+
this.verbCounts.set(type, current - 1);
|
|
746
|
+
}
|
|
747
|
+
else {
|
|
748
|
+
this.verbCounts.delete(type);
|
|
749
|
+
}
|
|
750
|
+
if (this.totalVerbCount > 0) {
|
|
751
|
+
this.totalVerbCount--;
|
|
752
|
+
}
|
|
753
|
+
// Update cache
|
|
754
|
+
this.countCache.set('verbs_count', {
|
|
755
|
+
count: this.totalVerbCount,
|
|
756
|
+
timestamp: Date.now()
|
|
757
|
+
});
|
|
758
|
+
}
|
|
759
|
+
/**
|
|
760
|
+
* Thread-safe decrement for verb counts (v4.1.2)
|
|
731
761
|
* @param type The verb type
|
|
732
762
|
*/
|
|
733
|
-
async
|
|
763
|
+
async decrementVerbCountSafe(type) {
|
|
734
764
|
const mutex = getGlobalMutex();
|
|
735
765
|
await mutex.runExclusive(`count-verb-${type}`, async () => {
|
|
736
|
-
|
|
737
|
-
if (current > 1) {
|
|
738
|
-
this.verbCounts.set(type, current - 1);
|
|
739
|
-
}
|
|
740
|
-
else {
|
|
741
|
-
this.verbCounts.delete(type);
|
|
742
|
-
}
|
|
743
|
-
if (this.totalVerbCount > 0) {
|
|
744
|
-
this.totalVerbCount--;
|
|
745
|
-
}
|
|
746
|
-
// Update cache
|
|
747
|
-
this.countCache.set('verbs_count', {
|
|
748
|
-
count: this.totalVerbCount,
|
|
749
|
-
timestamp: Date.now()
|
|
750
|
-
});
|
|
766
|
+
this.decrementVerbCount(type);
|
|
751
767
|
// Smart batching (v3.32.3+): Adapts to storage type
|
|
752
768
|
await this.scheduleCountPersist();
|
|
753
769
|
});
|
|
@@ -179,8 +179,6 @@ export class FileSystemStorage extends BaseStorage {
|
|
|
179
179
|
*/
|
|
180
180
|
async saveNode(node) {
|
|
181
181
|
await this.ensureInitialized();
|
|
182
|
-
// Check if this is a new node to update counts
|
|
183
|
-
const isNew = !(await this.fileExists(this.getNodePath(node.id)));
|
|
184
182
|
// Convert connections Map to a serializable format
|
|
185
183
|
// CRITICAL: Only save lightweight vector data (no metadata)
|
|
186
184
|
// Metadata is saved separately via saveNounMetadata() (2-file system)
|
|
@@ -194,17 +192,8 @@ export class FileSystemStorage extends BaseStorage {
|
|
|
194
192
|
const filePath = this.getNodePath(node.id);
|
|
195
193
|
await this.ensureDirectoryExists(path.dirname(filePath));
|
|
196
194
|
await fs.promises.writeFile(filePath, JSON.stringify(serializableNode, null, 2));
|
|
197
|
-
//
|
|
198
|
-
|
|
199
|
-
// v4.0.0: Get type from separate metadata storage
|
|
200
|
-
const metadata = await this.getNounMetadata(node.id);
|
|
201
|
-
const type = metadata?.noun || 'default';
|
|
202
|
-
this.incrementEntityCount(type);
|
|
203
|
-
// Persist counts periodically (every 10 operations for efficiency)
|
|
204
|
-
if (this.totalNounCount % 10 === 0) {
|
|
205
|
-
await this.persistCounts();
|
|
206
|
-
}
|
|
207
|
-
}
|
|
195
|
+
// Count tracking happens in baseStorage.saveNounMetadata_internal (v4.1.2)
|
|
196
|
+
// This fixes the race condition where metadata didn't exist yet
|
|
208
197
|
}
|
|
209
198
|
/**
|
|
210
199
|
* Get a node from storage
|
|
@@ -354,8 +343,6 @@ export class FileSystemStorage extends BaseStorage {
|
|
|
354
343
|
*/
|
|
355
344
|
async saveEdge(edge) {
|
|
356
345
|
await this.ensureInitialized();
|
|
357
|
-
// Check if this is a new edge to update counts
|
|
358
|
-
const isNew = !(await this.fileExists(this.getVerbPath(edge.id)));
|
|
359
346
|
// Convert connections Map to a serializable format
|
|
360
347
|
// ARCHITECTURAL FIX (v3.50.1): Include core relational fields in verb vector file
|
|
361
348
|
// These fields are essential for 90% of operations - no metadata lookup needed
|
|
@@ -373,14 +360,8 @@ export class FileSystemStorage extends BaseStorage {
|
|
|
373
360
|
const filePath = this.getVerbPath(edge.id);
|
|
374
361
|
await this.ensureDirectoryExists(path.dirname(filePath));
|
|
375
362
|
await fs.promises.writeFile(filePath, JSON.stringify(serializableEdge, null, 2));
|
|
376
|
-
//
|
|
377
|
-
|
|
378
|
-
this.totalVerbCount++;
|
|
379
|
-
// Persist counts periodically (every 10 operations for efficiency)
|
|
380
|
-
if (this.totalVerbCount % 10 === 0) {
|
|
381
|
-
this.persistCounts(); // Async persist, don't await
|
|
382
|
-
}
|
|
383
|
-
}
|
|
363
|
+
// Count tracking happens in baseStorage.saveVerbMetadata_internal (v4.1.2)
|
|
364
|
+
// This fixes the race condition where metadata didn't exist yet
|
|
384
365
|
}
|
|
385
366
|
/**
|
|
386
367
|
* Get an edge from storage
|
|
@@ -353,11 +353,8 @@ export class GcsStorage extends BaseStorage {
|
|
|
353
353
|
this.nounCacheManager.set(node.id, node);
|
|
354
354
|
}
|
|
355
355
|
// Note: Empty vectors are intentional during HNSW lazy mode - not logged
|
|
356
|
-
//
|
|
357
|
-
|
|
358
|
-
if (metadata && metadata.type) {
|
|
359
|
-
await this.incrementEntityCountSafe(metadata.type);
|
|
360
|
-
}
|
|
356
|
+
// Count tracking happens in baseStorage.saveNounMetadata_internal (v4.1.2)
|
|
357
|
+
// This fixes the race condition where metadata didn't exist yet
|
|
361
358
|
this.logger.trace(`Node ${node.id} saved successfully`);
|
|
362
359
|
this.releaseBackpressure(true, requestId);
|
|
363
360
|
}
|
|
@@ -663,11 +660,8 @@ export class GcsStorage extends BaseStorage {
|
|
|
663
660
|
});
|
|
664
661
|
// Update cache
|
|
665
662
|
this.verbCacheManager.set(edge.id, edge);
|
|
666
|
-
//
|
|
667
|
-
|
|
668
|
-
if (metadata && metadata.type) {
|
|
669
|
-
await this.incrementVerbCount(metadata.type);
|
|
670
|
-
}
|
|
663
|
+
// Count tracking happens in baseStorage.saveVerbMetadata_internal (v4.1.2)
|
|
664
|
+
// This fixes the race condition where metadata didn't exist yet
|
|
671
665
|
this.logger.trace(`Edge ${edge.id} saved successfully`);
|
|
672
666
|
this.releaseBackpressure(true, requestId);
|
|
673
667
|
}
|
|
@@ -56,7 +56,8 @@ export class MemoryStorage extends BaseStorage {
|
|
|
56
56
|
}
|
|
57
57
|
// Save the noun directly in the nouns map
|
|
58
58
|
this.nouns.set(noun.id, nounCopy);
|
|
59
|
-
//
|
|
59
|
+
// Count tracking happens in baseStorage.saveNounMetadata_internal (v4.1.2)
|
|
60
|
+
// This fixes the race condition where metadata didn't exist yet
|
|
60
61
|
}
|
|
61
62
|
/**
|
|
62
63
|
* Get a noun from storage (v4.0.0: returns pure vector only)
|
|
@@ -574,10 +574,8 @@ export class R2Storage extends BaseStorage {
|
|
|
574
574
|
ContentType: 'application/json'
|
|
575
575
|
}));
|
|
576
576
|
this.verbCacheManager.set(edge.id, edge);
|
|
577
|
-
|
|
578
|
-
|
|
579
|
-
await this.incrementVerbCount(metadata.type);
|
|
580
|
-
}
|
|
577
|
+
// Count tracking happens in baseStorage.saveVerbMetadata_internal (v4.1.2)
|
|
578
|
+
// This fixes the race condition where metadata didn't exist yet
|
|
581
579
|
this.releaseBackpressure(true, requestId);
|
|
582
580
|
}
|
|
583
581
|
catch (error) {
|
|
@@ -231,6 +231,11 @@ export declare abstract class BaseStorage extends BaseStorageAdapter {
|
|
|
231
231
|
/**
|
|
232
232
|
* Internal method for saving noun metadata (v4.0.0: now typed)
|
|
233
233
|
* Uses routing logic to handle both UUIDs (sharded) and system keys (unsharded)
|
|
234
|
+
*
|
|
235
|
+
* CRITICAL (v4.1.2): Count synchronization happens here
|
|
236
|
+
* This ensures counts are updated AFTER metadata exists, fixing the race condition
|
|
237
|
+
* where storage adapters tried to read metadata before it was saved.
|
|
238
|
+
*
|
|
234
239
|
* @protected
|
|
235
240
|
*/
|
|
236
241
|
protected saveNounMetadata_internal(id: string, metadata: NounMetadata): Promise<void>;
|
|
@@ -252,6 +257,13 @@ export declare abstract class BaseStorage extends BaseStorageAdapter {
|
|
|
252
257
|
/**
|
|
253
258
|
* Internal method for saving verb metadata (v4.0.0: now typed)
|
|
254
259
|
* Uses routing logic to handle both UUIDs (sharded) and system keys (unsharded)
|
|
260
|
+
*
|
|
261
|
+
* CRITICAL (v4.1.2): Count synchronization happens here
|
|
262
|
+
* This ensures verb counts are updated AFTER metadata exists, fixing the race condition
|
|
263
|
+
* where storage adapters tried to read metadata before it was saved.
|
|
264
|
+
*
|
|
265
|
+
* Note: Verb type is now stored in both HNSWVerb (vector file) and VerbMetadata for count tracking
|
|
266
|
+
*
|
|
255
267
|
* @protected
|
|
256
268
|
*/
|
|
257
269
|
protected saveVerbMetadata_internal(id: string, metadata: VerbMetadata): Promise<void>;
|
|
@@ -706,12 +706,32 @@ export class BaseStorage extends BaseStorageAdapter {
|
|
|
706
706
|
/**
|
|
707
707
|
* Internal method for saving noun metadata (v4.0.0: now typed)
|
|
708
708
|
* Uses routing logic to handle both UUIDs (sharded) and system keys (unsharded)
|
|
709
|
+
*
|
|
710
|
+
* CRITICAL (v4.1.2): Count synchronization happens here
|
|
711
|
+
* This ensures counts are updated AFTER metadata exists, fixing the race condition
|
|
712
|
+
* where storage adapters tried to read metadata before it was saved.
|
|
713
|
+
*
|
|
709
714
|
* @protected
|
|
710
715
|
*/
|
|
711
716
|
async saveNounMetadata_internal(id, metadata) {
|
|
712
717
|
await this.ensureInitialized();
|
|
718
|
+
// Determine if this is a new entity by checking if metadata already exists
|
|
713
719
|
const keyInfo = this.analyzeKey(id, 'noun-metadata');
|
|
714
|
-
|
|
720
|
+
const existingMetadata = await this.readObjectFromPath(keyInfo.fullPath);
|
|
721
|
+
const isNew = !existingMetadata;
|
|
722
|
+
// Save the metadata
|
|
723
|
+
await this.writeObjectToPath(keyInfo.fullPath, metadata);
|
|
724
|
+
// CRITICAL FIX (v4.1.2): Increment count for new entities
|
|
725
|
+
// This runs AFTER metadata is saved, guaranteeing type information is available
|
|
726
|
+
// Uses synchronous increment since storage operations are already serialized
|
|
727
|
+
// Fixes Bug #1: Count synchronization failure during add() and import()
|
|
728
|
+
if (isNew && metadata.noun) {
|
|
729
|
+
this.incrementEntityCount(metadata.noun);
|
|
730
|
+
// Persist counts asynchronously (fire and forget)
|
|
731
|
+
this.scheduleCountPersist().catch(() => {
|
|
732
|
+
// Ignore persist errors - will retry on next operation
|
|
733
|
+
});
|
|
734
|
+
}
|
|
715
735
|
}
|
|
716
736
|
/**
|
|
717
737
|
* Get noun metadata from storage (v4.0.0: now typed)
|
|
@@ -742,12 +762,35 @@ export class BaseStorage extends BaseStorageAdapter {
|
|
|
742
762
|
/**
|
|
743
763
|
* Internal method for saving verb metadata (v4.0.0: now typed)
|
|
744
764
|
* Uses routing logic to handle both UUIDs (sharded) and system keys (unsharded)
|
|
765
|
+
*
|
|
766
|
+
* CRITICAL (v4.1.2): Count synchronization happens here
|
|
767
|
+
* This ensures verb counts are updated AFTER metadata exists, fixing the race condition
|
|
768
|
+
* where storage adapters tried to read metadata before it was saved.
|
|
769
|
+
*
|
|
770
|
+
* Note: Verb type is now stored in both HNSWVerb (vector file) and VerbMetadata for count tracking
|
|
771
|
+
*
|
|
745
772
|
* @protected
|
|
746
773
|
*/
|
|
747
774
|
async saveVerbMetadata_internal(id, metadata) {
|
|
748
775
|
await this.ensureInitialized();
|
|
776
|
+
// Determine if this is a new verb by checking if metadata already exists
|
|
749
777
|
const keyInfo = this.analyzeKey(id, 'verb-metadata');
|
|
750
|
-
|
|
778
|
+
const existingMetadata = await this.readObjectFromPath(keyInfo.fullPath);
|
|
779
|
+
const isNew = !existingMetadata;
|
|
780
|
+
// Save the metadata
|
|
781
|
+
await this.writeObjectToPath(keyInfo.fullPath, metadata);
|
|
782
|
+
// CRITICAL FIX (v4.1.2): Increment verb count for new relationships
|
|
783
|
+
// This runs AFTER metadata is saved
|
|
784
|
+
// Verb type is now stored in metadata (as of v4.1.2) to avoid loading HNSWVerb
|
|
785
|
+
// Uses synchronous increment since storage operations are already serialized
|
|
786
|
+
// Fixes Bug #2: Count synchronization failure during relate() and import()
|
|
787
|
+
if (isNew && metadata.verb) {
|
|
788
|
+
this.incrementVerbCount(metadata.verb);
|
|
789
|
+
// Persist counts asynchronously (fire and forget)
|
|
790
|
+
this.scheduleCountPersist().catch(() => {
|
|
791
|
+
// Ignore persist errors - will retry on next operation
|
|
792
|
+
});
|
|
793
|
+
}
|
|
751
794
|
}
|
|
752
795
|
/**
|
|
753
796
|
* Get verb metadata from storage (v4.0.0: now typed)
|
package/dist/utils/embedding.js
CHANGED
|
@@ -10,7 +10,7 @@ import { pipeline, env } from '@huggingface/transformers';
|
|
|
10
10
|
if (typeof process !== 'undefined' && process.env) {
|
|
11
11
|
process.env.ORT_DISABLE_MEMORY_ARENA = '1';
|
|
12
12
|
process.env.ORT_DISABLE_MEMORY_PATTERN = '1';
|
|
13
|
-
// Force single-threaded operation for maximum stability (Node.js
|
|
13
|
+
// Force single-threaded operation for maximum stability (Node.js 22 LTS)
|
|
14
14
|
process.env.ORT_INTRA_OP_NUM_THREADS = '1'; // Single thread for operators
|
|
15
15
|
process.env.ORT_INTER_OP_NUM_THREADS = '1'; // Single thread for sessions
|
|
16
16
|
process.env.ORT_NUM_THREADS = '1'; // Additional safety override
|
|
@@ -56,8 +56,8 @@ export async function areWorkerThreadsAvailable() {
|
|
|
56
56
|
export function areWorkerThreadsAvailableSync() {
|
|
57
57
|
if (!isNode())
|
|
58
58
|
return false;
|
|
59
|
-
// In Node.js
|
|
60
|
-
return parseInt(process.versions.node.split('.')[0]) >=
|
|
59
|
+
// In Node.js 22+, worker_threads is always available
|
|
60
|
+
return parseInt(process.versions.node.split('.')[0]) >= 22;
|
|
61
61
|
}
|
|
62
62
|
/**
|
|
63
63
|
* Determine if threading is available in the current environment
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Rebuild Counts Utility
|
|
3
|
+
*
|
|
4
|
+
* Scans storage and rebuilds counts.json from actual data
|
|
5
|
+
* Use this to fix databases affected by the v4.1.1 count synchronization bug
|
|
6
|
+
*
|
|
7
|
+
* NO MOCKS - Production-ready implementation
|
|
8
|
+
*/
|
|
9
|
+
import type { BaseStorage } from '../storage/baseStorage.js';
|
|
10
|
+
export interface RebuildCountsResult {
|
|
11
|
+
/** Total number of entities (nouns) found */
|
|
12
|
+
nounCount: number;
|
|
13
|
+
/** Total number of relationships (verbs) found */
|
|
14
|
+
verbCount: number;
|
|
15
|
+
/** Entity counts by type */
|
|
16
|
+
entityCounts: Map<string, number>;
|
|
17
|
+
/** Verb counts by type */
|
|
18
|
+
verbCounts: Map<string, number>;
|
|
19
|
+
/** Processing time in milliseconds */
|
|
20
|
+
duration: number;
|
|
21
|
+
}
|
|
22
|
+
/**
|
|
23
|
+
* Rebuild counts.json from actual storage data
|
|
24
|
+
*
|
|
25
|
+
* This scans all entities and relationships in storage and reconstructs
|
|
26
|
+
* the counts index from scratch. Use this to fix count desynchronization.
|
|
27
|
+
*
|
|
28
|
+
* @param storage - The storage adapter to rebuild counts for
|
|
29
|
+
* @returns Promise that resolves to rebuild statistics
|
|
30
|
+
*
|
|
31
|
+
* @example
|
|
32
|
+
* ```typescript
|
|
33
|
+
* const brain = new Brainy({ storage: { type: 'filesystem', path: './brainy-data' } })
|
|
34
|
+
* await brain.init()
|
|
35
|
+
*
|
|
36
|
+
* const result = await rebuildCounts(brain.storage)
|
|
37
|
+
* console.log(`Rebuilt counts: ${result.nounCount} nouns, ${result.verbCount} verbs`)
|
|
38
|
+
* ```
|
|
39
|
+
*/
|
|
40
|
+
export declare function rebuildCounts(storage: BaseStorage): Promise<RebuildCountsResult>;
|
|
@@ -0,0 +1,112 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Rebuild Counts Utility
|
|
3
|
+
*
|
|
4
|
+
* Scans storage and rebuilds counts.json from actual data
|
|
5
|
+
* Use this to fix databases affected by the v4.1.1 count synchronization bug
|
|
6
|
+
*
|
|
7
|
+
* NO MOCKS - Production-ready implementation
|
|
8
|
+
*/
|
|
9
|
+
/**
|
|
10
|
+
* Rebuild counts.json from actual storage data
|
|
11
|
+
*
|
|
12
|
+
* This scans all entities and relationships in storage and reconstructs
|
|
13
|
+
* the counts index from scratch. Use this to fix count desynchronization.
|
|
14
|
+
*
|
|
15
|
+
* @param storage - The storage adapter to rebuild counts for
|
|
16
|
+
* @returns Promise that resolves to rebuild statistics
|
|
17
|
+
*
|
|
18
|
+
* @example
|
|
19
|
+
* ```typescript
|
|
20
|
+
* const brain = new Brainy({ storage: { type: 'filesystem', path: './brainy-data' } })
|
|
21
|
+
* await brain.init()
|
|
22
|
+
*
|
|
23
|
+
* const result = await rebuildCounts(brain.storage)
|
|
24
|
+
* console.log(`Rebuilt counts: ${result.nounCount} nouns, ${result.verbCount} verbs`)
|
|
25
|
+
* ```
|
|
26
|
+
*/
|
|
27
|
+
export async function rebuildCounts(storage) {
|
|
28
|
+
const startTime = Date.now();
|
|
29
|
+
console.log('π§ Rebuilding counts from storage...');
|
|
30
|
+
const entityCounts = new Map();
|
|
31
|
+
const verbCounts = new Map();
|
|
32
|
+
let totalNouns = 0;
|
|
33
|
+
let totalVerbs = 0;
|
|
34
|
+
// Scan all nouns using pagination
|
|
35
|
+
console.log('π Scanning entities...');
|
|
36
|
+
// Check if pagination method exists
|
|
37
|
+
const storageWithPagination = storage;
|
|
38
|
+
if (typeof storageWithPagination.getNounsWithPagination !== 'function') {
|
|
39
|
+
throw new Error('Storage adapter does not support getNounsWithPagination');
|
|
40
|
+
}
|
|
41
|
+
let hasMore = true;
|
|
42
|
+
let cursor;
|
|
43
|
+
while (hasMore) {
|
|
44
|
+
const result = await storageWithPagination.getNounsWithPagination({ limit: 100, cursor });
|
|
45
|
+
for (const noun of result.items) {
|
|
46
|
+
const metadata = await storage.getNounMetadata(noun.id);
|
|
47
|
+
if (metadata?.noun) {
|
|
48
|
+
const entityType = metadata.noun;
|
|
49
|
+
entityCounts.set(entityType, (entityCounts.get(entityType) || 0) + 1);
|
|
50
|
+
totalNouns++;
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
hasMore = result.hasMore;
|
|
54
|
+
cursor = result.nextCursor;
|
|
55
|
+
}
|
|
56
|
+
console.log(` Found ${totalNouns} entities across ${entityCounts.size} types`);
|
|
57
|
+
// Scan all verbs using pagination
|
|
58
|
+
console.log('π Scanning relationships...');
|
|
59
|
+
if (typeof storageWithPagination.getVerbsWithPagination !== 'function') {
|
|
60
|
+
throw new Error('Storage adapter does not support getVerbsWithPagination');
|
|
61
|
+
}
|
|
62
|
+
hasMore = true;
|
|
63
|
+
cursor = undefined;
|
|
64
|
+
while (hasMore) {
|
|
65
|
+
const result = await storageWithPagination.getVerbsWithPagination({ limit: 100, cursor });
|
|
66
|
+
for (const verb of result.items) {
|
|
67
|
+
if (verb.verb) {
|
|
68
|
+
const verbType = verb.verb;
|
|
69
|
+
verbCounts.set(verbType, (verbCounts.get(verbType) || 0) + 1);
|
|
70
|
+
totalVerbs++;
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
hasMore = result.hasMore;
|
|
74
|
+
cursor = result.nextCursor;
|
|
75
|
+
}
|
|
76
|
+
console.log(` Found ${totalVerbs} relationships across ${verbCounts.size} types`);
|
|
77
|
+
// Update storage adapter's in-memory counts FIRST
|
|
78
|
+
storageWithPagination.totalNounCount = totalNouns;
|
|
79
|
+
storageWithPagination.totalVerbCount = totalVerbs;
|
|
80
|
+
storageWithPagination.entityCounts = entityCounts;
|
|
81
|
+
storageWithPagination.verbCounts = verbCounts;
|
|
82
|
+
// Mark counts as pending persist (required for flushCounts to actually persist)
|
|
83
|
+
storageWithPagination.pendingCountPersist = true;
|
|
84
|
+
storageWithPagination.pendingCountOperations = 1;
|
|
85
|
+
// Persist counts using storage adapter's own persist method
|
|
86
|
+
// This ensures counts.json is written correctly (compressed or uncompressed)
|
|
87
|
+
await storageWithPagination.flushCounts();
|
|
88
|
+
const duration = Date.now() - startTime;
|
|
89
|
+
console.log(`β
Counts rebuilt successfully in ${duration}ms`);
|
|
90
|
+
console.log(` Entities: ${totalNouns}`);
|
|
91
|
+
console.log(` Relationships: ${totalVerbs}`);
|
|
92
|
+
console.log('');
|
|
93
|
+
console.log('Entity breakdown:');
|
|
94
|
+
entityCounts.forEach((count, entityType) => {
|
|
95
|
+
console.log(` ${entityType}: ${count}`);
|
|
96
|
+
});
|
|
97
|
+
if (verbCounts.size > 0) {
|
|
98
|
+
console.log('');
|
|
99
|
+
console.log('Relationship breakdown:');
|
|
100
|
+
verbCounts.forEach((count, verbType) => {
|
|
101
|
+
console.log(` ${verbType}: ${count}`);
|
|
102
|
+
});
|
|
103
|
+
}
|
|
104
|
+
return {
|
|
105
|
+
nounCount: totalNouns,
|
|
106
|
+
verbCount: totalVerbs,
|
|
107
|
+
entityCounts,
|
|
108
|
+
verbCounts,
|
|
109
|
+
duration
|
|
110
|
+
};
|
|
111
|
+
}
|
|
112
|
+
//# sourceMappingURL=rebuildCounts.js.map
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Utility functions for executing functions in Worker Threads (Node.js) or Web Workers (Browser)
|
|
3
|
-
* This implementation leverages Node.js
|
|
3
|
+
* This implementation leverages Node.js 22's stable Worker Threads API for maximum reliability
|
|
4
4
|
*/
|
|
5
5
|
/**
|
|
6
6
|
* Execute a function in a separate thread
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Utility functions for executing functions in Worker Threads (Node.js) or Web Workers (Browser)
|
|
3
|
-
* This implementation leverages Node.js
|
|
3
|
+
* This implementation leverages Node.js 22's stable Worker Threads API for maximum reliability
|
|
4
4
|
*/
|
|
5
5
|
import { isBrowser, isNode } from './environment.js';
|
|
6
6
|
// Worker pool to reuse workers
|
|
@@ -64,7 +64,7 @@ export function executeInThread(fnString, args) {
|
|
|
64
64
|
}
|
|
65
65
|
/**
|
|
66
66
|
* Execute a function in a Node.js Worker Thread
|
|
67
|
-
* Optimized for Node.js
|
|
67
|
+
* Optimized for Node.js 22 LTS with stable Worker Threads performance
|
|
68
68
|
*/
|
|
69
69
|
function executeInNodeWorker(fnString, args) {
|
|
70
70
|
return new Promise((resolve, reject) => {
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@soulcraft/brainy",
|
|
3
|
-
"version": "4.1.
|
|
3
|
+
"version": "4.1.2",
|
|
4
4
|
"description": "Universal Knowledge Protocolβ’ - World's first Triple Intelligence database unifying vector, graph, and document search in one API. 31 nouns Γ 40 verbs for infinite expressiveness.",
|
|
5
5
|
"main": "dist/index.js",
|
|
6
6
|
"module": "dist/index.js",
|