@soulcraft/brainy 6.2.0 → 6.2.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +6 -0
- package/dist/brainy.d.ts +17 -13
- package/dist/brainy.js +72 -8
- package/dist/hnsw/hnswIndex.d.ts +6 -0
- package/dist/hnsw/hnswIndex.js +81 -8
- package/dist/index.d.ts +2 -3
- package/dist/index.js +2 -3
- package/dist/storage/adapters/azureBlobStorage.js +9 -1
- package/dist/storage/adapters/gcsStorage.js +8 -1
- package/dist/storage/adapters/s3CompatibleStorage.js +7 -2
- package/dist/triple/TripleIntelligenceSystem.d.ts +1 -2
- package/dist/utils/metadataIndex.d.ts +17 -0
- package/dist/utils/metadataIndex.js +63 -0
- package/package.json +1 -1
package/CHANGELOG.md
CHANGED
|
@@ -2,6 +2,12 @@
|
|
|
2
2
|
|
|
3
3
|
All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines.
|
|
4
4
|
|
|
5
|
+
### [6.2.2](https://github.com/soulcraftlabs/brainy/compare/v6.2.1...v6.2.2) (2025-11-25)
|
|
6
|
+
|
|
7
|
+
- refactor: remove 3,700+ LOC of unused HNSW implementations (e3146ce)
|
|
8
|
+
- fix(hnsw): entry point recovery prevents import failures and log spam (52eae67)
|
|
9
|
+
|
|
10
|
+
|
|
5
11
|
## [6.2.0](https://github.com/soulcraftlabs/brainy/compare/v6.1.0...v6.2.0) (2025-11-20)
|
|
6
12
|
|
|
7
13
|
### ⚡ Critical Performance Fix
|
package/dist/brainy.d.ts
CHANGED
|
@@ -1566,9 +1566,11 @@ export declare class Brainy<T = any> implements BrainyInterface<T> {
|
|
|
1566
1566
|
get counts(): {
|
|
1567
1567
|
entities: () => number;
|
|
1568
1568
|
relationships: () => number;
|
|
1569
|
-
byType: (
|
|
1570
|
-
|
|
1571
|
-
}
|
|
1569
|
+
byType: (typeOrOptions?: string | {
|
|
1570
|
+
excludeVFS?: boolean;
|
|
1571
|
+
}, options?: {
|
|
1572
|
+
excludeVFS?: boolean;
|
|
1573
|
+
}) => Promise<number | Record<string, number>>;
|
|
1572
1574
|
byTypeEnum: (type: NounType) => number;
|
|
1573
1575
|
topTypes: (n?: number) => NounType[];
|
|
1574
1576
|
topVerbTypes: (n?: number) => VerbType[];
|
|
@@ -1579,12 +1581,12 @@ export declare class Brainy<T = any> implements BrainyInterface<T> {
|
|
|
1579
1581
|
};
|
|
1580
1582
|
byCriteria: (field: string, value: any) => Promise<number>;
|
|
1581
1583
|
getAllTypeCounts: () => Map<string, number>;
|
|
1582
|
-
getStats: (
|
|
1584
|
+
getStats: (options?: {
|
|
1585
|
+
excludeVFS?: boolean;
|
|
1586
|
+
}) => Promise<{
|
|
1583
1587
|
entities: {
|
|
1584
1588
|
total: number;
|
|
1585
|
-
byType:
|
|
1586
|
-
[k: string]: number;
|
|
1587
|
-
};
|
|
1589
|
+
byType: Record<string, number>;
|
|
1588
1590
|
};
|
|
1589
1591
|
relationships: {
|
|
1590
1592
|
totalRelationships: number;
|
|
@@ -1594,7 +1596,7 @@ export declare class Brainy<T = any> implements BrainyInterface<T> {
|
|
|
1594
1596
|
totalNodes: number;
|
|
1595
1597
|
};
|
|
1596
1598
|
density: number;
|
|
1597
|
-
}
|
|
1599
|
+
}>;
|
|
1598
1600
|
};
|
|
1599
1601
|
/**
|
|
1600
1602
|
* Augmentations API - Clean and simple
|
|
@@ -1607,14 +1609,16 @@ export declare class Brainy<T = any> implements BrainyInterface<T> {
|
|
|
1607
1609
|
/**
|
|
1608
1610
|
* Get complete statistics - convenience method
|
|
1609
1611
|
* For more granular counting, use brain.counts API
|
|
1612
|
+
* v6.2.1: Added optional excludeVFS using Roaring bitmap intersection
|
|
1613
|
+
* @param options Optional settings - excludeVFS: filter out VFS entities
|
|
1610
1614
|
* @returns Complete statistics including entities, relationships, and density
|
|
1611
1615
|
*/
|
|
1612
|
-
getStats(
|
|
1616
|
+
getStats(options?: {
|
|
1617
|
+
excludeVFS?: boolean;
|
|
1618
|
+
}): Promise<{
|
|
1613
1619
|
entities: {
|
|
1614
1620
|
total: number;
|
|
1615
|
-
byType:
|
|
1616
|
-
[k: string]: number;
|
|
1617
|
-
};
|
|
1621
|
+
byType: Record<string, number>;
|
|
1618
1622
|
};
|
|
1619
1623
|
relationships: {
|
|
1620
1624
|
totalRelationships: number;
|
|
@@ -1624,7 +1628,7 @@ export declare class Brainy<T = any> implements BrainyInterface<T> {
|
|
|
1624
1628
|
totalNodes: number;
|
|
1625
1629
|
};
|
|
1626
1630
|
density: number;
|
|
1627
|
-
}
|
|
1631
|
+
}>;
|
|
1628
1632
|
/**
|
|
1629
1633
|
* Parse natural language query using advanced NLP with 220+ patterns
|
|
1630
1634
|
* The embedding model is always available as it's core to Brainy's functionality
|
package/dist/brainy.js
CHANGED
|
@@ -6,7 +6,6 @@
|
|
|
6
6
|
*/
|
|
7
7
|
import { v4 as uuidv4 } from './universal/uuid.js';
|
|
8
8
|
import { HNSWIndex } from './hnsw/hnswIndex.js';
|
|
9
|
-
import { HNSWIndexOptimized } from './hnsw/hnswIndexOptimized.js';
|
|
10
9
|
import { TypeAwareHNSWIndex } from './hnsw/typeAwareHNSWIndex.js';
|
|
11
10
|
import { createStorage } from './storage/storageFactory.js';
|
|
12
11
|
import { defaultEmbeddingFunction, cosineDistance } from './utils/index.js';
|
|
@@ -768,7 +767,13 @@ export class Brainy {
|
|
|
768
767
|
}
|
|
769
768
|
}
|
|
770
769
|
// Operation 5-6: Update metadata index (remove old, add new)
|
|
771
|
-
|
|
770
|
+
// v6.2.1: Fix - Include type in removal metadata so noun index is properly updated
|
|
771
|
+
// existing.metadata only contains custom fields, not 'type' which maps to 'noun'
|
|
772
|
+
const removalMetadata = {
|
|
773
|
+
...existing.metadata,
|
|
774
|
+
type: existing.type // Include type so it maps to 'noun' during removal
|
|
775
|
+
};
|
|
776
|
+
tx.addOperation(new RemoveFromMetadataIndexOperation(this.metadataIndex, params.id, removalMetadata));
|
|
772
777
|
tx.addOperation(new AddToMetadataIndexOperation(this.metadataIndex, params.id, entityForIndexing));
|
|
773
778
|
});
|
|
774
779
|
});
|
|
@@ -796,7 +801,7 @@ export class Brainy {
|
|
|
796
801
|
if (this.index instanceof TypeAwareHNSWIndex && metadata.noun) {
|
|
797
802
|
tx.addOperation(new RemoveFromTypeAwareHNSWOperation(this.index, id, noun.vector, metadata.noun));
|
|
798
803
|
}
|
|
799
|
-
else if (this.index instanceof HNSWIndex
|
|
804
|
+
else if (this.index instanceof HNSWIndex) {
|
|
800
805
|
tx.addOperation(new RemoveFromHNSWOperation(this.index, id, noun.vector));
|
|
801
806
|
}
|
|
802
807
|
}
|
|
@@ -1881,7 +1886,7 @@ export class Brainy {
|
|
|
1881
1886
|
if (this.index instanceof TypeAwareHNSWIndex && metadata.noun) {
|
|
1882
1887
|
tx.addOperation(new RemoveFromTypeAwareHNSWOperation(this.index, id, noun.vector, metadata.noun));
|
|
1883
1888
|
}
|
|
1884
|
-
else if (this.index instanceof HNSWIndex
|
|
1889
|
+
else if (this.index instanceof HNSWIndex) {
|
|
1885
1890
|
tx.addOperation(new RemoveFromHNSWOperation(this.index, id, noun.vector));
|
|
1886
1891
|
}
|
|
1887
1892
|
}
|
|
@@ -3405,7 +3410,39 @@ export class Brainy {
|
|
|
3405
3410
|
// O(1) total relationship count
|
|
3406
3411
|
relationships: () => this.graphIndex.getTotalRelationshipCount(),
|
|
3407
3412
|
// O(1) count by type (string-based, backward compatible)
|
|
3408
|
-
|
|
3413
|
+
// v6.2.1: Added optional excludeVFS using Roaring bitmap intersection
|
|
3414
|
+
byType: async (typeOrOptions, options) => {
|
|
3415
|
+
// Handle overloaded signature: byType(type), byType({ excludeVFS }), byType(type, { excludeVFS })
|
|
3416
|
+
let type;
|
|
3417
|
+
let excludeVFS = false;
|
|
3418
|
+
if (typeof typeOrOptions === 'string') {
|
|
3419
|
+
type = typeOrOptions;
|
|
3420
|
+
excludeVFS = options?.excludeVFS ?? false;
|
|
3421
|
+
}
|
|
3422
|
+
else if (typeOrOptions && typeof typeOrOptions === 'object') {
|
|
3423
|
+
excludeVFS = typeOrOptions.excludeVFS ?? false;
|
|
3424
|
+
}
|
|
3425
|
+
if (excludeVFS) {
|
|
3426
|
+
const allCounts = this.metadataIndex.getAllEntityCounts();
|
|
3427
|
+
// Uses Roaring bitmap intersection - hardware accelerated
|
|
3428
|
+
const vfsCounts = await this.metadataIndex.getAllVFSEntityCounts();
|
|
3429
|
+
if (type) {
|
|
3430
|
+
const total = allCounts.get(type) || 0;
|
|
3431
|
+
const vfs = vfsCounts.get(type) || 0;
|
|
3432
|
+
return total - vfs;
|
|
3433
|
+
}
|
|
3434
|
+
// Return all counts with VFS subtracted
|
|
3435
|
+
const result = {};
|
|
3436
|
+
for (const [t, total] of allCounts) {
|
|
3437
|
+
const vfs = vfsCounts.get(t) || 0;
|
|
3438
|
+
const nonVfs = total - vfs;
|
|
3439
|
+
if (nonVfs > 0) {
|
|
3440
|
+
result[t] = nonVfs;
|
|
3441
|
+
}
|
|
3442
|
+
}
|
|
3443
|
+
return result;
|
|
3444
|
+
}
|
|
3445
|
+
// Default path (unchanged) - synchronous for backward compatibility
|
|
3409
3446
|
if (type) {
|
|
3410
3447
|
return this.metadataIndex.getEntityCountByType(type);
|
|
3411
3448
|
}
|
|
@@ -3447,7 +3484,32 @@ export class Brainy {
|
|
|
3447
3484
|
// Get all type counts as Map for performance-critical operations
|
|
3448
3485
|
getAllTypeCounts: () => this.metadataIndex.getAllEntityCounts(),
|
|
3449
3486
|
// Get complete statistics
|
|
3450
|
-
|
|
3487
|
+
// v6.2.1: Added optional excludeVFS using Roaring bitmap intersection
|
|
3488
|
+
getStats: async (options) => {
|
|
3489
|
+
if (options?.excludeVFS) {
|
|
3490
|
+
const allCounts = this.metadataIndex.getAllEntityCounts();
|
|
3491
|
+
// Uses Roaring bitmap intersection - hardware accelerated
|
|
3492
|
+
const vfsCounts = await this.metadataIndex.getAllVFSEntityCounts();
|
|
3493
|
+
// Compute non-VFS counts via subtraction
|
|
3494
|
+
const byType = {};
|
|
3495
|
+
let total = 0;
|
|
3496
|
+
for (const [type, count] of allCounts) {
|
|
3497
|
+
const vfs = vfsCounts.get(type) || 0;
|
|
3498
|
+
const nonVfs = count - vfs;
|
|
3499
|
+
if (nonVfs > 0) {
|
|
3500
|
+
byType[type] = nonVfs;
|
|
3501
|
+
total += nonVfs;
|
|
3502
|
+
}
|
|
3503
|
+
}
|
|
3504
|
+
const entityStats = { total, byType };
|
|
3505
|
+
const relationshipStats = this.graphIndex.getRelationshipStats();
|
|
3506
|
+
return {
|
|
3507
|
+
entities: entityStats,
|
|
3508
|
+
relationships: relationshipStats,
|
|
3509
|
+
density: total > 0 ? relationshipStats.totalRelationships / total : 0
|
|
3510
|
+
};
|
|
3511
|
+
}
|
|
3512
|
+
// Default path (unchanged) - synchronous for backward compatibility
|
|
3451
3513
|
const entityStats = {
|
|
3452
3514
|
total: this.metadataIndex.getTotalEntityCount(),
|
|
3453
3515
|
byType: Object.fromEntries(this.metadataIndex.getAllEntityCounts())
|
|
@@ -3474,10 +3536,12 @@ export class Brainy {
|
|
|
3474
3536
|
/**
|
|
3475
3537
|
* Get complete statistics - convenience method
|
|
3476
3538
|
* For more granular counting, use brain.counts API
|
|
3539
|
+
* v6.2.1: Added optional excludeVFS using Roaring bitmap intersection
|
|
3540
|
+
* @param options Optional settings - excludeVFS: filter out VFS entities
|
|
3477
3541
|
* @returns Complete statistics including entities, relationships, and density
|
|
3478
3542
|
*/
|
|
3479
|
-
getStats() {
|
|
3480
|
-
return this.counts.getStats();
|
|
3543
|
+
async getStats(options) {
|
|
3544
|
+
return this.counts.getStats(options);
|
|
3481
3545
|
}
|
|
3482
3546
|
// ============= HELPER METHODS =============
|
|
3483
3547
|
/**
|
package/dist/hnsw/hnswIndex.d.ts
CHANGED
|
@@ -83,6 +83,12 @@ export declare class HNSWIndex {
|
|
|
83
83
|
* Add a vector to the index
|
|
84
84
|
*/
|
|
85
85
|
addItem(item: VectorDocument): Promise<string>;
|
|
86
|
+
/**
|
|
87
|
+
* O(1) entry point recovery using highLevelNodes index (v6.2.3).
|
|
88
|
+
* At any reasonable scale (1000+ nodes), level 2+ nodes are guaranteed to exist.
|
|
89
|
+
* For tiny indexes with only level 0-1 nodes, any node works as entry point.
|
|
90
|
+
*/
|
|
91
|
+
private recoverEntryPointO1;
|
|
86
92
|
/**
|
|
87
93
|
* Search for nearest neighbors
|
|
88
94
|
*/
|
package/dist/hnsw/hnswIndex.js
CHANGED
|
@@ -210,9 +210,8 @@ export class HNSWIndex {
|
|
|
210
210
|
}
|
|
211
211
|
// Find entry point
|
|
212
212
|
if (!this.entryPointId) {
|
|
213
|
-
|
|
214
|
-
//
|
|
215
|
-
// This is a safety check
|
|
213
|
+
// No entry point but nouns exist - corrupted state, recover by using this item
|
|
214
|
+
// This shouldn't normally happen as first item sets entry point above
|
|
216
215
|
this.entryPointId = id;
|
|
217
216
|
this.maxLevel = nounLevel;
|
|
218
217
|
this.nouns.set(id, noun);
|
|
@@ -220,7 +219,7 @@ export class HNSWIndex {
|
|
|
220
219
|
}
|
|
221
220
|
const entryPoint = this.nouns.get(this.entryPointId);
|
|
222
221
|
if (!entryPoint) {
|
|
223
|
-
|
|
222
|
+
// Entry point was deleted but ID not updated - recover by using new item
|
|
224
223
|
// If the entry point doesn't exist, treat this as the first noun
|
|
225
224
|
this.entryPointId = id;
|
|
226
225
|
this.maxLevel = nounLevel;
|
|
@@ -383,6 +382,27 @@ export class HNSWIndex {
|
|
|
383
382
|
}
|
|
384
383
|
return id;
|
|
385
384
|
}
|
|
385
|
+
/**
|
|
386
|
+
* O(1) entry point recovery using highLevelNodes index (v6.2.3).
|
|
387
|
+
* At any reasonable scale (1000+ nodes), level 2+ nodes are guaranteed to exist.
|
|
388
|
+
* For tiny indexes with only level 0-1 nodes, any node works as entry point.
|
|
389
|
+
*/
|
|
390
|
+
recoverEntryPointO1() {
|
|
391
|
+
// O(1) recovery: check highLevelNodes from highest to lowest level
|
|
392
|
+
for (let level = this.MAX_TRACKED_LEVELS; level >= 2; level--) {
|
|
393
|
+
const nodesAtLevel = this.highLevelNodes.get(level);
|
|
394
|
+
if (nodesAtLevel && nodesAtLevel.size > 0) {
|
|
395
|
+
for (const nodeId of nodesAtLevel) {
|
|
396
|
+
if (this.nouns.has(nodeId)) {
|
|
397
|
+
return { id: nodeId, level };
|
|
398
|
+
}
|
|
399
|
+
}
|
|
400
|
+
}
|
|
401
|
+
}
|
|
402
|
+
// No high-level nodes - use any available node (works fine for HNSW)
|
|
403
|
+
const firstNode = this.nouns.keys().next().value;
|
|
404
|
+
return { id: firstNode ?? null, level: 0 };
|
|
405
|
+
}
|
|
386
406
|
/**
|
|
387
407
|
* Search for nearest neighbors
|
|
388
408
|
*/
|
|
@@ -398,14 +418,33 @@ export class HNSWIndex {
|
|
|
398
418
|
throw new Error(`Query vector dimension mismatch: expected ${this.dimension}, got ${queryVector.length}`);
|
|
399
419
|
}
|
|
400
420
|
// Start from the entry point
|
|
421
|
+
// If entry point is null but nouns exist, attempt O(1) recovery (v6.2.3)
|
|
422
|
+
if (!this.entryPointId && this.nouns.size > 0) {
|
|
423
|
+
const { id: recoveredId, level: recoveredLevel } = this.recoverEntryPointO1();
|
|
424
|
+
if (recoveredId) {
|
|
425
|
+
this.entryPointId = recoveredId;
|
|
426
|
+
this.maxLevel = recoveredLevel;
|
|
427
|
+
}
|
|
428
|
+
}
|
|
401
429
|
if (!this.entryPointId) {
|
|
402
|
-
|
|
430
|
+
// Truly empty index - return empty results silently
|
|
403
431
|
return [];
|
|
404
432
|
}
|
|
405
|
-
|
|
433
|
+
let entryPoint = this.nouns.get(this.entryPointId);
|
|
406
434
|
if (!entryPoint) {
|
|
407
|
-
|
|
408
|
-
|
|
435
|
+
// Entry point ID exists but noun was deleted - O(1) recovery (v6.2.3)
|
|
436
|
+
if (this.nouns.size > 0) {
|
|
437
|
+
const { id: recoveredId, level: recoveredLevel } = this.recoverEntryPointO1();
|
|
438
|
+
if (recoveredId) {
|
|
439
|
+
this.entryPointId = recoveredId;
|
|
440
|
+
this.maxLevel = recoveredLevel;
|
|
441
|
+
entryPoint = this.nouns.get(recoveredId);
|
|
442
|
+
}
|
|
443
|
+
}
|
|
444
|
+
// If still no entry point, return empty
|
|
445
|
+
if (!entryPoint) {
|
|
446
|
+
return [];
|
|
447
|
+
}
|
|
409
448
|
}
|
|
410
449
|
let currObj = entryPoint;
|
|
411
450
|
// OPTIMIZATION: Preload entry point vector
|
|
@@ -915,6 +954,40 @@ export class HNSWIndex {
|
|
|
915
954
|
offset += batchSize; // v5.7.11: Increment offset for next page
|
|
916
955
|
}
|
|
917
956
|
}
|
|
957
|
+
// Step 5: CRITICAL - Recover entry point if missing (v6.2.3 - O(1))
|
|
958
|
+
// This ensures consistency even if getHNSWSystem() returned null
|
|
959
|
+
if (this.nouns.size > 0 && this.entryPointId === null) {
|
|
960
|
+
prodLog.warn('HNSW rebuild: Entry point was null after loading nouns - recovering with O(1) lookup');
|
|
961
|
+
const { id: recoveredId, level: recoveredLevel } = this.recoverEntryPointO1();
|
|
962
|
+
this.entryPointId = recoveredId;
|
|
963
|
+
this.maxLevel = recoveredLevel;
|
|
964
|
+
prodLog.info(`HNSW entry point recovered: ${recoveredId} at level ${recoveredLevel}`);
|
|
965
|
+
// Persist recovered state to prevent future recovery
|
|
966
|
+
if (this.storage && recoveredId) {
|
|
967
|
+
await this.storage.saveHNSWSystem({
|
|
968
|
+
entryPointId: this.entryPointId,
|
|
969
|
+
maxLevel: this.maxLevel
|
|
970
|
+
}).catch((error) => {
|
|
971
|
+
prodLog.error('Failed to persist recovered HNSW system data:', error);
|
|
972
|
+
});
|
|
973
|
+
}
|
|
974
|
+
}
|
|
975
|
+
// Step 6: Validate entry point exists if set (handles stale/deleted entry point)
|
|
976
|
+
if (this.entryPointId && !this.nouns.has(this.entryPointId)) {
|
|
977
|
+
prodLog.warn(`HNSW: Entry point ${this.entryPointId} not found in loaded nouns - recovering with O(1) lookup`);
|
|
978
|
+
const { id: recoveredId, level: recoveredLevel } = this.recoverEntryPointO1();
|
|
979
|
+
this.entryPointId = recoveredId;
|
|
980
|
+
this.maxLevel = recoveredLevel;
|
|
981
|
+
// Persist corrected state
|
|
982
|
+
if (this.storage && recoveredId) {
|
|
983
|
+
await this.storage.saveHNSWSystem({
|
|
984
|
+
entryPointId: this.entryPointId,
|
|
985
|
+
maxLevel: this.maxLevel
|
|
986
|
+
}).catch((error) => {
|
|
987
|
+
prodLog.error('Failed to persist corrected HNSW system data:', error);
|
|
988
|
+
});
|
|
989
|
+
}
|
|
990
|
+
}
|
|
918
991
|
const cacheInfo = shouldPreload
|
|
919
992
|
? ` (vectors preloaded)`
|
|
920
993
|
: ` (adaptive caching - vectors loaded on-demand)`;
|
package/dist/index.d.ts
CHANGED
|
@@ -51,9 +51,8 @@ export { StorageAugmentation, DynamicStorageAugmentation, MemoryStorageAugmentat
|
|
|
51
51
|
export { WebSocketConduitAugmentation };
|
|
52
52
|
import type { Vector, VectorDocument, SearchResult, DistanceFunction, EmbeddingFunction, EmbeddingModel, HNSWNoun, HNSWVerb, HNSWConfig, StorageAdapter } from './coreTypes.js';
|
|
53
53
|
import { HNSWIndex } from './hnsw/hnswIndex.js';
|
|
54
|
-
|
|
55
|
-
export {
|
|
56
|
-
export type { Vector, VectorDocument, SearchResult, DistanceFunction, EmbeddingFunction, EmbeddingModel, HNSWNoun, HNSWVerb, HNSWConfig, HNSWOptimizedConfig, StorageAdapter };
|
|
54
|
+
export { HNSWIndex };
|
|
55
|
+
export type { Vector, VectorDocument, SearchResult, DistanceFunction, EmbeddingFunction, EmbeddingModel, HNSWNoun, HNSWVerb, HNSWConfig, StorageAdapter };
|
|
57
56
|
import type { AugmentationResponse, BrainyAugmentation, BaseAugmentation, AugmentationContext } from './types/augmentations.js';
|
|
58
57
|
export { AugmentationManager, type AugmentationInfo } from './augmentationManager.js';
|
|
59
58
|
export type { AugmentationResponse, BrainyAugmentation, BaseAugmentation, AugmentationContext };
|
package/dist/index.js
CHANGED
|
@@ -111,10 +111,9 @@ MemoryStorageAugmentation, FileSystemStorageAugmentation, OPFSStorageAugmentatio
|
|
|
111
111
|
createAutoStorageAugmentation, createStorageAugmentationFromConfig };
|
|
112
112
|
// Other augmentation exports
|
|
113
113
|
export { WebSocketConduitAugmentation };
|
|
114
|
-
// Export HNSW index
|
|
114
|
+
// Export HNSW index
|
|
115
115
|
import { HNSWIndex } from './hnsw/hnswIndex.js';
|
|
116
|
-
|
|
117
|
-
export { HNSWIndex, HNSWIndexOptimized };
|
|
116
|
+
export { HNSWIndex };
|
|
118
117
|
// Export augmentation manager for type-safe augmentation management
|
|
119
118
|
export { AugmentationManager } from './augmentationManager.js';
|
|
120
119
|
import { NounType, VerbType } from './types/graphTypes.js';
|
|
@@ -1269,7 +1269,15 @@ export class AzureBlobStorage extends BaseStorage {
|
|
|
1269
1269
|
return JSON.parse(downloaded.toString());
|
|
1270
1270
|
}
|
|
1271
1271
|
catch (error) {
|
|
1272
|
-
|
|
1272
|
+
// Azure may return not found errors in different formats
|
|
1273
|
+
const isNotFound = error.statusCode === 404 ||
|
|
1274
|
+
error.code === 'BlobNotFound' ||
|
|
1275
|
+
error.code === 404 ||
|
|
1276
|
+
error.details?.code === 'BlobNotFound' ||
|
|
1277
|
+
error.message?.includes('BlobNotFound') ||
|
|
1278
|
+
error.message?.includes('not found') ||
|
|
1279
|
+
error.message?.includes('404');
|
|
1280
|
+
if (isNotFound) {
|
|
1273
1281
|
return null;
|
|
1274
1282
|
}
|
|
1275
1283
|
this.logger.error('Failed to get HNSW system data:', error);
|
|
@@ -1260,7 +1260,14 @@ export class GcsStorage extends BaseStorage {
|
|
|
1260
1260
|
return JSON.parse(contents.toString());
|
|
1261
1261
|
}
|
|
1262
1262
|
catch (error) {
|
|
1263
|
-
|
|
1263
|
+
// GCS may return 404 in different formats depending on SDK version
|
|
1264
|
+
const is404 = error.code === 404 ||
|
|
1265
|
+
error.statusCode === 404 ||
|
|
1266
|
+
error.status === 404 ||
|
|
1267
|
+
error.message?.includes('No such object') ||
|
|
1268
|
+
error.message?.includes('not found') ||
|
|
1269
|
+
error.message?.includes('404');
|
|
1270
|
+
if (is404) {
|
|
1264
1271
|
return null;
|
|
1265
1272
|
}
|
|
1266
1273
|
this.logger.error('Failed to get HNSW system data:', error);
|
|
@@ -2829,9 +2829,14 @@ export class S3CompatibleStorage extends BaseStorage {
|
|
|
2829
2829
|
return JSON.parse(bodyContents);
|
|
2830
2830
|
}
|
|
2831
2831
|
catch (error) {
|
|
2832
|
-
|
|
2832
|
+
// S3 may return not found errors in different formats
|
|
2833
|
+
const isNotFound = error.name === 'NoSuchKey' ||
|
|
2834
|
+
error.code === 'NoSuchKey' ||
|
|
2835
|
+
error.$metadata?.httpStatusCode === 404 ||
|
|
2833
2836
|
error.message?.includes('NoSuchKey') ||
|
|
2834
|
-
error.message?.includes('not found')
|
|
2837
|
+
error.message?.includes('not found') ||
|
|
2838
|
+
error.message?.includes('404');
|
|
2839
|
+
if (isNotFound) {
|
|
2835
2840
|
return null;
|
|
2836
2841
|
}
|
|
2837
2842
|
this.logger.error('Failed to get HNSW system data:', error);
|
|
@@ -13,7 +13,6 @@
|
|
|
13
13
|
* - Fusion: O(k log k) where k = result count
|
|
14
14
|
*/
|
|
15
15
|
import { HNSWIndex } from '../hnsw/hnswIndex.js';
|
|
16
|
-
import { HNSWIndexOptimized } from '../hnsw/hnswIndexOptimized.js';
|
|
17
16
|
import { TypeAwareHNSWIndex } from '../hnsw/typeAwareHNSWIndex.js';
|
|
18
17
|
import { MetadataIndexManager } from '../utils/metadataIndex.js';
|
|
19
18
|
import { Vector } from '../coreTypes.js';
|
|
@@ -68,7 +67,7 @@ export declare class TripleIntelligenceSystem {
|
|
|
68
67
|
private planner;
|
|
69
68
|
private embedder;
|
|
70
69
|
private storage;
|
|
71
|
-
constructor(metadataIndex: MetadataIndexManager, hnswIndex: HNSWIndex |
|
|
70
|
+
constructor(metadataIndex: MetadataIndexManager, hnswIndex: HNSWIndex | TypeAwareHNSWIndex, graphIndex: GraphAdjacencyIndex, embedder: (text: string) => Promise<Vector>, storage: any);
|
|
72
71
|
/**
|
|
73
72
|
* Main find method - executes Triple Intelligence queries
|
|
74
73
|
* Phase 3: Now with automatic type inference for 40% latency reduction
|
|
@@ -436,6 +436,23 @@ export declare class MetadataIndexManager {
|
|
|
436
436
|
* Get all entity types and their counts - O(1) operation
|
|
437
437
|
*/
|
|
438
438
|
getAllEntityCounts(): Map<string, number>;
|
|
439
|
+
/**
|
|
440
|
+
* Get VFS entity count for a specific type using Roaring bitmap intersection
|
|
441
|
+
* Uses hardware-accelerated SIMD operations (AVX2/SSE4.2)
|
|
442
|
+
* @param type The noun type to query
|
|
443
|
+
* @returns Count of VFS entities of this type
|
|
444
|
+
*/
|
|
445
|
+
getVFSEntityCountByType(type: string): Promise<number>;
|
|
446
|
+
/**
|
|
447
|
+
* Get all VFS entity counts by type using Roaring bitmap operations
|
|
448
|
+
* @returns Map of type -> VFS entity count
|
|
449
|
+
*/
|
|
450
|
+
getAllVFSEntityCounts(): Promise<Map<string, number>>;
|
|
451
|
+
/**
|
|
452
|
+
* Get total count of VFS entities - O(1) using Roaring bitmap cardinality
|
|
453
|
+
* @returns Total VFS entity count
|
|
454
|
+
*/
|
|
455
|
+
getTotalVFSEntityCount(): Promise<number>;
|
|
439
456
|
/**
|
|
440
457
|
* Get entity count for a noun type using type enum (O(1) array access)
|
|
441
458
|
* More efficient than Map-based getEntityCountByType
|
|
@@ -1230,6 +1230,21 @@ export class MetadataIndexManager {
|
|
|
1230
1230
|
const subIds = await this.getIdsForFilter(subFilter);
|
|
1231
1231
|
subIds.forEach(id => unionIds.add(id));
|
|
1232
1232
|
}
|
|
1233
|
+
// v6.2.1: Fix - Check for outer-level field conditions that need AND application
|
|
1234
|
+
// This handles cases like { anyOf: [...], vfsType: { exists: false } }
|
|
1235
|
+
// where the anyOf results must be intersected with other field conditions
|
|
1236
|
+
const outerFields = Object.keys(filter).filter((k) => k !== 'anyOf' && k !== 'allOf' && k !== 'not');
|
|
1237
|
+
if (outerFields.length > 0) {
|
|
1238
|
+
// Build filter with just outer fields and get matching IDs
|
|
1239
|
+
const outerFilter = {};
|
|
1240
|
+
for (const field of outerFields) {
|
|
1241
|
+
outerFilter[field] = filter[field];
|
|
1242
|
+
}
|
|
1243
|
+
const outerIds = await this.getIdsForFilter(outerFilter);
|
|
1244
|
+
const outerIdSet = new Set(outerIds);
|
|
1245
|
+
// Intersect: anyOf union AND outer field conditions
|
|
1246
|
+
return Array.from(unionIds).filter((id) => outerIdSet.has(id));
|
|
1247
|
+
}
|
|
1233
1248
|
return Array.from(unionIds);
|
|
1234
1249
|
}
|
|
1235
1250
|
// Process field filters with range support
|
|
@@ -1865,6 +1880,54 @@ export class MetadataIndexManager {
|
|
|
1865
1880
|
return new Map(this.totalEntitiesByType);
|
|
1866
1881
|
}
|
|
1867
1882
|
// ============================================================================
|
|
1883
|
+
// v6.2.1: VFS Statistics Methods (uses existing Roaring bitmap infrastructure)
|
|
1884
|
+
// ============================================================================
|
|
1885
|
+
/**
|
|
1886
|
+
* Get VFS entity count for a specific type using Roaring bitmap intersection
|
|
1887
|
+
* Uses hardware-accelerated SIMD operations (AVX2/SSE4.2)
|
|
1888
|
+
* @param type The noun type to query
|
|
1889
|
+
* @returns Count of VFS entities of this type
|
|
1890
|
+
*/
|
|
1891
|
+
async getVFSEntityCountByType(type) {
|
|
1892
|
+
const vfsBitmap = await this.getBitmapFromChunks('isVFSEntity', true);
|
|
1893
|
+
const typeBitmap = await this.getBitmapFromChunks('noun', type);
|
|
1894
|
+
if (!vfsBitmap || !typeBitmap)
|
|
1895
|
+
return 0;
|
|
1896
|
+
// Hardware-accelerated intersection + O(1) cardinality
|
|
1897
|
+
const intersection = RoaringBitmap32.and(vfsBitmap, typeBitmap);
|
|
1898
|
+
return intersection.size;
|
|
1899
|
+
}
|
|
1900
|
+
/**
|
|
1901
|
+
* Get all VFS entity counts by type using Roaring bitmap operations
|
|
1902
|
+
* @returns Map of type -> VFS entity count
|
|
1903
|
+
*/
|
|
1904
|
+
async getAllVFSEntityCounts() {
|
|
1905
|
+
const vfsBitmap = await this.getBitmapFromChunks('isVFSEntity', true);
|
|
1906
|
+
if (!vfsBitmap || vfsBitmap.size === 0) {
|
|
1907
|
+
return new Map();
|
|
1908
|
+
}
|
|
1909
|
+
const result = new Map();
|
|
1910
|
+
// Iterate through all known types and compute VFS count via intersection
|
|
1911
|
+
for (const type of this.totalEntitiesByType.keys()) {
|
|
1912
|
+
const typeBitmap = await this.getBitmapFromChunks('noun', type);
|
|
1913
|
+
if (typeBitmap) {
|
|
1914
|
+
const intersection = RoaringBitmap32.and(vfsBitmap, typeBitmap);
|
|
1915
|
+
if (intersection.size > 0) {
|
|
1916
|
+
result.set(type, intersection.size);
|
|
1917
|
+
}
|
|
1918
|
+
}
|
|
1919
|
+
}
|
|
1920
|
+
return result;
|
|
1921
|
+
}
|
|
1922
|
+
/**
|
|
1923
|
+
* Get total count of VFS entities - O(1) using Roaring bitmap cardinality
|
|
1924
|
+
* @returns Total VFS entity count
|
|
1925
|
+
*/
|
|
1926
|
+
async getTotalVFSEntityCount() {
|
|
1927
|
+
const vfsBitmap = await this.getBitmapFromChunks('isVFSEntity', true);
|
|
1928
|
+
return vfsBitmap?.size ?? 0;
|
|
1929
|
+
}
|
|
1930
|
+
// ============================================================================
|
|
1868
1931
|
// Phase 1b: Type Enum Methods (O(1) access via Uint32Arrays)
|
|
1869
1932
|
// ============================================================================
|
|
1870
1933
|
/**
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@soulcraft/brainy",
|
|
3
|
-
"version": "6.2.
|
|
3
|
+
"version": "6.2.2",
|
|
4
4
|
"description": "Universal Knowledge Protocol™ - World's first Triple Intelligence database unifying vector, graph, and document search in one API. Stage 3 CANONICAL: 42 nouns × 127 verbs covering 96-97% of all human knowledge.",
|
|
5
5
|
"main": "dist/index.js",
|
|
6
6
|
"module": "dist/index.js",
|