@soulcraft/brainy 0.47.0 → 0.49.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +268 -554
- package/dist/brainyData.d.ts +83 -2
- package/dist/brainyData.js +536 -66
- package/dist/brainyData.js.map +1 -1
- package/dist/coreTypes.d.ts +74 -12
- package/dist/distributed/configManager.d.ts +9 -0
- package/dist/distributed/configManager.js +129 -10
- package/dist/distributed/configManager.js.map +1 -1
- package/dist/hnsw/hnswIndex.d.ts +2 -2
- package/dist/hnsw/hnswIndex.js +48 -29
- package/dist/hnsw/hnswIndex.js.map +1 -1
- package/dist/hnsw/optimizedHNSWIndex.d.ts +1 -1
- package/dist/hnsw/optimizedHNSWIndex.js +3 -3
- package/dist/hnsw/optimizedHNSWIndex.js.map +1 -1
- package/dist/storage/adapters/baseStorageAdapter.d.ts +18 -2
- package/dist/storage/adapters/baseStorageAdapter.js +69 -4
- package/dist/storage/adapters/baseStorageAdapter.js.map +1 -1
- package/dist/storage/adapters/fileSystemStorage.d.ts +14 -8
- package/dist/storage/adapters/fileSystemStorage.js +90 -22
- package/dist/storage/adapters/fileSystemStorage.js.map +1 -1
- package/dist/storage/adapters/memoryStorage.d.ts +0 -8
- package/dist/storage/adapters/memoryStorage.js +26 -45
- package/dist/storage/adapters/memoryStorage.js.map +1 -1
- package/dist/storage/adapters/opfsStorage.d.ts +40 -8
- package/dist/storage/adapters/opfsStorage.js +195 -44
- package/dist/storage/adapters/opfsStorage.js.map +1 -1
- package/dist/storage/adapters/optimizedS3Search.js +4 -3
- package/dist/storage/adapters/optimizedS3Search.js.map +1 -1
- package/dist/storage/adapters/s3CompatibleStorage.d.ts +3 -10
- package/dist/storage/adapters/s3CompatibleStorage.js +41 -44
- package/dist/storage/adapters/s3CompatibleStorage.js.map +1 -1
- package/dist/storage/backwardCompatibility.d.ts +84 -0
- package/dist/storage/backwardCompatibility.js +141 -0
- package/dist/storage/backwardCompatibility.js.map +1 -0
- package/dist/storage/baseStorage.d.ts +33 -19
- package/dist/storage/baseStorage.js +116 -195
- package/dist/storage/baseStorage.js.map +1 -1
- package/dist/utils/distance.d.ts +4 -4
- package/dist/utils/distance.js +4 -4
- package/dist/utils/embedding.d.ts +10 -0
- package/dist/utils/embedding.js +81 -9
- package/dist/utils/embedding.js.map +1 -1
- package/dist/utils/metadataFilter.d.ts +79 -0
- package/dist/utils/metadataFilter.js +229 -0
- package/dist/utils/metadataFilter.js.map +1 -0
- package/dist/utils/metadataIndex.d.ts +148 -0
- package/dist/utils/metadataIndex.js +639 -0
- package/dist/utils/metadataIndex.js.map +1 -0
- package/dist/utils/metadataIndexCache.d.ts +60 -0
- package/dist/utils/metadataIndexCache.js +119 -0
- package/dist/utils/metadataIndexCache.js.map +1 -0
- package/package.json +4 -4
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* MetadataIndexCache - Caches metadata index data for improved performance
|
|
3
|
+
* Reuses the same pattern as SearchCache for consistency
|
|
4
|
+
*/
|
|
5
|
+
export interface MetadataCacheEntry {
|
|
6
|
+
data: any;
|
|
7
|
+
timestamp: number;
|
|
8
|
+
hits: number;
|
|
9
|
+
}
|
|
10
|
+
export interface MetadataIndexCacheConfig {
|
|
11
|
+
maxAge?: number;
|
|
12
|
+
maxSize?: number;
|
|
13
|
+
enabled?: boolean;
|
|
14
|
+
hitCountWeight?: number;
|
|
15
|
+
}
|
|
16
|
+
export declare class MetadataIndexCache {
|
|
17
|
+
private cache;
|
|
18
|
+
private maxAge;
|
|
19
|
+
private maxSize;
|
|
20
|
+
private enabled;
|
|
21
|
+
private hitCountWeight;
|
|
22
|
+
private hits;
|
|
23
|
+
private misses;
|
|
24
|
+
private evictions;
|
|
25
|
+
constructor(config?: MetadataIndexCacheConfig);
|
|
26
|
+
/**
|
|
27
|
+
* Get cached entry
|
|
28
|
+
*/
|
|
29
|
+
get(key: string): any | undefined;
|
|
30
|
+
/**
|
|
31
|
+
* Set cache entry
|
|
32
|
+
*/
|
|
33
|
+
set(key: string, data: any): void;
|
|
34
|
+
/**
|
|
35
|
+
* Evict least valuable entry based on age and hit count
|
|
36
|
+
*/
|
|
37
|
+
private evictLeastValuable;
|
|
38
|
+
/**
|
|
39
|
+
* Invalidate cache entries matching a pattern
|
|
40
|
+
*/
|
|
41
|
+
invalidatePattern(pattern: string): void;
|
|
42
|
+
/**
|
|
43
|
+
* Clear all cache entries
|
|
44
|
+
*/
|
|
45
|
+
clear(): void;
|
|
46
|
+
/**
|
|
47
|
+
* Get cache statistics
|
|
48
|
+
*/
|
|
49
|
+
getStats(): {
|
|
50
|
+
size: number;
|
|
51
|
+
hits: number;
|
|
52
|
+
misses: number;
|
|
53
|
+
hitRate: number;
|
|
54
|
+
evictions: number;
|
|
55
|
+
};
|
|
56
|
+
/**
|
|
57
|
+
* Get estimated memory usage
|
|
58
|
+
*/
|
|
59
|
+
getMemoryUsage(): number;
|
|
60
|
+
}
|
|
@@ -0,0 +1,119 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* MetadataIndexCache - Caches metadata index data for improved performance
|
|
3
|
+
* Reuses the same pattern as SearchCache for consistency
|
|
4
|
+
*/
|
|
5
|
+
export class MetadataIndexCache {
|
|
6
|
+
constructor(config = {}) {
|
|
7
|
+
this.cache = new Map();
|
|
8
|
+
// Cache statistics
|
|
9
|
+
this.hits = 0;
|
|
10
|
+
this.misses = 0;
|
|
11
|
+
this.evictions = 0;
|
|
12
|
+
this.maxAge = config.maxAge ?? 5 * 60 * 1000; // 5 minutes
|
|
13
|
+
this.maxSize = config.maxSize ?? 500; // More entries than SearchCache since indexes are smaller
|
|
14
|
+
this.enabled = config.enabled ?? true;
|
|
15
|
+
this.hitCountWeight = config.hitCountWeight ?? 0.3;
|
|
16
|
+
}
|
|
17
|
+
/**
|
|
18
|
+
* Get cached entry
|
|
19
|
+
*/
|
|
20
|
+
get(key) {
|
|
21
|
+
if (!this.enabled)
|
|
22
|
+
return undefined;
|
|
23
|
+
const entry = this.cache.get(key);
|
|
24
|
+
if (!entry) {
|
|
25
|
+
this.misses++;
|
|
26
|
+
return undefined;
|
|
27
|
+
}
|
|
28
|
+
// Check if entry is expired
|
|
29
|
+
if (Date.now() - entry.timestamp > this.maxAge) {
|
|
30
|
+
this.cache.delete(key);
|
|
31
|
+
this.misses++;
|
|
32
|
+
return undefined;
|
|
33
|
+
}
|
|
34
|
+
// Update hit count
|
|
35
|
+
entry.hits++;
|
|
36
|
+
this.hits++;
|
|
37
|
+
return entry.data;
|
|
38
|
+
}
|
|
39
|
+
/**
|
|
40
|
+
* Set cache entry
|
|
41
|
+
*/
|
|
42
|
+
set(key, data) {
|
|
43
|
+
if (!this.enabled)
|
|
44
|
+
return;
|
|
45
|
+
// Evict entries if at max size
|
|
46
|
+
if (this.cache.size >= this.maxSize) {
|
|
47
|
+
this.evictLeastValuable();
|
|
48
|
+
}
|
|
49
|
+
this.cache.set(key, {
|
|
50
|
+
data,
|
|
51
|
+
timestamp: Date.now(),
|
|
52
|
+
hits: 0
|
|
53
|
+
});
|
|
54
|
+
}
|
|
55
|
+
/**
|
|
56
|
+
* Evict least valuable entry based on age and hit count
|
|
57
|
+
*/
|
|
58
|
+
evictLeastValuable() {
|
|
59
|
+
let leastValuableKey = null;
|
|
60
|
+
let lowestScore = Infinity;
|
|
61
|
+
for (const [key, entry] of this.cache.entries()) {
|
|
62
|
+
const age = Date.now() - entry.timestamp;
|
|
63
|
+
const ageScore = age / this.maxAge;
|
|
64
|
+
const hitScore = entry.hits * this.hitCountWeight;
|
|
65
|
+
const score = hitScore - ageScore;
|
|
66
|
+
if (score < lowestScore) {
|
|
67
|
+
lowestScore = score;
|
|
68
|
+
leastValuableKey = key;
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
if (leastValuableKey) {
|
|
72
|
+
this.cache.delete(leastValuableKey);
|
|
73
|
+
this.evictions++;
|
|
74
|
+
}
|
|
75
|
+
}
|
|
76
|
+
/**
|
|
77
|
+
* Invalidate cache entries matching a pattern
|
|
78
|
+
*/
|
|
79
|
+
invalidatePattern(pattern) {
|
|
80
|
+
const keysToDelete = [];
|
|
81
|
+
for (const key of this.cache.keys()) {
|
|
82
|
+
if (key.includes(pattern)) {
|
|
83
|
+
keysToDelete.push(key);
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
keysToDelete.forEach(key => this.cache.delete(key));
|
|
87
|
+
}
|
|
88
|
+
/**
|
|
89
|
+
* Clear all cache entries
|
|
90
|
+
*/
|
|
91
|
+
clear() {
|
|
92
|
+
this.cache.clear();
|
|
93
|
+
}
|
|
94
|
+
/**
|
|
95
|
+
* Get cache statistics
|
|
96
|
+
*/
|
|
97
|
+
getStats() {
|
|
98
|
+
return {
|
|
99
|
+
size: this.cache.size,
|
|
100
|
+
hits: this.hits,
|
|
101
|
+
misses: this.misses,
|
|
102
|
+
hitRate: this.hits / (this.hits + this.misses) || 0,
|
|
103
|
+
evictions: this.evictions
|
|
104
|
+
};
|
|
105
|
+
}
|
|
106
|
+
/**
|
|
107
|
+
* Get estimated memory usage
|
|
108
|
+
*/
|
|
109
|
+
getMemoryUsage() {
|
|
110
|
+
// Rough estimate: 100 bytes per entry + data size
|
|
111
|
+
let totalSize = 0;
|
|
112
|
+
for (const entry of this.cache.values()) {
|
|
113
|
+
totalSize += 100; // Base overhead
|
|
114
|
+
totalSize += JSON.stringify(entry.data).length * 2; // Unicode chars
|
|
115
|
+
}
|
|
116
|
+
return totalSize;
|
|
117
|
+
}
|
|
118
|
+
}
|
|
119
|
+
//# sourceMappingURL=metadataIndexCache.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"metadataIndexCache.js","sourceRoot":"","sources":["../../src/utils/metadataIndexCache.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAeH,MAAM,OAAO,kBAAkB;IAY7B,YAAY,SAAmC,EAAE;QAXzC,UAAK,GAAG,IAAI,GAAG,EAA8B,CAAA;QAMrD,mBAAmB;QACX,SAAI,GAAG,CAAC,CAAA;QACR,WAAM,GAAG,CAAC,CAAA;QACV,cAAS,GAAG,CAAC,CAAA;QAGnB,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,CAAA,CAAC,YAAY;QACzD,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC,OAAO,IAAI,GAAG,CAAA,CAAC,0DAA0D;QAC/F,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC,OAAO,IAAI,IAAI,CAAA;QACrC,IAAI,CAAC,cAAc,GAAG,MAAM,CAAC,cAAc,IAAI,GAAG,CAAA;IACpD,CAAC;IAED;;OAEG;IACH,GAAG,CAAC,GAAW;QACb,IAAI,CAAC,IAAI,CAAC,OAAO;YAAE,OAAO,SAAS,CAAA;QAEnC,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,CAAA;QACjC,IAAI,CAAC,KAAK,EAAE,CAAC;YACX,IAAI,CAAC,MAAM,EAAE,CAAA;YACb,OAAO,SAAS,CAAA;QAClB,CAAC;QAED,4BAA4B;QAC5B,IAAI,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,CAAC,SAAS,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC;YAC/C,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAA;YACtB,IAAI,CAAC,MAAM,EAAE,CAAA;YACb,OAAO,SAAS,CAAA;QAClB,CAAC;QAED,mBAAmB;QACnB,KAAK,CAAC,IAAI,EAAE,CAAA;QACZ,IAAI,CAAC,IAAI,EAAE,CAAA;QACX,OAAO,KAAK,CAAC,IAAI,CAAA;IACnB,CAAC;IAED;;OAEG;IACH,GAAG,CAAC,GAAW,EAAE,IAAS;QACxB,IAAI,CAAC,IAAI,CAAC,OAAO;YAAE,OAAM;QAEzB,+BAA+B;QAC/B,IAAI,IAAI,CAAC,KAAK,CAAC,IAAI,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;YACpC,IAAI,CAAC,kBAAkB,EAAE,CAAA;QAC3B,CAAC;QAED,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE;YAClB,IAAI;YACJ,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE;YACrB,IAAI,EAAE,CAAC;SACR,CAAC,CAAA;IACJ,CAAC;IAED;;OAEG;IACK,kBAAkB;QACxB,IAAI,gBAAgB,GAAkB,IAAI,CAAA;QAC1C,IAAI,WAAW,GAAG,QAAQ,CAAA;QAE1B,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,IAAI,CAAC,KAAK,CAAC,OAAO,EAAE,EAAE,CAAC;YAChD,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,CAAC,SAAS,CAAA;YACxC,MAAM,QAAQ,GAAG,GAAG,GAAG,IAAI,CAAC,MAAM,CAAA;YAClC,MAAM,QAAQ,GAAG,KAAK,CAAC,IAAI,GAAG,IAAI,CAAC,cAAc,CAAA;YACjD,MAAM,KAAK,GAAG,QAAQ,GAAG,QAAQ,CAAA;YAEjC,IAAI,KAAK,GAAG,WAAW,EAAE,CAAC;gBACxB,WAAW,GAAG,KAAK,CAAA;gBACnB,gBAAgB,GAAG,GAAG,CAAA;YACxB,CAAC;QACH,CAAC;QAED,IAAI,gBAAgB,EAAE,CAAC;YACrB,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,gBAAgB,CAAC,CAAA;YACnC,IAAI,CAAC,SAAS,EAAE,CAAA;QAClB,CAAC;IACH,CAAC;IAED;;OAEG;IACH,iBAAiB,CAAC,OAAe;QAC/B,MAAM,YAAY,GAAa,EAAE,CAAA;QACjC,KAAK,MAAM,GAAG,IAAI,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,EAAE,CAAC;YACpC,IAAI,GAAG,CAAC,QAAQ,CAAC,OAAO,CAAC,EAAE,CAAC;gBAC1B,YAAY,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;YACxB,CAAC;QACH,CAAC;QACD,YAAY,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAA;IACrD,CAAC;IAED;;OAEG;IACH,KAAK;QACH,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAA;IACpB,CAAC;IAED;;OAEG;IACH,QAAQ;QACN,OAAO;YACL,IAAI,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI;YACrB,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,MAAM,EAAE,IAAI,CAAC,MAAM;YACnB,OAAO,EAAE,IAAI,CAAC,IAAI,GAAG,CAAC,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC;YACnD,SAAS,EAAE,IAAI,CAAC,SAAS;SAC1B,CAAA;IACH,CAAC;IAED;;OAEG;IACH,cAAc;QACZ,kDAAkD;QAClD,IAAI,SAAS,GAAG,CAAC,CAAA;QACjB,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,EAAE,CAAC;YACxC,SAAS,IAAI,GAAG,CAAA,CAAC,gBAAgB;YACjC,SAAS,IAAI,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,MAAM,GAAG,CAAC,CAAA,CAAC,gBAAgB;QACrE,CAAC;QACD,OAAO,SAAS,CAAA;IAClB,CAAC;CACF"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@soulcraft/brainy",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.49.0",
|
|
4
4
|
"description": "A vector graph database using HNSW indexing with Origin Private File System storage",
|
|
5
5
|
"main": "dist/index.js",
|
|
6
6
|
"module": "dist/index.js",
|
|
@@ -143,11 +143,11 @@
|
|
|
143
143
|
"@types/jsdom": "^21.1.7",
|
|
144
144
|
"@types/node": "^20.11.30",
|
|
145
145
|
"@types/uuid": "^10.0.0",
|
|
146
|
-
"@typescript-eslint/eslint-plugin": "^
|
|
147
|
-
"@typescript-eslint/parser": "^
|
|
146
|
+
"@typescript-eslint/eslint-plugin": "^8.0.0",
|
|
147
|
+
"@typescript-eslint/parser": "^8.0.0",
|
|
148
148
|
"@vitest/coverage-v8": "^3.2.4",
|
|
149
149
|
"@vitest/ui": "^3.2.4",
|
|
150
|
-
"eslint": "^
|
|
150
|
+
"eslint": "^9.0.0",
|
|
151
151
|
"express": "^5.1.0",
|
|
152
152
|
"happy-dom": "^18.0.1",
|
|
153
153
|
"jsdom": "^26.1.0",
|