@soulcraft/brainy 0.40.0 → 0.43.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +605 -194
- package/dist/augmentations/conduitAugmentations.js +1158 -0
- package/dist/augmentations/conduitAugmentations.js.map +1 -0
- package/dist/augmentations/memoryAugmentations.d.ts +2 -0
- package/dist/augmentations/memoryAugmentations.d.ts.map +1 -1
- package/dist/augmentations/memoryAugmentations.js +270 -0
- package/dist/augmentations/memoryAugmentations.js.map +1 -0
- package/dist/augmentations/serverSearchAugmentations.js +531 -0
- package/dist/augmentations/serverSearchAugmentations.js.map +1 -0
- package/dist/browserFramework.d.ts +15 -0
- package/dist/demo.d.ts +106 -0
- package/dist/examples/basicUsage.js +118 -0
- package/dist/examples/basicUsage.js.map +1 -0
- package/dist/hnsw/distributedSearch.js +452 -0
- package/dist/hnsw/distributedSearch.js.map +1 -0
- package/dist/hnsw/hnswIndex.js +602 -0
- package/dist/hnsw/hnswIndex.js.map +1 -0
- package/dist/hnsw/hnswIndexOptimized.js +471 -0
- package/dist/hnsw/hnswIndexOptimized.js.map +1 -0
- package/dist/hnsw/optimizedHNSWIndex.js +313 -0
- package/dist/hnsw/optimizedHNSWIndex.js.map +1 -0
- package/dist/hnsw/partitionedHNSWIndex.js +304 -0
- package/dist/hnsw/partitionedHNSWIndex.js.map +1 -0
- package/dist/hnsw/scaledHNSWSystem.js +559 -0
- package/dist/hnsw/scaledHNSWSystem.js.map +1 -0
- package/dist/index.d.ts +5 -3
- package/dist/index.js +81 -0
- package/dist/mcp/brainyMCPAdapter.js +142 -0
- package/dist/mcp/brainyMCPAdapter.js.map +1 -0
- package/dist/mcp/brainyMCPService.js +248 -0
- package/dist/mcp/brainyMCPService.js.map +1 -0
- package/dist/mcp/index.js +17 -0
- package/dist/mcp/index.js.map +1 -0
- package/dist/mcp/mcpAugmentationToolset.js +180 -0
- package/dist/mcp/mcpAugmentationToolset.js.map +1 -0
- package/dist/storage/adapters/baseStorageAdapter.js +349 -0
- package/dist/storage/adapters/baseStorageAdapter.js.map +1 -0
- package/dist/storage/adapters/batchS3Operations.js +287 -0
- package/dist/storage/adapters/batchS3Operations.js.map +1 -0
- package/dist/storage/adapters/fileSystemStorage.js +846 -0
- package/dist/storage/adapters/fileSystemStorage.js.map +1 -0
- package/dist/storage/adapters/memoryStorage.js +532 -0
- package/dist/storage/adapters/memoryStorage.js.map +1 -0
- package/dist/storage/adapters/opfsStorage.d.ts.map +1 -1
- package/dist/storage/adapters/opfsStorage.js +1118 -0
- package/dist/storage/adapters/opfsStorage.js.map +1 -0
- package/dist/storage/adapters/optimizedS3Search.d.ts +79 -0
- package/dist/storage/adapters/optimizedS3Search.d.ts.map +1 -0
- package/dist/storage/adapters/optimizedS3Search.js +248 -0
- package/dist/storage/adapters/optimizedS3Search.js.map +1 -0
- package/dist/storage/adapters/s3CompatibleStorage.d.ts +21 -0
- package/dist/storage/adapters/s3CompatibleStorage.d.ts.map +1 -1
- package/dist/storage/adapters/s3CompatibleStorage.js +2026 -0
- package/dist/storage/adapters/s3CompatibleStorage.js.map +1 -0
- package/dist/storage/baseStorage.d.ts +1 -0
- package/dist/storage/baseStorage.d.ts.map +1 -1
- package/dist/storage/baseStorage.js +603 -0
- package/dist/storage/baseStorage.js.map +1 -0
- package/dist/storage/cacheManager.js +1306 -0
- package/dist/storage/cacheManager.js.map +1 -0
- package/dist/storage/enhancedCacheManager.js +520 -0
- package/dist/storage/enhancedCacheManager.js.map +1 -0
- package/dist/storage/readOnlyOptimizations.js +425 -0
- package/dist/storage/readOnlyOptimizations.js.map +1 -0
- package/dist/storage/storageFactory.d.ts +0 -1
- package/dist/storage/storageFactory.d.ts.map +1 -1
- package/dist/storage/storageFactory.js +227 -0
- package/dist/storage/storageFactory.js.map +1 -0
- package/dist/types/augmentations.js +16 -0
- package/dist/types/augmentations.js.map +1 -0
- package/dist/types/brainyDataInterface.js +8 -0
- package/dist/types/brainyDataInterface.js.map +1 -0
- package/dist/types/distributedTypes.js +6 -0
- package/dist/types/distributedTypes.js.map +1 -0
- package/dist/types/fileSystemTypes.js +8 -0
- package/dist/types/fileSystemTypes.js.map +1 -0
- package/dist/types/graphTypes.js +247 -0
- package/dist/types/graphTypes.js.map +1 -0
- package/dist/types/mcpTypes.js +22 -0
- package/dist/types/mcpTypes.js.map +1 -0
- package/dist/types/paginationTypes.js +5 -0
- package/dist/types/paginationTypes.js.map +1 -0
- package/dist/types/pipelineTypes.js +7 -0
- package/dist/types/pipelineTypes.js.map +1 -0
- package/dist/types/tensorflowTypes.js +6 -0
- package/dist/types/tensorflowTypes.js.map +1 -0
- package/dist/unified.js +52 -128048
- package/dist/utils/autoConfiguration.js +341 -0
- package/dist/utils/autoConfiguration.js.map +1 -0
- package/dist/utils/cacheAutoConfig.js +261 -0
- package/dist/utils/cacheAutoConfig.js.map +1 -0
- package/dist/utils/crypto.js +45 -0
- package/dist/utils/crypto.js.map +1 -0
- package/dist/utils/distance.js +239 -0
- package/dist/utils/distance.js.map +1 -0
- package/dist/utils/embedding.d.ts.map +1 -1
- package/dist/utils/embedding.js +702 -0
- package/dist/utils/embedding.js.map +1 -0
- package/dist/utils/environment.js +75 -0
- package/dist/utils/environment.js.map +1 -0
- package/dist/utils/fieldNameTracking.js +90 -0
- package/dist/utils/fieldNameTracking.js.map +1 -0
- package/dist/utils/index.d.ts +1 -0
- package/dist/utils/index.d.ts.map +1 -1
- package/dist/utils/index.js +8 -0
- package/dist/utils/index.js.map +1 -0
- package/dist/utils/jsonProcessing.js +179 -0
- package/dist/utils/jsonProcessing.js.map +1 -0
- package/dist/utils/logger.d.ts +45 -92
- package/dist/utils/logger.d.ts.map +1 -1
- package/dist/utils/logger.js +129 -0
- package/dist/utils/logger.js.map +1 -0
- package/dist/utils/operationUtils.js +126 -0
- package/dist/utils/operationUtils.js.map +1 -0
- package/dist/utils/robustModelLoader.d.ts +14 -0
- package/dist/utils/robustModelLoader.d.ts.map +1 -1
- package/dist/utils/robustModelLoader.js +537 -0
- package/dist/utils/robustModelLoader.js.map +1 -0
- package/dist/utils/searchCache.js +248 -0
- package/dist/utils/searchCache.js.map +1 -0
- package/dist/utils/statistics.js +25 -0
- package/dist/utils/statistics.js.map +1 -0
- package/dist/utils/statisticsCollector.js +224 -0
- package/dist/utils/statisticsCollector.js.map +1 -0
- package/dist/utils/textEncoding.js +309 -0
- package/dist/utils/textEncoding.js.map +1 -0
- package/dist/utils/typeUtils.js +40 -0
- package/dist/utils/typeUtils.js.map +1 -0
- package/dist/utils/version.d.ts +15 -3
- package/dist/utils/version.d.ts.map +1 -1
- package/dist/utils/version.js +24 -0
- package/dist/utils/version.js.map +1 -0
- package/dist/utils/workerUtils.js +458 -0
- package/dist/utils/workerUtils.js.map +1 -0
- package/package.json +23 -15
- package/dist/brainy.js +0 -90220
- package/dist/brainy.min.js +0 -12511
- package/dist/patched-platform-node.d.ts +0 -17
- package/dist/statistics/statisticsManager.d.ts +0 -121
- package/dist/storage/fileSystemStorage.d.ts +0 -73
- package/dist/storage/fileSystemStorage.d.ts.map +0 -1
- package/dist/storage/opfsStorage.d.ts +0 -236
- package/dist/storage/opfsStorage.d.ts.map +0 -1
- package/dist/storage/s3CompatibleStorage.d.ts +0 -157
- package/dist/storage/s3CompatibleStorage.d.ts.map +0 -1
- package/dist/testing/prettyReporter.d.ts +0 -23
- package/dist/testing/prettySummaryReporter.d.ts +0 -22
- package/dist/unified.min.js +0 -16153
- package/dist/utils/environmentDetection.d.ts +0 -47
- package/dist/utils/environmentDetection.d.ts.map +0 -1
- package/dist/utils/tensorflowUtils.d.ts +0 -17
- package/dist/utils/tensorflowUtils.d.ts.map +0 -1
|
@@ -0,0 +1,248 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* SearchCache - Caches search results for improved performance
|
|
3
|
+
*/
|
|
4
|
+
export class SearchCache {
|
|
5
|
+
constructor(config = {}) {
|
|
6
|
+
this.cache = new Map();
|
|
7
|
+
// Cache statistics
|
|
8
|
+
this.hits = 0;
|
|
9
|
+
this.misses = 0;
|
|
10
|
+
this.evictions = 0;
|
|
11
|
+
this.maxAge = config.maxAge ?? 5 * 60 * 1000; // 5 minutes
|
|
12
|
+
this.maxSize = config.maxSize ?? 100;
|
|
13
|
+
this.enabled = config.enabled ?? true;
|
|
14
|
+
this.hitCountWeight = config.hitCountWeight ?? 0.3;
|
|
15
|
+
}
|
|
16
|
+
/**
|
|
17
|
+
* Generate cache key from search parameters
|
|
18
|
+
*/
|
|
19
|
+
getCacheKey(query, k, options = {}) {
|
|
20
|
+
// Create a normalized key that ignores order of options
|
|
21
|
+
const normalizedOptions = Object.keys(options)
|
|
22
|
+
.sort()
|
|
23
|
+
.reduce((acc, key) => {
|
|
24
|
+
// Skip cache-related options
|
|
25
|
+
if (key === 'skipCache' || key === 'useStreaming')
|
|
26
|
+
return acc;
|
|
27
|
+
acc[key] = options[key];
|
|
28
|
+
return acc;
|
|
29
|
+
}, {});
|
|
30
|
+
return JSON.stringify({
|
|
31
|
+
query: typeof query === 'object' ? JSON.stringify(query) : query,
|
|
32
|
+
k,
|
|
33
|
+
...normalizedOptions
|
|
34
|
+
});
|
|
35
|
+
}
|
|
36
|
+
/**
|
|
37
|
+
* Get cached results if available and not expired
|
|
38
|
+
*/
|
|
39
|
+
get(key) {
|
|
40
|
+
if (!this.enabled)
|
|
41
|
+
return null;
|
|
42
|
+
const entry = this.cache.get(key);
|
|
43
|
+
if (!entry) {
|
|
44
|
+
this.misses++;
|
|
45
|
+
return null;
|
|
46
|
+
}
|
|
47
|
+
// Check if expired
|
|
48
|
+
if (Date.now() - entry.timestamp > this.maxAge) {
|
|
49
|
+
this.cache.delete(key);
|
|
50
|
+
this.misses++;
|
|
51
|
+
return null;
|
|
52
|
+
}
|
|
53
|
+
// Update hit count and statistics
|
|
54
|
+
entry.hits++;
|
|
55
|
+
this.hits++;
|
|
56
|
+
return entry.results;
|
|
57
|
+
}
|
|
58
|
+
/**
|
|
59
|
+
* Cache search results
|
|
60
|
+
*/
|
|
61
|
+
set(key, results) {
|
|
62
|
+
if (!this.enabled)
|
|
63
|
+
return;
|
|
64
|
+
// Evict if cache is full
|
|
65
|
+
if (this.cache.size >= this.maxSize) {
|
|
66
|
+
this.evictOldest();
|
|
67
|
+
}
|
|
68
|
+
this.cache.set(key, {
|
|
69
|
+
results: [...results], // Deep copy to prevent mutations
|
|
70
|
+
timestamp: Date.now(),
|
|
71
|
+
hits: 0
|
|
72
|
+
});
|
|
73
|
+
}
|
|
74
|
+
/**
|
|
75
|
+
* Evict the oldest entry based on timestamp and hit count
|
|
76
|
+
*/
|
|
77
|
+
evictOldest() {
|
|
78
|
+
let oldestKey = null;
|
|
79
|
+
let oldestScore = Infinity;
|
|
80
|
+
const now = Date.now();
|
|
81
|
+
for (const [key, entry] of this.cache.entries()) {
|
|
82
|
+
// Score combines age and inverse hit count
|
|
83
|
+
const age = now - entry.timestamp;
|
|
84
|
+
const hitScore = entry.hits > 0 ? 1 / entry.hits : 1;
|
|
85
|
+
const score = age + (hitScore * this.hitCountWeight * this.maxAge);
|
|
86
|
+
if (score < oldestScore) {
|
|
87
|
+
oldestScore = score;
|
|
88
|
+
oldestKey = key;
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
if (oldestKey) {
|
|
92
|
+
this.cache.delete(oldestKey);
|
|
93
|
+
this.evictions++;
|
|
94
|
+
}
|
|
95
|
+
}
|
|
96
|
+
/**
|
|
97
|
+
* Clear all cached results
|
|
98
|
+
*/
|
|
99
|
+
clear() {
|
|
100
|
+
this.cache.clear();
|
|
101
|
+
this.hits = 0;
|
|
102
|
+
this.misses = 0;
|
|
103
|
+
this.evictions = 0;
|
|
104
|
+
}
|
|
105
|
+
/**
|
|
106
|
+
* Invalidate cache entries that might be affected by data changes
|
|
107
|
+
*/
|
|
108
|
+
invalidate(pattern) {
|
|
109
|
+
if (!pattern) {
|
|
110
|
+
this.clear();
|
|
111
|
+
return;
|
|
112
|
+
}
|
|
113
|
+
const keysToDelete = [];
|
|
114
|
+
for (const key of this.cache.keys()) {
|
|
115
|
+
const shouldDelete = typeof pattern === 'string'
|
|
116
|
+
? key.includes(pattern)
|
|
117
|
+
: pattern.test(key);
|
|
118
|
+
if (shouldDelete) {
|
|
119
|
+
keysToDelete.push(key);
|
|
120
|
+
}
|
|
121
|
+
}
|
|
122
|
+
keysToDelete.forEach(key => this.cache.delete(key));
|
|
123
|
+
}
|
|
124
|
+
/**
|
|
125
|
+
* Smart invalidation for real-time data updates
|
|
126
|
+
* Only clears cache if it's getting stale or if data changes significantly
|
|
127
|
+
*/
|
|
128
|
+
invalidateOnDataChange(changeType) {
|
|
129
|
+
// For now, clear all caches on data changes to ensure consistency
|
|
130
|
+
// In the future, we could implement more sophisticated invalidation
|
|
131
|
+
// based on the type of change and affected data
|
|
132
|
+
this.clear();
|
|
133
|
+
}
|
|
134
|
+
/**
|
|
135
|
+
* Check if cache entries have expired and remove them
|
|
136
|
+
* This is especially important in distributed scenarios where
|
|
137
|
+
* real-time updates might be delayed or missed
|
|
138
|
+
*/
|
|
139
|
+
cleanupExpiredEntries() {
|
|
140
|
+
const now = Date.now();
|
|
141
|
+
const keysToDelete = [];
|
|
142
|
+
for (const [key, entry] of this.cache.entries()) {
|
|
143
|
+
if (now - entry.timestamp > this.maxAge) {
|
|
144
|
+
keysToDelete.push(key);
|
|
145
|
+
}
|
|
146
|
+
}
|
|
147
|
+
keysToDelete.forEach(key => this.cache.delete(key));
|
|
148
|
+
return keysToDelete.length;
|
|
149
|
+
}
|
|
150
|
+
/**
|
|
151
|
+
* Get cache statistics
|
|
152
|
+
*/
|
|
153
|
+
getStats() {
|
|
154
|
+
const total = this.hits + this.misses;
|
|
155
|
+
return {
|
|
156
|
+
hits: this.hits,
|
|
157
|
+
misses: this.misses,
|
|
158
|
+
evictions: this.evictions,
|
|
159
|
+
hitRate: total > 0 ? this.hits / total : 0,
|
|
160
|
+
size: this.cache.size,
|
|
161
|
+
maxSize: this.maxSize,
|
|
162
|
+
enabled: this.enabled
|
|
163
|
+
};
|
|
164
|
+
}
|
|
165
|
+
/**
|
|
166
|
+
* Enable or disable caching
|
|
167
|
+
*/
|
|
168
|
+
setEnabled(enabled) {
|
|
169
|
+
Object.defineProperty(this, 'enabled', { value: enabled, writable: false });
|
|
170
|
+
if (!enabled) {
|
|
171
|
+
this.clear();
|
|
172
|
+
}
|
|
173
|
+
}
|
|
174
|
+
/**
|
|
175
|
+
* Get memory usage estimate in bytes
|
|
176
|
+
*/
|
|
177
|
+
getMemoryUsage() {
|
|
178
|
+
let totalSize = 0;
|
|
179
|
+
for (const [key, entry] of this.cache.entries()) {
|
|
180
|
+
// Estimate key size
|
|
181
|
+
totalSize += key.length * 2; // UTF-16 characters
|
|
182
|
+
// Estimate entry size
|
|
183
|
+
totalSize += JSON.stringify(entry.results).length * 2;
|
|
184
|
+
totalSize += 16; // timestamp + hits (8 bytes each)
|
|
185
|
+
}
|
|
186
|
+
return totalSize;
|
|
187
|
+
}
|
|
188
|
+
/**
|
|
189
|
+
* Get current cache configuration
|
|
190
|
+
*/
|
|
191
|
+
getConfig() {
|
|
192
|
+
return {
|
|
193
|
+
enabled: this.enabled,
|
|
194
|
+
maxSize: this.maxSize,
|
|
195
|
+
maxAge: this.maxAge,
|
|
196
|
+
hitCountWeight: this.hitCountWeight
|
|
197
|
+
};
|
|
198
|
+
}
|
|
199
|
+
/**
|
|
200
|
+
* Update cache configuration dynamically
|
|
201
|
+
*/
|
|
202
|
+
updateConfig(newConfig) {
|
|
203
|
+
if (newConfig.enabled !== undefined) {
|
|
204
|
+
this.enabled = newConfig.enabled;
|
|
205
|
+
}
|
|
206
|
+
if (newConfig.maxSize !== undefined) {
|
|
207
|
+
this.maxSize = newConfig.maxSize;
|
|
208
|
+
// Trigger eviction if current size exceeds new limit
|
|
209
|
+
this.evictIfNeeded();
|
|
210
|
+
}
|
|
211
|
+
if (newConfig.maxAge !== undefined) {
|
|
212
|
+
this.maxAge = newConfig.maxAge;
|
|
213
|
+
// Clean up entries that are now expired with new TTL
|
|
214
|
+
this.cleanupExpiredEntries();
|
|
215
|
+
}
|
|
216
|
+
if (newConfig.hitCountWeight !== undefined) {
|
|
217
|
+
this.hitCountWeight = newConfig.hitCountWeight;
|
|
218
|
+
}
|
|
219
|
+
}
|
|
220
|
+
/**
|
|
221
|
+
* Evict entries if cache exceeds maxSize
|
|
222
|
+
*/
|
|
223
|
+
evictIfNeeded() {
|
|
224
|
+
if (this.cache.size <= this.maxSize) {
|
|
225
|
+
return;
|
|
226
|
+
}
|
|
227
|
+
// Calculate eviction score for each entry (same logic as existing eviction)
|
|
228
|
+
const entries = Array.from(this.cache.entries()).map(([key, entry]) => {
|
|
229
|
+
const age = Date.now() - entry.timestamp;
|
|
230
|
+
const hitCount = entry.hits;
|
|
231
|
+
// Eviction score: lower is more likely to be evicted
|
|
232
|
+
// Combines age and hit count (weighted by hitCountWeight)
|
|
233
|
+
const ageScore = age / this.maxAge;
|
|
234
|
+
const hitScore = 1 / (hitCount + 1); // Inverse of hits (more hits = lower score)
|
|
235
|
+
const score = ageScore * (1 - this.hitCountWeight) + hitScore * this.hitCountWeight;
|
|
236
|
+
return { key, entry, score };
|
|
237
|
+
});
|
|
238
|
+
// Sort by score (lowest first - these will be evicted)
|
|
239
|
+
entries.sort((a, b) => a.score - b.score);
|
|
240
|
+
// Evict entries until we're under the limit
|
|
241
|
+
const toEvict = entries.slice(0, this.cache.size - this.maxSize);
|
|
242
|
+
toEvict.forEach(({ key }) => {
|
|
243
|
+
this.cache.delete(key);
|
|
244
|
+
this.evictions++;
|
|
245
|
+
});
|
|
246
|
+
}
|
|
247
|
+
}
|
|
248
|
+
//# sourceMappingURL=searchCache.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"searchCache.js","sourceRoot":"","sources":["../../src/utils/searchCache.ts"],"names":[],"mappings":"AAAA;;GAEG;AAiBH,MAAM,OAAO,WAAW;IAYtB,YAAY,SAA4B,EAAE;QAXlC,UAAK,GAAG,IAAI,GAAG,EAAyB,CAAA;QAMhD,mBAAmB;QACX,SAAI,GAAG,CAAC,CAAA;QACR,WAAM,GAAG,CAAC,CAAA;QACV,cAAS,GAAG,CAAC,CAAA;QAGnB,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,CAAA,CAAC,YAAY;QACzD,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC,OAAO,IAAI,GAAG,CAAA;QACpC,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC,OAAO,IAAI,IAAI,CAAA;QACrC,IAAI,CAAC,cAAc,GAAG,MAAM,CAAC,cAAc,IAAI,GAAG,CAAA;IACpD,CAAC;IAED;;OAEG;IACH,WAAW,CACT,KAAU,EACV,CAAS,EACT,UAA+B,EAAE;QAEjC,wDAAwD;QACxD,MAAM,iBAAiB,GAAG,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC;aAC3C,IAAI,EAAE;aACN,MAAM,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE;YACnB,6BAA6B;YAC7B,IAAI,GAAG,KAAK,WAAW,IAAI,GAAG,KAAK,cAAc;gBAAE,OAAO,GAAG,CAAA;YAC7D,GAAG,CAAC,GAAG,CAAC,GAAG,OAAO,CAAC,GAAG,CAAC,CAAA;YACvB,OAAO,GAAG,CAAA;QACZ,CAAC,EAAE,EAAyB,CAAC,CAAA;QAE/B,OAAO,IAAI,CAAC,SAAS,CAAC;YACpB,KAAK,EAAE,OAAO,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK;YAChE,CAAC;YACD,GAAG,iBAAiB;SACrB,CAAC,CAAA;IACJ,CAAC;IAED;;OAEG;IACH,GAAG,CAAC,GAAW;QACb,IAAI,CAAC,IAAI,CAAC,OAAO;YAAE,OAAO,IAAI,CAAA;QAE9B,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,CAAA;QACjC,IAAI,CAAC,KAAK,EAAE,CAAC;YACX,IAAI,CAAC,MAAM,EAAE,CAAA;YACb,OAAO,IAAI,CAAA;QACb,CAAC;QAED,mBAAmB;QACnB,IAAI,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,CAAC,SAAS,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC;YAC/C,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAA;YACtB,IAAI,CAAC,MAAM,EAAE,CAAA;YACb,OAAO,IAAI,CAAA;QACb,CAAC;QAED,kCAAkC;QAClC,KAAK,CAAC,IAAI,EAAE,CAAA;QACZ,IAAI,CAAC,IAAI,EAAE,CAAA;QACX,OAAO,KAAK,CAAC,OAAO,CAAA;IACtB,CAAC;IAED;;OAEG;IACH,GAAG,CAAC,GAAW,EAAE,OAA0B;QACzC,IAAI,CAAC,IAAI,CAAC,OAAO;YAAE,OAAM;QAEzB,yBAAyB;QACzB,IAAI,IAAI,CAAC,KAAK,CAAC,IAAI,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;YACpC,IAAI,CAAC,WAAW,EAAE,CAAA;QACpB,CAAC;QAED,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE;YAClB,OAAO,EAAE,CAAC,GAAG,OAAO,CAAC,EAAE,iCAAiC;YACxD,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE;YACrB,IAAI,EAAE,CAAC;SACR,CAAC,CAAA;IACJ,CAAC;IAED;;OAEG;IACK,WAAW;QACjB,IAAI,SAAS,GAAkB,IAAI,CAAA;QACnC,IAAI,WAAW,GAAG,QAAQ,CAAA;QAE1B,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAA;QAEtB,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,IAAI,CAAC,KAAK,CAAC,OAAO,EAAE,EAAE,CAAC;YAChD,2CAA2C;YAC3C,MAAM,GAAG,GAAG,GAAG,GAAG,KAAK,CAAC,SAAS,CAAA;YACjC,MAAM,QAAQ,GAAG,KAAK,CAAC,IAAI,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAA;YACpD,MAAM,KAAK,GAAG,GAAG,GAAG,CAAC,QAAQ,GAAG,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC,MAAM,CAAC,CAAA;YAElE,IAAI,KAAK,GAAG,WAAW,EAAE,CAAC;gBACxB,WAAW,GAAG,KAAK,CAAA;gBACnB,SAAS,GAAG,GAAG,CAAA;YACjB,CAAC;QACH,CAAC;QAED,IAAI,SAAS,EAAE,CAAC;YACd,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,SAAS,CAAC,CAAA;YAC5B,IAAI,CAAC,SAAS,EAAE,CAAA;QAClB,CAAC;IACH,CAAC;IAED;;OAEG;IACH,KAAK;QACH,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAA;QAClB,IAAI,CAAC,IAAI,GAAG,CAAC,CAAA;QACb,IAAI,CAAC,MAAM,GAAG,CAAC,CAAA;QACf,IAAI,CAAC,SAAS,GAAG,CAAC,CAAA;IACpB,CAAC;IAED;;OAEG;IACH,UAAU,CAAC,OAAyB;QAClC,IAAI,CAAC,OAAO,EAAE,CAAC;YACb,IAAI,CAAC,KAAK,EAAE,CAAA;YACZ,OAAM;QACR,CAAC;QAED,MAAM,YAAY,GAAa,EAAE,CAAA;QAEjC,KAAK,MAAM,GAAG,IAAI,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,EAAE,CAAC;YACpC,MAAM,YAAY,GAAG,OAAO,OAAO,KAAK,QAAQ;gBAC9C,CAAC,CAAC,GAAG,CAAC,QAAQ,CAAC,OAAO,CAAC;gBACvB,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;YAErB,IAAI,YAAY,EAAE,CAAC;gBACjB,YAAY,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;YACxB,CAAC;QACH,CAAC;QAED,YAAY,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAA;IACrD,CAAC;IAED;;;OAGG;IACH,sBAAsB,CAAC,UAAwC;QAC7D,kEAAkE;QAClE,oEAAoE;QACpE,gDAAgD;QAChD,IAAI,CAAC,KAAK,EAAE,CAAA;IACd,CAAC;IAED;;;;OAIG;IACH,qBAAqB;QACnB,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAA;QACtB,MAAM,YAAY,GAAa,EAAE,CAAA;QAEjC,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,IAAI,CAAC,KAAK,CAAC,OAAO,EAAE,EAAE,CAAC;YAChD,IAAI,GAAG,GAAG,KAAK,CAAC,SAAS,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC;gBACxC,YAAY,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;YACxB,CAAC;QACH,CAAC;QAED,YAAY,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAA;QACnD,OAAO,YAAY,CAAC,MAAM,CAAA;IAC5B,CAAC;IAED;;OAEG;IACH,QAAQ;QACN,MAAM,KAAK,GAAG,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC,MAAM,CAAA;QACrC,OAAO;YACL,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,MAAM,EAAE,IAAI,CAAC,MAAM;YACnB,SAAS,EAAE,IAAI,CAAC,SAAS;YACzB,OAAO,EAAE,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;YAC1C,IAAI,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI;YACrB,OAAO,EAAE,IAAI,CAAC,OAAO;YACrB,OAAO,EAAE,IAAI,CAAC,OAAO;SACtB,CAAA;IACH,CAAC;IAED;;OAEG;IACH,UAAU,CAAC,OAAgB;QACzB,MAAM,CAAC,cAAc,CAAC,IAAI,EAAE,SAAS,EAAE,EAAE,KAAK,EAAE,OAAO,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC,CAAA;QAC3E,IAAI,CAAC,OAAO,EAAE,CAAC;YACb,IAAI,CAAC,KAAK,EAAE,CAAA;QACd,CAAC;IACH,CAAC;IAED;;OAEG;IACH,cAAc;QACZ,IAAI,SAAS,GAAG,CAAC,CAAA;QAEjB,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,IAAI,CAAC,KAAK,CAAC,OAAO,EAAE,EAAE,CAAC;YAChD,oBAAoB;YACpB,SAAS,IAAI,GAAG,CAAC,MAAM,GAAG,CAAC,CAAA,CAAC,oBAAoB;YAEhD,sBAAsB;YACtB,SAAS,IAAI,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,MAAM,GAAG,CAAC,CAAA;YACrD,SAAS,IAAI,EAAE,CAAA,CAAC,kCAAkC;QACpD,CAAC;QAED,OAAO,SAAS,CAAA;IAClB,CAAC;IAED;;OAEG;IACH,SAAS;QACP,OAAO;YACL,OAAO,EAAE,IAAI,CAAC,OAAO;YACrB,OAAO,EAAE,IAAI,CAAC,OAAO;YACrB,MAAM,EAAE,IAAI,CAAC,MAAM;YACnB,cAAc,EAAE,IAAI,CAAC,cAAc;SACpC,CAAA;IACH,CAAC;IAED;;OAEG;IACH,YAAY,CAAC,SAAqC;QAChD,IAAI,SAAS,CAAC,OAAO,KAAK,SAAS,EAAE,CAAC;YACpC,IAAI,CAAC,OAAO,GAAG,SAAS,CAAC,OAAO,CAAA;QAClC,CAAC;QACD,IAAI,SAAS,CAAC,OAAO,KAAK,SAAS,EAAE,CAAC;YACpC,IAAI,CAAC,OAAO,GAAG,SAAS,CAAC,OAAO,CAAA;YAChC,qDAAqD;YACrD,IAAI,CAAC,aAAa,EAAE,CAAA;QACtB,CAAC;QACD,IAAI,SAAS,CAAC,MAAM,KAAK,SAAS,EAAE,CAAC;YACnC,IAAI,CAAC,MAAM,GAAG,SAAS,CAAC,MAAM,CAAA;YAC9B,qDAAqD;YACrD,IAAI,CAAC,qBAAqB,EAAE,CAAA;QAC9B,CAAC;QACD,IAAI,SAAS,CAAC,cAAc,KAAK,SAAS,EAAE,CAAC;YAC3C,IAAI,CAAC,cAAc,GAAG,SAAS,CAAC,cAAc,CAAA;QAChD,CAAC;IACH,CAAC;IAED;;OAEG;IACK,aAAa;QACnB,IAAI,IAAI,CAAC,KAAK,CAAC,IAAI,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;YACpC,OAAM;QACR,CAAC;QAED,4EAA4E;QAC5E,MAAM,OAAO,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,EAAE,KAAK,CAAC,EAAE,EAAE;YACpE,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,CAAC,SAAS,CAAA;YACxC,MAAM,QAAQ,GAAG,KAAK,CAAC,IAAI,CAAA;YAE3B,qDAAqD;YACrD,0DAA0D;YAC1D,MAAM,QAAQ,GAAG,GAAG,GAAG,IAAI,CAAC,MAAM,CAAA;YAClC,MAAM,QAAQ,GAAG,CAAC,GAAG,CAAC,QAAQ,GAAG,CAAC,CAAC,CAAA,CAAC,4CAA4C;YAChF,MAAM,KAAK,GAAG,QAAQ,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC,cAAc,CAAC,GAAG,QAAQ,GAAG,IAAI,CAAC,cAAc,CAAA;YAEnF,OAAO,EAAE,GAAG,EAAE,KAAK,EAAE,KAAK,EAAE,CAAA;QAC9B,CAAC,CAAC,CAAA;QAEF,uDAAuD;QACvD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,CAAA;QAEzC,4CAA4C;QAC5C,MAAM,OAAO,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,CAAA;QAChE,OAAO,CAAC,OAAO,CAAC,CAAC,EAAE,GAAG,EAAE,EAAE,EAAE;YAC1B,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAA;YACtB,IAAI,CAAC,SAAS,EAAE,CAAA;QAClB,CAAC,CAAC,CAAA;IACJ,CAAC;CACF"}
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Utility functions for retrieving statistics from Brainy
|
|
3
|
+
*/
|
|
4
|
+
/**
|
|
5
|
+
* Get statistics about the current state of a BrainyData instance
|
|
6
|
+
* This function provides access to statistics at the root level of the library
|
|
7
|
+
*
|
|
8
|
+
* @param instance A BrainyData instance to get statistics from
|
|
9
|
+
* @param options Additional options for retrieving statistics
|
|
10
|
+
* @returns Object containing counts of nouns, verbs, metadata entries, and HNSW index size
|
|
11
|
+
* @throws Error if the instance is not provided or if statistics retrieval fails
|
|
12
|
+
*/
|
|
13
|
+
export async function getStatistics(instance, options = {}) {
|
|
14
|
+
if (!instance) {
|
|
15
|
+
throw new Error('BrainyData instance must be provided to getStatistics');
|
|
16
|
+
}
|
|
17
|
+
try {
|
|
18
|
+
return await instance.getStatistics(options);
|
|
19
|
+
}
|
|
20
|
+
catch (error) {
|
|
21
|
+
console.error('Failed to get statistics:', error);
|
|
22
|
+
throw new Error(`Failed to get statistics: ${error}`);
|
|
23
|
+
}
|
|
24
|
+
}
|
|
25
|
+
//# sourceMappingURL=statistics.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"statistics.js","sourceRoot":"","sources":["../../src/utils/statistics.ts"],"names":[],"mappings":"AAAA;;GAEG;AAIH;;;;;;;;GAQG;AACH,MAAM,CAAC,KAAK,UAAU,aAAa,CAC/B,QAAoB,EACpB,UAEI,EAAE;IAcN,IAAI,CAAC,QAAQ,EAAE,CAAC;QACZ,MAAM,IAAI,KAAK,CAAC,uDAAuD,CAAC,CAAA;IAC5E,CAAC;IAED,IAAI,CAAC;QACD,OAAO,MAAM,QAAQ,CAAC,aAAa,CAAC,OAAO,CAAC,CAAA;IAChD,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACb,OAAO,CAAC,KAAK,CAAC,2BAA2B,EAAE,KAAK,CAAC,CAAA;QACjD,MAAM,IAAI,KAAK,CAAC,6BAA6B,KAAK,EAAE,CAAC,CAAA;IACzD,CAAC;AACL,CAAC"}
|
|
@@ -0,0 +1,224 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Lightweight statistics collector for Brainy
|
|
3
|
+
* Designed to have minimal performance impact even with millions of entries
|
|
4
|
+
*/
|
|
5
|
+
export class StatisticsCollector {
|
|
6
|
+
constructor() {
|
|
7
|
+
// Content type tracking (lightweight counters)
|
|
8
|
+
this.contentTypes = new Map();
|
|
9
|
+
// Data freshness tracking (only track timestamps, not full data)
|
|
10
|
+
this.oldestTimestamp = Date.now();
|
|
11
|
+
this.newestTimestamp = Date.now();
|
|
12
|
+
this.updateTimestamps = [];
|
|
13
|
+
// Search performance tracking (rolling window)
|
|
14
|
+
this.searchMetrics = {
|
|
15
|
+
totalSearches: 0,
|
|
16
|
+
totalSearchTimeMs: 0,
|
|
17
|
+
searchTimestamps: [],
|
|
18
|
+
topSearchTerms: new Map()
|
|
19
|
+
};
|
|
20
|
+
// Verb type tracking
|
|
21
|
+
this.verbTypes = new Map();
|
|
22
|
+
// Storage size estimates (updated periodically, not on every operation)
|
|
23
|
+
this.storageSizeCache = {
|
|
24
|
+
lastUpdated: 0,
|
|
25
|
+
sizes: {
|
|
26
|
+
nouns: 0,
|
|
27
|
+
verbs: 0,
|
|
28
|
+
metadata: 0,
|
|
29
|
+
index: 0
|
|
30
|
+
}
|
|
31
|
+
};
|
|
32
|
+
this.MAX_TIMESTAMPS = 1000; // Keep last 1000 timestamps
|
|
33
|
+
this.MAX_SEARCH_TERMS = 100; // Track top 100 search terms
|
|
34
|
+
this.SIZE_UPDATE_INTERVAL = 60000; // Update sizes every minute
|
|
35
|
+
}
|
|
36
|
+
/**
|
|
37
|
+
* Track content type (very lightweight)
|
|
38
|
+
*/
|
|
39
|
+
trackContentType(type) {
|
|
40
|
+
this.contentTypes.set(type, (this.contentTypes.get(type) || 0) + 1);
|
|
41
|
+
}
|
|
42
|
+
/**
|
|
43
|
+
* Track data update timestamp (lightweight)
|
|
44
|
+
*/
|
|
45
|
+
trackUpdate(timestamp) {
|
|
46
|
+
const ts = timestamp || Date.now();
|
|
47
|
+
// Update oldest/newest
|
|
48
|
+
if (ts < this.oldestTimestamp)
|
|
49
|
+
this.oldestTimestamp = ts;
|
|
50
|
+
if (ts > this.newestTimestamp)
|
|
51
|
+
this.newestTimestamp = ts;
|
|
52
|
+
// Add to rolling window
|
|
53
|
+
this.updateTimestamps.push({ timestamp: ts, count: 1 });
|
|
54
|
+
// Keep window size limited
|
|
55
|
+
if (this.updateTimestamps.length > this.MAX_TIMESTAMPS) {
|
|
56
|
+
this.updateTimestamps.shift();
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
/**
|
|
60
|
+
* Track search performance (lightweight)
|
|
61
|
+
*/
|
|
62
|
+
trackSearch(searchTerm, durationMs) {
|
|
63
|
+
this.searchMetrics.totalSearches++;
|
|
64
|
+
this.searchMetrics.totalSearchTimeMs += durationMs;
|
|
65
|
+
// Add to rolling window
|
|
66
|
+
this.searchMetrics.searchTimestamps.push({
|
|
67
|
+
timestamp: Date.now(),
|
|
68
|
+
count: 1
|
|
69
|
+
});
|
|
70
|
+
// Keep window size limited
|
|
71
|
+
if (this.searchMetrics.searchTimestamps.length > this.MAX_TIMESTAMPS) {
|
|
72
|
+
this.searchMetrics.searchTimestamps.shift();
|
|
73
|
+
}
|
|
74
|
+
// Track search term (limit to top N)
|
|
75
|
+
const termCount = (this.searchMetrics.topSearchTerms.get(searchTerm) || 0) + 1;
|
|
76
|
+
this.searchMetrics.topSearchTerms.set(searchTerm, termCount);
|
|
77
|
+
// Prune if too many terms
|
|
78
|
+
if (this.searchMetrics.topSearchTerms.size > this.MAX_SEARCH_TERMS * 2) {
|
|
79
|
+
this.pruneSearchTerms();
|
|
80
|
+
}
|
|
81
|
+
}
|
|
82
|
+
/**
|
|
83
|
+
* Track verb type (lightweight)
|
|
84
|
+
*/
|
|
85
|
+
trackVerbType(type) {
|
|
86
|
+
this.verbTypes.set(type, (this.verbTypes.get(type) || 0) + 1);
|
|
87
|
+
}
|
|
88
|
+
/**
|
|
89
|
+
* Update storage size estimates (called periodically, not on every operation)
|
|
90
|
+
*/
|
|
91
|
+
updateStorageSizes(sizes) {
|
|
92
|
+
this.storageSizeCache = {
|
|
93
|
+
lastUpdated: Date.now(),
|
|
94
|
+
sizes
|
|
95
|
+
};
|
|
96
|
+
}
|
|
97
|
+
/**
|
|
98
|
+
* Get comprehensive statistics
|
|
99
|
+
*/
|
|
100
|
+
getStatistics() {
|
|
101
|
+
const now = Date.now();
|
|
102
|
+
const hourAgo = now - 3600000;
|
|
103
|
+
const dayAgo = now - 86400000;
|
|
104
|
+
const weekAgo = now - 604800000;
|
|
105
|
+
const monthAgo = now - 2592000000;
|
|
106
|
+
// Calculate data freshness
|
|
107
|
+
const updatesLastHour = this.updateTimestamps.filter(t => t.timestamp > hourAgo).length;
|
|
108
|
+
const updatesLastDay = this.updateTimestamps.filter(t => t.timestamp > dayAgo).length;
|
|
109
|
+
// Calculate age distribution
|
|
110
|
+
const ageDistribution = {
|
|
111
|
+
last24h: 0,
|
|
112
|
+
last7d: 0,
|
|
113
|
+
last30d: 0,
|
|
114
|
+
older: 0
|
|
115
|
+
};
|
|
116
|
+
// Estimate based on update patterns (not scanning all data)
|
|
117
|
+
const totalUpdates = this.updateTimestamps.length;
|
|
118
|
+
if (totalUpdates > 0) {
|
|
119
|
+
const recentUpdates = this.updateTimestamps.filter(t => t.timestamp > dayAgo).length;
|
|
120
|
+
const weekUpdates = this.updateTimestamps.filter(t => t.timestamp > weekAgo).length;
|
|
121
|
+
const monthUpdates = this.updateTimestamps.filter(t => t.timestamp > monthAgo).length;
|
|
122
|
+
ageDistribution.last24h = Math.round((recentUpdates / totalUpdates) * 100);
|
|
123
|
+
ageDistribution.last7d = Math.round(((weekUpdates - recentUpdates) / totalUpdates) * 100);
|
|
124
|
+
ageDistribution.last30d = Math.round(((monthUpdates - weekUpdates) / totalUpdates) * 100);
|
|
125
|
+
ageDistribution.older = 100 - ageDistribution.last24h - ageDistribution.last7d - ageDistribution.last30d;
|
|
126
|
+
}
|
|
127
|
+
// Calculate search metrics
|
|
128
|
+
const searchesLastHour = this.searchMetrics.searchTimestamps.filter(t => t.timestamp > hourAgo).length;
|
|
129
|
+
const searchesLastDay = this.searchMetrics.searchTimestamps.filter(t => t.timestamp > dayAgo).length;
|
|
130
|
+
const avgSearchTime = this.searchMetrics.totalSearches > 0
|
|
131
|
+
? this.searchMetrics.totalSearchTimeMs / this.searchMetrics.totalSearches
|
|
132
|
+
: 0;
|
|
133
|
+
// Get top search terms
|
|
134
|
+
const topSearchTerms = Array.from(this.searchMetrics.topSearchTerms.entries())
|
|
135
|
+
.sort((a, b) => b[1] - a[1])
|
|
136
|
+
.slice(0, 10)
|
|
137
|
+
.map(([term]) => term);
|
|
138
|
+
// Calculate storage metrics
|
|
139
|
+
const totalSize = Object.values(this.storageSizeCache.sizes).reduce((a, b) => a + b, 0);
|
|
140
|
+
return {
|
|
141
|
+
contentTypes: Object.fromEntries(this.contentTypes),
|
|
142
|
+
dataFreshness: {
|
|
143
|
+
oldestEntry: new Date(this.oldestTimestamp).toISOString(),
|
|
144
|
+
newestEntry: new Date(this.newestTimestamp).toISOString(),
|
|
145
|
+
updatesLastHour,
|
|
146
|
+
updatesLastDay,
|
|
147
|
+
ageDistribution
|
|
148
|
+
},
|
|
149
|
+
storageMetrics: {
|
|
150
|
+
totalSizeBytes: totalSize,
|
|
151
|
+
nounsSizeBytes: this.storageSizeCache.sizes.nouns,
|
|
152
|
+
verbsSizeBytes: this.storageSizeCache.sizes.verbs,
|
|
153
|
+
metadataSizeBytes: this.storageSizeCache.sizes.metadata,
|
|
154
|
+
indexSizeBytes: this.storageSizeCache.sizes.index
|
|
155
|
+
},
|
|
156
|
+
searchMetrics: {
|
|
157
|
+
totalSearches: this.searchMetrics.totalSearches,
|
|
158
|
+
averageSearchTimeMs: avgSearchTime,
|
|
159
|
+
searchesLastHour,
|
|
160
|
+
searchesLastDay,
|
|
161
|
+
topSearchTerms
|
|
162
|
+
},
|
|
163
|
+
verbStatistics: {
|
|
164
|
+
totalVerbs: Array.from(this.verbTypes.values()).reduce((a, b) => a + b, 0),
|
|
165
|
+
verbTypes: Object.fromEntries(this.verbTypes),
|
|
166
|
+
averageConnectionsPerVerb: 2 // Verbs connect 2 nouns
|
|
167
|
+
}
|
|
168
|
+
};
|
|
169
|
+
}
|
|
170
|
+
/**
|
|
171
|
+
* Merge statistics from storage (for distributed systems)
|
|
172
|
+
*/
|
|
173
|
+
mergeFromStorage(stored) {
|
|
174
|
+
// Merge content types
|
|
175
|
+
if (stored.contentTypes) {
|
|
176
|
+
for (const [type, count] of Object.entries(stored.contentTypes)) {
|
|
177
|
+
this.contentTypes.set(type, count);
|
|
178
|
+
}
|
|
179
|
+
}
|
|
180
|
+
// Merge verb types
|
|
181
|
+
if (stored.verbStatistics?.verbTypes) {
|
|
182
|
+
for (const [type, count] of Object.entries(stored.verbStatistics.verbTypes)) {
|
|
183
|
+
this.verbTypes.set(type, count);
|
|
184
|
+
}
|
|
185
|
+
}
|
|
186
|
+
// Merge search metrics
|
|
187
|
+
if (stored.searchMetrics) {
|
|
188
|
+
this.searchMetrics.totalSearches = stored.searchMetrics.totalSearches || 0;
|
|
189
|
+
this.searchMetrics.totalSearchTimeMs = (stored.searchMetrics.averageSearchTimeMs || 0) * this.searchMetrics.totalSearches;
|
|
190
|
+
}
|
|
191
|
+
// Merge data freshness
|
|
192
|
+
if (stored.dataFreshness) {
|
|
193
|
+
this.oldestTimestamp = new Date(stored.dataFreshness.oldestEntry).getTime();
|
|
194
|
+
this.newestTimestamp = new Date(stored.dataFreshness.newestEntry).getTime();
|
|
195
|
+
}
|
|
196
|
+
}
|
|
197
|
+
/**
|
|
198
|
+
* Reset statistics (for testing)
|
|
199
|
+
*/
|
|
200
|
+
reset() {
|
|
201
|
+
this.contentTypes.clear();
|
|
202
|
+
this.verbTypes.clear();
|
|
203
|
+
this.updateTimestamps = [];
|
|
204
|
+
this.searchMetrics = {
|
|
205
|
+
totalSearches: 0,
|
|
206
|
+
totalSearchTimeMs: 0,
|
|
207
|
+
searchTimestamps: [],
|
|
208
|
+
topSearchTerms: new Map()
|
|
209
|
+
};
|
|
210
|
+
this.oldestTimestamp = Date.now();
|
|
211
|
+
this.newestTimestamp = Date.now();
|
|
212
|
+
}
|
|
213
|
+
pruneSearchTerms() {
|
|
214
|
+
// Keep only top N search terms
|
|
215
|
+
const sorted = Array.from(this.searchMetrics.topSearchTerms.entries())
|
|
216
|
+
.sort((a, b) => b[1] - a[1])
|
|
217
|
+
.slice(0, this.MAX_SEARCH_TERMS);
|
|
218
|
+
this.searchMetrics.topSearchTerms.clear();
|
|
219
|
+
for (const [term, count] of sorted) {
|
|
220
|
+
this.searchMetrics.topSearchTerms.set(term, count);
|
|
221
|
+
}
|
|
222
|
+
}
|
|
223
|
+
}
|
|
224
|
+
//# sourceMappingURL=statisticsCollector.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"statisticsCollector.js","sourceRoot":"","sources":["../../src/utils/statisticsCollector.ts"],"names":[],"mappings":"AAAA;;;GAGG;AASH,MAAM,OAAO,mBAAmB;IAAhC;QACE,+CAA+C;QACvC,iBAAY,GAAwB,IAAI,GAAG,EAAE,CAAA;QAErD,iEAAiE;QACzD,oBAAe,GAAW,IAAI,CAAC,GAAG,EAAE,CAAA;QACpC,oBAAe,GAAW,IAAI,CAAC,GAAG,EAAE,CAAA;QACpC,qBAAgB,GAAqB,EAAE,CAAA;QAE/C,+CAA+C;QACvC,kBAAa,GAAG;YACtB,aAAa,EAAE,CAAC;YAChB,iBAAiB,EAAE,CAAC;YACpB,gBAAgB,EAAE,EAAsB;YACxC,cAAc,EAAE,IAAI,GAAG,EAAkB;SAC1C,CAAA;QAED,qBAAqB;QACb,cAAS,GAAwB,IAAI,GAAG,EAAE,CAAA;QAElD,wEAAwE;QAChE,qBAAgB,GAAG;YACzB,WAAW,EAAE,CAAC;YACd,KAAK,EAAE;gBACL,KAAK,EAAE,CAAC;gBACR,KAAK,EAAE,CAAC;gBACR,QAAQ,EAAE,CAAC;gBACX,KAAK,EAAE,CAAC;aACT;SACF,CAAA;QAEgB,mBAAc,GAAG,IAAI,CAAA,CAAC,4BAA4B;QAClD,qBAAgB,GAAG,GAAG,CAAA,CAAC,6BAA6B;QACpD,yBAAoB,GAAG,KAAK,CAAA,CAAC,4BAA4B;IA+N5E,CAAC;IA7NC;;OAEG;IACH,gBAAgB,CAAC,IAAY;QAC3B,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAA;IACrE,CAAC;IAED;;OAEG;IACH,WAAW,CAAC,SAAkB;QAC5B,MAAM,EAAE,GAAG,SAAS,IAAI,IAAI,CAAC,GAAG,EAAE,CAAA;QAElC,uBAAuB;QACvB,IAAI,EAAE,GAAG,IAAI,CAAC,eAAe;YAAE,IAAI,CAAC,eAAe,GAAG,EAAE,CAAA;QACxD,IAAI,EAAE,GAAG,IAAI,CAAC,eAAe;YAAE,IAAI,CAAC,eAAe,GAAG,EAAE,CAAA;QAExD,wBAAwB;QACxB,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,EAAE,SAAS,EAAE,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC,CAAA;QAEvD,2BAA2B;QAC3B,IAAI,IAAI,CAAC,gBAAgB,CAAC,MAAM,GAAG,IAAI,CAAC,cAAc,EAAE,CAAC;YACvD,IAAI,CAAC,gBAAgB,CAAC,KAAK,EAAE,CAAA;QAC/B,CAAC;IACH,CAAC;IAED;;OAEG;IACH,WAAW,CAAC,UAAkB,EAAE,UAAkB;QAChD,IAAI,CAAC,aAAa,CAAC,aAAa,EAAE,CAAA;QAClC,IAAI,CAAC,aAAa,CAAC,iBAAiB,IAAI,UAAU,CAAA;QAElD,wBAAwB;QACxB,IAAI,CAAC,aAAa,CAAC,gBAAgB,CAAC,IAAI,CAAC;YACvC,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE;YACrB,KAAK,EAAE,CAAC;SACT,CAAC,CAAA;QAEF,2BAA2B;QAC3B,IAAI,IAAI,CAAC,aAAa,CAAC,gBAAgB,CAAC,MAAM,GAAG,IAAI,CAAC,cAAc,EAAE,CAAC;YACrE,IAAI,CAAC,aAAa,CAAC,gBAAgB,CAAC,KAAK,EAAE,CAAA;QAC7C,CAAC;QAED,qCAAqC;QACrC,MAAM,SAAS,GAAG,CAAC,IAAI,CAAC,aAAa,CAAC,cAAc,CAAC,GAAG,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAA;QAC9E,IAAI,CAAC,aAAa,CAAC,cAAc,CAAC,GAAG,CAAC,UAAU,EAAE,SAAS,CAAC,CAAA;QAE5D,0BAA0B;QAC1B,IAAI,IAAI,CAAC,aAAa,CAAC,cAAc,CAAC,IAAI,GAAG,IAAI,CAAC,gBAAgB,GAAG,CAAC,EAAE,CAAC;YACvE,IAAI,CAAC,gBAAgB,EAAE,CAAA;QACzB,CAAC;IACH,CAAC;IAED;;OAEG;IACH,aAAa,CAAC,IAAY;QACxB,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAA;IAC/D,CAAC;IAED;;OAEG;IACH,kBAAkB,CAAC,KAKlB;QACC,IAAI,CAAC,gBAAgB,GAAG;YACtB,WAAW,EAAE,IAAI,CAAC,GAAG,EAAE;YACvB,KAAK;SACN,CAAA;IACH,CAAC;IAED;;OAEG;IACH,aAAa;QACX,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAA;QACtB,MAAM,OAAO,GAAG,GAAG,GAAG,OAAO,CAAA;QAC7B,MAAM,MAAM,GAAG,GAAG,GAAG,QAAQ,CAAA;QAC7B,MAAM,OAAO,GAAG,GAAG,GAAG,SAAS,CAAA;QAC/B,MAAM,QAAQ,GAAG,GAAG,GAAG,UAAU,CAAA;QAEjC,2BAA2B;QAC3B,MAAM,eAAe,GAAG,IAAI,CAAC,gBAAgB,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,SAAS,GAAG,OAAO,CAAC,CAAC,MAAM,CAAA;QACvF,MAAM,cAAc,GAAG,IAAI,CAAC,gBAAgB,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,SAAS,GAAG,MAAM,CAAC,CAAC,MAAM,CAAA;QAErF,6BAA6B;QAC7B,MAAM,eAAe,GAAG;YACtB,OAAO,EAAE,CAAC;YACV,MAAM,EAAE,CAAC;YACT,OAAO,EAAE,CAAC;YACV,KAAK,EAAE,CAAC;SACT,CAAA;QAED,4DAA4D;QAC5D,MAAM,YAAY,GAAG,IAAI,CAAC,gBAAgB,CAAC,MAAM,CAAA;QACjD,IAAI,YAAY,GAAG,CAAC,EAAE,CAAC;YACrB,MAAM,aAAa,GAAG,IAAI,CAAC,gBAAgB,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,SAAS,GAAG,MAAM,CAAC,CAAC,MAAM,CAAA;YACpF,MAAM,WAAW,GAAG,IAAI,CAAC,gBAAgB,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,SAAS,GAAG,OAAO,CAAC,CAAC,MAAM,CAAA;YACnF,MAAM,YAAY,GAAG,IAAI,CAAC,gBAAgB,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,SAAS,GAAG,QAAQ,CAAC,CAAC,MAAM,CAAA;YAErF,eAAe,CAAC,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,aAAa,GAAG,YAAY,CAAC,GAAG,GAAG,CAAC,CAAA;YAC1E,eAAe,CAAC,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,WAAW,GAAG,aAAa,CAAC,GAAG,YAAY,CAAC,GAAG,GAAG,CAAC,CAAA;YACzF,eAAe,CAAC,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,YAAY,GAAG,WAAW,CAAC,GAAG,YAAY,CAAC,GAAG,GAAG,CAAC,CAAA;YACzF,eAAe,CAAC,KAAK,GAAG,GAAG,GAAG,eAAe,CAAC,OAAO,GAAG,eAAe,CAAC,MAAM,GAAG,eAAe,CAAC,OAAO,CAAA;QAC1G,CAAC;QAED,2BAA2B;QAC3B,MAAM,gBAAgB,GAAG,IAAI,CAAC,aAAa,CAAC,gBAAgB,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,SAAS,GAAG,OAAO,CAAC,CAAC,MAAM,CAAA;QACtG,MAAM,eAAe,GAAG,IAAI,CAAC,aAAa,CAAC,gBAAgB,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,SAAS,GAAG,MAAM,CAAC,CAAC,MAAM,CAAA;QACpG,MAAM,aAAa,GAAG,IAAI,CAAC,aAAa,CAAC,aAAa,GAAG,CAAC;YACxD,CAAC,CAAC,IAAI,CAAC,aAAa,CAAC,iBAAiB,GAAG,IAAI,CAAC,aAAa,CAAC,aAAa;YACzE,CAAC,CAAC,CAAC,CAAA;QAEL,uBAAuB;QACvB,MAAM,cAAc,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,aAAa,CAAC,cAAc,CAAC,OAAO,EAAE,CAAC;aAC3E,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;aAC3B,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC;aACZ,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,CAAA;QAExB,4BAA4B;QAC5B,MAAM,SAAS,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,gBAAgB,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAA;QAEvF,OAAO;YACL,YAAY,EAAE,MAAM,CAAC,WAAW,CAAC,IAAI,CAAC,YAAY,CAAC;YAEnD,aAAa,EAAE;gBACb,WAAW,EAAE,IAAI,IAAI,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC,WAAW,EAAE;gBACzD,WAAW,EAAE,IAAI,IAAI,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC,WAAW,EAAE;gBACzD,eAAe;gBACf,cAAc;gBACd,eAAe;aAChB;YAED,cAAc,EAAE;gBACd,cAAc,EAAE,SAAS;gBACzB,cAAc,EAAE,IAAI,CAAC,gBAAgB,CAAC,KAAK,CAAC,KAAK;gBACjD,cAAc,EAAE,IAAI,CAAC,gBAAgB,CAAC,KAAK,CAAC,KAAK;gBACjD,iBAAiB,EAAE,IAAI,CAAC,gBAAgB,CAAC,KAAK,CAAC,QAAQ;gBACvD,cAAc,EAAE,IAAI,CAAC,gBAAgB,CAAC,KAAK,CAAC,KAAK;aAClD;YAED,aAAa,EAAE;gBACb,aAAa,EAAE,IAAI,CAAC,aAAa,CAAC,aAAa;gBAC/C,mBAAmB,EAAE,aAAa;gBAClC,gBAAgB;gBAChB,eAAe;gBACf,cAAc;aACf;YAED,cAAc,EAAE;gBACd,UAAU,EAAE,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;gBAC1E,SAAS,EAAE,MAAM,CAAC,WAAW,CAAC,IAAI,CAAC,SAAS,CAAC;gBAC7C,yBAAyB,EAAE,CAAC,CAAC,wBAAwB;aACtD;SACF,CAAA;IACH,CAAC;IAED;;OAEG;IACH,gBAAgB,CAAC,MAA+B;QAC9C,sBAAsB;QACtB,IAAI,MAAM,CAAC,YAAY,EAAE,CAAC;YACxB,KAAK,MAAM,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,YAAY,CAAC,EAAE,CAAC;gBAChE,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,IAAI,EAAE,KAAK,CAAC,CAAA;YACpC,CAAC;QACH,CAAC;QAED,mBAAmB;QACnB,IAAI,MAAM,CAAC,cAAc,EAAE,SAAS,EAAE,CAAC;YACrC,KAAK,MAAM,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,cAAc,CAAC,SAAS,CAAC,EAAE,CAAC;gBAC5E,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,IAAI,EAAE,KAAK,CAAC,CAAA;YACjC,CAAC;QACH,CAAC;QAED,uBAAuB;QACvB,IAAI,MAAM,CAAC,aAAa,EAAE,CAAC;YACzB,IAAI,CAAC,aAAa,CAAC,aAAa,GAAG,MAAM,CAAC,aAAa,CAAC,aAAa,IAAI,CAAC,CAAA;YAC1E,IAAI,CAAC,aAAa,CAAC,iBAAiB,GAAG,CAAC,MAAM,CAAC,aAAa,CAAC,mBAAmB,IAAI,CAAC,CAAC,GAAG,IAAI,CAAC,aAAa,CAAC,aAAa,CAAA;QAC3H,CAAC;QAED,uBAAuB;QACvB,IAAI,MAAM,CAAC,aAAa,EAAE,CAAC;YACzB,IAAI,CAAC,eAAe,GAAG,IAAI,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,WAAW,CAAC,CAAC,OAAO,EAAE,CAAA;YAC3E,IAAI,CAAC,eAAe,GAAG,IAAI,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,WAAW,CAAC,CAAC,OAAO,EAAE,CAAA;QAC7E,CAAC;IACH,CAAC;IAED;;OAEG;IACH,KAAK;QACH,IAAI,CAAC,YAAY,CAAC,KAAK,EAAE,CAAA;QACzB,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,CAAA;QACtB,IAAI,CAAC,gBAAgB,GAAG,EAAE,CAAA;QAC1B,IAAI,CAAC,aAAa,GAAG;YACnB,aAAa,EAAE,CAAC;YAChB,iBAAiB,EAAE,CAAC;YACpB,gBAAgB,EAAE,EAAE;YACpB,cAAc,EAAE,IAAI,GAAG,EAAE;SAC1B,CAAA;QACD,IAAI,CAAC,eAAe,GAAG,IAAI,CAAC,GAAG,EAAE,CAAA;QACjC,IAAI,CAAC,eAAe,GAAG,IAAI,CAAC,GAAG,EAAE,CAAA;IACnC,CAAC;IAEO,gBAAgB;QACtB,+BAA+B;QAC/B,MAAM,MAAM,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,aAAa,CAAC,cAAc,CAAC,OAAO,EAAE,CAAC;aACnE,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;aAC3B,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,gBAAgB,CAAC,CAAA;QAElC,IAAI,CAAC,aAAa,CAAC,cAAc,CAAC,KAAK,EAAE,CAAA;QACzC,KAAK,MAAM,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,MAAM,EAAE,CAAC;YACnC,IAAI,CAAC,aAAa,CAAC,cAAc,CAAC,GAAG,CAAC,IAAI,EAAE,KAAK,CAAC,CAAA;QACpD,CAAC;IACH,CAAC;CACF"}
|