@soulcraft/brainy 0.23.0 → 0.25.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +22 -16
- package/dist/brainy.js +4574 -4691
- package/dist/brainy.min.js +750 -750
- package/dist/brainyData.d.ts +86 -10
- package/dist/coreTypes.d.ts +11 -0
- package/dist/storage/adapters/fileSystemStorage.d.ts +78 -29
- package/dist/storage/adapters/fileSystemStorage.d.ts.map +1 -1
- package/dist/storage/adapters/memoryStorage.d.ts.map +1 -1
- package/dist/storage/fileSystemStorage.d.ts +2 -15
- package/dist/storage/fileSystemStorage.d.ts.map +1 -1
- package/dist/storage/opfsStorage.d.ts +3 -66
- package/dist/storage/opfsStorage.d.ts.map +1 -1
- package/dist/storage/s3CompatibleStorage.d.ts +2 -14
- package/dist/storage/s3CompatibleStorage.d.ts.map +1 -1
- package/dist/testing/prettyReporter.d.ts +23 -0
- package/dist/testing/prettySummaryReporter.d.ts +22 -0
- package/dist/types/tensorflowTypes.d.ts +0 -8
- package/dist/types/tensorflowTypes.d.ts.map +1 -1
- package/dist/unified.d.ts +4 -0
- package/dist/unified.js +846 -297
- package/dist/unified.min.js +528 -528
- package/dist/utils/embedding.d.ts +7 -0
- package/dist/utils/embedding.d.ts.map +1 -1
- package/dist/utils/logger.d.ts +99 -0
- package/dist/utils/logger.d.ts.map +1 -0
- package/dist/utils/textEncoding.d.ts +0 -7
- package/dist/utils/textEncoding.d.ts.map +1 -1
- package/dist/utils/version.d.ts +1 -1
- package/package.json +14 -3
- package/dist/augmentations/conduitAugmentations.js +0 -1158
- package/dist/augmentations/conduitAugmentations.js.map +0 -1
- package/dist/augmentations/memoryAugmentations.js +0 -255
- package/dist/augmentations/memoryAugmentations.js.map +0 -1
- package/dist/augmentations/serverSearchAugmentations.js +0 -531
- package/dist/augmentations/serverSearchAugmentations.js.map +0 -1
- package/dist/examples/basicUsage.js +0 -128
- package/dist/examples/basicUsage.js.map +0 -1
- package/dist/hnsw/hnswIndex.js +0 -550
- package/dist/hnsw/hnswIndex.js.map +0 -1
- package/dist/hnsw/hnswIndexOptimized.js +0 -441
- package/dist/hnsw/hnswIndexOptimized.js.map +0 -1
- package/dist/mcp/brainyMCPAdapter.js +0 -142
- package/dist/mcp/brainyMCPAdapter.js.map +0 -1
- package/dist/mcp/brainyMCPService.js +0 -248
- package/dist/mcp/brainyMCPService.js.map +0 -1
- package/dist/mcp/index.js +0 -17
- package/dist/mcp/index.js.map +0 -1
- package/dist/mcp/mcpAugmentationToolset.js +0 -180
- package/dist/mcp/mcpAugmentationToolset.js.map +0 -1
- package/dist/storage/adapters/baseStorageAdapter.js +0 -233
- package/dist/storage/adapters/baseStorageAdapter.js.map +0 -1
- package/dist/storage/adapters/fileSystemStorage.js +0 -568
- package/dist/storage/adapters/fileSystemStorage.js.map +0 -1
- package/dist/storage/adapters/memoryStorage.js +0 -300
- package/dist/storage/adapters/memoryStorage.js.map +0 -1
- package/dist/storage/adapters/opfsStorage.js +0 -778
- package/dist/storage/adapters/opfsStorage.js.map +0 -1
- package/dist/storage/adapters/s3CompatibleStorage.js +0 -1021
- package/dist/storage/adapters/s3CompatibleStorage.js.map +0 -1
- package/dist/storage/baseStorage.js +0 -126
- package/dist/storage/baseStorage.js.map +0 -1
- package/dist/storage/storageFactory.js +0 -183
- package/dist/storage/storageFactory.js.map +0 -1
- package/dist/types/augmentations.js +0 -16
- package/dist/types/augmentations.js.map +0 -1
- package/dist/types/brainyDataInterface.js +0 -8
- package/dist/types/brainyDataInterface.js.map +0 -1
- package/dist/types/fileSystemTypes.js +0 -8
- package/dist/types/fileSystemTypes.js.map +0 -1
- package/dist/types/graphTypes.js +0 -36
- package/dist/types/graphTypes.js.map +0 -1
- package/dist/types/mcpTypes.js +0 -22
- package/dist/types/mcpTypes.js.map +0 -1
- package/dist/types/pipelineTypes.js +0 -7
- package/dist/types/pipelineTypes.js.map +0 -1
- package/dist/types/tensorflowTypes.js +0 -6
- package/dist/types/tensorflowTypes.js.map +0 -1
- package/dist/utils/distance.js +0 -239
- package/dist/utils/distance.js.map +0 -1
- package/dist/utils/embedding.js +0 -622
- package/dist/utils/embedding.js.map +0 -1
- package/dist/utils/environment.js +0 -75
- package/dist/utils/environment.js.map +0 -1
- package/dist/utils/index.js +0 -5
- package/dist/utils/index.js.map +0 -1
- package/dist/utils/statistics.js +0 -25
- package/dist/utils/statistics.js.map +0 -1
- package/dist/utils/tensorflowUtils.js +0 -25
- package/dist/utils/tensorflowUtils.js.map +0 -1
- package/dist/utils/textEncoding.js +0 -281
- package/dist/utils/textEncoding.js.map +0 -1
- package/dist/utils/workerUtils.js +0 -458
- package/dist/utils/workerUtils.js.map +0 -1
|
@@ -1,441 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Optimized HNSW (Hierarchical Navigable Small World) Index implementation
|
|
3
|
-
* Extends the base HNSW implementation with support for large datasets
|
|
4
|
-
* Uses product quantization for dimensionality reduction and disk-based storage when needed
|
|
5
|
-
*/
|
|
6
|
-
import { HNSWIndex } from './hnswIndex.js';
|
|
7
|
-
// Default configuration for the optimized HNSW index
|
|
8
|
-
const DEFAULT_OPTIMIZED_CONFIG = {
|
|
9
|
-
M: 16,
|
|
10
|
-
efConstruction: 200,
|
|
11
|
-
efSearch: 50,
|
|
12
|
-
ml: 16,
|
|
13
|
-
memoryThreshold: 1024 * 1024 * 1024, // 1GB default threshold
|
|
14
|
-
productQuantization: {
|
|
15
|
-
enabled: false,
|
|
16
|
-
numSubvectors: 16,
|
|
17
|
-
numCentroids: 256
|
|
18
|
-
},
|
|
19
|
-
useDiskBasedIndex: false
|
|
20
|
-
};
|
|
21
|
-
/**
|
|
22
|
-
* Product Quantization implementation
|
|
23
|
-
* Reduces vector dimensionality by splitting vectors into subvectors
|
|
24
|
-
* and quantizing each subvector to the nearest centroid
|
|
25
|
-
*/
|
|
26
|
-
class ProductQuantizer {
|
|
27
|
-
constructor(numSubvectors = 16, numCentroids = 256) {
|
|
28
|
-
this.centroids = [];
|
|
29
|
-
this.subvectorSize = 0;
|
|
30
|
-
this.initialized = false;
|
|
31
|
-
this.dimension = 0;
|
|
32
|
-
this.numSubvectors = numSubvectors;
|
|
33
|
-
this.numCentroids = numCentroids;
|
|
34
|
-
}
|
|
35
|
-
/**
|
|
36
|
-
* Initialize the product quantizer with training data
|
|
37
|
-
* @param vectors Training vectors to use for learning centroids
|
|
38
|
-
*/
|
|
39
|
-
train(vectors) {
|
|
40
|
-
if (vectors.length === 0) {
|
|
41
|
-
throw new Error('Cannot train product quantizer with empty vector set');
|
|
42
|
-
}
|
|
43
|
-
this.dimension = vectors[0].length;
|
|
44
|
-
this.subvectorSize = Math.ceil(this.dimension / this.numSubvectors);
|
|
45
|
-
// Initialize centroids for each subvector
|
|
46
|
-
for (let i = 0; i < this.numSubvectors; i++) {
|
|
47
|
-
// Extract subvectors from training data
|
|
48
|
-
const subvectors = vectors.map((vector) => {
|
|
49
|
-
const start = i * this.subvectorSize;
|
|
50
|
-
const end = Math.min(start + this.subvectorSize, this.dimension);
|
|
51
|
-
return vector.slice(start, end);
|
|
52
|
-
});
|
|
53
|
-
// Initialize centroids for this subvector using k-means++
|
|
54
|
-
this.centroids[i] = this.kMeansPlusPlus(subvectors, this.numCentroids);
|
|
55
|
-
}
|
|
56
|
-
this.initialized = true;
|
|
57
|
-
}
|
|
58
|
-
/**
|
|
59
|
-
* Quantize a vector using product quantization
|
|
60
|
-
* @param vector Vector to quantize
|
|
61
|
-
* @returns Array of centroid indices, one for each subvector
|
|
62
|
-
*/
|
|
63
|
-
quantize(vector) {
|
|
64
|
-
if (!this.initialized) {
|
|
65
|
-
throw new Error('Product quantizer not initialized. Call train() first.');
|
|
66
|
-
}
|
|
67
|
-
if (vector.length !== this.dimension) {
|
|
68
|
-
throw new Error(`Vector dimension mismatch: expected ${this.dimension}, got ${vector.length}`);
|
|
69
|
-
}
|
|
70
|
-
const codes = [];
|
|
71
|
-
// Quantize each subvector
|
|
72
|
-
for (let i = 0; i < this.numSubvectors; i++) {
|
|
73
|
-
const start = i * this.subvectorSize;
|
|
74
|
-
const end = Math.min(start + this.subvectorSize, this.dimension);
|
|
75
|
-
const subvector = vector.slice(start, end);
|
|
76
|
-
// Find nearest centroid
|
|
77
|
-
let minDist = Number.MAX_VALUE;
|
|
78
|
-
let nearestCentroidIndex = 0;
|
|
79
|
-
for (let j = 0; j < this.centroids[i].length; j++) {
|
|
80
|
-
const centroid = this.centroids[i][j];
|
|
81
|
-
const dist = this.euclideanDistanceSquared(subvector, centroid);
|
|
82
|
-
if (dist < minDist) {
|
|
83
|
-
minDist = dist;
|
|
84
|
-
nearestCentroidIndex = j;
|
|
85
|
-
}
|
|
86
|
-
}
|
|
87
|
-
codes.push(nearestCentroidIndex);
|
|
88
|
-
}
|
|
89
|
-
return codes;
|
|
90
|
-
}
|
|
91
|
-
/**
|
|
92
|
-
* Reconstruct a vector from its quantized representation
|
|
93
|
-
* @param codes Array of centroid indices
|
|
94
|
-
* @returns Reconstructed vector
|
|
95
|
-
*/
|
|
96
|
-
reconstruct(codes) {
|
|
97
|
-
if (!this.initialized) {
|
|
98
|
-
throw new Error('Product quantizer not initialized. Call train() first.');
|
|
99
|
-
}
|
|
100
|
-
if (codes.length !== this.numSubvectors) {
|
|
101
|
-
throw new Error(`Code length mismatch: expected ${this.numSubvectors}, got ${codes.length}`);
|
|
102
|
-
}
|
|
103
|
-
const reconstructed = [];
|
|
104
|
-
// Reconstruct each subvector
|
|
105
|
-
for (let i = 0; i < this.numSubvectors; i++) {
|
|
106
|
-
const centroidIndex = codes[i];
|
|
107
|
-
const centroid = this.centroids[i][centroidIndex];
|
|
108
|
-
// Add centroid components to reconstructed vector
|
|
109
|
-
for (const component of centroid) {
|
|
110
|
-
reconstructed.push(component);
|
|
111
|
-
}
|
|
112
|
-
}
|
|
113
|
-
// Trim to original dimension if needed
|
|
114
|
-
return reconstructed.slice(0, this.dimension);
|
|
115
|
-
}
|
|
116
|
-
/**
|
|
117
|
-
* Compute squared Euclidean distance between two vectors
|
|
118
|
-
* @param a First vector
|
|
119
|
-
* @param b Second vector
|
|
120
|
-
* @returns Squared Euclidean distance
|
|
121
|
-
*/
|
|
122
|
-
euclideanDistanceSquared(a, b) {
|
|
123
|
-
let sum = 0;
|
|
124
|
-
const length = Math.min(a.length, b.length);
|
|
125
|
-
for (let i = 0; i < length; i++) {
|
|
126
|
-
const diff = a[i] - b[i];
|
|
127
|
-
sum += diff * diff;
|
|
128
|
-
}
|
|
129
|
-
return sum;
|
|
130
|
-
}
|
|
131
|
-
/**
|
|
132
|
-
* Implement k-means++ algorithm to initialize centroids
|
|
133
|
-
* @param vectors Vectors to cluster
|
|
134
|
-
* @param k Number of clusters
|
|
135
|
-
* @returns Array of centroids
|
|
136
|
-
*/
|
|
137
|
-
kMeansPlusPlus(vectors, k) {
|
|
138
|
-
if (vectors.length < k) {
|
|
139
|
-
// If we have fewer vectors than centroids, use the vectors as centroids
|
|
140
|
-
return [...vectors];
|
|
141
|
-
}
|
|
142
|
-
const centroids = [];
|
|
143
|
-
// Choose first centroid randomly
|
|
144
|
-
const firstIndex = Math.floor(Math.random() * vectors.length);
|
|
145
|
-
centroids.push([...vectors[firstIndex]]);
|
|
146
|
-
// Choose remaining centroids
|
|
147
|
-
for (let i = 1; i < k; i++) {
|
|
148
|
-
// Compute distances to nearest centroid for each vector
|
|
149
|
-
const distances = vectors.map((vector) => {
|
|
150
|
-
let minDist = Number.MAX_VALUE;
|
|
151
|
-
for (const centroid of centroids) {
|
|
152
|
-
const dist = this.euclideanDistanceSquared(vector, centroid);
|
|
153
|
-
minDist = Math.min(minDist, dist);
|
|
154
|
-
}
|
|
155
|
-
return minDist;
|
|
156
|
-
});
|
|
157
|
-
// Compute sum of distances
|
|
158
|
-
const distSum = distances.reduce((sum, dist) => sum + dist, 0);
|
|
159
|
-
// Choose next centroid with probability proportional to distance
|
|
160
|
-
let r = Math.random() * distSum;
|
|
161
|
-
let nextIndex = 0;
|
|
162
|
-
for (let j = 0; j < distances.length; j++) {
|
|
163
|
-
r -= distances[j];
|
|
164
|
-
if (r <= 0) {
|
|
165
|
-
nextIndex = j;
|
|
166
|
-
break;
|
|
167
|
-
}
|
|
168
|
-
}
|
|
169
|
-
centroids.push([...vectors[nextIndex]]);
|
|
170
|
-
}
|
|
171
|
-
return centroids;
|
|
172
|
-
}
|
|
173
|
-
/**
|
|
174
|
-
* Get the centroids for each subvector
|
|
175
|
-
* @returns Array of centroid arrays
|
|
176
|
-
*/
|
|
177
|
-
getCentroids() {
|
|
178
|
-
return this.centroids;
|
|
179
|
-
}
|
|
180
|
-
/**
|
|
181
|
-
* Set the centroids for each subvector
|
|
182
|
-
* @param centroids Array of centroid arrays
|
|
183
|
-
*/
|
|
184
|
-
setCentroids(centroids) {
|
|
185
|
-
this.centroids = centroids;
|
|
186
|
-
this.numSubvectors = centroids.length;
|
|
187
|
-
this.numCentroids = centroids[0].length;
|
|
188
|
-
this.initialized = true;
|
|
189
|
-
}
|
|
190
|
-
/**
|
|
191
|
-
* Get the dimension of the vectors
|
|
192
|
-
* @returns Dimension
|
|
193
|
-
*/
|
|
194
|
-
getDimension() {
|
|
195
|
-
return this.dimension;
|
|
196
|
-
}
|
|
197
|
-
/**
|
|
198
|
-
* Set the dimension of the vectors
|
|
199
|
-
* @param dimension Dimension
|
|
200
|
-
*/
|
|
201
|
-
setDimension(dimension) {
|
|
202
|
-
this.dimension = dimension;
|
|
203
|
-
this.subvectorSize = Math.ceil(dimension / this.numSubvectors);
|
|
204
|
-
}
|
|
205
|
-
}
|
|
206
|
-
/**
|
|
207
|
-
* Optimized HNSW Index implementation
|
|
208
|
-
* Extends the base HNSW implementation with support for large datasets
|
|
209
|
-
* Uses product quantization for dimensionality reduction and disk-based storage when needed
|
|
210
|
-
*/
|
|
211
|
-
export class HNSWIndexOptimized extends HNSWIndex {
|
|
212
|
-
constructor(config = {}, distanceFunction, storage = null) {
|
|
213
|
-
// Initialize base HNSW index with standard config
|
|
214
|
-
super(config, distanceFunction);
|
|
215
|
-
this.productQuantizer = null;
|
|
216
|
-
this.storage = null;
|
|
217
|
-
this.useDiskBasedIndex = false;
|
|
218
|
-
this.useProductQuantization = false;
|
|
219
|
-
this.quantizedVectors = new Map();
|
|
220
|
-
this.memoryUsage = 0;
|
|
221
|
-
this.vectorCount = 0;
|
|
222
|
-
// Set optimized config
|
|
223
|
-
this.optimizedConfig = { ...DEFAULT_OPTIMIZED_CONFIG, ...config };
|
|
224
|
-
// Set storage adapter
|
|
225
|
-
this.storage = storage;
|
|
226
|
-
// Initialize product quantizer if enabled
|
|
227
|
-
if (this.optimizedConfig.productQuantization?.enabled) {
|
|
228
|
-
this.useProductQuantization = true;
|
|
229
|
-
this.productQuantizer = new ProductQuantizer(this.optimizedConfig.productQuantization.numSubvectors, this.optimizedConfig.productQuantization.numCentroids);
|
|
230
|
-
}
|
|
231
|
-
// Set disk-based index flag
|
|
232
|
-
this.useDiskBasedIndex = this.optimizedConfig.useDiskBasedIndex || false;
|
|
233
|
-
}
|
|
234
|
-
/**
|
|
235
|
-
* Add a vector to the index
|
|
236
|
-
* Uses product quantization if enabled and memory threshold is exceeded
|
|
237
|
-
*/
|
|
238
|
-
async addItem(item) {
|
|
239
|
-
// Check if item is defined
|
|
240
|
-
if (!item) {
|
|
241
|
-
throw new Error('Item is undefined or null');
|
|
242
|
-
}
|
|
243
|
-
const { id, vector } = item;
|
|
244
|
-
// Check if vector is defined
|
|
245
|
-
if (!vector) {
|
|
246
|
-
throw new Error('Vector is undefined or null');
|
|
247
|
-
}
|
|
248
|
-
// Estimate memory usage for this vector
|
|
249
|
-
const vectorMemory = vector.length * 8; // 8 bytes per number (Float64)
|
|
250
|
-
const connectionsMemory = this.optimizedConfig.M * this.optimizedConfig.ml * 16; // Estimate for connections
|
|
251
|
-
const totalMemory = vectorMemory + connectionsMemory;
|
|
252
|
-
// Update memory usage estimate
|
|
253
|
-
this.memoryUsage += totalMemory;
|
|
254
|
-
this.vectorCount++;
|
|
255
|
-
// Check if we should switch to product quantization
|
|
256
|
-
if (this.useProductQuantization &&
|
|
257
|
-
this.memoryUsage > this.optimizedConfig.memoryThreshold &&
|
|
258
|
-
this.productQuantizer &&
|
|
259
|
-
!this.productQuantizer.getDimension()) {
|
|
260
|
-
// Initialize product quantizer with existing vectors
|
|
261
|
-
this.initializeProductQuantizer();
|
|
262
|
-
}
|
|
263
|
-
// If product quantization is active, quantize the vector
|
|
264
|
-
if (this.useProductQuantization &&
|
|
265
|
-
this.productQuantizer &&
|
|
266
|
-
this.productQuantizer.getDimension() > 0) {
|
|
267
|
-
// Quantize the vector
|
|
268
|
-
const codes = this.productQuantizer.quantize(vector);
|
|
269
|
-
// Store the quantized vector
|
|
270
|
-
this.quantizedVectors.set(id, codes);
|
|
271
|
-
// Reconstruct the vector for indexing
|
|
272
|
-
const reconstructedVector = this.productQuantizer.reconstruct(codes);
|
|
273
|
-
// Add the reconstructed vector to the index
|
|
274
|
-
return await super.addItem({ id, vector: reconstructedVector });
|
|
275
|
-
}
|
|
276
|
-
// If disk-based index is active and storage is available, store the vector
|
|
277
|
-
if (this.useDiskBasedIndex && this.storage) {
|
|
278
|
-
// Create a noun object
|
|
279
|
-
const noun = {
|
|
280
|
-
id,
|
|
281
|
-
vector,
|
|
282
|
-
connections: new Map()
|
|
283
|
-
};
|
|
284
|
-
// Store the noun
|
|
285
|
-
this.storage.saveNoun(noun).catch((error) => {
|
|
286
|
-
console.error(`Failed to save noun ${id} to storage:`, error);
|
|
287
|
-
});
|
|
288
|
-
}
|
|
289
|
-
// Add the vector to the in-memory index
|
|
290
|
-
return await super.addItem(item);
|
|
291
|
-
}
|
|
292
|
-
/**
|
|
293
|
-
* Search for nearest neighbors
|
|
294
|
-
* Uses product quantization if enabled
|
|
295
|
-
*/
|
|
296
|
-
async search(queryVector, k = 10) {
|
|
297
|
-
// Check if query vector is defined
|
|
298
|
-
if (!queryVector) {
|
|
299
|
-
throw new Error('Query vector is undefined or null');
|
|
300
|
-
}
|
|
301
|
-
// If product quantization is active, quantize the query vector
|
|
302
|
-
if (this.useProductQuantization &&
|
|
303
|
-
this.productQuantizer &&
|
|
304
|
-
this.productQuantizer.getDimension() > 0) {
|
|
305
|
-
// Quantize the query vector
|
|
306
|
-
const codes = this.productQuantizer.quantize(queryVector);
|
|
307
|
-
// Reconstruct the query vector
|
|
308
|
-
const reconstructedVector = this.productQuantizer.reconstruct(codes);
|
|
309
|
-
// Search with the reconstructed vector
|
|
310
|
-
return await super.search(reconstructedVector, k);
|
|
311
|
-
}
|
|
312
|
-
// Otherwise, use the standard search
|
|
313
|
-
return await super.search(queryVector, k);
|
|
314
|
-
}
|
|
315
|
-
/**
|
|
316
|
-
* Remove an item from the index
|
|
317
|
-
*/
|
|
318
|
-
removeItem(id) {
|
|
319
|
-
// If product quantization is active, remove the quantized vector
|
|
320
|
-
if (this.useProductQuantization) {
|
|
321
|
-
this.quantizedVectors.delete(id);
|
|
322
|
-
}
|
|
323
|
-
// If disk-based index is active and storage is available, remove the vector from storage
|
|
324
|
-
if (this.useDiskBasedIndex && this.storage) {
|
|
325
|
-
this.storage.deleteNoun(id).catch((error) => {
|
|
326
|
-
console.error(`Failed to delete noun ${id} from storage:`, error);
|
|
327
|
-
});
|
|
328
|
-
}
|
|
329
|
-
// Update memory usage estimate
|
|
330
|
-
if (this.vectorCount > 0) {
|
|
331
|
-
this.memoryUsage = Math.max(0, this.memoryUsage - this.memoryUsage / this.vectorCount);
|
|
332
|
-
this.vectorCount--;
|
|
333
|
-
}
|
|
334
|
-
// Remove the item from the in-memory index
|
|
335
|
-
return super.removeItem(id);
|
|
336
|
-
}
|
|
337
|
-
/**
|
|
338
|
-
* Clear the index
|
|
339
|
-
*/
|
|
340
|
-
clear() {
|
|
341
|
-
// Clear product quantization data
|
|
342
|
-
if (this.useProductQuantization) {
|
|
343
|
-
this.quantizedVectors.clear();
|
|
344
|
-
this.productQuantizer = new ProductQuantizer(this.optimizedConfig.productQuantization.numSubvectors, this.optimizedConfig.productQuantization.numCentroids);
|
|
345
|
-
}
|
|
346
|
-
// Reset memory usage
|
|
347
|
-
this.memoryUsage = 0;
|
|
348
|
-
this.vectorCount = 0;
|
|
349
|
-
// Clear the in-memory index
|
|
350
|
-
super.clear();
|
|
351
|
-
}
|
|
352
|
-
/**
|
|
353
|
-
* Initialize product quantizer with existing vectors
|
|
354
|
-
*/
|
|
355
|
-
initializeProductQuantizer() {
|
|
356
|
-
if (!this.productQuantizer) {
|
|
357
|
-
return;
|
|
358
|
-
}
|
|
359
|
-
// Get all vectors from the index
|
|
360
|
-
const nouns = super.getNouns();
|
|
361
|
-
const vectors = [];
|
|
362
|
-
// Extract vectors
|
|
363
|
-
for (const [_, noun] of nouns) {
|
|
364
|
-
vectors.push(noun.vector);
|
|
365
|
-
}
|
|
366
|
-
// Train the product quantizer
|
|
367
|
-
if (vectors.length > 0) {
|
|
368
|
-
this.productQuantizer.train(vectors);
|
|
369
|
-
// Quantize all existing vectors
|
|
370
|
-
for (const [id, noun] of nouns) {
|
|
371
|
-
const codes = this.productQuantizer.quantize(noun.vector);
|
|
372
|
-
this.quantizedVectors.set(id, codes);
|
|
373
|
-
}
|
|
374
|
-
console.log(`Initialized product quantizer with ${vectors.length} vectors`);
|
|
375
|
-
}
|
|
376
|
-
}
|
|
377
|
-
/**
|
|
378
|
-
* Get the product quantizer
|
|
379
|
-
* @returns Product quantizer or null if not enabled
|
|
380
|
-
*/
|
|
381
|
-
getProductQuantizer() {
|
|
382
|
-
return this.productQuantizer;
|
|
383
|
-
}
|
|
384
|
-
/**
|
|
385
|
-
* Get the optimized configuration
|
|
386
|
-
* @returns Optimized configuration
|
|
387
|
-
*/
|
|
388
|
-
getOptimizedConfig() {
|
|
389
|
-
return { ...this.optimizedConfig };
|
|
390
|
-
}
|
|
391
|
-
/**
|
|
392
|
-
* Get the estimated memory usage
|
|
393
|
-
* @returns Estimated memory usage in bytes
|
|
394
|
-
*/
|
|
395
|
-
getMemoryUsage() {
|
|
396
|
-
return this.memoryUsage;
|
|
397
|
-
}
|
|
398
|
-
/**
|
|
399
|
-
* Set the storage adapter
|
|
400
|
-
* @param storage Storage adapter
|
|
401
|
-
*/
|
|
402
|
-
setStorage(storage) {
|
|
403
|
-
this.storage = storage;
|
|
404
|
-
}
|
|
405
|
-
/**
|
|
406
|
-
* Get the storage adapter
|
|
407
|
-
* @returns Storage adapter or null if not set
|
|
408
|
-
*/
|
|
409
|
-
getStorage() {
|
|
410
|
-
return this.storage;
|
|
411
|
-
}
|
|
412
|
-
/**
|
|
413
|
-
* Set whether to use disk-based index
|
|
414
|
-
* @param useDiskBasedIndex Whether to use disk-based index
|
|
415
|
-
*/
|
|
416
|
-
setUseDiskBasedIndex(useDiskBasedIndex) {
|
|
417
|
-
this.useDiskBasedIndex = useDiskBasedIndex;
|
|
418
|
-
}
|
|
419
|
-
/**
|
|
420
|
-
* Get whether disk-based index is used
|
|
421
|
-
* @returns Whether disk-based index is used
|
|
422
|
-
*/
|
|
423
|
-
getUseDiskBasedIndex() {
|
|
424
|
-
return this.useDiskBasedIndex;
|
|
425
|
-
}
|
|
426
|
-
/**
|
|
427
|
-
* Set whether to use product quantization
|
|
428
|
-
* @param useProductQuantization Whether to use product quantization
|
|
429
|
-
*/
|
|
430
|
-
setUseProductQuantization(useProductQuantization) {
|
|
431
|
-
this.useProductQuantization = useProductQuantization;
|
|
432
|
-
}
|
|
433
|
-
/**
|
|
434
|
-
* Get whether product quantization is used
|
|
435
|
-
* @returns Whether product quantization is used
|
|
436
|
-
*/
|
|
437
|
-
getUseProductQuantization() {
|
|
438
|
-
return this.useProductQuantization;
|
|
439
|
-
}
|
|
440
|
-
}
|
|
441
|
-
//# sourceMappingURL=hnswIndexOptimized.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"hnswIndexOptimized.js","sourceRoot":"","sources":["../../src/hnsw/hnswIndexOptimized.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AASH,OAAO,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAA;AAsB1C,qDAAqD;AACrD,MAAM,wBAAwB,GAAwB;IACpD,CAAC,EAAE,EAAE;IACL,cAAc,EAAE,GAAG;IACnB,QAAQ,EAAE,EAAE;IACZ,EAAE,EAAE,EAAE;IACN,eAAe,EAAE,IAAI,GAAG,IAAI,GAAG,IAAI,EAAE,wBAAwB;IAC7D,mBAAmB,EAAE;QACnB,OAAO,EAAE,KAAK;QACd,aAAa,EAAE,EAAE;QACjB,YAAY,EAAE,GAAG;KAClB;IACD,iBAAiB,EAAE,KAAK;CACzB,CAAA;AAED;;;;GAIG;AACH,MAAM,gBAAgB;IAQpB,YAAY,gBAAwB,EAAE,EAAE,eAAuB,GAAG;QAL1D,cAAS,GAAe,EAAE,CAAA;QAC1B,kBAAa,GAAW,CAAC,CAAA;QACzB,gBAAW,GAAY,KAAK,CAAA;QAC5B,cAAS,GAAW,CAAC,CAAA;QAG3B,IAAI,CAAC,aAAa,GAAG,aAAa,CAAA;QAClC,IAAI,CAAC,YAAY,GAAG,YAAY,CAAA;IAClC,CAAC;IAED;;;OAGG;IACI,KAAK,CAAC,OAAiB;QAC5B,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACzB,MAAM,IAAI,KAAK,CAAC,sDAAsD,CAAC,CAAA;QACzE,CAAC;QAED,IAAI,CAAC,SAAS,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC,MAAM,CAAA;QAClC,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,aAAa,CAAC,CAAA;QAEnE,0CAA0C;QAC1C,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,aAAa,EAAE,CAAC,EAAE,EAAE,CAAC;YAC5C,wCAAwC;YACxC,MAAM,UAAU,GAAa,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE;gBAClD,MAAM,KAAK,GAAG,CAAC,GAAG,IAAI,CAAC,aAAa,CAAA;gBACpC,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,GAAG,IAAI,CAAC,aAAa,EAAE,IAAI,CAAC,SAAS,CAAC,CAAA;gBAChE,OAAO,MAAM,CAAC,KAAK,CAAC,KAAK,EAAE,GAAG,CAAC,CAAA;YACjC,CAAC,CAAC,CAAA;YAEF,0DAA0D;YAC1D,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,cAAc,CAAC,UAAU,EAAE,IAAI,CAAC,YAAY,CAAC,CAAA;QACxE,CAAC;QAED,IAAI,CAAC,WAAW,GAAG,IAAI,CAAA;IACzB,CAAC;IAED;;;;OAIG;IACI,QAAQ,CAAC,MAAc;QAC5B,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC;YACtB,MAAM,IAAI,KAAK,CAAC,wDAAwD,CAAC,CAAA;QAC3E,CAAC;QAED,IAAI,MAAM,CAAC,MAAM,KAAK,IAAI,CAAC,SAAS,EAAE,CAAC;YACrC,MAAM,IAAI,KAAK,CACb,uCAAuC,IAAI,CAAC,SAAS,SAAS,MAAM,CAAC,MAAM,EAAE,CAC9E,CAAA;QACH,CAAC;QAED,MAAM,KAAK,GAAa,EAAE,CAAA;QAE1B,0BAA0B;QAC1B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,aAAa,EAAE,CAAC,EAAE,EAAE,CAAC;YAC5C,MAAM,KAAK,GAAG,CAAC,GAAG,IAAI,CAAC,aAAa,CAAA;YACpC,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,GAAG,IAAI,CAAC,aAAa,EAAE,IAAI,CAAC,SAAS,CAAC,CAAA;YAChE,MAAM,SAAS,GAAG,MAAM,CAAC,KAAK,CAAC,KAAK,EAAE,GAAG,CAAC,CAAA;YAE1C,wBAAwB;YACxB,IAAI,OAAO,GAAG,MAAM,CAAC,SAAS,CAAA;YAC9B,IAAI,oBAAoB,GAAG,CAAC,CAAA;YAE5B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;gBAClD,MAAM,QAAQ,GAAG,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAA;gBACrC,MAAM,IAAI,GAAG,IAAI,CAAC,wBAAwB,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAA;gBAE/D,IAAI,IAAI,GAAG,OAAO,EAAE,CAAC;oBACnB,OAAO,GAAG,IAAI,CAAA;oBACd,oBAAoB,GAAG,CAAC,CAAA;gBAC1B,CAAC;YACH,CAAC;YAED,KAAK,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAA;QAClC,CAAC;QAED,OAAO,KAAK,CAAA;IACd,CAAC;IAED;;;;OAIG;IACI,WAAW,CAAC,KAAe;QAChC,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC;YACtB,MAAM,IAAI,KAAK,CAAC,wDAAwD,CAAC,CAAA;QAC3E,CAAC;QAED,IAAI,KAAK,CAAC,MAAM,KAAK,IAAI,CAAC,aAAa,EAAE,CAAC;YACxC,MAAM,IAAI,KAAK,CACb,kCAAkC,IAAI,CAAC,aAAa,SAAS,KAAK,CAAC,MAAM,EAAE,CAC5E,CAAA;QACH,CAAC;QAED,MAAM,aAAa,GAAW,EAAE,CAAA;QAEhC,6BAA6B;QAC7B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,aAAa,EAAE,CAAC,EAAE,EAAE,CAAC;YAC5C,MAAM,aAAa,GAAG,KAAK,CAAC,CAAC,CAAC,CAAA;YAC9B,MAAM,QAAQ,GAAG,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,aAAa,CAAC,CAAA;YAEjD,kDAAkD;YAClD,KAAK,MAAM,SAAS,IAAI,QAAQ,EAAE,CAAC;gBACjC,aAAa,CAAC,IAAI,CAAC,SAAS,CAAC,CAAA;YAC/B,CAAC;QACH,CAAC;QAED,uCAAuC;QACvC,OAAO,aAAa,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,SAAS,CAAC,CAAA;IAC/C,CAAC;IAED;;;;;OAKG;IACK,wBAAwB,CAAC,CAAS,EAAE,CAAS;QACnD,IAAI,GAAG,GAAG,CAAC,CAAA;QACX,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,MAAM,CAAC,CAAA;QAE3C,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YAChC,MAAM,IAAI,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAA;YACxB,GAAG,IAAI,IAAI,GAAG,IAAI,CAAA;QACpB,CAAC;QAED,OAAO,GAAG,CAAA;IACZ,CAAC;IAED;;;;;OAKG;IACK,cAAc,CAAC,OAAiB,EAAE,CAAS;QACjD,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACvB,wEAAwE;YACxE,OAAO,CAAC,GAAG,OAAO,CAAC,CAAA;QACrB,CAAC;QAED,MAAM,SAAS,GAAa,EAAE,CAAA;QAE9B,iCAAiC;QACjC,MAAM,UAAU,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,OAAO,CAAC,MAAM,CAAC,CAAA;QAC7D,SAAS,CAAC,IAAI,CAAC,CAAC,GAAG,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC,CAAA;QAExC,6BAA6B;QAC7B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;YAC3B,wDAAwD;YACxD,MAAM,SAAS,GAAa,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE;gBACjD,IAAI,OAAO,GAAG,MAAM,CAAC,SAAS,CAAA;gBAE9B,KAAK,MAAM,QAAQ,IAAI,SAAS,EAAE,CAAC;oBACjC,MAAM,IAAI,GAAG,IAAI,CAAC,wBAAwB,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAA;oBAC5D,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,OAAO,EAAE,IAAI,CAAC,CAAA;gBACnC,CAAC;gBAED,OAAO,OAAO,CAAA;YAChB,CAAC,CAAC,CAAA;YAEF,2BAA2B;YAC3B,MAAM,OAAO,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE,CAAC,GAAG,GAAG,IAAI,EAAE,CAAC,CAAC,CAAA;YAE9D,iEAAiE;YACjE,IAAI,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,GAAG,OAAO,CAAA;YAC/B,IAAI,SAAS,GAAG,CAAC,CAAA;YAEjB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,SAAS,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;gBAC1C,CAAC,IAAI,SAAS,CAAC,CAAC,CAAC,CAAA;gBACjB,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;oBACX,SAAS,GAAG,CAAC,CAAA;oBACb,MAAK;gBACP,CAAC;YACH,CAAC;YAED,SAAS,CAAC,IAAI,CAAC,CAAC,GAAG,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC,CAAA;QACzC,CAAC;QAED,OAAO,SAAS,CAAA;IAClB,CAAC;IAED;;;OAGG;IACI,YAAY;QACjB,OAAO,IAAI,CAAC,SAAS,CAAA;IACvB,CAAC;IAED;;;OAGG;IACI,YAAY,CAAC,SAAqB;QACvC,IAAI,CAAC,SAAS,GAAG,SAAS,CAAA;QAC1B,IAAI,CAAC,aAAa,GAAG,SAAS,CAAC,MAAM,CAAA;QACrC,IAAI,CAAC,YAAY,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC,MAAM,CAAA;QACvC,IAAI,CAAC,WAAW,GAAG,IAAI,CAAA;IACzB,CAAC;IAED;;;OAGG;IACI,YAAY;QACjB,OAAO,IAAI,CAAC,SAAS,CAAA;IACvB,CAAC;IAED;;;OAGG;IACI,YAAY,CAAC,SAAiB;QACnC,IAAI,CAAC,SAAS,GAAG,SAAS,CAAA;QAC1B,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,aAAa,CAAC,CAAA;IAChE,CAAC;CACF;AAED;;;;GAIG;AACH,MAAM,OAAO,kBAAmB,SAAQ,SAAS;IAU/C,YACE,SAAuC,EAAE,EACzC,gBAAkC,EAClC,UAAiC,IAAI;QAErC,kDAAkD;QAClD,KAAK,CAAC,MAAM,EAAE,gBAAgB,CAAC,CAAA;QAdzB,qBAAgB,GAA4B,IAAI,CAAA;QAChD,YAAO,GAA0B,IAAI,CAAA;QACrC,sBAAiB,GAAY,KAAK,CAAA;QAClC,2BAAsB,GAAY,KAAK,CAAA;QACvC,qBAAgB,GAA0B,IAAI,GAAG,EAAE,CAAA;QACnD,gBAAW,GAAW,CAAC,CAAA;QACvB,gBAAW,GAAW,CAAC,CAAA;QAU7B,uBAAuB;QACvB,IAAI,CAAC,eAAe,GAAG,EAAE,GAAG,wBAAwB,EAAE,GAAG,MAAM,EAAE,CAAA;QAEjE,sBAAsB;QACtB,IAAI,CAAC,OAAO,GAAG,OAAO,CAAA;QAEtB,0CAA0C;QAC1C,IAAI,IAAI,CAAC,eAAe,CAAC,mBAAmB,EAAE,OAAO,EAAE,CAAC;YACtD,IAAI,CAAC,sBAAsB,GAAG,IAAI,CAAA;YAClC,IAAI,CAAC,gBAAgB,GAAG,IAAI,gBAAgB,CAC1C,IAAI,CAAC,eAAe,CAAC,mBAAmB,CAAC,aAAa,EACtD,IAAI,CAAC,eAAe,CAAC,mBAAmB,CAAC,YAAY,CACtD,CAAA;QACH,CAAC;QAED,4BAA4B;QAC5B,IAAI,CAAC,iBAAiB,GAAG,IAAI,CAAC,eAAe,CAAC,iBAAiB,IAAI,KAAK,CAAA;IAC1E,CAAC;IAED;;;OAGG;IACa,KAAK,CAAC,OAAO,CAAC,IAAoB;QAChD,2BAA2B;QAC3B,IAAI,CAAC,IAAI,EAAE,CAAC;YACV,MAAM,IAAI,KAAK,CAAC,2BAA2B,CAAC,CAAA;QAC9C,CAAC;QAED,MAAM,EAAE,EAAE,EAAE,MAAM,EAAE,GAAG,IAAI,CAAA;QAE3B,6BAA6B;QAC7B,IAAI,CAAC,MAAM,EAAE,CAAC;YACZ,MAAM,IAAI,KAAK,CAAC,6BAA6B,CAAC,CAAA;QAChD,CAAC;QAED,wCAAwC;QACxC,MAAM,YAAY,GAAG,MAAM,CAAC,MAAM,GAAG,CAAC,CAAA,CAAC,+BAA+B;QACtE,MAAM,iBAAiB,GAAG,IAAI,CAAC,eAAe,CAAC,CAAC,GAAG,IAAI,CAAC,eAAe,CAAC,EAAE,GAAG,EAAE,CAAA,CAAC,2BAA2B;QAC3G,MAAM,WAAW,GAAG,YAAY,GAAG,iBAAiB,CAAA;QAEpD,+BAA+B;QAC/B,IAAI,CAAC,WAAW,IAAI,WAAW,CAAA;QAC/B,IAAI,CAAC,WAAW,EAAE,CAAA;QAElB,oDAAoD;QACpD,IACE,IAAI,CAAC,sBAAsB;YAC3B,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,eAAe,CAAC,eAAgB;YACxD,IAAI,CAAC,gBAAgB;YACrB,CAAC,IAAI,CAAC,gBAAgB,CAAC,YAAY,EAAE,EACrC,CAAC;YACD,qDAAqD;YACrD,IAAI,CAAC,0BAA0B,EAAE,CAAA;QACnC,CAAC;QAED,yDAAyD;QACzD,IACE,IAAI,CAAC,sBAAsB;YAC3B,IAAI,CAAC,gBAAgB;YACrB,IAAI,CAAC,gBAAgB,CAAC,YAAY,EAAE,GAAG,CAAC,EACxC,CAAC;YACD,sBAAsB;YACtB,MAAM,KAAK,GAAG,IAAI,CAAC,gBAAgB,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAA;YAEpD,6BAA6B;YAC7B,IAAI,CAAC,gBAAgB,CAAC,GAAG,CAAC,EAAE,EAAE,KAAK,CAAC,CAAA;YAEpC,sCAAsC;YACtC,MAAM,mBAAmB,GAAG,IAAI,CAAC,gBAAgB,CAAC,WAAW,CAAC,KAAK,CAAC,CAAA;YAEpE,4CAA4C;YAC5C,OAAO,MAAM,KAAK,CAAC,OAAO,CAAC,EAAE,EAAE,EAAE,MAAM,EAAE,mBAAmB,EAAE,CAAC,CAAA;QACjE,CAAC;QAED,2EAA2E;QAC3E,IAAI,IAAI,CAAC,iBAAiB,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;YAC3C,uBAAuB;YACvB,MAAM,IAAI,GAAa;gBACrB,EAAE;gBACF,MAAM;gBACN,WAAW,EAAE,IAAI,GAAG,EAAE;aACvB,CAAA;YAED,iBAAiB;YACjB,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE;gBAC1C,OAAO,CAAC,KAAK,CAAC,uBAAuB,EAAE,cAAc,EAAE,KAAK,CAAC,CAAA;YAC/D,CAAC,CAAC,CAAA;QACJ,CAAC;QAED,wCAAwC;QACxC,OAAO,MAAM,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,CAAA;IAClC,CAAC;IAED;;;OAGG;IACa,KAAK,CAAC,MAAM,CAC1B,WAAmB,EACnB,IAAY,EAAE;QAEd,mCAAmC;QACnC,IAAI,CAAC,WAAW,EAAE,CAAC;YACjB,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAA;QACtD,CAAC;QAED,+DAA+D;QAC/D,IACE,IAAI,CAAC,sBAAsB;YAC3B,IAAI,CAAC,gBAAgB;YACrB,IAAI,CAAC,gBAAgB,CAAC,YAAY,EAAE,GAAG,CAAC,EACxC,CAAC;YACD,4BAA4B;YAC5B,MAAM,KAAK,GAAG,IAAI,CAAC,gBAAgB,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAA;YAEzD,+BAA+B;YAC/B,MAAM,mBAAmB,GAAG,IAAI,CAAC,gBAAgB,CAAC,WAAW,CAAC,KAAK,CAAC,CAAA;YAEpE,uCAAuC;YACvC,OAAO,MAAM,KAAK,CAAC,MAAM,CAAC,mBAAmB,EAAE,CAAC,CAAC,CAAA;QACnD,CAAC;QAED,qCAAqC;QACrC,OAAO,MAAM,KAAK,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC,CAAC,CAAA;IAC3C,CAAC;IAED;;OAEG;IACa,UAAU,CAAC,EAAU;QACnC,iEAAiE;QACjE,IAAI,IAAI,CAAC,sBAAsB,EAAE,CAAC;YAChC,IAAI,CAAC,gBAAgB,CAAC,MAAM,CAAC,EAAE,CAAC,CAAA;QAClC,CAAC;QAED,yFAAyF;QACzF,IAAI,IAAI,CAAC,iBAAiB,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;YAC3C,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE;gBAC1C,OAAO,CAAC,KAAK,CAAC,yBAAyB,EAAE,gBAAgB,EAAE,KAAK,CAAC,CAAA;YACnE,CAAC,CAAC,CAAA;QACJ,CAAC;QAED,+BAA+B;QAC/B,IAAI,IAAI,CAAC,WAAW,GAAG,CAAC,EAAE,CAAC;YACzB,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,GAAG,CACzB,CAAC,EACD,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,WAAW,CACvD,CAAA;YACD,IAAI,CAAC,WAAW,EAAE,CAAA;QACpB,CAAC;QAED,2CAA2C;QAC3C,OAAO,KAAK,CAAC,UAAU,CAAC,EAAE,CAAC,CAAA;IAC7B,CAAC;IAED;;OAEG;IACa,KAAK;QACnB,kCAAkC;QAClC,IAAI,IAAI,CAAC,sBAAsB,EAAE,CAAC;YAChC,IAAI,CAAC,gBAAgB,CAAC,KAAK,EAAE,CAAA;YAC7B,IAAI,CAAC,gBAAgB,GAAG,IAAI,gBAAgB,CAC1C,IAAI,CAAC,eAAe,CAAC,mBAAoB,CAAC,aAAa,EACvD,IAAI,CAAC,eAAe,CAAC,mBAAoB,CAAC,YAAY,CACvD,CAAA;QACH,CAAC;QAED,qBAAqB;QACrB,IAAI,CAAC,WAAW,GAAG,CAAC,CAAA;QACpB,IAAI,CAAC,WAAW,GAAG,CAAC,CAAA;QAEpB,4BAA4B;QAC5B,KAAK,CAAC,KAAK,EAAE,CAAA;IACf,CAAC;IAED;;OAEG;IACK,0BAA0B;QAChC,IAAI,CAAC,IAAI,CAAC,gBAAgB,EAAE,CAAC;YAC3B,OAAM;QACR,CAAC;QAED,iCAAiC;QACjC,MAAM,KAAK,GAAG,KAAK,CAAC,QAAQ,EAAE,CAAA;QAC9B,MAAM,OAAO,GAAa,EAAE,CAAA;QAE5B,kBAAkB;QAClB,KAAK,MAAM,CAAC,CAAC,EAAE,IAAI,CAAC,IAAI,KAAK,EAAE,CAAC;YAC9B,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,CAAA;QAC3B,CAAC;QAED,8BAA8B;QAC9B,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACvB,IAAI,CAAC,gBAAgB,CAAC,KAAK,CAAC,OAAO,CAAC,CAAA;YAEpC,gCAAgC;YAChC,KAAK,MAAM,CAAC,EAAE,EAAE,IAAI,CAAC,IAAI,KAAK,EAAE,CAAC;gBAC/B,MAAM,KAAK,GAAG,IAAI,CAAC,gBAAgB,CAAC,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,CAAA;gBACzD,IAAI,CAAC,gBAAgB,CAAC,GAAG,CAAC,EAAE,EAAE,KAAK,CAAC,CAAA;YACtC,CAAC;YAED,OAAO,CAAC,GAAG,CACT,sCAAsC,OAAO,CAAC,MAAM,UAAU,CAC/D,CAAA;QACH,CAAC;IACH,CAAC;IAED;;;OAGG;IACI,mBAAmB;QACxB,OAAO,IAAI,CAAC,gBAAgB,CAAA;IAC9B,CAAC;IAED;;;OAGG;IACI,kBAAkB;QACvB,OAAO,EAAE,GAAG,IAAI,CAAC,eAAe,EAAE,CAAA;IACpC,CAAC;IAED;;;OAGG;IACI,cAAc;QACnB,OAAO,IAAI,CAAC,WAAW,CAAA;IACzB,CAAC;IAED;;;OAGG;IACI,UAAU,CAAC,OAAuB;QACvC,IAAI,CAAC,OAAO,GAAG,OAAO,CAAA;IACxB,CAAC;IAED;;;OAGG;IACI,UAAU;QACf,OAAO,IAAI,CAAC,OAAO,CAAA;IACrB,CAAC;IAED;;;OAGG;IACI,oBAAoB,CAAC,iBAA0B;QACpD,IAAI,CAAC,iBAAiB,GAAG,iBAAiB,CAAA;IAC5C,CAAC;IAED;;;OAGG;IACI,oBAAoB;QACzB,OAAO,IAAI,CAAC,iBAAiB,CAAA;IAC/B,CAAC;IAED;;;OAGG;IACI,yBAAyB,CAAC,sBAA+B;QAC9D,IAAI,CAAC,sBAAsB,GAAG,sBAAsB,CAAA;IACtD,CAAC;IAED;;;OAGG;IACI,yBAAyB;QAC9B,OAAO,IAAI,CAAC,sBAAsB,CAAA;IACpC,CAAC;CACF"}
|
|
@@ -1,142 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* BrainyMCPAdapter
|
|
3
|
-
*
|
|
4
|
-
* This class provides an adapter for accessing Brainy data through the Model Control Protocol (MCP).
|
|
5
|
-
* It wraps a BrainyData instance and exposes methods for getting vectors, searching similar items,
|
|
6
|
-
* and getting relationships.
|
|
7
|
-
*/
|
|
8
|
-
import { v4 as uuidv4 } from 'uuid';
|
|
9
|
-
import { MCP_VERSION } from '../types/mcpTypes.js';
|
|
10
|
-
export class BrainyMCPAdapter {
|
|
11
|
-
/**
|
|
12
|
-
* Creates a new BrainyMCPAdapter
|
|
13
|
-
* @param brainyData The BrainyData instance to wrap
|
|
14
|
-
*/
|
|
15
|
-
constructor(brainyData) {
|
|
16
|
-
this.brainyData = brainyData;
|
|
17
|
-
}
|
|
18
|
-
/**
|
|
19
|
-
* Handles an MCP data access request
|
|
20
|
-
* @param request The MCP request
|
|
21
|
-
* @returns An MCP response
|
|
22
|
-
*/
|
|
23
|
-
async handleRequest(request) {
|
|
24
|
-
try {
|
|
25
|
-
switch (request.operation) {
|
|
26
|
-
case 'get':
|
|
27
|
-
return await this.handleGetRequest(request);
|
|
28
|
-
case 'search':
|
|
29
|
-
return await this.handleSearchRequest(request);
|
|
30
|
-
case 'add':
|
|
31
|
-
return await this.handleAddRequest(request);
|
|
32
|
-
case 'getRelationships':
|
|
33
|
-
return await this.handleGetRelationshipsRequest(request);
|
|
34
|
-
default:
|
|
35
|
-
return this.createErrorResponse(request.requestId, 'UNSUPPORTED_OPERATION', `Operation ${request.operation} is not supported`);
|
|
36
|
-
}
|
|
37
|
-
}
|
|
38
|
-
catch (error) {
|
|
39
|
-
return this.createErrorResponse(request.requestId, 'INTERNAL_ERROR', error instanceof Error ? error.message : String(error));
|
|
40
|
-
}
|
|
41
|
-
}
|
|
42
|
-
/**
|
|
43
|
-
* Handles a get request
|
|
44
|
-
* @param request The MCP request
|
|
45
|
-
* @returns An MCP response
|
|
46
|
-
*/
|
|
47
|
-
async handleGetRequest(request) {
|
|
48
|
-
const { id } = request.parameters;
|
|
49
|
-
if (!id) {
|
|
50
|
-
return this.createErrorResponse(request.requestId, 'MISSING_PARAMETER', 'Parameter "id" is required');
|
|
51
|
-
}
|
|
52
|
-
const noun = await this.brainyData.get(id);
|
|
53
|
-
if (!noun) {
|
|
54
|
-
return this.createErrorResponse(request.requestId, 'NOT_FOUND', `No noun found with id ${id}`);
|
|
55
|
-
}
|
|
56
|
-
return this.createSuccessResponse(request.requestId, noun);
|
|
57
|
-
}
|
|
58
|
-
/**
|
|
59
|
-
* Handles a search request
|
|
60
|
-
* @param request The MCP request
|
|
61
|
-
* @returns An MCP response
|
|
62
|
-
*/
|
|
63
|
-
async handleSearchRequest(request) {
|
|
64
|
-
const { query, k = 10 } = request.parameters;
|
|
65
|
-
if (!query) {
|
|
66
|
-
return this.createErrorResponse(request.requestId, 'MISSING_PARAMETER', 'Parameter "query" is required');
|
|
67
|
-
}
|
|
68
|
-
const results = await this.brainyData.searchText(query, k);
|
|
69
|
-
return this.createSuccessResponse(request.requestId, results);
|
|
70
|
-
}
|
|
71
|
-
/**
|
|
72
|
-
* Handles an add request
|
|
73
|
-
* @param request The MCP request
|
|
74
|
-
* @returns An MCP response
|
|
75
|
-
*/
|
|
76
|
-
async handleAddRequest(request) {
|
|
77
|
-
const { text, metadata } = request.parameters;
|
|
78
|
-
if (!text) {
|
|
79
|
-
return this.createErrorResponse(request.requestId, 'MISSING_PARAMETER', 'Parameter "text" is required');
|
|
80
|
-
}
|
|
81
|
-
const id = await this.brainyData.add(text, metadata);
|
|
82
|
-
return this.createSuccessResponse(request.requestId, { id });
|
|
83
|
-
}
|
|
84
|
-
/**
|
|
85
|
-
* Handles a getRelationships request
|
|
86
|
-
* @param request The MCP request
|
|
87
|
-
* @returns An MCP response
|
|
88
|
-
*/
|
|
89
|
-
async handleGetRelationshipsRequest(request) {
|
|
90
|
-
const { id } = request.parameters;
|
|
91
|
-
if (!id) {
|
|
92
|
-
return this.createErrorResponse(request.requestId, 'MISSING_PARAMETER', 'Parameter "id" is required');
|
|
93
|
-
}
|
|
94
|
-
// This is a simplified implementation - in a real implementation, we would
|
|
95
|
-
// need to check if these methods exist on the BrainyDataInterface
|
|
96
|
-
const outgoing = await this.brainyData.getVerbsBySource?.(id) || [];
|
|
97
|
-
const incoming = await this.brainyData.getVerbsByTarget?.(id) || [];
|
|
98
|
-
return this.createSuccessResponse(request.requestId, { outgoing, incoming });
|
|
99
|
-
}
|
|
100
|
-
/**
|
|
101
|
-
* Creates a success response
|
|
102
|
-
* @param requestId The request ID
|
|
103
|
-
* @param data The response data
|
|
104
|
-
* @returns An MCP response
|
|
105
|
-
*/
|
|
106
|
-
createSuccessResponse(requestId, data) {
|
|
107
|
-
return {
|
|
108
|
-
success: true,
|
|
109
|
-
requestId,
|
|
110
|
-
version: MCP_VERSION,
|
|
111
|
-
data
|
|
112
|
-
};
|
|
113
|
-
}
|
|
114
|
-
/**
|
|
115
|
-
* Creates an error response
|
|
116
|
-
* @param requestId The request ID
|
|
117
|
-
* @param code The error code
|
|
118
|
-
* @param message The error message
|
|
119
|
-
* @param details Optional error details
|
|
120
|
-
* @returns An MCP response
|
|
121
|
-
*/
|
|
122
|
-
createErrorResponse(requestId, code, message, details) {
|
|
123
|
-
return {
|
|
124
|
-
success: false,
|
|
125
|
-
requestId,
|
|
126
|
-
version: MCP_VERSION,
|
|
127
|
-
error: {
|
|
128
|
-
code,
|
|
129
|
-
message,
|
|
130
|
-
details
|
|
131
|
-
}
|
|
132
|
-
};
|
|
133
|
-
}
|
|
134
|
-
/**
|
|
135
|
-
* Creates a new request ID
|
|
136
|
-
* @returns A new UUID
|
|
137
|
-
*/
|
|
138
|
-
generateRequestId() {
|
|
139
|
-
return uuidv4();
|
|
140
|
-
}
|
|
141
|
-
}
|
|
142
|
-
//# sourceMappingURL=brainyMCPAdapter.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"brainyMCPAdapter.js","sourceRoot":"","sources":["../../src/mcp/brainyMCPAdapter.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,OAAO,EAAE,EAAE,IAAI,MAAM,EAAE,MAAM,MAAM,CAAA;AAEnC,OAAO,EAKL,WAAW,EACZ,MAAM,sBAAsB,CAAA;AAE7B,MAAM,OAAO,gBAAgB;IAG3B;;;OAGG;IACH,YAAY,UAA+B;QACzC,IAAI,CAAC,UAAU,GAAG,UAAU,CAAA;IAC9B,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,aAAa,CAAC,OAA6B;QAC/C,IAAI,CAAC;YACH,QAAQ,OAAO,CAAC,SAAS,EAAE,CAAC;gBAC1B,KAAK,KAAK;oBACR,OAAO,MAAM,IAAI,CAAC,gBAAgB,CAAC,OAAO,CAAC,CAAA;gBAC7C,KAAK,QAAQ;oBACX,OAAO,MAAM,IAAI,CAAC,mBAAmB,CAAC,OAAO,CAAC,CAAA;gBAChD,KAAK,KAAK;oBACR,OAAO,MAAM,IAAI,CAAC,gBAAgB,CAAC,OAAO,CAAC,CAAA;gBAC7C,KAAK,kBAAkB;oBACrB,OAAO,MAAM,IAAI,CAAC,6BAA6B,CAAC,OAAO,CAAC,CAAA;gBAC1D;oBACE,OAAO,IAAI,CAAC,mBAAmB,CAC7B,OAAO,CAAC,SAAS,EACjB,uBAAuB,EACvB,aAAa,OAAO,CAAC,SAAS,mBAAmB,CAClD,CAAA;YACL,CAAC;QACH,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO,IAAI,CAAC,mBAAmB,CAC7B,OAAO,CAAC,SAAS,EACjB,gBAAgB,EAChB,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CACvD,CAAA;QACH,CAAC;IACH,CAAC;IAED;;;;OAIG;IACK,KAAK,CAAC,gBAAgB,CAAC,OAA6B;QAC1D,MAAM,EAAE,EAAE,EAAE,GAAG,OAAO,CAAC,UAAU,CAAA;QAEjC,IAAI,CAAC,EAAE,EAAE,CAAC;YACR,OAAO,IAAI,CAAC,mBAAmB,CAC7B,OAAO,CAAC,SAAS,EACjB,mBAAmB,EACnB,4BAA4B,CAC7B,CAAA;QACH,CAAC;QAED,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC,CAAA;QAE1C,IAAI,CAAC,IAAI,EAAE,CAAC;YACV,OAAO,IAAI,CAAC,mBAAmB,CAC7B,OAAO,CAAC,SAAS,EACjB,WAAW,EACX,yBAAyB,EAAE,EAAE,CAC9B,CAAA;QACH,CAAC;QAED,OAAO,IAAI,CAAC,qBAAqB,CAAC,OAAO,CAAC,SAAS,EAAE,IAAI,CAAC,CAAA;IAC5D,CAAC;IAED;;;;OAIG;IACK,KAAK,CAAC,mBAAmB,CAAC,OAA6B;QAC7D,MAAM,EAAE,KAAK,EAAE,CAAC,GAAG,EAAE,EAAE,GAAG,OAAO,CAAC,UAAU,CAAA;QAE5C,IAAI,CAAC,KAAK,EAAE,CAAC;YACX,OAAO,IAAI,CAAC,mBAAmB,CAC7B,OAAO,CAAC,SAAS,EACjB,mBAAmB,EACnB,+BAA+B,CAChC,CAAA;QACH,CAAC;QAED,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,UAAU,CAAC,KAAK,EAAE,CAAC,CAAC,CAAA;QAC1D,OAAO,IAAI,CAAC,qBAAqB,CAAC,OAAO,CAAC,SAAS,EAAE,OAAO,CAAC,CAAA;IAC/D,CAAC;IAED;;;;OAIG;IACK,KAAK,CAAC,gBAAgB,CAAC,OAA6B;QAC1D,MAAM,EAAE,IAAI,EAAE,QAAQ,EAAE,GAAG,OAAO,CAAC,UAAU,CAAA;QAE7C,IAAI,CAAC,IAAI,EAAE,CAAC;YACV,OAAO,IAAI,CAAC,mBAAmB,CAC7B,OAAO,CAAC,SAAS,EACjB,mBAAmB,EACnB,8BAA8B,CAC/B,CAAA;QACH,CAAC;QAED,MAAM,EAAE,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAA;QACpD,OAAO,IAAI,CAAC,qBAAqB,CAAC,OAAO,CAAC,SAAS,EAAE,EAAE,EAAE,EAAE,CAAC,CAAA;IAC9D,CAAC;IAED;;;;OAIG;IACK,KAAK,CAAC,6BAA6B,CAAC,OAA6B;QACvE,MAAM,EAAE,EAAE,EAAE,GAAG,OAAO,CAAC,UAAU,CAAA;QAEjC,IAAI,CAAC,EAAE,EAAE,CAAC;YACR,OAAO,IAAI,CAAC,mBAAmB,CAC7B,OAAO,CAAC,SAAS,EACjB,mBAAmB,EACnB,4BAA4B,CAC7B,CAAA;QACH,CAAC;QAED,2EAA2E;QAC3E,kEAAkE;QAClE,MAAM,QAAQ,GAAG,MAAO,IAAI,CAAC,UAAkB,CAAC,gBAAgB,EAAE,CAAC,EAAE,CAAC,IAAI,EAAE,CAAA;QAC5E,MAAM,QAAQ,GAAG,MAAO,IAAI,CAAC,UAAkB,CAAC,gBAAgB,EAAE,CAAC,EAAE,CAAC,IAAI,EAAE,CAAA;QAE5E,OAAO,IAAI,CAAC,qBAAqB,CAAC,OAAO,CAAC,SAAS,EAAE,EAAE,QAAQ,EAAE,QAAQ,EAAE,CAAC,CAAA;IAC9E,CAAC;IAED;;;;;OAKG;IACK,qBAAqB,CAAC,SAAiB,EAAE,IAAS;QACxD,OAAO;YACL,OAAO,EAAE,IAAI;YACb,SAAS;YACT,OAAO,EAAE,WAAW;YACpB,IAAI;SACL,CAAA;IACH,CAAC;IAED;;;;;;;OAOG;IACK,mBAAmB,CACzB,SAAiB,EACjB,IAAY,EACZ,OAAe,EACf,OAAa;QAEb,OAAO;YACL,OAAO,EAAE,KAAK;YACd,SAAS;YACT,OAAO,EAAE,WAAW;YACpB,KAAK,EAAE;gBACL,IAAI;gBACJ,OAAO;gBACP,OAAO;aACR;SACF,CAAA;IACH,CAAC;IAED;;;OAGG;IACH,iBAAiB;QACf,OAAO,MAAM,EAAE,CAAA;IACjB,CAAC;CACF"}
|