@lov3kaizen/agentsea-embeddings 0.5.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +475 -0
- package/dist/caching/index.d.mts +286 -0
- package/dist/caching/index.d.ts +286 -0
- package/dist/caching/index.js +1005 -0
- package/dist/caching/index.mjs +27 -0
- package/dist/chunk-3KM32UQK.mjs +207 -0
- package/dist/chunk-DJAURHAS.mjs +1117 -0
- package/dist/chunk-NBHIRTJT.mjs +895 -0
- package/dist/chunk-QAITLJ2E.mjs +259 -0
- package/dist/chunk-TER262ST.mjs +877 -0
- package/dist/chunk-VPSMDBHH.mjs +957 -0
- package/dist/chunking/index.d.mts +1 -0
- package/dist/chunking/index.d.ts +1 -0
- package/dist/chunking/index.js +1408 -0
- package/dist/chunking/index.mjs +37 -0
- package/dist/embedding.types-CCgPVxt1.d.mts +102 -0
- package/dist/embedding.types-CCgPVxt1.d.ts +102 -0
- package/dist/index-CeG6God2.d.mts +297 -0
- package/dist/index-DMaQRn2w.d.mts +172 -0
- package/dist/index-DMaQRn2w.d.ts +172 -0
- package/dist/index-DWddsKRi.d.ts +297 -0
- package/dist/index.d.mts +647 -0
- package/dist/index.d.ts +647 -0
- package/dist/index.js +5259 -0
- package/dist/index.mjs +1028 -0
- package/dist/providers/index.d.mts +2 -0
- package/dist/providers/index.d.ts +2 -0
- package/dist/providers/index.js +1235 -0
- package/dist/providers/index.mjs +32 -0
- package/dist/stores/index.d.mts +298 -0
- package/dist/stores/index.d.ts +298 -0
- package/dist/stores/index.js +1178 -0
- package/dist/stores/index.mjs +26 -0
- package/package.json +102 -0
|
@@ -0,0 +1,1178 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __create = Object.create;
|
|
3
|
+
var __defProp = Object.defineProperty;
|
|
4
|
+
var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
|
|
5
|
+
var __getOwnPropNames = Object.getOwnPropertyNames;
|
|
6
|
+
var __getProtoOf = Object.getPrototypeOf;
|
|
7
|
+
var __hasOwnProp = Object.prototype.hasOwnProperty;
|
|
8
|
+
var __export = (target, all) => {
|
|
9
|
+
for (var name in all)
|
|
10
|
+
__defProp(target, name, { get: all[name], enumerable: true });
|
|
11
|
+
};
|
|
12
|
+
var __copyProps = (to, from, except, desc) => {
|
|
13
|
+
if (from && typeof from === "object" || typeof from === "function") {
|
|
14
|
+
for (let key of __getOwnPropNames(from))
|
|
15
|
+
if (!__hasOwnProp.call(to, key) && key !== except)
|
|
16
|
+
__defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
|
|
17
|
+
}
|
|
18
|
+
return to;
|
|
19
|
+
};
|
|
20
|
+
var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
|
|
21
|
+
// If the importer is in node compatibility mode or this is not an ESM
|
|
22
|
+
// file that has been converted to a CommonJS file using a Babel-
|
|
23
|
+
// compatible transform (i.e. "__esModule" has not been set), then set
|
|
24
|
+
// "default" to the CommonJS "module.exports" for node compatibility.
|
|
25
|
+
isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
|
|
26
|
+
mod
|
|
27
|
+
));
|
|
28
|
+
var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
|
|
29
|
+
|
|
30
|
+
// src/stores/index.ts
|
|
31
|
+
var stores_exports = {};
|
|
32
|
+
__export(stores_exports, {
|
|
33
|
+
BaseStore: () => BaseStore,
|
|
34
|
+
ChromaStore: () => ChromaStore,
|
|
35
|
+
MemoryStore: () => MemoryStore,
|
|
36
|
+
PineconeStore: () => PineconeStore,
|
|
37
|
+
QdrantStore: () => QdrantStore,
|
|
38
|
+
createChromaStore: () => createChromaStore,
|
|
39
|
+
createMemoryStore: () => createMemoryStore,
|
|
40
|
+
createPineconeStore: () => createPineconeStore,
|
|
41
|
+
createQdrantStore: () => createQdrantStore,
|
|
42
|
+
createStore: () => createStore
|
|
43
|
+
});
|
|
44
|
+
module.exports = __toCommonJS(stores_exports);
|
|
45
|
+
|
|
46
|
+
// src/core/EmbeddingModel.ts
|
|
47
|
+
var EmbeddingModel = class {
|
|
48
|
+
/**
|
|
49
|
+
* Get model dimensions
|
|
50
|
+
*/
|
|
51
|
+
get dimensions() {
|
|
52
|
+
return this.info.dimensions;
|
|
53
|
+
}
|
|
54
|
+
/**
|
|
55
|
+
* Get max tokens
|
|
56
|
+
*/
|
|
57
|
+
get maxTokens() {
|
|
58
|
+
return this.info.maxTokens;
|
|
59
|
+
}
|
|
60
|
+
/**
|
|
61
|
+
* Get max batch size
|
|
62
|
+
*/
|
|
63
|
+
get maxBatchSize() {
|
|
64
|
+
return this.info.maxBatchSize;
|
|
65
|
+
}
|
|
66
|
+
/**
|
|
67
|
+
* Get model name
|
|
68
|
+
*/
|
|
69
|
+
get name() {
|
|
70
|
+
return this.info.name;
|
|
71
|
+
}
|
|
72
|
+
/**
|
|
73
|
+
* Get provider name
|
|
74
|
+
*/
|
|
75
|
+
get provider() {
|
|
76
|
+
return this.info.provider;
|
|
77
|
+
}
|
|
78
|
+
/**
|
|
79
|
+
* Count tokens in text (default implementation)
|
|
80
|
+
* Subclasses should override for accurate counting
|
|
81
|
+
*/
|
|
82
|
+
countTokens(text) {
|
|
83
|
+
return Math.ceil(text.length / 4);
|
|
84
|
+
}
|
|
85
|
+
/**
|
|
86
|
+
* Check if text exceeds max tokens
|
|
87
|
+
*/
|
|
88
|
+
exceedsMaxTokens(text) {
|
|
89
|
+
return this.countTokens(text) > this.maxTokens;
|
|
90
|
+
}
|
|
91
|
+
/**
|
|
92
|
+
* Truncate text to max tokens
|
|
93
|
+
*/
|
|
94
|
+
truncateToMaxTokens(text) {
|
|
95
|
+
const tokens = this.countTokens(text);
|
|
96
|
+
if (tokens <= this.maxTokens) {
|
|
97
|
+
return text;
|
|
98
|
+
}
|
|
99
|
+
const ratio = this.maxTokens / tokens;
|
|
100
|
+
const targetLength = Math.floor(text.length * ratio * 0.95);
|
|
101
|
+
return text.slice(0, targetLength);
|
|
102
|
+
}
|
|
103
|
+
/**
|
|
104
|
+
* Calculate similarity between two vectors
|
|
105
|
+
*/
|
|
106
|
+
static cosineSimilarity(a, b) {
|
|
107
|
+
if (a.length !== b.length) {
|
|
108
|
+
throw new Error(`Vector dimensions mismatch: ${a.length} vs ${b.length}`);
|
|
109
|
+
}
|
|
110
|
+
let dotProduct = 0;
|
|
111
|
+
let normA = 0;
|
|
112
|
+
let normB = 0;
|
|
113
|
+
for (let i = 0; i < a.length; i++) {
|
|
114
|
+
dotProduct += a[i] * b[i];
|
|
115
|
+
normA += a[i] * a[i];
|
|
116
|
+
normB += b[i] * b[i];
|
|
117
|
+
}
|
|
118
|
+
const magnitude = Math.sqrt(normA) * Math.sqrt(normB);
|
|
119
|
+
if (magnitude === 0) {
|
|
120
|
+
return 0;
|
|
121
|
+
}
|
|
122
|
+
return dotProduct / magnitude;
|
|
123
|
+
}
|
|
124
|
+
/**
|
|
125
|
+
* Calculate Euclidean distance between two vectors
|
|
126
|
+
*/
|
|
127
|
+
static euclideanDistance(a, b) {
|
|
128
|
+
if (a.length !== b.length) {
|
|
129
|
+
throw new Error(`Vector dimensions mismatch: ${a.length} vs ${b.length}`);
|
|
130
|
+
}
|
|
131
|
+
let sum = 0;
|
|
132
|
+
for (let i = 0; i < a.length; i++) {
|
|
133
|
+
const diff = a[i] - b[i];
|
|
134
|
+
sum += diff * diff;
|
|
135
|
+
}
|
|
136
|
+
return Math.sqrt(sum);
|
|
137
|
+
}
|
|
138
|
+
/**
|
|
139
|
+
* Calculate dot product of two vectors
|
|
140
|
+
*/
|
|
141
|
+
static dotProduct(a, b) {
|
|
142
|
+
if (a.length !== b.length) {
|
|
143
|
+
throw new Error(`Vector dimensions mismatch: ${a.length} vs ${b.length}`);
|
|
144
|
+
}
|
|
145
|
+
let result = 0;
|
|
146
|
+
for (let i = 0; i < a.length; i++) {
|
|
147
|
+
result += a[i] * b[i];
|
|
148
|
+
}
|
|
149
|
+
return result;
|
|
150
|
+
}
|
|
151
|
+
/**
|
|
152
|
+
* Normalize a vector to unit length
|
|
153
|
+
*/
|
|
154
|
+
static normalize(vector) {
|
|
155
|
+
let norm = 0;
|
|
156
|
+
for (let i = 0; i < vector.length; i++) {
|
|
157
|
+
norm += vector[i] * vector[i];
|
|
158
|
+
}
|
|
159
|
+
norm = Math.sqrt(norm);
|
|
160
|
+
if (norm === 0) {
|
|
161
|
+
return vector.slice();
|
|
162
|
+
}
|
|
163
|
+
return vector.map((v) => v / norm);
|
|
164
|
+
}
|
|
165
|
+
/**
|
|
166
|
+
* Average multiple vectors
|
|
167
|
+
*/
|
|
168
|
+
static average(vectors) {
|
|
169
|
+
if (vectors.length === 0) {
|
|
170
|
+
throw new Error("Cannot average empty array of vectors");
|
|
171
|
+
}
|
|
172
|
+
const dimensions = vectors[0].length;
|
|
173
|
+
const result = new Array(dimensions).fill(0);
|
|
174
|
+
for (const vector of vectors) {
|
|
175
|
+
if (vector.length !== dimensions) {
|
|
176
|
+
throw new Error(
|
|
177
|
+
`Vector dimensions mismatch: expected ${dimensions}, got ${vector.length}`
|
|
178
|
+
);
|
|
179
|
+
}
|
|
180
|
+
for (let i = 0; i < dimensions; i++) {
|
|
181
|
+
result[i] += vector[i];
|
|
182
|
+
}
|
|
183
|
+
}
|
|
184
|
+
for (let i = 0; i < dimensions; i++) {
|
|
185
|
+
result[i] /= vectors.length;
|
|
186
|
+
}
|
|
187
|
+
return result;
|
|
188
|
+
}
|
|
189
|
+
/**
|
|
190
|
+
* Weighted average of vectors
|
|
191
|
+
*/
|
|
192
|
+
static weightedAverage(vectors, weights) {
|
|
193
|
+
if (vectors.length === 0) {
|
|
194
|
+
throw new Error("Cannot average empty array of vectors");
|
|
195
|
+
}
|
|
196
|
+
if (vectors.length !== weights.length) {
|
|
197
|
+
throw new Error("Vectors and weights arrays must have same length");
|
|
198
|
+
}
|
|
199
|
+
const dimensions = vectors[0].length;
|
|
200
|
+
const result = new Array(dimensions).fill(0);
|
|
201
|
+
let totalWeight = 0;
|
|
202
|
+
for (let j = 0; j < vectors.length; j++) {
|
|
203
|
+
const vector = vectors[j];
|
|
204
|
+
const weight = weights[j];
|
|
205
|
+
totalWeight += weight;
|
|
206
|
+
if (vector.length !== dimensions) {
|
|
207
|
+
throw new Error(
|
|
208
|
+
`Vector dimensions mismatch: expected ${dimensions}, got ${vector.length}`
|
|
209
|
+
);
|
|
210
|
+
}
|
|
211
|
+
for (let i = 0; i < dimensions; i++) {
|
|
212
|
+
result[i] += vector[i] * weight;
|
|
213
|
+
}
|
|
214
|
+
}
|
|
215
|
+
if (totalWeight === 0) {
|
|
216
|
+
throw new Error("Total weight cannot be zero");
|
|
217
|
+
}
|
|
218
|
+
for (let i = 0; i < dimensions; i++) {
|
|
219
|
+
result[i] /= totalWeight;
|
|
220
|
+
}
|
|
221
|
+
return result;
|
|
222
|
+
}
|
|
223
|
+
};
|
|
224
|
+
var ModelRegistry = class {
|
|
225
|
+
models = /* @__PURE__ */ new Map();
|
|
226
|
+
defaultModel = null;
|
|
227
|
+
/**
|
|
228
|
+
* Register a model
|
|
229
|
+
*/
|
|
230
|
+
register(model, isDefault = false) {
|
|
231
|
+
const key = `${model.provider}:${model.name}`;
|
|
232
|
+
this.models.set(key, model);
|
|
233
|
+
if (isDefault || this.defaultModel === null) {
|
|
234
|
+
this.defaultModel = key;
|
|
235
|
+
}
|
|
236
|
+
}
|
|
237
|
+
/**
|
|
238
|
+
* Get a model by provider and name
|
|
239
|
+
*/
|
|
240
|
+
get(provider, name) {
|
|
241
|
+
return this.models.get(`${provider}:${name}`);
|
|
242
|
+
}
|
|
243
|
+
/**
|
|
244
|
+
* Get model by key
|
|
245
|
+
*/
|
|
246
|
+
getByKey(key) {
|
|
247
|
+
return this.models.get(key);
|
|
248
|
+
}
|
|
249
|
+
/**
|
|
250
|
+
* Get the default model
|
|
251
|
+
*/
|
|
252
|
+
getDefault() {
|
|
253
|
+
if (this.defaultModel === null) {
|
|
254
|
+
return void 0;
|
|
255
|
+
}
|
|
256
|
+
return this.models.get(this.defaultModel);
|
|
257
|
+
}
|
|
258
|
+
/**
|
|
259
|
+
* Set default model
|
|
260
|
+
*/
|
|
261
|
+
setDefault(provider, name) {
|
|
262
|
+
const key = `${provider}:${name}`;
|
|
263
|
+
if (!this.models.has(key)) {
|
|
264
|
+
throw new Error(`Model ${key} not found in registry`);
|
|
265
|
+
}
|
|
266
|
+
this.defaultModel = key;
|
|
267
|
+
}
|
|
268
|
+
/**
|
|
269
|
+
* List all registered models
|
|
270
|
+
*/
|
|
271
|
+
list() {
|
|
272
|
+
return Array.from(this.models.values()).map((m) => m.info);
|
|
273
|
+
}
|
|
274
|
+
/**
|
|
275
|
+
* Check if a model is registered
|
|
276
|
+
*/
|
|
277
|
+
has(provider, name) {
|
|
278
|
+
return this.models.has(`${provider}:${name}`);
|
|
279
|
+
}
|
|
280
|
+
/**
|
|
281
|
+
* Remove a model
|
|
282
|
+
*/
|
|
283
|
+
remove(provider, name) {
|
|
284
|
+
const key = `${provider}:${name}`;
|
|
285
|
+
if (this.defaultModel === key) {
|
|
286
|
+
this.defaultModel = null;
|
|
287
|
+
}
|
|
288
|
+
return this.models.delete(key);
|
|
289
|
+
}
|
|
290
|
+
/**
|
|
291
|
+
* Clear all models
|
|
292
|
+
*/
|
|
293
|
+
clear() {
|
|
294
|
+
this.models.clear();
|
|
295
|
+
this.defaultModel = null;
|
|
296
|
+
}
|
|
297
|
+
};
|
|
298
|
+
var modelRegistry = new ModelRegistry();
|
|
299
|
+
|
|
300
|
+
// src/stores/BaseStore.ts
|
|
301
|
+
var BaseStore = class {
|
|
302
|
+
/** Store configuration */
|
|
303
|
+
config;
|
|
304
|
+
constructor(config) {
|
|
305
|
+
this.config = {
|
|
306
|
+
namespace: config.namespace ?? "default",
|
|
307
|
+
dimensions: config.dimensions,
|
|
308
|
+
metric: config.metric ?? "cosine",
|
|
309
|
+
...config
|
|
310
|
+
};
|
|
311
|
+
}
|
|
312
|
+
/**
|
|
313
|
+
* Get namespace
|
|
314
|
+
*/
|
|
315
|
+
get namespace() {
|
|
316
|
+
return this.config.namespace ?? "default";
|
|
317
|
+
}
|
|
318
|
+
/**
|
|
319
|
+
* Get dimensions
|
|
320
|
+
*/
|
|
321
|
+
get dimensions() {
|
|
322
|
+
return this.config.dimensions;
|
|
323
|
+
}
|
|
324
|
+
/**
|
|
325
|
+
* Get distance metric
|
|
326
|
+
*/
|
|
327
|
+
get metric() {
|
|
328
|
+
return this.config.metric ?? "cosine";
|
|
329
|
+
}
|
|
330
|
+
/**
|
|
331
|
+
* Calculate similarity/distance between vectors
|
|
332
|
+
*/
|
|
333
|
+
calculateScore(a, b) {
|
|
334
|
+
switch (this.metric) {
|
|
335
|
+
case "cosine":
|
|
336
|
+
return EmbeddingModel.cosineSimilarity(a, b);
|
|
337
|
+
case "euclidean": {
|
|
338
|
+
const dist = EmbeddingModel.euclideanDistance(a, b);
|
|
339
|
+
return 1 / (1 + dist);
|
|
340
|
+
}
|
|
341
|
+
case "dot_product":
|
|
342
|
+
return EmbeddingModel.dotProduct(a, b);
|
|
343
|
+
default:
|
|
344
|
+
return EmbeddingModel.cosineSimilarity(a, b);
|
|
345
|
+
}
|
|
346
|
+
}
|
|
347
|
+
/**
|
|
348
|
+
* Filter records by metadata
|
|
349
|
+
*/
|
|
350
|
+
filterByMetadata(records, filter) {
|
|
351
|
+
if (!filter) return records;
|
|
352
|
+
return records.filter((record) => {
|
|
353
|
+
if (!record.metadata) return false;
|
|
354
|
+
for (const [key, value] of Object.entries(filter)) {
|
|
355
|
+
if (record.metadata[key] !== value) {
|
|
356
|
+
return false;
|
|
357
|
+
}
|
|
358
|
+
}
|
|
359
|
+
return true;
|
|
360
|
+
});
|
|
361
|
+
}
|
|
362
|
+
/**
|
|
363
|
+
* Convert records to search results
|
|
364
|
+
*/
|
|
365
|
+
toSearchResults(records, options) {
|
|
366
|
+
return records.map((record) => ({
|
|
367
|
+
id: record.id,
|
|
368
|
+
text: options?.includeText !== false ? record.text ?? "" : "",
|
|
369
|
+
score: record.score,
|
|
370
|
+
metadata: options?.includeMetadata !== false ? record.metadata ?? {} : {},
|
|
371
|
+
distance: this.metric !== "cosine" ? 1 - record.score : void 0
|
|
372
|
+
}));
|
|
373
|
+
}
|
|
374
|
+
};
|
|
375
|
+
|
|
376
|
+
// src/stores/MemoryStore.ts
|
|
377
|
+
var fs = __toESM(require("fs/promises"));
|
|
378
|
+
var MemoryStore = class extends BaseStore {
|
|
379
|
+
storeType = "memory";
|
|
380
|
+
vectors = /* @__PURE__ */ new Map();
|
|
381
|
+
namespaces = /* @__PURE__ */ new Map();
|
|
382
|
+
persistPath;
|
|
383
|
+
persistInterval;
|
|
384
|
+
maxVectors;
|
|
385
|
+
constructor(config = { type: "memory" }) {
|
|
386
|
+
super(config);
|
|
387
|
+
this.maxVectors = config.maxVectors ?? 1e5;
|
|
388
|
+
this.persistPath = config.persistPath;
|
|
389
|
+
if (config.persistInterval && config.persistPath) {
|
|
390
|
+
this.persistInterval = setInterval(
|
|
391
|
+
() => void this.persist().catch(() => {
|
|
392
|
+
}),
|
|
393
|
+
config.persistInterval
|
|
394
|
+
);
|
|
395
|
+
}
|
|
396
|
+
}
|
|
397
|
+
async upsert(records, options) {
|
|
398
|
+
const startTime = performance.now();
|
|
399
|
+
const namespace = options?.namespace ?? this.namespace;
|
|
400
|
+
const upsertedIds = [];
|
|
401
|
+
const errors = [];
|
|
402
|
+
if (!this.namespaces.has(namespace)) {
|
|
403
|
+
this.namespaces.set(namespace, /* @__PURE__ */ new Set());
|
|
404
|
+
}
|
|
405
|
+
const nsIds = this.namespaces.get(namespace);
|
|
406
|
+
for (const record of records) {
|
|
407
|
+
try {
|
|
408
|
+
if (this.vectors.size >= this.maxVectors && !this.vectors.has(record.id)) {
|
|
409
|
+
const oldestId = this.vectors.keys().next().value;
|
|
410
|
+
if (oldestId) {
|
|
411
|
+
this.vectors.delete(oldestId);
|
|
412
|
+
for (const ns of this.namespaces.values()) {
|
|
413
|
+
ns.delete(oldestId);
|
|
414
|
+
}
|
|
415
|
+
}
|
|
416
|
+
}
|
|
417
|
+
this.vectors.set(record.id, record);
|
|
418
|
+
nsIds.add(record.id);
|
|
419
|
+
upsertedIds.push(record.id);
|
|
420
|
+
} catch (error) {
|
|
421
|
+
errors.push({ id: record.id, error: error.message });
|
|
422
|
+
}
|
|
423
|
+
}
|
|
424
|
+
return Promise.resolve({
|
|
425
|
+
upsertedIds,
|
|
426
|
+
upsertedCount: upsertedIds.length,
|
|
427
|
+
errors,
|
|
428
|
+
durationMs: performance.now() - startTime
|
|
429
|
+
});
|
|
430
|
+
}
|
|
431
|
+
async query(vector, options) {
|
|
432
|
+
const startTime = performance.now();
|
|
433
|
+
const namespace = options?.namespace ?? this.namespace;
|
|
434
|
+
const topK = options?.topK ?? 10;
|
|
435
|
+
const minScore = options?.minScore ?? 0;
|
|
436
|
+
const nsIds = this.namespaces.get(namespace);
|
|
437
|
+
if (!nsIds || nsIds.size === 0) {
|
|
438
|
+
return {
|
|
439
|
+
matches: [],
|
|
440
|
+
namespace,
|
|
441
|
+
durationMs: performance.now() - startTime
|
|
442
|
+
};
|
|
443
|
+
}
|
|
444
|
+
let scoredRecords = [];
|
|
445
|
+
for (const id of nsIds) {
|
|
446
|
+
const record = this.vectors.get(id);
|
|
447
|
+
if (!record) continue;
|
|
448
|
+
const score = this.calculateScore(vector, record.vector);
|
|
449
|
+
if (score >= minScore) {
|
|
450
|
+
scoredRecords.push({ ...record, score });
|
|
451
|
+
}
|
|
452
|
+
}
|
|
453
|
+
if (options?.filter) {
|
|
454
|
+
scoredRecords = this.filterByMetadata(
|
|
455
|
+
scoredRecords,
|
|
456
|
+
options.filter
|
|
457
|
+
);
|
|
458
|
+
}
|
|
459
|
+
scoredRecords.sort((a, b) => b.score - a.score);
|
|
460
|
+
const topResults = scoredRecords.slice(0, topK);
|
|
461
|
+
return Promise.resolve({
|
|
462
|
+
matches: this.toSearchResults(topResults, options),
|
|
463
|
+
namespace,
|
|
464
|
+
durationMs: performance.now() - startTime
|
|
465
|
+
});
|
|
466
|
+
}
|
|
467
|
+
async delete(ids, options) {
|
|
468
|
+
const startTime = performance.now();
|
|
469
|
+
const namespace = options?.namespace ?? this.namespace;
|
|
470
|
+
let deletedCount = 0;
|
|
471
|
+
const nsIds = this.namespaces.get(namespace);
|
|
472
|
+
for (const id of ids) {
|
|
473
|
+
if (this.vectors.has(id)) {
|
|
474
|
+
this.vectors.delete(id);
|
|
475
|
+
nsIds?.delete(id);
|
|
476
|
+
deletedCount++;
|
|
477
|
+
}
|
|
478
|
+
}
|
|
479
|
+
return Promise.resolve({
|
|
480
|
+
deletedCount,
|
|
481
|
+
durationMs: performance.now() - startTime
|
|
482
|
+
});
|
|
483
|
+
}
|
|
484
|
+
async deleteAll(options) {
|
|
485
|
+
const startTime = performance.now();
|
|
486
|
+
const namespace = options?.namespace ?? this.namespace;
|
|
487
|
+
if (options?.deleteAll) {
|
|
488
|
+
const count2 = this.vectors.size;
|
|
489
|
+
this.vectors.clear();
|
|
490
|
+
this.namespaces.clear();
|
|
491
|
+
return Promise.resolve({
|
|
492
|
+
deletedCount: count2,
|
|
493
|
+
durationMs: performance.now() - startTime
|
|
494
|
+
});
|
|
495
|
+
}
|
|
496
|
+
const nsIds = this.namespaces.get(namespace);
|
|
497
|
+
if (!nsIds) {
|
|
498
|
+
return Promise.resolve({
|
|
499
|
+
deletedCount: 0,
|
|
500
|
+
durationMs: performance.now() - startTime
|
|
501
|
+
});
|
|
502
|
+
}
|
|
503
|
+
const count = nsIds.size;
|
|
504
|
+
for (const id of nsIds) {
|
|
505
|
+
this.vectors.delete(id);
|
|
506
|
+
}
|
|
507
|
+
this.namespaces.delete(namespace);
|
|
508
|
+
return Promise.resolve({
|
|
509
|
+
deletedCount: count,
|
|
510
|
+
durationMs: performance.now() - startTime
|
|
511
|
+
});
|
|
512
|
+
}
|
|
513
|
+
getStats() {
|
|
514
|
+
return Promise.resolve({
|
|
515
|
+
type: this.storeType,
|
|
516
|
+
vectorCount: this.vectors.size,
|
|
517
|
+
namespaceCount: this.namespaces.size,
|
|
518
|
+
dimensions: this.dimensions ?? 0,
|
|
519
|
+
metric: this.metric,
|
|
520
|
+
lastUpdated: Date.now()
|
|
521
|
+
});
|
|
522
|
+
}
|
|
523
|
+
checkHealth() {
|
|
524
|
+
return Promise.resolve({
|
|
525
|
+
healthy: true,
|
|
526
|
+
latencyMs: 0,
|
|
527
|
+
lastCheck: Date.now()
|
|
528
|
+
});
|
|
529
|
+
}
|
|
530
|
+
async close() {
|
|
531
|
+
if (this.persistInterval) {
|
|
532
|
+
clearInterval(this.persistInterval);
|
|
533
|
+
}
|
|
534
|
+
if (this.persistPath) {
|
|
535
|
+
await this.persist();
|
|
536
|
+
}
|
|
537
|
+
}
|
|
538
|
+
/**
|
|
539
|
+
* Persist store to file
|
|
540
|
+
*/
|
|
541
|
+
async persist() {
|
|
542
|
+
if (!this.persistPath) return;
|
|
543
|
+
const data = {
|
|
544
|
+
vectors: Array.from(this.vectors.entries()),
|
|
545
|
+
namespaces: Array.from(this.namespaces.entries()).map(([k, v]) => [
|
|
546
|
+
k,
|
|
547
|
+
Array.from(v)
|
|
548
|
+
])
|
|
549
|
+
};
|
|
550
|
+
await fs.writeFile(this.persistPath, JSON.stringify(data), "utf-8");
|
|
551
|
+
}
|
|
552
|
+
/**
|
|
553
|
+
* Load store from file
|
|
554
|
+
*/
|
|
555
|
+
async load() {
|
|
556
|
+
if (!this.persistPath) return;
|
|
557
|
+
try {
|
|
558
|
+
const data = JSON.parse(await fs.readFile(this.persistPath, "utf-8"));
|
|
559
|
+
this.vectors = new Map(data.vectors);
|
|
560
|
+
this.namespaces = new Map(
|
|
561
|
+
data.namespaces.map(([k, v]) => [k, new Set(v)])
|
|
562
|
+
);
|
|
563
|
+
} catch {
|
|
564
|
+
}
|
|
565
|
+
}
|
|
566
|
+
/**
|
|
567
|
+
* Get all vectors
|
|
568
|
+
*/
|
|
569
|
+
getAll() {
|
|
570
|
+
return Array.from(this.vectors.values());
|
|
571
|
+
}
|
|
572
|
+
/**
|
|
573
|
+
* Get vector by ID
|
|
574
|
+
*/
|
|
575
|
+
getById(id) {
|
|
576
|
+
return this.vectors.get(id);
|
|
577
|
+
}
|
|
578
|
+
};
|
|
579
|
+
function createMemoryStore(config) {
|
|
580
|
+
return new MemoryStore(config);
|
|
581
|
+
}
|
|
582
|
+
|
|
583
|
+
// src/core/utils.ts
|
|
584
|
+
function batch(items, batchSize) {
|
|
585
|
+
const batches = [];
|
|
586
|
+
for (let i = 0; i < items.length; i += batchSize) {
|
|
587
|
+
batches.push(items.slice(i, i + batchSize));
|
|
588
|
+
}
|
|
589
|
+
return batches;
|
|
590
|
+
}
|
|
591
|
+
|
|
592
|
+
// src/stores/PineconeStore.ts
|
|
593
|
+
var PineconeStore = class extends BaseStore {
|
|
594
|
+
storeType = "pinecone";
|
|
595
|
+
client;
|
|
596
|
+
index;
|
|
597
|
+
apiKey;
|
|
598
|
+
indexName;
|
|
599
|
+
initialized = false;
|
|
600
|
+
constructor(config) {
|
|
601
|
+
super(config);
|
|
602
|
+
if (!config.apiKey) {
|
|
603
|
+
throw new Error("Pinecone API key is required");
|
|
604
|
+
}
|
|
605
|
+
if (!config.indexName) {
|
|
606
|
+
throw new Error("Pinecone index name is required");
|
|
607
|
+
}
|
|
608
|
+
this.apiKey = config.apiKey;
|
|
609
|
+
this.indexName = config.indexName;
|
|
610
|
+
}
|
|
611
|
+
/**
|
|
612
|
+
* Initialize Pinecone client
|
|
613
|
+
*/
|
|
614
|
+
async init() {
|
|
615
|
+
if (this.initialized) return;
|
|
616
|
+
try {
|
|
617
|
+
const { Pinecone } = await import("@pinecone-database/pinecone");
|
|
618
|
+
this.client = new Pinecone({ apiKey: this.apiKey });
|
|
619
|
+
this.index = this.client.index(
|
|
620
|
+
this.indexName
|
|
621
|
+
);
|
|
622
|
+
this.initialized = true;
|
|
623
|
+
} catch (error) {
|
|
624
|
+
throw new Error(
|
|
625
|
+
`Failed to initialize Pinecone: ${error.message}`
|
|
626
|
+
);
|
|
627
|
+
}
|
|
628
|
+
}
|
|
629
|
+
async ensureInitialized() {
|
|
630
|
+
if (!this.initialized) {
|
|
631
|
+
await this.init();
|
|
632
|
+
}
|
|
633
|
+
}
|
|
634
|
+
async upsert(records, options) {
|
|
635
|
+
await this.ensureInitialized();
|
|
636
|
+
const startTime = performance.now();
|
|
637
|
+
const namespace = options?.namespace ?? this.namespace;
|
|
638
|
+
const batchSize = options?.batchSize ?? 100;
|
|
639
|
+
const upsertedIds = [];
|
|
640
|
+
const errors = [];
|
|
641
|
+
const vectors = records.map((record) => ({
|
|
642
|
+
id: record.id,
|
|
643
|
+
values: record.vector,
|
|
644
|
+
metadata: {
|
|
645
|
+
...record.metadata,
|
|
646
|
+
text: record.text
|
|
647
|
+
}
|
|
648
|
+
}));
|
|
649
|
+
const batches = batch(vectors, batchSize);
|
|
650
|
+
let completed = 0;
|
|
651
|
+
for (const batchVectors of batches) {
|
|
652
|
+
try {
|
|
653
|
+
const ns = this.index.namespace(namespace);
|
|
654
|
+
await ns.upsert(
|
|
655
|
+
batchVectors
|
|
656
|
+
);
|
|
657
|
+
upsertedIds.push(...batchVectors.map((v) => v.id));
|
|
658
|
+
} catch (error) {
|
|
659
|
+
for (const v of batchVectors) {
|
|
660
|
+
errors.push({ id: v.id, error: error.message });
|
|
661
|
+
}
|
|
662
|
+
}
|
|
663
|
+
completed += batchVectors.length;
|
|
664
|
+
options?.onProgress?.({ completed, total: records.length });
|
|
665
|
+
}
|
|
666
|
+
return {
|
|
667
|
+
upsertedIds,
|
|
668
|
+
upsertedCount: upsertedIds.length,
|
|
669
|
+
errors,
|
|
670
|
+
durationMs: performance.now() - startTime
|
|
671
|
+
};
|
|
672
|
+
}
|
|
673
|
+
async query(vector, options) {
|
|
674
|
+
await this.ensureInitialized();
|
|
675
|
+
const startTime = performance.now();
|
|
676
|
+
const namespace = options?.namespace ?? this.namespace;
|
|
677
|
+
const topK = options?.topK ?? 10;
|
|
678
|
+
const ns = this.index.namespace(
|
|
679
|
+
namespace
|
|
680
|
+
);
|
|
681
|
+
const result = await ns.query({
|
|
682
|
+
vector,
|
|
683
|
+
topK,
|
|
684
|
+
filter: options?.filter,
|
|
685
|
+
includeValues: options?.includeVectors ?? false,
|
|
686
|
+
includeMetadata: options?.includeMetadata ?? true
|
|
687
|
+
});
|
|
688
|
+
const matches = result.matches.map((match) => ({
|
|
689
|
+
id: match.id,
|
|
690
|
+
text: match.metadata?.text ?? "",
|
|
691
|
+
score: match.score,
|
|
692
|
+
metadata: match.metadata ?? {}
|
|
693
|
+
}));
|
|
694
|
+
const filtered = options?.minScore ? matches.filter((m) => m.score >= options.minScore) : matches;
|
|
695
|
+
return {
|
|
696
|
+
matches: filtered,
|
|
697
|
+
namespace,
|
|
698
|
+
durationMs: performance.now() - startTime
|
|
699
|
+
};
|
|
700
|
+
}
|
|
701
|
+
async delete(ids, options) {
|
|
702
|
+
await this.ensureInitialized();
|
|
703
|
+
const startTime = performance.now();
|
|
704
|
+
const namespace = options?.namespace ?? this.namespace;
|
|
705
|
+
const ns = this.index.namespace(
|
|
706
|
+
namespace
|
|
707
|
+
);
|
|
708
|
+
await ns.deleteMany(
|
|
709
|
+
ids
|
|
710
|
+
);
|
|
711
|
+
return {
|
|
712
|
+
deletedCount: ids.length,
|
|
713
|
+
durationMs: performance.now() - startTime
|
|
714
|
+
};
|
|
715
|
+
}
|
|
716
|
+
async deleteAll(options) {
|
|
717
|
+
await this.ensureInitialized();
|
|
718
|
+
const startTime = performance.now();
|
|
719
|
+
const namespace = options?.namespace ?? this.namespace;
|
|
720
|
+
const ns = this.index.namespace(
|
|
721
|
+
namespace
|
|
722
|
+
);
|
|
723
|
+
await ns.deleteAll();
|
|
724
|
+
return {
|
|
725
|
+
deletedCount: -1,
|
|
726
|
+
// Unknown count
|
|
727
|
+
durationMs: performance.now() - startTime
|
|
728
|
+
};
|
|
729
|
+
}
|
|
730
|
+
async getStats() {
|
|
731
|
+
await this.ensureInitialized();
|
|
732
|
+
const stats = await this.index.describeIndexStats();
|
|
733
|
+
return {
|
|
734
|
+
type: this.storeType,
|
|
735
|
+
vectorCount: stats.totalVectorCount ?? 0,
|
|
736
|
+
namespaceCount: Object.keys(stats.namespaces ?? {}).length,
|
|
737
|
+
dimensions: stats.dimension,
|
|
738
|
+
metric: this.metric,
|
|
739
|
+
lastUpdated: Date.now()
|
|
740
|
+
};
|
|
741
|
+
}
|
|
742
|
+
async checkHealth() {
|
|
743
|
+
const startTime = performance.now();
|
|
744
|
+
try {
|
|
745
|
+
await this.ensureInitialized();
|
|
746
|
+
await this.index.describeIndexStats();
|
|
747
|
+
return {
|
|
748
|
+
healthy: true,
|
|
749
|
+
latencyMs: performance.now() - startTime,
|
|
750
|
+
lastCheck: Date.now()
|
|
751
|
+
};
|
|
752
|
+
} catch (error) {
|
|
753
|
+
return {
|
|
754
|
+
healthy: false,
|
|
755
|
+
latencyMs: performance.now() - startTime,
|
|
756
|
+
lastCheck: Date.now(),
|
|
757
|
+
error: error.message
|
|
758
|
+
};
|
|
759
|
+
}
|
|
760
|
+
}
|
|
761
|
+
async close() {
|
|
762
|
+
this.initialized = false;
|
|
763
|
+
return Promise.resolve();
|
|
764
|
+
}
|
|
765
|
+
};
|
|
766
|
+
function createPineconeStore(config) {
|
|
767
|
+
return new PineconeStore(config);
|
|
768
|
+
}
|
|
769
|
+
|
|
770
|
+
// src/stores/ChromaStore.ts
|
|
771
|
+
var ChromaStore = class extends BaseStore {
|
|
772
|
+
storeType = "chroma";
|
|
773
|
+
client;
|
|
774
|
+
collection;
|
|
775
|
+
collectionName;
|
|
776
|
+
url;
|
|
777
|
+
initialized = false;
|
|
778
|
+
constructor(config) {
|
|
779
|
+
super(config);
|
|
780
|
+
if (!config.collectionName) {
|
|
781
|
+
throw new Error("Chroma collection name is required");
|
|
782
|
+
}
|
|
783
|
+
this.collectionName = config.collectionName;
|
|
784
|
+
this.url = config.url;
|
|
785
|
+
}
|
|
786
|
+
/**
|
|
787
|
+
* Initialize ChromaDB client
|
|
788
|
+
*/
|
|
789
|
+
async init() {
|
|
790
|
+
if (this.initialized) return;
|
|
791
|
+
try {
|
|
792
|
+
const { ChromaClient } = await import("chromadb");
|
|
793
|
+
this.client = this.url ? new ChromaClient({ path: this.url }) : new ChromaClient();
|
|
794
|
+
this.collection = await this.client.getOrCreateCollection({
|
|
795
|
+
name: this.collectionName,
|
|
796
|
+
metadata: {
|
|
797
|
+
"hnsw:space": this.metricToChroma(this.metric)
|
|
798
|
+
}
|
|
799
|
+
});
|
|
800
|
+
this.initialized = true;
|
|
801
|
+
} catch (error) {
|
|
802
|
+
throw new Error(
|
|
803
|
+
`Failed to initialize ChromaDB: ${error.message}`
|
|
804
|
+
);
|
|
805
|
+
}
|
|
806
|
+
}
|
|
807
|
+
metricToChroma(metric) {
|
|
808
|
+
switch (metric) {
|
|
809
|
+
case "cosine":
|
|
810
|
+
return "cosine";
|
|
811
|
+
case "euclidean":
|
|
812
|
+
return "l2";
|
|
813
|
+
case "dot_product":
|
|
814
|
+
return "ip";
|
|
815
|
+
default:
|
|
816
|
+
return "cosine";
|
|
817
|
+
}
|
|
818
|
+
}
|
|
819
|
+
async ensureInitialized() {
|
|
820
|
+
if (!this.initialized) {
|
|
821
|
+
await this.init();
|
|
822
|
+
}
|
|
823
|
+
}
|
|
824
|
+
async upsert(records, _options) {
|
|
825
|
+
await this.ensureInitialized();
|
|
826
|
+
const startTime = performance.now();
|
|
827
|
+
const ids = records.map((r) => r.id);
|
|
828
|
+
const embeddings = records.map((r) => r.vector);
|
|
829
|
+
const documents = records.map((r) => r.text ?? "");
|
|
830
|
+
const metadatas = records.map((r) => r.metadata ?? {});
|
|
831
|
+
const upsertedIds = [];
|
|
832
|
+
const errors = [];
|
|
833
|
+
try {
|
|
834
|
+
await this.collection.upsert({
|
|
835
|
+
ids,
|
|
836
|
+
embeddings,
|
|
837
|
+
documents,
|
|
838
|
+
metadatas
|
|
839
|
+
});
|
|
840
|
+
upsertedIds.push(...ids);
|
|
841
|
+
} catch (error) {
|
|
842
|
+
for (const id of ids) {
|
|
843
|
+
errors.push({ id, error: error.message });
|
|
844
|
+
}
|
|
845
|
+
}
|
|
846
|
+
return {
|
|
847
|
+
upsertedIds,
|
|
848
|
+
upsertedCount: upsertedIds.length,
|
|
849
|
+
errors,
|
|
850
|
+
durationMs: performance.now() - startTime
|
|
851
|
+
};
|
|
852
|
+
}
|
|
853
|
+
async query(vector, options) {
|
|
854
|
+
await this.ensureInitialized();
|
|
855
|
+
const startTime = performance.now();
|
|
856
|
+
const topK = options?.topK ?? 10;
|
|
857
|
+
const result = await this.collection.query({
|
|
858
|
+
queryEmbeddings: [vector],
|
|
859
|
+
nResults: topK,
|
|
860
|
+
where: options?.filter,
|
|
861
|
+
include: ["documents", "metadatas", "distances"]
|
|
862
|
+
});
|
|
863
|
+
const matches = (result.ids[0] ?? []).map((id, i) => {
|
|
864
|
+
const distance = result.distances?.[0]?.[i] ?? 0;
|
|
865
|
+
const score = 1 / (1 + distance);
|
|
866
|
+
return {
|
|
867
|
+
id,
|
|
868
|
+
text: result.documents?.[0]?.[i] ?? "",
|
|
869
|
+
score,
|
|
870
|
+
metadata: result.metadatas?.[0]?.[i] ?? {},
|
|
871
|
+
distance
|
|
872
|
+
};
|
|
873
|
+
});
|
|
874
|
+
const filtered = options?.minScore ? matches.filter((m) => m.score >= options.minScore) : matches;
|
|
875
|
+
return {
|
|
876
|
+
matches: filtered,
|
|
877
|
+
namespace: this.collectionName,
|
|
878
|
+
durationMs: performance.now() - startTime
|
|
879
|
+
};
|
|
880
|
+
}
|
|
881
|
+
async delete(ids, _options) {
|
|
882
|
+
await this.ensureInitialized();
|
|
883
|
+
const startTime = performance.now();
|
|
884
|
+
await this.collection.delete({ ids });
|
|
885
|
+
return {
|
|
886
|
+
deletedCount: ids.length,
|
|
887
|
+
durationMs: performance.now() - startTime
|
|
888
|
+
};
|
|
889
|
+
}
|
|
890
|
+
async deleteAll(_options) {
|
|
891
|
+
await this.ensureInitialized();
|
|
892
|
+
const startTime = performance.now();
|
|
893
|
+
await this.client.deleteCollection({ name: this.collectionName });
|
|
894
|
+
this.collection = await this.client.createCollection({
|
|
895
|
+
name: this.collectionName,
|
|
896
|
+
metadata: {
|
|
897
|
+
"hnsw:space": this.metricToChroma(this.metric)
|
|
898
|
+
}
|
|
899
|
+
});
|
|
900
|
+
return {
|
|
901
|
+
deletedCount: -1,
|
|
902
|
+
durationMs: performance.now() - startTime
|
|
903
|
+
};
|
|
904
|
+
}
|
|
905
|
+
async getStats() {
|
|
906
|
+
await this.ensureInitialized();
|
|
907
|
+
const count = await this.collection.count();
|
|
908
|
+
return {
|
|
909
|
+
type: this.storeType,
|
|
910
|
+
vectorCount: count,
|
|
911
|
+
namespaceCount: 1,
|
|
912
|
+
dimensions: this.dimensions ?? 0,
|
|
913
|
+
metric: this.metric,
|
|
914
|
+
lastUpdated: Date.now()
|
|
915
|
+
};
|
|
916
|
+
}
|
|
917
|
+
async checkHealth() {
|
|
918
|
+
const startTime = performance.now();
|
|
919
|
+
try {
|
|
920
|
+
await this.ensureInitialized();
|
|
921
|
+
await this.collection.count();
|
|
922
|
+
return {
|
|
923
|
+
healthy: true,
|
|
924
|
+
latencyMs: performance.now() - startTime,
|
|
925
|
+
lastCheck: Date.now()
|
|
926
|
+
};
|
|
927
|
+
} catch (error) {
|
|
928
|
+
return {
|
|
929
|
+
healthy: false,
|
|
930
|
+
latencyMs: performance.now() - startTime,
|
|
931
|
+
lastCheck: Date.now(),
|
|
932
|
+
error: error.message
|
|
933
|
+
};
|
|
934
|
+
}
|
|
935
|
+
}
|
|
936
|
+
async close() {
|
|
937
|
+
this.initialized = false;
|
|
938
|
+
return Promise.resolve();
|
|
939
|
+
}
|
|
940
|
+
};
|
|
941
|
+
function createChromaStore(config) {
|
|
942
|
+
return new ChromaStore(config);
|
|
943
|
+
}
|
|
944
|
+
|
|
945
|
+
// src/stores/QdrantStore.ts
|
|
946
|
+
var QdrantStore = class extends BaseStore {
|
|
947
|
+
storeType = "qdrant";
|
|
948
|
+
client;
|
|
949
|
+
collectionName;
|
|
950
|
+
url;
|
|
951
|
+
apiKey;
|
|
952
|
+
initialized = false;
|
|
953
|
+
constructor(config) {
|
|
954
|
+
super(config);
|
|
955
|
+
if (!config.url) {
|
|
956
|
+
throw new Error("Qdrant URL is required");
|
|
957
|
+
}
|
|
958
|
+
if (!config.collectionName) {
|
|
959
|
+
throw new Error("Qdrant collection name is required");
|
|
960
|
+
}
|
|
961
|
+
this.url = config.url;
|
|
962
|
+
this.collectionName = config.collectionName;
|
|
963
|
+
this.apiKey = config.apiKey;
|
|
964
|
+
}
|
|
965
|
+
/**
|
|
966
|
+
* Initialize Qdrant client
|
|
967
|
+
*/
|
|
968
|
+
async init() {
|
|
969
|
+
if (this.initialized) return;
|
|
970
|
+
try {
|
|
971
|
+
const { QdrantClient } = await import("@qdrant/js-client-rest");
|
|
972
|
+
this.client = new QdrantClient({
|
|
973
|
+
url: this.url,
|
|
974
|
+
apiKey: this.apiKey
|
|
975
|
+
});
|
|
976
|
+
const collections = await this.client.getCollections();
|
|
977
|
+
const exists = collections.collections.some(
|
|
978
|
+
(c) => c.name === this.collectionName
|
|
979
|
+
);
|
|
980
|
+
if (!exists && this.dimensions) {
|
|
981
|
+
await this.client.createCollection(this.collectionName, {
|
|
982
|
+
vectors: {
|
|
983
|
+
size: this.dimensions,
|
|
984
|
+
distance: this.metricToQdrant(this.metric)
|
|
985
|
+
}
|
|
986
|
+
});
|
|
987
|
+
}
|
|
988
|
+
this.initialized = true;
|
|
989
|
+
} catch (error) {
|
|
990
|
+
throw new Error(
|
|
991
|
+
`Failed to initialize Qdrant: ${error.message}`
|
|
992
|
+
);
|
|
993
|
+
}
|
|
994
|
+
}
|
|
995
|
+
metricToQdrant(metric) {
|
|
996
|
+
switch (metric) {
|
|
997
|
+
case "cosine":
|
|
998
|
+
return "Cosine";
|
|
999
|
+
case "euclidean":
|
|
1000
|
+
return "Euclid";
|
|
1001
|
+
case "dot_product":
|
|
1002
|
+
return "Dot";
|
|
1003
|
+
default:
|
|
1004
|
+
return "Cosine";
|
|
1005
|
+
}
|
|
1006
|
+
}
|
|
1007
|
+
async ensureInitialized() {
|
|
1008
|
+
if (!this.initialized) {
|
|
1009
|
+
await this.init();
|
|
1010
|
+
}
|
|
1011
|
+
}
|
|
1012
|
+
async upsert(records, options) {
|
|
1013
|
+
await this.ensureInitialized();
|
|
1014
|
+
const startTime = performance.now();
|
|
1015
|
+
const batchSize = options?.batchSize ?? 100;
|
|
1016
|
+
const upsertedIds = [];
|
|
1017
|
+
const errors = [];
|
|
1018
|
+
const points = records.map((record) => ({
|
|
1019
|
+
id: record.id,
|
|
1020
|
+
vector: record.vector,
|
|
1021
|
+
payload: {
|
|
1022
|
+
...record.metadata,
|
|
1023
|
+
text: record.text
|
|
1024
|
+
}
|
|
1025
|
+
}));
|
|
1026
|
+
const batches = batch(points, batchSize);
|
|
1027
|
+
let completed = 0;
|
|
1028
|
+
for (const batchPoints of batches) {
|
|
1029
|
+
try {
|
|
1030
|
+
await this.client.upsert(this.collectionName, {
|
|
1031
|
+
points: batchPoints
|
|
1032
|
+
});
|
|
1033
|
+
upsertedIds.push(...batchPoints.map((p) => p.id));
|
|
1034
|
+
} catch (error) {
|
|
1035
|
+
for (const p of batchPoints) {
|
|
1036
|
+
errors.push({ id: p.id, error: error.message });
|
|
1037
|
+
}
|
|
1038
|
+
}
|
|
1039
|
+
completed += batchPoints.length;
|
|
1040
|
+
options?.onProgress?.({ completed, total: records.length });
|
|
1041
|
+
}
|
|
1042
|
+
return {
|
|
1043
|
+
upsertedIds,
|
|
1044
|
+
upsertedCount: upsertedIds.length,
|
|
1045
|
+
errors,
|
|
1046
|
+
durationMs: performance.now() - startTime
|
|
1047
|
+
};
|
|
1048
|
+
}
|
|
1049
|
+
async query(vector, options) {
|
|
1050
|
+
await this.ensureInitialized();
|
|
1051
|
+
const startTime = performance.now();
|
|
1052
|
+
const topK = options?.topK ?? 10;
|
|
1053
|
+
const result = await this.client.search(this.collectionName, {
|
|
1054
|
+
vector,
|
|
1055
|
+
limit: topK,
|
|
1056
|
+
filter: options?.filter ? this.buildQdrantFilter(options.filter) : void 0,
|
|
1057
|
+
with_payload: options?.includeMetadata ?? true,
|
|
1058
|
+
with_vector: options?.includeVectors ?? false,
|
|
1059
|
+
score_threshold: options?.minScore
|
|
1060
|
+
});
|
|
1061
|
+
const matches = result.map((point) => ({
|
|
1062
|
+
id: point.id.toString(),
|
|
1063
|
+
text: point.payload?.text ?? "",
|
|
1064
|
+
score: point.score,
|
|
1065
|
+
metadata: point.payload ?? {}
|
|
1066
|
+
}));
|
|
1067
|
+
return {
|
|
1068
|
+
matches,
|
|
1069
|
+
namespace: this.collectionName,
|
|
1070
|
+
durationMs: performance.now() - startTime
|
|
1071
|
+
};
|
|
1072
|
+
}
|
|
1073
|
+
buildQdrantFilter(filter) {
|
|
1074
|
+
const must = [];
|
|
1075
|
+
for (const [key, value] of Object.entries(filter)) {
|
|
1076
|
+
must.push({
|
|
1077
|
+
key,
|
|
1078
|
+
match: { value }
|
|
1079
|
+
});
|
|
1080
|
+
}
|
|
1081
|
+
return { must };
|
|
1082
|
+
}
|
|
1083
|
+
async delete(ids, _options) {
|
|
1084
|
+
await this.ensureInitialized();
|
|
1085
|
+
const startTime = performance.now();
|
|
1086
|
+
await this.client.delete(this.collectionName, {
|
|
1087
|
+
points: ids
|
|
1088
|
+
});
|
|
1089
|
+
return {
|
|
1090
|
+
deletedCount: ids.length,
|
|
1091
|
+
durationMs: performance.now() - startTime
|
|
1092
|
+
};
|
|
1093
|
+
}
|
|
1094
|
+
async deleteAll(_options) {
|
|
1095
|
+
await this.ensureInitialized();
|
|
1096
|
+
const startTime = performance.now();
|
|
1097
|
+
await this.client.deleteCollection(this.collectionName);
|
|
1098
|
+
if (this.dimensions) {
|
|
1099
|
+
await this.client.createCollection(this.collectionName, {
|
|
1100
|
+
vectors: {
|
|
1101
|
+
size: this.dimensions,
|
|
1102
|
+
distance: this.metricToQdrant(this.metric)
|
|
1103
|
+
}
|
|
1104
|
+
});
|
|
1105
|
+
}
|
|
1106
|
+
return {
|
|
1107
|
+
deletedCount: -1,
|
|
1108
|
+
durationMs: performance.now() - startTime
|
|
1109
|
+
};
|
|
1110
|
+
}
|
|
1111
|
+
async getStats() {
|
|
1112
|
+
await this.ensureInitialized();
|
|
1113
|
+
const info = await this.client.getCollection(this.collectionName);
|
|
1114
|
+
return {
|
|
1115
|
+
type: this.storeType,
|
|
1116
|
+
vectorCount: info.vectors_count,
|
|
1117
|
+
namespaceCount: 1,
|
|
1118
|
+
dimensions: info.config?.params?.vectors?.size,
|
|
1119
|
+
metric: this.metric,
|
|
1120
|
+
lastUpdated: Date.now()
|
|
1121
|
+
};
|
|
1122
|
+
}
|
|
1123
|
+
async checkHealth() {
|
|
1124
|
+
const startTime = performance.now();
|
|
1125
|
+
try {
|
|
1126
|
+
await this.ensureInitialized();
|
|
1127
|
+
await this.client.getCollection(this.collectionName);
|
|
1128
|
+
return {
|
|
1129
|
+
healthy: true,
|
|
1130
|
+
latencyMs: performance.now() - startTime,
|
|
1131
|
+
lastCheck: Date.now()
|
|
1132
|
+
};
|
|
1133
|
+
} catch (error) {
|
|
1134
|
+
return {
|
|
1135
|
+
healthy: false,
|
|
1136
|
+
latencyMs: performance.now() - startTime,
|
|
1137
|
+
lastCheck: Date.now(),
|
|
1138
|
+
error: error.message
|
|
1139
|
+
};
|
|
1140
|
+
}
|
|
1141
|
+
}
|
|
1142
|
+
async close() {
|
|
1143
|
+
this.initialized = false;
|
|
1144
|
+
return Promise.resolve();
|
|
1145
|
+
}
|
|
1146
|
+
};
|
|
1147
|
+
function createQdrantStore(config) {
|
|
1148
|
+
return new QdrantStore(config);
|
|
1149
|
+
}
|
|
1150
|
+
|
|
1151
|
+
// src/stores/index.ts
|
|
1152
|
+
function createStore(type, config) {
|
|
1153
|
+
switch (type) {
|
|
1154
|
+
case "memory":
|
|
1155
|
+
return new MemoryStore(config);
|
|
1156
|
+
case "pinecone":
|
|
1157
|
+
return new PineconeStore(config);
|
|
1158
|
+
case "chroma":
|
|
1159
|
+
return new ChromaStore(config);
|
|
1160
|
+
case "qdrant":
|
|
1161
|
+
return new QdrantStore(config);
|
|
1162
|
+
default:
|
|
1163
|
+
return new MemoryStore(config);
|
|
1164
|
+
}
|
|
1165
|
+
}
|
|
1166
|
+
// Annotate the CommonJS export names for ESM import in node:
|
|
1167
|
+
0 && (module.exports = {
|
|
1168
|
+
BaseStore,
|
|
1169
|
+
ChromaStore,
|
|
1170
|
+
MemoryStore,
|
|
1171
|
+
PineconeStore,
|
|
1172
|
+
QdrantStore,
|
|
1173
|
+
createChromaStore,
|
|
1174
|
+
createMemoryStore,
|
|
1175
|
+
createPineconeStore,
|
|
1176
|
+
createQdrantStore,
|
|
1177
|
+
createStore
|
|
1178
|
+
});
|