@mhalder/qdrant-mcp-server 3.3.3 → 3.3.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.github/workflows/ci.yml +0 -2
- package/.github/workflows/claude-code-review.yml +1 -1
- package/CHANGELOG.md +6 -0
- package/README.md +1 -1
- package/biome.json +3 -2
- package/build/code/chunker/tree-sitter-chunker.d.ts.map +1 -1
- package/build/code/chunker/tree-sitter-chunker.js +2 -12
- package/build/code/chunker/tree-sitter-chunker.js.map +1 -1
- package/build/code/indexer.d.ts.map +1 -1
- package/build/code/indexer.js +12 -18
- package/build/code/indexer.js.map +1 -1
- package/build/code/scanner.js +1 -1
- package/build/code/scanner.js.map +1 -1
- package/build/embeddings/cohere.d.ts +1 -1
- package/build/embeddings/cohere.d.ts.map +1 -1
- package/build/embeddings/cohere.js +2 -2
- package/build/embeddings/cohere.js.map +1 -1
- package/build/embeddings/cohere.test.js +1 -5
- package/build/embeddings/cohere.test.js.map +1 -1
- package/build/embeddings/factory.d.ts +1 -1
- package/build/embeddings/factory.d.ts.map +1 -1
- package/build/embeddings/factory.js +7 -9
- package/build/embeddings/factory.js.map +1 -1
- package/build/embeddings/factory.test.js +3 -3
- package/build/embeddings/factory.test.js.map +1 -1
- package/build/embeddings/ollama.d.ts +1 -1
- package/build/embeddings/ollama.d.ts.map +1 -1
- package/build/embeddings/ollama.js +6 -8
- package/build/embeddings/ollama.js.map +1 -1
- package/build/embeddings/ollama.test.js +2 -6
- package/build/embeddings/ollama.test.js.map +1 -1
- package/build/embeddings/openai.d.ts +1 -1
- package/build/embeddings/openai.d.ts.map +1 -1
- package/build/embeddings/openai.js +4 -7
- package/build/embeddings/openai.js.map +1 -1
- package/build/embeddings/openai.test.js +3 -12
- package/build/embeddings/openai.test.js.map +1 -1
- package/build/embeddings/sparse.test.js +12 -2
- package/build/embeddings/sparse.test.js.map +1 -1
- package/build/embeddings/voyage.d.ts +1 -1
- package/build/embeddings/voyage.d.ts.map +1 -1
- package/build/embeddings/voyage.js +2 -3
- package/build/embeddings/voyage.js.map +1 -1
- package/build/embeddings/voyage.test.js +2 -6
- package/build/embeddings/voyage.test.js.map +1 -1
- package/build/git/chunker.d.ts.map +1 -1
- package/build/git/chunker.js +2 -2
- package/build/git/chunker.js.map +1 -1
- package/build/git/chunker.test.js +1 -1
- package/build/git/chunker.test.js.map +1 -1
- package/build/git/extractor.d.ts.map +1 -1
- package/build/git/extractor.integration.test.js +9 -5
- package/build/git/extractor.integration.test.js.map +1 -1
- package/build/git/extractor.js +1 -1
- package/build/git/extractor.js.map +1 -1
- package/build/git/extractor.test.js +2 -2
- package/build/git/extractor.test.js.map +1 -1
- package/build/git/index.d.ts +4 -4
- package/build/git/index.d.ts.map +1 -1
- package/build/git/index.js +3 -3
- package/build/git/index.js.map +1 -1
- package/build/git/indexer.d.ts.map +1 -1
- package/build/git/indexer.js +9 -21
- package/build/git/indexer.js.map +1 -1
- package/build/git/indexer.test.js +4 -8
- package/build/git/indexer.test.js.map +1 -1
- package/build/git/sync/synchronizer.d.ts.map +1 -1
- package/build/git/sync/synchronizer.js.map +1 -1
- package/build/git/sync/synchronizer.test.js +4 -2
- package/build/git/sync/synchronizer.test.js.map +1 -1
- package/build/index.js +5 -9
- package/build/index.js.map +1 -1
- package/build/index.test.js +3 -3
- package/build/index.test.js.map +1 -1
- package/build/logger.d.ts.map +1 -1
- package/build/logger.js +1 -9
- package/build/logger.js.map +1 -1
- package/build/prompts/register.d.ts.map +1 -1
- package/build/prompts/register.js.map +1 -1
- package/build/qdrant/client.d.ts.map +1 -1
- package/build/qdrant/client.js.map +1 -1
- package/build/qdrant/client.test.js +10 -34
- package/build/qdrant/client.test.js.map +1 -1
- package/build/resources/index.d.ts +1 -1
- package/build/resources/index.d.ts.map +1 -1
- package/build/resources/index.js +1 -1
- package/build/resources/index.js.map +1 -1
- package/build/tools/code.d.ts.map +1 -1
- package/build/tools/code.js +3 -9
- package/build/tools/code.js.map +1 -1
- package/build/tools/collection.d.ts.map +1 -1
- package/build/tools/collection.js +1 -3
- package/build/tools/collection.js.map +1 -1
- package/build/tools/document.d.ts.map +1 -1
- package/build/tools/document.js +1 -1
- package/build/tools/document.js.map +1 -1
- package/build/tools/federated.d.ts.map +1 -1
- package/build/tools/federated.js +15 -6
- package/build/tools/federated.js.map +1 -1
- package/build/tools/federated.test.js +18 -22
- package/build/tools/federated.test.js.map +1 -1
- package/build/tools/git-history.d.ts.map +1 -1
- package/build/tools/git-history.js +3 -7
- package/build/tools/git-history.js.map +1 -1
- package/build/tools/index.d.ts.map +1 -1
- package/build/tools/index.js.map +1 -1
- package/build/tools/logging.d.ts.map +1 -1
- package/build/tools/logging.js +1 -3
- package/build/tools/logging.js.map +1 -1
- package/build/tools/logging.test.js +1 -1
- package/build/tools/logging.test.js.map +1 -1
- package/build/tools/schemas.d.ts.map +1 -1
- package/build/tools/schemas.js +17 -64
- package/build/tools/schemas.js.map +1 -1
- package/build/tools/search.d.ts.map +1 -1
- package/build/tools/search.js +1 -1
- package/build/tools/search.js.map +1 -1
- package/commitlint.config.js +12 -23
- package/package.json +1 -1
- package/scripts/verify-providers.js +12 -32
- package/src/code/chunker/tree-sitter-chunker.ts +9 -35
- package/src/code/indexer.ts +45 -107
- package/src/code/scanner.ts +1 -1
- package/src/embeddings/cohere.test.ts +17 -45
- package/src/embeddings/cohere.ts +10 -17
- package/src/embeddings/factory.test.ts +18 -18
- package/src/embeddings/factory.ts +18 -25
- package/src/embeddings/ollama.test.ts +38 -67
- package/src/embeddings/ollama.ts +15 -27
- package/src/embeddings/openai.test.ts +17 -53
- package/src/embeddings/openai.ts +11 -22
- package/src/embeddings/sparse.test.ts +12 -2
- package/src/embeddings/voyage.test.ts +39 -80
- package/src/embeddings/voyage.ts +9 -13
- package/src/git/chunker.test.ts +1 -1
- package/src/git/chunker.ts +6 -22
- package/src/git/extractor.integration.test.ts +12 -16
- package/src/git/extractor.test.ts +21 -35
- package/src/git/extractor.ts +14 -36
- package/src/git/index.ts +9 -10
- package/src/git/indexer.test.ts +29 -57
- package/src/git/indexer.ts +38 -86
- package/src/git/sync/synchronizer.test.ts +6 -9
- package/src/git/sync/synchronizer.ts +2 -5
- package/src/index.test.ts +7 -9
- package/src/index.ts +34 -80
- package/src/logger.ts +3 -14
- package/src/prompts/register.ts +3 -10
- package/src/qdrant/client.test.ts +63 -169
- package/src/qdrant/client.ts +19 -45
- package/src/resources/index.ts +4 -10
- package/src/tools/code.ts +43 -66
- package/src/tools/collection.ts +19 -38
- package/src/tools/document.ts +10 -19
- package/src/tools/federated.test.ts +34 -57
- package/src/tools/federated.ts +88 -108
- package/src/tools/git-history.ts +32 -60
- package/src/tools/index.ts +1 -4
- package/src/tools/logging.test.ts +10 -10
- package/src/tools/logging.ts +8 -18
- package/src/tools/schemas.ts +23 -78
- package/src/tools/search.ts +77 -94
- package/tests/code/chunker/tree-sitter-chunker.test.ts +6 -19
- package/tests/code/indexer.test.ts +100 -192
- package/tests/code/integration.test.ts +61 -117
- package/tests/code/scanner.test.ts +12 -39
- package/tests/code/sync/snapshot.test.ts +4 -14
- package/tests/code/sync/synchronizer.test.ts +10 -40
|
@@ -30,9 +30,7 @@ describe("OpenAIEmbeddings", () => {
|
|
|
30
30
|
let embeddings: OpenAIEmbeddings;
|
|
31
31
|
|
|
32
32
|
beforeEach(() => {
|
|
33
|
-
mockOpenAI.embeddings.create
|
|
34
|
-
.mockReset()
|
|
35
|
-
.mockResolvedValue({ data: [{ embedding: [] }] });
|
|
33
|
+
mockOpenAI.embeddings.create.mockReset().mockResolvedValue({ data: [{ embedding: [] }] });
|
|
36
34
|
vi.mocked(OpenAI).mockClear();
|
|
37
35
|
embeddings = new OpenAIEmbeddings("test-api-key");
|
|
38
36
|
});
|
|
@@ -44,28 +42,18 @@ describe("OpenAIEmbeddings", () => {
|
|
|
44
42
|
});
|
|
45
43
|
|
|
46
44
|
it("should use custom model", () => {
|
|
47
|
-
const customEmbeddings = new OpenAIEmbeddings(
|
|
48
|
-
"test-api-key",
|
|
49
|
-
"text-embedding-3-large",
|
|
50
|
-
);
|
|
45
|
+
const customEmbeddings = new OpenAIEmbeddings("test-api-key", "text-embedding-3-large");
|
|
51
46
|
expect(customEmbeddings.getModel()).toBe("text-embedding-3-large");
|
|
52
47
|
expect(customEmbeddings.getDimensions()).toBe(3072);
|
|
53
48
|
});
|
|
54
49
|
|
|
55
50
|
it("should use custom dimensions", () => {
|
|
56
|
-
const customEmbeddings = new OpenAIEmbeddings(
|
|
57
|
-
"test-api-key",
|
|
58
|
-
"text-embedding-3-small",
|
|
59
|
-
512,
|
|
60
|
-
);
|
|
51
|
+
const customEmbeddings = new OpenAIEmbeddings("test-api-key", "text-embedding-3-small", 512);
|
|
61
52
|
expect(customEmbeddings.getDimensions()).toBe(512);
|
|
62
53
|
});
|
|
63
54
|
|
|
64
55
|
it("should use default dimensions for text-embedding-ada-002", () => {
|
|
65
|
-
const adaEmbeddings = new OpenAIEmbeddings(
|
|
66
|
-
"test-api-key",
|
|
67
|
-
"text-embedding-ada-002",
|
|
68
|
-
);
|
|
56
|
+
const adaEmbeddings = new OpenAIEmbeddings("test-api-key", "text-embedding-ada-002");
|
|
69
57
|
expect(adaEmbeddings.getDimensions()).toBe(1536);
|
|
70
58
|
});
|
|
71
59
|
});
|
|
@@ -110,11 +98,7 @@ describe("OpenAIEmbeddings", () => {
|
|
|
110
98
|
});
|
|
111
99
|
|
|
112
100
|
it("should use custom model configuration", async () => {
|
|
113
|
-
const customEmbeddings = new OpenAIEmbeddings(
|
|
114
|
-
"test-api-key",
|
|
115
|
-
"text-embedding-3-large",
|
|
116
|
-
3072,
|
|
117
|
-
);
|
|
101
|
+
const customEmbeddings = new OpenAIEmbeddings("test-api-key", "text-embedding-3-large", 3072);
|
|
118
102
|
const mockEmbedding = Array(3072).fill(0.1);
|
|
119
103
|
mockOpenAI.embeddings.create.mockResolvedValue({
|
|
120
104
|
data: [{ embedding: mockEmbedding }],
|
|
@@ -138,11 +122,7 @@ describe("OpenAIEmbeddings", () => {
|
|
|
138
122
|
|
|
139
123
|
describe("embedBatch", () => {
|
|
140
124
|
it("should generate embeddings for multiple texts", async () => {
|
|
141
|
-
const mockEmbeddings = [
|
|
142
|
-
Array(1536).fill(0.1),
|
|
143
|
-
Array(1536).fill(0.2),
|
|
144
|
-
Array(1536).fill(0.3),
|
|
145
|
-
];
|
|
125
|
+
const mockEmbeddings = [Array(1536).fill(0.1), Array(1536).fill(0.2), Array(1536).fill(0.3)];
|
|
146
126
|
mockOpenAI.embeddings.create.mockResolvedValue({
|
|
147
127
|
data: [
|
|
148
128
|
{ embedding: mockEmbeddings[0] },
|
|
@@ -207,13 +187,9 @@ describe("OpenAIEmbeddings", () => {
|
|
|
207
187
|
});
|
|
208
188
|
|
|
209
189
|
it("should propagate errors in batch", async () => {
|
|
210
|
-
mockOpenAI.embeddings.create.mockRejectedValue(
|
|
211
|
-
new Error("Batch API Error"),
|
|
212
|
-
);
|
|
190
|
+
mockOpenAI.embeddings.create.mockRejectedValue(new Error("Batch API Error"));
|
|
213
191
|
|
|
214
|
-
await expect(embeddings.embedBatch(["text1", "text2"])).rejects.toThrow(
|
|
215
|
-
"Batch API Error",
|
|
216
|
-
);
|
|
192
|
+
await expect(embeddings.embedBatch(["text1", "text2"])).rejects.toThrow("Batch API Error");
|
|
217
193
|
});
|
|
218
194
|
});
|
|
219
195
|
|
|
@@ -223,11 +199,7 @@ describe("OpenAIEmbeddings", () => {
|
|
|
223
199
|
});
|
|
224
200
|
|
|
225
201
|
it("should return custom dimensions", () => {
|
|
226
|
-
const customEmbeddings = new OpenAIEmbeddings(
|
|
227
|
-
"test-api-key",
|
|
228
|
-
"text-embedding-3-small",
|
|
229
|
-
512,
|
|
230
|
-
);
|
|
202
|
+
const customEmbeddings = new OpenAIEmbeddings("test-api-key", "text-embedding-3-small", 512);
|
|
231
203
|
expect(customEmbeddings.getDimensions()).toBe(512);
|
|
232
204
|
});
|
|
233
205
|
});
|
|
@@ -238,10 +210,7 @@ describe("OpenAIEmbeddings", () => {
|
|
|
238
210
|
});
|
|
239
211
|
|
|
240
212
|
it("should return custom model", () => {
|
|
241
|
-
const customEmbeddings = new OpenAIEmbeddings(
|
|
242
|
-
"test-api-key",
|
|
243
|
-
"text-embedding-3-large",
|
|
244
|
-
);
|
|
213
|
+
const customEmbeddings = new OpenAIEmbeddings("test-api-key", "text-embedding-3-large");
|
|
245
214
|
expect(customEmbeddings.getModel()).toBe("text-embedding-3-large");
|
|
246
215
|
});
|
|
247
216
|
});
|
|
@@ -290,7 +259,7 @@ describe("OpenAIEmbeddings", () => {
|
|
|
290
259
|
{
|
|
291
260
|
retryAttempts: 2,
|
|
292
261
|
retryDelayMs: 100, // 100ms for faster tests
|
|
293
|
-
}
|
|
262
|
+
}
|
|
294
263
|
);
|
|
295
264
|
|
|
296
265
|
const mockEmbedding = Array(1536).fill(0.5);
|
|
@@ -323,7 +292,7 @@ describe("OpenAIEmbeddings", () => {
|
|
|
323
292
|
{
|
|
324
293
|
retryAttempts: 3,
|
|
325
294
|
retryDelayMs: 100, // 100ms for faster tests
|
|
326
|
-
}
|
|
295
|
+
}
|
|
327
296
|
);
|
|
328
297
|
|
|
329
298
|
const mockEmbedding = Array(1536).fill(0.5);
|
|
@@ -353,7 +322,7 @@ describe("OpenAIEmbeddings", () => {
|
|
|
353
322
|
{
|
|
354
323
|
retryAttempts: 2,
|
|
355
324
|
retryDelayMs: 100,
|
|
356
|
-
}
|
|
325
|
+
}
|
|
357
326
|
);
|
|
358
327
|
|
|
359
328
|
const rateLimitError = {
|
|
@@ -364,7 +333,7 @@ describe("OpenAIEmbeddings", () => {
|
|
|
364
333
|
mockOpenAI.embeddings.create.mockRejectedValue(rateLimitError);
|
|
365
334
|
|
|
366
335
|
await expect(rateLimitEmbeddings.embed("test text")).rejects.toThrow(
|
|
367
|
-
"OpenAI API rate limit exceeded after 2 retry attempts"
|
|
336
|
+
"OpenAI API rate limit exceeded after 2 retry attempts"
|
|
368
337
|
);
|
|
369
338
|
|
|
370
339
|
// Should try initial + 2 retries = 3 total attempts
|
|
@@ -377,10 +346,7 @@ describe("OpenAIEmbeddings", () => {
|
|
|
377
346
|
mockOpenAI.embeddings.create
|
|
378
347
|
.mockRejectedValueOnce({ status: 429, message: "Rate limit exceeded" })
|
|
379
348
|
.mockResolvedValue({
|
|
380
|
-
data: [
|
|
381
|
-
{ embedding: mockEmbeddings[0] },
|
|
382
|
-
{ embedding: mockEmbeddings[1] },
|
|
383
|
-
],
|
|
349
|
+
data: [{ embedding: mockEmbeddings[0] }, { embedding: mockEmbeddings[1] }],
|
|
384
350
|
});
|
|
385
351
|
|
|
386
352
|
const results = await embeddings.embedBatch(["text1", "text2"]);
|
|
@@ -393,9 +359,7 @@ describe("OpenAIEmbeddings", () => {
|
|
|
393
359
|
const apiError = new Error("Invalid API key");
|
|
394
360
|
mockOpenAI.embeddings.create.mockRejectedValue(apiError);
|
|
395
361
|
|
|
396
|
-
await expect(embeddings.embed("test text")).rejects.toThrow(
|
|
397
|
-
"Invalid API key",
|
|
398
|
-
);
|
|
362
|
+
await expect(embeddings.embed("test text")).rejects.toThrow("Invalid API key");
|
|
399
363
|
expect(mockOpenAI.embeddings.create).toHaveBeenCalledTimes(1);
|
|
400
364
|
});
|
|
401
365
|
|
|
@@ -408,7 +372,7 @@ describe("OpenAIEmbeddings", () => {
|
|
|
408
372
|
maxRequestsPerMinute: 1000,
|
|
409
373
|
retryAttempts: 5,
|
|
410
374
|
retryDelayMs: 2000,
|
|
411
|
-
}
|
|
375
|
+
}
|
|
412
376
|
);
|
|
413
377
|
|
|
414
378
|
expect(customEmbeddings).toBeDefined();
|
package/src/embeddings/openai.ts
CHANGED
|
@@ -1,12 +1,7 @@
|
|
|
1
|
-
import OpenAI from "openai";
|
|
2
1
|
import Bottleneck from "bottleneck";
|
|
2
|
+
import OpenAI from "openai";
|
|
3
3
|
import logger from "../logger.js";
|
|
4
|
-
import {
|
|
5
|
-
EmbeddingProvider,
|
|
6
|
-
EmbeddingResult,
|
|
7
|
-
RateLimitConfig,
|
|
8
|
-
ProviderConfig,
|
|
9
|
-
} from "./base.js";
|
|
4
|
+
import type { EmbeddingProvider, EmbeddingResult, RateLimitConfig } from "./base.js";
|
|
10
5
|
|
|
11
6
|
interface OpenAIError {
|
|
12
7
|
status?: number;
|
|
@@ -31,7 +26,7 @@ export class OpenAIEmbeddings implements EmbeddingProvider {
|
|
|
31
26
|
apiKey: string,
|
|
32
27
|
model: string = "text-embedding-3-small",
|
|
33
28
|
dimensions?: number,
|
|
34
|
-
rateLimitConfig?: RateLimitConfig
|
|
29
|
+
rateLimitConfig?: RateLimitConfig
|
|
35
30
|
) {
|
|
36
31
|
this.client = new OpenAI({ apiKey });
|
|
37
32
|
this.model = model;
|
|
@@ -64,10 +59,7 @@ export class OpenAIEmbeddings implements EmbeddingProvider {
|
|
|
64
59
|
});
|
|
65
60
|
}
|
|
66
61
|
|
|
67
|
-
private async retryWithBackoff<T>(
|
|
68
|
-
fn: () => Promise<T>,
|
|
69
|
-
attempt: number = 0,
|
|
70
|
-
): Promise<T> {
|
|
62
|
+
private async retryWithBackoff<T>(fn: () => Promise<T>, attempt: number = 0): Promise<T> {
|
|
71
63
|
try {
|
|
72
64
|
return await fn();
|
|
73
65
|
} catch (error: unknown) {
|
|
@@ -80,20 +72,17 @@ export class OpenAIEmbeddings implements EmbeddingProvider {
|
|
|
80
72
|
if (isRateLimitError && attempt < this.retryAttempts) {
|
|
81
73
|
// Check for Retry-After header (different HTTP clients may nest differently)
|
|
82
74
|
const retryAfter =
|
|
83
|
-
apiError?.response?.headers?.["retry-after"] ||
|
|
84
|
-
apiError?.headers?.["retry-after"];
|
|
75
|
+
apiError?.response?.headers?.["retry-after"] || apiError?.headers?.["retry-after"];
|
|
85
76
|
let delayMs: number;
|
|
86
77
|
|
|
87
78
|
if (retryAfter) {
|
|
88
79
|
// Use Retry-After header if available (in seconds)
|
|
89
80
|
const parsed = parseInt(retryAfter, 10);
|
|
90
81
|
delayMs =
|
|
91
|
-
!isNaN(parsed) && parsed > 0
|
|
92
|
-
? parsed * 1000
|
|
93
|
-
: this.retryDelayMs * Math.pow(2, attempt);
|
|
82
|
+
!Number.isNaN(parsed) && parsed > 0 ? parsed * 1000 : this.retryDelayMs * 2 ** attempt;
|
|
94
83
|
} else {
|
|
95
84
|
// Exponential backoff: 1s, 2s, 4s, 8s...
|
|
96
|
-
delayMs = this.retryDelayMs *
|
|
85
|
+
delayMs = this.retryDelayMs * 2 ** attempt;
|
|
97
86
|
}
|
|
98
87
|
|
|
99
88
|
const waitTimeSeconds = (delayMs / 1000).toFixed(1);
|
|
@@ -103,7 +92,7 @@ export class OpenAIEmbeddings implements EmbeddingProvider {
|
|
|
103
92
|
attempt: attempt + 1,
|
|
104
93
|
maxAttempts: this.retryAttempts,
|
|
105
94
|
},
|
|
106
|
-
"Rate limit reached, retrying"
|
|
95
|
+
"Rate limit reached, retrying"
|
|
107
96
|
);
|
|
108
97
|
|
|
109
98
|
await new Promise((resolve) => setTimeout(resolve, delayMs));
|
|
@@ -113,7 +102,7 @@ export class OpenAIEmbeddings implements EmbeddingProvider {
|
|
|
113
102
|
// If not a rate limit error or max retries exceeded, throw
|
|
114
103
|
if (isRateLimitError) {
|
|
115
104
|
throw new Error(
|
|
116
|
-
`OpenAI API rate limit exceeded after ${this.retryAttempts} retry attempts. Please try again later or reduce request frequency
|
|
105
|
+
`OpenAI API rate limit exceeded after ${this.retryAttempts} retry attempts. Please try again later or reduce request frequency.`
|
|
117
106
|
);
|
|
118
107
|
}
|
|
119
108
|
|
|
@@ -134,7 +123,7 @@ export class OpenAIEmbeddings implements EmbeddingProvider {
|
|
|
134
123
|
embedding: response.data[0].embedding,
|
|
135
124
|
dimensions: this.dimensions,
|
|
136
125
|
};
|
|
137
|
-
})
|
|
126
|
+
})
|
|
138
127
|
);
|
|
139
128
|
}
|
|
140
129
|
|
|
@@ -152,7 +141,7 @@ export class OpenAIEmbeddings implements EmbeddingProvider {
|
|
|
152
141
|
embedding: item.embedding,
|
|
153
142
|
dimensions: this.dimensions,
|
|
154
143
|
}));
|
|
155
|
-
})
|
|
144
|
+
})
|
|
156
145
|
);
|
|
157
146
|
}
|
|
158
147
|
|
|
@@ -124,8 +124,18 @@ describe("BM25SparseVectorGenerator", () => {
|
|
|
124
124
|
it("should map different tokens to different indices (within hash collision tolerance)", () => {
|
|
125
125
|
const generator = new BM25SparseVectorGenerator();
|
|
126
126
|
// Use a set of clearly distinct tokens
|
|
127
|
-
const tokens = [
|
|
128
|
-
"
|
|
127
|
+
const tokens = [
|
|
128
|
+
"apple",
|
|
129
|
+
"banana",
|
|
130
|
+
"cherry",
|
|
131
|
+
"dragon",
|
|
132
|
+
"elephant",
|
|
133
|
+
"flamingo",
|
|
134
|
+
"giraffe",
|
|
135
|
+
"helicopter",
|
|
136
|
+
"igloo",
|
|
137
|
+
"jungle",
|
|
138
|
+
];
|
|
129
139
|
|
|
130
140
|
const indices = new Set<number>();
|
|
131
141
|
for (const token of tokens) {
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { beforeEach, describe, expect, it, vi } from "vitest";
|
|
2
2
|
import { VoyageEmbeddings } from "./voyage.js";
|
|
3
3
|
|
|
4
4
|
// Mock fetch globally
|
|
@@ -34,20 +34,13 @@ describe("VoyageEmbeddings", () => {
|
|
|
34
34
|
});
|
|
35
35
|
|
|
36
36
|
it("should use custom model", () => {
|
|
37
|
-
const customEmbeddings = new VoyageEmbeddings(
|
|
38
|
-
"test-api-key",
|
|
39
|
-
"voyage-large-2",
|
|
40
|
-
);
|
|
37
|
+
const customEmbeddings = new VoyageEmbeddings("test-api-key", "voyage-large-2");
|
|
41
38
|
expect(customEmbeddings.getModel()).toBe("voyage-large-2");
|
|
42
39
|
expect(customEmbeddings.getDimensions()).toBe(1536);
|
|
43
40
|
});
|
|
44
41
|
|
|
45
42
|
it("should use custom dimensions", () => {
|
|
46
|
-
const customEmbeddings = new VoyageEmbeddings(
|
|
47
|
-
"test-api-key",
|
|
48
|
-
"voyage-2",
|
|
49
|
-
512,
|
|
50
|
-
);
|
|
43
|
+
const customEmbeddings = new VoyageEmbeddings("test-api-key", "voyage-2", 512);
|
|
51
44
|
expect(customEmbeddings.getDimensions()).toBe(512);
|
|
52
45
|
});
|
|
53
46
|
|
|
@@ -62,16 +55,13 @@ describe("VoyageEmbeddings", () => {
|
|
|
62
55
|
"voyage-2",
|
|
63
56
|
undefined,
|
|
64
57
|
undefined,
|
|
65
|
-
"https://custom.voyage.com"
|
|
58
|
+
"https://custom.voyage.com"
|
|
66
59
|
);
|
|
67
60
|
expect(customEmbeddings).toBeDefined();
|
|
68
61
|
});
|
|
69
62
|
|
|
70
63
|
it("should default to 1024 for unknown models", () => {
|
|
71
|
-
const unknownEmbeddings = new VoyageEmbeddings(
|
|
72
|
-
"test-api-key",
|
|
73
|
-
"custom-model",
|
|
74
|
-
);
|
|
64
|
+
const unknownEmbeddings = new VoyageEmbeddings("test-api-key", "custom-model");
|
|
75
65
|
expect(unknownEmbeddings.getDimensions()).toBe(1024);
|
|
76
66
|
});
|
|
77
67
|
|
|
@@ -82,7 +72,7 @@ describe("VoyageEmbeddings", () => {
|
|
|
82
72
|
undefined,
|
|
83
73
|
undefined,
|
|
84
74
|
undefined,
|
|
85
|
-
"query"
|
|
75
|
+
"query"
|
|
86
76
|
);
|
|
87
77
|
expect(queryEmbeddings).toBeInstanceOf(VoyageEmbeddings);
|
|
88
78
|
});
|
|
@@ -108,20 +98,17 @@ describe("VoyageEmbeddings", () => {
|
|
|
108
98
|
embedding: mockEmbedding,
|
|
109
99
|
dimensions: 1024,
|
|
110
100
|
});
|
|
111
|
-
expect(mockFetch).toHaveBeenCalledWith(
|
|
112
|
-
"
|
|
113
|
-
{
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
"Content-Type": "application/json",
|
|
117
|
-
Authorization: "Bearer test-api-key",
|
|
118
|
-
},
|
|
119
|
-
body: JSON.stringify({
|
|
120
|
-
input: ["test text"],
|
|
121
|
-
model: "voyage-2",
|
|
122
|
-
}),
|
|
101
|
+
expect(mockFetch).toHaveBeenCalledWith("https://api.voyageai.com/v1/embeddings", {
|
|
102
|
+
method: "POST",
|
|
103
|
+
headers: {
|
|
104
|
+
"Content-Type": "application/json",
|
|
105
|
+
Authorization: "Bearer test-api-key",
|
|
123
106
|
},
|
|
124
|
-
|
|
107
|
+
body: JSON.stringify({
|
|
108
|
+
input: ["test text"],
|
|
109
|
+
model: "voyage-2",
|
|
110
|
+
}),
|
|
111
|
+
});
|
|
125
112
|
});
|
|
126
113
|
|
|
127
114
|
it("should include input_type when specified", async () => {
|
|
@@ -131,7 +118,7 @@ describe("VoyageEmbeddings", () => {
|
|
|
131
118
|
undefined,
|
|
132
119
|
undefined,
|
|
133
120
|
undefined,
|
|
134
|
-
"query"
|
|
121
|
+
"query"
|
|
135
122
|
);
|
|
136
123
|
|
|
137
124
|
const mockEmbedding = Array(1024).fill(0.5);
|
|
@@ -173,7 +160,7 @@ describe("VoyageEmbeddings", () => {
|
|
|
173
160
|
"voyage-2",
|
|
174
161
|
undefined,
|
|
175
162
|
undefined,
|
|
176
|
-
"https://custom.voyage.com/v1"
|
|
163
|
+
"https://custom.voyage.com/v1"
|
|
177
164
|
);
|
|
178
165
|
|
|
179
166
|
const mockEmbedding = Array(1024).fill(0.1);
|
|
@@ -190,7 +177,7 @@ describe("VoyageEmbeddings", () => {
|
|
|
190
177
|
|
|
191
178
|
expect(mockFetch).toHaveBeenCalledWith(
|
|
192
179
|
"https://custom.voyage.com/v1/embeddings",
|
|
193
|
-
expect.any(Object)
|
|
180
|
+
expect.any(Object)
|
|
194
181
|
);
|
|
195
182
|
});
|
|
196
183
|
|
|
@@ -205,7 +192,7 @@ describe("VoyageEmbeddings", () => {
|
|
|
205
192
|
});
|
|
206
193
|
|
|
207
194
|
await expect(embeddings.embed("test")).rejects.toThrow(
|
|
208
|
-
"No embedding returned from Voyage AI API"
|
|
195
|
+
"No embedding returned from Voyage AI API"
|
|
209
196
|
);
|
|
210
197
|
});
|
|
211
198
|
|
|
@@ -228,11 +215,7 @@ describe("VoyageEmbeddings", () => {
|
|
|
228
215
|
|
|
229
216
|
describe("embedBatch", () => {
|
|
230
217
|
it("should generate embeddings for multiple texts", async () => {
|
|
231
|
-
const mockEmbeddings = [
|
|
232
|
-
Array(1024).fill(0.1),
|
|
233
|
-
Array(1024).fill(0.2),
|
|
234
|
-
Array(1024).fill(0.3),
|
|
235
|
-
];
|
|
218
|
+
const mockEmbeddings = [Array(1024).fill(0.1), Array(1024).fill(0.2), Array(1024).fill(0.3)];
|
|
236
219
|
mockFetch.mockResolvedValue({
|
|
237
220
|
ok: true,
|
|
238
221
|
json: async () => ({
|
|
@@ -320,16 +303,14 @@ describe("VoyageEmbeddings", () => {
|
|
|
320
303
|
});
|
|
321
304
|
|
|
322
305
|
await expect(embeddings.embedBatch(["text1"])).rejects.toThrow(
|
|
323
|
-
"No embeddings returned from Voyage AI API"
|
|
306
|
+
"No embeddings returned from Voyage AI API"
|
|
324
307
|
);
|
|
325
308
|
});
|
|
326
309
|
|
|
327
310
|
it("should propagate errors in batch", async () => {
|
|
328
311
|
mockFetch.mockRejectedValue(new Error("Batch API Error"));
|
|
329
312
|
|
|
330
|
-
await expect(embeddings.embedBatch(["text1", "text2"])).rejects.toThrow(
|
|
331
|
-
"Batch API Error",
|
|
332
|
-
);
|
|
313
|
+
await expect(embeddings.embedBatch(["text1", "text2"])).rejects.toThrow("Batch API Error");
|
|
333
314
|
});
|
|
334
315
|
});
|
|
335
316
|
|
|
@@ -339,11 +320,7 @@ describe("VoyageEmbeddings", () => {
|
|
|
339
320
|
});
|
|
340
321
|
|
|
341
322
|
it("should return custom dimensions", () => {
|
|
342
|
-
const customEmbeddings = new VoyageEmbeddings(
|
|
343
|
-
"test-api-key",
|
|
344
|
-
"voyage-2",
|
|
345
|
-
512,
|
|
346
|
-
);
|
|
323
|
+
const customEmbeddings = new VoyageEmbeddings("test-api-key", "voyage-2", 512);
|
|
347
324
|
expect(customEmbeddings.getDimensions()).toBe(512);
|
|
348
325
|
});
|
|
349
326
|
});
|
|
@@ -354,10 +331,7 @@ describe("VoyageEmbeddings", () => {
|
|
|
354
331
|
});
|
|
355
332
|
|
|
356
333
|
it("should return custom model", () => {
|
|
357
|
-
const customEmbeddings = new VoyageEmbeddings(
|
|
358
|
-
"test-api-key",
|
|
359
|
-
"voyage-large-2",
|
|
360
|
-
);
|
|
334
|
+
const customEmbeddings = new VoyageEmbeddings("test-api-key", "voyage-large-2");
|
|
361
335
|
expect(customEmbeddings.getModel()).toBe("voyage-large-2");
|
|
362
336
|
});
|
|
363
337
|
});
|
|
@@ -415,15 +389,10 @@ describe("VoyageEmbeddings", () => {
|
|
|
415
389
|
});
|
|
416
390
|
|
|
417
391
|
it("should use exponential backoff", async () => {
|
|
418
|
-
const rateLimitEmbeddings = new VoyageEmbeddings(
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
{
|
|
423
|
-
retryAttempts: 3,
|
|
424
|
-
retryDelayMs: 100,
|
|
425
|
-
},
|
|
426
|
-
);
|
|
392
|
+
const rateLimitEmbeddings = new VoyageEmbeddings("test-api-key", "voyage-2", undefined, {
|
|
393
|
+
retryAttempts: 3,
|
|
394
|
+
retryDelayMs: 100,
|
|
395
|
+
});
|
|
427
396
|
|
|
428
397
|
const mockEmbedding = Array(1024).fill(0.5);
|
|
429
398
|
|
|
@@ -456,15 +425,10 @@ describe("VoyageEmbeddings", () => {
|
|
|
456
425
|
});
|
|
457
426
|
|
|
458
427
|
it("should throw error after max retries exceeded", async () => {
|
|
459
|
-
const rateLimitEmbeddings = new VoyageEmbeddings(
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
|
|
463
|
-
{
|
|
464
|
-
retryAttempts: 2,
|
|
465
|
-
retryDelayMs: 100,
|
|
466
|
-
},
|
|
467
|
-
);
|
|
428
|
+
const rateLimitEmbeddings = new VoyageEmbeddings("test-api-key", "voyage-2", undefined, {
|
|
429
|
+
retryAttempts: 2,
|
|
430
|
+
retryDelayMs: 100,
|
|
431
|
+
});
|
|
468
432
|
|
|
469
433
|
mockFetch.mockResolvedValue({
|
|
470
434
|
ok: false,
|
|
@@ -473,7 +437,7 @@ describe("VoyageEmbeddings", () => {
|
|
|
473
437
|
});
|
|
474
438
|
|
|
475
439
|
await expect(rateLimitEmbeddings.embed("test text")).rejects.toThrow(
|
|
476
|
-
"Voyage AI API rate limit exceeded after 2 retry attempts"
|
|
440
|
+
"Voyage AI API rate limit exceeded after 2 retry attempts"
|
|
477
441
|
);
|
|
478
442
|
|
|
479
443
|
expect(mockFetch).toHaveBeenCalledTimes(3);
|
|
@@ -515,16 +479,11 @@ describe("VoyageEmbeddings", () => {
|
|
|
515
479
|
});
|
|
516
480
|
|
|
517
481
|
it("should accept custom rate limit configuration", () => {
|
|
518
|
-
const customEmbeddings = new VoyageEmbeddings(
|
|
519
|
-
|
|
520
|
-
|
|
521
|
-
|
|
522
|
-
|
|
523
|
-
maxRequestsPerMinute: 500,
|
|
524
|
-
retryAttempts: 5,
|
|
525
|
-
retryDelayMs: 2000,
|
|
526
|
-
},
|
|
527
|
-
);
|
|
482
|
+
const customEmbeddings = new VoyageEmbeddings("test-api-key", "voyage-2", undefined, {
|
|
483
|
+
maxRequestsPerMinute: 500,
|
|
484
|
+
retryAttempts: 5,
|
|
485
|
+
retryDelayMs: 2000,
|
|
486
|
+
});
|
|
528
487
|
|
|
529
488
|
expect(customEmbeddings).toBeDefined();
|
|
530
489
|
});
|
package/src/embeddings/voyage.ts
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import Bottleneck from "bottleneck";
|
|
2
2
|
import logger from "../logger.js";
|
|
3
|
-
import { EmbeddingProvider, EmbeddingResult, RateLimitConfig } from "./base.js";
|
|
3
|
+
import type { EmbeddingProvider, EmbeddingResult, RateLimitConfig } from "./base.js";
|
|
4
4
|
|
|
5
5
|
interface VoyageError {
|
|
6
6
|
status?: number;
|
|
@@ -32,7 +32,7 @@ export class VoyageEmbeddings implements EmbeddingProvider {
|
|
|
32
32
|
dimensions?: number,
|
|
33
33
|
rateLimitConfig?: RateLimitConfig,
|
|
34
34
|
baseUrl: string = "https://api.voyageai.com/v1",
|
|
35
|
-
inputType?: "query" | "document"
|
|
35
|
+
inputType?: "query" | "document"
|
|
36
36
|
) {
|
|
37
37
|
this.apiKey = apiKey;
|
|
38
38
|
this.model = model;
|
|
@@ -63,20 +63,16 @@ export class VoyageEmbeddings implements EmbeddingProvider {
|
|
|
63
63
|
});
|
|
64
64
|
}
|
|
65
65
|
|
|
66
|
-
private async retryWithBackoff<T>(
|
|
67
|
-
fn: () => Promise<T>,
|
|
68
|
-
attempt: number = 0,
|
|
69
|
-
): Promise<T> {
|
|
66
|
+
private async retryWithBackoff<T>(fn: () => Promise<T>, attempt: number = 0): Promise<T> {
|
|
70
67
|
try {
|
|
71
68
|
return await fn();
|
|
72
69
|
} catch (error: unknown) {
|
|
73
70
|
const apiError = error as VoyageError;
|
|
74
71
|
const isRateLimitError =
|
|
75
|
-
apiError?.status === 429 ||
|
|
76
|
-
apiError?.message?.toLowerCase().includes("rate limit");
|
|
72
|
+
apiError?.status === 429 || apiError?.message?.toLowerCase().includes("rate limit");
|
|
77
73
|
|
|
78
74
|
if (isRateLimitError && attempt < this.retryAttempts) {
|
|
79
|
-
const delayMs = this.retryDelayMs *
|
|
75
|
+
const delayMs = this.retryDelayMs * 2 ** attempt;
|
|
80
76
|
const waitTimeSeconds = (delayMs / 1000).toFixed(1);
|
|
81
77
|
this.log.warn(
|
|
82
78
|
{
|
|
@@ -84,7 +80,7 @@ export class VoyageEmbeddings implements EmbeddingProvider {
|
|
|
84
80
|
attempt: attempt + 1,
|
|
85
81
|
maxAttempts: this.retryAttempts,
|
|
86
82
|
},
|
|
87
|
-
"Rate limit reached, retrying"
|
|
83
|
+
"Rate limit reached, retrying"
|
|
88
84
|
);
|
|
89
85
|
|
|
90
86
|
await new Promise((resolve) => setTimeout(resolve, delayMs));
|
|
@@ -93,7 +89,7 @@ export class VoyageEmbeddings implements EmbeddingProvider {
|
|
|
93
89
|
|
|
94
90
|
if (isRateLimitError) {
|
|
95
91
|
throw new Error(
|
|
96
|
-
`Voyage AI API rate limit exceeded after ${this.retryAttempts} retry attempts. Please try again later or reduce request frequency
|
|
92
|
+
`Voyage AI API rate limit exceeded after ${this.retryAttempts} retry attempts. Please try again later or reduce request frequency.`
|
|
97
93
|
);
|
|
98
94
|
}
|
|
99
95
|
|
|
@@ -144,7 +140,7 @@ export class VoyageEmbeddings implements EmbeddingProvider {
|
|
|
144
140
|
embedding: response.data[0].embedding,
|
|
145
141
|
dimensions: this.dimensions,
|
|
146
142
|
};
|
|
147
|
-
})
|
|
143
|
+
})
|
|
148
144
|
);
|
|
149
145
|
}
|
|
150
146
|
|
|
@@ -162,7 +158,7 @@ export class VoyageEmbeddings implements EmbeddingProvider {
|
|
|
162
158
|
embedding: item.embedding,
|
|
163
159
|
dimensions: this.dimensions,
|
|
164
160
|
}));
|
|
165
|
-
})
|
|
161
|
+
})
|
|
166
162
|
);
|
|
167
163
|
}
|
|
168
164
|
|
package/src/git/chunker.test.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { beforeEach, describe, expect, it } from "vitest";
|
|
2
2
|
import { CommitChunker } from "./chunker.js";
|
|
3
3
|
import { DEFAULT_GIT_CONFIG } from "./config.js";
|
|
4
4
|
import type { GitConfig, RawCommit } from "./types.js";
|
package/src/git/chunker.ts
CHANGED
|
@@ -32,22 +32,14 @@ export class CommitChunker {
|
|
|
32
32
|
* Currently produces one chunk per commit, but could be extended
|
|
33
33
|
* to handle very large commits differently
|
|
34
34
|
*/
|
|
35
|
-
createChunks(
|
|
36
|
-
commit: RawCommit,
|
|
37
|
-
repoPath: string,
|
|
38
|
-
diff?: string,
|
|
39
|
-
): CommitChunk[] {
|
|
35
|
+
createChunks(commit: RawCommit, repoPath: string, diff?: string): CommitChunk[] {
|
|
40
36
|
const commitType = this.classifyCommitType(commit);
|
|
41
37
|
const content = this.formatChunkContent(commit, commitType, diff);
|
|
42
38
|
|
|
43
39
|
// Check if content exceeds max chunk size
|
|
44
40
|
if (content.length > this.config.maxChunkSize) {
|
|
45
41
|
// Truncate content but keep essential metadata visible
|
|
46
|
-
const truncatedContent = this.truncateContent(
|
|
47
|
-
content,
|
|
48
|
-
commit,
|
|
49
|
-
commitType,
|
|
50
|
-
);
|
|
42
|
+
const truncatedContent = this.truncateContent(content, commit, commitType);
|
|
51
43
|
return [
|
|
52
44
|
{
|
|
53
45
|
content: truncatedContent,
|
|
@@ -77,11 +69,7 @@ export class CommitChunker {
|
|
|
77
69
|
/**
|
|
78
70
|
* Format the chunk content for embedding
|
|
79
71
|
*/
|
|
80
|
-
private formatChunkContent(
|
|
81
|
-
commit: RawCommit,
|
|
82
|
-
commitType: CommitType,
|
|
83
|
-
diff?: string,
|
|
84
|
-
): string {
|
|
72
|
+
private formatChunkContent(commit: RawCommit, commitType: CommitType, diff?: string): string {
|
|
85
73
|
const lines: string[] = [];
|
|
86
74
|
|
|
87
75
|
// Header section
|
|
@@ -181,11 +169,7 @@ export class CommitChunker {
|
|
|
181
169
|
/**
|
|
182
170
|
* Truncate content while preserving essential information
|
|
183
171
|
*/
|
|
184
|
-
private truncateContent(
|
|
185
|
-
content: string,
|
|
186
|
-
commit: RawCommit,
|
|
187
|
-
commitType: CommitType,
|
|
188
|
-
): string {
|
|
172
|
+
private truncateContent(_content: string, commit: RawCommit, commitType: CommitType): string {
|
|
189
173
|
// Keep the header and subject, truncate the rest
|
|
190
174
|
const essentialLines: string[] = [
|
|
191
175
|
`Commit: ${commit.shortHash}`,
|
|
@@ -203,7 +187,7 @@ export class CommitChunker {
|
|
|
203
187
|
const body = commit.body.trim();
|
|
204
188
|
if (body.length > maxBodyLength) {
|
|
205
189
|
essentialLines.push("Description:");
|
|
206
|
-
essentialLines.push(body.substring(0, maxBodyLength)
|
|
190
|
+
essentialLines.push(`${body.substring(0, maxBodyLength)}...`);
|
|
207
191
|
essentialLines.push("");
|
|
208
192
|
} else {
|
|
209
193
|
essentialLines.push("Description:");
|
|
@@ -237,7 +221,7 @@ export class CommitChunker {
|
|
|
237
221
|
private createMetadata(
|
|
238
222
|
commit: RawCommit,
|
|
239
223
|
commitType: CommitType,
|
|
240
|
-
repoPath: string
|
|
224
|
+
repoPath: string
|
|
241
225
|
): CommitChunk["metadata"] {
|
|
242
226
|
return {
|
|
243
227
|
commitHash: commit.hash,
|