@mcampa/ai-context-core 0.0.2-beta.2250c7e → 0.0.2-beta.a9337db
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +47 -55
- package/dist/.tsbuildinfo +1 -1
- package/dist/context.d.ts +5 -30
- package/dist/context.d.ts.map +1 -1
- package/dist/context.js +16 -160
- package/dist/context.js.map +1 -1
- package/dist/embedding/base-embedding.d.ts.map +1 -1
- package/dist/embedding/base-embedding.js +0 -4
- package/dist/embedding/base-embedding.js.map +1 -1
- package/dist/embedding/gemini-embedding.d.ts +1 -43
- package/dist/embedding/gemini-embedding.d.ts.map +1 -1
- package/dist/embedding/gemini-embedding.js +26 -164
- package/dist/embedding/gemini-embedding.js.map +1 -1
- package/dist/embedding/index.d.ts +2 -3
- package/dist/embedding/index.d.ts.map +1 -1
- package/dist/embedding/index.js +2 -3
- package/dist/embedding/index.js.map +1 -1
- package/dist/embedding/ollama-embedding.d.ts +1 -2
- package/dist/embedding/ollama-embedding.d.ts.map +1 -1
- package/dist/embedding/ollama-embedding.js +3 -2
- package/dist/embedding/ollama-embedding.js.map +1 -1
- package/dist/embedding/openai-embedding.d.ts +1 -2
- package/dist/embedding/openai-embedding.d.ts.map +1 -1
- package/dist/embedding/openai-embedding.js +3 -3
- package/dist/embedding/openai-embedding.js.map +1 -1
- package/dist/embedding/voyageai-embedding.d.ts +1 -2
- package/dist/embedding/voyageai-embedding.d.ts.map +1 -1
- package/dist/embedding/voyageai-embedding.js +2 -2
- package/dist/embedding/voyageai-embedding.js.map +1 -1
- package/dist/index.d.ts +4 -4
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +4 -4
- package/dist/index.js.map +1 -1
- package/dist/splitter/ast-splitter.d.ts +1 -1
- package/dist/splitter/ast-splitter.d.ts.map +1 -1
- package/dist/splitter/ast-splitter.js +7 -7
- package/dist/splitter/ast-splitter.js.map +1 -1
- package/dist/splitter/index.d.ts +4 -4
- package/dist/splitter/index.d.ts.map +1 -1
- package/dist/splitter/index.js +1 -1
- package/dist/splitter/index.js.map +1 -1
- package/dist/splitter/langchain-splitter.d.ts +1 -1
- package/dist/splitter/langchain-splitter.d.ts.map +1 -1
- package/dist/splitter/langchain-splitter.js +2 -2
- package/dist/splitter/langchain-splitter.js.map +1 -1
- package/dist/sync/merkle.js +1 -1
- package/dist/sync/merkle.js.map +1 -1
- package/dist/sync/synchronizer.js +6 -6
- package/dist/sync/synchronizer.js.map +1 -1
- package/dist/utils/env-manager.js +4 -4
- package/dist/utils/env-manager.js.map +1 -1
- package/dist/vectordb/index.d.ts +4 -13
- package/dist/vectordb/index.d.ts.map +1 -1
- package/dist/vectordb/index.js +5 -39
- package/dist/vectordb/index.js.map +1 -1
- package/dist/vectordb/milvus-restful-vectordb.d.ts +11 -12
- package/dist/vectordb/milvus-restful-vectordb.d.ts.map +1 -1
- package/dist/vectordb/milvus-restful-vectordb.js +31 -29
- package/dist/vectordb/milvus-restful-vectordb.js.map +1 -1
- package/dist/vectordb/milvus-vectordb.d.ts +12 -12
- package/dist/vectordb/milvus-vectordb.d.ts.map +1 -1
- package/dist/vectordb/milvus-vectordb.js +28 -31
- package/dist/vectordb/milvus-vectordb.js.map +1 -1
- package/dist/vectordb/types.d.ts +16 -16
- package/dist/vectordb/types.d.ts.map +1 -1
- package/dist/vectordb/types.js.map +1 -1
- package/dist/vectordb/zilliz-utils.js +3 -3
- package/dist/vectordb/zilliz-utils.js.map +1 -1
- package/package.json +22 -32
- package/dist/embedding/huggingface-embedding.d.ts +0 -70
- package/dist/embedding/huggingface-embedding.d.ts.map +0 -1
- package/dist/embedding/huggingface-embedding.js +0 -270
- package/dist/embedding/huggingface-embedding.js.map +0 -1
- package/dist/vectordb/base/base-vector-database.d.ts +0 -58
- package/dist/vectordb/base/base-vector-database.d.ts.map +0 -1
- package/dist/vectordb/base/base-vector-database.js +0 -32
- package/dist/vectordb/base/base-vector-database.js.map +0 -1
- package/dist/vectordb/factory.d.ts +0 -113
- package/dist/vectordb/factory.d.ts.map +0 -1
- package/dist/vectordb/factory.js +0 -170
- package/dist/vectordb/factory.js.map +0 -1
- package/dist/vectordb/faiss-vectordb.d.ts +0 -162
- package/dist/vectordb/faiss-vectordb.d.ts.map +0 -1
- package/dist/vectordb/faiss-vectordb.js +0 -777
- package/dist/vectordb/faiss-vectordb.js.map +0 -1
- package/dist/vectordb/libsql-vectordb.d.ts +0 -170
- package/dist/vectordb/libsql-vectordb.d.ts.map +0 -1
- package/dist/vectordb/libsql-vectordb.js +0 -837
- package/dist/vectordb/libsql-vectordb.js.map +0 -1
- package/dist/vectordb/qdrant-vectordb.d.ts +0 -149
- package/dist/vectordb/qdrant-vectordb.d.ts.map +0 -1
- package/dist/vectordb/qdrant-vectordb.js +0 -856
- package/dist/vectordb/qdrant-vectordb.js.map +0 -1
- package/dist/vectordb/sparse/index.d.ts +0 -4
- package/dist/vectordb/sparse/index.d.ts.map +0 -1
- package/dist/vectordb/sparse/index.js +0 -23
- package/dist/vectordb/sparse/index.js.map +0 -1
- package/dist/vectordb/sparse/simple-bm25.d.ts +0 -115
- package/dist/vectordb/sparse/simple-bm25.d.ts.map +0 -1
- package/dist/vectordb/sparse/simple-bm25.js +0 -249
- package/dist/vectordb/sparse/simple-bm25.js.map +0 -1
- package/dist/vectordb/sparse/sparse-vector-generator.d.ts +0 -54
- package/dist/vectordb/sparse/sparse-vector-generator.d.ts.map +0 -1
- package/dist/vectordb/sparse/sparse-vector-generator.js +0 -3
- package/dist/vectordb/sparse/sparse-vector-generator.js.map +0 -1
- package/dist/vectordb/sparse/types.d.ts +0 -38
- package/dist/vectordb/sparse/types.d.ts.map +0 -1
- package/dist/vectordb/sparse/types.js +0 -3
- package/dist/vectordb/sparse/types.js.map +0 -1
|
@@ -9,8 +9,6 @@ class GeminiEmbedding extends base_embedding_1.Embedding {
|
|
|
9
9
|
this.dimension = 3072; // Default dimension for gemini-embedding-001
|
|
10
10
|
this.maxTokens = 2048; // Maximum tokens for Gemini embedding models
|
|
11
11
|
this.config = config;
|
|
12
|
-
this.maxRetries = config.maxRetries ?? 3;
|
|
13
|
-
this.baseDelay = config.baseDelay ?? 1000;
|
|
14
12
|
this.client = new genai_1.GoogleGenAI({
|
|
15
13
|
apiKey: config.apiKey,
|
|
16
14
|
...(config.baseURL && {
|
|
@@ -39,91 +37,6 @@ class GeminiEmbedding extends base_embedding_1.Embedding {
|
|
|
39
37
|
this.maxTokens = 2048;
|
|
40
38
|
}
|
|
41
39
|
}
|
|
42
|
-
/**
|
|
43
|
-
* Determine if an error is retryable
|
|
44
|
-
* @param error Error object to check
|
|
45
|
-
* @returns True if error is retryable
|
|
46
|
-
*/
|
|
47
|
-
isRetryableError(error) {
|
|
48
|
-
if (typeof error !== "object" || error === null) {
|
|
49
|
-
return false;
|
|
50
|
-
}
|
|
51
|
-
// Network errors
|
|
52
|
-
const networkErrorCodes = [
|
|
53
|
-
"ECONNREFUSED",
|
|
54
|
-
"ETIMEDOUT",
|
|
55
|
-
"ENOTFOUND",
|
|
56
|
-
"EAI_AGAIN",
|
|
57
|
-
];
|
|
58
|
-
if ("code" in error &&
|
|
59
|
-
typeof error.code === "string" &&
|
|
60
|
-
networkErrorCodes.includes(error.code)) {
|
|
61
|
-
return true;
|
|
62
|
-
}
|
|
63
|
-
// HTTP status codes
|
|
64
|
-
const retryableStatusCodes = [429, 500, 502, 503, 504];
|
|
65
|
-
if ("status" in error &&
|
|
66
|
-
typeof error.status === "number" &&
|
|
67
|
-
retryableStatusCodes.includes(error.status)) {
|
|
68
|
-
return true;
|
|
69
|
-
}
|
|
70
|
-
// Error message patterns
|
|
71
|
-
const errorMessage = "message" in error && typeof error.message === "string"
|
|
72
|
-
? error.message.toLowerCase()
|
|
73
|
-
: "";
|
|
74
|
-
const retryablePatterns = [
|
|
75
|
-
"rate limit",
|
|
76
|
-
"quota exceeded",
|
|
77
|
-
"service unavailable",
|
|
78
|
-
"timeout",
|
|
79
|
-
"connection",
|
|
80
|
-
];
|
|
81
|
-
return retryablePatterns.some((pattern) => errorMessage.includes(pattern));
|
|
82
|
-
}
|
|
83
|
-
/**
|
|
84
|
-
* Sleep for specified milliseconds
|
|
85
|
-
* @param ms Milliseconds to sleep
|
|
86
|
-
*/
|
|
87
|
-
async sleep(ms) {
|
|
88
|
-
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
89
|
-
}
|
|
90
|
-
/**
|
|
91
|
-
* Execute operation with retry logic
|
|
92
|
-
* Only retries on retryable errors (network errors, rate limits, server errors)
|
|
93
|
-
* @param operation Operation to execute
|
|
94
|
-
* @param context Context string for error messages
|
|
95
|
-
* @returns Operation result
|
|
96
|
-
*/
|
|
97
|
-
async executeWithRetry(operation, context) {
|
|
98
|
-
let lastError;
|
|
99
|
-
for (let attempt = 0; attempt <= this.maxRetries; attempt++) {
|
|
100
|
-
try {
|
|
101
|
-
return await operation();
|
|
102
|
-
}
|
|
103
|
-
catch (error) {
|
|
104
|
-
lastError = error;
|
|
105
|
-
// If error is not retryable, fail immediately
|
|
106
|
-
if (!this.isRetryableError(error)) {
|
|
107
|
-
const err = new Error(`${context}: ${error instanceof Error ? error.message : "Unknown error"}`);
|
|
108
|
-
err.cause = error;
|
|
109
|
-
throw err;
|
|
110
|
-
}
|
|
111
|
-
// If we've exhausted all retries, throw the error
|
|
112
|
-
if (attempt === this.maxRetries) {
|
|
113
|
-
const err = new Error(`${context}: ${error instanceof Error ? error.message : "Unknown error"}`);
|
|
114
|
-
err.cause = error;
|
|
115
|
-
throw err;
|
|
116
|
-
}
|
|
117
|
-
// Calculate delay with exponential backoff (capped at 10s)
|
|
118
|
-
const delay = Math.min(this.baseDelay * 2 ** attempt, 10000);
|
|
119
|
-
await this.sleep(delay);
|
|
120
|
-
}
|
|
121
|
-
}
|
|
122
|
-
// This should never be reached, but TypeScript needs it
|
|
123
|
-
const err = new Error(`${context}: ${lastError instanceof Error ? lastError.message : "Unknown error"}`);
|
|
124
|
-
err.cause = lastError;
|
|
125
|
-
throw err;
|
|
126
|
-
}
|
|
127
40
|
async detectDimension() {
|
|
128
41
|
// Gemini doesn't need dynamic detection, return configured dimension
|
|
129
42
|
return this.dimension;
|
|
@@ -131,9 +44,9 @@ class GeminiEmbedding extends base_embedding_1.Embedding {
|
|
|
131
44
|
async embed(text) {
|
|
132
45
|
const processedText = this.preprocessText(text);
|
|
133
46
|
const model = this.config.model || "gemini-embedding-001";
|
|
134
|
-
|
|
47
|
+
try {
|
|
135
48
|
const response = await this.client.models.embedContent({
|
|
136
|
-
model,
|
|
49
|
+
model: model,
|
|
137
50
|
contents: processedText,
|
|
138
51
|
config: {
|
|
139
52
|
outputDimensionality: this.config.outputDimensionality || this.dimension,
|
|
@@ -148,58 +61,37 @@ class GeminiEmbedding extends base_embedding_1.Embedding {
|
|
|
148
61
|
vector: response.embeddings[0].values,
|
|
149
62
|
dimension: response.embeddings[0].values.length,
|
|
150
63
|
};
|
|
151
|
-
}
|
|
64
|
+
}
|
|
65
|
+
catch (error) {
|
|
66
|
+
throw new Error(`Gemini embedding failed: ${error instanceof Error ? error.message : "Unknown error"}`);
|
|
67
|
+
}
|
|
152
68
|
}
|
|
153
69
|
async embedBatch(texts) {
|
|
154
70
|
const processedTexts = this.preprocessTexts(texts);
|
|
155
71
|
const model = this.config.model || "gemini-embedding-001";
|
|
156
|
-
// Try batch processing with retry logic
|
|
157
72
|
try {
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
73
|
+
const response = await this.client.models.embedContent({
|
|
74
|
+
model: model,
|
|
75
|
+
contents: processedTexts,
|
|
76
|
+
config: {
|
|
77
|
+
outputDimensionality: this.config.outputDimensionality || this.dimension,
|
|
78
|
+
},
|
|
79
|
+
});
|
|
80
|
+
if (!response.embeddings) {
|
|
81
|
+
throw new Error("Gemini API returned invalid response");
|
|
82
|
+
}
|
|
83
|
+
return response.embeddings.map((embedding) => {
|
|
84
|
+
if (!embedding.values) {
|
|
85
|
+
throw new Error("Gemini API returned invalid embedding data");
|
|
168
86
|
}
|
|
169
|
-
return
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
vector: embedding.values,
|
|
175
|
-
dimension: embedding.values.length,
|
|
176
|
-
};
|
|
177
|
-
});
|
|
178
|
-
}, "Gemini batch embedding failed");
|
|
87
|
+
return {
|
|
88
|
+
vector: embedding.values,
|
|
89
|
+
dimension: embedding.values.length,
|
|
90
|
+
};
|
|
91
|
+
});
|
|
179
92
|
}
|
|
180
|
-
catch {
|
|
181
|
-
|
|
182
|
-
// Add delay between requests to avoid rate limiting
|
|
183
|
-
const results = [];
|
|
184
|
-
const FALLBACK_DELAY_MS = 100; // Delay between individual requests
|
|
185
|
-
for (let i = 0; i < processedTexts.length; i++) {
|
|
186
|
-
const text = processedTexts[i];
|
|
187
|
-
try {
|
|
188
|
-
// Add delay between requests (except for first)
|
|
189
|
-
if (i > 0) {
|
|
190
|
-
await new Promise((resolve) => setTimeout(resolve, FALLBACK_DELAY_MS));
|
|
191
|
-
}
|
|
192
|
-
const result = await this.embed(text);
|
|
193
|
-
results.push(result);
|
|
194
|
-
}
|
|
195
|
-
catch (individualError) {
|
|
196
|
-
// If individual request also fails, re-throw the error with cause
|
|
197
|
-
const error = new Error("Gemini batch embedding failed (both batch and individual attempts failed)");
|
|
198
|
-
error.cause = individualError;
|
|
199
|
-
throw error;
|
|
200
|
-
}
|
|
201
|
-
}
|
|
202
|
-
return results;
|
|
93
|
+
catch (error) {
|
|
94
|
+
throw new Error(`Gemini batch embedding failed: ${error instanceof Error ? error.message : "Unknown error"}`);
|
|
203
95
|
}
|
|
204
96
|
}
|
|
205
97
|
getDimension() {
|
|
@@ -257,36 +149,6 @@ class GeminiEmbedding extends base_embedding_1.Embedding {
|
|
|
257
149
|
const supportedDimensions = this.getSupportedDimensions();
|
|
258
150
|
return supportedDimensions.includes(dimension);
|
|
259
151
|
}
|
|
260
|
-
/**
|
|
261
|
-
* Get current retry configuration
|
|
262
|
-
* @returns Object containing maxRetries and baseDelay
|
|
263
|
-
*/
|
|
264
|
-
getRetryConfig() {
|
|
265
|
-
return {
|
|
266
|
-
maxRetries: this.maxRetries,
|
|
267
|
-
baseDelay: this.baseDelay,
|
|
268
|
-
};
|
|
269
|
-
}
|
|
270
|
-
/**
|
|
271
|
-
* Set maximum number of retry attempts
|
|
272
|
-
* @param maxRetries Maximum retry attempts
|
|
273
|
-
*/
|
|
274
|
-
setMaxRetries(maxRetries) {
|
|
275
|
-
if (maxRetries < 0) {
|
|
276
|
-
throw new Error("maxRetries must be non-negative");
|
|
277
|
-
}
|
|
278
|
-
this.maxRetries = maxRetries;
|
|
279
|
-
}
|
|
280
|
-
/**
|
|
281
|
-
* Set base delay for exponential backoff
|
|
282
|
-
* @param baseDelay Base delay in milliseconds
|
|
283
|
-
*/
|
|
284
|
-
setBaseDelay(baseDelay) {
|
|
285
|
-
if (baseDelay <= 0) {
|
|
286
|
-
throw new Error("baseDelay must be positive");
|
|
287
|
-
}
|
|
288
|
-
this.baseDelay = baseDelay;
|
|
289
|
-
}
|
|
290
152
|
}
|
|
291
153
|
exports.GeminiEmbedding = GeminiEmbedding;
|
|
292
154
|
//# sourceMappingURL=gemini-embedding.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"gemini-embedding.js","sourceRoot":"","sources":["../../src/embedding/gemini-embedding.ts"],"names":[],"mappings":";;;
|
|
1
|
+
{"version":3,"file":"gemini-embedding.js","sourceRoot":"","sources":["../../src/embedding/gemini-embedding.ts"],"names":[],"mappings":";;;AAAA,yCAA4C;AAC5C,qDAA8D;AAS9D,MAAa,eAAgB,SAAQ,0BAAS;IAM5C,YAAY,MAA6B;QACvC,KAAK,EAAE,CAAC;QAJF,cAAS,GAAW,IAAI,CAAC,CAAC,6CAA6C;QACrE,cAAS,GAAW,IAAI,CAAC,CAAC,6CAA6C;QAI/E,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,MAAM,GAAG,IAAI,mBAAW,CAAC;YAC5B,MAAM,EAAE,MAAM,CAAC,MAAM;YACrB,GAAG,CAAC,MAAM,CAAC,OAAO,IAAI;gBACpB,WAAW,EAAE;oBACX,OAAO,EAAE,MAAM,CAAC,OAAO;iBACxB;aACF,CAAC;SACH,CAAC,CAAC;QAEH,iDAAiD;QACjD,IAAI,CAAC,uBAAuB,CAAC,MAAM,CAAC,KAAK,IAAI,sBAAsB,CAAC,CAAC;QAErE,4CAA4C;QAC5C,IAAI,MAAM,CAAC,oBAAoB,EAAE,CAAC;YAChC,IAAI,CAAC,SAAS,GAAG,MAAM,CAAC,oBAAoB,CAAC;QAC/C,CAAC;IACH,CAAC;IAEO,uBAAuB,CAAC,KAAa;QAC3C,MAAM,eAAe,GAAG,eAAe,CAAC,kBAAkB,EAAE,CAAC;QAC7D,MAAM,SAAS,GAAG,eAAe,CAAC,KAAK,CAAC,CAAC;QAEzC,IAAI,SAAS,EAAE,CAAC;YACd,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC,SAAS,CAAC;YACrC,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC,aAAa,CAAC;QAC3C,CAAC;aAAM,CAAC;YACN,8DAA8D;YAC9D,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC;YACtB,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC;QACxB,CAAC;IACH,CAAC;IAED,KAAK,CAAC,eAAe;QACnB,qEAAqE;QACrE,OAAO,IAAI,CAAC,SAAS,CAAC;IACxB,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,IAAY;QACtB,MAAM,aAAa,GAAG,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,CAAC;QAChD,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,KAAK,IAAI,sBAAsB,CAAC;QAE1D,IAAI,CAAC;YACH,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,YAAY,CAAC;gBACrD,KAAK,EAAE,KAAK;gBACZ,QAAQ,EAAE,aAAa;gBACvB,MAAM,EAAE;oBACN,oBAAoB,EAClB,IAAI,CAAC,MAAM,CAAC,oBAAoB,IAAI,IAAI,CAAC,SAAS;iBACrD;aACF,CAAC,CAAC;YAEH,IACE,CAAC,QAAQ,CAAC,UAAU;gBACpB,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC,CAAC;gBACvB,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,MAAM,EAC9B,CAAC;gBACD,MAAM,IAAI,KAAK,CAAC,sCAAsC,CAAC,CAAC;YAC1D,CAAC;YAED,OAAO;gBACL,MAAM,EAAE,QAAQ,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,MAAM;gBACrC,SAAS,EAAE,QAAQ,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM;aAChD,CAAC;QACJ,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,IAAI,KAAK,CACb,4BAA4B,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe,EAAE,CACvF,CAAC;QACJ,CAAC;IACH,CAAC;IAED,KAAK,CAAC,UAAU,CAAC,KAAe;QAC9B,MAAM,cAAc,GAAG,IAAI,CAAC,eAAe,CAAC,KAAK,CAAC,CAAC;QACnD,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,KAAK,IAAI,sBAAsB,CAAC;QAE1D,IAAI,CAAC;YACH,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,YAAY,CAAC;gBACrD,KAAK,EAAE,KAAK;gBACZ,QAAQ,EAAE,cAAc;gBACxB,MAAM,EAAE;oBACN,oBAAoB,EAClB,IAAI,CAAC,MAAM,CAAC,oBAAoB,IAAI,IAAI,CAAC,SAAS;iBACrD;aACF,CAAC,CAAC;YAEH,IAAI,CAAC,QAAQ,CAAC,UAAU,EAAE,CAAC;gBACzB,MAAM,IAAI,KAAK,CAAC,sCAAsC,CAAC,CAAC;YAC1D,CAAC;YAED,OAAO,QAAQ,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,SAAS,EAAE,EAAE;gBAC3C,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,CAAC;oBACtB,MAAM,IAAI,KAAK,CAAC,4CAA4C,CAAC,CAAC;gBAChE,CAAC;gBACD,OAAO;oBACL,MAAM,EAAE,SAAS,CAAC,MAAM;oBACxB,SAAS,EAAE,SAAS,CAAC,MAAM,CAAC,MAAM;iBACnC,CAAC;YACJ,CAAC,CAAC,CAAC;QACL,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,IAAI,KAAK,CACb,kCAAkC,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe,EAAE,CAC7F,CAAC;QACJ,CAAC;IACH,CAAC;IAED,YAAY;QACV,OAAO,IAAI,CAAC,SAAS,CAAC;IACxB,CAAC;IAED,WAAW;QACT,OAAO,QAAQ,CAAC;IAClB,CAAC;IAED;;;OAGG;IACH,QAAQ,CAAC,KAAa;QACpB,IAAI,CAAC,MAAM,CAAC,KAAK,GAAG,KAAK,CAAC;QAC1B,IAAI,CAAC,uBAAuB,CAAC,KAAK,CAAC,CAAC;IACtC,CAAC;IAED;;;OAGG;IACH,uBAAuB,CAAC,SAAiB;QACvC,IAAI,CAAC,MAAM,CAAC,oBAAoB,GAAG,SAAS,CAAC;QAC7C,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;IAC7B,CAAC;IAED;;OAEG;IACH,SAAS;QACP,OAAO,IAAI,CAAC,MAAM,CAAC;IACrB,CAAC;IAED;;OAEG;IACH,MAAM,CAAC,kBAAkB;QASvB,OAAO;YACL,sBAAsB,EAAE;gBACtB,SAAS,EAAE,IAAI;gBACf,aAAa,EAAE,IAAI;gBACnB,WAAW,EACT,+EAA+E;gBACjF,mBAAmB,EAAE,CAAC,IAAI,EAAE,IAAI,EAAE,GAAG,EAAE,GAAG,CAAC,EAAE,6CAA6C;aAC3F;SACF,CAAC;IACJ,CAAC;IAED;;OAEG;IACH,sBAAsB;QACpB,MAAM,SAAS,GACb,eAAe,CAAC,kBAAkB,EAAE,CAClC,IAAI,CAAC,MAAM,CAAC,KAAK,IAAI,sBAAsB,CAC5C,CAAC;QACJ,OAAO,SAAS,EAAE,mBAAmB,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;IAC5D,CAAC;IAED;;OAEG;IACH,oBAAoB,CAAC,SAAiB;QACpC,MAAM,mBAAmB,GAAG,IAAI,CAAC,sBAAsB,EAAE,CAAC;QAC1D,OAAO,mBAAmB,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC;IACjD,CAAC;CACF;AA3LD,0CA2LC"}
|
|
@@ -1,7 +1,6 @@
|
|
|
1
1
|
export * from "./base-embedding";
|
|
2
|
-
export * from "./gemini-embedding";
|
|
3
|
-
export * from "./huggingface-embedding";
|
|
4
|
-
export * from "./ollama-embedding";
|
|
5
2
|
export * from "./openai-embedding";
|
|
6
3
|
export * from "./voyageai-embedding";
|
|
4
|
+
export * from "./ollama-embedding";
|
|
5
|
+
export * from "./gemini-embedding";
|
|
7
6
|
//# sourceMappingURL=index.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/embedding/index.ts"],"names":[],"mappings":"AACA,cAAc,kBAAkB,CAAC;
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/embedding/index.ts"],"names":[],"mappings":"AACA,cAAc,kBAAkB,CAAC;AAGjC,cAAc,oBAAoB,CAAC;AACnC,cAAc,sBAAsB,CAAC;AACrC,cAAc,oBAAoB,CAAC;AACnC,cAAc,oBAAoB,CAAC"}
|
package/dist/embedding/index.js
CHANGED
|
@@ -16,10 +16,9 @@ var __exportStar = (this && this.__exportStar) || function(m, exports) {
|
|
|
16
16
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
17
17
|
// Export base classes and interfaces
|
|
18
18
|
__exportStar(require("./base-embedding"), exports);
|
|
19
|
-
__exportStar(require("./gemini-embedding"), exports);
|
|
20
|
-
__exportStar(require("./huggingface-embedding"), exports);
|
|
21
|
-
__exportStar(require("./ollama-embedding"), exports);
|
|
22
19
|
// Implementation class exports
|
|
23
20
|
__exportStar(require("./openai-embedding"), exports);
|
|
24
21
|
__exportStar(require("./voyageai-embedding"), exports);
|
|
22
|
+
__exportStar(require("./ollama-embedding"), exports);
|
|
23
|
+
__exportStar(require("./gemini-embedding"), exports);
|
|
25
24
|
//# sourceMappingURL=index.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/embedding/index.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;AAAA,qCAAqC;AACrC,mDAAiC;AAEjC
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/embedding/index.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;AAAA,qCAAqC;AACrC,mDAAiC;AAEjC,+BAA+B;AAC/B,qDAAmC;AACnC,uDAAqC;AACrC,qDAAmC;AACnC,qDAAmC"}
|
|
@@ -1,6 +1,5 @@
|
|
|
1
|
-
import type { EmbeddingVector } from "./base-embedding";
|
|
2
1
|
import { Ollama } from "ollama";
|
|
3
|
-
import { Embedding } from "./base-embedding";
|
|
2
|
+
import { Embedding, EmbeddingVector } from "./base-embedding";
|
|
4
3
|
export interface OllamaEmbeddingConfig {
|
|
5
4
|
model: string;
|
|
6
5
|
host?: string;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"ollama-embedding.d.ts","sourceRoot":"","sources":["../../src/embedding/ollama-embedding.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"ollama-embedding.d.ts","sourceRoot":"","sources":["../../src/embedding/ollama-embedding.ts"],"names":[],"mappings":"AAAA,OAAO,EAAgB,MAAM,EAAE,MAAM,QAAQ,CAAC;AAC9C,OAAO,EAAE,SAAS,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;AAE9D,MAAM,WAAW,qBAAqB;IACpC,KAAK,EAAE,MAAM,CAAC;IACd,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,KAAK,CAAC,EAAE,OAAO,UAAU,CAAC,KAAK,CAAC;IAChC,SAAS,CAAC,EAAE,MAAM,GAAG,MAAM,CAAC;IAC5B,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAClC,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,qBAAa,eAAgB,SAAQ,SAAS;IAC5C,OAAO,CAAC,MAAM,CAAS;IACvB,OAAO,CAAC,MAAM,CAAwB;IACtC,OAAO,CAAC,SAAS,CAAe;IAChC,OAAO,CAAC,iBAAiB,CAAkB;IAC3C,SAAS,CAAC,SAAS,EAAE,MAAM,CAAQ;gBAEvB,MAAM,EAAE,qBAAqB;IAyBzC,OAAO,CAAC,2BAA2B;IAW7B,KAAK,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,eAAe,CAAC;IAoC7C,UAAU,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,eAAe,EAAE,CAAC;IAsC7D,YAAY,IAAI,MAAM;IAItB,WAAW,IAAI,MAAM;IAIrB;;;OAGG;IACG,QAAQ,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAoB5C;;;OAGG;IACH,OAAO,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI;IAQ3B;;;OAGG;IACH,YAAY,CAAC,SAAS,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI;IAI9C;;;OAGG;IACH,UAAU,CAAC,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI;IAIlD;;;OAGG;IACH,YAAY,CAAC,SAAS,EAAE,MAAM,GAAG,IAAI;IAKrC;;OAEG;IACH,SAAS,IAAI,MAAM;IAIb,eAAe,CAAC,QAAQ,GAAE,MAAe,GAAG,OAAO,CAAC,MAAM,CAAC;CAqClE"}
|
|
@@ -119,7 +119,8 @@ class OllamaEmbedding extends base_embedding_1.Embedding {
|
|
|
119
119
|
console.log(`[OllamaEmbedding] 📏 Detected Ollama embedding dimension: ${this.dimension} for model: ${this.config.model}`);
|
|
120
120
|
}
|
|
121
121
|
else {
|
|
122
|
-
console.log(
|
|
122
|
+
console.log("[OllamaEmbedding] Dimension already detected for model " +
|
|
123
|
+
this.config.model);
|
|
123
124
|
}
|
|
124
125
|
}
|
|
125
126
|
/**
|
|
@@ -129,7 +130,7 @@ class OllamaEmbedding extends base_embedding_1.Embedding {
|
|
|
129
130
|
setHost(host) {
|
|
130
131
|
this.config.host = host;
|
|
131
132
|
this.client = new ollama_1.Ollama({
|
|
132
|
-
host,
|
|
133
|
+
host: host,
|
|
133
134
|
fetch: this.config.fetch,
|
|
134
135
|
});
|
|
135
136
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"ollama-embedding.js","sourceRoot":"","sources":["../../src/embedding/ollama-embedding.ts"],"names":[],"mappings":";;;
|
|
1
|
+
{"version":3,"file":"ollama-embedding.js","sourceRoot":"","sources":["../../src/embedding/ollama-embedding.ts"],"names":[],"mappings":";;;AAAA,mCAA8C;AAC9C,qDAA8D;AAY9D,MAAa,eAAgB,SAAQ,0BAAS;IAO5C,YAAY,MAA6B;QACvC,KAAK,EAAE,CAAC;QALF,cAAS,GAAW,GAAG,CAAC,CAAC,8CAA8C;QACvE,sBAAiB,GAAY,KAAK,CAAC,CAAC,uCAAuC;QACzE,cAAS,GAAW,IAAI,CAAC,CAAC,oCAAoC;QAItE,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,MAAM,GAAG,IAAI,eAAM,CAAC;YACvB,IAAI,EAAE,MAAM,CAAC,IAAI,IAAI,wBAAwB;YAC7C,KAAK,EAAE,MAAM,CAAC,KAAK;SACpB,CAAC,CAAC;QAEH,iEAAiE;QACjE,IAAI,MAAM,CAAC,SAAS,EAAE,CAAC;YACrB,IAAI,CAAC,SAAS,GAAG,MAAM,CAAC,SAAS,CAAC;YAClC,IAAI,CAAC,iBAAiB,GAAG,IAAI,CAAC;QAChC,CAAC;QAED,gDAAgD;QAChD,IAAI,MAAM,CAAC,SAAS,EAAE,CAAC;YACrB,IAAI,CAAC,SAAS,GAAG,MAAM,CAAC,SAAS,CAAC;QACpC,CAAC;aAAM,CAAC;YACN,oCAAoC;YACpC,IAAI,CAAC,2BAA2B,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QACjD,CAAC;QAED,2EAA2E;IAC7E,CAAC;IAEO,2BAA2B,CAAC,KAAa;QAC/C,iDAAiD;QACjD,IAAI,KAAK,EAAE,QAAQ,CAAC,kBAAkB,CAAC,EAAE,CAAC;YACxC,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,CAAC,wCAAwC;QACjE,CAAC;aAAM,IAAI,KAAK,EAAE,QAAQ,CAAC,wBAAwB,CAAC,EAAE,CAAC;YACrD,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,CAAC,8CAA8C;QACvE,CAAC;aAAM,CAAC;YACN,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,CAAC,iCAAiC;QAC1D,CAAC;IACH,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,IAAY;QACtB,sBAAsB;QACtB,MAAM,aAAa,GAAG,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,CAAC;QAEhD,kDAAkD;QAClD,IAAI,CAAC,IAAI,CAAC,iBAAiB,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,SAAS,EAAE,CAAC;YACtD,IAAI,CAAC,SAAS,GAAG,MAAM,IAAI,CAAC,eAAe,EAAE,CAAC;YAC9C,IAAI,CAAC,iBAAiB,GAAG,IAAI,CAAC;YAC9B,OAAO,CAAC,GAAG,CACT,6DAA6D,IAAI,CAAC,SAAS,eAAe,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,CAC9G,CAAC;QACJ,CAAC;QAED,MAAM,YAAY,GAAiB;YACjC,KAAK,EAAE,IAAI,CAAC,MAAM,CAAC,KAAK;YACxB,KAAK,EAAE,aAAa;YACpB,OAAO,EAAE,IAAI,CAAC,MAAM,CAAC,OAAO;SAC7B,CAAC;QAEF,kDAAkD;QAClD,IAAI,IAAI,CAAC,MAAM,CAAC,SAAS,IAAI,IAAI,CAAC,MAAM,CAAC,SAAS,KAAK,EAAE,EAAE,CAAC;YAC1D,YAAY,CAAC,UAAU,GAAG,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC;QAClD,CAAC;QAED,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC;QAEvD,IAAI,CAAC,QAAQ,CAAC,UAAU,IAAI,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC,CAAC,EAAE,CAAC;YACpD,MAAM,IAAI,KAAK,CAAC,sCAAsC,CAAC,CAAC;QAC1D,CAAC;QAED,OAAO;YACL,MAAM,EAAE,QAAQ,CAAC,UAAU,CAAC,CAAC,CAAC;YAC9B,SAAS,EAAE,IAAI,CAAC,SAAS;SAC1B,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,UAAU,CAAC,KAAe;QAC9B,uBAAuB;QACvB,MAAM,cAAc,GAAG,IAAI,CAAC,eAAe,CAAC,KAAK,CAAC,CAAC;QAEnD,kDAAkD;QAClD,IAAI,CAAC,IAAI,CAAC,iBAAiB,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,SAAS,EAAE,CAAC;YACtD,IAAI,CAAC,SAAS,GAAG,MAAM,IAAI,CAAC,eAAe,EAAE,CAAC;YAC9C,IAAI,CAAC,iBAAiB,GAAG,IAAI,CAAC;YAC9B,OAAO,CAAC,GAAG,CACT,6DAA6D,IAAI,CAAC,SAAS,eAAe,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,CAC9G,CAAC;QACJ,CAAC;QAED,0CAA0C;QAC1C,MAAM,YAAY,GAAiB;YACjC,KAAK,EAAE,IAAI,CAAC,MAAM,CAAC,KAAK;YACxB,KAAK,EAAE,cAAc,EAAE,gCAAgC;YACvD,OAAO,EAAE,IAAI,CAAC,MAAM,CAAC,OAAO;SAC7B,CAAC;QAEF,kDAAkD;QAClD,IAAI,IAAI,CAAC,MAAM,CAAC,SAAS,IAAI,IAAI,CAAC,MAAM,CAAC,SAAS,KAAK,EAAE,EAAE,CAAC;YAC1D,YAAY,CAAC,UAAU,GAAG,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC;QAClD,CAAC;QAED,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC;QAEvD,IAAI,CAAC,QAAQ,CAAC,UAAU,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAC,EAAE,CAAC;YAChE,MAAM,IAAI,KAAK,CAAC,4CAA4C,CAAC,CAAC;QAChE,CAAC;QAED,oCAAoC;QACpC,OAAO,QAAQ,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,SAAmB,EAAE,EAAE,CAAC,CAAC;YACvD,MAAM,EAAE,SAAS;YACjB,SAAS,EAAE,IAAI,CAAC,SAAS;SAC1B,CAAC,CAAC,CAAC;IACN,CAAC;IAED,YAAY;QACV,OAAO,IAAI,CAAC,SAAS,CAAC;IACxB,CAAC;IAED,WAAW;QACT,OAAO,QAAQ,CAAC;IAClB,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,QAAQ,CAAC,KAAa;QAC1B,IAAI,CAAC,MAAM,CAAC,KAAK,GAAG,KAAK,CAAC;QAC1B,+CAA+C;QAC/C,IAAI,CAAC,iBAAiB,GAAG,KAAK,CAAC;QAC/B,kCAAkC;QAClC,IAAI,CAAC,2BAA2B,CAAC,KAAK,CAAC,CAAC;QACxC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,SAAS,EAAE,CAAC;YAC3B,IAAI,CAAC,SAAS,GAAG,MAAM,IAAI,CAAC,eAAe,EAAE,CAAC;YAC9C,IAAI,CAAC,iBAAiB,GAAG,IAAI,CAAC;YAC9B,OAAO,CAAC,GAAG,CACT,6DAA6D,IAAI,CAAC,SAAS,eAAe,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,CAC9G,CAAC;QACJ,CAAC;aAAM,CAAC;YACN,OAAO,CAAC,GAAG,CACT,yDAAyD;gBACvD,IAAI,CAAC,MAAM,CAAC,KAAK,CACpB,CAAC;QACJ,CAAC;IACH,CAAC;IAED;;;OAGG;IACH,OAAO,CAAC,IAAY;QAClB,IAAI,CAAC,MAAM,CAAC,IAAI,GAAG,IAAI,CAAC;QACxB,IAAI,CAAC,MAAM,GAAG,IAAI,eAAM,CAAC;YACvB,IAAI,EAAE,IAAI;YACV,KAAK,EAAE,IAAI,CAAC,MAAM,CAAC,KAAK;SACzB,CAAC,CAAC;IACL,CAAC;IAED;;;OAGG;IACH,YAAY,CAAC,SAA0B;QACrC,IAAI,CAAC,MAAM,CAAC,SAAS,GAAG,SAAS,CAAC;IACpC,CAAC;IAED;;;OAGG;IACH,UAAU,CAAC,OAAgC;QACzC,IAAI,CAAC,MAAM,CAAC,OAAO,GAAG,OAAO,CAAC;IAChC,CAAC;IAED;;;OAGG;IACH,YAAY,CAAC,SAAiB;QAC5B,IAAI,CAAC,MAAM,CAAC,SAAS,GAAG,SAAS,CAAC;QAClC,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;IAC7B,CAAC;IAED;;OAEG;IACH,SAAS;QACP,OAAO,IAAI,CAAC,MAAM,CAAC;IACrB,CAAC;IAED,KAAK,CAAC,eAAe,CAAC,WAAmB,MAAM;QAC7C,OAAO,CAAC,GAAG,CAAC,oDAAoD,CAAC,CAAC;QAElE,IAAI,CAAC;YACH,MAAM,aAAa,GAAG,IAAI,CAAC,cAAc,CAAC,QAAQ,CAAC,CAAC;YACpD,MAAM,YAAY,GAAiB;gBACjC,KAAK,EAAE,IAAI,CAAC,MAAM,CAAC,KAAK;gBACxB,KAAK,EAAE,aAAa;gBACpB,OAAO,EAAE,IAAI,CAAC,MAAM,CAAC,OAAO;aAC7B,CAAC;YAEF,IAAI,IAAI,CAAC,MAAM,CAAC,SAAS,IAAI,IAAI,CAAC,MAAM,CAAC,SAAS,KAAK,EAAE,EAAE,CAAC;gBAC1D,YAAY,CAAC,UAAU,GAAG,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC;YAClD,CAAC;YAED,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC;YAEvD,IAAI,CAAC,QAAQ,CAAC,UAAU,IAAI,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC,CAAC,EAAE,CAAC;gBACpD,MAAM,IAAI,KAAK,CAAC,sCAAsC,CAAC,CAAC;YAC1D,CAAC;YAED,MAAM,SAAS,GAAG,QAAQ,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC;YAChD,OAAO,CAAC,GAAG,CACT,gEAAgE,SAAS,EAAE,CAC5E,CAAC;YACF,OAAO,SAAS,CAAC;QACnB,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,YAAY,GAChB,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe,CAAC;YAC3D,OAAO,CAAC,KAAK,CACX,iDAAiD,YAAY,EAAE,CAChE,CAAC;YACF,MAAM,IAAI,KAAK,CACb,gDAAgD,YAAY,EAAE,CAC/D,CAAC;QACJ,CAAC;IACH,CAAC;CACF;AAtOD,0CAsOC"}
|
|
@@ -1,6 +1,5 @@
|
|
|
1
|
-
import type { EmbeddingVector } from "./base-embedding";
|
|
2
1
|
import OpenAI from "openai";
|
|
3
|
-
import { Embedding } from "./base-embedding";
|
|
2
|
+
import { Embedding, EmbeddingVector } from "./base-embedding";
|
|
4
3
|
export interface OpenAIEmbeddingConfig {
|
|
5
4
|
model: string;
|
|
6
5
|
apiKey: string;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"openai-embedding.d.ts","sourceRoot":"","sources":["../../src/embedding/openai-embedding.ts"],"names":[],"mappings":"AAAA,OAAO,
|
|
1
|
+
{"version":3,"file":"openai-embedding.d.ts","sourceRoot":"","sources":["../../src/embedding/openai-embedding.ts"],"names":[],"mappings":"AAAA,OAAO,MAAM,MAAM,QAAQ,CAAC;AAC5B,OAAO,EAAE,SAAS,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;AAE9D,MAAM,WAAW,qBAAqB;IACpC,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,qBAAa,eAAgB,SAAQ,SAAS;IAC5C,OAAO,CAAC,MAAM,CAAS;IACvB,OAAO,CAAC,MAAM,CAAwB;IACtC,OAAO,CAAC,SAAS,CAAgB;IACjC,SAAS,CAAC,SAAS,EAAE,MAAM,CAAQ;gBAEvB,MAAM,EAAE,qBAAqB;IASnC,eAAe,CAAC,QAAQ,GAAE,MAAe,GAAG,OAAO,CAAC,MAAM,CAAC;IAwC3D,KAAK,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,eAAe,CAAC;IAgC7C,UAAU,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,eAAe,EAAE,CAAC;IAiC7D,YAAY,IAAI,MAAM;IAkBtB,WAAW,IAAI,MAAM;IAIrB;;;OAGG;IACG,QAAQ,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAU5C;;OAEG;IACH,SAAS,IAAI,MAAM;IAInB;;OAEG;IACH,MAAM,CAAC,kBAAkB,IAAI,MAAM,CACjC,MAAM,EACN;QAAE,SAAS,EAAE,MAAM,CAAC;QAAC,WAAW,EAAE,MAAM,CAAA;KAAE,CAC3C;CAkBF"}
|
|
@@ -28,7 +28,7 @@ class OpenAIEmbedding extends base_embedding_1.Embedding {
|
|
|
28
28
|
try {
|
|
29
29
|
const processedText = this.preprocessText(testText);
|
|
30
30
|
const response = await this.client.embeddings.create({
|
|
31
|
-
model,
|
|
31
|
+
model: model,
|
|
32
32
|
input: processedText,
|
|
33
33
|
encoding_format: "float",
|
|
34
34
|
});
|
|
@@ -58,7 +58,7 @@ class OpenAIEmbedding extends base_embedding_1.Embedding {
|
|
|
58
58
|
}
|
|
59
59
|
try {
|
|
60
60
|
const response = await this.client.embeddings.create({
|
|
61
|
-
model,
|
|
61
|
+
model: model,
|
|
62
62
|
input: processedText,
|
|
63
63
|
encoding_format: "float",
|
|
64
64
|
});
|
|
@@ -86,7 +86,7 @@ class OpenAIEmbedding extends base_embedding_1.Embedding {
|
|
|
86
86
|
}
|
|
87
87
|
try {
|
|
88
88
|
const response = await this.client.embeddings.create({
|
|
89
|
-
model,
|
|
89
|
+
model: model,
|
|
90
90
|
input: processedTexts,
|
|
91
91
|
encoding_format: "float",
|
|
92
92
|
});
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"openai-embedding.js","sourceRoot":"","sources":["../../src/embedding/openai-embedding.ts"],"names":[],"mappings":";;;;;;
|
|
1
|
+
{"version":3,"file":"openai-embedding.js","sourceRoot":"","sources":["../../src/embedding/openai-embedding.ts"],"names":[],"mappings":";;;;;;AAAA,oDAA4B;AAC5B,qDAA8D;AAQ9D,MAAa,eAAgB,SAAQ,0BAAS;IAM5C,YAAY,MAA6B;QACvC,KAAK,EAAE,CAAC;QAJF,cAAS,GAAW,IAAI,CAAC,CAAC,+CAA+C;QACvE,cAAS,GAAW,IAAI,CAAC,CAAC,6CAA6C;QAI/E,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,MAAM,GAAG,IAAI,gBAAM,CAAC;YACvB,MAAM,EAAE,MAAM,CAAC,MAAM;YACrB,OAAO,EAAE,MAAM,CAAC,OAAO;SACxB,CAAC,CAAC;IACL,CAAC;IAED,KAAK,CAAC,eAAe,CAAC,WAAmB,MAAM;QAC7C,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,KAAK,IAAI,wBAAwB,CAAC;QAC5D,MAAM,WAAW,GAAG,eAAe,CAAC,kBAAkB,EAAE,CAAC;QAEzD,0CAA0C;QAC1C,IAAI,WAAW,CAAC,KAAK,CAAC,EAAE,CAAC;YACvB,OAAO,WAAW,CAAC,KAAK,CAAC,CAAC,SAAS,CAAC;QACtC,CAAC;QAED,uDAAuD;QACvD,IAAI,CAAC;YACH,MAAM,aAAa,GAAG,IAAI,CAAC,cAAc,CAAC,QAAQ,CAAC,CAAC;YACpD,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC,MAAM,CAAC;gBACnD,KAAK,EAAE,KAAK;gBACZ,KAAK,EAAE,aAAa;gBACpB,eAAe,EAAE,OAAO;aACzB,CAAC,CAAC;YACH,OAAO,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,MAAM,CAAC;QAC3C,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,YAAY,GAChB,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe,CAAC;YAE3D,iCAAiC;YACjC,IACE,YAAY,CAAC,QAAQ,CAAC,SAAS,CAAC;gBAChC,YAAY,CAAC,QAAQ,CAAC,cAAc,CAAC;gBACrC,YAAY,CAAC,QAAQ,CAAC,gBAAgB,CAAC,EACvC,CAAC;gBACD,MAAM,IAAI,KAAK,CACb,wCAAwC,KAAK,KAAK,YAAY,EAAE,CACjE,CAAC;YACJ,CAAC;YAED,8DAA8D;YAC9D,MAAM,IAAI,KAAK,CACb,wCAAwC,KAAK,KAAK,YAAY,EAAE,CACjE,CAAC;QACJ,CAAC;IACH,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,IAAY;QACtB,MAAM,aAAa,GAAG,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,CAAC;QAChD,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,KAAK,IAAI,wBAAwB,CAAC;QAE5D,MAAM,WAAW,GAAG,eAAe,CAAC,kBAAkB,EAAE,CAAC;QACzD,IAAI,WAAW,CAAC,KAAK,CAAC,IAAI,IAAI,CAAC,SAAS,KAAK,WAAW,CAAC,KAAK,CAAC,CAAC,SAAS,EAAE,CAAC;YAC1E,IAAI,CAAC,SAAS,GAAG,WAAW,CAAC,KAAK,CAAC,CAAC,SAAS,CAAC;QAChD,CAAC;aAAM,IAAI,CAAC,WAAW,CAAC,KAAK,CAAC,EAAE,CAAC;YAC/B,IAAI,CAAC,SAAS,GAAG,MAAM,IAAI,CAAC,eAAe,EAAE,CAAC;QAChD,CAAC;QAED,IAAI,CAAC;YACH,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC,MAAM,CAAC;gBACnD,KAAK,EAAE,KAAK;gBACZ,KAAK,EAAE,aAAa;gBACpB,eAAe,EAAE,OAAO;aACzB,CAAC,CAAC;YAEH,wCAAwC;YACxC,IAAI,CAAC,SAAS,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,MAAM,CAAC;YAEnD,OAAO;gBACL,MAAM,EAAE,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,SAAS;gBAClC,SAAS,EAAE,IAAI,CAAC,SAAS;aAC1B,CAAC;QACJ,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,YAAY,GAChB,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe,CAAC;YAC3D,MAAM,IAAI,KAAK,CAAC,wCAAwC,YAAY,EAAE,CAAC,CAAC;QAC1E,CAAC;IACH,CAAC;IAED,KAAK,CAAC,UAAU,CAAC,KAAe;QAC9B,MAAM,cAAc,GAAG,IAAI,CAAC,eAAe,CAAC,KAAK,CAAC,CAAC;QACnD,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,KAAK,IAAI,wBAAwB,CAAC;QAE5D,MAAM,WAAW,GAAG,eAAe,CAAC,kBAAkB,EAAE,CAAC;QACzD,IAAI,WAAW,CAAC,KAAK,CAAC,IAAI,IAAI,CAAC,SAAS,KAAK,WAAW,CAAC,KAAK,CAAC,CAAC,SAAS,EAAE,CAAC;YAC1E,IAAI,CAAC,SAAS,GAAG,WAAW,CAAC,KAAK,CAAC,CAAC,SAAS,CAAC;QAChD,CAAC;aAAM,IAAI,CAAC,WAAW,CAAC,KAAK,CAAC,EAAE,CAAC;YAC/B,IAAI,CAAC,SAAS,GAAG,MAAM,IAAI,CAAC,eAAe,EAAE,CAAC;QAChD,CAAC;QAED,IAAI,CAAC;YACH,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC,MAAM,CAAC;gBACnD,KAAK,EAAE,KAAK;gBACZ,KAAK,EAAE,cAAc;gBACrB,eAAe,EAAE,OAAO;aACzB,CAAC,CAAC;YAEH,IAAI,CAAC,SAAS,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,MAAM,CAAC;YAEnD,OAAO,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;gBAClC,MAAM,EAAE,IAAI,CAAC,SAAS;gBACtB,SAAS,EAAE,IAAI,CAAC,SAAS;aAC1B,CAAC,CAAC,CAAC;QACN,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,YAAY,GAChB,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe,CAAC;YAC3D,MAAM,IAAI,KAAK,CACb,+CAA+C,YAAY,EAAE,CAC9D,CAAC;QACJ,CAAC;IACH,CAAC;IAED,YAAY;QACV,2DAA2D;QAC3D,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,KAAK,IAAI,wBAAwB,CAAC;QAC5D,MAAM,WAAW,GAAG,eAAe,CAAC,kBAAkB,EAAE,CAAC;QAEzD,oDAAoD;QACpD,IAAI,WAAW,CAAC,KAAK,CAAC,EAAE,CAAC;YACvB,OAAO,WAAW,CAAC,KAAK,CAAC,CAAC,SAAS,CAAC;QACtC,CAAC;QAED,kDAAkD;QAClD,gEAAgE;QAChE,OAAO,CAAC,IAAI,CACV,gEAAgE,KAAK,iBAAiB,IAAI,CAAC,SAAS,wDAAwD,CAC7J,CAAC;QACF,OAAO,IAAI,CAAC,SAAS,CAAC;IACxB,CAAC;IAED,WAAW;QACT,OAAO,QAAQ,CAAC;IAClB,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,QAAQ,CAAC,KAAa;QAC1B,IAAI,CAAC,MAAM,CAAC,KAAK,GAAG,KAAK,CAAC;QAC1B,MAAM,WAAW,GAAG,eAAe,CAAC,kBAAkB,EAAE,CAAC;QACzD,IAAI,WAAW,CAAC,KAAK,CAAC,EAAE,CAAC;YACvB,IAAI,CAAC,SAAS,GAAG,WAAW,CAAC,KAAK,CAAC,CAAC,SAAS,CAAC;QAChD,CAAC;aAAM,CAAC;YACN,IAAI,CAAC,SAAS,GAAG,MAAM,IAAI,CAAC,eAAe,EAAE,CAAC;QAChD,CAAC;IACH,CAAC;IAED;;OAEG;IACH,SAAS;QACP,OAAO,IAAI,CAAC,MAAM,CAAC;IACrB,CAAC;IAED;;OAEG;IACH,MAAM,CAAC,kBAAkB;QAIvB,OAAO;YACL,wBAAwB,EAAE;gBACxB,SAAS,EAAE,IAAI;gBACf,WAAW,EACT,mEAAmE;aACtE;YACD,wBAAwB,EAAE;gBACxB,SAAS,EAAE,IAAI;gBACf,WAAW,EACT,4DAA4D;aAC/D;YACD,wBAAwB,EAAE;gBACxB,SAAS,EAAE,IAAI;gBACf,WAAW,EAAE,mDAAmD;aACjE;SACF,CAAC;IACJ,CAAC;CACF;AA3LD,0CA2LC"}
|
|
@@ -1,6 +1,5 @@
|
|
|
1
|
-
import type { EmbeddingVector } from "./base-embedding";
|
|
2
1
|
import { VoyageAIClient } from "voyageai";
|
|
3
|
-
import { Embedding } from "./base-embedding";
|
|
2
|
+
import { Embedding, EmbeddingVector } from "./base-embedding";
|
|
4
3
|
export interface VoyageAIEmbeddingConfig {
|
|
5
4
|
model: string;
|
|
6
5
|
apiKey: string;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"voyageai-embedding.d.ts","sourceRoot":"","sources":["../../src/embedding/voyageai-embedding.ts"],"names":[],"mappings":"AAAA,OAAO,
|
|
1
|
+
{"version":3,"file":"voyageai-embedding.d.ts","sourceRoot":"","sources":["../../src/embedding/voyageai-embedding.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,cAAc,EAAE,MAAM,UAAU,CAAC;AAC1C,OAAO,EAAE,SAAS,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;AAE9D,MAAM,WAAW,uBAAuB;IACtC,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;CAChB;AAED,qBAAa,iBAAkB,SAAQ,SAAS;IAC9C,OAAO,CAAC,MAAM,CAAiB;IAC/B,OAAO,CAAC,MAAM,CAA0B;IACxC,OAAO,CAAC,SAAS,CAAgB;IACjC,OAAO,CAAC,SAAS,CAAoC;IACrD,SAAS,CAAC,SAAS,EAAE,MAAM,CAAS;gBAExB,MAAM,EAAE,uBAAuB;IAW3C,OAAO,CAAC,mBAAmB;IAoB3B,OAAO,CAAC,uBAAuB;IAiBzB,eAAe,IAAI,OAAO,CAAC,MAAM,CAAC;IAKlC,KAAK,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,eAAe,CAAC;IAoB7C,UAAU,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,eAAe,EAAE,CAAC;IAyB7D,YAAY,IAAI,MAAM;IAItB,WAAW,IAAI,MAAM;IAIrB;;;OAGG;IACH,QAAQ,CAAC,KAAK,EAAE,MAAM,GAAG,IAAI;IAK7B;;;OAGG;IACH,YAAY,CAAC,SAAS,EAAE,UAAU,GAAG,OAAO,GAAG,IAAI;IAInD;;OAEG;IACH,SAAS,IAAI,cAAc;IAI3B;;OAEG;IACH,MAAM,CAAC,kBAAkB,IAAI,MAAM,CACjC,MAAM,EACN;QAAE,SAAS,EAAE,MAAM,GAAG,MAAM,CAAC;QAAC,aAAa,EAAE,MAAM,CAAC;QAAC,WAAW,EAAE,MAAM,CAAA;KAAE,CAC3E;CAoGF"}
|
|
@@ -62,7 +62,7 @@ class VoyageAIEmbedding extends base_embedding_1.Embedding {
|
|
|
62
62
|
const model = this.config.model || "voyage-code-3";
|
|
63
63
|
const response = await this.client.embed({
|
|
64
64
|
input: processedText,
|
|
65
|
-
model,
|
|
65
|
+
model: model,
|
|
66
66
|
inputType: this.inputType,
|
|
67
67
|
});
|
|
68
68
|
if (!response.data || !response.data[0] || !response.data[0].embedding) {
|
|
@@ -78,7 +78,7 @@ class VoyageAIEmbedding extends base_embedding_1.Embedding {
|
|
|
78
78
|
const model = this.config.model || "voyage-code-3";
|
|
79
79
|
const response = await this.client.embed({
|
|
80
80
|
input: processedTexts,
|
|
81
|
-
model,
|
|
81
|
+
model: model,
|
|
82
82
|
inputType: this.inputType,
|
|
83
83
|
});
|
|
84
84
|
if (!response.data) {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"voyageai-embedding.js","sourceRoot":"","sources":["../../src/embedding/voyageai-embedding.ts"],"names":[],"mappings":";;;
|
|
1
|
+
{"version":3,"file":"voyageai-embedding.js","sourceRoot":"","sources":["../../src/embedding/voyageai-embedding.ts"],"names":[],"mappings":";;;AAAA,uCAA0C;AAC1C,qDAA8D;AAO9D,MAAa,iBAAkB,SAAQ,0BAAS;IAO9C,YAAY,MAA+B;QACzC,KAAK,EAAE,CAAC;QALF,cAAS,GAAW,IAAI,CAAC,CAAC,sCAAsC;QAChE,cAAS,GAAyB,UAAU,CAAC;QAC3C,cAAS,GAAW,KAAK,CAAC,CAAC,qBAAqB;QAIxD,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,MAAM,GAAG,IAAI,yBAAc,CAAC;YAC/B,MAAM,EAAE,MAAM,CAAC,MAAM;SACtB,CAAC,CAAC;QAEH,6DAA6D;QAC7D,IAAI,CAAC,mBAAmB,CAAC,MAAM,CAAC,KAAK,IAAI,eAAe,CAAC,CAAC;IAC5D,CAAC;IAEO,mBAAmB,CAAC,KAAa;QACvC,MAAM,eAAe,GAAG,iBAAiB,CAAC,kBAAkB,EAAE,CAAC;QAC/D,MAAM,SAAS,GAAG,eAAe,CAAC,KAAK,CAAC,CAAC;QAEzC,IAAI,SAAS,EAAE,CAAC;YACd,mFAAmF;YACnF,IAAI,OAAO,SAAS,CAAC,SAAS,KAAK,QAAQ,EAAE,CAAC;gBAC5C,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,CAAC,oBAAoB;YAC7C,CAAC;iBAAM,CAAC;gBACN,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC,SAAS,CAAC;YACvC,CAAC;YACD,iDAAiD;YACjD,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC,aAAa,CAAC;QAC3C,CAAC;aAAM,CAAC;YACN,8DAA8D;YAC9D,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC;YACtB,IAAI,CAAC,SAAS,GAAG,KAAK,CAAC;QACzB,CAAC;IACH,CAAC;IAEO,uBAAuB,CAAC,KAAa;QAC3C,MAAM,eAAe,GAAG,iBAAiB,CAAC,kBAAkB,EAAE,CAAC;QAC/D,MAAM,SAAS,GAAG,eAAe,CAAC,KAAK,CAAC,CAAC;QAEzC,IAAI,SAAS,EAAE,CAAC;YACd,mFAAmF;YACnF,IAAI,OAAO,SAAS,CAAC,SAAS,KAAK,QAAQ,EAAE,CAAC;gBAC5C,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,CAAC,oBAAoB;YAC7C,CAAC;iBAAM,CAAC;gBACN,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC,SAAS,CAAC;YACvC,CAAC;QACH,CAAC;aAAM,CAAC;YACN,2CAA2C;YAC3C,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC;QACxB,CAAC;IACH,CAAC;IAED,KAAK,CAAC,eAAe;QACnB,uEAAuE;QACvE,OAAO,IAAI,CAAC,SAAS,CAAC;IACxB,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,IAAY;QACtB,MAAM,aAAa,GAAG,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,CAAC;QAChD,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,KAAK,IAAI,eAAe,CAAC;QAEnD,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC;YACvC,KAAK,EAAE,aAAa;YACpB,KAAK,EAAE,KAAK;YACZ,SAAS,EAAE,IAAI,CAAC,SAAS;SAC1B,CAAC,CAAC;QAEH,IAAI,CAAC,QAAQ,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,SAAS,EAAE,CAAC;YACvE,MAAM,IAAI,KAAK,CAAC,wCAAwC,CAAC,CAAC;QAC5D,CAAC;QAED,OAAO;YACL,MAAM,EAAE,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,SAAS;YAClC,SAAS,EAAE,IAAI,CAAC,SAAS;SAC1B,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,UAAU,CAAC,KAAe;QAC9B,MAAM,cAAc,GAAG,IAAI,CAAC,eAAe,CAAC,KAAK,CAAC,CAAC;QACnD,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,KAAK,IAAI,eAAe,CAAC;QAEnD,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC;YACvC,KAAK,EAAE,cAAc;YACrB,KAAK,EAAE,KAAK;YACZ,SAAS,EAAE,IAAI,CAAC,SAAS;SAC1B,CAAC,CAAC;QAEH,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC;YACnB,MAAM,IAAI,KAAK,CAAC,wCAAwC,CAAC,CAAC;QAC5D,CAAC;QAED,OAAO,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;YAChC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,CAAC;gBACpB,MAAM,IAAI,KAAK,CAAC,8CAA8C,CAAC,CAAC;YAClE,CAAC;YACD,OAAO;gBACL,MAAM,EAAE,IAAI,CAAC,SAAS;gBACtB,SAAS,EAAE,IAAI,CAAC,SAAS;aAC1B,CAAC;QACJ,CAAC,CAAC,CAAC;IACL,CAAC;IAED,YAAY;QACV,OAAO,IAAI,CAAC,SAAS,CAAC;IACxB,CAAC;IAED,WAAW;QACT,OAAO,UAAU,CAAC;IACpB,CAAC;IAED;;;OAGG;IACH,QAAQ,CAAC,KAAa;QACpB,IAAI,CAAC,MAAM,CAAC,KAAK,GAAG,KAAK,CAAC;QAC1B,IAAI,CAAC,mBAAmB,CAAC,KAAK,CAAC,CAAC;IAClC,CAAC;IAED;;;OAGG;IACH,YAAY,CAAC,SAA+B;QAC1C,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;IAC7B,CAAC;IAED;;OAEG;IACH,SAAS;QACP,OAAO,IAAI,CAAC,MAAM,CAAC;IACrB,CAAC;IAED;;OAEG;IACH,MAAM,CAAC,kBAAkB;QAIvB,OAAO;YACL,4BAA4B;YAC5B,gBAAgB,EAAE;gBAChB,SAAS,EAAE,gCAAgC;gBAC3C,aAAa,EAAE,KAAK;gBACpB,WAAW,EACT,6DAA6D;aAChE;YACD,YAAY,EAAE;gBACZ,SAAS,EAAE,gCAAgC;gBAC3C,aAAa,EAAE,KAAK;gBACpB,WAAW,EACT,kEAAkE;aACrE;YACD,iBAAiB,EAAE;gBACjB,SAAS,EAAE,gCAAgC;gBAC3C,aAAa,EAAE,KAAK;gBACpB,WAAW,EAAE,gCAAgC;aAC9C;YACD,eAAe,EAAE;gBACf,SAAS,EAAE,gCAAgC;gBAC3C,aAAa,EAAE,KAAK;gBACpB,WAAW,EAAE,qDAAqD;aACnE;YACD,6BAA6B;YAC7B,kBAAkB,EAAE;gBAClB,SAAS,EAAE,IAAI;gBACf,aAAa,EAAE,KAAK;gBACpB,WAAW,EAAE,yCAAyC;aACvD;YACD,cAAc,EAAE;gBACd,SAAS,EAAE,IAAI;gBACf,aAAa,EAAE,KAAK;gBACpB,WAAW,EAAE,uCAAuC;aACrD;YACD,uBAAuB,EAAE;gBACvB,SAAS,EAAE,IAAI;gBACf,aAAa,EAAE,KAAK;gBACpB,WAAW,EAAE,+CAA+C;aAC7D;YACD,yBAAyB,EAAE;gBACzB,SAAS,EAAE,IAAI;gBACf,aAAa,EAAE,KAAK;gBACpB,WAAW,EAAE,gCAAgC;aAC9C;YACD,gBAAgB;YAChB,gBAAgB,EAAE;gBAChB,SAAS,EAAE,IAAI;gBACf,aAAa,EAAE,KAAK;gBACpB,WAAW,EAAE,gCAAgC;aAC9C;YACD,eAAe,EAAE;gBACf,SAAS,EAAE,IAAI;gBACf,aAAa,EAAE,KAAK;gBACpB,WAAW,EAAE,wCAAwC;aACtD;YACD,UAAU,EAAE;gBACV,SAAS,EAAE,IAAI;gBACf,aAAa,EAAE,KAAK;gBACpB,WAAW,EAAE,gCAAgC;aAC9C;YACD,eAAe,EAAE;gBACf,SAAS,EAAE,GAAG;gBACd,aAAa,EAAE,KAAK;gBACpB,WAAW,EAAE,qCAAqC;aACnD;YACD,UAAU,EAAE;gBACV,SAAS,EAAE,IAAI;gBACf,aAAa,EAAE,IAAI;gBACnB,WAAW,EAAE,qCAAqC;aACnD;YACD,sBAAsB;YACtB,WAAW,EAAE;gBACX,SAAS,EAAE,IAAI;gBACf,aAAa,EAAE,IAAI;gBACnB,WAAW,EAAE,cAAc;aAC5B;YACD,WAAW,EAAE;gBACX,SAAS,EAAE,IAAI;gBACf,aAAa,EAAE,IAAI;gBACnB,WAAW,EAAE,cAAc;aAC5B;YACD,gBAAgB,EAAE;gBAChB,SAAS,EAAE,IAAI;gBACf,aAAa,EAAE,IAAI;gBACnB,WAAW,EAAE,cAAc;aAC5B;YACD,yBAAyB,EAAE;gBACzB,SAAS,EAAE,IAAI;gBACf,aAAa,EAAE,IAAI;gBACnB,WAAW,EAAE,cAAc;aAC5B;YACD,yBAAyB,EAAE;gBACzB,SAAS,EAAE,IAAI;gBACf,aAAa,EAAE,IAAI;gBACnB,WAAW,EAAE,cAAc;aAC5B;SACF,CAAC;IACJ,CAAC;CACF;AAnPD,8CAmPC"}
|
package/dist/index.d.ts
CHANGED
|
@@ -1,8 +1,8 @@
|
|
|
1
|
-
export * from "./context";
|
|
2
|
-
export * from "./embedding";
|
|
3
1
|
export * from "./splitter";
|
|
4
|
-
export * from "./
|
|
2
|
+
export * from "./embedding";
|
|
3
|
+
export * from "./vectordb";
|
|
5
4
|
export * from "./types";
|
|
5
|
+
export * from "./context";
|
|
6
|
+
export * from "./sync/synchronizer";
|
|
6
7
|
export * from "./utils";
|
|
7
|
-
export * from "./vectordb";
|
|
8
8
|
//# sourceMappingURL=index.d.ts.map
|
package/dist/index.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,YAAY,CAAC;AAC3B,cAAc,aAAa,CAAC;AAC5B,cAAc,YAAY,CAAC;AAC3B,cAAc,SAAS,CAAC;AACxB,cAAc,WAAW,CAAC;AAC1B,cAAc,qBAAqB,CAAC;AACpC,cAAc,SAAS,CAAC"}
|
package/dist/index.js
CHANGED
|
@@ -14,11 +14,11 @@ var __exportStar = (this && this.__exportStar) || function(m, exports) {
|
|
|
14
14
|
for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
|
|
15
15
|
};
|
|
16
16
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
17
|
-
__exportStar(require("./context"), exports);
|
|
18
|
-
__exportStar(require("./embedding"), exports);
|
|
19
17
|
__exportStar(require("./splitter"), exports);
|
|
20
|
-
__exportStar(require("./
|
|
18
|
+
__exportStar(require("./embedding"), exports);
|
|
19
|
+
__exportStar(require("./vectordb"), exports);
|
|
21
20
|
__exportStar(require("./types"), exports);
|
|
21
|
+
__exportStar(require("./context"), exports);
|
|
22
|
+
__exportStar(require("./sync/synchronizer"), exports);
|
|
22
23
|
__exportStar(require("./utils"), exports);
|
|
23
|
-
__exportStar(require("./vectordb"), exports);
|
|
24
24
|
//# sourceMappingURL=index.js.map
|
package/dist/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;AAAA,
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;AAAA,6CAA2B;AAC3B,8CAA4B;AAC5B,6CAA2B;AAC3B,0CAAwB;AACxB,4CAA0B;AAC1B,sDAAoC;AACpC,0CAAwB"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"ast-splitter.d.ts","sourceRoot":"","sources":["../../src/splitter/ast-splitter.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"ast-splitter.d.ts","sourceRoot":"","sources":["../../src/splitter/ast-splitter.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,QAAQ,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC;AAwF9C,qBAAa,eAAgB,YAAW,QAAQ;IAC9C,OAAO,CAAC,SAAS,CAAgB;IACjC,OAAO,CAAC,YAAY,CAAe;IACnC,OAAO,CAAC,MAAM,CAAS;IACvB,OAAO,CAAC,iBAAiB,CAAwB;gBAErC,SAAS,CAAC,EAAE,MAAM,EAAE,YAAY,CAAC,EAAE,MAAM;IAU/C,KAAK,CACT,IAAI,EAAE,MAAM,EACZ,QAAQ,EAAE,MAAM,EAChB,QAAQ,CAAC,EAAE,MAAM,GAChB,OAAO,CAAC,SAAS,EAAE,CAAC;IA8CvB,YAAY,CAAC,SAAS,EAAE,MAAM,GAAG,IAAI;IAKrC,eAAe,CAAC,YAAY,EAAE,MAAM,GAAG,IAAI;IAK3C,OAAO,CAAC,iBAAiB;IA6BzB,OAAO,CAAC,aAAa;YA0DP,YAAY;IAmB1B,OAAO,CAAC,eAAe;IAsDvB,OAAO,CAAC,UAAU;IA+BlB,OAAO,CAAC,YAAY;IAIpB;;OAEG;IACH,MAAM,CAAC,mBAAmB,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO;CAqBtD"}
|
|
@@ -7,14 +7,13 @@ exports.AstCodeSplitter = void 0;
|
|
|
7
7
|
const tree_sitter_1 = __importDefault(require("tree-sitter"));
|
|
8
8
|
// Language parsers
|
|
9
9
|
const JavaScript = require("tree-sitter-javascript");
|
|
10
|
-
const langchain_splitter_1 = require("./langchain-splitter");
|
|
11
10
|
const TypeScript = require("tree-sitter-typescript").typescript;
|
|
12
|
-
const
|
|
11
|
+
const Python = require("tree-sitter-python");
|
|
12
|
+
const Java = require("tree-sitter-java");
|
|
13
13
|
const Cpp = require("tree-sitter-cpp");
|
|
14
14
|
const Go = require("tree-sitter-go");
|
|
15
|
-
const Java = require("tree-sitter-java");
|
|
16
|
-
const Python = require("tree-sitter-python");
|
|
17
15
|
const Rust = require("tree-sitter-rust");
|
|
16
|
+
const CSharp = require("tree-sitter-c-sharp");
|
|
18
17
|
const Scala = require("tree-sitter-scala");
|
|
19
18
|
// Node types that represent logical code units
|
|
20
19
|
const SPLITTABLE_NODE_TYPES = {
|
|
@@ -91,7 +90,8 @@ class AstCodeSplitter {
|
|
|
91
90
|
this.chunkOverlap = chunkOverlap;
|
|
92
91
|
this.parser = new tree_sitter_1.default();
|
|
93
92
|
// Initialize fallback splitter
|
|
94
|
-
|
|
93
|
+
const { LangChainCodeSplitter } = require("./langchain-splitter");
|
|
94
|
+
this.langchainFallback = new LangChainCodeSplitter(chunkSize, chunkOverlap);
|
|
95
95
|
}
|
|
96
96
|
async split(code, language, filePath) {
|
|
97
97
|
// Check if language is supported by AST splitter
|
|
@@ -218,7 +218,7 @@ class AstCodeSplitter {
|
|
|
218
218
|
let currentLineCount = 0;
|
|
219
219
|
for (let i = 0; i < lines.length; i++) {
|
|
220
220
|
const line = lines[i];
|
|
221
|
-
const lineWithNewline = i === lines.length - 1 ? line :
|
|
221
|
+
const lineWithNewline = i === lines.length - 1 ? line : line + "\n";
|
|
222
222
|
if (currentChunk.length + lineWithNewline.length > this.chunkSize &&
|
|
223
223
|
currentChunk.length > 0) {
|
|
224
224
|
// Create a sub-chunk
|
|
@@ -266,7 +266,7 @@ class AstCodeSplitter {
|
|
|
266
266
|
if (i > 0 && this.chunkOverlap > 0) {
|
|
267
267
|
const prevChunk = chunks[i - 1];
|
|
268
268
|
const overlapText = prevChunk.content.slice(-this.chunkOverlap);
|
|
269
|
-
content =
|
|
269
|
+
content = overlapText + "\n" + content;
|
|
270
270
|
metadata.startLine = Math.max(1, metadata.startLine - this.getLineCount(overlapText));
|
|
271
271
|
}
|
|
272
272
|
overlappedChunks.push({
|