@zilliz/claude-context-core 0.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +275 -0
- package/dist/context.d.ts +234 -0
- package/dist/context.d.ts.map +1 -0
- package/dist/context.js +879 -0
- package/dist/context.js.map +1 -0
- package/dist/embedding/base-embedding.d.ts +45 -0
- package/dist/embedding/base-embedding.d.ts.map +1 -0
- package/dist/embedding/base-embedding.js +36 -0
- package/dist/embedding/base-embedding.js.map +1 -0
- package/dist/embedding/gemini-embedding.d.ts +51 -0
- package/dist/embedding/gemini-embedding.d.ts.map +1 -0
- package/dist/embedding/gemini-embedding.js +143 -0
- package/dist/embedding/gemini-embedding.js.map +1 -0
- package/dist/embedding/index.d.ts +6 -0
- package/dist/embedding/index.d.ts.map +1 -0
- package/dist/embedding/index.js +24 -0
- package/dist/embedding/index.js.map +1 -0
- package/dist/embedding/ollama-embedding.d.ts +64 -0
- package/dist/embedding/ollama-embedding.d.ts.map +1 -0
- package/dist/embedding/ollama-embedding.js +205 -0
- package/dist/embedding/ollama-embedding.js.map +1 -0
- package/dist/embedding/openai-embedding.d.ts +36 -0
- package/dist/embedding/openai-embedding.d.ts.map +1 -0
- package/dist/embedding/openai-embedding.js +103 -0
- package/dist/embedding/openai-embedding.js.map +1 -0
- package/dist/embedding/voyageai-embedding.d.ts +43 -0
- package/dist/embedding/voyageai-embedding.d.ts.map +1 -0
- package/dist/embedding/voyageai-embedding.js +223 -0
- package/dist/embedding/voyageai-embedding.js.map +1 -0
- package/dist/index.d.ts +8 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +24 -0
- package/dist/index.js.map +1 -0
- package/dist/splitter/ast-splitter.d.ts +22 -0
- package/dist/splitter/ast-splitter.d.ts.map +1 -0
- package/dist/splitter/ast-splitter.js +227 -0
- package/dist/splitter/ast-splitter.js.map +1 -0
- package/dist/splitter/index.d.ts +41 -0
- package/dist/splitter/index.d.ts.map +1 -0
- package/dist/splitter/index.js +27 -0
- package/dist/splitter/index.js.map +1 -0
- package/dist/splitter/langchain-splitter.d.ts +13 -0
- package/dist/splitter/langchain-splitter.d.ts.map +1 -0
- package/dist/splitter/langchain-splitter.js +118 -0
- package/dist/splitter/langchain-splitter.js.map +1 -0
- package/dist/sync/merkle.d.ts +26 -0
- package/dist/sync/merkle.d.ts.map +1 -0
- package/dist/sync/merkle.js +112 -0
- package/dist/sync/merkle.js.map +1 -0
- package/dist/sync/synchronizer.d.ts +30 -0
- package/dist/sync/synchronizer.d.ts.map +1 -0
- package/dist/sync/synchronizer.js +339 -0
- package/dist/sync/synchronizer.js.map +1 -0
- package/dist/types.d.ts +14 -0
- package/dist/types.d.ts.map +1 -0
- package/dist/types.js +3 -0
- package/dist/types.js.map +1 -0
- package/dist/utils/env-manager.d.ts +19 -0
- package/dist/utils/env-manager.d.ts.map +1 -0
- package/dist/utils/env-manager.js +125 -0
- package/dist/utils/env-manager.js.map +1 -0
- package/dist/utils/index.d.ts +2 -0
- package/dist/utils/index.d.ts.map +1 -0
- package/dist/utils/index.js +7 -0
- package/dist/utils/index.js.map +1 -0
- package/dist/vectordb/index.d.ts +5 -0
- package/dist/vectordb/index.d.ts.map +1 -0
- package/dist/vectordb/index.js +14 -0
- package/dist/vectordb/index.js.map +1 -0
- package/dist/vectordb/milvus-restful-vectordb.d.ts +51 -0
- package/dist/vectordb/milvus-restful-vectordb.d.ts.map +1 -0
- package/dist/vectordb/milvus-restful-vectordb.js +406 -0
- package/dist/vectordb/milvus-restful-vectordb.js.map +1 -0
- package/dist/vectordb/milvus-vectordb.d.ts +34 -0
- package/dist/vectordb/milvus-vectordb.d.ts.map +1 -0
- package/dist/vectordb/milvus-vectordb.js +248 -0
- package/dist/vectordb/milvus-vectordb.js.map +1 -0
- package/dist/vectordb/types.d.ts +75 -0
- package/dist/vectordb/types.d.ts.map +1 -0
- package/dist/vectordb/types.js +9 -0
- package/dist/vectordb/types.js.map +1 -0
- package/dist/vectordb/zilliz-utils.d.ts +135 -0
- package/dist/vectordb/zilliz-utils.d.ts.map +1 -0
- package/dist/vectordb/zilliz-utils.js +192 -0
- package/dist/vectordb/zilliz-utils.js.map +1 -0
- package/package.json +56 -0
|
@@ -0,0 +1,205 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.OllamaEmbedding = void 0;
|
|
4
|
+
const ollama_1 = require("ollama");
|
|
5
|
+
const base_embedding_1 = require("./base-embedding");
|
|
6
|
+
class OllamaEmbedding extends base_embedding_1.Embedding {
|
|
7
|
+
constructor(config) {
|
|
8
|
+
super();
|
|
9
|
+
this.dimension = 768; // Default dimension for many embedding models
|
|
10
|
+
this.dimensionDetected = false; // Track if dimension has been detected
|
|
11
|
+
this.maxTokens = 2048; // Default context window for Ollama
|
|
12
|
+
this.config = config;
|
|
13
|
+
this.client = new ollama_1.Ollama({
|
|
14
|
+
host: config.host || 'http://127.0.0.1:11434',
|
|
15
|
+
fetch: config.fetch,
|
|
16
|
+
});
|
|
17
|
+
// Set dimension based on config or will be detected on first use
|
|
18
|
+
if (config.dimension) {
|
|
19
|
+
this.dimension = config.dimension;
|
|
20
|
+
this.dimensionDetected = true;
|
|
21
|
+
}
|
|
22
|
+
// Set max tokens based on config or use default
|
|
23
|
+
if (config.maxTokens) {
|
|
24
|
+
this.maxTokens = config.maxTokens;
|
|
25
|
+
}
|
|
26
|
+
else {
|
|
27
|
+
// Set default based on known models
|
|
28
|
+
this.setDefaultMaxTokensForModel(config.model);
|
|
29
|
+
}
|
|
30
|
+
// If no dimension is provided, it will be detected in the first embed call
|
|
31
|
+
}
|
|
32
|
+
setDefaultMaxTokensForModel(model) {
|
|
33
|
+
// Set different max tokens based on known models
|
|
34
|
+
if (model?.includes('nomic-embed-text')) {
|
|
35
|
+
this.maxTokens = 8192; // nomic-embed-text supports 8192 tokens
|
|
36
|
+
}
|
|
37
|
+
else if (model?.includes('snowflake-arctic-embed')) {
|
|
38
|
+
this.maxTokens = 8192; // snowflake-arctic-embed supports 8192 tokens
|
|
39
|
+
}
|
|
40
|
+
else {
|
|
41
|
+
this.maxTokens = 2048; // Default for most Ollama models
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
async updateDimensionForModel(model) {
|
|
45
|
+
try {
|
|
46
|
+
// Use a dummy query to detect embedding dimension
|
|
47
|
+
const embedOptions = {
|
|
48
|
+
model: model,
|
|
49
|
+
input: 'test',
|
|
50
|
+
options: this.config.options,
|
|
51
|
+
};
|
|
52
|
+
// Only include keep_alive if it has a valid value
|
|
53
|
+
if (this.config.keepAlive && this.config.keepAlive !== '') {
|
|
54
|
+
embedOptions.keep_alive = this.config.keepAlive;
|
|
55
|
+
}
|
|
56
|
+
const response = await this.client.embed(embedOptions);
|
|
57
|
+
if (response.embeddings && response.embeddings[0]) {
|
|
58
|
+
this.dimension = response.embeddings[0].length;
|
|
59
|
+
this.dimensionDetected = true;
|
|
60
|
+
console.log(`📏 Detected embedding dimension: ${this.dimension} for model: ${model}`);
|
|
61
|
+
}
|
|
62
|
+
else {
|
|
63
|
+
// Fallback to default dimension
|
|
64
|
+
this.dimension = 768;
|
|
65
|
+
this.dimensionDetected = true;
|
|
66
|
+
console.warn(`⚠️ Could not detect dimension for model ${model}, using default: 768`);
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
catch (error) {
|
|
70
|
+
console.warn(`Failed to detect dimension for model ${model}, using default dimension 768:`, error);
|
|
71
|
+
this.dimension = 768;
|
|
72
|
+
this.dimensionDetected = true;
|
|
73
|
+
}
|
|
74
|
+
}
|
|
75
|
+
async embed(text) {
|
|
76
|
+
// Preprocess the text
|
|
77
|
+
const processedText = this.preprocessText(text);
|
|
78
|
+
// Detect dimension on first use if not configured
|
|
79
|
+
if (!this.dimensionDetected) {
|
|
80
|
+
await this.updateDimensionForModel(this.config.model);
|
|
81
|
+
}
|
|
82
|
+
const embedOptions = {
|
|
83
|
+
model: this.config.model,
|
|
84
|
+
input: processedText,
|
|
85
|
+
options: this.config.options,
|
|
86
|
+
};
|
|
87
|
+
// Only include keep_alive if it has a valid value
|
|
88
|
+
if (this.config.keepAlive && this.config.keepAlive !== '') {
|
|
89
|
+
embedOptions.keep_alive = this.config.keepAlive;
|
|
90
|
+
}
|
|
91
|
+
const response = await this.client.embed(embedOptions);
|
|
92
|
+
if (!response.embeddings || !response.embeddings[0]) {
|
|
93
|
+
throw new Error('Ollama API returned invalid response');
|
|
94
|
+
}
|
|
95
|
+
return {
|
|
96
|
+
vector: response.embeddings[0],
|
|
97
|
+
dimension: this.dimension
|
|
98
|
+
};
|
|
99
|
+
}
|
|
100
|
+
async embedBatch(texts) {
|
|
101
|
+
// Preprocess all texts
|
|
102
|
+
const processedTexts = this.preprocessTexts(texts);
|
|
103
|
+
// Detect dimension on first use if not configured
|
|
104
|
+
if (!this.dimensionDetected) {
|
|
105
|
+
await this.updateDimensionForModel(this.config.model);
|
|
106
|
+
}
|
|
107
|
+
// Use Ollama's native batch embedding API
|
|
108
|
+
const embedOptions = {
|
|
109
|
+
model: this.config.model,
|
|
110
|
+
input: processedTexts, // Pass array directly to Ollama
|
|
111
|
+
options: this.config.options,
|
|
112
|
+
};
|
|
113
|
+
// Only include keep_alive if it has a valid value
|
|
114
|
+
if (this.config.keepAlive && this.config.keepAlive !== '') {
|
|
115
|
+
embedOptions.keep_alive = this.config.keepAlive;
|
|
116
|
+
}
|
|
117
|
+
const response = await this.client.embed(embedOptions);
|
|
118
|
+
if (!response.embeddings || !Array.isArray(response.embeddings)) {
|
|
119
|
+
throw new Error('Ollama API returned invalid batch response');
|
|
120
|
+
}
|
|
121
|
+
// Convert to EmbeddingVector format
|
|
122
|
+
return response.embeddings.map((embedding) => ({
|
|
123
|
+
vector: embedding,
|
|
124
|
+
dimension: this.dimension
|
|
125
|
+
}));
|
|
126
|
+
}
|
|
127
|
+
getDimension() {
|
|
128
|
+
return this.dimension;
|
|
129
|
+
}
|
|
130
|
+
getProvider() {
|
|
131
|
+
return 'Ollama';
|
|
132
|
+
}
|
|
133
|
+
/**
|
|
134
|
+
* Set model type and detect its dimension
|
|
135
|
+
* @param model Model name
|
|
136
|
+
*/
|
|
137
|
+
async setModel(model) {
|
|
138
|
+
this.config.model = model;
|
|
139
|
+
// Reset dimension detection when model changes
|
|
140
|
+
this.dimensionDetected = false;
|
|
141
|
+
// Update max tokens for new model
|
|
142
|
+
this.setDefaultMaxTokensForModel(model);
|
|
143
|
+
if (!this.config.dimension) {
|
|
144
|
+
await this.updateDimensionForModel(model);
|
|
145
|
+
}
|
|
146
|
+
}
|
|
147
|
+
/**
|
|
148
|
+
* Set host URL
|
|
149
|
+
* @param host Ollama host URL
|
|
150
|
+
*/
|
|
151
|
+
setHost(host) {
|
|
152
|
+
this.config.host = host;
|
|
153
|
+
this.client = new ollama_1.Ollama({
|
|
154
|
+
host: host,
|
|
155
|
+
fetch: this.config.fetch,
|
|
156
|
+
});
|
|
157
|
+
}
|
|
158
|
+
/**
|
|
159
|
+
* Set keep alive duration
|
|
160
|
+
* @param keepAlive Keep alive duration
|
|
161
|
+
*/
|
|
162
|
+
setKeepAlive(keepAlive) {
|
|
163
|
+
this.config.keepAlive = keepAlive;
|
|
164
|
+
}
|
|
165
|
+
/**
|
|
166
|
+
* Set additional options
|
|
167
|
+
* @param options Additional options for the model
|
|
168
|
+
*/
|
|
169
|
+
setOptions(options) {
|
|
170
|
+
this.config.options = options;
|
|
171
|
+
}
|
|
172
|
+
/**
|
|
173
|
+
* Set dimension manually
|
|
174
|
+
* @param dimension Embedding dimension
|
|
175
|
+
*/
|
|
176
|
+
setDimension(dimension) {
|
|
177
|
+
this.config.dimension = dimension;
|
|
178
|
+
this.dimension = dimension;
|
|
179
|
+
this.dimensionDetected = true;
|
|
180
|
+
}
|
|
181
|
+
/**
|
|
182
|
+
* Set max tokens manually
|
|
183
|
+
* @param maxTokens Maximum number of tokens
|
|
184
|
+
*/
|
|
185
|
+
setMaxTokens(maxTokens) {
|
|
186
|
+
this.config.maxTokens = maxTokens;
|
|
187
|
+
this.maxTokens = maxTokens;
|
|
188
|
+
}
|
|
189
|
+
/**
|
|
190
|
+
* Get client instance (for advanced usage)
|
|
191
|
+
*/
|
|
192
|
+
getClient() {
|
|
193
|
+
return this.client;
|
|
194
|
+
}
|
|
195
|
+
/**
|
|
196
|
+
* Initialize dimension detection for the current model
|
|
197
|
+
*/
|
|
198
|
+
async initializeDimension() {
|
|
199
|
+
if (!this.config.dimension) {
|
|
200
|
+
await this.updateDimensionForModel(this.config.model);
|
|
201
|
+
}
|
|
202
|
+
}
|
|
203
|
+
}
|
|
204
|
+
exports.OllamaEmbedding = OllamaEmbedding;
|
|
205
|
+
//# sourceMappingURL=ollama-embedding.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"ollama-embedding.js","sourceRoot":"","sources":["../../src/embedding/ollama-embedding.ts"],"names":[],"mappings":";;;AAAA,mCAAgC;AAChC,qDAA8D;AAY9D,MAAa,eAAgB,SAAQ,0BAAS;IAO1C,YAAY,MAA6B;QACrC,KAAK,EAAE,CAAC;QALJ,cAAS,GAAW,GAAG,CAAC,CAAC,8CAA8C;QACvE,sBAAiB,GAAY,KAAK,CAAC,CAAC,uCAAuC;QACzE,cAAS,GAAW,IAAI,CAAC,CAAC,oCAAoC;QAIpE,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,MAAM,GAAG,IAAI,eAAM,CAAC;YACrB,IAAI,EAAE,MAAM,CAAC,IAAI,IAAI,wBAAwB;YAC7C,KAAK,EAAE,MAAM,CAAC,KAAK;SACtB,CAAC,CAAC;QAEH,iEAAiE;QACjE,IAAI,MAAM,CAAC,SAAS,EAAE,CAAC;YACnB,IAAI,CAAC,SAAS,GAAG,MAAM,CAAC,SAAS,CAAC;YAClC,IAAI,CAAC,iBAAiB,GAAG,IAAI,CAAC;QAClC,CAAC;QAED,gDAAgD;QAChD,IAAI,MAAM,CAAC,SAAS,EAAE,CAAC;YACnB,IAAI,CAAC,SAAS,GAAG,MAAM,CAAC,SAAS,CAAC;QACtC,CAAC;aAAM,CAAC;YACJ,oCAAoC;YACpC,IAAI,CAAC,2BAA2B,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QACnD,CAAC;QAED,2EAA2E;IAC/E,CAAC;IAEO,2BAA2B,CAAC,KAAa;QAC7C,iDAAiD;QACjD,IAAI,KAAK,EAAE,QAAQ,CAAC,kBAAkB,CAAC,EAAE,CAAC;YACtC,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,CAAC,wCAAwC;QACnE,CAAC;aAAM,IAAI,KAAK,EAAE,QAAQ,CAAC,wBAAwB,CAAC,EAAE,CAAC;YACnD,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,CAAC,8CAA8C;QACzE,CAAC;aAAM,CAAC;YACJ,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,CAAC,iCAAiC;QAC5D,CAAC;IACL,CAAC;IAEO,KAAK,CAAC,uBAAuB,CAAC,KAAa;QAC/C,IAAI,CAAC;YACD,kDAAkD;YAClD,MAAM,YAAY,GAAQ;gBACtB,KAAK,EAAE,KAAK;gBACZ,KAAK,EAAE,MAAM;gBACb,OAAO,EAAE,IAAI,CAAC,MAAM,CAAC,OAAO;aAC/B,CAAC;YAEF,kDAAkD;YAClD,IAAI,IAAI,CAAC,MAAM,CAAC,SAAS,IAAI,IAAI,CAAC,MAAM,CAAC,SAAS,KAAK,EAAE,EAAE,CAAC;gBACxD,YAAY,CAAC,UAAU,GAAG,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC;YACpD,CAAC;YAED,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC;YAEvD,IAAI,QAAQ,CAAC,UAAU,IAAI,QAAQ,CAAC,UAAU,CAAC,CAAC,CAAC,EAAE,CAAC;gBAChD,IAAI,CAAC,SAAS,GAAG,QAAQ,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC;gBAC/C,IAAI,CAAC,iBAAiB,GAAG,IAAI,CAAC;gBAC9B,OAAO,CAAC,GAAG,CAAC,oCAAoC,IAAI,CAAC,SAAS,eAAe,KAAK,EAAE,CAAC,CAAC;YAC1F,CAAC;iBAAM,CAAC;gBACJ,gCAAgC;gBAChC,IAAI,CAAC,SAAS,GAAG,GAAG,CAAC;gBACrB,IAAI,CAAC,iBAAiB,GAAG,IAAI,CAAC;gBAC9B,OAAO,CAAC,IAAI,CAAC,4CAA4C,KAAK,sBAAsB,CAAC,CAAC;YAC1F,CAAC;QACL,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACb,OAAO,CAAC,IAAI,CAAC,wCAAwC,KAAK,gCAAgC,EAAE,KAAK,CAAC,CAAC;YACnG,IAAI,CAAC,SAAS,GAAG,GAAG,CAAC;YACrB,IAAI,CAAC,iBAAiB,GAAG,IAAI,CAAC;QAClC,CAAC;IACL,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,IAAY;QACpB,sBAAsB;QACtB,MAAM,aAAa,GAAG,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,CAAC;QAEhD,kDAAkD;QAClD,IAAI,CAAC,IAAI,CAAC,iBAAiB,EAAE,CAAC;YAC1B,MAAM,IAAI,CAAC,uBAAuB,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QAC1D,CAAC;QAED,MAAM,YAAY,GAAQ;YACtB,KAAK,EAAE,IAAI,CAAC,MAAM,CAAC,KAAK;YACxB,KAAK,EAAE,aAAa;YACpB,OAAO,EAAE,IAAI,CAAC,MAAM,CAAC,OAAO;SAC/B,CAAC;QAEF,kDAAkD;QAClD,IAAI,IAAI,CAAC,MAAM,CAAC,SAAS,IAAI,IAAI,CAAC,MAAM,CAAC,SAAS,KAAK,EAAE,EAAE,CAAC;YACxD,YAAY,CAAC,UAAU,GAAG,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC;QACpD,CAAC;QAED,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC;QAEvD,IAAI,CAAC,QAAQ,CAAC,UAAU,IAAI,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC,CAAC,EAAE,CAAC;YAClD,MAAM,IAAI,KAAK,CAAC,sCAAsC,CAAC,CAAC;QAC5D,CAAC;QAED,OAAO;YACH,MAAM,EAAE,QAAQ,CAAC,UAAU,CAAC,CAAC,CAAC;YAC9B,SAAS,EAAE,IAAI,CAAC,SAAS;SAC5B,CAAC;IACN,CAAC;IAED,KAAK,CAAC,UAAU,CAAC,KAAe;QAC5B,uBAAuB;QACvB,MAAM,cAAc,GAAG,IAAI,CAAC,eAAe,CAAC,KAAK,CAAC,CAAC;QAEnD,kDAAkD;QAClD,IAAI,CAAC,IAAI,CAAC,iBAAiB,EAAE,CAAC;YAC1B,MAAM,IAAI,CAAC,uBAAuB,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QAC1D,CAAC;QAED,0CAA0C;QAC1C,MAAM,YAAY,GAAQ;YACtB,KAAK,EAAE,IAAI,CAAC,MAAM,CAAC,KAAK;YACxB,KAAK,EAAE,cAAc,EAAE,gCAAgC;YACvD,OAAO,EAAE,IAAI,CAAC,MAAM,CAAC,OAAO;SAC/B,CAAC;QAEF,kDAAkD;QAClD,IAAI,IAAI,CAAC,MAAM,CAAC,SAAS,IAAI,IAAI,CAAC,MAAM,CAAC,SAAS,KAAK,EAAE,EAAE,CAAC;YACxD,YAAY,CAAC,UAAU,GAAG,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC;QACpD,CAAC;QAED,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC;QAEvD,IAAI,CAAC,QAAQ,CAAC,UAAU,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAC,EAAE,CAAC;YAC9D,MAAM,IAAI,KAAK,CAAC,4CAA4C,CAAC,CAAC;QAClE,CAAC;QAED,oCAAoC;QACpC,OAAO,QAAQ,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,SAAmB,EAAE,EAAE,CAAC,CAAC;YACrD,MAAM,EAAE,SAAS;YACjB,SAAS,EAAE,IAAI,CAAC,SAAS;SAC5B,CAAC,CAAC,CAAC;IACR,CAAC;IAED,YAAY;QACR,OAAO,IAAI,CAAC,SAAS,CAAC;IAC1B,CAAC;IAED,WAAW;QACP,OAAO,QAAQ,CAAC;IACpB,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,QAAQ,CAAC,KAAa;QACxB,IAAI,CAAC,MAAM,CAAC,KAAK,GAAG,KAAK,CAAC;QAC1B,+CAA+C;QAC/C,IAAI,CAAC,iBAAiB,GAAG,KAAK,CAAC;QAC/B,kCAAkC;QAClC,IAAI,CAAC,2BAA2B,CAAC,KAAK,CAAC,CAAC;QACxC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,SAAS,EAAE,CAAC;YACzB,MAAM,IAAI,CAAC,uBAAuB,CAAC,KAAK,CAAC,CAAC;QAC9C,CAAC;IACL,CAAC;IAED;;;OAGG;IACH,OAAO,CAAC,IAAY;QAChB,IAAI,CAAC,MAAM,CAAC,IAAI,GAAG,IAAI,CAAC;QACxB,IAAI,CAAC,MAAM,GAAG,IAAI,eAAM,CAAC;YACrB,IAAI,EAAE,IAAI;YACV,KAAK,EAAE,IAAI,CAAC,MAAM,CAAC,KAAK;SAC3B,CAAC,CAAC;IACP,CAAC;IAED;;;OAGG;IACH,YAAY,CAAC,SAA0B;QACnC,IAAI,CAAC,MAAM,CAAC,SAAS,GAAG,SAAS,CAAC;IACtC,CAAC;IAED;;;OAGG;IACH,UAAU,CAAC,OAA4B;QACnC,IAAI,CAAC,MAAM,CAAC,OAAO,GAAG,OAAO,CAAC;IAClC,CAAC;IAED;;;OAGG;IACH,YAAY,CAAC,SAAiB;QAC1B,IAAI,CAAC,MAAM,CAAC,SAAS,GAAG,SAAS,CAAC;QAClC,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;QAC3B,IAAI,CAAC,iBAAiB,GAAG,IAAI,CAAC;IAClC,CAAC;IAED;;;OAGG;IACH,YAAY,CAAC,SAAiB;QAC1B,IAAI,CAAC,MAAM,CAAC,SAAS,GAAG,SAAS,CAAC;QAClC,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;IAC/B,CAAC;IAED;;OAEG;IACH,SAAS;QACL,OAAO,IAAI,CAAC,MAAM,CAAC;IACvB,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,mBAAmB;QACrB,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,SAAS,EAAE,CAAC;YACzB,MAAM,IAAI,CAAC,uBAAuB,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QAC1D,CAAC;IACL,CAAC;CAEJ;AApOD,0CAoOC"}
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
import OpenAI from 'openai';
|
|
2
|
+
import { Embedding, EmbeddingVector } from './base-embedding';
|
|
3
|
+
export interface OpenAIEmbeddingConfig {
|
|
4
|
+
model: string;
|
|
5
|
+
apiKey: string;
|
|
6
|
+
baseURL?: string;
|
|
7
|
+
}
|
|
8
|
+
export declare class OpenAIEmbedding extends Embedding {
|
|
9
|
+
private client;
|
|
10
|
+
private config;
|
|
11
|
+
private dimension;
|
|
12
|
+
protected maxTokens: number;
|
|
13
|
+
constructor(config: OpenAIEmbeddingConfig);
|
|
14
|
+
private updateDimensionForModel;
|
|
15
|
+
embed(text: string): Promise<EmbeddingVector>;
|
|
16
|
+
embedBatch(texts: string[]): Promise<EmbeddingVector[]>;
|
|
17
|
+
getDimension(): number;
|
|
18
|
+
getProvider(): string;
|
|
19
|
+
/**
|
|
20
|
+
* Set model type
|
|
21
|
+
* @param model Model name
|
|
22
|
+
*/
|
|
23
|
+
setModel(model: string): void;
|
|
24
|
+
/**
|
|
25
|
+
* Get client instance (for advanced usage)
|
|
26
|
+
*/
|
|
27
|
+
getClient(): OpenAI;
|
|
28
|
+
/**
|
|
29
|
+
* Get list of supported models
|
|
30
|
+
*/
|
|
31
|
+
static getSupportedModels(): Record<string, {
|
|
32
|
+
dimension: number;
|
|
33
|
+
description: string;
|
|
34
|
+
}>;
|
|
35
|
+
}
|
|
36
|
+
//# sourceMappingURL=openai-embedding.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"openai-embedding.d.ts","sourceRoot":"","sources":["../../src/embedding/openai-embedding.ts"],"names":[],"mappings":"AAAA,OAAO,MAAM,MAAM,QAAQ,CAAC;AAC5B,OAAO,EAAE,SAAS,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;AAE9D,MAAM,WAAW,qBAAqB;IAClC,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,qBAAa,eAAgB,SAAQ,SAAS;IAC1C,OAAO,CAAC,MAAM,CAAS;IACvB,OAAO,CAAC,MAAM,CAAwB;IACtC,OAAO,CAAC,SAAS,CAAgB;IACjC,SAAS,CAAC,SAAS,EAAE,MAAM,CAAQ;gBAEvB,MAAM,EAAE,qBAAqB;IAYzC,OAAO,CAAC,uBAAuB;IAYzB,KAAK,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,eAAe,CAAC;IAgB7C,UAAU,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,eAAe,EAAE,CAAC;IAgB7D,YAAY,IAAI,MAAM;IAItB,WAAW,IAAI,MAAM;IAIrB;;;OAGG;IACH,QAAQ,CAAC,KAAK,EAAE,MAAM,GAAG,IAAI;IAK7B;;OAEG;IACH,SAAS,IAAI,MAAM;IAInB;;OAEG;IACH,MAAM,CAAC,kBAAkB,IAAI,MAAM,CAAC,MAAM,EAAE;QAAE,SAAS,EAAE,MAAM,CAAC;QAAC,WAAW,EAAE,MAAM,CAAA;KAAE,CAAC;CAgB1F"}
|
|
@@ -0,0 +1,103 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
|
+
};
|
|
5
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
+
exports.OpenAIEmbedding = void 0;
|
|
7
|
+
const openai_1 = __importDefault(require("openai"));
|
|
8
|
+
const base_embedding_1 = require("./base-embedding");
|
|
9
|
+
class OpenAIEmbedding extends base_embedding_1.Embedding {
|
|
10
|
+
constructor(config) {
|
|
11
|
+
super();
|
|
12
|
+
this.dimension = 1536; // Default dimension for text-embedding-3-small
|
|
13
|
+
this.maxTokens = 8192; // Maximum tokens for OpenAI embedding models
|
|
14
|
+
this.config = config;
|
|
15
|
+
this.client = new openai_1.default({
|
|
16
|
+
apiKey: config.apiKey,
|
|
17
|
+
baseURL: config.baseURL,
|
|
18
|
+
});
|
|
19
|
+
// Set dimension based on model
|
|
20
|
+
this.updateDimensionForModel(config.model || 'text-embedding-3-small');
|
|
21
|
+
}
|
|
22
|
+
updateDimensionForModel(model) {
|
|
23
|
+
if (model === 'text-embedding-3-small') {
|
|
24
|
+
this.dimension = 1536;
|
|
25
|
+
}
|
|
26
|
+
else if (model === 'text-embedding-3-large') {
|
|
27
|
+
this.dimension = 3072;
|
|
28
|
+
}
|
|
29
|
+
else if (model === 'text-embedding-ada-002') {
|
|
30
|
+
this.dimension = 1536;
|
|
31
|
+
}
|
|
32
|
+
else {
|
|
33
|
+
this.dimension = 1536; // Default dimension
|
|
34
|
+
}
|
|
35
|
+
}
|
|
36
|
+
async embed(text) {
|
|
37
|
+
const processedText = this.preprocessText(text);
|
|
38
|
+
const model = this.config.model || 'text-embedding-3-small';
|
|
39
|
+
const response = await this.client.embeddings.create({
|
|
40
|
+
model: model,
|
|
41
|
+
input: processedText,
|
|
42
|
+
encoding_format: 'float',
|
|
43
|
+
});
|
|
44
|
+
return {
|
|
45
|
+
vector: response.data[0].embedding,
|
|
46
|
+
dimension: this.dimension
|
|
47
|
+
};
|
|
48
|
+
}
|
|
49
|
+
async embedBatch(texts) {
|
|
50
|
+
const processedTexts = this.preprocessTexts(texts);
|
|
51
|
+
const model = this.config.model || 'text-embedding-3-small';
|
|
52
|
+
const response = await this.client.embeddings.create({
|
|
53
|
+
model: model,
|
|
54
|
+
input: processedTexts,
|
|
55
|
+
encoding_format: 'float',
|
|
56
|
+
});
|
|
57
|
+
return response.data.map((item) => ({
|
|
58
|
+
vector: item.embedding,
|
|
59
|
+
dimension: this.dimension
|
|
60
|
+
}));
|
|
61
|
+
}
|
|
62
|
+
getDimension() {
|
|
63
|
+
return this.dimension;
|
|
64
|
+
}
|
|
65
|
+
getProvider() {
|
|
66
|
+
return 'OpenAI';
|
|
67
|
+
}
|
|
68
|
+
/**
|
|
69
|
+
* Set model type
|
|
70
|
+
* @param model Model name
|
|
71
|
+
*/
|
|
72
|
+
setModel(model) {
|
|
73
|
+
this.config.model = model;
|
|
74
|
+
this.updateDimensionForModel(model);
|
|
75
|
+
}
|
|
76
|
+
/**
|
|
77
|
+
* Get client instance (for advanced usage)
|
|
78
|
+
*/
|
|
79
|
+
getClient() {
|
|
80
|
+
return this.client;
|
|
81
|
+
}
|
|
82
|
+
/**
|
|
83
|
+
* Get list of supported models
|
|
84
|
+
*/
|
|
85
|
+
static getSupportedModels() {
|
|
86
|
+
return {
|
|
87
|
+
'text-embedding-3-small': {
|
|
88
|
+
dimension: 1536,
|
|
89
|
+
description: 'High performance and cost-effective embedding model (recommended)'
|
|
90
|
+
},
|
|
91
|
+
'text-embedding-3-large': {
|
|
92
|
+
dimension: 3072,
|
|
93
|
+
description: 'Highest performance embedding model with larger dimensions'
|
|
94
|
+
},
|
|
95
|
+
'text-embedding-ada-002': {
|
|
96
|
+
dimension: 1536,
|
|
97
|
+
description: 'Legacy model (use text-embedding-3-small instead)'
|
|
98
|
+
}
|
|
99
|
+
};
|
|
100
|
+
}
|
|
101
|
+
}
|
|
102
|
+
exports.OpenAIEmbedding = OpenAIEmbedding;
|
|
103
|
+
//# sourceMappingURL=openai-embedding.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"openai-embedding.js","sourceRoot":"","sources":["../../src/embedding/openai-embedding.ts"],"names":[],"mappings":";;;;;;AAAA,oDAA4B;AAC5B,qDAA8D;AAQ9D,MAAa,eAAgB,SAAQ,0BAAS;IAM1C,YAAY,MAA6B;QACrC,KAAK,EAAE,CAAC;QAJJ,cAAS,GAAW,IAAI,CAAC,CAAC,+CAA+C;QACvE,cAAS,GAAW,IAAI,CAAC,CAAC,6CAA6C;QAI7E,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,MAAM,GAAG,IAAI,gBAAM,CAAC;YACrB,MAAM,EAAE,MAAM,CAAC,MAAM;YACrB,OAAO,EAAE,MAAM,CAAC,OAAO;SAC1B,CAAC,CAAC;QAEH,+BAA+B;QAC/B,IAAI,CAAC,uBAAuB,CAAC,MAAM,CAAC,KAAK,IAAI,wBAAwB,CAAC,CAAC;IAC3E,CAAC;IAEO,uBAAuB,CAAC,KAAa;QACzC,IAAI,KAAK,KAAK,wBAAwB,EAAE,CAAC;YACrC,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC;QAC1B,CAAC;aAAM,IAAI,KAAK,KAAK,wBAAwB,EAAE,CAAC;YAC5C,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC;QAC1B,CAAC;aAAM,IAAI,KAAK,KAAK,wBAAwB,EAAE,CAAC;YAC5C,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC;QAC1B,CAAC;aAAM,CAAC;YACJ,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,CAAC,oBAAoB;QAC/C,CAAC;IACL,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,IAAY;QACpB,MAAM,aAAa,GAAG,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,CAAC;QAChD,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,KAAK,IAAI,wBAAwB,CAAC;QAE5D,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC,MAAM,CAAC;YACjD,KAAK,EAAE,KAAK;YACZ,KAAK,EAAE,aAAa;YACpB,eAAe,EAAE,OAAO;SAC3B,CAAC,CAAC;QAEH,OAAO;YACH,MAAM,EAAE,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,SAAS;YAClC,SAAS,EAAE,IAAI,CAAC,SAAS;SAC5B,CAAC;IACN,CAAC;IAED,KAAK,CAAC,UAAU,CAAC,KAAe;QAC5B,MAAM,cAAc,GAAG,IAAI,CAAC,eAAe,CAAC,KAAK,CAAC,CAAC;QACnD,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,KAAK,IAAI,wBAAwB,CAAC;QAE5D,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC,MAAM,CAAC;YACjD,KAAK,EAAE,KAAK;YACZ,KAAK,EAAE,cAAc;YACrB,eAAe,EAAE,OAAO;SAC3B,CAAC,CAAC;QAEH,OAAO,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;YAChC,MAAM,EAAE,IAAI,CAAC,SAAS;YACtB,SAAS,EAAE,IAAI,CAAC,SAAS;SAC5B,CAAC,CAAC,CAAC;IACR,CAAC;IAED,YAAY;QACR,OAAO,IAAI,CAAC,SAAS,CAAC;IAC1B,CAAC;IAED,WAAW;QACP,OAAO,QAAQ,CAAC;IACpB,CAAC;IAED;;;OAGG;IACH,QAAQ,CAAC,KAAa;QAClB,IAAI,CAAC,MAAM,CAAC,KAAK,GAAG,KAAK,CAAC;QAC1B,IAAI,CAAC,uBAAuB,CAAC,KAAK,CAAC,CAAC;IACxC,CAAC;IAED;;OAEG;IACH,SAAS;QACL,OAAO,IAAI,CAAC,MAAM,CAAC;IACvB,CAAC;IAED;;OAEG;IACH,MAAM,CAAC,kBAAkB;QACrB,OAAO;YACH,wBAAwB,EAAE;gBACtB,SAAS,EAAE,IAAI;gBACf,WAAW,EAAE,mEAAmE;aACnF;YACD,wBAAwB,EAAE;gBACtB,SAAS,EAAE,IAAI;gBACf,WAAW,EAAE,4DAA4D;aAC5E;YACD,wBAAwB,EAAE;gBACtB,SAAS,EAAE,IAAI;gBACf,WAAW,EAAE,mDAAmD;aACnE;SACJ,CAAC;IACN,CAAC;CACJ;AAzGD,0CAyGC"}
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
import { VoyageAIClient } from 'voyageai';
|
|
2
|
+
import { Embedding, EmbeddingVector } from './base-embedding';
|
|
3
|
+
export interface VoyageAIEmbeddingConfig {
|
|
4
|
+
model: string;
|
|
5
|
+
apiKey: string;
|
|
6
|
+
}
|
|
7
|
+
export declare class VoyageAIEmbedding extends Embedding {
|
|
8
|
+
private client;
|
|
9
|
+
private config;
|
|
10
|
+
private dimension;
|
|
11
|
+
private inputType;
|
|
12
|
+
protected maxTokens: number;
|
|
13
|
+
constructor(config: VoyageAIEmbeddingConfig);
|
|
14
|
+
private updateModelSettings;
|
|
15
|
+
private updateDimensionForModel;
|
|
16
|
+
embed(text: string): Promise<EmbeddingVector>;
|
|
17
|
+
embedBatch(texts: string[]): Promise<EmbeddingVector[]>;
|
|
18
|
+
getDimension(): number;
|
|
19
|
+
getProvider(): string;
|
|
20
|
+
/**
|
|
21
|
+
* Set model type
|
|
22
|
+
* @param model Model name
|
|
23
|
+
*/
|
|
24
|
+
setModel(model: string): void;
|
|
25
|
+
/**
|
|
26
|
+
* Set input type (VoyageAI specific feature)
|
|
27
|
+
* @param inputType Input type: 'document' | 'query'
|
|
28
|
+
*/
|
|
29
|
+
setInputType(inputType: 'document' | 'query'): void;
|
|
30
|
+
/**
|
|
31
|
+
* Get client instance (for advanced usage)
|
|
32
|
+
*/
|
|
33
|
+
getClient(): VoyageAIClient;
|
|
34
|
+
/**
|
|
35
|
+
* Get list of supported models
|
|
36
|
+
*/
|
|
37
|
+
static getSupportedModels(): Record<string, {
|
|
38
|
+
dimension: number | string;
|
|
39
|
+
contextLength: number;
|
|
40
|
+
description: string;
|
|
41
|
+
}>;
|
|
42
|
+
}
|
|
43
|
+
//# sourceMappingURL=voyageai-embedding.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"voyageai-embedding.d.ts","sourceRoot":"","sources":["../../src/embedding/voyageai-embedding.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,cAAc,EAAE,MAAM,UAAU,CAAC;AAC1C,OAAO,EAAE,SAAS,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;AAE9D,MAAM,WAAW,uBAAuB;IACpC,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;CAClB;AAED,qBAAa,iBAAkB,SAAQ,SAAS;IAC5C,OAAO,CAAC,MAAM,CAAiB;IAC/B,OAAO,CAAC,MAAM,CAA0B;IACxC,OAAO,CAAC,SAAS,CAAgB;IACjC,OAAO,CAAC,SAAS,CAAoC;IACrD,SAAS,CAAC,SAAS,EAAE,MAAM,CAAS;gBAExB,MAAM,EAAE,uBAAuB;IAW3C,OAAO,CAAC,mBAAmB;IAoB3B,OAAO,CAAC,uBAAuB;IAiBzB,KAAK,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,eAAe,CAAC;IAoB7C,UAAU,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,eAAe,EAAE,CAAC;IAyB7D,YAAY,IAAI,MAAM;IAItB,WAAW,IAAI,MAAM;IAIrB;;;OAGG;IACH,QAAQ,CAAC,KAAK,EAAE,MAAM,GAAG,IAAI;IAK7B;;;OAGG;IACH,YAAY,CAAC,SAAS,EAAE,UAAU,GAAG,OAAO,GAAG,IAAI;IAInD;;OAEG;IACH,SAAS,IAAI,cAAc;IAI3B;;OAEG;IACH,MAAM,CAAC,kBAAkB,IAAI,MAAM,CAAC,MAAM,EAAE;QAAE,SAAS,EAAE,MAAM,GAAG,MAAM,CAAC;QAAC,aAAa,EAAE,MAAM,CAAC;QAAC,WAAW,EAAE,MAAM,CAAA;KAAE,CAAC;CAkG1H"}
|
|
@@ -0,0 +1,223 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.VoyageAIEmbedding = void 0;
|
|
4
|
+
const voyageai_1 = require("voyageai");
|
|
5
|
+
const base_embedding_1 = require("./base-embedding");
|
|
6
|
+
class VoyageAIEmbedding extends base_embedding_1.Embedding {
|
|
7
|
+
constructor(config) {
|
|
8
|
+
super();
|
|
9
|
+
this.dimension = 1024; // Default dimension for voyage-code-3
|
|
10
|
+
this.inputType = 'document';
|
|
11
|
+
this.maxTokens = 32000; // Default max tokens
|
|
12
|
+
this.config = config;
|
|
13
|
+
this.client = new voyageai_1.VoyageAIClient({
|
|
14
|
+
apiKey: config.apiKey,
|
|
15
|
+
});
|
|
16
|
+
// Set dimension and context length based on different models
|
|
17
|
+
this.updateModelSettings(config.model || 'voyage-code-3');
|
|
18
|
+
}
|
|
19
|
+
updateModelSettings(model) {
|
|
20
|
+
const supportedModels = VoyageAIEmbedding.getSupportedModels();
|
|
21
|
+
const modelInfo = supportedModels[model];
|
|
22
|
+
if (modelInfo) {
|
|
23
|
+
// If dimension is a string (indicating variable dimension), use default value 1024
|
|
24
|
+
if (typeof modelInfo.dimension === 'string') {
|
|
25
|
+
this.dimension = 1024; // Default dimension
|
|
26
|
+
}
|
|
27
|
+
else {
|
|
28
|
+
this.dimension = modelInfo.dimension;
|
|
29
|
+
}
|
|
30
|
+
// Set max tokens based on model's context length
|
|
31
|
+
this.maxTokens = modelInfo.contextLength;
|
|
32
|
+
}
|
|
33
|
+
else {
|
|
34
|
+
// Use default dimension and context length for unknown models
|
|
35
|
+
this.dimension = 1024;
|
|
36
|
+
this.maxTokens = 32000;
|
|
37
|
+
}
|
|
38
|
+
}
|
|
39
|
+
updateDimensionForModel(model) {
|
|
40
|
+
const supportedModels = VoyageAIEmbedding.getSupportedModels();
|
|
41
|
+
const modelInfo = supportedModels[model];
|
|
42
|
+
if (modelInfo) {
|
|
43
|
+
// If dimension is a string (indicating variable dimension), use default value 1024
|
|
44
|
+
if (typeof modelInfo.dimension === 'string') {
|
|
45
|
+
this.dimension = 1024; // Default dimension
|
|
46
|
+
}
|
|
47
|
+
else {
|
|
48
|
+
this.dimension = modelInfo.dimension;
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
else {
|
|
52
|
+
// Use default dimension for unknown models
|
|
53
|
+
this.dimension = 1024;
|
|
54
|
+
}
|
|
55
|
+
}
|
|
56
|
+
async embed(text) {
|
|
57
|
+
const processedText = this.preprocessText(text);
|
|
58
|
+
const model = this.config.model || 'voyage-code-3';
|
|
59
|
+
const response = await this.client.embed({
|
|
60
|
+
input: processedText,
|
|
61
|
+
model: model,
|
|
62
|
+
inputType: this.inputType,
|
|
63
|
+
});
|
|
64
|
+
if (!response.data || !response.data[0] || !response.data[0].embedding) {
|
|
65
|
+
throw new Error('VoyageAI API returned invalid response');
|
|
66
|
+
}
|
|
67
|
+
return {
|
|
68
|
+
vector: response.data[0].embedding,
|
|
69
|
+
dimension: this.dimension
|
|
70
|
+
};
|
|
71
|
+
}
|
|
72
|
+
async embedBatch(texts) {
|
|
73
|
+
const processedTexts = this.preprocessTexts(texts);
|
|
74
|
+
const model = this.config.model || 'voyage-code-3';
|
|
75
|
+
const response = await this.client.embed({
|
|
76
|
+
input: processedTexts,
|
|
77
|
+
model: model,
|
|
78
|
+
inputType: this.inputType,
|
|
79
|
+
});
|
|
80
|
+
if (!response.data) {
|
|
81
|
+
throw new Error('VoyageAI API returned invalid response');
|
|
82
|
+
}
|
|
83
|
+
return response.data.map((item) => {
|
|
84
|
+
if (!item.embedding) {
|
|
85
|
+
throw new Error('VoyageAI API returned invalid embedding data');
|
|
86
|
+
}
|
|
87
|
+
return {
|
|
88
|
+
vector: item.embedding,
|
|
89
|
+
dimension: this.dimension
|
|
90
|
+
};
|
|
91
|
+
});
|
|
92
|
+
}
|
|
93
|
+
getDimension() {
|
|
94
|
+
return this.dimension;
|
|
95
|
+
}
|
|
96
|
+
getProvider() {
|
|
97
|
+
return 'VoyageAI';
|
|
98
|
+
}
|
|
99
|
+
/**
|
|
100
|
+
* Set model type
|
|
101
|
+
* @param model Model name
|
|
102
|
+
*/
|
|
103
|
+
setModel(model) {
|
|
104
|
+
this.config.model = model;
|
|
105
|
+
this.updateModelSettings(model);
|
|
106
|
+
}
|
|
107
|
+
/**
|
|
108
|
+
* Set input type (VoyageAI specific feature)
|
|
109
|
+
* @param inputType Input type: 'document' | 'query'
|
|
110
|
+
*/
|
|
111
|
+
setInputType(inputType) {
|
|
112
|
+
this.inputType = inputType;
|
|
113
|
+
}
|
|
114
|
+
/**
|
|
115
|
+
* Get client instance (for advanced usage)
|
|
116
|
+
*/
|
|
117
|
+
getClient() {
|
|
118
|
+
return this.client;
|
|
119
|
+
}
|
|
120
|
+
/**
|
|
121
|
+
* Get list of supported models
|
|
122
|
+
*/
|
|
123
|
+
static getSupportedModels() {
|
|
124
|
+
return {
|
|
125
|
+
// Latest recommended models
|
|
126
|
+
'voyage-3-large': {
|
|
127
|
+
dimension: '1024 (default), 256, 512, 2048',
|
|
128
|
+
contextLength: 32000,
|
|
129
|
+
description: 'The best general-purpose and multilingual retrieval quality'
|
|
130
|
+
},
|
|
131
|
+
'voyage-3.5': {
|
|
132
|
+
dimension: '1024 (default), 256, 512, 2048',
|
|
133
|
+
contextLength: 32000,
|
|
134
|
+
description: 'Optimized for general-purpose and multilingual retrieval quality'
|
|
135
|
+
},
|
|
136
|
+
'voyage-3.5-lite': {
|
|
137
|
+
dimension: '1024 (default), 256, 512, 2048',
|
|
138
|
+
contextLength: 32000,
|
|
139
|
+
description: 'Optimized for latency and cost'
|
|
140
|
+
},
|
|
141
|
+
'voyage-code-3': {
|
|
142
|
+
dimension: '1024 (default), 256, 512, 2048',
|
|
143
|
+
contextLength: 32000,
|
|
144
|
+
description: 'Optimized for code retrieval (recommended for code)'
|
|
145
|
+
},
|
|
146
|
+
// Professional domain models
|
|
147
|
+
'voyage-finance-2': {
|
|
148
|
+
dimension: 1024,
|
|
149
|
+
contextLength: 32000,
|
|
150
|
+
description: 'Optimized for finance retrieval and RAG'
|
|
151
|
+
},
|
|
152
|
+
'voyage-law-2': {
|
|
153
|
+
dimension: 1024,
|
|
154
|
+
contextLength: 16000,
|
|
155
|
+
description: 'Optimized for legal retrieval and RAG'
|
|
156
|
+
},
|
|
157
|
+
'voyage-multilingual-2': {
|
|
158
|
+
dimension: 1024,
|
|
159
|
+
contextLength: 32000,
|
|
160
|
+
description: 'Legacy: Use voyage-3.5 for multilingual tasks'
|
|
161
|
+
},
|
|
162
|
+
'voyage-large-2-instruct': {
|
|
163
|
+
dimension: 1024,
|
|
164
|
+
contextLength: 16000,
|
|
165
|
+
description: 'Legacy: Use voyage-3.5 instead'
|
|
166
|
+
},
|
|
167
|
+
// Legacy models
|
|
168
|
+
'voyage-large-2': {
|
|
169
|
+
dimension: 1536,
|
|
170
|
+
contextLength: 16000,
|
|
171
|
+
description: 'Legacy: Use voyage-3.5 instead'
|
|
172
|
+
},
|
|
173
|
+
'voyage-code-2': {
|
|
174
|
+
dimension: 1536,
|
|
175
|
+
contextLength: 16000,
|
|
176
|
+
description: 'Previous generation of code embeddings'
|
|
177
|
+
},
|
|
178
|
+
'voyage-3': {
|
|
179
|
+
dimension: 1024,
|
|
180
|
+
contextLength: 32000,
|
|
181
|
+
description: 'Legacy: Use voyage-3.5 instead'
|
|
182
|
+
},
|
|
183
|
+
'voyage-3-lite': {
|
|
184
|
+
dimension: 512,
|
|
185
|
+
contextLength: 32000,
|
|
186
|
+
description: 'Legacy: Use voyage-3.5-lite instead'
|
|
187
|
+
},
|
|
188
|
+
'voyage-2': {
|
|
189
|
+
dimension: 1024,
|
|
190
|
+
contextLength: 4000,
|
|
191
|
+
description: 'Legacy: Use voyage-3.5-lite instead'
|
|
192
|
+
},
|
|
193
|
+
// Other legacy models
|
|
194
|
+
'voyage-02': {
|
|
195
|
+
dimension: 1024,
|
|
196
|
+
contextLength: 4000,
|
|
197
|
+
description: 'Legacy model'
|
|
198
|
+
},
|
|
199
|
+
'voyage-01': {
|
|
200
|
+
dimension: 1024,
|
|
201
|
+
contextLength: 4000,
|
|
202
|
+
description: 'Legacy model'
|
|
203
|
+
},
|
|
204
|
+
'voyage-lite-01': {
|
|
205
|
+
dimension: 1024,
|
|
206
|
+
contextLength: 4000,
|
|
207
|
+
description: 'Legacy model'
|
|
208
|
+
},
|
|
209
|
+
'voyage-lite-01-instruct': {
|
|
210
|
+
dimension: 1024,
|
|
211
|
+
contextLength: 4000,
|
|
212
|
+
description: 'Legacy model'
|
|
213
|
+
},
|
|
214
|
+
'voyage-lite-02-instruct': {
|
|
215
|
+
dimension: 1024,
|
|
216
|
+
contextLength: 4000,
|
|
217
|
+
description: 'Legacy model'
|
|
218
|
+
}
|
|
219
|
+
};
|
|
220
|
+
}
|
|
221
|
+
}
|
|
222
|
+
exports.VoyageAIEmbedding = VoyageAIEmbedding;
|
|
223
|
+
//# sourceMappingURL=voyageai-embedding.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"voyageai-embedding.js","sourceRoot":"","sources":["../../src/embedding/voyageai-embedding.ts"],"names":[],"mappings":";;;AAAA,uCAA0C;AAC1C,qDAA8D;AAO9D,MAAa,iBAAkB,SAAQ,0BAAS;IAO5C,YAAY,MAA+B;QACvC,KAAK,EAAE,CAAC;QALJ,cAAS,GAAW,IAAI,CAAC,CAAC,sCAAsC;QAChE,cAAS,GAAyB,UAAU,CAAC;QAC3C,cAAS,GAAW,KAAK,CAAC,CAAC,qBAAqB;QAItD,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,MAAM,GAAG,IAAI,yBAAc,CAAC;YAC7B,MAAM,EAAE,MAAM,CAAC,MAAM;SACxB,CAAC,CAAC;QAEH,6DAA6D;QAC7D,IAAI,CAAC,mBAAmB,CAAC,MAAM,CAAC,KAAK,IAAI,eAAe,CAAC,CAAC;IAC9D,CAAC;IAEO,mBAAmB,CAAC,KAAa;QACrC,MAAM,eAAe,GAAG,iBAAiB,CAAC,kBAAkB,EAAE,CAAC;QAC/D,MAAM,SAAS,GAAG,eAAe,CAAC,KAAK,CAAC,CAAC;QAEzC,IAAI,SAAS,EAAE,CAAC;YACZ,mFAAmF;YACnF,IAAI,OAAO,SAAS,CAAC,SAAS,KAAK,QAAQ,EAAE,CAAC;gBAC1C,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,CAAC,oBAAoB;YAC/C,CAAC;iBAAM,CAAC;gBACJ,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC,SAAS,CAAC;YACzC,CAAC;YACD,iDAAiD;YACjD,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC,aAAa,CAAC;QAC7C,CAAC;aAAM,CAAC;YACJ,8DAA8D;YAC9D,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC;YACtB,IAAI,CAAC,SAAS,GAAG,KAAK,CAAC;QAC3B,CAAC;IACL,CAAC;IAEO,uBAAuB,CAAC,KAAa;QACzC,MAAM,eAAe,GAAG,iBAAiB,CAAC,kBAAkB,EAAE,CAAC;QAC/D,MAAM,SAAS,GAAG,eAAe,CAAC,KAAK,CAAC,CAAC;QAEzC,IAAI,SAAS,EAAE,CAAC;YACZ,mFAAmF;YACnF,IAAI,OAAO,SAAS,CAAC,SAAS,KAAK,QAAQ,EAAE,CAAC;gBAC1C,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,CAAC,oBAAoB;YAC/C,CAAC;iBAAM,CAAC;gBACJ,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC,SAAS,CAAC;YACzC,CAAC;QACL,CAAC;aAAM,CAAC;YACJ,2CAA2C;YAC3C,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC;QAC1B,CAAC;IACL,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,IAAY;QACpB,MAAM,aAAa,GAAG,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,CAAC;QAChD,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,KAAK,IAAI,eAAe,CAAC;QAEnD,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC;YACrC,KAAK,EAAE,aAAa;YACpB,KAAK,EAAE,KAAK;YACZ,SAAS,EAAE,IAAI,CAAC,SAAS;SAC5B,CAAC,CAAC;QAEH,IAAI,CAAC,QAAQ,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,SAAS,EAAE,CAAC;YACrE,MAAM,IAAI,KAAK,CAAC,wCAAwC,CAAC,CAAC;QAC9D,CAAC;QAED,OAAO;YACH,MAAM,EAAE,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,SAAS;YAClC,SAAS,EAAE,IAAI,CAAC,SAAS;SAC5B,CAAC;IACN,CAAC;IAED,KAAK,CAAC,UAAU,CAAC,KAAe;QAC5B,MAAM,cAAc,GAAG,IAAI,CAAC,eAAe,CAAC,KAAK,CAAC,CAAC;QACnD,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,KAAK,IAAI,eAAe,CAAC;QAEnD,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC;YACrC,KAAK,EAAE,cAAc;YACrB,KAAK,EAAE,KAAK;YACZ,SAAS,EAAE,IAAI,CAAC,SAAS;SAC5B,CAAC,CAAC;QAEH,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC;YACjB,MAAM,IAAI,KAAK,CAAC,wCAAwC,CAAC,CAAC;QAC9D,CAAC;QAED,OAAO,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;YAC9B,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,CAAC;gBAClB,MAAM,IAAI,KAAK,CAAC,8CAA8C,CAAC,CAAC;YACpE,CAAC;YACD,OAAO;gBACH,MAAM,EAAE,IAAI,CAAC,SAAS;gBACtB,SAAS,EAAE,IAAI,CAAC,SAAS;aAC5B,CAAC;QACN,CAAC,CAAC,CAAC;IACP,CAAC;IAED,YAAY;QACR,OAAO,IAAI,CAAC,SAAS,CAAC;IAC1B,CAAC;IAED,WAAW;QACP,OAAO,UAAU,CAAC;IACtB,CAAC;IAED;;;OAGG;IACH,QAAQ,CAAC,KAAa;QAClB,IAAI,CAAC,MAAM,CAAC,KAAK,GAAG,KAAK,CAAC;QAC1B,IAAI,CAAC,mBAAmB,CAAC,KAAK,CAAC,CAAC;IACpC,CAAC;IAED;;;OAGG;IACH,YAAY,CAAC,SAA+B;QACxC,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;IAC/B,CAAC;IAED;;OAEG;IACH,SAAS;QACL,OAAO,IAAI,CAAC,MAAM,CAAC;IACvB,CAAC;IAED;;OAEG;IACH,MAAM,CAAC,kBAAkB;QACrB,OAAO;YACH,4BAA4B;YAC5B,gBAAgB,EAAE;gBACd,SAAS,EAAE,gCAAgC;gBAC3C,aAAa,EAAE,KAAK;gBACpB,WAAW,EAAE,6DAA6D;aAC7E;YACD,YAAY,EAAE;gBACV,SAAS,EAAE,gCAAgC;gBAC3C,aAAa,EAAE,KAAK;gBACpB,WAAW,EAAE,kEAAkE;aAClF;YACD,iBAAiB,EAAE;gBACf,SAAS,EAAE,gCAAgC;gBAC3C,aAAa,EAAE,KAAK;gBACpB,WAAW,EAAE,gCAAgC;aAChD;YACD,eAAe,EAAE;gBACb,SAAS,EAAE,gCAAgC;gBAC3C,aAAa,EAAE,KAAK;gBACpB,WAAW,EAAE,qDAAqD;aACrE;YACD,6BAA6B;YAC7B,kBAAkB,EAAE;gBAChB,SAAS,EAAE,IAAI;gBACf,aAAa,EAAE,KAAK;gBACpB,WAAW,EAAE,yCAAyC;aACzD;YACD,cAAc,EAAE;gBACZ,SAAS,EAAE,IAAI;gBACf,aAAa,EAAE,KAAK;gBACpB,WAAW,EAAE,uCAAuC;aACvD;YACD,uBAAuB,EAAE;gBACrB,SAAS,EAAE,IAAI;gBACf,aAAa,EAAE,KAAK;gBACpB,WAAW,EAAE,+CAA+C;aAC/D;YACD,yBAAyB,EAAE;gBACvB,SAAS,EAAE,IAAI;gBACf,aAAa,EAAE,KAAK;gBACpB,WAAW,EAAE,gCAAgC;aAChD;YACD,gBAAgB;YAChB,gBAAgB,EAAE;gBACd,SAAS,EAAE,IAAI;gBACf,aAAa,EAAE,KAAK;gBACpB,WAAW,EAAE,gCAAgC;aAChD;YACD,eAAe,EAAE;gBACb,SAAS,EAAE,IAAI;gBACf,aAAa,EAAE,KAAK;gBACpB,WAAW,EAAE,wCAAwC;aACxD;YACD,UAAU,EAAE;gBACR,SAAS,EAAE,IAAI;gBACf,aAAa,EAAE,KAAK;gBACpB,WAAW,EAAE,gCAAgC;aAChD;YACD,eAAe,EAAE;gBACb,SAAS,EAAE,GAAG;gBACd,aAAa,EAAE,KAAK;gBACpB,WAAW,EAAE,qCAAqC;aACrD;YACD,UAAU,EAAE;gBACR,SAAS,EAAE,IAAI;gBACf,aAAa,EAAE,IAAI;gBACnB,WAAW,EAAE,qCAAqC;aACrD;YACD,sBAAsB;YACtB,WAAW,EAAE;gBACT,SAAS,EAAE,IAAI;gBACf,aAAa,EAAE,IAAI;gBACnB,WAAW,EAAE,cAAc;aAC9B;YACD,WAAW,EAAE;gBACT,SAAS,EAAE,IAAI;gBACf,aAAa,EAAE,IAAI;gBACnB,WAAW,EAAE,cAAc;aAC9B;YACD,gBAAgB,EAAE;gBACd,SAAS,EAAE,IAAI;gBACf,aAAa,EAAE,IAAI;gBACnB,WAAW,EAAE,cAAc;aAC9B;YACD,yBAAyB,EAAE;gBACvB,SAAS,EAAE,IAAI;gBACf,aAAa,EAAE,IAAI;gBACnB,WAAW,EAAE,cAAc;aAC9B;YACD,yBAAyB,EAAE;gBACvB,SAAS,EAAE,IAAI;gBACf,aAAa,EAAE,IAAI;gBACnB,WAAW,EAAE,cAAc;aAC9B;SACJ,CAAC;IACN,CAAC;CACJ;AAzOD,8CAyOC"}
|
package/dist/index.d.ts
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,YAAY,CAAC;AAC3B,cAAc,aAAa,CAAC;AAC5B,cAAc,YAAY,CAAC;AAC3B,cAAc,SAAS,CAAC;AACxB,cAAc,WAAW,CAAC;AAC1B,cAAc,qBAAqB,CAAC;AACpC,cAAc,SAAS,CAAC"}
|