@retrivora-ai/rag-engine 0.1.5 → 0.1.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{DocumentChunker-cfaMidtA.d.mts → DocumentChunker-BEyzadsv.d.mts} +2 -2
- package/dist/{DocumentChunker-cfaMidtA.d.ts → DocumentChunker-BEyzadsv.d.ts} +2 -2
- package/dist/{PineconeProvider-NJ675H7U.mjs → PineconeProvider-ZRAFNFEC.mjs} +1 -1
- package/dist/{PostgreSQLProvider-ISNMD3BE.mjs → PostgreSQLProvider-ZNXA67IM.mjs} +1 -1
- package/dist/{QdrantProvider-LJWOIGES.mjs → QdrantProvider-NYGHAA7C.mjs} +1 -1
- package/dist/{RagConfig-DG_0f8ka.d.mts → RagConfig-hBGXJmSx.d.mts} +3 -3
- package/dist/{RagConfig-DG_0f8ka.d.ts → RagConfig-hBGXJmSx.d.ts} +3 -3
- package/dist/{chunk-6FODXNUF.mjs → chunk-5K23H7JL.mjs} +27 -1
- package/dist/chunk-HSBXE2WV.mjs +1758 -0
- package/dist/{chunk-S5DRHETN.mjs → chunk-IUTAZ7QR.mjs} +31 -2
- package/dist/{chunk-AALIF3AL.mjs → chunk-ZM6TYIDH.mjs} +3 -3
- package/dist/handlers/index.d.mts +3 -44
- package/dist/handlers/index.d.ts +3 -44
- package/dist/handlers/index.js +1322 -57
- package/dist/handlers/index.mjs +1 -1
- package/dist/index-Bx182KKn.d.ts +64 -0
- package/dist/index-Ck2pt7-8.d.mts +64 -0
- package/dist/index.d.mts +4 -4
- package/dist/index.d.ts +4 -4
- package/dist/server.d.mts +65 -18
- package/dist/server.d.ts +65 -18
- package/dist/server.js +1322 -57
- package/dist/server.mjs +4 -4
- package/package.json +2 -1
- package/src/config/serverConfig.ts +3 -0
- package/src/core/BatchProcessor.ts +347 -0
- package/src/core/ConfigValidator.ts +560 -0
- package/src/core/Pipeline.ts +89 -39
- package/src/core/ProviderHealthCheck.ts +565 -0
- package/src/core/VectorPlugin.ts +49 -8
- package/src/handlers/index.ts +2 -2
- package/src/providers/vectordb/BaseVectorProvider.ts +1 -1
- package/src/providers/vectordb/MilvusProvider.ts +1 -1
- package/src/providers/vectordb/MongoDBProvider.ts +1 -1
- package/src/providers/vectordb/PineconeProvider.ts +4 -4
- package/src/providers/vectordb/PostgreSQLProvider.ts +35 -3
- package/src/providers/vectordb/QdrantProvider.ts +32 -2
- package/src/rag/DocumentChunker.ts +2 -2
- package/src/types/index.ts +3 -3
- package/dist/chunk-BP4U4TT5.mjs +0 -548
package/dist/chunk-BP4U4TT5.mjs
DELETED
|
@@ -1,548 +0,0 @@
|
|
|
1
|
-
import {
|
|
2
|
-
LLMFactory
|
|
3
|
-
} from "./chunk-JI6VD5TJ.mjs";
|
|
4
|
-
import {
|
|
5
|
-
mergeDefined
|
|
6
|
-
} from "./chunk-UKDXCXW7.mjs";
|
|
7
|
-
import {
|
|
8
|
-
__spreadProps,
|
|
9
|
-
__spreadValues
|
|
10
|
-
} from "./chunk-I4E63NIC.mjs";
|
|
11
|
-
|
|
12
|
-
// src/handlers/index.ts
|
|
13
|
-
import { NextResponse } from "next/server";
|
|
14
|
-
|
|
15
|
-
// src/config/serverConfig.ts
|
|
16
|
-
var VECTOR_DB_PROVIDERS = ["pinecone", "pgvector", "postgresql", "mongodb", "milvus", "qdrant", "chromadb", "redis", "weaviate", "rest", "universal_rest", "custom"];
|
|
17
|
-
var LLM_PROVIDERS = ["openai", "anthropic", "ollama", "rest", "universal_rest", "custom"];
|
|
18
|
-
var EMBEDDING_PROVIDERS = ["openai", "ollama", "rest", "universal_rest", "custom"];
|
|
19
|
-
function readString(env, name) {
|
|
20
|
-
var _a;
|
|
21
|
-
const value = (_a = env[name]) == null ? void 0 : _a.trim();
|
|
22
|
-
return value ? value : void 0;
|
|
23
|
-
}
|
|
24
|
-
function readNumber(env, name, fallback) {
|
|
25
|
-
const value = env[name];
|
|
26
|
-
if (!value) return fallback;
|
|
27
|
-
const parsed = Number(value);
|
|
28
|
-
if (!Number.isFinite(parsed)) {
|
|
29
|
-
throw new Error(`[getRagConfig] ${name} must be a valid number`);
|
|
30
|
-
}
|
|
31
|
-
return parsed;
|
|
32
|
-
}
|
|
33
|
-
function readEnum(env, name, fallback, allowed) {
|
|
34
|
-
var _a;
|
|
35
|
-
const value = (_a = readString(env, name)) != null ? _a : fallback;
|
|
36
|
-
if (allowed.includes(value)) {
|
|
37
|
-
return value;
|
|
38
|
-
}
|
|
39
|
-
throw new Error(`[getRagConfig] ${name} must be one of: ${allowed.join(", ")}`);
|
|
40
|
-
}
|
|
41
|
-
function getRagConfig(env = process.env) {
|
|
42
|
-
var _a, _b, _c, _d, _e, _f, _g, _h, _i, _j, _k, _l, _m, _n, _o, _p, _q, _r, _s, _t, _u, _v, _w, _x, _y, _z, _A, _B, _C, _D, _E, _F, _G, _H, _I, _J;
|
|
43
|
-
const projectId = (_b = (_a = readString(env, "RAG_PROJECT_ID")) != null ? _a : readString(env, "NEXT_PUBLIC_PROJECT_ID")) != null ? _b : "__default__";
|
|
44
|
-
const vectorProvider = readEnum(env, "VECTOR_DB_PROVIDER", "pinecone", VECTOR_DB_PROVIDERS);
|
|
45
|
-
const llmProvider = readEnum(env, "LLM_PROVIDER", "openai", LLM_PROVIDERS);
|
|
46
|
-
const embeddingProvider = readEnum(env, "EMBEDDING_PROVIDER", "openai", EMBEDDING_PROVIDERS);
|
|
47
|
-
const embeddingDimensions = readNumber(env, "EMBEDDING_DIMENSIONS", 1536);
|
|
48
|
-
const vectorDbOptions = {};
|
|
49
|
-
if (vectorProvider === "pinecone") {
|
|
50
|
-
vectorDbOptions.apiKey = (_c = readString(env, "PINECONE_API_KEY")) != null ? _c : "";
|
|
51
|
-
vectorDbOptions.environment = (_d = readString(env, "PINECONE_ENVIRONMENT")) != null ? _d : "";
|
|
52
|
-
} else if (vectorProvider === "pgvector") {
|
|
53
|
-
vectorDbOptions.connectionString = (_e = readString(env, "PGVECTOR_CONNECTION_STRING")) != null ? _e : "";
|
|
54
|
-
vectorDbOptions.dimensions = embeddingDimensions;
|
|
55
|
-
} else if (vectorProvider === "rest") {
|
|
56
|
-
vectorDbOptions.baseUrl = (_f = readString(env, "VECTOR_DB_REST_URL")) != null ? _f : "";
|
|
57
|
-
vectorDbOptions.headers = readString(env, "VECTOR_DB_REST_API_KEY") ? { "api-key": readString(env, "VECTOR_DB_REST_API_KEY") } : {};
|
|
58
|
-
} else if (vectorProvider === "universal_rest") {
|
|
59
|
-
vectorDbOptions.baseUrl = (_h = (_g = readString(env, "VECTOR_BASE_URL")) != null ? _g : readString(env, "VECTOR_DB_REST_URL")) != null ? _h : "";
|
|
60
|
-
vectorDbOptions.profile = readString(env, "VECTOR_UNIVERSAL_PROFILE");
|
|
61
|
-
vectorDbOptions.headers = readString(env, "VECTOR_DB_REST_API_KEY") ? { Authorization: `Bearer ${readString(env, "VECTOR_DB_REST_API_KEY")}` } : {};
|
|
62
|
-
} else if (vectorProvider === "mongodb") {
|
|
63
|
-
vectorDbOptions.uri = (_i = readString(env, "MONGODB_URI")) != null ? _i : "";
|
|
64
|
-
vectorDbOptions.database = (_j = readString(env, "MONGODB_DB")) != null ? _j : "";
|
|
65
|
-
vectorDbOptions.collection = (_k = readString(env, "MONGODB_COLLECTION")) != null ? _k : "";
|
|
66
|
-
vectorDbOptions.indexName = (_l = readString(env, "MONGODB_INDEX_NAME")) != null ? _l : "vector_index";
|
|
67
|
-
}
|
|
68
|
-
const llmApiKeyByProvider = {
|
|
69
|
-
openai: readString(env, "OPENAI_API_KEY"),
|
|
70
|
-
anthropic: readString(env, "ANTHROPIC_API_KEY"),
|
|
71
|
-
ollama: void 0,
|
|
72
|
-
universal_rest: readString(env, "LLM_API_KEY")
|
|
73
|
-
};
|
|
74
|
-
const embeddingApiKeyByProvider = {
|
|
75
|
-
openai: readString(env, "OPENAI_API_KEY"),
|
|
76
|
-
ollama: void 0,
|
|
77
|
-
custom: (_m = readString(env, "EMBEDDING_API_KEY")) != null ? _m : readString(env, "OPENAI_API_KEY")
|
|
78
|
-
};
|
|
79
|
-
return {
|
|
80
|
-
projectId,
|
|
81
|
-
vectorDb: {
|
|
82
|
-
provider: vectorProvider,
|
|
83
|
-
indexName: (_n = readString(env, "VECTOR_DB_INDEX")) != null ? _n : "rag-index",
|
|
84
|
-
options: vectorDbOptions
|
|
85
|
-
},
|
|
86
|
-
llm: {
|
|
87
|
-
provider: llmProvider,
|
|
88
|
-
model: (_o = readString(env, "LLM_MODEL")) != null ? _o : "gpt-4o",
|
|
89
|
-
apiKey: (_p = llmApiKeyByProvider[llmProvider]) != null ? _p : "",
|
|
90
|
-
baseUrl: readString(env, "LLM_BASE_URL"),
|
|
91
|
-
systemPrompt: readString(env, "LLM_SYSTEM_PROMPT"),
|
|
92
|
-
maxTokens: readNumber(env, "LLM_MAX_TOKENS", 1024),
|
|
93
|
-
temperature: readNumber(env, "LLM_TEMPERATURE", 0.7),
|
|
94
|
-
options: {
|
|
95
|
-
profile: readString(env, "LLM_UNIVERSAL_PROFILE")
|
|
96
|
-
}
|
|
97
|
-
},
|
|
98
|
-
embedding: {
|
|
99
|
-
provider: embeddingProvider,
|
|
100
|
-
model: (_q = readString(env, "EMBEDDING_MODEL")) != null ? _q : "text-embedding-3-small",
|
|
101
|
-
apiKey: embeddingApiKeyByProvider[embeddingProvider],
|
|
102
|
-
baseUrl: readString(env, "EMBEDDING_BASE_URL"),
|
|
103
|
-
dimensions: embeddingDimensions,
|
|
104
|
-
options: {
|
|
105
|
-
profile: readString(env, "EMBEDDING_UNIVERSAL_PROFILE")
|
|
106
|
-
}
|
|
107
|
-
},
|
|
108
|
-
ui: {
|
|
109
|
-
title: (_s = (_r = readString(env, "NEXT_PUBLIC_UI_TITLE")) != null ? _r : readString(env, "UI_TITLE")) != null ? _s : "AI Assistant",
|
|
110
|
-
subtitle: (_u = (_t = readString(env, "NEXT_PUBLIC_UI_SUBTITLE")) != null ? _t : readString(env, "UI_SUBTITLE")) != null ? _u : "Powered by RAG",
|
|
111
|
-
primaryColor: (_w = (_v = readString(env, "NEXT_PUBLIC_PRIMARY_COLOR")) != null ? _v : readString(env, "UI_PRIMARY_COLOR")) != null ? _w : "#10b981",
|
|
112
|
-
accentColor: (_y = (_x = readString(env, "NEXT_PUBLIC_ACCENT_COLOR")) != null ? _x : readString(env, "UI_ACCENT_COLOR")) != null ? _y : "#3b82f6",
|
|
113
|
-
logoUrl: (_z = readString(env, "NEXT_PUBLIC_LOGO_URL")) != null ? _z : readString(env, "UI_LOGO_URL"),
|
|
114
|
-
placeholder: (_B = (_A = readString(env, "NEXT_PUBLIC_PLACEHOLDER")) != null ? _A : readString(env, "UI_PLACEHOLDER")) != null ? _B : "Ask me anything\u2026",
|
|
115
|
-
showSources: ((_D = (_C = readString(env, "NEXT_PUBLIC_SHOW_SOURCES")) != null ? _C : readString(env, "UI_SHOW_SOURCES")) != null ? _D : "true") !== "false",
|
|
116
|
-
welcomeMessage: (_F = (_E = readString(env, "NEXT_PUBLIC_WELCOME_MESSAGE")) != null ? _E : readString(env, "UI_WELCOME_MESSAGE")) != null ? _F : "Hello! I'm your AI assistant. Ask me anything about your documents.",
|
|
117
|
-
visualStyle: (_H = (_G = readString(env, "NEXT_PUBLIC_UI_VISUAL_STYLE")) != null ? _G : readString(env, "UI_VISUAL_STYLE")) != null ? _H : "glass",
|
|
118
|
-
borderRadius: (_J = (_I = readString(env, "NEXT_PUBLIC_UI_BORDER_RADIUS")) != null ? _I : readString(env, "UI_BORDER_RADIUS")) != null ? _J : "xl"
|
|
119
|
-
},
|
|
120
|
-
rag: {
|
|
121
|
-
topK: readNumber(env, "RAG_TOP_K", 5),
|
|
122
|
-
scoreThreshold: readNumber(env, "RAG_SCORE_THRESHOLD", 0),
|
|
123
|
-
chunkSize: readNumber(env, "RAG_CHUNK_SIZE", 1e3),
|
|
124
|
-
chunkOverlap: readNumber(env, "RAG_CHUNK_OVERLAP", 200)
|
|
125
|
-
}
|
|
126
|
-
};
|
|
127
|
-
}
|
|
128
|
-
|
|
129
|
-
// src/core/ConfigResolver.ts
|
|
130
|
-
var ConfigResolver = class {
|
|
131
|
-
/**
|
|
132
|
-
* Resolves the final configuration by merging host-provided config with defaults.
|
|
133
|
-
* @param hostConfig - Partial configuration passed from the host application.
|
|
134
|
-
*/
|
|
135
|
-
static resolve(hostConfig) {
|
|
136
|
-
const envConfig = getRagConfig();
|
|
137
|
-
if (!hostConfig) {
|
|
138
|
-
return envConfig;
|
|
139
|
-
}
|
|
140
|
-
return __spreadProps(__spreadValues(__spreadValues({}, envConfig), hostConfig), {
|
|
141
|
-
projectId: hostConfig.projectId || envConfig.projectId,
|
|
142
|
-
vectorDb: hostConfig.vectorDb ? __spreadProps(__spreadValues(__spreadValues({}, envConfig.vectorDb), hostConfig.vectorDb), {
|
|
143
|
-
options: __spreadValues(__spreadValues({}, envConfig.vectorDb.options), hostConfig.vectorDb.options || {})
|
|
144
|
-
}) : envConfig.vectorDb,
|
|
145
|
-
llm: hostConfig.llm ? __spreadProps(__spreadValues(__spreadValues({}, envConfig.llm), hostConfig.llm), {
|
|
146
|
-
options: __spreadValues(__spreadValues({}, envConfig.llm.options || {}), hostConfig.llm.options || {})
|
|
147
|
-
}) : envConfig.llm,
|
|
148
|
-
embedding: hostConfig.embedding ? __spreadProps(__spreadValues(__spreadValues({}, envConfig.embedding), hostConfig.embedding), {
|
|
149
|
-
options: __spreadValues(__spreadValues({}, envConfig.embedding.options || {}), hostConfig.embedding.options || {})
|
|
150
|
-
}) : envConfig.embedding,
|
|
151
|
-
ui: hostConfig.ui ? mergeDefined(envConfig.ui, hostConfig.ui) : envConfig.ui,
|
|
152
|
-
rag: hostConfig.rag ? __spreadValues(__spreadValues({}, envConfig.rag), hostConfig.rag) : envConfig.rag
|
|
153
|
-
});
|
|
154
|
-
}
|
|
155
|
-
/**
|
|
156
|
-
* Validates the configuration for required fields.
|
|
157
|
-
*/
|
|
158
|
-
static validate(config) {
|
|
159
|
-
if (!config.projectId) {
|
|
160
|
-
throw new Error("[ConfigResolver] projectId is required");
|
|
161
|
-
}
|
|
162
|
-
if (!config.vectorDb.provider) {
|
|
163
|
-
throw new Error("[ConfigResolver] vectorDb.provider is required");
|
|
164
|
-
}
|
|
165
|
-
if (!config.llm.provider) {
|
|
166
|
-
throw new Error("[ConfigResolver] llm.provider is required");
|
|
167
|
-
}
|
|
168
|
-
}
|
|
169
|
-
};
|
|
170
|
-
|
|
171
|
-
// src/rag/DocumentChunker.ts
|
|
172
|
-
var DocumentChunker = class {
|
|
173
|
-
constructor(chunkSize = 1e3, chunkOverlap = 200) {
|
|
174
|
-
this.chunkSize = chunkSize;
|
|
175
|
-
this.chunkOverlap = chunkOverlap;
|
|
176
|
-
}
|
|
177
|
-
/**
|
|
178
|
-
* Split a single text string into overlapping chunks.
|
|
179
|
-
*/
|
|
180
|
-
chunk(text, options = {}) {
|
|
181
|
-
const {
|
|
182
|
-
chunkSize = this.chunkSize,
|
|
183
|
-
chunkOverlap = this.chunkOverlap,
|
|
184
|
-
docId = `doc_${Date.now()}`,
|
|
185
|
-
metadata = {}
|
|
186
|
-
} = options;
|
|
187
|
-
const cleaned = text.replace(/\r\n/g, "\n").trim();
|
|
188
|
-
if (!cleaned) return [];
|
|
189
|
-
const sentences = cleaned.split(new RegExp("(?<=[.!?])\\s+|\\n{2,}")).map((s) => s.trim()).filter(Boolean);
|
|
190
|
-
const chunks = [];
|
|
191
|
-
let current = "";
|
|
192
|
-
let chunkIndex = 0;
|
|
193
|
-
for (const sentence of sentences) {
|
|
194
|
-
if ((current + " " + sentence).trim().length <= chunkSize) {
|
|
195
|
-
current = current ? `${current} ${sentence}` : sentence;
|
|
196
|
-
} else {
|
|
197
|
-
if (current) {
|
|
198
|
-
chunks.push({
|
|
199
|
-
id: `${docId}_chunk_${chunkIndex++}`,
|
|
200
|
-
content: current.trim(),
|
|
201
|
-
metadata: __spreadProps(__spreadValues({}, metadata), { docId, chunkIndex: chunkIndex - 1 })
|
|
202
|
-
});
|
|
203
|
-
}
|
|
204
|
-
if (chunkOverlap > 0 && current.length > chunkOverlap) {
|
|
205
|
-
current = current.slice(-chunkOverlap) + " " + sentence;
|
|
206
|
-
} else {
|
|
207
|
-
current = sentence;
|
|
208
|
-
}
|
|
209
|
-
}
|
|
210
|
-
}
|
|
211
|
-
if (current.trim()) {
|
|
212
|
-
chunks.push({
|
|
213
|
-
id: `${docId}_chunk_${chunkIndex}`,
|
|
214
|
-
content: current.trim(),
|
|
215
|
-
metadata: __spreadProps(__spreadValues({}, metadata), { docId, chunkIndex })
|
|
216
|
-
});
|
|
217
|
-
}
|
|
218
|
-
return chunks;
|
|
219
|
-
}
|
|
220
|
-
/**
|
|
221
|
-
* Chunk multiple documents at once.
|
|
222
|
-
*/
|
|
223
|
-
chunkMany(documents) {
|
|
224
|
-
return documents.flatMap(
|
|
225
|
-
(doc) => this.chunk(doc.content, { docId: doc.docId, metadata: doc.metadata })
|
|
226
|
-
);
|
|
227
|
-
}
|
|
228
|
-
};
|
|
229
|
-
|
|
230
|
-
// src/core/ProviderRegistry.ts
|
|
231
|
-
var ProviderRegistry = class {
|
|
232
|
-
/**
|
|
233
|
-
* Register a custom vector provider.
|
|
234
|
-
*/
|
|
235
|
-
static registerVectorProvider(name, providerClass) {
|
|
236
|
-
this.vectorProviders[name] = providerClass;
|
|
237
|
-
}
|
|
238
|
-
/**
|
|
239
|
-
* Creates a vector database provider based on the configuration.
|
|
240
|
-
*/
|
|
241
|
-
static async createVectorProvider(config) {
|
|
242
|
-
const { provider } = config;
|
|
243
|
-
if (this.vectorProviders[provider]) {
|
|
244
|
-
return new this.vectorProviders[provider](config);
|
|
245
|
-
}
|
|
246
|
-
switch (provider) {
|
|
247
|
-
case "pinecone":
|
|
248
|
-
const { PineconeProvider } = await import("./PineconeProvider-NJ675H7U.mjs");
|
|
249
|
-
return new PineconeProvider(config);
|
|
250
|
-
case "pgvector":
|
|
251
|
-
case "postgresql":
|
|
252
|
-
const { PostgreSQLProvider } = await import("./PostgreSQLProvider-ISNMD3BE.mjs");
|
|
253
|
-
return new PostgreSQLProvider(config);
|
|
254
|
-
case "mongodb":
|
|
255
|
-
const { MongoDBProvider } = await import("./MongoDBProvider-WWVJG3WT.mjs");
|
|
256
|
-
return new MongoDBProvider(config);
|
|
257
|
-
case "milvus":
|
|
258
|
-
const { MilvusProvider } = await import("./MilvusProvider-OO6QGZDZ.mjs");
|
|
259
|
-
return new MilvusProvider(config);
|
|
260
|
-
case "qdrant":
|
|
261
|
-
const { QdrantProvider } = await import("./QdrantProvider-LJWOIGES.mjs");
|
|
262
|
-
return new QdrantProvider(config);
|
|
263
|
-
case "chromadb":
|
|
264
|
-
const { ChromaDBProvider } = await import("./ChromaDBProvider-QNI7UCX4.mjs");
|
|
265
|
-
return new ChromaDBProvider(config);
|
|
266
|
-
case "redis":
|
|
267
|
-
const { RedisProvider } = await import("./RedisProvider-ASONNYBI.mjs");
|
|
268
|
-
return new RedisProvider(config);
|
|
269
|
-
case "weaviate":
|
|
270
|
-
const { WeaviateProvider } = await import("./WeaviateProvider-PSDCUGC7.mjs");
|
|
271
|
-
return new WeaviateProvider(config);
|
|
272
|
-
default:
|
|
273
|
-
throw new Error(`[ProviderRegistry] Unsupported vector provider: ${provider}`);
|
|
274
|
-
}
|
|
275
|
-
}
|
|
276
|
-
/**
|
|
277
|
-
* Creates an LLM provider based on the configuration.
|
|
278
|
-
*/
|
|
279
|
-
static createLLMProvider(llmConfig, embeddingConfig) {
|
|
280
|
-
return LLMFactory.create(llmConfig, embeddingConfig);
|
|
281
|
-
}
|
|
282
|
-
};
|
|
283
|
-
ProviderRegistry.vectorProviders = {};
|
|
284
|
-
|
|
285
|
-
// src/core/Pipeline.ts
|
|
286
|
-
var Pipeline = class {
|
|
287
|
-
constructor(config) {
|
|
288
|
-
this.initialised = false;
|
|
289
|
-
var _a, _b, _c, _d;
|
|
290
|
-
this.config = config;
|
|
291
|
-
this.chunker = new DocumentChunker(
|
|
292
|
-
(_b = (_a = config.rag) == null ? void 0 : _a.chunkSize) != null ? _b : 1e3,
|
|
293
|
-
(_d = (_c = config.rag) == null ? void 0 : _c.chunkOverlap) != null ? _d : 200
|
|
294
|
-
);
|
|
295
|
-
}
|
|
296
|
-
async initialize() {
|
|
297
|
-
if (this.initialised) return;
|
|
298
|
-
this.vectorDB = await ProviderRegistry.createVectorProvider(this.config.vectorDb);
|
|
299
|
-
this.llmProvider = ProviderRegistry.createLLMProvider(this.config.llm, this.config.embedding);
|
|
300
|
-
if (this.config.llm.provider === "anthropic") {
|
|
301
|
-
const { LLMFactory: LLMFactory2 } = await import("./LLMFactory-JFOY2V4X.mjs");
|
|
302
|
-
this.embeddingProvider = LLMFactory2.createEmbeddingProvider(this.config.embedding);
|
|
303
|
-
} else {
|
|
304
|
-
this.embeddingProvider = this.llmProvider;
|
|
305
|
-
}
|
|
306
|
-
await this.vectorDB.initialize();
|
|
307
|
-
this.initialised = true;
|
|
308
|
-
}
|
|
309
|
-
async ingest(documents, namespace) {
|
|
310
|
-
await this.initialize();
|
|
311
|
-
const ns = namespace != null ? namespace : this.config.projectId;
|
|
312
|
-
const results = [];
|
|
313
|
-
for (const doc of documents) {
|
|
314
|
-
const chunks = this.chunker.chunk(doc.content, {
|
|
315
|
-
docId: doc.docId,
|
|
316
|
-
metadata: doc.metadata
|
|
317
|
-
});
|
|
318
|
-
const vectors = await this.embeddingProvider.batchEmbed(chunks.map((c) => c.content));
|
|
319
|
-
const upsertDocs = chunks.map((chunk, i) => ({
|
|
320
|
-
id: chunk.id,
|
|
321
|
-
vector: vectors[i],
|
|
322
|
-
content: chunk.content,
|
|
323
|
-
metadata: chunk.metadata
|
|
324
|
-
}));
|
|
325
|
-
await this.vectorDB.batchUpsert(upsertDocs, ns);
|
|
326
|
-
results.push({ docId: doc.docId, chunksIngested: chunks.length });
|
|
327
|
-
}
|
|
328
|
-
return results;
|
|
329
|
-
}
|
|
330
|
-
async ask(question, history = [], namespace) {
|
|
331
|
-
var _a, _b, _c, _d;
|
|
332
|
-
await this.initialize();
|
|
333
|
-
const ns = namespace != null ? namespace : this.config.projectId;
|
|
334
|
-
const topK = (_b = (_a = this.config.rag) == null ? void 0 : _a.topK) != null ? _b : 5;
|
|
335
|
-
const scoreThreshold = (_d = (_c = this.config.rag) == null ? void 0 : _c.scoreThreshold) != null ? _d : 0;
|
|
336
|
-
const queryVector = await this.embeddingProvider.embed(question);
|
|
337
|
-
const rawMatches = await this.vectorDB.query(queryVector, topK, ns);
|
|
338
|
-
const sources = rawMatches.filter((m) => m.score >= scoreThreshold);
|
|
339
|
-
const context = sources.length ? sources.map((m, i) => `[Source ${i + 1}]
|
|
340
|
-
${m.content}`).join("\n\n---\n\n") : "No relevant context found.";
|
|
341
|
-
const messages = [...history, { role: "user", content: question }];
|
|
342
|
-
const reply = await this.llmProvider.chat(messages, context);
|
|
343
|
-
return { reply, sources };
|
|
344
|
-
}
|
|
345
|
-
async health() {
|
|
346
|
-
await this.initialize();
|
|
347
|
-
const [vectorDB, llm] = await Promise.all([
|
|
348
|
-
this.vectorDB.ping().catch(() => false),
|
|
349
|
-
this.llmProvider.ping().catch(() => false)
|
|
350
|
-
]);
|
|
351
|
-
return { vectorDB, llm };
|
|
352
|
-
}
|
|
353
|
-
};
|
|
354
|
-
|
|
355
|
-
// src/core/VectorPlugin.ts
|
|
356
|
-
var VectorPlugin = class {
|
|
357
|
-
/**
|
|
358
|
-
* Initializes the plugin with the host configuration.
|
|
359
|
-
* @param hostConfig - Configuration object passed from the host application.
|
|
360
|
-
*/
|
|
361
|
-
constructor(hostConfig) {
|
|
362
|
-
this.config = ConfigResolver.resolve(hostConfig);
|
|
363
|
-
ConfigResolver.validate(this.config);
|
|
364
|
-
this.pipeline = new Pipeline(this.config);
|
|
365
|
-
}
|
|
366
|
-
/**
|
|
367
|
-
* Get the current resolved configuration.
|
|
368
|
-
*/
|
|
369
|
-
getConfig() {
|
|
370
|
-
return this.config;
|
|
371
|
-
}
|
|
372
|
-
/**
|
|
373
|
-
* Run a chat query.
|
|
374
|
-
*/
|
|
375
|
-
async chat(message, history = [], namespace) {
|
|
376
|
-
return this.pipeline.ask(message, history, namespace);
|
|
377
|
-
}
|
|
378
|
-
/**
|
|
379
|
-
* Ingest documents into the vector database.
|
|
380
|
-
*/
|
|
381
|
-
async ingest(documents, namespace) {
|
|
382
|
-
return this.pipeline.ingest(documents, namespace);
|
|
383
|
-
}
|
|
384
|
-
/**
|
|
385
|
-
* Check the health of the connected providers.
|
|
386
|
-
*/
|
|
387
|
-
async health() {
|
|
388
|
-
return this.pipeline.health();
|
|
389
|
-
}
|
|
390
|
-
};
|
|
391
|
-
|
|
392
|
-
// src/utils/DocumentParser.ts
|
|
393
|
-
var DocumentParser = class {
|
|
394
|
-
/**
|
|
395
|
-
* Extract text from a File or Buffer based on its type.
|
|
396
|
-
*/
|
|
397
|
-
static async parse(file, fileName, mimeType) {
|
|
398
|
-
var _a;
|
|
399
|
-
const extension = (_a = fileName.split(".").pop()) == null ? void 0 : _a.toLowerCase();
|
|
400
|
-
if (extension === "txt" || extension === "md" || mimeType === "text/plain" || mimeType === "text/markdown") {
|
|
401
|
-
return this.readAsText(file);
|
|
402
|
-
}
|
|
403
|
-
if (extension === "json" || mimeType === "application/json") {
|
|
404
|
-
const text = await this.readAsText(file);
|
|
405
|
-
try {
|
|
406
|
-
const obj = JSON.parse(text);
|
|
407
|
-
return JSON.stringify(obj, null, 2);
|
|
408
|
-
} catch (e) {
|
|
409
|
-
return text;
|
|
410
|
-
}
|
|
411
|
-
}
|
|
412
|
-
if (extension === "csv" || mimeType === "text/csv") {
|
|
413
|
-
return this.readAsText(file);
|
|
414
|
-
}
|
|
415
|
-
if (extension === "pdf" || mimeType === "application/pdf") {
|
|
416
|
-
try {
|
|
417
|
-
const pdf = await import("pdf-parse/lib/pdf-parse.js");
|
|
418
|
-
const buffer = Buffer.isBuffer(file) ? file : Buffer.from(await file.arrayBuffer());
|
|
419
|
-
const data = await pdf.default(buffer);
|
|
420
|
-
return data.text;
|
|
421
|
-
} catch (err) {
|
|
422
|
-
console.warn('[DocumentParser] PDF parsing failed. Make sure "pdf-parse" is installed.', err);
|
|
423
|
-
return `[PDF Parsing Error for ${fileName}]`;
|
|
424
|
-
}
|
|
425
|
-
}
|
|
426
|
-
if (extension === "docx" || mimeType === "application/vnd.openxmlformats-officedocument.wordprocessingml.document") {
|
|
427
|
-
try {
|
|
428
|
-
const mammoth = await import("mammoth");
|
|
429
|
-
const buffer = Buffer.isBuffer(file) ? file : Buffer.from(await file.arrayBuffer());
|
|
430
|
-
const result = await mammoth.extractRawText({ buffer });
|
|
431
|
-
return result.value;
|
|
432
|
-
} catch (err) {
|
|
433
|
-
console.warn('[DocumentParser] DOCX parsing failed. Make sure "mammoth" is installed.', err);
|
|
434
|
-
return `[DOCX Parsing Error for ${fileName}]`;
|
|
435
|
-
}
|
|
436
|
-
}
|
|
437
|
-
try {
|
|
438
|
-
return await this.readAsText(file);
|
|
439
|
-
} catch (e) {
|
|
440
|
-
throw new Error(`[DocumentParser] Unsupported file format: ${fileName} (${mimeType})`);
|
|
441
|
-
}
|
|
442
|
-
}
|
|
443
|
-
static async readAsText(file) {
|
|
444
|
-
if (Buffer.isBuffer(file)) {
|
|
445
|
-
return file.toString("utf-8");
|
|
446
|
-
}
|
|
447
|
-
return await file.text();
|
|
448
|
-
}
|
|
449
|
-
};
|
|
450
|
-
|
|
451
|
-
// src/handlers/index.ts
|
|
452
|
-
function createChatHandler(config) {
|
|
453
|
-
const plugin = new VectorPlugin(config);
|
|
454
|
-
return async function POST(req) {
|
|
455
|
-
try {
|
|
456
|
-
const body = await req.json();
|
|
457
|
-
const { message, history = [], namespace } = body;
|
|
458
|
-
if (!(message == null ? void 0 : message.trim())) {
|
|
459
|
-
return NextResponse.json({ error: "message is required" }, { status: 400 });
|
|
460
|
-
}
|
|
461
|
-
const result = await plugin.chat(message, history, namespace);
|
|
462
|
-
return NextResponse.json(result);
|
|
463
|
-
} catch (err) {
|
|
464
|
-
const message = err instanceof Error ? err.message : "Internal server error";
|
|
465
|
-
return NextResponse.json({ error: message }, { status: 500 });
|
|
466
|
-
}
|
|
467
|
-
};
|
|
468
|
-
}
|
|
469
|
-
function createIngestHandler(config) {
|
|
470
|
-
const plugin = new VectorPlugin(config);
|
|
471
|
-
return async function POST(req) {
|
|
472
|
-
try {
|
|
473
|
-
const body = await req.json();
|
|
474
|
-
const { documents, namespace } = body;
|
|
475
|
-
if (!Array.isArray(documents) || documents.length === 0) {
|
|
476
|
-
return NextResponse.json({ error: "documents array is required" }, { status: 400 });
|
|
477
|
-
}
|
|
478
|
-
const results = await plugin.ingest(documents, namespace);
|
|
479
|
-
return NextResponse.json({ results });
|
|
480
|
-
} catch (err) {
|
|
481
|
-
const message = err instanceof Error ? err.message : "Internal server error";
|
|
482
|
-
return NextResponse.json({ error: message }, { status: 500 });
|
|
483
|
-
}
|
|
484
|
-
};
|
|
485
|
-
}
|
|
486
|
-
function createHealthHandler(config) {
|
|
487
|
-
const plugin = new VectorPlugin(config);
|
|
488
|
-
return async function GET() {
|
|
489
|
-
try {
|
|
490
|
-
const health = await plugin.health();
|
|
491
|
-
const status = health.vectorDB && health.llm ? "ok" : "degraded";
|
|
492
|
-
return NextResponse.json(__spreadProps(__spreadValues({
|
|
493
|
-
status
|
|
494
|
-
}, health), {
|
|
495
|
-
timestamp: (/* @__PURE__ */ new Date()).toISOString()
|
|
496
|
-
}));
|
|
497
|
-
} catch (err) {
|
|
498
|
-
const message = err instanceof Error ? err.message : "Unknown error";
|
|
499
|
-
return NextResponse.json({ status: "error", error: message }, { status: 500 });
|
|
500
|
-
}
|
|
501
|
-
};
|
|
502
|
-
}
|
|
503
|
-
function createUploadHandler(config) {
|
|
504
|
-
const plugin = new VectorPlugin(config);
|
|
505
|
-
return async function POST(req) {
|
|
506
|
-
try {
|
|
507
|
-
const formData = await req.formData();
|
|
508
|
-
const files = formData.getAll("files");
|
|
509
|
-
const namespace = formData.get("namespace") || void 0;
|
|
510
|
-
if (!files || files.length === 0) {
|
|
511
|
-
return NextResponse.json({ error: "No files provided" }, { status: 400 });
|
|
512
|
-
}
|
|
513
|
-
const documents = await Promise.all(
|
|
514
|
-
files.map(async (file) => {
|
|
515
|
-
const content = await DocumentParser.parse(file, file.name, file.type);
|
|
516
|
-
return {
|
|
517
|
-
docId: file.name,
|
|
518
|
-
content,
|
|
519
|
-
metadata: {
|
|
520
|
-
fileName: file.name,
|
|
521
|
-
fileSize: file.size,
|
|
522
|
-
fileType: file.type,
|
|
523
|
-
uploadedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
524
|
-
}
|
|
525
|
-
};
|
|
526
|
-
})
|
|
527
|
-
);
|
|
528
|
-
const results = await plugin.ingest(documents, namespace);
|
|
529
|
-
return NextResponse.json({ message: "Upload successful", results });
|
|
530
|
-
} catch (err) {
|
|
531
|
-
const message = err instanceof Error ? err.message : "Upload failed";
|
|
532
|
-
return NextResponse.json({ error: message }, { status: 500 });
|
|
533
|
-
}
|
|
534
|
-
};
|
|
535
|
-
}
|
|
536
|
-
|
|
537
|
-
export {
|
|
538
|
-
getRagConfig,
|
|
539
|
-
ConfigResolver,
|
|
540
|
-
DocumentChunker,
|
|
541
|
-
ProviderRegistry,
|
|
542
|
-
Pipeline,
|
|
543
|
-
VectorPlugin,
|
|
544
|
-
createChatHandler,
|
|
545
|
-
createIngestHandler,
|
|
546
|
-
createHealthHandler,
|
|
547
|
-
createUploadHandler
|
|
548
|
-
};
|