@retrivora-ai/rag-engine 0.4.3 → 0.4.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{DocumentChunker-BICIjSuG.d.mts → DocumentChunker-3yElxTO3.d.mts} +9 -2
- package/dist/{DocumentChunker-BICIjSuG.d.ts → DocumentChunker-3yElxTO3.d.ts} +9 -2
- package/dist/{MongoDBProvider-ZKW34AEL.mjs → MongoDBProvider-RE3Q5S5B.mjs} +1 -1
- package/dist/{RagConfig-CVt24lbC.d.ts → RagConfig-BgRDL9Vy.d.mts} +37 -1
- package/dist/{RagConfig-CVt24lbC.d.mts → RagConfig-BgRDL9Vy.d.ts} +37 -1
- package/dist/SimpleGraphProvider-M6T7SE7D.mjs +62 -0
- package/dist/{chunk-IWHCAQEA.mjs → chunk-PQKTC73Y.mjs} +1 -1
- package/dist/{chunk-UUZ3F4WK.mjs → chunk-PRC5CZIZ.mjs} +197 -42
- package/dist/handlers/index.d.mts +2 -2
- package/dist/handlers/index.d.ts +2 -2
- package/dist/handlers/index.js +272 -42
- package/dist/handlers/index.mjs +1 -1
- package/dist/{index-OI2--lvT.d.mts → index-7qeLTPBL.d.mts} +1 -1
- package/dist/{index-D1hoNXMT.d.ts → index-DowY4_K0.d.ts} +1 -1
- package/dist/index.d.mts +15 -5
- package/dist/index.d.ts +15 -5
- package/dist/index.js +161 -1
- package/dist/index.mjs +158 -1
- package/dist/server.d.mts +52 -12
- package/dist/server.d.ts +52 -12
- package/dist/server.js +272 -42
- package/dist/server.mjs +2 -2
- package/package.json +1 -1
- package/src/components/ConfigProvider.tsx +1 -0
- package/src/components/DocumentUpload.tsx +192 -0
- package/src/config/RagConfig.ts +27 -0
- package/src/config/constants.ts +7 -0
- package/src/config/serverConfig.ts +1 -0
- package/src/core/Pipeline.ts +89 -10
- package/src/core/ProviderRegistry.ts +40 -8
- package/src/index.ts +1 -0
- package/src/providers/graphdb/BaseGraphProvider.ts +43 -0
- package/src/providers/graphdb/SimpleGraphProvider.ts +66 -0
- package/src/providers/vectordb/MongoDBProvider.ts +3 -3
- package/src/rag/DocumentChunker.ts +77 -34
- package/src/rag/EntityExtractor.ts +43 -0
- package/src/types/index.ts +19 -0
- package/src/utils/DocumentParser.ts +1 -1
package/dist/server.js
CHANGED
|
@@ -457,7 +457,7 @@ var init_MongoDBProvider = __esm({
|
|
|
457
457
|
return results.map((res) => ({
|
|
458
458
|
id: res._id,
|
|
459
459
|
content: res[this.contentKey],
|
|
460
|
-
metadata: res[this.metadataKey],
|
|
460
|
+
metadata: res[this.metadataKey] || {},
|
|
461
461
|
score: res.score
|
|
462
462
|
}));
|
|
463
463
|
}
|
|
@@ -1205,6 +1205,81 @@ var init_UniversalVectorProvider = __esm({
|
|
|
1205
1205
|
}
|
|
1206
1206
|
});
|
|
1207
1207
|
|
|
1208
|
+
// src/providers/graphdb/BaseGraphProvider.ts
|
|
1209
|
+
var BaseGraphProvider;
|
|
1210
|
+
var init_BaseGraphProvider = __esm({
|
|
1211
|
+
"src/providers/graphdb/BaseGraphProvider.ts"() {
|
|
1212
|
+
"use strict";
|
|
1213
|
+
BaseGraphProvider = class {
|
|
1214
|
+
constructor(config) {
|
|
1215
|
+
this.config = config;
|
|
1216
|
+
}
|
|
1217
|
+
};
|
|
1218
|
+
}
|
|
1219
|
+
});
|
|
1220
|
+
|
|
1221
|
+
// src/providers/graphdb/SimpleGraphProvider.ts
|
|
1222
|
+
var SimpleGraphProvider_exports = {};
|
|
1223
|
+
__export(SimpleGraphProvider_exports, {
|
|
1224
|
+
SimpleGraphProvider: () => SimpleGraphProvider
|
|
1225
|
+
});
|
|
1226
|
+
var SimpleGraphProvider;
|
|
1227
|
+
var init_SimpleGraphProvider = __esm({
|
|
1228
|
+
"src/providers/graphdb/SimpleGraphProvider.ts"() {
|
|
1229
|
+
"use strict";
|
|
1230
|
+
init_BaseGraphProvider();
|
|
1231
|
+
SimpleGraphProvider = class extends BaseGraphProvider {
|
|
1232
|
+
constructor() {
|
|
1233
|
+
super(...arguments);
|
|
1234
|
+
this.nodes = /* @__PURE__ */ new Map();
|
|
1235
|
+
this.edges = [];
|
|
1236
|
+
}
|
|
1237
|
+
async initialize() {
|
|
1238
|
+
console.log("[SimpleGraphProvider] Initialised in-memory graph store.");
|
|
1239
|
+
}
|
|
1240
|
+
async addNodes(nodes) {
|
|
1241
|
+
for (const node of nodes) {
|
|
1242
|
+
this.nodes.set(node.id, node);
|
|
1243
|
+
}
|
|
1244
|
+
}
|
|
1245
|
+
async addEdges(edges) {
|
|
1246
|
+
this.edges.push(...edges);
|
|
1247
|
+
}
|
|
1248
|
+
async query(queryText, limit = 5) {
|
|
1249
|
+
const q = queryText.toLowerCase();
|
|
1250
|
+
const matchedNodes = Array.from(this.nodes.values()).filter(
|
|
1251
|
+
(node) => node.id.toLowerCase().includes(q) || node.label.toLowerCase().includes(q) || JSON.stringify(node.properties).toLowerCase().includes(q)
|
|
1252
|
+
).slice(0, limit);
|
|
1253
|
+
const matchedNodeIds = new Set(matchedNodes.map((n) => n.id));
|
|
1254
|
+
const matchedEdges = this.edges.filter(
|
|
1255
|
+
(edge) => matchedNodeIds.has(edge.source) || matchedNodeIds.has(edge.target)
|
|
1256
|
+
);
|
|
1257
|
+
for (const edge of matchedEdges) {
|
|
1258
|
+
if (!matchedNodeIds.has(edge.source)) {
|
|
1259
|
+
const source = this.nodes.get(edge.source);
|
|
1260
|
+
if (source) matchedNodes.push(source);
|
|
1261
|
+
}
|
|
1262
|
+
if (!matchedNodeIds.has(edge.target)) {
|
|
1263
|
+
const target = this.nodes.get(edge.target);
|
|
1264
|
+
if (target) matchedNodes.push(target);
|
|
1265
|
+
}
|
|
1266
|
+
}
|
|
1267
|
+
return {
|
|
1268
|
+
nodes: Array.from(new Set(matchedNodes)),
|
|
1269
|
+
edges: matchedEdges
|
|
1270
|
+
};
|
|
1271
|
+
}
|
|
1272
|
+
async ping() {
|
|
1273
|
+
return true;
|
|
1274
|
+
}
|
|
1275
|
+
async disconnect() {
|
|
1276
|
+
this.nodes.clear();
|
|
1277
|
+
this.edges = [];
|
|
1278
|
+
}
|
|
1279
|
+
};
|
|
1280
|
+
}
|
|
1281
|
+
});
|
|
1282
|
+
|
|
1208
1283
|
// src/server.ts
|
|
1209
1284
|
var server_exports = {};
|
|
1210
1285
|
__export(server_exports, {
|
|
@@ -1317,7 +1392,7 @@ function readEnum(env, name, fallback, allowed) {
|
|
|
1317
1392
|
throw new Error(`[getRagConfig] ${name} must be one of: ${allowed.join(", ")}`);
|
|
1318
1393
|
}
|
|
1319
1394
|
function getRagConfig(env = process.env) {
|
|
1320
|
-
var _a, _b, _c, _d, _e, _f, _g, _h, _i, _j, _k, _l, _m, _n, _o, _p, _q, _r, _s, _t, _u, _v, _w, _x, _y, _z, _A, _B, _C, _D, _E, _F, _G, _H, _I, _J, _K, _L, _M, _N, _O;
|
|
1395
|
+
var _a, _b, _c, _d, _e, _f, _g, _h, _i, _j, _k, _l, _m, _n, _o, _p, _q, _r, _s, _t, _u, _v, _w, _x, _y, _z, _A, _B, _C, _D, _E, _F, _G, _H, _I, _J, _K, _L, _M, _N, _O, _P, _Q;
|
|
1321
1396
|
const projectId = (_b = (_a = readString(env, "RAG_PROJECT_ID")) != null ? _a : readString(env, "NEXT_PUBLIC_PROJECT_ID")) != null ? _b : "__default__";
|
|
1322
1397
|
const vectorProvider = readEnum(env, "VECTOR_DB_PROVIDER", "pinecone", VECTOR_DB_PROVIDERS);
|
|
1323
1398
|
const llmProvider = readEnum(env, "LLM_PROVIDER", "openai", LLM_PROVIDERS);
|
|
@@ -1414,7 +1489,8 @@ function getRagConfig(env = process.env) {
|
|
|
1414
1489
|
showSources: ((_I = (_H = readString(env, "NEXT_PUBLIC_SHOW_SOURCES")) != null ? _H : readString(env, "UI_SHOW_SOURCES")) != null ? _I : "true") !== "false",
|
|
1415
1490
|
welcomeMessage: (_K = (_J = readString(env, "NEXT_PUBLIC_WELCOME_MESSAGE")) != null ? _J : readString(env, "UI_WELCOME_MESSAGE")) != null ? _K : "Hello! I'm your AI assistant. Ask me anything about your documents.",
|
|
1416
1491
|
visualStyle: (_M = (_L = readString(env, "NEXT_PUBLIC_UI_VISUAL_STYLE")) != null ? _L : readString(env, "UI_VISUAL_STYLE")) != null ? _M : "glass",
|
|
1417
|
-
borderRadius: (_O = (_N = readString(env, "NEXT_PUBLIC_UI_BORDER_RADIUS")) != null ? _N : readString(env, "UI_BORDER_RADIUS")) != null ? _O : "xl"
|
|
1492
|
+
borderRadius: (_O = (_N = readString(env, "NEXT_PUBLIC_UI_BORDER_RADIUS")) != null ? _N : readString(env, "UI_BORDER_RADIUS")) != null ? _O : "xl",
|
|
1493
|
+
allowUpload: ((_Q = (_P = readString(env, "NEXT_PUBLIC_ALLOW_UPLOAD")) != null ? _P : readString(env, "UI_ALLOW_UPLOAD")) != null ? _Q : "false") === "true"
|
|
1418
1494
|
},
|
|
1419
1495
|
rag: {
|
|
1420
1496
|
topK: readNumber(env, "RAG_TOP_K", 5),
|
|
@@ -1988,52 +2064,84 @@ var ConfigValidator = class {
|
|
|
1988
2064
|
|
|
1989
2065
|
// src/rag/DocumentChunker.ts
|
|
1990
2066
|
var DocumentChunker = class {
|
|
1991
|
-
constructor(chunkSize = 1e3, chunkOverlap = 200) {
|
|
2067
|
+
constructor(chunkSize = 1e3, chunkOverlap = 200, separators = ["\n\n", "\n", " ", ""]) {
|
|
1992
2068
|
this.chunkSize = chunkSize;
|
|
1993
2069
|
this.chunkOverlap = chunkOverlap;
|
|
2070
|
+
this.separators = separators;
|
|
1994
2071
|
}
|
|
1995
2072
|
/**
|
|
1996
|
-
* Split a single text string into overlapping chunks.
|
|
2073
|
+
* Split a single text string into overlapping chunks using a recursive strategy.
|
|
1997
2074
|
*/
|
|
1998
2075
|
chunk(text, options = {}) {
|
|
1999
2076
|
const {
|
|
2000
2077
|
chunkSize = this.chunkSize,
|
|
2001
2078
|
chunkOverlap = this.chunkOverlap,
|
|
2002
2079
|
docId = `doc_${Date.now()}`,
|
|
2003
|
-
metadata = {}
|
|
2080
|
+
metadata = {},
|
|
2081
|
+
separators = this.separators
|
|
2004
2082
|
} = options;
|
|
2005
|
-
const
|
|
2006
|
-
|
|
2007
|
-
|
|
2008
|
-
|
|
2009
|
-
let current = "";
|
|
2083
|
+
const finalChunks = [];
|
|
2084
|
+
const splits = this.recursiveSplit(text, separators, chunkSize);
|
|
2085
|
+
let currentChunk = [];
|
|
2086
|
+
let currentLength = 0;
|
|
2010
2087
|
let chunkIndex = 0;
|
|
2011
|
-
for (const
|
|
2012
|
-
if (
|
|
2013
|
-
|
|
2014
|
-
|
|
2015
|
-
|
|
2016
|
-
|
|
2017
|
-
|
|
2018
|
-
|
|
2019
|
-
|
|
2020
|
-
|
|
2021
|
-
|
|
2022
|
-
|
|
2023
|
-
|
|
2024
|
-
|
|
2025
|
-
|
|
2088
|
+
for (const split of splits) {
|
|
2089
|
+
if (currentLength + split.length > chunkSize && currentChunk.length > 0) {
|
|
2090
|
+
finalChunks.push({
|
|
2091
|
+
id: `${docId}_chunk_${chunkIndex++}`,
|
|
2092
|
+
content: currentChunk.join("").trim(),
|
|
2093
|
+
metadata: __spreadProps(__spreadValues({}, metadata), { docId, chunkIndex: chunkIndex - 1 })
|
|
2094
|
+
});
|
|
2095
|
+
const overlapItems = [];
|
|
2096
|
+
let overlapLen = 0;
|
|
2097
|
+
for (let i = currentChunk.length - 1; i >= 0; i--) {
|
|
2098
|
+
if (overlapLen + currentChunk[i].length <= chunkOverlap) {
|
|
2099
|
+
overlapItems.unshift(currentChunk[i]);
|
|
2100
|
+
overlapLen += currentChunk[i].length;
|
|
2101
|
+
} else {
|
|
2102
|
+
break;
|
|
2103
|
+
}
|
|
2026
2104
|
}
|
|
2105
|
+
currentChunk = overlapItems;
|
|
2106
|
+
currentLength = overlapLen;
|
|
2027
2107
|
}
|
|
2108
|
+
currentChunk.push(split);
|
|
2109
|
+
currentLength += split.length;
|
|
2028
2110
|
}
|
|
2029
|
-
if (
|
|
2030
|
-
|
|
2111
|
+
if (currentChunk.length > 0) {
|
|
2112
|
+
finalChunks.push({
|
|
2031
2113
|
id: `${docId}_chunk_${chunkIndex}`,
|
|
2032
|
-
content:
|
|
2114
|
+
content: currentChunk.join("").trim(),
|
|
2033
2115
|
metadata: __spreadProps(__spreadValues({}, metadata), { docId, chunkIndex })
|
|
2034
2116
|
});
|
|
2035
2117
|
}
|
|
2036
|
-
return
|
|
2118
|
+
return finalChunks;
|
|
2119
|
+
}
|
|
2120
|
+
/**
|
|
2121
|
+
* Recursively split text based on separators.
|
|
2122
|
+
*/
|
|
2123
|
+
recursiveSplit(text, separators, chunkSize) {
|
|
2124
|
+
const finalSplits = [];
|
|
2125
|
+
let separator = separators[separators.length - 1];
|
|
2126
|
+
let nextSeparators = [];
|
|
2127
|
+
for (let i = 0; i < separators.length; i++) {
|
|
2128
|
+
if (text.includes(separators[i])) {
|
|
2129
|
+
separator = separators[i];
|
|
2130
|
+
nextSeparators = separators.slice(i + 1);
|
|
2131
|
+
break;
|
|
2132
|
+
}
|
|
2133
|
+
}
|
|
2134
|
+
const parts = text.split(separator);
|
|
2135
|
+
for (const part of parts) {
|
|
2136
|
+
if (part.length <= chunkSize) {
|
|
2137
|
+
finalSplits.push(part + separator);
|
|
2138
|
+
} else if (nextSeparators.length > 0) {
|
|
2139
|
+
finalSplits.push(...this.recursiveSplit(part, nextSeparators, chunkSize));
|
|
2140
|
+
} else {
|
|
2141
|
+
finalSplits.push(part);
|
|
2142
|
+
}
|
|
2143
|
+
}
|
|
2144
|
+
return finalSplits;
|
|
2037
2145
|
}
|
|
2038
2146
|
/**
|
|
2039
2147
|
* Chunk multiple documents at once.
|
|
@@ -2045,6 +2153,39 @@ var DocumentChunker = class {
|
|
|
2045
2153
|
}
|
|
2046
2154
|
};
|
|
2047
2155
|
|
|
2156
|
+
// src/rag/EntityExtractor.ts
|
|
2157
|
+
var EntityExtractor = class {
|
|
2158
|
+
constructor(llm) {
|
|
2159
|
+
this.llm = llm;
|
|
2160
|
+
}
|
|
2161
|
+
/**
|
|
2162
|
+
* Extract nodes and edges from a text chunk.
|
|
2163
|
+
*/
|
|
2164
|
+
async extract(text) {
|
|
2165
|
+
const prompt = `
|
|
2166
|
+
Extract entities and relationships from the following text.
|
|
2167
|
+
Format the output as JSON with two keys: "nodes" (array of {id, label, properties}) and "edges" (array of {source, target, type, properties}).
|
|
2168
|
+
Use the same ID for the same entity.
|
|
2169
|
+
|
|
2170
|
+
Text:
|
|
2171
|
+
"${text}"
|
|
2172
|
+
|
|
2173
|
+
Output JSON:
|
|
2174
|
+
`;
|
|
2175
|
+
const response = await this.llm.chat([
|
|
2176
|
+
{ role: "system", content: "You are an expert at knowledge graph extraction. Respond only with valid JSON." },
|
|
2177
|
+
{ role: "user", content: prompt }
|
|
2178
|
+
], "");
|
|
2179
|
+
try {
|
|
2180
|
+
const cleanJson = response.replace(/```json\n?|\n?```/g, "").trim();
|
|
2181
|
+
return JSON.parse(cleanJson);
|
|
2182
|
+
} catch (e) {
|
|
2183
|
+
console.warn("[EntityExtractor] Failed to parse LLM response as JSON:", response);
|
|
2184
|
+
return { nodes: [], edges: [] };
|
|
2185
|
+
}
|
|
2186
|
+
}
|
|
2187
|
+
};
|
|
2188
|
+
|
|
2048
2189
|
// src/llm/providers/OpenAIProvider.ts
|
|
2049
2190
|
var import_openai = __toESM(require("openai"));
|
|
2050
2191
|
var OpenAIProvider = class {
|
|
@@ -2589,16 +2730,15 @@ var LLMFactory = class _LLMFactory {
|
|
|
2589
2730
|
|
|
2590
2731
|
// src/core/ProviderRegistry.ts
|
|
2591
2732
|
var ProviderRegistry = class {
|
|
2592
|
-
/**
|
|
2593
|
-
* Register a custom vector provider class by name.
|
|
2594
|
-
* The name must match the provider value used in VectorDBConfig.provider.
|
|
2595
|
-
*
|
|
2596
|
-
* @example
|
|
2597
|
-
* ProviderRegistry.registerVectorProvider('my-db', MyCustomProvider);
|
|
2598
|
-
*/
|
|
2599
2733
|
static registerVectorProvider(name, providerClass) {
|
|
2600
2734
|
this.vectorProviders[name] = providerClass;
|
|
2601
2735
|
}
|
|
2736
|
+
/**
|
|
2737
|
+
* Register a custom graph provider class by name.
|
|
2738
|
+
*/
|
|
2739
|
+
static registerGraphProvider(name, providerClass) {
|
|
2740
|
+
this.graphProviders[name] = providerClass;
|
|
2741
|
+
}
|
|
2602
2742
|
/**
|
|
2603
2743
|
* Creates a vector database provider based on the configuration.
|
|
2604
2744
|
* Built-in providers are dynamically imported to avoid bundling all SDKs.
|
|
@@ -2653,6 +2793,28 @@ var ProviderRegistry = class {
|
|
|
2653
2793
|
);
|
|
2654
2794
|
}
|
|
2655
2795
|
}
|
|
2796
|
+
/**
|
|
2797
|
+
* Creates a graph database provider based on the configuration.
|
|
2798
|
+
*/
|
|
2799
|
+
static async createGraphProvider(config) {
|
|
2800
|
+
const { provider } = config;
|
|
2801
|
+
if (this.graphProviders[provider]) {
|
|
2802
|
+
return new this.graphProviders[provider](config);
|
|
2803
|
+
}
|
|
2804
|
+
switch (provider) {
|
|
2805
|
+
case "neo4j": {
|
|
2806
|
+
throw new Error("[ProviderRegistry] Neo4j provider not implemented yet.");
|
|
2807
|
+
}
|
|
2808
|
+
case "simple": {
|
|
2809
|
+
const { SimpleGraphProvider: SimpleGraphProvider2 } = await Promise.resolve().then(() => (init_SimpleGraphProvider(), SimpleGraphProvider_exports));
|
|
2810
|
+
return new SimpleGraphProvider2(config);
|
|
2811
|
+
}
|
|
2812
|
+
default:
|
|
2813
|
+
throw new Error(
|
|
2814
|
+
`[ProviderRegistry] Unsupported graph provider: "${provider}". Built-in providers: simple. For custom providers, call ProviderRegistry.registerGraphProvider("${provider}", YourClass).`
|
|
2815
|
+
);
|
|
2816
|
+
}
|
|
2817
|
+
}
|
|
2656
2818
|
/**
|
|
2657
2819
|
* Creates an LLM provider based on the configuration.
|
|
2658
2820
|
*/
|
|
@@ -2661,6 +2823,7 @@ var ProviderRegistry = class {
|
|
|
2661
2823
|
}
|
|
2662
2824
|
};
|
|
2663
2825
|
ProviderRegistry.vectorProviders = {};
|
|
2826
|
+
ProviderRegistry.graphProviders = {};
|
|
2664
2827
|
|
|
2665
2828
|
// src/core/BatchProcessor.ts
|
|
2666
2829
|
function isTransientError(error) {
|
|
@@ -3092,6 +3255,11 @@ var Pipeline = class {
|
|
|
3092
3255
|
);
|
|
3093
3256
|
this.llmProvider = llmProvider;
|
|
3094
3257
|
this.embeddingProvider = embeddingProvider;
|
|
3258
|
+
if (this.config.graphDb) {
|
|
3259
|
+
this.graphDB = await ProviderRegistry.createGraphProvider(this.config.graphDb);
|
|
3260
|
+
await this.graphDB.initialize();
|
|
3261
|
+
this.entityExtractor = new EntityExtractor(this.llmProvider);
|
|
3262
|
+
}
|
|
3095
3263
|
await this.vectorDB.initialize();
|
|
3096
3264
|
this.initialised = true;
|
|
3097
3265
|
}
|
|
@@ -3145,6 +3313,30 @@ var Pipeline = class {
|
|
|
3145
3313
|
docId: doc.docId,
|
|
3146
3314
|
chunksIngested: upsertResult.totalProcessed
|
|
3147
3315
|
});
|
|
3316
|
+
if (this.graphDB && this.entityExtractor) {
|
|
3317
|
+
console.log(`[Pipeline] Extracting entities for doc ${doc.docId} (${chunks.length} chunks)...`);
|
|
3318
|
+
const extractionOptions = {
|
|
3319
|
+
batchSize: 2,
|
|
3320
|
+
// Low concurrency for LLM extraction
|
|
3321
|
+
maxRetries: 1,
|
|
3322
|
+
initialDelayMs: 500
|
|
3323
|
+
};
|
|
3324
|
+
await BatchProcessor.processBatch(
|
|
3325
|
+
chunks,
|
|
3326
|
+
async (batch) => {
|
|
3327
|
+
for (const chunk of batch) {
|
|
3328
|
+
try {
|
|
3329
|
+
const { nodes, edges } = await this.entityExtractor.extract(chunk.content);
|
|
3330
|
+
if (nodes.length > 0) await this.graphDB.addNodes(nodes);
|
|
3331
|
+
if (edges.length > 0) await this.graphDB.addEdges(edges);
|
|
3332
|
+
} catch (err) {
|
|
3333
|
+
console.warn(`[Pipeline] Entity extraction failed for chunk:`, err);
|
|
3334
|
+
}
|
|
3335
|
+
}
|
|
3336
|
+
},
|
|
3337
|
+
extractionOptions
|
|
3338
|
+
);
|
|
3339
|
+
}
|
|
3148
3340
|
} catch (error) {
|
|
3149
3341
|
console.error(`[Pipeline] Failed to ingest document ${doc.docId}:`, error);
|
|
3150
3342
|
results.push({ docId: doc.docId, chunksIngested: 0 });
|
|
@@ -3153,27 +3345,65 @@ var Pipeline = class {
|
|
|
3153
3345
|
return results;
|
|
3154
3346
|
}
|
|
3155
3347
|
async ask(question, history = [], namespace) {
|
|
3156
|
-
var _a, _b, _c, _d;
|
|
3348
|
+
var _a, _b, _c, _d, _e, _f;
|
|
3157
3349
|
await this.initialize();
|
|
3158
3350
|
const ns = namespace != null ? namespace : this.config.projectId;
|
|
3159
3351
|
const topK = (_b = (_a = this.config.rag) == null ? void 0 : _a.topK) != null ? _b : 5;
|
|
3160
3352
|
const scoreThreshold = (_d = (_c = this.config.rag) == null ? void 0 : _c.scoreThreshold) != null ? _d : 0;
|
|
3161
3353
|
try {
|
|
3162
|
-
|
|
3354
|
+
let searchQuery = question;
|
|
3355
|
+
if ((_e = this.config.rag) == null ? void 0 : _e.useQueryTransformation) {
|
|
3356
|
+
searchQuery = await this.rewriteQuery(question, history);
|
|
3357
|
+
}
|
|
3358
|
+
const queryVector = await this.embeddingProvider.embed(searchQuery, { taskType: "query" });
|
|
3163
3359
|
const fieldHints = extractQueryFieldHints(question);
|
|
3164
3360
|
const filter = buildQueryFilter(question, fieldHints);
|
|
3165
3361
|
filter.__entityHints = fieldHints;
|
|
3166
3362
|
const rawMatches = await this.vectorDB.query(queryVector, topK, ns, filter);
|
|
3167
3363
|
const sources = rawMatches.filter((m) => m.score >= scoreThreshold);
|
|
3168
|
-
|
|
3364
|
+
let graphData;
|
|
3365
|
+
if (this.graphDB && ((_f = this.config.rag) == null ? void 0 : _f.useGraphRetrieval)) {
|
|
3366
|
+
graphData = await this.graphDB.query(searchQuery);
|
|
3367
|
+
}
|
|
3368
|
+
let context = sources.length ? sources.map((m, i) => `[Source ${i + 1}]
|
|
3169
3369
|
${m.content}`).join("\n\n---\n\n") : "No relevant context found.";
|
|
3370
|
+
if (graphData && graphData.nodes.length > 0) {
|
|
3371
|
+
const graphContext = graphData.nodes.map(
|
|
3372
|
+
(n) => `Entity: ${n.label} (${n.id})${n.properties ? " - " + JSON.stringify(n.properties) : ""}`
|
|
3373
|
+
).join("\n");
|
|
3374
|
+
context = `GRAPH KNOWLEDGE:
|
|
3375
|
+
${graphContext}
|
|
3376
|
+
|
|
3377
|
+
VECTOR CONTEXT:
|
|
3378
|
+
${context}`;
|
|
3379
|
+
}
|
|
3170
3380
|
const messages = [...history, { role: "user", content: question }];
|
|
3171
3381
|
const reply = await this.llmProvider.chat(messages, context);
|
|
3172
|
-
return { reply, sources };
|
|
3382
|
+
return { reply, sources, graphData };
|
|
3173
3383
|
} catch (error) {
|
|
3174
3384
|
throw new Error(`[Pipeline] Chat failed: ${error instanceof Error ? error.message : String(error)}`);
|
|
3175
3385
|
}
|
|
3176
3386
|
}
|
|
3387
|
+
/**
|
|
3388
|
+
* Rewrite the user query for better retrieval performance.
|
|
3389
|
+
*/
|
|
3390
|
+
async rewriteQuery(question, history) {
|
|
3391
|
+
const prompt = `
|
|
3392
|
+
Given the following conversation history and a new question, rewrite the question to be a better search query for a vector database.
|
|
3393
|
+
Focus on extracting the core intent and entities. Do not answer the question, just rewrite it.
|
|
3394
|
+
|
|
3395
|
+
History:
|
|
3396
|
+
${history.map((m) => `${m.role}: ${m.content}`).join("\n")}
|
|
3397
|
+
|
|
3398
|
+
New Question: ${question}
|
|
3399
|
+
|
|
3400
|
+
Optimized Search Query:`;
|
|
3401
|
+
const rewrite = await this.llmProvider.chat([
|
|
3402
|
+
{ role: "system", content: "You are an assistant that optimizes search queries for RAG systems." },
|
|
3403
|
+
{ role: "user", content: prompt }
|
|
3404
|
+
], "");
|
|
3405
|
+
return rewrite.trim() || question;
|
|
3406
|
+
}
|
|
3177
3407
|
};
|
|
3178
3408
|
|
|
3179
3409
|
// src/core/ProviderHealthCheck.ts
|
|
@@ -3974,7 +4204,7 @@ var DocumentParser = class {
|
|
|
3974
4204
|
}
|
|
3975
4205
|
if (extension === "pdf" || mimeType === "application/pdf") {
|
|
3976
4206
|
try {
|
|
3977
|
-
const pdf = await import("pdf-parse
|
|
4207
|
+
const pdf = await import("pdf-parse");
|
|
3978
4208
|
const buffer = Buffer.isBuffer(file) ? file : Buffer.from(await file.arrayBuffer());
|
|
3979
4209
|
const data = await pdf.default(buffer);
|
|
3980
4210
|
return data.text;
|
package/dist/server.mjs
CHANGED
|
@@ -34,7 +34,7 @@ import {
|
|
|
34
34
|
createIngestHandler,
|
|
35
35
|
createUploadHandler,
|
|
36
36
|
getRagConfig
|
|
37
|
-
} from "./chunk-
|
|
37
|
+
} from "./chunk-PRC5CZIZ.mjs";
|
|
38
38
|
import "./chunk-EDLTMSNY.mjs";
|
|
39
39
|
import {
|
|
40
40
|
PineconeProvider
|
|
@@ -44,7 +44,7 @@ import {
|
|
|
44
44
|
} from "./chunk-LJWWPTWE.mjs";
|
|
45
45
|
import {
|
|
46
46
|
MongoDBProvider
|
|
47
|
-
} from "./chunk-
|
|
47
|
+
} from "./chunk-PQKTC73Y.mjs";
|
|
48
48
|
import {
|
|
49
49
|
MilvusProvider
|
|
50
50
|
} from "./chunk-3QWAK3RZ.mjs";
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@retrivora-ai/rag-engine",
|
|
3
|
-
"version": "0.4.
|
|
3
|
+
"version": "0.4.5",
|
|
4
4
|
"description": "Retrivora AI is a plug-and-play AI engine for RAG chat experiences — generic vector DB + LLM provider, embeddable or standalone.",
|
|
5
5
|
"author": "Abhinav Alkuchi",
|
|
6
6
|
"license": "MIT",
|
|
@@ -0,0 +1,192 @@
|
|
|
1
|
+
'use client';
|
|
2
|
+
|
|
3
|
+
import React, { useState, useRef } from 'react';
|
|
4
|
+
import { Upload, File, X, CheckCircle, AlertCircle, Loader2 } from 'lucide-react';
|
|
5
|
+
import { useConfig } from './ConfigProvider';
|
|
6
|
+
|
|
7
|
+
interface DocumentUploadProps {
|
|
8
|
+
/** Optional namespace for the upload */
|
|
9
|
+
namespace?: string;
|
|
10
|
+
/** Callback when upload completes */
|
|
11
|
+
onUploadComplete?: (results: unknown) => void;
|
|
12
|
+
/** Additional className */
|
|
13
|
+
className?: string;
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
interface FileState {
|
|
17
|
+
file: File;
|
|
18
|
+
status: 'idle' | 'uploading' | 'success' | 'error';
|
|
19
|
+
error?: string;
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
export function DocumentUpload({ namespace, onUploadComplete, className = '' }: DocumentUploadProps) {
|
|
23
|
+
const { ui } = useConfig();
|
|
24
|
+
const [fileStates, setFileStates] = useState<FileState[]>([]);
|
|
25
|
+
const [isDragging, setIsDragging] = useState(false);
|
|
26
|
+
const fileInputRef = useRef<HTMLInputElement>(null);
|
|
27
|
+
|
|
28
|
+
const addFiles = (files: File[]) => {
|
|
29
|
+
const newStates = files.map(file => ({ file, status: 'idle' as const }));
|
|
30
|
+
setFileStates(prev => [...prev, ...newStates]);
|
|
31
|
+
};
|
|
32
|
+
|
|
33
|
+
const removeFile = (index: number) => {
|
|
34
|
+
setFileStates(prev => prev.filter((_, i) => i !== index));
|
|
35
|
+
};
|
|
36
|
+
|
|
37
|
+
const onDragOver = (e: React.DragEvent) => {
|
|
38
|
+
e.preventDefault();
|
|
39
|
+
setIsDragging(true);
|
|
40
|
+
};
|
|
41
|
+
|
|
42
|
+
const onDragLeave = () => {
|
|
43
|
+
setIsDragging(false);
|
|
44
|
+
};
|
|
45
|
+
|
|
46
|
+
const onDrop = (e: React.DragEvent) => {
|
|
47
|
+
e.preventDefault();
|
|
48
|
+
setIsDragging(false);
|
|
49
|
+
if (e.dataTransfer.files) {
|
|
50
|
+
addFiles(Array.from(e.dataTransfer.files));
|
|
51
|
+
}
|
|
52
|
+
};
|
|
53
|
+
|
|
54
|
+
const onFileSelect = (e: React.ChangeEvent<HTMLInputElement>) => {
|
|
55
|
+
if (e.target.files) {
|
|
56
|
+
addFiles(Array.from(e.target.files));
|
|
57
|
+
}
|
|
58
|
+
};
|
|
59
|
+
|
|
60
|
+
const uploadFiles = async () => {
|
|
61
|
+
const idleFiles = fileStates.filter(s => s.status === 'idle');
|
|
62
|
+
if (idleFiles.length === 0) return;
|
|
63
|
+
|
|
64
|
+
// Update status to uploading
|
|
65
|
+
setFileStates(prev => prev.map(s => s.status === 'idle' ? { ...s, status: 'uploading' } : s));
|
|
66
|
+
|
|
67
|
+
const formData = new FormData();
|
|
68
|
+
idleFiles.forEach(s => formData.append('files', s.file));
|
|
69
|
+
if (namespace) formData.append('namespace', namespace);
|
|
70
|
+
|
|
71
|
+
try {
|
|
72
|
+
const response = await fetch('/api/upload', {
|
|
73
|
+
method: 'POST',
|
|
74
|
+
body: formData,
|
|
75
|
+
});
|
|
76
|
+
|
|
77
|
+
const result = await response.json();
|
|
78
|
+
|
|
79
|
+
if (!response.ok) throw new Error(result.error || 'Upload failed');
|
|
80
|
+
|
|
81
|
+
setFileStates(prev => prev.map(s => s.status === 'uploading' ? { ...s, status: 'success' } : s));
|
|
82
|
+
if (onUploadComplete) onUploadComplete(result);
|
|
83
|
+
} catch (err) {
|
|
84
|
+
const message = err instanceof Error ? err.message : 'Upload failed';
|
|
85
|
+
setFileStates(prev => prev.map(s => s.status === 'uploading' ? { ...s, status: 'error', error: message } : s));
|
|
86
|
+
}
|
|
87
|
+
};
|
|
88
|
+
|
|
89
|
+
const isUploading = fileStates.some(s => s.status === 'uploading');
|
|
90
|
+
const hasIdle = fileStates.some(s => s.status === 'idle');
|
|
91
|
+
|
|
92
|
+
return (
|
|
93
|
+
<div
|
|
94
|
+
className={`p-6 border-2 border-dashed transition-all duration-300 rounded-2xl ${
|
|
95
|
+
isDragging
|
|
96
|
+
? 'border-emerald-500 bg-emerald-50/50 dark:bg-emerald-500/5'
|
|
97
|
+
: 'border-slate-200 dark:border-white/10 bg-white dark:bg-white/5'
|
|
98
|
+
} ${className}`}
|
|
99
|
+
onDragOver={onDragOver}
|
|
100
|
+
onDragLeave={onDragLeave}
|
|
101
|
+
onDrop={onDrop}
|
|
102
|
+
>
|
|
103
|
+
<div className="flex flex-col items-center justify-center text-center">
|
|
104
|
+
<div
|
|
105
|
+
className="w-12 h-12 rounded-full flex items-center justify-center mb-4 shadow-sm"
|
|
106
|
+
style={{ background: `linear-gradient(135deg, ${ui.primaryColor}20, ${ui.accentColor}20)` }}
|
|
107
|
+
>
|
|
108
|
+
<Upload className="w-6 h-6" style={{ color: ui.primaryColor }} />
|
|
109
|
+
</div>
|
|
110
|
+
<h3 className="text-slate-900 dark:text-white font-semibold mb-1">Upload Documents</h3>
|
|
111
|
+
<p className="text-slate-500 dark:text-white/40 text-sm mb-6 max-w-xs">
|
|
112
|
+
Drag and drop PDF, DOCX, TXT, or JSON files to train your AI on your own data.
|
|
113
|
+
</p>
|
|
114
|
+
|
|
115
|
+
<button
|
|
116
|
+
onClick={() => fileInputRef.current?.click()}
|
|
117
|
+
disabled={isUploading}
|
|
118
|
+
className="px-4 py-2 rounded-lg text-sm font-medium transition-all hover:scale-105 active:scale-95 disabled:opacity-50"
|
|
119
|
+
style={{
|
|
120
|
+
backgroundColor: `${ui.primaryColor}15`,
|
|
121
|
+
color: ui.primaryColor,
|
|
122
|
+
border: `1px solid ${ui.primaryColor}30`
|
|
123
|
+
}}
|
|
124
|
+
>
|
|
125
|
+
Select Files
|
|
126
|
+
</button>
|
|
127
|
+
<input
|
|
128
|
+
ref={fileInputRef}
|
|
129
|
+
type="file"
|
|
130
|
+
multiple
|
|
131
|
+
onChange={onFileSelect}
|
|
132
|
+
className="hidden"
|
|
133
|
+
accept=".pdf,.docx,.txt,.md,.json,.csv"
|
|
134
|
+
/>
|
|
135
|
+
</div>
|
|
136
|
+
|
|
137
|
+
{fileStates.length > 0 && (
|
|
138
|
+
<div className="mt-8 space-y-3">
|
|
139
|
+
{fileStates.map((state, i) => (
|
|
140
|
+
<div
|
|
141
|
+
key={i}
|
|
142
|
+
className="flex items-center justify-between p-3 rounded-xl bg-slate-50 dark:bg-white/5 border border-slate-100 dark:border-white/10"
|
|
143
|
+
>
|
|
144
|
+
<div className="flex items-center gap-3 overflow-hidden">
|
|
145
|
+
<div className="w-8 h-8 rounded-lg bg-white dark:bg-white/10 flex items-center justify-center shadow-sm">
|
|
146
|
+
<File className="w-4 h-4 text-slate-400" />
|
|
147
|
+
</div>
|
|
148
|
+
<div className="truncate">
|
|
149
|
+
<p className="text-xs font-medium text-slate-700 dark:text-white/80 truncate">
|
|
150
|
+
{state.file.name}
|
|
151
|
+
</p>
|
|
152
|
+
<p className="text-[10px] text-slate-400">
|
|
153
|
+
{(state.file.size / 1024).toFixed(1)} KB
|
|
154
|
+
</p>
|
|
155
|
+
</div>
|
|
156
|
+
</div>
|
|
157
|
+
|
|
158
|
+
<div className="flex items-center gap-2">
|
|
159
|
+
{state.status === 'uploading' && <Loader2 className="w-4 h-4 text-slate-400 animate-spin" />}
|
|
160
|
+
{state.status === 'success' && <CheckCircle className="w-4 h-4 text-emerald-500" />}
|
|
161
|
+
{state.status === 'error' && (
|
|
162
|
+
<div title={state.error}>
|
|
163
|
+
<AlertCircle className="w-4 h-4 text-rose-500" />
|
|
164
|
+
</div>
|
|
165
|
+
)}
|
|
166
|
+
|
|
167
|
+
{state.status !== 'uploading' && (
|
|
168
|
+
<button
|
|
169
|
+
onClick={() => removeFile(i)}
|
|
170
|
+
className="p-1 rounded-md hover:bg-slate-200 dark:hover:bg-white/10 transition-colors"
|
|
171
|
+
>
|
|
172
|
+
<X className="w-3.5 h-3.5 text-slate-400" />
|
|
173
|
+
</button>
|
|
174
|
+
)}
|
|
175
|
+
</div>
|
|
176
|
+
</div>
|
|
177
|
+
))}
|
|
178
|
+
|
|
179
|
+
{hasIdle && !isUploading && (
|
|
180
|
+
<button
|
|
181
|
+
onClick={uploadFiles}
|
|
182
|
+
className="w-full mt-4 py-2.5 rounded-xl text-sm font-semibold text-white shadow-lg transition-all hover:scale-[1.02] active:scale-[0.98]"
|
|
183
|
+
style={{ background: `linear-gradient(135deg, ${ui.primaryColor}, ${ui.accentColor})` }}
|
|
184
|
+
>
|
|
185
|
+
Start Upload
|
|
186
|
+
</button>
|
|
187
|
+
)}
|
|
188
|
+
</div>
|
|
189
|
+
)}
|
|
190
|
+
</div>
|
|
191
|
+
);
|
|
192
|
+
}
|