@retrivora-ai/rag-engine 0.4.3 → 0.4.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{DocumentChunker-BICIjSuG.d.mts → DocumentChunker-3yElxTO3.d.mts} +9 -2
- package/dist/{DocumentChunker-BICIjSuG.d.ts → DocumentChunker-3yElxTO3.d.ts} +9 -2
- package/dist/{RagConfig-CVt24lbC.d.mts → RagConfig-BgRDL9Vy.d.mts} +37 -1
- package/dist/{RagConfig-CVt24lbC.d.ts → RagConfig-BgRDL9Vy.d.ts} +37 -1
- package/dist/SimpleGraphProvider-M6T7SE7D.mjs +62 -0
- package/dist/{chunk-UUZ3F4WK.mjs → chunk-OKY5P6RA.mjs} +179 -40
- package/dist/handlers/index.d.mts +2 -2
- package/dist/handlers/index.d.ts +2 -2
- package/dist/handlers/index.js +254 -40
- package/dist/handlers/index.mjs +1 -1
- package/dist/{index-OI2--lvT.d.mts → index-7qeLTPBL.d.mts} +1 -1
- package/dist/{index-D1hoNXMT.d.ts → index-DowY4_K0.d.ts} +1 -1
- package/dist/index.d.mts +15 -5
- package/dist/index.d.ts +15 -5
- package/dist/index.js +161 -1
- package/dist/index.mjs +158 -1
- package/dist/server.d.mts +52 -12
- package/dist/server.d.ts +52 -12
- package/dist/server.js +254 -40
- package/dist/server.mjs +1 -1
- package/package.json +1 -1
- package/src/components/ConfigProvider.tsx +1 -0
- package/src/components/DocumentUpload.tsx +192 -0
- package/src/config/RagConfig.ts +27 -0
- package/src/config/constants.ts +7 -0
- package/src/config/serverConfig.ts +1 -0
- package/src/core/Pipeline.ts +71 -10
- package/src/core/ProviderRegistry.ts +40 -8
- package/src/index.ts +1 -0
- package/src/providers/graphdb/BaseGraphProvider.ts +43 -0
- package/src/providers/graphdb/SimpleGraphProvider.ts +66 -0
- package/src/rag/DocumentChunker.ts +77 -34
- package/src/rag/EntityExtractor.ts +43 -0
- package/src/types/index.ts +19 -0
package/dist/server.js
CHANGED
|
@@ -1205,6 +1205,81 @@ var init_UniversalVectorProvider = __esm({
|
|
|
1205
1205
|
}
|
|
1206
1206
|
});
|
|
1207
1207
|
|
|
1208
|
+
// src/providers/graphdb/BaseGraphProvider.ts
|
|
1209
|
+
var BaseGraphProvider;
|
|
1210
|
+
var init_BaseGraphProvider = __esm({
|
|
1211
|
+
"src/providers/graphdb/BaseGraphProvider.ts"() {
|
|
1212
|
+
"use strict";
|
|
1213
|
+
BaseGraphProvider = class {
|
|
1214
|
+
constructor(config) {
|
|
1215
|
+
this.config = config;
|
|
1216
|
+
}
|
|
1217
|
+
};
|
|
1218
|
+
}
|
|
1219
|
+
});
|
|
1220
|
+
|
|
1221
|
+
// src/providers/graphdb/SimpleGraphProvider.ts
|
|
1222
|
+
var SimpleGraphProvider_exports = {};
|
|
1223
|
+
__export(SimpleGraphProvider_exports, {
|
|
1224
|
+
SimpleGraphProvider: () => SimpleGraphProvider
|
|
1225
|
+
});
|
|
1226
|
+
var SimpleGraphProvider;
|
|
1227
|
+
var init_SimpleGraphProvider = __esm({
|
|
1228
|
+
"src/providers/graphdb/SimpleGraphProvider.ts"() {
|
|
1229
|
+
"use strict";
|
|
1230
|
+
init_BaseGraphProvider();
|
|
1231
|
+
SimpleGraphProvider = class extends BaseGraphProvider {
|
|
1232
|
+
constructor() {
|
|
1233
|
+
super(...arguments);
|
|
1234
|
+
this.nodes = /* @__PURE__ */ new Map();
|
|
1235
|
+
this.edges = [];
|
|
1236
|
+
}
|
|
1237
|
+
async initialize() {
|
|
1238
|
+
console.log("[SimpleGraphProvider] Initialised in-memory graph store.");
|
|
1239
|
+
}
|
|
1240
|
+
async addNodes(nodes) {
|
|
1241
|
+
for (const node of nodes) {
|
|
1242
|
+
this.nodes.set(node.id, node);
|
|
1243
|
+
}
|
|
1244
|
+
}
|
|
1245
|
+
async addEdges(edges) {
|
|
1246
|
+
this.edges.push(...edges);
|
|
1247
|
+
}
|
|
1248
|
+
async query(queryText, limit = 5) {
|
|
1249
|
+
const q = queryText.toLowerCase();
|
|
1250
|
+
const matchedNodes = Array.from(this.nodes.values()).filter(
|
|
1251
|
+
(node) => node.id.toLowerCase().includes(q) || node.label.toLowerCase().includes(q) || JSON.stringify(node.properties).toLowerCase().includes(q)
|
|
1252
|
+
).slice(0, limit);
|
|
1253
|
+
const matchedNodeIds = new Set(matchedNodes.map((n) => n.id));
|
|
1254
|
+
const matchedEdges = this.edges.filter(
|
|
1255
|
+
(edge) => matchedNodeIds.has(edge.source) || matchedNodeIds.has(edge.target)
|
|
1256
|
+
);
|
|
1257
|
+
for (const edge of matchedEdges) {
|
|
1258
|
+
if (!matchedNodeIds.has(edge.source)) {
|
|
1259
|
+
const source = this.nodes.get(edge.source);
|
|
1260
|
+
if (source) matchedNodes.push(source);
|
|
1261
|
+
}
|
|
1262
|
+
if (!matchedNodeIds.has(edge.target)) {
|
|
1263
|
+
const target = this.nodes.get(edge.target);
|
|
1264
|
+
if (target) matchedNodes.push(target);
|
|
1265
|
+
}
|
|
1266
|
+
}
|
|
1267
|
+
return {
|
|
1268
|
+
nodes: Array.from(new Set(matchedNodes)),
|
|
1269
|
+
edges: matchedEdges
|
|
1270
|
+
};
|
|
1271
|
+
}
|
|
1272
|
+
async ping() {
|
|
1273
|
+
return true;
|
|
1274
|
+
}
|
|
1275
|
+
async disconnect() {
|
|
1276
|
+
this.nodes.clear();
|
|
1277
|
+
this.edges = [];
|
|
1278
|
+
}
|
|
1279
|
+
};
|
|
1280
|
+
}
|
|
1281
|
+
});
|
|
1282
|
+
|
|
1208
1283
|
// src/server.ts
|
|
1209
1284
|
var server_exports = {};
|
|
1210
1285
|
__export(server_exports, {
|
|
@@ -1317,7 +1392,7 @@ function readEnum(env, name, fallback, allowed) {
|
|
|
1317
1392
|
throw new Error(`[getRagConfig] ${name} must be one of: ${allowed.join(", ")}`);
|
|
1318
1393
|
}
|
|
1319
1394
|
function getRagConfig(env = process.env) {
|
|
1320
|
-
var _a, _b, _c, _d, _e, _f, _g, _h, _i, _j, _k, _l, _m, _n, _o, _p, _q, _r, _s, _t, _u, _v, _w, _x, _y, _z, _A, _B, _C, _D, _E, _F, _G, _H, _I, _J, _K, _L, _M, _N, _O;
|
|
1395
|
+
var _a, _b, _c, _d, _e, _f, _g, _h, _i, _j, _k, _l, _m, _n, _o, _p, _q, _r, _s, _t, _u, _v, _w, _x, _y, _z, _A, _B, _C, _D, _E, _F, _G, _H, _I, _J, _K, _L, _M, _N, _O, _P, _Q;
|
|
1321
1396
|
const projectId = (_b = (_a = readString(env, "RAG_PROJECT_ID")) != null ? _a : readString(env, "NEXT_PUBLIC_PROJECT_ID")) != null ? _b : "__default__";
|
|
1322
1397
|
const vectorProvider = readEnum(env, "VECTOR_DB_PROVIDER", "pinecone", VECTOR_DB_PROVIDERS);
|
|
1323
1398
|
const llmProvider = readEnum(env, "LLM_PROVIDER", "openai", LLM_PROVIDERS);
|
|
@@ -1414,7 +1489,8 @@ function getRagConfig(env = process.env) {
|
|
|
1414
1489
|
showSources: ((_I = (_H = readString(env, "NEXT_PUBLIC_SHOW_SOURCES")) != null ? _H : readString(env, "UI_SHOW_SOURCES")) != null ? _I : "true") !== "false",
|
|
1415
1490
|
welcomeMessage: (_K = (_J = readString(env, "NEXT_PUBLIC_WELCOME_MESSAGE")) != null ? _J : readString(env, "UI_WELCOME_MESSAGE")) != null ? _K : "Hello! I'm your AI assistant. Ask me anything about your documents.",
|
|
1416
1491
|
visualStyle: (_M = (_L = readString(env, "NEXT_PUBLIC_UI_VISUAL_STYLE")) != null ? _L : readString(env, "UI_VISUAL_STYLE")) != null ? _M : "glass",
|
|
1417
|
-
borderRadius: (_O = (_N = readString(env, "NEXT_PUBLIC_UI_BORDER_RADIUS")) != null ? _N : readString(env, "UI_BORDER_RADIUS")) != null ? _O : "xl"
|
|
1492
|
+
borderRadius: (_O = (_N = readString(env, "NEXT_PUBLIC_UI_BORDER_RADIUS")) != null ? _N : readString(env, "UI_BORDER_RADIUS")) != null ? _O : "xl",
|
|
1493
|
+
allowUpload: ((_Q = (_P = readString(env, "NEXT_PUBLIC_ALLOW_UPLOAD")) != null ? _P : readString(env, "UI_ALLOW_UPLOAD")) != null ? _Q : "false") === "true"
|
|
1418
1494
|
},
|
|
1419
1495
|
rag: {
|
|
1420
1496
|
topK: readNumber(env, "RAG_TOP_K", 5),
|
|
@@ -1988,52 +2064,84 @@ var ConfigValidator = class {
|
|
|
1988
2064
|
|
|
1989
2065
|
// src/rag/DocumentChunker.ts
|
|
1990
2066
|
var DocumentChunker = class {
|
|
1991
|
-
constructor(chunkSize = 1e3, chunkOverlap = 200) {
|
|
2067
|
+
constructor(chunkSize = 1e3, chunkOverlap = 200, separators = ["\n\n", "\n", " ", ""]) {
|
|
1992
2068
|
this.chunkSize = chunkSize;
|
|
1993
2069
|
this.chunkOverlap = chunkOverlap;
|
|
2070
|
+
this.separators = separators;
|
|
1994
2071
|
}
|
|
1995
2072
|
/**
|
|
1996
|
-
* Split a single text string into overlapping chunks.
|
|
2073
|
+
* Split a single text string into overlapping chunks using a recursive strategy.
|
|
1997
2074
|
*/
|
|
1998
2075
|
chunk(text, options = {}) {
|
|
1999
2076
|
const {
|
|
2000
2077
|
chunkSize = this.chunkSize,
|
|
2001
2078
|
chunkOverlap = this.chunkOverlap,
|
|
2002
2079
|
docId = `doc_${Date.now()}`,
|
|
2003
|
-
metadata = {}
|
|
2080
|
+
metadata = {},
|
|
2081
|
+
separators = this.separators
|
|
2004
2082
|
} = options;
|
|
2005
|
-
const
|
|
2006
|
-
|
|
2007
|
-
|
|
2008
|
-
|
|
2009
|
-
let current = "";
|
|
2083
|
+
const finalChunks = [];
|
|
2084
|
+
const splits = this.recursiveSplit(text, separators, chunkSize);
|
|
2085
|
+
let currentChunk = [];
|
|
2086
|
+
let currentLength = 0;
|
|
2010
2087
|
let chunkIndex = 0;
|
|
2011
|
-
for (const
|
|
2012
|
-
if (
|
|
2013
|
-
|
|
2014
|
-
|
|
2015
|
-
|
|
2016
|
-
|
|
2017
|
-
|
|
2018
|
-
|
|
2019
|
-
|
|
2020
|
-
|
|
2021
|
-
|
|
2022
|
-
|
|
2023
|
-
|
|
2024
|
-
|
|
2025
|
-
|
|
2088
|
+
for (const split of splits) {
|
|
2089
|
+
if (currentLength + split.length > chunkSize && currentChunk.length > 0) {
|
|
2090
|
+
finalChunks.push({
|
|
2091
|
+
id: `${docId}_chunk_${chunkIndex++}`,
|
|
2092
|
+
content: currentChunk.join("").trim(),
|
|
2093
|
+
metadata: __spreadProps(__spreadValues({}, metadata), { docId, chunkIndex: chunkIndex - 1 })
|
|
2094
|
+
});
|
|
2095
|
+
const overlapItems = [];
|
|
2096
|
+
let overlapLen = 0;
|
|
2097
|
+
for (let i = currentChunk.length - 1; i >= 0; i--) {
|
|
2098
|
+
if (overlapLen + currentChunk[i].length <= chunkOverlap) {
|
|
2099
|
+
overlapItems.unshift(currentChunk[i]);
|
|
2100
|
+
overlapLen += currentChunk[i].length;
|
|
2101
|
+
} else {
|
|
2102
|
+
break;
|
|
2103
|
+
}
|
|
2026
2104
|
}
|
|
2105
|
+
currentChunk = overlapItems;
|
|
2106
|
+
currentLength = overlapLen;
|
|
2027
2107
|
}
|
|
2108
|
+
currentChunk.push(split);
|
|
2109
|
+
currentLength += split.length;
|
|
2028
2110
|
}
|
|
2029
|
-
if (
|
|
2030
|
-
|
|
2111
|
+
if (currentChunk.length > 0) {
|
|
2112
|
+
finalChunks.push({
|
|
2031
2113
|
id: `${docId}_chunk_${chunkIndex}`,
|
|
2032
|
-
content:
|
|
2114
|
+
content: currentChunk.join("").trim(),
|
|
2033
2115
|
metadata: __spreadProps(__spreadValues({}, metadata), { docId, chunkIndex })
|
|
2034
2116
|
});
|
|
2035
2117
|
}
|
|
2036
|
-
return
|
|
2118
|
+
return finalChunks;
|
|
2119
|
+
}
|
|
2120
|
+
/**
|
|
2121
|
+
* Recursively split text based on separators.
|
|
2122
|
+
*/
|
|
2123
|
+
recursiveSplit(text, separators, chunkSize) {
|
|
2124
|
+
const finalSplits = [];
|
|
2125
|
+
let separator = separators[separators.length - 1];
|
|
2126
|
+
let nextSeparators = [];
|
|
2127
|
+
for (let i = 0; i < separators.length; i++) {
|
|
2128
|
+
if (text.includes(separators[i])) {
|
|
2129
|
+
separator = separators[i];
|
|
2130
|
+
nextSeparators = separators.slice(i + 1);
|
|
2131
|
+
break;
|
|
2132
|
+
}
|
|
2133
|
+
}
|
|
2134
|
+
const parts = text.split(separator);
|
|
2135
|
+
for (const part of parts) {
|
|
2136
|
+
if (part.length <= chunkSize) {
|
|
2137
|
+
finalSplits.push(part + separator);
|
|
2138
|
+
} else if (nextSeparators.length > 0) {
|
|
2139
|
+
finalSplits.push(...this.recursiveSplit(part, nextSeparators, chunkSize));
|
|
2140
|
+
} else {
|
|
2141
|
+
finalSplits.push(part);
|
|
2142
|
+
}
|
|
2143
|
+
}
|
|
2144
|
+
return finalSplits;
|
|
2037
2145
|
}
|
|
2038
2146
|
/**
|
|
2039
2147
|
* Chunk multiple documents at once.
|
|
@@ -2045,6 +2153,39 @@ var DocumentChunker = class {
|
|
|
2045
2153
|
}
|
|
2046
2154
|
};
|
|
2047
2155
|
|
|
2156
|
+
// src/rag/EntityExtractor.ts
|
|
2157
|
+
var EntityExtractor = class {
|
|
2158
|
+
constructor(llm) {
|
|
2159
|
+
this.llm = llm;
|
|
2160
|
+
}
|
|
2161
|
+
/**
|
|
2162
|
+
* Extract nodes and edges from a text chunk.
|
|
2163
|
+
*/
|
|
2164
|
+
async extract(text) {
|
|
2165
|
+
const prompt = `
|
|
2166
|
+
Extract entities and relationships from the following text.
|
|
2167
|
+
Format the output as JSON with two keys: "nodes" (array of {id, label, properties}) and "edges" (array of {source, target, type, properties}).
|
|
2168
|
+
Use the same ID for the same entity.
|
|
2169
|
+
|
|
2170
|
+
Text:
|
|
2171
|
+
"${text}"
|
|
2172
|
+
|
|
2173
|
+
Output JSON:
|
|
2174
|
+
`;
|
|
2175
|
+
const response = await this.llm.chat([
|
|
2176
|
+
{ role: "system", content: "You are an expert at knowledge graph extraction. Respond only with valid JSON." },
|
|
2177
|
+
{ role: "user", content: prompt }
|
|
2178
|
+
], "");
|
|
2179
|
+
try {
|
|
2180
|
+
const cleanJson = response.replace(/```json\n?|\n?```/g, "").trim();
|
|
2181
|
+
return JSON.parse(cleanJson);
|
|
2182
|
+
} catch (error) {
|
|
2183
|
+
console.warn("[EntityExtractor] Failed to parse LLM response as JSON:", response);
|
|
2184
|
+
return { nodes: [], edges: [] };
|
|
2185
|
+
}
|
|
2186
|
+
}
|
|
2187
|
+
};
|
|
2188
|
+
|
|
2048
2189
|
// src/llm/providers/OpenAIProvider.ts
|
|
2049
2190
|
var import_openai = __toESM(require("openai"));
|
|
2050
2191
|
var OpenAIProvider = class {
|
|
@@ -2589,16 +2730,15 @@ var LLMFactory = class _LLMFactory {
|
|
|
2589
2730
|
|
|
2590
2731
|
// src/core/ProviderRegistry.ts
|
|
2591
2732
|
var ProviderRegistry = class {
|
|
2592
|
-
/**
|
|
2593
|
-
* Register a custom vector provider class by name.
|
|
2594
|
-
* The name must match the provider value used in VectorDBConfig.provider.
|
|
2595
|
-
*
|
|
2596
|
-
* @example
|
|
2597
|
-
* ProviderRegistry.registerVectorProvider('my-db', MyCustomProvider);
|
|
2598
|
-
*/
|
|
2599
2733
|
static registerVectorProvider(name, providerClass) {
|
|
2600
2734
|
this.vectorProviders[name] = providerClass;
|
|
2601
2735
|
}
|
|
2736
|
+
/**
|
|
2737
|
+
* Register a custom graph provider class by name.
|
|
2738
|
+
*/
|
|
2739
|
+
static registerGraphProvider(name, providerClass) {
|
|
2740
|
+
this.graphProviders[name] = providerClass;
|
|
2741
|
+
}
|
|
2602
2742
|
/**
|
|
2603
2743
|
* Creates a vector database provider based on the configuration.
|
|
2604
2744
|
* Built-in providers are dynamically imported to avoid bundling all SDKs.
|
|
@@ -2653,6 +2793,28 @@ var ProviderRegistry = class {
|
|
|
2653
2793
|
);
|
|
2654
2794
|
}
|
|
2655
2795
|
}
|
|
2796
|
+
/**
|
|
2797
|
+
* Creates a graph database provider based on the configuration.
|
|
2798
|
+
*/
|
|
2799
|
+
static async createGraphProvider(config) {
|
|
2800
|
+
const { provider } = config;
|
|
2801
|
+
if (this.graphProviders[provider]) {
|
|
2802
|
+
return new this.graphProviders[provider](config);
|
|
2803
|
+
}
|
|
2804
|
+
switch (provider) {
|
|
2805
|
+
case "neo4j": {
|
|
2806
|
+
throw new Error("[ProviderRegistry] Neo4j provider not implemented yet.");
|
|
2807
|
+
}
|
|
2808
|
+
case "simple": {
|
|
2809
|
+
const { SimpleGraphProvider: SimpleGraphProvider2 } = await Promise.resolve().then(() => (init_SimpleGraphProvider(), SimpleGraphProvider_exports));
|
|
2810
|
+
return new SimpleGraphProvider2(config);
|
|
2811
|
+
}
|
|
2812
|
+
default:
|
|
2813
|
+
throw new Error(
|
|
2814
|
+
`[ProviderRegistry] Unsupported graph provider: "${provider}". Built-in providers: simple. For custom providers, call ProviderRegistry.registerGraphProvider("${provider}", YourClass).`
|
|
2815
|
+
);
|
|
2816
|
+
}
|
|
2817
|
+
}
|
|
2656
2818
|
/**
|
|
2657
2819
|
* Creates an LLM provider based on the configuration.
|
|
2658
2820
|
*/
|
|
@@ -2661,6 +2823,7 @@ var ProviderRegistry = class {
|
|
|
2661
2823
|
}
|
|
2662
2824
|
};
|
|
2663
2825
|
ProviderRegistry.vectorProviders = {};
|
|
2826
|
+
ProviderRegistry.graphProviders = {};
|
|
2664
2827
|
|
|
2665
2828
|
// src/core/BatchProcessor.ts
|
|
2666
2829
|
function isTransientError(error) {
|
|
@@ -3092,6 +3255,11 @@ var Pipeline = class {
|
|
|
3092
3255
|
);
|
|
3093
3256
|
this.llmProvider = llmProvider;
|
|
3094
3257
|
this.embeddingProvider = embeddingProvider;
|
|
3258
|
+
if (this.config.graphDb) {
|
|
3259
|
+
this.graphDB = await ProviderRegistry.createGraphProvider(this.config.graphDb);
|
|
3260
|
+
await this.graphDB.initialize();
|
|
3261
|
+
this.entityExtractor = new EntityExtractor(this.llmProvider);
|
|
3262
|
+
}
|
|
3095
3263
|
await this.vectorDB.initialize();
|
|
3096
3264
|
this.initialised = true;
|
|
3097
3265
|
}
|
|
@@ -3145,6 +3313,14 @@ var Pipeline = class {
|
|
|
3145
3313
|
docId: doc.docId,
|
|
3146
3314
|
chunksIngested: upsertResult.totalProcessed
|
|
3147
3315
|
});
|
|
3316
|
+
if (this.graphDB && this.entityExtractor) {
|
|
3317
|
+
console.log(`[Pipeline] Extracting entities for doc ${doc.docId}...`);
|
|
3318
|
+
for (const chunk of chunks) {
|
|
3319
|
+
const { nodes, edges } = await this.entityExtractor.extract(chunk.content);
|
|
3320
|
+
if (nodes.length > 0) await this.graphDB.addNodes(nodes);
|
|
3321
|
+
if (edges.length > 0) await this.graphDB.addEdges(edges);
|
|
3322
|
+
}
|
|
3323
|
+
}
|
|
3148
3324
|
} catch (error) {
|
|
3149
3325
|
console.error(`[Pipeline] Failed to ingest document ${doc.docId}:`, error);
|
|
3150
3326
|
results.push({ docId: doc.docId, chunksIngested: 0 });
|
|
@@ -3153,27 +3329,65 @@ var Pipeline = class {
|
|
|
3153
3329
|
return results;
|
|
3154
3330
|
}
|
|
3155
3331
|
async ask(question, history = [], namespace) {
|
|
3156
|
-
var _a, _b, _c, _d;
|
|
3332
|
+
var _a, _b, _c, _d, _e, _f;
|
|
3157
3333
|
await this.initialize();
|
|
3158
3334
|
const ns = namespace != null ? namespace : this.config.projectId;
|
|
3159
3335
|
const topK = (_b = (_a = this.config.rag) == null ? void 0 : _a.topK) != null ? _b : 5;
|
|
3160
3336
|
const scoreThreshold = (_d = (_c = this.config.rag) == null ? void 0 : _c.scoreThreshold) != null ? _d : 0;
|
|
3161
3337
|
try {
|
|
3162
|
-
|
|
3338
|
+
let searchQuery = question;
|
|
3339
|
+
if ((_e = this.config.rag) == null ? void 0 : _e.useQueryTransformation) {
|
|
3340
|
+
searchQuery = await this.rewriteQuery(question, history);
|
|
3341
|
+
}
|
|
3342
|
+
const queryVector = await this.embeddingProvider.embed(searchQuery, { taskType: "query" });
|
|
3163
3343
|
const fieldHints = extractQueryFieldHints(question);
|
|
3164
3344
|
const filter = buildQueryFilter(question, fieldHints);
|
|
3165
3345
|
filter.__entityHints = fieldHints;
|
|
3166
3346
|
const rawMatches = await this.vectorDB.query(queryVector, topK, ns, filter);
|
|
3167
3347
|
const sources = rawMatches.filter((m) => m.score >= scoreThreshold);
|
|
3168
|
-
|
|
3348
|
+
let graphData;
|
|
3349
|
+
if (this.graphDB && ((_f = this.config.rag) == null ? void 0 : _f.useGraphRetrieval)) {
|
|
3350
|
+
graphData = await this.graphDB.query(searchQuery);
|
|
3351
|
+
}
|
|
3352
|
+
let context = sources.length ? sources.map((m, i) => `[Source ${i + 1}]
|
|
3169
3353
|
${m.content}`).join("\n\n---\n\n") : "No relevant context found.";
|
|
3354
|
+
if (graphData && graphData.nodes.length > 0) {
|
|
3355
|
+
const graphContext = graphData.nodes.map(
|
|
3356
|
+
(n) => `Entity: ${n.label} (${n.id})${n.properties ? " - " + JSON.stringify(n.properties) : ""}`
|
|
3357
|
+
).join("\n");
|
|
3358
|
+
context = `GRAPH KNOWLEDGE:
|
|
3359
|
+
${graphContext}
|
|
3360
|
+
|
|
3361
|
+
VECTOR CONTEXT:
|
|
3362
|
+
${context}`;
|
|
3363
|
+
}
|
|
3170
3364
|
const messages = [...history, { role: "user", content: question }];
|
|
3171
3365
|
const reply = await this.llmProvider.chat(messages, context);
|
|
3172
|
-
return { reply, sources };
|
|
3366
|
+
return { reply, sources, graphData };
|
|
3173
3367
|
} catch (error) {
|
|
3174
3368
|
throw new Error(`[Pipeline] Chat failed: ${error instanceof Error ? error.message : String(error)}`);
|
|
3175
3369
|
}
|
|
3176
3370
|
}
|
|
3371
|
+
/**
|
|
3372
|
+
* Rewrite the user query for better retrieval performance.
|
|
3373
|
+
*/
|
|
3374
|
+
async rewriteQuery(question, history) {
|
|
3375
|
+
const prompt = `
|
|
3376
|
+
Given the following conversation history and a new question, rewrite the question to be a better search query for a vector database.
|
|
3377
|
+
Focus on extracting the core intent and entities. Do not answer the question, just rewrite it.
|
|
3378
|
+
|
|
3379
|
+
History:
|
|
3380
|
+
${history.map((m) => `${m.role}: ${m.content}`).join("\n")}
|
|
3381
|
+
|
|
3382
|
+
New Question: ${question}
|
|
3383
|
+
|
|
3384
|
+
Optimized Search Query:`;
|
|
3385
|
+
const rewrite = await this.llmProvider.chat([
|
|
3386
|
+
{ role: "system", content: "You are an assistant that optimizes search queries for RAG systems." },
|
|
3387
|
+
{ role: "user", content: prompt }
|
|
3388
|
+
], "");
|
|
3389
|
+
return rewrite.trim() || question;
|
|
3390
|
+
}
|
|
3177
3391
|
};
|
|
3178
3392
|
|
|
3179
3393
|
// src/core/ProviderHealthCheck.ts
|
package/dist/server.mjs
CHANGED
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@retrivora-ai/rag-engine",
|
|
3
|
-
"version": "0.4.
|
|
3
|
+
"version": "0.4.4",
|
|
4
4
|
"description": "Retrivora AI is a plug-and-play AI engine for RAG chat experiences — generic vector DB + LLM provider, embeddable or standalone.",
|
|
5
5
|
"author": "Abhinav Alkuchi",
|
|
6
6
|
"license": "MIT",
|
|
@@ -0,0 +1,192 @@
|
|
|
1
|
+
'use client';
|
|
2
|
+
|
|
3
|
+
import React, { useState, useRef } from 'react';
|
|
4
|
+
import { Upload, File, X, CheckCircle, AlertCircle, Loader2 } from 'lucide-react';
|
|
5
|
+
import { useConfig } from './ConfigProvider';
|
|
6
|
+
|
|
7
|
+
interface DocumentUploadProps {
|
|
8
|
+
/** Optional namespace for the upload */
|
|
9
|
+
namespace?: string;
|
|
10
|
+
/** Callback when upload completes */
|
|
11
|
+
onUploadComplete?: (results: any) => void;
|
|
12
|
+
/** Additional className */
|
|
13
|
+
className?: string;
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
interface FileState {
|
|
17
|
+
file: File;
|
|
18
|
+
status: 'idle' | 'uploading' | 'success' | 'error';
|
|
19
|
+
error?: string;
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
export function DocumentUpload({ namespace, onUploadComplete, className = '' }: DocumentUploadProps) {
|
|
23
|
+
const { ui } = useConfig();
|
|
24
|
+
const [fileStates, setFileStates] = useState<FileState[]>([]);
|
|
25
|
+
const [isDragging, setIsDragging] = useState(false);
|
|
26
|
+
const fileInputRef = useRef<HTMLInputElement>(null);
|
|
27
|
+
|
|
28
|
+
const addFiles = (files: File[]) => {
|
|
29
|
+
const newStates = files.map(file => ({ file, status: 'idle' as const }));
|
|
30
|
+
setFileStates(prev => [...prev, ...newStates]);
|
|
31
|
+
};
|
|
32
|
+
|
|
33
|
+
const removeFile = (index: number) => {
|
|
34
|
+
setFileStates(prev => prev.filter((_, i) => i !== index));
|
|
35
|
+
};
|
|
36
|
+
|
|
37
|
+
const onDragOver = (e: React.DragEvent) => {
|
|
38
|
+
e.preventDefault();
|
|
39
|
+
setIsDragging(true);
|
|
40
|
+
};
|
|
41
|
+
|
|
42
|
+
const onDragLeave = () => {
|
|
43
|
+
setIsDragging(false);
|
|
44
|
+
};
|
|
45
|
+
|
|
46
|
+
const onDrop = (e: React.DragEvent) => {
|
|
47
|
+
e.preventDefault();
|
|
48
|
+
setIsDragging(false);
|
|
49
|
+
if (e.dataTransfer.files) {
|
|
50
|
+
addFiles(Array.from(e.dataTransfer.files));
|
|
51
|
+
}
|
|
52
|
+
};
|
|
53
|
+
|
|
54
|
+
const onFileSelect = (e: React.ChangeEvent<HTMLInputElement>) => {
|
|
55
|
+
if (e.target.files) {
|
|
56
|
+
addFiles(Array.from(e.target.files));
|
|
57
|
+
}
|
|
58
|
+
};
|
|
59
|
+
|
|
60
|
+
const uploadFiles = async () => {
|
|
61
|
+
const idleFiles = fileStates.filter(s => s.status === 'idle');
|
|
62
|
+
if (idleFiles.length === 0) return;
|
|
63
|
+
|
|
64
|
+
// Update status to uploading
|
|
65
|
+
setFileStates(prev => prev.map(s => s.status === 'idle' ? { ...s, status: 'uploading' } : s));
|
|
66
|
+
|
|
67
|
+
const formData = new FormData();
|
|
68
|
+
idleFiles.forEach(s => formData.append('files', s.file));
|
|
69
|
+
if (namespace) formData.append('namespace', namespace);
|
|
70
|
+
|
|
71
|
+
try {
|
|
72
|
+
const response = await fetch('/api/upload', {
|
|
73
|
+
method: 'POST',
|
|
74
|
+
body: formData,
|
|
75
|
+
});
|
|
76
|
+
|
|
77
|
+
const result = await response.json();
|
|
78
|
+
|
|
79
|
+
if (!response.ok) throw new Error(result.error || 'Upload failed');
|
|
80
|
+
|
|
81
|
+
setFileStates(prev => prev.map(s => s.status === 'uploading' ? { ...s, status: 'success' } : s));
|
|
82
|
+
if (onUploadComplete) onUploadComplete(result);
|
|
83
|
+
} catch (err) {
|
|
84
|
+
const message = err instanceof Error ? err.message : 'Upload failed';
|
|
85
|
+
setFileStates(prev => prev.map(s => s.status === 'uploading' ? { ...s, status: 'error', error: message } : s));
|
|
86
|
+
}
|
|
87
|
+
};
|
|
88
|
+
|
|
89
|
+
const isUploading = fileStates.some(s => s.status === 'uploading');
|
|
90
|
+
const hasIdle = fileStates.some(s => s.status === 'idle');
|
|
91
|
+
|
|
92
|
+
return (
|
|
93
|
+
<div
|
|
94
|
+
className={`p-6 border-2 border-dashed transition-all duration-300 rounded-2xl ${
|
|
95
|
+
isDragging
|
|
96
|
+
? 'border-emerald-500 bg-emerald-50/50 dark:bg-emerald-500/5'
|
|
97
|
+
: 'border-slate-200 dark:border-white/10 bg-white dark:bg-white/5'
|
|
98
|
+
} ${className}`}
|
|
99
|
+
onDragOver={onDragOver}
|
|
100
|
+
onDragLeave={onDragLeave}
|
|
101
|
+
onDrop={onDrop}
|
|
102
|
+
>
|
|
103
|
+
<div className="flex flex-col items-center justify-center text-center">
|
|
104
|
+
<div
|
|
105
|
+
className="w-12 h-12 rounded-full flex items-center justify-center mb-4 shadow-sm"
|
|
106
|
+
style={{ background: `linear-gradient(135deg, ${ui.primaryColor}20, ${ui.accentColor}20)` }}
|
|
107
|
+
>
|
|
108
|
+
<Upload className="w-6 h-6" style={{ color: ui.primaryColor }} />
|
|
109
|
+
</div>
|
|
110
|
+
<h3 className="text-slate-900 dark:text-white font-semibold mb-1">Upload Documents</h3>
|
|
111
|
+
<p className="text-slate-500 dark:text-white/40 text-sm mb-6 max-w-xs">
|
|
112
|
+
Drag and drop PDF, DOCX, TXT, or JSON files to train your AI on your own data.
|
|
113
|
+
</p>
|
|
114
|
+
|
|
115
|
+
<button
|
|
116
|
+
onClick={() => fileInputRef.current?.click()}
|
|
117
|
+
disabled={isUploading}
|
|
118
|
+
className="px-4 py-2 rounded-lg text-sm font-medium transition-all hover:scale-105 active:scale-95 disabled:opacity-50"
|
|
119
|
+
style={{
|
|
120
|
+
backgroundColor: `${ui.primaryColor}15`,
|
|
121
|
+
color: ui.primaryColor,
|
|
122
|
+
border: `1px solid ${ui.primaryColor}30`
|
|
123
|
+
}}
|
|
124
|
+
>
|
|
125
|
+
Select Files
|
|
126
|
+
</button>
|
|
127
|
+
<input
|
|
128
|
+
ref={fileInputRef}
|
|
129
|
+
type="file"
|
|
130
|
+
multiple
|
|
131
|
+
onChange={onFileSelect}
|
|
132
|
+
className="hidden"
|
|
133
|
+
accept=".pdf,.docx,.txt,.md,.json,.csv"
|
|
134
|
+
/>
|
|
135
|
+
</div>
|
|
136
|
+
|
|
137
|
+
{fileStates.length > 0 && (
|
|
138
|
+
<div className="mt-8 space-y-3">
|
|
139
|
+
{fileStates.map((state, i) => (
|
|
140
|
+
<div
|
|
141
|
+
key={i}
|
|
142
|
+
className="flex items-center justify-between p-3 rounded-xl bg-slate-50 dark:bg-white/5 border border-slate-100 dark:border-white/10"
|
|
143
|
+
>
|
|
144
|
+
<div className="flex items-center gap-3 overflow-hidden">
|
|
145
|
+
<div className="w-8 h-8 rounded-lg bg-white dark:bg-white/10 flex items-center justify-center shadow-sm">
|
|
146
|
+
<File className="w-4 h-4 text-slate-400" />
|
|
147
|
+
</div>
|
|
148
|
+
<div className="truncate">
|
|
149
|
+
<p className="text-xs font-medium text-slate-700 dark:text-white/80 truncate">
|
|
150
|
+
{state.file.name}
|
|
151
|
+
</p>
|
|
152
|
+
<p className="text-[10px] text-slate-400">
|
|
153
|
+
{(state.file.size / 1024).toFixed(1)} KB
|
|
154
|
+
</p>
|
|
155
|
+
</div>
|
|
156
|
+
</div>
|
|
157
|
+
|
|
158
|
+
<div className="flex items-center gap-2">
|
|
159
|
+
{state.status === 'uploading' && <Loader2 className="w-4 h-4 text-slate-400 animate-spin" />}
|
|
160
|
+
{state.status === 'success' && <CheckCircle className="w-4 h-4 text-emerald-500" />}
|
|
161
|
+
{state.status === 'error' && (
|
|
162
|
+
<div title={state.error}>
|
|
163
|
+
<AlertCircle className="w-4 h-4 text-rose-500" />
|
|
164
|
+
</div>
|
|
165
|
+
)}
|
|
166
|
+
|
|
167
|
+
{state.status !== 'uploading' && (
|
|
168
|
+
<button
|
|
169
|
+
onClick={() => removeFile(i)}
|
|
170
|
+
className="p-1 rounded-md hover:bg-slate-200 dark:hover:bg-white/10 transition-colors"
|
|
171
|
+
>
|
|
172
|
+
<X className="w-3.5 h-3.5 text-slate-400" />
|
|
173
|
+
</button>
|
|
174
|
+
)}
|
|
175
|
+
</div>
|
|
176
|
+
</div>
|
|
177
|
+
))}
|
|
178
|
+
|
|
179
|
+
{hasIdle && !isUploading && (
|
|
180
|
+
<button
|
|
181
|
+
onClick={uploadFiles}
|
|
182
|
+
className="w-full mt-4 py-2.5 rounded-xl text-sm font-semibold text-white shadow-lg transition-all hover:scale-[1.02] active:scale-[0.98]"
|
|
183
|
+
style={{ background: `linear-gradient(135deg, ${ui.primaryColor}, ${ui.accentColor})` }}
|
|
184
|
+
>
|
|
185
|
+
Start Upload
|
|
186
|
+
</button>
|
|
187
|
+
)}
|
|
188
|
+
</div>
|
|
189
|
+
)}
|
|
190
|
+
</div>
|
|
191
|
+
);
|
|
192
|
+
}
|
package/src/config/RagConfig.ts
CHANGED
|
@@ -12,6 +12,7 @@ import {
|
|
|
12
12
|
VECTOR_DB_PROVIDERS,
|
|
13
13
|
LLM_PROVIDERS,
|
|
14
14
|
EMBEDDING_PROVIDERS,
|
|
15
|
+
GRAPH_DB_PROVIDERS,
|
|
15
16
|
} from './constants';
|
|
16
17
|
|
|
17
18
|
export type VectorDBProvider = typeof VECTOR_DB_PROVIDERS[number];
|
|
@@ -42,6 +43,19 @@ export interface VectorDBConfig {
|
|
|
42
43
|
options: Record<string, unknown>;
|
|
43
44
|
}
|
|
44
45
|
|
|
46
|
+
// ---------------------------------------------------------------------------
|
|
47
|
+
// Graph DB
|
|
48
|
+
// ---------------------------------------------------------------------------
|
|
49
|
+
|
|
50
|
+
export type GraphDBProvider = typeof GRAPH_DB_PROVIDERS[number];
|
|
51
|
+
|
|
52
|
+
export interface GraphDBConfig {
|
|
53
|
+
/** Which graph database to use */
|
|
54
|
+
provider: GraphDBProvider;
|
|
55
|
+
/** Provider-specific options (URI, credentials, etc.) */
|
|
56
|
+
options: Record<string, unknown>;
|
|
57
|
+
}
|
|
58
|
+
|
|
45
59
|
// ---------------------------------------------------------------------------
|
|
46
60
|
// LLM Provider
|
|
47
61
|
// ---------------------------------------------------------------------------
|
|
@@ -128,6 +142,8 @@ export interface UIConfig {
|
|
|
128
142
|
visualStyle?: 'glass' | 'solid';
|
|
129
143
|
/** Border radius: 'none', 'sm', 'md', 'lg', 'xl', 'full' */
|
|
130
144
|
borderRadius?: 'none' | 'sm' | 'md' | 'lg' | 'xl' | 'full';
|
|
145
|
+
/** Whether to allow file uploads directly from the UI */
|
|
146
|
+
allowUpload?: boolean;
|
|
131
147
|
}
|
|
132
148
|
|
|
133
149
|
// ---------------------------------------------------------------------------
|
|
@@ -143,6 +159,14 @@ export interface RAGConfig {
|
|
|
143
159
|
chunkSize?: number;
|
|
144
160
|
/** Overlap between adjacent chunks in tokens */
|
|
145
161
|
chunkOverlap?: number;
|
|
162
|
+
/** Characters used to split text, in order of priority */
|
|
163
|
+
separators?: string[];
|
|
164
|
+
/** Whether to use query transformation (e.g. HyDE) */
|
|
165
|
+
useQueryTransformation?: boolean;
|
|
166
|
+
/** Whether to use graph-based retrieval */
|
|
167
|
+
useGraphRetrieval?: boolean;
|
|
168
|
+
/** Whether to perform reranking on retrieved results */
|
|
169
|
+
useReranking?: boolean;
|
|
146
170
|
}
|
|
147
171
|
|
|
148
172
|
// ---------------------------------------------------------------------------
|
|
@@ -170,4 +194,7 @@ export interface RagConfig {
|
|
|
170
194
|
|
|
171
195
|
/** Optional RAG pipeline tuning knobs */
|
|
172
196
|
rag?: RAGConfig;
|
|
197
|
+
|
|
198
|
+
/** Optional Graph database configuration */
|
|
199
|
+
graphDb?: GraphDBConfig;
|
|
173
200
|
}
|