@retrivora-ai/rag-engine 0.4.2 → 0.4.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{DocumentChunker-BICIjSuG.d.mts → DocumentChunker-3yElxTO3.d.mts} +9 -2
- package/dist/{DocumentChunker-BICIjSuG.d.ts → DocumentChunker-3yElxTO3.d.ts} +9 -2
- package/dist/{RagConfig-CVt24lbC.d.mts → RagConfig-BgRDL9Vy.d.mts} +37 -1
- package/dist/{RagConfig-CVt24lbC.d.ts → RagConfig-BgRDL9Vy.d.ts} +37 -1
- package/dist/SimpleGraphProvider-M6T7SE7D.mjs +62 -0
- package/dist/{chunk-2RLRKGHG.mjs → chunk-OKY5P6RA.mjs} +182 -43
- package/dist/handlers/index.d.mts +2 -2
- package/dist/handlers/index.d.ts +2 -2
- package/dist/handlers/index.js +257 -43
- package/dist/handlers/index.mjs +1 -1
- package/dist/{index-OI2--lvT.d.mts → index-7qeLTPBL.d.mts} +1 -1
- package/dist/{index-D1hoNXMT.d.ts → index-DowY4_K0.d.ts} +1 -1
- package/dist/index.d.mts +15 -5
- package/dist/index.d.ts +15 -5
- package/dist/index.js +161 -1
- package/dist/index.mjs +158 -1
- package/dist/server.d.mts +52 -12
- package/dist/server.d.ts +52 -12
- package/dist/server.js +257 -43
- package/dist/server.mjs +1 -1
- package/package.json +1 -1
- package/src/components/ConfigProvider.tsx +1 -0
- package/src/components/DocumentUpload.tsx +192 -0
- package/src/config/RagConfig.ts +27 -0
- package/src/config/constants.ts +7 -0
- package/src/config/serverConfig.ts +1 -0
- package/src/core/Pipeline.ts +71 -10
- package/src/core/ProviderRegistry.ts +40 -8
- package/src/index.ts +1 -0
- package/src/llm/providers/OllamaProvider.ts +3 -2
- package/src/providers/graphdb/BaseGraphProvider.ts +43 -0
- package/src/providers/graphdb/SimpleGraphProvider.ts +66 -0
- package/src/rag/DocumentChunker.ts +77 -34
- package/src/rag/EntityExtractor.ts +43 -0
- package/src/types/index.ts +19 -0
package/dist/handlers/index.js
CHANGED
|
@@ -1193,6 +1193,81 @@ var init_UniversalVectorProvider = __esm({
|
|
|
1193
1193
|
}
|
|
1194
1194
|
});
|
|
1195
1195
|
|
|
1196
|
+
// src/providers/graphdb/BaseGraphProvider.ts
|
|
1197
|
+
var BaseGraphProvider;
|
|
1198
|
+
var init_BaseGraphProvider = __esm({
|
|
1199
|
+
"src/providers/graphdb/BaseGraphProvider.ts"() {
|
|
1200
|
+
"use strict";
|
|
1201
|
+
BaseGraphProvider = class {
|
|
1202
|
+
constructor(config) {
|
|
1203
|
+
this.config = config;
|
|
1204
|
+
}
|
|
1205
|
+
};
|
|
1206
|
+
}
|
|
1207
|
+
});
|
|
1208
|
+
|
|
1209
|
+
// src/providers/graphdb/SimpleGraphProvider.ts
|
|
1210
|
+
var SimpleGraphProvider_exports = {};
|
|
1211
|
+
__export(SimpleGraphProvider_exports, {
|
|
1212
|
+
SimpleGraphProvider: () => SimpleGraphProvider
|
|
1213
|
+
});
|
|
1214
|
+
var SimpleGraphProvider;
|
|
1215
|
+
var init_SimpleGraphProvider = __esm({
|
|
1216
|
+
"src/providers/graphdb/SimpleGraphProvider.ts"() {
|
|
1217
|
+
"use strict";
|
|
1218
|
+
init_BaseGraphProvider();
|
|
1219
|
+
SimpleGraphProvider = class extends BaseGraphProvider {
|
|
1220
|
+
constructor() {
|
|
1221
|
+
super(...arguments);
|
|
1222
|
+
this.nodes = /* @__PURE__ */ new Map();
|
|
1223
|
+
this.edges = [];
|
|
1224
|
+
}
|
|
1225
|
+
async initialize() {
|
|
1226
|
+
console.log("[SimpleGraphProvider] Initialised in-memory graph store.");
|
|
1227
|
+
}
|
|
1228
|
+
async addNodes(nodes) {
|
|
1229
|
+
for (const node of nodes) {
|
|
1230
|
+
this.nodes.set(node.id, node);
|
|
1231
|
+
}
|
|
1232
|
+
}
|
|
1233
|
+
async addEdges(edges) {
|
|
1234
|
+
this.edges.push(...edges);
|
|
1235
|
+
}
|
|
1236
|
+
async query(queryText, limit = 5) {
|
|
1237
|
+
const q = queryText.toLowerCase();
|
|
1238
|
+
const matchedNodes = Array.from(this.nodes.values()).filter(
|
|
1239
|
+
(node) => node.id.toLowerCase().includes(q) || node.label.toLowerCase().includes(q) || JSON.stringify(node.properties).toLowerCase().includes(q)
|
|
1240
|
+
).slice(0, limit);
|
|
1241
|
+
const matchedNodeIds = new Set(matchedNodes.map((n) => n.id));
|
|
1242
|
+
const matchedEdges = this.edges.filter(
|
|
1243
|
+
(edge) => matchedNodeIds.has(edge.source) || matchedNodeIds.has(edge.target)
|
|
1244
|
+
);
|
|
1245
|
+
for (const edge of matchedEdges) {
|
|
1246
|
+
if (!matchedNodeIds.has(edge.source)) {
|
|
1247
|
+
const source = this.nodes.get(edge.source);
|
|
1248
|
+
if (source) matchedNodes.push(source);
|
|
1249
|
+
}
|
|
1250
|
+
if (!matchedNodeIds.has(edge.target)) {
|
|
1251
|
+
const target = this.nodes.get(edge.target);
|
|
1252
|
+
if (target) matchedNodes.push(target);
|
|
1253
|
+
}
|
|
1254
|
+
}
|
|
1255
|
+
return {
|
|
1256
|
+
nodes: Array.from(new Set(matchedNodes)),
|
|
1257
|
+
edges: matchedEdges
|
|
1258
|
+
};
|
|
1259
|
+
}
|
|
1260
|
+
async ping() {
|
|
1261
|
+
return true;
|
|
1262
|
+
}
|
|
1263
|
+
async disconnect() {
|
|
1264
|
+
this.nodes.clear();
|
|
1265
|
+
this.edges = [];
|
|
1266
|
+
}
|
|
1267
|
+
};
|
|
1268
|
+
}
|
|
1269
|
+
});
|
|
1270
|
+
|
|
1196
1271
|
// src/handlers/index.ts
|
|
1197
1272
|
var handlers_exports = {};
|
|
1198
1273
|
__export(handlers_exports, {
|
|
@@ -1273,7 +1348,7 @@ function readEnum(env, name, fallback, allowed) {
|
|
|
1273
1348
|
throw new Error(`[getRagConfig] ${name} must be one of: ${allowed.join(", ")}`);
|
|
1274
1349
|
}
|
|
1275
1350
|
function getRagConfig(env = process.env) {
|
|
1276
|
-
var _a, _b, _c, _d, _e, _f, _g, _h, _i, _j, _k, _l, _m, _n, _o, _p, _q, _r, _s, _t, _u, _v, _w, _x, _y, _z, _A, _B, _C, _D, _E, _F, _G, _H, _I, _J, _K, _L, _M, _N, _O;
|
|
1351
|
+
var _a, _b, _c, _d, _e, _f, _g, _h, _i, _j, _k, _l, _m, _n, _o, _p, _q, _r, _s, _t, _u, _v, _w, _x, _y, _z, _A, _B, _C, _D, _E, _F, _G, _H, _I, _J, _K, _L, _M, _N, _O, _P, _Q;
|
|
1277
1352
|
const projectId = (_b = (_a = readString(env, "RAG_PROJECT_ID")) != null ? _a : readString(env, "NEXT_PUBLIC_PROJECT_ID")) != null ? _b : "__default__";
|
|
1278
1353
|
const vectorProvider = readEnum(env, "VECTOR_DB_PROVIDER", "pinecone", VECTOR_DB_PROVIDERS);
|
|
1279
1354
|
const llmProvider = readEnum(env, "LLM_PROVIDER", "openai", LLM_PROVIDERS);
|
|
@@ -1370,7 +1445,8 @@ function getRagConfig(env = process.env) {
|
|
|
1370
1445
|
showSources: ((_I = (_H = readString(env, "NEXT_PUBLIC_SHOW_SOURCES")) != null ? _H : readString(env, "UI_SHOW_SOURCES")) != null ? _I : "true") !== "false",
|
|
1371
1446
|
welcomeMessage: (_K = (_J = readString(env, "NEXT_PUBLIC_WELCOME_MESSAGE")) != null ? _J : readString(env, "UI_WELCOME_MESSAGE")) != null ? _K : "Hello! I'm your AI assistant. Ask me anything about your documents.",
|
|
1372
1447
|
visualStyle: (_M = (_L = readString(env, "NEXT_PUBLIC_UI_VISUAL_STYLE")) != null ? _L : readString(env, "UI_VISUAL_STYLE")) != null ? _M : "glass",
|
|
1373
|
-
borderRadius: (_O = (_N = readString(env, "NEXT_PUBLIC_UI_BORDER_RADIUS")) != null ? _N : readString(env, "UI_BORDER_RADIUS")) != null ? _O : "xl"
|
|
1448
|
+
borderRadius: (_O = (_N = readString(env, "NEXT_PUBLIC_UI_BORDER_RADIUS")) != null ? _N : readString(env, "UI_BORDER_RADIUS")) != null ? _O : "xl",
|
|
1449
|
+
allowUpload: ((_Q = (_P = readString(env, "NEXT_PUBLIC_ALLOW_UPLOAD")) != null ? _P : readString(env, "UI_ALLOW_UPLOAD")) != null ? _Q : "false") === "true"
|
|
1374
1450
|
},
|
|
1375
1451
|
rag: {
|
|
1376
1452
|
topK: readNumber(env, "RAG_TOP_K", 5),
|
|
@@ -1944,52 +2020,84 @@ var ConfigValidator = class {
|
|
|
1944
2020
|
|
|
1945
2021
|
// src/rag/DocumentChunker.ts
|
|
1946
2022
|
var DocumentChunker = class {
|
|
1947
|
-
constructor(chunkSize = 1e3, chunkOverlap = 200) {
|
|
2023
|
+
constructor(chunkSize = 1e3, chunkOverlap = 200, separators = ["\n\n", "\n", " ", ""]) {
|
|
1948
2024
|
this.chunkSize = chunkSize;
|
|
1949
2025
|
this.chunkOverlap = chunkOverlap;
|
|
2026
|
+
this.separators = separators;
|
|
1950
2027
|
}
|
|
1951
2028
|
/**
|
|
1952
|
-
* Split a single text string into overlapping chunks.
|
|
2029
|
+
* Split a single text string into overlapping chunks using a recursive strategy.
|
|
1953
2030
|
*/
|
|
1954
2031
|
chunk(text, options = {}) {
|
|
1955
2032
|
const {
|
|
1956
2033
|
chunkSize = this.chunkSize,
|
|
1957
2034
|
chunkOverlap = this.chunkOverlap,
|
|
1958
2035
|
docId = `doc_${Date.now()}`,
|
|
1959
|
-
metadata = {}
|
|
2036
|
+
metadata = {},
|
|
2037
|
+
separators = this.separators
|
|
1960
2038
|
} = options;
|
|
1961
|
-
const
|
|
1962
|
-
|
|
1963
|
-
|
|
1964
|
-
|
|
1965
|
-
let current = "";
|
|
2039
|
+
const finalChunks = [];
|
|
2040
|
+
const splits = this.recursiveSplit(text, separators, chunkSize);
|
|
2041
|
+
let currentChunk = [];
|
|
2042
|
+
let currentLength = 0;
|
|
1966
2043
|
let chunkIndex = 0;
|
|
1967
|
-
for (const
|
|
1968
|
-
if (
|
|
1969
|
-
|
|
1970
|
-
|
|
1971
|
-
|
|
1972
|
-
|
|
1973
|
-
|
|
1974
|
-
|
|
1975
|
-
|
|
1976
|
-
|
|
1977
|
-
|
|
1978
|
-
|
|
1979
|
-
|
|
1980
|
-
|
|
1981
|
-
|
|
2044
|
+
for (const split of splits) {
|
|
2045
|
+
if (currentLength + split.length > chunkSize && currentChunk.length > 0) {
|
|
2046
|
+
finalChunks.push({
|
|
2047
|
+
id: `${docId}_chunk_${chunkIndex++}`,
|
|
2048
|
+
content: currentChunk.join("").trim(),
|
|
2049
|
+
metadata: __spreadProps(__spreadValues({}, metadata), { docId, chunkIndex: chunkIndex - 1 })
|
|
2050
|
+
});
|
|
2051
|
+
const overlapItems = [];
|
|
2052
|
+
let overlapLen = 0;
|
|
2053
|
+
for (let i = currentChunk.length - 1; i >= 0; i--) {
|
|
2054
|
+
if (overlapLen + currentChunk[i].length <= chunkOverlap) {
|
|
2055
|
+
overlapItems.unshift(currentChunk[i]);
|
|
2056
|
+
overlapLen += currentChunk[i].length;
|
|
2057
|
+
} else {
|
|
2058
|
+
break;
|
|
2059
|
+
}
|
|
1982
2060
|
}
|
|
2061
|
+
currentChunk = overlapItems;
|
|
2062
|
+
currentLength = overlapLen;
|
|
1983
2063
|
}
|
|
2064
|
+
currentChunk.push(split);
|
|
2065
|
+
currentLength += split.length;
|
|
1984
2066
|
}
|
|
1985
|
-
if (
|
|
1986
|
-
|
|
2067
|
+
if (currentChunk.length > 0) {
|
|
2068
|
+
finalChunks.push({
|
|
1987
2069
|
id: `${docId}_chunk_${chunkIndex}`,
|
|
1988
|
-
content:
|
|
2070
|
+
content: currentChunk.join("").trim(),
|
|
1989
2071
|
metadata: __spreadProps(__spreadValues({}, metadata), { docId, chunkIndex })
|
|
1990
2072
|
});
|
|
1991
2073
|
}
|
|
1992
|
-
return
|
|
2074
|
+
return finalChunks;
|
|
2075
|
+
}
|
|
2076
|
+
/**
|
|
2077
|
+
* Recursively split text based on separators.
|
|
2078
|
+
*/
|
|
2079
|
+
recursiveSplit(text, separators, chunkSize) {
|
|
2080
|
+
const finalSplits = [];
|
|
2081
|
+
let separator = separators[separators.length - 1];
|
|
2082
|
+
let nextSeparators = [];
|
|
2083
|
+
for (let i = 0; i < separators.length; i++) {
|
|
2084
|
+
if (text.includes(separators[i])) {
|
|
2085
|
+
separator = separators[i];
|
|
2086
|
+
nextSeparators = separators.slice(i + 1);
|
|
2087
|
+
break;
|
|
2088
|
+
}
|
|
2089
|
+
}
|
|
2090
|
+
const parts = text.split(separator);
|
|
2091
|
+
for (const part of parts) {
|
|
2092
|
+
if (part.length <= chunkSize) {
|
|
2093
|
+
finalSplits.push(part + separator);
|
|
2094
|
+
} else if (nextSeparators.length > 0) {
|
|
2095
|
+
finalSplits.push(...this.recursiveSplit(part, nextSeparators, chunkSize));
|
|
2096
|
+
} else {
|
|
2097
|
+
finalSplits.push(part);
|
|
2098
|
+
}
|
|
2099
|
+
}
|
|
2100
|
+
return finalSplits;
|
|
1993
2101
|
}
|
|
1994
2102
|
/**
|
|
1995
2103
|
* Chunk multiple documents at once.
|
|
@@ -2001,6 +2109,39 @@ var DocumentChunker = class {
|
|
|
2001
2109
|
}
|
|
2002
2110
|
};
|
|
2003
2111
|
|
|
2112
|
+
// src/rag/EntityExtractor.ts
|
|
2113
|
+
var EntityExtractor = class {
|
|
2114
|
+
constructor(llm) {
|
|
2115
|
+
this.llm = llm;
|
|
2116
|
+
}
|
|
2117
|
+
/**
|
|
2118
|
+
* Extract nodes and edges from a text chunk.
|
|
2119
|
+
*/
|
|
2120
|
+
async extract(text) {
|
|
2121
|
+
const prompt = `
|
|
2122
|
+
Extract entities and relationships from the following text.
|
|
2123
|
+
Format the output as JSON with two keys: "nodes" (array of {id, label, properties}) and "edges" (array of {source, target, type, properties}).
|
|
2124
|
+
Use the same ID for the same entity.
|
|
2125
|
+
|
|
2126
|
+
Text:
|
|
2127
|
+
"${text}"
|
|
2128
|
+
|
|
2129
|
+
Output JSON:
|
|
2130
|
+
`;
|
|
2131
|
+
const response = await this.llm.chat([
|
|
2132
|
+
{ role: "system", content: "You are an expert at knowledge graph extraction. Respond only with valid JSON." },
|
|
2133
|
+
{ role: "user", content: prompt }
|
|
2134
|
+
], "");
|
|
2135
|
+
try {
|
|
2136
|
+
const cleanJson = response.replace(/```json\n?|\n?```/g, "").trim();
|
|
2137
|
+
return JSON.parse(cleanJson);
|
|
2138
|
+
} catch (error) {
|
|
2139
|
+
console.warn("[EntityExtractor] Failed to parse LLM response as JSON:", response);
|
|
2140
|
+
return { nodes: [], edges: [] };
|
|
2141
|
+
}
|
|
2142
|
+
}
|
|
2143
|
+
};
|
|
2144
|
+
|
|
2004
2145
|
// src/llm/providers/OpenAIProvider.ts
|
|
2005
2146
|
var import_openai = __toESM(require("openai"));
|
|
2006
2147
|
var OpenAIProvider = class {
|
|
@@ -2159,13 +2300,13 @@ ${context}`;
|
|
|
2159
2300
|
return data.message.content;
|
|
2160
2301
|
}
|
|
2161
2302
|
async embed(text, options) {
|
|
2162
|
-
var _a, _b, _c, _d, _e, _f, _g, _h, _i;
|
|
2303
|
+
var _a, _b, _c, _d, _e, _f, _g, _h, _i, _j, _k;
|
|
2163
2304
|
const model = (_c = (_b = options == null ? void 0 : options.model) != null ? _b : (_a = this.embeddingConfig) == null ? void 0 : _a.model) != null ? _c : "nomic-embed-text";
|
|
2164
2305
|
const baseURL = (_f = (_e = (_d = this.embeddingConfig) == null ? void 0 : _d.baseUrl) != null ? _e : this.llmConfig.baseUrl) != null ? _f : "http://localhost:11434";
|
|
2165
2306
|
const client = baseURL !== ((_g = this.llmConfig.baseUrl) != null ? _g : "http://localhost:11434") ? import_axios.default.create({ baseURL, timeout: 6e4 }) : this.http;
|
|
2166
2307
|
let prompt = text;
|
|
2167
|
-
const queryPrefix = (_h = this.embeddingConfig) == null ? void 0 : _h.queryPrefix;
|
|
2168
|
-
const docPrefix = (
|
|
2308
|
+
const queryPrefix = (_i = (_h = this.embeddingConfig) == null ? void 0 : _h.queryPrefix) != null ? _i : model.includes("nomic") ? "search_query: " : "";
|
|
2309
|
+
const docPrefix = (_k = (_j = this.embeddingConfig) == null ? void 0 : _j.documentPrefix) != null ? _k : model.includes("nomic") ? "search_document: " : "";
|
|
2169
2310
|
if ((options == null ? void 0 : options.taskType) === "query" && queryPrefix) {
|
|
2170
2311
|
if (!prompt.startsWith(queryPrefix)) {
|
|
2171
2312
|
prompt = `${queryPrefix}${text}`;
|
|
@@ -2506,16 +2647,15 @@ var LLMFactory = class _LLMFactory {
|
|
|
2506
2647
|
|
|
2507
2648
|
// src/core/ProviderRegistry.ts
|
|
2508
2649
|
var ProviderRegistry = class {
|
|
2509
|
-
/**
|
|
2510
|
-
* Register a custom vector provider class by name.
|
|
2511
|
-
* The name must match the provider value used in VectorDBConfig.provider.
|
|
2512
|
-
*
|
|
2513
|
-
* @example
|
|
2514
|
-
* ProviderRegistry.registerVectorProvider('my-db', MyCustomProvider);
|
|
2515
|
-
*/
|
|
2516
2650
|
static registerVectorProvider(name, providerClass) {
|
|
2517
2651
|
this.vectorProviders[name] = providerClass;
|
|
2518
2652
|
}
|
|
2653
|
+
/**
|
|
2654
|
+
* Register a custom graph provider class by name.
|
|
2655
|
+
*/
|
|
2656
|
+
static registerGraphProvider(name, providerClass) {
|
|
2657
|
+
this.graphProviders[name] = providerClass;
|
|
2658
|
+
}
|
|
2519
2659
|
/**
|
|
2520
2660
|
* Creates a vector database provider based on the configuration.
|
|
2521
2661
|
* Built-in providers are dynamically imported to avoid bundling all SDKs.
|
|
@@ -2570,6 +2710,28 @@ var ProviderRegistry = class {
|
|
|
2570
2710
|
);
|
|
2571
2711
|
}
|
|
2572
2712
|
}
|
|
2713
|
+
/**
|
|
2714
|
+
* Creates a graph database provider based on the configuration.
|
|
2715
|
+
*/
|
|
2716
|
+
static async createGraphProvider(config) {
|
|
2717
|
+
const { provider } = config;
|
|
2718
|
+
if (this.graphProviders[provider]) {
|
|
2719
|
+
return new this.graphProviders[provider](config);
|
|
2720
|
+
}
|
|
2721
|
+
switch (provider) {
|
|
2722
|
+
case "neo4j": {
|
|
2723
|
+
throw new Error("[ProviderRegistry] Neo4j provider not implemented yet.");
|
|
2724
|
+
}
|
|
2725
|
+
case "simple": {
|
|
2726
|
+
const { SimpleGraphProvider: SimpleGraphProvider2 } = await Promise.resolve().then(() => (init_SimpleGraphProvider(), SimpleGraphProvider_exports));
|
|
2727
|
+
return new SimpleGraphProvider2(config);
|
|
2728
|
+
}
|
|
2729
|
+
default:
|
|
2730
|
+
throw new Error(
|
|
2731
|
+
`[ProviderRegistry] Unsupported graph provider: "${provider}". Built-in providers: simple. For custom providers, call ProviderRegistry.registerGraphProvider("${provider}", YourClass).`
|
|
2732
|
+
);
|
|
2733
|
+
}
|
|
2734
|
+
}
|
|
2573
2735
|
/**
|
|
2574
2736
|
* Creates an LLM provider based on the configuration.
|
|
2575
2737
|
*/
|
|
@@ -2578,6 +2740,7 @@ var ProviderRegistry = class {
|
|
|
2578
2740
|
}
|
|
2579
2741
|
};
|
|
2580
2742
|
ProviderRegistry.vectorProviders = {};
|
|
2743
|
+
ProviderRegistry.graphProviders = {};
|
|
2581
2744
|
|
|
2582
2745
|
// src/core/BatchProcessor.ts
|
|
2583
2746
|
function isTransientError(error) {
|
|
@@ -3003,6 +3166,11 @@ var Pipeline = class {
|
|
|
3003
3166
|
);
|
|
3004
3167
|
this.llmProvider = llmProvider;
|
|
3005
3168
|
this.embeddingProvider = embeddingProvider;
|
|
3169
|
+
if (this.config.graphDb) {
|
|
3170
|
+
this.graphDB = await ProviderRegistry.createGraphProvider(this.config.graphDb);
|
|
3171
|
+
await this.graphDB.initialize();
|
|
3172
|
+
this.entityExtractor = new EntityExtractor(this.llmProvider);
|
|
3173
|
+
}
|
|
3006
3174
|
await this.vectorDB.initialize();
|
|
3007
3175
|
this.initialised = true;
|
|
3008
3176
|
}
|
|
@@ -3056,6 +3224,14 @@ var Pipeline = class {
|
|
|
3056
3224
|
docId: doc.docId,
|
|
3057
3225
|
chunksIngested: upsertResult.totalProcessed
|
|
3058
3226
|
});
|
|
3227
|
+
if (this.graphDB && this.entityExtractor) {
|
|
3228
|
+
console.log(`[Pipeline] Extracting entities for doc ${doc.docId}...`);
|
|
3229
|
+
for (const chunk of chunks) {
|
|
3230
|
+
const { nodes, edges } = await this.entityExtractor.extract(chunk.content);
|
|
3231
|
+
if (nodes.length > 0) await this.graphDB.addNodes(nodes);
|
|
3232
|
+
if (edges.length > 0) await this.graphDB.addEdges(edges);
|
|
3233
|
+
}
|
|
3234
|
+
}
|
|
3059
3235
|
} catch (error) {
|
|
3060
3236
|
console.error(`[Pipeline] Failed to ingest document ${doc.docId}:`, error);
|
|
3061
3237
|
results.push({ docId: doc.docId, chunksIngested: 0 });
|
|
@@ -3064,27 +3240,65 @@ var Pipeline = class {
|
|
|
3064
3240
|
return results;
|
|
3065
3241
|
}
|
|
3066
3242
|
async ask(question, history = [], namespace) {
|
|
3067
|
-
var _a, _b, _c, _d;
|
|
3243
|
+
var _a, _b, _c, _d, _e, _f;
|
|
3068
3244
|
await this.initialize();
|
|
3069
3245
|
const ns = namespace != null ? namespace : this.config.projectId;
|
|
3070
3246
|
const topK = (_b = (_a = this.config.rag) == null ? void 0 : _a.topK) != null ? _b : 5;
|
|
3071
3247
|
const scoreThreshold = (_d = (_c = this.config.rag) == null ? void 0 : _c.scoreThreshold) != null ? _d : 0;
|
|
3072
3248
|
try {
|
|
3073
|
-
|
|
3249
|
+
let searchQuery = question;
|
|
3250
|
+
if ((_e = this.config.rag) == null ? void 0 : _e.useQueryTransformation) {
|
|
3251
|
+
searchQuery = await this.rewriteQuery(question, history);
|
|
3252
|
+
}
|
|
3253
|
+
const queryVector = await this.embeddingProvider.embed(searchQuery, { taskType: "query" });
|
|
3074
3254
|
const fieldHints = extractQueryFieldHints(question);
|
|
3075
3255
|
const filter = buildQueryFilter(question, fieldHints);
|
|
3076
3256
|
filter.__entityHints = fieldHints;
|
|
3077
3257
|
const rawMatches = await this.vectorDB.query(queryVector, topK, ns, filter);
|
|
3078
3258
|
const sources = rawMatches.filter((m) => m.score >= scoreThreshold);
|
|
3079
|
-
|
|
3259
|
+
let graphData;
|
|
3260
|
+
if (this.graphDB && ((_f = this.config.rag) == null ? void 0 : _f.useGraphRetrieval)) {
|
|
3261
|
+
graphData = await this.graphDB.query(searchQuery);
|
|
3262
|
+
}
|
|
3263
|
+
let context = sources.length ? sources.map((m, i) => `[Source ${i + 1}]
|
|
3080
3264
|
${m.content}`).join("\n\n---\n\n") : "No relevant context found.";
|
|
3265
|
+
if (graphData && graphData.nodes.length > 0) {
|
|
3266
|
+
const graphContext = graphData.nodes.map(
|
|
3267
|
+
(n) => `Entity: ${n.label} (${n.id})${n.properties ? " - " + JSON.stringify(n.properties) : ""}`
|
|
3268
|
+
).join("\n");
|
|
3269
|
+
context = `GRAPH KNOWLEDGE:
|
|
3270
|
+
${graphContext}
|
|
3271
|
+
|
|
3272
|
+
VECTOR CONTEXT:
|
|
3273
|
+
${context}`;
|
|
3274
|
+
}
|
|
3081
3275
|
const messages = [...history, { role: "user", content: question }];
|
|
3082
3276
|
const reply = await this.llmProvider.chat(messages, context);
|
|
3083
|
-
return { reply, sources };
|
|
3277
|
+
return { reply, sources, graphData };
|
|
3084
3278
|
} catch (error) {
|
|
3085
3279
|
throw new Error(`[Pipeline] Chat failed: ${error instanceof Error ? error.message : String(error)}`);
|
|
3086
3280
|
}
|
|
3087
3281
|
}
|
|
3282
|
+
/**
|
|
3283
|
+
* Rewrite the user query for better retrieval performance.
|
|
3284
|
+
*/
|
|
3285
|
+
async rewriteQuery(question, history) {
|
|
3286
|
+
const prompt = `
|
|
3287
|
+
Given the following conversation history and a new question, rewrite the question to be a better search query for a vector database.
|
|
3288
|
+
Focus on extracting the core intent and entities. Do not answer the question, just rewrite it.
|
|
3289
|
+
|
|
3290
|
+
History:
|
|
3291
|
+
${history.map((m) => `${m.role}: ${m.content}`).join("\n")}
|
|
3292
|
+
|
|
3293
|
+
New Question: ${question}
|
|
3294
|
+
|
|
3295
|
+
Optimized Search Query:`;
|
|
3296
|
+
const rewrite = await this.llmProvider.chat([
|
|
3297
|
+
{ role: "system", content: "You are an assistant that optimizes search queries for RAG systems." },
|
|
3298
|
+
{ role: "user", content: prompt }
|
|
3299
|
+
], "");
|
|
3300
|
+
return rewrite.trim() || question;
|
|
3301
|
+
}
|
|
3088
3302
|
};
|
|
3089
3303
|
|
|
3090
3304
|
// src/core/ProviderHealthCheck.ts
|
package/dist/handlers/index.mjs
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { e as VectorDBConfig, L as LLMConfig, E as EmbeddingConfig, c as RagConfig, C as ChatResponse } from './RagConfig-
|
|
1
|
+
import { e as VectorDBConfig, L as LLMConfig, E as EmbeddingConfig, c as RagConfig, C as ChatResponse } from './RagConfig-BgRDL9Vy.mjs';
|
|
2
2
|
import { NextRequest, NextResponse } from 'next/server';
|
|
3
3
|
|
|
4
4
|
/**
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { e as VectorDBConfig, L as LLMConfig, E as EmbeddingConfig, c as RagConfig, C as ChatResponse } from './RagConfig-
|
|
1
|
+
import { e as VectorDBConfig, L as LLMConfig, E as EmbeddingConfig, c as RagConfig, C as ChatResponse } from './RagConfig-BgRDL9Vy.js';
|
|
2
2
|
import { NextRequest, NextResponse } from 'next/server';
|
|
3
3
|
|
|
4
4
|
/**
|
package/dist/index.d.mts
CHANGED
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
import * as react_jsx_runtime from 'react/jsx-runtime';
|
|
2
2
|
import React$1, { ReactNode } from 'react';
|
|
3
|
-
import { C as ChatMessage } from './DocumentChunker-
|
|
4
|
-
export { a as ChatOptions, b as Chunk, c as ChunkOptions, E as EmbedOptions, I as ILLMProvider } from './DocumentChunker-
|
|
5
|
-
import { V as VectorMatch, U as UIConfig } from './RagConfig-
|
|
6
|
-
export { C as ChatResponse, E as EmbeddingConfig, a as EmbeddingProvider, I as IngestDocument, L as LLMConfig, b as LLMProvider, R as RAGConfig, c as RagConfig, d as UpsertDocument, e as VectorDBConfig, f as VectorDBProvider } from './RagConfig-
|
|
3
|
+
import { C as ChatMessage } from './DocumentChunker-3yElxTO3.mjs';
|
|
4
|
+
export { a as ChatOptions, b as Chunk, c as ChunkOptions, E as EmbedOptions, I as ILLMProvider } from './DocumentChunker-3yElxTO3.mjs';
|
|
5
|
+
import { V as VectorMatch, U as UIConfig } from './RagConfig-BgRDL9Vy.mjs';
|
|
6
|
+
export { C as ChatResponse, E as EmbeddingConfig, a as EmbeddingProvider, I as IngestDocument, L as LLMConfig, b as LLMProvider, R as RAGConfig, c as RagConfig, d as UpsertDocument, e as VectorDBConfig, f as VectorDBProvider } from './RagConfig-BgRDL9Vy.mjs';
|
|
7
7
|
|
|
8
8
|
interface ChatWidgetProps {
|
|
9
9
|
/** Position of the floating button. Defaults to bottom-right. */
|
|
@@ -23,6 +23,16 @@ interface ChatWindowProps {
|
|
|
23
23
|
}
|
|
24
24
|
declare function ChatWindow({ className, style, onClose, showClose }: ChatWindowProps): react_jsx_runtime.JSX.Element;
|
|
25
25
|
|
|
26
|
+
interface DocumentUploadProps {
|
|
27
|
+
/** Optional namespace for the upload */
|
|
28
|
+
namespace?: string;
|
|
29
|
+
/** Callback when upload completes */
|
|
30
|
+
onUploadComplete?: (results: any) => void;
|
|
31
|
+
/** Additional className */
|
|
32
|
+
className?: string;
|
|
33
|
+
}
|
|
34
|
+
declare function DocumentUpload({ namespace, onUploadComplete, className }: DocumentUploadProps): react_jsx_runtime.JSX.Element;
|
|
35
|
+
|
|
26
36
|
interface MessageBubbleProps {
|
|
27
37
|
message: ChatMessage;
|
|
28
38
|
sources?: VectorMatch[];
|
|
@@ -91,4 +101,4 @@ interface UseRagChatReturn {
|
|
|
91
101
|
}
|
|
92
102
|
declare function useRagChat(projectId: string, options?: UseRagChatOptions): UseRagChatReturn;
|
|
93
103
|
|
|
94
|
-
export { ChatMessage, ChatWidget, ChatWindow, type ClientConfig, ConfigProvider, MessageBubble, type RagMessage, SourceCard, UIConfig, type UseRagChatOptions, type UseRagChatReturn, VectorMatch, useConfig, useRagChat };
|
|
104
|
+
export { ChatMessage, ChatWidget, ChatWindow, type ClientConfig, ConfigProvider, DocumentUpload, MessageBubble, type RagMessage, SourceCard, UIConfig, type UseRagChatOptions, type UseRagChatReturn, VectorMatch, useConfig, useRagChat };
|
package/dist/index.d.ts
CHANGED
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
import * as react_jsx_runtime from 'react/jsx-runtime';
|
|
2
2
|
import React$1, { ReactNode } from 'react';
|
|
3
|
-
import { C as ChatMessage } from './DocumentChunker-
|
|
4
|
-
export { a as ChatOptions, b as Chunk, c as ChunkOptions, E as EmbedOptions, I as ILLMProvider } from './DocumentChunker-
|
|
5
|
-
import { V as VectorMatch, U as UIConfig } from './RagConfig-
|
|
6
|
-
export { C as ChatResponse, E as EmbeddingConfig, a as EmbeddingProvider, I as IngestDocument, L as LLMConfig, b as LLMProvider, R as RAGConfig, c as RagConfig, d as UpsertDocument, e as VectorDBConfig, f as VectorDBProvider } from './RagConfig-
|
|
3
|
+
import { C as ChatMessage } from './DocumentChunker-3yElxTO3.js';
|
|
4
|
+
export { a as ChatOptions, b as Chunk, c as ChunkOptions, E as EmbedOptions, I as ILLMProvider } from './DocumentChunker-3yElxTO3.js';
|
|
5
|
+
import { V as VectorMatch, U as UIConfig } from './RagConfig-BgRDL9Vy.js';
|
|
6
|
+
export { C as ChatResponse, E as EmbeddingConfig, a as EmbeddingProvider, I as IngestDocument, L as LLMConfig, b as LLMProvider, R as RAGConfig, c as RagConfig, d as UpsertDocument, e as VectorDBConfig, f as VectorDBProvider } from './RagConfig-BgRDL9Vy.js';
|
|
7
7
|
|
|
8
8
|
interface ChatWidgetProps {
|
|
9
9
|
/** Position of the floating button. Defaults to bottom-right. */
|
|
@@ -23,6 +23,16 @@ interface ChatWindowProps {
|
|
|
23
23
|
}
|
|
24
24
|
declare function ChatWindow({ className, style, onClose, showClose }: ChatWindowProps): react_jsx_runtime.JSX.Element;
|
|
25
25
|
|
|
26
|
+
interface DocumentUploadProps {
|
|
27
|
+
/** Optional namespace for the upload */
|
|
28
|
+
namespace?: string;
|
|
29
|
+
/** Callback when upload completes */
|
|
30
|
+
onUploadComplete?: (results: any) => void;
|
|
31
|
+
/** Additional className */
|
|
32
|
+
className?: string;
|
|
33
|
+
}
|
|
34
|
+
declare function DocumentUpload({ namespace, onUploadComplete, className }: DocumentUploadProps): react_jsx_runtime.JSX.Element;
|
|
35
|
+
|
|
26
36
|
interface MessageBubbleProps {
|
|
27
37
|
message: ChatMessage;
|
|
28
38
|
sources?: VectorMatch[];
|
|
@@ -91,4 +101,4 @@ interface UseRagChatReturn {
|
|
|
91
101
|
}
|
|
92
102
|
declare function useRagChat(projectId: string, options?: UseRagChatOptions): UseRagChatReturn;
|
|
93
103
|
|
|
94
|
-
export { ChatMessage, ChatWidget, ChatWindow, type ClientConfig, ConfigProvider, MessageBubble, type RagMessage, SourceCard, UIConfig, type UseRagChatOptions, type UseRagChatReturn, VectorMatch, useConfig, useRagChat };
|
|
104
|
+
export { ChatMessage, ChatWidget, ChatWindow, type ClientConfig, ConfigProvider, DocumentUpload, MessageBubble, type RagMessage, SourceCard, UIConfig, type UseRagChatOptions, type UseRagChatReturn, VectorMatch, useConfig, useRagChat };
|