@retrivora-ai/rag-engine 0.4.3 → 0.4.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (34) hide show
  1. package/dist/{DocumentChunker-BICIjSuG.d.mts → DocumentChunker-3yElxTO3.d.mts} +9 -2
  2. package/dist/{DocumentChunker-BICIjSuG.d.ts → DocumentChunker-3yElxTO3.d.ts} +9 -2
  3. package/dist/{RagConfig-CVt24lbC.d.mts → RagConfig-BgRDL9Vy.d.mts} +37 -1
  4. package/dist/{RagConfig-CVt24lbC.d.ts → RagConfig-BgRDL9Vy.d.ts} +37 -1
  5. package/dist/SimpleGraphProvider-M6T7SE7D.mjs +62 -0
  6. package/dist/{chunk-UUZ3F4WK.mjs → chunk-OKY5P6RA.mjs} +179 -40
  7. package/dist/handlers/index.d.mts +2 -2
  8. package/dist/handlers/index.d.ts +2 -2
  9. package/dist/handlers/index.js +254 -40
  10. package/dist/handlers/index.mjs +1 -1
  11. package/dist/{index-OI2--lvT.d.mts → index-7qeLTPBL.d.mts} +1 -1
  12. package/dist/{index-D1hoNXMT.d.ts → index-DowY4_K0.d.ts} +1 -1
  13. package/dist/index.d.mts +15 -5
  14. package/dist/index.d.ts +15 -5
  15. package/dist/index.js +161 -1
  16. package/dist/index.mjs +158 -1
  17. package/dist/server.d.mts +52 -12
  18. package/dist/server.d.ts +52 -12
  19. package/dist/server.js +254 -40
  20. package/dist/server.mjs +1 -1
  21. package/package.json +1 -1
  22. package/src/components/ConfigProvider.tsx +1 -0
  23. package/src/components/DocumentUpload.tsx +192 -0
  24. package/src/config/RagConfig.ts +27 -0
  25. package/src/config/constants.ts +7 -0
  26. package/src/config/serverConfig.ts +1 -0
  27. package/src/core/Pipeline.ts +71 -10
  28. package/src/core/ProviderRegistry.ts +40 -8
  29. package/src/index.ts +1 -0
  30. package/src/providers/graphdb/BaseGraphProvider.ts +43 -0
  31. package/src/providers/graphdb/SimpleGraphProvider.ts +66 -0
  32. package/src/rag/DocumentChunker.ts +77 -34
  33. package/src/rag/EntityExtractor.ts +43 -0
  34. package/src/types/index.ts +19 -0
@@ -73,15 +73,22 @@ interface ChunkOptions {
73
73
  docId?: string | number;
74
74
  /** Extra metadata to attach to every chunk */
75
75
  metadata?: Record<string, unknown>;
76
+ /** Characters used to split text, in order of priority */
77
+ separators?: string[];
76
78
  }
77
79
  declare class DocumentChunker {
78
80
  private readonly chunkSize;
79
81
  private readonly chunkOverlap;
80
- constructor(chunkSize?: number, chunkOverlap?: number);
82
+ private readonly separators;
83
+ constructor(chunkSize?: number, chunkOverlap?: number, separators?: string[]);
81
84
  /**
82
- * Split a single text string into overlapping chunks.
85
+ * Split a single text string into overlapping chunks using a recursive strategy.
83
86
  */
84
87
  chunk(text: string, options?: ChunkOptions): Chunk[];
88
+ /**
89
+ * Recursively split text based on separators.
90
+ */
91
+ private recursiveSplit;
85
92
  /**
86
93
  * Chunk multiple documents at once.
87
94
  */
@@ -73,15 +73,22 @@ interface ChunkOptions {
73
73
  docId?: string | number;
74
74
  /** Extra metadata to attach to every chunk */
75
75
  metadata?: Record<string, unknown>;
76
+ /** Characters used to split text, in order of priority */
77
+ separators?: string[];
76
78
  }
77
79
  declare class DocumentChunker {
78
80
  private readonly chunkSize;
79
81
  private readonly chunkOverlap;
80
- constructor(chunkSize?: number, chunkOverlap?: number);
82
+ private readonly separators;
83
+ constructor(chunkSize?: number, chunkOverlap?: number, separators?: string[]);
81
84
  /**
82
- * Split a single text string into overlapping chunks.
85
+ * Split a single text string into overlapping chunks using a recursive strategy.
83
86
  */
84
87
  chunk(text: string, options?: ChunkOptions): Chunk[];
88
+ /**
89
+ * Recursively split text based on separators.
90
+ */
91
+ private recursiveSplit;
85
92
  /**
86
93
  * Chunk multiple documents at once.
87
94
  */
@@ -18,6 +18,22 @@ interface IngestDocument {
18
18
  interface ChatResponse {
19
19
  reply: string;
20
20
  sources: VectorMatch[];
21
+ graphData?: GraphSearchResult;
22
+ }
23
+ interface GraphNode {
24
+ id: string;
25
+ label: string;
26
+ properties?: Record<string, unknown>;
27
+ }
28
+ interface Edge {
29
+ source: string;
30
+ target: string;
31
+ type: string;
32
+ properties?: Record<string, unknown>;
33
+ }
34
+ interface GraphSearchResult {
35
+ nodes: GraphNode[];
36
+ edges: Edge[];
21
37
  }
22
38
 
23
39
  /**
@@ -29,6 +45,7 @@ interface ChatResponse {
29
45
  declare const VECTOR_DB_PROVIDERS: readonly ["pinecone", "pgvector", "postgresql", "mongodb", "milvus", "qdrant", "chromadb", "redis", "weaviate", "rest", "universal_rest", "custom"];
30
46
  declare const LLM_PROVIDERS: readonly ["openai", "anthropic", "ollama", "gemini", "rest", "universal_rest", "custom"];
31
47
  declare const EMBEDDING_PROVIDERS: readonly ["openai", "ollama", "gemini", "rest", "universal_rest", "custom"];
48
+ declare const GRAPH_DB_PROVIDERS: readonly ["neo4j", "memgraph", "simple", "custom"];
32
49
 
33
50
  /**
34
51
  * Master configuration interface for Retrivora AI.
@@ -62,6 +79,13 @@ interface VectorDBConfig {
62
79
  */
63
80
  options: Record<string, unknown>;
64
81
  }
82
+ type GraphDBProvider = typeof GRAPH_DB_PROVIDERS[number];
83
+ interface GraphDBConfig {
84
+ /** Which graph database to use */
85
+ provider: GraphDBProvider;
86
+ /** Provider-specific options (URI, credentials, etc.) */
87
+ options: Record<string, unknown>;
88
+ }
65
89
  type LLMProvider = typeof LLM_PROVIDERS[number];
66
90
  interface LLMConfig {
67
91
  /** Which LLM provider to use */
@@ -132,6 +156,8 @@ interface UIConfig {
132
156
  visualStyle?: 'glass' | 'solid';
133
157
  /** Border radius: 'none', 'sm', 'md', 'lg', 'xl', 'full' */
134
158
  borderRadius?: 'none' | 'sm' | 'md' | 'lg' | 'xl' | 'full';
159
+ /** Whether to allow file uploads directly from the UI */
160
+ allowUpload?: boolean;
135
161
  }
136
162
  interface RAGConfig {
137
163
  /** Number of top-K chunks retrieved per query */
@@ -142,6 +168,14 @@ interface RAGConfig {
142
168
  chunkSize?: number;
143
169
  /** Overlap between adjacent chunks in tokens */
144
170
  chunkOverlap?: number;
171
+ /** Characters used to split text, in order of priority */
172
+ separators?: string[];
173
+ /** Whether to use query transformation (e.g. HyDE) */
174
+ useQueryTransformation?: boolean;
175
+ /** Whether to use graph-based retrieval */
176
+ useGraphRetrieval?: boolean;
177
+ /** Whether to perform reranking on retrieved results */
178
+ useReranking?: boolean;
145
179
  }
146
180
  interface RagConfig {
147
181
  /**
@@ -159,6 +193,8 @@ interface RagConfig {
159
193
  ui?: UIConfig;
160
194
  /** Optional RAG pipeline tuning knobs */
161
195
  rag?: RAGConfig;
196
+ /** Optional Graph database configuration */
197
+ graphDb?: GraphDBConfig;
162
198
  }
163
199
 
164
- export type { ChatResponse as C, EmbeddingConfig as E, IngestDocument as I, LLMConfig as L, RAGConfig as R, UIConfig as U, VectorMatch as V, EmbeddingProvider as a, LLMProvider as b, RagConfig as c, UpsertDocument as d, VectorDBConfig as e, VectorDBProvider as f };
200
+ export type { ChatResponse as C, EmbeddingConfig as E, GraphDBConfig as G, IngestDocument as I, LLMConfig as L, RAGConfig as R, UIConfig as U, VectorMatch as V, EmbeddingProvider as a, LLMProvider as b, RagConfig as c, UpsertDocument as d, VectorDBConfig as e, VectorDBProvider as f, GraphNode as g, Edge as h, GraphSearchResult as i };
@@ -18,6 +18,22 @@ interface IngestDocument {
18
18
  interface ChatResponse {
19
19
  reply: string;
20
20
  sources: VectorMatch[];
21
+ graphData?: GraphSearchResult;
22
+ }
23
+ interface GraphNode {
24
+ id: string;
25
+ label: string;
26
+ properties?: Record<string, unknown>;
27
+ }
28
+ interface Edge {
29
+ source: string;
30
+ target: string;
31
+ type: string;
32
+ properties?: Record<string, unknown>;
33
+ }
34
+ interface GraphSearchResult {
35
+ nodes: GraphNode[];
36
+ edges: Edge[];
21
37
  }
22
38
 
23
39
  /**
@@ -29,6 +45,7 @@ interface ChatResponse {
29
45
  declare const VECTOR_DB_PROVIDERS: readonly ["pinecone", "pgvector", "postgresql", "mongodb", "milvus", "qdrant", "chromadb", "redis", "weaviate", "rest", "universal_rest", "custom"];
30
46
  declare const LLM_PROVIDERS: readonly ["openai", "anthropic", "ollama", "gemini", "rest", "universal_rest", "custom"];
31
47
  declare const EMBEDDING_PROVIDERS: readonly ["openai", "ollama", "gemini", "rest", "universal_rest", "custom"];
48
+ declare const GRAPH_DB_PROVIDERS: readonly ["neo4j", "memgraph", "simple", "custom"];
32
49
 
33
50
  /**
34
51
  * Master configuration interface for Retrivora AI.
@@ -62,6 +79,13 @@ interface VectorDBConfig {
62
79
  */
63
80
  options: Record<string, unknown>;
64
81
  }
82
+ type GraphDBProvider = typeof GRAPH_DB_PROVIDERS[number];
83
+ interface GraphDBConfig {
84
+ /** Which graph database to use */
85
+ provider: GraphDBProvider;
86
+ /** Provider-specific options (URI, credentials, etc.) */
87
+ options: Record<string, unknown>;
88
+ }
65
89
  type LLMProvider = typeof LLM_PROVIDERS[number];
66
90
  interface LLMConfig {
67
91
  /** Which LLM provider to use */
@@ -132,6 +156,8 @@ interface UIConfig {
132
156
  visualStyle?: 'glass' | 'solid';
133
157
  /** Border radius: 'none', 'sm', 'md', 'lg', 'xl', 'full' */
134
158
  borderRadius?: 'none' | 'sm' | 'md' | 'lg' | 'xl' | 'full';
159
+ /** Whether to allow file uploads directly from the UI */
160
+ allowUpload?: boolean;
135
161
  }
136
162
  interface RAGConfig {
137
163
  /** Number of top-K chunks retrieved per query */
@@ -142,6 +168,14 @@ interface RAGConfig {
142
168
  chunkSize?: number;
143
169
  /** Overlap between adjacent chunks in tokens */
144
170
  chunkOverlap?: number;
171
+ /** Characters used to split text, in order of priority */
172
+ separators?: string[];
173
+ /** Whether to use query transformation (e.g. HyDE) */
174
+ useQueryTransformation?: boolean;
175
+ /** Whether to use graph-based retrieval */
176
+ useGraphRetrieval?: boolean;
177
+ /** Whether to perform reranking on retrieved results */
178
+ useReranking?: boolean;
145
179
  }
146
180
  interface RagConfig {
147
181
  /**
@@ -159,6 +193,8 @@ interface RagConfig {
159
193
  ui?: UIConfig;
160
194
  /** Optional RAG pipeline tuning knobs */
161
195
  rag?: RAGConfig;
196
+ /** Optional Graph database configuration */
197
+ graphDb?: GraphDBConfig;
162
198
  }
163
199
 
164
- export type { ChatResponse as C, EmbeddingConfig as E, IngestDocument as I, LLMConfig as L, RAGConfig as R, UIConfig as U, VectorMatch as V, EmbeddingProvider as a, LLMProvider as b, RagConfig as c, UpsertDocument as d, VectorDBConfig as e, VectorDBProvider as f };
200
+ export type { ChatResponse as C, EmbeddingConfig as E, GraphDBConfig as G, IngestDocument as I, LLMConfig as L, RAGConfig as R, UIConfig as U, VectorMatch as V, EmbeddingProvider as a, LLMProvider as b, RagConfig as c, UpsertDocument as d, VectorDBConfig as e, VectorDBProvider as f, GraphNode as g, Edge as h, GraphSearchResult as i };
@@ -0,0 +1,62 @@
1
+ import "./chunk-FWCSY2DS.mjs";
2
+
3
+ // src/providers/graphdb/BaseGraphProvider.ts
4
+ var BaseGraphProvider = class {
5
+ constructor(config) {
6
+ this.config = config;
7
+ }
8
+ };
9
+
10
+ // src/providers/graphdb/SimpleGraphProvider.ts
11
+ var SimpleGraphProvider = class extends BaseGraphProvider {
12
+ constructor() {
13
+ super(...arguments);
14
+ this.nodes = /* @__PURE__ */ new Map();
15
+ this.edges = [];
16
+ }
17
+ async initialize() {
18
+ console.log("[SimpleGraphProvider] Initialised in-memory graph store.");
19
+ }
20
+ async addNodes(nodes) {
21
+ for (const node of nodes) {
22
+ this.nodes.set(node.id, node);
23
+ }
24
+ }
25
+ async addEdges(edges) {
26
+ this.edges.push(...edges);
27
+ }
28
+ async query(queryText, limit = 5) {
29
+ const q = queryText.toLowerCase();
30
+ const matchedNodes = Array.from(this.nodes.values()).filter(
31
+ (node) => node.id.toLowerCase().includes(q) || node.label.toLowerCase().includes(q) || JSON.stringify(node.properties).toLowerCase().includes(q)
32
+ ).slice(0, limit);
33
+ const matchedNodeIds = new Set(matchedNodes.map((n) => n.id));
34
+ const matchedEdges = this.edges.filter(
35
+ (edge) => matchedNodeIds.has(edge.source) || matchedNodeIds.has(edge.target)
36
+ );
37
+ for (const edge of matchedEdges) {
38
+ if (!matchedNodeIds.has(edge.source)) {
39
+ const source = this.nodes.get(edge.source);
40
+ if (source) matchedNodes.push(source);
41
+ }
42
+ if (!matchedNodeIds.has(edge.target)) {
43
+ const target = this.nodes.get(edge.target);
44
+ if (target) matchedNodes.push(target);
45
+ }
46
+ }
47
+ return {
48
+ nodes: Array.from(new Set(matchedNodes)),
49
+ edges: matchedEdges
50
+ };
51
+ }
52
+ async ping() {
53
+ return true;
54
+ }
55
+ async disconnect() {
56
+ this.nodes.clear();
57
+ this.edges = [];
58
+ }
59
+ };
60
+ export {
61
+ SimpleGraphProvider
62
+ };
@@ -77,7 +77,7 @@ function readEnum(env, name, fallback, allowed) {
77
77
  throw new Error(`[getRagConfig] ${name} must be one of: ${allowed.join(", ")}`);
78
78
  }
79
79
  function getRagConfig(env = process.env) {
80
- var _a, _b, _c, _d, _e, _f, _g, _h, _i, _j, _k, _l, _m, _n, _o, _p, _q, _r, _s, _t, _u, _v, _w, _x, _y, _z, _A, _B, _C, _D, _E, _F, _G, _H, _I, _J, _K, _L, _M, _N, _O;
80
+ var _a, _b, _c, _d, _e, _f, _g, _h, _i, _j, _k, _l, _m, _n, _o, _p, _q, _r, _s, _t, _u, _v, _w, _x, _y, _z, _A, _B, _C, _D, _E, _F, _G, _H, _I, _J, _K, _L, _M, _N, _O, _P, _Q;
81
81
  const projectId = (_b = (_a = readString(env, "RAG_PROJECT_ID")) != null ? _a : readString(env, "NEXT_PUBLIC_PROJECT_ID")) != null ? _b : "__default__";
82
82
  const vectorProvider = readEnum(env, "VECTOR_DB_PROVIDER", "pinecone", VECTOR_DB_PROVIDERS);
83
83
  const llmProvider = readEnum(env, "LLM_PROVIDER", "openai", LLM_PROVIDERS);
@@ -174,7 +174,8 @@ function getRagConfig(env = process.env) {
174
174
  showSources: ((_I = (_H = readString(env, "NEXT_PUBLIC_SHOW_SOURCES")) != null ? _H : readString(env, "UI_SHOW_SOURCES")) != null ? _I : "true") !== "false",
175
175
  welcomeMessage: (_K = (_J = readString(env, "NEXT_PUBLIC_WELCOME_MESSAGE")) != null ? _J : readString(env, "UI_WELCOME_MESSAGE")) != null ? _K : "Hello! I'm your AI assistant. Ask me anything about your documents.",
176
176
  visualStyle: (_M = (_L = readString(env, "NEXT_PUBLIC_UI_VISUAL_STYLE")) != null ? _L : readString(env, "UI_VISUAL_STYLE")) != null ? _M : "glass",
177
- borderRadius: (_O = (_N = readString(env, "NEXT_PUBLIC_UI_BORDER_RADIUS")) != null ? _N : readString(env, "UI_BORDER_RADIUS")) != null ? _O : "xl"
177
+ borderRadius: (_O = (_N = readString(env, "NEXT_PUBLIC_UI_BORDER_RADIUS")) != null ? _N : readString(env, "UI_BORDER_RADIUS")) != null ? _O : "xl",
178
+ allowUpload: ((_Q = (_P = readString(env, "NEXT_PUBLIC_ALLOW_UPLOAD")) != null ? _P : readString(env, "UI_ALLOW_UPLOAD")) != null ? _Q : "false") === "true"
178
179
  },
179
180
  rag: {
180
181
  topK: readNumber(env, "RAG_TOP_K", 5),
@@ -748,52 +749,84 @@ var ConfigValidator = class {
748
749
 
749
750
  // src/rag/DocumentChunker.ts
750
751
  var DocumentChunker = class {
751
- constructor(chunkSize = 1e3, chunkOverlap = 200) {
752
+ constructor(chunkSize = 1e3, chunkOverlap = 200, separators = ["\n\n", "\n", " ", ""]) {
752
753
  this.chunkSize = chunkSize;
753
754
  this.chunkOverlap = chunkOverlap;
755
+ this.separators = separators;
754
756
  }
755
757
  /**
756
- * Split a single text string into overlapping chunks.
758
+ * Split a single text string into overlapping chunks using a recursive strategy.
757
759
  */
758
760
  chunk(text, options = {}) {
759
761
  const {
760
762
  chunkSize = this.chunkSize,
761
763
  chunkOverlap = this.chunkOverlap,
762
764
  docId = `doc_${Date.now()}`,
763
- metadata = {}
765
+ metadata = {},
766
+ separators = this.separators
764
767
  } = options;
765
- const cleaned = text.replace(/\r\n/g, "\n").trim();
766
- if (!cleaned) return [];
767
- const sentences = cleaned.split(new RegExp("(?<=[.!?])\\s+|\\n{2,}")).map((s) => s.trim()).filter(Boolean);
768
- const chunks = [];
769
- let current = "";
768
+ const finalChunks = [];
769
+ const splits = this.recursiveSplit(text, separators, chunkSize);
770
+ let currentChunk = [];
771
+ let currentLength = 0;
770
772
  let chunkIndex = 0;
771
- for (const sentence of sentences) {
772
- if ((current + " " + sentence).trim().length <= chunkSize) {
773
- current = current ? `${current} ${sentence}` : sentence;
774
- } else {
775
- if (current) {
776
- chunks.push({
777
- id: `${docId}_chunk_${chunkIndex++}`,
778
- content: current.trim(),
779
- metadata: __spreadProps(__spreadValues({}, metadata), { docId, chunkIndex: chunkIndex - 1 })
780
- });
781
- }
782
- if (chunkOverlap > 0 && current.length > chunkOverlap) {
783
- current = current.slice(-chunkOverlap) + " " + sentence;
784
- } else {
785
- current = sentence;
773
+ for (const split of splits) {
774
+ if (currentLength + split.length > chunkSize && currentChunk.length > 0) {
775
+ finalChunks.push({
776
+ id: `${docId}_chunk_${chunkIndex++}`,
777
+ content: currentChunk.join("").trim(),
778
+ metadata: __spreadProps(__spreadValues({}, metadata), { docId, chunkIndex: chunkIndex - 1 })
779
+ });
780
+ const overlapItems = [];
781
+ let overlapLen = 0;
782
+ for (let i = currentChunk.length - 1; i >= 0; i--) {
783
+ if (overlapLen + currentChunk[i].length <= chunkOverlap) {
784
+ overlapItems.unshift(currentChunk[i]);
785
+ overlapLen += currentChunk[i].length;
786
+ } else {
787
+ break;
788
+ }
786
789
  }
790
+ currentChunk = overlapItems;
791
+ currentLength = overlapLen;
787
792
  }
793
+ currentChunk.push(split);
794
+ currentLength += split.length;
788
795
  }
789
- if (current.trim()) {
790
- chunks.push({
796
+ if (currentChunk.length > 0) {
797
+ finalChunks.push({
791
798
  id: `${docId}_chunk_${chunkIndex}`,
792
- content: current.trim(),
799
+ content: currentChunk.join("").trim(),
793
800
  metadata: __spreadProps(__spreadValues({}, metadata), { docId, chunkIndex })
794
801
  });
795
802
  }
796
- return chunks;
803
+ return finalChunks;
804
+ }
805
+ /**
806
+ * Recursively split text based on separators.
807
+ */
808
+ recursiveSplit(text, separators, chunkSize) {
809
+ const finalSplits = [];
810
+ let separator = separators[separators.length - 1];
811
+ let nextSeparators = [];
812
+ for (let i = 0; i < separators.length; i++) {
813
+ if (text.includes(separators[i])) {
814
+ separator = separators[i];
815
+ nextSeparators = separators.slice(i + 1);
816
+ break;
817
+ }
818
+ }
819
+ const parts = text.split(separator);
820
+ for (const part of parts) {
821
+ if (part.length <= chunkSize) {
822
+ finalSplits.push(part + separator);
823
+ } else if (nextSeparators.length > 0) {
824
+ finalSplits.push(...this.recursiveSplit(part, nextSeparators, chunkSize));
825
+ } else {
826
+ finalSplits.push(part);
827
+ }
828
+ }
829
+ return finalSplits;
797
830
  }
798
831
  /**
799
832
  * Chunk multiple documents at once.
@@ -805,6 +838,39 @@ var DocumentChunker = class {
805
838
  }
806
839
  };
807
840
 
841
+ // src/rag/EntityExtractor.ts
842
+ var EntityExtractor = class {
843
+ constructor(llm) {
844
+ this.llm = llm;
845
+ }
846
+ /**
847
+ * Extract nodes and edges from a text chunk.
848
+ */
849
+ async extract(text) {
850
+ const prompt = `
851
+ Extract entities and relationships from the following text.
852
+ Format the output as JSON with two keys: "nodes" (array of {id, label, properties}) and "edges" (array of {source, target, type, properties}).
853
+ Use the same ID for the same entity.
854
+
855
+ Text:
856
+ "${text}"
857
+
858
+ Output JSON:
859
+ `;
860
+ const response = await this.llm.chat([
861
+ { role: "system", content: "You are an expert at knowledge graph extraction. Respond only with valid JSON." },
862
+ { role: "user", content: prompt }
863
+ ], "");
864
+ try {
865
+ const cleanJson = response.replace(/```json\n?|\n?```/g, "").trim();
866
+ return JSON.parse(cleanJson);
867
+ } catch (error) {
868
+ console.warn("[EntityExtractor] Failed to parse LLM response as JSON:", response);
869
+ return { nodes: [], edges: [] };
870
+ }
871
+ }
872
+ };
873
+
808
874
  // src/llm/providers/OpenAIProvider.ts
809
875
  import OpenAI from "openai";
810
876
  var OpenAIProvider = class {
@@ -1348,16 +1414,15 @@ var LLMFactory = class _LLMFactory {
1348
1414
 
1349
1415
  // src/core/ProviderRegistry.ts
1350
1416
  var ProviderRegistry = class {
1351
- /**
1352
- * Register a custom vector provider class by name.
1353
- * The name must match the provider value used in VectorDBConfig.provider.
1354
- *
1355
- * @example
1356
- * ProviderRegistry.registerVectorProvider('my-db', MyCustomProvider);
1357
- */
1358
1417
  static registerVectorProvider(name, providerClass) {
1359
1418
  this.vectorProviders[name] = providerClass;
1360
1419
  }
1420
+ /**
1421
+ * Register a custom graph provider class by name.
1422
+ */
1423
+ static registerGraphProvider(name, providerClass) {
1424
+ this.graphProviders[name] = providerClass;
1425
+ }
1361
1426
  /**
1362
1427
  * Creates a vector database provider based on the configuration.
1363
1428
  * Built-in providers are dynamically imported to avoid bundling all SDKs.
@@ -1412,6 +1477,28 @@ var ProviderRegistry = class {
1412
1477
  );
1413
1478
  }
1414
1479
  }
1480
+ /**
1481
+ * Creates a graph database provider based on the configuration.
1482
+ */
1483
+ static async createGraphProvider(config) {
1484
+ const { provider } = config;
1485
+ if (this.graphProviders[provider]) {
1486
+ return new this.graphProviders[provider](config);
1487
+ }
1488
+ switch (provider) {
1489
+ case "neo4j": {
1490
+ throw new Error("[ProviderRegistry] Neo4j provider not implemented yet.");
1491
+ }
1492
+ case "simple": {
1493
+ const { SimpleGraphProvider } = await import("./SimpleGraphProvider-M6T7SE7D.mjs");
1494
+ return new SimpleGraphProvider(config);
1495
+ }
1496
+ default:
1497
+ throw new Error(
1498
+ `[ProviderRegistry] Unsupported graph provider: "${provider}". Built-in providers: simple. For custom providers, call ProviderRegistry.registerGraphProvider("${provider}", YourClass).`
1499
+ );
1500
+ }
1501
+ }
1415
1502
  /**
1416
1503
  * Creates an LLM provider based on the configuration.
1417
1504
  */
@@ -1420,6 +1507,7 @@ var ProviderRegistry = class {
1420
1507
  }
1421
1508
  };
1422
1509
  ProviderRegistry.vectorProviders = {};
1510
+ ProviderRegistry.graphProviders = {};
1423
1511
 
1424
1512
  // src/core/BatchProcessor.ts
1425
1513
  function isTransientError(error) {
@@ -1851,6 +1939,11 @@ var Pipeline = class {
1851
1939
  );
1852
1940
  this.llmProvider = llmProvider;
1853
1941
  this.embeddingProvider = embeddingProvider;
1942
+ if (this.config.graphDb) {
1943
+ this.graphDB = await ProviderRegistry.createGraphProvider(this.config.graphDb);
1944
+ await this.graphDB.initialize();
1945
+ this.entityExtractor = new EntityExtractor(this.llmProvider);
1946
+ }
1854
1947
  await this.vectorDB.initialize();
1855
1948
  this.initialised = true;
1856
1949
  }
@@ -1904,6 +1997,14 @@ var Pipeline = class {
1904
1997
  docId: doc.docId,
1905
1998
  chunksIngested: upsertResult.totalProcessed
1906
1999
  });
2000
+ if (this.graphDB && this.entityExtractor) {
2001
+ console.log(`[Pipeline] Extracting entities for doc ${doc.docId}...`);
2002
+ for (const chunk of chunks) {
2003
+ const { nodes, edges } = await this.entityExtractor.extract(chunk.content);
2004
+ if (nodes.length > 0) await this.graphDB.addNodes(nodes);
2005
+ if (edges.length > 0) await this.graphDB.addEdges(edges);
2006
+ }
2007
+ }
1907
2008
  } catch (error) {
1908
2009
  console.error(`[Pipeline] Failed to ingest document ${doc.docId}:`, error);
1909
2010
  results.push({ docId: doc.docId, chunksIngested: 0 });
@@ -1912,27 +2013,65 @@ var Pipeline = class {
1912
2013
  return results;
1913
2014
  }
1914
2015
  async ask(question, history = [], namespace) {
1915
- var _a, _b, _c, _d;
2016
+ var _a, _b, _c, _d, _e, _f;
1916
2017
  await this.initialize();
1917
2018
  const ns = namespace != null ? namespace : this.config.projectId;
1918
2019
  const topK = (_b = (_a = this.config.rag) == null ? void 0 : _a.topK) != null ? _b : 5;
1919
2020
  const scoreThreshold = (_d = (_c = this.config.rag) == null ? void 0 : _c.scoreThreshold) != null ? _d : 0;
1920
2021
  try {
1921
- const queryVector = await this.embeddingProvider.embed(question, { taskType: "query" });
2022
+ let searchQuery = question;
2023
+ if ((_e = this.config.rag) == null ? void 0 : _e.useQueryTransformation) {
2024
+ searchQuery = await this.rewriteQuery(question, history);
2025
+ }
2026
+ const queryVector = await this.embeddingProvider.embed(searchQuery, { taskType: "query" });
1922
2027
  const fieldHints = extractQueryFieldHints(question);
1923
2028
  const filter = buildQueryFilter(question, fieldHints);
1924
2029
  filter.__entityHints = fieldHints;
1925
2030
  const rawMatches = await this.vectorDB.query(queryVector, topK, ns, filter);
1926
2031
  const sources = rawMatches.filter((m) => m.score >= scoreThreshold);
1927
- const context = sources.length ? sources.map((m, i) => `[Source ${i + 1}]
2032
+ let graphData;
2033
+ if (this.graphDB && ((_f = this.config.rag) == null ? void 0 : _f.useGraphRetrieval)) {
2034
+ graphData = await this.graphDB.query(searchQuery);
2035
+ }
2036
+ let context = sources.length ? sources.map((m, i) => `[Source ${i + 1}]
1928
2037
  ${m.content}`).join("\n\n---\n\n") : "No relevant context found.";
2038
+ if (graphData && graphData.nodes.length > 0) {
2039
+ const graphContext = graphData.nodes.map(
2040
+ (n) => `Entity: ${n.label} (${n.id})${n.properties ? " - " + JSON.stringify(n.properties) : ""}`
2041
+ ).join("\n");
2042
+ context = `GRAPH KNOWLEDGE:
2043
+ ${graphContext}
2044
+
2045
+ VECTOR CONTEXT:
2046
+ ${context}`;
2047
+ }
1929
2048
  const messages = [...history, { role: "user", content: question }];
1930
2049
  const reply = await this.llmProvider.chat(messages, context);
1931
- return { reply, sources };
2050
+ return { reply, sources, graphData };
1932
2051
  } catch (error) {
1933
2052
  throw new Error(`[Pipeline] Chat failed: ${error instanceof Error ? error.message : String(error)}`);
1934
2053
  }
1935
2054
  }
2055
+ /**
2056
+ * Rewrite the user query for better retrieval performance.
2057
+ */
2058
+ async rewriteQuery(question, history) {
2059
+ const prompt = `
2060
+ Given the following conversation history and a new question, rewrite the question to be a better search query for a vector database.
2061
+ Focus on extracting the core intent and entities. Do not answer the question, just rewrite it.
2062
+
2063
+ History:
2064
+ ${history.map((m) => `${m.role}: ${m.content}`).join("\n")}
2065
+
2066
+ New Question: ${question}
2067
+
2068
+ Optimized Search Query:`;
2069
+ const rewrite = await this.llmProvider.chat([
2070
+ { role: "system", content: "You are an assistant that optimizes search queries for RAG systems." },
2071
+ { role: "user", content: prompt }
2072
+ ], "");
2073
+ return rewrite.trim() || question;
2074
+ }
1936
2075
  };
1937
2076
 
1938
2077
  // src/core/ProviderHealthCheck.ts
@@ -1,3 +1,3 @@
1
- import '../RagConfig-CVt24lbC.mjs';
2
- export { c as createChatHandler, a as createHealthHandler, b as createIngestHandler, d as createUploadHandler } from '../index-OI2--lvT.mjs';
1
+ import '../RagConfig-BgRDL9Vy.mjs';
2
+ export { c as createChatHandler, a as createHealthHandler, b as createIngestHandler, d as createUploadHandler } from '../index-7qeLTPBL.mjs';
3
3
  import 'next/server';
@@ -1,3 +1,3 @@
1
- import '../RagConfig-CVt24lbC.js';
2
- export { c as createChatHandler, a as createHealthHandler, b as createIngestHandler, d as createUploadHandler } from '../index-D1hoNXMT.js';
1
+ import '../RagConfig-BgRDL9Vy.js';
2
+ export { c as createChatHandler, a as createHealthHandler, b as createIngestHandler, d as createUploadHandler } from '../index-DowY4_K0.js';
3
3
  import 'next/server';