@retrivora-ai/rag-engine 2.4.3 → 2.4.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (142) hide show
  1. package/README.md +22 -0
  2. package/dist/{BatchProcessor-zbQMaZHm.d.mts → BatchProcessor-BGslIR5l.d.mts} +65 -4
  3. package/dist/{BatchProcessor-D42E706a.d.ts → BatchProcessor-C3gY6kZ0.d.ts} +65 -4
  4. package/dist/handlers/index.js +668 -837
  5. package/dist/handlers/index.mjs +678 -837
  6. package/dist/index.css +5 -0
  7. package/dist/index.d.mts +4 -28
  8. package/dist/index.d.ts +4 -28
  9. package/dist/index.js +91 -43
  10. package/dist/index.min.css +1 -1
  11. package/dist/index.mjs +91 -43
  12. package/dist/server.d.mts +21 -7
  13. package/dist/server.d.ts +21 -7
  14. package/dist/server.js +731 -875
  15. package/dist/server.mjs +741 -875
  16. package/package.json +11 -4
  17. package/src/LicensedRetrivora.ts +0 -278
  18. package/src/app/api/chat/route.ts +0 -4
  19. package/src/app/api/health/route.ts +0 -5
  20. package/src/app/api/ingest/route.ts +0 -27
  21. package/src/app/api/suggestions/route.ts +0 -5
  22. package/src/app/api/upload/route.ts +0 -5
  23. package/src/app/constants.tsx +0 -270
  24. package/src/app/favicon.ico +0 -0
  25. package/src/app/globals.css +0 -449
  26. package/src/app/layout.tsx +0 -22
  27. package/src/app/page.tsx +0 -64
  28. package/src/app/types.ts +0 -30
  29. package/src/browser.tsx +0 -127
  30. package/src/components/ChatWidget.tsx +0 -393
  31. package/src/components/ChatWindow.tsx +0 -952
  32. package/src/components/CodeViewer.tsx +0 -56
  33. package/src/components/ConfigProvider.tsx +0 -141
  34. package/src/components/DocumentUpload.tsx +0 -354
  35. package/src/components/DynamicChart.tsx +0 -297
  36. package/src/components/HourglassLoader.tsx +0 -87
  37. package/src/components/MarkdownComponents.tsx +0 -186
  38. package/src/components/MessageBubble.tsx +0 -498
  39. package/src/components/ObservabilityPanel.tsx +0 -374
  40. package/src/components/ProductCard.tsx +0 -107
  41. package/src/components/ProductCarousel.tsx +0 -97
  42. package/src/components/SourceCard.tsx +0 -61
  43. package/src/components/ThemeProvider.tsx +0 -8
  44. package/src/components/ThemeToggle.tsx +0 -29
  45. package/src/components/ThinkingBlock.tsx +0 -75
  46. package/src/components/UIDispatcher.tsx +0 -342
  47. package/src/components/VisualizationRenderer.tsx +0 -621
  48. package/src/components/constants.tsx +0 -503
  49. package/src/config/ConfigBuilder.ts +0 -285
  50. package/src/config/EmbeddingStrategy.ts +0 -148
  51. package/src/config/LicenseConfigResolver.ts +0 -298
  52. package/src/config/RagConfig.ts +0 -328
  53. package/src/config/UniversalProfiles.ts +0 -82
  54. package/src/config/constants.ts +0 -64
  55. package/src/config/serverConfig.ts +0 -378
  56. package/src/config/uiConstants.ts +0 -47
  57. package/src/core/BatchProcessor.ts +0 -385
  58. package/src/core/CircuitBreaker.ts +0 -118
  59. package/src/core/ConfigFetcher.ts +0 -220
  60. package/src/core/ConfigResolver.ts +0 -165
  61. package/src/core/ConfigValidator.ts +0 -197
  62. package/src/core/DatabaseStorage.ts +0 -526
  63. package/src/core/FreeTierLimitsGuard.ts +0 -281
  64. package/src/core/LLMRouter.ts +0 -79
  65. package/src/core/LangChainAgent.ts +0 -132
  66. package/src/core/LicenseValidator.ts +0 -284
  67. package/src/core/LicenseVerifier.ts +0 -261
  68. package/src/core/MultiAgentCoordinator.ts +0 -239
  69. package/src/core/Pipeline.ts +0 -1279
  70. package/src/core/PluginManager.ts +0 -277
  71. package/src/core/ProjectSettings.ts +0 -247
  72. package/src/core/ProviderHealthCheck.ts +0 -133
  73. package/src/core/ProviderInterfaces.ts +0 -37
  74. package/src/core/ProviderRegistry.ts +0 -156
  75. package/src/core/QueryProcessor.ts +0 -399
  76. package/src/core/Retrivora.ts +0 -98
  77. package/src/core/VectorPlugin.ts +0 -193
  78. package/src/core/mcp.ts +0 -261
  79. package/src/exceptions/index.ts +0 -131
  80. package/src/handlers/index.ts +0 -1625
  81. package/src/hooks/useHydrated.ts +0 -13
  82. package/src/hooks/useProjectSettings.ts +0 -105
  83. package/src/hooks/useRagChat.ts +0 -436
  84. package/src/hooks/useStoredMessages.ts +0 -64
  85. package/src/index.css +0 -4269
  86. package/src/index.ts +0 -137
  87. package/src/lib/plugin.ts +0 -24
  88. package/src/llm/ILLMProvider.ts +0 -60
  89. package/src/llm/LLMFactory.ts +0 -141
  90. package/src/llm/providers/AnthropicProvider.ts +0 -215
  91. package/src/llm/providers/GeminiProvider.ts +0 -311
  92. package/src/llm/providers/GroqProvider.ts +0 -184
  93. package/src/llm/providers/OllamaProvider.ts +0 -232
  94. package/src/llm/providers/OpenAIProvider.ts +0 -188
  95. package/src/llm/providers/QwenProvider.ts +0 -199
  96. package/src/llm/providers/UniversalLLMAdapter.ts +0 -422
  97. package/src/llm/utils.ts +0 -46
  98. package/src/providers/graphdb/BaseGraphProvider.ts +0 -43
  99. package/src/providers/graphdb/SimpleGraphProvider.ts +0 -66
  100. package/src/providers/vectordb/BaseVectorProvider.ts +0 -98
  101. package/src/providers/vectordb/ChromaDBProvider.ts +0 -130
  102. package/src/providers/vectordb/MilvusProvider.ts +0 -134
  103. package/src/providers/vectordb/MongoDBProvider.ts +0 -250
  104. package/src/providers/vectordb/MultiTablePostgresProvider.ts +0 -461
  105. package/src/providers/vectordb/PineconeProvider.ts +0 -295
  106. package/src/providers/vectordb/PostgreSQLProvider.ts +0 -227
  107. package/src/providers/vectordb/QdrantProvider.ts +0 -294
  108. package/src/providers/vectordb/RedisProvider.ts +0 -105
  109. package/src/providers/vectordb/UniversalVectorProvider.ts +0 -221
  110. package/src/providers/vectordb/WeaviateProvider.ts +0 -172
  111. package/src/rag/DocumentChunker.ts +0 -144
  112. package/src/rag/EntityExtractor.ts +0 -90
  113. package/src/rag/LlamaIndexIngestor.ts +0 -60
  114. package/src/rag/Reranker.ts +0 -106
  115. package/src/rendering/IntentClassifier.ts +0 -167
  116. package/src/rendering/RendererRegistry.ts +0 -164
  117. package/src/rendering/RuleEngine.ts +0 -67
  118. package/src/rendering/VisualizationDecisionEngine.ts +0 -164
  119. package/src/rendering/__tests__/VisualizationDecisionEngine.test.ts +0 -237
  120. package/src/rendering/index.ts +0 -12
  121. package/src/rendering/rules/Rule1SpecificInfoRule.ts +0 -28
  122. package/src/rendering/rules/Rule2ComparisonRule.ts +0 -24
  123. package/src/rendering/rules/Rule3ProductDiscoveryRule.ts +0 -28
  124. package/src/rendering/rules/Rule4AnalyticalRule.ts +0 -38
  125. package/src/rendering/rules/Rule5MixedResponseRule.ts +0 -29
  126. package/src/rendering/rules/Rule6SmallResultSetRule.ts +0 -24
  127. package/src/rendering/rules/Rule7LargeTableRule.ts +0 -35
  128. package/src/rendering/types.ts +0 -96
  129. package/src/server.ts +0 -156
  130. package/src/tailwind.css +0 -194
  131. package/src/telemetry/TelemetryService.ts +0 -330
  132. package/src/types/chat.ts +0 -89
  133. package/src/types/index.ts +0 -278
  134. package/src/types/pdf-parse.d.ts +0 -13
  135. package/src/types/props.ts +0 -169
  136. package/src/utils/DocumentParser.ts +0 -111
  137. package/src/utils/ProductExtractor.ts +0 -350
  138. package/src/utils/SchemaMapper.ts +0 -176
  139. package/src/utils/UITransformer.ts +0 -1812
  140. package/src/utils/synonyms.ts +0 -80
  141. package/src/utils/templateUtils.ts +0 -78
  142. package/src/version.ts +0 -6
@@ -1,172 +0,0 @@
1
- import axios, { AxiosInstance } from 'axios';
2
- import { VectorDBConfig } from '../../config/RagConfig';
3
- import { BaseVectorProvider } from './BaseVectorProvider';
4
- import { VectorMatch, UpsertDocument } from '../../types';
5
-
6
- /**
7
- * WeaviateProvider — Weaviate implementation using its REST/GraphQL API.
8
- *
9
- * Required options:
10
- * { baseUrl: string } — e.g. "http://localhost:8080"
11
- * OR { url: string } — alias accepted for backwards compatibility
12
- *
13
- * Optional: { apiKey?: string }
14
- */
15
- export class WeaviateProvider extends BaseVectorProvider {
16
- private http: AxiosInstance;
17
-
18
- constructor(config: VectorDBConfig) {
19
- super(config);
20
- const opts = config.options as Record<string, unknown>;
21
-
22
- // Accept baseUrl or url alias
23
- const baseUrl = (opts.baseUrl as string) || (opts.url as string);
24
- if (!baseUrl) throw new Error('[WeaviateProvider] options.baseUrl is required');
25
-
26
- this.http = axios.create({
27
- baseURL: baseUrl,
28
- headers: {
29
- 'Content-Type': 'application/json',
30
- ...(opts.apiKey ? {
31
- 'X-OpenAI-Api-Key': opts.apiKey as string,
32
- 'Authorization': `Bearer ${opts.apiKey as string}`,
33
- } : {}),
34
- },
35
- });
36
- }
37
-
38
- async initialize(): Promise<void> {
39
- await this.ping();
40
- }
41
-
42
- async upsert(doc: UpsertDocument, namespace?: string): Promise<void> {
43
- const primitiveMetadata = this.extractPrimitiveMetadata(doc.metadata);
44
- const payload = {
45
- class: this.indexName,
46
- id: doc.id,
47
- vector: doc.vector,
48
- properties: {
49
- content: doc.content,
50
- metadata: JSON.stringify(doc.metadata || {}),
51
- ...primitiveMetadata,
52
- namespace: namespace || '',
53
- },
54
- };
55
- await this.http.post('/v1/objects', payload);
56
- }
57
-
58
- async batchUpsert(docs: UpsertDocument[], namespace?: string): Promise<void> {
59
- const payload = {
60
- objects: docs.map(doc => ({
61
- class: this.indexName,
62
- id: doc.id,
63
- vector: doc.vector,
64
- properties: {
65
- content: doc.content,
66
- metadata: JSON.stringify(doc.metadata || {}),
67
- ...this.extractPrimitiveMetadata(doc.metadata),
68
- namespace: namespace || '',
69
- },
70
- })),
71
- };
72
- await this.http.post('/v1/batch/objects', payload);
73
- }
74
-
75
- async query(vector: number[], topK: number, namespace?: string, _filter?: Record<string, unknown>): Promise<VectorMatch[]> {
76
- // IMPORTANT: read queryText BEFORE calling sanitizeFilter() because sanitizeFilter strips it.
77
- const queryText = _filter?.queryText as string | undefined;
78
- const sanitizedFilter = this.sanitizeFilter(_filter);
79
-
80
- const searchParams = queryText
81
- ? `hybrid: { query: ${JSON.stringify(queryText)}, alpha: 0.5 }`
82
- : `nearVector: { vector: ${JSON.stringify(vector)} }`;
83
- const where = this.buildWhereFilter(namespace, sanitizedFilter);
84
-
85
- const graphqlQuery = {
86
- query: `
87
- {
88
- Get {
89
- ${this.indexName}(
90
- ${searchParams}
91
- limit: ${topK}
92
- ${where ? `where: ${where}` : ''}
93
- ) {
94
- content
95
- metadata
96
- _additional {
97
- id
98
- distance
99
- }
100
- }
101
- }
102
- }
103
- `
104
- };
105
- const { data } = await this.http.post('/v1/graphql', graphqlQuery);
106
- const results = data.data?.Get?.[this.indexName] || [];
107
-
108
- return results.map((res: Record<string, unknown>) => ({
109
- id: (res['_additional'] as Record<string, unknown>).id as string,
110
- score: 1 - ((res['_additional'] as Record<string, unknown>).distance as number),
111
- content: res['content'] as string,
112
- metadata: res['metadata'] ? JSON.parse(res['metadata'] as string) : {},
113
- }));
114
- }
115
-
116
- async delete(id: string, _namespace?: string): Promise<void> {
117
- await this.http.delete(`/v1/objects/${this.indexName}/${id}`);
118
- }
119
-
120
- async deleteNamespace(namespace: string): Promise<void> {
121
- await this.http.post('/v1/batch/objects', {
122
- match: {
123
- class: this.indexName,
124
- where: { path: ['namespace'], operator: 'Equal', valueString: namespace },
125
- },
126
- });
127
- }
128
-
129
- async ping(): Promise<boolean> {
130
- try {
131
- await this.http.get('/v1/.well-known/live');
132
- return true;
133
- } catch {
134
- return false;
135
- }
136
- }
137
-
138
- async disconnect(): Promise<void> {}
139
-
140
- private extractPrimitiveMetadata(metadata?: Record<string, unknown>): Record<string, string | number | boolean> {
141
- const result: Record<string, string | number | boolean> = {};
142
- Object.entries(metadata || {}).forEach(([key, value]) => {
143
- if (typeof value === 'string' || typeof value === 'number' || typeof value === 'boolean') {
144
- result[key] = value;
145
- }
146
- });
147
- return result;
148
- }
149
-
150
- private buildWhereFilter(namespace?: string, filter?: Record<string, unknown>): string | undefined {
151
- const operands: string[] = [];
152
- if (namespace) operands.push(this.weaviateOperand('namespace', namespace));
153
-
154
- Object.entries(filter || {}).forEach(([key, value]) => {
155
- if (key === 'namespace') return;
156
- if (typeof value === 'string' || typeof value === 'number' || typeof value === 'boolean') {
157
- operands.push(this.weaviateOperand(key, value));
158
- }
159
- });
160
-
161
- if (operands.length === 0) return undefined;
162
- if (operands.length === 1) return operands[0];
163
- return `{ operator: And, operands: [${operands.join(', ')}] }`;
164
- }
165
-
166
- private weaviateOperand(key: string, value: string | number | boolean): string {
167
- const path = `path: [${JSON.stringify(key)}], operator: Equal`;
168
- if (typeof value === 'number') return `{ ${path}, valueNumber: ${value} }`;
169
- if (typeof value === 'boolean') return `{ ${path}, valueBoolean: ${value} }`;
170
- return `{ ${path}, valueString: ${JSON.stringify(value)} }`;
171
- }
172
- }
@@ -1,144 +0,0 @@
1
- /**
2
- * DocumentChunker — splits long text into overlapping chunks
3
- * suitable for vector embedding and retrieval.
4
- *
5
- * Strategy: sentence-aware sliding window
6
- * 1. Split on sentence boundaries
7
- * 2. Accumulate sentences into chunks up to `chunkSize` characters
8
- * 3. Carry over `chunkOverlap` characters from the previous chunk
9
- */
10
-
11
- export interface Chunk {
12
- id: string;
13
- content: string;
14
- metadata?: Record<string, unknown>;
15
- }
16
-
17
- export interface ChunkOptions {
18
- /** Target chunk size in characters (default 1000) */
19
- chunkSize?: number;
20
- /** Overlap between consecutive chunks in characters (default 200) */
21
- chunkOverlap?: number;
22
- /** Source document identifier used as ID prefix */
23
- docId?: string | number;
24
- /** Extra metadata to attach to every chunk */
25
- metadata?: Record<string, unknown>;
26
- /** Characters used to split text, in order of priority */
27
- separators?: string[];
28
- }
29
-
30
- export class DocumentChunker {
31
- private readonly chunkSize: number;
32
- private readonly chunkOverlap: number;
33
- private readonly separators: string[];
34
-
35
- constructor(
36
- chunkSize = 1000,
37
- chunkOverlap = 200,
38
- separators = ['\n# ', '\n## ', '\n### ', '\n#### ', '\n\n', '\n', ' ', '']
39
- ) {
40
- this.chunkSize = chunkSize;
41
- this.chunkOverlap = chunkOverlap;
42
- this.separators = separators;
43
- }
44
-
45
- /**
46
- * Split a single text string into overlapping chunks using a recursive strategy.
47
- * Preserves structural boundaries (Markdown headers) where possible.
48
- */
49
- chunk(text: string, options: ChunkOptions = {}): Chunk[] {
50
- const {
51
- chunkSize = this.chunkSize,
52
- chunkOverlap = this.chunkOverlap,
53
- docId = `doc_${Date.now()}`,
54
- metadata = {},
55
- separators = this.separators,
56
- } = options;
57
-
58
- const finalChunks: Chunk[] = [];
59
- const splits = this.recursiveSplit(text, separators, chunkSize);
60
-
61
- let currentChunk: string[] = [];
62
- let currentLength = 0;
63
- let chunkIndex = 0;
64
-
65
- for (const split of splits) {
66
- // If adding this split exceeds chunk size, and we already have content, close the chunk
67
- if (currentLength + split.length > chunkSize && currentChunk.length > 0) {
68
- finalChunks.push({
69
- id: `${docId}_chunk_${chunkIndex++}`,
70
- content: currentChunk.join('').trim(),
71
- metadata: { ...metadata, docId, chunkIndex: chunkIndex - 1 },
72
- });
73
-
74
- // Sliding window overlap logic
75
- const overlapItems = [];
76
- let overlapLen = 0;
77
- // Search backwards through current chunk to find segments that fit within the overlap window
78
- for (let i = currentChunk.length - 1; i >= 0; i--) {
79
- if (overlapLen + currentChunk[i].length <= chunkOverlap) {
80
- overlapItems.unshift(currentChunk[i]);
81
- overlapLen += currentChunk[i].length;
82
- } else {
83
- break;
84
- }
85
- }
86
- currentChunk = overlapItems;
87
- currentLength = overlapLen;
88
- }
89
-
90
- currentChunk.push(split);
91
- currentLength += split.length;
92
- }
93
-
94
- // Push the last remaining chunk
95
- if (currentChunk.length > 0) {
96
- finalChunks.push({
97
- id: `${docId}_chunk_${chunkIndex}`,
98
- content: currentChunk.join('').trim(),
99
- metadata: { ...metadata, docId, chunkIndex },
100
- });
101
- }
102
-
103
- return finalChunks;
104
- }
105
-
106
- private recursiveSplit(text: string, separators: string[], chunkSize: number): string[] {
107
- const finalSplits: string[] = [];
108
-
109
- let separator = separators[separators.length - 1];
110
- let nextSeparators: string[] = [];
111
-
112
- for (let i = 0; i < separators.length; i++) {
113
- const sep = separators[i];
114
- if (text.includes(sep)) {
115
- separator = sep;
116
- nextSeparators = separators.slice(i + 1);
117
- break;
118
- }
119
- }
120
-
121
- const parts = text.split(separator);
122
-
123
- for (const part of parts) {
124
- if (part.length <= chunkSize) {
125
- // Keep the separator to preserve formatting/meaning
126
- finalSplits.push(part + (part === parts[parts.length - 1] ? '' : separator));
127
- } else if (nextSeparators.length > 0) {
128
- finalSplits.push(...this.recursiveSplit(part, nextSeparators, chunkSize));
129
- } else {
130
- finalSplits.push(part);
131
- }
132
- }
133
-
134
- return finalSplits;
135
- }
136
-
137
- chunkMany(
138
- documents: Array<{ content: string; docId?: string | number; metadata?: Record<string, unknown> }>
139
- ): Chunk[] {
140
- return documents.flatMap((doc) =>
141
- this.chunk(doc.content, { docId: doc.docId, metadata: doc.metadata })
142
- );
143
- }
144
- }
@@ -1,90 +0,0 @@
1
- import { ILLMProvider } from '../llm/ILLMProvider';
2
- import { GraphNode, Edge } from '../types';
3
-
4
- /**
5
- * EntityExtractor — Uses an LLM to extract entities and relations from text.
6
- */
7
- export class EntityExtractor {
8
- private llm: ILLMProvider;
9
-
10
- constructor(llm: ILLMProvider) {
11
- this.llm = llm;
12
- }
13
-
14
- /**
15
- * Extract nodes and edges from a text chunk.
16
- */
17
- async extract(text: string): Promise<{ nodes: GraphNode[]; edges: Edge[] }> {
18
- const prompt = `
19
- Extract entities and relationships from the following text.
20
- Format the output as a JSON object with exactly two keys: "nodes" and "edges".
21
-
22
- Nodes: {id, label, properties}
23
- Edges: {source, target, type, properties}
24
-
25
- IMPORTANT:
26
- - Ensure all property values are simple JSON types (string, number, boolean).
27
- - Do not use mathematical expressions or complex nested objects in properties.
28
- - If no entities are found, return empty arrays.
29
- - RESPOND ONLY WITH THE JSON OBJECT.
30
-
31
- Text to extract from:
32
- "${text}"
33
- `;
34
-
35
- const response = await this.llm.chat([
36
- { role: 'system', content: 'You are a precise knowledge graph extraction engine. You always output valid JSON and nothing else.' },
37
- { role: 'user', content: prompt }
38
- ], '', { maxTokens: 4096, temperature: 0.1 });
39
-
40
- try {
41
- // 1. Try to find the JSON block using a regex
42
- const jsonMatch = response.match(/\{[\s\S]*\}/);
43
- const cleanJson = jsonMatch ? jsonMatch[0] : response.trim();
44
-
45
- return JSON.parse(cleanJson);
46
- } catch {
47
- // 2. If parsing fails, try to repair truncated JSON
48
- try {
49
- const repaired = this.repairTruncatedJson(response);
50
- if (repaired) return JSON.parse(repaired);
51
- } catch { /* ignore repair failure */ }
52
-
53
- console.warn(`[EntityExtractor] Failed to parse LLM response. Length: ${response.length} chars.`);
54
- console.warn('[EntityExtractor] Snippet:', response.substring(0, 100) + '...');
55
- return { nodes: [], edges: [] };
56
- }
57
- }
58
-
59
- /**
60
- * Attempts to fix JSON that was cut off mid-generation.
61
- * Strategy: Find the last valid element in an array and close the structure.
62
- */
63
- private repairTruncatedJson(json: string): string | null {
64
- let text = json.trim();
65
-
66
- // Find the first { and work from there
67
- const startIdx = text.indexOf('{');
68
- if (startIdx === -1) return null;
69
- text = text.substring(startIdx);
70
-
71
- // If it's truncated in an array like "nodes": [ ... , { ...
72
- // We try to find the last complete object before the truncation
73
- const lastCompleteObjectIdx = text.lastIndexOf('}');
74
- if (lastCompleteObjectIdx === -1) return null;
75
-
76
- // Truncate at the last complete object
77
- let repaired = text.substring(0, lastCompleteObjectIdx + 1);
78
-
79
- // Close any open structures (arrays or the main object)
80
- const openBrackets = (repaired.match(/\{/g) || []).length;
81
- const closedBrackets = (repaired.match(/\}/g) || []).length;
82
- const openSquares = (repaired.match(/\[/g) || []).length;
83
- const closedSquares = (repaired.match(/\]/g) || []).length;
84
-
85
- for (let i = 0; i < openSquares - closedSquares; i++) repaired += ']';
86
- for (let i = 0; i < openBrackets - closedBrackets; i++) repaired += '}';
87
-
88
- return repaired;
89
- }
90
- }
@@ -1,60 +0,0 @@
1
- import { Chunk, ChunkOptions } from './DocumentChunker';
2
-
3
- /**
4
- * LlamaIndexIngestor — Advanced ingestion provider using LlamaIndex (TS).
5
- *
6
- * Provides semantic chunking, metadata extraction, and node parsing
7
- * superior to basic recursive character splitting.
8
- *
9
- * Note: Requires 'llamaindex' package to be installed.
10
- */
11
- export class LlamaIndexIngestor {
12
- /**
13
- * Chunks document content using LlamaIndex SentenceSplitter.
14
- * This respects sentence and paragraph boundaries much more effectively
15
- * than standard character-count splitting.
16
- */
17
- async chunk(text: string, options: ChunkOptions = {}): Promise<Chunk[]> {
18
- try {
19
- // Dynamically import for optional dependency support using template string to hide from TS static analysis
20
- const { Document, SentenceSplitter, MetadataMode } = await import(`${'llamaindex'}`);
21
-
22
- const splitter = new SentenceSplitter({
23
- chunkSize: options.chunkSize ?? 1000,
24
- chunkOverlap: options.chunkOverlap ?? 200,
25
- });
26
-
27
- const doc = new Document({ text, metadata: options.metadata || {} });
28
- const nodes = splitter.getNodesFromDocuments([doc]);
29
-
30
- return (nodes as any[]).map((node: any, index: number) => ({
31
- id: `${options.docId || 'doc'}_node_${index}`,
32
- content: node.getContent(MetadataMode.ALL),
33
- metadata: {
34
- ...options.metadata,
35
- ...node.metadata,
36
- nodeId: node.id_,
37
- chunkIndex: index
38
- }
39
- }));
40
- } catch (error) {
41
- console.warn('[LlamaIndexIngestor] LlamaIndex package not found or failed. Falling back to default chunker.');
42
- throw error;
43
- }
44
- }
45
-
46
- /**
47
- * Batch processing for multiple documents.
48
- */
49
- async chunkMany(
50
- documents: Array<{ content: string; docId?: string | number; metadata?: Record<string, unknown> }>,
51
- options: ChunkOptions = {}
52
- ): Promise<Chunk[]> {
53
- const allChunks: Chunk[] = [];
54
- for (const doc of documents) {
55
- const chunks = await this.chunk(doc.content, { ...options, docId: doc.docId, metadata: doc.metadata });
56
- allChunks.push(...chunks);
57
- }
58
- return allChunks;
59
- }
60
- }
@@ -1,106 +0,0 @@
1
- import { VectorMatch } from '../types';
2
- import { ILLMProvider } from '../llm/ILLMProvider';
3
-
4
- export class Reranker {
5
- /**
6
- * Re-ranks matches based on a secondary relevance score using an LLM.
7
- */
8
- async rerank(
9
- matches: VectorMatch[],
10
- query: string,
11
- limit: number = 5,
12
- llm?: ILLMProvider
13
- ): Promise<VectorMatch[]> {
14
- if (!llm || matches.length <= 1) {
15
- // Fallback to deterministic keyword-based re-ranking for production speed
16
- const keywords = query.toLowerCase()
17
- .replace(/[^\w\s]/g, '')
18
- .split(/\s+/)
19
- .filter(w => w.length > 2); // Filter out short words/stop words
20
-
21
- if (keywords.length === 0) {
22
- return matches
23
- .sort((a, b) => b.score - a.score)
24
- .slice(0, limit);
25
- }
26
-
27
- const scoredMatches = matches.map(match => {
28
- const contentLower = (match?.content ?? '').toLowerCase();
29
- let keywordScore = 0;
30
-
31
- keywords.forEach(keyword => {
32
- if (contentLower.includes(keyword)) {
33
- keywordScore += 1;
34
- }
35
- });
36
-
37
- // Normalize keyword score (fraction of keywords found)
38
- const normalizedKeywordScore = keywordScore / keywords.length;
39
-
40
- // Combine scores: Vector score (usually 0-1) + Keyword score weighted
41
- // We give a significant weight to keyword matches to surface specific results
42
- const combinedScore = match.score + (normalizedKeywordScore * 0.5);
43
-
44
- return {
45
- ...match,
46
- score: combinedScore
47
- };
48
- });
49
-
50
- return scoredMatches
51
- .sort((a, b) => b.score - a.score)
52
- .slice(0, limit);
53
- }
54
-
55
- try {
56
- // Take top 10 matches to avoid large context and high latency
57
- const topN = matches.slice(0, 10);
58
-
59
- const response = await llm.chat(
60
- [{ role: 'user', content: `Query: "${query}"\n\nDocuments:\n${topN.map((m, i) => `[${i}] ${(m?.content ?? '').replace(/\n/g, ' ')}`).join('\n')}` }],
61
- '',
62
- {
63
- systemPrompt: 'You are a relevance ranking expert. Given the user query and a list of retrieved document chunks, rank the chunks by relevance to the query. Return the result as a comma-separated list of document indices in order of relevance, from most relevant to least relevant. Use the indices provided in brackets like [0], [1], etc. Only return the indices (e.g., "2,0,1"), nothing else. Do not include brackets in the output.',
64
- temperature: 0,
65
- maxTokens: 50
66
- }
67
- );
68
-
69
- const cleanedResponse = response.trim().replace(/[\[\]\s]/g, '');
70
- const rankedIndices = cleanedResponse.split(',').map(Number).filter(n => !isNaN(n));
71
-
72
- if (rankedIndices.length > 0) {
73
- const rankedMatches: VectorMatch[] = [];
74
- const usedIndices = new Set<number>();
75
-
76
- // Add matches in the order specified by the LLM
77
- for (const index of rankedIndices) {
78
- if (index >= 0 && index < topN.length && !usedIndices.has(index)) {
79
- rankedMatches.push(topN[index]);
80
- usedIndices.add(index);
81
- }
82
- }
83
-
84
- // Add any remaining matches that the LLM might have missed
85
- for (let i = 0; i < topN.length; i++) {
86
- if (!usedIndices.has(i)) {
87
- rankedMatches.push(topN[i]);
88
- }
89
- }
90
-
91
- // Add back the rest of the matches that weren't in the top N
92
- const rest = matches.slice(10);
93
- rankedMatches.push(...rest);
94
-
95
- return rankedMatches.slice(0, limit);
96
- }
97
- } catch (error) {
98
- console.warn('[Reranker] LLM re-ranking failed, falling back to score-based sorting:', error);
99
- }
100
-
101
- // Fallback if LLM fails or returns invalid response
102
- return matches
103
- .sort((a, b) => b.score - a.score)
104
- .slice(0, limit);
105
- }
106
- }