@retrivora-ai/rag-engine 2.4.2 → 2.4.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (140) hide show
  1. package/dist/{BatchProcessor-zbQMaZHm.d.mts → BatchProcessor-Csy_K1qQ.d.mts} +2 -0
  2. package/dist/{BatchProcessor-D42E706a.d.ts → BatchProcessor-Dg18zZGk.d.ts} +2 -0
  3. package/dist/handlers/index.js +31 -5
  4. package/dist/handlers/index.mjs +31 -5
  5. package/dist/index.d.mts +3 -3
  6. package/dist/index.d.ts +3 -3
  7. package/dist/index.js +41 -21
  8. package/dist/index.min.css +2 -0
  9. package/dist/index.mjs +41 -21
  10. package/dist/server.d.mts +2 -2
  11. package/dist/server.d.ts +2 -2
  12. package/dist/server.js +32 -5
  13. package/dist/server.mjs +32 -5
  14. package/package.json +5 -4
  15. package/src/LicensedRetrivora.ts +0 -278
  16. package/src/app/api/chat/route.ts +0 -4
  17. package/src/app/api/health/route.ts +0 -5
  18. package/src/app/api/ingest/route.ts +0 -27
  19. package/src/app/api/suggestions/route.ts +0 -5
  20. package/src/app/api/upload/route.ts +0 -5
  21. package/src/app/constants.tsx +0 -270
  22. package/src/app/favicon.ico +0 -0
  23. package/src/app/globals.css +0 -449
  24. package/src/app/layout.tsx +0 -22
  25. package/src/app/page.tsx +0 -64
  26. package/src/app/types.ts +0 -30
  27. package/src/browser.tsx +0 -139
  28. package/src/components/ChatWidget.tsx +0 -393
  29. package/src/components/ChatWindow.tsx +0 -952
  30. package/src/components/CodeViewer.tsx +0 -56
  31. package/src/components/ConfigProvider.tsx +0 -123
  32. package/src/components/DocumentUpload.tsx +0 -354
  33. package/src/components/DynamicChart.tsx +0 -297
  34. package/src/components/HourglassLoader.tsx +0 -87
  35. package/src/components/MarkdownComponents.tsx +0 -186
  36. package/src/components/MessageBubble.tsx +0 -498
  37. package/src/components/ObservabilityPanel.tsx +0 -374
  38. package/src/components/ProductCard.tsx +0 -107
  39. package/src/components/ProductCarousel.tsx +0 -97
  40. package/src/components/SourceCard.tsx +0 -61
  41. package/src/components/ThemeProvider.tsx +0 -8
  42. package/src/components/ThemeToggle.tsx +0 -29
  43. package/src/components/ThinkingBlock.tsx +0 -75
  44. package/src/components/UIDispatcher.tsx +0 -342
  45. package/src/components/VisualizationRenderer.tsx +0 -621
  46. package/src/components/constants.tsx +0 -503
  47. package/src/config/ConfigBuilder.ts +0 -285
  48. package/src/config/EmbeddingStrategy.ts +0 -148
  49. package/src/config/LicenseConfigResolver.ts +0 -298
  50. package/src/config/RagConfig.ts +0 -328
  51. package/src/config/UniversalProfiles.ts +0 -82
  52. package/src/config/constants.ts +0 -64
  53. package/src/config/serverConfig.ts +0 -378
  54. package/src/config/uiConstants.ts +0 -47
  55. package/src/core/BatchProcessor.ts +0 -385
  56. package/src/core/CircuitBreaker.ts +0 -118
  57. package/src/core/ConfigFetcher.ts +0 -220
  58. package/src/core/ConfigResolver.ts +0 -165
  59. package/src/core/ConfigValidator.ts +0 -197
  60. package/src/core/DatabaseStorage.ts +0 -526
  61. package/src/core/FreeTierLimitsGuard.ts +0 -281
  62. package/src/core/LLMRouter.ts +0 -79
  63. package/src/core/LangChainAgent.ts +0 -132
  64. package/src/core/LicenseValidator.ts +0 -284
  65. package/src/core/LicenseVerifier.ts +0 -261
  66. package/src/core/MultiAgentCoordinator.ts +0 -239
  67. package/src/core/Pipeline.ts +0 -1279
  68. package/src/core/PluginManager.ts +0 -277
  69. package/src/core/ProjectSettings.ts +0 -247
  70. package/src/core/ProviderHealthCheck.ts +0 -133
  71. package/src/core/ProviderInterfaces.ts +0 -37
  72. package/src/core/ProviderRegistry.ts +0 -156
  73. package/src/core/QueryProcessor.ts +0 -399
  74. package/src/core/Retrivora.ts +0 -98
  75. package/src/core/VectorPlugin.ts +0 -193
  76. package/src/core/mcp.ts +0 -261
  77. package/src/exceptions/index.ts +0 -131
  78. package/src/handlers/index.ts +0 -1625
  79. package/src/hooks/useHydrated.ts +0 -13
  80. package/src/hooks/useProjectSettings.ts +0 -105
  81. package/src/hooks/useRagChat.ts +0 -436
  82. package/src/hooks/useStoredMessages.ts +0 -64
  83. package/src/index.css +0 -4269
  84. package/src/index.ts +0 -137
  85. package/src/lib/plugin.ts +0 -24
  86. package/src/llm/ILLMProvider.ts +0 -60
  87. package/src/llm/LLMFactory.ts +0 -141
  88. package/src/llm/providers/AnthropicProvider.ts +0 -215
  89. package/src/llm/providers/GeminiProvider.ts +0 -311
  90. package/src/llm/providers/GroqProvider.ts +0 -184
  91. package/src/llm/providers/OllamaProvider.ts +0 -232
  92. package/src/llm/providers/OpenAIProvider.ts +0 -188
  93. package/src/llm/providers/QwenProvider.ts +0 -199
  94. package/src/llm/providers/UniversalLLMAdapter.ts +0 -422
  95. package/src/llm/utils.ts +0 -46
  96. package/src/providers/graphdb/BaseGraphProvider.ts +0 -43
  97. package/src/providers/graphdb/SimpleGraphProvider.ts +0 -66
  98. package/src/providers/vectordb/BaseVectorProvider.ts +0 -98
  99. package/src/providers/vectordb/ChromaDBProvider.ts +0 -130
  100. package/src/providers/vectordb/MilvusProvider.ts +0 -134
  101. package/src/providers/vectordb/MongoDBProvider.ts +0 -250
  102. package/src/providers/vectordb/MultiTablePostgresProvider.ts +0 -461
  103. package/src/providers/vectordb/PineconeProvider.ts +0 -295
  104. package/src/providers/vectordb/PostgreSQLProvider.ts +0 -227
  105. package/src/providers/vectordb/QdrantProvider.ts +0 -294
  106. package/src/providers/vectordb/RedisProvider.ts +0 -105
  107. package/src/providers/vectordb/UniversalVectorProvider.ts +0 -221
  108. package/src/providers/vectordb/WeaviateProvider.ts +0 -172
  109. package/src/rag/DocumentChunker.ts +0 -144
  110. package/src/rag/EntityExtractor.ts +0 -90
  111. package/src/rag/LlamaIndexIngestor.ts +0 -60
  112. package/src/rag/Reranker.ts +0 -106
  113. package/src/rendering/IntentClassifier.ts +0 -167
  114. package/src/rendering/RendererRegistry.ts +0 -164
  115. package/src/rendering/RuleEngine.ts +0 -67
  116. package/src/rendering/VisualizationDecisionEngine.ts +0 -164
  117. package/src/rendering/__tests__/VisualizationDecisionEngine.test.ts +0 -237
  118. package/src/rendering/index.ts +0 -12
  119. package/src/rendering/rules/Rule1SpecificInfoRule.ts +0 -28
  120. package/src/rendering/rules/Rule2ComparisonRule.ts +0 -24
  121. package/src/rendering/rules/Rule3ProductDiscoveryRule.ts +0 -28
  122. package/src/rendering/rules/Rule4AnalyticalRule.ts +0 -38
  123. package/src/rendering/rules/Rule5MixedResponseRule.ts +0 -29
  124. package/src/rendering/rules/Rule6SmallResultSetRule.ts +0 -24
  125. package/src/rendering/rules/Rule7LargeTableRule.ts +0 -35
  126. package/src/rendering/types.ts +0 -96
  127. package/src/server.ts +0 -156
  128. package/src/tailwind.css +0 -194
  129. package/src/telemetry/TelemetryService.ts +0 -330
  130. package/src/types/chat.ts +0 -89
  131. package/src/types/index.ts +0 -278
  132. package/src/types/pdf-parse.d.ts +0 -13
  133. package/src/types/props.ts +0 -169
  134. package/src/utils/DocumentParser.ts +0 -111
  135. package/src/utils/ProductExtractor.ts +0 -350
  136. package/src/utils/SchemaMapper.ts +0 -176
  137. package/src/utils/UITransformer.ts +0 -1812
  138. package/src/utils/synonyms.ts +0 -80
  139. package/src/utils/templateUtils.ts +0 -78
  140. package/src/version.ts +0 -6
@@ -1,172 +0,0 @@
1
- import axios, { AxiosInstance } from 'axios';
2
- import { VectorDBConfig } from '../../config/RagConfig';
3
- import { BaseVectorProvider } from './BaseVectorProvider';
4
- import { VectorMatch, UpsertDocument } from '../../types';
5
-
6
- /**
7
- * WeaviateProvider — Weaviate implementation using its REST/GraphQL API.
8
- *
9
- * Required options:
10
- * { baseUrl: string } — e.g. "http://localhost:8080"
11
- * OR { url: string } — alias accepted for backwards compatibility
12
- *
13
- * Optional: { apiKey?: string }
14
- */
15
- export class WeaviateProvider extends BaseVectorProvider {
16
- private http: AxiosInstance;
17
-
18
- constructor(config: VectorDBConfig) {
19
- super(config);
20
- const opts = config.options as Record<string, unknown>;
21
-
22
- // Accept baseUrl or url alias
23
- const baseUrl = (opts.baseUrl as string) || (opts.url as string);
24
- if (!baseUrl) throw new Error('[WeaviateProvider] options.baseUrl is required');
25
-
26
- this.http = axios.create({
27
- baseURL: baseUrl,
28
- headers: {
29
- 'Content-Type': 'application/json',
30
- ...(opts.apiKey ? {
31
- 'X-OpenAI-Api-Key': opts.apiKey as string,
32
- 'Authorization': `Bearer ${opts.apiKey as string}`,
33
- } : {}),
34
- },
35
- });
36
- }
37
-
38
- async initialize(): Promise<void> {
39
- await this.ping();
40
- }
41
-
42
- async upsert(doc: UpsertDocument, namespace?: string): Promise<void> {
43
- const primitiveMetadata = this.extractPrimitiveMetadata(doc.metadata);
44
- const payload = {
45
- class: this.indexName,
46
- id: doc.id,
47
- vector: doc.vector,
48
- properties: {
49
- content: doc.content,
50
- metadata: JSON.stringify(doc.metadata || {}),
51
- ...primitiveMetadata,
52
- namespace: namespace || '',
53
- },
54
- };
55
- await this.http.post('/v1/objects', payload);
56
- }
57
-
58
- async batchUpsert(docs: UpsertDocument[], namespace?: string): Promise<void> {
59
- const payload = {
60
- objects: docs.map(doc => ({
61
- class: this.indexName,
62
- id: doc.id,
63
- vector: doc.vector,
64
- properties: {
65
- content: doc.content,
66
- metadata: JSON.stringify(doc.metadata || {}),
67
- ...this.extractPrimitiveMetadata(doc.metadata),
68
- namespace: namespace || '',
69
- },
70
- })),
71
- };
72
- await this.http.post('/v1/batch/objects', payload);
73
- }
74
-
75
- async query(vector: number[], topK: number, namespace?: string, _filter?: Record<string, unknown>): Promise<VectorMatch[]> {
76
- // IMPORTANT: read queryText BEFORE calling sanitizeFilter() because sanitizeFilter strips it.
77
- const queryText = _filter?.queryText as string | undefined;
78
- const sanitizedFilter = this.sanitizeFilter(_filter);
79
-
80
- const searchParams = queryText
81
- ? `hybrid: { query: ${JSON.stringify(queryText)}, alpha: 0.5 }`
82
- : `nearVector: { vector: ${JSON.stringify(vector)} }`;
83
- const where = this.buildWhereFilter(namespace, sanitizedFilter);
84
-
85
- const graphqlQuery = {
86
- query: `
87
- {
88
- Get {
89
- ${this.indexName}(
90
- ${searchParams}
91
- limit: ${topK}
92
- ${where ? `where: ${where}` : ''}
93
- ) {
94
- content
95
- metadata
96
- _additional {
97
- id
98
- distance
99
- }
100
- }
101
- }
102
- }
103
- `
104
- };
105
- const { data } = await this.http.post('/v1/graphql', graphqlQuery);
106
- const results = data.data?.Get?.[this.indexName] || [];
107
-
108
- return results.map((res: Record<string, unknown>) => ({
109
- id: (res['_additional'] as Record<string, unknown>).id as string,
110
- score: 1 - ((res['_additional'] as Record<string, unknown>).distance as number),
111
- content: res['content'] as string,
112
- metadata: res['metadata'] ? JSON.parse(res['metadata'] as string) : {},
113
- }));
114
- }
115
-
116
- async delete(id: string, _namespace?: string): Promise<void> {
117
- await this.http.delete(`/v1/objects/${this.indexName}/${id}`);
118
- }
119
-
120
- async deleteNamespace(namespace: string): Promise<void> {
121
- await this.http.post('/v1/batch/objects', {
122
- match: {
123
- class: this.indexName,
124
- where: { path: ['namespace'], operator: 'Equal', valueString: namespace },
125
- },
126
- });
127
- }
128
-
129
- async ping(): Promise<boolean> {
130
- try {
131
- await this.http.get('/v1/.well-known/live');
132
- return true;
133
- } catch {
134
- return false;
135
- }
136
- }
137
-
138
- async disconnect(): Promise<void> {}
139
-
140
- private extractPrimitiveMetadata(metadata?: Record<string, unknown>): Record<string, string | number | boolean> {
141
- const result: Record<string, string | number | boolean> = {};
142
- Object.entries(metadata || {}).forEach(([key, value]) => {
143
- if (typeof value === 'string' || typeof value === 'number' || typeof value === 'boolean') {
144
- result[key] = value;
145
- }
146
- });
147
- return result;
148
- }
149
-
150
- private buildWhereFilter(namespace?: string, filter?: Record<string, unknown>): string | undefined {
151
- const operands: string[] = [];
152
- if (namespace) operands.push(this.weaviateOperand('namespace', namespace));
153
-
154
- Object.entries(filter || {}).forEach(([key, value]) => {
155
- if (key === 'namespace') return;
156
- if (typeof value === 'string' || typeof value === 'number' || typeof value === 'boolean') {
157
- operands.push(this.weaviateOperand(key, value));
158
- }
159
- });
160
-
161
- if (operands.length === 0) return undefined;
162
- if (operands.length === 1) return operands[0];
163
- return `{ operator: And, operands: [${operands.join(', ')}] }`;
164
- }
165
-
166
- private weaviateOperand(key: string, value: string | number | boolean): string {
167
- const path = `path: [${JSON.stringify(key)}], operator: Equal`;
168
- if (typeof value === 'number') return `{ ${path}, valueNumber: ${value} }`;
169
- if (typeof value === 'boolean') return `{ ${path}, valueBoolean: ${value} }`;
170
- return `{ ${path}, valueString: ${JSON.stringify(value)} }`;
171
- }
172
- }
@@ -1,144 +0,0 @@
1
- /**
2
- * DocumentChunker — splits long text into overlapping chunks
3
- * suitable for vector embedding and retrieval.
4
- *
5
- * Strategy: sentence-aware sliding window
6
- * 1. Split on sentence boundaries
7
- * 2. Accumulate sentences into chunks up to `chunkSize` characters
8
- * 3. Carry over `chunkOverlap` characters from the previous chunk
9
- */
10
-
11
- export interface Chunk {
12
- id: string;
13
- content: string;
14
- metadata?: Record<string, unknown>;
15
- }
16
-
17
- export interface ChunkOptions {
18
- /** Target chunk size in characters (default 1000) */
19
- chunkSize?: number;
20
- /** Overlap between consecutive chunks in characters (default 200) */
21
- chunkOverlap?: number;
22
- /** Source document identifier used as ID prefix */
23
- docId?: string | number;
24
- /** Extra metadata to attach to every chunk */
25
- metadata?: Record<string, unknown>;
26
- /** Characters used to split text, in order of priority */
27
- separators?: string[];
28
- }
29
-
30
- export class DocumentChunker {
31
- private readonly chunkSize: number;
32
- private readonly chunkOverlap: number;
33
- private readonly separators: string[];
34
-
35
- constructor(
36
- chunkSize = 1000,
37
- chunkOverlap = 200,
38
- separators = ['\n# ', '\n## ', '\n### ', '\n#### ', '\n\n', '\n', ' ', '']
39
- ) {
40
- this.chunkSize = chunkSize;
41
- this.chunkOverlap = chunkOverlap;
42
- this.separators = separators;
43
- }
44
-
45
- /**
46
- * Split a single text string into overlapping chunks using a recursive strategy.
47
- * Preserves structural boundaries (Markdown headers) where possible.
48
- */
49
- chunk(text: string, options: ChunkOptions = {}): Chunk[] {
50
- const {
51
- chunkSize = this.chunkSize,
52
- chunkOverlap = this.chunkOverlap,
53
- docId = `doc_${Date.now()}`,
54
- metadata = {},
55
- separators = this.separators,
56
- } = options;
57
-
58
- const finalChunks: Chunk[] = [];
59
- const splits = this.recursiveSplit(text, separators, chunkSize);
60
-
61
- let currentChunk: string[] = [];
62
- let currentLength = 0;
63
- let chunkIndex = 0;
64
-
65
- for (const split of splits) {
66
- // If adding this split exceeds chunk size, and we already have content, close the chunk
67
- if (currentLength + split.length > chunkSize && currentChunk.length > 0) {
68
- finalChunks.push({
69
- id: `${docId}_chunk_${chunkIndex++}`,
70
- content: currentChunk.join('').trim(),
71
- metadata: { ...metadata, docId, chunkIndex: chunkIndex - 1 },
72
- });
73
-
74
- // Sliding window overlap logic
75
- const overlapItems = [];
76
- let overlapLen = 0;
77
- // Search backwards through current chunk to find segments that fit within the overlap window
78
- for (let i = currentChunk.length - 1; i >= 0; i--) {
79
- if (overlapLen + currentChunk[i].length <= chunkOverlap) {
80
- overlapItems.unshift(currentChunk[i]);
81
- overlapLen += currentChunk[i].length;
82
- } else {
83
- break;
84
- }
85
- }
86
- currentChunk = overlapItems;
87
- currentLength = overlapLen;
88
- }
89
-
90
- currentChunk.push(split);
91
- currentLength += split.length;
92
- }
93
-
94
- // Push the last remaining chunk
95
- if (currentChunk.length > 0) {
96
- finalChunks.push({
97
- id: `${docId}_chunk_${chunkIndex}`,
98
- content: currentChunk.join('').trim(),
99
- metadata: { ...metadata, docId, chunkIndex },
100
- });
101
- }
102
-
103
- return finalChunks;
104
- }
105
-
106
- private recursiveSplit(text: string, separators: string[], chunkSize: number): string[] {
107
- const finalSplits: string[] = [];
108
-
109
- let separator = separators[separators.length - 1];
110
- let nextSeparators: string[] = [];
111
-
112
- for (let i = 0; i < separators.length; i++) {
113
- const sep = separators[i];
114
- if (text.includes(sep)) {
115
- separator = sep;
116
- nextSeparators = separators.slice(i + 1);
117
- break;
118
- }
119
- }
120
-
121
- const parts = text.split(separator);
122
-
123
- for (const part of parts) {
124
- if (part.length <= chunkSize) {
125
- // Keep the separator to preserve formatting/meaning
126
- finalSplits.push(part + (part === parts[parts.length - 1] ? '' : separator));
127
- } else if (nextSeparators.length > 0) {
128
- finalSplits.push(...this.recursiveSplit(part, nextSeparators, chunkSize));
129
- } else {
130
- finalSplits.push(part);
131
- }
132
- }
133
-
134
- return finalSplits;
135
- }
136
-
137
- chunkMany(
138
- documents: Array<{ content: string; docId?: string | number; metadata?: Record<string, unknown> }>
139
- ): Chunk[] {
140
- return documents.flatMap((doc) =>
141
- this.chunk(doc.content, { docId: doc.docId, metadata: doc.metadata })
142
- );
143
- }
144
- }
@@ -1,90 +0,0 @@
1
- import { ILLMProvider } from '../llm/ILLMProvider';
2
- import { GraphNode, Edge } from '../types';
3
-
4
- /**
5
- * EntityExtractor — Uses an LLM to extract entities and relations from text.
6
- */
7
- export class EntityExtractor {
8
- private llm: ILLMProvider;
9
-
10
- constructor(llm: ILLMProvider) {
11
- this.llm = llm;
12
- }
13
-
14
- /**
15
- * Extract nodes and edges from a text chunk.
16
- */
17
- async extract(text: string): Promise<{ nodes: GraphNode[]; edges: Edge[] }> {
18
- const prompt = `
19
- Extract entities and relationships from the following text.
20
- Format the output as a JSON object with exactly two keys: "nodes" and "edges".
21
-
22
- Nodes: {id, label, properties}
23
- Edges: {source, target, type, properties}
24
-
25
- IMPORTANT:
26
- - Ensure all property values are simple JSON types (string, number, boolean).
27
- - Do not use mathematical expressions or complex nested objects in properties.
28
- - If no entities are found, return empty arrays.
29
- - RESPOND ONLY WITH THE JSON OBJECT.
30
-
31
- Text to extract from:
32
- "${text}"
33
- `;
34
-
35
- const response = await this.llm.chat([
36
- { role: 'system', content: 'You are a precise knowledge graph extraction engine. You always output valid JSON and nothing else.' },
37
- { role: 'user', content: prompt }
38
- ], '', { maxTokens: 4096, temperature: 0.1 });
39
-
40
- try {
41
- // 1. Try to find the JSON block using a regex
42
- const jsonMatch = response.match(/\{[\s\S]*\}/);
43
- const cleanJson = jsonMatch ? jsonMatch[0] : response.trim();
44
-
45
- return JSON.parse(cleanJson);
46
- } catch {
47
- // 2. If parsing fails, try to repair truncated JSON
48
- try {
49
- const repaired = this.repairTruncatedJson(response);
50
- if (repaired) return JSON.parse(repaired);
51
- } catch { /* ignore repair failure */ }
52
-
53
- console.warn(`[EntityExtractor] Failed to parse LLM response. Length: ${response.length} chars.`);
54
- console.warn('[EntityExtractor] Snippet:', response.substring(0, 100) + '...');
55
- return { nodes: [], edges: [] };
56
- }
57
- }
58
-
59
- /**
60
- * Attempts to fix JSON that was cut off mid-generation.
61
- * Strategy: Find the last valid element in an array and close the structure.
62
- */
63
- private repairTruncatedJson(json: string): string | null {
64
- let text = json.trim();
65
-
66
- // Find the first { and work from there
67
- const startIdx = text.indexOf('{');
68
- if (startIdx === -1) return null;
69
- text = text.substring(startIdx);
70
-
71
- // If it's truncated in an array like "nodes": [ ... , { ...
72
- // We try to find the last complete object before the truncation
73
- const lastCompleteObjectIdx = text.lastIndexOf('}');
74
- if (lastCompleteObjectIdx === -1) return null;
75
-
76
- // Truncate at the last complete object
77
- let repaired = text.substring(0, lastCompleteObjectIdx + 1);
78
-
79
- // Close any open structures (arrays or the main object)
80
- const openBrackets = (repaired.match(/\{/g) || []).length;
81
- const closedBrackets = (repaired.match(/\}/g) || []).length;
82
- const openSquares = (repaired.match(/\[/g) || []).length;
83
- const closedSquares = (repaired.match(/\]/g) || []).length;
84
-
85
- for (let i = 0; i < openSquares - closedSquares; i++) repaired += ']';
86
- for (let i = 0; i < openBrackets - closedBrackets; i++) repaired += '}';
87
-
88
- return repaired;
89
- }
90
- }
@@ -1,60 +0,0 @@
1
- import { Chunk, ChunkOptions } from './DocumentChunker';
2
-
3
- /**
4
- * LlamaIndexIngestor — Advanced ingestion provider using LlamaIndex (TS).
5
- *
6
- * Provides semantic chunking, metadata extraction, and node parsing
7
- * superior to basic recursive character splitting.
8
- *
9
- * Note: Requires 'llamaindex' package to be installed.
10
- */
11
- export class LlamaIndexIngestor {
12
- /**
13
- * Chunks document content using LlamaIndex SentenceSplitter.
14
- * This respects sentence and paragraph boundaries much more effectively
15
- * than standard character-count splitting.
16
- */
17
- async chunk(text: string, options: ChunkOptions = {}): Promise<Chunk[]> {
18
- try {
19
- // Dynamically import for optional dependency support using template string to hide from TS static analysis
20
- const { Document, SentenceSplitter, MetadataMode } = await import(`${'llamaindex'}`);
21
-
22
- const splitter = new SentenceSplitter({
23
- chunkSize: options.chunkSize ?? 1000,
24
- chunkOverlap: options.chunkOverlap ?? 200,
25
- });
26
-
27
- const doc = new Document({ text, metadata: options.metadata || {} });
28
- const nodes = splitter.getNodesFromDocuments([doc]);
29
-
30
- return (nodes as any[]).map((node: any, index: number) => ({
31
- id: `${options.docId || 'doc'}_node_${index}`,
32
- content: node.getContent(MetadataMode.ALL),
33
- metadata: {
34
- ...options.metadata,
35
- ...node.metadata,
36
- nodeId: node.id_,
37
- chunkIndex: index
38
- }
39
- }));
40
- } catch (error) {
41
- console.warn('[LlamaIndexIngestor] LlamaIndex package not found or failed. Falling back to default chunker.');
42
- throw error;
43
- }
44
- }
45
-
46
- /**
47
- * Batch processing for multiple documents.
48
- */
49
- async chunkMany(
50
- documents: Array<{ content: string; docId?: string | number; metadata?: Record<string, unknown> }>,
51
- options: ChunkOptions = {}
52
- ): Promise<Chunk[]> {
53
- const allChunks: Chunk[] = [];
54
- for (const doc of documents) {
55
- const chunks = await this.chunk(doc.content, { ...options, docId: doc.docId, metadata: doc.metadata });
56
- allChunks.push(...chunks);
57
- }
58
- return allChunks;
59
- }
60
- }
@@ -1,106 +0,0 @@
1
- import { VectorMatch } from '../types';
2
- import { ILLMProvider } from '../llm/ILLMProvider';
3
-
4
- export class Reranker {
5
- /**
6
- * Re-ranks matches based on a secondary relevance score using an LLM.
7
- */
8
- async rerank(
9
- matches: VectorMatch[],
10
- query: string,
11
- limit: number = 5,
12
- llm?: ILLMProvider
13
- ): Promise<VectorMatch[]> {
14
- if (!llm || matches.length <= 1) {
15
- // Fallback to deterministic keyword-based re-ranking for production speed
16
- const keywords = query.toLowerCase()
17
- .replace(/[^\w\s]/g, '')
18
- .split(/\s+/)
19
- .filter(w => w.length > 2); // Filter out short words/stop words
20
-
21
- if (keywords.length === 0) {
22
- return matches
23
- .sort((a, b) => b.score - a.score)
24
- .slice(0, limit);
25
- }
26
-
27
- const scoredMatches = matches.map(match => {
28
- const contentLower = (match?.content ?? '').toLowerCase();
29
- let keywordScore = 0;
30
-
31
- keywords.forEach(keyword => {
32
- if (contentLower.includes(keyword)) {
33
- keywordScore += 1;
34
- }
35
- });
36
-
37
- // Normalize keyword score (fraction of keywords found)
38
- const normalizedKeywordScore = keywordScore / keywords.length;
39
-
40
- // Combine scores: Vector score (usually 0-1) + Keyword score weighted
41
- // We give a significant weight to keyword matches to surface specific results
42
- const combinedScore = match.score + (normalizedKeywordScore * 0.5);
43
-
44
- return {
45
- ...match,
46
- score: combinedScore
47
- };
48
- });
49
-
50
- return scoredMatches
51
- .sort((a, b) => b.score - a.score)
52
- .slice(0, limit);
53
- }
54
-
55
- try {
56
- // Take top 10 matches to avoid large context and high latency
57
- const topN = matches.slice(0, 10);
58
-
59
- const response = await llm.chat(
60
- [{ role: 'user', content: `Query: "${query}"\n\nDocuments:\n${topN.map((m, i) => `[${i}] ${(m?.content ?? '').replace(/\n/g, ' ')}`).join('\n')}` }],
61
- '',
62
- {
63
- systemPrompt: 'You are a relevance ranking expert. Given the user query and a list of retrieved document chunks, rank the chunks by relevance to the query. Return the result as a comma-separated list of document indices in order of relevance, from most relevant to least relevant. Use the indices provided in brackets like [0], [1], etc. Only return the indices (e.g., "2,0,1"), nothing else. Do not include brackets in the output.',
64
- temperature: 0,
65
- maxTokens: 50
66
- }
67
- );
68
-
69
- const cleanedResponse = response.trim().replace(/[\[\]\s]/g, '');
70
- const rankedIndices = cleanedResponse.split(',').map(Number).filter(n => !isNaN(n));
71
-
72
- if (rankedIndices.length > 0) {
73
- const rankedMatches: VectorMatch[] = [];
74
- const usedIndices = new Set<number>();
75
-
76
- // Add matches in the order specified by the LLM
77
- for (const index of rankedIndices) {
78
- if (index >= 0 && index < topN.length && !usedIndices.has(index)) {
79
- rankedMatches.push(topN[index]);
80
- usedIndices.add(index);
81
- }
82
- }
83
-
84
- // Add any remaining matches that the LLM might have missed
85
- for (let i = 0; i < topN.length; i++) {
86
- if (!usedIndices.has(i)) {
87
- rankedMatches.push(topN[i]);
88
- }
89
- }
90
-
91
- // Add back the rest of the matches that weren't in the top N
92
- const rest = matches.slice(10);
93
- rankedMatches.push(...rest);
94
-
95
- return rankedMatches.slice(0, limit);
96
- }
97
- } catch (error) {
98
- console.warn('[Reranker] LLM re-ranking failed, falling back to score-based sorting:', error);
99
- }
100
-
101
- // Fallback if LLM fails or returns invalid response
102
- return matches
103
- .sort((a, b) => b.score - a.score)
104
- .slice(0, limit);
105
- }
106
- }