@retrivora-ai/rag-engine 2.4.2 → 2.4.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{BatchProcessor-zbQMaZHm.d.mts → BatchProcessor-Csy_K1qQ.d.mts} +2 -0
- package/dist/{BatchProcessor-D42E706a.d.ts → BatchProcessor-Dg18zZGk.d.ts} +2 -0
- package/dist/handlers/index.js +31 -5
- package/dist/handlers/index.mjs +31 -5
- package/dist/index.d.mts +3 -3
- package/dist/index.d.ts +3 -3
- package/dist/index.js +41 -21
- package/dist/index.min.css +2 -0
- package/dist/index.mjs +41 -21
- package/dist/server.d.mts +2 -2
- package/dist/server.d.ts +2 -2
- package/dist/server.js +32 -5
- package/dist/server.mjs +32 -5
- package/package.json +5 -4
- package/src/LicensedRetrivora.ts +0 -278
- package/src/app/api/chat/route.ts +0 -4
- package/src/app/api/health/route.ts +0 -5
- package/src/app/api/ingest/route.ts +0 -27
- package/src/app/api/suggestions/route.ts +0 -5
- package/src/app/api/upload/route.ts +0 -5
- package/src/app/constants.tsx +0 -270
- package/src/app/favicon.ico +0 -0
- package/src/app/globals.css +0 -449
- package/src/app/layout.tsx +0 -22
- package/src/app/page.tsx +0 -64
- package/src/app/types.ts +0 -30
- package/src/browser.tsx +0 -139
- package/src/components/ChatWidget.tsx +0 -393
- package/src/components/ChatWindow.tsx +0 -952
- package/src/components/CodeViewer.tsx +0 -56
- package/src/components/ConfigProvider.tsx +0 -123
- package/src/components/DocumentUpload.tsx +0 -354
- package/src/components/DynamicChart.tsx +0 -297
- package/src/components/HourglassLoader.tsx +0 -87
- package/src/components/MarkdownComponents.tsx +0 -186
- package/src/components/MessageBubble.tsx +0 -498
- package/src/components/ObservabilityPanel.tsx +0 -374
- package/src/components/ProductCard.tsx +0 -107
- package/src/components/ProductCarousel.tsx +0 -97
- package/src/components/SourceCard.tsx +0 -61
- package/src/components/ThemeProvider.tsx +0 -8
- package/src/components/ThemeToggle.tsx +0 -29
- package/src/components/ThinkingBlock.tsx +0 -75
- package/src/components/UIDispatcher.tsx +0 -342
- package/src/components/VisualizationRenderer.tsx +0 -621
- package/src/components/constants.tsx +0 -503
- package/src/config/ConfigBuilder.ts +0 -285
- package/src/config/EmbeddingStrategy.ts +0 -148
- package/src/config/LicenseConfigResolver.ts +0 -298
- package/src/config/RagConfig.ts +0 -328
- package/src/config/UniversalProfiles.ts +0 -82
- package/src/config/constants.ts +0 -64
- package/src/config/serverConfig.ts +0 -378
- package/src/config/uiConstants.ts +0 -47
- package/src/core/BatchProcessor.ts +0 -385
- package/src/core/CircuitBreaker.ts +0 -118
- package/src/core/ConfigFetcher.ts +0 -220
- package/src/core/ConfigResolver.ts +0 -165
- package/src/core/ConfigValidator.ts +0 -197
- package/src/core/DatabaseStorage.ts +0 -526
- package/src/core/FreeTierLimitsGuard.ts +0 -281
- package/src/core/LLMRouter.ts +0 -79
- package/src/core/LangChainAgent.ts +0 -132
- package/src/core/LicenseValidator.ts +0 -284
- package/src/core/LicenseVerifier.ts +0 -261
- package/src/core/MultiAgentCoordinator.ts +0 -239
- package/src/core/Pipeline.ts +0 -1279
- package/src/core/PluginManager.ts +0 -277
- package/src/core/ProjectSettings.ts +0 -247
- package/src/core/ProviderHealthCheck.ts +0 -133
- package/src/core/ProviderInterfaces.ts +0 -37
- package/src/core/ProviderRegistry.ts +0 -156
- package/src/core/QueryProcessor.ts +0 -399
- package/src/core/Retrivora.ts +0 -98
- package/src/core/VectorPlugin.ts +0 -193
- package/src/core/mcp.ts +0 -261
- package/src/exceptions/index.ts +0 -131
- package/src/handlers/index.ts +0 -1625
- package/src/hooks/useHydrated.ts +0 -13
- package/src/hooks/useProjectSettings.ts +0 -105
- package/src/hooks/useRagChat.ts +0 -436
- package/src/hooks/useStoredMessages.ts +0 -64
- package/src/index.css +0 -4269
- package/src/index.ts +0 -137
- package/src/lib/plugin.ts +0 -24
- package/src/llm/ILLMProvider.ts +0 -60
- package/src/llm/LLMFactory.ts +0 -141
- package/src/llm/providers/AnthropicProvider.ts +0 -215
- package/src/llm/providers/GeminiProvider.ts +0 -311
- package/src/llm/providers/GroqProvider.ts +0 -184
- package/src/llm/providers/OllamaProvider.ts +0 -232
- package/src/llm/providers/OpenAIProvider.ts +0 -188
- package/src/llm/providers/QwenProvider.ts +0 -199
- package/src/llm/providers/UniversalLLMAdapter.ts +0 -422
- package/src/llm/utils.ts +0 -46
- package/src/providers/graphdb/BaseGraphProvider.ts +0 -43
- package/src/providers/graphdb/SimpleGraphProvider.ts +0 -66
- package/src/providers/vectordb/BaseVectorProvider.ts +0 -98
- package/src/providers/vectordb/ChromaDBProvider.ts +0 -130
- package/src/providers/vectordb/MilvusProvider.ts +0 -134
- package/src/providers/vectordb/MongoDBProvider.ts +0 -250
- package/src/providers/vectordb/MultiTablePostgresProvider.ts +0 -461
- package/src/providers/vectordb/PineconeProvider.ts +0 -295
- package/src/providers/vectordb/PostgreSQLProvider.ts +0 -227
- package/src/providers/vectordb/QdrantProvider.ts +0 -294
- package/src/providers/vectordb/RedisProvider.ts +0 -105
- package/src/providers/vectordb/UniversalVectorProvider.ts +0 -221
- package/src/providers/vectordb/WeaviateProvider.ts +0 -172
- package/src/rag/DocumentChunker.ts +0 -144
- package/src/rag/EntityExtractor.ts +0 -90
- package/src/rag/LlamaIndexIngestor.ts +0 -60
- package/src/rag/Reranker.ts +0 -106
- package/src/rendering/IntentClassifier.ts +0 -167
- package/src/rendering/RendererRegistry.ts +0 -164
- package/src/rendering/RuleEngine.ts +0 -67
- package/src/rendering/VisualizationDecisionEngine.ts +0 -164
- package/src/rendering/__tests__/VisualizationDecisionEngine.test.ts +0 -237
- package/src/rendering/index.ts +0 -12
- package/src/rendering/rules/Rule1SpecificInfoRule.ts +0 -28
- package/src/rendering/rules/Rule2ComparisonRule.ts +0 -24
- package/src/rendering/rules/Rule3ProductDiscoveryRule.ts +0 -28
- package/src/rendering/rules/Rule4AnalyticalRule.ts +0 -38
- package/src/rendering/rules/Rule5MixedResponseRule.ts +0 -29
- package/src/rendering/rules/Rule6SmallResultSetRule.ts +0 -24
- package/src/rendering/rules/Rule7LargeTableRule.ts +0 -35
- package/src/rendering/types.ts +0 -96
- package/src/server.ts +0 -156
- package/src/tailwind.css +0 -194
- package/src/telemetry/TelemetryService.ts +0 -330
- package/src/types/chat.ts +0 -89
- package/src/types/index.ts +0 -278
- package/src/types/pdf-parse.d.ts +0 -13
- package/src/types/props.ts +0 -169
- package/src/utils/DocumentParser.ts +0 -111
- package/src/utils/ProductExtractor.ts +0 -350
- package/src/utils/SchemaMapper.ts +0 -176
- package/src/utils/UITransformer.ts +0 -1812
- package/src/utils/synonyms.ts +0 -80
- package/src/utils/templateUtils.ts +0 -78
- package/src/version.ts +0 -6
|
@@ -1,172 +0,0 @@
|
|
|
1
|
-
import axios, { AxiosInstance } from 'axios';
|
|
2
|
-
import { VectorDBConfig } from '../../config/RagConfig';
|
|
3
|
-
import { BaseVectorProvider } from './BaseVectorProvider';
|
|
4
|
-
import { VectorMatch, UpsertDocument } from '../../types';
|
|
5
|
-
|
|
6
|
-
/**
|
|
7
|
-
* WeaviateProvider — Weaviate implementation using its REST/GraphQL API.
|
|
8
|
-
*
|
|
9
|
-
* Required options:
|
|
10
|
-
* { baseUrl: string } — e.g. "http://localhost:8080"
|
|
11
|
-
* OR { url: string } — alias accepted for backwards compatibility
|
|
12
|
-
*
|
|
13
|
-
* Optional: { apiKey?: string }
|
|
14
|
-
*/
|
|
15
|
-
export class WeaviateProvider extends BaseVectorProvider {
|
|
16
|
-
private http: AxiosInstance;
|
|
17
|
-
|
|
18
|
-
constructor(config: VectorDBConfig) {
|
|
19
|
-
super(config);
|
|
20
|
-
const opts = config.options as Record<string, unknown>;
|
|
21
|
-
|
|
22
|
-
// Accept baseUrl or url alias
|
|
23
|
-
const baseUrl = (opts.baseUrl as string) || (opts.url as string);
|
|
24
|
-
if (!baseUrl) throw new Error('[WeaviateProvider] options.baseUrl is required');
|
|
25
|
-
|
|
26
|
-
this.http = axios.create({
|
|
27
|
-
baseURL: baseUrl,
|
|
28
|
-
headers: {
|
|
29
|
-
'Content-Type': 'application/json',
|
|
30
|
-
...(opts.apiKey ? {
|
|
31
|
-
'X-OpenAI-Api-Key': opts.apiKey as string,
|
|
32
|
-
'Authorization': `Bearer ${opts.apiKey as string}`,
|
|
33
|
-
} : {}),
|
|
34
|
-
},
|
|
35
|
-
});
|
|
36
|
-
}
|
|
37
|
-
|
|
38
|
-
async initialize(): Promise<void> {
|
|
39
|
-
await this.ping();
|
|
40
|
-
}
|
|
41
|
-
|
|
42
|
-
async upsert(doc: UpsertDocument, namespace?: string): Promise<void> {
|
|
43
|
-
const primitiveMetadata = this.extractPrimitiveMetadata(doc.metadata);
|
|
44
|
-
const payload = {
|
|
45
|
-
class: this.indexName,
|
|
46
|
-
id: doc.id,
|
|
47
|
-
vector: doc.vector,
|
|
48
|
-
properties: {
|
|
49
|
-
content: doc.content,
|
|
50
|
-
metadata: JSON.stringify(doc.metadata || {}),
|
|
51
|
-
...primitiveMetadata,
|
|
52
|
-
namespace: namespace || '',
|
|
53
|
-
},
|
|
54
|
-
};
|
|
55
|
-
await this.http.post('/v1/objects', payload);
|
|
56
|
-
}
|
|
57
|
-
|
|
58
|
-
async batchUpsert(docs: UpsertDocument[], namespace?: string): Promise<void> {
|
|
59
|
-
const payload = {
|
|
60
|
-
objects: docs.map(doc => ({
|
|
61
|
-
class: this.indexName,
|
|
62
|
-
id: doc.id,
|
|
63
|
-
vector: doc.vector,
|
|
64
|
-
properties: {
|
|
65
|
-
content: doc.content,
|
|
66
|
-
metadata: JSON.stringify(doc.metadata || {}),
|
|
67
|
-
...this.extractPrimitiveMetadata(doc.metadata),
|
|
68
|
-
namespace: namespace || '',
|
|
69
|
-
},
|
|
70
|
-
})),
|
|
71
|
-
};
|
|
72
|
-
await this.http.post('/v1/batch/objects', payload);
|
|
73
|
-
}
|
|
74
|
-
|
|
75
|
-
async query(vector: number[], topK: number, namespace?: string, _filter?: Record<string, unknown>): Promise<VectorMatch[]> {
|
|
76
|
-
// IMPORTANT: read queryText BEFORE calling sanitizeFilter() because sanitizeFilter strips it.
|
|
77
|
-
const queryText = _filter?.queryText as string | undefined;
|
|
78
|
-
const sanitizedFilter = this.sanitizeFilter(_filter);
|
|
79
|
-
|
|
80
|
-
const searchParams = queryText
|
|
81
|
-
? `hybrid: { query: ${JSON.stringify(queryText)}, alpha: 0.5 }`
|
|
82
|
-
: `nearVector: { vector: ${JSON.stringify(vector)} }`;
|
|
83
|
-
const where = this.buildWhereFilter(namespace, sanitizedFilter);
|
|
84
|
-
|
|
85
|
-
const graphqlQuery = {
|
|
86
|
-
query: `
|
|
87
|
-
{
|
|
88
|
-
Get {
|
|
89
|
-
${this.indexName}(
|
|
90
|
-
${searchParams}
|
|
91
|
-
limit: ${topK}
|
|
92
|
-
${where ? `where: ${where}` : ''}
|
|
93
|
-
) {
|
|
94
|
-
content
|
|
95
|
-
metadata
|
|
96
|
-
_additional {
|
|
97
|
-
id
|
|
98
|
-
distance
|
|
99
|
-
}
|
|
100
|
-
}
|
|
101
|
-
}
|
|
102
|
-
}
|
|
103
|
-
`
|
|
104
|
-
};
|
|
105
|
-
const { data } = await this.http.post('/v1/graphql', graphqlQuery);
|
|
106
|
-
const results = data.data?.Get?.[this.indexName] || [];
|
|
107
|
-
|
|
108
|
-
return results.map((res: Record<string, unknown>) => ({
|
|
109
|
-
id: (res['_additional'] as Record<string, unknown>).id as string,
|
|
110
|
-
score: 1 - ((res['_additional'] as Record<string, unknown>).distance as number),
|
|
111
|
-
content: res['content'] as string,
|
|
112
|
-
metadata: res['metadata'] ? JSON.parse(res['metadata'] as string) : {},
|
|
113
|
-
}));
|
|
114
|
-
}
|
|
115
|
-
|
|
116
|
-
async delete(id: string, _namespace?: string): Promise<void> {
|
|
117
|
-
await this.http.delete(`/v1/objects/${this.indexName}/${id}`);
|
|
118
|
-
}
|
|
119
|
-
|
|
120
|
-
async deleteNamespace(namespace: string): Promise<void> {
|
|
121
|
-
await this.http.post('/v1/batch/objects', {
|
|
122
|
-
match: {
|
|
123
|
-
class: this.indexName,
|
|
124
|
-
where: { path: ['namespace'], operator: 'Equal', valueString: namespace },
|
|
125
|
-
},
|
|
126
|
-
});
|
|
127
|
-
}
|
|
128
|
-
|
|
129
|
-
async ping(): Promise<boolean> {
|
|
130
|
-
try {
|
|
131
|
-
await this.http.get('/v1/.well-known/live');
|
|
132
|
-
return true;
|
|
133
|
-
} catch {
|
|
134
|
-
return false;
|
|
135
|
-
}
|
|
136
|
-
}
|
|
137
|
-
|
|
138
|
-
async disconnect(): Promise<void> {}
|
|
139
|
-
|
|
140
|
-
private extractPrimitiveMetadata(metadata?: Record<string, unknown>): Record<string, string | number | boolean> {
|
|
141
|
-
const result: Record<string, string | number | boolean> = {};
|
|
142
|
-
Object.entries(metadata || {}).forEach(([key, value]) => {
|
|
143
|
-
if (typeof value === 'string' || typeof value === 'number' || typeof value === 'boolean') {
|
|
144
|
-
result[key] = value;
|
|
145
|
-
}
|
|
146
|
-
});
|
|
147
|
-
return result;
|
|
148
|
-
}
|
|
149
|
-
|
|
150
|
-
private buildWhereFilter(namespace?: string, filter?: Record<string, unknown>): string | undefined {
|
|
151
|
-
const operands: string[] = [];
|
|
152
|
-
if (namespace) operands.push(this.weaviateOperand('namespace', namespace));
|
|
153
|
-
|
|
154
|
-
Object.entries(filter || {}).forEach(([key, value]) => {
|
|
155
|
-
if (key === 'namespace') return;
|
|
156
|
-
if (typeof value === 'string' || typeof value === 'number' || typeof value === 'boolean') {
|
|
157
|
-
operands.push(this.weaviateOperand(key, value));
|
|
158
|
-
}
|
|
159
|
-
});
|
|
160
|
-
|
|
161
|
-
if (operands.length === 0) return undefined;
|
|
162
|
-
if (operands.length === 1) return operands[0];
|
|
163
|
-
return `{ operator: And, operands: [${operands.join(', ')}] }`;
|
|
164
|
-
}
|
|
165
|
-
|
|
166
|
-
private weaviateOperand(key: string, value: string | number | boolean): string {
|
|
167
|
-
const path = `path: [${JSON.stringify(key)}], operator: Equal`;
|
|
168
|
-
if (typeof value === 'number') return `{ ${path}, valueNumber: ${value} }`;
|
|
169
|
-
if (typeof value === 'boolean') return `{ ${path}, valueBoolean: ${value} }`;
|
|
170
|
-
return `{ ${path}, valueString: ${JSON.stringify(value)} }`;
|
|
171
|
-
}
|
|
172
|
-
}
|
|
@@ -1,144 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* DocumentChunker — splits long text into overlapping chunks
|
|
3
|
-
* suitable for vector embedding and retrieval.
|
|
4
|
-
*
|
|
5
|
-
* Strategy: sentence-aware sliding window
|
|
6
|
-
* 1. Split on sentence boundaries
|
|
7
|
-
* 2. Accumulate sentences into chunks up to `chunkSize` characters
|
|
8
|
-
* 3. Carry over `chunkOverlap` characters from the previous chunk
|
|
9
|
-
*/
|
|
10
|
-
|
|
11
|
-
export interface Chunk {
|
|
12
|
-
id: string;
|
|
13
|
-
content: string;
|
|
14
|
-
metadata?: Record<string, unknown>;
|
|
15
|
-
}
|
|
16
|
-
|
|
17
|
-
export interface ChunkOptions {
|
|
18
|
-
/** Target chunk size in characters (default 1000) */
|
|
19
|
-
chunkSize?: number;
|
|
20
|
-
/** Overlap between consecutive chunks in characters (default 200) */
|
|
21
|
-
chunkOverlap?: number;
|
|
22
|
-
/** Source document identifier used as ID prefix */
|
|
23
|
-
docId?: string | number;
|
|
24
|
-
/** Extra metadata to attach to every chunk */
|
|
25
|
-
metadata?: Record<string, unknown>;
|
|
26
|
-
/** Characters used to split text, in order of priority */
|
|
27
|
-
separators?: string[];
|
|
28
|
-
}
|
|
29
|
-
|
|
30
|
-
export class DocumentChunker {
|
|
31
|
-
private readonly chunkSize: number;
|
|
32
|
-
private readonly chunkOverlap: number;
|
|
33
|
-
private readonly separators: string[];
|
|
34
|
-
|
|
35
|
-
constructor(
|
|
36
|
-
chunkSize = 1000,
|
|
37
|
-
chunkOverlap = 200,
|
|
38
|
-
separators = ['\n# ', '\n## ', '\n### ', '\n#### ', '\n\n', '\n', ' ', '']
|
|
39
|
-
) {
|
|
40
|
-
this.chunkSize = chunkSize;
|
|
41
|
-
this.chunkOverlap = chunkOverlap;
|
|
42
|
-
this.separators = separators;
|
|
43
|
-
}
|
|
44
|
-
|
|
45
|
-
/**
|
|
46
|
-
* Split a single text string into overlapping chunks using a recursive strategy.
|
|
47
|
-
* Preserves structural boundaries (Markdown headers) where possible.
|
|
48
|
-
*/
|
|
49
|
-
chunk(text: string, options: ChunkOptions = {}): Chunk[] {
|
|
50
|
-
const {
|
|
51
|
-
chunkSize = this.chunkSize,
|
|
52
|
-
chunkOverlap = this.chunkOverlap,
|
|
53
|
-
docId = `doc_${Date.now()}`,
|
|
54
|
-
metadata = {},
|
|
55
|
-
separators = this.separators,
|
|
56
|
-
} = options;
|
|
57
|
-
|
|
58
|
-
const finalChunks: Chunk[] = [];
|
|
59
|
-
const splits = this.recursiveSplit(text, separators, chunkSize);
|
|
60
|
-
|
|
61
|
-
let currentChunk: string[] = [];
|
|
62
|
-
let currentLength = 0;
|
|
63
|
-
let chunkIndex = 0;
|
|
64
|
-
|
|
65
|
-
for (const split of splits) {
|
|
66
|
-
// If adding this split exceeds chunk size, and we already have content, close the chunk
|
|
67
|
-
if (currentLength + split.length > chunkSize && currentChunk.length > 0) {
|
|
68
|
-
finalChunks.push({
|
|
69
|
-
id: `${docId}_chunk_${chunkIndex++}`,
|
|
70
|
-
content: currentChunk.join('').trim(),
|
|
71
|
-
metadata: { ...metadata, docId, chunkIndex: chunkIndex - 1 },
|
|
72
|
-
});
|
|
73
|
-
|
|
74
|
-
// Sliding window overlap logic
|
|
75
|
-
const overlapItems = [];
|
|
76
|
-
let overlapLen = 0;
|
|
77
|
-
// Search backwards through current chunk to find segments that fit within the overlap window
|
|
78
|
-
for (let i = currentChunk.length - 1; i >= 0; i--) {
|
|
79
|
-
if (overlapLen + currentChunk[i].length <= chunkOverlap) {
|
|
80
|
-
overlapItems.unshift(currentChunk[i]);
|
|
81
|
-
overlapLen += currentChunk[i].length;
|
|
82
|
-
} else {
|
|
83
|
-
break;
|
|
84
|
-
}
|
|
85
|
-
}
|
|
86
|
-
currentChunk = overlapItems;
|
|
87
|
-
currentLength = overlapLen;
|
|
88
|
-
}
|
|
89
|
-
|
|
90
|
-
currentChunk.push(split);
|
|
91
|
-
currentLength += split.length;
|
|
92
|
-
}
|
|
93
|
-
|
|
94
|
-
// Push the last remaining chunk
|
|
95
|
-
if (currentChunk.length > 0) {
|
|
96
|
-
finalChunks.push({
|
|
97
|
-
id: `${docId}_chunk_${chunkIndex}`,
|
|
98
|
-
content: currentChunk.join('').trim(),
|
|
99
|
-
metadata: { ...metadata, docId, chunkIndex },
|
|
100
|
-
});
|
|
101
|
-
}
|
|
102
|
-
|
|
103
|
-
return finalChunks;
|
|
104
|
-
}
|
|
105
|
-
|
|
106
|
-
private recursiveSplit(text: string, separators: string[], chunkSize: number): string[] {
|
|
107
|
-
const finalSplits: string[] = [];
|
|
108
|
-
|
|
109
|
-
let separator = separators[separators.length - 1];
|
|
110
|
-
let nextSeparators: string[] = [];
|
|
111
|
-
|
|
112
|
-
for (let i = 0; i < separators.length; i++) {
|
|
113
|
-
const sep = separators[i];
|
|
114
|
-
if (text.includes(sep)) {
|
|
115
|
-
separator = sep;
|
|
116
|
-
nextSeparators = separators.slice(i + 1);
|
|
117
|
-
break;
|
|
118
|
-
}
|
|
119
|
-
}
|
|
120
|
-
|
|
121
|
-
const parts = text.split(separator);
|
|
122
|
-
|
|
123
|
-
for (const part of parts) {
|
|
124
|
-
if (part.length <= chunkSize) {
|
|
125
|
-
// Keep the separator to preserve formatting/meaning
|
|
126
|
-
finalSplits.push(part + (part === parts[parts.length - 1] ? '' : separator));
|
|
127
|
-
} else if (nextSeparators.length > 0) {
|
|
128
|
-
finalSplits.push(...this.recursiveSplit(part, nextSeparators, chunkSize));
|
|
129
|
-
} else {
|
|
130
|
-
finalSplits.push(part);
|
|
131
|
-
}
|
|
132
|
-
}
|
|
133
|
-
|
|
134
|
-
return finalSplits;
|
|
135
|
-
}
|
|
136
|
-
|
|
137
|
-
chunkMany(
|
|
138
|
-
documents: Array<{ content: string; docId?: string | number; metadata?: Record<string, unknown> }>
|
|
139
|
-
): Chunk[] {
|
|
140
|
-
return documents.flatMap((doc) =>
|
|
141
|
-
this.chunk(doc.content, { docId: doc.docId, metadata: doc.metadata })
|
|
142
|
-
);
|
|
143
|
-
}
|
|
144
|
-
}
|
|
@@ -1,90 +0,0 @@
|
|
|
1
|
-
import { ILLMProvider } from '../llm/ILLMProvider';
|
|
2
|
-
import { GraphNode, Edge } from '../types';
|
|
3
|
-
|
|
4
|
-
/**
|
|
5
|
-
* EntityExtractor — Uses an LLM to extract entities and relations from text.
|
|
6
|
-
*/
|
|
7
|
-
export class EntityExtractor {
|
|
8
|
-
private llm: ILLMProvider;
|
|
9
|
-
|
|
10
|
-
constructor(llm: ILLMProvider) {
|
|
11
|
-
this.llm = llm;
|
|
12
|
-
}
|
|
13
|
-
|
|
14
|
-
/**
|
|
15
|
-
* Extract nodes and edges from a text chunk.
|
|
16
|
-
*/
|
|
17
|
-
async extract(text: string): Promise<{ nodes: GraphNode[]; edges: Edge[] }> {
|
|
18
|
-
const prompt = `
|
|
19
|
-
Extract entities and relationships from the following text.
|
|
20
|
-
Format the output as a JSON object with exactly two keys: "nodes" and "edges".
|
|
21
|
-
|
|
22
|
-
Nodes: {id, label, properties}
|
|
23
|
-
Edges: {source, target, type, properties}
|
|
24
|
-
|
|
25
|
-
IMPORTANT:
|
|
26
|
-
- Ensure all property values are simple JSON types (string, number, boolean).
|
|
27
|
-
- Do not use mathematical expressions or complex nested objects in properties.
|
|
28
|
-
- If no entities are found, return empty arrays.
|
|
29
|
-
- RESPOND ONLY WITH THE JSON OBJECT.
|
|
30
|
-
|
|
31
|
-
Text to extract from:
|
|
32
|
-
"${text}"
|
|
33
|
-
`;
|
|
34
|
-
|
|
35
|
-
const response = await this.llm.chat([
|
|
36
|
-
{ role: 'system', content: 'You are a precise knowledge graph extraction engine. You always output valid JSON and nothing else.' },
|
|
37
|
-
{ role: 'user', content: prompt }
|
|
38
|
-
], '', { maxTokens: 4096, temperature: 0.1 });
|
|
39
|
-
|
|
40
|
-
try {
|
|
41
|
-
// 1. Try to find the JSON block using a regex
|
|
42
|
-
const jsonMatch = response.match(/\{[\s\S]*\}/);
|
|
43
|
-
const cleanJson = jsonMatch ? jsonMatch[0] : response.trim();
|
|
44
|
-
|
|
45
|
-
return JSON.parse(cleanJson);
|
|
46
|
-
} catch {
|
|
47
|
-
// 2. If parsing fails, try to repair truncated JSON
|
|
48
|
-
try {
|
|
49
|
-
const repaired = this.repairTruncatedJson(response);
|
|
50
|
-
if (repaired) return JSON.parse(repaired);
|
|
51
|
-
} catch { /* ignore repair failure */ }
|
|
52
|
-
|
|
53
|
-
console.warn(`[EntityExtractor] Failed to parse LLM response. Length: ${response.length} chars.`);
|
|
54
|
-
console.warn('[EntityExtractor] Snippet:', response.substring(0, 100) + '...');
|
|
55
|
-
return { nodes: [], edges: [] };
|
|
56
|
-
}
|
|
57
|
-
}
|
|
58
|
-
|
|
59
|
-
/**
|
|
60
|
-
* Attempts to fix JSON that was cut off mid-generation.
|
|
61
|
-
* Strategy: Find the last valid element in an array and close the structure.
|
|
62
|
-
*/
|
|
63
|
-
private repairTruncatedJson(json: string): string | null {
|
|
64
|
-
let text = json.trim();
|
|
65
|
-
|
|
66
|
-
// Find the first { and work from there
|
|
67
|
-
const startIdx = text.indexOf('{');
|
|
68
|
-
if (startIdx === -1) return null;
|
|
69
|
-
text = text.substring(startIdx);
|
|
70
|
-
|
|
71
|
-
// If it's truncated in an array like "nodes": [ ... , { ...
|
|
72
|
-
// We try to find the last complete object before the truncation
|
|
73
|
-
const lastCompleteObjectIdx = text.lastIndexOf('}');
|
|
74
|
-
if (lastCompleteObjectIdx === -1) return null;
|
|
75
|
-
|
|
76
|
-
// Truncate at the last complete object
|
|
77
|
-
let repaired = text.substring(0, lastCompleteObjectIdx + 1);
|
|
78
|
-
|
|
79
|
-
// Close any open structures (arrays or the main object)
|
|
80
|
-
const openBrackets = (repaired.match(/\{/g) || []).length;
|
|
81
|
-
const closedBrackets = (repaired.match(/\}/g) || []).length;
|
|
82
|
-
const openSquares = (repaired.match(/\[/g) || []).length;
|
|
83
|
-
const closedSquares = (repaired.match(/\]/g) || []).length;
|
|
84
|
-
|
|
85
|
-
for (let i = 0; i < openSquares - closedSquares; i++) repaired += ']';
|
|
86
|
-
for (let i = 0; i < openBrackets - closedBrackets; i++) repaired += '}';
|
|
87
|
-
|
|
88
|
-
return repaired;
|
|
89
|
-
}
|
|
90
|
-
}
|
|
@@ -1,60 +0,0 @@
|
|
|
1
|
-
import { Chunk, ChunkOptions } from './DocumentChunker';
|
|
2
|
-
|
|
3
|
-
/**
|
|
4
|
-
* LlamaIndexIngestor — Advanced ingestion provider using LlamaIndex (TS).
|
|
5
|
-
*
|
|
6
|
-
* Provides semantic chunking, metadata extraction, and node parsing
|
|
7
|
-
* superior to basic recursive character splitting.
|
|
8
|
-
*
|
|
9
|
-
* Note: Requires 'llamaindex' package to be installed.
|
|
10
|
-
*/
|
|
11
|
-
export class LlamaIndexIngestor {
|
|
12
|
-
/**
|
|
13
|
-
* Chunks document content using LlamaIndex SentenceSplitter.
|
|
14
|
-
* This respects sentence and paragraph boundaries much more effectively
|
|
15
|
-
* than standard character-count splitting.
|
|
16
|
-
*/
|
|
17
|
-
async chunk(text: string, options: ChunkOptions = {}): Promise<Chunk[]> {
|
|
18
|
-
try {
|
|
19
|
-
// Dynamically import for optional dependency support using template string to hide from TS static analysis
|
|
20
|
-
const { Document, SentenceSplitter, MetadataMode } = await import(`${'llamaindex'}`);
|
|
21
|
-
|
|
22
|
-
const splitter = new SentenceSplitter({
|
|
23
|
-
chunkSize: options.chunkSize ?? 1000,
|
|
24
|
-
chunkOverlap: options.chunkOverlap ?? 200,
|
|
25
|
-
});
|
|
26
|
-
|
|
27
|
-
const doc = new Document({ text, metadata: options.metadata || {} });
|
|
28
|
-
const nodes = splitter.getNodesFromDocuments([doc]);
|
|
29
|
-
|
|
30
|
-
return (nodes as any[]).map((node: any, index: number) => ({
|
|
31
|
-
id: `${options.docId || 'doc'}_node_${index}`,
|
|
32
|
-
content: node.getContent(MetadataMode.ALL),
|
|
33
|
-
metadata: {
|
|
34
|
-
...options.metadata,
|
|
35
|
-
...node.metadata,
|
|
36
|
-
nodeId: node.id_,
|
|
37
|
-
chunkIndex: index
|
|
38
|
-
}
|
|
39
|
-
}));
|
|
40
|
-
} catch (error) {
|
|
41
|
-
console.warn('[LlamaIndexIngestor] LlamaIndex package not found or failed. Falling back to default chunker.');
|
|
42
|
-
throw error;
|
|
43
|
-
}
|
|
44
|
-
}
|
|
45
|
-
|
|
46
|
-
/**
|
|
47
|
-
* Batch processing for multiple documents.
|
|
48
|
-
*/
|
|
49
|
-
async chunkMany(
|
|
50
|
-
documents: Array<{ content: string; docId?: string | number; metadata?: Record<string, unknown> }>,
|
|
51
|
-
options: ChunkOptions = {}
|
|
52
|
-
): Promise<Chunk[]> {
|
|
53
|
-
const allChunks: Chunk[] = [];
|
|
54
|
-
for (const doc of documents) {
|
|
55
|
-
const chunks = await this.chunk(doc.content, { ...options, docId: doc.docId, metadata: doc.metadata });
|
|
56
|
-
allChunks.push(...chunks);
|
|
57
|
-
}
|
|
58
|
-
return allChunks;
|
|
59
|
-
}
|
|
60
|
-
}
|
package/src/rag/Reranker.ts
DELETED
|
@@ -1,106 +0,0 @@
|
|
|
1
|
-
import { VectorMatch } from '../types';
|
|
2
|
-
import { ILLMProvider } from '../llm/ILLMProvider';
|
|
3
|
-
|
|
4
|
-
export class Reranker {
|
|
5
|
-
/**
|
|
6
|
-
* Re-ranks matches based on a secondary relevance score using an LLM.
|
|
7
|
-
*/
|
|
8
|
-
async rerank(
|
|
9
|
-
matches: VectorMatch[],
|
|
10
|
-
query: string,
|
|
11
|
-
limit: number = 5,
|
|
12
|
-
llm?: ILLMProvider
|
|
13
|
-
): Promise<VectorMatch[]> {
|
|
14
|
-
if (!llm || matches.length <= 1) {
|
|
15
|
-
// Fallback to deterministic keyword-based re-ranking for production speed
|
|
16
|
-
const keywords = query.toLowerCase()
|
|
17
|
-
.replace(/[^\w\s]/g, '')
|
|
18
|
-
.split(/\s+/)
|
|
19
|
-
.filter(w => w.length > 2); // Filter out short words/stop words
|
|
20
|
-
|
|
21
|
-
if (keywords.length === 0) {
|
|
22
|
-
return matches
|
|
23
|
-
.sort((a, b) => b.score - a.score)
|
|
24
|
-
.slice(0, limit);
|
|
25
|
-
}
|
|
26
|
-
|
|
27
|
-
const scoredMatches = matches.map(match => {
|
|
28
|
-
const contentLower = (match?.content ?? '').toLowerCase();
|
|
29
|
-
let keywordScore = 0;
|
|
30
|
-
|
|
31
|
-
keywords.forEach(keyword => {
|
|
32
|
-
if (contentLower.includes(keyword)) {
|
|
33
|
-
keywordScore += 1;
|
|
34
|
-
}
|
|
35
|
-
});
|
|
36
|
-
|
|
37
|
-
// Normalize keyword score (fraction of keywords found)
|
|
38
|
-
const normalizedKeywordScore = keywordScore / keywords.length;
|
|
39
|
-
|
|
40
|
-
// Combine scores: Vector score (usually 0-1) + Keyword score weighted
|
|
41
|
-
// We give a significant weight to keyword matches to surface specific results
|
|
42
|
-
const combinedScore = match.score + (normalizedKeywordScore * 0.5);
|
|
43
|
-
|
|
44
|
-
return {
|
|
45
|
-
...match,
|
|
46
|
-
score: combinedScore
|
|
47
|
-
};
|
|
48
|
-
});
|
|
49
|
-
|
|
50
|
-
return scoredMatches
|
|
51
|
-
.sort((a, b) => b.score - a.score)
|
|
52
|
-
.slice(0, limit);
|
|
53
|
-
}
|
|
54
|
-
|
|
55
|
-
try {
|
|
56
|
-
// Take top 10 matches to avoid large context and high latency
|
|
57
|
-
const topN = matches.slice(0, 10);
|
|
58
|
-
|
|
59
|
-
const response = await llm.chat(
|
|
60
|
-
[{ role: 'user', content: `Query: "${query}"\n\nDocuments:\n${topN.map((m, i) => `[${i}] ${(m?.content ?? '').replace(/\n/g, ' ')}`).join('\n')}` }],
|
|
61
|
-
'',
|
|
62
|
-
{
|
|
63
|
-
systemPrompt: 'You are a relevance ranking expert. Given the user query and a list of retrieved document chunks, rank the chunks by relevance to the query. Return the result as a comma-separated list of document indices in order of relevance, from most relevant to least relevant. Use the indices provided in brackets like [0], [1], etc. Only return the indices (e.g., "2,0,1"), nothing else. Do not include brackets in the output.',
|
|
64
|
-
temperature: 0,
|
|
65
|
-
maxTokens: 50
|
|
66
|
-
}
|
|
67
|
-
);
|
|
68
|
-
|
|
69
|
-
const cleanedResponse = response.trim().replace(/[\[\]\s]/g, '');
|
|
70
|
-
const rankedIndices = cleanedResponse.split(',').map(Number).filter(n => !isNaN(n));
|
|
71
|
-
|
|
72
|
-
if (rankedIndices.length > 0) {
|
|
73
|
-
const rankedMatches: VectorMatch[] = [];
|
|
74
|
-
const usedIndices = new Set<number>();
|
|
75
|
-
|
|
76
|
-
// Add matches in the order specified by the LLM
|
|
77
|
-
for (const index of rankedIndices) {
|
|
78
|
-
if (index >= 0 && index < topN.length && !usedIndices.has(index)) {
|
|
79
|
-
rankedMatches.push(topN[index]);
|
|
80
|
-
usedIndices.add(index);
|
|
81
|
-
}
|
|
82
|
-
}
|
|
83
|
-
|
|
84
|
-
// Add any remaining matches that the LLM might have missed
|
|
85
|
-
for (let i = 0; i < topN.length; i++) {
|
|
86
|
-
if (!usedIndices.has(i)) {
|
|
87
|
-
rankedMatches.push(topN[i]);
|
|
88
|
-
}
|
|
89
|
-
}
|
|
90
|
-
|
|
91
|
-
// Add back the rest of the matches that weren't in the top N
|
|
92
|
-
const rest = matches.slice(10);
|
|
93
|
-
rankedMatches.push(...rest);
|
|
94
|
-
|
|
95
|
-
return rankedMatches.slice(0, limit);
|
|
96
|
-
}
|
|
97
|
-
} catch (error) {
|
|
98
|
-
console.warn('[Reranker] LLM re-ranking failed, falling back to score-based sorting:', error);
|
|
99
|
-
}
|
|
100
|
-
|
|
101
|
-
// Fallback if LLM fails or returns invalid response
|
|
102
|
-
return matches
|
|
103
|
-
.sort((a, b) => b.score - a.score)
|
|
104
|
-
.slice(0, limit);
|
|
105
|
-
}
|
|
106
|
-
}
|