@retrivora-ai/rag-engine 2.4.2 → 2.4.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{BatchProcessor-zbQMaZHm.d.mts → BatchProcessor-Csy_K1qQ.d.mts} +2 -0
- package/dist/{BatchProcessor-D42E706a.d.ts → BatchProcessor-Dg18zZGk.d.ts} +2 -0
- package/dist/handlers/index.js +31 -5
- package/dist/handlers/index.mjs +31 -5
- package/dist/index.d.mts +3 -3
- package/dist/index.d.ts +3 -3
- package/dist/index.js +41 -21
- package/dist/index.min.css +2 -0
- package/dist/index.mjs +41 -21
- package/dist/server.d.mts +2 -2
- package/dist/server.d.ts +2 -2
- package/dist/server.js +32 -5
- package/dist/server.mjs +32 -5
- package/package.json +5 -4
- package/src/LicensedRetrivora.ts +0 -278
- package/src/app/api/chat/route.ts +0 -4
- package/src/app/api/health/route.ts +0 -5
- package/src/app/api/ingest/route.ts +0 -27
- package/src/app/api/suggestions/route.ts +0 -5
- package/src/app/api/upload/route.ts +0 -5
- package/src/app/constants.tsx +0 -270
- package/src/app/favicon.ico +0 -0
- package/src/app/globals.css +0 -449
- package/src/app/layout.tsx +0 -22
- package/src/app/page.tsx +0 -64
- package/src/app/types.ts +0 -30
- package/src/browser.tsx +0 -139
- package/src/components/ChatWidget.tsx +0 -393
- package/src/components/ChatWindow.tsx +0 -952
- package/src/components/CodeViewer.tsx +0 -56
- package/src/components/ConfigProvider.tsx +0 -123
- package/src/components/DocumentUpload.tsx +0 -354
- package/src/components/DynamicChart.tsx +0 -297
- package/src/components/HourglassLoader.tsx +0 -87
- package/src/components/MarkdownComponents.tsx +0 -186
- package/src/components/MessageBubble.tsx +0 -498
- package/src/components/ObservabilityPanel.tsx +0 -374
- package/src/components/ProductCard.tsx +0 -107
- package/src/components/ProductCarousel.tsx +0 -97
- package/src/components/SourceCard.tsx +0 -61
- package/src/components/ThemeProvider.tsx +0 -8
- package/src/components/ThemeToggle.tsx +0 -29
- package/src/components/ThinkingBlock.tsx +0 -75
- package/src/components/UIDispatcher.tsx +0 -342
- package/src/components/VisualizationRenderer.tsx +0 -621
- package/src/components/constants.tsx +0 -503
- package/src/config/ConfigBuilder.ts +0 -285
- package/src/config/EmbeddingStrategy.ts +0 -148
- package/src/config/LicenseConfigResolver.ts +0 -298
- package/src/config/RagConfig.ts +0 -328
- package/src/config/UniversalProfiles.ts +0 -82
- package/src/config/constants.ts +0 -64
- package/src/config/serverConfig.ts +0 -378
- package/src/config/uiConstants.ts +0 -47
- package/src/core/BatchProcessor.ts +0 -385
- package/src/core/CircuitBreaker.ts +0 -118
- package/src/core/ConfigFetcher.ts +0 -220
- package/src/core/ConfigResolver.ts +0 -165
- package/src/core/ConfigValidator.ts +0 -197
- package/src/core/DatabaseStorage.ts +0 -526
- package/src/core/FreeTierLimitsGuard.ts +0 -281
- package/src/core/LLMRouter.ts +0 -79
- package/src/core/LangChainAgent.ts +0 -132
- package/src/core/LicenseValidator.ts +0 -284
- package/src/core/LicenseVerifier.ts +0 -261
- package/src/core/MultiAgentCoordinator.ts +0 -239
- package/src/core/Pipeline.ts +0 -1279
- package/src/core/PluginManager.ts +0 -277
- package/src/core/ProjectSettings.ts +0 -247
- package/src/core/ProviderHealthCheck.ts +0 -133
- package/src/core/ProviderInterfaces.ts +0 -37
- package/src/core/ProviderRegistry.ts +0 -156
- package/src/core/QueryProcessor.ts +0 -399
- package/src/core/Retrivora.ts +0 -98
- package/src/core/VectorPlugin.ts +0 -193
- package/src/core/mcp.ts +0 -261
- package/src/exceptions/index.ts +0 -131
- package/src/handlers/index.ts +0 -1625
- package/src/hooks/useHydrated.ts +0 -13
- package/src/hooks/useProjectSettings.ts +0 -105
- package/src/hooks/useRagChat.ts +0 -436
- package/src/hooks/useStoredMessages.ts +0 -64
- package/src/index.css +0 -4269
- package/src/index.ts +0 -137
- package/src/lib/plugin.ts +0 -24
- package/src/llm/ILLMProvider.ts +0 -60
- package/src/llm/LLMFactory.ts +0 -141
- package/src/llm/providers/AnthropicProvider.ts +0 -215
- package/src/llm/providers/GeminiProvider.ts +0 -311
- package/src/llm/providers/GroqProvider.ts +0 -184
- package/src/llm/providers/OllamaProvider.ts +0 -232
- package/src/llm/providers/OpenAIProvider.ts +0 -188
- package/src/llm/providers/QwenProvider.ts +0 -199
- package/src/llm/providers/UniversalLLMAdapter.ts +0 -422
- package/src/llm/utils.ts +0 -46
- package/src/providers/graphdb/BaseGraphProvider.ts +0 -43
- package/src/providers/graphdb/SimpleGraphProvider.ts +0 -66
- package/src/providers/vectordb/BaseVectorProvider.ts +0 -98
- package/src/providers/vectordb/ChromaDBProvider.ts +0 -130
- package/src/providers/vectordb/MilvusProvider.ts +0 -134
- package/src/providers/vectordb/MongoDBProvider.ts +0 -250
- package/src/providers/vectordb/MultiTablePostgresProvider.ts +0 -461
- package/src/providers/vectordb/PineconeProvider.ts +0 -295
- package/src/providers/vectordb/PostgreSQLProvider.ts +0 -227
- package/src/providers/vectordb/QdrantProvider.ts +0 -294
- package/src/providers/vectordb/RedisProvider.ts +0 -105
- package/src/providers/vectordb/UniversalVectorProvider.ts +0 -221
- package/src/providers/vectordb/WeaviateProvider.ts +0 -172
- package/src/rag/DocumentChunker.ts +0 -144
- package/src/rag/EntityExtractor.ts +0 -90
- package/src/rag/LlamaIndexIngestor.ts +0 -60
- package/src/rag/Reranker.ts +0 -106
- package/src/rendering/IntentClassifier.ts +0 -167
- package/src/rendering/RendererRegistry.ts +0 -164
- package/src/rendering/RuleEngine.ts +0 -67
- package/src/rendering/VisualizationDecisionEngine.ts +0 -164
- package/src/rendering/__tests__/VisualizationDecisionEngine.test.ts +0 -237
- package/src/rendering/index.ts +0 -12
- package/src/rendering/rules/Rule1SpecificInfoRule.ts +0 -28
- package/src/rendering/rules/Rule2ComparisonRule.ts +0 -24
- package/src/rendering/rules/Rule3ProductDiscoveryRule.ts +0 -28
- package/src/rendering/rules/Rule4AnalyticalRule.ts +0 -38
- package/src/rendering/rules/Rule5MixedResponseRule.ts +0 -29
- package/src/rendering/rules/Rule6SmallResultSetRule.ts +0 -24
- package/src/rendering/rules/Rule7LargeTableRule.ts +0 -35
- package/src/rendering/types.ts +0 -96
- package/src/server.ts +0 -156
- package/src/tailwind.css +0 -194
- package/src/telemetry/TelemetryService.ts +0 -330
- package/src/types/chat.ts +0 -89
- package/src/types/index.ts +0 -278
- package/src/types/pdf-parse.d.ts +0 -13
- package/src/types/props.ts +0 -169
- package/src/utils/DocumentParser.ts +0 -111
- package/src/utils/ProductExtractor.ts +0 -350
- package/src/utils/SchemaMapper.ts +0 -176
- package/src/utils/UITransformer.ts +0 -1812
- package/src/utils/synonyms.ts +0 -80
- package/src/utils/templateUtils.ts +0 -78
- package/src/version.ts +0 -6
package/src/core/Pipeline.ts
DELETED
|
@@ -1,1279 +0,0 @@
|
|
|
1
|
-
import { BaseVectorProvider } from '../providers/vectordb/BaseVectorProvider';
|
|
2
|
-
import { SDK_VERSION } from '../version';
|
|
3
|
-
import { BaseGraphProvider } from '../providers/graphdb/BaseGraphProvider';
|
|
4
|
-
import { ILLMProvider } from '../llm/ILLMProvider';
|
|
5
|
-
import { ChatMessage } from '../types';
|
|
6
|
-
import { EmbeddingFailedException } from '../exceptions';
|
|
7
|
-
import { DocumentChunker, Chunk } from '../rag/DocumentChunker';
|
|
8
|
-
import { EntityExtractor } from '../rag/EntityExtractor';
|
|
9
|
-
import { Reranker } from '../rag/Reranker';
|
|
10
|
-
import { LlamaIndexIngestor } from '../rag/LlamaIndexIngestor';
|
|
11
|
-
import { LangChainAgent } from './LangChainAgent';
|
|
12
|
-
import { RagConfig } from '../config/RagConfig';
|
|
13
|
-
import { MCPRegistry } from './mcp';
|
|
14
|
-
import { MultiAgentCoordinator } from './MultiAgentCoordinator';
|
|
15
|
-
import { ProviderRegistry } from './ProviderRegistry';
|
|
16
|
-
import { BatchProcessor, BatchOptions } from './BatchProcessor';
|
|
17
|
-
import { EmbeddingStrategyResolver } from '../config/EmbeddingStrategy';
|
|
18
|
-
import { QueryProcessor, NumericPredicate, QueryFilter } from './QueryProcessor';
|
|
19
|
-
import { LLMRouter } from './LLMRouter';
|
|
20
|
-
import {
|
|
21
|
-
IngestDocument, ChatResponse, VectorMatch, GraphSearchResult,
|
|
22
|
-
RetrievalResult, UpsertDocument, ObservabilityTrace, LatencyBreakdown,
|
|
23
|
-
TokenUsage, RetrievedChunk,
|
|
24
|
-
} from '../types';
|
|
25
|
-
import { UITransformer } from '../utils/UITransformer';
|
|
26
|
-
import { SchemaMapper, SchemaMap } from '../utils/SchemaMapper';
|
|
27
|
-
|
|
28
|
-
// ─── Namespace Formatter ───────────────────────────────────────────────────────
|
|
29
|
-
|
|
30
|
-
/**
|
|
31
|
-
* Clean & format namespace string to satisfy vector DB identifier rules.
|
|
32
|
-
* Preserves the exact dynamic project ID passed by the user without forcing static prefixes.
|
|
33
|
-
*/
|
|
34
|
-
export function formatNamespace(raw?: string): string {
|
|
35
|
-
if (!raw || !raw.trim()) return 'default';
|
|
36
|
-
const trimmed = raw.trim();
|
|
37
|
-
const sanitized = trimmed
|
|
38
|
-
.replace(/[^a-zA-Z0-9_-]/g, '_')
|
|
39
|
-
.replace(/_+/g, '_')
|
|
40
|
-
.replace(/^_+|_+$/g, '');
|
|
41
|
-
return sanitized || 'default';
|
|
42
|
-
}
|
|
43
|
-
|
|
44
|
-
// ─── LRU Embedding Cache ───────────────────────────────────────────────────────
|
|
45
|
-
|
|
46
|
-
/**
|
|
47
|
-
* A simple size-capped LRU cache for embedding vectors.
|
|
48
|
-
* Prevents the unbounded `Map` from consuming infinite memory under production load.
|
|
49
|
-
*/
|
|
50
|
-
class LRUEmbeddingCache {
|
|
51
|
-
private readonly cache = new Map<string, number[]>();
|
|
52
|
-
private readonly maxSize: number;
|
|
53
|
-
|
|
54
|
-
constructor(maxSize = 500) {
|
|
55
|
-
this.maxSize = maxSize;
|
|
56
|
-
}
|
|
57
|
-
|
|
58
|
-
get(key: string): number[] | undefined {
|
|
59
|
-
const value = this.cache.get(key);
|
|
60
|
-
if (value !== undefined) {
|
|
61
|
-
// Promote to most-recently-used position
|
|
62
|
-
this.cache.delete(key);
|
|
63
|
-
this.cache.set(key, value);
|
|
64
|
-
}
|
|
65
|
-
return value;
|
|
66
|
-
}
|
|
67
|
-
|
|
68
|
-
set(key: string, value: number[]): void {
|
|
69
|
-
if (this.cache.has(key)) {
|
|
70
|
-
this.cache.delete(key);
|
|
71
|
-
} else if (this.cache.size >= this.maxSize) {
|
|
72
|
-
// Evict the oldest (first) entry
|
|
73
|
-
const oldestKey = this.cache.keys().next().value;
|
|
74
|
-
if (oldestKey !== undefined) this.cache.delete(oldestKey);
|
|
75
|
-
}
|
|
76
|
-
this.cache.set(key, value);
|
|
77
|
-
}
|
|
78
|
-
|
|
79
|
-
clear(): void {
|
|
80
|
-
this.cache.clear();
|
|
81
|
-
}
|
|
82
|
-
|
|
83
|
-
get size(): number {
|
|
84
|
-
return this.cache.size;
|
|
85
|
-
}
|
|
86
|
-
}
|
|
87
|
-
|
|
88
|
-
// ─── Token estimation helpers ──────────────────────────────────────────────────
|
|
89
|
-
|
|
90
|
-
/** Rough token count estimate when the provider doesn't expose usage data. */
|
|
91
|
-
function estimateTokens(text: string): number {
|
|
92
|
-
return Math.ceil(text.length / 4);
|
|
93
|
-
}
|
|
94
|
-
|
|
95
|
-
/**
|
|
96
|
-
* Model pricing table (cost per 1k tokens in USD).
|
|
97
|
-
* Used for estimating cost when providers don't return billing data.
|
|
98
|
-
*/
|
|
99
|
-
const MODEL_COST_PER_1K: Record<string, { input: number; output: number }> = {
|
|
100
|
-
'gpt-4o': { input: 0.0025, output: 0.010 },
|
|
101
|
-
'gpt-4o-mini': { input: 0.00015, output: 0.0006 },
|
|
102
|
-
'gpt-4-turbo': { input: 0.01, output: 0.03 },
|
|
103
|
-
'gpt-3.5-turbo': { input: 0.0005, output: 0.0015 },
|
|
104
|
-
'claude-3-5-sonnet': { input: 0.003, output: 0.015 },
|
|
105
|
-
'claude-3-haiku': { input: 0.00025, output: 0.00125 },
|
|
106
|
-
'gemini-1.5-flash': { input: 0.000075, output: 0.0003 },
|
|
107
|
-
'gemini-1.5-pro': { input: 0.00125, output: 0.005 },
|
|
108
|
-
};
|
|
109
|
-
|
|
110
|
-
function estimateCostUsd(
|
|
111
|
-
promptTokens: number,
|
|
112
|
-
completionTokens: number,
|
|
113
|
-
model?: string,
|
|
114
|
-
): number | undefined {
|
|
115
|
-
if (!model) return undefined;
|
|
116
|
-
const key = Object.keys(MODEL_COST_PER_1K).find((k) => model.toLowerCase().includes(k));
|
|
117
|
-
if (!key) return undefined;
|
|
118
|
-
const prices = MODEL_COST_PER_1K[key];
|
|
119
|
-
return (promptTokens / 1000) * prices.input + (completionTokens / 1000) * prices.output;
|
|
120
|
-
}
|
|
121
|
-
|
|
122
|
-
// ─── Hallucination scorer ──────────────────────────────────────────────────────
|
|
123
|
-
|
|
124
|
-
/**
|
|
125
|
-
* Ask the LLM to self-critique: rate how well the answer is supported by the context.
|
|
126
|
-
* Returns a score 0–1 and a brief reason. Silently returns undefined on failure.
|
|
127
|
-
*/
|
|
128
|
-
async function scoreHallucination(
|
|
129
|
-
llm: ILLMProvider,
|
|
130
|
-
answer: string,
|
|
131
|
-
context: string,
|
|
132
|
-
): Promise<{ score: number; reason: string } | undefined> {
|
|
133
|
-
const maxContextChars = 3000;
|
|
134
|
-
const truncatedContext = context.length > maxContextChars
|
|
135
|
-
? context.slice(0, maxContextChars) + '\n...[truncated]'
|
|
136
|
-
: context;
|
|
137
|
-
|
|
138
|
-
try {
|
|
139
|
-
const raw = await llm.chat(
|
|
140
|
-
[{ role: 'user', content: `CONTEXT:\n${truncatedContext}\n\nANSWER:\n${answer}` }],
|
|
141
|
-
'',
|
|
142
|
-
{
|
|
143
|
-
systemPrompt: 'You are an AI quality checker. Given the CONTEXT retrieved from a knowledge base and the ANSWER generated from it, rate how well-grounded the answer is. Return ONLY a valid JSON object with no markdown fences: {"score": <float 0-1>, "reason": "<one sentence>"} where score 0 = fully grounded in context, score 1 = likely hallucinated or unsupported.',
|
|
144
|
-
temperature: 0,
|
|
145
|
-
maxTokens: 120
|
|
146
|
-
},
|
|
147
|
-
);
|
|
148
|
-
|
|
149
|
-
const jsonMatch = raw.match(/\{[\s\S]*\}/);
|
|
150
|
-
if (!jsonMatch) return undefined;
|
|
151
|
-
|
|
152
|
-
const parsed = JSON.parse(jsonMatch[0]);
|
|
153
|
-
if (typeof parsed.score === 'number' && typeof parsed.reason === 'string') {
|
|
154
|
-
return { score: Math.min(1, Math.max(0, parsed.score)), reason: parsed.reason };
|
|
155
|
-
}
|
|
156
|
-
} catch {
|
|
157
|
-
// Non-critical — fail silently
|
|
158
|
-
}
|
|
159
|
-
return undefined;
|
|
160
|
-
}
|
|
161
|
-
|
|
162
|
-
// ─── Pipeline ─────────────────────────────────────────────────────────────────
|
|
163
|
-
|
|
164
|
-
/**
|
|
165
|
-
* Pipeline — orchestrates the RAG flow: Embed → Search → Augment → Generate.
|
|
166
|
-
*
|
|
167
|
-
* Features:
|
|
168
|
-
* - Lazy initialization of providers
|
|
169
|
-
* - Batch processing with retry logic
|
|
170
|
-
* - Smart embedding strategy (integrated / separate / external)
|
|
171
|
-
* - Error recovery for transient failures
|
|
172
|
-
* - Multi-tenancy support via namespacing
|
|
173
|
-
* - LRU-bounded embedding cache (max 500 entries, prevents memory leaks)
|
|
174
|
-
* - Full observability tracing (latency, tokens, hallucination scoring)
|
|
175
|
-
*/
|
|
176
|
-
export class Pipeline {
|
|
177
|
-
private vectorDB!: BaseVectorProvider;
|
|
178
|
-
private graphDB?: BaseGraphProvider;
|
|
179
|
-
private llmProvider!: ILLMProvider;
|
|
180
|
-
private embeddingProvider!: ILLMProvider;
|
|
181
|
-
private llmRouter!: LLMRouter;
|
|
182
|
-
private chunker: DocumentChunker;
|
|
183
|
-
private llamaIngestor?: LlamaIndexIngestor;
|
|
184
|
-
private entityExtractor?: EntityExtractor;
|
|
185
|
-
private reranker: Reranker;
|
|
186
|
-
private agent?: LangChainAgent;
|
|
187
|
-
private mcpRegistry?: MCPRegistry;
|
|
188
|
-
private multiAgentCoordinator?: MultiAgentCoordinator;
|
|
189
|
-
/** LRU-bounded cache: avoids re-embedding identical queries within the same process. */
|
|
190
|
-
private embeddingCache = new LRUEmbeddingCache(500);
|
|
191
|
-
private initialised = false;
|
|
192
|
-
/** Namespace-specific static cold context cache for CAG */
|
|
193
|
-
private coldContexts = new Map<string, string>();
|
|
194
|
-
|
|
195
|
-
constructor(private config: RagConfig) {
|
|
196
|
-
this.chunker = new DocumentChunker(
|
|
197
|
-
config.rag?.chunkSize ?? 1000,
|
|
198
|
-
config.rag?.chunkOverlap ?? 200
|
|
199
|
-
);
|
|
200
|
-
if (config.rag?.chunkingStrategy === 'llamaindex') {
|
|
201
|
-
this.llamaIngestor = new LlamaIndexIngestor();
|
|
202
|
-
}
|
|
203
|
-
this.reranker = new Reranker();
|
|
204
|
-
}
|
|
205
|
-
|
|
206
|
-
/**
|
|
207
|
-
* Expose the underlying LLM provider (set after initialize()).
|
|
208
|
-
* Used by the stream handler to pass to UITransformer.analyzeAndDecide().
|
|
209
|
-
*/
|
|
210
|
-
getLLMProvider(): ILLMProvider | undefined {
|
|
211
|
-
return this.initialised ? this.llmProvider : undefined;
|
|
212
|
-
}
|
|
213
|
-
|
|
214
|
-
async initialize(): Promise<void> {
|
|
215
|
-
if (this.initialised) return;
|
|
216
|
-
|
|
217
|
-
const chartInstruction = `\
|
|
218
|
-
You are a helpful assistant. Use the provided context to answer questions accurately.
|
|
219
|
-
|
|
220
|
-
### CRITICAL RULES:
|
|
221
|
-
- ONLY answer the user's specific question. Do NOT suggest or recommend unrelated products unless specifically asked.
|
|
222
|
-
- NEVER generate markdown tables. If you do, the UI will break.
|
|
223
|
-
- NEVER generate HTML tags like <figure>, <tbody>, <tr>, etc.
|
|
224
|
-
- NEVER generate text-based charts or graphs.
|
|
225
|
-
- NEVER say you cannot display, render, draw, or create a chart when the user asks for a visualization. The UI handles visual rendering separately.
|
|
226
|
-
- ONLY use plain text and bullet points.
|
|
227
|
-
- NEVER use the plus sign (+) as a separator between names, categories, or products. Use commas or bullet points.
|
|
228
|
-
- ONLY answer the question if the sources contain relevant information to answer it, else say that you cannot answer the question.
|
|
229
|
-
- If answer cannot be found in the sources, say that you cannot answer the question and do not hallucinate.
|
|
230
|
-
- Do not use information from previous turns to answer the question.
|
|
231
|
-
- You CAN use numbers for years and counts (e.g., 2006, 5800). But NEVER put a dollar sign ($) before non-monetary numbers like Stock quantities, EAN numbers, or years. Only use dollar signs for actual monetary prices.
|
|
232
|
-
|
|
233
|
-
### PRODUCT DISPLAY:
|
|
234
|
-
- Recommended Products should only be displayed when the user explicitly asks for products.
|
|
235
|
-
- When recommending products (ONLY when asked), simply list their names, prices, and features in a friendly, conversational manner.
|
|
236
|
-
- The UI will automatically detect these products and show high-quality product cards in a carousel below your message.
|
|
237
|
-
- For product descriptions, summarize customer-facing details only. Do NOT list internal catalog fields such as Handle, Body (HTML), Vendor, Type, Tags, Published, Option, Variant, SKU, or raw metadata labels.
|
|
238
|
-
- Do NOT try to format product lists as tables.
|
|
239
|
-
`;
|
|
240
|
-
|
|
241
|
-
// Merge chartInstruction with any user-supplied systemPrompt.
|
|
242
|
-
// We append rather than overwrite so plugin consumers who pass a custom
|
|
243
|
-
// systemPrompt do not have it silently replaced.
|
|
244
|
-
const userPromptPrefix = this.config.llm.systemPrompt
|
|
245
|
-
? `${this.config.llm.systemPrompt}\n\n`
|
|
246
|
-
: '';
|
|
247
|
-
this.config.llm.systemPrompt = `${userPromptPrefix}${chartInstruction}`;
|
|
248
|
-
|
|
249
|
-
this.vectorDB = await ProviderRegistry.createVectorProvider(this.config.vectorDb);
|
|
250
|
-
|
|
251
|
-
// Initialize LLM Router for multi-model support
|
|
252
|
-
this.llmRouter = new LLMRouter(this.config);
|
|
253
|
-
|
|
254
|
-
// EmbeddingStrategyResolver creates the providers with the correct configs
|
|
255
|
-
const { llmProvider: resolvedLLM, embeddingProvider } = await EmbeddingStrategyResolver.resolve(
|
|
256
|
-
this.config.llm,
|
|
257
|
-
this.config.embedding
|
|
258
|
-
);
|
|
259
|
-
this.embeddingProvider = embeddingProvider;
|
|
260
|
-
|
|
261
|
-
// Seed the router with the correctly initialised LLM provider, then initialise
|
|
262
|
-
// the rest of the router (fast/powerful roles) using provider-aware defaults.
|
|
263
|
-
await this.llmRouter.initialize(resolvedLLM);
|
|
264
|
-
|
|
265
|
-
// Primary LLM provider — use what the router resolved as 'default'
|
|
266
|
-
this.llmProvider = this.llmRouter.get('default');
|
|
267
|
-
|
|
268
|
-
if (this.config.graphDb) {
|
|
269
|
-
this.graphDB = await ProviderRegistry.createGraphProvider(this.config.graphDb);
|
|
270
|
-
await this.graphDB.initialize();
|
|
271
|
-
this.entityExtractor = new EntityExtractor(this.llmProvider);
|
|
272
|
-
}
|
|
273
|
-
|
|
274
|
-
await this.vectorDB.initialize();
|
|
275
|
-
|
|
276
|
-
if (this.config.rag?.architecture === 'agentic' || this.config.rag?.architecture === 'multi-agent' || (this.config.mcpServers && this.config.mcpServers.length > 0)) {
|
|
277
|
-
this.mcpRegistry = new MCPRegistry(this.config.mcpServers || []);
|
|
278
|
-
this.multiAgentCoordinator = new MultiAgentCoordinator({
|
|
279
|
-
llmProvider: this.llmProvider,
|
|
280
|
-
mcpRegistry: this.mcpRegistry,
|
|
281
|
-
documentSearch: async (query: string) => {
|
|
282
|
-
return this.runNormalQuery(query);
|
|
283
|
-
}
|
|
284
|
-
});
|
|
285
|
-
this.agent = new LangChainAgent(this, this.config);
|
|
286
|
-
await this.agent.initialize(this.llmProvider as unknown);
|
|
287
|
-
}
|
|
288
|
-
|
|
289
|
-
if (this.config.rag?.cag?.enabled) {
|
|
290
|
-
await this.loadColdContext(this.config.projectId);
|
|
291
|
-
}
|
|
292
|
-
|
|
293
|
-
this.initialised = true;
|
|
294
|
-
}
|
|
295
|
-
|
|
296
|
-
private async loadColdContext(ns: string): Promise<void> {
|
|
297
|
-
const cagConfig = this.config.rag?.cag;
|
|
298
|
-
if (!cagConfig || !cagConfig.enabled) return;
|
|
299
|
-
|
|
300
|
-
try {
|
|
301
|
-
const coldNs = cagConfig.coldNamespace || ns;
|
|
302
|
-
const limit = cagConfig.coldLimit ?? 20;
|
|
303
|
-
|
|
304
|
-
const queryText = cagConfig.coldQuery || 'documentation';
|
|
305
|
-
const queryVector = await this.embeddingProvider.embed(queryText, { taskType: 'query' });
|
|
306
|
-
const coldMatches = await this.vectorDB.query(queryVector, limit, coldNs);
|
|
307
|
-
|
|
308
|
-
if (coldMatches.length > 0) {
|
|
309
|
-
const formatted = coldMatches.map((m, i) => `[Cold Source ${i + 1}]\n${m.content}`).join('\n\n---\n\n');
|
|
310
|
-
this.coldContexts.set(ns, formatted);
|
|
311
|
-
console.log(`[Pipeline] Cold RAG (CAG) loaded ${coldMatches.length} documents into cached context for namespace: ${ns}`);
|
|
312
|
-
} else {
|
|
313
|
-
this.coldContexts.set(ns, 'No cold context found.');
|
|
314
|
-
console.warn(`[Pipeline] Cold RAG (CAG) initialized but no documents were found in namespace: ${coldNs}`);
|
|
315
|
-
}
|
|
316
|
-
} catch (err) {
|
|
317
|
-
console.error(`[Pipeline] Failed to load Cold RAG (CAG) context for namespace ${ns}:`, err);
|
|
318
|
-
this.coldContexts.set(ns, 'Failed to load cold context.');
|
|
319
|
-
}
|
|
320
|
-
}
|
|
321
|
-
|
|
322
|
-
/**
|
|
323
|
-
* Ingest documents with automatic chunking, embedding, and batch upsert.
|
|
324
|
-
*/
|
|
325
|
-
async ingest(
|
|
326
|
-
documents: IngestDocument[],
|
|
327
|
-
namespace?: string
|
|
328
|
-
): Promise<Array<{ docId: string | number; chunksIngested: number }>> {
|
|
329
|
-
await this.initialize();
|
|
330
|
-
const ns = formatNamespace(namespace ?? this.config.projectId);
|
|
331
|
-
const results = [];
|
|
332
|
-
|
|
333
|
-
for (const doc of documents) {
|
|
334
|
-
try {
|
|
335
|
-
const chunks = await this.prepareChunks(doc);
|
|
336
|
-
const vectors = await this.processEmbeddings(chunks);
|
|
337
|
-
|
|
338
|
-
const upsertDocs = chunks.map((chunk, i) => ({
|
|
339
|
-
id: chunk.id,
|
|
340
|
-
vector: vectors[i],
|
|
341
|
-
content: chunk.content,
|
|
342
|
-
metadata: chunk.metadata,
|
|
343
|
-
}));
|
|
344
|
-
|
|
345
|
-
const totalProcessed = await this.processUpserts(upsertDocs, ns);
|
|
346
|
-
|
|
347
|
-
results.push({ docId: doc.docId, chunksIngested: totalProcessed });
|
|
348
|
-
|
|
349
|
-
if (this.graphDB && this.entityExtractor) {
|
|
350
|
-
await this.processGraphIngestion(doc.docId, chunks);
|
|
351
|
-
}
|
|
352
|
-
} catch (error) {
|
|
353
|
-
console.error(`[Pipeline] Failed to ingest document ${doc.docId}:`, error);
|
|
354
|
-
results.push({ docId: doc.docId, chunksIngested: 0 });
|
|
355
|
-
}
|
|
356
|
-
}
|
|
357
|
-
|
|
358
|
-
return results;
|
|
359
|
-
}
|
|
360
|
-
|
|
361
|
-
/** Step 1: Chunk the document content. */
|
|
362
|
-
private async prepareChunks(doc: IngestDocument): Promise<Chunk[]> {
|
|
363
|
-
if (this.llamaIngestor) {
|
|
364
|
-
return this.llamaIngestor.chunk(doc.content, {
|
|
365
|
-
docId: doc.docId,
|
|
366
|
-
metadata: doc.metadata,
|
|
367
|
-
chunkSize: this.config.rag?.chunkSize,
|
|
368
|
-
chunkOverlap: this.config.rag?.chunkOverlap,
|
|
369
|
-
});
|
|
370
|
-
}
|
|
371
|
-
return this.chunker.chunk(doc.content, { docId: doc.docId, metadata: doc.metadata });
|
|
372
|
-
}
|
|
373
|
-
|
|
374
|
-
/**
|
|
375
|
-
* Step 2: Generate embeddings for chunks with retry logic.
|
|
376
|
-
* Uses batchEmbed when available for efficiency; falls back to sequential embedding.
|
|
377
|
-
*/
|
|
378
|
-
private async processEmbeddings(chunks: Chunk[]): Promise<number[][]> {
|
|
379
|
-
const embedBatchOptions: BatchOptions = {
|
|
380
|
-
batchSize: 50,
|
|
381
|
-
maxRetries: 3,
|
|
382
|
-
initialDelayMs: 100,
|
|
383
|
-
};
|
|
384
|
-
|
|
385
|
-
const vectors = await BatchProcessor.mapWithRetry(
|
|
386
|
-
chunks.map(c => c.content),
|
|
387
|
-
(text) => this.embeddingProvider.embed(text, { taskType: 'document' }),
|
|
388
|
-
embedBatchOptions
|
|
389
|
-
);
|
|
390
|
-
|
|
391
|
-
if (vectors.length !== chunks.length) {
|
|
392
|
-
throw new EmbeddingFailedException(
|
|
393
|
-
`[Pipeline] Embedding mismatch: got ${vectors.length} vectors for ${chunks.length} chunks. ` +
|
|
394
|
-
`Check embedding provider logs for individual chunk failures.`
|
|
395
|
-
);
|
|
396
|
-
}
|
|
397
|
-
|
|
398
|
-
return vectors;
|
|
399
|
-
}
|
|
400
|
-
|
|
401
|
-
/** Step 3: Upsert chunks to vector database with retry logic. */
|
|
402
|
-
private async processUpserts(upsertDocs: UpsertDocument[], namespace: string): Promise<number> {
|
|
403
|
-
const upsertBatchOptions: BatchOptions = {
|
|
404
|
-
batchSize: 100,
|
|
405
|
-
maxRetries: 3,
|
|
406
|
-
initialDelayMs: 100,
|
|
407
|
-
};
|
|
408
|
-
|
|
409
|
-
const upsertResult = await BatchProcessor.processBatch(
|
|
410
|
-
upsertDocs,
|
|
411
|
-
(batch) => this.vectorDB.batchUpsert(batch, namespace),
|
|
412
|
-
upsertBatchOptions
|
|
413
|
-
);
|
|
414
|
-
|
|
415
|
-
if (upsertResult.errors.length > 0) {
|
|
416
|
-
console.warn(`[Pipeline] Ingestion: ${upsertResult.errors.length} batches failed. Error details:`);
|
|
417
|
-
upsertResult.errors.forEach((err, idx) => {
|
|
418
|
-
console.warn(` Batch ${idx + 1} Error: ${err.error.message || String(err.error)}`);
|
|
419
|
-
});
|
|
420
|
-
}
|
|
421
|
-
|
|
422
|
-
return upsertResult.totalProcessed;
|
|
423
|
-
}
|
|
424
|
-
|
|
425
|
-
/** Step 4: Optional graph-based entity extraction and ingestion. */
|
|
426
|
-
private async processGraphIngestion(docId: string | number, chunks: Chunk[]): Promise<void> {
|
|
427
|
-
const extractionOptions: BatchOptions = {
|
|
428
|
-
batchSize: 2,
|
|
429
|
-
maxRetries: 1,
|
|
430
|
-
initialDelayMs: 500,
|
|
431
|
-
};
|
|
432
|
-
|
|
433
|
-
await BatchProcessor.processBatch(
|
|
434
|
-
chunks,
|
|
435
|
-
async (batch) => {
|
|
436
|
-
for (const chunk of batch) {
|
|
437
|
-
try {
|
|
438
|
-
const { nodes, edges } = await this.entityExtractor!.extract(chunk.content);
|
|
439
|
-
if (nodes.length > 0) await this.graphDB!.addNodes(nodes);
|
|
440
|
-
if (edges.length > 0) await this.graphDB!.addEdges(edges);
|
|
441
|
-
} catch (err) {
|
|
442
|
-
console.warn(`[Pipeline] Entity extraction failed for chunk in doc ${docId}:`, err);
|
|
443
|
-
}
|
|
444
|
-
}
|
|
445
|
-
},
|
|
446
|
-
extractionOptions
|
|
447
|
-
);
|
|
448
|
-
}
|
|
449
|
-
|
|
450
|
-
async runNormalQuery(question: string, history: ChatMessage[] = [], namespace?: string): Promise<{ reply: string; sources: any[] }> {
|
|
451
|
-
const stream = this.askStreamInternal(question, history, namespace);
|
|
452
|
-
let reply = '';
|
|
453
|
-
let sources: any[] = [];
|
|
454
|
-
for await (const chunk of stream) {
|
|
455
|
-
if (typeof chunk === 'string') {
|
|
456
|
-
reply += chunk;
|
|
457
|
-
} else if (typeof chunk === 'object' && chunk !== null) {
|
|
458
|
-
if ('reply' in chunk && chunk.reply) reply += chunk.reply;
|
|
459
|
-
if ('sources' in chunk && chunk.sources) sources = chunk.sources;
|
|
460
|
-
}
|
|
461
|
-
}
|
|
462
|
-
return { reply, sources };
|
|
463
|
-
}
|
|
464
|
-
|
|
465
|
-
async ask(question: string, history: ChatMessage[] = [], namespace?: string): Promise<ChatResponse> {
|
|
466
|
-
await this.initialize();
|
|
467
|
-
|
|
468
|
-
if ((this.config.rag?.architecture === 'agentic' || this.config.rag?.architecture === 'multi-agent' || (this.config.mcpServers && this.config.mcpServers.length > 0)) && this.multiAgentCoordinator) {
|
|
469
|
-
return await this.multiAgentCoordinator.run(question, history);
|
|
470
|
-
}
|
|
471
|
-
|
|
472
|
-
const stream = this.askStream(question, history, namespace);
|
|
473
|
-
let reply = '';
|
|
474
|
-
let sources: VectorMatch[] = [];
|
|
475
|
-
let graphData: GraphSearchResult | undefined;
|
|
476
|
-
let uiTransformation: unknown;
|
|
477
|
-
let trace: ObservabilityTrace | undefined;
|
|
478
|
-
|
|
479
|
-
for await (const chunk of stream) {
|
|
480
|
-
if (typeof chunk === 'string') {
|
|
481
|
-
reply += chunk;
|
|
482
|
-
} else if (typeof chunk === 'object' && chunk !== null) {
|
|
483
|
-
if ('sources' in chunk) sources = (chunk as ChatResponse).sources;
|
|
484
|
-
if ('graphData' in chunk) graphData = (chunk as ChatResponse).graphData;
|
|
485
|
-
if ('ui_transformation' in chunk) uiTransformation = (chunk as ChatResponse).ui_transformation;
|
|
486
|
-
if ('trace' in chunk) trace = (chunk as { trace: ObservabilityTrace }).trace;
|
|
487
|
-
}
|
|
488
|
-
}
|
|
489
|
-
|
|
490
|
-
return { reply, sources, graphData, ui_transformation: uiTransformation, trace };
|
|
491
|
-
}
|
|
492
|
-
|
|
493
|
-
async *askStream(question: string, history: ChatMessage[] = [], namespace?: string): AsyncIterable<string | ChatResponse> {
|
|
494
|
-
await this.initialize();
|
|
495
|
-
if ((this.config.rag?.architecture === 'agentic' || this.config.rag?.architecture === 'multi-agent' || (this.config.mcpServers && this.config.mcpServers.length > 0)) && this.multiAgentCoordinator) {
|
|
496
|
-
const stream = this.multiAgentCoordinator.runStream(question, history);
|
|
497
|
-
for await (const chunk of stream) {
|
|
498
|
-
yield chunk;
|
|
499
|
-
}
|
|
500
|
-
return;
|
|
501
|
-
}
|
|
502
|
-
const stream = this.askStreamInternal(question, history, namespace);
|
|
503
|
-
for await (const chunk of stream) {
|
|
504
|
-
yield chunk;
|
|
505
|
-
}
|
|
506
|
-
}
|
|
507
|
-
|
|
508
|
-
/**
|
|
509
|
-
* High-performance streaming RAG flow.
|
|
510
|
-
* Yields text chunks first, then the retrieval metadata + observability trace at the end.
|
|
511
|
-
*
|
|
512
|
-
* Latency optimizations:
|
|
513
|
-
* - Strategy classification runs in parallel with query embedding (saves ~400ms)
|
|
514
|
-
* - Hallucination scoring is fire-and-forget (doesn't block metadata yield)
|
|
515
|
-
* - UITransformation is computed after text streaming and emitted with metadata
|
|
516
|
-
* - SchemaMapper.train runs while answer generation streams
|
|
517
|
-
*/
|
|
518
|
-
async *askStreamInternal(question: string, history: ChatMessage[] = [], namespace?: string): AsyncIterable<string | ChatResponse> {
|
|
519
|
-
await this.initialize();
|
|
520
|
-
const ns = formatNamespace(namespace ?? this.config.projectId);
|
|
521
|
-
const topK = this.config.rag?.topK ?? 5;
|
|
522
|
-
// Default score threshold of 0.3 filters random matches while keeping
|
|
523
|
-
// genuinely relevant chunks. Users can override via rag.scoreThreshold.
|
|
524
|
-
const scoreThreshold = this.config.rag?.scoreThreshold ?? 0.3;
|
|
525
|
-
const requestId = `req_${Date.now()}_${Math.random().toString(36).slice(2, 8)}`;
|
|
526
|
-
const requestStart = performance.now();
|
|
527
|
-
|
|
528
|
-
try {
|
|
529
|
-
let searchQuery = question;
|
|
530
|
-
let rewrittenQuery: string | undefined;
|
|
531
|
-
|
|
532
|
-
// 1. Query Pre-processing
|
|
533
|
-
if (this.config.rag?.useQueryTransformation) {
|
|
534
|
-
searchQuery = await this.rewriteQuery(question, history);
|
|
535
|
-
rewrittenQuery = searchQuery !== question ? searchQuery : undefined;
|
|
536
|
-
}
|
|
537
|
-
|
|
538
|
-
// 2. Parallel: embed query + classify retrieval strategy at the same time.
|
|
539
|
-
// Previously strategy was awaited first (serialized LLM call), blocking embed+retrieve.
|
|
540
|
-
const hints = QueryProcessor.extractQueryFieldHints(question, this.config.rag?.filterableFields);
|
|
541
|
-
const filter = QueryProcessor.buildQueryFilter(question, hints);
|
|
542
|
-
const numericPredicates = QueryProcessor.extractNumericPredicates(question, this.config.rag?.filterableFields);
|
|
543
|
-
if (numericPredicates.length > 0) {
|
|
544
|
-
filter.__numericPredicates = numericPredicates;
|
|
545
|
-
}
|
|
546
|
-
|
|
547
|
-
const embedStart = performance.now();
|
|
548
|
-
|
|
549
|
-
// Parallelize: strategy classification + embedding run concurrently.
|
|
550
|
-
// Strategy is an LLM call (~300-800ms). Embedding is also async.
|
|
551
|
-
// Neither depends on the other's result, so both can run at the same time.
|
|
552
|
-
const cacheKey = `${ns}::${searchQuery}`;
|
|
553
|
-
const cachedVector = this.embeddingCache.get(cacheKey);
|
|
554
|
-
|
|
555
|
-
const arch = this.config.rag?.architecture;
|
|
556
|
-
const useGraph = arch !== 'naive' && arch !== 'retrieve-and-rerank' && this.graphDB && this.config.rag?.useGraphRetrieval;
|
|
557
|
-
const [strategyResult, embeddedVector] = await Promise.all([
|
|
558
|
-
QueryProcessor.determineRetrievalStrategy(
|
|
559
|
-
searchQuery,
|
|
560
|
-
useGraph ? this.llmRouter.get('fast') : undefined,
|
|
561
|
-
this.config.rag?.graphKeywords,
|
|
562
|
-
this.config.rag?.vectorKeywords
|
|
563
|
-
),
|
|
564
|
-
// Embed immediately regardless of strategy — costs nothing if cached.
|
|
565
|
-
// If strategy turns out to be 'graph'-only we just won't use the vector.
|
|
566
|
-
cachedVector
|
|
567
|
-
? Promise.resolve(cachedVector)
|
|
568
|
-
: this.embeddingProvider.embed(searchQuery, { taskType: 'query' }),
|
|
569
|
-
]);
|
|
570
|
-
|
|
571
|
-
const queryVector = embeddedVector;
|
|
572
|
-
if (!cachedVector && queryVector.length > 0) {
|
|
573
|
-
this.embeddingCache.set(cacheKey, queryVector);
|
|
574
|
-
}
|
|
575
|
-
|
|
576
|
-
const graphData = (strategyResult === 'graph' || strategyResult === 'both') && this.graphDB && this.config.rag?.useGraphRetrieval
|
|
577
|
-
? await this.graphDB.query(searchQuery)
|
|
578
|
-
: undefined;
|
|
579
|
-
|
|
580
|
-
const hasMetadataFilter = filter.metadata && Object.keys(filter.metadata).length > 0;
|
|
581
|
-
const hasNumericPredicates = Array.isArray(filter.__numericPredicates) && filter.__numericPredicates.length > 0;
|
|
582
|
-
const wantsExhaustiveList = true; // Always query all available resources across the database
|
|
583
|
-
const retrievalLimit = Math.max(topK * 50, 1000);
|
|
584
|
-
|
|
585
|
-
const rawSources = ((strategyResult === 'vector' || strategyResult === 'both') && queryVector && queryVector.length > 0
|
|
586
|
-
? await this.vectorDB.query(queryVector, retrievalLimit, ns, filter)
|
|
587
|
-
: []).filter((s): s is VectorMatch => Boolean(s && typeof s === 'object'));
|
|
588
|
-
|
|
589
|
-
// console.log('[Pipeline] Raw vectorDB.query response:', {
|
|
590
|
-
// rawCount: rawSources.length,
|
|
591
|
-
// sample: rawSources.slice(0, 2).map((s) => ({
|
|
592
|
-
// id: s?.id,
|
|
593
|
-
// score: s?.score,
|
|
594
|
-
// contentType: typeof s?.content,
|
|
595
|
-
// contentSnippet: String(s?.content ?? '').substring(0, 150),
|
|
596
|
-
// metadataKeys: Object.keys(s?.metadata || {}),
|
|
597
|
-
// })),
|
|
598
|
-
// });
|
|
599
|
-
|
|
600
|
-
const retrieveEnd = performance.now();
|
|
601
|
-
const embedMs = retrieveEnd - embedStart;
|
|
602
|
-
const retrieveMs = retrieveEnd - embedStart;
|
|
603
|
-
|
|
604
|
-
// 3. Reranking & Filtering across ALL matching resources
|
|
605
|
-
const rerankStart = performance.now();
|
|
606
|
-
const structuredSources = this.applyStructuredFilters(rawSources, filter).filter((s): s is VectorMatch => Boolean(s && typeof s === 'object'));
|
|
607
|
-
let fullSources = hasNumericPredicates
|
|
608
|
-
? structuredSources
|
|
609
|
-
: structuredSources.filter((m) => (m?.score ?? 0) >= scoreThreshold);
|
|
610
|
-
const rerankLimit = Math.max(retrievalLimit, fullSources.length);
|
|
611
|
-
const useReranking = arch !== 'naive' && (arch === 'retrieve-and-rerank' || this.config.rag?.useReranking);
|
|
612
|
-
if (!hasNumericPredicates && useReranking) {
|
|
613
|
-
fullSources = (await this.reranker.rerank(fullSources, question, rerankLimit)).filter((s): s is VectorMatch => Boolean(s && typeof s === 'object'));
|
|
614
|
-
}
|
|
615
|
-
const rerankMs = performance.now() - rerankStart;
|
|
616
|
-
|
|
617
|
-
// 4. Context Augmentation — Cap context sent to LLM to top 15 sources (~3000 tokens)
|
|
618
|
-
// to keep prompt under Groq's 6000 token limit while preserving ALL sources for UI citations.
|
|
619
|
-
const totalCount = fullSources.length;
|
|
620
|
-
const promptSources = fullSources.slice(0, 15);
|
|
621
|
-
let context = promptSources.length
|
|
622
|
-
? `[SEARCH SUMMARY: Found ${totalCount} matching record(s) out of ${rawSources.length} total resources searched in database. Context includes top ${promptSources.length} relevant records]\n\n` +
|
|
623
|
-
promptSources.map((m, i) => `[Source ${i + 1}]\n${m?.content ?? ''}`).join('\n\n---\n\n')
|
|
624
|
-
: 'No relevant context found.';
|
|
625
|
-
|
|
626
|
-
// Do NOT truncate UI sources — keep all qualified matches for UI citations & tables
|
|
627
|
-
const sources = [...fullSources]
|
|
628
|
-
.filter((s): s is VectorMatch => Boolean(s && typeof s === 'object'))
|
|
629
|
-
.sort((a, b) => (b?.score ?? 0) - (a?.score ?? 0));
|
|
630
|
-
|
|
631
|
-
// console.log('[Pipeline] Final sources for prompt & UI:', {
|
|
632
|
-
// count: sources.length,
|
|
633
|
-
// totalRawCount: rawSources.length,
|
|
634
|
-
// sources: sources.slice(0, 10).map((s, idx) => ({
|
|
635
|
-
// rank: idx + 1,
|
|
636
|
-
// id: s?.id,
|
|
637
|
-
// score: s?.score,
|
|
638
|
-
// contentType: typeof s?.content,
|
|
639
|
-
// contentSnippet: String(s?.content ?? '').substring(0, 150),
|
|
640
|
-
// metadata: s?.metadata,
|
|
641
|
-
// })),
|
|
642
|
-
// });
|
|
643
|
-
|
|
644
|
-
if (graphData && graphData.nodes.length > 0) {
|
|
645
|
-
const graphContext = graphData.nodes.map(n =>
|
|
646
|
-
`Entity: ${n.label} (${n.id})${n.properties ? ' - ' + JSON.stringify(n.properties) : ''}`
|
|
647
|
-
).join('\n');
|
|
648
|
-
context = `GRAPH KNOWLEDGE:\n${graphContext}\n\nVECTOR CONTEXT:\n${context}`;
|
|
649
|
-
}
|
|
650
|
-
|
|
651
|
-
if (this.config.rag?.cag?.enabled) {
|
|
652
|
-
if (!this.coldContexts.has(ns)) {
|
|
653
|
-
await this.loadColdContext(ns);
|
|
654
|
-
}
|
|
655
|
-
const coldContext = this.coldContexts.get(ns);
|
|
656
|
-
if (coldContext && coldContext !== 'No cold context found.' && coldContext !== 'Failed to load cold context.') {
|
|
657
|
-
context = `COLD CONTEXT (STATIC KNOWLEDGE):\n${coldContext}\n\nRETRIEVED HOT CONTEXT (REAL-TIME KNOWLEDGE):\n${context}`;
|
|
658
|
-
}
|
|
659
|
-
}
|
|
660
|
-
|
|
661
|
-
// Yield sources immediately to support early UI citation rendering (highly responsive)
|
|
662
|
-
yield {
|
|
663
|
-
reply: '',
|
|
664
|
-
sources: (sources || []).filter((s): s is VectorMatch => Boolean(s && typeof s === 'object')).map((s) => ({
|
|
665
|
-
id: s.id,
|
|
666
|
-
score: s.score ?? 0,
|
|
667
|
-
content: s?.content ?? '',
|
|
668
|
-
metadata: s?.metadata ?? {},
|
|
669
|
-
namespace: ns,
|
|
670
|
-
} satisfies RetrievedChunk)),
|
|
671
|
-
} as ChatResponse;
|
|
672
|
-
|
|
673
|
-
// 4.5 Schema Training — true fire-and-forget, never gates anything.
|
|
674
|
-
const allMetadataKeys = Array.from(new Set(sources.filter(Boolean).flatMap(s => Object.keys(s?.metadata || {}))));
|
|
675
|
-
const cachedSchema = allMetadataKeys.length > 0
|
|
676
|
-
? SchemaMapper.getCached(ns, allMetadataKeys)
|
|
677
|
-
: undefined;
|
|
678
|
-
|
|
679
|
-
// Kick off schema training in the background — don't await it before starting UI transform.
|
|
680
|
-
// The next request will benefit from the trained schema via the cache.
|
|
681
|
-
if (allMetadataKeys.length > 0 && !cachedSchema) {
|
|
682
|
-
SchemaMapper.train(this.llmProvider, ns, allMetadataKeys).catch(() => { /* non-critical */ });
|
|
683
|
-
}
|
|
684
|
-
|
|
685
|
-
// 4.6 Start UI Transformation concurrently with generation.
|
|
686
|
-
// For product queries: use the fast heuristic immediately — no LLM call needed.
|
|
687
|
-
// For analytics queries: fire analyzeAndDecide() in parallel with text streaming.
|
|
688
|
-
const isProductQ = UITransformer.isProductQueryPublic(question);
|
|
689
|
-
const uiTransformationPromise = isProductQ
|
|
690
|
-
// Product carousel: pure heuristic, zero LLM latency
|
|
691
|
-
? Promise.resolve(
|
|
692
|
-
UITransformer.transform(question, sources, this.config, cachedSchema,
|
|
693
|
-
{ visualizationHint: 'product_browse', recommendedChart: 'text', filterInStockOnly: false, wantsExplicitTable: false, isTemporal: false, isComparison: false, language: 'en' }
|
|
694
|
-
)
|
|
695
|
-
)
|
|
696
|
-
: this.generateUiTransformation(
|
|
697
|
-
question,
|
|
698
|
-
sources,
|
|
699
|
-
cachedSchema,
|
|
700
|
-
hasNumericPredicates,
|
|
701
|
-
).catch(uiError => {
|
|
702
|
-
console.warn('[Pipeline] UI transformation failed concurrently:', uiError);
|
|
703
|
-
return UITransformer.transform(question, sources, this.config, cachedSchema);
|
|
704
|
-
});
|
|
705
|
-
|
|
706
|
-
// 5. Generation (Streaming)
|
|
707
|
-
// Start hallucination scoring as a background promise BEFORE streaming
|
|
708
|
-
// so that the score runs concurrently with text delivery to the client.
|
|
709
|
-
// We await it together with uiTransformationPromise (see step 7) so the
|
|
710
|
-
// metadata frame is never yielded until the score is resolved — no
|
|
711
|
-
// post-yield mutation required.
|
|
712
|
-
let hallucinationScoringPromise: Promise<{ score: number; reason: string } | undefined> =
|
|
713
|
-
Promise.resolve(undefined);
|
|
714
|
-
|
|
715
|
-
const restrictionSuffix = '\n\n(IMPORTANT: Format your response beautifully using rich Markdown! Use bolding for emphasis, headings (##) for structure, bullet points for lists, and proper line breaks between paragraphs to make it highly readable. However, NEVER generate Markdown tables, HTML figures, or text charts (the UI renders requested charts separately, so NEVER say you cannot create, display, draw, render, or provide a chart/visualization). If listing products, use simple markdown bullet points or comma-separated names. NEVER use plus signs (+) to separate product names, category names, or list items. For product description/detail questions, provide customer-facing prose only and do NOT list internal catalog fields such as Handle, Body (HTML), Vendor, Type, Tags, Published, Option, Variant, SKU, or raw metadata labels. You CAN use numbers for years/counts like 2006 or 5800, but NEVER put a dollar sign ($) before them.)';
|
|
716
|
-
|
|
717
|
-
const isClaude37 = this.config.llm.model.includes('claude-3-7-sonnet');
|
|
718
|
-
const isNativeThinking = isClaude37 && (this.config.llm.options?.thinking === true || this.config.llm.options?.thinking === 'enabled' || this.config.llm.options?.thinking !== false);
|
|
719
|
-
|
|
720
|
-
const modelNameLower = this.config.llm.model.toLowerCase();
|
|
721
|
-
const isReasoningModel = modelNameLower.includes('-r1') || modelNameLower.includes('deepseek-r1') || modelNameLower.includes('reasoning') || modelNameLower.includes('thinking');
|
|
722
|
-
const isSimulatedThinking = !isNativeThinking && (
|
|
723
|
-
this.config.llm.options?.thinking === true ||
|
|
724
|
-
this.config.llm.options?.thinking === 'enabled' ||
|
|
725
|
-
this.config.llm.options?.thinking !== false
|
|
726
|
-
);
|
|
727
|
-
|
|
728
|
-
let finalRestrictionSuffix = restrictionSuffix;
|
|
729
|
-
if (isSimulatedThinking) {
|
|
730
|
-
finalRestrictionSuffix += '\n\n(IMPORTANT: You must think step-by-step before answering. Write your thinking process inside a `<think>...</think>` block. Write the `<think>` block first, then follow it with your final response. Keep the thinking process thorough and detailed. Example: `<think>Thinking details here...</think>Final answer text here.`)';
|
|
731
|
-
}
|
|
732
|
-
|
|
733
|
-
const hardenedHistory = (Array.isArray(history) ? history : [])
|
|
734
|
-
.filter((m): m is ChatMessage => Boolean(m && typeof m === 'object'))
|
|
735
|
-
.map((m) => ({
|
|
736
|
-
role: m.role || 'user',
|
|
737
|
-
content: typeof m.content === 'string' ? m.content : (m.content != null ? String(m.content) : ''),
|
|
738
|
-
}));
|
|
739
|
-
const userQuestion = { role: 'user', content: question + finalRestrictionSuffix } as ChatMessage;
|
|
740
|
-
const messages: ChatMessage[] = [...hardenedHistory, userQuestion];
|
|
741
|
-
|
|
742
|
-
const systemPrompt = this.config.llm.systemPrompt ?? '';
|
|
743
|
-
const userPrompt = messages.map(m => `${m.role}: ${m.content}`).join('\n');
|
|
744
|
-
|
|
745
|
-
let fullReply = '';
|
|
746
|
-
let thinkingText = '';
|
|
747
|
-
let thinkingStartMs = performance.now();
|
|
748
|
-
let thinkingDurationMs = 0;
|
|
749
|
-
const generateStart = performance.now();
|
|
750
|
-
|
|
751
|
-
if (this.llmProvider.chatStream) {
|
|
752
|
-
const stream = this.llmProvider.chatStream(messages, context);
|
|
753
|
-
if (!stream) throw new Error(`[Pipeline] ${this.config.llm.provider} chatStream returned undefined`);
|
|
754
|
-
|
|
755
|
-
let inThinking = false;
|
|
756
|
-
let textBuffer = '';
|
|
757
|
-
|
|
758
|
-
for await (const chunk of stream) {
|
|
759
|
-
// 1. Native Anthropic thinking
|
|
760
|
-
if (chunk.startsWith('\x00THINKING\x00')) {
|
|
761
|
-
const thinkingDelta = chunk.slice('\x00THINKING\x00'.length);
|
|
762
|
-
thinkingText += thinkingDelta;
|
|
763
|
-
thinkingDurationMs = performance.now() - thinkingStartMs;
|
|
764
|
-
yield { type: 'thinking', text: thinkingDelta } as any;
|
|
765
|
-
continue;
|
|
766
|
-
}
|
|
767
|
-
|
|
768
|
-
// 2. Simulated thinking tag parsing
|
|
769
|
-
if (isSimulatedThinking) {
|
|
770
|
-
textBuffer += chunk;
|
|
771
|
-
while (textBuffer.length > 0) {
|
|
772
|
-
if (!inThinking) {
|
|
773
|
-
const thinkIndex = textBuffer.indexOf('<think>');
|
|
774
|
-
if (thinkIndex !== -1) {
|
|
775
|
-
const before = textBuffer.slice(0, thinkIndex);
|
|
776
|
-
if (before) {
|
|
777
|
-
fullReply += before;
|
|
778
|
-
yield before;
|
|
779
|
-
}
|
|
780
|
-
inThinking = true;
|
|
781
|
-
thinkingStartMs = performance.now();
|
|
782
|
-
textBuffer = textBuffer.slice(thinkIndex + '<think>'.length);
|
|
783
|
-
} else {
|
|
784
|
-
const maxSafeLength = textBuffer.length - '<think>'.length;
|
|
785
|
-
if (maxSafeLength > 0) {
|
|
786
|
-
const safeText = textBuffer.slice(0, maxSafeLength);
|
|
787
|
-
fullReply += safeText;
|
|
788
|
-
yield safeText;
|
|
789
|
-
textBuffer = textBuffer.slice(maxSafeLength);
|
|
790
|
-
}
|
|
791
|
-
break;
|
|
792
|
-
}
|
|
793
|
-
} else {
|
|
794
|
-
const endThinkIndex = textBuffer.indexOf('</think>');
|
|
795
|
-
if (endThinkIndex !== -1) {
|
|
796
|
-
const thinkingDelta = textBuffer.slice(0, endThinkIndex);
|
|
797
|
-
if (thinkingDelta) {
|
|
798
|
-
thinkingText += thinkingDelta;
|
|
799
|
-
thinkingDurationMs = performance.now() - thinkingStartMs;
|
|
800
|
-
yield { type: 'thinking', text: thinkingDelta } as any;
|
|
801
|
-
}
|
|
802
|
-
inThinking = false;
|
|
803
|
-
textBuffer = textBuffer.slice(endThinkIndex + '</think>'.length);
|
|
804
|
-
} else {
|
|
805
|
-
const maxSafeLength = textBuffer.length - '</think>'.length;
|
|
806
|
-
if (maxSafeLength > 0) {
|
|
807
|
-
const thinkingDelta = textBuffer.slice(0, maxSafeLength);
|
|
808
|
-
thinkingText += thinkingDelta;
|
|
809
|
-
thinkingDurationMs = performance.now() - thinkingStartMs;
|
|
810
|
-
yield { type: 'thinking', text: thinkingDelta } as any;
|
|
811
|
-
textBuffer = textBuffer.slice(maxSafeLength);
|
|
812
|
-
}
|
|
813
|
-
break;
|
|
814
|
-
}
|
|
815
|
-
}
|
|
816
|
-
}
|
|
817
|
-
} else {
|
|
818
|
-
fullReply += chunk;
|
|
819
|
-
yield chunk;
|
|
820
|
-
}
|
|
821
|
-
}
|
|
822
|
-
|
|
823
|
-
// Flush any remaining buffer
|
|
824
|
-
if (isSimulatedThinking && textBuffer.length > 0) {
|
|
825
|
-
if (inThinking) {
|
|
826
|
-
thinkingText += textBuffer;
|
|
827
|
-
yield { type: 'thinking', text: textBuffer } as any;
|
|
828
|
-
} else {
|
|
829
|
-
fullReply += textBuffer;
|
|
830
|
-
yield textBuffer;
|
|
831
|
-
}
|
|
832
|
-
}
|
|
833
|
-
} else {
|
|
834
|
-
const reply = await this.llmProvider.chat(messages, context);
|
|
835
|
-
if (isSimulatedThinking) {
|
|
836
|
-
const thinkStart = reply.indexOf('<think>');
|
|
837
|
-
const thinkEnd = reply.indexOf('</think>');
|
|
838
|
-
if (thinkStart !== -1 && thinkEnd !== -1 && thinkEnd > thinkStart) {
|
|
839
|
-
thinkingText = reply.slice(thinkStart + '<think>'.length, thinkEnd);
|
|
840
|
-
fullReply = reply.slice(0, thinkStart) + reply.slice(thinkEnd + '</think>'.length);
|
|
841
|
-
thinkingDurationMs = 100;
|
|
842
|
-
} else {
|
|
843
|
-
fullReply = reply;
|
|
844
|
-
}
|
|
845
|
-
} else {
|
|
846
|
-
fullReply = reply;
|
|
847
|
-
}
|
|
848
|
-
yield fullReply;
|
|
849
|
-
}
|
|
850
|
-
|
|
851
|
-
// Kick off hallucination scoring ONLY when explicitly enabled by options
|
|
852
|
-
const runHallucination = this.config.llm.options?.hallucinationScoring === true;
|
|
853
|
-
if (runHallucination) {
|
|
854
|
-
hallucinationScoringPromise = scoreHallucination(this.llmProvider, fullReply, context)
|
|
855
|
-
.catch(() => undefined);
|
|
856
|
-
}
|
|
857
|
-
|
|
858
|
-
const generateMs = performance.now() - generateStart;
|
|
859
|
-
const totalMs = performance.now() - requestStart;
|
|
860
|
-
|
|
861
|
-
const latency: LatencyBreakdown = {
|
|
862
|
-
embedMs: Math.round(embedMs),
|
|
863
|
-
retrieveMs: Math.round(retrieveMs),
|
|
864
|
-
rerankMs: (arch !== 'naive' && (arch === 'retrieve-and-rerank' || this.config.rag?.useReranking)) ? Math.round(rerankMs) : undefined,
|
|
865
|
-
generateMs: Math.round(generateMs),
|
|
866
|
-
totalMs: Math.round(totalMs),
|
|
867
|
-
};
|
|
868
|
-
|
|
869
|
-
// 6. Token estimation
|
|
870
|
-
const promptText = systemPrompt + '\n' + context + '\n' + userPrompt;
|
|
871
|
-
const promptTokens = estimateTokens(promptText);
|
|
872
|
-
const completionTokens = estimateTokens(fullReply);
|
|
873
|
-
const tokens: TokenUsage = {
|
|
874
|
-
promptTokens,
|
|
875
|
-
completionTokens,
|
|
876
|
-
totalTokens: promptTokens + completionTokens,
|
|
877
|
-
estimatedCostUsd: estimateCostUsd(promptTokens, completionTokens, this.config.llm.model),
|
|
878
|
-
};
|
|
879
|
-
|
|
880
|
-
// 7. Await UI transformation and optionally hallucination score before yielding the metadata frame.
|
|
881
|
-
const awaitHallucination = this.config.llm.options?.awaitHallucination === true;
|
|
882
|
-
|
|
883
|
-
const [uiTransformation, hallucinationResult] = await Promise.all([
|
|
884
|
-
uiTransformationPromise,
|
|
885
|
-
awaitHallucination ? hallucinationScoringPromise : Promise.resolve(undefined),
|
|
886
|
-
]);
|
|
887
|
-
|
|
888
|
-
const buildTrace = (hScore?: { score: number; reason: string }): ObservabilityTrace => ({
|
|
889
|
-
requestId,
|
|
890
|
-
query: question,
|
|
891
|
-
rewrittenQuery,
|
|
892
|
-
systemPrompt,
|
|
893
|
-
userPrompt: question + finalRestrictionSuffix,
|
|
894
|
-
chunks: (sources || []).filter(Boolean).map((s) => ({
|
|
895
|
-
id: s.id,
|
|
896
|
-
score: s.score,
|
|
897
|
-
content: s?.content ?? '',
|
|
898
|
-
metadata: s?.metadata ?? {},
|
|
899
|
-
namespace: ns,
|
|
900
|
-
} satisfies RetrievedChunk)),
|
|
901
|
-
latency,
|
|
902
|
-
tokens,
|
|
903
|
-
timestamp: new Date().toISOString(),
|
|
904
|
-
...(hScore ? {
|
|
905
|
-
hallucinationScore: hScore.score,
|
|
906
|
-
hallucinationReason: hScore.reason,
|
|
907
|
-
} : {}),
|
|
908
|
-
});
|
|
909
|
-
|
|
910
|
-
const trace = buildTrace(hallucinationResult);
|
|
911
|
-
|
|
912
|
-
// Asynchronously trigger telemetry logging without blocking response yield
|
|
913
|
-
const isTelemetryActive = this.config.telemetry?.enabled ?? Boolean(this.config.licenseKey);
|
|
914
|
-
if (isTelemetryActive) {
|
|
915
|
-
const defaultUrl = 'https://www.retrivora.com/api/telemetry';
|
|
916
|
-
const telemetryUrl = this.config.telemetry?.url || process.env.TELEMETRY_URL || process.env.NEXT_PUBLIC_TELEMETRY_URL || defaultUrl;
|
|
917
|
-
const absoluteUrl = telemetryUrl.startsWith('http')
|
|
918
|
-
? telemetryUrl
|
|
919
|
-
: 'https://www.retrivora.com' + (telemetryUrl.startsWith('/') ? telemetryUrl : '/' + telemetryUrl);
|
|
920
|
-
|
|
921
|
-
// Start background process for telemetry, awaiting hallucination scoring if it was backgrounded
|
|
922
|
-
(async () => {
|
|
923
|
-
try {
|
|
924
|
-
let finalTrace = trace;
|
|
925
|
-
if (!awaitHallucination && runHallucination) {
|
|
926
|
-
const backgroundScoreResult = await hallucinationScoringPromise;
|
|
927
|
-
if (backgroundScoreResult) {
|
|
928
|
-
finalTrace = buildTrace(backgroundScoreResult);
|
|
929
|
-
}
|
|
930
|
-
}
|
|
931
|
-
|
|
932
|
-
const modelName = finalTrace?.model || this.config.llm?.model || 'llama-3.1-8b-instant';
|
|
933
|
-
const providerName = finalTrace?.provider || this.config.llm?.provider || 'groq';
|
|
934
|
-
const tokenCount = Number(finalTrace?.tokens?.totalTokens || 0);
|
|
935
|
-
const costEst = Number(finalTrace?.tokens?.estimatedCostUsd || 0);
|
|
936
|
-
const latencyDuration = Number(finalTrace?.latency?.totalMs || 0);
|
|
937
|
-
|
|
938
|
-
console.log(`[Retrivora Pipeline Telemetry] 📊 Dispatching RAG telemetry -> Project: "${ns}", Tokens: ${tokenCount}, Latency: ${latencyDuration}ms`);
|
|
939
|
-
|
|
940
|
-
await fetch(absoluteUrl, {
|
|
941
|
-
method: 'POST',
|
|
942
|
-
headers: {
|
|
943
|
-
'Content-Type': 'application/json',
|
|
944
|
-
},
|
|
945
|
-
body: JSON.stringify({
|
|
946
|
-
trace: finalTrace,
|
|
947
|
-
licenseKey: this.config.licenseKey,
|
|
948
|
-
projectId: ns,
|
|
949
|
-
organization: process.env.RETRIVORA_ORGANIZATION || 'default-org',
|
|
950
|
-
sdkVersion: SDK_VERSION,
|
|
951
|
-
model: modelName,
|
|
952
|
-
provider: providerName,
|
|
953
|
-
tokens: tokenCount,
|
|
954
|
-
cost: costEst,
|
|
955
|
-
costUsd: costEst,
|
|
956
|
-
latencyMs: latencyDuration,
|
|
957
|
-
status: 'success',
|
|
958
|
-
action: 'RAG_QUERY',
|
|
959
|
-
feature: 'RAG',
|
|
960
|
-
timestamp: new Date().toISOString(),
|
|
961
|
-
details: `Org: default-org | Model: ${providerName}/${modelName} | Tokens: ${tokenCount} | Latency: ${latencyDuration}ms | Cost: $${costEst} | SDK: v${SDK_VERSION} | Feature: RAG`
|
|
962
|
-
})
|
|
963
|
-
});
|
|
964
|
-
} catch (err: any) {
|
|
965
|
-
console.warn(`[Pipeline Telemetry] Failed to send trace async to ${absoluteUrl}:`, err.message);
|
|
966
|
-
}
|
|
967
|
-
})();
|
|
968
|
-
}
|
|
969
|
-
|
|
970
|
-
yield {
|
|
971
|
-
reply: '',
|
|
972
|
-
sources,
|
|
973
|
-
graphData,
|
|
974
|
-
ui_transformation: uiTransformation,
|
|
975
|
-
trace,
|
|
976
|
-
thinking: thinkingText || undefined,
|
|
977
|
-
thinkingMs: thinkingDurationMs > 0 ? Math.round(thinkingDurationMs) : undefined,
|
|
978
|
-
} as ChatResponse & { trace: ObservabilityTrace; thinking?: string; thinkingMs?: number };
|
|
979
|
-
|
|
980
|
-
} catch (error) {
|
|
981
|
-
throw new Error(`[Pipeline] Stream failed: ${error instanceof Error ? error.message : String(error)}`);
|
|
982
|
-
}
|
|
983
|
-
}
|
|
984
|
-
|
|
985
|
-
/**
|
|
986
|
-
* Universal retrieval method combining all enabled providers.
|
|
987
|
-
* Uses an LRU-bounded embedding cache to avoid re-embedding the same query.
|
|
988
|
-
*/
|
|
989
|
-
private async generateUiTransformation(
|
|
990
|
-
question: string,
|
|
991
|
-
sources: VectorMatch[],
|
|
992
|
-
cachedSchema?: SchemaMap,
|
|
993
|
-
forceDeterministic: boolean = false,
|
|
994
|
-
): Promise<unknown> {
|
|
995
|
-
if (!sources || sources.length === 0) {
|
|
996
|
-
return UITransformer.transform(question, sources, this.config, cachedSchema);
|
|
997
|
-
}
|
|
998
|
-
|
|
999
|
-
// Fast heuristic for product queries — skip both LLM calls entirely.
|
|
1000
|
-
if (UITransformer.isProductQueryPublic(question)) {
|
|
1001
|
-
return UITransformer.transform(question, sources, this.config, cachedSchema,
|
|
1002
|
-
{ visualizationHint: 'product_browse', recommendedChart: 'text', filterInStockOnly: false, wantsExplicitTable: false, isTemporal: false, isComparison: false, language: 'en' }
|
|
1003
|
-
);
|
|
1004
|
-
}
|
|
1005
|
-
|
|
1006
|
-
const enableLlmUiTransform = this.config.llm.options?.enableLlmUiTransform === true;
|
|
1007
|
-
if (forceDeterministic || !enableLlmUiTransform) {
|
|
1008
|
-
return UITransformer.transform(question, sources, this.config, cachedSchema);
|
|
1009
|
-
}
|
|
1010
|
-
|
|
1011
|
-
try {
|
|
1012
|
-
return await UITransformer.analyzeAndDecide(question, sources, this.llmRouter.get('fast'));
|
|
1013
|
-
} catch (err) {
|
|
1014
|
-
console.warn('[Pipeline] generateUiTransformation failed, using heuristic fallback:', err);
|
|
1015
|
-
return UITransformer.transform(question, sources, this.config, cachedSchema);
|
|
1016
|
-
}
|
|
1017
|
-
}
|
|
1018
|
-
|
|
1019
|
-
private applyStructuredFilters(sources: VectorMatch[], filter: QueryFilter): VectorMatch[] {
|
|
1020
|
-
const predicates = Array.isArray(filter.__numericPredicates)
|
|
1021
|
-
? filter.__numericPredicates
|
|
1022
|
-
: [];
|
|
1023
|
-
if (predicates.length === 0) return sources;
|
|
1024
|
-
|
|
1025
|
-
return sources
|
|
1026
|
-
.filter(source => predicates.every(predicate => {
|
|
1027
|
-
const value = this.resolveNumericPredicateValue(source, predicate);
|
|
1028
|
-
return value !== null && this.matchesNumericPredicate(value, predicate);
|
|
1029
|
-
}))
|
|
1030
|
-
.sort((a, b) => {
|
|
1031
|
-
const primary = predicates[0];
|
|
1032
|
-
const aValue = this.resolveNumericPredicateValue(a, primary) ?? 0;
|
|
1033
|
-
const bValue = this.resolveNumericPredicateValue(b, primary) ?? 0;
|
|
1034
|
-
return primary.operator === 'lt' || primary.operator === 'lte'
|
|
1035
|
-
? aValue - bValue
|
|
1036
|
-
: bValue - aValue;
|
|
1037
|
-
});
|
|
1038
|
-
}
|
|
1039
|
-
|
|
1040
|
-
private resolveNumericPredicateValue(source: VectorMatch, predicate: NumericPredicate): number | null {
|
|
1041
|
-
if (!source) return null;
|
|
1042
|
-
const meta = source.metadata || {};
|
|
1043
|
-
const field = predicate.field;
|
|
1044
|
-
const entries = Object.entries(meta)
|
|
1045
|
-
.filter(([, value]) => value !== null && value !== undefined && typeof value !== 'object');
|
|
1046
|
-
|
|
1047
|
-
if (field) {
|
|
1048
|
-
const normalizedField = this.normalizeComparableField(field);
|
|
1049
|
-
const exact = entries.find(([key]) => this.normalizeComparableField(key) === normalizedField);
|
|
1050
|
-
const fuzzy = exact ?? entries
|
|
1051
|
-
.map(([key, value]) => ({ key, value, score: this.fieldSimilarityScore(key, field) }))
|
|
1052
|
-
.filter(candidate => candidate.score > 0)
|
|
1053
|
-
.sort((a, b) => b.score - a.score)[0];
|
|
1054
|
-
if (fuzzy) {
|
|
1055
|
-
const value = this.toFiniteNumber(Array.isArray(fuzzy) ? fuzzy[1] : fuzzy.value);
|
|
1056
|
-
if (value !== null) return value;
|
|
1057
|
-
}
|
|
1058
|
-
|
|
1059
|
-
const contentValue = this.extractNumericValueFromContent(source.content ?? '', field);
|
|
1060
|
-
if (contentValue !== null) return contentValue;
|
|
1061
|
-
}
|
|
1062
|
-
|
|
1063
|
-
for (const [key, value] of entries) {
|
|
1064
|
-
if (/(id|sku|ean|upc|phone|zip|postal|code)/i.test(key)) continue;
|
|
1065
|
-
const numeric = this.toFiniteNumber(value);
|
|
1066
|
-
if (numeric !== null) return numeric;
|
|
1067
|
-
}
|
|
1068
|
-
|
|
1069
|
-
return null;
|
|
1070
|
-
}
|
|
1071
|
-
|
|
1072
|
-
private extractNumericValueFromContent(content: string, field: string): number | null {
|
|
1073
|
-
if (!content || typeof content !== 'string') return null;
|
|
1074
|
-
const escapedWords = field
|
|
1075
|
-
.split(/\s+|_+|-+/)
|
|
1076
|
-
.filter(Boolean)
|
|
1077
|
-
.map(word => word.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'));
|
|
1078
|
-
if (escapedWords.length === 0) return null;
|
|
1079
|
-
|
|
1080
|
-
const pattern = new RegExp(`(?:${escapedWords.join('[\\\\s_:-]*')})\\s*[:\\-–]?\\s*([\\d,]+(?:\\.\\d+)?)`, 'i');
|
|
1081
|
-
const match = content.match(pattern);
|
|
1082
|
-
return match ? this.toFiniteNumber(match[1]) : null;
|
|
1083
|
-
}
|
|
1084
|
-
|
|
1085
|
-
private matchesNumericPredicate(value: number, predicate: NumericPredicate): boolean {
|
|
1086
|
-
switch (predicate.operator) {
|
|
1087
|
-
case 'gt':
|
|
1088
|
-
return value > predicate.value;
|
|
1089
|
-
case 'gte':
|
|
1090
|
-
return value >= predicate.value;
|
|
1091
|
-
case 'lt':
|
|
1092
|
-
return value < predicate.value;
|
|
1093
|
-
case 'lte':
|
|
1094
|
-
return value <= predicate.value;
|
|
1095
|
-
case 'eq':
|
|
1096
|
-
default:
|
|
1097
|
-
return value === predicate.value;
|
|
1098
|
-
}
|
|
1099
|
-
}
|
|
1100
|
-
|
|
1101
|
-
private normalizeComparableField(value: string): string {
|
|
1102
|
-
return value.toLowerCase().replace(/[^a-z0-9]/g, '');
|
|
1103
|
-
}
|
|
1104
|
-
|
|
1105
|
-
private fieldSimilarityScore(candidate: string, requested: string): number {
|
|
1106
|
-
const normalizedCandidate = this.normalizeComparableField(candidate);
|
|
1107
|
-
const normalizedRequested = this.normalizeComparableField(requested);
|
|
1108
|
-
if (normalizedCandidate === normalizedRequested) return 10;
|
|
1109
|
-
if (normalizedCandidate.includes(normalizedRequested) || normalizedRequested.includes(normalizedCandidate)) return 8;
|
|
1110
|
-
|
|
1111
|
-
const candidateTokens = this.fieldTokens(candidate);
|
|
1112
|
-
const requestedTokens = this.fieldTokens(requested);
|
|
1113
|
-
const overlap = requestedTokens.filter(token => candidateTokens.includes(token)).length;
|
|
1114
|
-
if (overlap === 0) return 0;
|
|
1115
|
-
return overlap / Math.max(requestedTokens.length, candidateTokens.length);
|
|
1116
|
-
}
|
|
1117
|
-
|
|
1118
|
-
private fieldTokens(value: string): string[] {
|
|
1119
|
-
return value
|
|
1120
|
-
.toLowerCase()
|
|
1121
|
-
.split(/[^a-z0-9]+/)
|
|
1122
|
-
.map(token => token.replace(/ies$/, 'y').replace(/s$/, ''))
|
|
1123
|
-
.filter(token => token.length > 1);
|
|
1124
|
-
}
|
|
1125
|
-
|
|
1126
|
-
private toFiniteNumber(value: unknown): number | null {
|
|
1127
|
-
if (typeof value === 'number') return Number.isFinite(value) ? value : null;
|
|
1128
|
-
const numeric = Number(String(value ?? '').replace(/[$,% ,]/g, ''));
|
|
1129
|
-
return Number.isFinite(numeric) ? numeric : null;
|
|
1130
|
-
}
|
|
1131
|
-
|
|
1132
|
-
async retrieve(query: string, options: { namespace?: string; topK?: number; filter?: Record<string, unknown> }): Promise<RetrievalResult> {
|
|
1133
|
-
const ns = formatNamespace(options.namespace ?? this.config.projectId);
|
|
1134
|
-
const topK = options.topK ?? 5;
|
|
1135
|
-
|
|
1136
|
-
const cacheKey = `${ns}::${query}`;
|
|
1137
|
-
let queryVector = this.embeddingCache.get(cacheKey);
|
|
1138
|
-
|
|
1139
|
-
// Determine strategy
|
|
1140
|
-
const useGraph = this.graphDB && this.config.rag?.useGraphRetrieval;
|
|
1141
|
-
const strategy = await QueryProcessor.determineRetrievalStrategy(
|
|
1142
|
-
query,
|
|
1143
|
-
useGraph ? this.llmRouter.get('fast') : undefined
|
|
1144
|
-
);
|
|
1145
|
-
console.debug(`[Pipeline] Determined retrieval strategy: ${strategy}`);
|
|
1146
|
-
|
|
1147
|
-
const [retrievedVector, graphData] = await Promise.all([
|
|
1148
|
-
// Only embed if we need vector search (strategy is 'vector' or 'both')
|
|
1149
|
-
(strategy === 'vector' || strategy === 'both')
|
|
1150
|
-
? (queryVector ? Promise.resolve(queryVector) : this.embeddingProvider.embed(query, { taskType: 'query' }))
|
|
1151
|
-
: Promise.resolve([]),
|
|
1152
|
-
|
|
1153
|
-
// Only query graph if we need graph search (strategy is 'graph' or 'both')
|
|
1154
|
-
(strategy === 'graph' || strategy === 'both') && this.graphDB && this.config.rag?.useGraphRetrieval
|
|
1155
|
-
? this.graphDB.query(query)
|
|
1156
|
-
: Promise.resolve(undefined)
|
|
1157
|
-
]);
|
|
1158
|
-
|
|
1159
|
-
if ((strategy === 'vector' || strategy === 'both') && !queryVector && retrievedVector && retrievedVector.length > 0) {
|
|
1160
|
-
this.embeddingCache.set(cacheKey, retrievedVector);
|
|
1161
|
-
queryVector = retrievedVector;
|
|
1162
|
-
}
|
|
1163
|
-
|
|
1164
|
-
const baseFilter = { ...(options.filter ?? {}), queryText: query } as QueryFilter;
|
|
1165
|
-
const numericPredicates = QueryProcessor.extractNumericPredicates(query, this.config.rag?.filterableFields);
|
|
1166
|
-
if (numericPredicates.length > 0) {
|
|
1167
|
-
baseFilter.__numericPredicates = numericPredicates;
|
|
1168
|
-
}
|
|
1169
|
-
const retrievalLimit = numericPredicates.length > 0 ? Math.max(topK * 20, 100) : topK;
|
|
1170
|
-
|
|
1171
|
-
const sources = (strategy === 'vector' || strategy === 'both') && queryVector && queryVector.length > 0
|
|
1172
|
-
? this.applyStructuredFilters(
|
|
1173
|
-
await this.vectorDB.query(queryVector, retrievalLimit, ns, baseFilter),
|
|
1174
|
-
baseFilter,
|
|
1175
|
-
)
|
|
1176
|
-
: [];
|
|
1177
|
-
|
|
1178
|
-
// Multi-Vector Support: Check for parent_id in metadata
|
|
1179
|
-
const resolvedSources: VectorMatch[] = [];
|
|
1180
|
-
for (const source of sources) {
|
|
1181
|
-
const parentId = source.metadata?.parent_id;
|
|
1182
|
-
if (parentId) {
|
|
1183
|
-
console.log(`[Pipeline] Multi-Vector: Found child chunk. Parent ID: ${parentId}`);
|
|
1184
|
-
// In a full implementation, we would fetch the parent document here.
|
|
1185
|
-
// For now, we pass the child chunk forward.
|
|
1186
|
-
resolvedSources.push(source);
|
|
1187
|
-
} else {
|
|
1188
|
-
resolvedSources.push(source);
|
|
1189
|
-
}
|
|
1190
|
-
}
|
|
1191
|
-
|
|
1192
|
-
console.log('[Pipeline] retrieve() response:', {
|
|
1193
|
-
query,
|
|
1194
|
-
namespace: ns,
|
|
1195
|
-
count: resolvedSources.length,
|
|
1196
|
-
sample: resolvedSources.slice(0, 2).map((s) => ({
|
|
1197
|
-
id: s?.id,
|
|
1198
|
-
score: s?.score,
|
|
1199
|
-
contentSnippet: String(s?.content ?? '').substring(0, 150),
|
|
1200
|
-
metadata: s?.metadata,
|
|
1201
|
-
})),
|
|
1202
|
-
});
|
|
1203
|
-
|
|
1204
|
-
return { sources: resolvedSources, graphData };
|
|
1205
|
-
}
|
|
1206
|
-
|
|
1207
|
-
/** Rewrite the user query for better retrieval performance. */
|
|
1208
|
-
private async rewriteQuery(question: string, history: ChatMessage[]): Promise<string> {
|
|
1209
|
-
const prompt = `Given the following conversation history and a new question, rewrite the question to be a better search query for a vector database.
|
|
1210
|
-
Focus on extracting the core intent and entities. Do not answer the question, just rewrite it.
|
|
1211
|
-
|
|
1212
|
-
History:
|
|
1213
|
-
${(history || []).map(m => `${m?.role || 'user'}: ${m?.content ?? ''}`).join('\n')}
|
|
1214
|
-
|
|
1215
|
-
New Question: ${question}
|
|
1216
|
-
|
|
1217
|
-
Optimized Search Query:`;
|
|
1218
|
-
|
|
1219
|
-
const rewrite = await this.llmProvider.chat(
|
|
1220
|
-
[
|
|
1221
|
-
{ role: 'user', content: prompt }
|
|
1222
|
-
],
|
|
1223
|
-
'',
|
|
1224
|
-
{
|
|
1225
|
-
systemPrompt: 'You are an assistant that optimizes search queries for RAG systems. Focus on extracting the core intent and entities. Do not answer the question, just rewrite it.',
|
|
1226
|
-
temperature: 0
|
|
1227
|
-
}
|
|
1228
|
-
);
|
|
1229
|
-
|
|
1230
|
-
return rewrite.trim() || question;
|
|
1231
|
-
}
|
|
1232
|
-
|
|
1233
|
-
/** Generate 3-5 short, relevant questions based on the vector database content. */
|
|
1234
|
-
async getSuggestions(query: string, namespace?: string): Promise<string[]> {
|
|
1235
|
-
if (!query || query.trim().length < 3) return [];
|
|
1236
|
-
|
|
1237
|
-
await this.initialize();
|
|
1238
|
-
const ns = formatNamespace(namespace ?? this.config.projectId);
|
|
1239
|
-
|
|
1240
|
-
try {
|
|
1241
|
-
const { sources } = await this.retrieve(query, { namespace: ns, topK: 5 });
|
|
1242
|
-
if (!sources || sources.length === 0) return [];
|
|
1243
|
-
|
|
1244
|
-
const context = sources.map((s) => s?.content ?? '').join('\n\n---\n\n');
|
|
1245
|
-
const prompt = `Based on the following snippets from a document, what are at least 5 short, relevant questions a user might ask?
|
|
1246
|
-
Focus on questions that can be answered by the context.
|
|
1247
|
-
Keep each question under 10 words and make them very specific to the content.
|
|
1248
|
-
Return ONLY a JSON array of strings like ["Question 1", "Question 2", "Question 3", "Question 4", "Question 5"].
|
|
1249
|
-
|
|
1250
|
-
Context:
|
|
1251
|
-
${context}
|
|
1252
|
-
|
|
1253
|
-
Suggestions:`;
|
|
1254
|
-
|
|
1255
|
-
const response = await this.llmProvider.chat(
|
|
1256
|
-
[
|
|
1257
|
-
{ role: 'user', content: prompt }
|
|
1258
|
-
],
|
|
1259
|
-
'',
|
|
1260
|
-
{
|
|
1261
|
-
systemPrompt: 'You are a helpful assistant that generates search suggestions based on the provided snippets. Focus on questions that can be answered by the context, keep each under 10 words, and return ONLY a JSON array of strings.',
|
|
1262
|
-
temperature: 0
|
|
1263
|
-
}
|
|
1264
|
-
);
|
|
1265
|
-
|
|
1266
|
-
const match = response.match(/\[[\s\S]*\]/);
|
|
1267
|
-
if (match) {
|
|
1268
|
-
const suggestions = JSON.parse(match[0]);
|
|
1269
|
-
if (Array.isArray(suggestions)) {
|
|
1270
|
-
return suggestions.map(s => String(s)).slice(0, 5);
|
|
1271
|
-
}
|
|
1272
|
-
}
|
|
1273
|
-
} catch (error) {
|
|
1274
|
-
console.error('[Pipeline] Failed to generate suggestions:', error);
|
|
1275
|
-
}
|
|
1276
|
-
|
|
1277
|
-
return [];
|
|
1278
|
-
}
|
|
1279
|
-
}
|