@retrivora-ai/rag-engine 2.4.3 → 2.4.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{BatchProcessor-zbQMaZHm.d.mts → BatchProcessor-Csy_K1qQ.d.mts} +2 -0
- package/dist/{BatchProcessor-D42E706a.d.ts → BatchProcessor-Dg18zZGk.d.ts} +2 -0
- package/dist/handlers/index.js +27 -4
- package/dist/handlers/index.mjs +27 -4
- package/dist/index.d.mts +3 -3
- package/dist/index.d.ts +3 -3
- package/dist/index.js +12 -10
- package/dist/index.mjs +12 -10
- package/dist/server.d.mts +2 -2
- package/dist/server.d.ts +2 -2
- package/dist/server.js +28 -4
- package/dist/server.mjs +28 -4
- package/package.json +1 -3
- package/src/LicensedRetrivora.ts +0 -278
- package/src/app/api/chat/route.ts +0 -4
- package/src/app/api/health/route.ts +0 -5
- package/src/app/api/ingest/route.ts +0 -27
- package/src/app/api/suggestions/route.ts +0 -5
- package/src/app/api/upload/route.ts +0 -5
- package/src/app/constants.tsx +0 -270
- package/src/app/favicon.ico +0 -0
- package/src/app/globals.css +0 -449
- package/src/app/layout.tsx +0 -22
- package/src/app/page.tsx +0 -64
- package/src/app/types.ts +0 -30
- package/src/browser.tsx +0 -127
- package/src/components/ChatWidget.tsx +0 -393
- package/src/components/ChatWindow.tsx +0 -952
- package/src/components/CodeViewer.tsx +0 -56
- package/src/components/ConfigProvider.tsx +0 -141
- package/src/components/DocumentUpload.tsx +0 -354
- package/src/components/DynamicChart.tsx +0 -297
- package/src/components/HourglassLoader.tsx +0 -87
- package/src/components/MarkdownComponents.tsx +0 -186
- package/src/components/MessageBubble.tsx +0 -498
- package/src/components/ObservabilityPanel.tsx +0 -374
- package/src/components/ProductCard.tsx +0 -107
- package/src/components/ProductCarousel.tsx +0 -97
- package/src/components/SourceCard.tsx +0 -61
- package/src/components/ThemeProvider.tsx +0 -8
- package/src/components/ThemeToggle.tsx +0 -29
- package/src/components/ThinkingBlock.tsx +0 -75
- package/src/components/UIDispatcher.tsx +0 -342
- package/src/components/VisualizationRenderer.tsx +0 -621
- package/src/components/constants.tsx +0 -503
- package/src/config/ConfigBuilder.ts +0 -285
- package/src/config/EmbeddingStrategy.ts +0 -148
- package/src/config/LicenseConfigResolver.ts +0 -298
- package/src/config/RagConfig.ts +0 -328
- package/src/config/UniversalProfiles.ts +0 -82
- package/src/config/constants.ts +0 -64
- package/src/config/serverConfig.ts +0 -378
- package/src/config/uiConstants.ts +0 -47
- package/src/core/BatchProcessor.ts +0 -385
- package/src/core/CircuitBreaker.ts +0 -118
- package/src/core/ConfigFetcher.ts +0 -220
- package/src/core/ConfigResolver.ts +0 -165
- package/src/core/ConfigValidator.ts +0 -197
- package/src/core/DatabaseStorage.ts +0 -526
- package/src/core/FreeTierLimitsGuard.ts +0 -281
- package/src/core/LLMRouter.ts +0 -79
- package/src/core/LangChainAgent.ts +0 -132
- package/src/core/LicenseValidator.ts +0 -284
- package/src/core/LicenseVerifier.ts +0 -261
- package/src/core/MultiAgentCoordinator.ts +0 -239
- package/src/core/Pipeline.ts +0 -1279
- package/src/core/PluginManager.ts +0 -277
- package/src/core/ProjectSettings.ts +0 -247
- package/src/core/ProviderHealthCheck.ts +0 -133
- package/src/core/ProviderInterfaces.ts +0 -37
- package/src/core/ProviderRegistry.ts +0 -156
- package/src/core/QueryProcessor.ts +0 -399
- package/src/core/Retrivora.ts +0 -98
- package/src/core/VectorPlugin.ts +0 -193
- package/src/core/mcp.ts +0 -261
- package/src/exceptions/index.ts +0 -131
- package/src/handlers/index.ts +0 -1625
- package/src/hooks/useHydrated.ts +0 -13
- package/src/hooks/useProjectSettings.ts +0 -105
- package/src/hooks/useRagChat.ts +0 -436
- package/src/hooks/useStoredMessages.ts +0 -64
- package/src/index.css +0 -4269
- package/src/index.ts +0 -137
- package/src/lib/plugin.ts +0 -24
- package/src/llm/ILLMProvider.ts +0 -60
- package/src/llm/LLMFactory.ts +0 -141
- package/src/llm/providers/AnthropicProvider.ts +0 -215
- package/src/llm/providers/GeminiProvider.ts +0 -311
- package/src/llm/providers/GroqProvider.ts +0 -184
- package/src/llm/providers/OllamaProvider.ts +0 -232
- package/src/llm/providers/OpenAIProvider.ts +0 -188
- package/src/llm/providers/QwenProvider.ts +0 -199
- package/src/llm/providers/UniversalLLMAdapter.ts +0 -422
- package/src/llm/utils.ts +0 -46
- package/src/providers/graphdb/BaseGraphProvider.ts +0 -43
- package/src/providers/graphdb/SimpleGraphProvider.ts +0 -66
- package/src/providers/vectordb/BaseVectorProvider.ts +0 -98
- package/src/providers/vectordb/ChromaDBProvider.ts +0 -130
- package/src/providers/vectordb/MilvusProvider.ts +0 -134
- package/src/providers/vectordb/MongoDBProvider.ts +0 -250
- package/src/providers/vectordb/MultiTablePostgresProvider.ts +0 -461
- package/src/providers/vectordb/PineconeProvider.ts +0 -295
- package/src/providers/vectordb/PostgreSQLProvider.ts +0 -227
- package/src/providers/vectordb/QdrantProvider.ts +0 -294
- package/src/providers/vectordb/RedisProvider.ts +0 -105
- package/src/providers/vectordb/UniversalVectorProvider.ts +0 -221
- package/src/providers/vectordb/WeaviateProvider.ts +0 -172
- package/src/rag/DocumentChunker.ts +0 -144
- package/src/rag/EntityExtractor.ts +0 -90
- package/src/rag/LlamaIndexIngestor.ts +0 -60
- package/src/rag/Reranker.ts +0 -106
- package/src/rendering/IntentClassifier.ts +0 -167
- package/src/rendering/RendererRegistry.ts +0 -164
- package/src/rendering/RuleEngine.ts +0 -67
- package/src/rendering/VisualizationDecisionEngine.ts +0 -164
- package/src/rendering/__tests__/VisualizationDecisionEngine.test.ts +0 -237
- package/src/rendering/index.ts +0 -12
- package/src/rendering/rules/Rule1SpecificInfoRule.ts +0 -28
- package/src/rendering/rules/Rule2ComparisonRule.ts +0 -24
- package/src/rendering/rules/Rule3ProductDiscoveryRule.ts +0 -28
- package/src/rendering/rules/Rule4AnalyticalRule.ts +0 -38
- package/src/rendering/rules/Rule5MixedResponseRule.ts +0 -29
- package/src/rendering/rules/Rule6SmallResultSetRule.ts +0 -24
- package/src/rendering/rules/Rule7LargeTableRule.ts +0 -35
- package/src/rendering/types.ts +0 -96
- package/src/server.ts +0 -156
- package/src/tailwind.css +0 -194
- package/src/telemetry/TelemetryService.ts +0 -330
- package/src/types/chat.ts +0 -89
- package/src/types/index.ts +0 -278
- package/src/types/pdf-parse.d.ts +0 -13
- package/src/types/props.ts +0 -169
- package/src/utils/DocumentParser.ts +0 -111
- package/src/utils/ProductExtractor.ts +0 -350
- package/src/utils/SchemaMapper.ts +0 -176
- package/src/utils/UITransformer.ts +0 -1812
- package/src/utils/synonyms.ts +0 -80
- package/src/utils/templateUtils.ts +0 -78
- package/src/version.ts +0 -6
|
@@ -1,156 +0,0 @@
|
|
|
1
|
-
import { BaseVectorProvider } from '../providers/vectordb/BaseVectorProvider';
|
|
2
|
-
import { BaseGraphProvider } from '../providers/graphdb/BaseGraphProvider';
|
|
3
|
-
import { VectorDBConfig, GraphDBConfig, LLMConfig, EmbeddingConfig } from '../config/RagConfig';
|
|
4
|
-
import { LLMFactory } from '../llm/LLMFactory';
|
|
5
|
-
import { ILLMProvider } from '../llm/ILLMProvider';
|
|
6
|
-
import { IProviderValidator, IProviderHealthChecker } from './ProviderInterfaces';
|
|
7
|
-
import { ProviderNotFoundException } from '../exceptions';
|
|
8
|
-
|
|
9
|
-
/**
|
|
10
|
-
* ProviderRegistry — dynamic provider loader for Vector DBs and LLMs.
|
|
11
|
-
*/
|
|
12
|
-
type VectorProviderClass = {
|
|
13
|
-
new(config: VectorDBConfig): BaseVectorProvider;
|
|
14
|
-
getValidator?: () => IProviderValidator;
|
|
15
|
-
getHealthChecker?: () => IProviderHealthChecker;
|
|
16
|
-
};
|
|
17
|
-
|
|
18
|
-
type GraphProviderClass = new (config: GraphDBConfig) => BaseGraphProvider;
|
|
19
|
-
|
|
20
|
-
export class ProviderRegistry {
|
|
21
|
-
private static vectorProviders: Record<string, VectorProviderClass> = {};
|
|
22
|
-
private static graphProviders: Record<string, GraphProviderClass> = {};
|
|
23
|
-
|
|
24
|
-
private static vectorValidators: Record<string, IProviderValidator> = {};
|
|
25
|
-
private static vectorHealthCheckers: Record<string, IProviderHealthChecker> = {};
|
|
26
|
-
|
|
27
|
-
private static llmValidators: Record<string, IProviderValidator> = {};
|
|
28
|
-
private static llmHealthCheckers: Record<string, IProviderHealthChecker> = {};
|
|
29
|
-
|
|
30
|
-
static registerVectorProvider(name: string, providerClass: VectorProviderClass) {
|
|
31
|
-
this.vectorProviders[name] = providerClass;
|
|
32
|
-
if (providerClass.getValidator) {
|
|
33
|
-
this.vectorValidators[name] = providerClass.getValidator();
|
|
34
|
-
}
|
|
35
|
-
if (providerClass.getHealthChecker) {
|
|
36
|
-
this.vectorHealthCheckers[name] = providerClass.getHealthChecker();
|
|
37
|
-
}
|
|
38
|
-
}
|
|
39
|
-
|
|
40
|
-
static async getVectorValidator(provider: string): Promise<IProviderValidator | null> {
|
|
41
|
-
if (this.vectorValidators[provider]) return this.vectorValidators[provider];
|
|
42
|
-
|
|
43
|
-
// Try to load built-in validator
|
|
44
|
-
try {
|
|
45
|
-
const providerClass = await this.loadVectorProviderClass(provider);
|
|
46
|
-
if (providerClass.getValidator) {
|
|
47
|
-
this.vectorValidators[provider] = providerClass.getValidator();
|
|
48
|
-
return this.vectorValidators[provider];
|
|
49
|
-
}
|
|
50
|
-
} catch (e) {
|
|
51
|
-
console.warn(`[ProviderRegistry] Failed to load validator for ${provider}:`, e);
|
|
52
|
-
}
|
|
53
|
-
return null;
|
|
54
|
-
}
|
|
55
|
-
|
|
56
|
-
static async getVectorHealthChecker(provider: string): Promise<IProviderHealthChecker | null> {
|
|
57
|
-
if (this.vectorHealthCheckers[provider]) return this.vectorHealthCheckers[provider];
|
|
58
|
-
|
|
59
|
-
try {
|
|
60
|
-
const providerClass = await this.loadVectorProviderClass(provider);
|
|
61
|
-
if (providerClass.getHealthChecker) {
|
|
62
|
-
this.vectorHealthCheckers[provider] = providerClass.getHealthChecker();
|
|
63
|
-
return this.vectorHealthCheckers[provider];
|
|
64
|
-
}
|
|
65
|
-
} catch (e) {
|
|
66
|
-
console.warn(`[ProviderRegistry] Failed to load health checker for ${provider}:`, e);
|
|
67
|
-
}
|
|
68
|
-
return null;
|
|
69
|
-
}
|
|
70
|
-
|
|
71
|
-
private static async loadVectorProviderClass(provider: string): Promise<VectorProviderClass> {
|
|
72
|
-
if (this.vectorProviders[provider]) return this.vectorProviders[provider];
|
|
73
|
-
|
|
74
|
-
switch (provider) {
|
|
75
|
-
case 'pinecone': {
|
|
76
|
-
const { PineconeProvider } = await import('../providers/vectordb/PineconeProvider');
|
|
77
|
-
return PineconeProvider;
|
|
78
|
-
}
|
|
79
|
-
case 'pgvector':
|
|
80
|
-
case 'postgresql': {
|
|
81
|
-
// POSTGRES_MODE=single uses the simple single-table provider (default: multi).
|
|
82
|
-
const postgresMode = (process.env.POSTGRES_MODE ?? 'multi').toLowerCase();
|
|
83
|
-
if (postgresMode === 'single') {
|
|
84
|
-
const { PostgreSQLProvider } = await import('../providers/vectordb/PostgreSQLProvider');
|
|
85
|
-
return PostgreSQLProvider;
|
|
86
|
-
}
|
|
87
|
-
const { MultiTablePostgresProvider } = await import('../providers/vectordb/MultiTablePostgresProvider');
|
|
88
|
-
return MultiTablePostgresProvider as unknown as VectorProviderClass;
|
|
89
|
-
}
|
|
90
|
-
case 'mongodb': {
|
|
91
|
-
const { MongoDBProvider } = await import('../providers/vectordb/MongoDBProvider');
|
|
92
|
-
return MongoDBProvider;
|
|
93
|
-
}
|
|
94
|
-
case 'milvus': {
|
|
95
|
-
const { MilvusProvider } = await import('../providers/vectordb/MilvusProvider');
|
|
96
|
-
return MilvusProvider;
|
|
97
|
-
}
|
|
98
|
-
case 'qdrant': {
|
|
99
|
-
const { QdrantProvider } = await import('../providers/vectordb/QdrantProvider');
|
|
100
|
-
return QdrantProvider;
|
|
101
|
-
}
|
|
102
|
-
case 'chromadb': {
|
|
103
|
-
const { ChromaDBProvider } = await import('../providers/vectordb/ChromaDBProvider');
|
|
104
|
-
return ChromaDBProvider;
|
|
105
|
-
}
|
|
106
|
-
case 'redis': {
|
|
107
|
-
const { RedisProvider } = await import('../providers/vectordb/RedisProvider');
|
|
108
|
-
return RedisProvider;
|
|
109
|
-
}
|
|
110
|
-
case 'weaviate': {
|
|
111
|
-
const { WeaviateProvider } = await import('../providers/vectordb/WeaviateProvider');
|
|
112
|
-
return WeaviateProvider;
|
|
113
|
-
}
|
|
114
|
-
case 'universal_rest':
|
|
115
|
-
case 'rest': {
|
|
116
|
-
const { UniversalVectorProvider } = await import('../providers/vectordb/UniversalVectorProvider');
|
|
117
|
-
return UniversalVectorProvider;
|
|
118
|
-
}
|
|
119
|
-
default:
|
|
120
|
-
throw new ProviderNotFoundException('vector', provider);
|
|
121
|
-
}
|
|
122
|
-
}
|
|
123
|
-
|
|
124
|
-
static async createVectorProvider(config: VectorDBConfig): Promise<BaseVectorProvider> {
|
|
125
|
-
const providerClass = await this.loadVectorProviderClass(config.provider);
|
|
126
|
-
return new providerClass(config);
|
|
127
|
-
}
|
|
128
|
-
|
|
129
|
-
static async createGraphProvider(config: GraphDBConfig): Promise<BaseGraphProvider> {
|
|
130
|
-
const { provider } = config;
|
|
131
|
-
if (this.graphProviders[provider]) {
|
|
132
|
-
return new this.graphProviders[provider](config);
|
|
133
|
-
}
|
|
134
|
-
|
|
135
|
-
switch (provider) {
|
|
136
|
-
case 'simple': {
|
|
137
|
-
const { SimpleGraphProvider } = await import('../providers/graphdb/SimpleGraphProvider');
|
|
138
|
-
return new SimpleGraphProvider(config);
|
|
139
|
-
}
|
|
140
|
-
default:
|
|
141
|
-
throw new ProviderNotFoundException('graph', provider);
|
|
142
|
-
}
|
|
143
|
-
}
|
|
144
|
-
|
|
145
|
-
static createLLMProvider(llmConfig: LLMConfig, embeddingConfig?: EmbeddingConfig): ILLMProvider {
|
|
146
|
-
return LLMFactory.create(llmConfig, embeddingConfig);
|
|
147
|
-
}
|
|
148
|
-
|
|
149
|
-
static registerLLMProvider(name: string, factory: (config: LLMConfig) => ILLMProvider): void {
|
|
150
|
-
LLMFactory.register(name, factory);
|
|
151
|
-
}
|
|
152
|
-
|
|
153
|
-
static createEmbeddingProvider(embeddingConfig: EmbeddingConfig): ILLMProvider {
|
|
154
|
-
return LLMFactory.createEmbeddingProvider(embeddingConfig);
|
|
155
|
-
}
|
|
156
|
-
}
|
|
@@ -1,399 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* QueryProcessor.ts — Utilities for pre-processing user questions.
|
|
3
|
-
*
|
|
4
|
-
* Responsibilities:
|
|
5
|
-
* - Extracting semantic hints from natural language (emails, IDs, quoted phrases).
|
|
6
|
-
* - Normalizing text values for consistent filtering.
|
|
7
|
-
* - Building structured query filters for vector databases.
|
|
8
|
-
*/
|
|
9
|
-
|
|
10
|
-
export interface QueryFieldHint {
|
|
11
|
-
field?: string;
|
|
12
|
-
value: string;
|
|
13
|
-
}
|
|
14
|
-
|
|
15
|
-
export interface NumericPredicate {
|
|
16
|
-
field?: string;
|
|
17
|
-
operator: 'gt' | 'gte' | 'lt' | 'lte' | 'eq';
|
|
18
|
-
value: number;
|
|
19
|
-
}
|
|
20
|
-
|
|
21
|
-
export interface QueryFilter {
|
|
22
|
-
metadata?: Record<string, string>;
|
|
23
|
-
keywords?: string[];
|
|
24
|
-
queryText?: string;
|
|
25
|
-
__numericPredicates?: NumericPredicate[];
|
|
26
|
-
[key: string]: unknown;
|
|
27
|
-
}
|
|
28
|
-
|
|
29
|
-
export class QueryProcessor {
|
|
30
|
-
/**
|
|
31
|
-
* Normalizes a string value by collapsing whitespace and trimming.
|
|
32
|
-
*/
|
|
33
|
-
static normalizeHintValue(value: string): string {
|
|
34
|
-
return value.replace(/\s+/g, ' ').trim();
|
|
35
|
-
}
|
|
36
|
-
|
|
37
|
-
/**
|
|
38
|
-
* Checks if a string is likely a question word or common prompt phrase.
|
|
39
|
-
*/
|
|
40
|
-
static isLikelyPromptPhrase(value: string): boolean {
|
|
41
|
-
return /^(what|which|who|where|when|why|how|the|is|are|was|were|in|on|at|by|for|with|about|a|an|to|of)\b/i.test(value.trim());
|
|
42
|
-
}
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
/**
|
|
46
|
-
* Scans a natural language question for potential metadata hints and keywords.
|
|
47
|
-
* @param question The user's query
|
|
48
|
-
* @param validFields Optional list of known filterable fields to look for
|
|
49
|
-
*/
|
|
50
|
-
static extractQueryFieldHints(question: string, validFields: string[] = []): QueryFieldHint[] {
|
|
51
|
-
if (!question.trim()) return [];
|
|
52
|
-
|
|
53
|
-
const hints = new Map<string, QueryFieldHint>();
|
|
54
|
-
|
|
55
|
-
const addHint = (value: string, field?: string) => {
|
|
56
|
-
const normalizedValue = this.normalizeHintValue(value);
|
|
57
|
-
if (!normalizedValue) return;
|
|
58
|
-
|
|
59
|
-
const normalizedField = field
|
|
60
|
-
? field
|
|
61
|
-
.toLowerCase()
|
|
62
|
-
.replace(/[^a-z0-9]+/g, ' ')
|
|
63
|
-
.trim()
|
|
64
|
-
: undefined;
|
|
65
|
-
|
|
66
|
-
const key = `${normalizedField ?? '*'}::${normalizedValue.toLowerCase()}`;
|
|
67
|
-
if (!hints.has(key)) {
|
|
68
|
-
hints.set(key, {
|
|
69
|
-
value: normalizedValue,
|
|
70
|
-
...(normalizedField ? { field: normalizedField } : {}),
|
|
71
|
-
});
|
|
72
|
-
}
|
|
73
|
-
};
|
|
74
|
-
|
|
75
|
-
const activeValidFields = validFields.length > 0
|
|
76
|
-
? validFields
|
|
77
|
-
: ['brand', 'category', 'price', 'rating', 'stock', 'stock_quantity', 'status', 'name', 'title'];
|
|
78
|
-
|
|
79
|
-
const resolveValidField = (fieldStr: string): string | null => {
|
|
80
|
-
const cleaned = this.normalizeHintValue(fieldStr)
|
|
81
|
-
.replace(/\b(?:provide|show|get|give|list|all|the|a|an|of|organizations?|companies?|records?|items?|whose|with|having|where|that|which|have|has|is|are|was|were)\b/gi, ' ')
|
|
82
|
-
.replace(/\s+/g, ' ')
|
|
83
|
-
.trim();
|
|
84
|
-
|
|
85
|
-
const comparable = (v: string) => v.toLowerCase().replace(/[^a-z0-9]/g, '');
|
|
86
|
-
const cleanedComparable = comparable(cleaned);
|
|
87
|
-
if (!cleanedComparable) return null;
|
|
88
|
-
|
|
89
|
-
const matchedField = activeValidFields.find(fieldName => {
|
|
90
|
-
const candidate = comparable(fieldName);
|
|
91
|
-
return candidate === cleanedComparable ||
|
|
92
|
-
candidate.includes(cleanedComparable) ||
|
|
93
|
-
cleanedComparable.includes(candidate);
|
|
94
|
-
});
|
|
95
|
-
|
|
96
|
-
return matchedField ?? null;
|
|
97
|
-
};
|
|
98
|
-
|
|
99
|
-
// 1. Quoted phrases
|
|
100
|
-
for (const match of question.matchAll(/["']([^"']{2,100})["']/g)) {
|
|
101
|
-
addHint(match[1]);
|
|
102
|
-
}
|
|
103
|
-
|
|
104
|
-
// 2. Natural question patterns
|
|
105
|
-
const naturalQuestionPatterns = [
|
|
106
|
-
/\b(?:what|which)\s+(?:is|are|was|were)\s+(?:the\s+)?([^?.!,]{1,60}?)\s+of\s+["']?([^"'\n?.!,]{2,120})["']?(?=[?.!,]|$)/gi,
|
|
107
|
-
/\b(?:who|what)\s+(?:is|are|was|were)\s+["']?([^"'\n?.!,]{2,120})["']?(?=[?.!,]|$)/gi,
|
|
108
|
-
/\b(?:about|for|regarding)\s+["']?([^"'\n?.!,]{2,120})["']?(?=[?.!,]|$)/gi,
|
|
109
|
-
];
|
|
110
|
-
|
|
111
|
-
// 3. Person/Company targeted patterns
|
|
112
|
-
const personCompanyPatterns = [
|
|
113
|
-
/\bcompany(?:\s+name)?\s+(?:of|for)\s+["']?([^"'\n?.!,]{2,120})["']?(?=[?.!,]|$)/gi,
|
|
114
|
-
/\b(?:which|what)\s+company\s+does\s+["']?([^"'\n?.!,]{2,120})["']?\s+work(?:\s+for|\s+at)?(?=[?.!,]|$)/gi,
|
|
115
|
-
/\bwhere\s+does\s+["']?([^"'\n?.!,]{2,120})["']?\s+work(?:\s+for|\s+at)?(?=[?.!,]|$)/gi,
|
|
116
|
-
];
|
|
117
|
-
|
|
118
|
-
for (const pattern of personCompanyPatterns) {
|
|
119
|
-
for (const match of question.matchAll(pattern)) {
|
|
120
|
-
const name = match[1];
|
|
121
|
-
if (name) addHint(name, 'name');
|
|
122
|
-
}
|
|
123
|
-
}
|
|
124
|
-
|
|
125
|
-
// 4. Dynamic Schema-driven patterns
|
|
126
|
-
// We look for any word-like key followed by a colon or equals.
|
|
127
|
-
const genericPattern = /\b([a-z][a-z0-9_]{1,30})\s*[:=]\s*["']?([^"'\n?.!,]{1,60})["']?/gi;
|
|
128
|
-
for (const match of question.matchAll(genericPattern)) {
|
|
129
|
-
const field = match[1];
|
|
130
|
-
const value = match[2];
|
|
131
|
-
if (field && !this.isLikelyPromptPhrase(field)) {
|
|
132
|
-
const resolvedField = resolveValidField(field);
|
|
133
|
-
if (resolvedField) {
|
|
134
|
-
addHint(value, resolvedField);
|
|
135
|
-
} else {
|
|
136
|
-
addHint(value);
|
|
137
|
-
addHint(field);
|
|
138
|
-
}
|
|
139
|
-
}
|
|
140
|
-
}
|
|
141
|
-
|
|
142
|
-
// 5. Explicitly requested fields (if any)
|
|
143
|
-
if (validFields.length > 0) {
|
|
144
|
-
for (const field of validFields) {
|
|
145
|
-
// More aggressive pattern for explicitly allowed fields: "[field] [value]"
|
|
146
|
-
const pattern = new RegExp(`\\b${field}\\s*(?:is|are|:|of)?\\s+["']?([^"\\n?.!,]{1,60})["']?(?=[?.!,]|$)`, 'gi');
|
|
147
|
-
for (const match of question.matchAll(pattern)) {
|
|
148
|
-
const value = match[1];
|
|
149
|
-
if (value && !this.isLikelyPromptPhrase(value)) {
|
|
150
|
-
addHint(value, field);
|
|
151
|
-
}
|
|
152
|
-
}
|
|
153
|
-
}
|
|
154
|
-
}
|
|
155
|
-
|
|
156
|
-
// 5. Universal patterns (email, phone, etc.)
|
|
157
|
-
const universalPatterns: Array<{ regex: RegExp; field?: string; group?: number }> = [
|
|
158
|
-
{ regex: /([a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,})/gi, field: 'email', group: 1 },
|
|
159
|
-
{ regex: /(\+?\d[\d\-\.\s\(\)]{6,}\d)/g, field: 'phone', group: 1 },
|
|
160
|
-
{ regex: /(\b\d{4}-\d{2}-\d{2}\b|\b\d{1,2}\/\d{1,2}\/\d{2,4}\b)/g, field: 'date', group: 1 },
|
|
161
|
-
{ regex: /\b(ID|id|identifier)[: ]\s*([A-Za-z0-9\-]{3,})\b/gi, field: 'id', group: 2 },
|
|
162
|
-
{ regex: /"([^"]{2,120})"/g, group: 1 },
|
|
163
|
-
{ regex: /'([^']{2,120})'/g, group: 1 },
|
|
164
|
-
];
|
|
165
|
-
|
|
166
|
-
for (const p of universalPatterns) {
|
|
167
|
-
for (const match of question.matchAll(p.regex)) {
|
|
168
|
-
const val = p.group ? match[p.group] ?? match[0] : match[0];
|
|
169
|
-
if (!val) continue;
|
|
170
|
-
if (p.field) addHint(val, p.field);
|
|
171
|
-
else addHint(val);
|
|
172
|
-
}
|
|
173
|
-
}
|
|
174
|
-
|
|
175
|
-
for (const pattern of naturalQuestionPatterns) {
|
|
176
|
-
for (const match of question.matchAll(pattern)) {
|
|
177
|
-
const value = match[2] ?? match[1];
|
|
178
|
-
if (value) addHint(value);
|
|
179
|
-
}
|
|
180
|
-
}
|
|
181
|
-
|
|
182
|
-
// 5. Explicit field-value pairs (e.g., status=active, priority:high)
|
|
183
|
-
const fieldPattern = `([^\\n:=?.!,]{1,60}?)`;
|
|
184
|
-
const valuePattern = `([^\\n?.!,]{1,120}?)`;
|
|
185
|
-
const fieldValuePatterns = [
|
|
186
|
-
new RegExp(`\\b${fieldPattern}\\s*(?:=|:)\\s*["']?${valuePattern}["']?(?=[?.!,]|$)`, 'gi'),
|
|
187
|
-
new RegExp(`\\b${fieldPattern}\\s+(?:is|are|was|were|equals?|equal to|named|called)\\s+["']?${valuePattern}["']?(?=[?.!,]|$)`, 'gi'),
|
|
188
|
-
new RegExp(`\\bwith\\s+${fieldPattern}\\s+["']?${valuePattern}["']?(?=[?.!,]|$)`, 'gi'),
|
|
189
|
-
new RegExp(`\\b(?:under|in|for|from|of)\\s+${fieldPattern}\\s+["']?${valuePattern}["']?(?=[?.!,]|$)`, 'gi'),
|
|
190
|
-
];
|
|
191
|
-
|
|
192
|
-
for (const pattern of fieldValuePatterns) {
|
|
193
|
-
for (const match of question.matchAll(pattern)) {
|
|
194
|
-
const field = match[1] ?? '';
|
|
195
|
-
const value = match[2] ?? '';
|
|
196
|
-
|
|
197
|
-
const resolvedField = resolveValidField(field);
|
|
198
|
-
if (resolvedField) {
|
|
199
|
-
addHint(value, resolvedField);
|
|
200
|
-
} else {
|
|
201
|
-
addHint(value);
|
|
202
|
-
if (field && !this.isLikelyPromptPhrase(field)) {
|
|
203
|
-
addHint(field);
|
|
204
|
-
}
|
|
205
|
-
}
|
|
206
|
-
}
|
|
207
|
-
}
|
|
208
|
-
|
|
209
|
-
// 6. Capitalized proper nouns — added here so buildQueryFilter can reuse them
|
|
210
|
-
for (const match of question.matchAll(/\b[A-Z][a-z]+(?:\s+[A-Z][a-z]+){0,3}\b/g)) {
|
|
211
|
-
addHint(match[0]);
|
|
212
|
-
}
|
|
213
|
-
|
|
214
|
-
return [...hints.values()];
|
|
215
|
-
}
|
|
216
|
-
|
|
217
|
-
static extractNumericPredicates(question: string, validFields: string[] = []): NumericPredicate[] {
|
|
218
|
-
const predicates: NumericPredicate[] = [];
|
|
219
|
-
const seen = new Set<string>();
|
|
220
|
-
const comparatorPattern = [
|
|
221
|
-
'greater than or equal to',
|
|
222
|
-
'more than or equal to',
|
|
223
|
-
'less than or equal to',
|
|
224
|
-
'greater than',
|
|
225
|
-
'more than',
|
|
226
|
-
'less than',
|
|
227
|
-
'equal to',
|
|
228
|
-
'at least',
|
|
229
|
-
'at most',
|
|
230
|
-
'above',
|
|
231
|
-
'over',
|
|
232
|
-
'below',
|
|
233
|
-
'under',
|
|
234
|
-
'equals?',
|
|
235
|
-
'>=',
|
|
236
|
-
'<=',
|
|
237
|
-
'>',
|
|
238
|
-
'<',
|
|
239
|
-
'=',
|
|
240
|
-
].join('|');
|
|
241
|
-
|
|
242
|
-
const addPredicate = (rawField: string | undefined, rawOperator: string, rawValue: string) => {
|
|
243
|
-
const value = Number(rawValue.replace(/,/g, ''));
|
|
244
|
-
if (!Number.isFinite(value)) return;
|
|
245
|
-
|
|
246
|
-
const operator = this.normalizeNumericOperator(rawOperator);
|
|
247
|
-
const field = rawField ? this.normalizePredicateField(rawField, validFields) : undefined;
|
|
248
|
-
const key = `${field ?? '*'}::${operator}::${value}`;
|
|
249
|
-
if (seen.has(key)) return;
|
|
250
|
-
seen.add(key);
|
|
251
|
-
predicates.push({ ...(field ? { field } : {}), operator, value });
|
|
252
|
-
};
|
|
253
|
-
|
|
254
|
-
const scopedPatterns = [
|
|
255
|
-
new RegExp(`\\b(?:whose|with|having|where|that\\s+have|which\\s+have)\\s+([a-zA-Z][a-zA-Z0-9_\\s\\-/]{1,80}?)\\s+(?:is|are|was|were)?\\s*(?:${comparatorPattern})\\s+([\\d,]+(?:\\.\\d+)?)`, 'gi'),
|
|
256
|
-
new RegExp(`\\b([a-zA-Z][a-zA-Z0-9_\\s\\-/]{1,80}?)\\s+(?:is|are|was|were)?\\s*(?:${comparatorPattern})\\s+([\\d,]+(?:\\.\\d+)?)`, 'gi'),
|
|
257
|
-
];
|
|
258
|
-
|
|
259
|
-
for (const pattern of scopedPatterns) {
|
|
260
|
-
for (const match of question.matchAll(pattern)) {
|
|
261
|
-
const full = match[0];
|
|
262
|
-
const operatorMatch = full.match(new RegExp(`(${comparatorPattern})`, 'i'));
|
|
263
|
-
if (!operatorMatch) continue;
|
|
264
|
-
addPredicate(match[1], operatorMatch[1], match[2]);
|
|
265
|
-
}
|
|
266
|
-
}
|
|
267
|
-
|
|
268
|
-
for (const match of question.matchAll(/\b([a-zA-Z][a-zA-Z0-9_\s\-/]{1,80}?)\s*(>=|<=|>|<|=)\s*([\d,]+(?:\.\d+)?)/g)) {
|
|
269
|
-
addPredicate(match[1], match[2], match[3]);
|
|
270
|
-
}
|
|
271
|
-
|
|
272
|
-
return predicates;
|
|
273
|
-
}
|
|
274
|
-
|
|
275
|
-
private static normalizeNumericOperator(operator: string): NumericPredicate['operator'] {
|
|
276
|
-
const op = operator.toLowerCase().trim();
|
|
277
|
-
if (op === '>' || /\b(greater than|more than|above|over)\b/.test(op)) return 'gt';
|
|
278
|
-
if (op === '>=' || /\b(greater than or equal to|more than or equal to|at least)\b/.test(op)) return 'gte';
|
|
279
|
-
if (op === '<' || /\b(less than|below|under)\b/.test(op)) return 'lt';
|
|
280
|
-
if (op === '<=' || /\b(less than or equal to|at most)\b/.test(op)) return 'lte';
|
|
281
|
-
return 'eq';
|
|
282
|
-
}
|
|
283
|
-
|
|
284
|
-
private static normalizePredicateField(field: string, validFields: string[]): string {
|
|
285
|
-
const cleaned = this.normalizeHintValue(field)
|
|
286
|
-
.replace(/\b(?:provide|show|get|give|list|all|the|a|an|of|organizations?|companies?|records?|items?|whose|with|having|where|that|which|have|has|is|are|was|were)\b/gi, ' ')
|
|
287
|
-
.replace(/\s+/g, ' ')
|
|
288
|
-
.trim();
|
|
289
|
-
|
|
290
|
-
if (validFields.length === 0) return cleaned;
|
|
291
|
-
const comparable = (value: string) => value.toLowerCase().replace(/[^a-z0-9]/g, '');
|
|
292
|
-
const cleanedComparable = comparable(cleaned);
|
|
293
|
-
const matchedField = validFields.find(fieldName => {
|
|
294
|
-
const candidate = comparable(fieldName);
|
|
295
|
-
return candidate === cleanedComparable ||
|
|
296
|
-
candidate.includes(cleanedComparable) ||
|
|
297
|
-
cleanedComparable.includes(candidate);
|
|
298
|
-
});
|
|
299
|
-
|
|
300
|
-
return matchedField ?? cleaned;
|
|
301
|
-
}
|
|
302
|
-
|
|
303
|
-
/**
|
|
304
|
-
* Constructs a QueryFilter object from extracted hints.
|
|
305
|
-
*
|
|
306
|
-
* Note: proper nouns are extracted inside `extractQueryFieldHints` and surfaced
|
|
307
|
-
* as field-less hints; `buildQueryFilter` adds them to `keywords` without
|
|
308
|
-
* re-running the proper noun regex to avoid duplication.
|
|
309
|
-
*/
|
|
310
|
-
static buildQueryFilter(question: string, hints: QueryFieldHint[]): QueryFilter {
|
|
311
|
-
const filter: QueryFilter = { metadata: {}, keywords: [], queryText: question };
|
|
312
|
-
|
|
313
|
-
for (const hint of hints) {
|
|
314
|
-
if (hint.field) {
|
|
315
|
-
filter.metadata![hint.field] = hint.value;
|
|
316
|
-
} else {
|
|
317
|
-
// hint.value already includes proper nouns from extractQueryFieldHints
|
|
318
|
-
if (!filter.keywords!.includes(hint.value)) {
|
|
319
|
-
filter.keywords!.push(hint.value);
|
|
320
|
-
}
|
|
321
|
-
}
|
|
322
|
-
}
|
|
323
|
-
|
|
324
|
-
if (Object.keys(filter.metadata || {}).length === 0) delete filter.metadata;
|
|
325
|
-
if (filter.keywords && filter.keywords.length === 0) delete filter.keywords;
|
|
326
|
-
|
|
327
|
-
const numericPredicates = this.extractNumericPredicates(question);
|
|
328
|
-
if (numericPredicates.length > 0) {
|
|
329
|
-
filter.__numericPredicates = numericPredicates;
|
|
330
|
-
}
|
|
331
|
-
|
|
332
|
-
return filter;
|
|
333
|
-
}
|
|
334
|
-
|
|
335
|
-
/**
|
|
336
|
-
* Determines the retrieval strategy based on the query content.
|
|
337
|
-
*/
|
|
338
|
-
static async determineRetrievalStrategy(
|
|
339
|
-
query: string,
|
|
340
|
-
llm?: import('../llm/ILLMProvider').ILLMProvider,
|
|
341
|
-
customGraphKeywords?: string[],
|
|
342
|
-
customVectorKeywords?: string[]
|
|
343
|
-
): Promise<'vector' | 'graph' | 'both'> {
|
|
344
|
-
|
|
345
|
-
if (llm) {
|
|
346
|
-
try {
|
|
347
|
-
const prompt = `You are a routing expert in a RAG system.
|
|
348
|
-
Given the user query, determine which retrieval mechanism is needed:
|
|
349
|
-
- 'vector': For semantic search, finding specific documents or content based on similarity.
|
|
350
|
-
- 'graph': For finding relationships between entities, multi-hop questions, or network structures.
|
|
351
|
-
- 'both': If the query requires both specific content and relationship/structural analysis.
|
|
352
|
-
|
|
353
|
-
Query: "${query}"
|
|
354
|
-
|
|
355
|
-
Return ONLY 'vector', 'graph', or 'both'. No explanation.`;
|
|
356
|
-
|
|
357
|
-
const response = await llm.chat(
|
|
358
|
-
[{ role: 'user', content: prompt }],
|
|
359
|
-
'',
|
|
360
|
-
{ temperature: 0, maxTokens: 10 }
|
|
361
|
-
);
|
|
362
|
-
|
|
363
|
-
const cleanResponse = (response ?? '').trim().toLowerCase();
|
|
364
|
-
if (cleanResponse.includes('both')) return 'both';
|
|
365
|
-
if (cleanResponse.includes('graph')) return 'graph';
|
|
366
|
-
if (cleanResponse.includes('vector')) return 'vector';
|
|
367
|
-
} catch (error) {
|
|
368
|
-
console.warn('[QueryProcessor] LLM strategy classification failed, falling back to keywords:', error);
|
|
369
|
-
}
|
|
370
|
-
}
|
|
371
|
-
|
|
372
|
-
// Fallback to keywords
|
|
373
|
-
const normalized = query.toLowerCase();
|
|
374
|
-
|
|
375
|
-
// Graph keywords
|
|
376
|
-
const graphKeywords = customGraphKeywords || [
|
|
377
|
-
'relationship', 'connect', 'link', 'network', 'friend', 'colleague',
|
|
378
|
-
'manager', 'hierarchy', 'multi-hop', 'shortest path', 'between',
|
|
379
|
-
'related to', 'associated with', 'belongs to', 'part of'
|
|
380
|
-
];
|
|
381
|
-
|
|
382
|
-
const hasGraphKeyword = graphKeywords.some(kw => normalized.includes(kw));
|
|
383
|
-
|
|
384
|
-
// Vector keywords (usually generic search or content specific)
|
|
385
|
-
const vectorKeywords = customVectorKeywords || [
|
|
386
|
-
'find', 'search', 'tell me about', 'what is', 'how to', 'documents about'
|
|
387
|
-
];
|
|
388
|
-
|
|
389
|
-
const hasVectorKeyword = vectorKeywords.some(kw => normalized.includes(kw));
|
|
390
|
-
|
|
391
|
-
if (hasGraphKeyword && !hasVectorKeyword) {
|
|
392
|
-
return 'graph';
|
|
393
|
-
} else if (hasGraphKeyword && hasVectorKeyword) {
|
|
394
|
-
return 'both';
|
|
395
|
-
}
|
|
396
|
-
|
|
397
|
-
return 'vector';
|
|
398
|
-
}
|
|
399
|
-
}
|
package/src/core/Retrivora.ts
DELETED
|
@@ -1,98 +0,0 @@
|
|
|
1
|
-
import { RagConfig, RetrievalConfig, UniversalRagConfig } from '../config/RagConfig';
|
|
2
|
-
import { ChatMessage, ChatResponse, IngestDocument } from '../types';
|
|
3
|
-
import { ConfigResolver } from './ConfigResolver';
|
|
4
|
-
import { ConfigValidator } from './ConfigValidator';
|
|
5
|
-
import { LicenseVerifier } from './LicenseVerifier';
|
|
6
|
-
import { Pipeline } from './Pipeline';
|
|
7
|
-
import { wrapError, RetrivoraErrorCode } from '../exceptions';
|
|
8
|
-
|
|
9
|
-
/**
|
|
10
|
-
* Public SDK facade matching the prompt-level Retrivora API.
|
|
11
|
-
*
|
|
12
|
-
* It keeps provider details behind configuration and delegates execution to the
|
|
13
|
-
* existing Pipeline implementation.
|
|
14
|
-
*/
|
|
15
|
-
export class Retrivora {
|
|
16
|
-
private readonly pipeline: Pipeline;
|
|
17
|
-
readonly config: RagConfig;
|
|
18
|
-
|
|
19
|
-
constructor(config?: UniversalRagConfig) {
|
|
20
|
-
this.config = ConfigResolver.resolveUniversal(config);
|
|
21
|
-
this.pipeline = new Pipeline(this.config);
|
|
22
|
-
}
|
|
23
|
-
|
|
24
|
-
async initialize(): Promise<void> {
|
|
25
|
-
try {
|
|
26
|
-
await ConfigValidator.validateAndThrow(this.config);
|
|
27
|
-
await LicenseVerifier.verifyAsync(
|
|
28
|
-
this.config.licenseKey,
|
|
29
|
-
this.config.projectId,
|
|
30
|
-
this.config.vectorDb?.provider
|
|
31
|
-
);
|
|
32
|
-
await LicenseVerifier.verifyIP(this.config.licenseKey);
|
|
33
|
-
} catch (err) {
|
|
34
|
-
throw wrapError(err, 'CONFIGURATION_ERROR');
|
|
35
|
-
}
|
|
36
|
-
try {
|
|
37
|
-
await this.pipeline.initialize();
|
|
38
|
-
} catch (err) {
|
|
39
|
-
throw wrapError(err, 'AUTHENTICATION_ERROR');
|
|
40
|
-
}
|
|
41
|
-
}
|
|
42
|
-
|
|
43
|
-
async ingest(
|
|
44
|
-
documents: IngestDocument[],
|
|
45
|
-
namespace?: string,
|
|
46
|
-
): Promise<Array<{ docId: string | number; chunksIngested: number }>> {
|
|
47
|
-
try {
|
|
48
|
-
return await this.pipeline.ingest(documents, namespace);
|
|
49
|
-
} catch (err) {
|
|
50
|
-
const msg = String(err);
|
|
51
|
-
let defaultCode: RetrivoraErrorCode = 'RETRIEVAL_FAILED';
|
|
52
|
-
if (msg.includes('Embed') || msg.includes('embed')) {
|
|
53
|
-
defaultCode = 'EMBEDDING_FAILED';
|
|
54
|
-
}
|
|
55
|
-
throw wrapError(err, defaultCode);
|
|
56
|
-
}
|
|
57
|
-
}
|
|
58
|
-
|
|
59
|
-
async ask(question: string, history: ChatMessage[] = [], namespace?: string): Promise<ChatResponse> {
|
|
60
|
-
try {
|
|
61
|
-
return await this.pipeline.ask(question, history, namespace);
|
|
62
|
-
} catch (err) {
|
|
63
|
-
const msg = String(err);
|
|
64
|
-
let defaultCode: RetrivoraErrorCode = 'RETRIEVAL_FAILED';
|
|
65
|
-
if (msg.includes('Embed') || msg.includes('embed')) {
|
|
66
|
-
defaultCode = 'EMBEDDING_FAILED';
|
|
67
|
-
}
|
|
68
|
-
throw wrapError(err, defaultCode);
|
|
69
|
-
}
|
|
70
|
-
}
|
|
71
|
-
|
|
72
|
-
async *askStream(
|
|
73
|
-
question: string,
|
|
74
|
-
history: ChatMessage[] = [],
|
|
75
|
-
namespace?: string,
|
|
76
|
-
): AsyncIterable<string | ChatResponse> {
|
|
77
|
-
try {
|
|
78
|
-
const stream = this.pipeline.askStream(question, history, namespace);
|
|
79
|
-
for await (const chunk of stream) {
|
|
80
|
-
yield chunk;
|
|
81
|
-
}
|
|
82
|
-
} catch (err) {
|
|
83
|
-
const msg = String(err);
|
|
84
|
-
let defaultCode: RetrivoraErrorCode = 'RETRIEVAL_FAILED';
|
|
85
|
-
if (msg.includes('Embed') || msg.includes('embed')) {
|
|
86
|
-
defaultCode = 'EMBEDDING_FAILED';
|
|
87
|
-
}
|
|
88
|
-
throw wrapError(err, defaultCode);
|
|
89
|
-
}
|
|
90
|
-
}
|
|
91
|
-
|
|
92
|
-
getPipeline(): Pipeline {
|
|
93
|
-
return this.pipeline;
|
|
94
|
-
}
|
|
95
|
-
}
|
|
96
|
-
|
|
97
|
-
export type { RetrievalConfig, UniversalRagConfig };
|
|
98
|
-
|