@retrivora-ai/rag-engine 2.4.3 → 2.4.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +22 -0
- package/dist/{BatchProcessor-zbQMaZHm.d.mts → BatchProcessor-BGslIR5l.d.mts} +65 -4
- package/dist/{BatchProcessor-D42E706a.d.ts → BatchProcessor-C3gY6kZ0.d.ts} +65 -4
- package/dist/handlers/index.js +668 -837
- package/dist/handlers/index.mjs +678 -837
- package/dist/index.css +5 -0
- package/dist/index.d.mts +4 -28
- package/dist/index.d.ts +4 -28
- package/dist/index.js +91 -43
- package/dist/index.min.css +1 -1
- package/dist/index.mjs +91 -43
- package/dist/server.d.mts +21 -7
- package/dist/server.d.ts +21 -7
- package/dist/server.js +731 -875
- package/dist/server.mjs +741 -875
- package/package.json +11 -4
- package/src/LicensedRetrivora.ts +0 -278
- package/src/app/api/chat/route.ts +0 -4
- package/src/app/api/health/route.ts +0 -5
- package/src/app/api/ingest/route.ts +0 -27
- package/src/app/api/suggestions/route.ts +0 -5
- package/src/app/api/upload/route.ts +0 -5
- package/src/app/constants.tsx +0 -270
- package/src/app/favicon.ico +0 -0
- package/src/app/globals.css +0 -449
- package/src/app/layout.tsx +0 -22
- package/src/app/page.tsx +0 -64
- package/src/app/types.ts +0 -30
- package/src/browser.tsx +0 -127
- package/src/components/ChatWidget.tsx +0 -393
- package/src/components/ChatWindow.tsx +0 -952
- package/src/components/CodeViewer.tsx +0 -56
- package/src/components/ConfigProvider.tsx +0 -141
- package/src/components/DocumentUpload.tsx +0 -354
- package/src/components/DynamicChart.tsx +0 -297
- package/src/components/HourglassLoader.tsx +0 -87
- package/src/components/MarkdownComponents.tsx +0 -186
- package/src/components/MessageBubble.tsx +0 -498
- package/src/components/ObservabilityPanel.tsx +0 -374
- package/src/components/ProductCard.tsx +0 -107
- package/src/components/ProductCarousel.tsx +0 -97
- package/src/components/SourceCard.tsx +0 -61
- package/src/components/ThemeProvider.tsx +0 -8
- package/src/components/ThemeToggle.tsx +0 -29
- package/src/components/ThinkingBlock.tsx +0 -75
- package/src/components/UIDispatcher.tsx +0 -342
- package/src/components/VisualizationRenderer.tsx +0 -621
- package/src/components/constants.tsx +0 -503
- package/src/config/ConfigBuilder.ts +0 -285
- package/src/config/EmbeddingStrategy.ts +0 -148
- package/src/config/LicenseConfigResolver.ts +0 -298
- package/src/config/RagConfig.ts +0 -328
- package/src/config/UniversalProfiles.ts +0 -82
- package/src/config/constants.ts +0 -64
- package/src/config/serverConfig.ts +0 -378
- package/src/config/uiConstants.ts +0 -47
- package/src/core/BatchProcessor.ts +0 -385
- package/src/core/CircuitBreaker.ts +0 -118
- package/src/core/ConfigFetcher.ts +0 -220
- package/src/core/ConfigResolver.ts +0 -165
- package/src/core/ConfigValidator.ts +0 -197
- package/src/core/DatabaseStorage.ts +0 -526
- package/src/core/FreeTierLimitsGuard.ts +0 -281
- package/src/core/LLMRouter.ts +0 -79
- package/src/core/LangChainAgent.ts +0 -132
- package/src/core/LicenseValidator.ts +0 -284
- package/src/core/LicenseVerifier.ts +0 -261
- package/src/core/MultiAgentCoordinator.ts +0 -239
- package/src/core/Pipeline.ts +0 -1279
- package/src/core/PluginManager.ts +0 -277
- package/src/core/ProjectSettings.ts +0 -247
- package/src/core/ProviderHealthCheck.ts +0 -133
- package/src/core/ProviderInterfaces.ts +0 -37
- package/src/core/ProviderRegistry.ts +0 -156
- package/src/core/QueryProcessor.ts +0 -399
- package/src/core/Retrivora.ts +0 -98
- package/src/core/VectorPlugin.ts +0 -193
- package/src/core/mcp.ts +0 -261
- package/src/exceptions/index.ts +0 -131
- package/src/handlers/index.ts +0 -1625
- package/src/hooks/useHydrated.ts +0 -13
- package/src/hooks/useProjectSettings.ts +0 -105
- package/src/hooks/useRagChat.ts +0 -436
- package/src/hooks/useStoredMessages.ts +0 -64
- package/src/index.css +0 -4269
- package/src/index.ts +0 -137
- package/src/lib/plugin.ts +0 -24
- package/src/llm/ILLMProvider.ts +0 -60
- package/src/llm/LLMFactory.ts +0 -141
- package/src/llm/providers/AnthropicProvider.ts +0 -215
- package/src/llm/providers/GeminiProvider.ts +0 -311
- package/src/llm/providers/GroqProvider.ts +0 -184
- package/src/llm/providers/OllamaProvider.ts +0 -232
- package/src/llm/providers/OpenAIProvider.ts +0 -188
- package/src/llm/providers/QwenProvider.ts +0 -199
- package/src/llm/providers/UniversalLLMAdapter.ts +0 -422
- package/src/llm/utils.ts +0 -46
- package/src/providers/graphdb/BaseGraphProvider.ts +0 -43
- package/src/providers/graphdb/SimpleGraphProvider.ts +0 -66
- package/src/providers/vectordb/BaseVectorProvider.ts +0 -98
- package/src/providers/vectordb/ChromaDBProvider.ts +0 -130
- package/src/providers/vectordb/MilvusProvider.ts +0 -134
- package/src/providers/vectordb/MongoDBProvider.ts +0 -250
- package/src/providers/vectordb/MultiTablePostgresProvider.ts +0 -461
- package/src/providers/vectordb/PineconeProvider.ts +0 -295
- package/src/providers/vectordb/PostgreSQLProvider.ts +0 -227
- package/src/providers/vectordb/QdrantProvider.ts +0 -294
- package/src/providers/vectordb/RedisProvider.ts +0 -105
- package/src/providers/vectordb/UniversalVectorProvider.ts +0 -221
- package/src/providers/vectordb/WeaviateProvider.ts +0 -172
- package/src/rag/DocumentChunker.ts +0 -144
- package/src/rag/EntityExtractor.ts +0 -90
- package/src/rag/LlamaIndexIngestor.ts +0 -60
- package/src/rag/Reranker.ts +0 -106
- package/src/rendering/IntentClassifier.ts +0 -167
- package/src/rendering/RendererRegistry.ts +0 -164
- package/src/rendering/RuleEngine.ts +0 -67
- package/src/rendering/VisualizationDecisionEngine.ts +0 -164
- package/src/rendering/__tests__/VisualizationDecisionEngine.test.ts +0 -237
- package/src/rendering/index.ts +0 -12
- package/src/rendering/rules/Rule1SpecificInfoRule.ts +0 -28
- package/src/rendering/rules/Rule2ComparisonRule.ts +0 -24
- package/src/rendering/rules/Rule3ProductDiscoveryRule.ts +0 -28
- package/src/rendering/rules/Rule4AnalyticalRule.ts +0 -38
- package/src/rendering/rules/Rule5MixedResponseRule.ts +0 -29
- package/src/rendering/rules/Rule6SmallResultSetRule.ts +0 -24
- package/src/rendering/rules/Rule7LargeTableRule.ts +0 -35
- package/src/rendering/types.ts +0 -96
- package/src/server.ts +0 -156
- package/src/tailwind.css +0 -194
- package/src/telemetry/TelemetryService.ts +0 -330
- package/src/types/chat.ts +0 -89
- package/src/types/index.ts +0 -278
- package/src/types/pdf-parse.d.ts +0 -13
- package/src/types/props.ts +0 -169
- package/src/utils/DocumentParser.ts +0 -111
- package/src/utils/ProductExtractor.ts +0 -350
- package/src/utils/SchemaMapper.ts +0 -176
- package/src/utils/UITransformer.ts +0 -1812
- package/src/utils/synonyms.ts +0 -80
- package/src/utils/templateUtils.ts +0 -78
- package/src/version.ts +0 -6
|
@@ -1,1812 +0,0 @@
|
|
|
1
|
-
import {
|
|
2
|
-
VectorMatch,
|
|
3
|
-
UITransformationResponse,
|
|
4
|
-
PieChartData,
|
|
5
|
-
BarChartData,
|
|
6
|
-
LineChartDataPoint,
|
|
7
|
-
ScatterPlotDataPoint,
|
|
8
|
-
MetricCardData,
|
|
9
|
-
TableData,
|
|
10
|
-
CarouselProduct,
|
|
11
|
-
VisualizationType,
|
|
12
|
-
} from '../types';
|
|
13
|
-
import { ILLMProvider } from '../llm/ILLMProvider';
|
|
14
|
-
import { resolveMetadataValue } from './synonyms';
|
|
15
|
-
import { VisualizationDecisionEngine } from '../rendering/VisualizationDecisionEngine';
|
|
16
|
-
|
|
17
|
-
// ─── Intent Types ─────────────────────────────────────────────────────────────
|
|
18
|
-
|
|
19
|
-
/**
|
|
20
|
-
* Structured intent extracted from the user query.
|
|
21
|
-
* Produced either by the LLM (preferred) or by heuristic fallback.
|
|
22
|
-
*/
|
|
23
|
-
export interface QueryIntent {
|
|
24
|
-
/** Primary display mode the user is asking for */
|
|
25
|
-
visualizationHint:
|
|
26
|
-
| 'trend'
|
|
27
|
-
| 'comparison'
|
|
28
|
-
| 'distribution'
|
|
29
|
-
| 'composition'
|
|
30
|
-
| 'correlation'
|
|
31
|
-
| 'ranking'
|
|
32
|
-
| 'kpi'
|
|
33
|
-
| 'tabular'
|
|
34
|
-
| 'geographic'
|
|
35
|
-
| 'category_breakdown'
|
|
36
|
-
| 'product_browse'
|
|
37
|
-
| 'table'
|
|
38
|
-
| 'text';
|
|
39
|
-
|
|
40
|
-
/** Chart family that best matches the analytical intent */
|
|
41
|
-
recommendedChart:
|
|
42
|
-
| 'line_chart'
|
|
43
|
-
| 'bar_chart'
|
|
44
|
-
| 'histogram'
|
|
45
|
-
| 'pie_chart'
|
|
46
|
-
| 'donut_chart'
|
|
47
|
-
| 'scatter_plot'
|
|
48
|
-
| 'horizontal_bar'
|
|
49
|
-
| 'metric_card'
|
|
50
|
-
| 'table'
|
|
51
|
-
| 'geo_map'
|
|
52
|
-
| 'text';
|
|
53
|
-
|
|
54
|
-
/** User wants only in-stock / available items */
|
|
55
|
-
filterInStockOnly: boolean;
|
|
56
|
-
|
|
57
|
-
/** User wants a tabular / list-style answer explicitly */
|
|
58
|
-
wantsExplicitTable: boolean;
|
|
59
|
-
|
|
60
|
-
/** User is asking about time-based changes */
|
|
61
|
-
isTemporal: boolean;
|
|
62
|
-
|
|
63
|
-
/** User is comparing, ranking, or contrasting entities/categories */
|
|
64
|
-
isComparison: boolean;
|
|
65
|
-
|
|
66
|
-
/** The detected language of the query (BCP-47 tag, e.g. "en", "de", "hi") */
|
|
67
|
-
language: string;
|
|
68
|
-
|
|
69
|
-
/** Free-form note the LLM can add for transparency (not used in logic) */
|
|
70
|
-
reasoning?: string;
|
|
71
|
-
}
|
|
72
|
-
|
|
73
|
-
type PrimitiveValue = string | number | boolean;
|
|
74
|
-
type AggregationOperation = 'sum' | 'average' | 'count' | 'min' | 'max' | 'median';
|
|
75
|
-
|
|
76
|
-
interface DataRecord {
|
|
77
|
-
id: string | number;
|
|
78
|
-
content: string;
|
|
79
|
-
score: number;
|
|
80
|
-
fields: Record<string, PrimitiveValue>;
|
|
81
|
-
source: VectorMatch;
|
|
82
|
-
}
|
|
83
|
-
|
|
84
|
-
interface FieldProfile {
|
|
85
|
-
key: string;
|
|
86
|
-
label: string;
|
|
87
|
-
kind: 'number' | 'date' | 'category' | 'boolean' | 'text';
|
|
88
|
-
values: PrimitiveValue[];
|
|
89
|
-
uniqueCount: number;
|
|
90
|
-
}
|
|
91
|
-
|
|
92
|
-
interface DataProfile {
|
|
93
|
-
records: DataRecord[];
|
|
94
|
-
fields: FieldProfile[];
|
|
95
|
-
numericFields: FieldProfile[];
|
|
96
|
-
dateFields: FieldProfile[];
|
|
97
|
-
categoricalFields: FieldProfile[];
|
|
98
|
-
booleanFields: FieldProfile[];
|
|
99
|
-
}
|
|
100
|
-
|
|
101
|
-
// ─── UITransformer ────────────────────────────────────────────────────────────
|
|
102
|
-
|
|
103
|
-
/**
|
|
104
|
-
* UITransformer — Converts vector database results into structured UI representations.
|
|
105
|
-
*
|
|
106
|
-
* Intent detection is now delegated to the LLM via `detectIntent()`.
|
|
107
|
-
* Static keyword lists are used only as a last-resort fallback when no LLM
|
|
108
|
-
* is available (e.g. during unit tests or offline mode).
|
|
109
|
-
*/
|
|
110
|
-
export class UITransformer {
|
|
111
|
-
|
|
112
|
-
// ─── Public Entry Points ─────────────────────────────────────────────────
|
|
113
|
-
|
|
114
|
-
/**
|
|
115
|
-
* Heuristic-only transform (no LLM required).
|
|
116
|
-
* Uses the lightweight heuristic intent detector as a fallback.
|
|
117
|
-
* Prefer `analyzeAndDecide()` in production.
|
|
118
|
-
*/
|
|
119
|
-
static transform(
|
|
120
|
-
userQuery: string,
|
|
121
|
-
retrievedData: VectorMatch[],
|
|
122
|
-
config?: import('../config/RagConfig').RagConfig,
|
|
123
|
-
trainedSchema?: import('./SchemaMapper').SchemaMap,
|
|
124
|
-
/** Optionally pass a pre-computed intent to skip re-detection */
|
|
125
|
-
intent?: QueryIntent,
|
|
126
|
-
): UITransformationResponse {
|
|
127
|
-
if (!retrievedData || retrievedData.length === 0) {
|
|
128
|
-
return this.createTextResponse('No data available', 'No relevant data found for your query.');
|
|
129
|
-
}
|
|
130
|
-
|
|
131
|
-
// //console.log('[UITransformer.transform] Processing retrievedData:', {
|
|
132
|
-
// userQuery,
|
|
133
|
-
// retrievedCount: retrievedData.length,
|
|
134
|
-
// sample: retrievedData.slice(0, 2).map((item) => ({
|
|
135
|
-
// id: item?.id,
|
|
136
|
-
// contentType: typeof item?.content,
|
|
137
|
-
// contentSnippet: String(item?.content ?? '').substring(0, 150),
|
|
138
|
-
// metadataKeys: Object.keys(item?.metadata || {}),
|
|
139
|
-
// })),
|
|
140
|
-
// });
|
|
141
|
-
|
|
142
|
-
const resolvedIntent = intent ?? this.detectIntentHeuristic(userQuery);
|
|
143
|
-
|
|
144
|
-
const filteredData = resolvedIntent.filterInStockOnly
|
|
145
|
-
? retrievedData.filter(item => this.determineStockStatus(item))
|
|
146
|
-
: retrievedData;
|
|
147
|
-
|
|
148
|
-
const profile = this.profileData(filteredData);
|
|
149
|
-
const hasProducts = filteredData.some(item => this.isProductData(item));
|
|
150
|
-
const wantsPieLikeChart = ['pie_chart', 'donut_chart'].includes(resolvedIntent.recommendedChart);
|
|
151
|
-
|
|
152
|
-
// Priority 1 – Explicit trends
|
|
153
|
-
if (resolvedIntent.visualizationHint === 'trend' && profile.dateFields.length > 0 && profile.numericFields.length > 0) {
|
|
154
|
-
return this.transformToLineChart(profile);
|
|
155
|
-
}
|
|
156
|
-
|
|
157
|
-
// Priority 2 – Explicit pie/donut or composition/category distribution
|
|
158
|
-
if (wantsPieLikeChart || ['composition', 'category_breakdown'].includes(resolvedIntent.visualizationHint)) {
|
|
159
|
-
const pieChart = this.transformToPieChart(filteredData, profile, userQuery);
|
|
160
|
-
if (pieChart) return pieChart;
|
|
161
|
-
}
|
|
162
|
-
|
|
163
|
-
// Priority 3 – Comparison/ranking → bar chart
|
|
164
|
-
if (['comparison', 'ranking'].includes(resolvedIntent.visualizationHint)) {
|
|
165
|
-
return this.transformToBarChart(filteredData, profile, userQuery, resolvedIntent.visualizationHint === 'ranking');
|
|
166
|
-
}
|
|
167
|
-
|
|
168
|
-
// Priority 4 – Distribution/correlation/geographic/KPI fallback to supported views
|
|
169
|
-
if (resolvedIntent.visualizationHint === 'distribution') {
|
|
170
|
-
return this.transformToHistogram(profile, userQuery) ?? this.transformToBarChart(filteredData, profile, userQuery);
|
|
171
|
-
}
|
|
172
|
-
|
|
173
|
-
if (resolvedIntent.visualizationHint === 'correlation') {
|
|
174
|
-
return this.transformToScatterPlot(profile, userQuery) ?? this.transformToBarChart(filteredData, profile, userQuery);
|
|
175
|
-
}
|
|
176
|
-
|
|
177
|
-
if (resolvedIntent.visualizationHint === 'geographic') {
|
|
178
|
-
return this.transformToBarChart(filteredData, profile, userQuery);
|
|
179
|
-
}
|
|
180
|
-
|
|
181
|
-
// Priority 5 – Structured list/filter questions should stay tabular even if
|
|
182
|
-
// source metadata happens to contain product-like fields such as category.
|
|
183
|
-
// This must run before KPI detection because fields such as "employee count"
|
|
184
|
-
// contain the word "count" but still describe records to list/filter.
|
|
185
|
-
if (this.isStructuredListQuery(userQuery)) {
|
|
186
|
-
return this.hasMultipleFields(filteredData)
|
|
187
|
-
? this.transformToTable(filteredData, userQuery)
|
|
188
|
-
: this.transformToText(filteredData);
|
|
189
|
-
}
|
|
190
|
-
|
|
191
|
-
if (resolvedIntent.visualizationHint === 'kpi') {
|
|
192
|
-
return this.transformToMetricCard(profile, userQuery) ?? this.transformToText(filteredData);
|
|
193
|
-
}
|
|
194
|
-
|
|
195
|
-
// Priority 6 – Explicit tabular/detail request
|
|
196
|
-
if (
|
|
197
|
-
['tabular', 'table'].includes(resolvedIntent.visualizationHint) ||
|
|
198
|
-
resolvedIntent.wantsExplicitTable
|
|
199
|
-
) {
|
|
200
|
-
return this.hasMultipleFields(filteredData)
|
|
201
|
-
? this.transformToTable(filteredData, userQuery)
|
|
202
|
-
: this.transformToText(filteredData);
|
|
203
|
-
}
|
|
204
|
-
|
|
205
|
-
// Priority 7 – Product carousel only for explicit product intent.
|
|
206
|
-
if ((hasProducts || this.isProductQuery(userQuery)) && resolvedIntent.visualizationHint === 'product_browse') {
|
|
207
|
-
if (!this.isProductQuery(userQuery)) {
|
|
208
|
-
return this.transformToText(filteredData);
|
|
209
|
-
}
|
|
210
|
-
return this.transformToProductCarousel(filteredData, config, trainedSchema);
|
|
211
|
-
}
|
|
212
|
-
|
|
213
|
-
const automatic = this.chooseAutomaticVisualization(filteredData, profile, userQuery);
|
|
214
|
-
if (automatic) return automatic;
|
|
215
|
-
|
|
216
|
-
// Fallback – plain text
|
|
217
|
-
return this.transformToText(filteredData);
|
|
218
|
-
}
|
|
219
|
-
|
|
220
|
-
/**
|
|
221
|
-
* LLM-driven entry point (recommended for production).
|
|
222
|
-
*
|
|
223
|
-
* Step 1 — Detect intent via a dedicated, lightweight LLM call.
|
|
224
|
-
* Step 2 — Pass the intent + data to the visualization-selection prompt.
|
|
225
|
-
* Step 3 — Fall back to the heuristic `transform()` if either LLM call fails.
|
|
226
|
-
*/
|
|
227
|
-
static async analyzeAndDecide(
|
|
228
|
-
query: string,
|
|
229
|
-
sources: VectorMatch[],
|
|
230
|
-
llm: ILLMProvider,
|
|
231
|
-
): Promise<UITransformationResponse> {
|
|
232
|
-
// ── Step 0: Fast Visualization Decision Engine check ────────────────────
|
|
233
|
-
const decision = VisualizationDecisionEngine.decideVisualization(query, sources);
|
|
234
|
-
if (decision.renderType === 'text') {
|
|
235
|
-
console.debug('[UITransformer] Decision Engine selected plain text rendering — skipping visual LLM calls.');
|
|
236
|
-
return this.transformToText(sources);
|
|
237
|
-
}
|
|
238
|
-
|
|
239
|
-
// ── Step 1: Detect intent ───────────────────────────────────────────────
|
|
240
|
-
let intent: QueryIntent;
|
|
241
|
-
try {
|
|
242
|
-
intent = await this.detectIntent(query, llm, sources);
|
|
243
|
-
console.debug('[UITransformer] Detected intent:', intent);
|
|
244
|
-
} catch (err) {
|
|
245
|
-
console.warn('[UITransformer] Intent detection failed; using heuristic.', err);
|
|
246
|
-
intent = this.detectIntentHeuristic(query);
|
|
247
|
-
}
|
|
248
|
-
|
|
249
|
-
if (this.isProductQuery(query) && ['text', 'product_browse'].includes(intent.visualizationHint)) {
|
|
250
|
-
return this.transform(
|
|
251
|
-
query,
|
|
252
|
-
sources,
|
|
253
|
-
undefined,
|
|
254
|
-
undefined,
|
|
255
|
-
{ ...intent, visualizationHint: 'product_browse', recommendedChart: 'text' },
|
|
256
|
-
);
|
|
257
|
-
}
|
|
258
|
-
|
|
259
|
-
// ── Step 2: Ask the LLM to pick a visualization ─────────────────────────
|
|
260
|
-
try {
|
|
261
|
-
const context = this.buildContextSummary(sources);
|
|
262
|
-
const systemPrompt = this.buildVisualizationSystemPrompt();
|
|
263
|
-
|
|
264
|
-
const profile = this.profileData(sources);
|
|
265
|
-
const schemaContext = `
|
|
266
|
-
RETRIEVED DATA SCHEMA:
|
|
267
|
-
- Numeric fields (measures): ${profile.numericFields.map(f => `${f.key} (unique values: ${f.uniqueCount})`).join(', ') || 'None'}
|
|
268
|
-
- Category fields (dimensions): ${profile.categoricalFields.map(f => `${f.key} (unique values: ${f.uniqueCount})`).join(', ') || 'None'}
|
|
269
|
-
- Date fields (temporal): ${profile.dateFields.map(f => `${f.key} (unique values: ${f.uniqueCount})`).join(', ') || 'None'}
|
|
270
|
-
- Boolean fields: ${profile.booleanFields.map(f => `${f.key} (unique values: ${f.uniqueCount})`).join(', ') || 'None'}
|
|
271
|
-
- Text/Other fields: ${profile.fields.filter(f => f.kind === 'text').map(f => f.key).join(', ') || 'None'}
|
|
272
|
-
`;
|
|
273
|
-
|
|
274
|
-
const userPrompt = [
|
|
275
|
-
`USER QUESTION: ${query}`,
|
|
276
|
-
'',
|
|
277
|
-
`DETECTED INTENT (JSON): ${JSON.stringify(intent)}`,
|
|
278
|
-
'',
|
|
279
|
-
`RETRIEVED DATA SCHEMA:`,
|
|
280
|
-
schemaContext,
|
|
281
|
-
'',
|
|
282
|
-
'RETRIEVED DATA (JSON):',
|
|
283
|
-
context,
|
|
284
|
-
].join('\n');
|
|
285
|
-
|
|
286
|
-
const rawResponse = await llm.chat(
|
|
287
|
-
[{ role: 'user', content: userPrompt }],
|
|
288
|
-
'',
|
|
289
|
-
{ systemPrompt, temperature: 0 },
|
|
290
|
-
);
|
|
291
|
-
|
|
292
|
-
const parsed = this.parseTransformationResponse(rawResponse);
|
|
293
|
-
if (parsed) {
|
|
294
|
-
// Validate suitability of selected chart type based on schema
|
|
295
|
-
let isCompatible = true;
|
|
296
|
-
if (parsed.type === 'line_chart' && (profile.dateFields.length === 0 || profile.numericFields.length === 0)) {
|
|
297
|
-
isCompatible = false;
|
|
298
|
-
} else if (
|
|
299
|
-
['bar_chart', 'horizontal_bar', 'pie_chart', 'donut_chart'].includes(parsed.type) &&
|
|
300
|
-
(profile.categoricalFields.length === 0 && profile.numericFields.length === 0)
|
|
301
|
-
) {
|
|
302
|
-
isCompatible = false;
|
|
303
|
-
} else if (parsed.type === 'scatter_plot' && profile.numericFields.length < 2) {
|
|
304
|
-
isCompatible = false;
|
|
305
|
-
} else if (parsed.type === 'metric_card' && profile.numericFields.length === 0) {
|
|
306
|
-
isCompatible = false;
|
|
307
|
-
}
|
|
308
|
-
|
|
309
|
-
if (!isCompatible) {
|
|
310
|
-
console.warn(`[UITransformer] LLM chose incompatible visualization type "${parsed.type}" for retrieved schema. Falling back to heuristic.`);
|
|
311
|
-
return this.transform(query, sources, undefined, undefined, intent);
|
|
312
|
-
}
|
|
313
|
-
|
|
314
|
-
const intentAllowsTable =
|
|
315
|
-
intent.wantsExplicitTable ||
|
|
316
|
-
['tabular', 'table', 'geographic'].includes(intent.visualizationHint);
|
|
317
|
-
const intentWantsPieLikeChart =
|
|
318
|
-
['pie_chart', 'donut_chart'].includes(intent.recommendedChart) ||
|
|
319
|
-
['composition', 'category_breakdown'].includes(intent.visualizationHint);
|
|
320
|
-
// Respect the intent: never sneak in a table the user didn't ask for
|
|
321
|
-
if (parsed.type === 'table' && !intentAllowsTable) {
|
|
322
|
-
console.debug('[UITransformer] LLM chose table but intent says no. Falling back to text.');
|
|
323
|
-
return this.transform(query, sources, undefined, undefined, intent);
|
|
324
|
-
}
|
|
325
|
-
if (intentWantsPieLikeChart && parsed.type !== 'pie_chart' && this.detectCategories(sources).length > 1) {
|
|
326
|
-
console.debug('[UITransformer] LLM ignored pie/composition intent. Using deterministic pie chart.');
|
|
327
|
-
return this.transform(query, sources, undefined, undefined, intent);
|
|
328
|
-
}
|
|
329
|
-
console.debug('[UITransformer] LLM chose visualization type:', parsed.type);
|
|
330
|
-
return parsed;
|
|
331
|
-
}
|
|
332
|
-
|
|
333
|
-
console.warn('[UITransformer] LLM returned unparseable response; falling back to heuristic.');
|
|
334
|
-
} catch (err) {
|
|
335
|
-
console.warn('[UITransformer] analyzeAndDecide LLM call failed; falling back to heuristic.', err);
|
|
336
|
-
}
|
|
337
|
-
|
|
338
|
-
// ── Step 3: Heuristic fallback ──────────────────────────────────────────
|
|
339
|
-
return this.transform(query, sources, undefined, undefined, intent);
|
|
340
|
-
}
|
|
341
|
-
|
|
342
|
-
// ─── Dynamic Intent Detection ─────────────────────────────────────────────
|
|
343
|
-
|
|
344
|
-
/**
|
|
345
|
-
* Calls the LLM with a compact, focused prompt to extract a structured
|
|
346
|
-
* `QueryIntent` from the user's query.
|
|
347
|
-
*
|
|
348
|
-
* Keeping this as a *separate* call from visualization selection means:
|
|
349
|
-
* - The prompt is shorter and more reliable.
|
|
350
|
-
* - The intent object can be reused across both the heuristic and LLM paths.
|
|
351
|
-
* - It is easy to unit-test intent detection in isolation.
|
|
352
|
-
*/
|
|
353
|
-
static async detectIntent(query: string, llm: ILLMProvider, sources?: VectorMatch[]): Promise<QueryIntent> {
|
|
354
|
-
let schemaProfileText = '';
|
|
355
|
-
if (sources && sources.length > 0) {
|
|
356
|
-
const profile = this.profileData(sources);
|
|
357
|
-
const hasProducts = sources.some(item => this.isProductData(item));
|
|
358
|
-
schemaProfileText = `
|
|
359
|
-
RETRIEVED DATA SCHEMA:
|
|
360
|
-
- Numeric fields (measures): ${profile.numericFields.map(f => `${f.key} (unique values: ${f.uniqueCount})`).join(', ') || 'None'}
|
|
361
|
-
- Category fields (dimensions): ${profile.categoricalFields.map(f => `${f.key} (unique values: ${f.uniqueCount})`).join(', ') || 'None'}
|
|
362
|
-
- Date fields (temporal): ${profile.dateFields.map(f => `${f.key} (unique values: ${f.uniqueCount})`).join(', ') || 'None'}
|
|
363
|
-
- Boolean fields: ${profile.booleanFields.map(f => `${f.key} (unique values: ${f.uniqueCount})`).join(', ') || 'None'}
|
|
364
|
-
- Text fields: ${profile.fields.filter(f => f.kind === 'text').map(f => f.key).join(', ') || 'None'}
|
|
365
|
-
|
|
366
|
-
Please choose visualizationHint and recommendedChart dynamically based on the available schema.
|
|
367
|
-
- Do NOT recommend "trend" or "line_chart" if there are no Date fields.
|
|
368
|
-
- Do NOT recommend numeric-dependent visualizations (like bar_chart, line_chart, pie_chart, scatter_plot, histogram, metric_card) if there are no Numeric fields. Fall back to tabular/table or text.
|
|
369
|
-
- If there are multiple metadata fields but no clear categories or numbers, "tabular"/"table" is a highly accurate default.
|
|
370
|
-
${hasProducts ? `- Note: The retrieved data matches product/catalog structure. If the user query is looking for, exploring, searching, browsing, or listing products/items, you MUST recommend visualizationHint: "product_browse" and recommendedChart: "text".` : ''}
|
|
371
|
-
`;
|
|
372
|
-
}
|
|
373
|
-
|
|
374
|
-
const systemPrompt = `You are an intent classifier for a product-search RAG system.
|
|
375
|
-
Given a user query, return ONLY a valid JSON object with this exact shape — no prose, no markdown:
|
|
376
|
-
|
|
377
|
-
{
|
|
378
|
-
"visualizationHint": "trend" | "comparison" | "distribution" | "composition" | "correlation" | "ranking" | "kpi" | "tabular" | "geographic" | "product_browse" | "table" | "text",
|
|
379
|
-
"recommendedChart": "line_chart" | "bar_chart" | "histogram" | "pie_chart" | "donut_chart" | "scatter_plot" | "horizontal_bar" | "metric_card" | "table" | "geo_map" | "text",
|
|
380
|
-
"filterInStockOnly": boolean,
|
|
381
|
-
"wantsExplicitTable": boolean,
|
|
382
|
-
"isTemporal": boolean,
|
|
383
|
-
"isComparison": boolean,
|
|
384
|
-
"language": "<BCP-47 language tag, e.g. en, de, hi, ja>",
|
|
385
|
-
"reasoning": "<one short sentence — why you chose these values>"
|
|
386
|
-
}
|
|
387
|
-
|
|
388
|
-
RULES:
|
|
389
|
-
- visualizationHint and recommendedChart
|
|
390
|
-
"trend" → keywords: trend, growth, over time; recommendedChart "line_chart"
|
|
391
|
-
"comparison" → keywords: compare, versus, top; recommendedChart "bar_chart"
|
|
392
|
-
"distribution" → keywords: distribution, spread; recommendedChart "histogram"
|
|
393
|
-
"composition" → keywords: share, percentage, breakup, breakdown; recommendedChart "pie_chart" or "donut_chart"
|
|
394
|
-
"correlation" → keywords: relation, correlation; recommendedChart "scatter_plot"
|
|
395
|
-
"ranking" → keywords: highest, lowest, top 10, ranking; recommendedChart "horizontal_bar"
|
|
396
|
-
"kpi" → keywords: total, average, count; recommendedChart "metric_card"
|
|
397
|
-
"tabular" → keywords: detailed records, table, grid, spreadsheet; recommendedChart "table"
|
|
398
|
-
"geographic" → keywords: region, country, map; recommendedChart "geo_map"
|
|
399
|
-
"product_browse" → user is browsing, searching, describing, viewing details for, or asking about one or more products without analytical visualization intent
|
|
400
|
-
"table" → legacy alias for "tabular"; use only if the query literally says table
|
|
401
|
-
"text" → conversational, factual, or other intent
|
|
402
|
-
- If the user explicitly asks for a chart type, honor that chart type in recommendedChart.
|
|
403
|
-
- If a query says "distribution ... in a pie chart", use visualizationHint "composition" and recommendedChart "pie_chart".
|
|
404
|
-
- filterInStockOnly: true only if user mentions stock, availability, in stock, etc.
|
|
405
|
-
- wantsExplicitTable: true if the user asks for a table, list, grid, spreadsheet, or detailed records.
|
|
406
|
-
- isTemporal: true if time words appear (trend, historical, over time, last year, monthly, etc.)
|
|
407
|
-
- isComparison: true if user compares, ranks, or contrasts entities or categories.
|
|
408
|
-
- language: detect from the query text itself; default "en" if uncertain.`;
|
|
409
|
-
|
|
410
|
-
const userPrompt = `QUERY: ${query}${schemaProfileText ? `\n\n${schemaProfileText}` : ''}`;
|
|
411
|
-
|
|
412
|
-
const rawResponse = await llm.chat(
|
|
413
|
-
[{ role: 'user', content: userPrompt }],
|
|
414
|
-
'',
|
|
415
|
-
{ systemPrompt, temperature: 0 },
|
|
416
|
-
);
|
|
417
|
-
|
|
418
|
-
const parsed = this.parseIntentResponse(rawResponse);
|
|
419
|
-
if (!parsed) {
|
|
420
|
-
throw new Error(`Could not parse intent JSON from LLM response: ${rawResponse}`);
|
|
421
|
-
}
|
|
422
|
-
return parsed;
|
|
423
|
-
}
|
|
424
|
-
|
|
425
|
-
/**
|
|
426
|
-
* Parse and validate the raw LLM response into a `QueryIntent`.
|
|
427
|
-
*/
|
|
428
|
-
private static parseIntentResponse(raw: string): QueryIntent | null {
|
|
429
|
-
const jsonStr = this.extractJsonCandidate(raw);
|
|
430
|
-
if (!jsonStr) return null;
|
|
431
|
-
|
|
432
|
-
try {
|
|
433
|
-
const obj = JSON.parse(jsonStr) as Partial<QueryIntent>;
|
|
434
|
-
|
|
435
|
-
const validHints = [
|
|
436
|
-
'trend', 'comparison', 'distribution', 'composition', 'correlation',
|
|
437
|
-
'ranking', 'kpi', 'tabular', 'geographic', 'category_breakdown',
|
|
438
|
-
'product_browse', 'table', 'text',
|
|
439
|
-
] as const;
|
|
440
|
-
const validCharts = [
|
|
441
|
-
'line_chart', 'bar_chart', 'histogram', 'pie_chart', 'donut_chart',
|
|
442
|
-
'scatter_plot', 'horizontal_bar', 'metric_card', 'table', 'geo_map', 'text',
|
|
443
|
-
] as const;
|
|
444
|
-
|
|
445
|
-
const hint = obj.visualizationHint;
|
|
446
|
-
if (!hint || !validHints.includes(hint as typeof validHints[number])) return null;
|
|
447
|
-
const normalizedHint = hint === 'category_breakdown' ? 'composition' : hint;
|
|
448
|
-
const recommendedChart =
|
|
449
|
-
typeof obj.recommendedChart === 'string' &&
|
|
450
|
-
validCharts.includes(obj.recommendedChart as typeof validCharts[number])
|
|
451
|
-
? obj.recommendedChart as QueryIntent['recommendedChart']
|
|
452
|
-
: this.getRecommendedChartForIntent(normalizedHint as QueryIntent['visualizationHint']);
|
|
453
|
-
|
|
454
|
-
return {
|
|
455
|
-
visualizationHint: normalizedHint as QueryIntent['visualizationHint'],
|
|
456
|
-
recommendedChart,
|
|
457
|
-
filterInStockOnly: Boolean(obj.filterInStockOnly),
|
|
458
|
-
wantsExplicitTable: Boolean(obj.wantsExplicitTable),
|
|
459
|
-
isTemporal: Boolean(obj.isTemporal),
|
|
460
|
-
isComparison: Boolean(obj.isComparison),
|
|
461
|
-
language: typeof obj.language === 'string' && obj.language ? obj.language : 'en',
|
|
462
|
-
reasoning: typeof obj.reasoning === 'string' ? obj.reasoning : undefined,
|
|
463
|
-
};
|
|
464
|
-
} catch {
|
|
465
|
-
return null;
|
|
466
|
-
}
|
|
467
|
-
}
|
|
468
|
-
|
|
469
|
-
/**
|
|
470
|
-
* Heuristic intent detector — used when the LLM is unavailable.
|
|
471
|
-
* Intentionally minimal: it should only catch the most obvious signals.
|
|
472
|
-
* The LLM path handles everything subtle.
|
|
473
|
-
*/
|
|
474
|
-
static detectIntentHeuristic(query: string): QueryIntent {
|
|
475
|
-
const q = query.toLowerCase();
|
|
476
|
-
|
|
477
|
-
const isTemporal = /\b(trend|trends|over time|historical|history|growth|decline|monthly|yearly|weekly|daily|last (year|month|week)|timeline|forecast)\b/.test(q);
|
|
478
|
-
|
|
479
|
-
const isRanking = /\b(highest|lowest|top\s*\d+|bottom\s*\d+|rank(?:ing)?|best|worst|leading)\b/.test(q);
|
|
480
|
-
|
|
481
|
-
const isComparison = /\b(compare|comparison|vs\.?|versus|against|differ|difference|contrast|top)\b/.test(q) || isRanking;
|
|
482
|
-
|
|
483
|
-
const isDistribution = /\b(distribution|spread|histogram|frequency|variance|range)\b/.test(q);
|
|
484
|
-
|
|
485
|
-
const wantsPieLikeChart = /\b(pie|donut|doughnut)(?:\s+chart)?\b/.test(q);
|
|
486
|
-
|
|
487
|
-
const isComposition = wantsPieLikeChart || /\b(share|percentage|percent|breakup|breakdown|composition|split|segmentation|by category|by type|proportion)\b/.test(q);
|
|
488
|
-
|
|
489
|
-
const isCorrelation = /\b(relation|relationship|correlation|correlate|scatter|association|impact of|depend(?:s|ence)? on)\b/.test(q);
|
|
490
|
-
|
|
491
|
-
const isKpi = /\b(total|average|avg|count|sum|median|minimum|maximum|metric|kpi|how many|number of)\b/.test(q);
|
|
492
|
-
|
|
493
|
-
const isGeographic = /\b(region|country|countries|state|city|location|map|geo|geographic|territory)\b/.test(q);
|
|
494
|
-
|
|
495
|
-
const filterInStockOnly = /\b(in[- ]?stock|available|availability|inventory|stock status)\b/.test(q);
|
|
496
|
-
|
|
497
|
-
const wantsExplicitTable = /\b(table|spreadsheet|grid|detailed records|record details|list all|compare all)\b/.test(q);
|
|
498
|
-
|
|
499
|
-
let visualizationHint: QueryIntent['visualizationHint'] = 'text';
|
|
500
|
-
if (wantsExplicitTable) visualizationHint = 'tabular';
|
|
501
|
-
else if (isTemporal) visualizationHint = 'trend';
|
|
502
|
-
else if (wantsPieLikeChart) visualizationHint = 'composition';
|
|
503
|
-
else if (isRanking) visualizationHint = 'ranking';
|
|
504
|
-
else if (isComparison) visualizationHint = 'comparison';
|
|
505
|
-
else if (isDistribution) visualizationHint = 'distribution';
|
|
506
|
-
else if (isComposition) visualizationHint = 'composition';
|
|
507
|
-
else if (isCorrelation) visualizationHint = 'correlation';
|
|
508
|
-
else if (isGeographic) visualizationHint = 'geographic';
|
|
509
|
-
else if (isKpi) visualizationHint = 'kpi';
|
|
510
|
-
else if (this.isProductQuery(query)) visualizationHint = 'product_browse';
|
|
511
|
-
|
|
512
|
-
return {
|
|
513
|
-
visualizationHint,
|
|
514
|
-
recommendedChart: this.getRecommendedChartForIntent(visualizationHint),
|
|
515
|
-
filterInStockOnly,
|
|
516
|
-
wantsExplicitTable,
|
|
517
|
-
isTemporal,
|
|
518
|
-
isComparison,
|
|
519
|
-
language: 'en', // heuristic cannot reliably detect language
|
|
520
|
-
};
|
|
521
|
-
}
|
|
522
|
-
|
|
523
|
-
private static getRecommendedChartForIntent(
|
|
524
|
-
intent: QueryIntent['visualizationHint'],
|
|
525
|
-
): QueryIntent['recommendedChart'] {
|
|
526
|
-
switch (intent) {
|
|
527
|
-
case 'trend':
|
|
528
|
-
return 'line_chart';
|
|
529
|
-
case 'comparison':
|
|
530
|
-
return 'bar_chart';
|
|
531
|
-
case 'distribution':
|
|
532
|
-
return 'histogram';
|
|
533
|
-
case 'composition':
|
|
534
|
-
case 'category_breakdown':
|
|
535
|
-
return 'pie_chart';
|
|
536
|
-
case 'correlation':
|
|
537
|
-
return 'scatter_plot';
|
|
538
|
-
case 'ranking':
|
|
539
|
-
return 'horizontal_bar';
|
|
540
|
-
case 'kpi':
|
|
541
|
-
return 'metric_card';
|
|
542
|
-
case 'tabular':
|
|
543
|
-
case 'table':
|
|
544
|
-
return 'table';
|
|
545
|
-
case 'geographic':
|
|
546
|
-
return 'geo_map';
|
|
547
|
-
case 'product_browse':
|
|
548
|
-
case 'text':
|
|
549
|
-
default:
|
|
550
|
-
return 'text';
|
|
551
|
-
}
|
|
552
|
-
}
|
|
553
|
-
|
|
554
|
-
// ─── Transform Helpers ────────────────────────────────────────────────────
|
|
555
|
-
|
|
556
|
-
private static transformToProductCarousel(
|
|
557
|
-
data: VectorMatch[],
|
|
558
|
-
config?: import('../config/RagConfig').RagConfig,
|
|
559
|
-
trainedSchema?: import('./SchemaMapper').SchemaMap,
|
|
560
|
-
): UITransformationResponse {
|
|
561
|
-
const products: CarouselProduct[] = data
|
|
562
|
-
.map(item => this.extractProductInfo(item, config, trainedSchema))
|
|
563
|
-
.filter((p): p is CarouselProduct => p !== null);
|
|
564
|
-
|
|
565
|
-
return {
|
|
566
|
-
type: 'product_carousel',
|
|
567
|
-
title: 'Recommended Products',
|
|
568
|
-
description: `Found ${products.length} relevant products`,
|
|
569
|
-
data: products,
|
|
570
|
-
};
|
|
571
|
-
}
|
|
572
|
-
|
|
573
|
-
private static transformToPieChart(
|
|
574
|
-
data: VectorMatch[],
|
|
575
|
-
profile?: DataProfile,
|
|
576
|
-
query = '',
|
|
577
|
-
): UITransformationResponse | null {
|
|
578
|
-
const dimension = profile ? this.selectDimensionField(profile, query) : undefined;
|
|
579
|
-
const categories = dimension
|
|
580
|
-
? Array.from(new Set(profile!.records.map(record => String(record.fields[dimension.key] ?? '')).filter(Boolean)))
|
|
581
|
-
: this.detectCategories(data);
|
|
582
|
-
if (categories.length === 0) return null;
|
|
583
|
-
|
|
584
|
-
const categoryData = dimension && profile
|
|
585
|
-
? this.aggregateProfileByDimension(profile, dimension.key)
|
|
586
|
-
: this.aggregateByCategory(data, categories);
|
|
587
|
-
|
|
588
|
-
const pieData: PieChartData[] = Object.entries(categoryData).map(([label, count]) => {
|
|
589
|
-
const { inStockCount, outOfStockCount } = this.calculateStockCounts(label, data);
|
|
590
|
-
return { label, value: count as number, inStockCount, outOfStockCount };
|
|
591
|
-
});
|
|
592
|
-
|
|
593
|
-
return {
|
|
594
|
-
type: 'pie_chart',
|
|
595
|
-
title: `Distribution by ${dimension?.label ?? 'Category'}`,
|
|
596
|
-
description: `Showing breakdown across ${pieData.length} categories`,
|
|
597
|
-
data: pieData,
|
|
598
|
-
};
|
|
599
|
-
}
|
|
600
|
-
|
|
601
|
-
private static transformToLineChart(profile: DataProfile): UITransformationResponse {
|
|
602
|
-
const dateField = profile.dateFields[0];
|
|
603
|
-
const valueField = profile.numericFields[0];
|
|
604
|
-
const buckets = new Map<string, number>();
|
|
605
|
-
|
|
606
|
-
profile.records.forEach(record => {
|
|
607
|
-
const timestamp = String(record.fields[dateField.key] ?? '');
|
|
608
|
-
const value = this.toFiniteNumber(record.fields[valueField.key]) ?? 0;
|
|
609
|
-
buckets.set(timestamp, (buckets.get(timestamp) ?? 0) + value);
|
|
610
|
-
});
|
|
611
|
-
|
|
612
|
-
const lineData: LineChartDataPoint[] = Array.from(buckets.entries())
|
|
613
|
-
.sort(([a], [b]) => Date.parse(a) - Date.parse(b))
|
|
614
|
-
.slice(0, 24)
|
|
615
|
-
.map(([timestamp, value]) => ({ timestamp, value, label: timestamp }));
|
|
616
|
-
|
|
617
|
-
return {
|
|
618
|
-
type: 'line_chart',
|
|
619
|
-
title: `${valueField.label} Over Time`,
|
|
620
|
-
description: `Showing ${lineData.length} data points`,
|
|
621
|
-
data: lineData,
|
|
622
|
-
};
|
|
623
|
-
}
|
|
624
|
-
|
|
625
|
-
private static transformToBarChart(
|
|
626
|
-
data: VectorMatch[],
|
|
627
|
-
profile?: DataProfile,
|
|
628
|
-
query = '',
|
|
629
|
-
horizontal = false,
|
|
630
|
-
): UITransformationResponse {
|
|
631
|
-
const dimension = profile ? this.selectDimensionField(profile, query) : undefined;
|
|
632
|
-
const measure = profile ? this.selectNumericField(profile, query) : undefined;
|
|
633
|
-
const aggregate = dimension && profile
|
|
634
|
-
? this.aggregateProfileByDimension(profile, dimension.key, measure?.key)
|
|
635
|
-
: this.aggregateByCategory(data, this.detectCategories(data));
|
|
636
|
-
|
|
637
|
-
const barData: BarChartData[] = Object.entries(aggregate)
|
|
638
|
-
.map(([category, value]) => ({ category, value: Number(value) }))
|
|
639
|
-
.sort((a, b) => horizontal ? b.value - a.value : 0)
|
|
640
|
-
.slice(0, 12);
|
|
641
|
-
|
|
642
|
-
const fallbackData: BarChartData[] = barData.length > 0
|
|
643
|
-
? barData
|
|
644
|
-
: data.slice(0, 12).map((item, index) => {
|
|
645
|
-
const meta = item.metadata || {};
|
|
646
|
-
const label = String(
|
|
647
|
-
this.getDynamicVal(meta, 'name') ??
|
|
648
|
-
meta.label ??
|
|
649
|
-
meta.title ??
|
|
650
|
-
`Result ${index + 1}`,
|
|
651
|
-
);
|
|
652
|
-
const value = this.extractNumericValue(meta) ?? item.score ?? 0;
|
|
653
|
-
return { category: label, value: Number(value) };
|
|
654
|
-
});
|
|
655
|
-
|
|
656
|
-
return {
|
|
657
|
-
type: horizontal ? 'horizontal_bar' : 'bar_chart',
|
|
658
|
-
title: dimension ? `${measure?.label ?? 'Count'} by ${dimension.label}` : 'Comparison',
|
|
659
|
-
description: `Showing ${fallbackData.length} comparable values`,
|
|
660
|
-
data: fallbackData,
|
|
661
|
-
};
|
|
662
|
-
}
|
|
663
|
-
|
|
664
|
-
private static transformToHistogram(profile: DataProfile, query = ''): UITransformationResponse | null {
|
|
665
|
-
const field = this.selectNumericField(profile, query);
|
|
666
|
-
if (!field) return null;
|
|
667
|
-
|
|
668
|
-
const values = profile.records
|
|
669
|
-
.map(record => this.toFiniteNumber(record.fields[field.key]))
|
|
670
|
-
.filter((value): value is number => value !== null)
|
|
671
|
-
.sort((a, b) => a - b);
|
|
672
|
-
if (values.length === 0) return null;
|
|
673
|
-
|
|
674
|
-
const min = values[0];
|
|
675
|
-
const max = values[values.length - 1];
|
|
676
|
-
const bucketCount = Math.min(10, Math.max(3, Math.ceil(Math.sqrt(values.length))));
|
|
677
|
-
const width = max === min ? 1 : (max - min) / bucketCount;
|
|
678
|
-
const buckets = Array.from({ length: bucketCount }, (_, index) => {
|
|
679
|
-
const start = min + index * width;
|
|
680
|
-
const end = index === bucketCount - 1 ? max : start + width;
|
|
681
|
-
return { category: `${this.formatNumber(start)}-${this.formatNumber(end)}`, value: 0 };
|
|
682
|
-
});
|
|
683
|
-
|
|
684
|
-
values.forEach(value => {
|
|
685
|
-
const bucketIndex = max === min ? 0 : Math.min(bucketCount - 1, Math.floor((value - min) / width));
|
|
686
|
-
buckets[bucketIndex].value += 1;
|
|
687
|
-
});
|
|
688
|
-
|
|
689
|
-
return {
|
|
690
|
-
type: 'histogram',
|
|
691
|
-
title: `${field.label} Distribution`,
|
|
692
|
-
description: `Showing ${values.length} values across ${bucketCount} buckets`,
|
|
693
|
-
data: buckets,
|
|
694
|
-
};
|
|
695
|
-
}
|
|
696
|
-
|
|
697
|
-
private static transformToScatterPlot(profile: DataProfile, query = ''): UITransformationResponse | null {
|
|
698
|
-
const fields = this.rankFieldsByQuery(profile.numericFields, query).slice(0, 2);
|
|
699
|
-
if (fields.length < 2) return null;
|
|
700
|
-
const [xField, yField] = fields;
|
|
701
|
-
|
|
702
|
-
const points = profile.records
|
|
703
|
-
.map((record): ScatterPlotDataPoint | null => {
|
|
704
|
-
const x = this.toFiniteNumber(record.fields[xField.key]);
|
|
705
|
-
const y = this.toFiniteNumber(record.fields[yField.key]);
|
|
706
|
-
if (x === null || y === null) return null;
|
|
707
|
-
return { x, y, label: String(this.getRecordLabel(record) ?? record.id) };
|
|
708
|
-
})
|
|
709
|
-
.filter((point): point is ScatterPlotDataPoint => point !== null)
|
|
710
|
-
.slice(0, 100);
|
|
711
|
-
|
|
712
|
-
if (points.length === 0) return null;
|
|
713
|
-
|
|
714
|
-
return {
|
|
715
|
-
type: 'scatter_plot',
|
|
716
|
-
title: `${xField.label} vs ${yField.label}`,
|
|
717
|
-
description: `Showing ${points.length} paired values`,
|
|
718
|
-
data: points,
|
|
719
|
-
};
|
|
720
|
-
}
|
|
721
|
-
|
|
722
|
-
private static transformToMetricCard(profile: DataProfile, query = ''): UITransformationResponse | null {
|
|
723
|
-
const operation = this.detectAggregationOperation(query);
|
|
724
|
-
const numericField = this.selectNumericField(profile, query);
|
|
725
|
-
const values = numericField
|
|
726
|
-
? profile.records
|
|
727
|
-
.map(record => this.toFiniteNumber(record.fields[numericField.key]))
|
|
728
|
-
.filter((value): value is number => value !== null)
|
|
729
|
-
: [];
|
|
730
|
-
|
|
731
|
-
const value = operation === 'count' || values.length === 0
|
|
732
|
-
? profile.records.length
|
|
733
|
-
: this.calculateAggregate(values, operation);
|
|
734
|
-
|
|
735
|
-
const metric: MetricCardData = {
|
|
736
|
-
label: numericField ? `${operation} ${numericField.label}` : 'Count',
|
|
737
|
-
value,
|
|
738
|
-
operation,
|
|
739
|
-
};
|
|
740
|
-
|
|
741
|
-
return {
|
|
742
|
-
type: 'metric_card',
|
|
743
|
-
title: metric.label,
|
|
744
|
-
description: `Calculated from ${profile.records.length} result${profile.records.length === 1 ? '' : 's'}`,
|
|
745
|
-
data: metric,
|
|
746
|
-
};
|
|
747
|
-
}
|
|
748
|
-
|
|
749
|
-
private static transformToRadarChart(data: VectorMatch[]): UITransformationResponse {
|
|
750
|
-
// Aggregate attribute scores across items for radar comparison
|
|
751
|
-
const attributeMap: Record<string, Record<string, number>> = {};
|
|
752
|
-
|
|
753
|
-
data.forEach(item => {
|
|
754
|
-
const meta = item.metadata || {};
|
|
755
|
-
const seriesName = String(meta.name ?? meta.product ?? item.id ?? 'Item');
|
|
756
|
-
Object.entries(meta).forEach(([key, val]) => {
|
|
757
|
-
if (typeof val === 'number' && !['price', 'id'].includes(key.toLowerCase())) {
|
|
758
|
-
if (!attributeMap[key]) attributeMap[key] = {};
|
|
759
|
-
attributeMap[key][seriesName] = val;
|
|
760
|
-
}
|
|
761
|
-
});
|
|
762
|
-
});
|
|
763
|
-
|
|
764
|
-
const radarData = Object.entries(attributeMap).map(([attribute, series]) => ({
|
|
765
|
-
attribute,
|
|
766
|
-
...series,
|
|
767
|
-
}));
|
|
768
|
-
|
|
769
|
-
return {
|
|
770
|
-
type: 'radar_chart' as VisualizationType,
|
|
771
|
-
title: 'Product Comparison',
|
|
772
|
-
description: `Comparing ${data.length} items across ${radarData.length} attributes`,
|
|
773
|
-
data: radarData.length > 0 ? radarData : data.map(d => ({ attribute: (d?.content ?? '').substring(0, 40) })),
|
|
774
|
-
};
|
|
775
|
-
}
|
|
776
|
-
|
|
777
|
-
private static transformToTable(data: VectorMatch[], query = ''): UITransformationResponse {
|
|
778
|
-
const columns = this.extractTableColumns(data, query);
|
|
779
|
-
const rows = data.map(item => this.extractTableRow(item, columns));
|
|
780
|
-
const tableData: TableData = { columns, rows };
|
|
781
|
-
|
|
782
|
-
return {
|
|
783
|
-
type: 'table',
|
|
784
|
-
title: 'Detailed Results',
|
|
785
|
-
description: `Showing ${data.length} results`,
|
|
786
|
-
data: tableData,
|
|
787
|
-
};
|
|
788
|
-
}
|
|
789
|
-
|
|
790
|
-
private static transformToText(data: VectorMatch[]): UITransformationResponse {
|
|
791
|
-
return this.createTextResponse(
|
|
792
|
-
'Retrieved Context',
|
|
793
|
-
data.map(item => item?.content ?? '').join('\n\n'),
|
|
794
|
-
`Found ${data.length} relevant results`,
|
|
795
|
-
);
|
|
796
|
-
}
|
|
797
|
-
|
|
798
|
-
private static createTextResponse(
|
|
799
|
-
title: string,
|
|
800
|
-
content: string,
|
|
801
|
-
description?: string,
|
|
802
|
-
): UITransformationResponse {
|
|
803
|
-
return { type: 'text', title, description, data: { content } };
|
|
804
|
-
}
|
|
805
|
-
|
|
806
|
-
// ─── LLM Response Parsing ─────────────────────────────────────────────────
|
|
807
|
-
|
|
808
|
-
static parseTransformationResponse(raw: string): UITransformationResponse | null {
|
|
809
|
-
const payloadText = this.extractJsonCandidate(raw);
|
|
810
|
-
if (!payloadText) return null;
|
|
811
|
-
|
|
812
|
-
try {
|
|
813
|
-
const parsed = JSON.parse(payloadText);
|
|
814
|
-
return this.normalizeTransformation(parsed);
|
|
815
|
-
} catch {
|
|
816
|
-
return null;
|
|
817
|
-
}
|
|
818
|
-
}
|
|
819
|
-
|
|
820
|
-
private static extractJsonCandidate(raw: string): string | null {
|
|
821
|
-
if (!raw) return null;
|
|
822
|
-
|
|
823
|
-
const cleaned = raw
|
|
824
|
-
.replace(/```(?:json|chart|ui)?\s*/gi, '')
|
|
825
|
-
.replace(/```/g, '')
|
|
826
|
-
.trim();
|
|
827
|
-
|
|
828
|
-
const start = cleaned.indexOf('{');
|
|
829
|
-
if (start === -1) return null;
|
|
830
|
-
|
|
831
|
-
let depth = 0;
|
|
832
|
-
let inString = false;
|
|
833
|
-
let escape = false;
|
|
834
|
-
|
|
835
|
-
for (let i = start; i < cleaned.length; i += 1) {
|
|
836
|
-
const char = cleaned[i];
|
|
837
|
-
if (escape) { escape = false; continue; }
|
|
838
|
-
if (char === '\\') { escape = true; continue; }
|
|
839
|
-
if (char === '"') { inString = !inString; continue; }
|
|
840
|
-
if (inString) continue;
|
|
841
|
-
if (char === '{') depth += 1;
|
|
842
|
-
if (char === '}') {
|
|
843
|
-
depth -= 1;
|
|
844
|
-
if (depth === 0) return cleaned.slice(start, i + 1);
|
|
845
|
-
}
|
|
846
|
-
}
|
|
847
|
-
|
|
848
|
-
return null;
|
|
849
|
-
}
|
|
850
|
-
|
|
851
|
-
private static normalizeTransformation(payload: unknown): UITransformationResponse | null {
|
|
852
|
-
if (!payload || typeof payload !== 'object') return null;
|
|
853
|
-
|
|
854
|
-
const p = payload as Record<string, unknown>;
|
|
855
|
-
const type = this.normalizeVisualizationType(
|
|
856
|
-
String(p.type ?? p.view ?? p.chartType ?? ''),
|
|
857
|
-
);
|
|
858
|
-
if (!type) return null;
|
|
859
|
-
|
|
860
|
-
const title = String(p.title ?? p.heading ?? 'Visualization');
|
|
861
|
-
const description = p.description ? String(p.description) : undefined;
|
|
862
|
-
const rawData = p.data ?? p.table ?? p.rows ?? p.items ?? p.content ?? null;
|
|
863
|
-
const data = type === 'text' && typeof rawData === 'string'
|
|
864
|
-
? { content: rawData }
|
|
865
|
-
: rawData;
|
|
866
|
-
|
|
867
|
-
const transformation: UITransformationResponse = { type, title, description, data };
|
|
868
|
-
return this.validateTransformation(transformation) ? transformation : null;
|
|
869
|
-
}
|
|
870
|
-
|
|
871
|
-
private static normalizeVisualizationType(type: string): VisualizationType | null {
|
|
872
|
-
const mapping: Record<string, VisualizationType> = {
|
|
873
|
-
pie: 'pie_chart',
|
|
874
|
-
pie_chart: 'pie_chart',
|
|
875
|
-
bar: 'bar_chart',
|
|
876
|
-
bar_chart: 'bar_chart',
|
|
877
|
-
histogram: 'histogram',
|
|
878
|
-
horizontal_bar: 'horizontal_bar',
|
|
879
|
-
horizontalbar: 'horizontal_bar',
|
|
880
|
-
line: 'line_chart',
|
|
881
|
-
line_chart: 'line_chart',
|
|
882
|
-
scatter: 'scatter_plot',
|
|
883
|
-
scatter_plot: 'scatter_plot',
|
|
884
|
-
scatterplot: 'scatter_plot',
|
|
885
|
-
radar: 'radar_chart',
|
|
886
|
-
radar_chart: 'radar_chart',
|
|
887
|
-
metric: 'metric_card',
|
|
888
|
-
metric_card: 'metric_card',
|
|
889
|
-
card: 'metric_card',
|
|
890
|
-
kpi: 'metric_card',
|
|
891
|
-
geo: 'geo_map',
|
|
892
|
-
geo_map: 'geo_map',
|
|
893
|
-
map: 'geo_map',
|
|
894
|
-
table: 'table',
|
|
895
|
-
text: 'text',
|
|
896
|
-
product_carousel: 'product_carousel',
|
|
897
|
-
carousel: 'carousel',
|
|
898
|
-
};
|
|
899
|
-
return mapping[type.toLowerCase()] ?? null;
|
|
900
|
-
}
|
|
901
|
-
|
|
902
|
-
private static validateTransformation(t: UITransformationResponse): boolean {
|
|
903
|
-
const { type, data } = t;
|
|
904
|
-
switch (type) {
|
|
905
|
-
case 'pie_chart':
|
|
906
|
-
case 'bar_chart':
|
|
907
|
-
case 'histogram':
|
|
908
|
-
case 'horizontal_bar':
|
|
909
|
-
return Array.isArray(data) && data.every(
|
|
910
|
-
(i): i is Record<string, unknown> =>
|
|
911
|
-
i !== null && typeof i === 'object' &&
|
|
912
|
-
(typeof i.value === 'number' || !Number.isNaN(Number(i.value))),
|
|
913
|
-
);
|
|
914
|
-
case 'scatter_plot':
|
|
915
|
-
return Array.isArray(data) && data.every(
|
|
916
|
-
(i): i is Record<string, unknown> =>
|
|
917
|
-
i !== null && typeof i === 'object' &&
|
|
918
|
-
(typeof i.x === 'number' || !Number.isNaN(Number(i.x))) &&
|
|
919
|
-
(typeof i.y === 'number' || !Number.isNaN(Number(i.y))),
|
|
920
|
-
);
|
|
921
|
-
case 'line_chart':
|
|
922
|
-
return Array.isArray(data) && data.every(
|
|
923
|
-
(i): i is Record<string, unknown> =>
|
|
924
|
-
i !== null && typeof i === 'object' && 'timestamp' in i &&
|
|
925
|
-
(typeof i.value === 'number' || !Number.isNaN(Number(i.value))),
|
|
926
|
-
);
|
|
927
|
-
case 'metric_card':
|
|
928
|
-
return (
|
|
929
|
-
typeof data === 'object' && data !== null &&
|
|
930
|
-
(typeof (data as Record<string, unknown>).value === 'number' ||
|
|
931
|
-
!Number.isNaN(Number((data as Record<string, unknown>).value)))
|
|
932
|
-
);
|
|
933
|
-
case 'geo_map':
|
|
934
|
-
return Array.isArray(data) || (typeof data === 'object' && data !== null);
|
|
935
|
-
case 'radar_chart':
|
|
936
|
-
return Array.isArray(data) && data.every(
|
|
937
|
-
(i): i is Record<string, unknown> =>
|
|
938
|
-
i !== null && typeof i === 'object' && 'attribute' in i,
|
|
939
|
-
);
|
|
940
|
-
case 'table':
|
|
941
|
-
return (
|
|
942
|
-
typeof data === 'object' && data !== null &&
|
|
943
|
-
Array.isArray((data as Record<string, unknown>).columns) &&
|
|
944
|
-
Array.isArray((data as Record<string, unknown>).rows)
|
|
945
|
-
);
|
|
946
|
-
case 'text':
|
|
947
|
-
return (
|
|
948
|
-
typeof data === 'object' && data !== null &&
|
|
949
|
-
typeof (data as Record<string, unknown>).content === 'string'
|
|
950
|
-
);
|
|
951
|
-
case 'product_carousel':
|
|
952
|
-
case 'carousel':
|
|
953
|
-
return Array.isArray(data) && data.every(
|
|
954
|
-
(i): i is Record<string, unknown> =>
|
|
955
|
-
i !== null && typeof i === 'object' && ('id' in i || 'name' in i),
|
|
956
|
-
);
|
|
957
|
-
default:
|
|
958
|
-
return false;
|
|
959
|
-
}
|
|
960
|
-
}
|
|
961
|
-
|
|
962
|
-
// ─── Data Inspection Helpers ──────────────────────────────────────────────
|
|
963
|
-
|
|
964
|
-
private static isStructuredListQuery(query: string): boolean {
|
|
965
|
-
const q = query.toLowerCase();
|
|
966
|
-
const asksForRecords = /\b(list|provide|show|give|get|find|which|whose|records?|details?)\b/.test(q);
|
|
967
|
-
const hasNumericPredicate = /\b(greater than|more than|above|over|less than|below|under|at least|at most|minimum|maximum|>=|<=|>|<|equals?|equal to)\b\s*\$?\d/i.test(q) ||
|
|
968
|
-
/\b\d+\b\s*(?:or more|or less|and above|and below)\b/i.test(q);
|
|
969
|
-
const hasEntityTerms = /\b(organization|organisations?|organizations?|company|companies|employee|employees|staff|headcount|workforce)\b/.test(q);
|
|
970
|
-
return asksForRecords && (hasNumericPredicate || hasEntityTerms);
|
|
971
|
-
}
|
|
972
|
-
|
|
973
|
-
/** @internal kept private for transform() internal logic */
|
|
974
|
-
private static isProductQuery(query: string): boolean {
|
|
975
|
-
return UITransformer._productQueryTest(query);
|
|
976
|
-
}
|
|
977
|
-
|
|
978
|
-
/**
|
|
979
|
-
* Public entry-point for external callers (e.g. Pipeline)
|
|
980
|
-
* to check whether a query maps to product_browse without triggering an LLM call.
|
|
981
|
-
*/
|
|
982
|
-
static isProductQueryPublic(query: string): boolean {
|
|
983
|
-
return UITransformer._productQueryTest(query);
|
|
984
|
-
}
|
|
985
|
-
|
|
986
|
-
private static _productQueryTest(query: string): boolean {
|
|
987
|
-
const q = query.toLowerCase();
|
|
988
|
-
const productTerms = /\b(product|products|item|items|sku|catalog|catalogue|price|prices|brand|model|stock|inventory|in stock|out of stock|buy|shop)\b/.test(q);
|
|
989
|
-
const productAction = /\b(recommend|suggest|describe|description|detail|details|about|show|find|search|browse|list|explore|exploring)\b/.test(q);
|
|
990
|
-
const nonProductEntityTerms = /\b(organization|organisations?|organizations?|company|companies|employee|employees|staff|headcount|workforce)\b/.test(q);
|
|
991
|
-
return productTerms && productAction && !nonProductEntityTerms;
|
|
992
|
-
}
|
|
993
|
-
|
|
994
|
-
|
|
995
|
-
private static isProductData(item: VectorMatch): boolean {
|
|
996
|
-
if (!item) return false;
|
|
997
|
-
const content = (item.content || '').toLowerCase();
|
|
998
|
-
const productKeywords = [
|
|
999
|
-
'product', 'price', 'stock', 'sku', 'brand', 'msrp',
|
|
1000
|
-
'inventory', 'buy', 'shop', 'beauty', 'cosmetic', 'facial',
|
|
1001
|
-
'cream', 'serum', 'mask', 'makeup', 'fragrance',
|
|
1002
|
-
];
|
|
1003
|
-
const hasKeywords = productKeywords.some(kw => content.includes(kw));
|
|
1004
|
-
|
|
1005
|
-
const metadata = item.metadata || {};
|
|
1006
|
-
const hasMetadataKey = Object.keys(metadata).some(k => {
|
|
1007
|
-
const val = metadata[k];
|
|
1008
|
-
return (
|
|
1009
|
-
['price', 'product', 'sku', 'brand', 'cost'].includes(k.toLowerCase()) &&
|
|
1010
|
-
val !== null && val !== undefined && val !== ''
|
|
1011
|
-
);
|
|
1012
|
-
});
|
|
1013
|
-
|
|
1014
|
-
const hasPricePattern = /\$\s*\d+\.\d{2}/.test(content);
|
|
1015
|
-
return hasKeywords || hasMetadataKey || hasPricePattern;
|
|
1016
|
-
}
|
|
1017
|
-
|
|
1018
|
-
private static isTimeSeriesData(item: VectorMatch): boolean {
|
|
1019
|
-
const metadata = item.metadata || {};
|
|
1020
|
-
const maybeDateKeys = Object.keys(metadata).filter(k =>
|
|
1021
|
-
['date', 'timestamp', 'time', 'period'].includes(k.toLowerCase()),
|
|
1022
|
-
);
|
|
1023
|
-
return maybeDateKeys.some(key => {
|
|
1024
|
-
const value = metadata[key];
|
|
1025
|
-
if (typeof value === 'string') return !Number.isNaN(Date.parse(value)) && value.trim().length > 0;
|
|
1026
|
-
return typeof value === 'number';
|
|
1027
|
-
});
|
|
1028
|
-
}
|
|
1029
|
-
|
|
1030
|
-
private static hasMultipleFields(data: VectorMatch[]): boolean {
|
|
1031
|
-
const fieldCount = new Set<string>();
|
|
1032
|
-
data.forEach(item => Object.keys(item.metadata || {}).forEach(k => fieldCount.add(k)));
|
|
1033
|
-
return fieldCount.size > 2;
|
|
1034
|
-
}
|
|
1035
|
-
|
|
1036
|
-
private static profileData(data: VectorMatch[]): DataProfile {
|
|
1037
|
-
const records = (data || []).filter((item): item is VectorMatch => Boolean(item && typeof item === 'object')).map((item): DataRecord => {
|
|
1038
|
-
const fields: Record<string, PrimitiveValue> = {};
|
|
1039
|
-
Object.entries(item.metadata || {}).forEach(([key, value]) => {
|
|
1040
|
-
const primitive = this.toPrimitive(value);
|
|
1041
|
-
if (primitive !== null) fields[key] = primitive;
|
|
1042
|
-
});
|
|
1043
|
-
if (!fields.content && item.content) fields.content = item.content.substring(0, 500);
|
|
1044
|
-
return {
|
|
1045
|
-
id: item.id,
|
|
1046
|
-
content: item.content ?? '',
|
|
1047
|
-
score: item.score ?? 0,
|
|
1048
|
-
fields,
|
|
1049
|
-
source: item,
|
|
1050
|
-
};
|
|
1051
|
-
});
|
|
1052
|
-
|
|
1053
|
-
const keys = Array.from(new Set(records.flatMap(record => Object.keys(record.fields))));
|
|
1054
|
-
const fields = keys.map((key): FieldProfile => {
|
|
1055
|
-
const values = records
|
|
1056
|
-
.map(record => record.fields[key])
|
|
1057
|
-
.filter((value): value is PrimitiveValue => value !== undefined && value !== null && String(value).trim() !== '');
|
|
1058
|
-
const uniqueCount = new Set(values.map(value => String(value).toLowerCase())).size;
|
|
1059
|
-
return {
|
|
1060
|
-
key,
|
|
1061
|
-
label: this.humanizeFieldName(key),
|
|
1062
|
-
kind: this.inferFieldKind(key, values, records.length, uniqueCount),
|
|
1063
|
-
values,
|
|
1064
|
-
uniqueCount,
|
|
1065
|
-
};
|
|
1066
|
-
});
|
|
1067
|
-
|
|
1068
|
-
return {
|
|
1069
|
-
records,
|
|
1070
|
-
fields,
|
|
1071
|
-
numericFields: fields.filter(field => field.kind === 'number'),
|
|
1072
|
-
dateFields: fields.filter(field => field.kind === 'date'),
|
|
1073
|
-
categoricalFields: fields.filter(field => field.kind === 'category'),
|
|
1074
|
-
booleanFields: fields.filter(field => field.kind === 'boolean'),
|
|
1075
|
-
};
|
|
1076
|
-
}
|
|
1077
|
-
|
|
1078
|
-
private static toPrimitive(value: unknown): PrimitiveValue | null {
|
|
1079
|
-
if (typeof value === 'string' || typeof value === 'number' || typeof value === 'boolean') return value;
|
|
1080
|
-
if (value === null || value === undefined) return null;
|
|
1081
|
-
if (value instanceof Date) return value.toISOString();
|
|
1082
|
-
return null;
|
|
1083
|
-
}
|
|
1084
|
-
|
|
1085
|
-
private static inferFieldKind(
|
|
1086
|
-
key: string,
|
|
1087
|
-
values: PrimitiveValue[],
|
|
1088
|
-
rowCount: number,
|
|
1089
|
-
uniqueCount: number,
|
|
1090
|
-
): FieldProfile['kind'] {
|
|
1091
|
-
const lowerKey = key.toLowerCase();
|
|
1092
|
-
if (values.length === 0) return 'text';
|
|
1093
|
-
if (values.every(value => typeof value === 'boolean' || /^(true|false|yes|no|in_stock|out_of_stock|available|unavailable)$/i.test(String(value)))) {
|
|
1094
|
-
return 'boolean';
|
|
1095
|
-
}
|
|
1096
|
-
if (/(date|time|timestamp|created|updated|month|year|period)/i.test(lowerKey) && values.some(value => this.isDateLike(value))) {
|
|
1097
|
-
return 'date';
|
|
1098
|
-
}
|
|
1099
|
-
const numericCount = values.filter(value => this.toFiniteNumber(value) !== null).length;
|
|
1100
|
-
if (numericCount / values.length >= 0.85 && !/(id|sku|ean|upc|zip|postal|phone|code)/i.test(lowerKey)) {
|
|
1101
|
-
return 'number';
|
|
1102
|
-
}
|
|
1103
|
-
if (
|
|
1104
|
-
uniqueCount <= Math.max(20, Math.ceil(rowCount * 0.7)) &&
|
|
1105
|
-
!/(id|sku|ean|upc|description|content|url|image|thumbnail)/i.test(lowerKey)
|
|
1106
|
-
) {
|
|
1107
|
-
return 'category';
|
|
1108
|
-
}
|
|
1109
|
-
return 'text';
|
|
1110
|
-
}
|
|
1111
|
-
|
|
1112
|
-
private static isDateLike(value: PrimitiveValue): boolean {
|
|
1113
|
-
if (typeof value === 'number') return value > 1900 && value < 3000;
|
|
1114
|
-
const text = String(value).trim();
|
|
1115
|
-
return text.length > 3 && !Number.isNaN(Date.parse(text));
|
|
1116
|
-
}
|
|
1117
|
-
|
|
1118
|
-
private static chooseAutomaticVisualization(
|
|
1119
|
-
data: VectorMatch[],
|
|
1120
|
-
profile: DataProfile,
|
|
1121
|
-
query: string,
|
|
1122
|
-
): UITransformationResponse | null {
|
|
1123
|
-
if (profile.records.length === 0) return null;
|
|
1124
|
-
|
|
1125
|
-
const q = query.toLowerCase().trim();
|
|
1126
|
-
// Do not automatically force charts unless query contains visual or analytical intent
|
|
1127
|
-
const isExplicitVisualOrAnalytic = /\b(chart|graph|plot|visualize|trend|monthly|revenue|top\s*\d+|breakdown|analytics|compare|versus|vs\.?|histogram|pie|bar|line|distribution|percentage|share|summary of metrics)\b/i.test(q);
|
|
1128
|
-
|
|
1129
|
-
if (!isExplicitVisualOrAnalytic) {
|
|
1130
|
-
return null;
|
|
1131
|
-
}
|
|
1132
|
-
|
|
1133
|
-
if (profile.dateFields.length > 0 && profile.numericFields.length > 0) {
|
|
1134
|
-
return this.transformToLineChart(profile);
|
|
1135
|
-
}
|
|
1136
|
-
if (profile.categoricalFields.length > 0 && profile.numericFields.length > 0) {
|
|
1137
|
-
return this.transformToBarChart(data, profile, query);
|
|
1138
|
-
}
|
|
1139
|
-
if (profile.categoricalFields.length > 0) {
|
|
1140
|
-
return this.transformToPieChart(data, profile, query);
|
|
1141
|
-
}
|
|
1142
|
-
if (profile.numericFields.length >= 2) {
|
|
1143
|
-
return this.transformToScatterPlot(profile, query);
|
|
1144
|
-
}
|
|
1145
|
-
if (profile.numericFields.length === 1) {
|
|
1146
|
-
return this.transformToMetricCard(profile, query);
|
|
1147
|
-
}
|
|
1148
|
-
return null;
|
|
1149
|
-
}
|
|
1150
|
-
|
|
1151
|
-
private static selectDimensionField(profile: DataProfile, query: string): FieldProfile | undefined {
|
|
1152
|
-
const productCategory = profile.categoricalFields.find(field =>
|
|
1153
|
-
/category|department|collection|type|group|segment|region|country|state|city/i.test(field.key),
|
|
1154
|
-
);
|
|
1155
|
-
const ranked = this.rankFieldsByQuery(profile.categoricalFields, query);
|
|
1156
|
-
return ranked[0] ?? productCategory ?? profile.categoricalFields[0];
|
|
1157
|
-
}
|
|
1158
|
-
|
|
1159
|
-
private static selectNumericField(profile: DataProfile, query: string): FieldProfile | undefined {
|
|
1160
|
-
return this.rankFieldsByQuery(profile.numericFields, query)[0] ?? profile.numericFields[0];
|
|
1161
|
-
}
|
|
1162
|
-
|
|
1163
|
-
private static rankFieldsByQuery(fields: FieldProfile[], query: string): FieldProfile[] {
|
|
1164
|
-
const q = query.toLowerCase();
|
|
1165
|
-
return [...fields].sort((a, b) => this.fieldScore(b, q) - this.fieldScore(a, q));
|
|
1166
|
-
}
|
|
1167
|
-
|
|
1168
|
-
private static fieldScore(field: FieldProfile, query: string): number {
|
|
1169
|
-
const key = field.key.toLowerCase();
|
|
1170
|
-
const label = field.label.toLowerCase();
|
|
1171
|
-
let score = 0;
|
|
1172
|
-
if (query.includes(key)) score += 4;
|
|
1173
|
-
if (query.includes(label)) score += 4;
|
|
1174
|
-
key.split(/[_\s-]+/).forEach(part => {
|
|
1175
|
-
if (part && query.includes(part)) score += 1;
|
|
1176
|
-
});
|
|
1177
|
-
if (/category|department|collection|type|group|segment/.test(key)) score += 2;
|
|
1178
|
-
if (/count|total|quantity|stock|inventory|sales|revenue|amount|price|value|score/.test(key)) score += 2;
|
|
1179
|
-
if (/id|sku|ean|upc|code/.test(key)) score -= 5;
|
|
1180
|
-
return score;
|
|
1181
|
-
}
|
|
1182
|
-
|
|
1183
|
-
private static normalizeComparableField(field: string): string {
|
|
1184
|
-
return field
|
|
1185
|
-
.toLowerCase()
|
|
1186
|
-
.replace(/&/g, 'and')
|
|
1187
|
-
.replace(/[^a-z0-9]+/g, '')
|
|
1188
|
-
.trim();
|
|
1189
|
-
}
|
|
1190
|
-
|
|
1191
|
-
private static fieldTokens(field: string): string[] {
|
|
1192
|
-
return field
|
|
1193
|
-
.toLowerCase()
|
|
1194
|
-
.replace(/[_-]+/g, ' ')
|
|
1195
|
-
.split(/\s+/)
|
|
1196
|
-
.map(token => token.replace(/[^a-z0-9]/g, ''))
|
|
1197
|
-
.filter(Boolean);
|
|
1198
|
-
}
|
|
1199
|
-
|
|
1200
|
-
private static aggregateProfileByDimension(
|
|
1201
|
-
profile: DataProfile,
|
|
1202
|
-
dimensionKey: string,
|
|
1203
|
-
measureKey?: string,
|
|
1204
|
-
): Record<string, number> {
|
|
1205
|
-
const result: Record<string, number> = {};
|
|
1206
|
-
profile.records.forEach(record => {
|
|
1207
|
-
const category = String(record.fields[dimensionKey] ?? 'Other').trim() || 'Other';
|
|
1208
|
-
const value = measureKey ? this.toFiniteNumber(record.fields[measureKey]) ?? 0 : 1;
|
|
1209
|
-
result[category] = (result[category] ?? 0) + value;
|
|
1210
|
-
});
|
|
1211
|
-
return result;
|
|
1212
|
-
}
|
|
1213
|
-
|
|
1214
|
-
private static getRecordLabel(record: DataRecord): PrimitiveValue | undefined {
|
|
1215
|
-
const labelKeys = ['name', 'title', 'label', 'product', 'item', 'brand'];
|
|
1216
|
-
const key = Object.keys(record.fields).find(fieldKey =>
|
|
1217
|
-
labelKeys.some(labelKey => fieldKey.toLowerCase().includes(labelKey)),
|
|
1218
|
-
);
|
|
1219
|
-
return key ? record.fields[key] : undefined;
|
|
1220
|
-
}
|
|
1221
|
-
|
|
1222
|
-
private static detectAggregationOperation(query: string): AggregationOperation {
|
|
1223
|
-
const q = query.toLowerCase();
|
|
1224
|
-
if (/\b(avg|average|mean)\b/.test(q)) return 'average';
|
|
1225
|
-
if (/\b(count|how many|number of)\b/.test(q)) return 'count';
|
|
1226
|
-
if (/\b(min|minimum|lowest)\b/.test(q)) return 'min';
|
|
1227
|
-
if (/\b(max|maximum|highest)\b/.test(q)) return 'max';
|
|
1228
|
-
if (/\bmedian\b/.test(q)) return 'median';
|
|
1229
|
-
return 'sum';
|
|
1230
|
-
}
|
|
1231
|
-
|
|
1232
|
-
private static calculateAggregate(values: number[], operation: AggregationOperation): number {
|
|
1233
|
-
if (values.length === 0) return 0;
|
|
1234
|
-
switch (operation) {
|
|
1235
|
-
case 'average':
|
|
1236
|
-
return values.reduce((sum, value) => sum + value, 0) / values.length;
|
|
1237
|
-
case 'count':
|
|
1238
|
-
return values.length;
|
|
1239
|
-
case 'min':
|
|
1240
|
-
return Math.min(...values);
|
|
1241
|
-
case 'max':
|
|
1242
|
-
return Math.max(...values);
|
|
1243
|
-
case 'median': {
|
|
1244
|
-
const sorted = [...values].sort((a, b) => a - b);
|
|
1245
|
-
const middle = Math.floor(sorted.length / 2);
|
|
1246
|
-
return sorted.length % 2 === 0 ? (sorted[middle - 1] + sorted[middle]) / 2 : sorted[middle];
|
|
1247
|
-
}
|
|
1248
|
-
case 'sum':
|
|
1249
|
-
default:
|
|
1250
|
-
return values.reduce((sum, value) => sum + value, 0);
|
|
1251
|
-
}
|
|
1252
|
-
}
|
|
1253
|
-
|
|
1254
|
-
private static humanizeFieldName(key: string): string {
|
|
1255
|
-
return key
|
|
1256
|
-
.replace(/[_-]+/g, ' ')
|
|
1257
|
-
.replace(/([a-z])([A-Z])/g, '$1 $2')
|
|
1258
|
-
.replace(/\s+/g, ' ')
|
|
1259
|
-
.trim()
|
|
1260
|
-
.replace(/\b\w/g, char => char.toUpperCase());
|
|
1261
|
-
}
|
|
1262
|
-
|
|
1263
|
-
private static formatNumber(value: number): string {
|
|
1264
|
-
return Number.isInteger(value) ? String(value) : value.toFixed(1);
|
|
1265
|
-
}
|
|
1266
|
-
|
|
1267
|
-
private static detectCategories(data: VectorMatch[]): string[] {
|
|
1268
|
-
const categories = new Set<string>();
|
|
1269
|
-
data.forEach(item => {
|
|
1270
|
-
const category = this.getProductCategory(item);
|
|
1271
|
-
if (category) categories.add(category);
|
|
1272
|
-
});
|
|
1273
|
-
return Array.from(categories);
|
|
1274
|
-
}
|
|
1275
|
-
|
|
1276
|
-
private static getProductCategory(item: VectorMatch): string | null {
|
|
1277
|
-
if (!item) return null;
|
|
1278
|
-
const meta = item.metadata || {};
|
|
1279
|
-
const metadataCategory = resolveMetadataValue(meta, 'category');
|
|
1280
|
-
if (this.isUsableCategory(metadataCategory)) return String(metadataCategory).trim();
|
|
1281
|
-
|
|
1282
|
-
const content = item.content || '';
|
|
1283
|
-
const explicitCategoryMatch = content.match(
|
|
1284
|
-
/(?:^|\n)\s*(?:product\s*)?(?:category|department|collection)\s*[:\-–]\s*([^\n]+)/i,
|
|
1285
|
-
);
|
|
1286
|
-
if (explicitCategoryMatch && this.isUsableCategory(explicitCategoryMatch[1])) {
|
|
1287
|
-
return explicitCategoryMatch[1].trim();
|
|
1288
|
-
}
|
|
1289
|
-
|
|
1290
|
-
return null;
|
|
1291
|
-
}
|
|
1292
|
-
|
|
1293
|
-
private static isUsableCategory(value: unknown): value is string | number {
|
|
1294
|
-
if (value === null || value === undefined) return false;
|
|
1295
|
-
const category = String(value).trim();
|
|
1296
|
-
if (!category) return false;
|
|
1297
|
-
return !/^(category|type|tag|price|cost|stock|availability|description|product|item|name|brand|sku|id|ean|currency|color|size|index|internal id|other|\[?type\]?|\[?products?_?\d*\]?)$/i.test(category);
|
|
1298
|
-
}
|
|
1299
|
-
|
|
1300
|
-
private static aggregateByCategory(
|
|
1301
|
-
data: VectorMatch[],
|
|
1302
|
-
categories: string[],
|
|
1303
|
-
): Record<string, number> {
|
|
1304
|
-
const result: Record<string, number> = Object.fromEntries(categories.map(c => [c, 0]));
|
|
1305
|
-
data.forEach(item => {
|
|
1306
|
-
const cat = this.getProductCategory(item) ?? 'Other';
|
|
1307
|
-
if (Object.prototype.hasOwnProperty.call(result, cat)) result[cat]++;
|
|
1308
|
-
else result['Other'] = (result['Other'] || 0) + 1;
|
|
1309
|
-
});
|
|
1310
|
-
return result;
|
|
1311
|
-
}
|
|
1312
|
-
|
|
1313
|
-
private static extractTimeSeriesData(
|
|
1314
|
-
data: VectorMatch[],
|
|
1315
|
-
): Array<{ timestamp: string | number; value: number; label?: string }> {
|
|
1316
|
-
return data.map(item => {
|
|
1317
|
-
const meta = item.metadata || {};
|
|
1318
|
-
return {
|
|
1319
|
-
timestamp: (meta.timestamp ?? meta.date ?? new Date().toISOString()) as string | number,
|
|
1320
|
-
value: (meta.value ?? item.score ?? 0) as number,
|
|
1321
|
-
label: (meta.label ?? (item?.content ?? '').substring(0, 50)) as string,
|
|
1322
|
-
};
|
|
1323
|
-
});
|
|
1324
|
-
}
|
|
1325
|
-
|
|
1326
|
-
private static extractNumericValue(meta: Record<string, unknown>): number | null {
|
|
1327
|
-
const preferredKeys = [
|
|
1328
|
-
'value', 'count', 'total', 'average', 'avg', 'amount', 'sales',
|
|
1329
|
-
'revenue', 'score', 'quantity', 'price',
|
|
1330
|
-
];
|
|
1331
|
-
|
|
1332
|
-
for (const key of preferredKeys) {
|
|
1333
|
-
const raw = resolveMetadataValue(meta, key) ?? meta[key];
|
|
1334
|
-
const value = typeof raw === 'number' ? raw : Number(String(raw ?? '').replace(/[$,% ,]/g, ''));
|
|
1335
|
-
if (Number.isFinite(value)) return value;
|
|
1336
|
-
}
|
|
1337
|
-
|
|
1338
|
-
for (const value of Object.values(meta)) {
|
|
1339
|
-
const numeric = typeof value === 'number' ? value : Number(String(value ?? '').replace(/[$,% ,]/g, ''));
|
|
1340
|
-
if (Number.isFinite(numeric)) return numeric;
|
|
1341
|
-
}
|
|
1342
|
-
|
|
1343
|
-
return null;
|
|
1344
|
-
}
|
|
1345
|
-
|
|
1346
|
-
private static extractTableColumns(data: VectorMatch[], query = ''): string[] {
|
|
1347
|
-
const q = query.toLowerCase();
|
|
1348
|
-
const availableFields = this.extractAvailableTableFields(data);
|
|
1349
|
-
|
|
1350
|
-
if (/\b(organization|organisations?|organizations?|company|companies)\b/.test(q) &&
|
|
1351
|
-
/\b(employee|employees|staff|headcount|workforce)\b/.test(q)) {
|
|
1352
|
-
const nameField = this.pickTableField(availableFields, [
|
|
1353
|
-
'company name',
|
|
1354
|
-
'organization name',
|
|
1355
|
-
'organisation name',
|
|
1356
|
-
'name',
|
|
1357
|
-
'company',
|
|
1358
|
-
'organization',
|
|
1359
|
-
'organisation',
|
|
1360
|
-
]);
|
|
1361
|
-
const employeeField = this.pickTableField(availableFields, [
|
|
1362
|
-
'number of employees',
|
|
1363
|
-
'employee count',
|
|
1364
|
-
'employees',
|
|
1365
|
-
'employee_count',
|
|
1366
|
-
'number_employees',
|
|
1367
|
-
'num employees',
|
|
1368
|
-
'staff count',
|
|
1369
|
-
'headcount',
|
|
1370
|
-
'workforce',
|
|
1371
|
-
]);
|
|
1372
|
-
|
|
1373
|
-
const columns = [nameField, employeeField].filter((field): field is string => Boolean(field));
|
|
1374
|
-
if (columns.length > 0) return columns;
|
|
1375
|
-
}
|
|
1376
|
-
|
|
1377
|
-
const requestedFields = availableFields
|
|
1378
|
-
.map(field => ({
|
|
1379
|
-
field,
|
|
1380
|
-
score: this.tableFieldQueryScore(field, q),
|
|
1381
|
-
}))
|
|
1382
|
-
.filter(item => item.score > 0)
|
|
1383
|
-
.sort((a, b) => b.score - a.score)
|
|
1384
|
-
.map(item => item.field);
|
|
1385
|
-
|
|
1386
|
-
if (requestedFields.length > 0) {
|
|
1387
|
-
return requestedFields.slice(0, Math.min(6, requestedFields.length));
|
|
1388
|
-
}
|
|
1389
|
-
|
|
1390
|
-
const metadataFields = Array.from(new Set(
|
|
1391
|
-
data.flatMap(item => Object.keys(item.metadata || {}).map(k => this.humanizeFieldName(k))),
|
|
1392
|
-
));
|
|
1393
|
-
|
|
1394
|
-
return metadataFields.length > 0 ? metadataFields : ['Content'];
|
|
1395
|
-
}
|
|
1396
|
-
|
|
1397
|
-
private static extractTableRow(
|
|
1398
|
-
item: VectorMatch,
|
|
1399
|
-
columns: string[],
|
|
1400
|
-
): (string | number | boolean)[] {
|
|
1401
|
-
return columns.map(col => this.resolveTableCellValue(item, col));
|
|
1402
|
-
}
|
|
1403
|
-
|
|
1404
|
-
private static extractAvailableTableFields(data: VectorMatch[]): string[] {
|
|
1405
|
-
const fields = new Map<string, string>();
|
|
1406
|
-
const addField = (field: string) => {
|
|
1407
|
-
const clean = this.humanizeFieldName(field);
|
|
1408
|
-
if (!clean || /^(content|\[?type\]?)$/i.test(clean)) return;
|
|
1409
|
-
const normalized = this.normalizeComparableField(clean);
|
|
1410
|
-
if (!fields.has(normalized)) fields.set(normalized, clean);
|
|
1411
|
-
};
|
|
1412
|
-
|
|
1413
|
-
data.forEach(item => {
|
|
1414
|
-
if (!item) return;
|
|
1415
|
-
Object.keys(item.metadata || {}).forEach(addField);
|
|
1416
|
-
for (const match of (item.content || '').matchAll(/(?:^|\n)\s*([A-Za-z][A-Za-z0-9_ /-]{1,50})\s*[:\-–]\s*([^\n]+)/g)) {
|
|
1417
|
-
addField(match[1]);
|
|
1418
|
-
}
|
|
1419
|
-
});
|
|
1420
|
-
|
|
1421
|
-
return Array.from(fields.values());
|
|
1422
|
-
}
|
|
1423
|
-
|
|
1424
|
-
private static pickTableField(fields: string[], aliases: string[]): string | undefined {
|
|
1425
|
-
const normalizedAliases = aliases.map(alias => this.normalizeComparableField(alias));
|
|
1426
|
-
for (const alias of normalizedAliases) {
|
|
1427
|
-
const exact = fields.find(field => this.normalizeComparableField(field) === alias);
|
|
1428
|
-
if (exact) return exact;
|
|
1429
|
-
}
|
|
1430
|
-
|
|
1431
|
-
for (const alias of normalizedAliases) {
|
|
1432
|
-
const fuzzy = fields.find(field => {
|
|
1433
|
-
const normalizedField = this.normalizeComparableField(field);
|
|
1434
|
-
if (/(id|code|uuid)$/.test(normalizedField) && !/(id|code|uuid)$/.test(alias)) return false;
|
|
1435
|
-
return normalizedField.includes(alias) || alias.includes(normalizedField);
|
|
1436
|
-
});
|
|
1437
|
-
if (fuzzy) return fuzzy;
|
|
1438
|
-
}
|
|
1439
|
-
|
|
1440
|
-
return undefined;
|
|
1441
|
-
}
|
|
1442
|
-
|
|
1443
|
-
private static tableFieldQueryScore(field: string, query: string): number {
|
|
1444
|
-
const normalizedField = this.normalizeComparableField(field);
|
|
1445
|
-
if (!normalizedField) return 0;
|
|
1446
|
-
const fieldTokens = this.fieldTokens(field);
|
|
1447
|
-
return fieldTokens.reduce((score, token) => {
|
|
1448
|
-
if (token.length < 3) return score;
|
|
1449
|
-
return query.includes(token) ? score + 1 : score;
|
|
1450
|
-
}, query.includes(normalizedField) ? 3 : 0);
|
|
1451
|
-
}
|
|
1452
|
-
|
|
1453
|
-
private static resolveTableCellValue(item: VectorMatch, column: string): string | number | boolean {
|
|
1454
|
-
if (column === 'Content') return (item?.content ?? '').substring(0, 100);
|
|
1455
|
-
|
|
1456
|
-
const meta = item.metadata || {};
|
|
1457
|
-
const normalizedColumn = this.normalizeComparableField(column);
|
|
1458
|
-
const exactMetadata = Object.entries(meta).find(([key]) =>
|
|
1459
|
-
this.normalizeComparableField(key) === normalizedColumn ||
|
|
1460
|
-
this.normalizeComparableField(this.humanizeFieldName(key)) === normalizedColumn,
|
|
1461
|
-
);
|
|
1462
|
-
if (exactMetadata && exactMetadata[1] !== undefined && exactMetadata[1] !== null) {
|
|
1463
|
-
return this.toDisplayValue(exactMetadata[1]);
|
|
1464
|
-
}
|
|
1465
|
-
|
|
1466
|
-
const aliasValue = this.resolveAliasedTableCell(meta, column);
|
|
1467
|
-
if (aliasValue !== undefined) return this.toDisplayValue(aliasValue);
|
|
1468
|
-
|
|
1469
|
-
const contentValue = this.extractContentFieldValue(item?.content ?? '', column);
|
|
1470
|
-
if (contentValue !== null) return contentValue;
|
|
1471
|
-
|
|
1472
|
-
const aliasContentValue = this.extractAliasedContentFieldValue(item?.content ?? '', column);
|
|
1473
|
-
return aliasContentValue ?? '';
|
|
1474
|
-
}
|
|
1475
|
-
|
|
1476
|
-
private static resolveAliasedTableCell(meta: Record<string, unknown>, column: string): unknown {
|
|
1477
|
-
const normalizedColumn = this.normalizeComparableField(column);
|
|
1478
|
-
const aliases = normalizedColumn.includes('employee')
|
|
1479
|
-
? ['number of employees', 'employee count', 'employees', 'headcount', 'staff count', 'workforce']
|
|
1480
|
-
: normalizedColumn.includes('name')
|
|
1481
|
-
? ['company name', 'organization name', 'organisation name', 'name', 'company', 'organization', 'organisation']
|
|
1482
|
-
: [];
|
|
1483
|
-
|
|
1484
|
-
for (const alias of aliases) {
|
|
1485
|
-
const normalizedAlias = this.normalizeComparableField(alias);
|
|
1486
|
-
const match = Object.entries(meta).find(([key]) => {
|
|
1487
|
-
const normalizedKey = this.normalizeComparableField(key);
|
|
1488
|
-
return normalizedKey === normalizedAlias ||
|
|
1489
|
-
normalizedKey.includes(normalizedAlias) ||
|
|
1490
|
-
normalizedAlias.includes(normalizedKey);
|
|
1491
|
-
});
|
|
1492
|
-
if (match && match[1] !== undefined && match[1] !== null) return match[1];
|
|
1493
|
-
}
|
|
1494
|
-
|
|
1495
|
-
return undefined;
|
|
1496
|
-
}
|
|
1497
|
-
|
|
1498
|
-
private static extractContentFieldValue(content: string, column: string): string | null {
|
|
1499
|
-
const escaped = column.replace(/[.*+?^${}()|[\]\\]/g, '\\$&').replace(/\s+/g, '[\\s_ -]+');
|
|
1500
|
-
const pattern = new RegExp(`(?:^|\\n)\\s*${escaped}\\s*[:\\-–]\\s*([^\\n]+)`, 'i');
|
|
1501
|
-
const match = content.match(pattern);
|
|
1502
|
-
return match ? match[1].trim() : null;
|
|
1503
|
-
}
|
|
1504
|
-
|
|
1505
|
-
private static extractAliasedContentFieldValue(content: string, column: string): string | null {
|
|
1506
|
-
const normalizedColumn = this.normalizeComparableField(column);
|
|
1507
|
-
const aliases = normalizedColumn.includes('employee')
|
|
1508
|
-
? ['Number Of Employees', 'Number of Employees', 'Employee Count', 'Employees', 'Headcount', 'Staff Count', 'Workforce']
|
|
1509
|
-
: normalizedColumn.includes('name')
|
|
1510
|
-
? ['Company Name', 'Organization Name', 'Organisation Name', 'Name', 'Company', 'Organization', 'Organisation']
|
|
1511
|
-
: [];
|
|
1512
|
-
|
|
1513
|
-
for (const alias of aliases) {
|
|
1514
|
-
const value = this.extractContentFieldValue(content, alias);
|
|
1515
|
-
if (value !== null) return value;
|
|
1516
|
-
}
|
|
1517
|
-
|
|
1518
|
-
return null;
|
|
1519
|
-
}
|
|
1520
|
-
|
|
1521
|
-
private static toDisplayValue(value: unknown): string | number | boolean {
|
|
1522
|
-
if (typeof value === 'string' || typeof value === 'number' || typeof value === 'boolean') return value;
|
|
1523
|
-
return String(value ?? '');
|
|
1524
|
-
}
|
|
1525
|
-
|
|
1526
|
-
private static calculateStockCounts(
|
|
1527
|
-
category: string,
|
|
1528
|
-
data: VectorMatch[],
|
|
1529
|
-
): { inStockCount: number; outOfStockCount: number } {
|
|
1530
|
-
let inStock = 0;
|
|
1531
|
-
let outOfStock = 0;
|
|
1532
|
-
data.forEach(d => {
|
|
1533
|
-
const cat = this.getProductCategory(d) ?? 'Other';
|
|
1534
|
-
if (cat === category) {
|
|
1535
|
-
const quantity = this.extractStockQuantity(d) ?? 1;
|
|
1536
|
-
if (this.determineStockStatus(d)) inStock += quantity;
|
|
1537
|
-
else outOfStock += quantity;
|
|
1538
|
-
}
|
|
1539
|
-
});
|
|
1540
|
-
return { inStockCount: inStock, outOfStockCount: outOfStock };
|
|
1541
|
-
}
|
|
1542
|
-
|
|
1543
|
-
private static determineStockStatus(item: VectorMatch): boolean {
|
|
1544
|
-
if (!item) return true;
|
|
1545
|
-
const meta = item.metadata || {};
|
|
1546
|
-
const stockValue = resolveMetadataValue(meta, 'stock');
|
|
1547
|
-
if (stockValue !== undefined) {
|
|
1548
|
-
const normalized = String(stockValue).toLowerCase();
|
|
1549
|
-
if (/out[_\s-]?of[_\s-]?stock|unavailable|false|no|sold out|0/.test(normalized)) return false;
|
|
1550
|
-
if (/in[_\s-]?stock|available|true|yes/.test(normalized)) return true;
|
|
1551
|
-
const numeric = this.toFiniteNumber(stockValue);
|
|
1552
|
-
if (numeric !== null) return numeric > 0;
|
|
1553
|
-
}
|
|
1554
|
-
if (meta.inStock !== undefined) return Boolean(meta.inStock);
|
|
1555
|
-
if (meta.stock !== undefined) return Boolean(meta.stock);
|
|
1556
|
-
if (meta.available !== undefined) return Boolean(meta.available);
|
|
1557
|
-
|
|
1558
|
-
const content = (item.content || '').toLowerCase();
|
|
1559
|
-
if (/out[_\s-]?of[_\s-]?stock|unavailable|sold out/.test(content)) return false;
|
|
1560
|
-
if (/in[_\s-]?stock|available/.test(content)) return true;
|
|
1561
|
-
return true;
|
|
1562
|
-
}
|
|
1563
|
-
|
|
1564
|
-
private static extractStockQuantity(item: VectorMatch): number | null {
|
|
1565
|
-
if (!item) return null;
|
|
1566
|
-
const meta = item.metadata || {};
|
|
1567
|
-
const stockValue = resolveMetadataValue(meta, 'stock');
|
|
1568
|
-
const numericStock = this.toFiniteNumber(stockValue);
|
|
1569
|
-
if (numericStock !== null) return numericStock;
|
|
1570
|
-
|
|
1571
|
-
const content = item.content || '';
|
|
1572
|
-
const stockMatch = content.match(/\b(?:stock|inventory|quantity|count)\s*[:\-–]?\s*([\d,]+)/i);
|
|
1573
|
-
if (!stockMatch) return null;
|
|
1574
|
-
|
|
1575
|
-
return this.toFiniteNumber(stockMatch[1]);
|
|
1576
|
-
}
|
|
1577
|
-
|
|
1578
|
-
private static toFiniteNumber(value: unknown): number | null {
|
|
1579
|
-
if (typeof value === 'number') return Number.isFinite(value) ? value : null;
|
|
1580
|
-
const numeric = Number(String(value ?? '').replace(/[$,% ,]/g, ''));
|
|
1581
|
-
return Number.isFinite(numeric) ? numeric : null;
|
|
1582
|
-
}
|
|
1583
|
-
|
|
1584
|
-
// ─── Product Extraction ───────────────────────────────────────────────────
|
|
1585
|
-
|
|
1586
|
-
private static getDynamicVal(
|
|
1587
|
-
meta: Record<string, unknown> | undefined,
|
|
1588
|
-
uiKey: string,
|
|
1589
|
-
config?: import('../config/RagConfig').RagConfig,
|
|
1590
|
-
trainedSchema?: import('./SchemaMapper').SchemaMap,
|
|
1591
|
-
): unknown {
|
|
1592
|
-
if (!meta) return undefined;
|
|
1593
|
-
|
|
1594
|
-
const mapping = config?.rag?.uiMapping;
|
|
1595
|
-
if (mapping?.[uiKey] && meta[mapping[uiKey]] !== undefined) return meta[mapping[uiKey]];
|
|
1596
|
-
|
|
1597
|
-
if (trainedSchema && typeof trainedSchema === 'object') {
|
|
1598
|
-
const trainedKey = (trainedSchema as Record<string, unknown>)[uiKey];
|
|
1599
|
-
if (trainedKey && meta[trainedKey as string] !== undefined) return meta[trainedKey as string];
|
|
1600
|
-
}
|
|
1601
|
-
|
|
1602
|
-
return resolveMetadataValue(meta, uiKey);
|
|
1603
|
-
}
|
|
1604
|
-
|
|
1605
|
-
private static extractProductInfo(
|
|
1606
|
-
item: VectorMatch,
|
|
1607
|
-
config?: import('../config/RagConfig').RagConfig,
|
|
1608
|
-
trainedSchema?: import('./SchemaMapper').SchemaMap,
|
|
1609
|
-
): CarouselProduct | null {
|
|
1610
|
-
if (!item) return null;
|
|
1611
|
-
const meta = item.metadata || {};
|
|
1612
|
-
const content = item.content || '';
|
|
1613
|
-
const name = this.getDynamicVal(meta, 'name', config, trainedSchema);
|
|
1614
|
-
const price = this.getDynamicVal(meta, 'price', config, trainedSchema);
|
|
1615
|
-
const brand = this.getDynamicVal(meta, 'brand', config, trainedSchema);
|
|
1616
|
-
const description = this.cleanProductDescription(
|
|
1617
|
-
this.extractProductDescriptionFromContent(content) ??
|
|
1618
|
-
this.getProductDescriptionValue(meta, config, trainedSchema),
|
|
1619
|
-
);
|
|
1620
|
-
|
|
1621
|
-
if (name || this.isProductData(item)) {
|
|
1622
|
-
let finalName = name ? String(name) : undefined;
|
|
1623
|
-
if (!finalName && content) {
|
|
1624
|
-
const nameMatch = content.match(/(?:Product|Item|Name):\s*([^\n]+)/i);
|
|
1625
|
-
finalName = nameMatch ? nameMatch[1].trim() : content.split('\n')[0]?.substring(0, 60);
|
|
1626
|
-
}
|
|
1627
|
-
|
|
1628
|
-
if (!finalName) {
|
|
1629
|
-
console.warn(
|
|
1630
|
-
`[UITransformer] Unable to read product name from metadata or content for item ${item.id}. Defaulting to 'Unnamed Product'.`,
|
|
1631
|
-
{ metadataKeys: Object.keys(meta), contentLength: content.length },
|
|
1632
|
-
);
|
|
1633
|
-
}
|
|
1634
|
-
|
|
1635
|
-
let finalPrice: string | number | undefined =
|
|
1636
|
-
typeof price === 'number' || typeof price === 'string' ? price : undefined;
|
|
1637
|
-
if (!finalPrice && content) {
|
|
1638
|
-
const priceMatch =
|
|
1639
|
-
content.match(/(?:Price|Cost):\s*\$?([\d,.]+)/i) ||
|
|
1640
|
-
content.match(/\$\s*([\d,.]+)/);
|
|
1641
|
-
if (priceMatch) finalPrice = priceMatch[1].replace(/,/g, '');
|
|
1642
|
-
}
|
|
1643
|
-
|
|
1644
|
-
if (finalPrice === undefined) {
|
|
1645
|
-
console.warn(
|
|
1646
|
-
`[UITransformer] Unable to read product price from metadata or content for item ${item.id}.`,
|
|
1647
|
-
{ metadataKeys: Object.keys(meta), contentSnippet: content.substring(0, 100) },
|
|
1648
|
-
);
|
|
1649
|
-
}
|
|
1650
|
-
|
|
1651
|
-
const imageValue = this.getDynamicVal(meta, 'image', config, trainedSchema);
|
|
1652
|
-
if (!imageValue) {
|
|
1653
|
-
console.debug(
|
|
1654
|
-
`[UITransformer] Unable to read product image URL from metadata for item ${item.id}.`,
|
|
1655
|
-
);
|
|
1656
|
-
}
|
|
1657
|
-
|
|
1658
|
-
if (!description) {
|
|
1659
|
-
console.debug(
|
|
1660
|
-
`[UITransformer] Unable to read product description from metadata or content for item ${item.id}.`,
|
|
1661
|
-
);
|
|
1662
|
-
}
|
|
1663
|
-
|
|
1664
|
-
return {
|
|
1665
|
-
id: item.id,
|
|
1666
|
-
name: finalName || 'Unnamed Product',
|
|
1667
|
-
price: finalPrice,
|
|
1668
|
-
image: typeof imageValue === 'string' ? imageValue : undefined,
|
|
1669
|
-
brand: brand ? String(brand) : undefined,
|
|
1670
|
-
description,
|
|
1671
|
-
inStock: this.determineStockStatus(item),
|
|
1672
|
-
};
|
|
1673
|
-
} else {
|
|
1674
|
-
console.warn(
|
|
1675
|
-
`[UITransformer] Item ${item.id} skipped: unable to read name or identify as product data from metadata/content.`,
|
|
1676
|
-
{ metadataKeys: Object.keys(meta), contentSnippet: content.substring(0, 100) },
|
|
1677
|
-
);
|
|
1678
|
-
}
|
|
1679
|
-
|
|
1680
|
-
return null;
|
|
1681
|
-
}
|
|
1682
|
-
|
|
1683
|
-
private static extractProductDescriptionFromContent(content: string): string | null {
|
|
1684
|
-
const bodyMatch = content.match(
|
|
1685
|
-
/\bBody\s*\(HTML\)\s*:\s*([\s\S]*?)(?=\s*[,.]?\s+\b(?:Handle|Title|Vendor|Type|Tags|Published|Option\d*\s+Name|Option\d*\s+Value|Option|Variant|Image|SKU|Price|Status|Category)\b\s*:|$)/i,
|
|
1686
|
-
);
|
|
1687
|
-
if (bodyMatch?.[1]) return bodyMatch[1];
|
|
1688
|
-
|
|
1689
|
-
const descriptionMatch = content.match(
|
|
1690
|
-
/\b(?:Description|Summary|Details|Body|Content)\s*:\s*([\s\S]*?)(?=\s*[,.]?\s+\b(?:Handle|Title|Vendor|Type|Tags|Published|Option\d*\s+Name|Option\d*\s+Value|Option|Variant|Image|SKU|Price|Status|Category)\b\s*:|$)/i,
|
|
1691
|
-
);
|
|
1692
|
-
if (descriptionMatch?.[1]) return descriptionMatch[1];
|
|
1693
|
-
|
|
1694
|
-
return null;
|
|
1695
|
-
}
|
|
1696
|
-
|
|
1697
|
-
private static getProductDescriptionValue(
|
|
1698
|
-
meta: Record<string, unknown>,
|
|
1699
|
-
config?: import('../config/RagConfig').RagConfig,
|
|
1700
|
-
trainedSchema?: import('./SchemaMapper').SchemaMap,
|
|
1701
|
-
): unknown {
|
|
1702
|
-
const mapped = this.getDynamicVal(meta, 'description', config, trainedSchema);
|
|
1703
|
-
if (mapped !== undefined && mapped !== meta.content && mapped !== meta.text) return mapped;
|
|
1704
|
-
|
|
1705
|
-
const preferredKeys = [
|
|
1706
|
-
'body_html',
|
|
1707
|
-
'body html',
|
|
1708
|
-
'bodyHtml',
|
|
1709
|
-
'description',
|
|
1710
|
-
'summary',
|
|
1711
|
-
'details',
|
|
1712
|
-
'body',
|
|
1713
|
-
];
|
|
1714
|
-
|
|
1715
|
-
for (const key of preferredKeys) {
|
|
1716
|
-
const match = Object.keys(meta).find(candidate =>
|
|
1717
|
-
this.normalizeComparableField(candidate) === this.normalizeComparableField(key),
|
|
1718
|
-
);
|
|
1719
|
-
if (match && meta[match] !== undefined && meta[match] !== null) return meta[match];
|
|
1720
|
-
}
|
|
1721
|
-
|
|
1722
|
-
return undefined;
|
|
1723
|
-
}
|
|
1724
|
-
|
|
1725
|
-
private static cleanProductDescription(raw: unknown): string | undefined {
|
|
1726
|
-
if (raw === null || raw === undefined) return undefined;
|
|
1727
|
-
|
|
1728
|
-
const extracted = this.extractProductDescriptionFromContent(String(raw));
|
|
1729
|
-
if (extracted && extracted !== String(raw)) return this.cleanProductDescription(extracted);
|
|
1730
|
-
|
|
1731
|
-
const text = String(raw)
|
|
1732
|
-
.replace(/<[^>]*>/g, ' ')
|
|
1733
|
-
.replace(/ /gi, ' ')
|
|
1734
|
-
.replace(/&/gi, '&')
|
|
1735
|
-
.replace(/"/gi, '"')
|
|
1736
|
-
.replace(/'/gi, "'")
|
|
1737
|
-
.replace(/\[[^\]]*TYPE[^\]]*\]/gi, ' ')
|
|
1738
|
-
.replace(/\b(?:Handle|Title|Vendor|Type|Tags|Published|Option\d*\s+Name|Option\d*\s+Value|Option|Variant|Image|SKU|Price|Status|Category)\s*:\s*[^,.\n:]+[,.]?/gi, ' ')
|
|
1739
|
-
.replace(/\bBody\s*\(HTML\)\s*:\s*/gi, ' ')
|
|
1740
|
-
.replace(/\b(?:Description|Summary|Details|Body|Content)\s*:\s*/gi, ' ')
|
|
1741
|
-
.replace(/\s+/g, ' ')
|
|
1742
|
-
.trim();
|
|
1743
|
-
|
|
1744
|
-
return text || undefined;
|
|
1745
|
-
}
|
|
1746
|
-
|
|
1747
|
-
// ─── Visualization Prompt ─────────────────────────────────────────────────
|
|
1748
|
-
|
|
1749
|
-
private static buildVisualizationSystemPrompt(): string {
|
|
1750
|
-
return `You are a data visualization expert embedded in a RAG chat system.
|
|
1751
|
-
You will receive a user question, a pre-computed INTENT object, and structured data from a vector database.
|
|
1752
|
-
Your ONLY job is to return a single JSON object describing how to visualize the data.
|
|
1753
|
-
The INTENT object is authoritative — honor it. For example:
|
|
1754
|
-
- If intent.wantsExplicitTable is false, never return type "table".
|
|
1755
|
-
- If intent.isTemporal is true, prefer line_chart.
|
|
1756
|
-
- If intent.visualizationHint is "comparison" or "ranking", prefer bar_chart.
|
|
1757
|
-
- If intent.visualizationHint is "composition", prefer pie_chart.
|
|
1758
|
-
- If intent.recommendedChart is unsupported by the response schema, use the nearest supported type.
|
|
1759
|
-
- Always respond in the language specified by intent.language.
|
|
1760
|
-
|
|
1761
|
-
Return ONLY a valid JSON object — no markdown fences, no prose.
|
|
1762
|
-
|
|
1763
|
-
{
|
|
1764
|
-
"type": "bar_chart" | "horizontal_bar" | "line_chart" | "pie_chart" | "histogram" | "scatter_plot" | "radar_chart" | "metric_card" | "geo_map" | "table" | "text",
|
|
1765
|
-
"title": "concise descriptive title",
|
|
1766
|
-
"description": "one sentence describing the visualization",
|
|
1767
|
-
"data": <structured data per type below>
|
|
1768
|
-
}
|
|
1769
|
-
|
|
1770
|
-
DATA SHAPES:
|
|
1771
|
-
- bar_chart: [{ "category": string, "value": number }]
|
|
1772
|
-
- horizontal_bar: [{ "category": string, "value": number }]
|
|
1773
|
-
- histogram: [{ "category": string, "value": number }]
|
|
1774
|
-
- line_chart: [{ "timestamp": string, "value": number, "label": string }]
|
|
1775
|
-
- pie_chart: [{ "label": string, "value": number }]
|
|
1776
|
-
- scatter_plot:[{ "x": number, "y": number, "label": string }]
|
|
1777
|
-
- radar_chart: [{ "attribute": string, "<series1>": number, "<series2>": number, ... }]
|
|
1778
|
-
- metric_card: { "label": string, "value": number, "operation": "sum"|"average"|"count"|"min"|"max"|"median" }
|
|
1779
|
-
- geo_map: [{ "category": string, "value": number }]
|
|
1780
|
-
- table: { "columns": string[], "rows": (string|number)[][] }
|
|
1781
|
-
- text: { "content": "A concise plain-language answer." }
|
|
1782
|
-
|
|
1783
|
-
RULES:
|
|
1784
|
-
1. Aggregate values — never pass raw nested objects in chart arrays.
|
|
1785
|
-
2. All "value" fields must be numbers.
|
|
1786
|
-
3. Cap bar/line/pie at 12 data points.
|
|
1787
|
-
4. Use dollar signs only for monetary prices, not for stock quantities or years.
|
|
1788
|
-
5. If no relevant data is found, return text: "I cannot answer this question based on the available information."`;
|
|
1789
|
-
}
|
|
1790
|
-
|
|
1791
|
-
private static buildContextSummary(sources: VectorMatch[], maxChars = 6000): string {
|
|
1792
|
-
const items = (sources || []).filter(Boolean).map((s, i) => ({
|
|
1793
|
-
index: i + 1,
|
|
1794
|
-
content: s?.content?.substring(0, 400) ?? '',
|
|
1795
|
-
metadata: s?.metadata ?? {},
|
|
1796
|
-
score: s?.score ?? 0,
|
|
1797
|
-
}));
|
|
1798
|
-
|
|
1799
|
-
const full = JSON.stringify(items, null, 2);
|
|
1800
|
-
if (full.length <= maxChars) return full;
|
|
1801
|
-
|
|
1802
|
-
const partial: typeof items = [];
|
|
1803
|
-
let chars = 2;
|
|
1804
|
-
for (const item of items) {
|
|
1805
|
-
const chunk = JSON.stringify(item);
|
|
1806
|
-
if (chars + chunk.length + 2 > maxChars) break;
|
|
1807
|
-
partial.push(item);
|
|
1808
|
-
chars += chunk.length + 2;
|
|
1809
|
-
}
|
|
1810
|
-
return JSON.stringify(partial, null, 2) + '\n// ... truncated';
|
|
1811
|
-
}
|
|
1812
|
-
}
|