@retrivora-ai/rag-engine 2.4.2 → 2.4.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (140) hide show
  1. package/dist/{BatchProcessor-zbQMaZHm.d.mts → BatchProcessor-Csy_K1qQ.d.mts} +2 -0
  2. package/dist/{BatchProcessor-D42E706a.d.ts → BatchProcessor-Dg18zZGk.d.ts} +2 -0
  3. package/dist/handlers/index.js +31 -5
  4. package/dist/handlers/index.mjs +31 -5
  5. package/dist/index.d.mts +3 -3
  6. package/dist/index.d.ts +3 -3
  7. package/dist/index.js +41 -21
  8. package/dist/index.min.css +2 -0
  9. package/dist/index.mjs +41 -21
  10. package/dist/server.d.mts +2 -2
  11. package/dist/server.d.ts +2 -2
  12. package/dist/server.js +32 -5
  13. package/dist/server.mjs +32 -5
  14. package/package.json +5 -4
  15. package/src/LicensedRetrivora.ts +0 -278
  16. package/src/app/api/chat/route.ts +0 -4
  17. package/src/app/api/health/route.ts +0 -5
  18. package/src/app/api/ingest/route.ts +0 -27
  19. package/src/app/api/suggestions/route.ts +0 -5
  20. package/src/app/api/upload/route.ts +0 -5
  21. package/src/app/constants.tsx +0 -270
  22. package/src/app/favicon.ico +0 -0
  23. package/src/app/globals.css +0 -449
  24. package/src/app/layout.tsx +0 -22
  25. package/src/app/page.tsx +0 -64
  26. package/src/app/types.ts +0 -30
  27. package/src/browser.tsx +0 -139
  28. package/src/components/ChatWidget.tsx +0 -393
  29. package/src/components/ChatWindow.tsx +0 -952
  30. package/src/components/CodeViewer.tsx +0 -56
  31. package/src/components/ConfigProvider.tsx +0 -123
  32. package/src/components/DocumentUpload.tsx +0 -354
  33. package/src/components/DynamicChart.tsx +0 -297
  34. package/src/components/HourglassLoader.tsx +0 -87
  35. package/src/components/MarkdownComponents.tsx +0 -186
  36. package/src/components/MessageBubble.tsx +0 -498
  37. package/src/components/ObservabilityPanel.tsx +0 -374
  38. package/src/components/ProductCard.tsx +0 -107
  39. package/src/components/ProductCarousel.tsx +0 -97
  40. package/src/components/SourceCard.tsx +0 -61
  41. package/src/components/ThemeProvider.tsx +0 -8
  42. package/src/components/ThemeToggle.tsx +0 -29
  43. package/src/components/ThinkingBlock.tsx +0 -75
  44. package/src/components/UIDispatcher.tsx +0 -342
  45. package/src/components/VisualizationRenderer.tsx +0 -621
  46. package/src/components/constants.tsx +0 -503
  47. package/src/config/ConfigBuilder.ts +0 -285
  48. package/src/config/EmbeddingStrategy.ts +0 -148
  49. package/src/config/LicenseConfigResolver.ts +0 -298
  50. package/src/config/RagConfig.ts +0 -328
  51. package/src/config/UniversalProfiles.ts +0 -82
  52. package/src/config/constants.ts +0 -64
  53. package/src/config/serverConfig.ts +0 -378
  54. package/src/config/uiConstants.ts +0 -47
  55. package/src/core/BatchProcessor.ts +0 -385
  56. package/src/core/CircuitBreaker.ts +0 -118
  57. package/src/core/ConfigFetcher.ts +0 -220
  58. package/src/core/ConfigResolver.ts +0 -165
  59. package/src/core/ConfigValidator.ts +0 -197
  60. package/src/core/DatabaseStorage.ts +0 -526
  61. package/src/core/FreeTierLimitsGuard.ts +0 -281
  62. package/src/core/LLMRouter.ts +0 -79
  63. package/src/core/LangChainAgent.ts +0 -132
  64. package/src/core/LicenseValidator.ts +0 -284
  65. package/src/core/LicenseVerifier.ts +0 -261
  66. package/src/core/MultiAgentCoordinator.ts +0 -239
  67. package/src/core/Pipeline.ts +0 -1279
  68. package/src/core/PluginManager.ts +0 -277
  69. package/src/core/ProjectSettings.ts +0 -247
  70. package/src/core/ProviderHealthCheck.ts +0 -133
  71. package/src/core/ProviderInterfaces.ts +0 -37
  72. package/src/core/ProviderRegistry.ts +0 -156
  73. package/src/core/QueryProcessor.ts +0 -399
  74. package/src/core/Retrivora.ts +0 -98
  75. package/src/core/VectorPlugin.ts +0 -193
  76. package/src/core/mcp.ts +0 -261
  77. package/src/exceptions/index.ts +0 -131
  78. package/src/handlers/index.ts +0 -1625
  79. package/src/hooks/useHydrated.ts +0 -13
  80. package/src/hooks/useProjectSettings.ts +0 -105
  81. package/src/hooks/useRagChat.ts +0 -436
  82. package/src/hooks/useStoredMessages.ts +0 -64
  83. package/src/index.css +0 -4269
  84. package/src/index.ts +0 -137
  85. package/src/lib/plugin.ts +0 -24
  86. package/src/llm/ILLMProvider.ts +0 -60
  87. package/src/llm/LLMFactory.ts +0 -141
  88. package/src/llm/providers/AnthropicProvider.ts +0 -215
  89. package/src/llm/providers/GeminiProvider.ts +0 -311
  90. package/src/llm/providers/GroqProvider.ts +0 -184
  91. package/src/llm/providers/OllamaProvider.ts +0 -232
  92. package/src/llm/providers/OpenAIProvider.ts +0 -188
  93. package/src/llm/providers/QwenProvider.ts +0 -199
  94. package/src/llm/providers/UniversalLLMAdapter.ts +0 -422
  95. package/src/llm/utils.ts +0 -46
  96. package/src/providers/graphdb/BaseGraphProvider.ts +0 -43
  97. package/src/providers/graphdb/SimpleGraphProvider.ts +0 -66
  98. package/src/providers/vectordb/BaseVectorProvider.ts +0 -98
  99. package/src/providers/vectordb/ChromaDBProvider.ts +0 -130
  100. package/src/providers/vectordb/MilvusProvider.ts +0 -134
  101. package/src/providers/vectordb/MongoDBProvider.ts +0 -250
  102. package/src/providers/vectordb/MultiTablePostgresProvider.ts +0 -461
  103. package/src/providers/vectordb/PineconeProvider.ts +0 -295
  104. package/src/providers/vectordb/PostgreSQLProvider.ts +0 -227
  105. package/src/providers/vectordb/QdrantProvider.ts +0 -294
  106. package/src/providers/vectordb/RedisProvider.ts +0 -105
  107. package/src/providers/vectordb/UniversalVectorProvider.ts +0 -221
  108. package/src/providers/vectordb/WeaviateProvider.ts +0 -172
  109. package/src/rag/DocumentChunker.ts +0 -144
  110. package/src/rag/EntityExtractor.ts +0 -90
  111. package/src/rag/LlamaIndexIngestor.ts +0 -60
  112. package/src/rag/Reranker.ts +0 -106
  113. package/src/rendering/IntentClassifier.ts +0 -167
  114. package/src/rendering/RendererRegistry.ts +0 -164
  115. package/src/rendering/RuleEngine.ts +0 -67
  116. package/src/rendering/VisualizationDecisionEngine.ts +0 -164
  117. package/src/rendering/__tests__/VisualizationDecisionEngine.test.ts +0 -237
  118. package/src/rendering/index.ts +0 -12
  119. package/src/rendering/rules/Rule1SpecificInfoRule.ts +0 -28
  120. package/src/rendering/rules/Rule2ComparisonRule.ts +0 -24
  121. package/src/rendering/rules/Rule3ProductDiscoveryRule.ts +0 -28
  122. package/src/rendering/rules/Rule4AnalyticalRule.ts +0 -38
  123. package/src/rendering/rules/Rule5MixedResponseRule.ts +0 -29
  124. package/src/rendering/rules/Rule6SmallResultSetRule.ts +0 -24
  125. package/src/rendering/rules/Rule7LargeTableRule.ts +0 -35
  126. package/src/rendering/types.ts +0 -96
  127. package/src/server.ts +0 -156
  128. package/src/tailwind.css +0 -194
  129. package/src/telemetry/TelemetryService.ts +0 -330
  130. package/src/types/chat.ts +0 -89
  131. package/src/types/index.ts +0 -278
  132. package/src/types/pdf-parse.d.ts +0 -13
  133. package/src/types/props.ts +0 -169
  134. package/src/utils/DocumentParser.ts +0 -111
  135. package/src/utils/ProductExtractor.ts +0 -350
  136. package/src/utils/SchemaMapper.ts +0 -176
  137. package/src/utils/UITransformer.ts +0 -1812
  138. package/src/utils/synonyms.ts +0 -80
  139. package/src/utils/templateUtils.ts +0 -78
  140. package/src/version.ts +0 -6
@@ -1,1812 +0,0 @@
1
- import {
2
- VectorMatch,
3
- UITransformationResponse,
4
- PieChartData,
5
- BarChartData,
6
- LineChartDataPoint,
7
- ScatterPlotDataPoint,
8
- MetricCardData,
9
- TableData,
10
- CarouselProduct,
11
- VisualizationType,
12
- } from '../types';
13
- import { ILLMProvider } from '../llm/ILLMProvider';
14
- import { resolveMetadataValue } from './synonyms';
15
- import { VisualizationDecisionEngine } from '../rendering/VisualizationDecisionEngine';
16
-
17
- // ─── Intent Types ─────────────────────────────────────────────────────────────
18
-
19
- /**
20
- * Structured intent extracted from the user query.
21
- * Produced either by the LLM (preferred) or by heuristic fallback.
22
- */
23
- export interface QueryIntent {
24
- /** Primary display mode the user is asking for */
25
- visualizationHint:
26
- | 'trend'
27
- | 'comparison'
28
- | 'distribution'
29
- | 'composition'
30
- | 'correlation'
31
- | 'ranking'
32
- | 'kpi'
33
- | 'tabular'
34
- | 'geographic'
35
- | 'category_breakdown'
36
- | 'product_browse'
37
- | 'table'
38
- | 'text';
39
-
40
- /** Chart family that best matches the analytical intent */
41
- recommendedChart:
42
- | 'line_chart'
43
- | 'bar_chart'
44
- | 'histogram'
45
- | 'pie_chart'
46
- | 'donut_chart'
47
- | 'scatter_plot'
48
- | 'horizontal_bar'
49
- | 'metric_card'
50
- | 'table'
51
- | 'geo_map'
52
- | 'text';
53
-
54
- /** User wants only in-stock / available items */
55
- filterInStockOnly: boolean;
56
-
57
- /** User wants a tabular / list-style answer explicitly */
58
- wantsExplicitTable: boolean;
59
-
60
- /** User is asking about time-based changes */
61
- isTemporal: boolean;
62
-
63
- /** User is comparing, ranking, or contrasting entities/categories */
64
- isComparison: boolean;
65
-
66
- /** The detected language of the query (BCP-47 tag, e.g. "en", "de", "hi") */
67
- language: string;
68
-
69
- /** Free-form note the LLM can add for transparency (not used in logic) */
70
- reasoning?: string;
71
- }
72
-
73
- type PrimitiveValue = string | number | boolean;
74
- type AggregationOperation = 'sum' | 'average' | 'count' | 'min' | 'max' | 'median';
75
-
76
- interface DataRecord {
77
- id: string | number;
78
- content: string;
79
- score: number;
80
- fields: Record<string, PrimitiveValue>;
81
- source: VectorMatch;
82
- }
83
-
84
- interface FieldProfile {
85
- key: string;
86
- label: string;
87
- kind: 'number' | 'date' | 'category' | 'boolean' | 'text';
88
- values: PrimitiveValue[];
89
- uniqueCount: number;
90
- }
91
-
92
- interface DataProfile {
93
- records: DataRecord[];
94
- fields: FieldProfile[];
95
- numericFields: FieldProfile[];
96
- dateFields: FieldProfile[];
97
- categoricalFields: FieldProfile[];
98
- booleanFields: FieldProfile[];
99
- }
100
-
101
- // ─── UITransformer ────────────────────────────────────────────────────────────
102
-
103
- /**
104
- * UITransformer — Converts vector database results into structured UI representations.
105
- *
106
- * Intent detection is now delegated to the LLM via `detectIntent()`.
107
- * Static keyword lists are used only as a last-resort fallback when no LLM
108
- * is available (e.g. during unit tests or offline mode).
109
- */
110
- export class UITransformer {
111
-
112
- // ─── Public Entry Points ─────────────────────────────────────────────────
113
-
114
- /**
115
- * Heuristic-only transform (no LLM required).
116
- * Uses the lightweight heuristic intent detector as a fallback.
117
- * Prefer `analyzeAndDecide()` in production.
118
- */
119
- static transform(
120
- userQuery: string,
121
- retrievedData: VectorMatch[],
122
- config?: import('../config/RagConfig').RagConfig,
123
- trainedSchema?: import('./SchemaMapper').SchemaMap,
124
- /** Optionally pass a pre-computed intent to skip re-detection */
125
- intent?: QueryIntent,
126
- ): UITransformationResponse {
127
- if (!retrievedData || retrievedData.length === 0) {
128
- return this.createTextResponse('No data available', 'No relevant data found for your query.');
129
- }
130
-
131
- // //console.log('[UITransformer.transform] Processing retrievedData:', {
132
- // userQuery,
133
- // retrievedCount: retrievedData.length,
134
- // sample: retrievedData.slice(0, 2).map((item) => ({
135
- // id: item?.id,
136
- // contentType: typeof item?.content,
137
- // contentSnippet: String(item?.content ?? '').substring(0, 150),
138
- // metadataKeys: Object.keys(item?.metadata || {}),
139
- // })),
140
- // });
141
-
142
- const resolvedIntent = intent ?? this.detectIntentHeuristic(userQuery);
143
-
144
- const filteredData = resolvedIntent.filterInStockOnly
145
- ? retrievedData.filter(item => this.determineStockStatus(item))
146
- : retrievedData;
147
-
148
- const profile = this.profileData(filteredData);
149
- const hasProducts = filteredData.some(item => this.isProductData(item));
150
- const wantsPieLikeChart = ['pie_chart', 'donut_chart'].includes(resolvedIntent.recommendedChart);
151
-
152
- // Priority 1 – Explicit trends
153
- if (resolvedIntent.visualizationHint === 'trend' && profile.dateFields.length > 0 && profile.numericFields.length > 0) {
154
- return this.transformToLineChart(profile);
155
- }
156
-
157
- // Priority 2 – Explicit pie/donut or composition/category distribution
158
- if (wantsPieLikeChart || ['composition', 'category_breakdown'].includes(resolvedIntent.visualizationHint)) {
159
- const pieChart = this.transformToPieChart(filteredData, profile, userQuery);
160
- if (pieChart) return pieChart;
161
- }
162
-
163
- // Priority 3 – Comparison/ranking → bar chart
164
- if (['comparison', 'ranking'].includes(resolvedIntent.visualizationHint)) {
165
- return this.transformToBarChart(filteredData, profile, userQuery, resolvedIntent.visualizationHint === 'ranking');
166
- }
167
-
168
- // Priority 4 – Distribution/correlation/geographic/KPI fallback to supported views
169
- if (resolvedIntent.visualizationHint === 'distribution') {
170
- return this.transformToHistogram(profile, userQuery) ?? this.transformToBarChart(filteredData, profile, userQuery);
171
- }
172
-
173
- if (resolvedIntent.visualizationHint === 'correlation') {
174
- return this.transformToScatterPlot(profile, userQuery) ?? this.transformToBarChart(filteredData, profile, userQuery);
175
- }
176
-
177
- if (resolvedIntent.visualizationHint === 'geographic') {
178
- return this.transformToBarChart(filteredData, profile, userQuery);
179
- }
180
-
181
- // Priority 5 – Structured list/filter questions should stay tabular even if
182
- // source metadata happens to contain product-like fields such as category.
183
- // This must run before KPI detection because fields such as "employee count"
184
- // contain the word "count" but still describe records to list/filter.
185
- if (this.isStructuredListQuery(userQuery)) {
186
- return this.hasMultipleFields(filteredData)
187
- ? this.transformToTable(filteredData, userQuery)
188
- : this.transformToText(filteredData);
189
- }
190
-
191
- if (resolvedIntent.visualizationHint === 'kpi') {
192
- return this.transformToMetricCard(profile, userQuery) ?? this.transformToText(filteredData);
193
- }
194
-
195
- // Priority 6 – Explicit tabular/detail request
196
- if (
197
- ['tabular', 'table'].includes(resolvedIntent.visualizationHint) ||
198
- resolvedIntent.wantsExplicitTable
199
- ) {
200
- return this.hasMultipleFields(filteredData)
201
- ? this.transformToTable(filteredData, userQuery)
202
- : this.transformToText(filteredData);
203
- }
204
-
205
- // Priority 7 – Product carousel only for explicit product intent.
206
- if ((hasProducts || this.isProductQuery(userQuery)) && resolvedIntent.visualizationHint === 'product_browse') {
207
- if (!this.isProductQuery(userQuery)) {
208
- return this.transformToText(filteredData);
209
- }
210
- return this.transformToProductCarousel(filteredData, config, trainedSchema);
211
- }
212
-
213
- const automatic = this.chooseAutomaticVisualization(filteredData, profile, userQuery);
214
- if (automatic) return automatic;
215
-
216
- // Fallback – plain text
217
- return this.transformToText(filteredData);
218
- }
219
-
220
- /**
221
- * LLM-driven entry point (recommended for production).
222
- *
223
- * Step 1 — Detect intent via a dedicated, lightweight LLM call.
224
- * Step 2 — Pass the intent + data to the visualization-selection prompt.
225
- * Step 3 — Fall back to the heuristic `transform()` if either LLM call fails.
226
- */
227
- static async analyzeAndDecide(
228
- query: string,
229
- sources: VectorMatch[],
230
- llm: ILLMProvider,
231
- ): Promise<UITransformationResponse> {
232
- // ── Step 0: Fast Visualization Decision Engine check ────────────────────
233
- const decision = VisualizationDecisionEngine.decideVisualization(query, sources);
234
- if (decision.renderType === 'text') {
235
- console.debug('[UITransformer] Decision Engine selected plain text rendering — skipping visual LLM calls.');
236
- return this.transformToText(sources);
237
- }
238
-
239
- // ── Step 1: Detect intent ───────────────────────────────────────────────
240
- let intent: QueryIntent;
241
- try {
242
- intent = await this.detectIntent(query, llm, sources);
243
- console.debug('[UITransformer] Detected intent:', intent);
244
- } catch (err) {
245
- console.warn('[UITransformer] Intent detection failed; using heuristic.', err);
246
- intent = this.detectIntentHeuristic(query);
247
- }
248
-
249
- if (this.isProductQuery(query) && ['text', 'product_browse'].includes(intent.visualizationHint)) {
250
- return this.transform(
251
- query,
252
- sources,
253
- undefined,
254
- undefined,
255
- { ...intent, visualizationHint: 'product_browse', recommendedChart: 'text' },
256
- );
257
- }
258
-
259
- // ── Step 2: Ask the LLM to pick a visualization ─────────────────────────
260
- try {
261
- const context = this.buildContextSummary(sources);
262
- const systemPrompt = this.buildVisualizationSystemPrompt();
263
-
264
- const profile = this.profileData(sources);
265
- const schemaContext = `
266
- RETRIEVED DATA SCHEMA:
267
- - Numeric fields (measures): ${profile.numericFields.map(f => `${f.key} (unique values: ${f.uniqueCount})`).join(', ') || 'None'}
268
- - Category fields (dimensions): ${profile.categoricalFields.map(f => `${f.key} (unique values: ${f.uniqueCount})`).join(', ') || 'None'}
269
- - Date fields (temporal): ${profile.dateFields.map(f => `${f.key} (unique values: ${f.uniqueCount})`).join(', ') || 'None'}
270
- - Boolean fields: ${profile.booleanFields.map(f => `${f.key} (unique values: ${f.uniqueCount})`).join(', ') || 'None'}
271
- - Text/Other fields: ${profile.fields.filter(f => f.kind === 'text').map(f => f.key).join(', ') || 'None'}
272
- `;
273
-
274
- const userPrompt = [
275
- `USER QUESTION: ${query}`,
276
- '',
277
- `DETECTED INTENT (JSON): ${JSON.stringify(intent)}`,
278
- '',
279
- `RETRIEVED DATA SCHEMA:`,
280
- schemaContext,
281
- '',
282
- 'RETRIEVED DATA (JSON):',
283
- context,
284
- ].join('\n');
285
-
286
- const rawResponse = await llm.chat(
287
- [{ role: 'user', content: userPrompt }],
288
- '',
289
- { systemPrompt, temperature: 0 },
290
- );
291
-
292
- const parsed = this.parseTransformationResponse(rawResponse);
293
- if (parsed) {
294
- // Validate suitability of selected chart type based on schema
295
- let isCompatible = true;
296
- if (parsed.type === 'line_chart' && (profile.dateFields.length === 0 || profile.numericFields.length === 0)) {
297
- isCompatible = false;
298
- } else if (
299
- ['bar_chart', 'horizontal_bar', 'pie_chart', 'donut_chart'].includes(parsed.type) &&
300
- (profile.categoricalFields.length === 0 && profile.numericFields.length === 0)
301
- ) {
302
- isCompatible = false;
303
- } else if (parsed.type === 'scatter_plot' && profile.numericFields.length < 2) {
304
- isCompatible = false;
305
- } else if (parsed.type === 'metric_card' && profile.numericFields.length === 0) {
306
- isCompatible = false;
307
- }
308
-
309
- if (!isCompatible) {
310
- console.warn(`[UITransformer] LLM chose incompatible visualization type "${parsed.type}" for retrieved schema. Falling back to heuristic.`);
311
- return this.transform(query, sources, undefined, undefined, intent);
312
- }
313
-
314
- const intentAllowsTable =
315
- intent.wantsExplicitTable ||
316
- ['tabular', 'table', 'geographic'].includes(intent.visualizationHint);
317
- const intentWantsPieLikeChart =
318
- ['pie_chart', 'donut_chart'].includes(intent.recommendedChart) ||
319
- ['composition', 'category_breakdown'].includes(intent.visualizationHint);
320
- // Respect the intent: never sneak in a table the user didn't ask for
321
- if (parsed.type === 'table' && !intentAllowsTable) {
322
- console.debug('[UITransformer] LLM chose table but intent says no. Falling back to text.');
323
- return this.transform(query, sources, undefined, undefined, intent);
324
- }
325
- if (intentWantsPieLikeChart && parsed.type !== 'pie_chart' && this.detectCategories(sources).length > 1) {
326
- console.debug('[UITransformer] LLM ignored pie/composition intent. Using deterministic pie chart.');
327
- return this.transform(query, sources, undefined, undefined, intent);
328
- }
329
- console.debug('[UITransformer] LLM chose visualization type:', parsed.type);
330
- return parsed;
331
- }
332
-
333
- console.warn('[UITransformer] LLM returned unparseable response; falling back to heuristic.');
334
- } catch (err) {
335
- console.warn('[UITransformer] analyzeAndDecide LLM call failed; falling back to heuristic.', err);
336
- }
337
-
338
- // ── Step 3: Heuristic fallback ──────────────────────────────────────────
339
- return this.transform(query, sources, undefined, undefined, intent);
340
- }
341
-
342
- // ─── Dynamic Intent Detection ─────────────────────────────────────────────
343
-
344
- /**
345
- * Calls the LLM with a compact, focused prompt to extract a structured
346
- * `QueryIntent` from the user's query.
347
- *
348
- * Keeping this as a *separate* call from visualization selection means:
349
- * - The prompt is shorter and more reliable.
350
- * - The intent object can be reused across both the heuristic and LLM paths.
351
- * - It is easy to unit-test intent detection in isolation.
352
- */
353
- static async detectIntent(query: string, llm: ILLMProvider, sources?: VectorMatch[]): Promise<QueryIntent> {
354
- let schemaProfileText = '';
355
- if (sources && sources.length > 0) {
356
- const profile = this.profileData(sources);
357
- const hasProducts = sources.some(item => this.isProductData(item));
358
- schemaProfileText = `
359
- RETRIEVED DATA SCHEMA:
360
- - Numeric fields (measures): ${profile.numericFields.map(f => `${f.key} (unique values: ${f.uniqueCount})`).join(', ') || 'None'}
361
- - Category fields (dimensions): ${profile.categoricalFields.map(f => `${f.key} (unique values: ${f.uniqueCount})`).join(', ') || 'None'}
362
- - Date fields (temporal): ${profile.dateFields.map(f => `${f.key} (unique values: ${f.uniqueCount})`).join(', ') || 'None'}
363
- - Boolean fields: ${profile.booleanFields.map(f => `${f.key} (unique values: ${f.uniqueCount})`).join(', ') || 'None'}
364
- - Text fields: ${profile.fields.filter(f => f.kind === 'text').map(f => f.key).join(', ') || 'None'}
365
-
366
- Please choose visualizationHint and recommendedChart dynamically based on the available schema.
367
- - Do NOT recommend "trend" or "line_chart" if there are no Date fields.
368
- - Do NOT recommend numeric-dependent visualizations (like bar_chart, line_chart, pie_chart, scatter_plot, histogram, metric_card) if there are no Numeric fields. Fall back to tabular/table or text.
369
- - If there are multiple metadata fields but no clear categories or numbers, "tabular"/"table" is a highly accurate default.
370
- ${hasProducts ? `- Note: The retrieved data matches product/catalog structure. If the user query is looking for, exploring, searching, browsing, or listing products/items, you MUST recommend visualizationHint: "product_browse" and recommendedChart: "text".` : ''}
371
- `;
372
- }
373
-
374
- const systemPrompt = `You are an intent classifier for a product-search RAG system.
375
- Given a user query, return ONLY a valid JSON object with this exact shape — no prose, no markdown:
376
-
377
- {
378
- "visualizationHint": "trend" | "comparison" | "distribution" | "composition" | "correlation" | "ranking" | "kpi" | "tabular" | "geographic" | "product_browse" | "table" | "text",
379
- "recommendedChart": "line_chart" | "bar_chart" | "histogram" | "pie_chart" | "donut_chart" | "scatter_plot" | "horizontal_bar" | "metric_card" | "table" | "geo_map" | "text",
380
- "filterInStockOnly": boolean,
381
- "wantsExplicitTable": boolean,
382
- "isTemporal": boolean,
383
- "isComparison": boolean,
384
- "language": "<BCP-47 language tag, e.g. en, de, hi, ja>",
385
- "reasoning": "<one short sentence — why you chose these values>"
386
- }
387
-
388
- RULES:
389
- - visualizationHint and recommendedChart
390
- "trend" → keywords: trend, growth, over time; recommendedChart "line_chart"
391
- "comparison" → keywords: compare, versus, top; recommendedChart "bar_chart"
392
- "distribution" → keywords: distribution, spread; recommendedChart "histogram"
393
- "composition" → keywords: share, percentage, breakup, breakdown; recommendedChart "pie_chart" or "donut_chart"
394
- "correlation" → keywords: relation, correlation; recommendedChart "scatter_plot"
395
- "ranking" → keywords: highest, lowest, top 10, ranking; recommendedChart "horizontal_bar"
396
- "kpi" → keywords: total, average, count; recommendedChart "metric_card"
397
- "tabular" → keywords: detailed records, table, grid, spreadsheet; recommendedChart "table"
398
- "geographic" → keywords: region, country, map; recommendedChart "geo_map"
399
- "product_browse" → user is browsing, searching, describing, viewing details for, or asking about one or more products without analytical visualization intent
400
- "table" → legacy alias for "tabular"; use only if the query literally says table
401
- "text" → conversational, factual, or other intent
402
- - If the user explicitly asks for a chart type, honor that chart type in recommendedChart.
403
- - If a query says "distribution ... in a pie chart", use visualizationHint "composition" and recommendedChart "pie_chart".
404
- - filterInStockOnly: true only if user mentions stock, availability, in stock, etc.
405
- - wantsExplicitTable: true if the user asks for a table, list, grid, spreadsheet, or detailed records.
406
- - isTemporal: true if time words appear (trend, historical, over time, last year, monthly, etc.)
407
- - isComparison: true if user compares, ranks, or contrasts entities or categories.
408
- - language: detect from the query text itself; default "en" if uncertain.`;
409
-
410
- const userPrompt = `QUERY: ${query}${schemaProfileText ? `\n\n${schemaProfileText}` : ''}`;
411
-
412
- const rawResponse = await llm.chat(
413
- [{ role: 'user', content: userPrompt }],
414
- '',
415
- { systemPrompt, temperature: 0 },
416
- );
417
-
418
- const parsed = this.parseIntentResponse(rawResponse);
419
- if (!parsed) {
420
- throw new Error(`Could not parse intent JSON from LLM response: ${rawResponse}`);
421
- }
422
- return parsed;
423
- }
424
-
425
- /**
426
- * Parse and validate the raw LLM response into a `QueryIntent`.
427
- */
428
- private static parseIntentResponse(raw: string): QueryIntent | null {
429
- const jsonStr = this.extractJsonCandidate(raw);
430
- if (!jsonStr) return null;
431
-
432
- try {
433
- const obj = JSON.parse(jsonStr) as Partial<QueryIntent>;
434
-
435
- const validHints = [
436
- 'trend', 'comparison', 'distribution', 'composition', 'correlation',
437
- 'ranking', 'kpi', 'tabular', 'geographic', 'category_breakdown',
438
- 'product_browse', 'table', 'text',
439
- ] as const;
440
- const validCharts = [
441
- 'line_chart', 'bar_chart', 'histogram', 'pie_chart', 'donut_chart',
442
- 'scatter_plot', 'horizontal_bar', 'metric_card', 'table', 'geo_map', 'text',
443
- ] as const;
444
-
445
- const hint = obj.visualizationHint;
446
- if (!hint || !validHints.includes(hint as typeof validHints[number])) return null;
447
- const normalizedHint = hint === 'category_breakdown' ? 'composition' : hint;
448
- const recommendedChart =
449
- typeof obj.recommendedChart === 'string' &&
450
- validCharts.includes(obj.recommendedChart as typeof validCharts[number])
451
- ? obj.recommendedChart as QueryIntent['recommendedChart']
452
- : this.getRecommendedChartForIntent(normalizedHint as QueryIntent['visualizationHint']);
453
-
454
- return {
455
- visualizationHint: normalizedHint as QueryIntent['visualizationHint'],
456
- recommendedChart,
457
- filterInStockOnly: Boolean(obj.filterInStockOnly),
458
- wantsExplicitTable: Boolean(obj.wantsExplicitTable),
459
- isTemporal: Boolean(obj.isTemporal),
460
- isComparison: Boolean(obj.isComparison),
461
- language: typeof obj.language === 'string' && obj.language ? obj.language : 'en',
462
- reasoning: typeof obj.reasoning === 'string' ? obj.reasoning : undefined,
463
- };
464
- } catch {
465
- return null;
466
- }
467
- }
468
-
469
- /**
470
- * Heuristic intent detector — used when the LLM is unavailable.
471
- * Intentionally minimal: it should only catch the most obvious signals.
472
- * The LLM path handles everything subtle.
473
- */
474
- static detectIntentHeuristic(query: string): QueryIntent {
475
- const q = query.toLowerCase();
476
-
477
- const isTemporal = /\b(trend|trends|over time|historical|history|growth|decline|monthly|yearly|weekly|daily|last (year|month|week)|timeline|forecast)\b/.test(q);
478
-
479
- const isRanking = /\b(highest|lowest|top\s*\d+|bottom\s*\d+|rank(?:ing)?|best|worst|leading)\b/.test(q);
480
-
481
- const isComparison = /\b(compare|comparison|vs\.?|versus|against|differ|difference|contrast|top)\b/.test(q) || isRanking;
482
-
483
- const isDistribution = /\b(distribution|spread|histogram|frequency|variance|range)\b/.test(q);
484
-
485
- const wantsPieLikeChart = /\b(pie|donut|doughnut)(?:\s+chart)?\b/.test(q);
486
-
487
- const isComposition = wantsPieLikeChart || /\b(share|percentage|percent|breakup|breakdown|composition|split|segmentation|by category|by type|proportion)\b/.test(q);
488
-
489
- const isCorrelation = /\b(relation|relationship|correlation|correlate|scatter|association|impact of|depend(?:s|ence)? on)\b/.test(q);
490
-
491
- const isKpi = /\b(total|average|avg|count|sum|median|minimum|maximum|metric|kpi|how many|number of)\b/.test(q);
492
-
493
- const isGeographic = /\b(region|country|countries|state|city|location|map|geo|geographic|territory)\b/.test(q);
494
-
495
- const filterInStockOnly = /\b(in[- ]?stock|available|availability|inventory|stock status)\b/.test(q);
496
-
497
- const wantsExplicitTable = /\b(table|spreadsheet|grid|detailed records|record details|list all|compare all)\b/.test(q);
498
-
499
- let visualizationHint: QueryIntent['visualizationHint'] = 'text';
500
- if (wantsExplicitTable) visualizationHint = 'tabular';
501
- else if (isTemporal) visualizationHint = 'trend';
502
- else if (wantsPieLikeChart) visualizationHint = 'composition';
503
- else if (isRanking) visualizationHint = 'ranking';
504
- else if (isComparison) visualizationHint = 'comparison';
505
- else if (isDistribution) visualizationHint = 'distribution';
506
- else if (isComposition) visualizationHint = 'composition';
507
- else if (isCorrelation) visualizationHint = 'correlation';
508
- else if (isGeographic) visualizationHint = 'geographic';
509
- else if (isKpi) visualizationHint = 'kpi';
510
- else if (this.isProductQuery(query)) visualizationHint = 'product_browse';
511
-
512
- return {
513
- visualizationHint,
514
- recommendedChart: this.getRecommendedChartForIntent(visualizationHint),
515
- filterInStockOnly,
516
- wantsExplicitTable,
517
- isTemporal,
518
- isComparison,
519
- language: 'en', // heuristic cannot reliably detect language
520
- };
521
- }
522
-
523
- private static getRecommendedChartForIntent(
524
- intent: QueryIntent['visualizationHint'],
525
- ): QueryIntent['recommendedChart'] {
526
- switch (intent) {
527
- case 'trend':
528
- return 'line_chart';
529
- case 'comparison':
530
- return 'bar_chart';
531
- case 'distribution':
532
- return 'histogram';
533
- case 'composition':
534
- case 'category_breakdown':
535
- return 'pie_chart';
536
- case 'correlation':
537
- return 'scatter_plot';
538
- case 'ranking':
539
- return 'horizontal_bar';
540
- case 'kpi':
541
- return 'metric_card';
542
- case 'tabular':
543
- case 'table':
544
- return 'table';
545
- case 'geographic':
546
- return 'geo_map';
547
- case 'product_browse':
548
- case 'text':
549
- default:
550
- return 'text';
551
- }
552
- }
553
-
554
- // ─── Transform Helpers ────────────────────────────────────────────────────
555
-
556
- private static transformToProductCarousel(
557
- data: VectorMatch[],
558
- config?: import('../config/RagConfig').RagConfig,
559
- trainedSchema?: import('./SchemaMapper').SchemaMap,
560
- ): UITransformationResponse {
561
- const products: CarouselProduct[] = data
562
- .map(item => this.extractProductInfo(item, config, trainedSchema))
563
- .filter((p): p is CarouselProduct => p !== null);
564
-
565
- return {
566
- type: 'product_carousel',
567
- title: 'Recommended Products',
568
- description: `Found ${products.length} relevant products`,
569
- data: products,
570
- };
571
- }
572
-
573
- private static transformToPieChart(
574
- data: VectorMatch[],
575
- profile?: DataProfile,
576
- query = '',
577
- ): UITransformationResponse | null {
578
- const dimension = profile ? this.selectDimensionField(profile, query) : undefined;
579
- const categories = dimension
580
- ? Array.from(new Set(profile!.records.map(record => String(record.fields[dimension.key] ?? '')).filter(Boolean)))
581
- : this.detectCategories(data);
582
- if (categories.length === 0) return null;
583
-
584
- const categoryData = dimension && profile
585
- ? this.aggregateProfileByDimension(profile, dimension.key)
586
- : this.aggregateByCategory(data, categories);
587
-
588
- const pieData: PieChartData[] = Object.entries(categoryData).map(([label, count]) => {
589
- const { inStockCount, outOfStockCount } = this.calculateStockCounts(label, data);
590
- return { label, value: count as number, inStockCount, outOfStockCount };
591
- });
592
-
593
- return {
594
- type: 'pie_chart',
595
- title: `Distribution by ${dimension?.label ?? 'Category'}`,
596
- description: `Showing breakdown across ${pieData.length} categories`,
597
- data: pieData,
598
- };
599
- }
600
-
601
- private static transformToLineChart(profile: DataProfile): UITransformationResponse {
602
- const dateField = profile.dateFields[0];
603
- const valueField = profile.numericFields[0];
604
- const buckets = new Map<string, number>();
605
-
606
- profile.records.forEach(record => {
607
- const timestamp = String(record.fields[dateField.key] ?? '');
608
- const value = this.toFiniteNumber(record.fields[valueField.key]) ?? 0;
609
- buckets.set(timestamp, (buckets.get(timestamp) ?? 0) + value);
610
- });
611
-
612
- const lineData: LineChartDataPoint[] = Array.from(buckets.entries())
613
- .sort(([a], [b]) => Date.parse(a) - Date.parse(b))
614
- .slice(0, 24)
615
- .map(([timestamp, value]) => ({ timestamp, value, label: timestamp }));
616
-
617
- return {
618
- type: 'line_chart',
619
- title: `${valueField.label} Over Time`,
620
- description: `Showing ${lineData.length} data points`,
621
- data: lineData,
622
- };
623
- }
624
-
625
- private static transformToBarChart(
626
- data: VectorMatch[],
627
- profile?: DataProfile,
628
- query = '',
629
- horizontal = false,
630
- ): UITransformationResponse {
631
- const dimension = profile ? this.selectDimensionField(profile, query) : undefined;
632
- const measure = profile ? this.selectNumericField(profile, query) : undefined;
633
- const aggregate = dimension && profile
634
- ? this.aggregateProfileByDimension(profile, dimension.key, measure?.key)
635
- : this.aggregateByCategory(data, this.detectCategories(data));
636
-
637
- const barData: BarChartData[] = Object.entries(aggregate)
638
- .map(([category, value]) => ({ category, value: Number(value) }))
639
- .sort((a, b) => horizontal ? b.value - a.value : 0)
640
- .slice(0, 12);
641
-
642
- const fallbackData: BarChartData[] = barData.length > 0
643
- ? barData
644
- : data.slice(0, 12).map((item, index) => {
645
- const meta = item.metadata || {};
646
- const label = String(
647
- this.getDynamicVal(meta, 'name') ??
648
- meta.label ??
649
- meta.title ??
650
- `Result ${index + 1}`,
651
- );
652
- const value = this.extractNumericValue(meta) ?? item.score ?? 0;
653
- return { category: label, value: Number(value) };
654
- });
655
-
656
- return {
657
- type: horizontal ? 'horizontal_bar' : 'bar_chart',
658
- title: dimension ? `${measure?.label ?? 'Count'} by ${dimension.label}` : 'Comparison',
659
- description: `Showing ${fallbackData.length} comparable values`,
660
- data: fallbackData,
661
- };
662
- }
663
-
664
- private static transformToHistogram(profile: DataProfile, query = ''): UITransformationResponse | null {
665
- const field = this.selectNumericField(profile, query);
666
- if (!field) return null;
667
-
668
- const values = profile.records
669
- .map(record => this.toFiniteNumber(record.fields[field.key]))
670
- .filter((value): value is number => value !== null)
671
- .sort((a, b) => a - b);
672
- if (values.length === 0) return null;
673
-
674
- const min = values[0];
675
- const max = values[values.length - 1];
676
- const bucketCount = Math.min(10, Math.max(3, Math.ceil(Math.sqrt(values.length))));
677
- const width = max === min ? 1 : (max - min) / bucketCount;
678
- const buckets = Array.from({ length: bucketCount }, (_, index) => {
679
- const start = min + index * width;
680
- const end = index === bucketCount - 1 ? max : start + width;
681
- return { category: `${this.formatNumber(start)}-${this.formatNumber(end)}`, value: 0 };
682
- });
683
-
684
- values.forEach(value => {
685
- const bucketIndex = max === min ? 0 : Math.min(bucketCount - 1, Math.floor((value - min) / width));
686
- buckets[bucketIndex].value += 1;
687
- });
688
-
689
- return {
690
- type: 'histogram',
691
- title: `${field.label} Distribution`,
692
- description: `Showing ${values.length} values across ${bucketCount} buckets`,
693
- data: buckets,
694
- };
695
- }
696
-
697
- private static transformToScatterPlot(profile: DataProfile, query = ''): UITransformationResponse | null {
698
- const fields = this.rankFieldsByQuery(profile.numericFields, query).slice(0, 2);
699
- if (fields.length < 2) return null;
700
- const [xField, yField] = fields;
701
-
702
- const points = profile.records
703
- .map((record): ScatterPlotDataPoint | null => {
704
- const x = this.toFiniteNumber(record.fields[xField.key]);
705
- const y = this.toFiniteNumber(record.fields[yField.key]);
706
- if (x === null || y === null) return null;
707
- return { x, y, label: String(this.getRecordLabel(record) ?? record.id) };
708
- })
709
- .filter((point): point is ScatterPlotDataPoint => point !== null)
710
- .slice(0, 100);
711
-
712
- if (points.length === 0) return null;
713
-
714
- return {
715
- type: 'scatter_plot',
716
- title: `${xField.label} vs ${yField.label}`,
717
- description: `Showing ${points.length} paired values`,
718
- data: points,
719
- };
720
- }
721
-
722
- private static transformToMetricCard(profile: DataProfile, query = ''): UITransformationResponse | null {
723
- const operation = this.detectAggregationOperation(query);
724
- const numericField = this.selectNumericField(profile, query);
725
- const values = numericField
726
- ? profile.records
727
- .map(record => this.toFiniteNumber(record.fields[numericField.key]))
728
- .filter((value): value is number => value !== null)
729
- : [];
730
-
731
- const value = operation === 'count' || values.length === 0
732
- ? profile.records.length
733
- : this.calculateAggregate(values, operation);
734
-
735
- const metric: MetricCardData = {
736
- label: numericField ? `${operation} ${numericField.label}` : 'Count',
737
- value,
738
- operation,
739
- };
740
-
741
- return {
742
- type: 'metric_card',
743
- title: metric.label,
744
- description: `Calculated from ${profile.records.length} result${profile.records.length === 1 ? '' : 's'}`,
745
- data: metric,
746
- };
747
- }
748
-
749
- private static transformToRadarChart(data: VectorMatch[]): UITransformationResponse {
750
- // Aggregate attribute scores across items for radar comparison
751
- const attributeMap: Record<string, Record<string, number>> = {};
752
-
753
- data.forEach(item => {
754
- const meta = item.metadata || {};
755
- const seriesName = String(meta.name ?? meta.product ?? item.id ?? 'Item');
756
- Object.entries(meta).forEach(([key, val]) => {
757
- if (typeof val === 'number' && !['price', 'id'].includes(key.toLowerCase())) {
758
- if (!attributeMap[key]) attributeMap[key] = {};
759
- attributeMap[key][seriesName] = val;
760
- }
761
- });
762
- });
763
-
764
- const radarData = Object.entries(attributeMap).map(([attribute, series]) => ({
765
- attribute,
766
- ...series,
767
- }));
768
-
769
- return {
770
- type: 'radar_chart' as VisualizationType,
771
- title: 'Product Comparison',
772
- description: `Comparing ${data.length} items across ${radarData.length} attributes`,
773
- data: radarData.length > 0 ? radarData : data.map(d => ({ attribute: (d?.content ?? '').substring(0, 40) })),
774
- };
775
- }
776
-
777
- private static transformToTable(data: VectorMatch[], query = ''): UITransformationResponse {
778
- const columns = this.extractTableColumns(data, query);
779
- const rows = data.map(item => this.extractTableRow(item, columns));
780
- const tableData: TableData = { columns, rows };
781
-
782
- return {
783
- type: 'table',
784
- title: 'Detailed Results',
785
- description: `Showing ${data.length} results`,
786
- data: tableData,
787
- };
788
- }
789
-
790
- private static transformToText(data: VectorMatch[]): UITransformationResponse {
791
- return this.createTextResponse(
792
- 'Retrieved Context',
793
- data.map(item => item?.content ?? '').join('\n\n'),
794
- `Found ${data.length} relevant results`,
795
- );
796
- }
797
-
798
- private static createTextResponse(
799
- title: string,
800
- content: string,
801
- description?: string,
802
- ): UITransformationResponse {
803
- return { type: 'text', title, description, data: { content } };
804
- }
805
-
806
- // ─── LLM Response Parsing ─────────────────────────────────────────────────
807
-
808
- static parseTransformationResponse(raw: string): UITransformationResponse | null {
809
- const payloadText = this.extractJsonCandidate(raw);
810
- if (!payloadText) return null;
811
-
812
- try {
813
- const parsed = JSON.parse(payloadText);
814
- return this.normalizeTransformation(parsed);
815
- } catch {
816
- return null;
817
- }
818
- }
819
-
820
- private static extractJsonCandidate(raw: string): string | null {
821
- if (!raw) return null;
822
-
823
- const cleaned = raw
824
- .replace(/```(?:json|chart|ui)?\s*/gi, '')
825
- .replace(/```/g, '')
826
- .trim();
827
-
828
- const start = cleaned.indexOf('{');
829
- if (start === -1) return null;
830
-
831
- let depth = 0;
832
- let inString = false;
833
- let escape = false;
834
-
835
- for (let i = start; i < cleaned.length; i += 1) {
836
- const char = cleaned[i];
837
- if (escape) { escape = false; continue; }
838
- if (char === '\\') { escape = true; continue; }
839
- if (char === '"') { inString = !inString; continue; }
840
- if (inString) continue;
841
- if (char === '{') depth += 1;
842
- if (char === '}') {
843
- depth -= 1;
844
- if (depth === 0) return cleaned.slice(start, i + 1);
845
- }
846
- }
847
-
848
- return null;
849
- }
850
-
851
- private static normalizeTransformation(payload: unknown): UITransformationResponse | null {
852
- if (!payload || typeof payload !== 'object') return null;
853
-
854
- const p = payload as Record<string, unknown>;
855
- const type = this.normalizeVisualizationType(
856
- String(p.type ?? p.view ?? p.chartType ?? ''),
857
- );
858
- if (!type) return null;
859
-
860
- const title = String(p.title ?? p.heading ?? 'Visualization');
861
- const description = p.description ? String(p.description) : undefined;
862
- const rawData = p.data ?? p.table ?? p.rows ?? p.items ?? p.content ?? null;
863
- const data = type === 'text' && typeof rawData === 'string'
864
- ? { content: rawData }
865
- : rawData;
866
-
867
- const transformation: UITransformationResponse = { type, title, description, data };
868
- return this.validateTransformation(transformation) ? transformation : null;
869
- }
870
-
871
- private static normalizeVisualizationType(type: string): VisualizationType | null {
872
- const mapping: Record<string, VisualizationType> = {
873
- pie: 'pie_chart',
874
- pie_chart: 'pie_chart',
875
- bar: 'bar_chart',
876
- bar_chart: 'bar_chart',
877
- histogram: 'histogram',
878
- horizontal_bar: 'horizontal_bar',
879
- horizontalbar: 'horizontal_bar',
880
- line: 'line_chart',
881
- line_chart: 'line_chart',
882
- scatter: 'scatter_plot',
883
- scatter_plot: 'scatter_plot',
884
- scatterplot: 'scatter_plot',
885
- radar: 'radar_chart',
886
- radar_chart: 'radar_chart',
887
- metric: 'metric_card',
888
- metric_card: 'metric_card',
889
- card: 'metric_card',
890
- kpi: 'metric_card',
891
- geo: 'geo_map',
892
- geo_map: 'geo_map',
893
- map: 'geo_map',
894
- table: 'table',
895
- text: 'text',
896
- product_carousel: 'product_carousel',
897
- carousel: 'carousel',
898
- };
899
- return mapping[type.toLowerCase()] ?? null;
900
- }
901
-
902
- private static validateTransformation(t: UITransformationResponse): boolean {
903
- const { type, data } = t;
904
- switch (type) {
905
- case 'pie_chart':
906
- case 'bar_chart':
907
- case 'histogram':
908
- case 'horizontal_bar':
909
- return Array.isArray(data) && data.every(
910
- (i): i is Record<string, unknown> =>
911
- i !== null && typeof i === 'object' &&
912
- (typeof i.value === 'number' || !Number.isNaN(Number(i.value))),
913
- );
914
- case 'scatter_plot':
915
- return Array.isArray(data) && data.every(
916
- (i): i is Record<string, unknown> =>
917
- i !== null && typeof i === 'object' &&
918
- (typeof i.x === 'number' || !Number.isNaN(Number(i.x))) &&
919
- (typeof i.y === 'number' || !Number.isNaN(Number(i.y))),
920
- );
921
- case 'line_chart':
922
- return Array.isArray(data) && data.every(
923
- (i): i is Record<string, unknown> =>
924
- i !== null && typeof i === 'object' && 'timestamp' in i &&
925
- (typeof i.value === 'number' || !Number.isNaN(Number(i.value))),
926
- );
927
- case 'metric_card':
928
- return (
929
- typeof data === 'object' && data !== null &&
930
- (typeof (data as Record<string, unknown>).value === 'number' ||
931
- !Number.isNaN(Number((data as Record<string, unknown>).value)))
932
- );
933
- case 'geo_map':
934
- return Array.isArray(data) || (typeof data === 'object' && data !== null);
935
- case 'radar_chart':
936
- return Array.isArray(data) && data.every(
937
- (i): i is Record<string, unknown> =>
938
- i !== null && typeof i === 'object' && 'attribute' in i,
939
- );
940
- case 'table':
941
- return (
942
- typeof data === 'object' && data !== null &&
943
- Array.isArray((data as Record<string, unknown>).columns) &&
944
- Array.isArray((data as Record<string, unknown>).rows)
945
- );
946
- case 'text':
947
- return (
948
- typeof data === 'object' && data !== null &&
949
- typeof (data as Record<string, unknown>).content === 'string'
950
- );
951
- case 'product_carousel':
952
- case 'carousel':
953
- return Array.isArray(data) && data.every(
954
- (i): i is Record<string, unknown> =>
955
- i !== null && typeof i === 'object' && ('id' in i || 'name' in i),
956
- );
957
- default:
958
- return false;
959
- }
960
- }
961
-
962
- // ─── Data Inspection Helpers ──────────────────────────────────────────────
963
-
964
- private static isStructuredListQuery(query: string): boolean {
965
- const q = query.toLowerCase();
966
- const asksForRecords = /\b(list|provide|show|give|get|find|which|whose|records?|details?)\b/.test(q);
967
- const hasNumericPredicate = /\b(greater than|more than|above|over|less than|below|under|at least|at most|minimum|maximum|>=|<=|>|<|equals?|equal to)\b\s*\$?\d/i.test(q) ||
968
- /\b\d+\b\s*(?:or more|or less|and above|and below)\b/i.test(q);
969
- const hasEntityTerms = /\b(organization|organisations?|organizations?|company|companies|employee|employees|staff|headcount|workforce)\b/.test(q);
970
- return asksForRecords && (hasNumericPredicate || hasEntityTerms);
971
- }
972
-
973
- /** @internal kept private for transform() internal logic */
974
- private static isProductQuery(query: string): boolean {
975
- return UITransformer._productQueryTest(query);
976
- }
977
-
978
- /**
979
- * Public entry-point for external callers (e.g. Pipeline)
980
- * to check whether a query maps to product_browse without triggering an LLM call.
981
- */
982
- static isProductQueryPublic(query: string): boolean {
983
- return UITransformer._productQueryTest(query);
984
- }
985
-
986
- private static _productQueryTest(query: string): boolean {
987
- const q = query.toLowerCase();
988
- const productTerms = /\b(product|products|item|items|sku|catalog|catalogue|price|prices|brand|model|stock|inventory|in stock|out of stock|buy|shop)\b/.test(q);
989
- const productAction = /\b(recommend|suggest|describe|description|detail|details|about|show|find|search|browse|list|explore|exploring)\b/.test(q);
990
- const nonProductEntityTerms = /\b(organization|organisations?|organizations?|company|companies|employee|employees|staff|headcount|workforce)\b/.test(q);
991
- return productTerms && productAction && !nonProductEntityTerms;
992
- }
993
-
994
-
995
- private static isProductData(item: VectorMatch): boolean {
996
- if (!item) return false;
997
- const content = (item.content || '').toLowerCase();
998
- const productKeywords = [
999
- 'product', 'price', 'stock', 'sku', 'brand', 'msrp',
1000
- 'inventory', 'buy', 'shop', 'beauty', 'cosmetic', 'facial',
1001
- 'cream', 'serum', 'mask', 'makeup', 'fragrance',
1002
- ];
1003
- const hasKeywords = productKeywords.some(kw => content.includes(kw));
1004
-
1005
- const metadata = item.metadata || {};
1006
- const hasMetadataKey = Object.keys(metadata).some(k => {
1007
- const val = metadata[k];
1008
- return (
1009
- ['price', 'product', 'sku', 'brand', 'cost'].includes(k.toLowerCase()) &&
1010
- val !== null && val !== undefined && val !== ''
1011
- );
1012
- });
1013
-
1014
- const hasPricePattern = /\$\s*\d+\.\d{2}/.test(content);
1015
- return hasKeywords || hasMetadataKey || hasPricePattern;
1016
- }
1017
-
1018
- private static isTimeSeriesData(item: VectorMatch): boolean {
1019
- const metadata = item.metadata || {};
1020
- const maybeDateKeys = Object.keys(metadata).filter(k =>
1021
- ['date', 'timestamp', 'time', 'period'].includes(k.toLowerCase()),
1022
- );
1023
- return maybeDateKeys.some(key => {
1024
- const value = metadata[key];
1025
- if (typeof value === 'string') return !Number.isNaN(Date.parse(value)) && value.trim().length > 0;
1026
- return typeof value === 'number';
1027
- });
1028
- }
1029
-
1030
- private static hasMultipleFields(data: VectorMatch[]): boolean {
1031
- const fieldCount = new Set<string>();
1032
- data.forEach(item => Object.keys(item.metadata || {}).forEach(k => fieldCount.add(k)));
1033
- return fieldCount.size > 2;
1034
- }
1035
-
1036
- private static profileData(data: VectorMatch[]): DataProfile {
1037
- const records = (data || []).filter((item): item is VectorMatch => Boolean(item && typeof item === 'object')).map((item): DataRecord => {
1038
- const fields: Record<string, PrimitiveValue> = {};
1039
- Object.entries(item.metadata || {}).forEach(([key, value]) => {
1040
- const primitive = this.toPrimitive(value);
1041
- if (primitive !== null) fields[key] = primitive;
1042
- });
1043
- if (!fields.content && item.content) fields.content = item.content.substring(0, 500);
1044
- return {
1045
- id: item.id,
1046
- content: item.content ?? '',
1047
- score: item.score ?? 0,
1048
- fields,
1049
- source: item,
1050
- };
1051
- });
1052
-
1053
- const keys = Array.from(new Set(records.flatMap(record => Object.keys(record.fields))));
1054
- const fields = keys.map((key): FieldProfile => {
1055
- const values = records
1056
- .map(record => record.fields[key])
1057
- .filter((value): value is PrimitiveValue => value !== undefined && value !== null && String(value).trim() !== '');
1058
- const uniqueCount = new Set(values.map(value => String(value).toLowerCase())).size;
1059
- return {
1060
- key,
1061
- label: this.humanizeFieldName(key),
1062
- kind: this.inferFieldKind(key, values, records.length, uniqueCount),
1063
- values,
1064
- uniqueCount,
1065
- };
1066
- });
1067
-
1068
- return {
1069
- records,
1070
- fields,
1071
- numericFields: fields.filter(field => field.kind === 'number'),
1072
- dateFields: fields.filter(field => field.kind === 'date'),
1073
- categoricalFields: fields.filter(field => field.kind === 'category'),
1074
- booleanFields: fields.filter(field => field.kind === 'boolean'),
1075
- };
1076
- }
1077
-
1078
- private static toPrimitive(value: unknown): PrimitiveValue | null {
1079
- if (typeof value === 'string' || typeof value === 'number' || typeof value === 'boolean') return value;
1080
- if (value === null || value === undefined) return null;
1081
- if (value instanceof Date) return value.toISOString();
1082
- return null;
1083
- }
1084
-
1085
- private static inferFieldKind(
1086
- key: string,
1087
- values: PrimitiveValue[],
1088
- rowCount: number,
1089
- uniqueCount: number,
1090
- ): FieldProfile['kind'] {
1091
- const lowerKey = key.toLowerCase();
1092
- if (values.length === 0) return 'text';
1093
- if (values.every(value => typeof value === 'boolean' || /^(true|false|yes|no|in_stock|out_of_stock|available|unavailable)$/i.test(String(value)))) {
1094
- return 'boolean';
1095
- }
1096
- if (/(date|time|timestamp|created|updated|month|year|period)/i.test(lowerKey) && values.some(value => this.isDateLike(value))) {
1097
- return 'date';
1098
- }
1099
- const numericCount = values.filter(value => this.toFiniteNumber(value) !== null).length;
1100
- if (numericCount / values.length >= 0.85 && !/(id|sku|ean|upc|zip|postal|phone|code)/i.test(lowerKey)) {
1101
- return 'number';
1102
- }
1103
- if (
1104
- uniqueCount <= Math.max(20, Math.ceil(rowCount * 0.7)) &&
1105
- !/(id|sku|ean|upc|description|content|url|image|thumbnail)/i.test(lowerKey)
1106
- ) {
1107
- return 'category';
1108
- }
1109
- return 'text';
1110
- }
1111
-
1112
- private static isDateLike(value: PrimitiveValue): boolean {
1113
- if (typeof value === 'number') return value > 1900 && value < 3000;
1114
- const text = String(value).trim();
1115
- return text.length > 3 && !Number.isNaN(Date.parse(text));
1116
- }
1117
-
1118
- private static chooseAutomaticVisualization(
1119
- data: VectorMatch[],
1120
- profile: DataProfile,
1121
- query: string,
1122
- ): UITransformationResponse | null {
1123
- if (profile.records.length === 0) return null;
1124
-
1125
- const q = query.toLowerCase().trim();
1126
- // Do not automatically force charts unless query contains visual or analytical intent
1127
- const isExplicitVisualOrAnalytic = /\b(chart|graph|plot|visualize|trend|monthly|revenue|top\s*\d+|breakdown|analytics|compare|versus|vs\.?|histogram|pie|bar|line|distribution|percentage|share|summary of metrics)\b/i.test(q);
1128
-
1129
- if (!isExplicitVisualOrAnalytic) {
1130
- return null;
1131
- }
1132
-
1133
- if (profile.dateFields.length > 0 && profile.numericFields.length > 0) {
1134
- return this.transformToLineChart(profile);
1135
- }
1136
- if (profile.categoricalFields.length > 0 && profile.numericFields.length > 0) {
1137
- return this.transformToBarChart(data, profile, query);
1138
- }
1139
- if (profile.categoricalFields.length > 0) {
1140
- return this.transformToPieChart(data, profile, query);
1141
- }
1142
- if (profile.numericFields.length >= 2) {
1143
- return this.transformToScatterPlot(profile, query);
1144
- }
1145
- if (profile.numericFields.length === 1) {
1146
- return this.transformToMetricCard(profile, query);
1147
- }
1148
- return null;
1149
- }
1150
-
1151
- private static selectDimensionField(profile: DataProfile, query: string): FieldProfile | undefined {
1152
- const productCategory = profile.categoricalFields.find(field =>
1153
- /category|department|collection|type|group|segment|region|country|state|city/i.test(field.key),
1154
- );
1155
- const ranked = this.rankFieldsByQuery(profile.categoricalFields, query);
1156
- return ranked[0] ?? productCategory ?? profile.categoricalFields[0];
1157
- }
1158
-
1159
- private static selectNumericField(profile: DataProfile, query: string): FieldProfile | undefined {
1160
- return this.rankFieldsByQuery(profile.numericFields, query)[0] ?? profile.numericFields[0];
1161
- }
1162
-
1163
- private static rankFieldsByQuery(fields: FieldProfile[], query: string): FieldProfile[] {
1164
- const q = query.toLowerCase();
1165
- return [...fields].sort((a, b) => this.fieldScore(b, q) - this.fieldScore(a, q));
1166
- }
1167
-
1168
- private static fieldScore(field: FieldProfile, query: string): number {
1169
- const key = field.key.toLowerCase();
1170
- const label = field.label.toLowerCase();
1171
- let score = 0;
1172
- if (query.includes(key)) score += 4;
1173
- if (query.includes(label)) score += 4;
1174
- key.split(/[_\s-]+/).forEach(part => {
1175
- if (part && query.includes(part)) score += 1;
1176
- });
1177
- if (/category|department|collection|type|group|segment/.test(key)) score += 2;
1178
- if (/count|total|quantity|stock|inventory|sales|revenue|amount|price|value|score/.test(key)) score += 2;
1179
- if (/id|sku|ean|upc|code/.test(key)) score -= 5;
1180
- return score;
1181
- }
1182
-
1183
- private static normalizeComparableField(field: string): string {
1184
- return field
1185
- .toLowerCase()
1186
- .replace(/&/g, 'and')
1187
- .replace(/[^a-z0-9]+/g, '')
1188
- .trim();
1189
- }
1190
-
1191
- private static fieldTokens(field: string): string[] {
1192
- return field
1193
- .toLowerCase()
1194
- .replace(/[_-]+/g, ' ')
1195
- .split(/\s+/)
1196
- .map(token => token.replace(/[^a-z0-9]/g, ''))
1197
- .filter(Boolean);
1198
- }
1199
-
1200
- private static aggregateProfileByDimension(
1201
- profile: DataProfile,
1202
- dimensionKey: string,
1203
- measureKey?: string,
1204
- ): Record<string, number> {
1205
- const result: Record<string, number> = {};
1206
- profile.records.forEach(record => {
1207
- const category = String(record.fields[dimensionKey] ?? 'Other').trim() || 'Other';
1208
- const value = measureKey ? this.toFiniteNumber(record.fields[measureKey]) ?? 0 : 1;
1209
- result[category] = (result[category] ?? 0) + value;
1210
- });
1211
- return result;
1212
- }
1213
-
1214
- private static getRecordLabel(record: DataRecord): PrimitiveValue | undefined {
1215
- const labelKeys = ['name', 'title', 'label', 'product', 'item', 'brand'];
1216
- const key = Object.keys(record.fields).find(fieldKey =>
1217
- labelKeys.some(labelKey => fieldKey.toLowerCase().includes(labelKey)),
1218
- );
1219
- return key ? record.fields[key] : undefined;
1220
- }
1221
-
1222
- private static detectAggregationOperation(query: string): AggregationOperation {
1223
- const q = query.toLowerCase();
1224
- if (/\b(avg|average|mean)\b/.test(q)) return 'average';
1225
- if (/\b(count|how many|number of)\b/.test(q)) return 'count';
1226
- if (/\b(min|minimum|lowest)\b/.test(q)) return 'min';
1227
- if (/\b(max|maximum|highest)\b/.test(q)) return 'max';
1228
- if (/\bmedian\b/.test(q)) return 'median';
1229
- return 'sum';
1230
- }
1231
-
1232
- private static calculateAggregate(values: number[], operation: AggregationOperation): number {
1233
- if (values.length === 0) return 0;
1234
- switch (operation) {
1235
- case 'average':
1236
- return values.reduce((sum, value) => sum + value, 0) / values.length;
1237
- case 'count':
1238
- return values.length;
1239
- case 'min':
1240
- return Math.min(...values);
1241
- case 'max':
1242
- return Math.max(...values);
1243
- case 'median': {
1244
- const sorted = [...values].sort((a, b) => a - b);
1245
- const middle = Math.floor(sorted.length / 2);
1246
- return sorted.length % 2 === 0 ? (sorted[middle - 1] + sorted[middle]) / 2 : sorted[middle];
1247
- }
1248
- case 'sum':
1249
- default:
1250
- return values.reduce((sum, value) => sum + value, 0);
1251
- }
1252
- }
1253
-
1254
- private static humanizeFieldName(key: string): string {
1255
- return key
1256
- .replace(/[_-]+/g, ' ')
1257
- .replace(/([a-z])([A-Z])/g, '$1 $2')
1258
- .replace(/\s+/g, ' ')
1259
- .trim()
1260
- .replace(/\b\w/g, char => char.toUpperCase());
1261
- }
1262
-
1263
- private static formatNumber(value: number): string {
1264
- return Number.isInteger(value) ? String(value) : value.toFixed(1);
1265
- }
1266
-
1267
- private static detectCategories(data: VectorMatch[]): string[] {
1268
- const categories = new Set<string>();
1269
- data.forEach(item => {
1270
- const category = this.getProductCategory(item);
1271
- if (category) categories.add(category);
1272
- });
1273
- return Array.from(categories);
1274
- }
1275
-
1276
- private static getProductCategory(item: VectorMatch): string | null {
1277
- if (!item) return null;
1278
- const meta = item.metadata || {};
1279
- const metadataCategory = resolveMetadataValue(meta, 'category');
1280
- if (this.isUsableCategory(metadataCategory)) return String(metadataCategory).trim();
1281
-
1282
- const content = item.content || '';
1283
- const explicitCategoryMatch = content.match(
1284
- /(?:^|\n)\s*(?:product\s*)?(?:category|department|collection)\s*[:\-–]\s*([^\n]+)/i,
1285
- );
1286
- if (explicitCategoryMatch && this.isUsableCategory(explicitCategoryMatch[1])) {
1287
- return explicitCategoryMatch[1].trim();
1288
- }
1289
-
1290
- return null;
1291
- }
1292
-
1293
- private static isUsableCategory(value: unknown): value is string | number {
1294
- if (value === null || value === undefined) return false;
1295
- const category = String(value).trim();
1296
- if (!category) return false;
1297
- return !/^(category|type|tag|price|cost|stock|availability|description|product|item|name|brand|sku|id|ean|currency|color|size|index|internal id|other|\[?type\]?|\[?products?_?\d*\]?)$/i.test(category);
1298
- }
1299
-
1300
- private static aggregateByCategory(
1301
- data: VectorMatch[],
1302
- categories: string[],
1303
- ): Record<string, number> {
1304
- const result: Record<string, number> = Object.fromEntries(categories.map(c => [c, 0]));
1305
- data.forEach(item => {
1306
- const cat = this.getProductCategory(item) ?? 'Other';
1307
- if (Object.prototype.hasOwnProperty.call(result, cat)) result[cat]++;
1308
- else result['Other'] = (result['Other'] || 0) + 1;
1309
- });
1310
- return result;
1311
- }
1312
-
1313
- private static extractTimeSeriesData(
1314
- data: VectorMatch[],
1315
- ): Array<{ timestamp: string | number; value: number; label?: string }> {
1316
- return data.map(item => {
1317
- const meta = item.metadata || {};
1318
- return {
1319
- timestamp: (meta.timestamp ?? meta.date ?? new Date().toISOString()) as string | number,
1320
- value: (meta.value ?? item.score ?? 0) as number,
1321
- label: (meta.label ?? (item?.content ?? '').substring(0, 50)) as string,
1322
- };
1323
- });
1324
- }
1325
-
1326
- private static extractNumericValue(meta: Record<string, unknown>): number | null {
1327
- const preferredKeys = [
1328
- 'value', 'count', 'total', 'average', 'avg', 'amount', 'sales',
1329
- 'revenue', 'score', 'quantity', 'price',
1330
- ];
1331
-
1332
- for (const key of preferredKeys) {
1333
- const raw = resolveMetadataValue(meta, key) ?? meta[key];
1334
- const value = typeof raw === 'number' ? raw : Number(String(raw ?? '').replace(/[$,% ,]/g, ''));
1335
- if (Number.isFinite(value)) return value;
1336
- }
1337
-
1338
- for (const value of Object.values(meta)) {
1339
- const numeric = typeof value === 'number' ? value : Number(String(value ?? '').replace(/[$,% ,]/g, ''));
1340
- if (Number.isFinite(numeric)) return numeric;
1341
- }
1342
-
1343
- return null;
1344
- }
1345
-
1346
- private static extractTableColumns(data: VectorMatch[], query = ''): string[] {
1347
- const q = query.toLowerCase();
1348
- const availableFields = this.extractAvailableTableFields(data);
1349
-
1350
- if (/\b(organization|organisations?|organizations?|company|companies)\b/.test(q) &&
1351
- /\b(employee|employees|staff|headcount|workforce)\b/.test(q)) {
1352
- const nameField = this.pickTableField(availableFields, [
1353
- 'company name',
1354
- 'organization name',
1355
- 'organisation name',
1356
- 'name',
1357
- 'company',
1358
- 'organization',
1359
- 'organisation',
1360
- ]);
1361
- const employeeField = this.pickTableField(availableFields, [
1362
- 'number of employees',
1363
- 'employee count',
1364
- 'employees',
1365
- 'employee_count',
1366
- 'number_employees',
1367
- 'num employees',
1368
- 'staff count',
1369
- 'headcount',
1370
- 'workforce',
1371
- ]);
1372
-
1373
- const columns = [nameField, employeeField].filter((field): field is string => Boolean(field));
1374
- if (columns.length > 0) return columns;
1375
- }
1376
-
1377
- const requestedFields = availableFields
1378
- .map(field => ({
1379
- field,
1380
- score: this.tableFieldQueryScore(field, q),
1381
- }))
1382
- .filter(item => item.score > 0)
1383
- .sort((a, b) => b.score - a.score)
1384
- .map(item => item.field);
1385
-
1386
- if (requestedFields.length > 0) {
1387
- return requestedFields.slice(0, Math.min(6, requestedFields.length));
1388
- }
1389
-
1390
- const metadataFields = Array.from(new Set(
1391
- data.flatMap(item => Object.keys(item.metadata || {}).map(k => this.humanizeFieldName(k))),
1392
- ));
1393
-
1394
- return metadataFields.length > 0 ? metadataFields : ['Content'];
1395
- }
1396
-
1397
- private static extractTableRow(
1398
- item: VectorMatch,
1399
- columns: string[],
1400
- ): (string | number | boolean)[] {
1401
- return columns.map(col => this.resolveTableCellValue(item, col));
1402
- }
1403
-
1404
- private static extractAvailableTableFields(data: VectorMatch[]): string[] {
1405
- const fields = new Map<string, string>();
1406
- const addField = (field: string) => {
1407
- const clean = this.humanizeFieldName(field);
1408
- if (!clean || /^(content|\[?type\]?)$/i.test(clean)) return;
1409
- const normalized = this.normalizeComparableField(clean);
1410
- if (!fields.has(normalized)) fields.set(normalized, clean);
1411
- };
1412
-
1413
- data.forEach(item => {
1414
- if (!item) return;
1415
- Object.keys(item.metadata || {}).forEach(addField);
1416
- for (const match of (item.content || '').matchAll(/(?:^|\n)\s*([A-Za-z][A-Za-z0-9_ /-]{1,50})\s*[:\-–]\s*([^\n]+)/g)) {
1417
- addField(match[1]);
1418
- }
1419
- });
1420
-
1421
- return Array.from(fields.values());
1422
- }
1423
-
1424
- private static pickTableField(fields: string[], aliases: string[]): string | undefined {
1425
- const normalizedAliases = aliases.map(alias => this.normalizeComparableField(alias));
1426
- for (const alias of normalizedAliases) {
1427
- const exact = fields.find(field => this.normalizeComparableField(field) === alias);
1428
- if (exact) return exact;
1429
- }
1430
-
1431
- for (const alias of normalizedAliases) {
1432
- const fuzzy = fields.find(field => {
1433
- const normalizedField = this.normalizeComparableField(field);
1434
- if (/(id|code|uuid)$/.test(normalizedField) && !/(id|code|uuid)$/.test(alias)) return false;
1435
- return normalizedField.includes(alias) || alias.includes(normalizedField);
1436
- });
1437
- if (fuzzy) return fuzzy;
1438
- }
1439
-
1440
- return undefined;
1441
- }
1442
-
1443
- private static tableFieldQueryScore(field: string, query: string): number {
1444
- const normalizedField = this.normalizeComparableField(field);
1445
- if (!normalizedField) return 0;
1446
- const fieldTokens = this.fieldTokens(field);
1447
- return fieldTokens.reduce((score, token) => {
1448
- if (token.length < 3) return score;
1449
- return query.includes(token) ? score + 1 : score;
1450
- }, query.includes(normalizedField) ? 3 : 0);
1451
- }
1452
-
1453
- private static resolveTableCellValue(item: VectorMatch, column: string): string | number | boolean {
1454
- if (column === 'Content') return (item?.content ?? '').substring(0, 100);
1455
-
1456
- const meta = item.metadata || {};
1457
- const normalizedColumn = this.normalizeComparableField(column);
1458
- const exactMetadata = Object.entries(meta).find(([key]) =>
1459
- this.normalizeComparableField(key) === normalizedColumn ||
1460
- this.normalizeComparableField(this.humanizeFieldName(key)) === normalizedColumn,
1461
- );
1462
- if (exactMetadata && exactMetadata[1] !== undefined && exactMetadata[1] !== null) {
1463
- return this.toDisplayValue(exactMetadata[1]);
1464
- }
1465
-
1466
- const aliasValue = this.resolveAliasedTableCell(meta, column);
1467
- if (aliasValue !== undefined) return this.toDisplayValue(aliasValue);
1468
-
1469
- const contentValue = this.extractContentFieldValue(item?.content ?? '', column);
1470
- if (contentValue !== null) return contentValue;
1471
-
1472
- const aliasContentValue = this.extractAliasedContentFieldValue(item?.content ?? '', column);
1473
- return aliasContentValue ?? '';
1474
- }
1475
-
1476
- private static resolveAliasedTableCell(meta: Record<string, unknown>, column: string): unknown {
1477
- const normalizedColumn = this.normalizeComparableField(column);
1478
- const aliases = normalizedColumn.includes('employee')
1479
- ? ['number of employees', 'employee count', 'employees', 'headcount', 'staff count', 'workforce']
1480
- : normalizedColumn.includes('name')
1481
- ? ['company name', 'organization name', 'organisation name', 'name', 'company', 'organization', 'organisation']
1482
- : [];
1483
-
1484
- for (const alias of aliases) {
1485
- const normalizedAlias = this.normalizeComparableField(alias);
1486
- const match = Object.entries(meta).find(([key]) => {
1487
- const normalizedKey = this.normalizeComparableField(key);
1488
- return normalizedKey === normalizedAlias ||
1489
- normalizedKey.includes(normalizedAlias) ||
1490
- normalizedAlias.includes(normalizedKey);
1491
- });
1492
- if (match && match[1] !== undefined && match[1] !== null) return match[1];
1493
- }
1494
-
1495
- return undefined;
1496
- }
1497
-
1498
- private static extractContentFieldValue(content: string, column: string): string | null {
1499
- const escaped = column.replace(/[.*+?^${}()|[\]\\]/g, '\\$&').replace(/\s+/g, '[\\s_ -]+');
1500
- const pattern = new RegExp(`(?:^|\\n)\\s*${escaped}\\s*[:\\-–]\\s*([^\\n]+)`, 'i');
1501
- const match = content.match(pattern);
1502
- return match ? match[1].trim() : null;
1503
- }
1504
-
1505
- private static extractAliasedContentFieldValue(content: string, column: string): string | null {
1506
- const normalizedColumn = this.normalizeComparableField(column);
1507
- const aliases = normalizedColumn.includes('employee')
1508
- ? ['Number Of Employees', 'Number of Employees', 'Employee Count', 'Employees', 'Headcount', 'Staff Count', 'Workforce']
1509
- : normalizedColumn.includes('name')
1510
- ? ['Company Name', 'Organization Name', 'Organisation Name', 'Name', 'Company', 'Organization', 'Organisation']
1511
- : [];
1512
-
1513
- for (const alias of aliases) {
1514
- const value = this.extractContentFieldValue(content, alias);
1515
- if (value !== null) return value;
1516
- }
1517
-
1518
- return null;
1519
- }
1520
-
1521
- private static toDisplayValue(value: unknown): string | number | boolean {
1522
- if (typeof value === 'string' || typeof value === 'number' || typeof value === 'boolean') return value;
1523
- return String(value ?? '');
1524
- }
1525
-
1526
- private static calculateStockCounts(
1527
- category: string,
1528
- data: VectorMatch[],
1529
- ): { inStockCount: number; outOfStockCount: number } {
1530
- let inStock = 0;
1531
- let outOfStock = 0;
1532
- data.forEach(d => {
1533
- const cat = this.getProductCategory(d) ?? 'Other';
1534
- if (cat === category) {
1535
- const quantity = this.extractStockQuantity(d) ?? 1;
1536
- if (this.determineStockStatus(d)) inStock += quantity;
1537
- else outOfStock += quantity;
1538
- }
1539
- });
1540
- return { inStockCount: inStock, outOfStockCount: outOfStock };
1541
- }
1542
-
1543
- private static determineStockStatus(item: VectorMatch): boolean {
1544
- if (!item) return true;
1545
- const meta = item.metadata || {};
1546
- const stockValue = resolveMetadataValue(meta, 'stock');
1547
- if (stockValue !== undefined) {
1548
- const normalized = String(stockValue).toLowerCase();
1549
- if (/out[_\s-]?of[_\s-]?stock|unavailable|false|no|sold out|0/.test(normalized)) return false;
1550
- if (/in[_\s-]?stock|available|true|yes/.test(normalized)) return true;
1551
- const numeric = this.toFiniteNumber(stockValue);
1552
- if (numeric !== null) return numeric > 0;
1553
- }
1554
- if (meta.inStock !== undefined) return Boolean(meta.inStock);
1555
- if (meta.stock !== undefined) return Boolean(meta.stock);
1556
- if (meta.available !== undefined) return Boolean(meta.available);
1557
-
1558
- const content = (item.content || '').toLowerCase();
1559
- if (/out[_\s-]?of[_\s-]?stock|unavailable|sold out/.test(content)) return false;
1560
- if (/in[_\s-]?stock|available/.test(content)) return true;
1561
- return true;
1562
- }
1563
-
1564
- private static extractStockQuantity(item: VectorMatch): number | null {
1565
- if (!item) return null;
1566
- const meta = item.metadata || {};
1567
- const stockValue = resolveMetadataValue(meta, 'stock');
1568
- const numericStock = this.toFiniteNumber(stockValue);
1569
- if (numericStock !== null) return numericStock;
1570
-
1571
- const content = item.content || '';
1572
- const stockMatch = content.match(/\b(?:stock|inventory|quantity|count)\s*[:\-–]?\s*([\d,]+)/i);
1573
- if (!stockMatch) return null;
1574
-
1575
- return this.toFiniteNumber(stockMatch[1]);
1576
- }
1577
-
1578
- private static toFiniteNumber(value: unknown): number | null {
1579
- if (typeof value === 'number') return Number.isFinite(value) ? value : null;
1580
- const numeric = Number(String(value ?? '').replace(/[$,% ,]/g, ''));
1581
- return Number.isFinite(numeric) ? numeric : null;
1582
- }
1583
-
1584
- // ─── Product Extraction ───────────────────────────────────────────────────
1585
-
1586
- private static getDynamicVal(
1587
- meta: Record<string, unknown> | undefined,
1588
- uiKey: string,
1589
- config?: import('../config/RagConfig').RagConfig,
1590
- trainedSchema?: import('./SchemaMapper').SchemaMap,
1591
- ): unknown {
1592
- if (!meta) return undefined;
1593
-
1594
- const mapping = config?.rag?.uiMapping;
1595
- if (mapping?.[uiKey] && meta[mapping[uiKey]] !== undefined) return meta[mapping[uiKey]];
1596
-
1597
- if (trainedSchema && typeof trainedSchema === 'object') {
1598
- const trainedKey = (trainedSchema as Record<string, unknown>)[uiKey];
1599
- if (trainedKey && meta[trainedKey as string] !== undefined) return meta[trainedKey as string];
1600
- }
1601
-
1602
- return resolveMetadataValue(meta, uiKey);
1603
- }
1604
-
1605
- private static extractProductInfo(
1606
- item: VectorMatch,
1607
- config?: import('../config/RagConfig').RagConfig,
1608
- trainedSchema?: import('./SchemaMapper').SchemaMap,
1609
- ): CarouselProduct | null {
1610
- if (!item) return null;
1611
- const meta = item.metadata || {};
1612
- const content = item.content || '';
1613
- const name = this.getDynamicVal(meta, 'name', config, trainedSchema);
1614
- const price = this.getDynamicVal(meta, 'price', config, trainedSchema);
1615
- const brand = this.getDynamicVal(meta, 'brand', config, trainedSchema);
1616
- const description = this.cleanProductDescription(
1617
- this.extractProductDescriptionFromContent(content) ??
1618
- this.getProductDescriptionValue(meta, config, trainedSchema),
1619
- );
1620
-
1621
- if (name || this.isProductData(item)) {
1622
- let finalName = name ? String(name) : undefined;
1623
- if (!finalName && content) {
1624
- const nameMatch = content.match(/(?:Product|Item|Name):\s*([^\n]+)/i);
1625
- finalName = nameMatch ? nameMatch[1].trim() : content.split('\n')[0]?.substring(0, 60);
1626
- }
1627
-
1628
- if (!finalName) {
1629
- console.warn(
1630
- `[UITransformer] Unable to read product name from metadata or content for item ${item.id}. Defaulting to 'Unnamed Product'.`,
1631
- { metadataKeys: Object.keys(meta), contentLength: content.length },
1632
- );
1633
- }
1634
-
1635
- let finalPrice: string | number | undefined =
1636
- typeof price === 'number' || typeof price === 'string' ? price : undefined;
1637
- if (!finalPrice && content) {
1638
- const priceMatch =
1639
- content.match(/(?:Price|Cost):\s*\$?([\d,.]+)/i) ||
1640
- content.match(/\$\s*([\d,.]+)/);
1641
- if (priceMatch) finalPrice = priceMatch[1].replace(/,/g, '');
1642
- }
1643
-
1644
- if (finalPrice === undefined) {
1645
- console.warn(
1646
- `[UITransformer] Unable to read product price from metadata or content for item ${item.id}.`,
1647
- { metadataKeys: Object.keys(meta), contentSnippet: content.substring(0, 100) },
1648
- );
1649
- }
1650
-
1651
- const imageValue = this.getDynamicVal(meta, 'image', config, trainedSchema);
1652
- if (!imageValue) {
1653
- console.debug(
1654
- `[UITransformer] Unable to read product image URL from metadata for item ${item.id}.`,
1655
- );
1656
- }
1657
-
1658
- if (!description) {
1659
- console.debug(
1660
- `[UITransformer] Unable to read product description from metadata or content for item ${item.id}.`,
1661
- );
1662
- }
1663
-
1664
- return {
1665
- id: item.id,
1666
- name: finalName || 'Unnamed Product',
1667
- price: finalPrice,
1668
- image: typeof imageValue === 'string' ? imageValue : undefined,
1669
- brand: brand ? String(brand) : undefined,
1670
- description,
1671
- inStock: this.determineStockStatus(item),
1672
- };
1673
- } else {
1674
- console.warn(
1675
- `[UITransformer] Item ${item.id} skipped: unable to read name or identify as product data from metadata/content.`,
1676
- { metadataKeys: Object.keys(meta), contentSnippet: content.substring(0, 100) },
1677
- );
1678
- }
1679
-
1680
- return null;
1681
- }
1682
-
1683
- private static extractProductDescriptionFromContent(content: string): string | null {
1684
- const bodyMatch = content.match(
1685
- /\bBody\s*\(HTML\)\s*:\s*([\s\S]*?)(?=\s*[,.]?\s+\b(?:Handle|Title|Vendor|Type|Tags|Published|Option\d*\s+Name|Option\d*\s+Value|Option|Variant|Image|SKU|Price|Status|Category)\b\s*:|$)/i,
1686
- );
1687
- if (bodyMatch?.[1]) return bodyMatch[1];
1688
-
1689
- const descriptionMatch = content.match(
1690
- /\b(?:Description|Summary|Details|Body|Content)\s*:\s*([\s\S]*?)(?=\s*[,.]?\s+\b(?:Handle|Title|Vendor|Type|Tags|Published|Option\d*\s+Name|Option\d*\s+Value|Option|Variant|Image|SKU|Price|Status|Category)\b\s*:|$)/i,
1691
- );
1692
- if (descriptionMatch?.[1]) return descriptionMatch[1];
1693
-
1694
- return null;
1695
- }
1696
-
1697
- private static getProductDescriptionValue(
1698
- meta: Record<string, unknown>,
1699
- config?: import('../config/RagConfig').RagConfig,
1700
- trainedSchema?: import('./SchemaMapper').SchemaMap,
1701
- ): unknown {
1702
- const mapped = this.getDynamicVal(meta, 'description', config, trainedSchema);
1703
- if (mapped !== undefined && mapped !== meta.content && mapped !== meta.text) return mapped;
1704
-
1705
- const preferredKeys = [
1706
- 'body_html',
1707
- 'body html',
1708
- 'bodyHtml',
1709
- 'description',
1710
- 'summary',
1711
- 'details',
1712
- 'body',
1713
- ];
1714
-
1715
- for (const key of preferredKeys) {
1716
- const match = Object.keys(meta).find(candidate =>
1717
- this.normalizeComparableField(candidate) === this.normalizeComparableField(key),
1718
- );
1719
- if (match && meta[match] !== undefined && meta[match] !== null) return meta[match];
1720
- }
1721
-
1722
- return undefined;
1723
- }
1724
-
1725
- private static cleanProductDescription(raw: unknown): string | undefined {
1726
- if (raw === null || raw === undefined) return undefined;
1727
-
1728
- const extracted = this.extractProductDescriptionFromContent(String(raw));
1729
- if (extracted && extracted !== String(raw)) return this.cleanProductDescription(extracted);
1730
-
1731
- const text = String(raw)
1732
- .replace(/<[^>]*>/g, ' ')
1733
- .replace(/&nbsp;/gi, ' ')
1734
- .replace(/&amp;/gi, '&')
1735
- .replace(/&quot;/gi, '"')
1736
- .replace(/&#39;/gi, "'")
1737
- .replace(/\[[^\]]*TYPE[^\]]*\]/gi, ' ')
1738
- .replace(/\b(?:Handle|Title|Vendor|Type|Tags|Published|Option\d*\s+Name|Option\d*\s+Value|Option|Variant|Image|SKU|Price|Status|Category)\s*:\s*[^,.\n:]+[,.]?/gi, ' ')
1739
- .replace(/\bBody\s*\(HTML\)\s*:\s*/gi, ' ')
1740
- .replace(/\b(?:Description|Summary|Details|Body|Content)\s*:\s*/gi, ' ')
1741
- .replace(/\s+/g, ' ')
1742
- .trim();
1743
-
1744
- return text || undefined;
1745
- }
1746
-
1747
- // ─── Visualization Prompt ─────────────────────────────────────────────────
1748
-
1749
- private static buildVisualizationSystemPrompt(): string {
1750
- return `You are a data visualization expert embedded in a RAG chat system.
1751
- You will receive a user question, a pre-computed INTENT object, and structured data from a vector database.
1752
- Your ONLY job is to return a single JSON object describing how to visualize the data.
1753
- The INTENT object is authoritative — honor it. For example:
1754
- - If intent.wantsExplicitTable is false, never return type "table".
1755
- - If intent.isTemporal is true, prefer line_chart.
1756
- - If intent.visualizationHint is "comparison" or "ranking", prefer bar_chart.
1757
- - If intent.visualizationHint is "composition", prefer pie_chart.
1758
- - If intent.recommendedChart is unsupported by the response schema, use the nearest supported type.
1759
- - Always respond in the language specified by intent.language.
1760
-
1761
- Return ONLY a valid JSON object — no markdown fences, no prose.
1762
-
1763
- {
1764
- "type": "bar_chart" | "horizontal_bar" | "line_chart" | "pie_chart" | "histogram" | "scatter_plot" | "radar_chart" | "metric_card" | "geo_map" | "table" | "text",
1765
- "title": "concise descriptive title",
1766
- "description": "one sentence describing the visualization",
1767
- "data": <structured data per type below>
1768
- }
1769
-
1770
- DATA SHAPES:
1771
- - bar_chart: [{ "category": string, "value": number }]
1772
- - horizontal_bar: [{ "category": string, "value": number }]
1773
- - histogram: [{ "category": string, "value": number }]
1774
- - line_chart: [{ "timestamp": string, "value": number, "label": string }]
1775
- - pie_chart: [{ "label": string, "value": number }]
1776
- - scatter_plot:[{ "x": number, "y": number, "label": string }]
1777
- - radar_chart: [{ "attribute": string, "<series1>": number, "<series2>": number, ... }]
1778
- - metric_card: { "label": string, "value": number, "operation": "sum"|"average"|"count"|"min"|"max"|"median" }
1779
- - geo_map: [{ "category": string, "value": number }]
1780
- - table: { "columns": string[], "rows": (string|number)[][] }
1781
- - text: { "content": "A concise plain-language answer." }
1782
-
1783
- RULES:
1784
- 1. Aggregate values — never pass raw nested objects in chart arrays.
1785
- 2. All "value" fields must be numbers.
1786
- 3. Cap bar/line/pie at 12 data points.
1787
- 4. Use dollar signs only for monetary prices, not for stock quantities or years.
1788
- 5. If no relevant data is found, return text: "I cannot answer this question based on the available information."`;
1789
- }
1790
-
1791
- private static buildContextSummary(sources: VectorMatch[], maxChars = 6000): string {
1792
- const items = (sources || []).filter(Boolean).map((s, i) => ({
1793
- index: i + 1,
1794
- content: s?.content?.substring(0, 400) ?? '',
1795
- metadata: s?.metadata ?? {},
1796
- score: s?.score ?? 0,
1797
- }));
1798
-
1799
- const full = JSON.stringify(items, null, 2);
1800
- if (full.length <= maxChars) return full;
1801
-
1802
- const partial: typeof items = [];
1803
- let chars = 2;
1804
- for (const item of items) {
1805
- const chunk = JSON.stringify(item);
1806
- if (chars + chunk.length + 2 > maxChars) break;
1807
- partial.push(item);
1808
- chars += chunk.length + 2;
1809
- }
1810
- return JSON.stringify(partial, null, 2) + '\n// ... truncated';
1811
- }
1812
- }