@retrivora-ai/rag-engine 2.4.3 → 2.4.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{BatchProcessor-zbQMaZHm.d.mts → BatchProcessor-Csy_K1qQ.d.mts} +2 -0
- package/dist/{BatchProcessor-D42E706a.d.ts → BatchProcessor-Dg18zZGk.d.ts} +2 -0
- package/dist/handlers/index.js +27 -4
- package/dist/handlers/index.mjs +27 -4
- package/dist/index.d.mts +3 -3
- package/dist/index.d.ts +3 -3
- package/dist/index.js +12 -10
- package/dist/index.mjs +12 -10
- package/dist/server.d.mts +2 -2
- package/dist/server.d.ts +2 -2
- package/dist/server.js +28 -4
- package/dist/server.mjs +28 -4
- package/package.json +1 -3
- package/src/LicensedRetrivora.ts +0 -278
- package/src/app/api/chat/route.ts +0 -4
- package/src/app/api/health/route.ts +0 -5
- package/src/app/api/ingest/route.ts +0 -27
- package/src/app/api/suggestions/route.ts +0 -5
- package/src/app/api/upload/route.ts +0 -5
- package/src/app/constants.tsx +0 -270
- package/src/app/favicon.ico +0 -0
- package/src/app/globals.css +0 -449
- package/src/app/layout.tsx +0 -22
- package/src/app/page.tsx +0 -64
- package/src/app/types.ts +0 -30
- package/src/browser.tsx +0 -127
- package/src/components/ChatWidget.tsx +0 -393
- package/src/components/ChatWindow.tsx +0 -952
- package/src/components/CodeViewer.tsx +0 -56
- package/src/components/ConfigProvider.tsx +0 -141
- package/src/components/DocumentUpload.tsx +0 -354
- package/src/components/DynamicChart.tsx +0 -297
- package/src/components/HourglassLoader.tsx +0 -87
- package/src/components/MarkdownComponents.tsx +0 -186
- package/src/components/MessageBubble.tsx +0 -498
- package/src/components/ObservabilityPanel.tsx +0 -374
- package/src/components/ProductCard.tsx +0 -107
- package/src/components/ProductCarousel.tsx +0 -97
- package/src/components/SourceCard.tsx +0 -61
- package/src/components/ThemeProvider.tsx +0 -8
- package/src/components/ThemeToggle.tsx +0 -29
- package/src/components/ThinkingBlock.tsx +0 -75
- package/src/components/UIDispatcher.tsx +0 -342
- package/src/components/VisualizationRenderer.tsx +0 -621
- package/src/components/constants.tsx +0 -503
- package/src/config/ConfigBuilder.ts +0 -285
- package/src/config/EmbeddingStrategy.ts +0 -148
- package/src/config/LicenseConfigResolver.ts +0 -298
- package/src/config/RagConfig.ts +0 -328
- package/src/config/UniversalProfiles.ts +0 -82
- package/src/config/constants.ts +0 -64
- package/src/config/serverConfig.ts +0 -378
- package/src/config/uiConstants.ts +0 -47
- package/src/core/BatchProcessor.ts +0 -385
- package/src/core/CircuitBreaker.ts +0 -118
- package/src/core/ConfigFetcher.ts +0 -220
- package/src/core/ConfigResolver.ts +0 -165
- package/src/core/ConfigValidator.ts +0 -197
- package/src/core/DatabaseStorage.ts +0 -526
- package/src/core/FreeTierLimitsGuard.ts +0 -281
- package/src/core/LLMRouter.ts +0 -79
- package/src/core/LangChainAgent.ts +0 -132
- package/src/core/LicenseValidator.ts +0 -284
- package/src/core/LicenseVerifier.ts +0 -261
- package/src/core/MultiAgentCoordinator.ts +0 -239
- package/src/core/Pipeline.ts +0 -1279
- package/src/core/PluginManager.ts +0 -277
- package/src/core/ProjectSettings.ts +0 -247
- package/src/core/ProviderHealthCheck.ts +0 -133
- package/src/core/ProviderInterfaces.ts +0 -37
- package/src/core/ProviderRegistry.ts +0 -156
- package/src/core/QueryProcessor.ts +0 -399
- package/src/core/Retrivora.ts +0 -98
- package/src/core/VectorPlugin.ts +0 -193
- package/src/core/mcp.ts +0 -261
- package/src/exceptions/index.ts +0 -131
- package/src/handlers/index.ts +0 -1625
- package/src/hooks/useHydrated.ts +0 -13
- package/src/hooks/useProjectSettings.ts +0 -105
- package/src/hooks/useRagChat.ts +0 -436
- package/src/hooks/useStoredMessages.ts +0 -64
- package/src/index.css +0 -4269
- package/src/index.ts +0 -137
- package/src/lib/plugin.ts +0 -24
- package/src/llm/ILLMProvider.ts +0 -60
- package/src/llm/LLMFactory.ts +0 -141
- package/src/llm/providers/AnthropicProvider.ts +0 -215
- package/src/llm/providers/GeminiProvider.ts +0 -311
- package/src/llm/providers/GroqProvider.ts +0 -184
- package/src/llm/providers/OllamaProvider.ts +0 -232
- package/src/llm/providers/OpenAIProvider.ts +0 -188
- package/src/llm/providers/QwenProvider.ts +0 -199
- package/src/llm/providers/UniversalLLMAdapter.ts +0 -422
- package/src/llm/utils.ts +0 -46
- package/src/providers/graphdb/BaseGraphProvider.ts +0 -43
- package/src/providers/graphdb/SimpleGraphProvider.ts +0 -66
- package/src/providers/vectordb/BaseVectorProvider.ts +0 -98
- package/src/providers/vectordb/ChromaDBProvider.ts +0 -130
- package/src/providers/vectordb/MilvusProvider.ts +0 -134
- package/src/providers/vectordb/MongoDBProvider.ts +0 -250
- package/src/providers/vectordb/MultiTablePostgresProvider.ts +0 -461
- package/src/providers/vectordb/PineconeProvider.ts +0 -295
- package/src/providers/vectordb/PostgreSQLProvider.ts +0 -227
- package/src/providers/vectordb/QdrantProvider.ts +0 -294
- package/src/providers/vectordb/RedisProvider.ts +0 -105
- package/src/providers/vectordb/UniversalVectorProvider.ts +0 -221
- package/src/providers/vectordb/WeaviateProvider.ts +0 -172
- package/src/rag/DocumentChunker.ts +0 -144
- package/src/rag/EntityExtractor.ts +0 -90
- package/src/rag/LlamaIndexIngestor.ts +0 -60
- package/src/rag/Reranker.ts +0 -106
- package/src/rendering/IntentClassifier.ts +0 -167
- package/src/rendering/RendererRegistry.ts +0 -164
- package/src/rendering/RuleEngine.ts +0 -67
- package/src/rendering/VisualizationDecisionEngine.ts +0 -164
- package/src/rendering/__tests__/VisualizationDecisionEngine.test.ts +0 -237
- package/src/rendering/index.ts +0 -12
- package/src/rendering/rules/Rule1SpecificInfoRule.ts +0 -28
- package/src/rendering/rules/Rule2ComparisonRule.ts +0 -24
- package/src/rendering/rules/Rule3ProductDiscoveryRule.ts +0 -28
- package/src/rendering/rules/Rule4AnalyticalRule.ts +0 -38
- package/src/rendering/rules/Rule5MixedResponseRule.ts +0 -29
- package/src/rendering/rules/Rule6SmallResultSetRule.ts +0 -24
- package/src/rendering/rules/Rule7LargeTableRule.ts +0 -35
- package/src/rendering/types.ts +0 -96
- package/src/server.ts +0 -156
- package/src/tailwind.css +0 -194
- package/src/telemetry/TelemetryService.ts +0 -330
- package/src/types/chat.ts +0 -89
- package/src/types/index.ts +0 -278
- package/src/types/pdf-parse.d.ts +0 -13
- package/src/types/props.ts +0 -169
- package/src/utils/DocumentParser.ts +0 -111
- package/src/utils/ProductExtractor.ts +0 -350
- package/src/utils/SchemaMapper.ts +0 -176
- package/src/utils/UITransformer.ts +0 -1812
- package/src/utils/synonyms.ts +0 -80
- package/src/utils/templateUtils.ts +0 -78
- package/src/version.ts +0 -6
|
@@ -1,350 +0,0 @@
|
|
|
1
|
-
import { Product, VectorMatch } from '../types';
|
|
2
|
-
import { resolveMetadataValue } from './synonyms';
|
|
3
|
-
|
|
4
|
-
// ─── JSON sanitization ────────────────────────────────────────────────────────
|
|
5
|
-
|
|
6
|
-
export function sanitizeJson(raw: string): string {
|
|
7
|
-
let s = raw.trim();
|
|
8
|
-
|
|
9
|
-
// 1. Evaluate simple arithmetic expressions in values e.g. "v": 495 / 1000
|
|
10
|
-
s = s.replace(
|
|
11
|
-
/:\s*(-?\d+(?:\.\d+)?)\s*([+\-*/])\s*(-?\d+(?:\.\d+)?)/g,
|
|
12
|
-
(_, a, op, b) => {
|
|
13
|
-
const n1 = parseFloat(a);
|
|
14
|
-
const n2 = parseFloat(b);
|
|
15
|
-
const result =
|
|
16
|
-
op === '+' ? n1 + n2
|
|
17
|
-
: op === '-' ? n1 - n2
|
|
18
|
-
: op === '*' ? n1 * n2
|
|
19
|
-
: n2 !== 0 ? n1 / n2
|
|
20
|
-
: n1;
|
|
21
|
-
return `: ${result}`;
|
|
22
|
-
},
|
|
23
|
-
);
|
|
24
|
-
|
|
25
|
-
// 2. Strip JS-style comments while preserving strings (e.g. URLs)
|
|
26
|
-
s = s.replace(/("(?:\\.|[^\\"])*")|\/\/[^\n]*|\/\*[\s\S]*?\*\//g, (m, g1) => g1 || '');
|
|
27
|
-
|
|
28
|
-
// 3. Quote bare (unquoted) object keys
|
|
29
|
-
s = s.replace(/([{,]\s*)([A-Za-z_$][A-Za-z0-9_$]*)\s*:/g, '$1"$2":');
|
|
30
|
-
|
|
31
|
-
// 4. Replace single-quoted strings with double-quoted ones
|
|
32
|
-
s = s.replace(/'([^'\\]*(?:\\.[^'\\]*)*)'/g, '"$1"');
|
|
33
|
-
|
|
34
|
-
// 5. Remove trailing commas before ] or }
|
|
35
|
-
s = s.replace(/,\s*([}\]])/g, '$1');
|
|
36
|
-
|
|
37
|
-
// 6. Remove completely empty key-value pairs left by prior cleanup
|
|
38
|
-
s = s.replace(/([{,])\s*""\s*(?=[,}])/g, '$1');
|
|
39
|
-
|
|
40
|
-
// 7. Stack-based structure balancer.
|
|
41
|
-
const stack: ('{' | '[')[] = [];
|
|
42
|
-
let inString = false;
|
|
43
|
-
let escape = false;
|
|
44
|
-
|
|
45
|
-
for (const ch of s) {
|
|
46
|
-
if (escape) { escape = false; continue; }
|
|
47
|
-
if (ch === '\\' && inString) { escape = true; continue; }
|
|
48
|
-
if (ch === '"') { inString = !inString; continue; }
|
|
49
|
-
if (inString) continue;
|
|
50
|
-
|
|
51
|
-
if (ch === '{' || ch === '[') {
|
|
52
|
-
stack.push(ch);
|
|
53
|
-
} else if (ch === '}') {
|
|
54
|
-
if (stack[stack.length - 1] === '{') stack.pop();
|
|
55
|
-
} else if (ch === ']') {
|
|
56
|
-
if (stack[stack.length - 1] === '[') stack.pop();
|
|
57
|
-
}
|
|
58
|
-
}
|
|
59
|
-
|
|
60
|
-
if (inString) s += '"';
|
|
61
|
-
for (let i = stack.length - 1; i >= 0; i--) {
|
|
62
|
-
s += stack[i] === '{' ? '}' : ']';
|
|
63
|
-
}
|
|
64
|
-
|
|
65
|
-
const lastClose = Math.max(s.lastIndexOf('}'), s.lastIndexOf(']'));
|
|
66
|
-
if (lastClose !== -1 && lastClose < s.length - 1 && /\S/.test(s.slice(lastClose + 1))) {
|
|
67
|
-
s = s.substring(0, lastClose + 1);
|
|
68
|
-
}
|
|
69
|
-
|
|
70
|
-
return s;
|
|
71
|
-
}
|
|
72
|
-
|
|
73
|
-
export function extractLikelyJsonBlock(raw: string): string {
|
|
74
|
-
const trimmed = raw.trim();
|
|
75
|
-
const objectStart = trimmed.indexOf('{');
|
|
76
|
-
const arrayStart = trimmed.indexOf('[');
|
|
77
|
-
|
|
78
|
-
const starts = [objectStart, arrayStart].filter((index) => index >= 0);
|
|
79
|
-
if (starts.length === 0) return trimmed;
|
|
80
|
-
|
|
81
|
-
return trimmed.slice(Math.min(...starts));
|
|
82
|
-
}
|
|
83
|
-
|
|
84
|
-
export function looksLikeStructuredPayload(raw: string): boolean {
|
|
85
|
-
return /"?(view|chartType|type|data|items|rows|products|results)"?\s*:/.test(raw);
|
|
86
|
-
}
|
|
87
|
-
|
|
88
|
-
export function stripMarkdownTables(raw: string): string {
|
|
89
|
-
return raw.replace(
|
|
90
|
-
/(?:^|\n)\|[^\n]+\|\n\|(?:\s*:?-+:?\s*\|)+\n(?:\|[^\n]*\|\n?)*/g,
|
|
91
|
-
'\n\n',
|
|
92
|
-
);
|
|
93
|
-
}
|
|
94
|
-
|
|
95
|
-
export function stripStructuredUiLabels(raw: string): string {
|
|
96
|
-
return raw.replace(
|
|
97
|
-
/(?:^|\n)\*\*(?:Chart Data|Table Data|Visualization|Visual Representation)\*\:\s*(?=\n|$)/gi,
|
|
98
|
-
'\n',
|
|
99
|
-
).replace(
|
|
100
|
-
/(?:^|\n)(?:Chart Data|Table Data|Visualization|Visual Representation)\s*\:\s*(?=\n|$)/gi,
|
|
101
|
-
'\n',
|
|
102
|
-
);
|
|
103
|
-
}
|
|
104
|
-
|
|
105
|
-
function extractBareJsonObject(raw: string): string | null {
|
|
106
|
-
let start = raw.indexOf('{');
|
|
107
|
-
if (start === -1) return null;
|
|
108
|
-
|
|
109
|
-
let bestJson: string | null = null;
|
|
110
|
-
let maxLen = 0;
|
|
111
|
-
|
|
112
|
-
while (start !== -1) {
|
|
113
|
-
let depth = 0;
|
|
114
|
-
let inString = false;
|
|
115
|
-
let escape = false;
|
|
116
|
-
|
|
117
|
-
for (let i = start; i < raw.length; i += 1) {
|
|
118
|
-
const ch = raw[i];
|
|
119
|
-
if (escape) { escape = false; continue; }
|
|
120
|
-
if (ch === '\\' && inString) { escape = true; continue; }
|
|
121
|
-
if (ch === '"') { inString = !inString; continue; }
|
|
122
|
-
if (inString) continue;
|
|
123
|
-
|
|
124
|
-
if (ch === '{') depth += 1;
|
|
125
|
-
else if (ch === '}') {
|
|
126
|
-
depth -= 1;
|
|
127
|
-
if (depth === 0) {
|
|
128
|
-
const candidate = raw.slice(start, i + 1);
|
|
129
|
-
if (candidate.length > maxLen) {
|
|
130
|
-
maxLen = candidate.length;
|
|
131
|
-
bestJson = candidate;
|
|
132
|
-
}
|
|
133
|
-
break;
|
|
134
|
-
}
|
|
135
|
-
}
|
|
136
|
-
}
|
|
137
|
-
start = raw.indexOf('{', start + 1);
|
|
138
|
-
}
|
|
139
|
-
|
|
140
|
-
return bestJson;
|
|
141
|
-
}
|
|
142
|
-
|
|
143
|
-
export function extractStructuredPayload(raw: string): { payload: string | null; text: string } {
|
|
144
|
-
let working = raw;
|
|
145
|
-
const fenceRegex = /```(?:[a-z]+)?\s*([\s\S]*?)```/gi;
|
|
146
|
-
|
|
147
|
-
for (const match of Array.from(raw.matchAll(fenceRegex))) {
|
|
148
|
-
const body = match[1]?.trim() ?? '';
|
|
149
|
-
if (!looksLikeStructuredPayload(body)) continue;
|
|
150
|
-
|
|
151
|
-
working = working.replace(match[0], '');
|
|
152
|
-
return {
|
|
153
|
-
payload: body,
|
|
154
|
-
text: stripStructuredUiLabels(working).replace(/\n{3,}/g, '\n\n').trim(),
|
|
155
|
-
};
|
|
156
|
-
}
|
|
157
|
-
|
|
158
|
-
const bareJson = extractBareJsonObject(raw);
|
|
159
|
-
if (bareJson && looksLikeStructuredPayload(bareJson)) {
|
|
160
|
-
working = working.replace(bareJson, '');
|
|
161
|
-
return {
|
|
162
|
-
payload: bareJson,
|
|
163
|
-
text: stripStructuredUiLabels(working).replace(/\n{3,}/g, '\n\n').trim(),
|
|
164
|
-
};
|
|
165
|
-
}
|
|
166
|
-
|
|
167
|
-
return {
|
|
168
|
-
payload: null,
|
|
169
|
-
text: stripStructuredUiLabels(raw).replace(/\n{3,}/g, '\n\n').trim(),
|
|
170
|
-
};
|
|
171
|
-
}
|
|
172
|
-
|
|
173
|
-
export function resolveImage(data: Record<string, unknown>): string | undefined {
|
|
174
|
-
const isPlaceholder = (val: unknown) =>
|
|
175
|
-
typeof val === 'string' && (val === '...' || val === '' || val.toLowerCase() === 'null' || val.toLowerCase() === 'undefined');
|
|
176
|
-
|
|
177
|
-
for (const key of ['image', 'img', 'thumbnail', 'images', 'product_image', 'Image', 'Thumbnail']) {
|
|
178
|
-
const v = data[key];
|
|
179
|
-
if (typeof v === 'string' && v && !isPlaceholder(v)) return v;
|
|
180
|
-
if (Array.isArray(v) && typeof v[0] === 'string' && v[0] && !isPlaceholder(v[0])) return v[0];
|
|
181
|
-
}
|
|
182
|
-
return undefined;
|
|
183
|
-
}
|
|
184
|
-
|
|
185
|
-
export function isLikelyUiOnlyMessage(text: string): boolean {
|
|
186
|
-
const trimmed = text.trim();
|
|
187
|
-
return (
|
|
188
|
-
trimmed.length === 0 ||
|
|
189
|
-
trimmed.length < 30 ||
|
|
190
|
-
/^(chart data|table data|visualization|visual representation|product catalog|product recommendations|catalog summary|availability|inventory status|details)\s*:?\s*$/i.test(trimmed)
|
|
191
|
-
);
|
|
192
|
-
}
|
|
193
|
-
|
|
194
|
-
// ─── Extractors ───────────────────────────────────────────────────────────────
|
|
195
|
-
|
|
196
|
-
export function extractProductsFromSources(sources: VectorMatch[] | undefined, isUser: boolean): Product[] {
|
|
197
|
-
if (isUser || !sources) return [];
|
|
198
|
-
return sources
|
|
199
|
-
.filter((s): s is VectorMatch => Boolean(s && typeof s === 'object'))
|
|
200
|
-
.filter(s => {
|
|
201
|
-
const m = s.metadata ?? {};
|
|
202
|
-
const keys = Object.keys(m).map(k => k.toLowerCase());
|
|
203
|
-
const hasStrongProductKey = keys.some(k =>
|
|
204
|
-
['price', 'image', 'img', 'thumbnail', 'images', 'product', 'sku', 'cost'].includes(k)
|
|
205
|
-
);
|
|
206
|
-
const hasProductIdentity = keys.some(k => ['brand', 'model'].includes(k)) &&
|
|
207
|
-
keys.some(k => ['name', 'title', 'product_name', 'product'].includes(k));
|
|
208
|
-
const hasPricePattern = /\$\s*\d+/.test(s?.content ?? '');
|
|
209
|
-
return hasStrongProductKey || hasProductIdentity || hasPricePattern || m.type === 'product';
|
|
210
|
-
})
|
|
211
|
-
.map(s => {
|
|
212
|
-
const m = (s.metadata ?? {}) as Record<string, unknown>;
|
|
213
|
-
const name = resolveMetadataValue(m, 'name') as string;
|
|
214
|
-
const brand = resolveMetadataValue(m, 'brand') as string;
|
|
215
|
-
const price = resolveMetadataValue(m, 'price') as string | number;
|
|
216
|
-
const description = resolveMetadataValue(m, 'description') as string;
|
|
217
|
-
|
|
218
|
-
return {
|
|
219
|
-
id: s?.id ?? `prod_${Math.random().toString(36).substring(2, 8)}`,
|
|
220
|
-
name: name ?? (s?.content ?? '').split('\n')[0] ?? 'Unknown Product',
|
|
221
|
-
brand: brand,
|
|
222
|
-
price: price,
|
|
223
|
-
image: resolveImage(m),
|
|
224
|
-
link: resolveMetadataValue(m, 'link') as string,
|
|
225
|
-
description: description ? description : '',
|
|
226
|
-
};
|
|
227
|
-
});
|
|
228
|
-
}
|
|
229
|
-
|
|
230
|
-
export function extractProductsFromContent(
|
|
231
|
-
content: string,
|
|
232
|
-
payloadText: string,
|
|
233
|
-
payloadCandidates: string[],
|
|
234
|
-
productsFromSources: Product[],
|
|
235
|
-
isStreaming: boolean = false
|
|
236
|
-
): { productsFromContent: Product[]; cleanContent: string } {
|
|
237
|
-
const jsonRegex = /```(?:json|ui)?\s*([\s\S]*?)\s*```/g;
|
|
238
|
-
const products: Product[] = [];
|
|
239
|
-
let cleanContent = payloadText;
|
|
240
|
-
|
|
241
|
-
for (const candidate of payloadCandidates) {
|
|
242
|
-
try {
|
|
243
|
-
const data = JSON.parse(sanitizeJson(candidate));
|
|
244
|
-
const itemSet =
|
|
245
|
-
Array.isArray(data.data) ? data.data
|
|
246
|
-
: Array.isArray(data.items) ? data.items
|
|
247
|
-
: Array.isArray(data.products) ? data.products
|
|
248
|
-
: null;
|
|
249
|
-
|
|
250
|
-
if (
|
|
251
|
-
(data.type === 'products' || data.view === 'carousel' || data.view === 'table' || Array.isArray(data.products)) &&
|
|
252
|
-
itemSet
|
|
253
|
-
) {
|
|
254
|
-
products.push(
|
|
255
|
-
...itemSet.map((item: Record<string, unknown>) => ({
|
|
256
|
-
...item,
|
|
257
|
-
image: (item.image as string) ?? resolveImage(item),
|
|
258
|
-
})) as Product[],
|
|
259
|
-
);
|
|
260
|
-
}
|
|
261
|
-
} catch {
|
|
262
|
-
// Ignore malformed product payloads
|
|
263
|
-
}
|
|
264
|
-
}
|
|
265
|
-
|
|
266
|
-
if (cleanContent.includes('"type": "products"') || cleanContent.includes('"type":"products"')) {
|
|
267
|
-
for (const match of Array.from(cleanContent.matchAll(jsonRegex))) {
|
|
268
|
-
try {
|
|
269
|
-
const data = JSON.parse(sanitizeJson(match[1]));
|
|
270
|
-
if (data.type === 'products' && Array.isArray(data.items)) {
|
|
271
|
-
products.push(
|
|
272
|
-
...data.items.map((item: Record<string, unknown>) => ({
|
|
273
|
-
...item,
|
|
274
|
-
image: (item.image as string) ?? resolveImage(item),
|
|
275
|
-
})) as Product[],
|
|
276
|
-
);
|
|
277
|
-
cleanContent = cleanContent.replace(match[0], '');
|
|
278
|
-
}
|
|
279
|
-
} catch {
|
|
280
|
-
// Malformed product block — skip silently
|
|
281
|
-
}
|
|
282
|
-
}
|
|
283
|
-
}
|
|
284
|
-
|
|
285
|
-
// Fallback: Scan for plain text bulleted products
|
|
286
|
-
// Requires a bullet or number at the start of the line to prevent matching plain text paragraphs.
|
|
287
|
-
const bulletRegex = /^[ \t]*(?:[•*-]|\d+\.)\s+(?:\*\*)?([^\n\-\$–—\(]+?)(?:\*\*)?(?:\s*\(?Price\s*[:\-–—]?\s*)?(?:\s*[:\-–—]\s*|\s+)\$?([\d,.]+)(?:\s*USD)?/gim;
|
|
288
|
-
const matches = Array.from(cleanContent.matchAll(bulletRegex));
|
|
289
|
-
|
|
290
|
-
if (!isStreaming && matches.length >= 2) {
|
|
291
|
-
const normalize = (s: string) => s.toLowerCase().replace(/[^a-z0-9]/g, '');
|
|
292
|
-
|
|
293
|
-
for (const match of matches) {
|
|
294
|
-
let name = match[1]?.trim() || '';
|
|
295
|
-
name = name.replace(/\s*\(?Price\s*$/i, '').replace(/[:\-–—]\s*$/g, '').trim();
|
|
296
|
-
name = name.replace(/^\*\*|\*\*$/g, '').replace(/^\*|\*$/g, '').trim();
|
|
297
|
-
|
|
298
|
-
let price = match[2]?.trim() || '';
|
|
299
|
-
price = price.replace(/[,\s]+$/, '').trim();
|
|
300
|
-
|
|
301
|
-
if (name && price) {
|
|
302
|
-
const newProduct: Product = {
|
|
303
|
-
id: `text-prod-${name}-${price}`,
|
|
304
|
-
name,
|
|
305
|
-
price: `$${price}`,
|
|
306
|
-
};
|
|
307
|
-
|
|
308
|
-
const normName = normalize(name);
|
|
309
|
-
const sourceMatch = productsFromSources.find(s => {
|
|
310
|
-
const sn = normalize(s.name);
|
|
311
|
-
return sn.includes(normName) || normName.includes(sn);
|
|
312
|
-
});
|
|
313
|
-
|
|
314
|
-
if (sourceMatch) {
|
|
315
|
-
newProduct.image = sourceMatch.image;
|
|
316
|
-
newProduct.brand = sourceMatch.brand;
|
|
317
|
-
newProduct.link = sourceMatch.link;
|
|
318
|
-
newProduct.description = sourceMatch.description;
|
|
319
|
-
}
|
|
320
|
-
|
|
321
|
-
products.push(newProduct);
|
|
322
|
-
cleanContent = cleanContent.replace(match[0], `\n**${name}** — $${price}`);
|
|
323
|
-
}
|
|
324
|
-
}
|
|
325
|
-
cleanContent = cleanContent.replace(/\n{3,}/g, '\n\n').trim();
|
|
326
|
-
}
|
|
327
|
-
|
|
328
|
-
return { productsFromContent: products, cleanContent: cleanContent.trim() };
|
|
329
|
-
}
|
|
330
|
-
|
|
331
|
-
export function deduplicateProducts(
|
|
332
|
-
productsFromSources: Product[],
|
|
333
|
-
productsFromContent: Product[],
|
|
334
|
-
messageContent: string
|
|
335
|
-
): Product[] {
|
|
336
|
-
if (productsFromContent.length > 0) return productsFromContent;
|
|
337
|
-
|
|
338
|
-
const seen = new Set<string>();
|
|
339
|
-
const contentLower = messageContent.toLowerCase();
|
|
340
|
-
|
|
341
|
-
return productsFromSources.filter(p => {
|
|
342
|
-
const id = String(p.id ?? p.name);
|
|
343
|
-
if (seen.has(id)) return false;
|
|
344
|
-
const mentioned =
|
|
345
|
-
contentLower.includes(p.name.toLowerCase()) ||
|
|
346
|
-
(p.brand ? contentLower.includes(p.brand.toLowerCase()) : false);
|
|
347
|
-
if (mentioned) { seen.add(id); return true; }
|
|
348
|
-
return false;
|
|
349
|
-
});
|
|
350
|
-
}
|
|
@@ -1,176 +0,0 @@
|
|
|
1
|
-
import { ILLMProvider } from '../llm/ILLMProvider';
|
|
2
|
-
|
|
3
|
-
export interface SchemaMap {
|
|
4
|
-
name?: string;
|
|
5
|
-
price?: string;
|
|
6
|
-
brand?: string;
|
|
7
|
-
image?: string;
|
|
8
|
-
stock?: string;
|
|
9
|
-
description?: string;
|
|
10
|
-
category?: string;
|
|
11
|
-
}
|
|
12
|
-
|
|
13
|
-
/**
|
|
14
|
-
* SchemaMapper — Uses AI to "train" the plugin on unknown database schemas.
|
|
15
|
-
* It maps arbitrary metadata keys to standard UI properties.
|
|
16
|
-
*
|
|
17
|
-
* Design notes:
|
|
18
|
-
* - Bypasses ILLMProvider.chat() entirely to avoid injecting the full chat
|
|
19
|
-
* system prompt (which can exceed Groq's free-tier TPM limit).
|
|
20
|
-
* - Deduplicates concurrent in-flight training calls for the same schema.
|
|
21
|
-
* - Falls back to heuristics on any failure (non-blocking).
|
|
22
|
-
*/
|
|
23
|
-
export class SchemaMapper {
|
|
24
|
-
private static cache = new Map<string, SchemaMap>();
|
|
25
|
-
/** In-flight deduplication — prevents concurrent identical LLM calls. */
|
|
26
|
-
private static pending = new Map<string, Promise<SchemaMap>>();
|
|
27
|
-
|
|
28
|
-
private static readonly TARGET_PROPERTIES: Record<string, string> = {
|
|
29
|
-
name: 'The primary title, name, or label of the item',
|
|
30
|
-
price: 'The numeric cost, price, MSRP, or amount',
|
|
31
|
-
brand: 'The manufacturer, vendor, brand, or maker',
|
|
32
|
-
image: 'The URL to an image, thumbnail, photo, or picture',
|
|
33
|
-
stock: 'The availability, inventory count, quantity, or stock level',
|
|
34
|
-
description: 'The detailed text content, summary, or body info',
|
|
35
|
-
category: 'The group, department, type, or category name',
|
|
36
|
-
};
|
|
37
|
-
|
|
38
|
-
/**
|
|
39
|
-
* Trains the plugin on a set of keys.
|
|
40
|
-
* Results are cached in-process; concurrent calls for the same key set are
|
|
41
|
-
* deduplicated to a single LLM request.
|
|
42
|
-
*/
|
|
43
|
-
static async train(
|
|
44
|
-
llm: ILLMProvider,
|
|
45
|
-
projectId: string,
|
|
46
|
-
keys: string[]
|
|
47
|
-
): Promise<SchemaMap> {
|
|
48
|
-
const cacheKey = `${projectId}:${keys.slice().sort().join(',')}`;
|
|
49
|
-
|
|
50
|
-
// 1. Return from in-process cache (fast path)
|
|
51
|
-
if (this.cache.has(cacheKey)) {
|
|
52
|
-
return this.cache.get(cacheKey)!;
|
|
53
|
-
}
|
|
54
|
-
|
|
55
|
-
// 2. Deduplicate concurrent in-flight requests for the same schema
|
|
56
|
-
if (this.pending.has(cacheKey)) {
|
|
57
|
-
return this.pending.get(cacheKey)!;
|
|
58
|
-
}
|
|
59
|
-
|
|
60
|
-
const promise = this._doTrain(llm, cacheKey, keys);
|
|
61
|
-
this.pending.set(cacheKey, promise);
|
|
62
|
-
promise.finally(() => this.pending.delete(cacheKey));
|
|
63
|
-
return promise;
|
|
64
|
-
}
|
|
65
|
-
|
|
66
|
-
private static async _doTrain(
|
|
67
|
-
llm: ILLMProvider,
|
|
68
|
-
cacheKey: string,
|
|
69
|
-
keys: string[]
|
|
70
|
-
): Promise<SchemaMap> {
|
|
71
|
-
//console.log(`[SchemaMapper] 🧠 Training on new schema keys: ${keys.join(', ')}`);
|
|
72
|
-
|
|
73
|
-
const propertyList = Object.entries(this.TARGET_PROPERTIES)
|
|
74
|
-
.map(([prop, desc]) => `- ${prop} (${desc})`)
|
|
75
|
-
.join('\n');
|
|
76
|
-
|
|
77
|
-
// Compact prompt — no chat system prompt injected.
|
|
78
|
-
const messages = [
|
|
79
|
-
{ role: 'system', content: 'You are a database schema expert. Respond ONLY with valid JSON.' },
|
|
80
|
-
{
|
|
81
|
-
role: 'user',
|
|
82
|
-
content:
|
|
83
|
-
`Keys: [${keys.join(', ')}]\n\nMap to:\n${propertyList}\n\n` +
|
|
84
|
-
`Return a JSON object like {"name":"Title","price":"Price",...}. Omit unmapped properties.`,
|
|
85
|
-
},
|
|
86
|
-
];
|
|
87
|
-
|
|
88
|
-
try {
|
|
89
|
-
// Use a direct fetch against the resolved base URL + auth headers,
|
|
90
|
-
// bypassing ILLMProvider.chat() which injects the full chat system prompt.
|
|
91
|
-
const baseUrl: string = (llm as any)?.baseUrl ?? '';
|
|
92
|
-
const apiKey: string = (llm as any)?.apiKey ?? '';
|
|
93
|
-
const model: string = (llm as any)?.model ?? 'llama-3.1-8b-instant';
|
|
94
|
-
|
|
95
|
-
let responseText: string | undefined;
|
|
96
|
-
|
|
97
|
-
if (baseUrl) {
|
|
98
|
-
const endpoint = `${baseUrl.replace(/\/$/, '')}/chat/completions`;
|
|
99
|
-
const headers: Record<string, string> = { 'Content-Type': 'application/json' };
|
|
100
|
-
if (apiKey) headers['Authorization'] = `Bearer ${apiKey}`;
|
|
101
|
-
|
|
102
|
-
const res = await fetch(endpoint, {
|
|
103
|
-
method: 'POST',
|
|
104
|
-
headers,
|
|
105
|
-
body: JSON.stringify({
|
|
106
|
-
model,
|
|
107
|
-
messages,
|
|
108
|
-
max_tokens: 256, // Only needs a small JSON object back
|
|
109
|
-
temperature: 0,
|
|
110
|
-
stream: false,
|
|
111
|
-
}),
|
|
112
|
-
signal: AbortSignal.timeout(8000),
|
|
113
|
-
});
|
|
114
|
-
|
|
115
|
-
if (res.ok) {
|
|
116
|
-
const data = await res.json();
|
|
117
|
-
responseText =
|
|
118
|
-
data?.choices?.[0]?.message?.content ??
|
|
119
|
-
data?.choices?.[0]?.delta?.content ??
|
|
120
|
-
undefined;
|
|
121
|
-
} else {
|
|
122
|
-
console.warn(`[SchemaMapper] LLM returned ${res.status}, falling back to heuristics.`);
|
|
123
|
-
}
|
|
124
|
-
}
|
|
125
|
-
|
|
126
|
-
if (responseText) {
|
|
127
|
-
const mapping = this._parseJson(responseText);
|
|
128
|
-
if (mapping) {
|
|
129
|
-
this.cache.set(cacheKey, mapping);
|
|
130
|
-
return mapping;
|
|
131
|
-
}
|
|
132
|
-
}
|
|
133
|
-
} catch (error) {
|
|
134
|
-
console.warn('[SchemaMapper] AI training failed, falling back to heuristics:', error);
|
|
135
|
-
}
|
|
136
|
-
|
|
137
|
-
// Store empty result so we don't keep retrying on the same schema
|
|
138
|
-
this.cache.set(cacheKey, {});
|
|
139
|
-
return {};
|
|
140
|
-
}
|
|
141
|
-
|
|
142
|
-
private static _parseJson(text: string): SchemaMap | null {
|
|
143
|
-
const startIdx = text.indexOf('{');
|
|
144
|
-
if (startIdx === -1) return null;
|
|
145
|
-
let braceCount = 0;
|
|
146
|
-
let endIdx = -1;
|
|
147
|
-
for (let i = startIdx; i < text.length; i++) {
|
|
148
|
-
if (text[i] === '{') braceCount++;
|
|
149
|
-
else if (text[i] === '}') {
|
|
150
|
-
braceCount--;
|
|
151
|
-
if (braceCount === 0) { endIdx = i; break; }
|
|
152
|
-
}
|
|
153
|
-
}
|
|
154
|
-
if (endIdx === -1) return null;
|
|
155
|
-
try {
|
|
156
|
-
return JSON.parse(this._sanitizeJson(text.substring(startIdx, endIdx + 1))) as SchemaMap;
|
|
157
|
-
} catch {
|
|
158
|
-
return null;
|
|
159
|
-
}
|
|
160
|
-
}
|
|
161
|
-
|
|
162
|
-
private static _sanitizeJson(s: string): string {
|
|
163
|
-
return s
|
|
164
|
-
.replace(/\/\*[\s\S]*?\*\/|([^:]|^)\/\/.*$/gm, '')
|
|
165
|
-
.replace(/[\u201C\u201D\u2018\u2019]/g, '"')
|
|
166
|
-
.replace(/'/g, '"')
|
|
167
|
-
.replace(/([{,]\s*)([a-zA-Z0-9_]+?)\s*:/g, '$1"$2":')
|
|
168
|
-
.replace(/,\s*([}\]])/g, '$1')
|
|
169
|
-
.trim();
|
|
170
|
-
}
|
|
171
|
-
|
|
172
|
-
static getCached(projectId: string, keys: string[]): SchemaMap | undefined {
|
|
173
|
-
const cacheKey = `${projectId}:${keys.slice().sort().join(',')}`;
|
|
174
|
-
return this.cache.get(cacheKey);
|
|
175
|
-
}
|
|
176
|
-
}
|