@retrivora-ai/rag-engine 1.9.0 → 1.9.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{ILLMProvider-BOJFz3Na.d.mts → ILLMProvider-BQyKZLnd.d.mts} +10 -0
- package/dist/{ILLMProvider-BOJFz3Na.d.ts → ILLMProvider-BQyKZLnd.d.ts} +10 -0
- package/dist/handlers/index.d.mts +2 -2
- package/dist/handlers/index.d.ts +2 -2
- package/dist/handlers/index.js +1719 -467
- package/dist/handlers/index.mjs +1718 -466
- package/dist/{index-BwpcaziY.d.ts → index-A0GqPsaG.d.ts} +1 -1
- package/dist/{index-D3V9Et2M.d.mts → index-CDftK3qs.d.mts} +1 -1
- package/dist/index.css +26 -0
- package/dist/index.d.mts +2 -2
- package/dist/index.d.ts +2 -2
- package/dist/index.js +246 -76
- package/dist/index.mjs +248 -76
- package/dist/server.d.mts +32 -5
- package/dist/server.d.ts +32 -5
- package/dist/server.js +1726 -671
- package/dist/server.mjs +1724 -669
- package/package.json +1 -1
- package/src/components/MarkdownComponents.tsx +3 -3
- package/src/components/MessageBubble.tsx +49 -2
- package/src/components/ProductCard.tsx +33 -6
- package/src/components/UIDispatcher.tsx +1 -0
- package/src/components/VisualizationRenderer.tsx +143 -11
- package/src/config/EmbeddingStrategy.ts +5 -4
- package/src/config/RagConfig.ts +10 -0
- package/src/config/serverConfig.ts +16 -1
- package/src/core/LLMRouter.ts +79 -0
- package/src/core/Pipeline.ts +262 -45
- package/src/core/ProviderRegistry.ts +6 -0
- package/src/core/QueryProcessor.ts +98 -0
- package/src/handlers/index.ts +8 -5
- package/src/hooks/useRagChat.ts +53 -17
- package/src/llm/providers/UniversalLLMAdapter.ts +110 -13
- package/src/providers/vectordb/ChromaDBProvider.ts +13 -2
- package/src/providers/vectordb/MilvusProvider.ts +18 -2
- package/src/providers/vectordb/MultiTablePostgresProvider.ts +12 -12
- package/src/providers/vectordb/PostgreSQLProvider.ts +1 -1
- package/src/providers/vectordb/QdrantProvider.ts +1 -1
- package/src/providers/vectordb/RedisProvider.ts +3 -4
- package/src/providers/vectordb/WeaviateProvider.ts +41 -3
- package/src/types/index.ts +26 -0
- package/src/utils/ProductExtractor.ts +5 -3
- package/src/utils/UITransformer.ts +1348 -489
- package/src/utils/synonyms.ts +2 -0
package/dist/handlers/index.mjs
CHANGED
|
@@ -313,12 +313,218 @@ var init_PineconeProvider = __esm({
|
|
|
313
313
|
}
|
|
314
314
|
});
|
|
315
315
|
|
|
316
|
+
// src/providers/vectordb/PostgreSQLProvider.ts
|
|
317
|
+
var PostgreSQLProvider_exports = {};
|
|
318
|
+
__export(PostgreSQLProvider_exports, {
|
|
319
|
+
PostgreSQLProvider: () => PostgreSQLProvider
|
|
320
|
+
});
|
|
321
|
+
import { Pool } from "pg";
|
|
322
|
+
var PostgreSQLProvider;
|
|
323
|
+
var init_PostgreSQLProvider = __esm({
|
|
324
|
+
"src/providers/vectordb/PostgreSQLProvider.ts"() {
|
|
325
|
+
"use strict";
|
|
326
|
+
init_BaseVectorProvider();
|
|
327
|
+
PostgreSQLProvider = class extends BaseVectorProvider {
|
|
328
|
+
constructor(config) {
|
|
329
|
+
var _a;
|
|
330
|
+
super(config);
|
|
331
|
+
this.tableName = this.indexName.replace(/[^a-z0-9_]/gi, "_");
|
|
332
|
+
const opts = config.options;
|
|
333
|
+
if (!opts.connectionString) throw new Error("[PostgreSQLProvider] options.connectionString is required");
|
|
334
|
+
this.connectionString = opts.connectionString;
|
|
335
|
+
this.dimensions = (_a = opts.dimensions) != null ? _a : 1536;
|
|
336
|
+
}
|
|
337
|
+
static getValidator() {
|
|
338
|
+
return {
|
|
339
|
+
validate(config) {
|
|
340
|
+
const errors = [];
|
|
341
|
+
const opts = config.options || {};
|
|
342
|
+
if (!opts.connectionString) {
|
|
343
|
+
errors.push({
|
|
344
|
+
field: "vectorDb.options.connectionString",
|
|
345
|
+
message: "PostgreSQL connection string is required",
|
|
346
|
+
suggestion: "Set PGVECTOR_CONNECTION_STRING environment variable",
|
|
347
|
+
severity: "error"
|
|
348
|
+
});
|
|
349
|
+
}
|
|
350
|
+
if (opts.tables && typeof opts.tables !== "string" && !Array.isArray(opts.tables)) {
|
|
351
|
+
errors.push({
|
|
352
|
+
field: "vectorDb.options.tables",
|
|
353
|
+
message: "PostgreSQL tables must be a string or a string array",
|
|
354
|
+
severity: "error"
|
|
355
|
+
});
|
|
356
|
+
}
|
|
357
|
+
return errors;
|
|
358
|
+
}
|
|
359
|
+
};
|
|
360
|
+
}
|
|
361
|
+
static getHealthChecker() {
|
|
362
|
+
return {
|
|
363
|
+
async check(config) {
|
|
364
|
+
const opts = config.options || {};
|
|
365
|
+
const timestamp = Date.now();
|
|
366
|
+
try {
|
|
367
|
+
const { Client } = await import("pg");
|
|
368
|
+
const client = new Client({ connectionString: opts.connectionString });
|
|
369
|
+
await client.connect();
|
|
370
|
+
const result = await client.query(`
|
|
371
|
+
SELECT EXISTS(SELECT 1 FROM pg_extension WHERE extname = 'vector');
|
|
372
|
+
`);
|
|
373
|
+
const hasVector = result.rows[0].exists;
|
|
374
|
+
await client.end();
|
|
375
|
+
return {
|
|
376
|
+
healthy: true,
|
|
377
|
+
provider: "postgresql",
|
|
378
|
+
capabilities: { pgvectorInstalled: hasVector },
|
|
379
|
+
timestamp
|
|
380
|
+
};
|
|
381
|
+
} catch (error) {
|
|
382
|
+
return {
|
|
383
|
+
healthy: false,
|
|
384
|
+
provider: "postgresql",
|
|
385
|
+
error: `Connection failed: ${error instanceof Error ? error.message : String(error)}`,
|
|
386
|
+
timestamp
|
|
387
|
+
};
|
|
388
|
+
}
|
|
389
|
+
}
|
|
390
|
+
};
|
|
391
|
+
}
|
|
392
|
+
async initialize() {
|
|
393
|
+
this.pool = new Pool({ connectionString: this.connectionString });
|
|
394
|
+
const client = await this.pool.connect();
|
|
395
|
+
try {
|
|
396
|
+
await client.query("CREATE EXTENSION IF NOT EXISTS vector");
|
|
397
|
+
await client.query(`
|
|
398
|
+
CREATE TABLE IF NOT EXISTS ${this.tableName} (
|
|
399
|
+
id TEXT PRIMARY KEY,
|
|
400
|
+
namespace TEXT NOT NULL DEFAULT '',
|
|
401
|
+
content TEXT NOT NULL,
|
|
402
|
+
metadata JSONB,
|
|
403
|
+
embedding VECTOR(${this.dimensions})
|
|
404
|
+
)
|
|
405
|
+
`);
|
|
406
|
+
await client.query(`
|
|
407
|
+
CREATE INDEX IF NOT EXISTS ${this.tableName}_embedding_idx
|
|
408
|
+
ON ${this.tableName}
|
|
409
|
+
USING hnsw (embedding vector_cosine_ops)
|
|
410
|
+
`);
|
|
411
|
+
} finally {
|
|
412
|
+
client.release();
|
|
413
|
+
}
|
|
414
|
+
}
|
|
415
|
+
async upsert(doc, namespace = "") {
|
|
416
|
+
var _a;
|
|
417
|
+
const vectorLiteral = `[${doc.vector.join(",")}]`;
|
|
418
|
+
await this.pool.query(
|
|
419
|
+
`INSERT INTO ${this.tableName} (id, namespace, content, metadata, embedding)
|
|
420
|
+
VALUES ($1, $2, $3, $4, $5::vector)
|
|
421
|
+
ON CONFLICT (id) DO UPDATE
|
|
422
|
+
SET namespace = EXCLUDED.namespace,
|
|
423
|
+
content = EXCLUDED.content,
|
|
424
|
+
metadata = EXCLUDED.metadata,
|
|
425
|
+
embedding = EXCLUDED.embedding`,
|
|
426
|
+
[doc.id, namespace, doc.content, JSON.stringify((_a = doc.metadata) != null ? _a : {}), vectorLiteral]
|
|
427
|
+
);
|
|
428
|
+
}
|
|
429
|
+
async batchUpsert(docs, namespace = "") {
|
|
430
|
+
if (docs.length === 0) return;
|
|
431
|
+
const client = await this.pool.connect();
|
|
432
|
+
try {
|
|
433
|
+
await client.query("BEGIN");
|
|
434
|
+
const BATCH_SIZE = 50;
|
|
435
|
+
for (let i = 0; i < docs.length; i += BATCH_SIZE) {
|
|
436
|
+
const batch = docs.slice(i, i + BATCH_SIZE);
|
|
437
|
+
const values = [];
|
|
438
|
+
const valuePlaceholders = batch.map((doc, idx) => {
|
|
439
|
+
var _a;
|
|
440
|
+
const offset = idx * 5;
|
|
441
|
+
values.push(doc.id, namespace, doc.content, JSON.stringify((_a = doc.metadata) != null ? _a : {}), `[${doc.vector.join(",")}]`);
|
|
442
|
+
return `($${offset + 1}, $${offset + 2}, $${offset + 3}, $${offset + 4}, $${offset + 5}::vector)`;
|
|
443
|
+
}).join(", ");
|
|
444
|
+
const query = `
|
|
445
|
+
INSERT INTO ${this.tableName} (id, namespace, content, metadata, embedding)
|
|
446
|
+
VALUES ${valuePlaceholders}
|
|
447
|
+
ON CONFLICT (id) DO UPDATE
|
|
448
|
+
SET namespace = EXCLUDED.namespace,
|
|
449
|
+
content = EXCLUDED.content,
|
|
450
|
+
metadata = EXCLUDED.metadata,
|
|
451
|
+
embedding = EXCLUDED.embedding
|
|
452
|
+
`;
|
|
453
|
+
await client.query(query, values);
|
|
454
|
+
}
|
|
455
|
+
await client.query("COMMIT");
|
|
456
|
+
} catch (error) {
|
|
457
|
+
await client.query("ROLLBACK");
|
|
458
|
+
throw error;
|
|
459
|
+
} finally {
|
|
460
|
+
client.release();
|
|
461
|
+
}
|
|
462
|
+
}
|
|
463
|
+
async query(vector, topK, namespace, filter) {
|
|
464
|
+
const vectorLiteral = `[${vector.join(",")}]`;
|
|
465
|
+
let whereClause = namespace ? `WHERE namespace = $3` : "";
|
|
466
|
+
const params = [vectorLiteral, topK];
|
|
467
|
+
if (namespace) params.push(namespace);
|
|
468
|
+
const publicFilter = this.sanitizeFilter(filter);
|
|
469
|
+
if (Object.keys(publicFilter).length > 0) {
|
|
470
|
+
const filterConditions = Object.entries(publicFilter).map(([key, val]) => {
|
|
471
|
+
const paramIdx = params.length + 1;
|
|
472
|
+
params.push(String(val));
|
|
473
|
+
return `metadata->>'${key}' = $${paramIdx}`;
|
|
474
|
+
}).join(" AND ");
|
|
475
|
+
whereClause = whereClause ? `${whereClause} AND ${filterConditions}` : `WHERE ${filterConditions}`;
|
|
476
|
+
}
|
|
477
|
+
const client = await this.pool.connect();
|
|
478
|
+
try {
|
|
479
|
+
const efSearch = this.config.options.efSearch || Math.max(topK * 10, 40);
|
|
480
|
+
await client.query(`SET LOCAL hnsw.ef_search = ${efSearch}`);
|
|
481
|
+
const result = await client.query(
|
|
482
|
+
`SELECT id, content, metadata, 1 - (embedding <=> $1::vector) AS score
|
|
483
|
+
FROM ${this.tableName}
|
|
484
|
+
${whereClause}
|
|
485
|
+
ORDER BY embedding <=> $1::vector
|
|
486
|
+
LIMIT $2`,
|
|
487
|
+
params
|
|
488
|
+
);
|
|
489
|
+
return result.rows.map((row) => ({
|
|
490
|
+
id: String(row["id"]),
|
|
491
|
+
score: parseFloat(String(row["score"])),
|
|
492
|
+
content: String(row["content"]),
|
|
493
|
+
metadata: row["metadata"]
|
|
494
|
+
}));
|
|
495
|
+
} finally {
|
|
496
|
+
client.release();
|
|
497
|
+
}
|
|
498
|
+
}
|
|
499
|
+
async delete(id, namespace) {
|
|
500
|
+
const where = namespace ? "WHERE id = $1 AND namespace = $2" : "WHERE id = $1";
|
|
501
|
+
const params = namespace ? [id, namespace] : [id];
|
|
502
|
+
await this.pool.query(`DELETE FROM ${this.tableName} ${where}`, params);
|
|
503
|
+
}
|
|
504
|
+
async deleteNamespace(namespace) {
|
|
505
|
+
await this.pool.query(`DELETE FROM ${this.tableName} WHERE namespace = $1`, [namespace]);
|
|
506
|
+
}
|
|
507
|
+
async ping() {
|
|
508
|
+
try {
|
|
509
|
+
await this.pool.query("SELECT 1");
|
|
510
|
+
return true;
|
|
511
|
+
} catch (e) {
|
|
512
|
+
return false;
|
|
513
|
+
}
|
|
514
|
+
}
|
|
515
|
+
async disconnect() {
|
|
516
|
+
await this.pool.end();
|
|
517
|
+
}
|
|
518
|
+
};
|
|
519
|
+
}
|
|
520
|
+
});
|
|
521
|
+
|
|
316
522
|
// src/providers/vectordb/MultiTablePostgresProvider.ts
|
|
317
523
|
var MultiTablePostgresProvider_exports = {};
|
|
318
524
|
__export(MultiTablePostgresProvider_exports, {
|
|
319
525
|
MultiTablePostgresProvider: () => MultiTablePostgresProvider
|
|
320
526
|
});
|
|
321
|
-
import { Pool } from "pg";
|
|
527
|
+
import { Pool as Pool2 } from "pg";
|
|
322
528
|
var MultiTablePostgresProvider;
|
|
323
529
|
var init_MultiTablePostgresProvider = __esm({
|
|
324
530
|
"src/providers/vectordb/MultiTablePostgresProvider.ts"() {
|
|
@@ -340,7 +546,7 @@ var init_MultiTablePostgresProvider = __esm({
|
|
|
340
546
|
this.uploadTable = opts.uploadTable || "document_chunks";
|
|
341
547
|
}
|
|
342
548
|
async initialize() {
|
|
343
|
-
this.pool = new
|
|
549
|
+
this.pool = new Pool2({ connectionString: this.connectionString });
|
|
344
550
|
const client = await this.pool.connect();
|
|
345
551
|
try {
|
|
346
552
|
await client.query("CREATE EXTENSION IF NOT EXISTS vector");
|
|
@@ -484,26 +690,25 @@ var init_MultiTablePostgresProvider = __esm({
|
|
|
484
690
|
const allResults = [];
|
|
485
691
|
console.log(`[MultiTablePostgresProvider] --- Starting Multi-Table Search ---`);
|
|
486
692
|
const queryText = _filter == null ? void 0 : _filter.queryText;
|
|
487
|
-
const entityHints = Array.isArray(_filter == null ? void 0 : _filter.
|
|
488
|
-
(hint) => typeof hint === "object" && hint !== null && typeof hint.value === "string"
|
|
489
|
-
).map((hint) => hint.value.trim().toLowerCase()).filter(Boolean) : [];
|
|
693
|
+
const entityHints = Array.isArray(_filter == null ? void 0 : _filter.keywords) ? _filter.keywords.map((k) => k.trim().toLowerCase()).filter(Boolean) : [];
|
|
490
694
|
console.log(`[MultiTablePostgresProvider] queryText: "${queryText}"`);
|
|
491
695
|
console.log(`[MultiTablePostgresProvider] entityHints: [${entityHints.join(", ")}]`);
|
|
492
696
|
const getDynamicKeywordQuery = () => {
|
|
493
697
|
if (entityHints.length > 0) {
|
|
494
|
-
return entityHints.map((h) => h.
|
|
698
|
+
return entityHints.map((h) => h.replace(/\s+/g, " & ")).join(" & ");
|
|
495
699
|
}
|
|
496
700
|
if (queryText) {
|
|
497
|
-
return queryText.toLowerCase().replace(/\b(show|me|the|product[s]?|item[s]?|card[s]?|of|category|find|list|browse|what|are|display|all|under|for|in|with|about)\b/g, "").trim().replace(/\s+/g, " & ");
|
|
701
|
+
return queryText.toLowerCase().replace(/\b(show|me|the|product[s]?|item[s]?|card[s]?|organization[s]?|comp(?:any|anies)|of|category|find|list|browse|what|are|display|all|whose|that|which|have|has|having|greater|than|more|less|equal|equals|above|below|over|under|at|least|most|for|in|with|about|get|details|information|info)\b/g, "").replace(/\b\d+(?:\.\d+)?\b/g, "").trim().replace(/\s+/g, " & ");
|
|
498
702
|
}
|
|
499
703
|
return "";
|
|
500
704
|
};
|
|
501
705
|
const dynamicKeywordQuery = getDynamicKeywordQuery();
|
|
706
|
+
const tableLimit = Math.max(topK, 50);
|
|
502
707
|
const queryPromises = this.tables.map(async (table) => {
|
|
503
708
|
try {
|
|
504
709
|
let sqlQuery = "";
|
|
505
710
|
let params = [];
|
|
506
|
-
if (queryText) {
|
|
711
|
+
if (queryText && dynamicKeywordQuery) {
|
|
507
712
|
const hasEntityHints = entityHints.length > 0;
|
|
508
713
|
const exactNameScoreExpr = hasEntityHints ? `+ (
|
|
509
714
|
SELECT COALESCE(MAX(
|
|
@@ -520,16 +725,16 @@ var init_MultiTablePostgresProvider = __esm({
|
|
|
520
725
|
((1 - (embedding <=> $1::vector)) + (COALESCE(ts_rank(to_tsvector('english', (to_jsonb(t) - 'embedding')::text), to_tsquery('english', $2)), 0) * 2.0) ${exactNameScoreExpr}) AS hybrid_score
|
|
521
726
|
FROM "${table}" t
|
|
522
727
|
ORDER BY hybrid_score DESC
|
|
523
|
-
LIMIT
|
|
728
|
+
LIMIT ${tableLimit}
|
|
524
729
|
`;
|
|
525
730
|
params = [vectorLiteral, dynamicKeywordQuery];
|
|
526
731
|
} else {
|
|
527
732
|
sqlQuery = `
|
|
528
733
|
SELECT *,
|
|
529
|
-
|
|
734
|
+
(1 - (embedding <=> $1::vector)) AS hybrid_score
|
|
530
735
|
FROM "${table}" t
|
|
531
736
|
ORDER BY hybrid_score DESC
|
|
532
|
-
LIMIT
|
|
737
|
+
LIMIT ${tableLimit}
|
|
533
738
|
`;
|
|
534
739
|
params = [vectorLiteral];
|
|
535
740
|
}
|
|
@@ -854,13 +1059,25 @@ var init_MilvusProvider = __esm({
|
|
|
854
1059
|
};
|
|
855
1060
|
await this.http.post("/v1/vector/upsert", payload);
|
|
856
1061
|
}
|
|
857
|
-
// eslint-disable-next-line @typescript-eslint/no-unused-vars
|
|
858
1062
|
async query(vector, topK, namespace, _filter) {
|
|
1063
|
+
const sanitizedFilter = this.sanitizeFilter(_filter);
|
|
1064
|
+
const filterParts = [];
|
|
1065
|
+
if (namespace) {
|
|
1066
|
+
filterParts.push(`namespace == "${namespace}"`);
|
|
1067
|
+
}
|
|
1068
|
+
for (const [key, value] of Object.entries(sanitizedFilter)) {
|
|
1069
|
+
if (key === "queryText" || key === "keywords") continue;
|
|
1070
|
+
if (typeof value === "string") {
|
|
1071
|
+
filterParts.push(`metadata["${key}"] == "${value.replace(/"/g, '\\"')}"`);
|
|
1072
|
+
} else if (typeof value === "number") {
|
|
1073
|
+
filterParts.push(`metadata["${key}"] == ${value}`);
|
|
1074
|
+
}
|
|
1075
|
+
}
|
|
859
1076
|
const payload = {
|
|
860
1077
|
collectionName: this.indexName,
|
|
861
1078
|
vector,
|
|
862
1079
|
limit: topK,
|
|
863
|
-
filter:
|
|
1080
|
+
filter: filterParts.length > 0 ? filterParts.join(" && ") : void 0,
|
|
864
1081
|
outputFields: ["content", "metadata"],
|
|
865
1082
|
searchParams: {
|
|
866
1083
|
nprobe: this.config.options.nprobe || 16,
|
|
@@ -1059,6 +1276,7 @@ var init_QdrantProvider = __esm({
|
|
|
1059
1276
|
await this.http.put(`/collections/${this.indexName}/points`, payload);
|
|
1060
1277
|
}
|
|
1061
1278
|
async query(vector, topK, namespace, _filter) {
|
|
1279
|
+
var _a;
|
|
1062
1280
|
const must = [];
|
|
1063
1281
|
if (namespace) {
|
|
1064
1282
|
must.push({ key: "namespace", match: { value: namespace } });
|
|
@@ -1080,7 +1298,7 @@ var init_QdrantProvider = __esm({
|
|
|
1080
1298
|
limit: topK,
|
|
1081
1299
|
with_payload: true,
|
|
1082
1300
|
params: {
|
|
1083
|
-
hnsw_ef: this.config.options.efSearch || Math.max(topK * 20, 128),
|
|
1301
|
+
hnsw_ef: ((_a = this.config.options) == null ? void 0 : _a.efSearch) || Math.max(topK * 20, 128),
|
|
1084
1302
|
exact: false
|
|
1085
1303
|
},
|
|
1086
1304
|
filter: must.length > 0 ? { must } : void 0
|
|
@@ -1205,12 +1423,20 @@ var init_ChromaDBProvider = __esm({
|
|
|
1205
1423
|
};
|
|
1206
1424
|
await this.http.post(`/api/v1/collections/${this.collectionId}/add`, payload);
|
|
1207
1425
|
}
|
|
1208
|
-
// eslint-disable-next-line @typescript-eslint/no-unused-vars
|
|
1209
1426
|
async query(vector, topK, namespace, _filter) {
|
|
1427
|
+
const sanitizedFilter = this.sanitizeFilter(_filter);
|
|
1428
|
+
const whereClauses = [];
|
|
1429
|
+
if (namespace) whereClauses.push({ namespace: { $eq: namespace } });
|
|
1430
|
+
Object.entries(sanitizedFilter).forEach(([key, value]) => {
|
|
1431
|
+
if (key === "namespace") return;
|
|
1432
|
+
if (typeof value === "string" || typeof value === "number" || typeof value === "boolean") {
|
|
1433
|
+
whereClauses.push({ [key]: { $eq: value } });
|
|
1434
|
+
}
|
|
1435
|
+
});
|
|
1210
1436
|
const payload = {
|
|
1211
1437
|
query_embeddings: [vector],
|
|
1212
1438
|
n_results: topK,
|
|
1213
|
-
where:
|
|
1439
|
+
where: whereClauses.length > 1 ? { $and: whereClauses } : whereClauses[0]
|
|
1214
1440
|
};
|
|
1215
1441
|
const { data } = await this.http.post(`/api/v1/collections/${this.collectionId}/query`, payload);
|
|
1216
1442
|
const matches = [];
|
|
@@ -1315,15 +1541,15 @@ var init_RedisProvider = __esm({
|
|
|
1315
1541
|
console.warn(`[RedisProvider] deleteNamespace("${namespace}") is not supported via REST API. Use Redis CLI: SCAN + DEL`);
|
|
1316
1542
|
}
|
|
1317
1543
|
/**
|
|
1318
|
-
*
|
|
1319
|
-
* Returns
|
|
1544
|
+
* Check reachability via a PING command.
|
|
1545
|
+
* Returns false on connection failure so health checks surface real problems.
|
|
1320
1546
|
*/
|
|
1321
1547
|
async ping() {
|
|
1322
1548
|
try {
|
|
1323
1549
|
await this.http.post("/", ["PING"]);
|
|
1324
1550
|
return true;
|
|
1325
1551
|
} catch (e) {
|
|
1326
|
-
return
|
|
1552
|
+
return false;
|
|
1327
1553
|
}
|
|
1328
1554
|
}
|
|
1329
1555
|
async disconnect() {
|
|
@@ -1363,15 +1589,17 @@ var init_WeaviateProvider = __esm({
|
|
|
1363
1589
|
await this.ping();
|
|
1364
1590
|
}
|
|
1365
1591
|
async upsert(doc, namespace) {
|
|
1592
|
+
const primitiveMetadata = this.extractPrimitiveMetadata(doc.metadata);
|
|
1366
1593
|
const payload = {
|
|
1367
1594
|
class: this.indexName,
|
|
1368
1595
|
id: doc.id,
|
|
1369
1596
|
vector: doc.vector,
|
|
1370
|
-
properties: {
|
|
1597
|
+
properties: __spreadProps(__spreadValues({
|
|
1371
1598
|
content: doc.content,
|
|
1372
|
-
metadata: JSON.stringify(doc.metadata || {})
|
|
1599
|
+
metadata: JSON.stringify(doc.metadata || {})
|
|
1600
|
+
}, primitiveMetadata), {
|
|
1373
1601
|
namespace: namespace || ""
|
|
1374
|
-
}
|
|
1602
|
+
})
|
|
1375
1603
|
};
|
|
1376
1604
|
await this.http.post("/v1/objects", payload);
|
|
1377
1605
|
}
|
|
@@ -1381,20 +1609,22 @@ var init_WeaviateProvider = __esm({
|
|
|
1381
1609
|
class: this.indexName,
|
|
1382
1610
|
id: doc.id,
|
|
1383
1611
|
vector: doc.vector,
|
|
1384
|
-
properties: {
|
|
1612
|
+
properties: __spreadProps(__spreadValues({
|
|
1385
1613
|
content: doc.content,
|
|
1386
|
-
metadata: JSON.stringify(doc.metadata || {})
|
|
1614
|
+
metadata: JSON.stringify(doc.metadata || {})
|
|
1615
|
+
}, this.extractPrimitiveMetadata(doc.metadata)), {
|
|
1387
1616
|
namespace: namespace || ""
|
|
1388
|
-
}
|
|
1617
|
+
})
|
|
1389
1618
|
}))
|
|
1390
1619
|
};
|
|
1391
1620
|
await this.http.post("/v1/batch/objects", payload);
|
|
1392
1621
|
}
|
|
1393
1622
|
async query(vector, topK, namespace, _filter) {
|
|
1394
1623
|
var _a, _b;
|
|
1624
|
+
const queryText = _filter == null ? void 0 : _filter.queryText;
|
|
1395
1625
|
const sanitizedFilter = this.sanitizeFilter(_filter);
|
|
1396
|
-
const queryText = sanitizedFilter.queryText;
|
|
1397
1626
|
const searchParams = queryText ? `hybrid: { query: ${JSON.stringify(queryText)}, alpha: 0.5 }` : `nearVector: { vector: ${JSON.stringify(vector)} }`;
|
|
1627
|
+
const where = this.buildWhereFilter(namespace, sanitizedFilter);
|
|
1398
1628
|
const graphqlQuery = {
|
|
1399
1629
|
query: `
|
|
1400
1630
|
{
|
|
@@ -1402,7 +1632,7 @@ var init_WeaviateProvider = __esm({
|
|
|
1402
1632
|
${this.indexName}(
|
|
1403
1633
|
${searchParams}
|
|
1404
1634
|
limit: ${topK}
|
|
1405
|
-
${
|
|
1635
|
+
${where ? `where: ${where}` : ""}
|
|
1406
1636
|
) {
|
|
1407
1637
|
content
|
|
1408
1638
|
metadata
|
|
@@ -1446,6 +1676,34 @@ var init_WeaviateProvider = __esm({
|
|
|
1446
1676
|
}
|
|
1447
1677
|
async disconnect() {
|
|
1448
1678
|
}
|
|
1679
|
+
extractPrimitiveMetadata(metadata) {
|
|
1680
|
+
const result = {};
|
|
1681
|
+
Object.entries(metadata || {}).forEach(([key, value]) => {
|
|
1682
|
+
if (typeof value === "string" || typeof value === "number" || typeof value === "boolean") {
|
|
1683
|
+
result[key] = value;
|
|
1684
|
+
}
|
|
1685
|
+
});
|
|
1686
|
+
return result;
|
|
1687
|
+
}
|
|
1688
|
+
buildWhereFilter(namespace, filter) {
|
|
1689
|
+
const operands = [];
|
|
1690
|
+
if (namespace) operands.push(this.weaviateOperand("namespace", namespace));
|
|
1691
|
+
Object.entries(filter || {}).forEach(([key, value]) => {
|
|
1692
|
+
if (key === "namespace") return;
|
|
1693
|
+
if (typeof value === "string" || typeof value === "number" || typeof value === "boolean") {
|
|
1694
|
+
operands.push(this.weaviateOperand(key, value));
|
|
1695
|
+
}
|
|
1696
|
+
});
|
|
1697
|
+
if (operands.length === 0) return void 0;
|
|
1698
|
+
if (operands.length === 1) return operands[0];
|
|
1699
|
+
return `{ operator: And, operands: [${operands.join(", ")}] }`;
|
|
1700
|
+
}
|
|
1701
|
+
weaviateOperand(key, value) {
|
|
1702
|
+
const path = `path: [${JSON.stringify(key)}], operator: Equal`;
|
|
1703
|
+
if (typeof value === "number") return `{ ${path}, valueNumber: ${value} }`;
|
|
1704
|
+
if (typeof value === "boolean") return `{ ${path}, valueBoolean: ${value} }`;
|
|
1705
|
+
return `{ ${path}, valueString: ${JSON.stringify(value)} }`;
|
|
1706
|
+
}
|
|
1449
1707
|
};
|
|
1450
1708
|
}
|
|
1451
1709
|
});
|
|
@@ -1751,7 +2009,7 @@ function readEnum(env, name, fallback, allowed) {
|
|
|
1751
2009
|
throw new Error(`[getRagConfig] ${name} must be one of: ${allowed.join(", ")}`);
|
|
1752
2010
|
}
|
|
1753
2011
|
function getEnvConfig(env = process.env, base) {
|
|
1754
|
-
var _a, _b, _c, _d, _e, _f, _g, _h, _i, _j, _k, _l, _m, _n, _o, _p, _q, _r, _s, _t, _u, _v, _w, _x, _y, _z, _A, _B, _C, _D, _E, _F, _G, _H, _I, _J, _K, _L, _M, _N, _O, _P, _Q, _R, _S, _T, _U, _V, _W, _X, _Y, _Z, __, _$, _aa, _ba, _ca, _da, _ea, _fa, _ga, _ha, _ia, _ja, _ka, _la, _ma, _na, _oa, _pa, _qa, _ra, _sa, _ta, _ua, _va, _wa, _xa, _ya, _za, _Aa, _Ba, _Ca, _Da, _Ea, _Fa;
|
|
2012
|
+
var _a, _b, _c, _d, _e, _f, _g, _h, _i, _j, _k, _l, _m, _n, _o, _p, _q, _r, _s, _t, _u, _v, _w, _x, _y, _z, _A, _B, _C, _D, _E, _F, _G, _H, _I, _J, _K, _L, _M, _N, _O, _P, _Q, _R, _S, _T, _U, _V, _W, _X, _Y, _Z, __, _$, _aa, _ba, _ca, _da, _ea, _fa, _ga, _ha, _ia, _ja, _ka, _la, _ma, _na, _oa, _pa, _qa, _ra, _sa, _ta, _ua, _va, _wa, _xa, _ya, _za, _Aa, _Ba, _Ca, _Da, _Ea, _Fa, _Ga;
|
|
1755
2013
|
const projectId = (_c = (_b = (_a = readString(env, "RAG_PROJECT_ID")) != null ? _a : readString(env, "NEXT_PUBLIC_PROJECT_ID")) != null ? _b : base == null ? void 0 : base.projectId) != null ? _c : "__default__";
|
|
1756
2014
|
const vectorProvider = readEnum(env, "VECTOR_DB_PROVIDER", "pinecone", VECTOR_DB_PROVIDERS);
|
|
1757
2015
|
const llmProvider = readEnum(env, "LLM_PROVIDER", "openai", LLM_PROVIDERS);
|
|
@@ -1805,11 +2063,22 @@ function getEnvConfig(env = process.env, base) {
|
|
|
1805
2063
|
};
|
|
1806
2064
|
const embeddingApiKeyByProvider = {
|
|
1807
2065
|
openai: readString(env, "OPENAI_API_KEY"),
|
|
2066
|
+
anthropic: readString(env, "ANTHROPIC_API_KEY"),
|
|
2067
|
+
// Anthropic needs a separate embedding provider; key kept for completeness
|
|
1808
2068
|
gemini: readString(env, "GEMINI_API_KEY"),
|
|
1809
2069
|
ollama: void 0,
|
|
1810
2070
|
universal_rest: (_ea = readString(env, "EMBEDDING_API_KEY")) != null ? _ea : readString(env, "OPENAI_API_KEY"),
|
|
1811
2071
|
custom: (_fa = readString(env, "EMBEDDING_API_KEY")) != null ? _fa : readString(env, "OPENAI_API_KEY")
|
|
1812
2072
|
};
|
|
2073
|
+
const DEFAULT_MODEL_BY_PROVIDER = {
|
|
2074
|
+
openai: "gpt-4o",
|
|
2075
|
+
anthropic: "claude-3-5-sonnet-20241022",
|
|
2076
|
+
gemini: "gemini-2.0-flash",
|
|
2077
|
+
ollama: "llama3",
|
|
2078
|
+
universal_rest: "default",
|
|
2079
|
+
rest: "default",
|
|
2080
|
+
custom: "default"
|
|
2081
|
+
};
|
|
1813
2082
|
return {
|
|
1814
2083
|
projectId,
|
|
1815
2084
|
vectorDb: {
|
|
@@ -1819,8 +2088,8 @@ function getEnvConfig(env = process.env, base) {
|
|
|
1819
2088
|
},
|
|
1820
2089
|
llm: {
|
|
1821
2090
|
provider: llmProvider,
|
|
1822
|
-
model: (_ia = readString(env, "LLM_MODEL")) != null ? _ia : "gpt-4o",
|
|
1823
|
-
apiKey: (
|
|
2091
|
+
model: (_ja = (_ia = readString(env, "LLM_MODEL")) != null ? _ia : DEFAULT_MODEL_BY_PROVIDER[llmProvider]) != null ? _ja : "gpt-4o",
|
|
2092
|
+
apiKey: (_ka = llmApiKeyByProvider[llmProvider]) != null ? _ka : "",
|
|
1824
2093
|
baseUrl: readString(env, "LLM_BASE_URL"),
|
|
1825
2094
|
systemPrompt: readString(env, "LLM_SYSTEM_PROMPT"),
|
|
1826
2095
|
maxTokens: readNumber(env, "LLM_MAX_TOKENS", 4096),
|
|
@@ -1831,7 +2100,7 @@ function getEnvConfig(env = process.env, base) {
|
|
|
1831
2100
|
},
|
|
1832
2101
|
embedding: {
|
|
1833
2102
|
provider: embeddingProvider,
|
|
1834
|
-
model: (
|
|
2103
|
+
model: (_la = readString(env, "EMBEDDING_MODEL")) != null ? _la : "text-embedding-3-small",
|
|
1835
2104
|
apiKey: embeddingApiKeyByProvider[embeddingProvider],
|
|
1836
2105
|
baseUrl: readString(env, "EMBEDDING_BASE_URL"),
|
|
1837
2106
|
dimensions: embeddingDimensions,
|
|
@@ -1842,17 +2111,17 @@ function getEnvConfig(env = process.env, base) {
|
|
|
1842
2111
|
}
|
|
1843
2112
|
},
|
|
1844
2113
|
ui: {
|
|
1845
|
-
title: (
|
|
1846
|
-
subtitle: (
|
|
1847
|
-
primaryColor: (
|
|
1848
|
-
accentColor: (
|
|
1849
|
-
logoUrl: (
|
|
1850
|
-
placeholder: (
|
|
1851
|
-
showSources: ((
|
|
1852
|
-
welcomeMessage: (
|
|
1853
|
-
visualStyle: (
|
|
1854
|
-
borderRadius: (
|
|
1855
|
-
allowUpload: ((
|
|
2114
|
+
title: (_na = (_ma = readString(env, "NEXT_PUBLIC_UI_TITLE")) != null ? _ma : readString(env, "UI_TITLE")) != null ? _na : "AI Assistant",
|
|
2115
|
+
subtitle: (_pa = (_oa = readString(env, "NEXT_PUBLIC_UI_SUBTITLE")) != null ? _oa : readString(env, "UI_SUBTITLE")) != null ? _pa : "Powered by RAG",
|
|
2116
|
+
primaryColor: (_ra = (_qa = readString(env, "NEXT_PUBLIC_PRIMARY_COLOR")) != null ? _qa : readString(env, "UI_PRIMARY_COLOR")) != null ? _ra : "#10b981",
|
|
2117
|
+
accentColor: (_ta = (_sa = readString(env, "NEXT_PUBLIC_ACCENT_COLOR")) != null ? _sa : readString(env, "UI_ACCENT_COLOR")) != null ? _ta : "#3b82f6",
|
|
2118
|
+
logoUrl: (_ua = readString(env, "NEXT_PUBLIC_LOGO_URL")) != null ? _ua : readString(env, "UI_LOGO_URL"),
|
|
2119
|
+
placeholder: (_wa = (_va = readString(env, "NEXT_PUBLIC_PLACEHOLDER")) != null ? _va : readString(env, "UI_PLACEHOLDER")) != null ? _wa : "Ask me anything\u2026",
|
|
2120
|
+
showSources: ((_ya = (_xa = readString(env, "NEXT_PUBLIC_SHOW_SOURCES")) != null ? _xa : readString(env, "UI_SHOW_SOURCES")) != null ? _ya : "true") !== "false",
|
|
2121
|
+
welcomeMessage: (_Aa = (_za = readString(env, "NEXT_PUBLIC_WELCOME_MESSAGE")) != null ? _za : readString(env, "UI_WELCOME_MESSAGE")) != null ? _Aa : "Hello! I'm your AI assistant. Ask me anything about your documents.",
|
|
2122
|
+
visualStyle: (_Ca = (_Ba = readString(env, "NEXT_PUBLIC_UI_VISUAL_STYLE")) != null ? _Ba : readString(env, "UI_VISUAL_STYLE")) != null ? _Ca : "glass",
|
|
2123
|
+
borderRadius: (_Ea = (_Da = readString(env, "NEXT_PUBLIC_UI_BORDER_RADIUS")) != null ? _Da : readString(env, "UI_BORDER_RADIUS")) != null ? _Ea : "xl",
|
|
2124
|
+
allowUpload: ((_Ga = (_Fa = readString(env, "NEXT_PUBLIC_ALLOW_UPLOAD")) != null ? _Fa : readString(env, "UI_ALLOW_UPLOAD")) != null ? _Ga : "false") === "true"
|
|
1856
2125
|
},
|
|
1857
2126
|
rag: {
|
|
1858
2127
|
topK: readNumber(env, "RAG_TOP_K", 5),
|
|
@@ -2714,7 +2983,7 @@ var LLM_PROFILES = {
|
|
|
2714
2983
|
// src/llm/providers/UniversalLLMAdapter.ts
|
|
2715
2984
|
var UniversalLLMAdapter = class {
|
|
2716
2985
|
constructor(config) {
|
|
2717
|
-
var _a, _b, _c, _d, _e, _f, _g;
|
|
2986
|
+
var _a, _b, _c, _d, _e, _f, _g, _h;
|
|
2718
2987
|
this.model = config.model;
|
|
2719
2988
|
const llmConfig = config;
|
|
2720
2989
|
const options = (_a = llmConfig.options) != null ? _a : {};
|
|
@@ -2728,16 +2997,18 @@ var UniversalLLMAdapter = class {
|
|
|
2728
2997
|
this.systemPrompt = (_c = llmConfig.systemPrompt) != null ? _c : "You are a helpful AI assistant. Use the provided context to answer the user.";
|
|
2729
2998
|
this.maxTokens = (_d = llmConfig.maxTokens) != null ? _d : 1024;
|
|
2730
2999
|
this.temperature = (_e = llmConfig.temperature) != null ? _e : 0;
|
|
2731
|
-
|
|
2732
|
-
|
|
3000
|
+
this.apiKey = config.apiKey;
|
|
3001
|
+
this.baseUrl = (_g = (_f = llmConfig.baseUrl) != null ? _f : this.opts.baseUrl) != null ? _g : "";
|
|
3002
|
+
if (!this.baseUrl) {
|
|
2733
3003
|
throw new Error("[UniversalLLMAdapter] baseUrl is required in config or config.options");
|
|
2734
3004
|
}
|
|
3005
|
+
this.resolvedHeaders = __spreadValues(__spreadValues({
|
|
3006
|
+
"Content-Type": "application/json"
|
|
3007
|
+
}, config.apiKey ? { Authorization: `Bearer ${config.apiKey}` } : {}), this.opts.headers || {});
|
|
2735
3008
|
this.http = axios2.create({
|
|
2736
|
-
baseURL: baseUrl,
|
|
2737
|
-
headers:
|
|
2738
|
-
|
|
2739
|
-
}, config.apiKey ? { Authorization: `Bearer ${config.apiKey}` } : {}), this.opts.headers || {}),
|
|
2740
|
-
timeout: (_g = this.opts.timeout) != null ? _g : 6e4
|
|
3009
|
+
baseURL: this.baseUrl,
|
|
3010
|
+
headers: this.resolvedHeaders,
|
|
3011
|
+
timeout: (_h = this.opts.timeout) != null ? _h : 6e4
|
|
2741
3012
|
});
|
|
2742
3013
|
}
|
|
2743
3014
|
async chat(messages, context) {
|
|
@@ -2774,6 +3045,92 @@ ${context != null ? context : "None"}` },
|
|
|
2774
3045
|
}
|
|
2775
3046
|
return String(result);
|
|
2776
3047
|
}
|
|
3048
|
+
/**
|
|
3049
|
+
* Streaming chat using native fetch + ReadableStream.
|
|
3050
|
+
* Parses OpenAI-compatible SSE frames: `data: {...}\n\n`
|
|
3051
|
+
* Works with vLLM, LMStudio, Together AI, Fireworks, and any OpenAI-compatible API.
|
|
3052
|
+
*/
|
|
3053
|
+
chatStream(messages, context) {
|
|
3054
|
+
return __asyncGenerator(this, null, function* () {
|
|
3055
|
+
var _a, _b, _c;
|
|
3056
|
+
const path = (_a = this.opts.chatPath) != null ? _a : "/chat/completions";
|
|
3057
|
+
const url = `${this.baseUrl.replace(/\/$/, "")}${path}`;
|
|
3058
|
+
const extractPath = ((_b = this.opts.responseExtractPath) != null ? _b : "choices[0].message.content").replace("message.content", "delta.content");
|
|
3059
|
+
const formattedMessages = [
|
|
3060
|
+
{ role: "system", content: `${this.systemPrompt}
|
|
3061
|
+
|
|
3062
|
+
Context:
|
|
3063
|
+
${context != null ? context : "None"}` },
|
|
3064
|
+
...messages
|
|
3065
|
+
];
|
|
3066
|
+
let payload;
|
|
3067
|
+
if (this.opts.chatPayloadTemplate) {
|
|
3068
|
+
payload = buildPayload(this.opts.chatPayloadTemplate, {
|
|
3069
|
+
model: this.model,
|
|
3070
|
+
messages: formattedMessages,
|
|
3071
|
+
maxTokens: this.maxTokens,
|
|
3072
|
+
temperature: this.temperature
|
|
3073
|
+
});
|
|
3074
|
+
if (typeof payload === "object" && payload !== null) {
|
|
3075
|
+
payload.stream = true;
|
|
3076
|
+
}
|
|
3077
|
+
} else {
|
|
3078
|
+
payload = {
|
|
3079
|
+
model: this.model,
|
|
3080
|
+
messages: formattedMessages,
|
|
3081
|
+
max_tokens: this.maxTokens,
|
|
3082
|
+
temperature: this.temperature,
|
|
3083
|
+
stream: true
|
|
3084
|
+
};
|
|
3085
|
+
}
|
|
3086
|
+
const response = yield new __await(fetch(url, {
|
|
3087
|
+
method: "POST",
|
|
3088
|
+
headers: this.resolvedHeaders,
|
|
3089
|
+
body: JSON.stringify(payload)
|
|
3090
|
+
}));
|
|
3091
|
+
if (!response.ok) {
|
|
3092
|
+
const errorText = yield new __await(response.text().catch(() => response.statusText));
|
|
3093
|
+
throw new Error(`[UniversalLLMAdapter] Streaming request failed (${response.status}): ${errorText}`);
|
|
3094
|
+
}
|
|
3095
|
+
if (!response.body) {
|
|
3096
|
+
throw new Error("[UniversalLLMAdapter] Response body is null \u2014 server did not send a streaming response.");
|
|
3097
|
+
}
|
|
3098
|
+
const reader = response.body.getReader();
|
|
3099
|
+
const decoder = new TextDecoder("utf-8");
|
|
3100
|
+
let buffer = "";
|
|
3101
|
+
try {
|
|
3102
|
+
while (true) {
|
|
3103
|
+
const { done, value } = yield new __await(reader.read());
|
|
3104
|
+
if (done) break;
|
|
3105
|
+
buffer += decoder.decode(value, { stream: true });
|
|
3106
|
+
const lines = buffer.split("\n");
|
|
3107
|
+
buffer = (_c = lines.pop()) != null ? _c : "";
|
|
3108
|
+
for (const line of lines) {
|
|
3109
|
+
const trimmed = line.trim();
|
|
3110
|
+
if (!trimmed || trimmed === "data: [DONE]") continue;
|
|
3111
|
+
if (!trimmed.startsWith("data:")) continue;
|
|
3112
|
+
try {
|
|
3113
|
+
const json = JSON.parse(trimmed.slice(5).trim());
|
|
3114
|
+
const text = resolvePath(json, extractPath);
|
|
3115
|
+
if (text && typeof text === "string") yield text;
|
|
3116
|
+
} catch (e) {
|
|
3117
|
+
}
|
|
3118
|
+
}
|
|
3119
|
+
}
|
|
3120
|
+
if (buffer.trim() && buffer.trim() !== "data: [DONE]") {
|
|
3121
|
+
const jsonStr = buffer.replace(/^data:\s*/, "").trim();
|
|
3122
|
+
try {
|
|
3123
|
+
const json = JSON.parse(jsonStr);
|
|
3124
|
+
const text = resolvePath(json, extractPath);
|
|
3125
|
+
if (text && typeof text === "string") yield text;
|
|
3126
|
+
} catch (e) {
|
|
3127
|
+
}
|
|
3128
|
+
}
|
|
3129
|
+
} finally {
|
|
3130
|
+
reader.releaseLock();
|
|
3131
|
+
}
|
|
3132
|
+
});
|
|
3133
|
+
}
|
|
2777
3134
|
async embed(text) {
|
|
2778
3135
|
var _a, _b;
|
|
2779
3136
|
const path = (_a = this.opts.embedPath) != null ? _a : "/embeddings";
|
|
@@ -2970,6 +3327,7 @@ var ProviderRegistry = class {
|
|
|
2970
3327
|
return null;
|
|
2971
3328
|
}
|
|
2972
3329
|
static async loadVectorProviderClass(provider) {
|
|
3330
|
+
var _a;
|
|
2973
3331
|
if (this.vectorProviders[provider]) return this.vectorProviders[provider];
|
|
2974
3332
|
switch (provider) {
|
|
2975
3333
|
case "pinecone": {
|
|
@@ -2978,6 +3336,11 @@ var ProviderRegistry = class {
|
|
|
2978
3336
|
}
|
|
2979
3337
|
case "pgvector":
|
|
2980
3338
|
case "postgresql": {
|
|
3339
|
+
const postgresMode = ((_a = process.env.POSTGRES_MODE) != null ? _a : "multi").toLowerCase();
|
|
3340
|
+
if (postgresMode === "single") {
|
|
3341
|
+
const { PostgreSQLProvider: PostgreSQLProvider2 } = await Promise.resolve().then(() => (init_PostgreSQLProvider(), PostgreSQLProvider_exports));
|
|
3342
|
+
return PostgreSQLProvider2;
|
|
3343
|
+
}
|
|
2981
3344
|
const { MultiTablePostgresProvider: MultiTablePostgresProvider2 } = await Promise.resolve().then(() => (init_MultiTablePostgresProvider(), MultiTablePostgresProvider_exports));
|
|
2982
3345
|
return MultiTablePostgresProvider2;
|
|
2983
3346
|
}
|
|
@@ -3971,6 +4334,76 @@ var QueryProcessor = class {
|
|
|
3971
4334
|
}
|
|
3972
4335
|
return [...hints.values()];
|
|
3973
4336
|
}
|
|
4337
|
+
static extractNumericPredicates(question, validFields = []) {
|
|
4338
|
+
const predicates = [];
|
|
4339
|
+
const seen = /* @__PURE__ */ new Set();
|
|
4340
|
+
const comparatorPattern = [
|
|
4341
|
+
"greater than or equal to",
|
|
4342
|
+
"more than or equal to",
|
|
4343
|
+
"less than or equal to",
|
|
4344
|
+
"greater than",
|
|
4345
|
+
"more than",
|
|
4346
|
+
"less than",
|
|
4347
|
+
"equal to",
|
|
4348
|
+
"at least",
|
|
4349
|
+
"at most",
|
|
4350
|
+
"above",
|
|
4351
|
+
"over",
|
|
4352
|
+
"below",
|
|
4353
|
+
"under",
|
|
4354
|
+
"equals?",
|
|
4355
|
+
">=",
|
|
4356
|
+
"<=",
|
|
4357
|
+
">",
|
|
4358
|
+
"<",
|
|
4359
|
+
"="
|
|
4360
|
+
].join("|");
|
|
4361
|
+
const addPredicate = (rawField, rawOperator, rawValue) => {
|
|
4362
|
+
const value = Number(rawValue.replace(/,/g, ""));
|
|
4363
|
+
if (!Number.isFinite(value)) return;
|
|
4364
|
+
const operator = this.normalizeNumericOperator(rawOperator);
|
|
4365
|
+
const field = rawField ? this.normalizePredicateField(rawField, validFields) : void 0;
|
|
4366
|
+
const key = `${field != null ? field : "*"}::${operator}::${value}`;
|
|
4367
|
+
if (seen.has(key)) return;
|
|
4368
|
+
seen.add(key);
|
|
4369
|
+
predicates.push(__spreadProps(__spreadValues({}, field ? { field } : {}), { operator, value }));
|
|
4370
|
+
};
|
|
4371
|
+
const scopedPatterns = [
|
|
4372
|
+
new RegExp(`\\b(?:whose|with|having|where|that\\s+have|which\\s+have)\\s+([a-zA-Z][a-zA-Z0-9_\\s\\-/]{1,80}?)\\s+(?:is|are|was|were)?\\s*(?:${comparatorPattern})\\s+([\\d,]+(?:\\.\\d+)?)`, "gi"),
|
|
4373
|
+
new RegExp(`\\b([a-zA-Z][a-zA-Z0-9_\\s\\-/]{1,80}?)\\s+(?:is|are|was|were)?\\s*(?:${comparatorPattern})\\s+([\\d,]+(?:\\.\\d+)?)`, "gi")
|
|
4374
|
+
];
|
|
4375
|
+
for (const pattern of scopedPatterns) {
|
|
4376
|
+
for (const match of question.matchAll(pattern)) {
|
|
4377
|
+
const full = match[0];
|
|
4378
|
+
const operatorMatch = full.match(new RegExp(`(${comparatorPattern})`, "i"));
|
|
4379
|
+
if (!operatorMatch) continue;
|
|
4380
|
+
addPredicate(match[1], operatorMatch[1], match[2]);
|
|
4381
|
+
}
|
|
4382
|
+
}
|
|
4383
|
+
for (const match of question.matchAll(/\b([a-zA-Z][a-zA-Z0-9_\s\-/]{1,80}?)\s*(>=|<=|>|<|=)\s*([\d,]+(?:\.\d+)?)/g)) {
|
|
4384
|
+
addPredicate(match[1], match[2], match[3]);
|
|
4385
|
+
}
|
|
4386
|
+
return predicates;
|
|
4387
|
+
}
|
|
4388
|
+
static normalizeNumericOperator(operator) {
|
|
4389
|
+
const op = operator.toLowerCase().trim();
|
|
4390
|
+
if (op === ">" || /\b(greater than|more than|above|over)\b/.test(op)) return "gt";
|
|
4391
|
+
if (op === ">=" || /\b(greater than or equal to|more than or equal to|at least)\b/.test(op)) return "gte";
|
|
4392
|
+
if (op === "<" || /\b(less than|below|under)\b/.test(op)) return "lt";
|
|
4393
|
+
if (op === "<=" || /\b(less than or equal to|at most)\b/.test(op)) return "lte";
|
|
4394
|
+
return "eq";
|
|
4395
|
+
}
|
|
4396
|
+
static normalizePredicateField(field, validFields) {
|
|
4397
|
+
const cleaned = this.normalizeHintValue(field).replace(/\b(?:provide|show|get|give|list|all|the|a|an|of|organizations?|companies?|records?|items?|whose|with|having|where|that|which|have|has|is|are|was|were)\b/gi, " ").replace(/\s+/g, " ").trim();
|
|
4398
|
+
if (validFields.length === 0) return cleaned;
|
|
4399
|
+
const comparable = (value) => value.toLowerCase().replace(/[^a-z0-9]/g, "");
|
|
4400
|
+
const cleanedComparable = comparable(cleaned);
|
|
4401
|
+
const matchedField = validFields.find((fieldName) => {
|
|
4402
|
+
const candidate = comparable(fieldName);
|
|
4403
|
+
return candidate === cleanedComparable || candidate.includes(cleanedComparable) || cleanedComparable.includes(candidate);
|
|
4404
|
+
});
|
|
4405
|
+
return matchedField != null ? matchedField : cleaned;
|
|
4406
|
+
}
|
|
3974
4407
|
/**
|
|
3975
4408
|
* Constructs a QueryFilter object from extracted hints.
|
|
3976
4409
|
*
|
|
@@ -3991,6 +4424,10 @@ var QueryProcessor = class {
|
|
|
3991
4424
|
}
|
|
3992
4425
|
if (Object.keys(filter.metadata || {}).length === 0) delete filter.metadata;
|
|
3993
4426
|
if (filter.keywords && filter.keywords.length === 0) delete filter.keywords;
|
|
4427
|
+
const numericPredicates = this.extractNumericPredicates(question);
|
|
4428
|
+
if (numericPredicates.length > 0) {
|
|
4429
|
+
filter.__numericPredicates = numericPredicates;
|
|
4430
|
+
}
|
|
3994
4431
|
return filter;
|
|
3995
4432
|
}
|
|
3996
4433
|
/**
|
|
@@ -4058,6 +4495,61 @@ Return ONLY 'vector', 'graph', or 'both'. No explanation.`;
|
|
|
4058
4495
|
}
|
|
4059
4496
|
};
|
|
4060
4497
|
|
|
4498
|
+
// src/core/LLMRouter.ts
|
|
4499
|
+
var FAST_MODEL_DEFAULTS = {
|
|
4500
|
+
openai: "gpt-4o-mini",
|
|
4501
|
+
gemini: "gemini-2.0-flash",
|
|
4502
|
+
anthropic: "claude-3-haiku-20240307",
|
|
4503
|
+
ollama: "",
|
|
4504
|
+
// Ollama has no universal lightweight default — reuse main model
|
|
4505
|
+
rest: "",
|
|
4506
|
+
universal_rest: "",
|
|
4507
|
+
custom: ""
|
|
4508
|
+
};
|
|
4509
|
+
var LLMRouter = class {
|
|
4510
|
+
constructor(config) {
|
|
4511
|
+
this.config = config;
|
|
4512
|
+
this.models = /* @__PURE__ */ new Map();
|
|
4513
|
+
}
|
|
4514
|
+
/**
|
|
4515
|
+
* Initialize all LLM roles.
|
|
4516
|
+
*
|
|
4517
|
+
* @param prebuiltDefault - optional pre-built provider (from EmbeddingStrategyResolver).
|
|
4518
|
+
* When provided it is used directly as the 'default' role without re-constructing.
|
|
4519
|
+
*/
|
|
4520
|
+
async initialize(prebuiltDefault) {
|
|
4521
|
+
var _a;
|
|
4522
|
+
const defaultModel = prebuiltDefault != null ? prebuiltDefault : LLMFactory.create(this.config.llm, this.config.embedding);
|
|
4523
|
+
this.models.set("default", defaultModel);
|
|
4524
|
+
const envFastModel = process.env.FAST_LLM_MODEL;
|
|
4525
|
+
const providerFastDefault = (_a = FAST_MODEL_DEFAULTS[this.config.llm.provider]) != null ? _a : "";
|
|
4526
|
+
const fastModelName = envFastModel || providerFastDefault;
|
|
4527
|
+
if (fastModelName && fastModelName !== this.config.llm.model) {
|
|
4528
|
+
console.log(`[LLMRouter] Fast role \u2192 provider="${this.config.llm.provider}" model="${fastModelName}"`);
|
|
4529
|
+
const fastConfig = __spreadProps(__spreadValues({}, this.config.llm), {
|
|
4530
|
+
model: fastModelName
|
|
4531
|
+
});
|
|
4532
|
+
this.models.set("fast", LLMFactory.create(fastConfig, this.config.embedding));
|
|
4533
|
+
} else {
|
|
4534
|
+
console.log(`[LLMRouter] Fast role \u2192 reusing default model (no lightweight alternative configured).`);
|
|
4535
|
+
this.models.set("fast", defaultModel);
|
|
4536
|
+
}
|
|
4537
|
+
this.models.set("powerful", defaultModel);
|
|
4538
|
+
}
|
|
4539
|
+
/**
|
|
4540
|
+
* Retrieve a model provider by its task role.
|
|
4541
|
+
* Falls back to 'default' if the requested role is not registered.
|
|
4542
|
+
*/
|
|
4543
|
+
get(role) {
|
|
4544
|
+
var _a;
|
|
4545
|
+
const provider = (_a = this.models.get(role)) != null ? _a : this.models.get("default");
|
|
4546
|
+
if (!provider) {
|
|
4547
|
+
throw new Error(`[LLMRouter] No provider registered for role: "${role}". Did you call initialize()?`);
|
|
4548
|
+
}
|
|
4549
|
+
return provider;
|
|
4550
|
+
}
|
|
4551
|
+
};
|
|
4552
|
+
|
|
4061
4553
|
// src/utils/synonyms.ts
|
|
4062
4554
|
var FIELD_SYNONYMS = {
|
|
4063
4555
|
name: ["product", "item", "title", "label", "heading", "subject", "product name", "item name"],
|
|
@@ -4087,6 +4579,14 @@ var FIELD_SYNONYMS = {
|
|
|
4087
4579
|
"in stock",
|
|
4088
4580
|
"status"
|
|
4089
4581
|
],
|
|
4582
|
+
category: [
|
|
4583
|
+
"product_category",
|
|
4584
|
+
"product category",
|
|
4585
|
+
"category_name",
|
|
4586
|
+
"category name",
|
|
4587
|
+
"department",
|
|
4588
|
+
"collection"
|
|
4589
|
+
],
|
|
4090
4590
|
description: ["summary", "content", "body", "text", "info", "details"],
|
|
4091
4591
|
link: ["url", "href", "product_url", "page_url", "link"]
|
|
4092
4592
|
};
|
|
@@ -4118,98 +4618,457 @@ function resolveMetadataValue(meta, uiKey) {
|
|
|
4118
4618
|
|
|
4119
4619
|
// src/utils/UITransformer.ts
|
|
4120
4620
|
var UITransformer = class {
|
|
4621
|
+
// ─── Public Entry Points ─────────────────────────────────────────────────
|
|
4121
4622
|
/**
|
|
4122
|
-
*
|
|
4123
|
-
*
|
|
4623
|
+
* Heuristic-only transform (no LLM required).
|
|
4624
|
+
* Uses the lightweight heuristic intent detector as a fallback.
|
|
4625
|
+
* Prefer `analyzeAndDecide()` in production.
|
|
4124
4626
|
*/
|
|
4125
|
-
static transform(userQuery, retrievedData, config, trainedSchema) {
|
|
4627
|
+
static transform(userQuery, retrievedData, config, trainedSchema, intent) {
|
|
4628
|
+
var _a, _b, _c;
|
|
4126
4629
|
if (!retrievedData || retrievedData.length === 0) {
|
|
4127
4630
|
return this.createTextResponse("No data available", "No relevant data found for your query.");
|
|
4128
4631
|
}
|
|
4129
|
-
const
|
|
4130
|
-
const filteredData =
|
|
4131
|
-
const
|
|
4632
|
+
const resolvedIntent = intent != null ? intent : this.detectIntentHeuristic(userQuery);
|
|
4633
|
+
const filteredData = resolvedIntent.filterInStockOnly ? retrievedData.filter((item) => this.determineStockStatus(item)) : retrievedData;
|
|
4634
|
+
const profile = this.profileData(filteredData);
|
|
4132
4635
|
const hasProducts = filteredData.some((item) => this.isProductData(item));
|
|
4133
|
-
const
|
|
4134
|
-
|
|
4135
|
-
|
|
4136
|
-
return this.transformToLineChart(filteredData);
|
|
4636
|
+
const wantsPieLikeChart = ["pie_chart", "donut_chart"].includes(resolvedIntent.recommendedChart);
|
|
4637
|
+
if (resolvedIntent.visualizationHint === "trend" && profile.dateFields.length > 0 && profile.numericFields.length > 0) {
|
|
4638
|
+
return this.transformToLineChart(profile);
|
|
4137
4639
|
}
|
|
4138
|
-
if (
|
|
4139
|
-
|
|
4640
|
+
if (wantsPieLikeChart || ["composition", "category_breakdown"].includes(resolvedIntent.visualizationHint)) {
|
|
4641
|
+
const pieChart = this.transformToPieChart(filteredData, profile, userQuery);
|
|
4642
|
+
if (pieChart) return pieChart;
|
|
4140
4643
|
}
|
|
4141
|
-
if (
|
|
4142
|
-
return this.
|
|
4644
|
+
if (["comparison", "ranking"].includes(resolvedIntent.visualizationHint)) {
|
|
4645
|
+
return this.transformToBarChart(filteredData, profile, userQuery, resolvedIntent.visualizationHint === "ranking");
|
|
4143
4646
|
}
|
|
4144
|
-
if (
|
|
4145
|
-
return this.
|
|
4647
|
+
if (resolvedIntent.visualizationHint === "distribution") {
|
|
4648
|
+
return (_a = this.transformToHistogram(profile, userQuery)) != null ? _a : this.transformToBarChart(filteredData, profile, userQuery);
|
|
4649
|
+
}
|
|
4650
|
+
if (resolvedIntent.visualizationHint === "correlation") {
|
|
4651
|
+
return (_b = this.transformToScatterPlot(profile, userQuery)) != null ? _b : this.transformToBarChart(filteredData, profile, userQuery);
|
|
4652
|
+
}
|
|
4653
|
+
if (resolvedIntent.visualizationHint === "geographic") {
|
|
4654
|
+
return this.transformToBarChart(filteredData, profile, userQuery);
|
|
4655
|
+
}
|
|
4656
|
+
if (this.isStructuredListQuery(userQuery)) {
|
|
4657
|
+
return this.hasMultipleFields(filteredData) ? this.transformToTable(filteredData, userQuery) : this.transformToText(filteredData);
|
|
4658
|
+
}
|
|
4659
|
+
if (resolvedIntent.visualizationHint === "kpi") {
|
|
4660
|
+
return (_c = this.transformToMetricCard(profile, userQuery)) != null ? _c : this.transformToText(filteredData);
|
|
4661
|
+
}
|
|
4662
|
+
if (["tabular", "table"].includes(resolvedIntent.visualizationHint) || resolvedIntent.wantsExplicitTable) {
|
|
4663
|
+
return this.hasMultipleFields(filteredData) ? this.transformToTable(filteredData, userQuery) : this.transformToText(filteredData);
|
|
4146
4664
|
}
|
|
4147
|
-
if (this.
|
|
4148
|
-
return this.
|
|
4665
|
+
if ((hasProducts || this.isProductQuery(userQuery)) && resolvedIntent.visualizationHint === "product_browse") {
|
|
4666
|
+
return this.transformToProductCarousel(filteredData, config, trainedSchema);
|
|
4149
4667
|
}
|
|
4668
|
+
const automatic = this.chooseAutomaticVisualization(filteredData, profile, userQuery);
|
|
4669
|
+
if (automatic) return automatic;
|
|
4150
4670
|
return this.transformToText(filteredData);
|
|
4151
4671
|
}
|
|
4152
4672
|
/**
|
|
4153
|
-
*
|
|
4673
|
+
* LLM-driven entry point (recommended for production).
|
|
4674
|
+
*
|
|
4675
|
+
* Step 1 — Detect intent via a dedicated, lightweight LLM call.
|
|
4676
|
+
* Step 2 — Pass the intent + data to the visualization-selection prompt.
|
|
4677
|
+
* Step 3 — Fall back to the heuristic `transform()` if either LLM call fails.
|
|
4154
4678
|
*/
|
|
4155
|
-
static
|
|
4156
|
-
|
|
4157
|
-
|
|
4158
|
-
|
|
4159
|
-
|
|
4160
|
-
|
|
4161
|
-
|
|
4162
|
-
|
|
4679
|
+
static async analyzeAndDecide(query, sources, llm) {
|
|
4680
|
+
let intent;
|
|
4681
|
+
try {
|
|
4682
|
+
intent = await this.detectIntent(query, llm);
|
|
4683
|
+
console.debug("[UITransformer] Detected intent:", intent);
|
|
4684
|
+
} catch (err) {
|
|
4685
|
+
console.warn("[UITransformer] Intent detection failed; using heuristic.", err);
|
|
4686
|
+
intent = this.detectIntentHeuristic(query);
|
|
4687
|
+
}
|
|
4688
|
+
if (this.isProductQuery(query) && ["text", "product_browse"].includes(intent.visualizationHint)) {
|
|
4689
|
+
return this.transform(
|
|
4690
|
+
query,
|
|
4691
|
+
sources,
|
|
4692
|
+
void 0,
|
|
4693
|
+
void 0,
|
|
4694
|
+
__spreadProps(__spreadValues({}, intent), { visualizationHint: "product_browse", recommendedChart: "text" })
|
|
4695
|
+
);
|
|
4696
|
+
}
|
|
4697
|
+
try {
|
|
4698
|
+
const context = this.buildContextSummary(sources);
|
|
4699
|
+
const systemPrompt = this.buildVisualizationSystemPrompt();
|
|
4700
|
+
const userPrompt = [
|
|
4701
|
+
`USER QUESTION: ${query}`,
|
|
4702
|
+
"",
|
|
4703
|
+
`DETECTED INTENT (JSON): ${JSON.stringify(intent)}`,
|
|
4704
|
+
"",
|
|
4705
|
+
"RETRIEVED DATA (JSON):",
|
|
4706
|
+
context
|
|
4707
|
+
].join("\n");
|
|
4708
|
+
const rawResponse = await llm.chat(
|
|
4709
|
+
[{ role: "user", content: userPrompt }],
|
|
4710
|
+
"",
|
|
4711
|
+
{ systemPrompt, temperature: 0 }
|
|
4712
|
+
);
|
|
4713
|
+
const parsed = this.parseTransformationResponse(rawResponse);
|
|
4714
|
+
if (parsed) {
|
|
4715
|
+
const intentAllowsTable = intent.wantsExplicitTable || ["tabular", "table", "geographic"].includes(intent.visualizationHint);
|
|
4716
|
+
const intentWantsPieLikeChart = ["pie_chart", "donut_chart"].includes(intent.recommendedChart) || ["composition", "category_breakdown"].includes(intent.visualizationHint);
|
|
4717
|
+
if (parsed.type === "table" && !intentAllowsTable) {
|
|
4718
|
+
console.debug("[UITransformer] LLM chose table but intent says no. Falling back to text.");
|
|
4719
|
+
return this.transform(query, sources, void 0, void 0, intent);
|
|
4720
|
+
}
|
|
4721
|
+
if (intentWantsPieLikeChart && parsed.type !== "pie_chart" && this.detectCategories(sources).length > 1) {
|
|
4722
|
+
console.debug("[UITransformer] LLM ignored pie/composition intent. Using deterministic pie chart.");
|
|
4723
|
+
return this.transform(query, sources, void 0, void 0, intent);
|
|
4724
|
+
}
|
|
4725
|
+
console.debug("[UITransformer] LLM chose visualization type:", parsed.type);
|
|
4726
|
+
return parsed;
|
|
4727
|
+
}
|
|
4728
|
+
console.warn("[UITransformer] LLM returned unparseable response; falling back to heuristic.");
|
|
4729
|
+
} catch (err) {
|
|
4730
|
+
console.warn("[UITransformer] analyzeAndDecide LLM call failed; falling back to heuristic.", err);
|
|
4731
|
+
}
|
|
4732
|
+
return this.transform(query, sources, void 0, void 0, intent);
|
|
4163
4733
|
}
|
|
4734
|
+
// ─── Dynamic Intent Detection ─────────────────────────────────────────────
|
|
4164
4735
|
/**
|
|
4165
|
-
*
|
|
4736
|
+
* Calls the LLM with a compact, focused prompt to extract a structured
|
|
4737
|
+
* `QueryIntent` from the user's query.
|
|
4738
|
+
*
|
|
4739
|
+
* Keeping this as a *separate* call from visualization selection means:
|
|
4740
|
+
* - The prompt is shorter and more reliable.
|
|
4741
|
+
* - The intent object can be reused across both the heuristic and LLM paths.
|
|
4742
|
+
* - It is easy to unit-test intent detection in isolation.
|
|
4166
4743
|
*/
|
|
4167
|
-
static
|
|
4168
|
-
const
|
|
4169
|
-
|
|
4170
|
-
|
|
4171
|
-
|
|
4744
|
+
static async detectIntent(query, llm) {
|
|
4745
|
+
const systemPrompt = `You are an intent classifier for a product-search RAG system.
|
|
4746
|
+
Given a user query, return ONLY a valid JSON object with this exact shape \u2014 no prose, no markdown:
|
|
4747
|
+
|
|
4748
|
+
{
|
|
4749
|
+
"visualizationHint": "trend" | "comparison" | "distribution" | "composition" | "correlation" | "ranking" | "kpi" | "tabular" | "geographic" | "product_browse" | "table" | "text",
|
|
4750
|
+
"recommendedChart": "line_chart" | "bar_chart" | "histogram" | "pie_chart" | "donut_chart" | "scatter_plot" | "horizontal_bar" | "metric_card" | "table" | "geo_map" | "text",
|
|
4751
|
+
"filterInStockOnly": boolean,
|
|
4752
|
+
"wantsExplicitTable": boolean,
|
|
4753
|
+
"isTemporal": boolean,
|
|
4754
|
+
"isComparison": boolean,
|
|
4755
|
+
"language": "<BCP-47 language tag, e.g. en, de, hi, ja>",
|
|
4756
|
+
"reasoning": "<one short sentence \u2014 why you chose these values>"
|
|
4757
|
+
}
|
|
4758
|
+
|
|
4759
|
+
RULES:
|
|
4760
|
+
- visualizationHint and recommendedChart
|
|
4761
|
+
"trend" \u2192 keywords: trend, growth, over time; recommendedChart "line_chart"
|
|
4762
|
+
"comparison" \u2192 keywords: compare, versus, top; recommendedChart "bar_chart"
|
|
4763
|
+
"distribution" \u2192 keywords: distribution, spread; recommendedChart "histogram"
|
|
4764
|
+
"composition" \u2192 keywords: share, percentage, breakup, breakdown; recommendedChart "pie_chart" or "donut_chart"
|
|
4765
|
+
"correlation" \u2192 keywords: relation, correlation; recommendedChart "scatter_plot"
|
|
4766
|
+
"ranking" \u2192 keywords: highest, lowest, top 10, ranking; recommendedChart "horizontal_bar"
|
|
4767
|
+
"kpi" \u2192 keywords: total, average, count; recommendedChart "metric_card"
|
|
4768
|
+
"tabular" \u2192 keywords: detailed records, table, grid, spreadsheet; recommendedChart "table"
|
|
4769
|
+
"geographic" \u2192 keywords: region, country, map; recommendedChart "geo_map"
|
|
4770
|
+
"product_browse" \u2192 user is browsing, searching, describing, viewing details for, or asking about one or more products without analytical visualization intent
|
|
4771
|
+
"table" \u2192 legacy alias for "tabular"; use only if the query literally says table
|
|
4772
|
+
"text" \u2192 conversational, factual, or other intent
|
|
4773
|
+
- If the user explicitly asks for a chart type, honor that chart type in recommendedChart.
|
|
4774
|
+
- If a query says "distribution ... in a pie chart", use visualizationHint "composition" and recommendedChart "pie_chart".
|
|
4775
|
+
- filterInStockOnly: true only if user mentions stock, availability, in stock, etc.
|
|
4776
|
+
- wantsExplicitTable: true if the user asks for a table, list, grid, spreadsheet, or detailed records.
|
|
4777
|
+
- isTemporal: true if time words appear (trend, historical, over time, last year, monthly, etc.)
|
|
4778
|
+
- isComparison: true if user compares, ranks, or contrasts entities or categories.
|
|
4779
|
+
- language: detect from the query text itself; default "en" if uncertain.`;
|
|
4780
|
+
const rawResponse = await llm.chat(
|
|
4781
|
+
[{ role: "user", content: `QUERY: ${query}` }],
|
|
4782
|
+
"",
|
|
4783
|
+
{ systemPrompt, temperature: 0 }
|
|
4784
|
+
);
|
|
4785
|
+
const parsed = this.parseIntentResponse(rawResponse);
|
|
4786
|
+
if (!parsed) {
|
|
4787
|
+
throw new Error(`Could not parse intent JSON from LLM response: ${rawResponse}`);
|
|
4788
|
+
}
|
|
4789
|
+
return parsed;
|
|
4790
|
+
}
|
|
4791
|
+
/**
|
|
4792
|
+
* Parse and validate the raw LLM response into a `QueryIntent`.
|
|
4793
|
+
*/
|
|
4794
|
+
static parseIntentResponse(raw) {
|
|
4795
|
+
const jsonStr = this.extractJsonCandidate(raw);
|
|
4796
|
+
if (!jsonStr) return null;
|
|
4797
|
+
try {
|
|
4798
|
+
const obj = JSON.parse(jsonStr);
|
|
4799
|
+
const validHints = [
|
|
4800
|
+
"trend",
|
|
4801
|
+
"comparison",
|
|
4802
|
+
"distribution",
|
|
4803
|
+
"composition",
|
|
4804
|
+
"correlation",
|
|
4805
|
+
"ranking",
|
|
4806
|
+
"kpi",
|
|
4807
|
+
"tabular",
|
|
4808
|
+
"geographic",
|
|
4809
|
+
"category_breakdown",
|
|
4810
|
+
"product_browse",
|
|
4811
|
+
"table",
|
|
4812
|
+
"text"
|
|
4813
|
+
];
|
|
4814
|
+
const validCharts = [
|
|
4815
|
+
"line_chart",
|
|
4816
|
+
"bar_chart",
|
|
4817
|
+
"histogram",
|
|
4818
|
+
"pie_chart",
|
|
4819
|
+
"donut_chart",
|
|
4820
|
+
"scatter_plot",
|
|
4821
|
+
"horizontal_bar",
|
|
4822
|
+
"metric_card",
|
|
4823
|
+
"table",
|
|
4824
|
+
"geo_map",
|
|
4825
|
+
"text"
|
|
4826
|
+
];
|
|
4827
|
+
const hint = obj.visualizationHint;
|
|
4828
|
+
if (!hint || !validHints.includes(hint)) return null;
|
|
4829
|
+
const normalizedHint = hint === "category_breakdown" ? "composition" : hint;
|
|
4830
|
+
const recommendedChart = typeof obj.recommendedChart === "string" && validCharts.includes(obj.recommendedChart) ? obj.recommendedChart : this.getRecommendedChartForIntent(normalizedHint);
|
|
4172
4831
|
return {
|
|
4173
|
-
|
|
4174
|
-
|
|
4175
|
-
|
|
4176
|
-
|
|
4832
|
+
visualizationHint: normalizedHint,
|
|
4833
|
+
recommendedChart,
|
|
4834
|
+
filterInStockOnly: Boolean(obj.filterInStockOnly),
|
|
4835
|
+
wantsExplicitTable: Boolean(obj.wantsExplicitTable),
|
|
4836
|
+
isTemporal: Boolean(obj.isTemporal),
|
|
4837
|
+
isComparison: Boolean(obj.isComparison),
|
|
4838
|
+
language: typeof obj.language === "string" && obj.language ? obj.language : "en",
|
|
4839
|
+
reasoning: typeof obj.reasoning === "string" ? obj.reasoning : void 0
|
|
4177
4840
|
};
|
|
4841
|
+
} catch (e) {
|
|
4842
|
+
return null;
|
|
4843
|
+
}
|
|
4844
|
+
}
|
|
4845
|
+
/**
|
|
4846
|
+
* Heuristic intent detector — used when the LLM is unavailable.
|
|
4847
|
+
* Intentionally minimal: it should only catch the most obvious signals.
|
|
4848
|
+
* The LLM path handles everything subtle.
|
|
4849
|
+
*/
|
|
4850
|
+
static detectIntentHeuristic(query) {
|
|
4851
|
+
const q = query.toLowerCase();
|
|
4852
|
+
const isTemporal = /\b(trend|trends|over time|historical|history|growth|decline|monthly|yearly|weekly|daily|last (year|month|week)|timeline|forecast)\b/.test(q);
|
|
4853
|
+
const isRanking = /\b(highest|lowest|top\s*\d+|bottom\s*\d+|rank(?:ing)?|best|worst|leading)\b/.test(q);
|
|
4854
|
+
const isComparison = /\b(compare|comparison|vs\.?|versus|against|differ|difference|contrast|top)\b/.test(q) || isRanking;
|
|
4855
|
+
const isDistribution = /\b(distribution|spread|histogram|frequency|variance|range)\b/.test(q);
|
|
4856
|
+
const wantsPieLikeChart = /\b(pie|donut|doughnut)(?:\s+chart)?\b/.test(q);
|
|
4857
|
+
const isComposition = wantsPieLikeChart || /\b(share|percentage|percent|breakup|breakdown|composition|split|segmentation|by category|by type|proportion)\b/.test(q);
|
|
4858
|
+
const isCorrelation = /\b(relation|relationship|correlation|correlate|scatter|association|impact of|depend(?:s|ence)? on)\b/.test(q);
|
|
4859
|
+
const isKpi = /\b(total|average|avg|count|sum|median|minimum|maximum|metric|kpi|how many|number of)\b/.test(q);
|
|
4860
|
+
const isGeographic = /\b(region|country|countries|state|city|location|map|geo|geographic|territory)\b/.test(q);
|
|
4861
|
+
const filterInStockOnly = /\b(in[- ]?stock|available|availability|inventory|stock status)\b/.test(q);
|
|
4862
|
+
const wantsExplicitTable = /\b(table|spreadsheet|grid|detailed records|record details|list all|compare all)\b/.test(q);
|
|
4863
|
+
let visualizationHint = "text";
|
|
4864
|
+
if (wantsExplicitTable) visualizationHint = "tabular";
|
|
4865
|
+
else if (isTemporal) visualizationHint = "trend";
|
|
4866
|
+
else if (wantsPieLikeChart) visualizationHint = "composition";
|
|
4867
|
+
else if (isRanking) visualizationHint = "ranking";
|
|
4868
|
+
else if (isComparison) visualizationHint = "comparison";
|
|
4869
|
+
else if (isDistribution) visualizationHint = "distribution";
|
|
4870
|
+
else if (isComposition) visualizationHint = "composition";
|
|
4871
|
+
else if (isCorrelation) visualizationHint = "correlation";
|
|
4872
|
+
else if (isGeographic) visualizationHint = "geographic";
|
|
4873
|
+
else if (isKpi) visualizationHint = "kpi";
|
|
4874
|
+
else if (this.isProductQuery(query)) visualizationHint = "product_browse";
|
|
4875
|
+
return {
|
|
4876
|
+
visualizationHint,
|
|
4877
|
+
recommendedChart: this.getRecommendedChartForIntent(visualizationHint),
|
|
4878
|
+
filterInStockOnly,
|
|
4879
|
+
wantsExplicitTable,
|
|
4880
|
+
isTemporal,
|
|
4881
|
+
isComparison,
|
|
4882
|
+
language: "en"
|
|
4883
|
+
// heuristic cannot reliably detect language
|
|
4884
|
+
};
|
|
4885
|
+
}
|
|
4886
|
+
static getRecommendedChartForIntent(intent) {
|
|
4887
|
+
switch (intent) {
|
|
4888
|
+
case "trend":
|
|
4889
|
+
return "line_chart";
|
|
4890
|
+
case "comparison":
|
|
4891
|
+
return "bar_chart";
|
|
4892
|
+
case "distribution":
|
|
4893
|
+
return "histogram";
|
|
4894
|
+
case "composition":
|
|
4895
|
+
case "category_breakdown":
|
|
4896
|
+
return "pie_chart";
|
|
4897
|
+
case "correlation":
|
|
4898
|
+
return "scatter_plot";
|
|
4899
|
+
case "ranking":
|
|
4900
|
+
return "horizontal_bar";
|
|
4901
|
+
case "kpi":
|
|
4902
|
+
return "metric_card";
|
|
4903
|
+
case "tabular":
|
|
4904
|
+
case "table":
|
|
4905
|
+
return "table";
|
|
4906
|
+
case "geographic":
|
|
4907
|
+
return "geo_map";
|
|
4908
|
+
case "product_browse":
|
|
4909
|
+
case "text":
|
|
4910
|
+
default:
|
|
4911
|
+
return "text";
|
|
4912
|
+
}
|
|
4913
|
+
}
|
|
4914
|
+
// ─── Transform Helpers ────────────────────────────────────────────────────
|
|
4915
|
+
static transformToProductCarousel(data, config, trainedSchema) {
|
|
4916
|
+
const products = data.map((item) => this.extractProductInfo(item, config, trainedSchema)).filter((p) => p !== null);
|
|
4917
|
+
return {
|
|
4918
|
+
type: "product_carousel",
|
|
4919
|
+
title: "Recommended Products",
|
|
4920
|
+
description: `Found ${products.length} relevant products`,
|
|
4921
|
+
data: products
|
|
4922
|
+
};
|
|
4923
|
+
}
|
|
4924
|
+
static transformToPieChart(data, profile, query = "") {
|
|
4925
|
+
var _a;
|
|
4926
|
+
const dimension = profile ? this.selectDimensionField(profile, query) : void 0;
|
|
4927
|
+
const categories = dimension ? Array.from(new Set(profile.records.map((record) => {
|
|
4928
|
+
var _a2;
|
|
4929
|
+
return String((_a2 = record.fields[dimension.key]) != null ? _a2 : "");
|
|
4930
|
+
}).filter(Boolean))) : this.detectCategories(data);
|
|
4931
|
+
if (categories.length === 0) return null;
|
|
4932
|
+
const categoryData = dimension && profile ? this.aggregateProfileByDimension(profile, dimension.key) : this.aggregateByCategory(data, categories);
|
|
4933
|
+
const pieData = Object.entries(categoryData).map(([label, count]) => {
|
|
4934
|
+
const { inStockCount, outOfStockCount } = this.calculateStockCounts(label, data);
|
|
4935
|
+
return { label, value: count, inStockCount, outOfStockCount };
|
|
4178
4936
|
});
|
|
4179
4937
|
return {
|
|
4180
4938
|
type: "pie_chart",
|
|
4181
|
-
title:
|
|
4182
|
-
description: `Showing breakdown across ${
|
|
4939
|
+
title: `Distribution by ${(_a = dimension == null ? void 0 : dimension.label) != null ? _a : "Category"}`,
|
|
4940
|
+
description: `Showing breakdown across ${pieData.length} categories`,
|
|
4183
4941
|
data: pieData
|
|
4184
4942
|
};
|
|
4185
4943
|
}
|
|
4186
|
-
|
|
4187
|
-
|
|
4188
|
-
|
|
4189
|
-
|
|
4190
|
-
|
|
4191
|
-
|
|
4192
|
-
timestamp:
|
|
4193
|
-
value:
|
|
4194
|
-
|
|
4195
|
-
})
|
|
4944
|
+
static transformToLineChart(profile) {
|
|
4945
|
+
const dateField = profile.dateFields[0];
|
|
4946
|
+
const valueField = profile.numericFields[0];
|
|
4947
|
+
const buckets = /* @__PURE__ */ new Map();
|
|
4948
|
+
profile.records.forEach((record) => {
|
|
4949
|
+
var _a, _b, _c;
|
|
4950
|
+
const timestamp = String((_a = record.fields[dateField.key]) != null ? _a : "");
|
|
4951
|
+
const value = (_b = this.toFiniteNumber(record.fields[valueField.key])) != null ? _b : 0;
|
|
4952
|
+
buckets.set(timestamp, ((_c = buckets.get(timestamp)) != null ? _c : 0) + value);
|
|
4953
|
+
});
|
|
4954
|
+
const lineData = Array.from(buckets.entries()).sort(([a], [b]) => Date.parse(a) - Date.parse(b)).slice(0, 24).map(([timestamp, value]) => ({ timestamp, value, label: timestamp }));
|
|
4196
4955
|
return {
|
|
4197
4956
|
type: "line_chart",
|
|
4198
|
-
title:
|
|
4957
|
+
title: `${valueField.label} Over Time`,
|
|
4199
4958
|
description: `Showing ${lineData.length} data points`,
|
|
4200
4959
|
data: lineData
|
|
4201
4960
|
};
|
|
4202
4961
|
}
|
|
4203
|
-
|
|
4204
|
-
|
|
4205
|
-
|
|
4206
|
-
|
|
4207
|
-
const
|
|
4208
|
-
const
|
|
4209
|
-
const
|
|
4210
|
-
|
|
4211
|
-
|
|
4962
|
+
static transformToBarChart(data, profile, query = "", horizontal = false) {
|
|
4963
|
+
var _a;
|
|
4964
|
+
const dimension = profile ? this.selectDimensionField(profile, query) : void 0;
|
|
4965
|
+
const measure = profile ? this.selectNumericField(profile, query) : void 0;
|
|
4966
|
+
const aggregate = dimension && profile ? this.aggregateProfileByDimension(profile, dimension.key, measure == null ? void 0 : measure.key) : this.aggregateByCategory(data, this.detectCategories(data));
|
|
4967
|
+
const barData = Object.entries(aggregate).map(([category, value]) => ({ category, value: Number(value) })).sort((a, b) => horizontal ? b.value - a.value : 0).slice(0, 12);
|
|
4968
|
+
const fallbackData = barData.length > 0 ? barData : data.slice(0, 12).map((item, index) => {
|
|
4969
|
+
var _a2, _b, _c, _d, _e;
|
|
4970
|
+
const meta = item.metadata || {};
|
|
4971
|
+
const label = String(
|
|
4972
|
+
(_c = (_b = (_a2 = this.getDynamicVal(meta, "name")) != null ? _a2 : meta.label) != null ? _b : meta.title) != null ? _c : `Result ${index + 1}`
|
|
4973
|
+
);
|
|
4974
|
+
const value = (_e = (_d = this.extractNumericValue(meta)) != null ? _d : item.score) != null ? _e : 0;
|
|
4975
|
+
return { category: label, value: Number(value) };
|
|
4976
|
+
});
|
|
4977
|
+
return {
|
|
4978
|
+
type: horizontal ? "horizontal_bar" : "bar_chart",
|
|
4979
|
+
title: dimension ? `${(_a = measure == null ? void 0 : measure.label) != null ? _a : "Count"} by ${dimension.label}` : "Comparison",
|
|
4980
|
+
description: `Showing ${fallbackData.length} comparable values`,
|
|
4981
|
+
data: fallbackData
|
|
4212
4982
|
};
|
|
4983
|
+
}
|
|
4984
|
+
static transformToHistogram(profile, query = "") {
|
|
4985
|
+
const field = this.selectNumericField(profile, query);
|
|
4986
|
+
if (!field) return null;
|
|
4987
|
+
const values = profile.records.map((record) => this.toFiniteNumber(record.fields[field.key])).filter((value) => value !== null).sort((a, b) => a - b);
|
|
4988
|
+
if (values.length === 0) return null;
|
|
4989
|
+
const min = values[0];
|
|
4990
|
+
const max = values[values.length - 1];
|
|
4991
|
+
const bucketCount = Math.min(10, Math.max(3, Math.ceil(Math.sqrt(values.length))));
|
|
4992
|
+
const width = max === min ? 1 : (max - min) / bucketCount;
|
|
4993
|
+
const buckets = Array.from({ length: bucketCount }, (_, index) => {
|
|
4994
|
+
const start = min + index * width;
|
|
4995
|
+
const end = index === bucketCount - 1 ? max : start + width;
|
|
4996
|
+
return { category: `${this.formatNumber(start)}-${this.formatNumber(end)}`, value: 0 };
|
|
4997
|
+
});
|
|
4998
|
+
values.forEach((value) => {
|
|
4999
|
+
const bucketIndex = max === min ? 0 : Math.min(bucketCount - 1, Math.floor((value - min) / width));
|
|
5000
|
+
buckets[bucketIndex].value += 1;
|
|
5001
|
+
});
|
|
5002
|
+
return {
|
|
5003
|
+
type: "histogram",
|
|
5004
|
+
title: `${field.label} Distribution`,
|
|
5005
|
+
description: `Showing ${values.length} values across ${bucketCount} buckets`,
|
|
5006
|
+
data: buckets
|
|
5007
|
+
};
|
|
5008
|
+
}
|
|
5009
|
+
static transformToScatterPlot(profile, query = "") {
|
|
5010
|
+
const fields = this.rankFieldsByQuery(profile.numericFields, query).slice(0, 2);
|
|
5011
|
+
if (fields.length < 2) return null;
|
|
5012
|
+
const [xField, yField] = fields;
|
|
5013
|
+
const points = profile.records.map((record) => {
|
|
5014
|
+
var _a;
|
|
5015
|
+
const x = this.toFiniteNumber(record.fields[xField.key]);
|
|
5016
|
+
const y = this.toFiniteNumber(record.fields[yField.key]);
|
|
5017
|
+
if (x === null || y === null) return null;
|
|
5018
|
+
return { x, y, label: String((_a = this.getRecordLabel(record)) != null ? _a : record.id) };
|
|
5019
|
+
}).filter((point) => point !== null).slice(0, 100);
|
|
5020
|
+
if (points.length === 0) return null;
|
|
5021
|
+
return {
|
|
5022
|
+
type: "scatter_plot",
|
|
5023
|
+
title: `${xField.label} vs ${yField.label}`,
|
|
5024
|
+
description: `Showing ${points.length} paired values`,
|
|
5025
|
+
data: points
|
|
5026
|
+
};
|
|
5027
|
+
}
|
|
5028
|
+
static transformToMetricCard(profile, query = "") {
|
|
5029
|
+
const operation = this.detectAggregationOperation(query);
|
|
5030
|
+
const numericField = this.selectNumericField(profile, query);
|
|
5031
|
+
const values = numericField ? profile.records.map((record) => this.toFiniteNumber(record.fields[numericField.key])).filter((value2) => value2 !== null) : [];
|
|
5032
|
+
const value = operation === "count" || values.length === 0 ? profile.records.length : this.calculateAggregate(values, operation);
|
|
5033
|
+
const metric = {
|
|
5034
|
+
label: numericField ? `${operation} ${numericField.label}` : "Count",
|
|
5035
|
+
value,
|
|
5036
|
+
operation
|
|
5037
|
+
};
|
|
5038
|
+
return {
|
|
5039
|
+
type: "metric_card",
|
|
5040
|
+
title: metric.label,
|
|
5041
|
+
description: `Calculated from ${profile.records.length} result${profile.records.length === 1 ? "" : "s"}`,
|
|
5042
|
+
data: metric
|
|
5043
|
+
};
|
|
5044
|
+
}
|
|
5045
|
+
static transformToRadarChart(data) {
|
|
5046
|
+
const attributeMap = {};
|
|
5047
|
+
data.forEach((item) => {
|
|
5048
|
+
var _a, _b, _c;
|
|
5049
|
+
const meta = item.metadata || {};
|
|
5050
|
+
const seriesName = String((_c = (_b = (_a = meta.name) != null ? _a : meta.product) != null ? _b : item.id) != null ? _c : "Item");
|
|
5051
|
+
Object.entries(meta).forEach(([key, val]) => {
|
|
5052
|
+
if (typeof val === "number" && !["price", "id"].includes(key.toLowerCase())) {
|
|
5053
|
+
if (!attributeMap[key]) attributeMap[key] = {};
|
|
5054
|
+
attributeMap[key][seriesName] = val;
|
|
5055
|
+
}
|
|
5056
|
+
});
|
|
5057
|
+
});
|
|
5058
|
+
const radarData = Object.entries(attributeMap).map(([attribute, series]) => __spreadValues({
|
|
5059
|
+
attribute
|
|
5060
|
+
}, series));
|
|
5061
|
+
return {
|
|
5062
|
+
type: "radar_chart",
|
|
5063
|
+
title: "Product Comparison",
|
|
5064
|
+
description: `Comparing ${data.length} items across ${radarData.length} attributes`,
|
|
5065
|
+
data: radarData.length > 0 ? radarData : data.map((d) => ({ attribute: d.content.substring(0, 40) }))
|
|
5066
|
+
};
|
|
5067
|
+
}
|
|
5068
|
+
static transformToTable(data, query = "") {
|
|
5069
|
+
const columns = this.extractTableColumns(data, query);
|
|
5070
|
+
const rows = data.map((item) => this.extractTableRow(item, columns));
|
|
5071
|
+
const tableData = { columns, rows };
|
|
4213
5072
|
return {
|
|
4214
5073
|
type: "table",
|
|
4215
5074
|
title: "Detailed Results",
|
|
@@ -4217,28 +5076,17 @@ var UITransformer = class {
|
|
|
4217
5076
|
data: tableData
|
|
4218
5077
|
};
|
|
4219
5078
|
}
|
|
4220
|
-
/**
|
|
4221
|
-
* Transform data to text format (fallback)
|
|
4222
|
-
*/
|
|
4223
5079
|
static transformToText(data) {
|
|
4224
|
-
const textContent = data.map((item) => item.content).join("\n\n");
|
|
4225
5080
|
return this.createTextResponse(
|
|
4226
5081
|
"Search Results",
|
|
4227
|
-
|
|
5082
|
+
data.map((item) => item.content).join("\n\n"),
|
|
4228
5083
|
`Found ${data.length} relevant results`
|
|
4229
5084
|
);
|
|
4230
5085
|
}
|
|
4231
|
-
/**
|
|
4232
|
-
* Helper: Create text response
|
|
4233
|
-
*/
|
|
4234
5086
|
static createTextResponse(title, content, description) {
|
|
4235
|
-
return {
|
|
4236
|
-
type: "text",
|
|
4237
|
-
title,
|
|
4238
|
-
description,
|
|
4239
|
-
data: { content }
|
|
4240
|
-
};
|
|
5087
|
+
return { type: "text", title, description, data: { content } };
|
|
4241
5088
|
}
|
|
5089
|
+
// ─── LLM Response Parsing ─────────────────────────────────────────────────
|
|
4242
5090
|
static parseTransformationResponse(raw) {
|
|
4243
5091
|
const payloadText = this.extractJsonCandidate(raw);
|
|
4244
5092
|
if (!payloadText) return null;
|
|
@@ -4275,9 +5123,7 @@ var UITransformer = class {
|
|
|
4275
5123
|
if (char === "{") depth += 1;
|
|
4276
5124
|
if (char === "}") {
|
|
4277
5125
|
depth -= 1;
|
|
4278
|
-
if (depth === 0)
|
|
4279
|
-
return cleaned.slice(start, i + 1);
|
|
4280
|
-
}
|
|
5126
|
+
if (depth === 0) return cleaned.slice(start, i + 1);
|
|
4281
5127
|
}
|
|
4282
5128
|
}
|
|
4283
5129
|
return null;
|
|
@@ -4285,19 +5131,16 @@ var UITransformer = class {
|
|
|
4285
5131
|
static normalizeTransformation(payload) {
|
|
4286
5132
|
var _a, _b, _c, _d, _e, _f, _g, _h, _i, _j;
|
|
4287
5133
|
if (!payload || typeof payload !== "object") return null;
|
|
4288
|
-
const
|
|
4289
|
-
const type = this.normalizeVisualizationType(
|
|
5134
|
+
const p = payload;
|
|
5135
|
+
const type = this.normalizeVisualizationType(
|
|
5136
|
+
String((_c = (_b = (_a = p.type) != null ? _a : p.view) != null ? _b : p.chartType) != null ? _c : "")
|
|
5137
|
+
);
|
|
4290
5138
|
if (!type) return null;
|
|
4291
|
-
const title = String((_e = (_d =
|
|
4292
|
-
const description =
|
|
4293
|
-
const rawData = (_j = (_i = (_h = (_g = (_f =
|
|
5139
|
+
const title = String((_e = (_d = p.title) != null ? _d : p.heading) != null ? _e : "Visualization");
|
|
5140
|
+
const description = p.description ? String(p.description) : void 0;
|
|
5141
|
+
const rawData = (_j = (_i = (_h = (_g = (_f = p.data) != null ? _f : p.table) != null ? _g : p.rows) != null ? _h : p.items) != null ? _i : p.content) != null ? _j : null;
|
|
4294
5142
|
const data = type === "text" && typeof rawData === "string" ? { content: rawData } : rawData;
|
|
4295
|
-
const transformation = {
|
|
4296
|
-
type,
|
|
4297
|
-
title,
|
|
4298
|
-
description,
|
|
4299
|
-
data
|
|
4300
|
-
};
|
|
5143
|
+
const transformation = { type, title, description, data };
|
|
4301
5144
|
return this.validateTransformation(transformation) ? transformation : null;
|
|
4302
5145
|
}
|
|
4303
5146
|
static normalizeVisualizationType(type) {
|
|
@@ -4307,10 +5150,23 @@ var UITransformer = class {
|
|
|
4307
5150
|
pie_chart: "pie_chart",
|
|
4308
5151
|
bar: "bar_chart",
|
|
4309
5152
|
bar_chart: "bar_chart",
|
|
5153
|
+
histogram: "histogram",
|
|
5154
|
+
horizontal_bar: "horizontal_bar",
|
|
5155
|
+
horizontalbar: "horizontal_bar",
|
|
4310
5156
|
line: "line_chart",
|
|
4311
5157
|
line_chart: "line_chart",
|
|
5158
|
+
scatter: "scatter_plot",
|
|
5159
|
+
scatter_plot: "scatter_plot",
|
|
5160
|
+
scatterplot: "scatter_plot",
|
|
4312
5161
|
radar: "radar_chart",
|
|
4313
5162
|
radar_chart: "radar_chart",
|
|
5163
|
+
metric: "metric_card",
|
|
5164
|
+
metric_card: "metric_card",
|
|
5165
|
+
card: "metric_card",
|
|
5166
|
+
kpi: "metric_card",
|
|
5167
|
+
geo: "geo_map",
|
|
5168
|
+
geo_map: "geo_map",
|
|
5169
|
+
map: "geo_map",
|
|
4314
5170
|
table: "table",
|
|
4315
5171
|
text: "text",
|
|
4316
5172
|
product_carousel: "product_carousel",
|
|
@@ -4318,21 +5174,31 @@ var UITransformer = class {
|
|
|
4318
5174
|
};
|
|
4319
5175
|
return (_a = mapping[type.toLowerCase()]) != null ? _a : null;
|
|
4320
5176
|
}
|
|
4321
|
-
static validateTransformation(
|
|
4322
|
-
const { type, data } =
|
|
5177
|
+
static validateTransformation(t) {
|
|
5178
|
+
const { type, data } = t;
|
|
4323
5179
|
switch (type) {
|
|
4324
5180
|
case "pie_chart":
|
|
4325
5181
|
case "bar_chart":
|
|
5182
|
+
case "histogram":
|
|
5183
|
+
case "horizontal_bar":
|
|
4326
5184
|
return Array.isArray(data) && data.every(
|
|
4327
|
-
(
|
|
5185
|
+
(i) => i !== null && typeof i === "object" && (typeof i.value === "number" || !Number.isNaN(Number(i.value)))
|
|
5186
|
+
);
|
|
5187
|
+
case "scatter_plot":
|
|
5188
|
+
return Array.isArray(data) && data.every(
|
|
5189
|
+
(i) => i !== null && typeof i === "object" && (typeof i.x === "number" || !Number.isNaN(Number(i.x))) && (typeof i.y === "number" || !Number.isNaN(Number(i.y)))
|
|
4328
5190
|
);
|
|
4329
5191
|
case "line_chart":
|
|
4330
5192
|
return Array.isArray(data) && data.every(
|
|
4331
|
-
(
|
|
5193
|
+
(i) => i !== null && typeof i === "object" && "timestamp" in i && (typeof i.value === "number" || !Number.isNaN(Number(i.value)))
|
|
4332
5194
|
);
|
|
5195
|
+
case "metric_card":
|
|
5196
|
+
return typeof data === "object" && data !== null && (typeof data.value === "number" || !Number.isNaN(Number(data.value)));
|
|
5197
|
+
case "geo_map":
|
|
5198
|
+
return Array.isArray(data) || typeof data === "object" && data !== null;
|
|
4333
5199
|
case "radar_chart":
|
|
4334
5200
|
return Array.isArray(data) && data.every(
|
|
4335
|
-
(
|
|
5201
|
+
(i) => i !== null && typeof i === "object" && "attribute" in i
|
|
4336
5202
|
);
|
|
4337
5203
|
case "table":
|
|
4338
5204
|
return typeof data === "object" && data !== null && Array.isArray(data.columns) && Array.isArray(data.rows);
|
|
@@ -4341,15 +5207,27 @@ var UITransformer = class {
|
|
|
4341
5207
|
case "product_carousel":
|
|
4342
5208
|
case "carousel":
|
|
4343
5209
|
return Array.isArray(data) && data.every(
|
|
4344
|
-
(
|
|
5210
|
+
(i) => i !== null && typeof i === "object" && ("id" in i || "name" in i)
|
|
4345
5211
|
);
|
|
4346
5212
|
default:
|
|
4347
5213
|
return false;
|
|
4348
5214
|
}
|
|
4349
5215
|
}
|
|
4350
|
-
|
|
4351
|
-
|
|
4352
|
-
|
|
5216
|
+
// ─── Data Inspection Helpers ──────────────────────────────────────────────
|
|
5217
|
+
static isStructuredListQuery(query) {
|
|
5218
|
+
const q = query.toLowerCase();
|
|
5219
|
+
const asksForRecords = /\b(list|provide|show|give|get|find|which|whose|records?|details?)\b/.test(q);
|
|
5220
|
+
const hasNumericPredicate = /\b(greater than|more than|above|over|less than|below|under|at least|at most|minimum|maximum|>=|<=|>|<|equals?|equal to)\b\s*\$?\d/i.test(q) || /\b\d+\b\s*(?:or more|or less|and above|and below)\b/i.test(q);
|
|
5221
|
+
const hasEntityTerms = /\b(organization|organisations?|organizations?|company|companies|employee|employees|staff|headcount|workforce)\b/.test(q);
|
|
5222
|
+
return asksForRecords && (hasNumericPredicate || hasEntityTerms);
|
|
5223
|
+
}
|
|
5224
|
+
static isProductQuery(query) {
|
|
5225
|
+
const q = query.toLowerCase();
|
|
5226
|
+
const productTerms = /\b(product|products|item|items|sku|catalog|catalogue|price|prices|brand|model|stock|inventory|in stock|out of stock|buy|shop)\b/.test(q);
|
|
5227
|
+
const productAction = /\b(recommend|suggest|describe|description|detail|details|about|show|find|search|browse|list)\b/.test(q);
|
|
5228
|
+
const nonProductEntityTerms = /\b(organization|organisations?|organizations?|company|companies|employee|employees|staff|headcount|workforce)\b/.test(q);
|
|
5229
|
+
return productTerms && productAction && !nonProductEntityTerms;
|
|
5230
|
+
}
|
|
4353
5231
|
static isProductData(item) {
|
|
4354
5232
|
const content = (item.content || "").toLowerCase();
|
|
4355
5233
|
const productKeywords = [
|
|
@@ -4375,360 +5253,582 @@ var UITransformer = class {
|
|
|
4375
5253
|
"fragrance"
|
|
4376
5254
|
];
|
|
4377
5255
|
const hasKeywords = productKeywords.some((kw) => content.includes(kw));
|
|
4378
|
-
const
|
|
4379
|
-
|
|
4380
|
-
|
|
4381
|
-
|
|
5256
|
+
const metadata = item.metadata || {};
|
|
5257
|
+
const hasMetadataKey = Object.keys(metadata).some((k) => {
|
|
5258
|
+
const val = metadata[k];
|
|
5259
|
+
return ["price", "product", "sku", "brand", "model", "cost", "item"].includes(k.toLowerCase()) && val !== null && val !== void 0 && val !== "";
|
|
5260
|
+
});
|
|
5261
|
+
const hasPricePattern = /\$\s*\d+\.\d{2}/.test(content);
|
|
4382
5262
|
return hasKeywords || hasMetadataKey || hasPricePattern;
|
|
4383
5263
|
}
|
|
4384
|
-
/**
|
|
4385
|
-
* Helper: Check if data contains time series
|
|
4386
|
-
*/
|
|
4387
5264
|
static isTimeSeriesData(item) {
|
|
4388
|
-
const content = (item.content || "").toLowerCase();
|
|
4389
|
-
const timeKeywords = ["trend", "historical", "growth", "decline", "change", "increase", "decrease"];
|
|
4390
|
-
const hasTimeKeyword = timeKeywords.some((kw) => content.includes(kw));
|
|
4391
5265
|
const metadata = item.metadata || {};
|
|
4392
5266
|
const maybeDateKeys = Object.keys(metadata).filter(
|
|
4393
5267
|
(k) => ["date", "timestamp", "time", "period"].includes(k.toLowerCase())
|
|
4394
5268
|
);
|
|
4395
|
-
|
|
5269
|
+
return maybeDateKeys.some((key) => {
|
|
4396
5270
|
const value = metadata[key];
|
|
4397
|
-
if (typeof value === "string")
|
|
4398
|
-
return !Number.isNaN(Date.parse(value)) && value.trim().length > 0;
|
|
4399
|
-
}
|
|
5271
|
+
if (typeof value === "string") return !Number.isNaN(Date.parse(value)) && value.trim().length > 0;
|
|
4400
5272
|
return typeof value === "number";
|
|
4401
5273
|
});
|
|
4402
|
-
return hasTimeKeyword || hasValidDateValue;
|
|
4403
5274
|
}
|
|
4404
|
-
static
|
|
4405
|
-
|
|
4406
|
-
|
|
4407
|
-
|
|
4408
|
-
const normalized = query.toLowerCase();
|
|
4409
|
-
const chartKeywords = [
|
|
4410
|
-
"distribution",
|
|
4411
|
-
"breakdown",
|
|
4412
|
-
"by category",
|
|
4413
|
-
"by type",
|
|
4414
|
-
"compare",
|
|
4415
|
-
"share",
|
|
4416
|
-
"percentage",
|
|
4417
|
-
"segmentation",
|
|
4418
|
-
"split",
|
|
4419
|
-
"category breakdown",
|
|
4420
|
-
"category distribution"
|
|
4421
|
-
];
|
|
4422
|
-
return chartKeywords.some((keyword) => normalized.includes(keyword));
|
|
5275
|
+
static hasMultipleFields(data) {
|
|
5276
|
+
const fieldCount = /* @__PURE__ */ new Set();
|
|
5277
|
+
data.forEach((item) => Object.keys(item.metadata || {}).forEach((k) => fieldCount.add(k)));
|
|
5278
|
+
return fieldCount.size > 2;
|
|
4423
5279
|
}
|
|
4424
|
-
static
|
|
4425
|
-
const
|
|
4426
|
-
|
|
4427
|
-
|
|
4428
|
-
|
|
4429
|
-
|
|
4430
|
-
|
|
4431
|
-
|
|
4432
|
-
|
|
4433
|
-
|
|
4434
|
-
|
|
4435
|
-
|
|
4436
|
-
|
|
4437
|
-
|
|
4438
|
-
|
|
4439
|
-
|
|
4440
|
-
|
|
4441
|
-
|
|
4442
|
-
|
|
4443
|
-
|
|
5280
|
+
static profileData(data) {
|
|
5281
|
+
const records = data.map((item) => {
|
|
5282
|
+
const fields2 = {};
|
|
5283
|
+
Object.entries(item.metadata || {}).forEach(([key, value]) => {
|
|
5284
|
+
const primitive = this.toPrimitive(value);
|
|
5285
|
+
if (primitive !== null) fields2[key] = primitive;
|
|
5286
|
+
});
|
|
5287
|
+
if (!fields2.content && item.content) fields2.content = item.content.substring(0, 500);
|
|
5288
|
+
return {
|
|
5289
|
+
id: item.id,
|
|
5290
|
+
content: item.content,
|
|
5291
|
+
score: item.score,
|
|
5292
|
+
fields: fields2,
|
|
5293
|
+
source: item
|
|
5294
|
+
};
|
|
5295
|
+
});
|
|
5296
|
+
const keys = Array.from(new Set(records.flatMap((record) => Object.keys(record.fields))));
|
|
5297
|
+
const fields = keys.map((key) => {
|
|
5298
|
+
const values = records.map((record) => record.fields[key]).filter((value) => value !== void 0 && value !== null && String(value).trim() !== "");
|
|
5299
|
+
const uniqueCount = new Set(values.map((value) => String(value).toLowerCase())).size;
|
|
5300
|
+
return {
|
|
5301
|
+
key,
|
|
5302
|
+
label: this.humanizeFieldName(key),
|
|
5303
|
+
kind: this.inferFieldKind(key, values, records.length, uniqueCount),
|
|
5304
|
+
values,
|
|
5305
|
+
uniqueCount
|
|
5306
|
+
};
|
|
5307
|
+
});
|
|
5308
|
+
return {
|
|
5309
|
+
records,
|
|
5310
|
+
fields,
|
|
5311
|
+
numericFields: fields.filter((field) => field.kind === "number"),
|
|
5312
|
+
dateFields: fields.filter((field) => field.kind === "date"),
|
|
5313
|
+
categoricalFields: fields.filter((field) => field.kind === "category"),
|
|
5314
|
+
booleanFields: fields.filter((field) => field.kind === "boolean")
|
|
5315
|
+
};
|
|
4444
5316
|
}
|
|
4445
|
-
static
|
|
4446
|
-
|
|
4447
|
-
|
|
5317
|
+
static toPrimitive(value) {
|
|
5318
|
+
if (typeof value === "string" || typeof value === "number" || typeof value === "boolean") return value;
|
|
5319
|
+
if (value === null || value === void 0) return null;
|
|
5320
|
+
if (value instanceof Date) return value.toISOString();
|
|
5321
|
+
return null;
|
|
4448
5322
|
}
|
|
4449
|
-
|
|
4450
|
-
|
|
4451
|
-
|
|
4452
|
-
|
|
4453
|
-
|
|
4454
|
-
if (!meta) return void 0;
|
|
4455
|
-
const mapping = (_a = config == null ? void 0 : config.rag) == null ? void 0 : _a.uiMapping;
|
|
4456
|
-
if (mapping && mapping[uiKey]) {
|
|
4457
|
-
const mappedKey = mapping[uiKey];
|
|
4458
|
-
if (meta[mappedKey] !== void 0) return meta[mappedKey];
|
|
5323
|
+
static inferFieldKind(key, values, rowCount, uniqueCount) {
|
|
5324
|
+
const lowerKey = key.toLowerCase();
|
|
5325
|
+
if (values.length === 0) return "text";
|
|
5326
|
+
if (values.every((value) => typeof value === "boolean" || /^(true|false|yes|no|in_stock|out_of_stock|available|unavailable)$/i.test(String(value)))) {
|
|
5327
|
+
return "boolean";
|
|
4459
5328
|
}
|
|
4460
|
-
if (
|
|
4461
|
-
|
|
4462
|
-
if (trainedKey && meta[trainedKey] !== void 0) return meta[trainedKey];
|
|
5329
|
+
if (/(date|time|timestamp|created|updated|month|year|period)/i.test(lowerKey) && values.some((value) => this.isDateLike(value))) {
|
|
5330
|
+
return "date";
|
|
4463
5331
|
}
|
|
4464
|
-
|
|
5332
|
+
const numericCount = values.filter((value) => this.toFiniteNumber(value) !== null).length;
|
|
5333
|
+
if (numericCount / values.length >= 0.85 && !/(id|sku|ean|upc|zip|postal|phone|code)/i.test(lowerKey)) {
|
|
5334
|
+
return "number";
|
|
5335
|
+
}
|
|
5336
|
+
if (uniqueCount <= Math.max(20, Math.ceil(rowCount * 0.7)) && !/(id|sku|ean|upc|description|content|url|image|thumbnail)/i.test(lowerKey)) {
|
|
5337
|
+
return "category";
|
|
5338
|
+
}
|
|
5339
|
+
return "text";
|
|
4465
5340
|
}
|
|
4466
|
-
static
|
|
4467
|
-
|
|
4468
|
-
const
|
|
4469
|
-
|
|
4470
|
-
|
|
4471
|
-
|
|
4472
|
-
|
|
4473
|
-
|
|
4474
|
-
|
|
4475
|
-
|
|
4476
|
-
|
|
4477
|
-
|
|
4478
|
-
|
|
4479
|
-
|
|
4480
|
-
|
|
4481
|
-
|
|
4482
|
-
|
|
4483
|
-
return
|
|
4484
|
-
|
|
4485
|
-
|
|
4486
|
-
|
|
4487
|
-
image: typeof imageValue === "string" ? imageValue : void 0,
|
|
4488
|
-
brand: brand ? String(brand) : void 0,
|
|
4489
|
-
description: item.content,
|
|
4490
|
-
inStock: this.determineStockStatus(item)
|
|
4491
|
-
};
|
|
5341
|
+
static isDateLike(value) {
|
|
5342
|
+
if (typeof value === "number") return value > 1900 && value < 3e3;
|
|
5343
|
+
const text = String(value).trim();
|
|
5344
|
+
return text.length > 3 && !Number.isNaN(Date.parse(text));
|
|
5345
|
+
}
|
|
5346
|
+
static chooseAutomaticVisualization(data, profile, query) {
|
|
5347
|
+
if (profile.records.length === 0) return null;
|
|
5348
|
+
if (profile.dateFields.length > 0 && profile.numericFields.length > 0) {
|
|
5349
|
+
return this.transformToLineChart(profile);
|
|
5350
|
+
}
|
|
5351
|
+
if (profile.categoricalFields.length > 0 && profile.numericFields.length > 0) {
|
|
5352
|
+
return this.transformToBarChart(data, profile, query);
|
|
5353
|
+
}
|
|
5354
|
+
if (profile.categoricalFields.length > 0) {
|
|
5355
|
+
return this.transformToPieChart(data, profile, query);
|
|
5356
|
+
}
|
|
5357
|
+
if (profile.numericFields.length >= 2) {
|
|
5358
|
+
return this.transformToScatterPlot(profile, query);
|
|
5359
|
+
}
|
|
5360
|
+
if (profile.numericFields.length === 1) {
|
|
5361
|
+
return this.transformToMetricCard(profile, query);
|
|
4492
5362
|
}
|
|
4493
5363
|
return null;
|
|
4494
5364
|
}
|
|
4495
|
-
|
|
4496
|
-
|
|
4497
|
-
|
|
5365
|
+
static selectDimensionField(profile, query) {
|
|
5366
|
+
var _a, _b;
|
|
5367
|
+
const productCategory = profile.categoricalFields.find(
|
|
5368
|
+
(field) => /category|department|collection|type|group|segment|region|country|state|city/i.test(field.key)
|
|
5369
|
+
);
|
|
5370
|
+
const ranked = this.rankFieldsByQuery(profile.categoricalFields, query);
|
|
5371
|
+
return (_b = (_a = ranked[0]) != null ? _a : productCategory) != null ? _b : profile.categoricalFields[0];
|
|
5372
|
+
}
|
|
5373
|
+
static selectNumericField(profile, query) {
|
|
5374
|
+
var _a;
|
|
5375
|
+
return (_a = this.rankFieldsByQuery(profile.numericFields, query)[0]) != null ? _a : profile.numericFields[0];
|
|
5376
|
+
}
|
|
5377
|
+
static rankFieldsByQuery(fields, query) {
|
|
5378
|
+
const q = query.toLowerCase();
|
|
5379
|
+
return [...fields].sort((a, b) => this.fieldScore(b, q) - this.fieldScore(a, q));
|
|
5380
|
+
}
|
|
5381
|
+
static fieldScore(field, query) {
|
|
5382
|
+
const key = field.key.toLowerCase();
|
|
5383
|
+
const label = field.label.toLowerCase();
|
|
5384
|
+
let score = 0;
|
|
5385
|
+
if (query.includes(key)) score += 4;
|
|
5386
|
+
if (query.includes(label)) score += 4;
|
|
5387
|
+
key.split(/[_\s-]+/).forEach((part) => {
|
|
5388
|
+
if (part && query.includes(part)) score += 1;
|
|
5389
|
+
});
|
|
5390
|
+
if (/category|department|collection|type|group|segment/.test(key)) score += 2;
|
|
5391
|
+
if (/count|total|quantity|stock|inventory|sales|revenue|amount|price|value|score/.test(key)) score += 2;
|
|
5392
|
+
if (/id|sku|ean|upc|code/.test(key)) score -= 5;
|
|
5393
|
+
return score;
|
|
5394
|
+
}
|
|
5395
|
+
static normalizeComparableField(field) {
|
|
5396
|
+
return field.toLowerCase().replace(/&/g, "and").replace(/[^a-z0-9]+/g, "").trim();
|
|
5397
|
+
}
|
|
5398
|
+
static fieldTokens(field) {
|
|
5399
|
+
return field.toLowerCase().replace(/[_-]+/g, " ").split(/\s+/).map((token) => token.replace(/[^a-z0-9]/g, "")).filter(Boolean);
|
|
5400
|
+
}
|
|
5401
|
+
static aggregateProfileByDimension(profile, dimensionKey, measureKey) {
|
|
5402
|
+
const result = {};
|
|
5403
|
+
profile.records.forEach((record) => {
|
|
5404
|
+
var _a, _b, _c;
|
|
5405
|
+
const category = String((_a = record.fields[dimensionKey]) != null ? _a : "Other").trim() || "Other";
|
|
5406
|
+
const value = measureKey ? (_b = this.toFiniteNumber(record.fields[measureKey])) != null ? _b : 0 : 1;
|
|
5407
|
+
result[category] = ((_c = result[category]) != null ? _c : 0) + value;
|
|
5408
|
+
});
|
|
5409
|
+
return result;
|
|
5410
|
+
}
|
|
5411
|
+
static getRecordLabel(record) {
|
|
5412
|
+
const labelKeys = ["name", "title", "label", "product", "item", "brand"];
|
|
5413
|
+
const key = Object.keys(record.fields).find(
|
|
5414
|
+
(fieldKey) => labelKeys.some((labelKey) => fieldKey.toLowerCase().includes(labelKey))
|
|
5415
|
+
);
|
|
5416
|
+
return key ? record.fields[key] : void 0;
|
|
5417
|
+
}
|
|
5418
|
+
static detectAggregationOperation(query) {
|
|
5419
|
+
const q = query.toLowerCase();
|
|
5420
|
+
if (/\b(avg|average|mean)\b/.test(q)) return "average";
|
|
5421
|
+
if (/\b(count|how many|number of)\b/.test(q)) return "count";
|
|
5422
|
+
if (/\b(min|minimum|lowest)\b/.test(q)) return "min";
|
|
5423
|
+
if (/\b(max|maximum|highest)\b/.test(q)) return "max";
|
|
5424
|
+
if (/\bmedian\b/.test(q)) return "median";
|
|
5425
|
+
return "sum";
|
|
5426
|
+
}
|
|
5427
|
+
static calculateAggregate(values, operation) {
|
|
5428
|
+
if (values.length === 0) return 0;
|
|
5429
|
+
switch (operation) {
|
|
5430
|
+
case "average":
|
|
5431
|
+
return values.reduce((sum, value) => sum + value, 0) / values.length;
|
|
5432
|
+
case "count":
|
|
5433
|
+
return values.length;
|
|
5434
|
+
case "min":
|
|
5435
|
+
return Math.min(...values);
|
|
5436
|
+
case "max":
|
|
5437
|
+
return Math.max(...values);
|
|
5438
|
+
case "median": {
|
|
5439
|
+
const sorted = [...values].sort((a, b) => a - b);
|
|
5440
|
+
const middle = Math.floor(sorted.length / 2);
|
|
5441
|
+
return sorted.length % 2 === 0 ? (sorted[middle - 1] + sorted[middle]) / 2 : sorted[middle];
|
|
5442
|
+
}
|
|
5443
|
+
case "sum":
|
|
5444
|
+
default:
|
|
5445
|
+
return values.reduce((sum, value) => sum + value, 0);
|
|
5446
|
+
}
|
|
5447
|
+
}
|
|
5448
|
+
static humanizeFieldName(key) {
|
|
5449
|
+
return key.replace(/[_-]+/g, " ").replace(/([a-z])([A-Z])/g, "$1 $2").replace(/\s+/g, " ").trim().replace(/\b\w/g, (char) => char.toUpperCase());
|
|
5450
|
+
}
|
|
5451
|
+
static formatNumber(value) {
|
|
5452
|
+
return Number.isInteger(value) ? String(value) : value.toFixed(1);
|
|
5453
|
+
}
|
|
4498
5454
|
static detectCategories(data) {
|
|
4499
5455
|
const categories = /* @__PURE__ */ new Set();
|
|
4500
5456
|
data.forEach((item) => {
|
|
4501
|
-
const
|
|
4502
|
-
if (
|
|
4503
|
-
categories.add(String(meta.category));
|
|
4504
|
-
}
|
|
4505
|
-
if (meta.type) {
|
|
4506
|
-
categories.add(String(meta.type));
|
|
4507
|
-
}
|
|
4508
|
-
if (meta.tag) {
|
|
4509
|
-
const tags = Array.isArray(meta.tag) ? meta.tag : [meta.tag];
|
|
4510
|
-
tags.forEach((t) => categories.add(String(t)));
|
|
4511
|
-
}
|
|
4512
|
-
const contentCategories = Array.from(new Set(
|
|
4513
|
-
Array.from(item.content.matchAll(/^\s*([^:\n]+):\s*(?:•|\*|-)*/gm)).map((match) => match[1].trim()).filter(Boolean)
|
|
4514
|
-
));
|
|
4515
|
-
contentCategories.forEach((category) => categories.add(category));
|
|
4516
|
-
const categoryMatch = item.content.match(/(?:Category|Type|Class):\s*([^\n]+)/i);
|
|
4517
|
-
if (categoryMatch) {
|
|
4518
|
-
categories.add(categoryMatch[1].trim());
|
|
4519
|
-
}
|
|
5457
|
+
const category = this.getProductCategory(item);
|
|
5458
|
+
if (category) categories.add(category);
|
|
4520
5459
|
});
|
|
4521
5460
|
return Array.from(categories);
|
|
4522
5461
|
}
|
|
4523
|
-
|
|
4524
|
-
|
|
4525
|
-
|
|
5462
|
+
static getProductCategory(item) {
|
|
5463
|
+
const meta = item.metadata || {};
|
|
5464
|
+
const metadataCategory = resolveMetadataValue(meta, "category");
|
|
5465
|
+
if (this.isUsableCategory(metadataCategory)) return String(metadataCategory).trim();
|
|
5466
|
+
const content = item.content || "";
|
|
5467
|
+
const explicitCategoryMatch = content.match(
|
|
5468
|
+
/(?:^|\n)\s*(?:product\s*)?(?:category|department|collection)\s*[:\-–]\s*([^\n]+)/i
|
|
5469
|
+
);
|
|
5470
|
+
if (explicitCategoryMatch && this.isUsableCategory(explicitCategoryMatch[1])) {
|
|
5471
|
+
return explicitCategoryMatch[1].trim();
|
|
5472
|
+
}
|
|
5473
|
+
return null;
|
|
5474
|
+
}
|
|
5475
|
+
static isUsableCategory(value) {
|
|
5476
|
+
if (value === null || value === void 0) return false;
|
|
5477
|
+
const category = String(value).trim();
|
|
5478
|
+
if (!category) return false;
|
|
5479
|
+
return !/^(category|type|tag|price|cost|stock|availability|description|product|item|name|brand|sku|id|ean|currency|color|size|index|internal id|other|\[?type\]?|\[?products?_?\d*\]?)$/i.test(category);
|
|
5480
|
+
}
|
|
4526
5481
|
static aggregateByCategory(data, categories) {
|
|
4527
|
-
const result =
|
|
4528
|
-
categories.forEach((cat) => {
|
|
4529
|
-
result[cat] = 0;
|
|
4530
|
-
});
|
|
5482
|
+
const result = Object.fromEntries(categories.map((c) => [c, 0]));
|
|
4531
5483
|
data.forEach((item) => {
|
|
4532
|
-
|
|
4533
|
-
const
|
|
4534
|
-
if (Object.prototype.hasOwnProperty.call(result,
|
|
4535
|
-
|
|
4536
|
-
} else {
|
|
4537
|
-
result["Other"] = (result["Other"] || 0) + 1;
|
|
4538
|
-
}
|
|
5484
|
+
var _a;
|
|
5485
|
+
const cat = (_a = this.getProductCategory(item)) != null ? _a : "Other";
|
|
5486
|
+
if (Object.prototype.hasOwnProperty.call(result, cat)) result[cat]++;
|
|
5487
|
+
else result["Other"] = (result["Other"] || 0) + 1;
|
|
4539
5488
|
});
|
|
4540
5489
|
return result;
|
|
4541
5490
|
}
|
|
4542
|
-
/**
|
|
4543
|
-
* Helper: Extract time series data
|
|
4544
|
-
*/
|
|
4545
5491
|
static extractTimeSeriesData(data) {
|
|
4546
5492
|
return data.map((item) => {
|
|
5493
|
+
var _a, _b, _c, _d, _e;
|
|
4547
5494
|
const meta = item.metadata || {};
|
|
4548
5495
|
return {
|
|
4549
|
-
timestamp: meta.timestamp
|
|
4550
|
-
value: meta.value
|
|
4551
|
-
label: meta.label
|
|
5496
|
+
timestamp: (_b = (_a = meta.timestamp) != null ? _a : meta.date) != null ? _b : (/* @__PURE__ */ new Date()).toISOString(),
|
|
5497
|
+
value: (_d = (_c = meta.value) != null ? _c : item.score) != null ? _d : 0,
|
|
5498
|
+
label: (_e = meta.label) != null ? _e : item.content.substring(0, 50)
|
|
4552
5499
|
};
|
|
4553
5500
|
});
|
|
4554
5501
|
}
|
|
4555
|
-
|
|
4556
|
-
|
|
4557
|
-
|
|
4558
|
-
|
|
4559
|
-
|
|
4560
|
-
|
|
4561
|
-
|
|
4562
|
-
|
|
4563
|
-
|
|
4564
|
-
|
|
4565
|
-
|
|
4566
|
-
|
|
5502
|
+
static extractNumericValue(meta) {
|
|
5503
|
+
var _a;
|
|
5504
|
+
const preferredKeys = [
|
|
5505
|
+
"value",
|
|
5506
|
+
"count",
|
|
5507
|
+
"total",
|
|
5508
|
+
"average",
|
|
5509
|
+
"avg",
|
|
5510
|
+
"amount",
|
|
5511
|
+
"sales",
|
|
5512
|
+
"revenue",
|
|
5513
|
+
"score",
|
|
5514
|
+
"quantity",
|
|
5515
|
+
"price"
|
|
5516
|
+
];
|
|
5517
|
+
for (const key of preferredKeys) {
|
|
5518
|
+
const raw = (_a = resolveMetadataValue(meta, key)) != null ? _a : meta[key];
|
|
5519
|
+
const value = typeof raw === "number" ? raw : Number(String(raw != null ? raw : "").replace(/[$,% ,]/g, ""));
|
|
5520
|
+
if (Number.isFinite(value)) return value;
|
|
5521
|
+
}
|
|
5522
|
+
for (const value of Object.values(meta)) {
|
|
5523
|
+
const numeric = typeof value === "number" ? value : Number(String(value != null ? value : "").replace(/[$,% ,]/g, ""));
|
|
5524
|
+
if (Number.isFinite(numeric)) return numeric;
|
|
5525
|
+
}
|
|
5526
|
+
return null;
|
|
5527
|
+
}
|
|
5528
|
+
static extractTableColumns(data, query = "") {
|
|
5529
|
+
const q = query.toLowerCase();
|
|
5530
|
+
const availableFields = this.extractAvailableTableFields(data);
|
|
5531
|
+
if (/\b(organization|organisations?|organizations?|company|companies)\b/.test(q) && /\b(employee|employees|staff|headcount|workforce)\b/.test(q)) {
|
|
5532
|
+
const nameField = this.pickTableField(availableFields, [
|
|
5533
|
+
"company name",
|
|
5534
|
+
"organization name",
|
|
5535
|
+
"organisation name",
|
|
5536
|
+
"name",
|
|
5537
|
+
"company",
|
|
5538
|
+
"organization",
|
|
5539
|
+
"organisation"
|
|
5540
|
+
]);
|
|
5541
|
+
const employeeField = this.pickTableField(availableFields, [
|
|
5542
|
+
"number of employees",
|
|
5543
|
+
"employee count",
|
|
5544
|
+
"employees",
|
|
5545
|
+
"employee_count",
|
|
5546
|
+
"number_employees",
|
|
5547
|
+
"num employees",
|
|
5548
|
+
"staff count",
|
|
5549
|
+
"headcount",
|
|
5550
|
+
"workforce"
|
|
5551
|
+
]);
|
|
5552
|
+
const columns = [nameField, employeeField].filter((field) => Boolean(field));
|
|
5553
|
+
if (columns.length > 0) return columns;
|
|
5554
|
+
}
|
|
5555
|
+
const requestedFields = availableFields.map((field) => ({
|
|
5556
|
+
field,
|
|
5557
|
+
score: this.tableFieldQueryScore(field, q)
|
|
5558
|
+
})).filter((item) => item.score > 0).sort((a, b) => b.score - a.score).map((item) => item.field);
|
|
5559
|
+
if (requestedFields.length > 0) {
|
|
5560
|
+
return requestedFields.slice(0, Math.min(6, requestedFields.length));
|
|
5561
|
+
}
|
|
5562
|
+
const metadataFields = Array.from(new Set(
|
|
5563
|
+
data.flatMap((item) => Object.keys(item.metadata || {}).map((k) => this.humanizeFieldName(k)))
|
|
5564
|
+
));
|
|
5565
|
+
return metadataFields.length > 0 ? metadataFields : ["Content"];
|
|
4567
5566
|
}
|
|
4568
|
-
/**
|
|
4569
|
-
* Helper: Extract table row
|
|
4570
|
-
*/
|
|
4571
5567
|
static extractTableRow(item, columns) {
|
|
4572
|
-
|
|
4573
|
-
|
|
4574
|
-
|
|
4575
|
-
|
|
5568
|
+
return columns.map((col) => this.resolveTableCellValue(item, col));
|
|
5569
|
+
}
|
|
5570
|
+
static extractAvailableTableFields(data) {
|
|
5571
|
+
const fields = /* @__PURE__ */ new Map();
|
|
5572
|
+
const addField = (field) => {
|
|
5573
|
+
const clean = this.humanizeFieldName(field);
|
|
5574
|
+
if (!clean || /^(content|\[?type\]?)$/i.test(clean)) return;
|
|
5575
|
+
const normalized = this.normalizeComparableField(clean);
|
|
5576
|
+
if (!fields.has(normalized)) fields.set(normalized, clean);
|
|
5577
|
+
};
|
|
5578
|
+
data.forEach((item) => {
|
|
5579
|
+
Object.keys(item.metadata || {}).forEach(addField);
|
|
5580
|
+
for (const match of (item.content || "").matchAll(/(?:^|\n)\s*([A-Za-z][A-Za-z0-9_ /-]{1,50})\s*[:\-–]\s*([^\n]+)/g)) {
|
|
5581
|
+
addField(match[1]);
|
|
4576
5582
|
}
|
|
4577
|
-
const metaKey = col.charAt(0).toLowerCase() + col.slice(1);
|
|
4578
|
-
const value = meta[metaKey];
|
|
4579
|
-
return value !== void 0 ? String(value) : "";
|
|
4580
5583
|
});
|
|
5584
|
+
return Array.from(fields.values());
|
|
5585
|
+
}
|
|
5586
|
+
static pickTableField(fields, aliases) {
|
|
5587
|
+
const normalizedAliases = aliases.map((alias) => this.normalizeComparableField(alias));
|
|
5588
|
+
for (const alias of normalizedAliases) {
|
|
5589
|
+
const exact = fields.find((field) => this.normalizeComparableField(field) === alias);
|
|
5590
|
+
if (exact) return exact;
|
|
5591
|
+
}
|
|
5592
|
+
for (const alias of normalizedAliases) {
|
|
5593
|
+
const fuzzy = fields.find((field) => {
|
|
5594
|
+
const normalizedField = this.normalizeComparableField(field);
|
|
5595
|
+
if (/(id|code|uuid)$/.test(normalizedField) && !/(id|code|uuid)$/.test(alias)) return false;
|
|
5596
|
+
return normalizedField.includes(alias) || alias.includes(normalizedField);
|
|
5597
|
+
});
|
|
5598
|
+
if (fuzzy) return fuzzy;
|
|
5599
|
+
}
|
|
5600
|
+
return void 0;
|
|
5601
|
+
}
|
|
5602
|
+
static tableFieldQueryScore(field, query) {
|
|
5603
|
+
const normalizedField = this.normalizeComparableField(field);
|
|
5604
|
+
if (!normalizedField) return 0;
|
|
5605
|
+
const fieldTokens = this.fieldTokens(field);
|
|
5606
|
+
return fieldTokens.reduce((score, token) => {
|
|
5607
|
+
if (token.length < 3) return score;
|
|
5608
|
+
return query.includes(token) ? score + 1 : score;
|
|
5609
|
+
}, query.includes(normalizedField) ? 3 : 0);
|
|
5610
|
+
}
|
|
5611
|
+
static resolveTableCellValue(item, column) {
|
|
5612
|
+
if (column === "Content") return item.content.substring(0, 100);
|
|
5613
|
+
const meta = item.metadata || {};
|
|
5614
|
+
const normalizedColumn = this.normalizeComparableField(column);
|
|
5615
|
+
const exactMetadata = Object.entries(meta).find(
|
|
5616
|
+
([key]) => this.normalizeComparableField(key) === normalizedColumn || this.normalizeComparableField(this.humanizeFieldName(key)) === normalizedColumn
|
|
5617
|
+
);
|
|
5618
|
+
if (exactMetadata && exactMetadata[1] !== void 0 && exactMetadata[1] !== null) {
|
|
5619
|
+
return this.toDisplayValue(exactMetadata[1]);
|
|
5620
|
+
}
|
|
5621
|
+
const aliasValue = this.resolveAliasedTableCell(meta, column);
|
|
5622
|
+
if (aliasValue !== void 0) return this.toDisplayValue(aliasValue);
|
|
5623
|
+
const contentValue = this.extractContentFieldValue(item.content, column);
|
|
5624
|
+
if (contentValue !== null) return contentValue;
|
|
5625
|
+
const aliasContentValue = this.extractAliasedContentFieldValue(item.content, column);
|
|
5626
|
+
return aliasContentValue != null ? aliasContentValue : "";
|
|
5627
|
+
}
|
|
5628
|
+
static resolveAliasedTableCell(meta, column) {
|
|
5629
|
+
const normalizedColumn = this.normalizeComparableField(column);
|
|
5630
|
+
const aliases = normalizedColumn.includes("employee") ? ["number of employees", "employee count", "employees", "headcount", "staff count", "workforce"] : normalizedColumn.includes("name") ? ["company name", "organization name", "organisation name", "name", "company", "organization", "organisation"] : [];
|
|
5631
|
+
for (const alias of aliases) {
|
|
5632
|
+
const normalizedAlias = this.normalizeComparableField(alias);
|
|
5633
|
+
const match = Object.entries(meta).find(([key]) => {
|
|
5634
|
+
const normalizedKey = this.normalizeComparableField(key);
|
|
5635
|
+
return normalizedKey === normalizedAlias || normalizedKey.includes(normalizedAlias) || normalizedAlias.includes(normalizedKey);
|
|
5636
|
+
});
|
|
5637
|
+
if (match && match[1] !== void 0 && match[1] !== null) return match[1];
|
|
5638
|
+
}
|
|
5639
|
+
return void 0;
|
|
5640
|
+
}
|
|
5641
|
+
static extractContentFieldValue(content, column) {
|
|
5642
|
+
const escaped = column.replace(/[.*+?^${}()|[\]\\]/g, "\\$&").replace(/\s+/g, "[\\s_ -]+");
|
|
5643
|
+
const pattern = new RegExp(`(?:^|\\n)\\s*${escaped}\\s*[:\\-\u2013]\\s*([^\\n]+)`, "i");
|
|
5644
|
+
const match = content.match(pattern);
|
|
5645
|
+
return match ? match[1].trim() : null;
|
|
5646
|
+
}
|
|
5647
|
+
static extractAliasedContentFieldValue(content, column) {
|
|
5648
|
+
const normalizedColumn = this.normalizeComparableField(column);
|
|
5649
|
+
const aliases = normalizedColumn.includes("employee") ? ["Number Of Employees", "Number of Employees", "Employee Count", "Employees", "Headcount", "Staff Count", "Workforce"] : normalizedColumn.includes("name") ? ["Company Name", "Organization Name", "Organisation Name", "Name", "Company", "Organization", "Organisation"] : [];
|
|
5650
|
+
for (const alias of aliases) {
|
|
5651
|
+
const value = this.extractContentFieldValue(content, alias);
|
|
5652
|
+
if (value !== null) return value;
|
|
5653
|
+
}
|
|
5654
|
+
return null;
|
|
5655
|
+
}
|
|
5656
|
+
static toDisplayValue(value) {
|
|
5657
|
+
if (typeof value === "string" || typeof value === "number" || typeof value === "boolean") return value;
|
|
5658
|
+
return String(value != null ? value : "");
|
|
4581
5659
|
}
|
|
4582
|
-
/**
|
|
4583
|
-
* Helper: Calculate stock counts by category
|
|
4584
|
-
*/
|
|
4585
5660
|
static calculateStockCounts(category, data) {
|
|
4586
5661
|
let inStock = 0;
|
|
4587
5662
|
let outOfStock = 0;
|
|
4588
5663
|
data.forEach((d) => {
|
|
4589
|
-
|
|
4590
|
-
const
|
|
4591
|
-
if (
|
|
4592
|
-
|
|
4593
|
-
|
|
4594
|
-
|
|
4595
|
-
outOfStock++;
|
|
4596
|
-
}
|
|
5664
|
+
var _a, _b;
|
|
5665
|
+
const cat = (_a = this.getProductCategory(d)) != null ? _a : "Other";
|
|
5666
|
+
if (cat === category) {
|
|
5667
|
+
const quantity = (_b = this.extractStockQuantity(d)) != null ? _b : 1;
|
|
5668
|
+
if (this.determineStockStatus(d)) inStock += quantity;
|
|
5669
|
+
else outOfStock += quantity;
|
|
4597
5670
|
}
|
|
4598
5671
|
});
|
|
4599
5672
|
return { inStockCount: inStock, outOfStockCount: outOfStock };
|
|
4600
5673
|
}
|
|
4601
|
-
/**
|
|
4602
|
-
* Helper: Determine if item is in stock
|
|
4603
|
-
*/
|
|
4604
5674
|
static determineStockStatus(item) {
|
|
4605
5675
|
const meta = item.metadata || {};
|
|
4606
|
-
|
|
4607
|
-
|
|
4608
|
-
|
|
4609
|
-
|
|
4610
|
-
|
|
4611
|
-
|
|
4612
|
-
|
|
4613
|
-
|
|
4614
|
-
|
|
5676
|
+
const stockValue = resolveMetadataValue(meta, "stock");
|
|
5677
|
+
if (stockValue !== void 0) {
|
|
5678
|
+
const normalized = String(stockValue).toLowerCase();
|
|
5679
|
+
if (/out[_\s-]?of[_\s-]?stock|unavailable|false|no|sold out|0/.test(normalized)) return false;
|
|
5680
|
+
if (/in[_\s-]?stock|available|true|yes/.test(normalized)) return true;
|
|
5681
|
+
const numeric = this.toFiniteNumber(stockValue);
|
|
5682
|
+
if (numeric !== null) return numeric > 0;
|
|
5683
|
+
}
|
|
5684
|
+
if (meta.inStock !== void 0) return Boolean(meta.inStock);
|
|
5685
|
+
if (meta.stock !== void 0) return Boolean(meta.stock);
|
|
5686
|
+
if (meta.available !== void 0) return Boolean(meta.available);
|
|
4615
5687
|
const content = (item.content || "").toLowerCase();
|
|
4616
|
-
if (
|
|
4617
|
-
|
|
5688
|
+
if (/out[_\s-]?of[_\s-]?stock|unavailable|sold out/.test(content)) return false;
|
|
5689
|
+
if (/in[_\s-]?stock|available/.test(content)) return true;
|
|
5690
|
+
return true;
|
|
5691
|
+
}
|
|
5692
|
+
static extractStockQuantity(item) {
|
|
5693
|
+
const meta = item.metadata || {};
|
|
5694
|
+
const stockValue = resolveMetadataValue(meta, "stock");
|
|
5695
|
+
const numericStock = this.toFiniteNumber(stockValue);
|
|
5696
|
+
if (numericStock !== null) return numericStock;
|
|
5697
|
+
const content = item.content || "";
|
|
5698
|
+
const stockMatch = content.match(/\b(?:stock|inventory|quantity|count)\s*[:\-–]?\s*([\d,]+)/i);
|
|
5699
|
+
if (!stockMatch) return null;
|
|
5700
|
+
return this.toFiniteNumber(stockMatch[1]);
|
|
5701
|
+
}
|
|
5702
|
+
static toFiniteNumber(value) {
|
|
5703
|
+
if (typeof value === "number") return Number.isFinite(value) ? value : null;
|
|
5704
|
+
const numeric = Number(String(value != null ? value : "").replace(/[$,% ,]/g, ""));
|
|
5705
|
+
return Number.isFinite(numeric) ? numeric : null;
|
|
5706
|
+
}
|
|
5707
|
+
// ─── Product Extraction ───────────────────────────────────────────────────
|
|
5708
|
+
static getDynamicVal(meta, uiKey, config, trainedSchema) {
|
|
5709
|
+
var _a;
|
|
5710
|
+
if (!meta) return void 0;
|
|
5711
|
+
const mapping = (_a = config == null ? void 0 : config.rag) == null ? void 0 : _a.uiMapping;
|
|
5712
|
+
if ((mapping == null ? void 0 : mapping[uiKey]) && meta[mapping[uiKey]] !== void 0) return meta[mapping[uiKey]];
|
|
5713
|
+
if (trainedSchema && typeof trainedSchema === "object") {
|
|
5714
|
+
const trainedKey = trainedSchema[uiKey];
|
|
5715
|
+
if (trainedKey && meta[trainedKey] !== void 0) return meta[trainedKey];
|
|
4618
5716
|
}
|
|
4619
|
-
|
|
4620
|
-
|
|
5717
|
+
return resolveMetadataValue(meta, uiKey);
|
|
5718
|
+
}
|
|
5719
|
+
static extractProductInfo(item, config, trainedSchema) {
|
|
5720
|
+
var _a;
|
|
5721
|
+
const meta = item.metadata || {};
|
|
5722
|
+
const name = this.getDynamicVal(meta, "name", config, trainedSchema);
|
|
5723
|
+
const price = this.getDynamicVal(meta, "price", config, trainedSchema);
|
|
5724
|
+
const brand = this.getDynamicVal(meta, "brand", config, trainedSchema);
|
|
5725
|
+
const description = this.cleanProductDescription(
|
|
5726
|
+
(_a = this.extractProductDescriptionFromContent(item.content)) != null ? _a : this.getProductDescriptionValue(meta, config, trainedSchema)
|
|
5727
|
+
);
|
|
5728
|
+
if (name || this.isProductData(item)) {
|
|
5729
|
+
let finalName = name ? String(name) : void 0;
|
|
5730
|
+
if (!finalName) {
|
|
5731
|
+
const nameMatch = item.content.match(/(?:Product|Item|Name):\s*([^\n]+)/i);
|
|
5732
|
+
finalName = nameMatch ? nameMatch[1].trim() : item.content.split("\n")[0].substring(0, 60);
|
|
5733
|
+
}
|
|
5734
|
+
let finalPrice = typeof price === "number" || typeof price === "string" ? price : void 0;
|
|
5735
|
+
if (!finalPrice) {
|
|
5736
|
+
const priceMatch = item.content.match(/(?:Price|Cost):\s*\$?([\d,.]+)/i) || item.content.match(/\$\s*([\d,.]+)/);
|
|
5737
|
+
if (priceMatch) finalPrice = priceMatch[1].replace(/,/g, "");
|
|
5738
|
+
}
|
|
5739
|
+
const imageValue = this.getDynamicVal(meta, "image", config, trainedSchema);
|
|
5740
|
+
return {
|
|
5741
|
+
id: item.id,
|
|
5742
|
+
name: finalName,
|
|
5743
|
+
price: finalPrice,
|
|
5744
|
+
image: typeof imageValue === "string" ? imageValue : void 0,
|
|
5745
|
+
brand: brand ? String(brand) : void 0,
|
|
5746
|
+
description,
|
|
5747
|
+
inStock: this.determineStockStatus(item)
|
|
5748
|
+
};
|
|
4621
5749
|
}
|
|
4622
|
-
return
|
|
5750
|
+
return null;
|
|
4623
5751
|
}
|
|
4624
|
-
|
|
4625
|
-
|
|
4626
|
-
|
|
4627
|
-
|
|
4628
|
-
|
|
4629
|
-
|
|
4630
|
-
|
|
4631
|
-
|
|
4632
|
-
|
|
4633
|
-
|
|
4634
|
-
return fieldCount.size > 2;
|
|
5752
|
+
static extractProductDescriptionFromContent(content) {
|
|
5753
|
+
const bodyMatch = content.match(
|
|
5754
|
+
/\bBody\s*\(HTML\)\s*:\s*([\s\S]*?)(?=\s*[,.]?\s+\b(?:Handle|Title|Vendor|Type|Tags|Published|Option\d*\s+Name|Option\d*\s+Value|Option|Variant|Image|SKU|Price|Status|Category)\b\s*:|$)/i
|
|
5755
|
+
);
|
|
5756
|
+
if (bodyMatch == null ? void 0 : bodyMatch[1]) return bodyMatch[1];
|
|
5757
|
+
const descriptionMatch = content.match(
|
|
5758
|
+
/\b(?:Description|Summary|Details|Body|Content)\s*:\s*([\s\S]*?)(?=\s*[,.]?\s+\b(?:Handle|Title|Vendor|Type|Tags|Published|Option\d*\s+Name|Option\d*\s+Value|Option|Variant|Image|SKU|Price|Status|Category)\b\s*:|$)/i
|
|
5759
|
+
);
|
|
5760
|
+
if (descriptionMatch == null ? void 0 : descriptionMatch[1]) return descriptionMatch[1];
|
|
5761
|
+
return null;
|
|
4635
5762
|
}
|
|
4636
|
-
|
|
4637
|
-
|
|
4638
|
-
|
|
4639
|
-
|
|
4640
|
-
|
|
4641
|
-
|
|
4642
|
-
|
|
4643
|
-
|
|
4644
|
-
|
|
4645
|
-
|
|
4646
|
-
|
|
4647
|
-
|
|
4648
|
-
|
|
4649
|
-
|
|
4650
|
-
|
|
4651
|
-
* @param sources - vector DB matches returned by RAG retrieval
|
|
4652
|
-
* @param llm - any ILLMProvider instance (OpenAI, Anthropic, Ollama, Gemini, REST…)
|
|
4653
|
-
* @returns - a validated UITransformationResponse (type + title + description + data)
|
|
4654
|
-
*/
|
|
4655
|
-
static async analyzeAndDecide(query, sources, llm) {
|
|
4656
|
-
try {
|
|
4657
|
-
const context = this.buildContextSummary(sources);
|
|
4658
|
-
const systemPrompt = this.buildVisualizationSystemPrompt();
|
|
4659
|
-
const userPrompt = [
|
|
4660
|
-
`USER QUESTION: ${query}`,
|
|
4661
|
-
"",
|
|
4662
|
-
"RETRIEVED DATA (JSON):",
|
|
4663
|
-
context
|
|
4664
|
-
].join("\n");
|
|
4665
|
-
const rawResponse = await llm.chat(
|
|
4666
|
-
[{ role: "user", content: userPrompt }],
|
|
4667
|
-
"",
|
|
4668
|
-
{ systemPrompt, temperature: 0 }
|
|
5763
|
+
static getProductDescriptionValue(meta, config, trainedSchema) {
|
|
5764
|
+
const mapped = this.getDynamicVal(meta, "description", config, trainedSchema);
|
|
5765
|
+
if (mapped !== void 0 && mapped !== meta.content && mapped !== meta.text) return mapped;
|
|
5766
|
+
const preferredKeys = [
|
|
5767
|
+
"body_html",
|
|
5768
|
+
"body html",
|
|
5769
|
+
"bodyHtml",
|
|
5770
|
+
"description",
|
|
5771
|
+
"summary",
|
|
5772
|
+
"details",
|
|
5773
|
+
"body"
|
|
5774
|
+
];
|
|
5775
|
+
for (const key of preferredKeys) {
|
|
5776
|
+
const match = Object.keys(meta).find(
|
|
5777
|
+
(candidate) => this.normalizeComparableField(candidate) === this.normalizeComparableField(key)
|
|
4669
5778
|
);
|
|
4670
|
-
|
|
4671
|
-
if (parsed) {
|
|
4672
|
-
console.debug("[UITransformer] LLM chose visualization type:", parsed.type);
|
|
4673
|
-
return parsed;
|
|
4674
|
-
}
|
|
4675
|
-
console.warn("[UITransformer] LLM returned unparseable response; falling back to heuristic.");
|
|
4676
|
-
} catch (err) {
|
|
4677
|
-
console.warn("[UITransformer] analyzeAndDecide LLM call failed; falling back to heuristic.", err);
|
|
5779
|
+
if (match && meta[match] !== void 0 && meta[match] !== null) return meta[match];
|
|
4678
5780
|
}
|
|
4679
|
-
return
|
|
5781
|
+
return void 0;
|
|
4680
5782
|
}
|
|
4681
|
-
|
|
4682
|
-
|
|
4683
|
-
|
|
5783
|
+
static cleanProductDescription(raw) {
|
|
5784
|
+
if (raw === null || raw === void 0) return void 0;
|
|
5785
|
+
const extracted = this.extractProductDescriptionFromContent(String(raw));
|
|
5786
|
+
if (extracted && extracted !== String(raw)) return this.cleanProductDescription(extracted);
|
|
5787
|
+
const text = String(raw).replace(/<[^>]*>/g, " ").replace(/ /gi, " ").replace(/&/gi, "&").replace(/"/gi, '"').replace(/'/gi, "'").replace(/\[[^\]]*TYPE[^\]]*\]/gi, " ").replace(/\b(?:Handle|Title|Vendor|Type|Tags|Published|Option\d*\s+Name|Option\d*\s+Value|Option|Variant|Image|SKU|Price|Status|Category)\s*:\s*[^,.\n:]+[,.]?/gi, " ").replace(/\bBody\s*\(HTML\)\s*:\s*/gi, " ").replace(/\b(?:Description|Summary|Details|Body|Content)\s*:\s*/gi, " ").replace(/\s+/g, " ").trim();
|
|
5788
|
+
return text || void 0;
|
|
5789
|
+
}
|
|
5790
|
+
// ─── Visualization Prompt ─────────────────────────────────────────────────
|
|
4684
5791
|
static buildVisualizationSystemPrompt() {
|
|
4685
5792
|
return `You are a data visualization expert embedded in a RAG chat system.
|
|
4686
|
-
You will receive a user question and structured data
|
|
4687
|
-
Your ONLY job is to
|
|
4688
|
-
|
|
5793
|
+
You will receive a user question, a pre-computed INTENT object, and structured data from a vector database.
|
|
5794
|
+
Your ONLY job is to return a single JSON object describing how to visualize the data.
|
|
5795
|
+
The INTENT object is authoritative \u2014 honor it. For example:
|
|
5796
|
+
- If intent.wantsExplicitTable is false, never return type "table".
|
|
5797
|
+
- If intent.isTemporal is true, prefer line_chart.
|
|
5798
|
+
- If intent.visualizationHint is "comparison" or "ranking", prefer bar_chart.
|
|
5799
|
+
- If intent.visualizationHint is "composition", prefer pie_chart.
|
|
5800
|
+
- If intent.recommendedChart is unsupported by the response schema, use the nearest supported type.
|
|
5801
|
+
- Always respond in the language specified by intent.language.
|
|
4689
5802
|
|
|
4690
|
-
Return ONLY a valid JSON object \u2014 no markdown
|
|
5803
|
+
Return ONLY a valid JSON object \u2014 no markdown fences, no prose.
|
|
4691
5804
|
|
|
4692
|
-
The JSON must have this exact shape:
|
|
4693
5805
|
{
|
|
4694
|
-
"type": "bar_chart" | "line_chart" | "pie_chart" | "radar_chart" | "table" | "text",
|
|
4695
|
-
"title": "
|
|
4696
|
-
"description": "
|
|
4697
|
-
"data": <structured data
|
|
5806
|
+
"type": "bar_chart" | "horizontal_bar" | "line_chart" | "pie_chart" | "histogram" | "scatter_plot" | "radar_chart" | "metric_card" | "geo_map" | "table" | "text",
|
|
5807
|
+
"title": "concise descriptive title",
|
|
5808
|
+
"description": "one sentence describing the visualization",
|
|
5809
|
+
"data": <structured data per type below>
|
|
4698
5810
|
}
|
|
4699
5811
|
|
|
4700
|
-
DATA
|
|
4701
|
-
- bar_chart:
|
|
4702
|
-
-
|
|
4703
|
-
-
|
|
4704
|
-
-
|
|
4705
|
-
|
|
4706
|
-
|
|
4707
|
-
|
|
4708
|
-
|
|
4709
|
-
|
|
4710
|
-
|
|
4711
|
-
-
|
|
4712
|
-
- text: { "content": "<prose answer>" }
|
|
4713
|
-
|
|
4714
|
-
DECISION RULES (follow strictly):
|
|
4715
|
-
1. bar_chart \u2192 comparing quantities across categories (e.g. sales by region, price by product). Use this when there is only ONE value per category.
|
|
4716
|
-
2. line_chart \u2192 trends or changes over time (dates, months, years, sequential events)
|
|
4717
|
-
3. pie_chart \u2192 proportional breakdown or percentage distribution
|
|
4718
|
-
4. radar_chart \u2192 comparing 2 or more products across MULTIPLE attributes (e.g. comparing features, ratings, or characteristics of 2 specific products). If the user asks to "compare" products and the data contains multiple dimensions or ratings for them, you MUST use this.
|
|
4719
|
-
5. table \u2192 multi-field structured records where each item has \u2265 3 attributes
|
|
4720
|
-
6. text \u2192 conversational or free-form answers where no chart adds value
|
|
5812
|
+
DATA SHAPES:
|
|
5813
|
+
- bar_chart: [{ "category": string, "value": number }]
|
|
5814
|
+
- horizontal_bar: [{ "category": string, "value": number }]
|
|
5815
|
+
- histogram: [{ "category": string, "value": number }]
|
|
5816
|
+
- line_chart: [{ "timestamp": string, "value": number, "label": string }]
|
|
5817
|
+
- pie_chart: [{ "label": string, "value": number }]
|
|
5818
|
+
- scatter_plot:[{ "x": number, "y": number, "label": string }]
|
|
5819
|
+
- radar_chart: [{ "attribute": string, "<series1>": number, "<series2>": number, ... }]
|
|
5820
|
+
- metric_card: { "label": string, "value": number, "operation": "sum"|"average"|"count"|"min"|"max"|"median" }
|
|
5821
|
+
- geo_map: [{ "category": string, "value": number }]
|
|
5822
|
+
- table: { "columns": string[], "rows": (string|number)[][] }
|
|
5823
|
+
- text: { "content": "A concise plain-language answer." }
|
|
4721
5824
|
|
|
4722
|
-
|
|
4723
|
-
|
|
4724
|
-
|
|
4725
|
-
|
|
4726
|
-
|
|
5825
|
+
RULES:
|
|
5826
|
+
1. Aggregate values \u2014 never pass raw nested objects in chart arrays.
|
|
5827
|
+
2. All "value" fields must be numbers.
|
|
5828
|
+
3. Cap bar/line/pie at 12 data points.
|
|
5829
|
+
4. Use dollar signs only for monetary prices, not for stock quantities or years.
|
|
5830
|
+
5. If no relevant data is found, return text: "I cannot answer this question based on the available information."`;
|
|
4727
5831
|
}
|
|
4728
|
-
/**
|
|
4729
|
-
* Serialize retrieved vector matches into a compact JSON context string.
|
|
4730
|
-
* Limits the total character count to avoid exceeding LLM context windows.
|
|
4731
|
-
*/
|
|
4732
5832
|
static buildContextSummary(sources, maxChars = 6e3) {
|
|
4733
5833
|
const items = sources.map((s, i) => {
|
|
4734
5834
|
var _a, _b, _c, _d;
|
|
@@ -4949,27 +6049,38 @@ var Pipeline = class {
|
|
|
4949
6049
|
async initialize() {
|
|
4950
6050
|
var _a;
|
|
4951
6051
|
if (this.initialised) return;
|
|
4952
|
-
const chartInstruction = `You are a helpful
|
|
6052
|
+
const chartInstruction = `You are a helpful assistant. Use the provided context to answer questions accurately.
|
|
4953
6053
|
|
|
4954
|
-
###
|
|
6054
|
+
### CRITICAL RULES:
|
|
6055
|
+
- ONLY answer the user's specific question. Do NOT suggest or recommend unrelated products unless specifically asked.
|
|
4955
6056
|
- NEVER generate markdown tables. If you do, the UI will break.
|
|
4956
6057
|
- NEVER generate HTML tags like <figure>, <tbody>, <tr>, etc.
|
|
4957
6058
|
- NEVER generate text-based charts or graphs.
|
|
6059
|
+
- NEVER say you cannot display, render, draw, or create a chart when the user asks for a visualization. The UI handles visual rendering separately.
|
|
4958
6060
|
- ONLY use plain text and bullet points.
|
|
6061
|
+
- NEVER use the plus sign (+) as a separator between names, categories, or products. Use commas or bullet points.
|
|
6062
|
+
- ONLY answer the question if the sources contain relevant information to answer it, else say that you cannot answer the question.
|
|
6063
|
+
- If answer cannot be found in the sources, say that you cannot answer the question and do not hallucinate.
|
|
6064
|
+
- Do not use information from previous turns to answer the question.
|
|
6065
|
+
- You CAN use numbers for years and counts (e.g., 2006, 5800). But NEVER put a dollar sign ($) before non-monetary numbers like Stock quantities, EAN numbers, or years. Only use dollar signs for actual monetary prices.
|
|
4959
6066
|
|
|
4960
6067
|
### PRODUCT DISPLAY:
|
|
4961
|
-
-
|
|
6068
|
+
- Recommended Products should only be displayed when the user explicitly asks for products.
|
|
6069
|
+
- When recommending products (ONLY when asked), simply list their names, prices, and features in a friendly, conversational manner.
|
|
4962
6070
|
- The UI will automatically detect these products and show high-quality product cards in a carousel below your message.
|
|
6071
|
+
- For product descriptions, summarize customer-facing details only. Do NOT list internal catalog fields such as Handle, Body (HTML), Vendor, Type, Tags, Published, Option, Variant, SKU, or raw metadata labels.
|
|
4963
6072
|
- Do NOT try to format product lists as tables.
|
|
4964
6073
|
`;
|
|
4965
6074
|
this.config.llm.systemPrompt = chartInstruction;
|
|
4966
6075
|
this.vectorDB = await ProviderRegistry.createVectorProvider(this.config.vectorDb);
|
|
4967
|
-
|
|
6076
|
+
this.llmRouter = new LLMRouter(this.config);
|
|
6077
|
+
const { llmProvider: resolvedLLM, embeddingProvider } = await EmbeddingStrategyResolver.resolve(
|
|
4968
6078
|
this.config.llm,
|
|
4969
6079
|
this.config.embedding
|
|
4970
6080
|
);
|
|
4971
|
-
this.llmProvider = llmProvider;
|
|
4972
6081
|
this.embeddingProvider = embeddingProvider;
|
|
6082
|
+
await this.llmRouter.initialize(resolvedLLM);
|
|
6083
|
+
this.llmProvider = this.llmRouter.get("default");
|
|
4973
6084
|
if (this.config.graphDb) {
|
|
4974
6085
|
this.graphDB = await ProviderRegistry.createGraphProvider(this.config.graphDb);
|
|
4975
6086
|
await this.graphDB.initialize();
|
|
@@ -5126,10 +6237,16 @@ var Pipeline = class {
|
|
|
5126
6237
|
/**
|
|
5127
6238
|
* High-performance streaming RAG flow.
|
|
5128
6239
|
* Yields text chunks first, then the retrieval metadata + observability trace at the end.
|
|
6240
|
+
*
|
|
6241
|
+
* Latency optimizations:
|
|
6242
|
+
* - Strategy classification runs in parallel with query embedding (saves ~400ms)
|
|
6243
|
+
* - Hallucination scoring is fire-and-forget (doesn't block metadata yield)
|
|
6244
|
+
* - UITransformation is computed after text streaming and emitted with metadata
|
|
6245
|
+
* - SchemaMapper.train runs while answer generation streams
|
|
5129
6246
|
*/
|
|
5130
6247
|
askStream(_0) {
|
|
5131
6248
|
return __asyncGenerator(this, arguments, function* (question, history = [], namespace) {
|
|
5132
|
-
var _a, _b, _c, _d, _e, _f, _g, _h, _i;
|
|
6249
|
+
var _a, _b, _c, _d, _e, _f, _g, _h, _i, _j, _k, _l, _m;
|
|
5133
6250
|
yield new __await(this.initialize());
|
|
5134
6251
|
const ns = namespace != null ? namespace : this.config.projectId;
|
|
5135
6252
|
const topK = (_b = (_a = this.config.rag) == null ? void 0 : _a.topK) != null ? _b : 5;
|
|
@@ -5145,27 +6262,48 @@ var Pipeline = class {
|
|
|
5145
6262
|
}
|
|
5146
6263
|
const hints = QueryProcessor.extractQueryFieldHints(question, (_f = this.config.rag) == null ? void 0 : _f.filterableFields);
|
|
5147
6264
|
const filter = QueryProcessor.buildQueryFilter(question, hints);
|
|
6265
|
+
const numericPredicates = QueryProcessor.extractNumericPredicates(question, (_g = this.config.rag) == null ? void 0 : _g.filterableFields);
|
|
6266
|
+
if (numericPredicates.length > 0) {
|
|
6267
|
+
filter.__numericPredicates = numericPredicates;
|
|
6268
|
+
}
|
|
5148
6269
|
const embedStart = performance.now();
|
|
5149
|
-
const
|
|
5150
|
-
|
|
5151
|
-
|
|
5152
|
-
|
|
5153
|
-
|
|
6270
|
+
const cacheKey = `${ns}::${searchQuery}`;
|
|
6271
|
+
const cachedVector = this.embeddingCache.get(cacheKey);
|
|
6272
|
+
const [strategyResult, embeddedVector] = yield new __await(Promise.all([
|
|
6273
|
+
QueryProcessor.determineRetrievalStrategy(
|
|
6274
|
+
searchQuery,
|
|
6275
|
+
this.llmRouter.get("fast"),
|
|
6276
|
+
(_h = this.config.rag) == null ? void 0 : _h.graphKeywords,
|
|
6277
|
+
(_i = this.config.rag) == null ? void 0 : _i.vectorKeywords
|
|
6278
|
+
),
|
|
6279
|
+
// Embed immediately regardless of strategy — costs nothing if cached.
|
|
6280
|
+
// If strategy turns out to be 'graph'-only we just won't use the vector.
|
|
6281
|
+
cachedVector ? Promise.resolve(cachedVector) : this.embeddingProvider.embed(searchQuery, { taskType: "query" })
|
|
6282
|
+
]));
|
|
6283
|
+
const queryVector = embeddedVector;
|
|
6284
|
+
if (!cachedVector && queryVector.length > 0) {
|
|
6285
|
+
this.embeddingCache.set(cacheKey, queryVector);
|
|
6286
|
+
}
|
|
6287
|
+
const graphData = (strategyResult === "graph" || strategyResult === "both") && this.graphDB && ((_j = this.config.rag) == null ? void 0 : _j.useGraphRetrieval) ? yield new __await(this.graphDB.query(searchQuery)) : void 0;
|
|
6288
|
+
const hasNumericPredicates = Array.isArray(filter.__numericPredicates) && filter.__numericPredicates.length > 0;
|
|
6289
|
+
const wantsExhaustiveList = hasNumericPredicates || /\b(list|all|show|provide|give|get)\b/i.test(question);
|
|
6290
|
+
const retrievalLimit = wantsExhaustiveList ? Math.max(topK * 20, 100) : topK * 2;
|
|
6291
|
+
const rawSources = (strategyResult === "vector" || strategyResult === "both") && queryVector && queryVector.length > 0 ? yield new __await(this.vectorDB.query(queryVector, retrievalLimit, ns, filter)) : [];
|
|
5154
6292
|
const retrieveEnd = performance.now();
|
|
5155
6293
|
const embedMs = retrieveEnd - embedStart;
|
|
5156
6294
|
const retrieveMs = retrieveEnd - embedStart;
|
|
5157
6295
|
const rerankStart = performance.now();
|
|
5158
|
-
|
|
5159
|
-
|
|
6296
|
+
const structuredSources = this.applyStructuredFilters(rawSources, filter);
|
|
6297
|
+
let sources = hasNumericPredicates ? structuredSources : structuredSources.filter((m) => m.score >= scoreThreshold);
|
|
6298
|
+
if (!hasNumericPredicates && ((_k = this.config.rag) == null ? void 0 : _k.useReranking)) {
|
|
5160
6299
|
sources = yield new __await(this.reranker.rerank(sources, question, topK));
|
|
5161
|
-
} else {
|
|
6300
|
+
} else if (!wantsExhaustiveList) {
|
|
5162
6301
|
sources = sources.slice(0, topK);
|
|
5163
6302
|
}
|
|
5164
6303
|
const rerankMs = performance.now() - rerankStart;
|
|
5165
6304
|
let context = sources.length ? sources.map((m, i) => `[Source ${i + 1}]
|
|
5166
6305
|
${m.content}`).join("\n\n---\n\n") : "No relevant context found.";
|
|
5167
6306
|
if (graphData && graphData.nodes.length > 0) {
|
|
5168
|
-
console.log(`[Graph Retrieval] Found ${graphData.nodes.length} relevant entities.`);
|
|
5169
6307
|
const graphContext = graphData.nodes.map(
|
|
5170
6308
|
(n) => `Entity: ${n.label} (${n.id})${n.properties ? " - " + JSON.stringify(n.properties) : ""}`
|
|
5171
6309
|
).join("\n");
|
|
@@ -5176,20 +6314,18 @@ VECTOR CONTEXT:
|
|
|
5176
6314
|
${context}`;
|
|
5177
6315
|
}
|
|
5178
6316
|
const allMetadataKeys = Array.from(new Set(sources.flatMap((s) => Object.keys(s.metadata || {}))));
|
|
5179
|
-
const
|
|
5180
|
-
const restrictionSuffix = "\n\n(IMPORTANT: Use plain text only. NEVER generate tables, HTML figures, or text charts. If listing products, use simple bullet points.)";
|
|
6317
|
+
const trainedSchemaPromise = allMetadataKeys.length > 0 ? SchemaMapper.train(this.llmProvider, ns, allMetadataKeys).catch(() => void 0) : Promise.resolve(void 0);
|
|
6318
|
+
const restrictionSuffix = "\n\n(IMPORTANT: Use plain text only. NEVER generate tables, HTML figures, or text charts. The UI renders requested charts separately, so NEVER say you cannot create, display, draw, render, or provide a chart/visualization. If listing products, use simple bullet points or comma-separated names. NEVER use plus signs (+) to separate product names, category names, or list items. For product description/detail questions, provide customer-facing prose only and do NOT list internal catalog fields such as Handle, Body (HTML), Vendor, Type, Tags, Published, Option, Variant, SKU, or raw metadata labels. You CAN use numbers for years/counts like 2006 or 5800, but NEVER put a dollar sign ($) before them.)";
|
|
5181
6319
|
const hardenedHistory = [...history];
|
|
5182
6320
|
const userQuestion = { role: "user", content: question + restrictionSuffix };
|
|
5183
6321
|
const messages = [...hardenedHistory, userQuestion];
|
|
5184
|
-
const systemPrompt = (
|
|
6322
|
+
const systemPrompt = (_l = this.config.llm.systemPrompt) != null ? _l : "";
|
|
5185
6323
|
const userPrompt = messages.map((m) => `${m.role}: ${m.content}`).join("\n");
|
|
5186
6324
|
let fullReply = "";
|
|
5187
6325
|
const generateStart = performance.now();
|
|
5188
6326
|
if (this.llmProvider.chatStream) {
|
|
5189
6327
|
const stream = this.llmProvider.chatStream(messages, context);
|
|
5190
|
-
if (!stream) {
|
|
5191
|
-
throw new Error(`[Pipeline] ${this.config.llm.provider} chatStream returned undefined`);
|
|
5192
|
-
}
|
|
6328
|
+
if (!stream) throw new Error(`[Pipeline] ${this.config.llm.provider} chatStream returned undefined`);
|
|
5193
6329
|
try {
|
|
5194
6330
|
for (var iter = __forAwait(stream), more, temp, error; more = !(temp = yield new __await(iter.next())).done; more = false) {
|
|
5195
6331
|
const chunk = temp.value;
|
|
@@ -5216,7 +6352,7 @@ ${context}`;
|
|
|
5216
6352
|
const latency = {
|
|
5217
6353
|
embedMs: Math.round(embedMs),
|
|
5218
6354
|
retrieveMs: Math.round(retrieveMs),
|
|
5219
|
-
rerankMs: ((
|
|
6355
|
+
rerankMs: ((_m = this.config.rag) == null ? void 0 : _m.useReranking) ? Math.round(rerankMs) : void 0,
|
|
5220
6356
|
generateMs: Math.round(generateMs),
|
|
5221
6357
|
totalMs: Math.round(totalMs)
|
|
5222
6358
|
};
|
|
@@ -5229,9 +6365,6 @@ ${context}`;
|
|
|
5229
6365
|
totalTokens: promptTokens + completionTokens,
|
|
5230
6366
|
estimatedCostUsd: estimateCostUsd(promptTokens, completionTokens, this.config.llm.model)
|
|
5231
6367
|
};
|
|
5232
|
-
const hallucinationResult = yield new __await(scoreHallucination(this.llmProvider, fullReply, context).catch(() => void 0));
|
|
5233
|
-
const trainedSchema = yield new __await(trainingPromise);
|
|
5234
|
-
const uiTransformation = yield new __await(this.generateUiTransformation(question, sources, trainedSchema));
|
|
5235
6368
|
const trace = {
|
|
5236
6369
|
requestId,
|
|
5237
6370
|
query: question,
|
|
@@ -5250,10 +6383,21 @@ ${context}`;
|
|
|
5250
6383
|
}),
|
|
5251
6384
|
latency,
|
|
5252
6385
|
tokens,
|
|
5253
|
-
hallucinationScore: hallucinationResult == null ? void 0 : hallucinationResult.score,
|
|
5254
|
-
hallucinationReason: hallucinationResult == null ? void 0 : hallucinationResult.reason,
|
|
5255
6386
|
timestamp: (/* @__PURE__ */ new Date()).toISOString()
|
|
5256
6387
|
};
|
|
6388
|
+
let uiTransformation;
|
|
6389
|
+
try {
|
|
6390
|
+
const trainedSchema = yield new __await(trainedSchemaPromise);
|
|
6391
|
+
uiTransformation = yield new __await(this.generateUiTransformation(
|
|
6392
|
+
question,
|
|
6393
|
+
sources,
|
|
6394
|
+
trainedSchema,
|
|
6395
|
+
hasNumericPredicates
|
|
6396
|
+
));
|
|
6397
|
+
} catch (uiError) {
|
|
6398
|
+
console.warn("[Pipeline] UI transformation failed before metadata yield:", uiError);
|
|
6399
|
+
uiTransformation = UITransformer.transform(question, sources, this.config);
|
|
6400
|
+
}
|
|
5257
6401
|
yield {
|
|
5258
6402
|
reply: "",
|
|
5259
6403
|
sources,
|
|
@@ -5261,6 +6405,13 @@ ${context}`;
|
|
|
5261
6405
|
ui_transformation: uiTransformation,
|
|
5262
6406
|
trace
|
|
5263
6407
|
};
|
|
6408
|
+
scoreHallucination(this.llmProvider, fullReply, context).then((hallucinationResult) => {
|
|
6409
|
+
if (hallucinationResult) {
|
|
6410
|
+
trace.hallucinationScore = hallucinationResult.score;
|
|
6411
|
+
trace.hallucinationReason = hallucinationResult.reason;
|
|
6412
|
+
}
|
|
6413
|
+
}).catch(() => {
|
|
6414
|
+
});
|
|
5264
6415
|
} catch (error2) {
|
|
5265
6416
|
throw new Error(`[Pipeline] Stream failed: ${error2 instanceof Error ? error2.message : String(error2)}`);
|
|
5266
6417
|
}
|
|
@@ -5270,24 +6421,107 @@ ${context}`;
|
|
|
5270
6421
|
* Universal retrieval method combining all enabled providers.
|
|
5271
6422
|
* Uses an LRU-bounded embedding cache to avoid re-embedding the same query.
|
|
5272
6423
|
*/
|
|
5273
|
-
async generateUiTransformation(question, sources, trainedSchema) {
|
|
6424
|
+
async generateUiTransformation(question, sources, trainedSchema, forceDeterministic = false) {
|
|
5274
6425
|
if (!sources || sources.length === 0) {
|
|
5275
6426
|
return UITransformer.transform(question, sources, this.config, trainedSchema);
|
|
5276
6427
|
}
|
|
6428
|
+
if (forceDeterministic) {
|
|
6429
|
+
return UITransformer.transform(question, sources, this.config, trainedSchema);
|
|
6430
|
+
}
|
|
5277
6431
|
try {
|
|
5278
|
-
return await UITransformer.analyzeAndDecide(question, sources, this.
|
|
6432
|
+
return await UITransformer.analyzeAndDecide(question, sources, this.llmRouter.get("fast"));
|
|
5279
6433
|
} catch (err) {
|
|
5280
6434
|
console.warn("[Pipeline] generateUiTransformation failed, using heuristic fallback:", err);
|
|
5281
6435
|
return UITransformer.transform(question, sources, this.config, trainedSchema);
|
|
5282
6436
|
}
|
|
5283
6437
|
}
|
|
6438
|
+
applyStructuredFilters(sources, filter) {
|
|
6439
|
+
const predicates = Array.isArray(filter.__numericPredicates) ? filter.__numericPredicates : [];
|
|
6440
|
+
if (predicates.length === 0) return sources;
|
|
6441
|
+
return sources.filter((source) => predicates.every((predicate) => {
|
|
6442
|
+
const value = this.resolveNumericPredicateValue(source, predicate);
|
|
6443
|
+
return value !== null && this.matchesNumericPredicate(value, predicate);
|
|
6444
|
+
})).sort((a, b) => {
|
|
6445
|
+
var _a, _b;
|
|
6446
|
+
const primary = predicates[0];
|
|
6447
|
+
const aValue = (_a = this.resolveNumericPredicateValue(a, primary)) != null ? _a : 0;
|
|
6448
|
+
const bValue = (_b = this.resolveNumericPredicateValue(b, primary)) != null ? _b : 0;
|
|
6449
|
+
return primary.operator === "lt" || primary.operator === "lte" ? aValue - bValue : bValue - aValue;
|
|
6450
|
+
});
|
|
6451
|
+
}
|
|
6452
|
+
resolveNumericPredicateValue(source, predicate) {
|
|
6453
|
+
const meta = source.metadata || {};
|
|
6454
|
+
const field = predicate.field;
|
|
6455
|
+
const entries = Object.entries(meta).filter(([, value]) => value !== null && value !== void 0 && typeof value !== "object");
|
|
6456
|
+
if (field) {
|
|
6457
|
+
const normalizedField = this.normalizeComparableField(field);
|
|
6458
|
+
const exact = entries.find(([key]) => this.normalizeComparableField(key) === normalizedField);
|
|
6459
|
+
const fuzzy = exact != null ? exact : entries.map(([key, value]) => ({ key, value, score: this.fieldSimilarityScore(key, field) })).filter((candidate) => candidate.score > 0).sort((a, b) => b.score - a.score)[0];
|
|
6460
|
+
if (fuzzy) {
|
|
6461
|
+
const value = this.toFiniteNumber(Array.isArray(fuzzy) ? fuzzy[1] : fuzzy.value);
|
|
6462
|
+
if (value !== null) return value;
|
|
6463
|
+
}
|
|
6464
|
+
const contentValue = this.extractNumericValueFromContent(source.content, field);
|
|
6465
|
+
if (contentValue !== null) return contentValue;
|
|
6466
|
+
}
|
|
6467
|
+
for (const [key, value] of entries) {
|
|
6468
|
+
if (/(id|sku|ean|upc|phone|zip|postal|code)/i.test(key)) continue;
|
|
6469
|
+
const numeric = this.toFiniteNumber(value);
|
|
6470
|
+
if (numeric !== null) return numeric;
|
|
6471
|
+
}
|
|
6472
|
+
return null;
|
|
6473
|
+
}
|
|
6474
|
+
extractNumericValueFromContent(content, field) {
|
|
6475
|
+
const escapedWords = field.split(/\s+|_+|-+/).filter(Boolean).map((word) => word.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"));
|
|
6476
|
+
if (escapedWords.length === 0) return null;
|
|
6477
|
+
const pattern = new RegExp(`(?:${escapedWords.join("[\\\\s_:-]*")})\\s*[:\\-\u2013]?\\s*([\\d,]+(?:\\.\\d+)?)`, "i");
|
|
6478
|
+
const match = content.match(pattern);
|
|
6479
|
+
return match ? this.toFiniteNumber(match[1]) : null;
|
|
6480
|
+
}
|
|
6481
|
+
matchesNumericPredicate(value, predicate) {
|
|
6482
|
+
switch (predicate.operator) {
|
|
6483
|
+
case "gt":
|
|
6484
|
+
return value > predicate.value;
|
|
6485
|
+
case "gte":
|
|
6486
|
+
return value >= predicate.value;
|
|
6487
|
+
case "lt":
|
|
6488
|
+
return value < predicate.value;
|
|
6489
|
+
case "lte":
|
|
6490
|
+
return value <= predicate.value;
|
|
6491
|
+
case "eq":
|
|
6492
|
+
default:
|
|
6493
|
+
return value === predicate.value;
|
|
6494
|
+
}
|
|
6495
|
+
}
|
|
6496
|
+
normalizeComparableField(value) {
|
|
6497
|
+
return value.toLowerCase().replace(/[^a-z0-9]/g, "");
|
|
6498
|
+
}
|
|
6499
|
+
fieldSimilarityScore(candidate, requested) {
|
|
6500
|
+
const normalizedCandidate = this.normalizeComparableField(candidate);
|
|
6501
|
+
const normalizedRequested = this.normalizeComparableField(requested);
|
|
6502
|
+
if (normalizedCandidate === normalizedRequested) return 10;
|
|
6503
|
+
if (normalizedCandidate.includes(normalizedRequested) || normalizedRequested.includes(normalizedCandidate)) return 8;
|
|
6504
|
+
const candidateTokens = this.fieldTokens(candidate);
|
|
6505
|
+
const requestedTokens = this.fieldTokens(requested);
|
|
6506
|
+
const overlap = requestedTokens.filter((token) => candidateTokens.includes(token)).length;
|
|
6507
|
+
if (overlap === 0) return 0;
|
|
6508
|
+
return overlap / Math.max(requestedTokens.length, candidateTokens.length);
|
|
6509
|
+
}
|
|
6510
|
+
fieldTokens(value) {
|
|
6511
|
+
return value.toLowerCase().split(/[^a-z0-9]+/).map((token) => token.replace(/ies$/, "y").replace(/s$/, "")).filter((token) => token.length > 1);
|
|
6512
|
+
}
|
|
6513
|
+
toFiniteNumber(value) {
|
|
6514
|
+
if (typeof value === "number") return Number.isFinite(value) ? value : null;
|
|
6515
|
+
const numeric = Number(String(value != null ? value : "").replace(/[$,% ,]/g, ""));
|
|
6516
|
+
return Number.isFinite(numeric) ? numeric : null;
|
|
6517
|
+
}
|
|
5284
6518
|
async retrieve(query, options) {
|
|
5285
|
-
var _a, _b, _c;
|
|
6519
|
+
var _a, _b, _c, _d, _e, _f;
|
|
5286
6520
|
const ns = (_a = options.namespace) != null ? _a : this.config.projectId;
|
|
5287
6521
|
const topK = (_b = options.topK) != null ? _b : 5;
|
|
5288
6522
|
const cacheKey = `${ns}::${query}`;
|
|
5289
6523
|
let queryVector = this.embeddingCache.get(cacheKey);
|
|
5290
|
-
const strategy = await QueryProcessor.determineRetrievalStrategy(query, this.
|
|
6524
|
+
const strategy = await QueryProcessor.determineRetrievalStrategy(query, this.llmRouter.get("fast"));
|
|
5291
6525
|
console.debug(`[Pipeline] Determined retrieval strategy: ${strategy}`);
|
|
5292
6526
|
const [retrievedVector, graphData] = await Promise.all([
|
|
5293
6527
|
// Only embed if we need vector search (strategy is 'vector' or 'both')
|
|
@@ -5299,8 +6533,27 @@ ${context}`;
|
|
|
5299
6533
|
this.embeddingCache.set(cacheKey, retrievedVector);
|
|
5300
6534
|
queryVector = retrievedVector;
|
|
5301
6535
|
}
|
|
5302
|
-
const
|
|
5303
|
-
|
|
6536
|
+
const baseFilter = __spreadProps(__spreadValues({}, (_d = options.filter) != null ? _d : {}), { queryText: query });
|
|
6537
|
+
const numericPredicates = QueryProcessor.extractNumericPredicates(query, (_e = this.config.rag) == null ? void 0 : _e.filterableFields);
|
|
6538
|
+
if (numericPredicates.length > 0) {
|
|
6539
|
+
baseFilter.__numericPredicates = numericPredicates;
|
|
6540
|
+
}
|
|
6541
|
+
const retrievalLimit = numericPredicates.length > 0 ? Math.max(topK * 20, 100) : topK;
|
|
6542
|
+
const sources = (strategy === "vector" || strategy === "both") && queryVector && queryVector.length > 0 ? this.applyStructuredFilters(
|
|
6543
|
+
await this.vectorDB.query(queryVector, retrievalLimit, ns, baseFilter),
|
|
6544
|
+
baseFilter
|
|
6545
|
+
) : [];
|
|
6546
|
+
const resolvedSources = [];
|
|
6547
|
+
for (const source of sources) {
|
|
6548
|
+
const parentId = (_f = source.metadata) == null ? void 0 : _f.parent_id;
|
|
6549
|
+
if (parentId) {
|
|
6550
|
+
console.log(`[Pipeline] Multi-Vector: Found child chunk. Parent ID: ${parentId}`);
|
|
6551
|
+
resolvedSources.push(source);
|
|
6552
|
+
} else {
|
|
6553
|
+
resolvedSources.push(source);
|
|
6554
|
+
}
|
|
6555
|
+
}
|
|
6556
|
+
return { sources: resolvedSources, graphData };
|
|
5304
6557
|
}
|
|
5305
6558
|
/** Rewrite the user query for better retrieval performance. */
|
|
5306
6559
|
async rewriteQuery(question, history) {
|
|
@@ -5666,7 +6919,7 @@ function createStreamHandler(configOrPlugin) {
|
|
|
5666
6919
|
const encoder = new TextEncoder();
|
|
5667
6920
|
const stream = new ReadableStream({
|
|
5668
6921
|
async start(controller) {
|
|
5669
|
-
var _a
|
|
6922
|
+
var _a;
|
|
5670
6923
|
const enqueue = (text) => controller.enqueue(encoder.encode(text));
|
|
5671
6924
|
try {
|
|
5672
6925
|
const pipelineStream = plugin.chatStream(message, history, namespace);
|
|
@@ -5684,8 +6937,7 @@ function createStreamHandler(configOrPlugin) {
|
|
|
5684
6937
|
}
|
|
5685
6938
|
if (sources.length > 0) {
|
|
5686
6939
|
try {
|
|
5687
|
-
const
|
|
5688
|
-
const uiTransformation = (_b = responseChunk == null ? void 0 : responseChunk.ui_transformation) != null ? _b : llmProvider ? await UITransformer.analyzeAndDecide(message, sources, llmProvider) : UITransformer.transform(message, sources, plugin.getConfig());
|
|
6940
|
+
const uiTransformation = (_a = responseChunk == null ? void 0 : responseChunk.ui_transformation) != null ? _a : UITransformer.transform(message, sources, plugin.getConfig());
|
|
5689
6941
|
if (uiTransformation) {
|
|
5690
6942
|
enqueue(sseUIFrame(uiTransformation));
|
|
5691
6943
|
}
|