@retrivora-ai/rag-engine 1.9.0 → 1.9.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{ILLMProvider-BOJFz3Na.d.mts → ILLMProvider-BQyKZLnd.d.mts} +10 -0
- package/dist/{ILLMProvider-BOJFz3Na.d.ts → ILLMProvider-BQyKZLnd.d.ts} +10 -0
- package/dist/handlers/index.d.mts +2 -2
- package/dist/handlers/index.d.ts +2 -2
- package/dist/handlers/index.js +1719 -467
- package/dist/handlers/index.mjs +1718 -466
- package/dist/{index-BwpcaziY.d.ts → index-A0GqPsaG.d.ts} +1 -1
- package/dist/{index-D3V9Et2M.d.mts → index-CDftK3qs.d.mts} +1 -1
- package/dist/index.css +26 -0
- package/dist/index.d.mts +2 -2
- package/dist/index.d.ts +2 -2
- package/dist/index.js +246 -76
- package/dist/index.mjs +248 -76
- package/dist/server.d.mts +32 -5
- package/dist/server.d.ts +32 -5
- package/dist/server.js +1726 -671
- package/dist/server.mjs +1724 -669
- package/package.json +1 -1
- package/src/components/MarkdownComponents.tsx +3 -3
- package/src/components/MessageBubble.tsx +49 -2
- package/src/components/ProductCard.tsx +33 -6
- package/src/components/UIDispatcher.tsx +1 -0
- package/src/components/VisualizationRenderer.tsx +143 -11
- package/src/config/EmbeddingStrategy.ts +5 -4
- package/src/config/RagConfig.ts +10 -0
- package/src/config/serverConfig.ts +16 -1
- package/src/core/LLMRouter.ts +79 -0
- package/src/core/Pipeline.ts +262 -45
- package/src/core/ProviderRegistry.ts +6 -0
- package/src/core/QueryProcessor.ts +98 -0
- package/src/handlers/index.ts +8 -5
- package/src/hooks/useRagChat.ts +53 -17
- package/src/llm/providers/UniversalLLMAdapter.ts +110 -13
- package/src/providers/vectordb/ChromaDBProvider.ts +13 -2
- package/src/providers/vectordb/MilvusProvider.ts +18 -2
- package/src/providers/vectordb/MultiTablePostgresProvider.ts +12 -12
- package/src/providers/vectordb/PostgreSQLProvider.ts +1 -1
- package/src/providers/vectordb/QdrantProvider.ts +1 -1
- package/src/providers/vectordb/RedisProvider.ts +3 -4
- package/src/providers/vectordb/WeaviateProvider.ts +41 -3
- package/src/types/index.ts +26 -0
- package/src/utils/ProductExtractor.ts +5 -3
- package/src/utils/UITransformer.ts +1348 -489
- package/src/utils/synonyms.ts +2 -0
package/dist/handlers/index.js
CHANGED
|
@@ -334,16 +334,222 @@ var init_PineconeProvider = __esm({
|
|
|
334
334
|
}
|
|
335
335
|
});
|
|
336
336
|
|
|
337
|
+
// src/providers/vectordb/PostgreSQLProvider.ts
|
|
338
|
+
var PostgreSQLProvider_exports = {};
|
|
339
|
+
__export(PostgreSQLProvider_exports, {
|
|
340
|
+
PostgreSQLProvider: () => PostgreSQLProvider
|
|
341
|
+
});
|
|
342
|
+
var import_pg, PostgreSQLProvider;
|
|
343
|
+
var init_PostgreSQLProvider = __esm({
|
|
344
|
+
"src/providers/vectordb/PostgreSQLProvider.ts"() {
|
|
345
|
+
"use strict";
|
|
346
|
+
import_pg = require("pg");
|
|
347
|
+
init_BaseVectorProvider();
|
|
348
|
+
PostgreSQLProvider = class extends BaseVectorProvider {
|
|
349
|
+
constructor(config) {
|
|
350
|
+
var _a;
|
|
351
|
+
super(config);
|
|
352
|
+
this.tableName = this.indexName.replace(/[^a-z0-9_]/gi, "_");
|
|
353
|
+
const opts = config.options;
|
|
354
|
+
if (!opts.connectionString) throw new Error("[PostgreSQLProvider] options.connectionString is required");
|
|
355
|
+
this.connectionString = opts.connectionString;
|
|
356
|
+
this.dimensions = (_a = opts.dimensions) != null ? _a : 1536;
|
|
357
|
+
}
|
|
358
|
+
static getValidator() {
|
|
359
|
+
return {
|
|
360
|
+
validate(config) {
|
|
361
|
+
const errors = [];
|
|
362
|
+
const opts = config.options || {};
|
|
363
|
+
if (!opts.connectionString) {
|
|
364
|
+
errors.push({
|
|
365
|
+
field: "vectorDb.options.connectionString",
|
|
366
|
+
message: "PostgreSQL connection string is required",
|
|
367
|
+
suggestion: "Set PGVECTOR_CONNECTION_STRING environment variable",
|
|
368
|
+
severity: "error"
|
|
369
|
+
});
|
|
370
|
+
}
|
|
371
|
+
if (opts.tables && typeof opts.tables !== "string" && !Array.isArray(opts.tables)) {
|
|
372
|
+
errors.push({
|
|
373
|
+
field: "vectorDb.options.tables",
|
|
374
|
+
message: "PostgreSQL tables must be a string or a string array",
|
|
375
|
+
severity: "error"
|
|
376
|
+
});
|
|
377
|
+
}
|
|
378
|
+
return errors;
|
|
379
|
+
}
|
|
380
|
+
};
|
|
381
|
+
}
|
|
382
|
+
static getHealthChecker() {
|
|
383
|
+
return {
|
|
384
|
+
async check(config) {
|
|
385
|
+
const opts = config.options || {};
|
|
386
|
+
const timestamp = Date.now();
|
|
387
|
+
try {
|
|
388
|
+
const { Client } = await import("pg");
|
|
389
|
+
const client = new Client({ connectionString: opts.connectionString });
|
|
390
|
+
await client.connect();
|
|
391
|
+
const result = await client.query(`
|
|
392
|
+
SELECT EXISTS(SELECT 1 FROM pg_extension WHERE extname = 'vector');
|
|
393
|
+
`);
|
|
394
|
+
const hasVector = result.rows[0].exists;
|
|
395
|
+
await client.end();
|
|
396
|
+
return {
|
|
397
|
+
healthy: true,
|
|
398
|
+
provider: "postgresql",
|
|
399
|
+
capabilities: { pgvectorInstalled: hasVector },
|
|
400
|
+
timestamp
|
|
401
|
+
};
|
|
402
|
+
} catch (error) {
|
|
403
|
+
return {
|
|
404
|
+
healthy: false,
|
|
405
|
+
provider: "postgresql",
|
|
406
|
+
error: `Connection failed: ${error instanceof Error ? error.message : String(error)}`,
|
|
407
|
+
timestamp
|
|
408
|
+
};
|
|
409
|
+
}
|
|
410
|
+
}
|
|
411
|
+
};
|
|
412
|
+
}
|
|
413
|
+
async initialize() {
|
|
414
|
+
this.pool = new import_pg.Pool({ connectionString: this.connectionString });
|
|
415
|
+
const client = await this.pool.connect();
|
|
416
|
+
try {
|
|
417
|
+
await client.query("CREATE EXTENSION IF NOT EXISTS vector");
|
|
418
|
+
await client.query(`
|
|
419
|
+
CREATE TABLE IF NOT EXISTS ${this.tableName} (
|
|
420
|
+
id TEXT PRIMARY KEY,
|
|
421
|
+
namespace TEXT NOT NULL DEFAULT '',
|
|
422
|
+
content TEXT NOT NULL,
|
|
423
|
+
metadata JSONB,
|
|
424
|
+
embedding VECTOR(${this.dimensions})
|
|
425
|
+
)
|
|
426
|
+
`);
|
|
427
|
+
await client.query(`
|
|
428
|
+
CREATE INDEX IF NOT EXISTS ${this.tableName}_embedding_idx
|
|
429
|
+
ON ${this.tableName}
|
|
430
|
+
USING hnsw (embedding vector_cosine_ops)
|
|
431
|
+
`);
|
|
432
|
+
} finally {
|
|
433
|
+
client.release();
|
|
434
|
+
}
|
|
435
|
+
}
|
|
436
|
+
async upsert(doc, namespace = "") {
|
|
437
|
+
var _a;
|
|
438
|
+
const vectorLiteral = `[${doc.vector.join(",")}]`;
|
|
439
|
+
await this.pool.query(
|
|
440
|
+
`INSERT INTO ${this.tableName} (id, namespace, content, metadata, embedding)
|
|
441
|
+
VALUES ($1, $2, $3, $4, $5::vector)
|
|
442
|
+
ON CONFLICT (id) DO UPDATE
|
|
443
|
+
SET namespace = EXCLUDED.namespace,
|
|
444
|
+
content = EXCLUDED.content,
|
|
445
|
+
metadata = EXCLUDED.metadata,
|
|
446
|
+
embedding = EXCLUDED.embedding`,
|
|
447
|
+
[doc.id, namespace, doc.content, JSON.stringify((_a = doc.metadata) != null ? _a : {}), vectorLiteral]
|
|
448
|
+
);
|
|
449
|
+
}
|
|
450
|
+
async batchUpsert(docs, namespace = "") {
|
|
451
|
+
if (docs.length === 0) return;
|
|
452
|
+
const client = await this.pool.connect();
|
|
453
|
+
try {
|
|
454
|
+
await client.query("BEGIN");
|
|
455
|
+
const BATCH_SIZE = 50;
|
|
456
|
+
for (let i = 0; i < docs.length; i += BATCH_SIZE) {
|
|
457
|
+
const batch = docs.slice(i, i + BATCH_SIZE);
|
|
458
|
+
const values = [];
|
|
459
|
+
const valuePlaceholders = batch.map((doc, idx) => {
|
|
460
|
+
var _a;
|
|
461
|
+
const offset = idx * 5;
|
|
462
|
+
values.push(doc.id, namespace, doc.content, JSON.stringify((_a = doc.metadata) != null ? _a : {}), `[${doc.vector.join(",")}]`);
|
|
463
|
+
return `($${offset + 1}, $${offset + 2}, $${offset + 3}, $${offset + 4}, $${offset + 5}::vector)`;
|
|
464
|
+
}).join(", ");
|
|
465
|
+
const query = `
|
|
466
|
+
INSERT INTO ${this.tableName} (id, namespace, content, metadata, embedding)
|
|
467
|
+
VALUES ${valuePlaceholders}
|
|
468
|
+
ON CONFLICT (id) DO UPDATE
|
|
469
|
+
SET namespace = EXCLUDED.namespace,
|
|
470
|
+
content = EXCLUDED.content,
|
|
471
|
+
metadata = EXCLUDED.metadata,
|
|
472
|
+
embedding = EXCLUDED.embedding
|
|
473
|
+
`;
|
|
474
|
+
await client.query(query, values);
|
|
475
|
+
}
|
|
476
|
+
await client.query("COMMIT");
|
|
477
|
+
} catch (error) {
|
|
478
|
+
await client.query("ROLLBACK");
|
|
479
|
+
throw error;
|
|
480
|
+
} finally {
|
|
481
|
+
client.release();
|
|
482
|
+
}
|
|
483
|
+
}
|
|
484
|
+
async query(vector, topK, namespace, filter) {
|
|
485
|
+
const vectorLiteral = `[${vector.join(",")}]`;
|
|
486
|
+
let whereClause = namespace ? `WHERE namespace = $3` : "";
|
|
487
|
+
const params = [vectorLiteral, topK];
|
|
488
|
+
if (namespace) params.push(namespace);
|
|
489
|
+
const publicFilter = this.sanitizeFilter(filter);
|
|
490
|
+
if (Object.keys(publicFilter).length > 0) {
|
|
491
|
+
const filterConditions = Object.entries(publicFilter).map(([key, val]) => {
|
|
492
|
+
const paramIdx = params.length + 1;
|
|
493
|
+
params.push(String(val));
|
|
494
|
+
return `metadata->>'${key}' = $${paramIdx}`;
|
|
495
|
+
}).join(" AND ");
|
|
496
|
+
whereClause = whereClause ? `${whereClause} AND ${filterConditions}` : `WHERE ${filterConditions}`;
|
|
497
|
+
}
|
|
498
|
+
const client = await this.pool.connect();
|
|
499
|
+
try {
|
|
500
|
+
const efSearch = this.config.options.efSearch || Math.max(topK * 10, 40);
|
|
501
|
+
await client.query(`SET LOCAL hnsw.ef_search = ${efSearch}`);
|
|
502
|
+
const result = await client.query(
|
|
503
|
+
`SELECT id, content, metadata, 1 - (embedding <=> $1::vector) AS score
|
|
504
|
+
FROM ${this.tableName}
|
|
505
|
+
${whereClause}
|
|
506
|
+
ORDER BY embedding <=> $1::vector
|
|
507
|
+
LIMIT $2`,
|
|
508
|
+
params
|
|
509
|
+
);
|
|
510
|
+
return result.rows.map((row) => ({
|
|
511
|
+
id: String(row["id"]),
|
|
512
|
+
score: parseFloat(String(row["score"])),
|
|
513
|
+
content: String(row["content"]),
|
|
514
|
+
metadata: row["metadata"]
|
|
515
|
+
}));
|
|
516
|
+
} finally {
|
|
517
|
+
client.release();
|
|
518
|
+
}
|
|
519
|
+
}
|
|
520
|
+
async delete(id, namespace) {
|
|
521
|
+
const where = namespace ? "WHERE id = $1 AND namespace = $2" : "WHERE id = $1";
|
|
522
|
+
const params = namespace ? [id, namespace] : [id];
|
|
523
|
+
await this.pool.query(`DELETE FROM ${this.tableName} ${where}`, params);
|
|
524
|
+
}
|
|
525
|
+
async deleteNamespace(namespace) {
|
|
526
|
+
await this.pool.query(`DELETE FROM ${this.tableName} WHERE namespace = $1`, [namespace]);
|
|
527
|
+
}
|
|
528
|
+
async ping() {
|
|
529
|
+
try {
|
|
530
|
+
await this.pool.query("SELECT 1");
|
|
531
|
+
return true;
|
|
532
|
+
} catch (e) {
|
|
533
|
+
return false;
|
|
534
|
+
}
|
|
535
|
+
}
|
|
536
|
+
async disconnect() {
|
|
537
|
+
await this.pool.end();
|
|
538
|
+
}
|
|
539
|
+
};
|
|
540
|
+
}
|
|
541
|
+
});
|
|
542
|
+
|
|
337
543
|
// src/providers/vectordb/MultiTablePostgresProvider.ts
|
|
338
544
|
var MultiTablePostgresProvider_exports = {};
|
|
339
545
|
__export(MultiTablePostgresProvider_exports, {
|
|
340
546
|
MultiTablePostgresProvider: () => MultiTablePostgresProvider
|
|
341
547
|
});
|
|
342
|
-
var
|
|
548
|
+
var import_pg2, MultiTablePostgresProvider;
|
|
343
549
|
var init_MultiTablePostgresProvider = __esm({
|
|
344
550
|
"src/providers/vectordb/MultiTablePostgresProvider.ts"() {
|
|
345
551
|
"use strict";
|
|
346
|
-
|
|
552
|
+
import_pg2 = require("pg");
|
|
347
553
|
init_BaseVectorProvider();
|
|
348
554
|
MultiTablePostgresProvider = class extends BaseVectorProvider {
|
|
349
555
|
constructor(config) {
|
|
@@ -361,7 +567,7 @@ var init_MultiTablePostgresProvider = __esm({
|
|
|
361
567
|
this.uploadTable = opts.uploadTable || "document_chunks";
|
|
362
568
|
}
|
|
363
569
|
async initialize() {
|
|
364
|
-
this.pool = new
|
|
570
|
+
this.pool = new import_pg2.Pool({ connectionString: this.connectionString });
|
|
365
571
|
const client = await this.pool.connect();
|
|
366
572
|
try {
|
|
367
573
|
await client.query("CREATE EXTENSION IF NOT EXISTS vector");
|
|
@@ -505,26 +711,25 @@ var init_MultiTablePostgresProvider = __esm({
|
|
|
505
711
|
const allResults = [];
|
|
506
712
|
console.log(`[MultiTablePostgresProvider] --- Starting Multi-Table Search ---`);
|
|
507
713
|
const queryText = _filter == null ? void 0 : _filter.queryText;
|
|
508
|
-
const entityHints = Array.isArray(_filter == null ? void 0 : _filter.
|
|
509
|
-
(hint) => typeof hint === "object" && hint !== null && typeof hint.value === "string"
|
|
510
|
-
).map((hint) => hint.value.trim().toLowerCase()).filter(Boolean) : [];
|
|
714
|
+
const entityHints = Array.isArray(_filter == null ? void 0 : _filter.keywords) ? _filter.keywords.map((k) => k.trim().toLowerCase()).filter(Boolean) : [];
|
|
511
715
|
console.log(`[MultiTablePostgresProvider] queryText: "${queryText}"`);
|
|
512
716
|
console.log(`[MultiTablePostgresProvider] entityHints: [${entityHints.join(", ")}]`);
|
|
513
717
|
const getDynamicKeywordQuery = () => {
|
|
514
718
|
if (entityHints.length > 0) {
|
|
515
|
-
return entityHints.map((h) => h.
|
|
719
|
+
return entityHints.map((h) => h.replace(/\s+/g, " & ")).join(" & ");
|
|
516
720
|
}
|
|
517
721
|
if (queryText) {
|
|
518
|
-
return queryText.toLowerCase().replace(/\b(show|me|the|product[s]?|item[s]?|card[s]?|of|category|find|list|browse|what|are|display|all|under|for|in|with|about)\b/g, "").trim().replace(/\s+/g, " & ");
|
|
722
|
+
return queryText.toLowerCase().replace(/\b(show|me|the|product[s]?|item[s]?|card[s]?|organization[s]?|comp(?:any|anies)|of|category|find|list|browse|what|are|display|all|whose|that|which|have|has|having|greater|than|more|less|equal|equals|above|below|over|under|at|least|most|for|in|with|about|get|details|information|info)\b/g, "").replace(/\b\d+(?:\.\d+)?\b/g, "").trim().replace(/\s+/g, " & ");
|
|
519
723
|
}
|
|
520
724
|
return "";
|
|
521
725
|
};
|
|
522
726
|
const dynamicKeywordQuery = getDynamicKeywordQuery();
|
|
727
|
+
const tableLimit = Math.max(topK, 50);
|
|
523
728
|
const queryPromises = this.tables.map(async (table) => {
|
|
524
729
|
try {
|
|
525
730
|
let sqlQuery = "";
|
|
526
731
|
let params = [];
|
|
527
|
-
if (queryText) {
|
|
732
|
+
if (queryText && dynamicKeywordQuery) {
|
|
528
733
|
const hasEntityHints = entityHints.length > 0;
|
|
529
734
|
const exactNameScoreExpr = hasEntityHints ? `+ (
|
|
530
735
|
SELECT COALESCE(MAX(
|
|
@@ -541,16 +746,16 @@ var init_MultiTablePostgresProvider = __esm({
|
|
|
541
746
|
((1 - (embedding <=> $1::vector)) + (COALESCE(ts_rank(to_tsvector('english', (to_jsonb(t) - 'embedding')::text), to_tsquery('english', $2)), 0) * 2.0) ${exactNameScoreExpr}) AS hybrid_score
|
|
542
747
|
FROM "${table}" t
|
|
543
748
|
ORDER BY hybrid_score DESC
|
|
544
|
-
LIMIT
|
|
749
|
+
LIMIT ${tableLimit}
|
|
545
750
|
`;
|
|
546
751
|
params = [vectorLiteral, dynamicKeywordQuery];
|
|
547
752
|
} else {
|
|
548
753
|
sqlQuery = `
|
|
549
754
|
SELECT *,
|
|
550
|
-
|
|
755
|
+
(1 - (embedding <=> $1::vector)) AS hybrid_score
|
|
551
756
|
FROM "${table}" t
|
|
552
757
|
ORDER BY hybrid_score DESC
|
|
553
|
-
LIMIT
|
|
758
|
+
LIMIT ${tableLimit}
|
|
554
759
|
`;
|
|
555
760
|
params = [vectorLiteral];
|
|
556
761
|
}
|
|
@@ -875,13 +1080,25 @@ var init_MilvusProvider = __esm({
|
|
|
875
1080
|
};
|
|
876
1081
|
await this.http.post("/v1/vector/upsert", payload);
|
|
877
1082
|
}
|
|
878
|
-
// eslint-disable-next-line @typescript-eslint/no-unused-vars
|
|
879
1083
|
async query(vector, topK, namespace, _filter) {
|
|
1084
|
+
const sanitizedFilter = this.sanitizeFilter(_filter);
|
|
1085
|
+
const filterParts = [];
|
|
1086
|
+
if (namespace) {
|
|
1087
|
+
filterParts.push(`namespace == "${namespace}"`);
|
|
1088
|
+
}
|
|
1089
|
+
for (const [key, value] of Object.entries(sanitizedFilter)) {
|
|
1090
|
+
if (key === "queryText" || key === "keywords") continue;
|
|
1091
|
+
if (typeof value === "string") {
|
|
1092
|
+
filterParts.push(`metadata["${key}"] == "${value.replace(/"/g, '\\"')}"`);
|
|
1093
|
+
} else if (typeof value === "number") {
|
|
1094
|
+
filterParts.push(`metadata["${key}"] == ${value}`);
|
|
1095
|
+
}
|
|
1096
|
+
}
|
|
880
1097
|
const payload = {
|
|
881
1098
|
collectionName: this.indexName,
|
|
882
1099
|
vector,
|
|
883
1100
|
limit: topK,
|
|
884
|
-
filter:
|
|
1101
|
+
filter: filterParts.length > 0 ? filterParts.join(" && ") : void 0,
|
|
885
1102
|
outputFields: ["content", "metadata"],
|
|
886
1103
|
searchParams: {
|
|
887
1104
|
nprobe: this.config.options.nprobe || 16,
|
|
@@ -1080,6 +1297,7 @@ var init_QdrantProvider = __esm({
|
|
|
1080
1297
|
await this.http.put(`/collections/${this.indexName}/points`, payload);
|
|
1081
1298
|
}
|
|
1082
1299
|
async query(vector, topK, namespace, _filter) {
|
|
1300
|
+
var _a;
|
|
1083
1301
|
const must = [];
|
|
1084
1302
|
if (namespace) {
|
|
1085
1303
|
must.push({ key: "namespace", match: { value: namespace } });
|
|
@@ -1101,7 +1319,7 @@ var init_QdrantProvider = __esm({
|
|
|
1101
1319
|
limit: topK,
|
|
1102
1320
|
with_payload: true,
|
|
1103
1321
|
params: {
|
|
1104
|
-
hnsw_ef: this.config.options.efSearch || Math.max(topK * 20, 128),
|
|
1322
|
+
hnsw_ef: ((_a = this.config.options) == null ? void 0 : _a.efSearch) || Math.max(topK * 20, 128),
|
|
1105
1323
|
exact: false
|
|
1106
1324
|
},
|
|
1107
1325
|
filter: must.length > 0 ? { must } : void 0
|
|
@@ -1226,12 +1444,20 @@ var init_ChromaDBProvider = __esm({
|
|
|
1226
1444
|
};
|
|
1227
1445
|
await this.http.post(`/api/v1/collections/${this.collectionId}/add`, payload);
|
|
1228
1446
|
}
|
|
1229
|
-
// eslint-disable-next-line @typescript-eslint/no-unused-vars
|
|
1230
1447
|
async query(vector, topK, namespace, _filter) {
|
|
1448
|
+
const sanitizedFilter = this.sanitizeFilter(_filter);
|
|
1449
|
+
const whereClauses = [];
|
|
1450
|
+
if (namespace) whereClauses.push({ namespace: { $eq: namespace } });
|
|
1451
|
+
Object.entries(sanitizedFilter).forEach(([key, value]) => {
|
|
1452
|
+
if (key === "namespace") return;
|
|
1453
|
+
if (typeof value === "string" || typeof value === "number" || typeof value === "boolean") {
|
|
1454
|
+
whereClauses.push({ [key]: { $eq: value } });
|
|
1455
|
+
}
|
|
1456
|
+
});
|
|
1231
1457
|
const payload = {
|
|
1232
1458
|
query_embeddings: [vector],
|
|
1233
1459
|
n_results: topK,
|
|
1234
|
-
where:
|
|
1460
|
+
where: whereClauses.length > 1 ? { $and: whereClauses } : whereClauses[0]
|
|
1235
1461
|
};
|
|
1236
1462
|
const { data } = await this.http.post(`/api/v1/collections/${this.collectionId}/query`, payload);
|
|
1237
1463
|
const matches = [];
|
|
@@ -1336,15 +1562,15 @@ var init_RedisProvider = __esm({
|
|
|
1336
1562
|
console.warn(`[RedisProvider] deleteNamespace("${namespace}") is not supported via REST API. Use Redis CLI: SCAN + DEL`);
|
|
1337
1563
|
}
|
|
1338
1564
|
/**
|
|
1339
|
-
*
|
|
1340
|
-
* Returns
|
|
1565
|
+
* Check reachability via a PING command.
|
|
1566
|
+
* Returns false on connection failure so health checks surface real problems.
|
|
1341
1567
|
*/
|
|
1342
1568
|
async ping() {
|
|
1343
1569
|
try {
|
|
1344
1570
|
await this.http.post("/", ["PING"]);
|
|
1345
1571
|
return true;
|
|
1346
1572
|
} catch (e) {
|
|
1347
|
-
return
|
|
1573
|
+
return false;
|
|
1348
1574
|
}
|
|
1349
1575
|
}
|
|
1350
1576
|
async disconnect() {
|
|
@@ -1384,15 +1610,17 @@ var init_WeaviateProvider = __esm({
|
|
|
1384
1610
|
await this.ping();
|
|
1385
1611
|
}
|
|
1386
1612
|
async upsert(doc, namespace) {
|
|
1613
|
+
const primitiveMetadata = this.extractPrimitiveMetadata(doc.metadata);
|
|
1387
1614
|
const payload = {
|
|
1388
1615
|
class: this.indexName,
|
|
1389
1616
|
id: doc.id,
|
|
1390
1617
|
vector: doc.vector,
|
|
1391
|
-
properties: {
|
|
1618
|
+
properties: __spreadProps(__spreadValues({
|
|
1392
1619
|
content: doc.content,
|
|
1393
|
-
metadata: JSON.stringify(doc.metadata || {})
|
|
1620
|
+
metadata: JSON.stringify(doc.metadata || {})
|
|
1621
|
+
}, primitiveMetadata), {
|
|
1394
1622
|
namespace: namespace || ""
|
|
1395
|
-
}
|
|
1623
|
+
})
|
|
1396
1624
|
};
|
|
1397
1625
|
await this.http.post("/v1/objects", payload);
|
|
1398
1626
|
}
|
|
@@ -1402,20 +1630,22 @@ var init_WeaviateProvider = __esm({
|
|
|
1402
1630
|
class: this.indexName,
|
|
1403
1631
|
id: doc.id,
|
|
1404
1632
|
vector: doc.vector,
|
|
1405
|
-
properties: {
|
|
1633
|
+
properties: __spreadProps(__spreadValues({
|
|
1406
1634
|
content: doc.content,
|
|
1407
|
-
metadata: JSON.stringify(doc.metadata || {})
|
|
1635
|
+
metadata: JSON.stringify(doc.metadata || {})
|
|
1636
|
+
}, this.extractPrimitiveMetadata(doc.metadata)), {
|
|
1408
1637
|
namespace: namespace || ""
|
|
1409
|
-
}
|
|
1638
|
+
})
|
|
1410
1639
|
}))
|
|
1411
1640
|
};
|
|
1412
1641
|
await this.http.post("/v1/batch/objects", payload);
|
|
1413
1642
|
}
|
|
1414
1643
|
async query(vector, topK, namespace, _filter) {
|
|
1415
1644
|
var _a, _b;
|
|
1645
|
+
const queryText = _filter == null ? void 0 : _filter.queryText;
|
|
1416
1646
|
const sanitizedFilter = this.sanitizeFilter(_filter);
|
|
1417
|
-
const queryText = sanitizedFilter.queryText;
|
|
1418
1647
|
const searchParams = queryText ? `hybrid: { query: ${JSON.stringify(queryText)}, alpha: 0.5 }` : `nearVector: { vector: ${JSON.stringify(vector)} }`;
|
|
1648
|
+
const where = this.buildWhereFilter(namespace, sanitizedFilter);
|
|
1419
1649
|
const graphqlQuery = {
|
|
1420
1650
|
query: `
|
|
1421
1651
|
{
|
|
@@ -1423,7 +1653,7 @@ var init_WeaviateProvider = __esm({
|
|
|
1423
1653
|
${this.indexName}(
|
|
1424
1654
|
${searchParams}
|
|
1425
1655
|
limit: ${topK}
|
|
1426
|
-
${
|
|
1656
|
+
${where ? `where: ${where}` : ""}
|
|
1427
1657
|
) {
|
|
1428
1658
|
content
|
|
1429
1659
|
metadata
|
|
@@ -1467,6 +1697,34 @@ var init_WeaviateProvider = __esm({
|
|
|
1467
1697
|
}
|
|
1468
1698
|
async disconnect() {
|
|
1469
1699
|
}
|
|
1700
|
+
extractPrimitiveMetadata(metadata) {
|
|
1701
|
+
const result = {};
|
|
1702
|
+
Object.entries(metadata || {}).forEach(([key, value]) => {
|
|
1703
|
+
if (typeof value === "string" || typeof value === "number" || typeof value === "boolean") {
|
|
1704
|
+
result[key] = value;
|
|
1705
|
+
}
|
|
1706
|
+
});
|
|
1707
|
+
return result;
|
|
1708
|
+
}
|
|
1709
|
+
buildWhereFilter(namespace, filter) {
|
|
1710
|
+
const operands = [];
|
|
1711
|
+
if (namespace) operands.push(this.weaviateOperand("namespace", namespace));
|
|
1712
|
+
Object.entries(filter || {}).forEach(([key, value]) => {
|
|
1713
|
+
if (key === "namespace") return;
|
|
1714
|
+
if (typeof value === "string" || typeof value === "number" || typeof value === "boolean") {
|
|
1715
|
+
operands.push(this.weaviateOperand(key, value));
|
|
1716
|
+
}
|
|
1717
|
+
});
|
|
1718
|
+
if (operands.length === 0) return void 0;
|
|
1719
|
+
if (operands.length === 1) return operands[0];
|
|
1720
|
+
return `{ operator: And, operands: [${operands.join(", ")}] }`;
|
|
1721
|
+
}
|
|
1722
|
+
weaviateOperand(key, value) {
|
|
1723
|
+
const path = `path: [${JSON.stringify(key)}], operator: Equal`;
|
|
1724
|
+
if (typeof value === "number") return `{ ${path}, valueNumber: ${value} }`;
|
|
1725
|
+
if (typeof value === "boolean") return `{ ${path}, valueBoolean: ${value} }`;
|
|
1726
|
+
return `{ ${path}, valueString: ${JSON.stringify(value)} }`;
|
|
1727
|
+
}
|
|
1470
1728
|
};
|
|
1471
1729
|
}
|
|
1472
1730
|
});
|
|
@@ -1788,7 +2046,7 @@ function readEnum(env, name, fallback, allowed) {
|
|
|
1788
2046
|
throw new Error(`[getRagConfig] ${name} must be one of: ${allowed.join(", ")}`);
|
|
1789
2047
|
}
|
|
1790
2048
|
function getEnvConfig(env = process.env, base) {
|
|
1791
|
-
var _a, _b, _c, _d, _e, _f, _g, _h, _i, _j, _k, _l, _m, _n, _o, _p, _q, _r, _s, _t, _u, _v, _w, _x, _y, _z, _A, _B, _C, _D, _E, _F, _G, _H, _I, _J, _K, _L, _M, _N, _O, _P, _Q, _R, _S, _T, _U, _V, _W, _X, _Y, _Z, __, _$, _aa, _ba, _ca, _da, _ea, _fa, _ga, _ha, _ia, _ja, _ka, _la, _ma, _na, _oa, _pa, _qa, _ra, _sa, _ta, _ua, _va, _wa, _xa, _ya, _za, _Aa, _Ba, _Ca, _Da, _Ea, _Fa;
|
|
2049
|
+
var _a, _b, _c, _d, _e, _f, _g, _h, _i, _j, _k, _l, _m, _n, _o, _p, _q, _r, _s, _t, _u, _v, _w, _x, _y, _z, _A, _B, _C, _D, _E, _F, _G, _H, _I, _J, _K, _L, _M, _N, _O, _P, _Q, _R, _S, _T, _U, _V, _W, _X, _Y, _Z, __, _$, _aa, _ba, _ca, _da, _ea, _fa, _ga, _ha, _ia, _ja, _ka, _la, _ma, _na, _oa, _pa, _qa, _ra, _sa, _ta, _ua, _va, _wa, _xa, _ya, _za, _Aa, _Ba, _Ca, _Da, _Ea, _Fa, _Ga;
|
|
1792
2050
|
const projectId = (_c = (_b = (_a = readString(env, "RAG_PROJECT_ID")) != null ? _a : readString(env, "NEXT_PUBLIC_PROJECT_ID")) != null ? _b : base == null ? void 0 : base.projectId) != null ? _c : "__default__";
|
|
1793
2051
|
const vectorProvider = readEnum(env, "VECTOR_DB_PROVIDER", "pinecone", VECTOR_DB_PROVIDERS);
|
|
1794
2052
|
const llmProvider = readEnum(env, "LLM_PROVIDER", "openai", LLM_PROVIDERS);
|
|
@@ -1842,11 +2100,22 @@ function getEnvConfig(env = process.env, base) {
|
|
|
1842
2100
|
};
|
|
1843
2101
|
const embeddingApiKeyByProvider = {
|
|
1844
2102
|
openai: readString(env, "OPENAI_API_KEY"),
|
|
2103
|
+
anthropic: readString(env, "ANTHROPIC_API_KEY"),
|
|
2104
|
+
// Anthropic needs a separate embedding provider; key kept for completeness
|
|
1845
2105
|
gemini: readString(env, "GEMINI_API_KEY"),
|
|
1846
2106
|
ollama: void 0,
|
|
1847
2107
|
universal_rest: (_ea = readString(env, "EMBEDDING_API_KEY")) != null ? _ea : readString(env, "OPENAI_API_KEY"),
|
|
1848
2108
|
custom: (_fa = readString(env, "EMBEDDING_API_KEY")) != null ? _fa : readString(env, "OPENAI_API_KEY")
|
|
1849
2109
|
};
|
|
2110
|
+
const DEFAULT_MODEL_BY_PROVIDER = {
|
|
2111
|
+
openai: "gpt-4o",
|
|
2112
|
+
anthropic: "claude-3-5-sonnet-20241022",
|
|
2113
|
+
gemini: "gemini-2.0-flash",
|
|
2114
|
+
ollama: "llama3",
|
|
2115
|
+
universal_rest: "default",
|
|
2116
|
+
rest: "default",
|
|
2117
|
+
custom: "default"
|
|
2118
|
+
};
|
|
1850
2119
|
return {
|
|
1851
2120
|
projectId,
|
|
1852
2121
|
vectorDb: {
|
|
@@ -1856,8 +2125,8 @@ function getEnvConfig(env = process.env, base) {
|
|
|
1856
2125
|
},
|
|
1857
2126
|
llm: {
|
|
1858
2127
|
provider: llmProvider,
|
|
1859
|
-
model: (_ia = readString(env, "LLM_MODEL")) != null ? _ia : "gpt-4o",
|
|
1860
|
-
apiKey: (
|
|
2128
|
+
model: (_ja = (_ia = readString(env, "LLM_MODEL")) != null ? _ia : DEFAULT_MODEL_BY_PROVIDER[llmProvider]) != null ? _ja : "gpt-4o",
|
|
2129
|
+
apiKey: (_ka = llmApiKeyByProvider[llmProvider]) != null ? _ka : "",
|
|
1861
2130
|
baseUrl: readString(env, "LLM_BASE_URL"),
|
|
1862
2131
|
systemPrompt: readString(env, "LLM_SYSTEM_PROMPT"),
|
|
1863
2132
|
maxTokens: readNumber(env, "LLM_MAX_TOKENS", 4096),
|
|
@@ -1868,7 +2137,7 @@ function getEnvConfig(env = process.env, base) {
|
|
|
1868
2137
|
},
|
|
1869
2138
|
embedding: {
|
|
1870
2139
|
provider: embeddingProvider,
|
|
1871
|
-
model: (
|
|
2140
|
+
model: (_la = readString(env, "EMBEDDING_MODEL")) != null ? _la : "text-embedding-3-small",
|
|
1872
2141
|
apiKey: embeddingApiKeyByProvider[embeddingProvider],
|
|
1873
2142
|
baseUrl: readString(env, "EMBEDDING_BASE_URL"),
|
|
1874
2143
|
dimensions: embeddingDimensions,
|
|
@@ -1879,17 +2148,17 @@ function getEnvConfig(env = process.env, base) {
|
|
|
1879
2148
|
}
|
|
1880
2149
|
},
|
|
1881
2150
|
ui: {
|
|
1882
|
-
title: (
|
|
1883
|
-
subtitle: (
|
|
1884
|
-
primaryColor: (
|
|
1885
|
-
accentColor: (
|
|
1886
|
-
logoUrl: (
|
|
1887
|
-
placeholder: (
|
|
1888
|
-
showSources: ((
|
|
1889
|
-
welcomeMessage: (
|
|
1890
|
-
visualStyle: (
|
|
1891
|
-
borderRadius: (
|
|
1892
|
-
allowUpload: ((
|
|
2151
|
+
title: (_na = (_ma = readString(env, "NEXT_PUBLIC_UI_TITLE")) != null ? _ma : readString(env, "UI_TITLE")) != null ? _na : "AI Assistant",
|
|
2152
|
+
subtitle: (_pa = (_oa = readString(env, "NEXT_PUBLIC_UI_SUBTITLE")) != null ? _oa : readString(env, "UI_SUBTITLE")) != null ? _pa : "Powered by RAG",
|
|
2153
|
+
primaryColor: (_ra = (_qa = readString(env, "NEXT_PUBLIC_PRIMARY_COLOR")) != null ? _qa : readString(env, "UI_PRIMARY_COLOR")) != null ? _ra : "#10b981",
|
|
2154
|
+
accentColor: (_ta = (_sa = readString(env, "NEXT_PUBLIC_ACCENT_COLOR")) != null ? _sa : readString(env, "UI_ACCENT_COLOR")) != null ? _ta : "#3b82f6",
|
|
2155
|
+
logoUrl: (_ua = readString(env, "NEXT_PUBLIC_LOGO_URL")) != null ? _ua : readString(env, "UI_LOGO_URL"),
|
|
2156
|
+
placeholder: (_wa = (_va = readString(env, "NEXT_PUBLIC_PLACEHOLDER")) != null ? _va : readString(env, "UI_PLACEHOLDER")) != null ? _wa : "Ask me anything\u2026",
|
|
2157
|
+
showSources: ((_ya = (_xa = readString(env, "NEXT_PUBLIC_SHOW_SOURCES")) != null ? _xa : readString(env, "UI_SHOW_SOURCES")) != null ? _ya : "true") !== "false",
|
|
2158
|
+
welcomeMessage: (_Aa = (_za = readString(env, "NEXT_PUBLIC_WELCOME_MESSAGE")) != null ? _za : readString(env, "UI_WELCOME_MESSAGE")) != null ? _Aa : "Hello! I'm your AI assistant. Ask me anything about your documents.",
|
|
2159
|
+
visualStyle: (_Ca = (_Ba = readString(env, "NEXT_PUBLIC_UI_VISUAL_STYLE")) != null ? _Ba : readString(env, "UI_VISUAL_STYLE")) != null ? _Ca : "glass",
|
|
2160
|
+
borderRadius: (_Ea = (_Da = readString(env, "NEXT_PUBLIC_UI_BORDER_RADIUS")) != null ? _Da : readString(env, "UI_BORDER_RADIUS")) != null ? _Ea : "xl",
|
|
2161
|
+
allowUpload: ((_Ga = (_Fa = readString(env, "NEXT_PUBLIC_ALLOW_UPLOAD")) != null ? _Fa : readString(env, "UI_ALLOW_UPLOAD")) != null ? _Ga : "false") === "true"
|
|
1893
2162
|
},
|
|
1894
2163
|
rag: {
|
|
1895
2164
|
topK: readNumber(env, "RAG_TOP_K", 5),
|
|
@@ -2751,7 +3020,7 @@ var LLM_PROFILES = {
|
|
|
2751
3020
|
// src/llm/providers/UniversalLLMAdapter.ts
|
|
2752
3021
|
var UniversalLLMAdapter = class {
|
|
2753
3022
|
constructor(config) {
|
|
2754
|
-
var _a, _b, _c, _d, _e, _f, _g;
|
|
3023
|
+
var _a, _b, _c, _d, _e, _f, _g, _h;
|
|
2755
3024
|
this.model = config.model;
|
|
2756
3025
|
const llmConfig = config;
|
|
2757
3026
|
const options = (_a = llmConfig.options) != null ? _a : {};
|
|
@@ -2765,16 +3034,18 @@ var UniversalLLMAdapter = class {
|
|
|
2765
3034
|
this.systemPrompt = (_c = llmConfig.systemPrompt) != null ? _c : "You are a helpful AI assistant. Use the provided context to answer the user.";
|
|
2766
3035
|
this.maxTokens = (_d = llmConfig.maxTokens) != null ? _d : 1024;
|
|
2767
3036
|
this.temperature = (_e = llmConfig.temperature) != null ? _e : 0;
|
|
2768
|
-
|
|
2769
|
-
|
|
3037
|
+
this.apiKey = config.apiKey;
|
|
3038
|
+
this.baseUrl = (_g = (_f = llmConfig.baseUrl) != null ? _f : this.opts.baseUrl) != null ? _g : "";
|
|
3039
|
+
if (!this.baseUrl) {
|
|
2770
3040
|
throw new Error("[UniversalLLMAdapter] baseUrl is required in config or config.options");
|
|
2771
3041
|
}
|
|
3042
|
+
this.resolvedHeaders = __spreadValues(__spreadValues({
|
|
3043
|
+
"Content-Type": "application/json"
|
|
3044
|
+
}, config.apiKey ? { Authorization: `Bearer ${config.apiKey}` } : {}), this.opts.headers || {});
|
|
2772
3045
|
this.http = import_axios2.default.create({
|
|
2773
|
-
baseURL: baseUrl,
|
|
2774
|
-
headers:
|
|
2775
|
-
|
|
2776
|
-
}, config.apiKey ? { Authorization: `Bearer ${config.apiKey}` } : {}), this.opts.headers || {}),
|
|
2777
|
-
timeout: (_g = this.opts.timeout) != null ? _g : 6e4
|
|
3046
|
+
baseURL: this.baseUrl,
|
|
3047
|
+
headers: this.resolvedHeaders,
|
|
3048
|
+
timeout: (_h = this.opts.timeout) != null ? _h : 6e4
|
|
2778
3049
|
});
|
|
2779
3050
|
}
|
|
2780
3051
|
async chat(messages, context) {
|
|
@@ -2811,6 +3082,92 @@ ${context != null ? context : "None"}` },
|
|
|
2811
3082
|
}
|
|
2812
3083
|
return String(result);
|
|
2813
3084
|
}
|
|
3085
|
+
/**
|
|
3086
|
+
* Streaming chat using native fetch + ReadableStream.
|
|
3087
|
+
* Parses OpenAI-compatible SSE frames: `data: {...}\n\n`
|
|
3088
|
+
* Works with vLLM, LMStudio, Together AI, Fireworks, and any OpenAI-compatible API.
|
|
3089
|
+
*/
|
|
3090
|
+
chatStream(messages, context) {
|
|
3091
|
+
return __asyncGenerator(this, null, function* () {
|
|
3092
|
+
var _a, _b, _c;
|
|
3093
|
+
const path = (_a = this.opts.chatPath) != null ? _a : "/chat/completions";
|
|
3094
|
+
const url = `${this.baseUrl.replace(/\/$/, "")}${path}`;
|
|
3095
|
+
const extractPath = ((_b = this.opts.responseExtractPath) != null ? _b : "choices[0].message.content").replace("message.content", "delta.content");
|
|
3096
|
+
const formattedMessages = [
|
|
3097
|
+
{ role: "system", content: `${this.systemPrompt}
|
|
3098
|
+
|
|
3099
|
+
Context:
|
|
3100
|
+
${context != null ? context : "None"}` },
|
|
3101
|
+
...messages
|
|
3102
|
+
];
|
|
3103
|
+
let payload;
|
|
3104
|
+
if (this.opts.chatPayloadTemplate) {
|
|
3105
|
+
payload = buildPayload(this.opts.chatPayloadTemplate, {
|
|
3106
|
+
model: this.model,
|
|
3107
|
+
messages: formattedMessages,
|
|
3108
|
+
maxTokens: this.maxTokens,
|
|
3109
|
+
temperature: this.temperature
|
|
3110
|
+
});
|
|
3111
|
+
if (typeof payload === "object" && payload !== null) {
|
|
3112
|
+
payload.stream = true;
|
|
3113
|
+
}
|
|
3114
|
+
} else {
|
|
3115
|
+
payload = {
|
|
3116
|
+
model: this.model,
|
|
3117
|
+
messages: formattedMessages,
|
|
3118
|
+
max_tokens: this.maxTokens,
|
|
3119
|
+
temperature: this.temperature,
|
|
3120
|
+
stream: true
|
|
3121
|
+
};
|
|
3122
|
+
}
|
|
3123
|
+
const response = yield new __await(fetch(url, {
|
|
3124
|
+
method: "POST",
|
|
3125
|
+
headers: this.resolvedHeaders,
|
|
3126
|
+
body: JSON.stringify(payload)
|
|
3127
|
+
}));
|
|
3128
|
+
if (!response.ok) {
|
|
3129
|
+
const errorText = yield new __await(response.text().catch(() => response.statusText));
|
|
3130
|
+
throw new Error(`[UniversalLLMAdapter] Streaming request failed (${response.status}): ${errorText}`);
|
|
3131
|
+
}
|
|
3132
|
+
if (!response.body) {
|
|
3133
|
+
throw new Error("[UniversalLLMAdapter] Response body is null \u2014 server did not send a streaming response.");
|
|
3134
|
+
}
|
|
3135
|
+
const reader = response.body.getReader();
|
|
3136
|
+
const decoder = new TextDecoder("utf-8");
|
|
3137
|
+
let buffer = "";
|
|
3138
|
+
try {
|
|
3139
|
+
while (true) {
|
|
3140
|
+
const { done, value } = yield new __await(reader.read());
|
|
3141
|
+
if (done) break;
|
|
3142
|
+
buffer += decoder.decode(value, { stream: true });
|
|
3143
|
+
const lines = buffer.split("\n");
|
|
3144
|
+
buffer = (_c = lines.pop()) != null ? _c : "";
|
|
3145
|
+
for (const line of lines) {
|
|
3146
|
+
const trimmed = line.trim();
|
|
3147
|
+
if (!trimmed || trimmed === "data: [DONE]") continue;
|
|
3148
|
+
if (!trimmed.startsWith("data:")) continue;
|
|
3149
|
+
try {
|
|
3150
|
+
const json = JSON.parse(trimmed.slice(5).trim());
|
|
3151
|
+
const text = resolvePath(json, extractPath);
|
|
3152
|
+
if (text && typeof text === "string") yield text;
|
|
3153
|
+
} catch (e) {
|
|
3154
|
+
}
|
|
3155
|
+
}
|
|
3156
|
+
}
|
|
3157
|
+
if (buffer.trim() && buffer.trim() !== "data: [DONE]") {
|
|
3158
|
+
const jsonStr = buffer.replace(/^data:\s*/, "").trim();
|
|
3159
|
+
try {
|
|
3160
|
+
const json = JSON.parse(jsonStr);
|
|
3161
|
+
const text = resolvePath(json, extractPath);
|
|
3162
|
+
if (text && typeof text === "string") yield text;
|
|
3163
|
+
} catch (e) {
|
|
3164
|
+
}
|
|
3165
|
+
}
|
|
3166
|
+
} finally {
|
|
3167
|
+
reader.releaseLock();
|
|
3168
|
+
}
|
|
3169
|
+
});
|
|
3170
|
+
}
|
|
2814
3171
|
async embed(text) {
|
|
2815
3172
|
var _a, _b;
|
|
2816
3173
|
const path = (_a = this.opts.embedPath) != null ? _a : "/embeddings";
|
|
@@ -3007,6 +3364,7 @@ var ProviderRegistry = class {
|
|
|
3007
3364
|
return null;
|
|
3008
3365
|
}
|
|
3009
3366
|
static async loadVectorProviderClass(provider) {
|
|
3367
|
+
var _a;
|
|
3010
3368
|
if (this.vectorProviders[provider]) return this.vectorProviders[provider];
|
|
3011
3369
|
switch (provider) {
|
|
3012
3370
|
case "pinecone": {
|
|
@@ -3015,6 +3373,11 @@ var ProviderRegistry = class {
|
|
|
3015
3373
|
}
|
|
3016
3374
|
case "pgvector":
|
|
3017
3375
|
case "postgresql": {
|
|
3376
|
+
const postgresMode = ((_a = process.env.POSTGRES_MODE) != null ? _a : "multi").toLowerCase();
|
|
3377
|
+
if (postgresMode === "single") {
|
|
3378
|
+
const { PostgreSQLProvider: PostgreSQLProvider2 } = await Promise.resolve().then(() => (init_PostgreSQLProvider(), PostgreSQLProvider_exports));
|
|
3379
|
+
return PostgreSQLProvider2;
|
|
3380
|
+
}
|
|
3018
3381
|
const { MultiTablePostgresProvider: MultiTablePostgresProvider2 } = await Promise.resolve().then(() => (init_MultiTablePostgresProvider(), MultiTablePostgresProvider_exports));
|
|
3019
3382
|
return MultiTablePostgresProvider2;
|
|
3020
3383
|
}
|
|
@@ -4008,6 +4371,76 @@ var QueryProcessor = class {
|
|
|
4008
4371
|
}
|
|
4009
4372
|
return [...hints.values()];
|
|
4010
4373
|
}
|
|
4374
|
+
static extractNumericPredicates(question, validFields = []) {
|
|
4375
|
+
const predicates = [];
|
|
4376
|
+
const seen = /* @__PURE__ */ new Set();
|
|
4377
|
+
const comparatorPattern = [
|
|
4378
|
+
"greater than or equal to",
|
|
4379
|
+
"more than or equal to",
|
|
4380
|
+
"less than or equal to",
|
|
4381
|
+
"greater than",
|
|
4382
|
+
"more than",
|
|
4383
|
+
"less than",
|
|
4384
|
+
"equal to",
|
|
4385
|
+
"at least",
|
|
4386
|
+
"at most",
|
|
4387
|
+
"above",
|
|
4388
|
+
"over",
|
|
4389
|
+
"below",
|
|
4390
|
+
"under",
|
|
4391
|
+
"equals?",
|
|
4392
|
+
">=",
|
|
4393
|
+
"<=",
|
|
4394
|
+
">",
|
|
4395
|
+
"<",
|
|
4396
|
+
"="
|
|
4397
|
+
].join("|");
|
|
4398
|
+
const addPredicate = (rawField, rawOperator, rawValue) => {
|
|
4399
|
+
const value = Number(rawValue.replace(/,/g, ""));
|
|
4400
|
+
if (!Number.isFinite(value)) return;
|
|
4401
|
+
const operator = this.normalizeNumericOperator(rawOperator);
|
|
4402
|
+
const field = rawField ? this.normalizePredicateField(rawField, validFields) : void 0;
|
|
4403
|
+
const key = `${field != null ? field : "*"}::${operator}::${value}`;
|
|
4404
|
+
if (seen.has(key)) return;
|
|
4405
|
+
seen.add(key);
|
|
4406
|
+
predicates.push(__spreadProps(__spreadValues({}, field ? { field } : {}), { operator, value }));
|
|
4407
|
+
};
|
|
4408
|
+
const scopedPatterns = [
|
|
4409
|
+
new RegExp(`\\b(?:whose|with|having|where|that\\s+have|which\\s+have)\\s+([a-zA-Z][a-zA-Z0-9_\\s\\-/]{1,80}?)\\s+(?:is|are|was|were)?\\s*(?:${comparatorPattern})\\s+([\\d,]+(?:\\.\\d+)?)`, "gi"),
|
|
4410
|
+
new RegExp(`\\b([a-zA-Z][a-zA-Z0-9_\\s\\-/]{1,80}?)\\s+(?:is|are|was|were)?\\s*(?:${comparatorPattern})\\s+([\\d,]+(?:\\.\\d+)?)`, "gi")
|
|
4411
|
+
];
|
|
4412
|
+
for (const pattern of scopedPatterns) {
|
|
4413
|
+
for (const match of question.matchAll(pattern)) {
|
|
4414
|
+
const full = match[0];
|
|
4415
|
+
const operatorMatch = full.match(new RegExp(`(${comparatorPattern})`, "i"));
|
|
4416
|
+
if (!operatorMatch) continue;
|
|
4417
|
+
addPredicate(match[1], operatorMatch[1], match[2]);
|
|
4418
|
+
}
|
|
4419
|
+
}
|
|
4420
|
+
for (const match of question.matchAll(/\b([a-zA-Z][a-zA-Z0-9_\s\-/]{1,80}?)\s*(>=|<=|>|<|=)\s*([\d,]+(?:\.\d+)?)/g)) {
|
|
4421
|
+
addPredicate(match[1], match[2], match[3]);
|
|
4422
|
+
}
|
|
4423
|
+
return predicates;
|
|
4424
|
+
}
|
|
4425
|
+
static normalizeNumericOperator(operator) {
|
|
4426
|
+
const op = operator.toLowerCase().trim();
|
|
4427
|
+
if (op === ">" || /\b(greater than|more than|above|over)\b/.test(op)) return "gt";
|
|
4428
|
+
if (op === ">=" || /\b(greater than or equal to|more than or equal to|at least)\b/.test(op)) return "gte";
|
|
4429
|
+
if (op === "<" || /\b(less than|below|under)\b/.test(op)) return "lt";
|
|
4430
|
+
if (op === "<=" || /\b(less than or equal to|at most)\b/.test(op)) return "lte";
|
|
4431
|
+
return "eq";
|
|
4432
|
+
}
|
|
4433
|
+
static normalizePredicateField(field, validFields) {
|
|
4434
|
+
const cleaned = this.normalizeHintValue(field).replace(/\b(?:provide|show|get|give|list|all|the|a|an|of|organizations?|companies?|records?|items?|whose|with|having|where|that|which|have|has|is|are|was|were)\b/gi, " ").replace(/\s+/g, " ").trim();
|
|
4435
|
+
if (validFields.length === 0) return cleaned;
|
|
4436
|
+
const comparable = (value) => value.toLowerCase().replace(/[^a-z0-9]/g, "");
|
|
4437
|
+
const cleanedComparable = comparable(cleaned);
|
|
4438
|
+
const matchedField = validFields.find((fieldName) => {
|
|
4439
|
+
const candidate = comparable(fieldName);
|
|
4440
|
+
return candidate === cleanedComparable || candidate.includes(cleanedComparable) || cleanedComparable.includes(candidate);
|
|
4441
|
+
});
|
|
4442
|
+
return matchedField != null ? matchedField : cleaned;
|
|
4443
|
+
}
|
|
4011
4444
|
/**
|
|
4012
4445
|
* Constructs a QueryFilter object from extracted hints.
|
|
4013
4446
|
*
|
|
@@ -4028,6 +4461,10 @@ var QueryProcessor = class {
|
|
|
4028
4461
|
}
|
|
4029
4462
|
if (Object.keys(filter.metadata || {}).length === 0) delete filter.metadata;
|
|
4030
4463
|
if (filter.keywords && filter.keywords.length === 0) delete filter.keywords;
|
|
4464
|
+
const numericPredicates = this.extractNumericPredicates(question);
|
|
4465
|
+
if (numericPredicates.length > 0) {
|
|
4466
|
+
filter.__numericPredicates = numericPredicates;
|
|
4467
|
+
}
|
|
4031
4468
|
return filter;
|
|
4032
4469
|
}
|
|
4033
4470
|
/**
|
|
@@ -4095,6 +4532,61 @@ Return ONLY 'vector', 'graph', or 'both'. No explanation.`;
|
|
|
4095
4532
|
}
|
|
4096
4533
|
};
|
|
4097
4534
|
|
|
4535
|
+
// src/core/LLMRouter.ts
|
|
4536
|
+
var FAST_MODEL_DEFAULTS = {
|
|
4537
|
+
openai: "gpt-4o-mini",
|
|
4538
|
+
gemini: "gemini-2.0-flash",
|
|
4539
|
+
anthropic: "claude-3-haiku-20240307",
|
|
4540
|
+
ollama: "",
|
|
4541
|
+
// Ollama has no universal lightweight default — reuse main model
|
|
4542
|
+
rest: "",
|
|
4543
|
+
universal_rest: "",
|
|
4544
|
+
custom: ""
|
|
4545
|
+
};
|
|
4546
|
+
var LLMRouter = class {
|
|
4547
|
+
constructor(config) {
|
|
4548
|
+
this.config = config;
|
|
4549
|
+
this.models = /* @__PURE__ */ new Map();
|
|
4550
|
+
}
|
|
4551
|
+
/**
|
|
4552
|
+
* Initialize all LLM roles.
|
|
4553
|
+
*
|
|
4554
|
+
* @param prebuiltDefault - optional pre-built provider (from EmbeddingStrategyResolver).
|
|
4555
|
+
* When provided it is used directly as the 'default' role without re-constructing.
|
|
4556
|
+
*/
|
|
4557
|
+
async initialize(prebuiltDefault) {
|
|
4558
|
+
var _a;
|
|
4559
|
+
const defaultModel = prebuiltDefault != null ? prebuiltDefault : LLMFactory.create(this.config.llm, this.config.embedding);
|
|
4560
|
+
this.models.set("default", defaultModel);
|
|
4561
|
+
const envFastModel = process.env.FAST_LLM_MODEL;
|
|
4562
|
+
const providerFastDefault = (_a = FAST_MODEL_DEFAULTS[this.config.llm.provider]) != null ? _a : "";
|
|
4563
|
+
const fastModelName = envFastModel || providerFastDefault;
|
|
4564
|
+
if (fastModelName && fastModelName !== this.config.llm.model) {
|
|
4565
|
+
console.log(`[LLMRouter] Fast role \u2192 provider="${this.config.llm.provider}" model="${fastModelName}"`);
|
|
4566
|
+
const fastConfig = __spreadProps(__spreadValues({}, this.config.llm), {
|
|
4567
|
+
model: fastModelName
|
|
4568
|
+
});
|
|
4569
|
+
this.models.set("fast", LLMFactory.create(fastConfig, this.config.embedding));
|
|
4570
|
+
} else {
|
|
4571
|
+
console.log(`[LLMRouter] Fast role \u2192 reusing default model (no lightweight alternative configured).`);
|
|
4572
|
+
this.models.set("fast", defaultModel);
|
|
4573
|
+
}
|
|
4574
|
+
this.models.set("powerful", defaultModel);
|
|
4575
|
+
}
|
|
4576
|
+
/**
|
|
4577
|
+
* Retrieve a model provider by its task role.
|
|
4578
|
+
* Falls back to 'default' if the requested role is not registered.
|
|
4579
|
+
*/
|
|
4580
|
+
get(role) {
|
|
4581
|
+
var _a;
|
|
4582
|
+
const provider = (_a = this.models.get(role)) != null ? _a : this.models.get("default");
|
|
4583
|
+
if (!provider) {
|
|
4584
|
+
throw new Error(`[LLMRouter] No provider registered for role: "${role}". Did you call initialize()?`);
|
|
4585
|
+
}
|
|
4586
|
+
return provider;
|
|
4587
|
+
}
|
|
4588
|
+
};
|
|
4589
|
+
|
|
4098
4590
|
// src/utils/synonyms.ts
|
|
4099
4591
|
var FIELD_SYNONYMS = {
|
|
4100
4592
|
name: ["product", "item", "title", "label", "heading", "subject", "product name", "item name"],
|
|
@@ -4124,6 +4616,14 @@ var FIELD_SYNONYMS = {
|
|
|
4124
4616
|
"in stock",
|
|
4125
4617
|
"status"
|
|
4126
4618
|
],
|
|
4619
|
+
category: [
|
|
4620
|
+
"product_category",
|
|
4621
|
+
"product category",
|
|
4622
|
+
"category_name",
|
|
4623
|
+
"category name",
|
|
4624
|
+
"department",
|
|
4625
|
+
"collection"
|
|
4626
|
+
],
|
|
4127
4627
|
description: ["summary", "content", "body", "text", "info", "details"],
|
|
4128
4628
|
link: ["url", "href", "product_url", "page_url", "link"]
|
|
4129
4629
|
};
|
|
@@ -4155,98 +4655,457 @@ function resolveMetadataValue(meta, uiKey) {
|
|
|
4155
4655
|
|
|
4156
4656
|
// src/utils/UITransformer.ts
|
|
4157
4657
|
var UITransformer = class {
|
|
4658
|
+
// ─── Public Entry Points ─────────────────────────────────────────────────
|
|
4158
4659
|
/**
|
|
4159
|
-
*
|
|
4160
|
-
*
|
|
4660
|
+
* Heuristic-only transform (no LLM required).
|
|
4661
|
+
* Uses the lightweight heuristic intent detector as a fallback.
|
|
4662
|
+
* Prefer `analyzeAndDecide()` in production.
|
|
4161
4663
|
*/
|
|
4162
|
-
static transform(userQuery, retrievedData, config, trainedSchema) {
|
|
4664
|
+
static transform(userQuery, retrievedData, config, trainedSchema, intent) {
|
|
4665
|
+
var _a, _b, _c;
|
|
4163
4666
|
if (!retrievedData || retrievedData.length === 0) {
|
|
4164
4667
|
return this.createTextResponse("No data available", "No relevant data found for your query.");
|
|
4165
4668
|
}
|
|
4166
|
-
const
|
|
4167
|
-
const filteredData =
|
|
4168
|
-
const
|
|
4669
|
+
const resolvedIntent = intent != null ? intent : this.detectIntentHeuristic(userQuery);
|
|
4670
|
+
const filteredData = resolvedIntent.filterInStockOnly ? retrievedData.filter((item) => this.determineStockStatus(item)) : retrievedData;
|
|
4671
|
+
const profile = this.profileData(filteredData);
|
|
4169
4672
|
const hasProducts = filteredData.some((item) => this.isProductData(item));
|
|
4170
|
-
const
|
|
4171
|
-
|
|
4172
|
-
|
|
4173
|
-
return this.transformToLineChart(filteredData);
|
|
4673
|
+
const wantsPieLikeChart = ["pie_chart", "donut_chart"].includes(resolvedIntent.recommendedChart);
|
|
4674
|
+
if (resolvedIntent.visualizationHint === "trend" && profile.dateFields.length > 0 && profile.numericFields.length > 0) {
|
|
4675
|
+
return this.transformToLineChart(profile);
|
|
4174
4676
|
}
|
|
4175
|
-
if (
|
|
4176
|
-
|
|
4677
|
+
if (wantsPieLikeChart || ["composition", "category_breakdown"].includes(resolvedIntent.visualizationHint)) {
|
|
4678
|
+
const pieChart = this.transformToPieChart(filteredData, profile, userQuery);
|
|
4679
|
+
if (pieChart) return pieChart;
|
|
4177
4680
|
}
|
|
4178
|
-
if (
|
|
4179
|
-
return this.
|
|
4681
|
+
if (["comparison", "ranking"].includes(resolvedIntent.visualizationHint)) {
|
|
4682
|
+
return this.transformToBarChart(filteredData, profile, userQuery, resolvedIntent.visualizationHint === "ranking");
|
|
4180
4683
|
}
|
|
4181
|
-
if (
|
|
4182
|
-
return this.
|
|
4684
|
+
if (resolvedIntent.visualizationHint === "distribution") {
|
|
4685
|
+
return (_a = this.transformToHistogram(profile, userQuery)) != null ? _a : this.transformToBarChart(filteredData, profile, userQuery);
|
|
4686
|
+
}
|
|
4687
|
+
if (resolvedIntent.visualizationHint === "correlation") {
|
|
4688
|
+
return (_b = this.transformToScatterPlot(profile, userQuery)) != null ? _b : this.transformToBarChart(filteredData, profile, userQuery);
|
|
4689
|
+
}
|
|
4690
|
+
if (resolvedIntent.visualizationHint === "geographic") {
|
|
4691
|
+
return this.transformToBarChart(filteredData, profile, userQuery);
|
|
4692
|
+
}
|
|
4693
|
+
if (this.isStructuredListQuery(userQuery)) {
|
|
4694
|
+
return this.hasMultipleFields(filteredData) ? this.transformToTable(filteredData, userQuery) : this.transformToText(filteredData);
|
|
4695
|
+
}
|
|
4696
|
+
if (resolvedIntent.visualizationHint === "kpi") {
|
|
4697
|
+
return (_c = this.transformToMetricCard(profile, userQuery)) != null ? _c : this.transformToText(filteredData);
|
|
4698
|
+
}
|
|
4699
|
+
if (["tabular", "table"].includes(resolvedIntent.visualizationHint) || resolvedIntent.wantsExplicitTable) {
|
|
4700
|
+
return this.hasMultipleFields(filteredData) ? this.transformToTable(filteredData, userQuery) : this.transformToText(filteredData);
|
|
4183
4701
|
}
|
|
4184
|
-
if (this.
|
|
4185
|
-
return this.
|
|
4702
|
+
if ((hasProducts || this.isProductQuery(userQuery)) && resolvedIntent.visualizationHint === "product_browse") {
|
|
4703
|
+
return this.transformToProductCarousel(filteredData, config, trainedSchema);
|
|
4186
4704
|
}
|
|
4705
|
+
const automatic = this.chooseAutomaticVisualization(filteredData, profile, userQuery);
|
|
4706
|
+
if (automatic) return automatic;
|
|
4187
4707
|
return this.transformToText(filteredData);
|
|
4188
4708
|
}
|
|
4189
4709
|
/**
|
|
4190
|
-
*
|
|
4710
|
+
* LLM-driven entry point (recommended for production).
|
|
4711
|
+
*
|
|
4712
|
+
* Step 1 — Detect intent via a dedicated, lightweight LLM call.
|
|
4713
|
+
* Step 2 — Pass the intent + data to the visualization-selection prompt.
|
|
4714
|
+
* Step 3 — Fall back to the heuristic `transform()` if either LLM call fails.
|
|
4191
4715
|
*/
|
|
4192
|
-
static
|
|
4193
|
-
|
|
4194
|
-
|
|
4195
|
-
|
|
4196
|
-
|
|
4197
|
-
|
|
4198
|
-
|
|
4199
|
-
|
|
4716
|
+
static async analyzeAndDecide(query, sources, llm) {
|
|
4717
|
+
let intent;
|
|
4718
|
+
try {
|
|
4719
|
+
intent = await this.detectIntent(query, llm);
|
|
4720
|
+
console.debug("[UITransformer] Detected intent:", intent);
|
|
4721
|
+
} catch (err) {
|
|
4722
|
+
console.warn("[UITransformer] Intent detection failed; using heuristic.", err);
|
|
4723
|
+
intent = this.detectIntentHeuristic(query);
|
|
4724
|
+
}
|
|
4725
|
+
if (this.isProductQuery(query) && ["text", "product_browse"].includes(intent.visualizationHint)) {
|
|
4726
|
+
return this.transform(
|
|
4727
|
+
query,
|
|
4728
|
+
sources,
|
|
4729
|
+
void 0,
|
|
4730
|
+
void 0,
|
|
4731
|
+
__spreadProps(__spreadValues({}, intent), { visualizationHint: "product_browse", recommendedChart: "text" })
|
|
4732
|
+
);
|
|
4733
|
+
}
|
|
4734
|
+
try {
|
|
4735
|
+
const context = this.buildContextSummary(sources);
|
|
4736
|
+
const systemPrompt = this.buildVisualizationSystemPrompt();
|
|
4737
|
+
const userPrompt = [
|
|
4738
|
+
`USER QUESTION: ${query}`,
|
|
4739
|
+
"",
|
|
4740
|
+
`DETECTED INTENT (JSON): ${JSON.stringify(intent)}`,
|
|
4741
|
+
"",
|
|
4742
|
+
"RETRIEVED DATA (JSON):",
|
|
4743
|
+
context
|
|
4744
|
+
].join("\n");
|
|
4745
|
+
const rawResponse = await llm.chat(
|
|
4746
|
+
[{ role: "user", content: userPrompt }],
|
|
4747
|
+
"",
|
|
4748
|
+
{ systemPrompt, temperature: 0 }
|
|
4749
|
+
);
|
|
4750
|
+
const parsed = this.parseTransformationResponse(rawResponse);
|
|
4751
|
+
if (parsed) {
|
|
4752
|
+
const intentAllowsTable = intent.wantsExplicitTable || ["tabular", "table", "geographic"].includes(intent.visualizationHint);
|
|
4753
|
+
const intentWantsPieLikeChart = ["pie_chart", "donut_chart"].includes(intent.recommendedChart) || ["composition", "category_breakdown"].includes(intent.visualizationHint);
|
|
4754
|
+
if (parsed.type === "table" && !intentAllowsTable) {
|
|
4755
|
+
console.debug("[UITransformer] LLM chose table but intent says no. Falling back to text.");
|
|
4756
|
+
return this.transform(query, sources, void 0, void 0, intent);
|
|
4757
|
+
}
|
|
4758
|
+
if (intentWantsPieLikeChart && parsed.type !== "pie_chart" && this.detectCategories(sources).length > 1) {
|
|
4759
|
+
console.debug("[UITransformer] LLM ignored pie/composition intent. Using deterministic pie chart.");
|
|
4760
|
+
return this.transform(query, sources, void 0, void 0, intent);
|
|
4761
|
+
}
|
|
4762
|
+
console.debug("[UITransformer] LLM chose visualization type:", parsed.type);
|
|
4763
|
+
return parsed;
|
|
4764
|
+
}
|
|
4765
|
+
console.warn("[UITransformer] LLM returned unparseable response; falling back to heuristic.");
|
|
4766
|
+
} catch (err) {
|
|
4767
|
+
console.warn("[UITransformer] analyzeAndDecide LLM call failed; falling back to heuristic.", err);
|
|
4768
|
+
}
|
|
4769
|
+
return this.transform(query, sources, void 0, void 0, intent);
|
|
4200
4770
|
}
|
|
4771
|
+
// ─── Dynamic Intent Detection ─────────────────────────────────────────────
|
|
4201
4772
|
/**
|
|
4202
|
-
*
|
|
4773
|
+
* Calls the LLM with a compact, focused prompt to extract a structured
|
|
4774
|
+
* `QueryIntent` from the user's query.
|
|
4775
|
+
*
|
|
4776
|
+
* Keeping this as a *separate* call from visualization selection means:
|
|
4777
|
+
* - The prompt is shorter and more reliable.
|
|
4778
|
+
* - The intent object can be reused across both the heuristic and LLM paths.
|
|
4779
|
+
* - It is easy to unit-test intent detection in isolation.
|
|
4203
4780
|
*/
|
|
4204
|
-
static
|
|
4205
|
-
const
|
|
4206
|
-
|
|
4207
|
-
|
|
4208
|
-
|
|
4781
|
+
static async detectIntent(query, llm) {
|
|
4782
|
+
const systemPrompt = `You are an intent classifier for a product-search RAG system.
|
|
4783
|
+
Given a user query, return ONLY a valid JSON object with this exact shape \u2014 no prose, no markdown:
|
|
4784
|
+
|
|
4785
|
+
{
|
|
4786
|
+
"visualizationHint": "trend" | "comparison" | "distribution" | "composition" | "correlation" | "ranking" | "kpi" | "tabular" | "geographic" | "product_browse" | "table" | "text",
|
|
4787
|
+
"recommendedChart": "line_chart" | "bar_chart" | "histogram" | "pie_chart" | "donut_chart" | "scatter_plot" | "horizontal_bar" | "metric_card" | "table" | "geo_map" | "text",
|
|
4788
|
+
"filterInStockOnly": boolean,
|
|
4789
|
+
"wantsExplicitTable": boolean,
|
|
4790
|
+
"isTemporal": boolean,
|
|
4791
|
+
"isComparison": boolean,
|
|
4792
|
+
"language": "<BCP-47 language tag, e.g. en, de, hi, ja>",
|
|
4793
|
+
"reasoning": "<one short sentence \u2014 why you chose these values>"
|
|
4794
|
+
}
|
|
4795
|
+
|
|
4796
|
+
RULES:
|
|
4797
|
+
- visualizationHint and recommendedChart
|
|
4798
|
+
"trend" \u2192 keywords: trend, growth, over time; recommendedChart "line_chart"
|
|
4799
|
+
"comparison" \u2192 keywords: compare, versus, top; recommendedChart "bar_chart"
|
|
4800
|
+
"distribution" \u2192 keywords: distribution, spread; recommendedChart "histogram"
|
|
4801
|
+
"composition" \u2192 keywords: share, percentage, breakup, breakdown; recommendedChart "pie_chart" or "donut_chart"
|
|
4802
|
+
"correlation" \u2192 keywords: relation, correlation; recommendedChart "scatter_plot"
|
|
4803
|
+
"ranking" \u2192 keywords: highest, lowest, top 10, ranking; recommendedChart "horizontal_bar"
|
|
4804
|
+
"kpi" \u2192 keywords: total, average, count; recommendedChart "metric_card"
|
|
4805
|
+
"tabular" \u2192 keywords: detailed records, table, grid, spreadsheet; recommendedChart "table"
|
|
4806
|
+
"geographic" \u2192 keywords: region, country, map; recommendedChart "geo_map"
|
|
4807
|
+
"product_browse" \u2192 user is browsing, searching, describing, viewing details for, or asking about one or more products without analytical visualization intent
|
|
4808
|
+
"table" \u2192 legacy alias for "tabular"; use only if the query literally says table
|
|
4809
|
+
"text" \u2192 conversational, factual, or other intent
|
|
4810
|
+
- If the user explicitly asks for a chart type, honor that chart type in recommendedChart.
|
|
4811
|
+
- If a query says "distribution ... in a pie chart", use visualizationHint "composition" and recommendedChart "pie_chart".
|
|
4812
|
+
- filterInStockOnly: true only if user mentions stock, availability, in stock, etc.
|
|
4813
|
+
- wantsExplicitTable: true if the user asks for a table, list, grid, spreadsheet, or detailed records.
|
|
4814
|
+
- isTemporal: true if time words appear (trend, historical, over time, last year, monthly, etc.)
|
|
4815
|
+
- isComparison: true if user compares, ranks, or contrasts entities or categories.
|
|
4816
|
+
- language: detect from the query text itself; default "en" if uncertain.`;
|
|
4817
|
+
const rawResponse = await llm.chat(
|
|
4818
|
+
[{ role: "user", content: `QUERY: ${query}` }],
|
|
4819
|
+
"",
|
|
4820
|
+
{ systemPrompt, temperature: 0 }
|
|
4821
|
+
);
|
|
4822
|
+
const parsed = this.parseIntentResponse(rawResponse);
|
|
4823
|
+
if (!parsed) {
|
|
4824
|
+
throw new Error(`Could not parse intent JSON from LLM response: ${rawResponse}`);
|
|
4825
|
+
}
|
|
4826
|
+
return parsed;
|
|
4827
|
+
}
|
|
4828
|
+
/**
|
|
4829
|
+
* Parse and validate the raw LLM response into a `QueryIntent`.
|
|
4830
|
+
*/
|
|
4831
|
+
static parseIntentResponse(raw) {
|
|
4832
|
+
const jsonStr = this.extractJsonCandidate(raw);
|
|
4833
|
+
if (!jsonStr) return null;
|
|
4834
|
+
try {
|
|
4835
|
+
const obj = JSON.parse(jsonStr);
|
|
4836
|
+
const validHints = [
|
|
4837
|
+
"trend",
|
|
4838
|
+
"comparison",
|
|
4839
|
+
"distribution",
|
|
4840
|
+
"composition",
|
|
4841
|
+
"correlation",
|
|
4842
|
+
"ranking",
|
|
4843
|
+
"kpi",
|
|
4844
|
+
"tabular",
|
|
4845
|
+
"geographic",
|
|
4846
|
+
"category_breakdown",
|
|
4847
|
+
"product_browse",
|
|
4848
|
+
"table",
|
|
4849
|
+
"text"
|
|
4850
|
+
];
|
|
4851
|
+
const validCharts = [
|
|
4852
|
+
"line_chart",
|
|
4853
|
+
"bar_chart",
|
|
4854
|
+
"histogram",
|
|
4855
|
+
"pie_chart",
|
|
4856
|
+
"donut_chart",
|
|
4857
|
+
"scatter_plot",
|
|
4858
|
+
"horizontal_bar",
|
|
4859
|
+
"metric_card",
|
|
4860
|
+
"table",
|
|
4861
|
+
"geo_map",
|
|
4862
|
+
"text"
|
|
4863
|
+
];
|
|
4864
|
+
const hint = obj.visualizationHint;
|
|
4865
|
+
if (!hint || !validHints.includes(hint)) return null;
|
|
4866
|
+
const normalizedHint = hint === "category_breakdown" ? "composition" : hint;
|
|
4867
|
+
const recommendedChart = typeof obj.recommendedChart === "string" && validCharts.includes(obj.recommendedChart) ? obj.recommendedChart : this.getRecommendedChartForIntent(normalizedHint);
|
|
4209
4868
|
return {
|
|
4210
|
-
|
|
4211
|
-
|
|
4212
|
-
|
|
4213
|
-
|
|
4869
|
+
visualizationHint: normalizedHint,
|
|
4870
|
+
recommendedChart,
|
|
4871
|
+
filterInStockOnly: Boolean(obj.filterInStockOnly),
|
|
4872
|
+
wantsExplicitTable: Boolean(obj.wantsExplicitTable),
|
|
4873
|
+
isTemporal: Boolean(obj.isTemporal),
|
|
4874
|
+
isComparison: Boolean(obj.isComparison),
|
|
4875
|
+
language: typeof obj.language === "string" && obj.language ? obj.language : "en",
|
|
4876
|
+
reasoning: typeof obj.reasoning === "string" ? obj.reasoning : void 0
|
|
4214
4877
|
};
|
|
4878
|
+
} catch (e) {
|
|
4879
|
+
return null;
|
|
4880
|
+
}
|
|
4881
|
+
}
|
|
4882
|
+
/**
|
|
4883
|
+
* Heuristic intent detector — used when the LLM is unavailable.
|
|
4884
|
+
* Intentionally minimal: it should only catch the most obvious signals.
|
|
4885
|
+
* The LLM path handles everything subtle.
|
|
4886
|
+
*/
|
|
4887
|
+
static detectIntentHeuristic(query) {
|
|
4888
|
+
const q = query.toLowerCase();
|
|
4889
|
+
const isTemporal = /\b(trend|trends|over time|historical|history|growth|decline|monthly|yearly|weekly|daily|last (year|month|week)|timeline|forecast)\b/.test(q);
|
|
4890
|
+
const isRanking = /\b(highest|lowest|top\s*\d+|bottom\s*\d+|rank(?:ing)?|best|worst|leading)\b/.test(q);
|
|
4891
|
+
const isComparison = /\b(compare|comparison|vs\.?|versus|against|differ|difference|contrast|top)\b/.test(q) || isRanking;
|
|
4892
|
+
const isDistribution = /\b(distribution|spread|histogram|frequency|variance|range)\b/.test(q);
|
|
4893
|
+
const wantsPieLikeChart = /\b(pie|donut|doughnut)(?:\s+chart)?\b/.test(q);
|
|
4894
|
+
const isComposition = wantsPieLikeChart || /\b(share|percentage|percent|breakup|breakdown|composition|split|segmentation|by category|by type|proportion)\b/.test(q);
|
|
4895
|
+
const isCorrelation = /\b(relation|relationship|correlation|correlate|scatter|association|impact of|depend(?:s|ence)? on)\b/.test(q);
|
|
4896
|
+
const isKpi = /\b(total|average|avg|count|sum|median|minimum|maximum|metric|kpi|how many|number of)\b/.test(q);
|
|
4897
|
+
const isGeographic = /\b(region|country|countries|state|city|location|map|geo|geographic|territory)\b/.test(q);
|
|
4898
|
+
const filterInStockOnly = /\b(in[- ]?stock|available|availability|inventory|stock status)\b/.test(q);
|
|
4899
|
+
const wantsExplicitTable = /\b(table|spreadsheet|grid|detailed records|record details|list all|compare all)\b/.test(q);
|
|
4900
|
+
let visualizationHint = "text";
|
|
4901
|
+
if (wantsExplicitTable) visualizationHint = "tabular";
|
|
4902
|
+
else if (isTemporal) visualizationHint = "trend";
|
|
4903
|
+
else if (wantsPieLikeChart) visualizationHint = "composition";
|
|
4904
|
+
else if (isRanking) visualizationHint = "ranking";
|
|
4905
|
+
else if (isComparison) visualizationHint = "comparison";
|
|
4906
|
+
else if (isDistribution) visualizationHint = "distribution";
|
|
4907
|
+
else if (isComposition) visualizationHint = "composition";
|
|
4908
|
+
else if (isCorrelation) visualizationHint = "correlation";
|
|
4909
|
+
else if (isGeographic) visualizationHint = "geographic";
|
|
4910
|
+
else if (isKpi) visualizationHint = "kpi";
|
|
4911
|
+
else if (this.isProductQuery(query)) visualizationHint = "product_browse";
|
|
4912
|
+
return {
|
|
4913
|
+
visualizationHint,
|
|
4914
|
+
recommendedChart: this.getRecommendedChartForIntent(visualizationHint),
|
|
4915
|
+
filterInStockOnly,
|
|
4916
|
+
wantsExplicitTable,
|
|
4917
|
+
isTemporal,
|
|
4918
|
+
isComparison,
|
|
4919
|
+
language: "en"
|
|
4920
|
+
// heuristic cannot reliably detect language
|
|
4921
|
+
};
|
|
4922
|
+
}
|
|
4923
|
+
static getRecommendedChartForIntent(intent) {
|
|
4924
|
+
switch (intent) {
|
|
4925
|
+
case "trend":
|
|
4926
|
+
return "line_chart";
|
|
4927
|
+
case "comparison":
|
|
4928
|
+
return "bar_chart";
|
|
4929
|
+
case "distribution":
|
|
4930
|
+
return "histogram";
|
|
4931
|
+
case "composition":
|
|
4932
|
+
case "category_breakdown":
|
|
4933
|
+
return "pie_chart";
|
|
4934
|
+
case "correlation":
|
|
4935
|
+
return "scatter_plot";
|
|
4936
|
+
case "ranking":
|
|
4937
|
+
return "horizontal_bar";
|
|
4938
|
+
case "kpi":
|
|
4939
|
+
return "metric_card";
|
|
4940
|
+
case "tabular":
|
|
4941
|
+
case "table":
|
|
4942
|
+
return "table";
|
|
4943
|
+
case "geographic":
|
|
4944
|
+
return "geo_map";
|
|
4945
|
+
case "product_browse":
|
|
4946
|
+
case "text":
|
|
4947
|
+
default:
|
|
4948
|
+
return "text";
|
|
4949
|
+
}
|
|
4950
|
+
}
|
|
4951
|
+
// ─── Transform Helpers ────────────────────────────────────────────────────
|
|
4952
|
+
static transformToProductCarousel(data, config, trainedSchema) {
|
|
4953
|
+
const products = data.map((item) => this.extractProductInfo(item, config, trainedSchema)).filter((p) => p !== null);
|
|
4954
|
+
return {
|
|
4955
|
+
type: "product_carousel",
|
|
4956
|
+
title: "Recommended Products",
|
|
4957
|
+
description: `Found ${products.length} relevant products`,
|
|
4958
|
+
data: products
|
|
4959
|
+
};
|
|
4960
|
+
}
|
|
4961
|
+
static transformToPieChart(data, profile, query = "") {
|
|
4962
|
+
var _a;
|
|
4963
|
+
const dimension = profile ? this.selectDimensionField(profile, query) : void 0;
|
|
4964
|
+
const categories = dimension ? Array.from(new Set(profile.records.map((record) => {
|
|
4965
|
+
var _a2;
|
|
4966
|
+
return String((_a2 = record.fields[dimension.key]) != null ? _a2 : "");
|
|
4967
|
+
}).filter(Boolean))) : this.detectCategories(data);
|
|
4968
|
+
if (categories.length === 0) return null;
|
|
4969
|
+
const categoryData = dimension && profile ? this.aggregateProfileByDimension(profile, dimension.key) : this.aggregateByCategory(data, categories);
|
|
4970
|
+
const pieData = Object.entries(categoryData).map(([label, count]) => {
|
|
4971
|
+
const { inStockCount, outOfStockCount } = this.calculateStockCounts(label, data);
|
|
4972
|
+
return { label, value: count, inStockCount, outOfStockCount };
|
|
4215
4973
|
});
|
|
4216
4974
|
return {
|
|
4217
4975
|
type: "pie_chart",
|
|
4218
|
-
title:
|
|
4219
|
-
description: `Showing breakdown across ${
|
|
4976
|
+
title: `Distribution by ${(_a = dimension == null ? void 0 : dimension.label) != null ? _a : "Category"}`,
|
|
4977
|
+
description: `Showing breakdown across ${pieData.length} categories`,
|
|
4220
4978
|
data: pieData
|
|
4221
4979
|
};
|
|
4222
4980
|
}
|
|
4223
|
-
|
|
4224
|
-
|
|
4225
|
-
|
|
4226
|
-
|
|
4227
|
-
|
|
4228
|
-
|
|
4229
|
-
timestamp:
|
|
4230
|
-
value:
|
|
4231
|
-
|
|
4232
|
-
})
|
|
4981
|
+
static transformToLineChart(profile) {
|
|
4982
|
+
const dateField = profile.dateFields[0];
|
|
4983
|
+
const valueField = profile.numericFields[0];
|
|
4984
|
+
const buckets = /* @__PURE__ */ new Map();
|
|
4985
|
+
profile.records.forEach((record) => {
|
|
4986
|
+
var _a, _b, _c;
|
|
4987
|
+
const timestamp = String((_a = record.fields[dateField.key]) != null ? _a : "");
|
|
4988
|
+
const value = (_b = this.toFiniteNumber(record.fields[valueField.key])) != null ? _b : 0;
|
|
4989
|
+
buckets.set(timestamp, ((_c = buckets.get(timestamp)) != null ? _c : 0) + value);
|
|
4990
|
+
});
|
|
4991
|
+
const lineData = Array.from(buckets.entries()).sort(([a], [b]) => Date.parse(a) - Date.parse(b)).slice(0, 24).map(([timestamp, value]) => ({ timestamp, value, label: timestamp }));
|
|
4233
4992
|
return {
|
|
4234
4993
|
type: "line_chart",
|
|
4235
|
-
title:
|
|
4994
|
+
title: `${valueField.label} Over Time`,
|
|
4236
4995
|
description: `Showing ${lineData.length} data points`,
|
|
4237
4996
|
data: lineData
|
|
4238
4997
|
};
|
|
4239
4998
|
}
|
|
4240
|
-
|
|
4241
|
-
|
|
4242
|
-
|
|
4243
|
-
|
|
4244
|
-
const
|
|
4245
|
-
const
|
|
4246
|
-
const
|
|
4247
|
-
|
|
4248
|
-
|
|
4999
|
+
static transformToBarChart(data, profile, query = "", horizontal = false) {
|
|
5000
|
+
var _a;
|
|
5001
|
+
const dimension = profile ? this.selectDimensionField(profile, query) : void 0;
|
|
5002
|
+
const measure = profile ? this.selectNumericField(profile, query) : void 0;
|
|
5003
|
+
const aggregate = dimension && profile ? this.aggregateProfileByDimension(profile, dimension.key, measure == null ? void 0 : measure.key) : this.aggregateByCategory(data, this.detectCategories(data));
|
|
5004
|
+
const barData = Object.entries(aggregate).map(([category, value]) => ({ category, value: Number(value) })).sort((a, b) => horizontal ? b.value - a.value : 0).slice(0, 12);
|
|
5005
|
+
const fallbackData = barData.length > 0 ? barData : data.slice(0, 12).map((item, index) => {
|
|
5006
|
+
var _a2, _b, _c, _d, _e;
|
|
5007
|
+
const meta = item.metadata || {};
|
|
5008
|
+
const label = String(
|
|
5009
|
+
(_c = (_b = (_a2 = this.getDynamicVal(meta, "name")) != null ? _a2 : meta.label) != null ? _b : meta.title) != null ? _c : `Result ${index + 1}`
|
|
5010
|
+
);
|
|
5011
|
+
const value = (_e = (_d = this.extractNumericValue(meta)) != null ? _d : item.score) != null ? _e : 0;
|
|
5012
|
+
return { category: label, value: Number(value) };
|
|
5013
|
+
});
|
|
5014
|
+
return {
|
|
5015
|
+
type: horizontal ? "horizontal_bar" : "bar_chart",
|
|
5016
|
+
title: dimension ? `${(_a = measure == null ? void 0 : measure.label) != null ? _a : "Count"} by ${dimension.label}` : "Comparison",
|
|
5017
|
+
description: `Showing ${fallbackData.length} comparable values`,
|
|
5018
|
+
data: fallbackData
|
|
4249
5019
|
};
|
|
5020
|
+
}
|
|
5021
|
+
static transformToHistogram(profile, query = "") {
|
|
5022
|
+
const field = this.selectNumericField(profile, query);
|
|
5023
|
+
if (!field) return null;
|
|
5024
|
+
const values = profile.records.map((record) => this.toFiniteNumber(record.fields[field.key])).filter((value) => value !== null).sort((a, b) => a - b);
|
|
5025
|
+
if (values.length === 0) return null;
|
|
5026
|
+
const min = values[0];
|
|
5027
|
+
const max = values[values.length - 1];
|
|
5028
|
+
const bucketCount = Math.min(10, Math.max(3, Math.ceil(Math.sqrt(values.length))));
|
|
5029
|
+
const width = max === min ? 1 : (max - min) / bucketCount;
|
|
5030
|
+
const buckets = Array.from({ length: bucketCount }, (_, index) => {
|
|
5031
|
+
const start = min + index * width;
|
|
5032
|
+
const end = index === bucketCount - 1 ? max : start + width;
|
|
5033
|
+
return { category: `${this.formatNumber(start)}-${this.formatNumber(end)}`, value: 0 };
|
|
5034
|
+
});
|
|
5035
|
+
values.forEach((value) => {
|
|
5036
|
+
const bucketIndex = max === min ? 0 : Math.min(bucketCount - 1, Math.floor((value - min) / width));
|
|
5037
|
+
buckets[bucketIndex].value += 1;
|
|
5038
|
+
});
|
|
5039
|
+
return {
|
|
5040
|
+
type: "histogram",
|
|
5041
|
+
title: `${field.label} Distribution`,
|
|
5042
|
+
description: `Showing ${values.length} values across ${bucketCount} buckets`,
|
|
5043
|
+
data: buckets
|
|
5044
|
+
};
|
|
5045
|
+
}
|
|
5046
|
+
static transformToScatterPlot(profile, query = "") {
|
|
5047
|
+
const fields = this.rankFieldsByQuery(profile.numericFields, query).slice(0, 2);
|
|
5048
|
+
if (fields.length < 2) return null;
|
|
5049
|
+
const [xField, yField] = fields;
|
|
5050
|
+
const points = profile.records.map((record) => {
|
|
5051
|
+
var _a;
|
|
5052
|
+
const x = this.toFiniteNumber(record.fields[xField.key]);
|
|
5053
|
+
const y = this.toFiniteNumber(record.fields[yField.key]);
|
|
5054
|
+
if (x === null || y === null) return null;
|
|
5055
|
+
return { x, y, label: String((_a = this.getRecordLabel(record)) != null ? _a : record.id) };
|
|
5056
|
+
}).filter((point) => point !== null).slice(0, 100);
|
|
5057
|
+
if (points.length === 0) return null;
|
|
5058
|
+
return {
|
|
5059
|
+
type: "scatter_plot",
|
|
5060
|
+
title: `${xField.label} vs ${yField.label}`,
|
|
5061
|
+
description: `Showing ${points.length} paired values`,
|
|
5062
|
+
data: points
|
|
5063
|
+
};
|
|
5064
|
+
}
|
|
5065
|
+
static transformToMetricCard(profile, query = "") {
|
|
5066
|
+
const operation = this.detectAggregationOperation(query);
|
|
5067
|
+
const numericField = this.selectNumericField(profile, query);
|
|
5068
|
+
const values = numericField ? profile.records.map((record) => this.toFiniteNumber(record.fields[numericField.key])).filter((value2) => value2 !== null) : [];
|
|
5069
|
+
const value = operation === "count" || values.length === 0 ? profile.records.length : this.calculateAggregate(values, operation);
|
|
5070
|
+
const metric = {
|
|
5071
|
+
label: numericField ? `${operation} ${numericField.label}` : "Count",
|
|
5072
|
+
value,
|
|
5073
|
+
operation
|
|
5074
|
+
};
|
|
5075
|
+
return {
|
|
5076
|
+
type: "metric_card",
|
|
5077
|
+
title: metric.label,
|
|
5078
|
+
description: `Calculated from ${profile.records.length} result${profile.records.length === 1 ? "" : "s"}`,
|
|
5079
|
+
data: metric
|
|
5080
|
+
};
|
|
5081
|
+
}
|
|
5082
|
+
static transformToRadarChart(data) {
|
|
5083
|
+
const attributeMap = {};
|
|
5084
|
+
data.forEach((item) => {
|
|
5085
|
+
var _a, _b, _c;
|
|
5086
|
+
const meta = item.metadata || {};
|
|
5087
|
+
const seriesName = String((_c = (_b = (_a = meta.name) != null ? _a : meta.product) != null ? _b : item.id) != null ? _c : "Item");
|
|
5088
|
+
Object.entries(meta).forEach(([key, val]) => {
|
|
5089
|
+
if (typeof val === "number" && !["price", "id"].includes(key.toLowerCase())) {
|
|
5090
|
+
if (!attributeMap[key]) attributeMap[key] = {};
|
|
5091
|
+
attributeMap[key][seriesName] = val;
|
|
5092
|
+
}
|
|
5093
|
+
});
|
|
5094
|
+
});
|
|
5095
|
+
const radarData = Object.entries(attributeMap).map(([attribute, series]) => __spreadValues({
|
|
5096
|
+
attribute
|
|
5097
|
+
}, series));
|
|
5098
|
+
return {
|
|
5099
|
+
type: "radar_chart",
|
|
5100
|
+
title: "Product Comparison",
|
|
5101
|
+
description: `Comparing ${data.length} items across ${radarData.length} attributes`,
|
|
5102
|
+
data: radarData.length > 0 ? radarData : data.map((d) => ({ attribute: d.content.substring(0, 40) }))
|
|
5103
|
+
};
|
|
5104
|
+
}
|
|
5105
|
+
static transformToTable(data, query = "") {
|
|
5106
|
+
const columns = this.extractTableColumns(data, query);
|
|
5107
|
+
const rows = data.map((item) => this.extractTableRow(item, columns));
|
|
5108
|
+
const tableData = { columns, rows };
|
|
4250
5109
|
return {
|
|
4251
5110
|
type: "table",
|
|
4252
5111
|
title: "Detailed Results",
|
|
@@ -4254,28 +5113,17 @@ var UITransformer = class {
|
|
|
4254
5113
|
data: tableData
|
|
4255
5114
|
};
|
|
4256
5115
|
}
|
|
4257
|
-
/**
|
|
4258
|
-
* Transform data to text format (fallback)
|
|
4259
|
-
*/
|
|
4260
5116
|
static transformToText(data) {
|
|
4261
|
-
const textContent = data.map((item) => item.content).join("\n\n");
|
|
4262
5117
|
return this.createTextResponse(
|
|
4263
5118
|
"Search Results",
|
|
4264
|
-
|
|
5119
|
+
data.map((item) => item.content).join("\n\n"),
|
|
4265
5120
|
`Found ${data.length} relevant results`
|
|
4266
5121
|
);
|
|
4267
5122
|
}
|
|
4268
|
-
/**
|
|
4269
|
-
* Helper: Create text response
|
|
4270
|
-
*/
|
|
4271
5123
|
static createTextResponse(title, content, description) {
|
|
4272
|
-
return {
|
|
4273
|
-
type: "text",
|
|
4274
|
-
title,
|
|
4275
|
-
description,
|
|
4276
|
-
data: { content }
|
|
4277
|
-
};
|
|
5124
|
+
return { type: "text", title, description, data: { content } };
|
|
4278
5125
|
}
|
|
5126
|
+
// ─── LLM Response Parsing ─────────────────────────────────────────────────
|
|
4279
5127
|
static parseTransformationResponse(raw) {
|
|
4280
5128
|
const payloadText = this.extractJsonCandidate(raw);
|
|
4281
5129
|
if (!payloadText) return null;
|
|
@@ -4312,9 +5160,7 @@ var UITransformer = class {
|
|
|
4312
5160
|
if (char === "{") depth += 1;
|
|
4313
5161
|
if (char === "}") {
|
|
4314
5162
|
depth -= 1;
|
|
4315
|
-
if (depth === 0)
|
|
4316
|
-
return cleaned.slice(start, i + 1);
|
|
4317
|
-
}
|
|
5163
|
+
if (depth === 0) return cleaned.slice(start, i + 1);
|
|
4318
5164
|
}
|
|
4319
5165
|
}
|
|
4320
5166
|
return null;
|
|
@@ -4322,19 +5168,16 @@ var UITransformer = class {
|
|
|
4322
5168
|
static normalizeTransformation(payload) {
|
|
4323
5169
|
var _a, _b, _c, _d, _e, _f, _g, _h, _i, _j;
|
|
4324
5170
|
if (!payload || typeof payload !== "object") return null;
|
|
4325
|
-
const
|
|
4326
|
-
const type = this.normalizeVisualizationType(
|
|
5171
|
+
const p = payload;
|
|
5172
|
+
const type = this.normalizeVisualizationType(
|
|
5173
|
+
String((_c = (_b = (_a = p.type) != null ? _a : p.view) != null ? _b : p.chartType) != null ? _c : "")
|
|
5174
|
+
);
|
|
4327
5175
|
if (!type) return null;
|
|
4328
|
-
const title = String((_e = (_d =
|
|
4329
|
-
const description =
|
|
4330
|
-
const rawData = (_j = (_i = (_h = (_g = (_f =
|
|
5176
|
+
const title = String((_e = (_d = p.title) != null ? _d : p.heading) != null ? _e : "Visualization");
|
|
5177
|
+
const description = p.description ? String(p.description) : void 0;
|
|
5178
|
+
const rawData = (_j = (_i = (_h = (_g = (_f = p.data) != null ? _f : p.table) != null ? _g : p.rows) != null ? _h : p.items) != null ? _i : p.content) != null ? _j : null;
|
|
4331
5179
|
const data = type === "text" && typeof rawData === "string" ? { content: rawData } : rawData;
|
|
4332
|
-
const transformation = {
|
|
4333
|
-
type,
|
|
4334
|
-
title,
|
|
4335
|
-
description,
|
|
4336
|
-
data
|
|
4337
|
-
};
|
|
5180
|
+
const transformation = { type, title, description, data };
|
|
4338
5181
|
return this.validateTransformation(transformation) ? transformation : null;
|
|
4339
5182
|
}
|
|
4340
5183
|
static normalizeVisualizationType(type) {
|
|
@@ -4344,10 +5187,23 @@ var UITransformer = class {
|
|
|
4344
5187
|
pie_chart: "pie_chart",
|
|
4345
5188
|
bar: "bar_chart",
|
|
4346
5189
|
bar_chart: "bar_chart",
|
|
5190
|
+
histogram: "histogram",
|
|
5191
|
+
horizontal_bar: "horizontal_bar",
|
|
5192
|
+
horizontalbar: "horizontal_bar",
|
|
4347
5193
|
line: "line_chart",
|
|
4348
5194
|
line_chart: "line_chart",
|
|
5195
|
+
scatter: "scatter_plot",
|
|
5196
|
+
scatter_plot: "scatter_plot",
|
|
5197
|
+
scatterplot: "scatter_plot",
|
|
4349
5198
|
radar: "radar_chart",
|
|
4350
5199
|
radar_chart: "radar_chart",
|
|
5200
|
+
metric: "metric_card",
|
|
5201
|
+
metric_card: "metric_card",
|
|
5202
|
+
card: "metric_card",
|
|
5203
|
+
kpi: "metric_card",
|
|
5204
|
+
geo: "geo_map",
|
|
5205
|
+
geo_map: "geo_map",
|
|
5206
|
+
map: "geo_map",
|
|
4351
5207
|
table: "table",
|
|
4352
5208
|
text: "text",
|
|
4353
5209
|
product_carousel: "product_carousel",
|
|
@@ -4355,21 +5211,31 @@ var UITransformer = class {
|
|
|
4355
5211
|
};
|
|
4356
5212
|
return (_a = mapping[type.toLowerCase()]) != null ? _a : null;
|
|
4357
5213
|
}
|
|
4358
|
-
static validateTransformation(
|
|
4359
|
-
const { type, data } =
|
|
5214
|
+
static validateTransformation(t) {
|
|
5215
|
+
const { type, data } = t;
|
|
4360
5216
|
switch (type) {
|
|
4361
5217
|
case "pie_chart":
|
|
4362
5218
|
case "bar_chart":
|
|
5219
|
+
case "histogram":
|
|
5220
|
+
case "horizontal_bar":
|
|
4363
5221
|
return Array.isArray(data) && data.every(
|
|
4364
|
-
(
|
|
5222
|
+
(i) => i !== null && typeof i === "object" && (typeof i.value === "number" || !Number.isNaN(Number(i.value)))
|
|
5223
|
+
);
|
|
5224
|
+
case "scatter_plot":
|
|
5225
|
+
return Array.isArray(data) && data.every(
|
|
5226
|
+
(i) => i !== null && typeof i === "object" && (typeof i.x === "number" || !Number.isNaN(Number(i.x))) && (typeof i.y === "number" || !Number.isNaN(Number(i.y)))
|
|
4365
5227
|
);
|
|
4366
5228
|
case "line_chart":
|
|
4367
5229
|
return Array.isArray(data) && data.every(
|
|
4368
|
-
(
|
|
5230
|
+
(i) => i !== null && typeof i === "object" && "timestamp" in i && (typeof i.value === "number" || !Number.isNaN(Number(i.value)))
|
|
4369
5231
|
);
|
|
5232
|
+
case "metric_card":
|
|
5233
|
+
return typeof data === "object" && data !== null && (typeof data.value === "number" || !Number.isNaN(Number(data.value)));
|
|
5234
|
+
case "geo_map":
|
|
5235
|
+
return Array.isArray(data) || typeof data === "object" && data !== null;
|
|
4370
5236
|
case "radar_chart":
|
|
4371
5237
|
return Array.isArray(data) && data.every(
|
|
4372
|
-
(
|
|
5238
|
+
(i) => i !== null && typeof i === "object" && "attribute" in i
|
|
4373
5239
|
);
|
|
4374
5240
|
case "table":
|
|
4375
5241
|
return typeof data === "object" && data !== null && Array.isArray(data.columns) && Array.isArray(data.rows);
|
|
@@ -4378,15 +5244,27 @@ var UITransformer = class {
|
|
|
4378
5244
|
case "product_carousel":
|
|
4379
5245
|
case "carousel":
|
|
4380
5246
|
return Array.isArray(data) && data.every(
|
|
4381
|
-
(
|
|
5247
|
+
(i) => i !== null && typeof i === "object" && ("id" in i || "name" in i)
|
|
4382
5248
|
);
|
|
4383
5249
|
default:
|
|
4384
5250
|
return false;
|
|
4385
5251
|
}
|
|
4386
5252
|
}
|
|
4387
|
-
|
|
4388
|
-
|
|
4389
|
-
|
|
5253
|
+
// ─── Data Inspection Helpers ──────────────────────────────────────────────
|
|
5254
|
+
static isStructuredListQuery(query) {
|
|
5255
|
+
const q = query.toLowerCase();
|
|
5256
|
+
const asksForRecords = /\b(list|provide|show|give|get|find|which|whose|records?|details?)\b/.test(q);
|
|
5257
|
+
const hasNumericPredicate = /\b(greater than|more than|above|over|less than|below|under|at least|at most|minimum|maximum|>=|<=|>|<|equals?|equal to)\b\s*\$?\d/i.test(q) || /\b\d+\b\s*(?:or more|or less|and above|and below)\b/i.test(q);
|
|
5258
|
+
const hasEntityTerms = /\b(organization|organisations?|organizations?|company|companies|employee|employees|staff|headcount|workforce)\b/.test(q);
|
|
5259
|
+
return asksForRecords && (hasNumericPredicate || hasEntityTerms);
|
|
5260
|
+
}
|
|
5261
|
+
static isProductQuery(query) {
|
|
5262
|
+
const q = query.toLowerCase();
|
|
5263
|
+
const productTerms = /\b(product|products|item|items|sku|catalog|catalogue|price|prices|brand|model|stock|inventory|in stock|out of stock|buy|shop)\b/.test(q);
|
|
5264
|
+
const productAction = /\b(recommend|suggest|describe|description|detail|details|about|show|find|search|browse|list)\b/.test(q);
|
|
5265
|
+
const nonProductEntityTerms = /\b(organization|organisations?|organizations?|company|companies|employee|employees|staff|headcount|workforce)\b/.test(q);
|
|
5266
|
+
return productTerms && productAction && !nonProductEntityTerms;
|
|
5267
|
+
}
|
|
4390
5268
|
static isProductData(item) {
|
|
4391
5269
|
const content = (item.content || "").toLowerCase();
|
|
4392
5270
|
const productKeywords = [
|
|
@@ -4412,360 +5290,582 @@ var UITransformer = class {
|
|
|
4412
5290
|
"fragrance"
|
|
4413
5291
|
];
|
|
4414
5292
|
const hasKeywords = productKeywords.some((kw) => content.includes(kw));
|
|
4415
|
-
const
|
|
4416
|
-
|
|
4417
|
-
|
|
4418
|
-
|
|
5293
|
+
const metadata = item.metadata || {};
|
|
5294
|
+
const hasMetadataKey = Object.keys(metadata).some((k) => {
|
|
5295
|
+
const val = metadata[k];
|
|
5296
|
+
return ["price", "product", "sku", "brand", "model", "cost", "item"].includes(k.toLowerCase()) && val !== null && val !== void 0 && val !== "";
|
|
5297
|
+
});
|
|
5298
|
+
const hasPricePattern = /\$\s*\d+\.\d{2}/.test(content);
|
|
4419
5299
|
return hasKeywords || hasMetadataKey || hasPricePattern;
|
|
4420
5300
|
}
|
|
4421
|
-
/**
|
|
4422
|
-
* Helper: Check if data contains time series
|
|
4423
|
-
*/
|
|
4424
5301
|
static isTimeSeriesData(item) {
|
|
4425
|
-
const content = (item.content || "").toLowerCase();
|
|
4426
|
-
const timeKeywords = ["trend", "historical", "growth", "decline", "change", "increase", "decrease"];
|
|
4427
|
-
const hasTimeKeyword = timeKeywords.some((kw) => content.includes(kw));
|
|
4428
5302
|
const metadata = item.metadata || {};
|
|
4429
5303
|
const maybeDateKeys = Object.keys(metadata).filter(
|
|
4430
5304
|
(k) => ["date", "timestamp", "time", "period"].includes(k.toLowerCase())
|
|
4431
5305
|
);
|
|
4432
|
-
|
|
5306
|
+
return maybeDateKeys.some((key) => {
|
|
4433
5307
|
const value = metadata[key];
|
|
4434
|
-
if (typeof value === "string")
|
|
4435
|
-
return !Number.isNaN(Date.parse(value)) && value.trim().length > 0;
|
|
4436
|
-
}
|
|
5308
|
+
if (typeof value === "string") return !Number.isNaN(Date.parse(value)) && value.trim().length > 0;
|
|
4437
5309
|
return typeof value === "number";
|
|
4438
5310
|
});
|
|
4439
|
-
return hasTimeKeyword || hasValidDateValue;
|
|
4440
5311
|
}
|
|
4441
|
-
static
|
|
4442
|
-
|
|
4443
|
-
|
|
4444
|
-
|
|
4445
|
-
const normalized = query.toLowerCase();
|
|
4446
|
-
const chartKeywords = [
|
|
4447
|
-
"distribution",
|
|
4448
|
-
"breakdown",
|
|
4449
|
-
"by category",
|
|
4450
|
-
"by type",
|
|
4451
|
-
"compare",
|
|
4452
|
-
"share",
|
|
4453
|
-
"percentage",
|
|
4454
|
-
"segmentation",
|
|
4455
|
-
"split",
|
|
4456
|
-
"category breakdown",
|
|
4457
|
-
"category distribution"
|
|
4458
|
-
];
|
|
4459
|
-
return chartKeywords.some((keyword) => normalized.includes(keyword));
|
|
5312
|
+
static hasMultipleFields(data) {
|
|
5313
|
+
const fieldCount = /* @__PURE__ */ new Set();
|
|
5314
|
+
data.forEach((item) => Object.keys(item.metadata || {}).forEach((k) => fieldCount.add(k)));
|
|
5315
|
+
return fieldCount.size > 2;
|
|
4460
5316
|
}
|
|
4461
|
-
static
|
|
4462
|
-
const
|
|
4463
|
-
|
|
4464
|
-
|
|
4465
|
-
|
|
4466
|
-
|
|
4467
|
-
|
|
4468
|
-
|
|
4469
|
-
|
|
4470
|
-
|
|
4471
|
-
|
|
4472
|
-
|
|
4473
|
-
|
|
4474
|
-
|
|
4475
|
-
|
|
4476
|
-
|
|
4477
|
-
|
|
4478
|
-
|
|
4479
|
-
|
|
4480
|
-
|
|
5317
|
+
static profileData(data) {
|
|
5318
|
+
const records = data.map((item) => {
|
|
5319
|
+
const fields2 = {};
|
|
5320
|
+
Object.entries(item.metadata || {}).forEach(([key, value]) => {
|
|
5321
|
+
const primitive = this.toPrimitive(value);
|
|
5322
|
+
if (primitive !== null) fields2[key] = primitive;
|
|
5323
|
+
});
|
|
5324
|
+
if (!fields2.content && item.content) fields2.content = item.content.substring(0, 500);
|
|
5325
|
+
return {
|
|
5326
|
+
id: item.id,
|
|
5327
|
+
content: item.content,
|
|
5328
|
+
score: item.score,
|
|
5329
|
+
fields: fields2,
|
|
5330
|
+
source: item
|
|
5331
|
+
};
|
|
5332
|
+
});
|
|
5333
|
+
const keys = Array.from(new Set(records.flatMap((record) => Object.keys(record.fields))));
|
|
5334
|
+
const fields = keys.map((key) => {
|
|
5335
|
+
const values = records.map((record) => record.fields[key]).filter((value) => value !== void 0 && value !== null && String(value).trim() !== "");
|
|
5336
|
+
const uniqueCount = new Set(values.map((value) => String(value).toLowerCase())).size;
|
|
5337
|
+
return {
|
|
5338
|
+
key,
|
|
5339
|
+
label: this.humanizeFieldName(key),
|
|
5340
|
+
kind: this.inferFieldKind(key, values, records.length, uniqueCount),
|
|
5341
|
+
values,
|
|
5342
|
+
uniqueCount
|
|
5343
|
+
};
|
|
5344
|
+
});
|
|
5345
|
+
return {
|
|
5346
|
+
records,
|
|
5347
|
+
fields,
|
|
5348
|
+
numericFields: fields.filter((field) => field.kind === "number"),
|
|
5349
|
+
dateFields: fields.filter((field) => field.kind === "date"),
|
|
5350
|
+
categoricalFields: fields.filter((field) => field.kind === "category"),
|
|
5351
|
+
booleanFields: fields.filter((field) => field.kind === "boolean")
|
|
5352
|
+
};
|
|
4481
5353
|
}
|
|
4482
|
-
static
|
|
4483
|
-
|
|
4484
|
-
|
|
5354
|
+
static toPrimitive(value) {
|
|
5355
|
+
if (typeof value === "string" || typeof value === "number" || typeof value === "boolean") return value;
|
|
5356
|
+
if (value === null || value === void 0) return null;
|
|
5357
|
+
if (value instanceof Date) return value.toISOString();
|
|
5358
|
+
return null;
|
|
4485
5359
|
}
|
|
4486
|
-
|
|
4487
|
-
|
|
4488
|
-
|
|
4489
|
-
|
|
4490
|
-
|
|
4491
|
-
if (!meta) return void 0;
|
|
4492
|
-
const mapping = (_a = config == null ? void 0 : config.rag) == null ? void 0 : _a.uiMapping;
|
|
4493
|
-
if (mapping && mapping[uiKey]) {
|
|
4494
|
-
const mappedKey = mapping[uiKey];
|
|
4495
|
-
if (meta[mappedKey] !== void 0) return meta[mappedKey];
|
|
5360
|
+
static inferFieldKind(key, values, rowCount, uniqueCount) {
|
|
5361
|
+
const lowerKey = key.toLowerCase();
|
|
5362
|
+
if (values.length === 0) return "text";
|
|
5363
|
+
if (values.every((value) => typeof value === "boolean" || /^(true|false|yes|no|in_stock|out_of_stock|available|unavailable)$/i.test(String(value)))) {
|
|
5364
|
+
return "boolean";
|
|
4496
5365
|
}
|
|
4497
|
-
if (
|
|
4498
|
-
|
|
4499
|
-
if (trainedKey && meta[trainedKey] !== void 0) return meta[trainedKey];
|
|
5366
|
+
if (/(date|time|timestamp|created|updated|month|year|period)/i.test(lowerKey) && values.some((value) => this.isDateLike(value))) {
|
|
5367
|
+
return "date";
|
|
4500
5368
|
}
|
|
4501
|
-
|
|
5369
|
+
const numericCount = values.filter((value) => this.toFiniteNumber(value) !== null).length;
|
|
5370
|
+
if (numericCount / values.length >= 0.85 && !/(id|sku|ean|upc|zip|postal|phone|code)/i.test(lowerKey)) {
|
|
5371
|
+
return "number";
|
|
5372
|
+
}
|
|
5373
|
+
if (uniqueCount <= Math.max(20, Math.ceil(rowCount * 0.7)) && !/(id|sku|ean|upc|description|content|url|image|thumbnail)/i.test(lowerKey)) {
|
|
5374
|
+
return "category";
|
|
5375
|
+
}
|
|
5376
|
+
return "text";
|
|
4502
5377
|
}
|
|
4503
|
-
static
|
|
4504
|
-
|
|
4505
|
-
const
|
|
4506
|
-
|
|
4507
|
-
|
|
4508
|
-
|
|
4509
|
-
|
|
4510
|
-
|
|
4511
|
-
|
|
4512
|
-
|
|
4513
|
-
|
|
4514
|
-
|
|
4515
|
-
|
|
4516
|
-
|
|
4517
|
-
|
|
4518
|
-
|
|
4519
|
-
|
|
4520
|
-
return
|
|
4521
|
-
|
|
4522
|
-
|
|
4523
|
-
|
|
4524
|
-
image: typeof imageValue === "string" ? imageValue : void 0,
|
|
4525
|
-
brand: brand ? String(brand) : void 0,
|
|
4526
|
-
description: item.content,
|
|
4527
|
-
inStock: this.determineStockStatus(item)
|
|
4528
|
-
};
|
|
5378
|
+
static isDateLike(value) {
|
|
5379
|
+
if (typeof value === "number") return value > 1900 && value < 3e3;
|
|
5380
|
+
const text = String(value).trim();
|
|
5381
|
+
return text.length > 3 && !Number.isNaN(Date.parse(text));
|
|
5382
|
+
}
|
|
5383
|
+
static chooseAutomaticVisualization(data, profile, query) {
|
|
5384
|
+
if (profile.records.length === 0) return null;
|
|
5385
|
+
if (profile.dateFields.length > 0 && profile.numericFields.length > 0) {
|
|
5386
|
+
return this.transformToLineChart(profile);
|
|
5387
|
+
}
|
|
5388
|
+
if (profile.categoricalFields.length > 0 && profile.numericFields.length > 0) {
|
|
5389
|
+
return this.transformToBarChart(data, profile, query);
|
|
5390
|
+
}
|
|
5391
|
+
if (profile.categoricalFields.length > 0) {
|
|
5392
|
+
return this.transformToPieChart(data, profile, query);
|
|
5393
|
+
}
|
|
5394
|
+
if (profile.numericFields.length >= 2) {
|
|
5395
|
+
return this.transformToScatterPlot(profile, query);
|
|
5396
|
+
}
|
|
5397
|
+
if (profile.numericFields.length === 1) {
|
|
5398
|
+
return this.transformToMetricCard(profile, query);
|
|
4529
5399
|
}
|
|
4530
5400
|
return null;
|
|
4531
5401
|
}
|
|
4532
|
-
|
|
4533
|
-
|
|
4534
|
-
|
|
5402
|
+
static selectDimensionField(profile, query) {
|
|
5403
|
+
var _a, _b;
|
|
5404
|
+
const productCategory = profile.categoricalFields.find(
|
|
5405
|
+
(field) => /category|department|collection|type|group|segment|region|country|state|city/i.test(field.key)
|
|
5406
|
+
);
|
|
5407
|
+
const ranked = this.rankFieldsByQuery(profile.categoricalFields, query);
|
|
5408
|
+
return (_b = (_a = ranked[0]) != null ? _a : productCategory) != null ? _b : profile.categoricalFields[0];
|
|
5409
|
+
}
|
|
5410
|
+
static selectNumericField(profile, query) {
|
|
5411
|
+
var _a;
|
|
5412
|
+
return (_a = this.rankFieldsByQuery(profile.numericFields, query)[0]) != null ? _a : profile.numericFields[0];
|
|
5413
|
+
}
|
|
5414
|
+
static rankFieldsByQuery(fields, query) {
|
|
5415
|
+
const q = query.toLowerCase();
|
|
5416
|
+
return [...fields].sort((a, b) => this.fieldScore(b, q) - this.fieldScore(a, q));
|
|
5417
|
+
}
|
|
5418
|
+
static fieldScore(field, query) {
|
|
5419
|
+
const key = field.key.toLowerCase();
|
|
5420
|
+
const label = field.label.toLowerCase();
|
|
5421
|
+
let score = 0;
|
|
5422
|
+
if (query.includes(key)) score += 4;
|
|
5423
|
+
if (query.includes(label)) score += 4;
|
|
5424
|
+
key.split(/[_\s-]+/).forEach((part) => {
|
|
5425
|
+
if (part && query.includes(part)) score += 1;
|
|
5426
|
+
});
|
|
5427
|
+
if (/category|department|collection|type|group|segment/.test(key)) score += 2;
|
|
5428
|
+
if (/count|total|quantity|stock|inventory|sales|revenue|amount|price|value|score/.test(key)) score += 2;
|
|
5429
|
+
if (/id|sku|ean|upc|code/.test(key)) score -= 5;
|
|
5430
|
+
return score;
|
|
5431
|
+
}
|
|
5432
|
+
static normalizeComparableField(field) {
|
|
5433
|
+
return field.toLowerCase().replace(/&/g, "and").replace(/[^a-z0-9]+/g, "").trim();
|
|
5434
|
+
}
|
|
5435
|
+
static fieldTokens(field) {
|
|
5436
|
+
return field.toLowerCase().replace(/[_-]+/g, " ").split(/\s+/).map((token) => token.replace(/[^a-z0-9]/g, "")).filter(Boolean);
|
|
5437
|
+
}
|
|
5438
|
+
static aggregateProfileByDimension(profile, dimensionKey, measureKey) {
|
|
5439
|
+
const result = {};
|
|
5440
|
+
profile.records.forEach((record) => {
|
|
5441
|
+
var _a, _b, _c;
|
|
5442
|
+
const category = String((_a = record.fields[dimensionKey]) != null ? _a : "Other").trim() || "Other";
|
|
5443
|
+
const value = measureKey ? (_b = this.toFiniteNumber(record.fields[measureKey])) != null ? _b : 0 : 1;
|
|
5444
|
+
result[category] = ((_c = result[category]) != null ? _c : 0) + value;
|
|
5445
|
+
});
|
|
5446
|
+
return result;
|
|
5447
|
+
}
|
|
5448
|
+
static getRecordLabel(record) {
|
|
5449
|
+
const labelKeys = ["name", "title", "label", "product", "item", "brand"];
|
|
5450
|
+
const key = Object.keys(record.fields).find(
|
|
5451
|
+
(fieldKey) => labelKeys.some((labelKey) => fieldKey.toLowerCase().includes(labelKey))
|
|
5452
|
+
);
|
|
5453
|
+
return key ? record.fields[key] : void 0;
|
|
5454
|
+
}
|
|
5455
|
+
static detectAggregationOperation(query) {
|
|
5456
|
+
const q = query.toLowerCase();
|
|
5457
|
+
if (/\b(avg|average|mean)\b/.test(q)) return "average";
|
|
5458
|
+
if (/\b(count|how many|number of)\b/.test(q)) return "count";
|
|
5459
|
+
if (/\b(min|minimum|lowest)\b/.test(q)) return "min";
|
|
5460
|
+
if (/\b(max|maximum|highest)\b/.test(q)) return "max";
|
|
5461
|
+
if (/\bmedian\b/.test(q)) return "median";
|
|
5462
|
+
return "sum";
|
|
5463
|
+
}
|
|
5464
|
+
static calculateAggregate(values, operation) {
|
|
5465
|
+
if (values.length === 0) return 0;
|
|
5466
|
+
switch (operation) {
|
|
5467
|
+
case "average":
|
|
5468
|
+
return values.reduce((sum, value) => sum + value, 0) / values.length;
|
|
5469
|
+
case "count":
|
|
5470
|
+
return values.length;
|
|
5471
|
+
case "min":
|
|
5472
|
+
return Math.min(...values);
|
|
5473
|
+
case "max":
|
|
5474
|
+
return Math.max(...values);
|
|
5475
|
+
case "median": {
|
|
5476
|
+
const sorted = [...values].sort((a, b) => a - b);
|
|
5477
|
+
const middle = Math.floor(sorted.length / 2);
|
|
5478
|
+
return sorted.length % 2 === 0 ? (sorted[middle - 1] + sorted[middle]) / 2 : sorted[middle];
|
|
5479
|
+
}
|
|
5480
|
+
case "sum":
|
|
5481
|
+
default:
|
|
5482
|
+
return values.reduce((sum, value) => sum + value, 0);
|
|
5483
|
+
}
|
|
5484
|
+
}
|
|
5485
|
+
static humanizeFieldName(key) {
|
|
5486
|
+
return key.replace(/[_-]+/g, " ").replace(/([a-z])([A-Z])/g, "$1 $2").replace(/\s+/g, " ").trim().replace(/\b\w/g, (char) => char.toUpperCase());
|
|
5487
|
+
}
|
|
5488
|
+
static formatNumber(value) {
|
|
5489
|
+
return Number.isInteger(value) ? String(value) : value.toFixed(1);
|
|
5490
|
+
}
|
|
4535
5491
|
static detectCategories(data) {
|
|
4536
5492
|
const categories = /* @__PURE__ */ new Set();
|
|
4537
5493
|
data.forEach((item) => {
|
|
4538
|
-
const
|
|
4539
|
-
if (
|
|
4540
|
-
categories.add(String(meta.category));
|
|
4541
|
-
}
|
|
4542
|
-
if (meta.type) {
|
|
4543
|
-
categories.add(String(meta.type));
|
|
4544
|
-
}
|
|
4545
|
-
if (meta.tag) {
|
|
4546
|
-
const tags = Array.isArray(meta.tag) ? meta.tag : [meta.tag];
|
|
4547
|
-
tags.forEach((t) => categories.add(String(t)));
|
|
4548
|
-
}
|
|
4549
|
-
const contentCategories = Array.from(new Set(
|
|
4550
|
-
Array.from(item.content.matchAll(/^\s*([^:\n]+):\s*(?:•|\*|-)*/gm)).map((match) => match[1].trim()).filter(Boolean)
|
|
4551
|
-
));
|
|
4552
|
-
contentCategories.forEach((category) => categories.add(category));
|
|
4553
|
-
const categoryMatch = item.content.match(/(?:Category|Type|Class):\s*([^\n]+)/i);
|
|
4554
|
-
if (categoryMatch) {
|
|
4555
|
-
categories.add(categoryMatch[1].trim());
|
|
4556
|
-
}
|
|
5494
|
+
const category = this.getProductCategory(item);
|
|
5495
|
+
if (category) categories.add(category);
|
|
4557
5496
|
});
|
|
4558
5497
|
return Array.from(categories);
|
|
4559
5498
|
}
|
|
4560
|
-
|
|
4561
|
-
|
|
4562
|
-
|
|
5499
|
+
static getProductCategory(item) {
|
|
5500
|
+
const meta = item.metadata || {};
|
|
5501
|
+
const metadataCategory = resolveMetadataValue(meta, "category");
|
|
5502
|
+
if (this.isUsableCategory(metadataCategory)) return String(metadataCategory).trim();
|
|
5503
|
+
const content = item.content || "";
|
|
5504
|
+
const explicitCategoryMatch = content.match(
|
|
5505
|
+
/(?:^|\n)\s*(?:product\s*)?(?:category|department|collection)\s*[:\-–]\s*([^\n]+)/i
|
|
5506
|
+
);
|
|
5507
|
+
if (explicitCategoryMatch && this.isUsableCategory(explicitCategoryMatch[1])) {
|
|
5508
|
+
return explicitCategoryMatch[1].trim();
|
|
5509
|
+
}
|
|
5510
|
+
return null;
|
|
5511
|
+
}
|
|
5512
|
+
static isUsableCategory(value) {
|
|
5513
|
+
if (value === null || value === void 0) return false;
|
|
5514
|
+
const category = String(value).trim();
|
|
5515
|
+
if (!category) return false;
|
|
5516
|
+
return !/^(category|type|tag|price|cost|stock|availability|description|product|item|name|brand|sku|id|ean|currency|color|size|index|internal id|other|\[?type\]?|\[?products?_?\d*\]?)$/i.test(category);
|
|
5517
|
+
}
|
|
4563
5518
|
static aggregateByCategory(data, categories) {
|
|
4564
|
-
const result =
|
|
4565
|
-
categories.forEach((cat) => {
|
|
4566
|
-
result[cat] = 0;
|
|
4567
|
-
});
|
|
5519
|
+
const result = Object.fromEntries(categories.map((c) => [c, 0]));
|
|
4568
5520
|
data.forEach((item) => {
|
|
4569
|
-
|
|
4570
|
-
const
|
|
4571
|
-
if (Object.prototype.hasOwnProperty.call(result,
|
|
4572
|
-
|
|
4573
|
-
} else {
|
|
4574
|
-
result["Other"] = (result["Other"] || 0) + 1;
|
|
4575
|
-
}
|
|
5521
|
+
var _a;
|
|
5522
|
+
const cat = (_a = this.getProductCategory(item)) != null ? _a : "Other";
|
|
5523
|
+
if (Object.prototype.hasOwnProperty.call(result, cat)) result[cat]++;
|
|
5524
|
+
else result["Other"] = (result["Other"] || 0) + 1;
|
|
4576
5525
|
});
|
|
4577
5526
|
return result;
|
|
4578
5527
|
}
|
|
4579
|
-
/**
|
|
4580
|
-
* Helper: Extract time series data
|
|
4581
|
-
*/
|
|
4582
5528
|
static extractTimeSeriesData(data) {
|
|
4583
5529
|
return data.map((item) => {
|
|
5530
|
+
var _a, _b, _c, _d, _e;
|
|
4584
5531
|
const meta = item.metadata || {};
|
|
4585
5532
|
return {
|
|
4586
|
-
timestamp: meta.timestamp
|
|
4587
|
-
value: meta.value
|
|
4588
|
-
label: meta.label
|
|
5533
|
+
timestamp: (_b = (_a = meta.timestamp) != null ? _a : meta.date) != null ? _b : (/* @__PURE__ */ new Date()).toISOString(),
|
|
5534
|
+
value: (_d = (_c = meta.value) != null ? _c : item.score) != null ? _d : 0,
|
|
5535
|
+
label: (_e = meta.label) != null ? _e : item.content.substring(0, 50)
|
|
4589
5536
|
};
|
|
4590
5537
|
});
|
|
4591
5538
|
}
|
|
4592
|
-
|
|
4593
|
-
|
|
4594
|
-
|
|
4595
|
-
|
|
4596
|
-
|
|
4597
|
-
|
|
4598
|
-
|
|
4599
|
-
|
|
4600
|
-
|
|
4601
|
-
|
|
4602
|
-
|
|
4603
|
-
|
|
5539
|
+
static extractNumericValue(meta) {
|
|
5540
|
+
var _a;
|
|
5541
|
+
const preferredKeys = [
|
|
5542
|
+
"value",
|
|
5543
|
+
"count",
|
|
5544
|
+
"total",
|
|
5545
|
+
"average",
|
|
5546
|
+
"avg",
|
|
5547
|
+
"amount",
|
|
5548
|
+
"sales",
|
|
5549
|
+
"revenue",
|
|
5550
|
+
"score",
|
|
5551
|
+
"quantity",
|
|
5552
|
+
"price"
|
|
5553
|
+
];
|
|
5554
|
+
for (const key of preferredKeys) {
|
|
5555
|
+
const raw = (_a = resolveMetadataValue(meta, key)) != null ? _a : meta[key];
|
|
5556
|
+
const value = typeof raw === "number" ? raw : Number(String(raw != null ? raw : "").replace(/[$,% ,]/g, ""));
|
|
5557
|
+
if (Number.isFinite(value)) return value;
|
|
5558
|
+
}
|
|
5559
|
+
for (const value of Object.values(meta)) {
|
|
5560
|
+
const numeric = typeof value === "number" ? value : Number(String(value != null ? value : "").replace(/[$,% ,]/g, ""));
|
|
5561
|
+
if (Number.isFinite(numeric)) return numeric;
|
|
5562
|
+
}
|
|
5563
|
+
return null;
|
|
5564
|
+
}
|
|
5565
|
+
static extractTableColumns(data, query = "") {
|
|
5566
|
+
const q = query.toLowerCase();
|
|
5567
|
+
const availableFields = this.extractAvailableTableFields(data);
|
|
5568
|
+
if (/\b(organization|organisations?|organizations?|company|companies)\b/.test(q) && /\b(employee|employees|staff|headcount|workforce)\b/.test(q)) {
|
|
5569
|
+
const nameField = this.pickTableField(availableFields, [
|
|
5570
|
+
"company name",
|
|
5571
|
+
"organization name",
|
|
5572
|
+
"organisation name",
|
|
5573
|
+
"name",
|
|
5574
|
+
"company",
|
|
5575
|
+
"organization",
|
|
5576
|
+
"organisation"
|
|
5577
|
+
]);
|
|
5578
|
+
const employeeField = this.pickTableField(availableFields, [
|
|
5579
|
+
"number of employees",
|
|
5580
|
+
"employee count",
|
|
5581
|
+
"employees",
|
|
5582
|
+
"employee_count",
|
|
5583
|
+
"number_employees",
|
|
5584
|
+
"num employees",
|
|
5585
|
+
"staff count",
|
|
5586
|
+
"headcount",
|
|
5587
|
+
"workforce"
|
|
5588
|
+
]);
|
|
5589
|
+
const columns = [nameField, employeeField].filter((field) => Boolean(field));
|
|
5590
|
+
if (columns.length > 0) return columns;
|
|
5591
|
+
}
|
|
5592
|
+
const requestedFields = availableFields.map((field) => ({
|
|
5593
|
+
field,
|
|
5594
|
+
score: this.tableFieldQueryScore(field, q)
|
|
5595
|
+
})).filter((item) => item.score > 0).sort((a, b) => b.score - a.score).map((item) => item.field);
|
|
5596
|
+
if (requestedFields.length > 0) {
|
|
5597
|
+
return requestedFields.slice(0, Math.min(6, requestedFields.length));
|
|
5598
|
+
}
|
|
5599
|
+
const metadataFields = Array.from(new Set(
|
|
5600
|
+
data.flatMap((item) => Object.keys(item.metadata || {}).map((k) => this.humanizeFieldName(k)))
|
|
5601
|
+
));
|
|
5602
|
+
return metadataFields.length > 0 ? metadataFields : ["Content"];
|
|
4604
5603
|
}
|
|
4605
|
-
/**
|
|
4606
|
-
* Helper: Extract table row
|
|
4607
|
-
*/
|
|
4608
5604
|
static extractTableRow(item, columns) {
|
|
4609
|
-
|
|
4610
|
-
|
|
4611
|
-
|
|
4612
|
-
|
|
5605
|
+
return columns.map((col) => this.resolveTableCellValue(item, col));
|
|
5606
|
+
}
|
|
5607
|
+
static extractAvailableTableFields(data) {
|
|
5608
|
+
const fields = /* @__PURE__ */ new Map();
|
|
5609
|
+
const addField = (field) => {
|
|
5610
|
+
const clean = this.humanizeFieldName(field);
|
|
5611
|
+
if (!clean || /^(content|\[?type\]?)$/i.test(clean)) return;
|
|
5612
|
+
const normalized = this.normalizeComparableField(clean);
|
|
5613
|
+
if (!fields.has(normalized)) fields.set(normalized, clean);
|
|
5614
|
+
};
|
|
5615
|
+
data.forEach((item) => {
|
|
5616
|
+
Object.keys(item.metadata || {}).forEach(addField);
|
|
5617
|
+
for (const match of (item.content || "").matchAll(/(?:^|\n)\s*([A-Za-z][A-Za-z0-9_ /-]{1,50})\s*[:\-–]\s*([^\n]+)/g)) {
|
|
5618
|
+
addField(match[1]);
|
|
4613
5619
|
}
|
|
4614
|
-
const metaKey = col.charAt(0).toLowerCase() + col.slice(1);
|
|
4615
|
-
const value = meta[metaKey];
|
|
4616
|
-
return value !== void 0 ? String(value) : "";
|
|
4617
5620
|
});
|
|
5621
|
+
return Array.from(fields.values());
|
|
5622
|
+
}
|
|
5623
|
+
static pickTableField(fields, aliases) {
|
|
5624
|
+
const normalizedAliases = aliases.map((alias) => this.normalizeComparableField(alias));
|
|
5625
|
+
for (const alias of normalizedAliases) {
|
|
5626
|
+
const exact = fields.find((field) => this.normalizeComparableField(field) === alias);
|
|
5627
|
+
if (exact) return exact;
|
|
5628
|
+
}
|
|
5629
|
+
for (const alias of normalizedAliases) {
|
|
5630
|
+
const fuzzy = fields.find((field) => {
|
|
5631
|
+
const normalizedField = this.normalizeComparableField(field);
|
|
5632
|
+
if (/(id|code|uuid)$/.test(normalizedField) && !/(id|code|uuid)$/.test(alias)) return false;
|
|
5633
|
+
return normalizedField.includes(alias) || alias.includes(normalizedField);
|
|
5634
|
+
});
|
|
5635
|
+
if (fuzzy) return fuzzy;
|
|
5636
|
+
}
|
|
5637
|
+
return void 0;
|
|
5638
|
+
}
|
|
5639
|
+
static tableFieldQueryScore(field, query) {
|
|
5640
|
+
const normalizedField = this.normalizeComparableField(field);
|
|
5641
|
+
if (!normalizedField) return 0;
|
|
5642
|
+
const fieldTokens = this.fieldTokens(field);
|
|
5643
|
+
return fieldTokens.reduce((score, token) => {
|
|
5644
|
+
if (token.length < 3) return score;
|
|
5645
|
+
return query.includes(token) ? score + 1 : score;
|
|
5646
|
+
}, query.includes(normalizedField) ? 3 : 0);
|
|
5647
|
+
}
|
|
5648
|
+
static resolveTableCellValue(item, column) {
|
|
5649
|
+
if (column === "Content") return item.content.substring(0, 100);
|
|
5650
|
+
const meta = item.metadata || {};
|
|
5651
|
+
const normalizedColumn = this.normalizeComparableField(column);
|
|
5652
|
+
const exactMetadata = Object.entries(meta).find(
|
|
5653
|
+
([key]) => this.normalizeComparableField(key) === normalizedColumn || this.normalizeComparableField(this.humanizeFieldName(key)) === normalizedColumn
|
|
5654
|
+
);
|
|
5655
|
+
if (exactMetadata && exactMetadata[1] !== void 0 && exactMetadata[1] !== null) {
|
|
5656
|
+
return this.toDisplayValue(exactMetadata[1]);
|
|
5657
|
+
}
|
|
5658
|
+
const aliasValue = this.resolveAliasedTableCell(meta, column);
|
|
5659
|
+
if (aliasValue !== void 0) return this.toDisplayValue(aliasValue);
|
|
5660
|
+
const contentValue = this.extractContentFieldValue(item.content, column);
|
|
5661
|
+
if (contentValue !== null) return contentValue;
|
|
5662
|
+
const aliasContentValue = this.extractAliasedContentFieldValue(item.content, column);
|
|
5663
|
+
return aliasContentValue != null ? aliasContentValue : "";
|
|
5664
|
+
}
|
|
5665
|
+
static resolveAliasedTableCell(meta, column) {
|
|
5666
|
+
const normalizedColumn = this.normalizeComparableField(column);
|
|
5667
|
+
const aliases = normalizedColumn.includes("employee") ? ["number of employees", "employee count", "employees", "headcount", "staff count", "workforce"] : normalizedColumn.includes("name") ? ["company name", "organization name", "organisation name", "name", "company", "organization", "organisation"] : [];
|
|
5668
|
+
for (const alias of aliases) {
|
|
5669
|
+
const normalizedAlias = this.normalizeComparableField(alias);
|
|
5670
|
+
const match = Object.entries(meta).find(([key]) => {
|
|
5671
|
+
const normalizedKey = this.normalizeComparableField(key);
|
|
5672
|
+
return normalizedKey === normalizedAlias || normalizedKey.includes(normalizedAlias) || normalizedAlias.includes(normalizedKey);
|
|
5673
|
+
});
|
|
5674
|
+
if (match && match[1] !== void 0 && match[1] !== null) return match[1];
|
|
5675
|
+
}
|
|
5676
|
+
return void 0;
|
|
5677
|
+
}
|
|
5678
|
+
static extractContentFieldValue(content, column) {
|
|
5679
|
+
const escaped = column.replace(/[.*+?^${}()|[\]\\]/g, "\\$&").replace(/\s+/g, "[\\s_ -]+");
|
|
5680
|
+
const pattern = new RegExp(`(?:^|\\n)\\s*${escaped}\\s*[:\\-\u2013]\\s*([^\\n]+)`, "i");
|
|
5681
|
+
const match = content.match(pattern);
|
|
5682
|
+
return match ? match[1].trim() : null;
|
|
5683
|
+
}
|
|
5684
|
+
static extractAliasedContentFieldValue(content, column) {
|
|
5685
|
+
const normalizedColumn = this.normalizeComparableField(column);
|
|
5686
|
+
const aliases = normalizedColumn.includes("employee") ? ["Number Of Employees", "Number of Employees", "Employee Count", "Employees", "Headcount", "Staff Count", "Workforce"] : normalizedColumn.includes("name") ? ["Company Name", "Organization Name", "Organisation Name", "Name", "Company", "Organization", "Organisation"] : [];
|
|
5687
|
+
for (const alias of aliases) {
|
|
5688
|
+
const value = this.extractContentFieldValue(content, alias);
|
|
5689
|
+
if (value !== null) return value;
|
|
5690
|
+
}
|
|
5691
|
+
return null;
|
|
5692
|
+
}
|
|
5693
|
+
static toDisplayValue(value) {
|
|
5694
|
+
if (typeof value === "string" || typeof value === "number" || typeof value === "boolean") return value;
|
|
5695
|
+
return String(value != null ? value : "");
|
|
4618
5696
|
}
|
|
4619
|
-
/**
|
|
4620
|
-
* Helper: Calculate stock counts by category
|
|
4621
|
-
*/
|
|
4622
5697
|
static calculateStockCounts(category, data) {
|
|
4623
5698
|
let inStock = 0;
|
|
4624
5699
|
let outOfStock = 0;
|
|
4625
5700
|
data.forEach((d) => {
|
|
4626
|
-
|
|
4627
|
-
const
|
|
4628
|
-
if (
|
|
4629
|
-
|
|
4630
|
-
|
|
4631
|
-
|
|
4632
|
-
outOfStock++;
|
|
4633
|
-
}
|
|
5701
|
+
var _a, _b;
|
|
5702
|
+
const cat = (_a = this.getProductCategory(d)) != null ? _a : "Other";
|
|
5703
|
+
if (cat === category) {
|
|
5704
|
+
const quantity = (_b = this.extractStockQuantity(d)) != null ? _b : 1;
|
|
5705
|
+
if (this.determineStockStatus(d)) inStock += quantity;
|
|
5706
|
+
else outOfStock += quantity;
|
|
4634
5707
|
}
|
|
4635
5708
|
});
|
|
4636
5709
|
return { inStockCount: inStock, outOfStockCount: outOfStock };
|
|
4637
5710
|
}
|
|
4638
|
-
/**
|
|
4639
|
-
* Helper: Determine if item is in stock
|
|
4640
|
-
*/
|
|
4641
5711
|
static determineStockStatus(item) {
|
|
4642
5712
|
const meta = item.metadata || {};
|
|
4643
|
-
|
|
4644
|
-
|
|
4645
|
-
|
|
4646
|
-
|
|
4647
|
-
|
|
4648
|
-
|
|
4649
|
-
|
|
4650
|
-
|
|
4651
|
-
|
|
5713
|
+
const stockValue = resolveMetadataValue(meta, "stock");
|
|
5714
|
+
if (stockValue !== void 0) {
|
|
5715
|
+
const normalized = String(stockValue).toLowerCase();
|
|
5716
|
+
if (/out[_\s-]?of[_\s-]?stock|unavailable|false|no|sold out|0/.test(normalized)) return false;
|
|
5717
|
+
if (/in[_\s-]?stock|available|true|yes/.test(normalized)) return true;
|
|
5718
|
+
const numeric = this.toFiniteNumber(stockValue);
|
|
5719
|
+
if (numeric !== null) return numeric > 0;
|
|
5720
|
+
}
|
|
5721
|
+
if (meta.inStock !== void 0) return Boolean(meta.inStock);
|
|
5722
|
+
if (meta.stock !== void 0) return Boolean(meta.stock);
|
|
5723
|
+
if (meta.available !== void 0) return Boolean(meta.available);
|
|
4652
5724
|
const content = (item.content || "").toLowerCase();
|
|
4653
|
-
if (
|
|
4654
|
-
|
|
5725
|
+
if (/out[_\s-]?of[_\s-]?stock|unavailable|sold out/.test(content)) return false;
|
|
5726
|
+
if (/in[_\s-]?stock|available/.test(content)) return true;
|
|
5727
|
+
return true;
|
|
5728
|
+
}
|
|
5729
|
+
static extractStockQuantity(item) {
|
|
5730
|
+
const meta = item.metadata || {};
|
|
5731
|
+
const stockValue = resolveMetadataValue(meta, "stock");
|
|
5732
|
+
const numericStock = this.toFiniteNumber(stockValue);
|
|
5733
|
+
if (numericStock !== null) return numericStock;
|
|
5734
|
+
const content = item.content || "";
|
|
5735
|
+
const stockMatch = content.match(/\b(?:stock|inventory|quantity|count)\s*[:\-–]?\s*([\d,]+)/i);
|
|
5736
|
+
if (!stockMatch) return null;
|
|
5737
|
+
return this.toFiniteNumber(stockMatch[1]);
|
|
5738
|
+
}
|
|
5739
|
+
static toFiniteNumber(value) {
|
|
5740
|
+
if (typeof value === "number") return Number.isFinite(value) ? value : null;
|
|
5741
|
+
const numeric = Number(String(value != null ? value : "").replace(/[$,% ,]/g, ""));
|
|
5742
|
+
return Number.isFinite(numeric) ? numeric : null;
|
|
5743
|
+
}
|
|
5744
|
+
// ─── Product Extraction ───────────────────────────────────────────────────
|
|
5745
|
+
static getDynamicVal(meta, uiKey, config, trainedSchema) {
|
|
5746
|
+
var _a;
|
|
5747
|
+
if (!meta) return void 0;
|
|
5748
|
+
const mapping = (_a = config == null ? void 0 : config.rag) == null ? void 0 : _a.uiMapping;
|
|
5749
|
+
if ((mapping == null ? void 0 : mapping[uiKey]) && meta[mapping[uiKey]] !== void 0) return meta[mapping[uiKey]];
|
|
5750
|
+
if (trainedSchema && typeof trainedSchema === "object") {
|
|
5751
|
+
const trainedKey = trainedSchema[uiKey];
|
|
5752
|
+
if (trainedKey && meta[trainedKey] !== void 0) return meta[trainedKey];
|
|
4655
5753
|
}
|
|
4656
|
-
|
|
4657
|
-
|
|
5754
|
+
return resolveMetadataValue(meta, uiKey);
|
|
5755
|
+
}
|
|
5756
|
+
static extractProductInfo(item, config, trainedSchema) {
|
|
5757
|
+
var _a;
|
|
5758
|
+
const meta = item.metadata || {};
|
|
5759
|
+
const name = this.getDynamicVal(meta, "name", config, trainedSchema);
|
|
5760
|
+
const price = this.getDynamicVal(meta, "price", config, trainedSchema);
|
|
5761
|
+
const brand = this.getDynamicVal(meta, "brand", config, trainedSchema);
|
|
5762
|
+
const description = this.cleanProductDescription(
|
|
5763
|
+
(_a = this.extractProductDescriptionFromContent(item.content)) != null ? _a : this.getProductDescriptionValue(meta, config, trainedSchema)
|
|
5764
|
+
);
|
|
5765
|
+
if (name || this.isProductData(item)) {
|
|
5766
|
+
let finalName = name ? String(name) : void 0;
|
|
5767
|
+
if (!finalName) {
|
|
5768
|
+
const nameMatch = item.content.match(/(?:Product|Item|Name):\s*([^\n]+)/i);
|
|
5769
|
+
finalName = nameMatch ? nameMatch[1].trim() : item.content.split("\n")[0].substring(0, 60);
|
|
5770
|
+
}
|
|
5771
|
+
let finalPrice = typeof price === "number" || typeof price === "string" ? price : void 0;
|
|
5772
|
+
if (!finalPrice) {
|
|
5773
|
+
const priceMatch = item.content.match(/(?:Price|Cost):\s*\$?([\d,.]+)/i) || item.content.match(/\$\s*([\d,.]+)/);
|
|
5774
|
+
if (priceMatch) finalPrice = priceMatch[1].replace(/,/g, "");
|
|
5775
|
+
}
|
|
5776
|
+
const imageValue = this.getDynamicVal(meta, "image", config, trainedSchema);
|
|
5777
|
+
return {
|
|
5778
|
+
id: item.id,
|
|
5779
|
+
name: finalName,
|
|
5780
|
+
price: finalPrice,
|
|
5781
|
+
image: typeof imageValue === "string" ? imageValue : void 0,
|
|
5782
|
+
brand: brand ? String(brand) : void 0,
|
|
5783
|
+
description,
|
|
5784
|
+
inStock: this.determineStockStatus(item)
|
|
5785
|
+
};
|
|
4658
5786
|
}
|
|
4659
|
-
return
|
|
5787
|
+
return null;
|
|
4660
5788
|
}
|
|
4661
|
-
|
|
4662
|
-
|
|
4663
|
-
|
|
4664
|
-
|
|
4665
|
-
|
|
4666
|
-
|
|
4667
|
-
|
|
4668
|
-
|
|
4669
|
-
|
|
4670
|
-
|
|
4671
|
-
return fieldCount.size > 2;
|
|
5789
|
+
static extractProductDescriptionFromContent(content) {
|
|
5790
|
+
const bodyMatch = content.match(
|
|
5791
|
+
/\bBody\s*\(HTML\)\s*:\s*([\s\S]*?)(?=\s*[,.]?\s+\b(?:Handle|Title|Vendor|Type|Tags|Published|Option\d*\s+Name|Option\d*\s+Value|Option|Variant|Image|SKU|Price|Status|Category)\b\s*:|$)/i
|
|
5792
|
+
);
|
|
5793
|
+
if (bodyMatch == null ? void 0 : bodyMatch[1]) return bodyMatch[1];
|
|
5794
|
+
const descriptionMatch = content.match(
|
|
5795
|
+
/\b(?:Description|Summary|Details|Body|Content)\s*:\s*([\s\S]*?)(?=\s*[,.]?\s+\b(?:Handle|Title|Vendor|Type|Tags|Published|Option\d*\s+Name|Option\d*\s+Value|Option|Variant|Image|SKU|Price|Status|Category)\b\s*:|$)/i
|
|
5796
|
+
);
|
|
5797
|
+
if (descriptionMatch == null ? void 0 : descriptionMatch[1]) return descriptionMatch[1];
|
|
5798
|
+
return null;
|
|
4672
5799
|
}
|
|
4673
|
-
|
|
4674
|
-
|
|
4675
|
-
|
|
4676
|
-
|
|
4677
|
-
|
|
4678
|
-
|
|
4679
|
-
|
|
4680
|
-
|
|
4681
|
-
|
|
4682
|
-
|
|
4683
|
-
|
|
4684
|
-
|
|
4685
|
-
|
|
4686
|
-
|
|
4687
|
-
|
|
4688
|
-
* @param sources - vector DB matches returned by RAG retrieval
|
|
4689
|
-
* @param llm - any ILLMProvider instance (OpenAI, Anthropic, Ollama, Gemini, REST…)
|
|
4690
|
-
* @returns - a validated UITransformationResponse (type + title + description + data)
|
|
4691
|
-
*/
|
|
4692
|
-
static async analyzeAndDecide(query, sources, llm) {
|
|
4693
|
-
try {
|
|
4694
|
-
const context = this.buildContextSummary(sources);
|
|
4695
|
-
const systemPrompt = this.buildVisualizationSystemPrompt();
|
|
4696
|
-
const userPrompt = [
|
|
4697
|
-
`USER QUESTION: ${query}`,
|
|
4698
|
-
"",
|
|
4699
|
-
"RETRIEVED DATA (JSON):",
|
|
4700
|
-
context
|
|
4701
|
-
].join("\n");
|
|
4702
|
-
const rawResponse = await llm.chat(
|
|
4703
|
-
[{ role: "user", content: userPrompt }],
|
|
4704
|
-
"",
|
|
4705
|
-
{ systemPrompt, temperature: 0 }
|
|
5800
|
+
static getProductDescriptionValue(meta, config, trainedSchema) {
|
|
5801
|
+
const mapped = this.getDynamicVal(meta, "description", config, trainedSchema);
|
|
5802
|
+
if (mapped !== void 0 && mapped !== meta.content && mapped !== meta.text) return mapped;
|
|
5803
|
+
const preferredKeys = [
|
|
5804
|
+
"body_html",
|
|
5805
|
+
"body html",
|
|
5806
|
+
"bodyHtml",
|
|
5807
|
+
"description",
|
|
5808
|
+
"summary",
|
|
5809
|
+
"details",
|
|
5810
|
+
"body"
|
|
5811
|
+
];
|
|
5812
|
+
for (const key of preferredKeys) {
|
|
5813
|
+
const match = Object.keys(meta).find(
|
|
5814
|
+
(candidate) => this.normalizeComparableField(candidate) === this.normalizeComparableField(key)
|
|
4706
5815
|
);
|
|
4707
|
-
|
|
4708
|
-
if (parsed) {
|
|
4709
|
-
console.debug("[UITransformer] LLM chose visualization type:", parsed.type);
|
|
4710
|
-
return parsed;
|
|
4711
|
-
}
|
|
4712
|
-
console.warn("[UITransformer] LLM returned unparseable response; falling back to heuristic.");
|
|
4713
|
-
} catch (err) {
|
|
4714
|
-
console.warn("[UITransformer] analyzeAndDecide LLM call failed; falling back to heuristic.", err);
|
|
5816
|
+
if (match && meta[match] !== void 0 && meta[match] !== null) return meta[match];
|
|
4715
5817
|
}
|
|
4716
|
-
return
|
|
5818
|
+
return void 0;
|
|
4717
5819
|
}
|
|
4718
|
-
|
|
4719
|
-
|
|
4720
|
-
|
|
5820
|
+
static cleanProductDescription(raw) {
|
|
5821
|
+
if (raw === null || raw === void 0) return void 0;
|
|
5822
|
+
const extracted = this.extractProductDescriptionFromContent(String(raw));
|
|
5823
|
+
if (extracted && extracted !== String(raw)) return this.cleanProductDescription(extracted);
|
|
5824
|
+
const text = String(raw).replace(/<[^>]*>/g, " ").replace(/ /gi, " ").replace(/&/gi, "&").replace(/"/gi, '"').replace(/'/gi, "'").replace(/\[[^\]]*TYPE[^\]]*\]/gi, " ").replace(/\b(?:Handle|Title|Vendor|Type|Tags|Published|Option\d*\s+Name|Option\d*\s+Value|Option|Variant|Image|SKU|Price|Status|Category)\s*:\s*[^,.\n:]+[,.]?/gi, " ").replace(/\bBody\s*\(HTML\)\s*:\s*/gi, " ").replace(/\b(?:Description|Summary|Details|Body|Content)\s*:\s*/gi, " ").replace(/\s+/g, " ").trim();
|
|
5825
|
+
return text || void 0;
|
|
5826
|
+
}
|
|
5827
|
+
// ─── Visualization Prompt ─────────────────────────────────────────────────
|
|
4721
5828
|
static buildVisualizationSystemPrompt() {
|
|
4722
5829
|
return `You are a data visualization expert embedded in a RAG chat system.
|
|
4723
|
-
You will receive a user question and structured data
|
|
4724
|
-
Your ONLY job is to
|
|
4725
|
-
|
|
5830
|
+
You will receive a user question, a pre-computed INTENT object, and structured data from a vector database.
|
|
5831
|
+
Your ONLY job is to return a single JSON object describing how to visualize the data.
|
|
5832
|
+
The INTENT object is authoritative \u2014 honor it. For example:
|
|
5833
|
+
- If intent.wantsExplicitTable is false, never return type "table".
|
|
5834
|
+
- If intent.isTemporal is true, prefer line_chart.
|
|
5835
|
+
- If intent.visualizationHint is "comparison" or "ranking", prefer bar_chart.
|
|
5836
|
+
- If intent.visualizationHint is "composition", prefer pie_chart.
|
|
5837
|
+
- If intent.recommendedChart is unsupported by the response schema, use the nearest supported type.
|
|
5838
|
+
- Always respond in the language specified by intent.language.
|
|
4726
5839
|
|
|
4727
|
-
Return ONLY a valid JSON object \u2014 no markdown
|
|
5840
|
+
Return ONLY a valid JSON object \u2014 no markdown fences, no prose.
|
|
4728
5841
|
|
|
4729
|
-
The JSON must have this exact shape:
|
|
4730
5842
|
{
|
|
4731
|
-
"type": "bar_chart" | "line_chart" | "pie_chart" | "radar_chart" | "table" | "text",
|
|
4732
|
-
"title": "
|
|
4733
|
-
"description": "
|
|
4734
|
-
"data": <structured data
|
|
5843
|
+
"type": "bar_chart" | "horizontal_bar" | "line_chart" | "pie_chart" | "histogram" | "scatter_plot" | "radar_chart" | "metric_card" | "geo_map" | "table" | "text",
|
|
5844
|
+
"title": "concise descriptive title",
|
|
5845
|
+
"description": "one sentence describing the visualization",
|
|
5846
|
+
"data": <structured data per type below>
|
|
4735
5847
|
}
|
|
4736
5848
|
|
|
4737
|
-
DATA
|
|
4738
|
-
- bar_chart:
|
|
4739
|
-
-
|
|
4740
|
-
-
|
|
4741
|
-
-
|
|
4742
|
-
|
|
4743
|
-
|
|
4744
|
-
|
|
4745
|
-
|
|
4746
|
-
|
|
4747
|
-
|
|
4748
|
-
-
|
|
4749
|
-
- text: { "content": "<prose answer>" }
|
|
4750
|
-
|
|
4751
|
-
DECISION RULES (follow strictly):
|
|
4752
|
-
1. bar_chart \u2192 comparing quantities across categories (e.g. sales by region, price by product). Use this when there is only ONE value per category.
|
|
4753
|
-
2. line_chart \u2192 trends or changes over time (dates, months, years, sequential events)
|
|
4754
|
-
3. pie_chart \u2192 proportional breakdown or percentage distribution
|
|
4755
|
-
4. radar_chart \u2192 comparing 2 or more products across MULTIPLE attributes (e.g. comparing features, ratings, or characteristics of 2 specific products). If the user asks to "compare" products and the data contains multiple dimensions or ratings for them, you MUST use this.
|
|
4756
|
-
5. table \u2192 multi-field structured records where each item has \u2265 3 attributes
|
|
4757
|
-
6. text \u2192 conversational or free-form answers where no chart adds value
|
|
5849
|
+
DATA SHAPES:
|
|
5850
|
+
- bar_chart: [{ "category": string, "value": number }]
|
|
5851
|
+
- horizontal_bar: [{ "category": string, "value": number }]
|
|
5852
|
+
- histogram: [{ "category": string, "value": number }]
|
|
5853
|
+
- line_chart: [{ "timestamp": string, "value": number, "label": string }]
|
|
5854
|
+
- pie_chart: [{ "label": string, "value": number }]
|
|
5855
|
+
- scatter_plot:[{ "x": number, "y": number, "label": string }]
|
|
5856
|
+
- radar_chart: [{ "attribute": string, "<series1>": number, "<series2>": number, ... }]
|
|
5857
|
+
- metric_card: { "label": string, "value": number, "operation": "sum"|"average"|"count"|"min"|"max"|"median" }
|
|
5858
|
+
- geo_map: [{ "category": string, "value": number }]
|
|
5859
|
+
- table: { "columns": string[], "rows": (string|number)[][] }
|
|
5860
|
+
- text: { "content": "A concise plain-language answer." }
|
|
4758
5861
|
|
|
4759
|
-
|
|
4760
|
-
|
|
4761
|
-
|
|
4762
|
-
|
|
4763
|
-
|
|
5862
|
+
RULES:
|
|
5863
|
+
1. Aggregate values \u2014 never pass raw nested objects in chart arrays.
|
|
5864
|
+
2. All "value" fields must be numbers.
|
|
5865
|
+
3. Cap bar/line/pie at 12 data points.
|
|
5866
|
+
4. Use dollar signs only for monetary prices, not for stock quantities or years.
|
|
5867
|
+
5. If no relevant data is found, return text: "I cannot answer this question based on the available information."`;
|
|
4764
5868
|
}
|
|
4765
|
-
/**
|
|
4766
|
-
* Serialize retrieved vector matches into a compact JSON context string.
|
|
4767
|
-
* Limits the total character count to avoid exceeding LLM context windows.
|
|
4768
|
-
*/
|
|
4769
5869
|
static buildContextSummary(sources, maxChars = 6e3) {
|
|
4770
5870
|
const items = sources.map((s, i) => {
|
|
4771
5871
|
var _a, _b, _c, _d;
|
|
@@ -4986,27 +6086,38 @@ var Pipeline = class {
|
|
|
4986
6086
|
async initialize() {
|
|
4987
6087
|
var _a;
|
|
4988
6088
|
if (this.initialised) return;
|
|
4989
|
-
const chartInstruction = `You are a helpful
|
|
6089
|
+
const chartInstruction = `You are a helpful assistant. Use the provided context to answer questions accurately.
|
|
4990
6090
|
|
|
4991
|
-
###
|
|
6091
|
+
### CRITICAL RULES:
|
|
6092
|
+
- ONLY answer the user's specific question. Do NOT suggest or recommend unrelated products unless specifically asked.
|
|
4992
6093
|
- NEVER generate markdown tables. If you do, the UI will break.
|
|
4993
6094
|
- NEVER generate HTML tags like <figure>, <tbody>, <tr>, etc.
|
|
4994
6095
|
- NEVER generate text-based charts or graphs.
|
|
6096
|
+
- NEVER say you cannot display, render, draw, or create a chart when the user asks for a visualization. The UI handles visual rendering separately.
|
|
4995
6097
|
- ONLY use plain text and bullet points.
|
|
6098
|
+
- NEVER use the plus sign (+) as a separator between names, categories, or products. Use commas or bullet points.
|
|
6099
|
+
- ONLY answer the question if the sources contain relevant information to answer it, else say that you cannot answer the question.
|
|
6100
|
+
- If answer cannot be found in the sources, say that you cannot answer the question and do not hallucinate.
|
|
6101
|
+
- Do not use information from previous turns to answer the question.
|
|
6102
|
+
- You CAN use numbers for years and counts (e.g., 2006, 5800). But NEVER put a dollar sign ($) before non-monetary numbers like Stock quantities, EAN numbers, or years. Only use dollar signs for actual monetary prices.
|
|
4996
6103
|
|
|
4997
6104
|
### PRODUCT DISPLAY:
|
|
4998
|
-
-
|
|
6105
|
+
- Recommended Products should only be displayed when the user explicitly asks for products.
|
|
6106
|
+
- When recommending products (ONLY when asked), simply list their names, prices, and features in a friendly, conversational manner.
|
|
4999
6107
|
- The UI will automatically detect these products and show high-quality product cards in a carousel below your message.
|
|
6108
|
+
- For product descriptions, summarize customer-facing details only. Do NOT list internal catalog fields such as Handle, Body (HTML), Vendor, Type, Tags, Published, Option, Variant, SKU, or raw metadata labels.
|
|
5000
6109
|
- Do NOT try to format product lists as tables.
|
|
5001
6110
|
`;
|
|
5002
6111
|
this.config.llm.systemPrompt = chartInstruction;
|
|
5003
6112
|
this.vectorDB = await ProviderRegistry.createVectorProvider(this.config.vectorDb);
|
|
5004
|
-
|
|
6113
|
+
this.llmRouter = new LLMRouter(this.config);
|
|
6114
|
+
const { llmProvider: resolvedLLM, embeddingProvider } = await EmbeddingStrategyResolver.resolve(
|
|
5005
6115
|
this.config.llm,
|
|
5006
6116
|
this.config.embedding
|
|
5007
6117
|
);
|
|
5008
|
-
this.llmProvider = llmProvider;
|
|
5009
6118
|
this.embeddingProvider = embeddingProvider;
|
|
6119
|
+
await this.llmRouter.initialize(resolvedLLM);
|
|
6120
|
+
this.llmProvider = this.llmRouter.get("default");
|
|
5010
6121
|
if (this.config.graphDb) {
|
|
5011
6122
|
this.graphDB = await ProviderRegistry.createGraphProvider(this.config.graphDb);
|
|
5012
6123
|
await this.graphDB.initialize();
|
|
@@ -5163,10 +6274,16 @@ var Pipeline = class {
|
|
|
5163
6274
|
/**
|
|
5164
6275
|
* High-performance streaming RAG flow.
|
|
5165
6276
|
* Yields text chunks first, then the retrieval metadata + observability trace at the end.
|
|
6277
|
+
*
|
|
6278
|
+
* Latency optimizations:
|
|
6279
|
+
* - Strategy classification runs in parallel with query embedding (saves ~400ms)
|
|
6280
|
+
* - Hallucination scoring is fire-and-forget (doesn't block metadata yield)
|
|
6281
|
+
* - UITransformation is computed after text streaming and emitted with metadata
|
|
6282
|
+
* - SchemaMapper.train runs while answer generation streams
|
|
5166
6283
|
*/
|
|
5167
6284
|
askStream(_0) {
|
|
5168
6285
|
return __asyncGenerator(this, arguments, function* (question, history = [], namespace) {
|
|
5169
|
-
var _a, _b, _c, _d, _e, _f, _g, _h, _i;
|
|
6286
|
+
var _a, _b, _c, _d, _e, _f, _g, _h, _i, _j, _k, _l, _m;
|
|
5170
6287
|
yield new __await(this.initialize());
|
|
5171
6288
|
const ns = namespace != null ? namespace : this.config.projectId;
|
|
5172
6289
|
const topK = (_b = (_a = this.config.rag) == null ? void 0 : _a.topK) != null ? _b : 5;
|
|
@@ -5182,27 +6299,48 @@ var Pipeline = class {
|
|
|
5182
6299
|
}
|
|
5183
6300
|
const hints = QueryProcessor.extractQueryFieldHints(question, (_f = this.config.rag) == null ? void 0 : _f.filterableFields);
|
|
5184
6301
|
const filter = QueryProcessor.buildQueryFilter(question, hints);
|
|
6302
|
+
const numericPredicates = QueryProcessor.extractNumericPredicates(question, (_g = this.config.rag) == null ? void 0 : _g.filterableFields);
|
|
6303
|
+
if (numericPredicates.length > 0) {
|
|
6304
|
+
filter.__numericPredicates = numericPredicates;
|
|
6305
|
+
}
|
|
5185
6306
|
const embedStart = performance.now();
|
|
5186
|
-
const
|
|
5187
|
-
|
|
5188
|
-
|
|
5189
|
-
|
|
5190
|
-
|
|
6307
|
+
const cacheKey = `${ns}::${searchQuery}`;
|
|
6308
|
+
const cachedVector = this.embeddingCache.get(cacheKey);
|
|
6309
|
+
const [strategyResult, embeddedVector] = yield new __await(Promise.all([
|
|
6310
|
+
QueryProcessor.determineRetrievalStrategy(
|
|
6311
|
+
searchQuery,
|
|
6312
|
+
this.llmRouter.get("fast"),
|
|
6313
|
+
(_h = this.config.rag) == null ? void 0 : _h.graphKeywords,
|
|
6314
|
+
(_i = this.config.rag) == null ? void 0 : _i.vectorKeywords
|
|
6315
|
+
),
|
|
6316
|
+
// Embed immediately regardless of strategy — costs nothing if cached.
|
|
6317
|
+
// If strategy turns out to be 'graph'-only we just won't use the vector.
|
|
6318
|
+
cachedVector ? Promise.resolve(cachedVector) : this.embeddingProvider.embed(searchQuery, { taskType: "query" })
|
|
6319
|
+
]));
|
|
6320
|
+
const queryVector = embeddedVector;
|
|
6321
|
+
if (!cachedVector && queryVector.length > 0) {
|
|
6322
|
+
this.embeddingCache.set(cacheKey, queryVector);
|
|
6323
|
+
}
|
|
6324
|
+
const graphData = (strategyResult === "graph" || strategyResult === "both") && this.graphDB && ((_j = this.config.rag) == null ? void 0 : _j.useGraphRetrieval) ? yield new __await(this.graphDB.query(searchQuery)) : void 0;
|
|
6325
|
+
const hasNumericPredicates = Array.isArray(filter.__numericPredicates) && filter.__numericPredicates.length > 0;
|
|
6326
|
+
const wantsExhaustiveList = hasNumericPredicates || /\b(list|all|show|provide|give|get)\b/i.test(question);
|
|
6327
|
+
const retrievalLimit = wantsExhaustiveList ? Math.max(topK * 20, 100) : topK * 2;
|
|
6328
|
+
const rawSources = (strategyResult === "vector" || strategyResult === "both") && queryVector && queryVector.length > 0 ? yield new __await(this.vectorDB.query(queryVector, retrievalLimit, ns, filter)) : [];
|
|
5191
6329
|
const retrieveEnd = performance.now();
|
|
5192
6330
|
const embedMs = retrieveEnd - embedStart;
|
|
5193
6331
|
const retrieveMs = retrieveEnd - embedStart;
|
|
5194
6332
|
const rerankStart = performance.now();
|
|
5195
|
-
|
|
5196
|
-
|
|
6333
|
+
const structuredSources = this.applyStructuredFilters(rawSources, filter);
|
|
6334
|
+
let sources = hasNumericPredicates ? structuredSources : structuredSources.filter((m) => m.score >= scoreThreshold);
|
|
6335
|
+
if (!hasNumericPredicates && ((_k = this.config.rag) == null ? void 0 : _k.useReranking)) {
|
|
5197
6336
|
sources = yield new __await(this.reranker.rerank(sources, question, topK));
|
|
5198
|
-
} else {
|
|
6337
|
+
} else if (!wantsExhaustiveList) {
|
|
5199
6338
|
sources = sources.slice(0, topK);
|
|
5200
6339
|
}
|
|
5201
6340
|
const rerankMs = performance.now() - rerankStart;
|
|
5202
6341
|
let context = sources.length ? sources.map((m, i) => `[Source ${i + 1}]
|
|
5203
6342
|
${m.content}`).join("\n\n---\n\n") : "No relevant context found.";
|
|
5204
6343
|
if (graphData && graphData.nodes.length > 0) {
|
|
5205
|
-
console.log(`[Graph Retrieval] Found ${graphData.nodes.length} relevant entities.`);
|
|
5206
6344
|
const graphContext = graphData.nodes.map(
|
|
5207
6345
|
(n) => `Entity: ${n.label} (${n.id})${n.properties ? " - " + JSON.stringify(n.properties) : ""}`
|
|
5208
6346
|
).join("\n");
|
|
@@ -5213,20 +6351,18 @@ VECTOR CONTEXT:
|
|
|
5213
6351
|
${context}`;
|
|
5214
6352
|
}
|
|
5215
6353
|
const allMetadataKeys = Array.from(new Set(sources.flatMap((s) => Object.keys(s.metadata || {}))));
|
|
5216
|
-
const
|
|
5217
|
-
const restrictionSuffix = "\n\n(IMPORTANT: Use plain text only. NEVER generate tables, HTML figures, or text charts. If listing products, use simple bullet points.)";
|
|
6354
|
+
const trainedSchemaPromise = allMetadataKeys.length > 0 ? SchemaMapper.train(this.llmProvider, ns, allMetadataKeys).catch(() => void 0) : Promise.resolve(void 0);
|
|
6355
|
+
const restrictionSuffix = "\n\n(IMPORTANT: Use plain text only. NEVER generate tables, HTML figures, or text charts. The UI renders requested charts separately, so NEVER say you cannot create, display, draw, render, or provide a chart/visualization. If listing products, use simple bullet points or comma-separated names. NEVER use plus signs (+) to separate product names, category names, or list items. For product description/detail questions, provide customer-facing prose only and do NOT list internal catalog fields such as Handle, Body (HTML), Vendor, Type, Tags, Published, Option, Variant, SKU, or raw metadata labels. You CAN use numbers for years/counts like 2006 or 5800, but NEVER put a dollar sign ($) before them.)";
|
|
5218
6356
|
const hardenedHistory = [...history];
|
|
5219
6357
|
const userQuestion = { role: "user", content: question + restrictionSuffix };
|
|
5220
6358
|
const messages = [...hardenedHistory, userQuestion];
|
|
5221
|
-
const systemPrompt = (
|
|
6359
|
+
const systemPrompt = (_l = this.config.llm.systemPrompt) != null ? _l : "";
|
|
5222
6360
|
const userPrompt = messages.map((m) => `${m.role}: ${m.content}`).join("\n");
|
|
5223
6361
|
let fullReply = "";
|
|
5224
6362
|
const generateStart = performance.now();
|
|
5225
6363
|
if (this.llmProvider.chatStream) {
|
|
5226
6364
|
const stream = this.llmProvider.chatStream(messages, context);
|
|
5227
|
-
if (!stream) {
|
|
5228
|
-
throw new Error(`[Pipeline] ${this.config.llm.provider} chatStream returned undefined`);
|
|
5229
|
-
}
|
|
6365
|
+
if (!stream) throw new Error(`[Pipeline] ${this.config.llm.provider} chatStream returned undefined`);
|
|
5230
6366
|
try {
|
|
5231
6367
|
for (var iter = __forAwait(stream), more, temp, error; more = !(temp = yield new __await(iter.next())).done; more = false) {
|
|
5232
6368
|
const chunk = temp.value;
|
|
@@ -5253,7 +6389,7 @@ ${context}`;
|
|
|
5253
6389
|
const latency = {
|
|
5254
6390
|
embedMs: Math.round(embedMs),
|
|
5255
6391
|
retrieveMs: Math.round(retrieveMs),
|
|
5256
|
-
rerankMs: ((
|
|
6392
|
+
rerankMs: ((_m = this.config.rag) == null ? void 0 : _m.useReranking) ? Math.round(rerankMs) : void 0,
|
|
5257
6393
|
generateMs: Math.round(generateMs),
|
|
5258
6394
|
totalMs: Math.round(totalMs)
|
|
5259
6395
|
};
|
|
@@ -5266,9 +6402,6 @@ ${context}`;
|
|
|
5266
6402
|
totalTokens: promptTokens + completionTokens,
|
|
5267
6403
|
estimatedCostUsd: estimateCostUsd(promptTokens, completionTokens, this.config.llm.model)
|
|
5268
6404
|
};
|
|
5269
|
-
const hallucinationResult = yield new __await(scoreHallucination(this.llmProvider, fullReply, context).catch(() => void 0));
|
|
5270
|
-
const trainedSchema = yield new __await(trainingPromise);
|
|
5271
|
-
const uiTransformation = yield new __await(this.generateUiTransformation(question, sources, trainedSchema));
|
|
5272
6405
|
const trace = {
|
|
5273
6406
|
requestId,
|
|
5274
6407
|
query: question,
|
|
@@ -5287,10 +6420,21 @@ ${context}`;
|
|
|
5287
6420
|
}),
|
|
5288
6421
|
latency,
|
|
5289
6422
|
tokens,
|
|
5290
|
-
hallucinationScore: hallucinationResult == null ? void 0 : hallucinationResult.score,
|
|
5291
|
-
hallucinationReason: hallucinationResult == null ? void 0 : hallucinationResult.reason,
|
|
5292
6423
|
timestamp: (/* @__PURE__ */ new Date()).toISOString()
|
|
5293
6424
|
};
|
|
6425
|
+
let uiTransformation;
|
|
6426
|
+
try {
|
|
6427
|
+
const trainedSchema = yield new __await(trainedSchemaPromise);
|
|
6428
|
+
uiTransformation = yield new __await(this.generateUiTransformation(
|
|
6429
|
+
question,
|
|
6430
|
+
sources,
|
|
6431
|
+
trainedSchema,
|
|
6432
|
+
hasNumericPredicates
|
|
6433
|
+
));
|
|
6434
|
+
} catch (uiError) {
|
|
6435
|
+
console.warn("[Pipeline] UI transformation failed before metadata yield:", uiError);
|
|
6436
|
+
uiTransformation = UITransformer.transform(question, sources, this.config);
|
|
6437
|
+
}
|
|
5294
6438
|
yield {
|
|
5295
6439
|
reply: "",
|
|
5296
6440
|
sources,
|
|
@@ -5298,6 +6442,13 @@ ${context}`;
|
|
|
5298
6442
|
ui_transformation: uiTransformation,
|
|
5299
6443
|
trace
|
|
5300
6444
|
};
|
|
6445
|
+
scoreHallucination(this.llmProvider, fullReply, context).then((hallucinationResult) => {
|
|
6446
|
+
if (hallucinationResult) {
|
|
6447
|
+
trace.hallucinationScore = hallucinationResult.score;
|
|
6448
|
+
trace.hallucinationReason = hallucinationResult.reason;
|
|
6449
|
+
}
|
|
6450
|
+
}).catch(() => {
|
|
6451
|
+
});
|
|
5301
6452
|
} catch (error2) {
|
|
5302
6453
|
throw new Error(`[Pipeline] Stream failed: ${error2 instanceof Error ? error2.message : String(error2)}`);
|
|
5303
6454
|
}
|
|
@@ -5307,24 +6458,107 @@ ${context}`;
|
|
|
5307
6458
|
* Universal retrieval method combining all enabled providers.
|
|
5308
6459
|
* Uses an LRU-bounded embedding cache to avoid re-embedding the same query.
|
|
5309
6460
|
*/
|
|
5310
|
-
async generateUiTransformation(question, sources, trainedSchema) {
|
|
6461
|
+
async generateUiTransformation(question, sources, trainedSchema, forceDeterministic = false) {
|
|
5311
6462
|
if (!sources || sources.length === 0) {
|
|
5312
6463
|
return UITransformer.transform(question, sources, this.config, trainedSchema);
|
|
5313
6464
|
}
|
|
6465
|
+
if (forceDeterministic) {
|
|
6466
|
+
return UITransformer.transform(question, sources, this.config, trainedSchema);
|
|
6467
|
+
}
|
|
5314
6468
|
try {
|
|
5315
|
-
return await UITransformer.analyzeAndDecide(question, sources, this.
|
|
6469
|
+
return await UITransformer.analyzeAndDecide(question, sources, this.llmRouter.get("fast"));
|
|
5316
6470
|
} catch (err) {
|
|
5317
6471
|
console.warn("[Pipeline] generateUiTransformation failed, using heuristic fallback:", err);
|
|
5318
6472
|
return UITransformer.transform(question, sources, this.config, trainedSchema);
|
|
5319
6473
|
}
|
|
5320
6474
|
}
|
|
6475
|
+
applyStructuredFilters(sources, filter) {
|
|
6476
|
+
const predicates = Array.isArray(filter.__numericPredicates) ? filter.__numericPredicates : [];
|
|
6477
|
+
if (predicates.length === 0) return sources;
|
|
6478
|
+
return sources.filter((source) => predicates.every((predicate) => {
|
|
6479
|
+
const value = this.resolveNumericPredicateValue(source, predicate);
|
|
6480
|
+
return value !== null && this.matchesNumericPredicate(value, predicate);
|
|
6481
|
+
})).sort((a, b) => {
|
|
6482
|
+
var _a, _b;
|
|
6483
|
+
const primary = predicates[0];
|
|
6484
|
+
const aValue = (_a = this.resolveNumericPredicateValue(a, primary)) != null ? _a : 0;
|
|
6485
|
+
const bValue = (_b = this.resolveNumericPredicateValue(b, primary)) != null ? _b : 0;
|
|
6486
|
+
return primary.operator === "lt" || primary.operator === "lte" ? aValue - bValue : bValue - aValue;
|
|
6487
|
+
});
|
|
6488
|
+
}
|
|
6489
|
+
resolveNumericPredicateValue(source, predicate) {
|
|
6490
|
+
const meta = source.metadata || {};
|
|
6491
|
+
const field = predicate.field;
|
|
6492
|
+
const entries = Object.entries(meta).filter(([, value]) => value !== null && value !== void 0 && typeof value !== "object");
|
|
6493
|
+
if (field) {
|
|
6494
|
+
const normalizedField = this.normalizeComparableField(field);
|
|
6495
|
+
const exact = entries.find(([key]) => this.normalizeComparableField(key) === normalizedField);
|
|
6496
|
+
const fuzzy = exact != null ? exact : entries.map(([key, value]) => ({ key, value, score: this.fieldSimilarityScore(key, field) })).filter((candidate) => candidate.score > 0).sort((a, b) => b.score - a.score)[0];
|
|
6497
|
+
if (fuzzy) {
|
|
6498
|
+
const value = this.toFiniteNumber(Array.isArray(fuzzy) ? fuzzy[1] : fuzzy.value);
|
|
6499
|
+
if (value !== null) return value;
|
|
6500
|
+
}
|
|
6501
|
+
const contentValue = this.extractNumericValueFromContent(source.content, field);
|
|
6502
|
+
if (contentValue !== null) return contentValue;
|
|
6503
|
+
}
|
|
6504
|
+
for (const [key, value] of entries) {
|
|
6505
|
+
if (/(id|sku|ean|upc|phone|zip|postal|code)/i.test(key)) continue;
|
|
6506
|
+
const numeric = this.toFiniteNumber(value);
|
|
6507
|
+
if (numeric !== null) return numeric;
|
|
6508
|
+
}
|
|
6509
|
+
return null;
|
|
6510
|
+
}
|
|
6511
|
+
extractNumericValueFromContent(content, field) {
|
|
6512
|
+
const escapedWords = field.split(/\s+|_+|-+/).filter(Boolean).map((word) => word.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"));
|
|
6513
|
+
if (escapedWords.length === 0) return null;
|
|
6514
|
+
const pattern = new RegExp(`(?:${escapedWords.join("[\\\\s_:-]*")})\\s*[:\\-\u2013]?\\s*([\\d,]+(?:\\.\\d+)?)`, "i");
|
|
6515
|
+
const match = content.match(pattern);
|
|
6516
|
+
return match ? this.toFiniteNumber(match[1]) : null;
|
|
6517
|
+
}
|
|
6518
|
+
matchesNumericPredicate(value, predicate) {
|
|
6519
|
+
switch (predicate.operator) {
|
|
6520
|
+
case "gt":
|
|
6521
|
+
return value > predicate.value;
|
|
6522
|
+
case "gte":
|
|
6523
|
+
return value >= predicate.value;
|
|
6524
|
+
case "lt":
|
|
6525
|
+
return value < predicate.value;
|
|
6526
|
+
case "lte":
|
|
6527
|
+
return value <= predicate.value;
|
|
6528
|
+
case "eq":
|
|
6529
|
+
default:
|
|
6530
|
+
return value === predicate.value;
|
|
6531
|
+
}
|
|
6532
|
+
}
|
|
6533
|
+
normalizeComparableField(value) {
|
|
6534
|
+
return value.toLowerCase().replace(/[^a-z0-9]/g, "");
|
|
6535
|
+
}
|
|
6536
|
+
fieldSimilarityScore(candidate, requested) {
|
|
6537
|
+
const normalizedCandidate = this.normalizeComparableField(candidate);
|
|
6538
|
+
const normalizedRequested = this.normalizeComparableField(requested);
|
|
6539
|
+
if (normalizedCandidate === normalizedRequested) return 10;
|
|
6540
|
+
if (normalizedCandidate.includes(normalizedRequested) || normalizedRequested.includes(normalizedCandidate)) return 8;
|
|
6541
|
+
const candidateTokens = this.fieldTokens(candidate);
|
|
6542
|
+
const requestedTokens = this.fieldTokens(requested);
|
|
6543
|
+
const overlap = requestedTokens.filter((token) => candidateTokens.includes(token)).length;
|
|
6544
|
+
if (overlap === 0) return 0;
|
|
6545
|
+
return overlap / Math.max(requestedTokens.length, candidateTokens.length);
|
|
6546
|
+
}
|
|
6547
|
+
fieldTokens(value) {
|
|
6548
|
+
return value.toLowerCase().split(/[^a-z0-9]+/).map((token) => token.replace(/ies$/, "y").replace(/s$/, "")).filter((token) => token.length > 1);
|
|
6549
|
+
}
|
|
6550
|
+
toFiniteNumber(value) {
|
|
6551
|
+
if (typeof value === "number") return Number.isFinite(value) ? value : null;
|
|
6552
|
+
const numeric = Number(String(value != null ? value : "").replace(/[$,% ,]/g, ""));
|
|
6553
|
+
return Number.isFinite(numeric) ? numeric : null;
|
|
6554
|
+
}
|
|
5321
6555
|
async retrieve(query, options) {
|
|
5322
|
-
var _a, _b, _c;
|
|
6556
|
+
var _a, _b, _c, _d, _e, _f;
|
|
5323
6557
|
const ns = (_a = options.namespace) != null ? _a : this.config.projectId;
|
|
5324
6558
|
const topK = (_b = options.topK) != null ? _b : 5;
|
|
5325
6559
|
const cacheKey = `${ns}::${query}`;
|
|
5326
6560
|
let queryVector = this.embeddingCache.get(cacheKey);
|
|
5327
|
-
const strategy = await QueryProcessor.determineRetrievalStrategy(query, this.
|
|
6561
|
+
const strategy = await QueryProcessor.determineRetrievalStrategy(query, this.llmRouter.get("fast"));
|
|
5328
6562
|
console.debug(`[Pipeline] Determined retrieval strategy: ${strategy}`);
|
|
5329
6563
|
const [retrievedVector, graphData] = await Promise.all([
|
|
5330
6564
|
// Only embed if we need vector search (strategy is 'vector' or 'both')
|
|
@@ -5336,8 +6570,27 @@ ${context}`;
|
|
|
5336
6570
|
this.embeddingCache.set(cacheKey, retrievedVector);
|
|
5337
6571
|
queryVector = retrievedVector;
|
|
5338
6572
|
}
|
|
5339
|
-
const
|
|
5340
|
-
|
|
6573
|
+
const baseFilter = __spreadProps(__spreadValues({}, (_d = options.filter) != null ? _d : {}), { queryText: query });
|
|
6574
|
+
const numericPredicates = QueryProcessor.extractNumericPredicates(query, (_e = this.config.rag) == null ? void 0 : _e.filterableFields);
|
|
6575
|
+
if (numericPredicates.length > 0) {
|
|
6576
|
+
baseFilter.__numericPredicates = numericPredicates;
|
|
6577
|
+
}
|
|
6578
|
+
const retrievalLimit = numericPredicates.length > 0 ? Math.max(topK * 20, 100) : topK;
|
|
6579
|
+
const sources = (strategy === "vector" || strategy === "both") && queryVector && queryVector.length > 0 ? this.applyStructuredFilters(
|
|
6580
|
+
await this.vectorDB.query(queryVector, retrievalLimit, ns, baseFilter),
|
|
6581
|
+
baseFilter
|
|
6582
|
+
) : [];
|
|
6583
|
+
const resolvedSources = [];
|
|
6584
|
+
for (const source of sources) {
|
|
6585
|
+
const parentId = (_f = source.metadata) == null ? void 0 : _f.parent_id;
|
|
6586
|
+
if (parentId) {
|
|
6587
|
+
console.log(`[Pipeline] Multi-Vector: Found child chunk. Parent ID: ${parentId}`);
|
|
6588
|
+
resolvedSources.push(source);
|
|
6589
|
+
} else {
|
|
6590
|
+
resolvedSources.push(source);
|
|
6591
|
+
}
|
|
6592
|
+
}
|
|
6593
|
+
return { sources: resolvedSources, graphData };
|
|
5341
6594
|
}
|
|
5342
6595
|
/** Rewrite the user query for better retrieval performance. */
|
|
5343
6596
|
async rewriteQuery(question, history) {
|
|
@@ -5703,7 +6956,7 @@ function createStreamHandler(configOrPlugin) {
|
|
|
5703
6956
|
const encoder = new TextEncoder();
|
|
5704
6957
|
const stream = new ReadableStream({
|
|
5705
6958
|
async start(controller) {
|
|
5706
|
-
var _a
|
|
6959
|
+
var _a;
|
|
5707
6960
|
const enqueue = (text) => controller.enqueue(encoder.encode(text));
|
|
5708
6961
|
try {
|
|
5709
6962
|
const pipelineStream = plugin.chatStream(message, history, namespace);
|
|
@@ -5721,8 +6974,7 @@ function createStreamHandler(configOrPlugin) {
|
|
|
5721
6974
|
}
|
|
5722
6975
|
if (sources.length > 0) {
|
|
5723
6976
|
try {
|
|
5724
|
-
const
|
|
5725
|
-
const uiTransformation = (_b = responseChunk == null ? void 0 : responseChunk.ui_transformation) != null ? _b : llmProvider ? await UITransformer.analyzeAndDecide(message, sources, llmProvider) : UITransformer.transform(message, sources, plugin.getConfig());
|
|
6977
|
+
const uiTransformation = (_a = responseChunk == null ? void 0 : responseChunk.ui_transformation) != null ? _a : UITransformer.transform(message, sources, plugin.getConfig());
|
|
5726
6978
|
if (uiTransformation) {
|
|
5727
6979
|
enqueue(sseUIFrame(uiTransformation));
|
|
5728
6980
|
}
|