@retrivora-ai/rag-engine 1.9.0 → 1.9.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{ILLMProvider-BOJFz3Na.d.mts → ILLMProvider-Bw2A28nU.d.mts} +12 -0
- package/dist/{ILLMProvider-BOJFz3Na.d.ts → ILLMProvider-Bw2A28nU.d.ts} +12 -0
- package/dist/handlers/index.d.mts +2 -2
- package/dist/handlers/index.d.ts +2 -2
- package/dist/handlers/index.js +1874 -542
- package/dist/handlers/index.mjs +1873 -541
- package/dist/{index-D3V9Et2M.d.mts → index-B70ZLkfG.d.mts} +1 -1
- package/dist/{index-BwpcaziY.d.ts → index-DVu-mkAM.d.ts} +1 -1
- package/dist/index.css +83 -0
- package/dist/index.d.mts +2 -2
- package/dist/index.d.ts +2 -2
- package/dist/index.js +330 -106
- package/dist/index.mjs +333 -107
- package/dist/server.d.mts +32 -5
- package/dist/server.d.ts +32 -5
- package/dist/server.js +1871 -736
- package/dist/server.mjs +1870 -735
- package/package.json +1 -1
- package/src/components/ChatWindow.tsx +24 -14
- package/src/components/MarkdownComponents.tsx +3 -3
- package/src/components/MessageBubble.tsx +89 -7
- package/src/components/ProductCard.tsx +29 -2
- package/src/components/UIDispatcher.tsx +1 -0
- package/src/components/VisualizationRenderer.tsx +143 -11
- package/src/config/EmbeddingStrategy.ts +5 -4
- package/src/config/RagConfig.ts +10 -0
- package/src/config/serverConfig.ts +16 -1
- package/src/core/LLMRouter.ts +79 -0
- package/src/core/Pipeline.ts +295 -51
- package/src/core/ProviderRegistry.ts +6 -0
- package/src/core/QueryProcessor.ts +108 -9
- package/src/handlers/index.ts +37 -11
- package/src/hooks/useRagChat.ts +77 -17
- package/src/llm/providers/UniversalLLMAdapter.ts +110 -13
- package/src/providers/vectordb/ChromaDBProvider.ts +13 -2
- package/src/providers/vectordb/MilvusProvider.ts +18 -2
- package/src/providers/vectordb/MultiTablePostgresProvider.ts +48 -16
- package/src/providers/vectordb/PostgreSQLProvider.ts +1 -1
- package/src/providers/vectordb/QdrantProvider.ts +1 -1
- package/src/providers/vectordb/RedisProvider.ts +3 -4
- package/src/providers/vectordb/WeaviateProvider.ts +41 -3
- package/src/types/chat.ts +2 -0
- package/src/types/index.ts +26 -0
- package/src/utils/ProductExtractor.ts +5 -3
- package/src/utils/SchemaMapper.ts +6 -4
- package/src/utils/UITransformer.ts +1350 -490
- package/src/utils/synonyms.ts +6 -4
package/dist/server.js
CHANGED
|
@@ -334,17 +334,300 @@ var init_PineconeProvider = __esm({
|
|
|
334
334
|
}
|
|
335
335
|
});
|
|
336
336
|
|
|
337
|
+
// src/providers/vectordb/PostgreSQLProvider.ts
|
|
338
|
+
var PostgreSQLProvider_exports = {};
|
|
339
|
+
__export(PostgreSQLProvider_exports, {
|
|
340
|
+
PostgreSQLProvider: () => PostgreSQLProvider
|
|
341
|
+
});
|
|
342
|
+
var import_pg, PostgreSQLProvider;
|
|
343
|
+
var init_PostgreSQLProvider = __esm({
|
|
344
|
+
"src/providers/vectordb/PostgreSQLProvider.ts"() {
|
|
345
|
+
"use strict";
|
|
346
|
+
import_pg = require("pg");
|
|
347
|
+
init_BaseVectorProvider();
|
|
348
|
+
PostgreSQLProvider = class extends BaseVectorProvider {
|
|
349
|
+
constructor(config) {
|
|
350
|
+
var _a;
|
|
351
|
+
super(config);
|
|
352
|
+
this.tableName = this.indexName.replace(/[^a-z0-9_]/gi, "_");
|
|
353
|
+
const opts = config.options;
|
|
354
|
+
if (!opts.connectionString) throw new Error("[PostgreSQLProvider] options.connectionString is required");
|
|
355
|
+
this.connectionString = opts.connectionString;
|
|
356
|
+
this.dimensions = (_a = opts.dimensions) != null ? _a : 1536;
|
|
357
|
+
}
|
|
358
|
+
static getValidator() {
|
|
359
|
+
return {
|
|
360
|
+
validate(config) {
|
|
361
|
+
const errors = [];
|
|
362
|
+
const opts = config.options || {};
|
|
363
|
+
if (!opts.connectionString) {
|
|
364
|
+
errors.push({
|
|
365
|
+
field: "vectorDb.options.connectionString",
|
|
366
|
+
message: "PostgreSQL connection string is required",
|
|
367
|
+
suggestion: "Set PGVECTOR_CONNECTION_STRING environment variable",
|
|
368
|
+
severity: "error"
|
|
369
|
+
});
|
|
370
|
+
}
|
|
371
|
+
if (opts.tables && typeof opts.tables !== "string" && !Array.isArray(opts.tables)) {
|
|
372
|
+
errors.push({
|
|
373
|
+
field: "vectorDb.options.tables",
|
|
374
|
+
message: "PostgreSQL tables must be a string or a string array",
|
|
375
|
+
severity: "error"
|
|
376
|
+
});
|
|
377
|
+
}
|
|
378
|
+
return errors;
|
|
379
|
+
}
|
|
380
|
+
};
|
|
381
|
+
}
|
|
382
|
+
static getHealthChecker() {
|
|
383
|
+
return {
|
|
384
|
+
async check(config) {
|
|
385
|
+
const opts = config.options || {};
|
|
386
|
+
const timestamp = Date.now();
|
|
387
|
+
try {
|
|
388
|
+
const { Client } = await import("pg");
|
|
389
|
+
const client = new Client({ connectionString: opts.connectionString });
|
|
390
|
+
await client.connect();
|
|
391
|
+
const result = await client.query(`
|
|
392
|
+
SELECT EXISTS(SELECT 1 FROM pg_extension WHERE extname = 'vector');
|
|
393
|
+
`);
|
|
394
|
+
const hasVector = result.rows[0].exists;
|
|
395
|
+
await client.end();
|
|
396
|
+
return {
|
|
397
|
+
healthy: true,
|
|
398
|
+
provider: "postgresql",
|
|
399
|
+
capabilities: { pgvectorInstalled: hasVector },
|
|
400
|
+
timestamp
|
|
401
|
+
};
|
|
402
|
+
} catch (error) {
|
|
403
|
+
return {
|
|
404
|
+
healthy: false,
|
|
405
|
+
provider: "postgresql",
|
|
406
|
+
error: `Connection failed: ${error instanceof Error ? error.message : String(error)}`,
|
|
407
|
+
timestamp
|
|
408
|
+
};
|
|
409
|
+
}
|
|
410
|
+
}
|
|
411
|
+
};
|
|
412
|
+
}
|
|
413
|
+
async initialize() {
|
|
414
|
+
this.pool = new import_pg.Pool({ connectionString: this.connectionString });
|
|
415
|
+
const client = await this.pool.connect();
|
|
416
|
+
try {
|
|
417
|
+
await client.query("CREATE EXTENSION IF NOT EXISTS vector");
|
|
418
|
+
await client.query(`
|
|
419
|
+
CREATE TABLE IF NOT EXISTS ${this.tableName} (
|
|
420
|
+
id TEXT PRIMARY KEY,
|
|
421
|
+
namespace TEXT NOT NULL DEFAULT '',
|
|
422
|
+
content TEXT NOT NULL,
|
|
423
|
+
metadata JSONB,
|
|
424
|
+
embedding VECTOR(${this.dimensions})
|
|
425
|
+
)
|
|
426
|
+
`);
|
|
427
|
+
await client.query(`
|
|
428
|
+
CREATE INDEX IF NOT EXISTS ${this.tableName}_embedding_idx
|
|
429
|
+
ON ${this.tableName}
|
|
430
|
+
USING hnsw (embedding vector_cosine_ops)
|
|
431
|
+
`);
|
|
432
|
+
} finally {
|
|
433
|
+
client.release();
|
|
434
|
+
}
|
|
435
|
+
}
|
|
436
|
+
async upsert(doc, namespace = "") {
|
|
437
|
+
var _a;
|
|
438
|
+
const vectorLiteral = `[${doc.vector.join(",")}]`;
|
|
439
|
+
await this.pool.query(
|
|
440
|
+
`INSERT INTO ${this.tableName} (id, namespace, content, metadata, embedding)
|
|
441
|
+
VALUES ($1, $2, $3, $4, $5::vector)
|
|
442
|
+
ON CONFLICT (id) DO UPDATE
|
|
443
|
+
SET namespace = EXCLUDED.namespace,
|
|
444
|
+
content = EXCLUDED.content,
|
|
445
|
+
metadata = EXCLUDED.metadata,
|
|
446
|
+
embedding = EXCLUDED.embedding`,
|
|
447
|
+
[doc.id, namespace, doc.content, JSON.stringify((_a = doc.metadata) != null ? _a : {}), vectorLiteral]
|
|
448
|
+
);
|
|
449
|
+
}
|
|
450
|
+
async batchUpsert(docs, namespace = "") {
|
|
451
|
+
if (docs.length === 0) return;
|
|
452
|
+
const client = await this.pool.connect();
|
|
453
|
+
try {
|
|
454
|
+
await client.query("BEGIN");
|
|
455
|
+
const BATCH_SIZE = 50;
|
|
456
|
+
for (let i = 0; i < docs.length; i += BATCH_SIZE) {
|
|
457
|
+
const batch = docs.slice(i, i + BATCH_SIZE);
|
|
458
|
+
const values = [];
|
|
459
|
+
const valuePlaceholders = batch.map((doc, idx) => {
|
|
460
|
+
var _a;
|
|
461
|
+
const offset = idx * 5;
|
|
462
|
+
values.push(doc.id, namespace, doc.content, JSON.stringify((_a = doc.metadata) != null ? _a : {}), `[${doc.vector.join(",")}]`);
|
|
463
|
+
return `($${offset + 1}, $${offset + 2}, $${offset + 3}, $${offset + 4}, $${offset + 5}::vector)`;
|
|
464
|
+
}).join(", ");
|
|
465
|
+
const query = `
|
|
466
|
+
INSERT INTO ${this.tableName} (id, namespace, content, metadata, embedding)
|
|
467
|
+
VALUES ${valuePlaceholders}
|
|
468
|
+
ON CONFLICT (id) DO UPDATE
|
|
469
|
+
SET namespace = EXCLUDED.namespace,
|
|
470
|
+
content = EXCLUDED.content,
|
|
471
|
+
metadata = EXCLUDED.metadata,
|
|
472
|
+
embedding = EXCLUDED.embedding
|
|
473
|
+
`;
|
|
474
|
+
await client.query(query, values);
|
|
475
|
+
}
|
|
476
|
+
await client.query("COMMIT");
|
|
477
|
+
} catch (error) {
|
|
478
|
+
await client.query("ROLLBACK");
|
|
479
|
+
throw error;
|
|
480
|
+
} finally {
|
|
481
|
+
client.release();
|
|
482
|
+
}
|
|
483
|
+
}
|
|
484
|
+
async query(vector, topK, namespace, filter) {
|
|
485
|
+
const vectorLiteral = `[${vector.join(",")}]`;
|
|
486
|
+
let whereClause = namespace ? `WHERE namespace = $3` : "";
|
|
487
|
+
const params = [vectorLiteral, topK];
|
|
488
|
+
if (namespace) params.push(namespace);
|
|
489
|
+
const publicFilter = this.sanitizeFilter(filter);
|
|
490
|
+
if (Object.keys(publicFilter).length > 0) {
|
|
491
|
+
const filterConditions = Object.entries(publicFilter).map(([key, val]) => {
|
|
492
|
+
const paramIdx = params.length + 1;
|
|
493
|
+
params.push(String(val));
|
|
494
|
+
return `metadata->>'${key}' = $${paramIdx}`;
|
|
495
|
+
}).join(" AND ");
|
|
496
|
+
whereClause = whereClause ? `${whereClause} AND ${filterConditions}` : `WHERE ${filterConditions}`;
|
|
497
|
+
}
|
|
498
|
+
const client = await this.pool.connect();
|
|
499
|
+
try {
|
|
500
|
+
const efSearch = this.config.options.efSearch || Math.max(topK * 10, 40);
|
|
501
|
+
await client.query(`SET LOCAL hnsw.ef_search = ${efSearch}`);
|
|
502
|
+
const result = await client.query(
|
|
503
|
+
`SELECT id, content, metadata, 1 - (embedding <=> $1::vector) AS score
|
|
504
|
+
FROM ${this.tableName}
|
|
505
|
+
${whereClause}
|
|
506
|
+
ORDER BY embedding <=> $1::vector
|
|
507
|
+
LIMIT $2`,
|
|
508
|
+
params
|
|
509
|
+
);
|
|
510
|
+
return result.rows.map((row) => ({
|
|
511
|
+
id: String(row["id"]),
|
|
512
|
+
score: parseFloat(String(row["score"])),
|
|
513
|
+
content: String(row["content"]),
|
|
514
|
+
metadata: row["metadata"]
|
|
515
|
+
}));
|
|
516
|
+
} finally {
|
|
517
|
+
client.release();
|
|
518
|
+
}
|
|
519
|
+
}
|
|
520
|
+
async delete(id, namespace) {
|
|
521
|
+
const where = namespace ? "WHERE id = $1 AND namespace = $2" : "WHERE id = $1";
|
|
522
|
+
const params = namespace ? [id, namespace] : [id];
|
|
523
|
+
await this.pool.query(`DELETE FROM ${this.tableName} ${where}`, params);
|
|
524
|
+
}
|
|
525
|
+
async deleteNamespace(namespace) {
|
|
526
|
+
await this.pool.query(`DELETE FROM ${this.tableName} WHERE namespace = $1`, [namespace]);
|
|
527
|
+
}
|
|
528
|
+
async ping() {
|
|
529
|
+
try {
|
|
530
|
+
await this.pool.query("SELECT 1");
|
|
531
|
+
return true;
|
|
532
|
+
} catch (e) {
|
|
533
|
+
return false;
|
|
534
|
+
}
|
|
535
|
+
}
|
|
536
|
+
async disconnect() {
|
|
537
|
+
await this.pool.end();
|
|
538
|
+
}
|
|
539
|
+
};
|
|
540
|
+
}
|
|
541
|
+
});
|
|
542
|
+
|
|
543
|
+
// src/utils/synonyms.ts
|
|
544
|
+
function resolveMetadataValue(meta, uiKey) {
|
|
545
|
+
var _a;
|
|
546
|
+
const synonyms = (_a = FIELD_SYNONYMS[uiKey]) != null ? _a : [];
|
|
547
|
+
const keys = Object.keys(meta);
|
|
548
|
+
const systemKeys = ["namespace", "filename", "filesize", "filetype", "docid", "uploadedat", "dimension", "chunkindex"];
|
|
549
|
+
let match = keys.find((k) => k.toLowerCase() === uiKey.toLowerCase());
|
|
550
|
+
if (match !== void 0) return meta[match];
|
|
551
|
+
match = keys.find((k) => synonyms.map((s) => s.toLowerCase()).includes(k.toLowerCase()));
|
|
552
|
+
if (match !== void 0) return meta[match];
|
|
553
|
+
const isBlacklisted = (kl) => {
|
|
554
|
+
return systemKeys.includes(kl) || kl.startsWith("option1") || kl.startsWith("option2") || kl.startsWith("option3");
|
|
555
|
+
};
|
|
556
|
+
match = keys.find((k) => {
|
|
557
|
+
const kl = k.toLowerCase();
|
|
558
|
+
if (isBlacklisted(kl)) return false;
|
|
559
|
+
return kl.includes(uiKey.toLowerCase());
|
|
560
|
+
});
|
|
561
|
+
if (match !== void 0) return meta[match];
|
|
562
|
+
match = keys.find((k) => {
|
|
563
|
+
const kl = k.toLowerCase();
|
|
564
|
+
if (isBlacklisted(kl)) return false;
|
|
565
|
+
return synonyms.some((sk) => kl.includes(sk.toLowerCase()) || sk.toLowerCase().includes(kl));
|
|
566
|
+
});
|
|
567
|
+
return match !== void 0 ? meta[match] : void 0;
|
|
568
|
+
}
|
|
569
|
+
var FIELD_SYNONYMS;
|
|
570
|
+
var init_synonyms = __esm({
|
|
571
|
+
"src/utils/synonyms.ts"() {
|
|
572
|
+
"use strict";
|
|
573
|
+
FIELD_SYNONYMS = {
|
|
574
|
+
name: ["product", "item", "title", "label", "heading", "subject", "product name", "item name"],
|
|
575
|
+
price: ["cost", "amount", "msrp", "price", "rate", "value", "price_usd", "variant price"],
|
|
576
|
+
brand: ["manufacturer", "vendor", "make", "company", "brand_name", "supplier"],
|
|
577
|
+
image: [
|
|
578
|
+
"imageUrl",
|
|
579
|
+
"thumbnail",
|
|
580
|
+
"img",
|
|
581
|
+
"url",
|
|
582
|
+
"photo",
|
|
583
|
+
"picture",
|
|
584
|
+
"media",
|
|
585
|
+
"image_url",
|
|
586
|
+
"main_image",
|
|
587
|
+
"product_image",
|
|
588
|
+
"thumb",
|
|
589
|
+
"image src",
|
|
590
|
+
"variant image"
|
|
591
|
+
],
|
|
592
|
+
stock: [
|
|
593
|
+
"inventory",
|
|
594
|
+
"quantity",
|
|
595
|
+
"count",
|
|
596
|
+
"availability",
|
|
597
|
+
"stock_level",
|
|
598
|
+
"inStock",
|
|
599
|
+
"is_available",
|
|
600
|
+
"in stock",
|
|
601
|
+
"status",
|
|
602
|
+
"variant inventory qty"
|
|
603
|
+
],
|
|
604
|
+
category: [
|
|
605
|
+
"product_category",
|
|
606
|
+
"product category",
|
|
607
|
+
"category_name",
|
|
608
|
+
"category name",
|
|
609
|
+
"department",
|
|
610
|
+
"collection",
|
|
611
|
+
"type"
|
|
612
|
+
],
|
|
613
|
+
description: ["summary", "content", "body", "text", "info", "details", "body (html)", "seo description"],
|
|
614
|
+
link: ["url", "href", "product_url", "page_url", "link"]
|
|
615
|
+
};
|
|
616
|
+
}
|
|
617
|
+
});
|
|
618
|
+
|
|
337
619
|
// src/providers/vectordb/MultiTablePostgresProvider.ts
|
|
338
620
|
var MultiTablePostgresProvider_exports = {};
|
|
339
621
|
__export(MultiTablePostgresProvider_exports, {
|
|
340
622
|
MultiTablePostgresProvider: () => MultiTablePostgresProvider
|
|
341
623
|
});
|
|
342
|
-
var
|
|
624
|
+
var import_pg2, MultiTablePostgresProvider;
|
|
343
625
|
var init_MultiTablePostgresProvider = __esm({
|
|
344
626
|
"src/providers/vectordb/MultiTablePostgresProvider.ts"() {
|
|
345
627
|
"use strict";
|
|
346
|
-
|
|
628
|
+
import_pg2 = require("pg");
|
|
347
629
|
init_BaseVectorProvider();
|
|
630
|
+
init_synonyms();
|
|
348
631
|
MultiTablePostgresProvider = class extends BaseVectorProvider {
|
|
349
632
|
constructor(config) {
|
|
350
633
|
var _a, _b, _c;
|
|
@@ -361,7 +644,7 @@ var init_MultiTablePostgresProvider = __esm({
|
|
|
361
644
|
this.uploadTable = opts.uploadTable || "document_chunks";
|
|
362
645
|
}
|
|
363
646
|
async initialize() {
|
|
364
|
-
this.pool = new
|
|
647
|
+
this.pool = new import_pg2.Pool({ connectionString: this.connectionString });
|
|
365
648
|
const client = await this.pool.connect();
|
|
366
649
|
try {
|
|
367
650
|
await client.query("CREATE EXTENSION IF NOT EXISTS vector");
|
|
@@ -425,8 +708,13 @@ var init_MultiTablePostgresProvider = __esm({
|
|
|
425
708
|
}
|
|
426
709
|
tableName = tableName.replace(/[^a-z0-9_]/gi, "_").toLowerCase() || this.uploadTable;
|
|
427
710
|
const firstMeta = fileDocs[0].metadata || {};
|
|
428
|
-
|
|
429
|
-
|
|
711
|
+
let csvHeaders = [];
|
|
712
|
+
if (Array.isArray(firstMeta.csvHeaders)) {
|
|
713
|
+
csvHeaders = firstMeta.csvHeaders;
|
|
714
|
+
} else {
|
|
715
|
+
const systemKeys = ["fileName", "fileSize", "fileType", "uploadedAt", "dimension", "chunkIndex", "id", "namespace", "content", "metadata", "embedding", "docId", "docid", "chunkId", "chunkid", "csvHeaders", "csvheaders"];
|
|
716
|
+
csvHeaders = Object.keys(firstMeta).filter((k) => !systemKeys.includes(k) && !systemKeys.includes(k.toLowerCase()));
|
|
717
|
+
}
|
|
430
718
|
const columnDefs = csvHeaders.map((h) => `"${h}" TEXT`).join(",\n ");
|
|
431
719
|
const createTableSql = `
|
|
432
720
|
CREATE TABLE IF NOT EXISTS "${tableName}" (
|
|
@@ -505,26 +793,41 @@ var init_MultiTablePostgresProvider = __esm({
|
|
|
505
793
|
const allResults = [];
|
|
506
794
|
console.log(`[MultiTablePostgresProvider] --- Starting Multi-Table Search ---`);
|
|
507
795
|
const queryText = _filter == null ? void 0 : _filter.queryText;
|
|
508
|
-
const entityHints = Array.isArray(_filter == null ? void 0 : _filter.
|
|
509
|
-
(hint) => typeof hint === "object" && hint !== null && typeof hint.value === "string"
|
|
510
|
-
).map((hint) => hint.value.trim().toLowerCase()).filter(Boolean) : [];
|
|
796
|
+
const entityHints = Array.isArray(_filter == null ? void 0 : _filter.keywords) ? _filter.keywords.map((k) => k.trim().toLowerCase()).filter(Boolean) : [];
|
|
511
797
|
console.log(`[MultiTablePostgresProvider] queryText: "${queryText}"`);
|
|
512
798
|
console.log(`[MultiTablePostgresProvider] entityHints: [${entityHints.join(", ")}]`);
|
|
513
799
|
const getDynamicKeywordQuery = () => {
|
|
514
800
|
if (entityHints.length > 0) {
|
|
515
|
-
return entityHints.map((h) => h.
|
|
801
|
+
return entityHints.map((h) => h.replace(/\s+/g, " & ")).join(" & ");
|
|
516
802
|
}
|
|
517
803
|
if (queryText) {
|
|
518
|
-
return queryText.toLowerCase().replace(/\b(show|me|the|product[s]?|item[s]?|card[s]?|of|category|find|list|browse|what|are|display|all|under|for|in|with|about)\b/g, "").trim().replace(/\s+/g, " & ");
|
|
804
|
+
return queryText.toLowerCase().replace(/\b(show|me|the|product[s]?|item[s]?|card[s]?|organization[s]?|comp(?:any|anies)|of|category|find|list|browse|what|are|display|all|whose|that|which|have|has|having|greater|than|more|less|equal|equals|above|below|over|under|at|least|most|for|in|with|about|get|details|information|info)\b/g, "").replace(/\b\d+(?:\.\d+)?\b/g, "").trim().replace(/\s+/g, " & ");
|
|
519
805
|
}
|
|
520
806
|
return "";
|
|
521
807
|
};
|
|
522
808
|
const dynamicKeywordQuery = getDynamicKeywordQuery();
|
|
809
|
+
const tableLimit = Math.max(topK, 50);
|
|
810
|
+
const metadataFilters = _filter == null ? void 0 : _filter.metadata;
|
|
523
811
|
const queryPromises = this.tables.map(async (table) => {
|
|
524
812
|
try {
|
|
525
813
|
let sqlQuery = "";
|
|
526
814
|
let params = [];
|
|
527
|
-
|
|
815
|
+
let whereClause = "";
|
|
816
|
+
const filterParams = [];
|
|
817
|
+
if (metadataFilters && Object.keys(metadataFilters).length > 0) {
|
|
818
|
+
const conditions = Object.entries(metadataFilters).map(([key, val]) => {
|
|
819
|
+
var _a3;
|
|
820
|
+
filterParams.push(String(val));
|
|
821
|
+
const baseOffset = queryText && dynamicKeywordQuery ? 2 : 1;
|
|
822
|
+
const paramIdx = baseOffset + filterParams.length;
|
|
823
|
+
const synonyms = (_a3 = FIELD_SYNONYMS[key]) != null ? _a3 : [];
|
|
824
|
+
const keysToCheck = [key, ...synonyms];
|
|
825
|
+
const coalesceExprs = keysToCheck.map((k) => `LOWER(metadata->>'${k}')`);
|
|
826
|
+
return `COALESCE(${coalesceExprs.join(", ")}) = LOWER($${paramIdx})`;
|
|
827
|
+
});
|
|
828
|
+
whereClause = `WHERE ${conditions.join(" AND ")}`;
|
|
829
|
+
}
|
|
830
|
+
if (queryText && dynamicKeywordQuery) {
|
|
528
831
|
const hasEntityHints = entityHints.length > 0;
|
|
529
832
|
const exactNameScoreExpr = hasEntityHints ? `+ (
|
|
530
833
|
SELECT COALESCE(MAX(
|
|
@@ -540,19 +843,21 @@ var init_MultiTablePostgresProvider = __esm({
|
|
|
540
843
|
COALESCE(ts_rank(to_tsvector('english', (to_jsonb(t) - 'embedding')::text), to_tsquery('english', $2)), 0) AS keyword_score,
|
|
541
844
|
((1 - (embedding <=> $1::vector)) + (COALESCE(ts_rank(to_tsvector('english', (to_jsonb(t) - 'embedding')::text), to_tsquery('english', $2)), 0) * 2.0) ${exactNameScoreExpr}) AS hybrid_score
|
|
542
845
|
FROM "${table}" t
|
|
846
|
+
${whereClause}
|
|
543
847
|
ORDER BY hybrid_score DESC
|
|
544
|
-
LIMIT
|
|
848
|
+
LIMIT ${tableLimit}
|
|
545
849
|
`;
|
|
546
|
-
params = [vectorLiteral, dynamicKeywordQuery];
|
|
850
|
+
params = [vectorLiteral, dynamicKeywordQuery, ...filterParams];
|
|
547
851
|
} else {
|
|
548
852
|
sqlQuery = `
|
|
549
853
|
SELECT *,
|
|
550
|
-
|
|
854
|
+
(1 - (embedding <=> $1::vector)) AS hybrid_score
|
|
551
855
|
FROM "${table}" t
|
|
856
|
+
${whereClause}
|
|
552
857
|
ORDER BY hybrid_score DESC
|
|
553
|
-
LIMIT
|
|
858
|
+
LIMIT ${tableLimit}
|
|
554
859
|
`;
|
|
555
|
-
params = [vectorLiteral];
|
|
860
|
+
params = [vectorLiteral, ...filterParams];
|
|
556
861
|
}
|
|
557
862
|
const result = await this.pool.query(sqlQuery, params);
|
|
558
863
|
if (result.rowCount && result.rowCount > 0) {
|
|
@@ -875,13 +1180,25 @@ var init_MilvusProvider = __esm({
|
|
|
875
1180
|
};
|
|
876
1181
|
await this.http.post("/v1/vector/upsert", payload);
|
|
877
1182
|
}
|
|
878
|
-
// eslint-disable-next-line @typescript-eslint/no-unused-vars
|
|
879
1183
|
async query(vector, topK, namespace, _filter) {
|
|
1184
|
+
const sanitizedFilter = this.sanitizeFilter(_filter);
|
|
1185
|
+
const filterParts = [];
|
|
1186
|
+
if (namespace) {
|
|
1187
|
+
filterParts.push(`namespace == "${namespace}"`);
|
|
1188
|
+
}
|
|
1189
|
+
for (const [key, value] of Object.entries(sanitizedFilter)) {
|
|
1190
|
+
if (key === "queryText" || key === "keywords") continue;
|
|
1191
|
+
if (typeof value === "string") {
|
|
1192
|
+
filterParts.push(`metadata["${key}"] == "${value.replace(/"/g, '\\"')}"`);
|
|
1193
|
+
} else if (typeof value === "number") {
|
|
1194
|
+
filterParts.push(`metadata["${key}"] == ${value}`);
|
|
1195
|
+
}
|
|
1196
|
+
}
|
|
880
1197
|
const payload = {
|
|
881
1198
|
collectionName: this.indexName,
|
|
882
1199
|
vector,
|
|
883
1200
|
limit: topK,
|
|
884
|
-
filter:
|
|
1201
|
+
filter: filterParts.length > 0 ? filterParts.join(" && ") : void 0,
|
|
885
1202
|
outputFields: ["content", "metadata"],
|
|
886
1203
|
searchParams: {
|
|
887
1204
|
nprobe: this.config.options.nprobe || 16,
|
|
@@ -1080,6 +1397,7 @@ var init_QdrantProvider = __esm({
|
|
|
1080
1397
|
await this.http.put(`/collections/${this.indexName}/points`, payload);
|
|
1081
1398
|
}
|
|
1082
1399
|
async query(vector, topK, namespace, _filter) {
|
|
1400
|
+
var _a;
|
|
1083
1401
|
const must = [];
|
|
1084
1402
|
if (namespace) {
|
|
1085
1403
|
must.push({ key: "namespace", match: { value: namespace } });
|
|
@@ -1101,7 +1419,7 @@ var init_QdrantProvider = __esm({
|
|
|
1101
1419
|
limit: topK,
|
|
1102
1420
|
with_payload: true,
|
|
1103
1421
|
params: {
|
|
1104
|
-
hnsw_ef: this.config.options.efSearch || Math.max(topK * 20, 128),
|
|
1422
|
+
hnsw_ef: ((_a = this.config.options) == null ? void 0 : _a.efSearch) || Math.max(topK * 20, 128),
|
|
1105
1423
|
exact: false
|
|
1106
1424
|
},
|
|
1107
1425
|
filter: must.length > 0 ? { must } : void 0
|
|
@@ -1226,12 +1544,20 @@ var init_ChromaDBProvider = __esm({
|
|
|
1226
1544
|
};
|
|
1227
1545
|
await this.http.post(`/api/v1/collections/${this.collectionId}/add`, payload);
|
|
1228
1546
|
}
|
|
1229
|
-
// eslint-disable-next-line @typescript-eslint/no-unused-vars
|
|
1230
1547
|
async query(vector, topK, namespace, _filter) {
|
|
1548
|
+
const sanitizedFilter = this.sanitizeFilter(_filter);
|
|
1549
|
+
const whereClauses = [];
|
|
1550
|
+
if (namespace) whereClauses.push({ namespace: { $eq: namespace } });
|
|
1551
|
+
Object.entries(sanitizedFilter).forEach(([key, value]) => {
|
|
1552
|
+
if (key === "namespace") return;
|
|
1553
|
+
if (typeof value === "string" || typeof value === "number" || typeof value === "boolean") {
|
|
1554
|
+
whereClauses.push({ [key]: { $eq: value } });
|
|
1555
|
+
}
|
|
1556
|
+
});
|
|
1231
1557
|
const payload = {
|
|
1232
1558
|
query_embeddings: [vector],
|
|
1233
1559
|
n_results: topK,
|
|
1234
|
-
where:
|
|
1560
|
+
where: whereClauses.length > 1 ? { $and: whereClauses } : whereClauses[0]
|
|
1235
1561
|
};
|
|
1236
1562
|
const { data } = await this.http.post(`/api/v1/collections/${this.collectionId}/query`, payload);
|
|
1237
1563
|
const matches = [];
|
|
@@ -1336,15 +1662,15 @@ var init_RedisProvider = __esm({
|
|
|
1336
1662
|
console.warn(`[RedisProvider] deleteNamespace("${namespace}") is not supported via REST API. Use Redis CLI: SCAN + DEL`);
|
|
1337
1663
|
}
|
|
1338
1664
|
/**
|
|
1339
|
-
*
|
|
1340
|
-
* Returns
|
|
1665
|
+
* Check reachability via a PING command.
|
|
1666
|
+
* Returns false on connection failure so health checks surface real problems.
|
|
1341
1667
|
*/
|
|
1342
1668
|
async ping() {
|
|
1343
1669
|
try {
|
|
1344
1670
|
await this.http.post("/", ["PING"]);
|
|
1345
1671
|
return true;
|
|
1346
1672
|
} catch (e) {
|
|
1347
|
-
return
|
|
1673
|
+
return false;
|
|
1348
1674
|
}
|
|
1349
1675
|
}
|
|
1350
1676
|
async disconnect() {
|
|
@@ -1384,15 +1710,17 @@ var init_WeaviateProvider = __esm({
|
|
|
1384
1710
|
await this.ping();
|
|
1385
1711
|
}
|
|
1386
1712
|
async upsert(doc, namespace) {
|
|
1713
|
+
const primitiveMetadata = this.extractPrimitiveMetadata(doc.metadata);
|
|
1387
1714
|
const payload = {
|
|
1388
1715
|
class: this.indexName,
|
|
1389
1716
|
id: doc.id,
|
|
1390
1717
|
vector: doc.vector,
|
|
1391
|
-
properties: {
|
|
1718
|
+
properties: __spreadProps(__spreadValues({
|
|
1392
1719
|
content: doc.content,
|
|
1393
|
-
metadata: JSON.stringify(doc.metadata || {})
|
|
1720
|
+
metadata: JSON.stringify(doc.metadata || {})
|
|
1721
|
+
}, primitiveMetadata), {
|
|
1394
1722
|
namespace: namespace || ""
|
|
1395
|
-
}
|
|
1723
|
+
})
|
|
1396
1724
|
};
|
|
1397
1725
|
await this.http.post("/v1/objects", payload);
|
|
1398
1726
|
}
|
|
@@ -1402,20 +1730,22 @@ var init_WeaviateProvider = __esm({
|
|
|
1402
1730
|
class: this.indexName,
|
|
1403
1731
|
id: doc.id,
|
|
1404
1732
|
vector: doc.vector,
|
|
1405
|
-
properties: {
|
|
1733
|
+
properties: __spreadProps(__spreadValues({
|
|
1406
1734
|
content: doc.content,
|
|
1407
|
-
metadata: JSON.stringify(doc.metadata || {})
|
|
1735
|
+
metadata: JSON.stringify(doc.metadata || {})
|
|
1736
|
+
}, this.extractPrimitiveMetadata(doc.metadata)), {
|
|
1408
1737
|
namespace: namespace || ""
|
|
1409
|
-
}
|
|
1738
|
+
})
|
|
1410
1739
|
}))
|
|
1411
1740
|
};
|
|
1412
1741
|
await this.http.post("/v1/batch/objects", payload);
|
|
1413
1742
|
}
|
|
1414
1743
|
async query(vector, topK, namespace, _filter) {
|
|
1415
1744
|
var _a, _b;
|
|
1745
|
+
const queryText = _filter == null ? void 0 : _filter.queryText;
|
|
1416
1746
|
const sanitizedFilter = this.sanitizeFilter(_filter);
|
|
1417
|
-
const queryText = sanitizedFilter.queryText;
|
|
1418
1747
|
const searchParams = queryText ? `hybrid: { query: ${JSON.stringify(queryText)}, alpha: 0.5 }` : `nearVector: { vector: ${JSON.stringify(vector)} }`;
|
|
1748
|
+
const where = this.buildWhereFilter(namespace, sanitizedFilter);
|
|
1419
1749
|
const graphqlQuery = {
|
|
1420
1750
|
query: `
|
|
1421
1751
|
{
|
|
@@ -1423,7 +1753,7 @@ var init_WeaviateProvider = __esm({
|
|
|
1423
1753
|
${this.indexName}(
|
|
1424
1754
|
${searchParams}
|
|
1425
1755
|
limit: ${topK}
|
|
1426
|
-
${
|
|
1756
|
+
${where ? `where: ${where}` : ""}
|
|
1427
1757
|
) {
|
|
1428
1758
|
content
|
|
1429
1759
|
metadata
|
|
@@ -1467,6 +1797,34 @@ var init_WeaviateProvider = __esm({
|
|
|
1467
1797
|
}
|
|
1468
1798
|
async disconnect() {
|
|
1469
1799
|
}
|
|
1800
|
+
extractPrimitiveMetadata(metadata) {
|
|
1801
|
+
const result = {};
|
|
1802
|
+
Object.entries(metadata || {}).forEach(([key, value]) => {
|
|
1803
|
+
if (typeof value === "string" || typeof value === "number" || typeof value === "boolean") {
|
|
1804
|
+
result[key] = value;
|
|
1805
|
+
}
|
|
1806
|
+
});
|
|
1807
|
+
return result;
|
|
1808
|
+
}
|
|
1809
|
+
buildWhereFilter(namespace, filter) {
|
|
1810
|
+
const operands = [];
|
|
1811
|
+
if (namespace) operands.push(this.weaviateOperand("namespace", namespace));
|
|
1812
|
+
Object.entries(filter || {}).forEach(([key, value]) => {
|
|
1813
|
+
if (key === "namespace") return;
|
|
1814
|
+
if (typeof value === "string" || typeof value === "number" || typeof value === "boolean") {
|
|
1815
|
+
operands.push(this.weaviateOperand(key, value));
|
|
1816
|
+
}
|
|
1817
|
+
});
|
|
1818
|
+
if (operands.length === 0) return void 0;
|
|
1819
|
+
if (operands.length === 1) return operands[0];
|
|
1820
|
+
return `{ operator: And, operands: [${operands.join(", ")}] }`;
|
|
1821
|
+
}
|
|
1822
|
+
weaviateOperand(key, value) {
|
|
1823
|
+
const path = `path: [${JSON.stringify(key)}], operator: Equal`;
|
|
1824
|
+
if (typeof value === "number") return `{ ${path}, valueNumber: ${value} }`;
|
|
1825
|
+
if (typeof value === "boolean") return `{ ${path}, valueBoolean: ${value} }`;
|
|
1826
|
+
return `{ ${path}, valueString: ${JSON.stringify(value)} }`;
|
|
1827
|
+
}
|
|
1470
1828
|
};
|
|
1471
1829
|
}
|
|
1472
1830
|
});
|
|
@@ -1820,7 +2178,7 @@ function getRagConfig(baseConfig, env = process.env) {
|
|
|
1820
2178
|
return getEnvConfig(env, baseConfig);
|
|
1821
2179
|
}
|
|
1822
2180
|
function getEnvConfig(env = process.env, base) {
|
|
1823
|
-
var _a, _b, _c, _d, _e, _f, _g, _h, _i, _j, _k, _l, _m, _n, _o, _p, _q, _r, _s, _t, _u, _v, _w, _x, _y, _z, _A, _B, _C, _D, _E, _F, _G, _H, _I, _J, _K, _L, _M, _N, _O, _P, _Q, _R, _S, _T, _U, _V, _W, _X, _Y, _Z, __, _$, _aa, _ba, _ca, _da, _ea, _fa, _ga, _ha, _ia, _ja, _ka, _la, _ma, _na, _oa, _pa, _qa, _ra, _sa, _ta, _ua, _va, _wa, _xa, _ya, _za, _Aa, _Ba, _Ca, _Da, _Ea, _Fa;
|
|
2181
|
+
var _a, _b, _c, _d, _e, _f, _g, _h, _i, _j, _k, _l, _m, _n, _o, _p, _q, _r, _s, _t, _u, _v, _w, _x, _y, _z, _A, _B, _C, _D, _E, _F, _G, _H, _I, _J, _K, _L, _M, _N, _O, _P, _Q, _R, _S, _T, _U, _V, _W, _X, _Y, _Z, __, _$, _aa, _ba, _ca, _da, _ea, _fa, _ga, _ha, _ia, _ja, _ka, _la, _ma, _na, _oa, _pa, _qa, _ra, _sa, _ta, _ua, _va, _wa, _xa, _ya, _za, _Aa, _Ba, _Ca, _Da, _Ea, _Fa, _Ga;
|
|
1824
2182
|
const projectId = (_c = (_b = (_a = readString(env, "RAG_PROJECT_ID")) != null ? _a : readString(env, "NEXT_PUBLIC_PROJECT_ID")) != null ? _b : base == null ? void 0 : base.projectId) != null ? _c : "__default__";
|
|
1825
2183
|
const vectorProvider = readEnum(env, "VECTOR_DB_PROVIDER", "pinecone", VECTOR_DB_PROVIDERS);
|
|
1826
2184
|
const llmProvider = readEnum(env, "LLM_PROVIDER", "openai", LLM_PROVIDERS);
|
|
@@ -1874,11 +2232,22 @@ function getEnvConfig(env = process.env, base) {
|
|
|
1874
2232
|
};
|
|
1875
2233
|
const embeddingApiKeyByProvider = {
|
|
1876
2234
|
openai: readString(env, "OPENAI_API_KEY"),
|
|
2235
|
+
anthropic: readString(env, "ANTHROPIC_API_KEY"),
|
|
2236
|
+
// Anthropic needs a separate embedding provider; key kept for completeness
|
|
1877
2237
|
gemini: readString(env, "GEMINI_API_KEY"),
|
|
1878
2238
|
ollama: void 0,
|
|
1879
2239
|
universal_rest: (_ea = readString(env, "EMBEDDING_API_KEY")) != null ? _ea : readString(env, "OPENAI_API_KEY"),
|
|
1880
2240
|
custom: (_fa = readString(env, "EMBEDDING_API_KEY")) != null ? _fa : readString(env, "OPENAI_API_KEY")
|
|
1881
2241
|
};
|
|
2242
|
+
const DEFAULT_MODEL_BY_PROVIDER = {
|
|
2243
|
+
openai: "gpt-4o",
|
|
2244
|
+
anthropic: "claude-3-5-sonnet-20241022",
|
|
2245
|
+
gemini: "gemini-2.0-flash",
|
|
2246
|
+
ollama: "llama3",
|
|
2247
|
+
universal_rest: "default",
|
|
2248
|
+
rest: "default",
|
|
2249
|
+
custom: "default"
|
|
2250
|
+
};
|
|
1882
2251
|
return {
|
|
1883
2252
|
projectId,
|
|
1884
2253
|
vectorDb: {
|
|
@@ -1888,8 +2257,8 @@ function getEnvConfig(env = process.env, base) {
|
|
|
1888
2257
|
},
|
|
1889
2258
|
llm: {
|
|
1890
2259
|
provider: llmProvider,
|
|
1891
|
-
model: (_ia = readString(env, "LLM_MODEL")) != null ? _ia : "gpt-4o",
|
|
1892
|
-
apiKey: (
|
|
2260
|
+
model: (_ja = (_ia = readString(env, "LLM_MODEL")) != null ? _ia : DEFAULT_MODEL_BY_PROVIDER[llmProvider]) != null ? _ja : "gpt-4o",
|
|
2261
|
+
apiKey: (_ka = llmApiKeyByProvider[llmProvider]) != null ? _ka : "",
|
|
1893
2262
|
baseUrl: readString(env, "LLM_BASE_URL"),
|
|
1894
2263
|
systemPrompt: readString(env, "LLM_SYSTEM_PROMPT"),
|
|
1895
2264
|
maxTokens: readNumber(env, "LLM_MAX_TOKENS", 4096),
|
|
@@ -1900,7 +2269,7 @@ function getEnvConfig(env = process.env, base) {
|
|
|
1900
2269
|
},
|
|
1901
2270
|
embedding: {
|
|
1902
2271
|
provider: embeddingProvider,
|
|
1903
|
-
model: (
|
|
2272
|
+
model: (_la = readString(env, "EMBEDDING_MODEL")) != null ? _la : "text-embedding-3-small",
|
|
1904
2273
|
apiKey: embeddingApiKeyByProvider[embeddingProvider],
|
|
1905
2274
|
baseUrl: readString(env, "EMBEDDING_BASE_URL"),
|
|
1906
2275
|
dimensions: embeddingDimensions,
|
|
@@ -1911,17 +2280,17 @@ function getEnvConfig(env = process.env, base) {
|
|
|
1911
2280
|
}
|
|
1912
2281
|
},
|
|
1913
2282
|
ui: {
|
|
1914
|
-
title: (
|
|
1915
|
-
subtitle: (
|
|
1916
|
-
primaryColor: (
|
|
1917
|
-
accentColor: (
|
|
1918
|
-
logoUrl: (
|
|
1919
|
-
placeholder: (
|
|
1920
|
-
showSources: ((
|
|
1921
|
-
welcomeMessage: (
|
|
1922
|
-
visualStyle: (
|
|
1923
|
-
borderRadius: (
|
|
1924
|
-
allowUpload: ((
|
|
2283
|
+
title: (_na = (_ma = readString(env, "NEXT_PUBLIC_UI_TITLE")) != null ? _ma : readString(env, "UI_TITLE")) != null ? _na : "AI Assistant",
|
|
2284
|
+
subtitle: (_pa = (_oa = readString(env, "NEXT_PUBLIC_UI_SUBTITLE")) != null ? _oa : readString(env, "UI_SUBTITLE")) != null ? _pa : "Powered by RAG",
|
|
2285
|
+
primaryColor: (_ra = (_qa = readString(env, "NEXT_PUBLIC_PRIMARY_COLOR")) != null ? _qa : readString(env, "UI_PRIMARY_COLOR")) != null ? _ra : "#10b981",
|
|
2286
|
+
accentColor: (_ta = (_sa = readString(env, "NEXT_PUBLIC_ACCENT_COLOR")) != null ? _sa : readString(env, "UI_ACCENT_COLOR")) != null ? _ta : "#3b82f6",
|
|
2287
|
+
logoUrl: (_ua = readString(env, "NEXT_PUBLIC_LOGO_URL")) != null ? _ua : readString(env, "UI_LOGO_URL"),
|
|
2288
|
+
placeholder: (_wa = (_va = readString(env, "NEXT_PUBLIC_PLACEHOLDER")) != null ? _va : readString(env, "UI_PLACEHOLDER")) != null ? _wa : "Ask me anything\u2026",
|
|
2289
|
+
showSources: ((_ya = (_xa = readString(env, "NEXT_PUBLIC_SHOW_SOURCES")) != null ? _xa : readString(env, "UI_SHOW_SOURCES")) != null ? _ya : "true") !== "false",
|
|
2290
|
+
welcomeMessage: (_Aa = (_za = readString(env, "NEXT_PUBLIC_WELCOME_MESSAGE")) != null ? _za : readString(env, "UI_WELCOME_MESSAGE")) != null ? _Aa : "Hello! I'm your AI assistant. Ask me anything about your documents.",
|
|
2291
|
+
visualStyle: (_Ca = (_Ba = readString(env, "NEXT_PUBLIC_UI_VISUAL_STYLE")) != null ? _Ba : readString(env, "UI_VISUAL_STYLE")) != null ? _Ca : "glass",
|
|
2292
|
+
borderRadius: (_Ea = (_Da = readString(env, "NEXT_PUBLIC_UI_BORDER_RADIUS")) != null ? _Da : readString(env, "UI_BORDER_RADIUS")) != null ? _Ea : "xl",
|
|
2293
|
+
allowUpload: ((_Ga = (_Fa = readString(env, "NEXT_PUBLIC_ALLOW_UPLOAD")) != null ? _Fa : readString(env, "UI_ALLOW_UPLOAD")) != null ? _Ga : "false") === "true"
|
|
1925
2294
|
},
|
|
1926
2295
|
rag: {
|
|
1927
2296
|
topK: readNumber(env, "RAG_TOP_K", 5),
|
|
@@ -2822,7 +3191,7 @@ var VECTOR_PROFILES = {
|
|
|
2822
3191
|
// src/llm/providers/UniversalLLMAdapter.ts
|
|
2823
3192
|
var UniversalLLMAdapter = class {
|
|
2824
3193
|
constructor(config) {
|
|
2825
|
-
var _a, _b, _c, _d, _e, _f, _g;
|
|
3194
|
+
var _a, _b, _c, _d, _e, _f, _g, _h;
|
|
2826
3195
|
this.model = config.model;
|
|
2827
3196
|
const llmConfig = config;
|
|
2828
3197
|
const options = (_a = llmConfig.options) != null ? _a : {};
|
|
@@ -2836,16 +3205,18 @@ var UniversalLLMAdapter = class {
|
|
|
2836
3205
|
this.systemPrompt = (_c = llmConfig.systemPrompt) != null ? _c : "You are a helpful AI assistant. Use the provided context to answer the user.";
|
|
2837
3206
|
this.maxTokens = (_d = llmConfig.maxTokens) != null ? _d : 1024;
|
|
2838
3207
|
this.temperature = (_e = llmConfig.temperature) != null ? _e : 0;
|
|
2839
|
-
|
|
2840
|
-
|
|
3208
|
+
this.apiKey = config.apiKey;
|
|
3209
|
+
this.baseUrl = (_g = (_f = llmConfig.baseUrl) != null ? _f : this.opts.baseUrl) != null ? _g : "";
|
|
3210
|
+
if (!this.baseUrl) {
|
|
2841
3211
|
throw new Error("[UniversalLLMAdapter] baseUrl is required in config or config.options");
|
|
2842
3212
|
}
|
|
3213
|
+
this.resolvedHeaders = __spreadValues(__spreadValues({
|
|
3214
|
+
"Content-Type": "application/json"
|
|
3215
|
+
}, config.apiKey ? { Authorization: `Bearer ${config.apiKey}` } : {}), this.opts.headers || {});
|
|
2843
3216
|
this.http = import_axios2.default.create({
|
|
2844
|
-
baseURL: baseUrl,
|
|
2845
|
-
headers:
|
|
2846
|
-
|
|
2847
|
-
}, config.apiKey ? { Authorization: `Bearer ${config.apiKey}` } : {}), this.opts.headers || {}),
|
|
2848
|
-
timeout: (_g = this.opts.timeout) != null ? _g : 6e4
|
|
3217
|
+
baseURL: this.baseUrl,
|
|
3218
|
+
headers: this.resolvedHeaders,
|
|
3219
|
+
timeout: (_h = this.opts.timeout) != null ? _h : 6e4
|
|
2849
3220
|
});
|
|
2850
3221
|
}
|
|
2851
3222
|
async chat(messages, context) {
|
|
@@ -2882,6 +3253,92 @@ ${context != null ? context : "None"}` },
|
|
|
2882
3253
|
}
|
|
2883
3254
|
return String(result);
|
|
2884
3255
|
}
|
|
3256
|
+
/**
|
|
3257
|
+
* Streaming chat using native fetch + ReadableStream.
|
|
3258
|
+
* Parses OpenAI-compatible SSE frames: `data: {...}\n\n`
|
|
3259
|
+
* Works with vLLM, LMStudio, Together AI, Fireworks, and any OpenAI-compatible API.
|
|
3260
|
+
*/
|
|
3261
|
+
chatStream(messages, context) {
|
|
3262
|
+
return __asyncGenerator(this, null, function* () {
|
|
3263
|
+
var _a, _b, _c;
|
|
3264
|
+
const path = (_a = this.opts.chatPath) != null ? _a : "/chat/completions";
|
|
3265
|
+
const url = `${this.baseUrl.replace(/\/$/, "")}${path}`;
|
|
3266
|
+
const extractPath = ((_b = this.opts.responseExtractPath) != null ? _b : "choices[0].message.content").replace("message.content", "delta.content");
|
|
3267
|
+
const formattedMessages = [
|
|
3268
|
+
{ role: "system", content: `${this.systemPrompt}
|
|
3269
|
+
|
|
3270
|
+
Context:
|
|
3271
|
+
${context != null ? context : "None"}` },
|
|
3272
|
+
...messages
|
|
3273
|
+
];
|
|
3274
|
+
let payload;
|
|
3275
|
+
if (this.opts.chatPayloadTemplate) {
|
|
3276
|
+
payload = buildPayload(this.opts.chatPayloadTemplate, {
|
|
3277
|
+
model: this.model,
|
|
3278
|
+
messages: formattedMessages,
|
|
3279
|
+
maxTokens: this.maxTokens,
|
|
3280
|
+
temperature: this.temperature
|
|
3281
|
+
});
|
|
3282
|
+
if (typeof payload === "object" && payload !== null) {
|
|
3283
|
+
payload.stream = true;
|
|
3284
|
+
}
|
|
3285
|
+
} else {
|
|
3286
|
+
payload = {
|
|
3287
|
+
model: this.model,
|
|
3288
|
+
messages: formattedMessages,
|
|
3289
|
+
max_tokens: this.maxTokens,
|
|
3290
|
+
temperature: this.temperature,
|
|
3291
|
+
stream: true
|
|
3292
|
+
};
|
|
3293
|
+
}
|
|
3294
|
+
const response = yield new __await(fetch(url, {
|
|
3295
|
+
method: "POST",
|
|
3296
|
+
headers: this.resolvedHeaders,
|
|
3297
|
+
body: JSON.stringify(payload)
|
|
3298
|
+
}));
|
|
3299
|
+
if (!response.ok) {
|
|
3300
|
+
const errorText = yield new __await(response.text().catch(() => response.statusText));
|
|
3301
|
+
throw new Error(`[UniversalLLMAdapter] Streaming request failed (${response.status}): ${errorText}`);
|
|
3302
|
+
}
|
|
3303
|
+
if (!response.body) {
|
|
3304
|
+
throw new Error("[UniversalLLMAdapter] Response body is null \u2014 server did not send a streaming response.");
|
|
3305
|
+
}
|
|
3306
|
+
const reader = response.body.getReader();
|
|
3307
|
+
const decoder = new TextDecoder("utf-8");
|
|
3308
|
+
let buffer = "";
|
|
3309
|
+
try {
|
|
3310
|
+
while (true) {
|
|
3311
|
+
const { done, value } = yield new __await(reader.read());
|
|
3312
|
+
if (done) break;
|
|
3313
|
+
buffer += decoder.decode(value, { stream: true });
|
|
3314
|
+
const lines = buffer.split("\n");
|
|
3315
|
+
buffer = (_c = lines.pop()) != null ? _c : "";
|
|
3316
|
+
for (const line of lines) {
|
|
3317
|
+
const trimmed = line.trim();
|
|
3318
|
+
if (!trimmed || trimmed === "data: [DONE]") continue;
|
|
3319
|
+
if (!trimmed.startsWith("data:")) continue;
|
|
3320
|
+
try {
|
|
3321
|
+
const json = JSON.parse(trimmed.slice(5).trim());
|
|
3322
|
+
const text = resolvePath(json, extractPath);
|
|
3323
|
+
if (text && typeof text === "string") yield text;
|
|
3324
|
+
} catch (e) {
|
|
3325
|
+
}
|
|
3326
|
+
}
|
|
3327
|
+
}
|
|
3328
|
+
if (buffer.trim() && buffer.trim() !== "data: [DONE]") {
|
|
3329
|
+
const jsonStr = buffer.replace(/^data:\s*/, "").trim();
|
|
3330
|
+
try {
|
|
3331
|
+
const json = JSON.parse(jsonStr);
|
|
3332
|
+
const text = resolvePath(json, extractPath);
|
|
3333
|
+
if (text && typeof text === "string") yield text;
|
|
3334
|
+
} catch (e) {
|
|
3335
|
+
}
|
|
3336
|
+
}
|
|
3337
|
+
} finally {
|
|
3338
|
+
reader.releaseLock();
|
|
3339
|
+
}
|
|
3340
|
+
});
|
|
3341
|
+
}
|
|
2885
3342
|
async embed(text) {
|
|
2886
3343
|
var _a, _b;
|
|
2887
3344
|
const path = (_a = this.opts.embedPath) != null ? _a : "/embeddings";
|
|
@@ -3078,6 +3535,7 @@ var ProviderRegistry = class {
|
|
|
3078
3535
|
return null;
|
|
3079
3536
|
}
|
|
3080
3537
|
static async loadVectorProviderClass(provider) {
|
|
3538
|
+
var _a;
|
|
3081
3539
|
if (this.vectorProviders[provider]) return this.vectorProviders[provider];
|
|
3082
3540
|
switch (provider) {
|
|
3083
3541
|
case "pinecone": {
|
|
@@ -3086,6 +3544,11 @@ var ProviderRegistry = class {
|
|
|
3086
3544
|
}
|
|
3087
3545
|
case "pgvector":
|
|
3088
3546
|
case "postgresql": {
|
|
3547
|
+
const postgresMode = ((_a = process.env.POSTGRES_MODE) != null ? _a : "multi").toLowerCase();
|
|
3548
|
+
if (postgresMode === "single") {
|
|
3549
|
+
const { PostgreSQLProvider: PostgreSQLProvider2 } = await Promise.resolve().then(() => (init_PostgreSQLProvider(), PostgreSQLProvider_exports));
|
|
3550
|
+
return PostgreSQLProvider2;
|
|
3551
|
+
}
|
|
3089
3552
|
const { MultiTablePostgresProvider: MultiTablePostgresProvider2 } = await Promise.resolve().then(() => (init_MultiTablePostgresProvider(), MultiTablePostgresProvider_exports));
|
|
3090
3553
|
return MultiTablePostgresProvider2;
|
|
3091
3554
|
}
|
|
@@ -4067,7 +4530,8 @@ var QueryProcessor = class {
|
|
|
4067
4530
|
const fieldValuePatterns = [
|
|
4068
4531
|
new RegExp(`\\b${fieldPattern}\\s*(?:=|:)\\s*["']?${valuePattern}["']?(?=[?.!,]|$)`, "gi"),
|
|
4069
4532
|
new RegExp(`\\b${fieldPattern}\\s+(?:is|are|was|were|equals?|equal to|named|called)\\s+["']?${valuePattern}["']?(?=[?.!,]|$)`, "gi"),
|
|
4070
|
-
new RegExp(`\\bwith\\s+${fieldPattern}\\s+["']?${valuePattern}["']?(?=[?.!,]|$)`, "gi")
|
|
4533
|
+
new RegExp(`\\bwith\\s+${fieldPattern}\\s+["']?${valuePattern}["']?(?=[?.!,]|$)`, "gi"),
|
|
4534
|
+
new RegExp(`\\b(?:under|in|for|from|of)\\s+${fieldPattern}\\s+["']?${valuePattern}["']?(?=[?.!,]|$)`, "gi")
|
|
4071
4535
|
];
|
|
4072
4536
|
for (const pattern of fieldValuePatterns) {
|
|
4073
4537
|
for (const match of question.matchAll(pattern)) {
|
|
@@ -4085,6 +4549,76 @@ var QueryProcessor = class {
|
|
|
4085
4549
|
}
|
|
4086
4550
|
return [...hints.values()];
|
|
4087
4551
|
}
|
|
4552
|
+
static extractNumericPredicates(question, validFields = []) {
|
|
4553
|
+
const predicates = [];
|
|
4554
|
+
const seen = /* @__PURE__ */ new Set();
|
|
4555
|
+
const comparatorPattern = [
|
|
4556
|
+
"greater than or equal to",
|
|
4557
|
+
"more than or equal to",
|
|
4558
|
+
"less than or equal to",
|
|
4559
|
+
"greater than",
|
|
4560
|
+
"more than",
|
|
4561
|
+
"less than",
|
|
4562
|
+
"equal to",
|
|
4563
|
+
"at least",
|
|
4564
|
+
"at most",
|
|
4565
|
+
"above",
|
|
4566
|
+
"over",
|
|
4567
|
+
"below",
|
|
4568
|
+
"under",
|
|
4569
|
+
"equals?",
|
|
4570
|
+
">=",
|
|
4571
|
+
"<=",
|
|
4572
|
+
">",
|
|
4573
|
+
"<",
|
|
4574
|
+
"="
|
|
4575
|
+
].join("|");
|
|
4576
|
+
const addPredicate = (rawField, rawOperator, rawValue) => {
|
|
4577
|
+
const value = Number(rawValue.replace(/,/g, ""));
|
|
4578
|
+
if (!Number.isFinite(value)) return;
|
|
4579
|
+
const operator = this.normalizeNumericOperator(rawOperator);
|
|
4580
|
+
const field = rawField ? this.normalizePredicateField(rawField, validFields) : void 0;
|
|
4581
|
+
const key = `${field != null ? field : "*"}::${operator}::${value}`;
|
|
4582
|
+
if (seen.has(key)) return;
|
|
4583
|
+
seen.add(key);
|
|
4584
|
+
predicates.push(__spreadProps(__spreadValues({}, field ? { field } : {}), { operator, value }));
|
|
4585
|
+
};
|
|
4586
|
+
const scopedPatterns = [
|
|
4587
|
+
new RegExp(`\\b(?:whose|with|having|where|that\\s+have|which\\s+have)\\s+([a-zA-Z][a-zA-Z0-9_\\s\\-/]{1,80}?)\\s+(?:is|are|was|were)?\\s*(?:${comparatorPattern})\\s+([\\d,]+(?:\\.\\d+)?)`, "gi"),
|
|
4588
|
+
new RegExp(`\\b([a-zA-Z][a-zA-Z0-9_\\s\\-/]{1,80}?)\\s+(?:is|are|was|were)?\\s*(?:${comparatorPattern})\\s+([\\d,]+(?:\\.\\d+)?)`, "gi")
|
|
4589
|
+
];
|
|
4590
|
+
for (const pattern of scopedPatterns) {
|
|
4591
|
+
for (const match of question.matchAll(pattern)) {
|
|
4592
|
+
const full = match[0];
|
|
4593
|
+
const operatorMatch = full.match(new RegExp(`(${comparatorPattern})`, "i"));
|
|
4594
|
+
if (!operatorMatch) continue;
|
|
4595
|
+
addPredicate(match[1], operatorMatch[1], match[2]);
|
|
4596
|
+
}
|
|
4597
|
+
}
|
|
4598
|
+
for (const match of question.matchAll(/\b([a-zA-Z][a-zA-Z0-9_\s\-/]{1,80}?)\s*(>=|<=|>|<|=)\s*([\d,]+(?:\.\d+)?)/g)) {
|
|
4599
|
+
addPredicate(match[1], match[2], match[3]);
|
|
4600
|
+
}
|
|
4601
|
+
return predicates;
|
|
4602
|
+
}
|
|
4603
|
+
static normalizeNumericOperator(operator) {
|
|
4604
|
+
const op = operator.toLowerCase().trim();
|
|
4605
|
+
if (op === ">" || /\b(greater than|more than|above|over)\b/.test(op)) return "gt";
|
|
4606
|
+
if (op === ">=" || /\b(greater than or equal to|more than or equal to|at least)\b/.test(op)) return "gte";
|
|
4607
|
+
if (op === "<" || /\b(less than|below|under)\b/.test(op)) return "lt";
|
|
4608
|
+
if (op === "<=" || /\b(less than or equal to|at most)\b/.test(op)) return "lte";
|
|
4609
|
+
return "eq";
|
|
4610
|
+
}
|
|
4611
|
+
static normalizePredicateField(field, validFields) {
|
|
4612
|
+
const cleaned = this.normalizeHintValue(field).replace(/\b(?:provide|show|get|give|list|all|the|a|an|of|organizations?|companies?|records?|items?|whose|with|having|where|that|which|have|has|is|are|was|were)\b/gi, " ").replace(/\s+/g, " ").trim();
|
|
4613
|
+
if (validFields.length === 0) return cleaned;
|
|
4614
|
+
const comparable = (value) => value.toLowerCase().replace(/[^a-z0-9]/g, "");
|
|
4615
|
+
const cleanedComparable = comparable(cleaned);
|
|
4616
|
+
const matchedField = validFields.find((fieldName) => {
|
|
4617
|
+
const candidate = comparable(fieldName);
|
|
4618
|
+
return candidate === cleanedComparable || candidate.includes(cleanedComparable) || cleanedComparable.includes(candidate);
|
|
4619
|
+
});
|
|
4620
|
+
return matchedField != null ? matchedField : cleaned;
|
|
4621
|
+
}
|
|
4088
4622
|
/**
|
|
4089
4623
|
* Constructs a QueryFilter object from extracted hints.
|
|
4090
4624
|
*
|
|
@@ -4105,6 +4639,10 @@ var QueryProcessor = class {
|
|
|
4105
4639
|
}
|
|
4106
4640
|
if (Object.keys(filter.metadata || {}).length === 0) delete filter.metadata;
|
|
4107
4641
|
if (filter.keywords && filter.keywords.length === 0) delete filter.keywords;
|
|
4642
|
+
const numericPredicates = this.extractNumericPredicates(question);
|
|
4643
|
+
if (numericPredicates.length > 0) {
|
|
4644
|
+
filter.__numericPredicates = numericPredicates;
|
|
4645
|
+
}
|
|
4108
4646
|
return filter;
|
|
4109
4647
|
}
|
|
4110
4648
|
/**
|
|
@@ -4172,102 +4710,360 @@ Return ONLY 'vector', 'graph', or 'both'. No explanation.`;
|
|
|
4172
4710
|
}
|
|
4173
4711
|
};
|
|
4174
4712
|
|
|
4175
|
-
// src/
|
|
4176
|
-
var
|
|
4177
|
-
|
|
4178
|
-
|
|
4179
|
-
|
|
4180
|
-
|
|
4181
|
-
|
|
4182
|
-
|
|
4183
|
-
|
|
4184
|
-
|
|
4185
|
-
|
|
4186
|
-
|
|
4187
|
-
|
|
4188
|
-
|
|
4189
|
-
|
|
4190
|
-
|
|
4191
|
-
|
|
4192
|
-
|
|
4193
|
-
|
|
4194
|
-
|
|
4195
|
-
|
|
4196
|
-
|
|
4197
|
-
|
|
4198
|
-
|
|
4199
|
-
|
|
4200
|
-
"
|
|
4201
|
-
|
|
4202
|
-
"
|
|
4203
|
-
|
|
4204
|
-
|
|
4205
|
-
|
|
4713
|
+
// src/core/LLMRouter.ts
|
|
4714
|
+
var FAST_MODEL_DEFAULTS = {
|
|
4715
|
+
openai: "gpt-4o-mini",
|
|
4716
|
+
gemini: "gemini-2.0-flash",
|
|
4717
|
+
anthropic: "claude-3-haiku-20240307",
|
|
4718
|
+
ollama: "",
|
|
4719
|
+
// Ollama has no universal lightweight default — reuse main model
|
|
4720
|
+
rest: "",
|
|
4721
|
+
universal_rest: "",
|
|
4722
|
+
custom: ""
|
|
4723
|
+
};
|
|
4724
|
+
var LLMRouter = class {
|
|
4725
|
+
constructor(config) {
|
|
4726
|
+
this.config = config;
|
|
4727
|
+
this.models = /* @__PURE__ */ new Map();
|
|
4728
|
+
}
|
|
4729
|
+
/**
|
|
4730
|
+
* Initialize all LLM roles.
|
|
4731
|
+
*
|
|
4732
|
+
* @param prebuiltDefault - optional pre-built provider (from EmbeddingStrategyResolver).
|
|
4733
|
+
* When provided it is used directly as the 'default' role without re-constructing.
|
|
4734
|
+
*/
|
|
4735
|
+
async initialize(prebuiltDefault) {
|
|
4736
|
+
var _a;
|
|
4737
|
+
const defaultModel = prebuiltDefault != null ? prebuiltDefault : LLMFactory.create(this.config.llm, this.config.embedding);
|
|
4738
|
+
this.models.set("default", defaultModel);
|
|
4739
|
+
const envFastModel = process.env.FAST_LLM_MODEL;
|
|
4740
|
+
const providerFastDefault = (_a = FAST_MODEL_DEFAULTS[this.config.llm.provider]) != null ? _a : "";
|
|
4741
|
+
const fastModelName = envFastModel || providerFastDefault;
|
|
4742
|
+
if (fastModelName && fastModelName !== this.config.llm.model) {
|
|
4743
|
+
console.log(`[LLMRouter] Fast role \u2192 provider="${this.config.llm.provider}" model="${fastModelName}"`);
|
|
4744
|
+
const fastConfig = __spreadProps(__spreadValues({}, this.config.llm), {
|
|
4745
|
+
model: fastModelName
|
|
4746
|
+
});
|
|
4747
|
+
this.models.set("fast", LLMFactory.create(fastConfig, this.config.embedding));
|
|
4748
|
+
} else {
|
|
4749
|
+
console.log(`[LLMRouter] Fast role \u2192 reusing default model (no lightweight alternative configured).`);
|
|
4750
|
+
this.models.set("fast", defaultModel);
|
|
4751
|
+
}
|
|
4752
|
+
this.models.set("powerful", defaultModel);
|
|
4753
|
+
}
|
|
4754
|
+
/**
|
|
4755
|
+
* Retrieve a model provider by its task role.
|
|
4756
|
+
* Falls back to 'default' if the requested role is not registered.
|
|
4757
|
+
*/
|
|
4758
|
+
get(role) {
|
|
4759
|
+
var _a;
|
|
4760
|
+
const provider = (_a = this.models.get(role)) != null ? _a : this.models.get("default");
|
|
4761
|
+
if (!provider) {
|
|
4762
|
+
throw new Error(`[LLMRouter] No provider registered for role: "${role}". Did you call initialize()?`);
|
|
4763
|
+
}
|
|
4764
|
+
return provider;
|
|
4765
|
+
}
|
|
4206
4766
|
};
|
|
4207
|
-
function resolveMetadataValue(meta, uiKey) {
|
|
4208
|
-
var _a;
|
|
4209
|
-
const synonyms = (_a = FIELD_SYNONYMS[uiKey]) != null ? _a : [];
|
|
4210
|
-
const keys = Object.keys(meta);
|
|
4211
|
-
const systemKeys = ["namespace", "filename", "filesize", "filetype", "docid", "uploadedat", "dimension", "chunkindex"];
|
|
4212
|
-
let match = keys.find((k) => k.toLowerCase() === uiKey.toLowerCase());
|
|
4213
|
-
if (match !== void 0) return meta[match];
|
|
4214
|
-
match = keys.find((k) => synonyms.map((s) => s.toLowerCase()).includes(k.toLowerCase()));
|
|
4215
|
-
if (match !== void 0) return meta[match];
|
|
4216
|
-
const isBlacklisted = (kl) => {
|
|
4217
|
-
return systemKeys.includes(kl) || kl.startsWith("option1") || kl.startsWith("option2") || kl.startsWith("option3");
|
|
4218
|
-
};
|
|
4219
|
-
match = keys.find((k) => {
|
|
4220
|
-
const kl = k.toLowerCase();
|
|
4221
|
-
if (isBlacklisted(kl)) return false;
|
|
4222
|
-
return kl.includes(uiKey.toLowerCase());
|
|
4223
|
-
});
|
|
4224
|
-
if (match !== void 0) return meta[match];
|
|
4225
|
-
match = keys.find((k) => {
|
|
4226
|
-
const kl = k.toLowerCase();
|
|
4227
|
-
if (isBlacklisted(kl)) return false;
|
|
4228
|
-
return synonyms.some((sk) => kl.includes(sk.toLowerCase()) || sk.toLowerCase().includes(kl));
|
|
4229
|
-
});
|
|
4230
|
-
return match !== void 0 ? meta[match] : void 0;
|
|
4231
|
-
}
|
|
4232
4767
|
|
|
4233
4768
|
// src/utils/UITransformer.ts
|
|
4769
|
+
init_synonyms();
|
|
4234
4770
|
var UITransformer = class {
|
|
4771
|
+
// ─── Public Entry Points ─────────────────────────────────────────────────
|
|
4235
4772
|
/**
|
|
4236
|
-
*
|
|
4237
|
-
*
|
|
4773
|
+
* Heuristic-only transform (no LLM required).
|
|
4774
|
+
* Uses the lightweight heuristic intent detector as a fallback.
|
|
4775
|
+
* Prefer `analyzeAndDecide()` in production.
|
|
4238
4776
|
*/
|
|
4239
|
-
static transform(userQuery, retrievedData, config, trainedSchema) {
|
|
4777
|
+
static transform(userQuery, retrievedData, config, trainedSchema, intent) {
|
|
4778
|
+
var _a, _b, _c;
|
|
4240
4779
|
if (!retrievedData || retrievedData.length === 0) {
|
|
4241
4780
|
return this.createTextResponse("No data available", "No relevant data found for your query.");
|
|
4242
4781
|
}
|
|
4243
|
-
const
|
|
4244
|
-
const filteredData =
|
|
4245
|
-
const
|
|
4782
|
+
const resolvedIntent = intent != null ? intent : this.detectIntentHeuristic(userQuery);
|
|
4783
|
+
const filteredData = resolvedIntent.filterInStockOnly ? retrievedData.filter((item) => this.determineStockStatus(item)) : retrievedData;
|
|
4784
|
+
const profile = this.profileData(filteredData);
|
|
4246
4785
|
const hasProducts = filteredData.some((item) => this.isProductData(item));
|
|
4247
|
-
const
|
|
4248
|
-
|
|
4249
|
-
|
|
4250
|
-
return this.transformToLineChart(filteredData);
|
|
4786
|
+
const wantsPieLikeChart = ["pie_chart", "donut_chart"].includes(resolvedIntent.recommendedChart);
|
|
4787
|
+
if (resolvedIntent.visualizationHint === "trend" && profile.dateFields.length > 0 && profile.numericFields.length > 0) {
|
|
4788
|
+
return this.transformToLineChart(profile);
|
|
4251
4789
|
}
|
|
4252
|
-
if (
|
|
4253
|
-
|
|
4790
|
+
if (wantsPieLikeChart || ["composition", "category_breakdown"].includes(resolvedIntent.visualizationHint)) {
|
|
4791
|
+
const pieChart = this.transformToPieChart(filteredData, profile, userQuery);
|
|
4792
|
+
if (pieChart) return pieChart;
|
|
4254
4793
|
}
|
|
4255
|
-
if (
|
|
4256
|
-
return this.
|
|
4794
|
+
if (["comparison", "ranking"].includes(resolvedIntent.visualizationHint)) {
|
|
4795
|
+
return this.transformToBarChart(filteredData, profile, userQuery, resolvedIntent.visualizationHint === "ranking");
|
|
4257
4796
|
}
|
|
4258
|
-
if (
|
|
4259
|
-
return this.
|
|
4797
|
+
if (resolvedIntent.visualizationHint === "distribution") {
|
|
4798
|
+
return (_a = this.transformToHistogram(profile, userQuery)) != null ? _a : this.transformToBarChart(filteredData, profile, userQuery);
|
|
4260
4799
|
}
|
|
4261
|
-
if (
|
|
4262
|
-
return this.
|
|
4800
|
+
if (resolvedIntent.visualizationHint === "correlation") {
|
|
4801
|
+
return (_b = this.transformToScatterPlot(profile, userQuery)) != null ? _b : this.transformToBarChart(filteredData, profile, userQuery);
|
|
4263
4802
|
}
|
|
4803
|
+
if (resolvedIntent.visualizationHint === "geographic") {
|
|
4804
|
+
return this.transformToBarChart(filteredData, profile, userQuery);
|
|
4805
|
+
}
|
|
4806
|
+
if (this.isStructuredListQuery(userQuery)) {
|
|
4807
|
+
return this.hasMultipleFields(filteredData) ? this.transformToTable(filteredData, userQuery) : this.transformToText(filteredData);
|
|
4808
|
+
}
|
|
4809
|
+
if (resolvedIntent.visualizationHint === "kpi") {
|
|
4810
|
+
return (_c = this.transformToMetricCard(profile, userQuery)) != null ? _c : this.transformToText(filteredData);
|
|
4811
|
+
}
|
|
4812
|
+
if (["tabular", "table"].includes(resolvedIntent.visualizationHint) || resolvedIntent.wantsExplicitTable) {
|
|
4813
|
+
return this.hasMultipleFields(filteredData) ? this.transformToTable(filteredData, userQuery) : this.transformToText(filteredData);
|
|
4814
|
+
}
|
|
4815
|
+
if ((hasProducts || this.isProductQuery(userQuery)) && resolvedIntent.visualizationHint === "product_browse") {
|
|
4816
|
+
return this.transformToProductCarousel(filteredData, config, trainedSchema);
|
|
4817
|
+
}
|
|
4818
|
+
const automatic = this.chooseAutomaticVisualization(filteredData, profile, userQuery);
|
|
4819
|
+
if (automatic) return automatic;
|
|
4264
4820
|
return this.transformToText(filteredData);
|
|
4265
4821
|
}
|
|
4266
4822
|
/**
|
|
4267
|
-
*
|
|
4823
|
+
* LLM-driven entry point (recommended for production).
|
|
4824
|
+
*
|
|
4825
|
+
* Step 1 — Detect intent via a dedicated, lightweight LLM call.
|
|
4826
|
+
* Step 2 — Pass the intent + data to the visualization-selection prompt.
|
|
4827
|
+
* Step 3 — Fall back to the heuristic `transform()` if either LLM call fails.
|
|
4828
|
+
*/
|
|
4829
|
+
static async analyzeAndDecide(query, sources, llm) {
|
|
4830
|
+
let intent;
|
|
4831
|
+
try {
|
|
4832
|
+
intent = await this.detectIntent(query, llm);
|
|
4833
|
+
console.debug("[UITransformer] Detected intent:", intent);
|
|
4834
|
+
} catch (err) {
|
|
4835
|
+
console.warn("[UITransformer] Intent detection failed; using heuristic.", err);
|
|
4836
|
+
intent = this.detectIntentHeuristic(query);
|
|
4837
|
+
}
|
|
4838
|
+
if (this.isProductQuery(query) && ["text", "product_browse"].includes(intent.visualizationHint)) {
|
|
4839
|
+
return this.transform(
|
|
4840
|
+
query,
|
|
4841
|
+
sources,
|
|
4842
|
+
void 0,
|
|
4843
|
+
void 0,
|
|
4844
|
+
__spreadProps(__spreadValues({}, intent), { visualizationHint: "product_browse", recommendedChart: "text" })
|
|
4845
|
+
);
|
|
4846
|
+
}
|
|
4847
|
+
try {
|
|
4848
|
+
const context = this.buildContextSummary(sources);
|
|
4849
|
+
const systemPrompt = this.buildVisualizationSystemPrompt();
|
|
4850
|
+
const userPrompt = [
|
|
4851
|
+
`USER QUESTION: ${query}`,
|
|
4852
|
+
"",
|
|
4853
|
+
`DETECTED INTENT (JSON): ${JSON.stringify(intent)}`,
|
|
4854
|
+
"",
|
|
4855
|
+
"RETRIEVED DATA (JSON):",
|
|
4856
|
+
context
|
|
4857
|
+
].join("\n");
|
|
4858
|
+
const rawResponse = await llm.chat(
|
|
4859
|
+
[{ role: "user", content: userPrompt }],
|
|
4860
|
+
"",
|
|
4861
|
+
{ systemPrompt, temperature: 0 }
|
|
4862
|
+
);
|
|
4863
|
+
const parsed = this.parseTransformationResponse(rawResponse);
|
|
4864
|
+
if (parsed) {
|
|
4865
|
+
const intentAllowsTable = intent.wantsExplicitTable || ["tabular", "table", "geographic"].includes(intent.visualizationHint);
|
|
4866
|
+
const intentWantsPieLikeChart = ["pie_chart", "donut_chart"].includes(intent.recommendedChart) || ["composition", "category_breakdown"].includes(intent.visualizationHint);
|
|
4867
|
+
if (parsed.type === "table" && !intentAllowsTable) {
|
|
4868
|
+
console.debug("[UITransformer] LLM chose table but intent says no. Falling back to text.");
|
|
4869
|
+
return this.transform(query, sources, void 0, void 0, intent);
|
|
4870
|
+
}
|
|
4871
|
+
if (intentWantsPieLikeChart && parsed.type !== "pie_chart" && this.detectCategories(sources).length > 1) {
|
|
4872
|
+
console.debug("[UITransformer] LLM ignored pie/composition intent. Using deterministic pie chart.");
|
|
4873
|
+
return this.transform(query, sources, void 0, void 0, intent);
|
|
4874
|
+
}
|
|
4875
|
+
console.debug("[UITransformer] LLM chose visualization type:", parsed.type);
|
|
4876
|
+
return parsed;
|
|
4877
|
+
}
|
|
4878
|
+
console.warn("[UITransformer] LLM returned unparseable response; falling back to heuristic.");
|
|
4879
|
+
} catch (err) {
|
|
4880
|
+
console.warn("[UITransformer] analyzeAndDecide LLM call failed; falling back to heuristic.", err);
|
|
4881
|
+
}
|
|
4882
|
+
return this.transform(query, sources, void 0, void 0, intent);
|
|
4883
|
+
}
|
|
4884
|
+
// ─── Dynamic Intent Detection ─────────────────────────────────────────────
|
|
4885
|
+
/**
|
|
4886
|
+
* Calls the LLM with a compact, focused prompt to extract a structured
|
|
4887
|
+
* `QueryIntent` from the user's query.
|
|
4888
|
+
*
|
|
4889
|
+
* Keeping this as a *separate* call from visualization selection means:
|
|
4890
|
+
* - The prompt is shorter and more reliable.
|
|
4891
|
+
* - The intent object can be reused across both the heuristic and LLM paths.
|
|
4892
|
+
* - It is easy to unit-test intent detection in isolation.
|
|
4893
|
+
*/
|
|
4894
|
+
static async detectIntent(query, llm) {
|
|
4895
|
+
const systemPrompt = `You are an intent classifier for a product-search RAG system.
|
|
4896
|
+
Given a user query, return ONLY a valid JSON object with this exact shape \u2014 no prose, no markdown:
|
|
4897
|
+
|
|
4898
|
+
{
|
|
4899
|
+
"visualizationHint": "trend" | "comparison" | "distribution" | "composition" | "correlation" | "ranking" | "kpi" | "tabular" | "geographic" | "product_browse" | "table" | "text",
|
|
4900
|
+
"recommendedChart": "line_chart" | "bar_chart" | "histogram" | "pie_chart" | "donut_chart" | "scatter_plot" | "horizontal_bar" | "metric_card" | "table" | "geo_map" | "text",
|
|
4901
|
+
"filterInStockOnly": boolean,
|
|
4902
|
+
"wantsExplicitTable": boolean,
|
|
4903
|
+
"isTemporal": boolean,
|
|
4904
|
+
"isComparison": boolean,
|
|
4905
|
+
"language": "<BCP-47 language tag, e.g. en, de, hi, ja>",
|
|
4906
|
+
"reasoning": "<one short sentence \u2014 why you chose these values>"
|
|
4907
|
+
}
|
|
4908
|
+
|
|
4909
|
+
RULES:
|
|
4910
|
+
- visualizationHint and recommendedChart
|
|
4911
|
+
"trend" \u2192 keywords: trend, growth, over time; recommendedChart "line_chart"
|
|
4912
|
+
"comparison" \u2192 keywords: compare, versus, top; recommendedChart "bar_chart"
|
|
4913
|
+
"distribution" \u2192 keywords: distribution, spread; recommendedChart "histogram"
|
|
4914
|
+
"composition" \u2192 keywords: share, percentage, breakup, breakdown; recommendedChart "pie_chart" or "donut_chart"
|
|
4915
|
+
"correlation" \u2192 keywords: relation, correlation; recommendedChart "scatter_plot"
|
|
4916
|
+
"ranking" \u2192 keywords: highest, lowest, top 10, ranking; recommendedChart "horizontal_bar"
|
|
4917
|
+
"kpi" \u2192 keywords: total, average, count; recommendedChart "metric_card"
|
|
4918
|
+
"tabular" \u2192 keywords: detailed records, table, grid, spreadsheet; recommendedChart "table"
|
|
4919
|
+
"geographic" \u2192 keywords: region, country, map; recommendedChart "geo_map"
|
|
4920
|
+
"product_browse" \u2192 user is browsing, searching, describing, viewing details for, or asking about one or more products without analytical visualization intent
|
|
4921
|
+
"table" \u2192 legacy alias for "tabular"; use only if the query literally says table
|
|
4922
|
+
"text" \u2192 conversational, factual, or other intent
|
|
4923
|
+
- If the user explicitly asks for a chart type, honor that chart type in recommendedChart.
|
|
4924
|
+
- If a query says "distribution ... in a pie chart", use visualizationHint "composition" and recommendedChart "pie_chart".
|
|
4925
|
+
- filterInStockOnly: true only if user mentions stock, availability, in stock, etc.
|
|
4926
|
+
- wantsExplicitTable: true if the user asks for a table, list, grid, spreadsheet, or detailed records.
|
|
4927
|
+
- isTemporal: true if time words appear (trend, historical, over time, last year, monthly, etc.)
|
|
4928
|
+
- isComparison: true if user compares, ranks, or contrasts entities or categories.
|
|
4929
|
+
- language: detect from the query text itself; default "en" if uncertain.`;
|
|
4930
|
+
const rawResponse = await llm.chat(
|
|
4931
|
+
[{ role: "user", content: `QUERY: ${query}` }],
|
|
4932
|
+
"",
|
|
4933
|
+
{ systemPrompt, temperature: 0 }
|
|
4934
|
+
);
|
|
4935
|
+
const parsed = this.parseIntentResponse(rawResponse);
|
|
4936
|
+
if (!parsed) {
|
|
4937
|
+
throw new Error(`Could not parse intent JSON from LLM response: ${rawResponse}`);
|
|
4938
|
+
}
|
|
4939
|
+
return parsed;
|
|
4940
|
+
}
|
|
4941
|
+
/**
|
|
4942
|
+
* Parse and validate the raw LLM response into a `QueryIntent`.
|
|
4268
4943
|
*/
|
|
4944
|
+
static parseIntentResponse(raw) {
|
|
4945
|
+
const jsonStr = this.extractJsonCandidate(raw);
|
|
4946
|
+
if (!jsonStr) return null;
|
|
4947
|
+
try {
|
|
4948
|
+
const obj = JSON.parse(jsonStr);
|
|
4949
|
+
const validHints = [
|
|
4950
|
+
"trend",
|
|
4951
|
+
"comparison",
|
|
4952
|
+
"distribution",
|
|
4953
|
+
"composition",
|
|
4954
|
+
"correlation",
|
|
4955
|
+
"ranking",
|
|
4956
|
+
"kpi",
|
|
4957
|
+
"tabular",
|
|
4958
|
+
"geographic",
|
|
4959
|
+
"category_breakdown",
|
|
4960
|
+
"product_browse",
|
|
4961
|
+
"table",
|
|
4962
|
+
"text"
|
|
4963
|
+
];
|
|
4964
|
+
const validCharts = [
|
|
4965
|
+
"line_chart",
|
|
4966
|
+
"bar_chart",
|
|
4967
|
+
"histogram",
|
|
4968
|
+
"pie_chart",
|
|
4969
|
+
"donut_chart",
|
|
4970
|
+
"scatter_plot",
|
|
4971
|
+
"horizontal_bar",
|
|
4972
|
+
"metric_card",
|
|
4973
|
+
"table",
|
|
4974
|
+
"geo_map",
|
|
4975
|
+
"text"
|
|
4976
|
+
];
|
|
4977
|
+
const hint = obj.visualizationHint;
|
|
4978
|
+
if (!hint || !validHints.includes(hint)) return null;
|
|
4979
|
+
const normalizedHint = hint === "category_breakdown" ? "composition" : hint;
|
|
4980
|
+
const recommendedChart = typeof obj.recommendedChart === "string" && validCharts.includes(obj.recommendedChart) ? obj.recommendedChart : this.getRecommendedChartForIntent(normalizedHint);
|
|
4981
|
+
return {
|
|
4982
|
+
visualizationHint: normalizedHint,
|
|
4983
|
+
recommendedChart,
|
|
4984
|
+
filterInStockOnly: Boolean(obj.filterInStockOnly),
|
|
4985
|
+
wantsExplicitTable: Boolean(obj.wantsExplicitTable),
|
|
4986
|
+
isTemporal: Boolean(obj.isTemporal),
|
|
4987
|
+
isComparison: Boolean(obj.isComparison),
|
|
4988
|
+
language: typeof obj.language === "string" && obj.language ? obj.language : "en",
|
|
4989
|
+
reasoning: typeof obj.reasoning === "string" ? obj.reasoning : void 0
|
|
4990
|
+
};
|
|
4991
|
+
} catch (e) {
|
|
4992
|
+
return null;
|
|
4993
|
+
}
|
|
4994
|
+
}
|
|
4995
|
+
/**
|
|
4996
|
+
* Heuristic intent detector — used when the LLM is unavailable.
|
|
4997
|
+
* Intentionally minimal: it should only catch the most obvious signals.
|
|
4998
|
+
* The LLM path handles everything subtle.
|
|
4999
|
+
*/
|
|
5000
|
+
static detectIntentHeuristic(query) {
|
|
5001
|
+
const q = query.toLowerCase();
|
|
5002
|
+
const isTemporal = /\b(trend|trends|over time|historical|history|growth|decline|monthly|yearly|weekly|daily|last (year|month|week)|timeline|forecast)\b/.test(q);
|
|
5003
|
+
const isRanking = /\b(highest|lowest|top\s*\d+|bottom\s*\d+|rank(?:ing)?|best|worst|leading)\b/.test(q);
|
|
5004
|
+
const isComparison = /\b(compare|comparison|vs\.?|versus|against|differ|difference|contrast|top)\b/.test(q) || isRanking;
|
|
5005
|
+
const isDistribution = /\b(distribution|spread|histogram|frequency|variance|range)\b/.test(q);
|
|
5006
|
+
const wantsPieLikeChart = /\b(pie|donut|doughnut)(?:\s+chart)?\b/.test(q);
|
|
5007
|
+
const isComposition = wantsPieLikeChart || /\b(share|percentage|percent|breakup|breakdown|composition|split|segmentation|by category|by type|proportion)\b/.test(q);
|
|
5008
|
+
const isCorrelation = /\b(relation|relationship|correlation|correlate|scatter|association|impact of|depend(?:s|ence)? on)\b/.test(q);
|
|
5009
|
+
const isKpi = /\b(total|average|avg|count|sum|median|minimum|maximum|metric|kpi|how many|number of)\b/.test(q);
|
|
5010
|
+
const isGeographic = /\b(region|country|countries|state|city|location|map|geo|geographic|territory)\b/.test(q);
|
|
5011
|
+
const filterInStockOnly = /\b(in[- ]?stock|available|availability|inventory|stock status)\b/.test(q);
|
|
5012
|
+
const wantsExplicitTable = /\b(table|spreadsheet|grid|detailed records|record details|list all|compare all)\b/.test(q);
|
|
5013
|
+
let visualizationHint = "text";
|
|
5014
|
+
if (wantsExplicitTable) visualizationHint = "tabular";
|
|
5015
|
+
else if (isTemporal) visualizationHint = "trend";
|
|
5016
|
+
else if (wantsPieLikeChart) visualizationHint = "composition";
|
|
5017
|
+
else if (isRanking) visualizationHint = "ranking";
|
|
5018
|
+
else if (isComparison) visualizationHint = "comparison";
|
|
5019
|
+
else if (isDistribution) visualizationHint = "distribution";
|
|
5020
|
+
else if (isComposition) visualizationHint = "composition";
|
|
5021
|
+
else if (isCorrelation) visualizationHint = "correlation";
|
|
5022
|
+
else if (isGeographic) visualizationHint = "geographic";
|
|
5023
|
+
else if (isKpi) visualizationHint = "kpi";
|
|
5024
|
+
else if (this.isProductQuery(query)) visualizationHint = "product_browse";
|
|
5025
|
+
return {
|
|
5026
|
+
visualizationHint,
|
|
5027
|
+
recommendedChart: this.getRecommendedChartForIntent(visualizationHint),
|
|
5028
|
+
filterInStockOnly,
|
|
5029
|
+
wantsExplicitTable,
|
|
5030
|
+
isTemporal,
|
|
5031
|
+
isComparison,
|
|
5032
|
+
language: "en"
|
|
5033
|
+
// heuristic cannot reliably detect language
|
|
5034
|
+
};
|
|
5035
|
+
}
|
|
5036
|
+
static getRecommendedChartForIntent(intent) {
|
|
5037
|
+
switch (intent) {
|
|
5038
|
+
case "trend":
|
|
5039
|
+
return "line_chart";
|
|
5040
|
+
case "comparison":
|
|
5041
|
+
return "bar_chart";
|
|
5042
|
+
case "distribution":
|
|
5043
|
+
return "histogram";
|
|
5044
|
+
case "composition":
|
|
5045
|
+
case "category_breakdown":
|
|
5046
|
+
return "pie_chart";
|
|
5047
|
+
case "correlation":
|
|
5048
|
+
return "scatter_plot";
|
|
5049
|
+
case "ranking":
|
|
5050
|
+
return "horizontal_bar";
|
|
5051
|
+
case "kpi":
|
|
5052
|
+
return "metric_card";
|
|
5053
|
+
case "tabular":
|
|
5054
|
+
case "table":
|
|
5055
|
+
return "table";
|
|
5056
|
+
case "geographic":
|
|
5057
|
+
return "geo_map";
|
|
5058
|
+
case "product_browse":
|
|
5059
|
+
case "text":
|
|
5060
|
+
default:
|
|
5061
|
+
return "text";
|
|
5062
|
+
}
|
|
5063
|
+
}
|
|
5064
|
+
// ─── Transform Helpers ────────────────────────────────────────────────────
|
|
4269
5065
|
static transformToProductCarousel(data, config, trainedSchema) {
|
|
4270
|
-
const products = data.
|
|
5066
|
+
const products = data.map((item) => this.extractProductInfo(item, config, trainedSchema)).filter((p) => p !== null).slice(0, 15);
|
|
4271
5067
|
return {
|
|
4272
5068
|
type: "product_carousel",
|
|
4273
5069
|
title: "Recommended Products",
|
|
@@ -4275,55 +5071,154 @@ var UITransformer = class {
|
|
|
4275
5071
|
data: products
|
|
4276
5072
|
};
|
|
4277
5073
|
}
|
|
4278
|
-
|
|
4279
|
-
|
|
4280
|
-
|
|
4281
|
-
|
|
4282
|
-
|
|
4283
|
-
|
|
5074
|
+
static transformToPieChart(data, profile, query = "") {
|
|
5075
|
+
var _a;
|
|
5076
|
+
const dimension = profile ? this.selectDimensionField(profile, query) : void 0;
|
|
5077
|
+
const categories = dimension ? Array.from(new Set(profile.records.map((record) => {
|
|
5078
|
+
var _a2;
|
|
5079
|
+
return String((_a2 = record.fields[dimension.key]) != null ? _a2 : "");
|
|
5080
|
+
}).filter(Boolean))) : this.detectCategories(data);
|
|
5081
|
+
if (categories.length === 0) return null;
|
|
5082
|
+
const categoryData = dimension && profile ? this.aggregateProfileByDimension(profile, dimension.key) : this.aggregateByCategory(data, categories);
|
|
4284
5083
|
const pieData = Object.entries(categoryData).map(([label, count]) => {
|
|
4285
5084
|
const { inStockCount, outOfStockCount } = this.calculateStockCounts(label, data);
|
|
4286
|
-
return {
|
|
4287
|
-
label,
|
|
4288
|
-
value: count,
|
|
4289
|
-
inStockCount,
|
|
4290
|
-
outOfStockCount
|
|
4291
|
-
};
|
|
5085
|
+
return { label, value: count, inStockCount, outOfStockCount };
|
|
4292
5086
|
});
|
|
4293
5087
|
return {
|
|
4294
5088
|
type: "pie_chart",
|
|
4295
|
-
title:
|
|
4296
|
-
description: `Showing breakdown across ${
|
|
5089
|
+
title: `Distribution by ${(_a = dimension == null ? void 0 : dimension.label) != null ? _a : "Category"}`,
|
|
5090
|
+
description: `Showing breakdown across ${pieData.length} categories`,
|
|
4297
5091
|
data: pieData
|
|
4298
5092
|
};
|
|
4299
5093
|
}
|
|
4300
|
-
|
|
4301
|
-
|
|
4302
|
-
|
|
4303
|
-
|
|
4304
|
-
|
|
4305
|
-
|
|
4306
|
-
timestamp:
|
|
4307
|
-
value:
|
|
4308
|
-
|
|
4309
|
-
})
|
|
5094
|
+
static transformToLineChart(profile) {
|
|
5095
|
+
const dateField = profile.dateFields[0];
|
|
5096
|
+
const valueField = profile.numericFields[0];
|
|
5097
|
+
const buckets = /* @__PURE__ */ new Map();
|
|
5098
|
+
profile.records.forEach((record) => {
|
|
5099
|
+
var _a, _b, _c;
|
|
5100
|
+
const timestamp = String((_a = record.fields[dateField.key]) != null ? _a : "");
|
|
5101
|
+
const value = (_b = this.toFiniteNumber(record.fields[valueField.key])) != null ? _b : 0;
|
|
5102
|
+
buckets.set(timestamp, ((_c = buckets.get(timestamp)) != null ? _c : 0) + value);
|
|
5103
|
+
});
|
|
5104
|
+
const lineData = Array.from(buckets.entries()).sort(([a], [b]) => Date.parse(a) - Date.parse(b)).slice(0, 24).map(([timestamp, value]) => ({ timestamp, value, label: timestamp }));
|
|
4310
5105
|
return {
|
|
4311
5106
|
type: "line_chart",
|
|
4312
|
-
title:
|
|
5107
|
+
title: `${valueField.label} Over Time`,
|
|
4313
5108
|
description: `Showing ${lineData.length} data points`,
|
|
4314
5109
|
data: lineData
|
|
4315
5110
|
};
|
|
4316
5111
|
}
|
|
4317
|
-
|
|
4318
|
-
|
|
4319
|
-
|
|
4320
|
-
|
|
4321
|
-
const
|
|
4322
|
-
const
|
|
4323
|
-
const
|
|
4324
|
-
|
|
4325
|
-
|
|
5112
|
+
static transformToBarChart(data, profile, query = "", horizontal = false) {
|
|
5113
|
+
var _a;
|
|
5114
|
+
const dimension = profile ? this.selectDimensionField(profile, query) : void 0;
|
|
5115
|
+
const measure = profile ? this.selectNumericField(profile, query) : void 0;
|
|
5116
|
+
const aggregate = dimension && profile ? this.aggregateProfileByDimension(profile, dimension.key, measure == null ? void 0 : measure.key) : this.aggregateByCategory(data, this.detectCategories(data));
|
|
5117
|
+
const barData = Object.entries(aggregate).map(([category, value]) => ({ category, value: Number(value) })).sort((a, b) => horizontal ? b.value - a.value : 0).slice(0, 12);
|
|
5118
|
+
const fallbackData = barData.length > 0 ? barData : data.slice(0, 12).map((item, index) => {
|
|
5119
|
+
var _a2, _b, _c, _d, _e;
|
|
5120
|
+
const meta = item.metadata || {};
|
|
5121
|
+
const label = String(
|
|
5122
|
+
(_c = (_b = (_a2 = this.getDynamicVal(meta, "name")) != null ? _a2 : meta.label) != null ? _b : meta.title) != null ? _c : `Result ${index + 1}`
|
|
5123
|
+
);
|
|
5124
|
+
const value = (_e = (_d = this.extractNumericValue(meta)) != null ? _d : item.score) != null ? _e : 0;
|
|
5125
|
+
return { category: label, value: Number(value) };
|
|
5126
|
+
});
|
|
5127
|
+
return {
|
|
5128
|
+
type: horizontal ? "horizontal_bar" : "bar_chart",
|
|
5129
|
+
title: dimension ? `${(_a = measure == null ? void 0 : measure.label) != null ? _a : "Count"} by ${dimension.label}` : "Comparison",
|
|
5130
|
+
description: `Showing ${fallbackData.length} comparable values`,
|
|
5131
|
+
data: fallbackData
|
|
5132
|
+
};
|
|
5133
|
+
}
|
|
5134
|
+
static transformToHistogram(profile, query = "") {
|
|
5135
|
+
const field = this.selectNumericField(profile, query);
|
|
5136
|
+
if (!field) return null;
|
|
5137
|
+
const values = profile.records.map((record) => this.toFiniteNumber(record.fields[field.key])).filter((value) => value !== null).sort((a, b) => a - b);
|
|
5138
|
+
if (values.length === 0) return null;
|
|
5139
|
+
const min = values[0];
|
|
5140
|
+
const max = values[values.length - 1];
|
|
5141
|
+
const bucketCount = Math.min(10, Math.max(3, Math.ceil(Math.sqrt(values.length))));
|
|
5142
|
+
const width = max === min ? 1 : (max - min) / bucketCount;
|
|
5143
|
+
const buckets = Array.from({ length: bucketCount }, (_, index) => {
|
|
5144
|
+
const start = min + index * width;
|
|
5145
|
+
const end = index === bucketCount - 1 ? max : start + width;
|
|
5146
|
+
return { category: `${this.formatNumber(start)}-${this.formatNumber(end)}`, value: 0 };
|
|
5147
|
+
});
|
|
5148
|
+
values.forEach((value) => {
|
|
5149
|
+
const bucketIndex = max === min ? 0 : Math.min(bucketCount - 1, Math.floor((value - min) / width));
|
|
5150
|
+
buckets[bucketIndex].value += 1;
|
|
5151
|
+
});
|
|
5152
|
+
return {
|
|
5153
|
+
type: "histogram",
|
|
5154
|
+
title: `${field.label} Distribution`,
|
|
5155
|
+
description: `Showing ${values.length} values across ${bucketCount} buckets`,
|
|
5156
|
+
data: buckets
|
|
5157
|
+
};
|
|
5158
|
+
}
|
|
5159
|
+
static transformToScatterPlot(profile, query = "") {
|
|
5160
|
+
const fields = this.rankFieldsByQuery(profile.numericFields, query).slice(0, 2);
|
|
5161
|
+
if (fields.length < 2) return null;
|
|
5162
|
+
const [xField, yField] = fields;
|
|
5163
|
+
const points = profile.records.map((record) => {
|
|
5164
|
+
var _a;
|
|
5165
|
+
const x = this.toFiniteNumber(record.fields[xField.key]);
|
|
5166
|
+
const y = this.toFiniteNumber(record.fields[yField.key]);
|
|
5167
|
+
if (x === null || y === null) return null;
|
|
5168
|
+
return { x, y, label: String((_a = this.getRecordLabel(record)) != null ? _a : record.id) };
|
|
5169
|
+
}).filter((point) => point !== null).slice(0, 100);
|
|
5170
|
+
if (points.length === 0) return null;
|
|
5171
|
+
return {
|
|
5172
|
+
type: "scatter_plot",
|
|
5173
|
+
title: `${xField.label} vs ${yField.label}`,
|
|
5174
|
+
description: `Showing ${points.length} paired values`,
|
|
5175
|
+
data: points
|
|
5176
|
+
};
|
|
5177
|
+
}
|
|
5178
|
+
static transformToMetricCard(profile, query = "") {
|
|
5179
|
+
const operation = this.detectAggregationOperation(query);
|
|
5180
|
+
const numericField = this.selectNumericField(profile, query);
|
|
5181
|
+
const values = numericField ? profile.records.map((record) => this.toFiniteNumber(record.fields[numericField.key])).filter((value2) => value2 !== null) : [];
|
|
5182
|
+
const value = operation === "count" || values.length === 0 ? profile.records.length : this.calculateAggregate(values, operation);
|
|
5183
|
+
const metric = {
|
|
5184
|
+
label: numericField ? `${operation} ${numericField.label}` : "Count",
|
|
5185
|
+
value,
|
|
5186
|
+
operation
|
|
4326
5187
|
};
|
|
5188
|
+
return {
|
|
5189
|
+
type: "metric_card",
|
|
5190
|
+
title: metric.label,
|
|
5191
|
+
description: `Calculated from ${profile.records.length} result${profile.records.length === 1 ? "" : "s"}`,
|
|
5192
|
+
data: metric
|
|
5193
|
+
};
|
|
5194
|
+
}
|
|
5195
|
+
static transformToRadarChart(data) {
|
|
5196
|
+
const attributeMap = {};
|
|
5197
|
+
data.forEach((item) => {
|
|
5198
|
+
var _a, _b, _c;
|
|
5199
|
+
const meta = item.metadata || {};
|
|
5200
|
+
const seriesName = String((_c = (_b = (_a = meta.name) != null ? _a : meta.product) != null ? _b : item.id) != null ? _c : "Item");
|
|
5201
|
+
Object.entries(meta).forEach(([key, val]) => {
|
|
5202
|
+
if (typeof val === "number" && !["price", "id"].includes(key.toLowerCase())) {
|
|
5203
|
+
if (!attributeMap[key]) attributeMap[key] = {};
|
|
5204
|
+
attributeMap[key][seriesName] = val;
|
|
5205
|
+
}
|
|
5206
|
+
});
|
|
5207
|
+
});
|
|
5208
|
+
const radarData = Object.entries(attributeMap).map(([attribute, series]) => __spreadValues({
|
|
5209
|
+
attribute
|
|
5210
|
+
}, series));
|
|
5211
|
+
return {
|
|
5212
|
+
type: "radar_chart",
|
|
5213
|
+
title: "Product Comparison",
|
|
5214
|
+
description: `Comparing ${data.length} items across ${radarData.length} attributes`,
|
|
5215
|
+
data: radarData.length > 0 ? radarData : data.map((d) => ({ attribute: d.content.substring(0, 40) }))
|
|
5216
|
+
};
|
|
5217
|
+
}
|
|
5218
|
+
static transformToTable(data, query = "") {
|
|
5219
|
+
const columns = this.extractTableColumns(data, query);
|
|
5220
|
+
const rows = data.map((item) => this.extractTableRow(item, columns));
|
|
5221
|
+
const tableData = { columns, rows };
|
|
4327
5222
|
return {
|
|
4328
5223
|
type: "table",
|
|
4329
5224
|
title: "Detailed Results",
|
|
@@ -4331,28 +5226,17 @@ var UITransformer = class {
|
|
|
4331
5226
|
data: tableData
|
|
4332
5227
|
};
|
|
4333
5228
|
}
|
|
4334
|
-
/**
|
|
4335
|
-
* Transform data to text format (fallback)
|
|
4336
|
-
*/
|
|
4337
5229
|
static transformToText(data) {
|
|
4338
|
-
const textContent = data.map((item) => item.content).join("\n\n");
|
|
4339
5230
|
return this.createTextResponse(
|
|
4340
5231
|
"Search Results",
|
|
4341
|
-
|
|
5232
|
+
data.map((item) => item.content).join("\n\n"),
|
|
4342
5233
|
`Found ${data.length} relevant results`
|
|
4343
5234
|
);
|
|
4344
5235
|
}
|
|
4345
|
-
/**
|
|
4346
|
-
* Helper: Create text response
|
|
4347
|
-
*/
|
|
4348
5236
|
static createTextResponse(title, content, description) {
|
|
4349
|
-
return {
|
|
4350
|
-
type: "text",
|
|
4351
|
-
title,
|
|
4352
|
-
description,
|
|
4353
|
-
data: { content }
|
|
4354
|
-
};
|
|
5237
|
+
return { type: "text", title, description, data: { content } };
|
|
4355
5238
|
}
|
|
5239
|
+
// ─── LLM Response Parsing ─────────────────────────────────────────────────
|
|
4356
5240
|
static parseTransformationResponse(raw) {
|
|
4357
5241
|
const payloadText = this.extractJsonCandidate(raw);
|
|
4358
5242
|
if (!payloadText) return null;
|
|
@@ -4389,9 +5273,7 @@ var UITransformer = class {
|
|
|
4389
5273
|
if (char === "{") depth += 1;
|
|
4390
5274
|
if (char === "}") {
|
|
4391
5275
|
depth -= 1;
|
|
4392
|
-
if (depth === 0)
|
|
4393
|
-
return cleaned.slice(start, i + 1);
|
|
4394
|
-
}
|
|
5276
|
+
if (depth === 0) return cleaned.slice(start, i + 1);
|
|
4395
5277
|
}
|
|
4396
5278
|
}
|
|
4397
5279
|
return null;
|
|
@@ -4399,19 +5281,16 @@ var UITransformer = class {
|
|
|
4399
5281
|
static normalizeTransformation(payload) {
|
|
4400
5282
|
var _a, _b, _c, _d, _e, _f, _g, _h, _i, _j;
|
|
4401
5283
|
if (!payload || typeof payload !== "object") return null;
|
|
4402
|
-
const
|
|
4403
|
-
const type = this.normalizeVisualizationType(
|
|
5284
|
+
const p = payload;
|
|
5285
|
+
const type = this.normalizeVisualizationType(
|
|
5286
|
+
String((_c = (_b = (_a = p.type) != null ? _a : p.view) != null ? _b : p.chartType) != null ? _c : "")
|
|
5287
|
+
);
|
|
4404
5288
|
if (!type) return null;
|
|
4405
|
-
const title = String((_e = (_d =
|
|
4406
|
-
const description =
|
|
4407
|
-
const rawData = (_j = (_i = (_h = (_g = (_f =
|
|
5289
|
+
const title = String((_e = (_d = p.title) != null ? _d : p.heading) != null ? _e : "Visualization");
|
|
5290
|
+
const description = p.description ? String(p.description) : void 0;
|
|
5291
|
+
const rawData = (_j = (_i = (_h = (_g = (_f = p.data) != null ? _f : p.table) != null ? _g : p.rows) != null ? _h : p.items) != null ? _i : p.content) != null ? _j : null;
|
|
4408
5292
|
const data = type === "text" && typeof rawData === "string" ? { content: rawData } : rawData;
|
|
4409
|
-
const transformation = {
|
|
4410
|
-
type,
|
|
4411
|
-
title,
|
|
4412
|
-
description,
|
|
4413
|
-
data
|
|
4414
|
-
};
|
|
5293
|
+
const transformation = { type, title, description, data };
|
|
4415
5294
|
return this.validateTransformation(transformation) ? transformation : null;
|
|
4416
5295
|
}
|
|
4417
5296
|
static normalizeVisualizationType(type) {
|
|
@@ -4421,10 +5300,23 @@ var UITransformer = class {
|
|
|
4421
5300
|
pie_chart: "pie_chart",
|
|
4422
5301
|
bar: "bar_chart",
|
|
4423
5302
|
bar_chart: "bar_chart",
|
|
5303
|
+
histogram: "histogram",
|
|
5304
|
+
horizontal_bar: "horizontal_bar",
|
|
5305
|
+
horizontalbar: "horizontal_bar",
|
|
4424
5306
|
line: "line_chart",
|
|
4425
5307
|
line_chart: "line_chart",
|
|
5308
|
+
scatter: "scatter_plot",
|
|
5309
|
+
scatter_plot: "scatter_plot",
|
|
5310
|
+
scatterplot: "scatter_plot",
|
|
4426
5311
|
radar: "radar_chart",
|
|
4427
5312
|
radar_chart: "radar_chart",
|
|
5313
|
+
metric: "metric_card",
|
|
5314
|
+
metric_card: "metric_card",
|
|
5315
|
+
card: "metric_card",
|
|
5316
|
+
kpi: "metric_card",
|
|
5317
|
+
geo: "geo_map",
|
|
5318
|
+
geo_map: "geo_map",
|
|
5319
|
+
map: "geo_map",
|
|
4428
5320
|
table: "table",
|
|
4429
5321
|
text: "text",
|
|
4430
5322
|
product_carousel: "product_carousel",
|
|
@@ -4432,21 +5324,31 @@ var UITransformer = class {
|
|
|
4432
5324
|
};
|
|
4433
5325
|
return (_a = mapping[type.toLowerCase()]) != null ? _a : null;
|
|
4434
5326
|
}
|
|
4435
|
-
static validateTransformation(
|
|
4436
|
-
const { type, data } =
|
|
5327
|
+
static validateTransformation(t) {
|
|
5328
|
+
const { type, data } = t;
|
|
4437
5329
|
switch (type) {
|
|
4438
5330
|
case "pie_chart":
|
|
4439
5331
|
case "bar_chart":
|
|
5332
|
+
case "histogram":
|
|
5333
|
+
case "horizontal_bar":
|
|
4440
5334
|
return Array.isArray(data) && data.every(
|
|
4441
|
-
(
|
|
5335
|
+
(i) => i !== null && typeof i === "object" && (typeof i.value === "number" || !Number.isNaN(Number(i.value)))
|
|
5336
|
+
);
|
|
5337
|
+
case "scatter_plot":
|
|
5338
|
+
return Array.isArray(data) && data.every(
|
|
5339
|
+
(i) => i !== null && typeof i === "object" && (typeof i.x === "number" || !Number.isNaN(Number(i.x))) && (typeof i.y === "number" || !Number.isNaN(Number(i.y)))
|
|
4442
5340
|
);
|
|
4443
5341
|
case "line_chart":
|
|
4444
5342
|
return Array.isArray(data) && data.every(
|
|
4445
|
-
(
|
|
5343
|
+
(i) => i !== null && typeof i === "object" && "timestamp" in i && (typeof i.value === "number" || !Number.isNaN(Number(i.value)))
|
|
4446
5344
|
);
|
|
5345
|
+
case "metric_card":
|
|
5346
|
+
return typeof data === "object" && data !== null && (typeof data.value === "number" || !Number.isNaN(Number(data.value)));
|
|
5347
|
+
case "geo_map":
|
|
5348
|
+
return Array.isArray(data) || typeof data === "object" && data !== null;
|
|
4447
5349
|
case "radar_chart":
|
|
4448
5350
|
return Array.isArray(data) && data.every(
|
|
4449
|
-
(
|
|
5351
|
+
(i) => i !== null && typeof i === "object" && "attribute" in i
|
|
4450
5352
|
);
|
|
4451
5353
|
case "table":
|
|
4452
5354
|
return typeof data === "object" && data !== null && Array.isArray(data.columns) && Array.isArray(data.rows);
|
|
@@ -4455,15 +5357,27 @@ var UITransformer = class {
|
|
|
4455
5357
|
case "product_carousel":
|
|
4456
5358
|
case "carousel":
|
|
4457
5359
|
return Array.isArray(data) && data.every(
|
|
4458
|
-
(
|
|
5360
|
+
(i) => i !== null && typeof i === "object" && ("id" in i || "name" in i)
|
|
4459
5361
|
);
|
|
4460
5362
|
default:
|
|
4461
5363
|
return false;
|
|
4462
5364
|
}
|
|
4463
5365
|
}
|
|
4464
|
-
|
|
4465
|
-
|
|
4466
|
-
|
|
5366
|
+
// ─── Data Inspection Helpers ──────────────────────────────────────────────
|
|
5367
|
+
static isStructuredListQuery(query) {
|
|
5368
|
+
const q = query.toLowerCase();
|
|
5369
|
+
const asksForRecords = /\b(list|provide|show|give|get|find|which|whose|records?|details?)\b/.test(q);
|
|
5370
|
+
const hasNumericPredicate = /\b(greater than|more than|above|over|less than|below|under|at least|at most|minimum|maximum|>=|<=|>|<|equals?|equal to)\b\s*\$?\d/i.test(q) || /\b\d+\b\s*(?:or more|or less|and above|and below)\b/i.test(q);
|
|
5371
|
+
const hasEntityTerms = /\b(organization|organisations?|organizations?|company|companies|employee|employees|staff|headcount|workforce)\b/.test(q);
|
|
5372
|
+
return asksForRecords && (hasNumericPredicate || hasEntityTerms);
|
|
5373
|
+
}
|
|
5374
|
+
static isProductQuery(query) {
|
|
5375
|
+
const q = query.toLowerCase();
|
|
5376
|
+
const productTerms = /\b(product|products|item|items|sku|catalog|catalogue|price|prices|brand|model|stock|inventory|in stock|out of stock|buy|shop)\b/.test(q);
|
|
5377
|
+
const productAction = /\b(recommend|suggest|describe|description|detail|details|about|show|find|search|browse|list)\b/.test(q);
|
|
5378
|
+
const nonProductEntityTerms = /\b(organization|organisations?|organizations?|company|companies|employee|employees|staff|headcount|workforce)\b/.test(q);
|
|
5379
|
+
return productTerms && productAction && !nonProductEntityTerms;
|
|
5380
|
+
}
|
|
4467
5381
|
static isProductData(item) {
|
|
4468
5382
|
const content = (item.content || "").toLowerCase();
|
|
4469
5383
|
const productKeywords = [
|
|
@@ -4489,360 +5403,582 @@ var UITransformer = class {
|
|
|
4489
5403
|
"fragrance"
|
|
4490
5404
|
];
|
|
4491
5405
|
const hasKeywords = productKeywords.some((kw) => content.includes(kw));
|
|
4492
|
-
const
|
|
4493
|
-
|
|
4494
|
-
|
|
4495
|
-
|
|
5406
|
+
const metadata = item.metadata || {};
|
|
5407
|
+
const hasMetadataKey = Object.keys(metadata).some((k) => {
|
|
5408
|
+
const val = metadata[k];
|
|
5409
|
+
return ["price", "product", "sku", "brand", "model", "cost", "item"].includes(k.toLowerCase()) && val !== null && val !== void 0 && val !== "";
|
|
5410
|
+
});
|
|
5411
|
+
const hasPricePattern = /\$\s*\d+\.\d{2}/.test(content);
|
|
4496
5412
|
return hasKeywords || hasMetadataKey || hasPricePattern;
|
|
4497
5413
|
}
|
|
4498
|
-
/**
|
|
4499
|
-
* Helper: Check if data contains time series
|
|
4500
|
-
*/
|
|
4501
5414
|
static isTimeSeriesData(item) {
|
|
4502
|
-
const content = (item.content || "").toLowerCase();
|
|
4503
|
-
const timeKeywords = ["trend", "historical", "growth", "decline", "change", "increase", "decrease"];
|
|
4504
|
-
const hasTimeKeyword = timeKeywords.some((kw) => content.includes(kw));
|
|
4505
5415
|
const metadata = item.metadata || {};
|
|
4506
5416
|
const maybeDateKeys = Object.keys(metadata).filter(
|
|
4507
5417
|
(k) => ["date", "timestamp", "time", "period"].includes(k.toLowerCase())
|
|
4508
5418
|
);
|
|
4509
|
-
|
|
5419
|
+
return maybeDateKeys.some((key) => {
|
|
4510
5420
|
const value = metadata[key];
|
|
4511
|
-
if (typeof value === "string")
|
|
4512
|
-
return !Number.isNaN(Date.parse(value)) && value.trim().length > 0;
|
|
4513
|
-
}
|
|
5421
|
+
if (typeof value === "string") return !Number.isNaN(Date.parse(value)) && value.trim().length > 0;
|
|
4514
5422
|
return typeof value === "number";
|
|
4515
5423
|
});
|
|
4516
|
-
return hasTimeKeyword || hasValidDateValue;
|
|
4517
5424
|
}
|
|
4518
|
-
static
|
|
4519
|
-
|
|
4520
|
-
|
|
4521
|
-
|
|
4522
|
-
const normalized = query.toLowerCase();
|
|
4523
|
-
const chartKeywords = [
|
|
4524
|
-
"distribution",
|
|
4525
|
-
"breakdown",
|
|
4526
|
-
"by category",
|
|
4527
|
-
"by type",
|
|
4528
|
-
"compare",
|
|
4529
|
-
"share",
|
|
4530
|
-
"percentage",
|
|
4531
|
-
"segmentation",
|
|
4532
|
-
"split",
|
|
4533
|
-
"category breakdown",
|
|
4534
|
-
"category distribution"
|
|
4535
|
-
];
|
|
4536
|
-
return chartKeywords.some((keyword) => normalized.includes(keyword));
|
|
5425
|
+
static hasMultipleFields(data) {
|
|
5426
|
+
const fieldCount = /* @__PURE__ */ new Set();
|
|
5427
|
+
data.forEach((item) => Object.keys(item.metadata || {}).forEach((k) => fieldCount.add(k)));
|
|
5428
|
+
return fieldCount.size > 2;
|
|
4537
5429
|
}
|
|
4538
|
-
static
|
|
4539
|
-
const
|
|
4540
|
-
|
|
4541
|
-
|
|
4542
|
-
|
|
4543
|
-
|
|
4544
|
-
|
|
4545
|
-
|
|
4546
|
-
|
|
4547
|
-
|
|
4548
|
-
|
|
4549
|
-
|
|
4550
|
-
|
|
4551
|
-
|
|
4552
|
-
|
|
4553
|
-
|
|
4554
|
-
|
|
4555
|
-
|
|
4556
|
-
|
|
4557
|
-
|
|
5430
|
+
static profileData(data) {
|
|
5431
|
+
const records = data.map((item) => {
|
|
5432
|
+
const fields2 = {};
|
|
5433
|
+
Object.entries(item.metadata || {}).forEach(([key, value]) => {
|
|
5434
|
+
const primitive = this.toPrimitive(value);
|
|
5435
|
+
if (primitive !== null) fields2[key] = primitive;
|
|
5436
|
+
});
|
|
5437
|
+
if (!fields2.content && item.content) fields2.content = item.content.substring(0, 500);
|
|
5438
|
+
return {
|
|
5439
|
+
id: item.id,
|
|
5440
|
+
content: item.content,
|
|
5441
|
+
score: item.score,
|
|
5442
|
+
fields: fields2,
|
|
5443
|
+
source: item
|
|
5444
|
+
};
|
|
5445
|
+
});
|
|
5446
|
+
const keys = Array.from(new Set(records.flatMap((record) => Object.keys(record.fields))));
|
|
5447
|
+
const fields = keys.map((key) => {
|
|
5448
|
+
const values = records.map((record) => record.fields[key]).filter((value) => value !== void 0 && value !== null && String(value).trim() !== "");
|
|
5449
|
+
const uniqueCount = new Set(values.map((value) => String(value).toLowerCase())).size;
|
|
5450
|
+
return {
|
|
5451
|
+
key,
|
|
5452
|
+
label: this.humanizeFieldName(key),
|
|
5453
|
+
kind: this.inferFieldKind(key, values, records.length, uniqueCount),
|
|
5454
|
+
values,
|
|
5455
|
+
uniqueCount
|
|
5456
|
+
};
|
|
5457
|
+
});
|
|
5458
|
+
return {
|
|
5459
|
+
records,
|
|
5460
|
+
fields,
|
|
5461
|
+
numericFields: fields.filter((field) => field.kind === "number"),
|
|
5462
|
+
dateFields: fields.filter((field) => field.kind === "date"),
|
|
5463
|
+
categoricalFields: fields.filter((field) => field.kind === "category"),
|
|
5464
|
+
booleanFields: fields.filter((field) => field.kind === "boolean")
|
|
5465
|
+
};
|
|
4558
5466
|
}
|
|
4559
|
-
static
|
|
4560
|
-
|
|
4561
|
-
|
|
5467
|
+
static toPrimitive(value) {
|
|
5468
|
+
if (typeof value === "string" || typeof value === "number" || typeof value === "boolean") return value;
|
|
5469
|
+
if (value === null || value === void 0) return null;
|
|
5470
|
+
if (value instanceof Date) return value.toISOString();
|
|
5471
|
+
return null;
|
|
4562
5472
|
}
|
|
4563
|
-
|
|
4564
|
-
|
|
4565
|
-
|
|
4566
|
-
|
|
4567
|
-
|
|
4568
|
-
if (!meta) return void 0;
|
|
4569
|
-
const mapping = (_a = config == null ? void 0 : config.rag) == null ? void 0 : _a.uiMapping;
|
|
4570
|
-
if (mapping && mapping[uiKey]) {
|
|
4571
|
-
const mappedKey = mapping[uiKey];
|
|
4572
|
-
if (meta[mappedKey] !== void 0) return meta[mappedKey];
|
|
5473
|
+
static inferFieldKind(key, values, rowCount, uniqueCount) {
|
|
5474
|
+
const lowerKey = key.toLowerCase();
|
|
5475
|
+
if (values.length === 0) return "text";
|
|
5476
|
+
if (values.every((value) => typeof value === "boolean" || /^(true|false|yes|no|in_stock|out_of_stock|available|unavailable)$/i.test(String(value)))) {
|
|
5477
|
+
return "boolean";
|
|
4573
5478
|
}
|
|
4574
|
-
if (
|
|
4575
|
-
|
|
4576
|
-
if (trainedKey && meta[trainedKey] !== void 0) return meta[trainedKey];
|
|
5479
|
+
if (/(date|time|timestamp|created|updated|month|year|period)/i.test(lowerKey) && values.some((value) => this.isDateLike(value))) {
|
|
5480
|
+
return "date";
|
|
4577
5481
|
}
|
|
4578
|
-
|
|
5482
|
+
const numericCount = values.filter((value) => this.toFiniteNumber(value) !== null).length;
|
|
5483
|
+
if (numericCount / values.length >= 0.85 && !/(id|sku|ean|upc|zip|postal|phone|code)/i.test(lowerKey)) {
|
|
5484
|
+
return "number";
|
|
5485
|
+
}
|
|
5486
|
+
if (uniqueCount <= Math.max(20, Math.ceil(rowCount * 0.7)) && !/(id|sku|ean|upc|description|content|url|image|thumbnail)/i.test(lowerKey)) {
|
|
5487
|
+
return "category";
|
|
5488
|
+
}
|
|
5489
|
+
return "text";
|
|
4579
5490
|
}
|
|
4580
|
-
static
|
|
4581
|
-
|
|
4582
|
-
const
|
|
4583
|
-
|
|
4584
|
-
|
|
4585
|
-
|
|
4586
|
-
|
|
4587
|
-
|
|
4588
|
-
|
|
4589
|
-
|
|
4590
|
-
|
|
4591
|
-
|
|
4592
|
-
|
|
4593
|
-
|
|
4594
|
-
|
|
4595
|
-
|
|
4596
|
-
|
|
4597
|
-
return
|
|
4598
|
-
|
|
4599
|
-
|
|
4600
|
-
|
|
4601
|
-
image: typeof imageValue === "string" ? imageValue : void 0,
|
|
4602
|
-
brand: brand ? String(brand) : void 0,
|
|
4603
|
-
description: item.content,
|
|
4604
|
-
inStock: this.determineStockStatus(item)
|
|
4605
|
-
};
|
|
5491
|
+
static isDateLike(value) {
|
|
5492
|
+
if (typeof value === "number") return value > 1900 && value < 3e3;
|
|
5493
|
+
const text = String(value).trim();
|
|
5494
|
+
return text.length > 3 && !Number.isNaN(Date.parse(text));
|
|
5495
|
+
}
|
|
5496
|
+
static chooseAutomaticVisualization(data, profile, query) {
|
|
5497
|
+
if (profile.records.length === 0) return null;
|
|
5498
|
+
if (profile.dateFields.length > 0 && profile.numericFields.length > 0) {
|
|
5499
|
+
return this.transformToLineChart(profile);
|
|
5500
|
+
}
|
|
5501
|
+
if (profile.categoricalFields.length > 0 && profile.numericFields.length > 0) {
|
|
5502
|
+
return this.transformToBarChart(data, profile, query);
|
|
5503
|
+
}
|
|
5504
|
+
if (profile.categoricalFields.length > 0) {
|
|
5505
|
+
return this.transformToPieChart(data, profile, query);
|
|
5506
|
+
}
|
|
5507
|
+
if (profile.numericFields.length >= 2) {
|
|
5508
|
+
return this.transformToScatterPlot(profile, query);
|
|
5509
|
+
}
|
|
5510
|
+
if (profile.numericFields.length === 1) {
|
|
5511
|
+
return this.transformToMetricCard(profile, query);
|
|
4606
5512
|
}
|
|
4607
5513
|
return null;
|
|
4608
5514
|
}
|
|
4609
|
-
|
|
4610
|
-
|
|
4611
|
-
|
|
5515
|
+
static selectDimensionField(profile, query) {
|
|
5516
|
+
var _a, _b;
|
|
5517
|
+
const productCategory = profile.categoricalFields.find(
|
|
5518
|
+
(field) => /category|department|collection|type|group|segment|region|country|state|city/i.test(field.key)
|
|
5519
|
+
);
|
|
5520
|
+
const ranked = this.rankFieldsByQuery(profile.categoricalFields, query);
|
|
5521
|
+
return (_b = (_a = ranked[0]) != null ? _a : productCategory) != null ? _b : profile.categoricalFields[0];
|
|
5522
|
+
}
|
|
5523
|
+
static selectNumericField(profile, query) {
|
|
5524
|
+
var _a;
|
|
5525
|
+
return (_a = this.rankFieldsByQuery(profile.numericFields, query)[0]) != null ? _a : profile.numericFields[0];
|
|
5526
|
+
}
|
|
5527
|
+
static rankFieldsByQuery(fields, query) {
|
|
5528
|
+
const q = query.toLowerCase();
|
|
5529
|
+
return [...fields].sort((a, b) => this.fieldScore(b, q) - this.fieldScore(a, q));
|
|
5530
|
+
}
|
|
5531
|
+
static fieldScore(field, query) {
|
|
5532
|
+
const key = field.key.toLowerCase();
|
|
5533
|
+
const label = field.label.toLowerCase();
|
|
5534
|
+
let score = 0;
|
|
5535
|
+
if (query.includes(key)) score += 4;
|
|
5536
|
+
if (query.includes(label)) score += 4;
|
|
5537
|
+
key.split(/[_\s-]+/).forEach((part) => {
|
|
5538
|
+
if (part && query.includes(part)) score += 1;
|
|
5539
|
+
});
|
|
5540
|
+
if (/category|department|collection|type|group|segment/.test(key)) score += 2;
|
|
5541
|
+
if (/count|total|quantity|stock|inventory|sales|revenue|amount|price|value|score/.test(key)) score += 2;
|
|
5542
|
+
if (/id|sku|ean|upc|code/.test(key)) score -= 5;
|
|
5543
|
+
return score;
|
|
5544
|
+
}
|
|
5545
|
+
static normalizeComparableField(field) {
|
|
5546
|
+
return field.toLowerCase().replace(/&/g, "and").replace(/[^a-z0-9]+/g, "").trim();
|
|
5547
|
+
}
|
|
5548
|
+
static fieldTokens(field) {
|
|
5549
|
+
return field.toLowerCase().replace(/[_-]+/g, " ").split(/\s+/).map((token) => token.replace(/[^a-z0-9]/g, "")).filter(Boolean);
|
|
5550
|
+
}
|
|
5551
|
+
static aggregateProfileByDimension(profile, dimensionKey, measureKey) {
|
|
5552
|
+
const result = {};
|
|
5553
|
+
profile.records.forEach((record) => {
|
|
5554
|
+
var _a, _b, _c;
|
|
5555
|
+
const category = String((_a = record.fields[dimensionKey]) != null ? _a : "Other").trim() || "Other";
|
|
5556
|
+
const value = measureKey ? (_b = this.toFiniteNumber(record.fields[measureKey])) != null ? _b : 0 : 1;
|
|
5557
|
+
result[category] = ((_c = result[category]) != null ? _c : 0) + value;
|
|
5558
|
+
});
|
|
5559
|
+
return result;
|
|
5560
|
+
}
|
|
5561
|
+
static getRecordLabel(record) {
|
|
5562
|
+
const labelKeys = ["name", "title", "label", "product", "item", "brand"];
|
|
5563
|
+
const key = Object.keys(record.fields).find(
|
|
5564
|
+
(fieldKey) => labelKeys.some((labelKey) => fieldKey.toLowerCase().includes(labelKey))
|
|
5565
|
+
);
|
|
5566
|
+
return key ? record.fields[key] : void 0;
|
|
5567
|
+
}
|
|
5568
|
+
static detectAggregationOperation(query) {
|
|
5569
|
+
const q = query.toLowerCase();
|
|
5570
|
+
if (/\b(avg|average|mean)\b/.test(q)) return "average";
|
|
5571
|
+
if (/\b(count|how many|number of)\b/.test(q)) return "count";
|
|
5572
|
+
if (/\b(min|minimum|lowest)\b/.test(q)) return "min";
|
|
5573
|
+
if (/\b(max|maximum|highest)\b/.test(q)) return "max";
|
|
5574
|
+
if (/\bmedian\b/.test(q)) return "median";
|
|
5575
|
+
return "sum";
|
|
5576
|
+
}
|
|
5577
|
+
static calculateAggregate(values, operation) {
|
|
5578
|
+
if (values.length === 0) return 0;
|
|
5579
|
+
switch (operation) {
|
|
5580
|
+
case "average":
|
|
5581
|
+
return values.reduce((sum, value) => sum + value, 0) / values.length;
|
|
5582
|
+
case "count":
|
|
5583
|
+
return values.length;
|
|
5584
|
+
case "min":
|
|
5585
|
+
return Math.min(...values);
|
|
5586
|
+
case "max":
|
|
5587
|
+
return Math.max(...values);
|
|
5588
|
+
case "median": {
|
|
5589
|
+
const sorted = [...values].sort((a, b) => a - b);
|
|
5590
|
+
const middle = Math.floor(sorted.length / 2);
|
|
5591
|
+
return sorted.length % 2 === 0 ? (sorted[middle - 1] + sorted[middle]) / 2 : sorted[middle];
|
|
5592
|
+
}
|
|
5593
|
+
case "sum":
|
|
5594
|
+
default:
|
|
5595
|
+
return values.reduce((sum, value) => sum + value, 0);
|
|
5596
|
+
}
|
|
5597
|
+
}
|
|
5598
|
+
static humanizeFieldName(key) {
|
|
5599
|
+
return key.replace(/[_-]+/g, " ").replace(/([a-z])([A-Z])/g, "$1 $2").replace(/\s+/g, " ").trim().replace(/\b\w/g, (char) => char.toUpperCase());
|
|
5600
|
+
}
|
|
5601
|
+
static formatNumber(value) {
|
|
5602
|
+
return Number.isInteger(value) ? String(value) : value.toFixed(1);
|
|
5603
|
+
}
|
|
4612
5604
|
static detectCategories(data) {
|
|
4613
5605
|
const categories = /* @__PURE__ */ new Set();
|
|
4614
5606
|
data.forEach((item) => {
|
|
4615
|
-
const
|
|
4616
|
-
if (
|
|
4617
|
-
categories.add(String(meta.category));
|
|
4618
|
-
}
|
|
4619
|
-
if (meta.type) {
|
|
4620
|
-
categories.add(String(meta.type));
|
|
4621
|
-
}
|
|
4622
|
-
if (meta.tag) {
|
|
4623
|
-
const tags = Array.isArray(meta.tag) ? meta.tag : [meta.tag];
|
|
4624
|
-
tags.forEach((t) => categories.add(String(t)));
|
|
4625
|
-
}
|
|
4626
|
-
const contentCategories = Array.from(new Set(
|
|
4627
|
-
Array.from(item.content.matchAll(/^\s*([^:\n]+):\s*(?:•|\*|-)*/gm)).map((match) => match[1].trim()).filter(Boolean)
|
|
4628
|
-
));
|
|
4629
|
-
contentCategories.forEach((category) => categories.add(category));
|
|
4630
|
-
const categoryMatch = item.content.match(/(?:Category|Type|Class):\s*([^\n]+)/i);
|
|
4631
|
-
if (categoryMatch) {
|
|
4632
|
-
categories.add(categoryMatch[1].trim());
|
|
4633
|
-
}
|
|
5607
|
+
const category = this.getProductCategory(item);
|
|
5608
|
+
if (category) categories.add(category);
|
|
4634
5609
|
});
|
|
4635
5610
|
return Array.from(categories);
|
|
4636
5611
|
}
|
|
4637
|
-
|
|
4638
|
-
|
|
4639
|
-
|
|
5612
|
+
static getProductCategory(item) {
|
|
5613
|
+
const meta = item.metadata || {};
|
|
5614
|
+
const metadataCategory = resolveMetadataValue(meta, "category");
|
|
5615
|
+
if (this.isUsableCategory(metadataCategory)) return String(metadataCategory).trim();
|
|
5616
|
+
const content = item.content || "";
|
|
5617
|
+
const explicitCategoryMatch = content.match(
|
|
5618
|
+
/(?:^|\n)\s*(?:product\s*)?(?:category|department|collection)\s*[:\-–]\s*([^\n]+)/i
|
|
5619
|
+
);
|
|
5620
|
+
if (explicitCategoryMatch && this.isUsableCategory(explicitCategoryMatch[1])) {
|
|
5621
|
+
return explicitCategoryMatch[1].trim();
|
|
5622
|
+
}
|
|
5623
|
+
return null;
|
|
5624
|
+
}
|
|
5625
|
+
static isUsableCategory(value) {
|
|
5626
|
+
if (value === null || value === void 0) return false;
|
|
5627
|
+
const category = String(value).trim();
|
|
5628
|
+
if (!category) return false;
|
|
5629
|
+
return !/^(category|type|tag|price|cost|stock|availability|description|product|item|name|brand|sku|id|ean|currency|color|size|index|internal id|other|\[?type\]?|\[?products?_?\d*\]?)$/i.test(category);
|
|
5630
|
+
}
|
|
4640
5631
|
static aggregateByCategory(data, categories) {
|
|
4641
|
-
const result =
|
|
4642
|
-
categories.forEach((cat) => {
|
|
4643
|
-
result[cat] = 0;
|
|
4644
|
-
});
|
|
5632
|
+
const result = Object.fromEntries(categories.map((c) => [c, 0]));
|
|
4645
5633
|
data.forEach((item) => {
|
|
4646
|
-
|
|
4647
|
-
const
|
|
4648
|
-
if (Object.prototype.hasOwnProperty.call(result,
|
|
4649
|
-
|
|
4650
|
-
} else {
|
|
4651
|
-
result["Other"] = (result["Other"] || 0) + 1;
|
|
4652
|
-
}
|
|
5634
|
+
var _a;
|
|
5635
|
+
const cat = (_a = this.getProductCategory(item)) != null ? _a : "Other";
|
|
5636
|
+
if (Object.prototype.hasOwnProperty.call(result, cat)) result[cat]++;
|
|
5637
|
+
else result["Other"] = (result["Other"] || 0) + 1;
|
|
4653
5638
|
});
|
|
4654
5639
|
return result;
|
|
4655
5640
|
}
|
|
4656
|
-
/**
|
|
4657
|
-
* Helper: Extract time series data
|
|
4658
|
-
*/
|
|
4659
5641
|
static extractTimeSeriesData(data) {
|
|
4660
5642
|
return data.map((item) => {
|
|
5643
|
+
var _a, _b, _c, _d, _e;
|
|
4661
5644
|
const meta = item.metadata || {};
|
|
4662
5645
|
return {
|
|
4663
|
-
timestamp: meta.timestamp
|
|
4664
|
-
value: meta.value
|
|
4665
|
-
label: meta.label
|
|
5646
|
+
timestamp: (_b = (_a = meta.timestamp) != null ? _a : meta.date) != null ? _b : (/* @__PURE__ */ new Date()).toISOString(),
|
|
5647
|
+
value: (_d = (_c = meta.value) != null ? _c : item.score) != null ? _d : 0,
|
|
5648
|
+
label: (_e = meta.label) != null ? _e : item.content.substring(0, 50)
|
|
4666
5649
|
};
|
|
4667
5650
|
});
|
|
4668
5651
|
}
|
|
4669
|
-
|
|
4670
|
-
|
|
4671
|
-
|
|
4672
|
-
|
|
4673
|
-
|
|
4674
|
-
|
|
4675
|
-
|
|
4676
|
-
|
|
4677
|
-
|
|
4678
|
-
|
|
4679
|
-
|
|
4680
|
-
|
|
5652
|
+
static extractNumericValue(meta) {
|
|
5653
|
+
var _a;
|
|
5654
|
+
const preferredKeys = [
|
|
5655
|
+
"value",
|
|
5656
|
+
"count",
|
|
5657
|
+
"total",
|
|
5658
|
+
"average",
|
|
5659
|
+
"avg",
|
|
5660
|
+
"amount",
|
|
5661
|
+
"sales",
|
|
5662
|
+
"revenue",
|
|
5663
|
+
"score",
|
|
5664
|
+
"quantity",
|
|
5665
|
+
"price"
|
|
5666
|
+
];
|
|
5667
|
+
for (const key of preferredKeys) {
|
|
5668
|
+
const raw = (_a = resolveMetadataValue(meta, key)) != null ? _a : meta[key];
|
|
5669
|
+
const value = typeof raw === "number" ? raw : Number(String(raw != null ? raw : "").replace(/[$,% ,]/g, ""));
|
|
5670
|
+
if (Number.isFinite(value)) return value;
|
|
5671
|
+
}
|
|
5672
|
+
for (const value of Object.values(meta)) {
|
|
5673
|
+
const numeric = typeof value === "number" ? value : Number(String(value != null ? value : "").replace(/[$,% ,]/g, ""));
|
|
5674
|
+
if (Number.isFinite(numeric)) return numeric;
|
|
5675
|
+
}
|
|
5676
|
+
return null;
|
|
5677
|
+
}
|
|
5678
|
+
static extractTableColumns(data, query = "") {
|
|
5679
|
+
const q = query.toLowerCase();
|
|
5680
|
+
const availableFields = this.extractAvailableTableFields(data);
|
|
5681
|
+
if (/\b(organization|organisations?|organizations?|company|companies)\b/.test(q) && /\b(employee|employees|staff|headcount|workforce)\b/.test(q)) {
|
|
5682
|
+
const nameField = this.pickTableField(availableFields, [
|
|
5683
|
+
"company name",
|
|
5684
|
+
"organization name",
|
|
5685
|
+
"organisation name",
|
|
5686
|
+
"name",
|
|
5687
|
+
"company",
|
|
5688
|
+
"organization",
|
|
5689
|
+
"organisation"
|
|
5690
|
+
]);
|
|
5691
|
+
const employeeField = this.pickTableField(availableFields, [
|
|
5692
|
+
"number of employees",
|
|
5693
|
+
"employee count",
|
|
5694
|
+
"employees",
|
|
5695
|
+
"employee_count",
|
|
5696
|
+
"number_employees",
|
|
5697
|
+
"num employees",
|
|
5698
|
+
"staff count",
|
|
5699
|
+
"headcount",
|
|
5700
|
+
"workforce"
|
|
5701
|
+
]);
|
|
5702
|
+
const columns = [nameField, employeeField].filter((field) => Boolean(field));
|
|
5703
|
+
if (columns.length > 0) return columns;
|
|
5704
|
+
}
|
|
5705
|
+
const requestedFields = availableFields.map((field) => ({
|
|
5706
|
+
field,
|
|
5707
|
+
score: this.tableFieldQueryScore(field, q)
|
|
5708
|
+
})).filter((item) => item.score > 0).sort((a, b) => b.score - a.score).map((item) => item.field);
|
|
5709
|
+
if (requestedFields.length > 0) {
|
|
5710
|
+
return requestedFields.slice(0, Math.min(6, requestedFields.length));
|
|
5711
|
+
}
|
|
5712
|
+
const metadataFields = Array.from(new Set(
|
|
5713
|
+
data.flatMap((item) => Object.keys(item.metadata || {}).map((k) => this.humanizeFieldName(k)))
|
|
5714
|
+
));
|
|
5715
|
+
return metadataFields.length > 0 ? metadataFields : ["Content"];
|
|
4681
5716
|
}
|
|
4682
|
-
/**
|
|
4683
|
-
* Helper: Extract table row
|
|
4684
|
-
*/
|
|
4685
5717
|
static extractTableRow(item, columns) {
|
|
4686
|
-
|
|
4687
|
-
|
|
4688
|
-
|
|
4689
|
-
|
|
5718
|
+
return columns.map((col) => this.resolveTableCellValue(item, col));
|
|
5719
|
+
}
|
|
5720
|
+
static extractAvailableTableFields(data) {
|
|
5721
|
+
const fields = /* @__PURE__ */ new Map();
|
|
5722
|
+
const addField = (field) => {
|
|
5723
|
+
const clean = this.humanizeFieldName(field);
|
|
5724
|
+
if (!clean || /^(content|\[?type\]?)$/i.test(clean)) return;
|
|
5725
|
+
const normalized = this.normalizeComparableField(clean);
|
|
5726
|
+
if (!fields.has(normalized)) fields.set(normalized, clean);
|
|
5727
|
+
};
|
|
5728
|
+
data.forEach((item) => {
|
|
5729
|
+
Object.keys(item.metadata || {}).forEach(addField);
|
|
5730
|
+
for (const match of (item.content || "").matchAll(/(?:^|\n)\s*([A-Za-z][A-Za-z0-9_ /-]{1,50})\s*[:\-–]\s*([^\n]+)/g)) {
|
|
5731
|
+
addField(match[1]);
|
|
4690
5732
|
}
|
|
4691
|
-
const metaKey = col.charAt(0).toLowerCase() + col.slice(1);
|
|
4692
|
-
const value = meta[metaKey];
|
|
4693
|
-
return value !== void 0 ? String(value) : "";
|
|
4694
5733
|
});
|
|
5734
|
+
return Array.from(fields.values());
|
|
5735
|
+
}
|
|
5736
|
+
static pickTableField(fields, aliases) {
|
|
5737
|
+
const normalizedAliases = aliases.map((alias) => this.normalizeComparableField(alias));
|
|
5738
|
+
for (const alias of normalizedAliases) {
|
|
5739
|
+
const exact = fields.find((field) => this.normalizeComparableField(field) === alias);
|
|
5740
|
+
if (exact) return exact;
|
|
5741
|
+
}
|
|
5742
|
+
for (const alias of normalizedAliases) {
|
|
5743
|
+
const fuzzy = fields.find((field) => {
|
|
5744
|
+
const normalizedField = this.normalizeComparableField(field);
|
|
5745
|
+
if (/(id|code|uuid)$/.test(normalizedField) && !/(id|code|uuid)$/.test(alias)) return false;
|
|
5746
|
+
return normalizedField.includes(alias) || alias.includes(normalizedField);
|
|
5747
|
+
});
|
|
5748
|
+
if (fuzzy) return fuzzy;
|
|
5749
|
+
}
|
|
5750
|
+
return void 0;
|
|
5751
|
+
}
|
|
5752
|
+
static tableFieldQueryScore(field, query) {
|
|
5753
|
+
const normalizedField = this.normalizeComparableField(field);
|
|
5754
|
+
if (!normalizedField) return 0;
|
|
5755
|
+
const fieldTokens = this.fieldTokens(field);
|
|
5756
|
+
return fieldTokens.reduce((score, token) => {
|
|
5757
|
+
if (token.length < 3) return score;
|
|
5758
|
+
return query.includes(token) ? score + 1 : score;
|
|
5759
|
+
}, query.includes(normalizedField) ? 3 : 0);
|
|
5760
|
+
}
|
|
5761
|
+
static resolveTableCellValue(item, column) {
|
|
5762
|
+
if (column === "Content") return item.content.substring(0, 100);
|
|
5763
|
+
const meta = item.metadata || {};
|
|
5764
|
+
const normalizedColumn = this.normalizeComparableField(column);
|
|
5765
|
+
const exactMetadata = Object.entries(meta).find(
|
|
5766
|
+
([key]) => this.normalizeComparableField(key) === normalizedColumn || this.normalizeComparableField(this.humanizeFieldName(key)) === normalizedColumn
|
|
5767
|
+
);
|
|
5768
|
+
if (exactMetadata && exactMetadata[1] !== void 0 && exactMetadata[1] !== null) {
|
|
5769
|
+
return this.toDisplayValue(exactMetadata[1]);
|
|
5770
|
+
}
|
|
5771
|
+
const aliasValue = this.resolveAliasedTableCell(meta, column);
|
|
5772
|
+
if (aliasValue !== void 0) return this.toDisplayValue(aliasValue);
|
|
5773
|
+
const contentValue = this.extractContentFieldValue(item.content, column);
|
|
5774
|
+
if (contentValue !== null) return contentValue;
|
|
5775
|
+
const aliasContentValue = this.extractAliasedContentFieldValue(item.content, column);
|
|
5776
|
+
return aliasContentValue != null ? aliasContentValue : "";
|
|
5777
|
+
}
|
|
5778
|
+
static resolveAliasedTableCell(meta, column) {
|
|
5779
|
+
const normalizedColumn = this.normalizeComparableField(column);
|
|
5780
|
+
const aliases = normalizedColumn.includes("employee") ? ["number of employees", "employee count", "employees", "headcount", "staff count", "workforce"] : normalizedColumn.includes("name") ? ["company name", "organization name", "organisation name", "name", "company", "organization", "organisation"] : [];
|
|
5781
|
+
for (const alias of aliases) {
|
|
5782
|
+
const normalizedAlias = this.normalizeComparableField(alias);
|
|
5783
|
+
const match = Object.entries(meta).find(([key]) => {
|
|
5784
|
+
const normalizedKey = this.normalizeComparableField(key);
|
|
5785
|
+
return normalizedKey === normalizedAlias || normalizedKey.includes(normalizedAlias) || normalizedAlias.includes(normalizedKey);
|
|
5786
|
+
});
|
|
5787
|
+
if (match && match[1] !== void 0 && match[1] !== null) return match[1];
|
|
5788
|
+
}
|
|
5789
|
+
return void 0;
|
|
5790
|
+
}
|
|
5791
|
+
static extractContentFieldValue(content, column) {
|
|
5792
|
+
const escaped = column.replace(/[.*+?^${}()|[\]\\]/g, "\\$&").replace(/\s+/g, "[\\s_ -]+");
|
|
5793
|
+
const pattern = new RegExp(`(?:^|\\n)\\s*${escaped}\\s*[:\\-\u2013]\\s*([^\\n]+)`, "i");
|
|
5794
|
+
const match = content.match(pattern);
|
|
5795
|
+
return match ? match[1].trim() : null;
|
|
5796
|
+
}
|
|
5797
|
+
static extractAliasedContentFieldValue(content, column) {
|
|
5798
|
+
const normalizedColumn = this.normalizeComparableField(column);
|
|
5799
|
+
const aliases = normalizedColumn.includes("employee") ? ["Number Of Employees", "Number of Employees", "Employee Count", "Employees", "Headcount", "Staff Count", "Workforce"] : normalizedColumn.includes("name") ? ["Company Name", "Organization Name", "Organisation Name", "Name", "Company", "Organization", "Organisation"] : [];
|
|
5800
|
+
for (const alias of aliases) {
|
|
5801
|
+
const value = this.extractContentFieldValue(content, alias);
|
|
5802
|
+
if (value !== null) return value;
|
|
5803
|
+
}
|
|
5804
|
+
return null;
|
|
5805
|
+
}
|
|
5806
|
+
static toDisplayValue(value) {
|
|
5807
|
+
if (typeof value === "string" || typeof value === "number" || typeof value === "boolean") return value;
|
|
5808
|
+
return String(value != null ? value : "");
|
|
4695
5809
|
}
|
|
4696
|
-
/**
|
|
4697
|
-
* Helper: Calculate stock counts by category
|
|
4698
|
-
*/
|
|
4699
5810
|
static calculateStockCounts(category, data) {
|
|
4700
5811
|
let inStock = 0;
|
|
4701
5812
|
let outOfStock = 0;
|
|
4702
5813
|
data.forEach((d) => {
|
|
4703
|
-
|
|
4704
|
-
const
|
|
4705
|
-
if (
|
|
4706
|
-
|
|
4707
|
-
|
|
4708
|
-
|
|
4709
|
-
outOfStock++;
|
|
4710
|
-
}
|
|
5814
|
+
var _a, _b;
|
|
5815
|
+
const cat = (_a = this.getProductCategory(d)) != null ? _a : "Other";
|
|
5816
|
+
if (cat === category) {
|
|
5817
|
+
const quantity = (_b = this.extractStockQuantity(d)) != null ? _b : 1;
|
|
5818
|
+
if (this.determineStockStatus(d)) inStock += quantity;
|
|
5819
|
+
else outOfStock += quantity;
|
|
4711
5820
|
}
|
|
4712
5821
|
});
|
|
4713
5822
|
return { inStockCount: inStock, outOfStockCount: outOfStock };
|
|
4714
5823
|
}
|
|
4715
|
-
/**
|
|
4716
|
-
* Helper: Determine if item is in stock
|
|
4717
|
-
*/
|
|
4718
5824
|
static determineStockStatus(item) {
|
|
4719
5825
|
const meta = item.metadata || {};
|
|
4720
|
-
|
|
4721
|
-
|
|
4722
|
-
|
|
4723
|
-
|
|
4724
|
-
|
|
4725
|
-
|
|
4726
|
-
|
|
4727
|
-
|
|
4728
|
-
|
|
5826
|
+
const stockValue = resolveMetadataValue(meta, "stock");
|
|
5827
|
+
if (stockValue !== void 0) {
|
|
5828
|
+
const normalized = String(stockValue).toLowerCase();
|
|
5829
|
+
if (/out[_\s-]?of[_\s-]?stock|unavailable|false|no|sold out|0/.test(normalized)) return false;
|
|
5830
|
+
if (/in[_\s-]?stock|available|true|yes/.test(normalized)) return true;
|
|
5831
|
+
const numeric = this.toFiniteNumber(stockValue);
|
|
5832
|
+
if (numeric !== null) return numeric > 0;
|
|
5833
|
+
}
|
|
5834
|
+
if (meta.inStock !== void 0) return Boolean(meta.inStock);
|
|
5835
|
+
if (meta.stock !== void 0) return Boolean(meta.stock);
|
|
5836
|
+
if (meta.available !== void 0) return Boolean(meta.available);
|
|
4729
5837
|
const content = (item.content || "").toLowerCase();
|
|
4730
|
-
if (
|
|
4731
|
-
|
|
5838
|
+
if (/out[_\s-]?of[_\s-]?stock|unavailable|sold out/.test(content)) return false;
|
|
5839
|
+
if (/in[_\s-]?stock|available/.test(content)) return true;
|
|
5840
|
+
return true;
|
|
5841
|
+
}
|
|
5842
|
+
static extractStockQuantity(item) {
|
|
5843
|
+
const meta = item.metadata || {};
|
|
5844
|
+
const stockValue = resolveMetadataValue(meta, "stock");
|
|
5845
|
+
const numericStock = this.toFiniteNumber(stockValue);
|
|
5846
|
+
if (numericStock !== null) return numericStock;
|
|
5847
|
+
const content = item.content || "";
|
|
5848
|
+
const stockMatch = content.match(/\b(?:stock|inventory|quantity|count)\s*[:\-–]?\s*([\d,]+)/i);
|
|
5849
|
+
if (!stockMatch) return null;
|
|
5850
|
+
return this.toFiniteNumber(stockMatch[1]);
|
|
5851
|
+
}
|
|
5852
|
+
static toFiniteNumber(value) {
|
|
5853
|
+
if (typeof value === "number") return Number.isFinite(value) ? value : null;
|
|
5854
|
+
const numeric = Number(String(value != null ? value : "").replace(/[$,% ,]/g, ""));
|
|
5855
|
+
return Number.isFinite(numeric) ? numeric : null;
|
|
5856
|
+
}
|
|
5857
|
+
// ─── Product Extraction ───────────────────────────────────────────────────
|
|
5858
|
+
static getDynamicVal(meta, uiKey, config, trainedSchema) {
|
|
5859
|
+
var _a;
|
|
5860
|
+
if (!meta) return void 0;
|
|
5861
|
+
const mapping = (_a = config == null ? void 0 : config.rag) == null ? void 0 : _a.uiMapping;
|
|
5862
|
+
if ((mapping == null ? void 0 : mapping[uiKey]) && meta[mapping[uiKey]] !== void 0) return meta[mapping[uiKey]];
|
|
5863
|
+
if (trainedSchema && typeof trainedSchema === "object") {
|
|
5864
|
+
const trainedKey = trainedSchema[uiKey];
|
|
5865
|
+
if (trainedKey && meta[trainedKey] !== void 0) return meta[trainedKey];
|
|
4732
5866
|
}
|
|
4733
|
-
|
|
4734
|
-
|
|
5867
|
+
return resolveMetadataValue(meta, uiKey);
|
|
5868
|
+
}
|
|
5869
|
+
static extractProductInfo(item, config, trainedSchema) {
|
|
5870
|
+
var _a;
|
|
5871
|
+
const meta = item.metadata || {};
|
|
5872
|
+
const name = this.getDynamicVal(meta, "name", config, trainedSchema);
|
|
5873
|
+
const price = this.getDynamicVal(meta, "price", config, trainedSchema);
|
|
5874
|
+
const brand = this.getDynamicVal(meta, "brand", config, trainedSchema);
|
|
5875
|
+
const description = this.cleanProductDescription(
|
|
5876
|
+
(_a = this.extractProductDescriptionFromContent(item.content)) != null ? _a : this.getProductDescriptionValue(meta, config, trainedSchema)
|
|
5877
|
+
);
|
|
5878
|
+
if (name || this.isProductData(item)) {
|
|
5879
|
+
let finalName = name ? String(name) : void 0;
|
|
5880
|
+
if (!finalName) {
|
|
5881
|
+
const nameMatch = item.content.match(/(?:Product|Item|Name):\s*([^\n]+)/i);
|
|
5882
|
+
finalName = nameMatch ? nameMatch[1].trim() : item.content.split("\n")[0].substring(0, 60);
|
|
5883
|
+
}
|
|
5884
|
+
let finalPrice = typeof price === "number" || typeof price === "string" ? price : void 0;
|
|
5885
|
+
if (!finalPrice) {
|
|
5886
|
+
const priceMatch = item.content.match(/(?:Price|Cost):\s*\$?([\d,.]+)/i) || item.content.match(/\$\s*([\d,.]+)/);
|
|
5887
|
+
if (priceMatch) finalPrice = priceMatch[1].replace(/,/g, "");
|
|
5888
|
+
}
|
|
5889
|
+
const imageValue = this.getDynamicVal(meta, "image", config, trainedSchema);
|
|
5890
|
+
return {
|
|
5891
|
+
id: item.id,
|
|
5892
|
+
name: finalName,
|
|
5893
|
+
price: finalPrice,
|
|
5894
|
+
image: typeof imageValue === "string" ? imageValue : void 0,
|
|
5895
|
+
brand: brand ? String(brand) : void 0,
|
|
5896
|
+
description,
|
|
5897
|
+
inStock: this.determineStockStatus(item)
|
|
5898
|
+
};
|
|
4735
5899
|
}
|
|
4736
|
-
return
|
|
5900
|
+
return null;
|
|
4737
5901
|
}
|
|
4738
|
-
|
|
4739
|
-
|
|
4740
|
-
|
|
4741
|
-
|
|
4742
|
-
|
|
4743
|
-
|
|
4744
|
-
|
|
4745
|
-
|
|
4746
|
-
|
|
4747
|
-
|
|
4748
|
-
return fieldCount.size > 2;
|
|
5902
|
+
static extractProductDescriptionFromContent(content) {
|
|
5903
|
+
const bodyMatch = content.match(
|
|
5904
|
+
/\bBody\s*\(HTML\)\s*:\s*([\s\S]*?)(?=\s*[,.]?\s+\b(?:Handle|Title|Vendor|Type|Tags|Published|Option\d*\s+Name|Option\d*\s+Value|Option|Variant|Image|SKU|Price|Status|Category)\b\s*:|$)/i
|
|
5905
|
+
);
|
|
5906
|
+
if (bodyMatch == null ? void 0 : bodyMatch[1]) return bodyMatch[1];
|
|
5907
|
+
const descriptionMatch = content.match(
|
|
5908
|
+
/\b(?:Description|Summary|Details|Body|Content)\s*:\s*([\s\S]*?)(?=\s*[,.]?\s+\b(?:Handle|Title|Vendor|Type|Tags|Published|Option\d*\s+Name|Option\d*\s+Value|Option|Variant|Image|SKU|Price|Status|Category)\b\s*:|$)/i
|
|
5909
|
+
);
|
|
5910
|
+
if (descriptionMatch == null ? void 0 : descriptionMatch[1]) return descriptionMatch[1];
|
|
5911
|
+
return null;
|
|
4749
5912
|
}
|
|
4750
|
-
|
|
4751
|
-
|
|
4752
|
-
|
|
4753
|
-
|
|
4754
|
-
|
|
4755
|
-
|
|
4756
|
-
|
|
4757
|
-
|
|
4758
|
-
|
|
4759
|
-
|
|
4760
|
-
|
|
4761
|
-
|
|
4762
|
-
|
|
4763
|
-
|
|
4764
|
-
|
|
4765
|
-
* @param sources - vector DB matches returned by RAG retrieval
|
|
4766
|
-
* @param llm - any ILLMProvider instance (OpenAI, Anthropic, Ollama, Gemini, REST…)
|
|
4767
|
-
* @returns - a validated UITransformationResponse (type + title + description + data)
|
|
4768
|
-
*/
|
|
4769
|
-
static async analyzeAndDecide(query, sources, llm) {
|
|
4770
|
-
try {
|
|
4771
|
-
const context = this.buildContextSummary(sources);
|
|
4772
|
-
const systemPrompt = this.buildVisualizationSystemPrompt();
|
|
4773
|
-
const userPrompt = [
|
|
4774
|
-
`USER QUESTION: ${query}`,
|
|
4775
|
-
"",
|
|
4776
|
-
"RETRIEVED DATA (JSON):",
|
|
4777
|
-
context
|
|
4778
|
-
].join("\n");
|
|
4779
|
-
const rawResponse = await llm.chat(
|
|
4780
|
-
[{ role: "user", content: userPrompt }],
|
|
4781
|
-
"",
|
|
4782
|
-
{ systemPrompt, temperature: 0 }
|
|
5913
|
+
static getProductDescriptionValue(meta, config, trainedSchema) {
|
|
5914
|
+
const mapped = this.getDynamicVal(meta, "description", config, trainedSchema);
|
|
5915
|
+
if (mapped !== void 0 && mapped !== meta.content && mapped !== meta.text) return mapped;
|
|
5916
|
+
const preferredKeys = [
|
|
5917
|
+
"body_html",
|
|
5918
|
+
"body html",
|
|
5919
|
+
"bodyHtml",
|
|
5920
|
+
"description",
|
|
5921
|
+
"summary",
|
|
5922
|
+
"details",
|
|
5923
|
+
"body"
|
|
5924
|
+
];
|
|
5925
|
+
for (const key of preferredKeys) {
|
|
5926
|
+
const match = Object.keys(meta).find(
|
|
5927
|
+
(candidate) => this.normalizeComparableField(candidate) === this.normalizeComparableField(key)
|
|
4783
5928
|
);
|
|
4784
|
-
|
|
4785
|
-
if (parsed) {
|
|
4786
|
-
console.debug("[UITransformer] LLM chose visualization type:", parsed.type);
|
|
4787
|
-
return parsed;
|
|
4788
|
-
}
|
|
4789
|
-
console.warn("[UITransformer] LLM returned unparseable response; falling back to heuristic.");
|
|
4790
|
-
} catch (err) {
|
|
4791
|
-
console.warn("[UITransformer] analyzeAndDecide LLM call failed; falling back to heuristic.", err);
|
|
5929
|
+
if (match && meta[match] !== void 0 && meta[match] !== null) return meta[match];
|
|
4792
5930
|
}
|
|
4793
|
-
return
|
|
5931
|
+
return void 0;
|
|
4794
5932
|
}
|
|
4795
|
-
|
|
4796
|
-
|
|
4797
|
-
|
|
5933
|
+
static cleanProductDescription(raw) {
|
|
5934
|
+
if (raw === null || raw === void 0) return void 0;
|
|
5935
|
+
const extracted = this.extractProductDescriptionFromContent(String(raw));
|
|
5936
|
+
if (extracted && extracted !== String(raw)) return this.cleanProductDescription(extracted);
|
|
5937
|
+
const text = String(raw).replace(/<[^>]*>/g, " ").replace(/ /gi, " ").replace(/&/gi, "&").replace(/"/gi, '"').replace(/'/gi, "'").replace(/\[[^\]]*TYPE[^\]]*\]/gi, " ").replace(/\b(?:Handle|Title|Vendor|Type|Tags|Published|Option\d*\s+Name|Option\d*\s+Value|Option|Variant|Image|SKU|Price|Status|Category)\s*:\s*[^,.\n:]+[,.]?/gi, " ").replace(/\bBody\s*\(HTML\)\s*:\s*/gi, " ").replace(/\b(?:Description|Summary|Details|Body|Content)\s*:\s*/gi, " ").replace(/\s+/g, " ").trim();
|
|
5938
|
+
return text || void 0;
|
|
5939
|
+
}
|
|
5940
|
+
// ─── Visualization Prompt ─────────────────────────────────────────────────
|
|
4798
5941
|
static buildVisualizationSystemPrompt() {
|
|
4799
5942
|
return `You are a data visualization expert embedded in a RAG chat system.
|
|
4800
|
-
You will receive a user question and structured data
|
|
4801
|
-
Your ONLY job is to
|
|
4802
|
-
|
|
5943
|
+
You will receive a user question, a pre-computed INTENT object, and structured data from a vector database.
|
|
5944
|
+
Your ONLY job is to return a single JSON object describing how to visualize the data.
|
|
5945
|
+
The INTENT object is authoritative \u2014 honor it. For example:
|
|
5946
|
+
- If intent.wantsExplicitTable is false, never return type "table".
|
|
5947
|
+
- If intent.isTemporal is true, prefer line_chart.
|
|
5948
|
+
- If intent.visualizationHint is "comparison" or "ranking", prefer bar_chart.
|
|
5949
|
+
- If intent.visualizationHint is "composition", prefer pie_chart.
|
|
5950
|
+
- If intent.recommendedChart is unsupported by the response schema, use the nearest supported type.
|
|
5951
|
+
- Always respond in the language specified by intent.language.
|
|
4803
5952
|
|
|
4804
|
-
Return ONLY a valid JSON object \u2014 no markdown
|
|
5953
|
+
Return ONLY a valid JSON object \u2014 no markdown fences, no prose.
|
|
4805
5954
|
|
|
4806
|
-
The JSON must have this exact shape:
|
|
4807
5955
|
{
|
|
4808
|
-
"type": "bar_chart" | "line_chart" | "pie_chart" | "radar_chart" | "table" | "text",
|
|
4809
|
-
"title": "
|
|
4810
|
-
"description": "
|
|
4811
|
-
"data": <structured data
|
|
5956
|
+
"type": "bar_chart" | "horizontal_bar" | "line_chart" | "pie_chart" | "histogram" | "scatter_plot" | "radar_chart" | "metric_card" | "geo_map" | "table" | "text",
|
|
5957
|
+
"title": "concise descriptive title",
|
|
5958
|
+
"description": "one sentence describing the visualization",
|
|
5959
|
+
"data": <structured data per type below>
|
|
4812
5960
|
}
|
|
4813
5961
|
|
|
4814
|
-
DATA
|
|
4815
|
-
- bar_chart:
|
|
4816
|
-
-
|
|
4817
|
-
-
|
|
4818
|
-
-
|
|
4819
|
-
|
|
4820
|
-
|
|
4821
|
-
|
|
4822
|
-
|
|
4823
|
-
|
|
4824
|
-
|
|
4825
|
-
-
|
|
4826
|
-
- text: { "content": "<prose answer>" }
|
|
5962
|
+
DATA SHAPES:
|
|
5963
|
+
- bar_chart: [{ "category": string, "value": number }]
|
|
5964
|
+
- horizontal_bar: [{ "category": string, "value": number }]
|
|
5965
|
+
- histogram: [{ "category": string, "value": number }]
|
|
5966
|
+
- line_chart: [{ "timestamp": string, "value": number, "label": string }]
|
|
5967
|
+
- pie_chart: [{ "label": string, "value": number }]
|
|
5968
|
+
- scatter_plot:[{ "x": number, "y": number, "label": string }]
|
|
5969
|
+
- radar_chart: [{ "attribute": string, "<series1>": number, "<series2>": number, ... }]
|
|
5970
|
+
- metric_card: { "label": string, "value": number, "operation": "sum"|"average"|"count"|"min"|"max"|"median" }
|
|
5971
|
+
- geo_map: [{ "category": string, "value": number }]
|
|
5972
|
+
- table: { "columns": string[], "rows": (string|number)[][] }
|
|
5973
|
+
- text: { "content": "A concise plain-language answer." }
|
|
4827
5974
|
|
|
4828
|
-
|
|
4829
|
-
1.
|
|
4830
|
-
2.
|
|
4831
|
-
3.
|
|
4832
|
-
4.
|
|
4833
|
-
5.
|
|
4834
|
-
6. text \u2192 conversational or free-form answers where no chart adds value
|
|
4835
|
-
|
|
4836
|
-
IMPORTANT:
|
|
4837
|
-
- Aggregate numeric values from the raw data \u2014 do not pass raw object arrays as data.
|
|
4838
|
-
- Ensure all "value" fields are numbers, not strings.
|
|
4839
|
-
- For bar/line/pie, keep at most 12 data points for readability.
|
|
4840
|
-
- Never include nested objects or arrays inside bar_chart / line_chart / pie_chart data items.`;
|
|
5975
|
+
RULES:
|
|
5976
|
+
1. Aggregate values \u2014 never pass raw nested objects in chart arrays.
|
|
5977
|
+
2. All "value" fields must be numbers.
|
|
5978
|
+
3. Cap bar/line/pie at 12 data points.
|
|
5979
|
+
4. Use dollar signs only for monetary prices, not for stock quantities or years.
|
|
5980
|
+
5. If no relevant data is found, return text: "I cannot answer this question based on the available information."`;
|
|
4841
5981
|
}
|
|
4842
|
-
/**
|
|
4843
|
-
* Serialize retrieved vector matches into a compact JSON context string.
|
|
4844
|
-
* Limits the total character count to avoid exceeding LLM context windows.
|
|
4845
|
-
*/
|
|
4846
5982
|
static buildContextSummary(sources, maxChars = 6e3) {
|
|
4847
5983
|
const items = sources.map((s, i) => {
|
|
4848
5984
|
var _a, _b, _c, _d;
|
|
@@ -4886,14 +6022,16 @@ Given these metadata keys from a database: [${keys.join(", ")}]
|
|
|
4886
6022
|
Identify which keys best correspond to these standard UI properties:
|
|
4887
6023
|
${propertyList}
|
|
4888
6024
|
|
|
4889
|
-
Return ONLY a JSON object where the keys are the UI properties and the values are the matching database keys.
|
|
6025
|
+
Return ONLY a valid JSON object where the keys are the UI properties and the values are the EXACT matching database keys from the list above.
|
|
4890
6026
|
If no good match is found for a property, omit it.
|
|
4891
6027
|
|
|
4892
6028
|
Example:
|
|
4893
6029
|
{
|
|
4894
|
-
"name": "
|
|
4895
|
-
"price": "
|
|
4896
|
-
"brand": "
|
|
6030
|
+
"name": "Title",
|
|
6031
|
+
"price": "Variant Price",
|
|
6032
|
+
"brand": "Vendor",
|
|
6033
|
+
"image": "Image Src",
|
|
6034
|
+
"stock": "Variant Inventory Qty"
|
|
4897
6035
|
}
|
|
4898
6036
|
`;
|
|
4899
6037
|
try {
|
|
@@ -5063,27 +6201,38 @@ var Pipeline = class {
|
|
|
5063
6201
|
async initialize() {
|
|
5064
6202
|
var _a;
|
|
5065
6203
|
if (this.initialised) return;
|
|
5066
|
-
const chartInstruction = `You are a helpful
|
|
6204
|
+
const chartInstruction = `You are a helpful assistant. Use the provided context to answer questions accurately.
|
|
5067
6205
|
|
|
5068
|
-
###
|
|
6206
|
+
### CRITICAL RULES:
|
|
6207
|
+
- ONLY answer the user's specific question. Do NOT suggest or recommend unrelated products unless specifically asked.
|
|
5069
6208
|
- NEVER generate markdown tables. If you do, the UI will break.
|
|
5070
6209
|
- NEVER generate HTML tags like <figure>, <tbody>, <tr>, etc.
|
|
5071
6210
|
- NEVER generate text-based charts or graphs.
|
|
6211
|
+
- NEVER say you cannot display, render, draw, or create a chart when the user asks for a visualization. The UI handles visual rendering separately.
|
|
5072
6212
|
- ONLY use plain text and bullet points.
|
|
6213
|
+
- NEVER use the plus sign (+) as a separator between names, categories, or products. Use commas or bullet points.
|
|
6214
|
+
- ONLY answer the question if the sources contain relevant information to answer it, else say that you cannot answer the question.
|
|
6215
|
+
- If answer cannot be found in the sources, say that you cannot answer the question and do not hallucinate.
|
|
6216
|
+
- Do not use information from previous turns to answer the question.
|
|
6217
|
+
- You CAN use numbers for years and counts (e.g., 2006, 5800). But NEVER put a dollar sign ($) before non-monetary numbers like Stock quantities, EAN numbers, or years. Only use dollar signs for actual monetary prices.
|
|
5073
6218
|
|
|
5074
6219
|
### PRODUCT DISPLAY:
|
|
5075
|
-
-
|
|
6220
|
+
- Recommended Products should only be displayed when the user explicitly asks for products.
|
|
6221
|
+
- When recommending products (ONLY when asked), simply list their names, prices, and features in a friendly, conversational manner.
|
|
5076
6222
|
- The UI will automatically detect these products and show high-quality product cards in a carousel below your message.
|
|
6223
|
+
- For product descriptions, summarize customer-facing details only. Do NOT list internal catalog fields such as Handle, Body (HTML), Vendor, Type, Tags, Published, Option, Variant, SKU, or raw metadata labels.
|
|
5077
6224
|
- Do NOT try to format product lists as tables.
|
|
5078
6225
|
`;
|
|
5079
6226
|
this.config.llm.systemPrompt = chartInstruction;
|
|
5080
6227
|
this.vectorDB = await ProviderRegistry.createVectorProvider(this.config.vectorDb);
|
|
5081
|
-
|
|
6228
|
+
this.llmRouter = new LLMRouter(this.config);
|
|
6229
|
+
const { llmProvider: resolvedLLM, embeddingProvider } = await EmbeddingStrategyResolver.resolve(
|
|
5082
6230
|
this.config.llm,
|
|
5083
6231
|
this.config.embedding
|
|
5084
6232
|
);
|
|
5085
|
-
this.llmProvider = llmProvider;
|
|
5086
6233
|
this.embeddingProvider = embeddingProvider;
|
|
6234
|
+
await this.llmRouter.initialize(resolvedLLM);
|
|
6235
|
+
this.llmProvider = this.llmRouter.get("default");
|
|
5087
6236
|
if (this.config.graphDb) {
|
|
5088
6237
|
this.graphDB = await ProviderRegistry.createGraphProvider(this.config.graphDb);
|
|
5089
6238
|
await this.graphDB.initialize();
|
|
@@ -5173,7 +6322,10 @@ var Pipeline = class {
|
|
|
5173
6322
|
upsertBatchOptions
|
|
5174
6323
|
);
|
|
5175
6324
|
if (upsertResult.errors.length > 0) {
|
|
5176
|
-
console.warn(`[Pipeline] Ingestion: ${upsertResult.errors.length} batches failed
|
|
6325
|
+
console.warn(`[Pipeline] Ingestion: ${upsertResult.errors.length} batches failed. Error details:`);
|
|
6326
|
+
upsertResult.errors.forEach((err, idx) => {
|
|
6327
|
+
console.warn(` Batch ${idx + 1} Error: ${err.error.message || String(err.error)}`);
|
|
6328
|
+
});
|
|
5177
6329
|
}
|
|
5178
6330
|
return upsertResult.totalProcessed;
|
|
5179
6331
|
}
|
|
@@ -5240,10 +6392,16 @@ var Pipeline = class {
|
|
|
5240
6392
|
/**
|
|
5241
6393
|
* High-performance streaming RAG flow.
|
|
5242
6394
|
* Yields text chunks first, then the retrieval metadata + observability trace at the end.
|
|
6395
|
+
*
|
|
6396
|
+
* Latency optimizations:
|
|
6397
|
+
* - Strategy classification runs in parallel with query embedding (saves ~400ms)
|
|
6398
|
+
* - Hallucination scoring is fire-and-forget (doesn't block metadata yield)
|
|
6399
|
+
* - UITransformation is computed after text streaming and emitted with metadata
|
|
6400
|
+
* - SchemaMapper.train runs while answer generation streams
|
|
5243
6401
|
*/
|
|
5244
6402
|
askStream(_0) {
|
|
5245
6403
|
return __asyncGenerator(this, arguments, function* (question, history = [], namespace) {
|
|
5246
|
-
var _a, _b, _c, _d, _e, _f, _g, _h, _i;
|
|
6404
|
+
var _a, _b, _c, _d, _e, _f, _g, _h, _i, _j, _k, _l, _m;
|
|
5247
6405
|
yield new __await(this.initialize());
|
|
5248
6406
|
const ns = namespace != null ? namespace : this.config.projectId;
|
|
5249
6407
|
const topK = (_b = (_a = this.config.rag) == null ? void 0 : _a.topK) != null ? _b : 5;
|
|
@@ -5259,27 +6417,61 @@ var Pipeline = class {
|
|
|
5259
6417
|
}
|
|
5260
6418
|
const hints = QueryProcessor.extractQueryFieldHints(question, (_f = this.config.rag) == null ? void 0 : _f.filterableFields);
|
|
5261
6419
|
const filter = QueryProcessor.buildQueryFilter(question, hints);
|
|
6420
|
+
const numericPredicates = QueryProcessor.extractNumericPredicates(question, (_g = this.config.rag) == null ? void 0 : _g.filterableFields);
|
|
6421
|
+
if (numericPredicates.length > 0) {
|
|
6422
|
+
filter.__numericPredicates = numericPredicates;
|
|
6423
|
+
}
|
|
5262
6424
|
const embedStart = performance.now();
|
|
5263
|
-
const
|
|
5264
|
-
|
|
5265
|
-
|
|
5266
|
-
|
|
5267
|
-
|
|
6425
|
+
const cacheKey = `${ns}::${searchQuery}`;
|
|
6426
|
+
const cachedVector = this.embeddingCache.get(cacheKey);
|
|
6427
|
+
const [strategyResult, embeddedVector] = yield new __await(Promise.all([
|
|
6428
|
+
QueryProcessor.determineRetrievalStrategy(
|
|
6429
|
+
searchQuery,
|
|
6430
|
+
this.llmRouter.get("fast"),
|
|
6431
|
+
(_h = this.config.rag) == null ? void 0 : _h.graphKeywords,
|
|
6432
|
+
(_i = this.config.rag) == null ? void 0 : _i.vectorKeywords
|
|
6433
|
+
),
|
|
6434
|
+
// Embed immediately regardless of strategy — costs nothing if cached.
|
|
6435
|
+
// If strategy turns out to be 'graph'-only we just won't use the vector.
|
|
6436
|
+
cachedVector ? Promise.resolve(cachedVector) : this.embeddingProvider.embed(searchQuery, { taskType: "query" })
|
|
6437
|
+
]));
|
|
6438
|
+
const queryVector = embeddedVector;
|
|
6439
|
+
if (!cachedVector && queryVector.length > 0) {
|
|
6440
|
+
this.embeddingCache.set(cacheKey, queryVector);
|
|
6441
|
+
}
|
|
6442
|
+
const graphData = (strategyResult === "graph" || strategyResult === "both") && this.graphDB && ((_j = this.config.rag) == null ? void 0 : _j.useGraphRetrieval) ? yield new __await(this.graphDB.query(searchQuery)) : void 0;
|
|
6443
|
+
const hasMetadataFilter = filter.metadata && Object.keys(filter.metadata).length > 0;
|
|
6444
|
+
const hasNumericPredicates = Array.isArray(filter.__numericPredicates) && filter.__numericPredicates.length > 0;
|
|
6445
|
+
const wantsExhaustiveList = hasNumericPredicates || hasMetadataFilter || /\b(list|all|show|provide|give|get|browse|find|search|display)\b/i.test(question);
|
|
6446
|
+
const retrievalLimit = wantsExhaustiveList ? Math.max(topK * 20, 100) : topK * 2;
|
|
6447
|
+
const rawSources = (strategyResult === "vector" || strategyResult === "both") && queryVector && queryVector.length > 0 ? yield new __await(this.vectorDB.query(queryVector, retrievalLimit, ns, filter)) : [];
|
|
5268
6448
|
const retrieveEnd = performance.now();
|
|
5269
6449
|
const embedMs = retrieveEnd - embedStart;
|
|
5270
6450
|
const retrieveMs = retrieveEnd - embedStart;
|
|
5271
6451
|
const rerankStart = performance.now();
|
|
5272
|
-
|
|
5273
|
-
|
|
5274
|
-
|
|
5275
|
-
|
|
5276
|
-
|
|
6452
|
+
const structuredSources = this.applyStructuredFilters(rawSources, filter);
|
|
6453
|
+
let fullSources = hasNumericPredicates ? structuredSources : structuredSources.filter((m) => m.score >= scoreThreshold);
|
|
6454
|
+
const rerankLimit = wantsExhaustiveList ? retrievalLimit : topK;
|
|
6455
|
+
if (!hasNumericPredicates && ((_k = this.config.rag) == null ? void 0 : _k.useReranking)) {
|
|
6456
|
+
fullSources = yield new __await(this.reranker.rerank(fullSources, question, rerankLimit));
|
|
6457
|
+
} else if (!wantsExhaustiveList) {
|
|
6458
|
+
fullSources = fullSources.slice(0, topK);
|
|
5277
6459
|
}
|
|
5278
6460
|
const rerankMs = performance.now() - rerankStart;
|
|
5279
|
-
let context =
|
|
6461
|
+
let context = fullSources.length ? fullSources.map((m, i) => `[Source ${i + 1}]
|
|
5280
6462
|
${m.content}`).join("\n\n---\n\n") : "No relevant context found.";
|
|
6463
|
+
let displayCount = 15;
|
|
6464
|
+
if (hasMetadataFilter) {
|
|
6465
|
+
displayCount = fullSources.length;
|
|
6466
|
+
} else {
|
|
6467
|
+
const highlyRelevant = fullSources.filter((m) => m.score >= 0.4);
|
|
6468
|
+
displayCount = Math.max(highlyRelevant.length, topK);
|
|
6469
|
+
if (displayCount > 15) {
|
|
6470
|
+
displayCount = 15;
|
|
6471
|
+
}
|
|
6472
|
+
}
|
|
6473
|
+
const sources = [...fullSources].sort((a, b) => b.score - a.score).slice(0, displayCount);
|
|
5281
6474
|
if (graphData && graphData.nodes.length > 0) {
|
|
5282
|
-
console.log(`[Graph Retrieval] Found ${graphData.nodes.length} relevant entities.`);
|
|
5283
6475
|
const graphContext = graphData.nodes.map(
|
|
5284
6476
|
(n) => `Entity: ${n.label} (${n.id})${n.properties ? " - " + JSON.stringify(n.properties) : ""}`
|
|
5285
6477
|
).join("\n");
|
|
@@ -5290,20 +6482,29 @@ VECTOR CONTEXT:
|
|
|
5290
6482
|
${context}`;
|
|
5291
6483
|
}
|
|
5292
6484
|
const allMetadataKeys = Array.from(new Set(sources.flatMap((s) => Object.keys(s.metadata || {}))));
|
|
5293
|
-
const
|
|
5294
|
-
const
|
|
6485
|
+
const trainedSchemaPromise = allMetadataKeys.length > 0 ? SchemaMapper.train(this.llmProvider, ns, allMetadataKeys).catch(() => void 0) : Promise.resolve(void 0);
|
|
6486
|
+
const uiTransformationPromise = trainedSchemaPromise.then(
|
|
6487
|
+
(trainedSchema) => this.generateUiTransformation(
|
|
6488
|
+
question,
|
|
6489
|
+
sources,
|
|
6490
|
+
trainedSchema,
|
|
6491
|
+
hasNumericPredicates
|
|
6492
|
+
)
|
|
6493
|
+
).catch((uiError) => {
|
|
6494
|
+
console.warn("[Pipeline] UI transformation failed concurrently:", uiError);
|
|
6495
|
+
return UITransformer.transform(question, sources, this.config);
|
|
6496
|
+
});
|
|
6497
|
+
const restrictionSuffix = "\n\n(IMPORTANT: Format your response beautifully using rich Markdown! Use bolding for emphasis, headings (##) for structure, bullet points for lists, and proper line breaks between paragraphs to make it highly readable. However, NEVER generate Markdown tables, HTML figures, or text charts (the UI renders requested charts separately, so NEVER say you cannot create, display, draw, render, or provide a chart/visualization). If listing products, use simple markdown bullet points or comma-separated names. NEVER use plus signs (+) to separate product names, category names, or list items. For product description/detail questions, provide customer-facing prose only and do NOT list internal catalog fields such as Handle, Body (HTML), Vendor, Type, Tags, Published, Option, Variant, SKU, or raw metadata labels. You CAN use numbers for years/counts like 2006 or 5800, but NEVER put a dollar sign ($) before them.)";
|
|
5295
6498
|
const hardenedHistory = [...history];
|
|
5296
6499
|
const userQuestion = { role: "user", content: question + restrictionSuffix };
|
|
5297
6500
|
const messages = [...hardenedHistory, userQuestion];
|
|
5298
|
-
const systemPrompt = (
|
|
6501
|
+
const systemPrompt = (_l = this.config.llm.systemPrompt) != null ? _l : "";
|
|
5299
6502
|
const userPrompt = messages.map((m) => `${m.role}: ${m.content}`).join("\n");
|
|
5300
6503
|
let fullReply = "";
|
|
5301
6504
|
const generateStart = performance.now();
|
|
5302
6505
|
if (this.llmProvider.chatStream) {
|
|
5303
6506
|
const stream = this.llmProvider.chatStream(messages, context);
|
|
5304
|
-
if (!stream) {
|
|
5305
|
-
throw new Error(`[Pipeline] ${this.config.llm.provider} chatStream returned undefined`);
|
|
5306
|
-
}
|
|
6507
|
+
if (!stream) throw new Error(`[Pipeline] ${this.config.llm.provider} chatStream returned undefined`);
|
|
5307
6508
|
try {
|
|
5308
6509
|
for (var iter = __forAwait(stream), more, temp, error; more = !(temp = yield new __await(iter.next())).done; more = false) {
|
|
5309
6510
|
const chunk = temp.value;
|
|
@@ -5330,7 +6531,7 @@ ${context}`;
|
|
|
5330
6531
|
const latency = {
|
|
5331
6532
|
embedMs: Math.round(embedMs),
|
|
5332
6533
|
retrieveMs: Math.round(retrieveMs),
|
|
5333
|
-
rerankMs: ((
|
|
6534
|
+
rerankMs: ((_m = this.config.rag) == null ? void 0 : _m.useReranking) ? Math.round(rerankMs) : void 0,
|
|
5334
6535
|
generateMs: Math.round(generateMs),
|
|
5335
6536
|
totalMs: Math.round(totalMs)
|
|
5336
6537
|
};
|
|
@@ -5343,9 +6544,6 @@ ${context}`;
|
|
|
5343
6544
|
totalTokens: promptTokens + completionTokens,
|
|
5344
6545
|
estimatedCostUsd: estimateCostUsd(promptTokens, completionTokens, this.config.llm.model)
|
|
5345
6546
|
};
|
|
5346
|
-
const hallucinationResult = yield new __await(scoreHallucination(this.llmProvider, fullReply, context).catch(() => void 0));
|
|
5347
|
-
const trainedSchema = yield new __await(trainingPromise);
|
|
5348
|
-
const uiTransformation = yield new __await(this.generateUiTransformation(question, sources, trainedSchema));
|
|
5349
6547
|
const trace = {
|
|
5350
6548
|
requestId,
|
|
5351
6549
|
query: question,
|
|
@@ -5364,10 +6562,9 @@ ${context}`;
|
|
|
5364
6562
|
}),
|
|
5365
6563
|
latency,
|
|
5366
6564
|
tokens,
|
|
5367
|
-
hallucinationScore: hallucinationResult == null ? void 0 : hallucinationResult.score,
|
|
5368
|
-
hallucinationReason: hallucinationResult == null ? void 0 : hallucinationResult.reason,
|
|
5369
6565
|
timestamp: (/* @__PURE__ */ new Date()).toISOString()
|
|
5370
6566
|
};
|
|
6567
|
+
const uiTransformation = yield new __await(uiTransformationPromise);
|
|
5371
6568
|
yield {
|
|
5372
6569
|
reply: "",
|
|
5373
6570
|
sources,
|
|
@@ -5375,6 +6572,13 @@ ${context}`;
|
|
|
5375
6572
|
ui_transformation: uiTransformation,
|
|
5376
6573
|
trace
|
|
5377
6574
|
};
|
|
6575
|
+
scoreHallucination(this.llmProvider, fullReply, context).then((hallucinationResult) => {
|
|
6576
|
+
if (hallucinationResult) {
|
|
6577
|
+
trace.hallucinationScore = hallucinationResult.score;
|
|
6578
|
+
trace.hallucinationReason = hallucinationResult.reason;
|
|
6579
|
+
}
|
|
6580
|
+
}).catch(() => {
|
|
6581
|
+
});
|
|
5378
6582
|
} catch (error2) {
|
|
5379
6583
|
throw new Error(`[Pipeline] Stream failed: ${error2 instanceof Error ? error2.message : String(error2)}`);
|
|
5380
6584
|
}
|
|
@@ -5384,24 +6588,107 @@ ${context}`;
|
|
|
5384
6588
|
* Universal retrieval method combining all enabled providers.
|
|
5385
6589
|
* Uses an LRU-bounded embedding cache to avoid re-embedding the same query.
|
|
5386
6590
|
*/
|
|
5387
|
-
async generateUiTransformation(question, sources, trainedSchema) {
|
|
6591
|
+
async generateUiTransformation(question, sources, trainedSchema, forceDeterministic = false) {
|
|
5388
6592
|
if (!sources || sources.length === 0) {
|
|
5389
6593
|
return UITransformer.transform(question, sources, this.config, trainedSchema);
|
|
5390
6594
|
}
|
|
6595
|
+
if (forceDeterministic) {
|
|
6596
|
+
return UITransformer.transform(question, sources, this.config, trainedSchema);
|
|
6597
|
+
}
|
|
5391
6598
|
try {
|
|
5392
|
-
return await UITransformer.analyzeAndDecide(question, sources, this.
|
|
6599
|
+
return await UITransformer.analyzeAndDecide(question, sources, this.llmRouter.get("fast"));
|
|
5393
6600
|
} catch (err) {
|
|
5394
6601
|
console.warn("[Pipeline] generateUiTransformation failed, using heuristic fallback:", err);
|
|
5395
6602
|
return UITransformer.transform(question, sources, this.config, trainedSchema);
|
|
5396
6603
|
}
|
|
5397
6604
|
}
|
|
6605
|
+
applyStructuredFilters(sources, filter) {
|
|
6606
|
+
const predicates = Array.isArray(filter.__numericPredicates) ? filter.__numericPredicates : [];
|
|
6607
|
+
if (predicates.length === 0) return sources;
|
|
6608
|
+
return sources.filter((source) => predicates.every((predicate) => {
|
|
6609
|
+
const value = this.resolveNumericPredicateValue(source, predicate);
|
|
6610
|
+
return value !== null && this.matchesNumericPredicate(value, predicate);
|
|
6611
|
+
})).sort((a, b) => {
|
|
6612
|
+
var _a, _b;
|
|
6613
|
+
const primary = predicates[0];
|
|
6614
|
+
const aValue = (_a = this.resolveNumericPredicateValue(a, primary)) != null ? _a : 0;
|
|
6615
|
+
const bValue = (_b = this.resolveNumericPredicateValue(b, primary)) != null ? _b : 0;
|
|
6616
|
+
return primary.operator === "lt" || primary.operator === "lte" ? aValue - bValue : bValue - aValue;
|
|
6617
|
+
});
|
|
6618
|
+
}
|
|
6619
|
+
resolveNumericPredicateValue(source, predicate) {
|
|
6620
|
+
const meta = source.metadata || {};
|
|
6621
|
+
const field = predicate.field;
|
|
6622
|
+
const entries = Object.entries(meta).filter(([, value]) => value !== null && value !== void 0 && typeof value !== "object");
|
|
6623
|
+
if (field) {
|
|
6624
|
+
const normalizedField = this.normalizeComparableField(field);
|
|
6625
|
+
const exact = entries.find(([key]) => this.normalizeComparableField(key) === normalizedField);
|
|
6626
|
+
const fuzzy = exact != null ? exact : entries.map(([key, value]) => ({ key, value, score: this.fieldSimilarityScore(key, field) })).filter((candidate) => candidate.score > 0).sort((a, b) => b.score - a.score)[0];
|
|
6627
|
+
if (fuzzy) {
|
|
6628
|
+
const value = this.toFiniteNumber(Array.isArray(fuzzy) ? fuzzy[1] : fuzzy.value);
|
|
6629
|
+
if (value !== null) return value;
|
|
6630
|
+
}
|
|
6631
|
+
const contentValue = this.extractNumericValueFromContent(source.content, field);
|
|
6632
|
+
if (contentValue !== null) return contentValue;
|
|
6633
|
+
}
|
|
6634
|
+
for (const [key, value] of entries) {
|
|
6635
|
+
if (/(id|sku|ean|upc|phone|zip|postal|code)/i.test(key)) continue;
|
|
6636
|
+
const numeric = this.toFiniteNumber(value);
|
|
6637
|
+
if (numeric !== null) return numeric;
|
|
6638
|
+
}
|
|
6639
|
+
return null;
|
|
6640
|
+
}
|
|
6641
|
+
extractNumericValueFromContent(content, field) {
|
|
6642
|
+
const escapedWords = field.split(/\s+|_+|-+/).filter(Boolean).map((word) => word.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"));
|
|
6643
|
+
if (escapedWords.length === 0) return null;
|
|
6644
|
+
const pattern = new RegExp(`(?:${escapedWords.join("[\\\\s_:-]*")})\\s*[:\\-\u2013]?\\s*([\\d,]+(?:\\.\\d+)?)`, "i");
|
|
6645
|
+
const match = content.match(pattern);
|
|
6646
|
+
return match ? this.toFiniteNumber(match[1]) : null;
|
|
6647
|
+
}
|
|
6648
|
+
matchesNumericPredicate(value, predicate) {
|
|
6649
|
+
switch (predicate.operator) {
|
|
6650
|
+
case "gt":
|
|
6651
|
+
return value > predicate.value;
|
|
6652
|
+
case "gte":
|
|
6653
|
+
return value >= predicate.value;
|
|
6654
|
+
case "lt":
|
|
6655
|
+
return value < predicate.value;
|
|
6656
|
+
case "lte":
|
|
6657
|
+
return value <= predicate.value;
|
|
6658
|
+
case "eq":
|
|
6659
|
+
default:
|
|
6660
|
+
return value === predicate.value;
|
|
6661
|
+
}
|
|
6662
|
+
}
|
|
6663
|
+
normalizeComparableField(value) {
|
|
6664
|
+
return value.toLowerCase().replace(/[^a-z0-9]/g, "");
|
|
6665
|
+
}
|
|
6666
|
+
fieldSimilarityScore(candidate, requested) {
|
|
6667
|
+
const normalizedCandidate = this.normalizeComparableField(candidate);
|
|
6668
|
+
const normalizedRequested = this.normalizeComparableField(requested);
|
|
6669
|
+
if (normalizedCandidate === normalizedRequested) return 10;
|
|
6670
|
+
if (normalizedCandidate.includes(normalizedRequested) || normalizedRequested.includes(normalizedCandidate)) return 8;
|
|
6671
|
+
const candidateTokens = this.fieldTokens(candidate);
|
|
6672
|
+
const requestedTokens = this.fieldTokens(requested);
|
|
6673
|
+
const overlap = requestedTokens.filter((token) => candidateTokens.includes(token)).length;
|
|
6674
|
+
if (overlap === 0) return 0;
|
|
6675
|
+
return overlap / Math.max(requestedTokens.length, candidateTokens.length);
|
|
6676
|
+
}
|
|
6677
|
+
fieldTokens(value) {
|
|
6678
|
+
return value.toLowerCase().split(/[^a-z0-9]+/).map((token) => token.replace(/ies$/, "y").replace(/s$/, "")).filter((token) => token.length > 1);
|
|
6679
|
+
}
|
|
6680
|
+
toFiniteNumber(value) {
|
|
6681
|
+
if (typeof value === "number") return Number.isFinite(value) ? value : null;
|
|
6682
|
+
const numeric = Number(String(value != null ? value : "").replace(/[$,% ,]/g, ""));
|
|
6683
|
+
return Number.isFinite(numeric) ? numeric : null;
|
|
6684
|
+
}
|
|
5398
6685
|
async retrieve(query, options) {
|
|
5399
|
-
var _a, _b, _c;
|
|
6686
|
+
var _a, _b, _c, _d, _e, _f;
|
|
5400
6687
|
const ns = (_a = options.namespace) != null ? _a : this.config.projectId;
|
|
5401
6688
|
const topK = (_b = options.topK) != null ? _b : 5;
|
|
5402
6689
|
const cacheKey = `${ns}::${query}`;
|
|
5403
6690
|
let queryVector = this.embeddingCache.get(cacheKey);
|
|
5404
|
-
const strategy = await QueryProcessor.determineRetrievalStrategy(query, this.
|
|
6691
|
+
const strategy = await QueryProcessor.determineRetrievalStrategy(query, this.llmRouter.get("fast"));
|
|
5405
6692
|
console.debug(`[Pipeline] Determined retrieval strategy: ${strategy}`);
|
|
5406
6693
|
const [retrievedVector, graphData] = await Promise.all([
|
|
5407
6694
|
// Only embed if we need vector search (strategy is 'vector' or 'both')
|
|
@@ -5413,8 +6700,27 @@ ${context}`;
|
|
|
5413
6700
|
this.embeddingCache.set(cacheKey, retrievedVector);
|
|
5414
6701
|
queryVector = retrievedVector;
|
|
5415
6702
|
}
|
|
5416
|
-
const
|
|
5417
|
-
|
|
6703
|
+
const baseFilter = __spreadProps(__spreadValues({}, (_d = options.filter) != null ? _d : {}), { queryText: query });
|
|
6704
|
+
const numericPredicates = QueryProcessor.extractNumericPredicates(query, (_e = this.config.rag) == null ? void 0 : _e.filterableFields);
|
|
6705
|
+
if (numericPredicates.length > 0) {
|
|
6706
|
+
baseFilter.__numericPredicates = numericPredicates;
|
|
6707
|
+
}
|
|
6708
|
+
const retrievalLimit = numericPredicates.length > 0 ? Math.max(topK * 20, 100) : topK;
|
|
6709
|
+
const sources = (strategy === "vector" || strategy === "both") && queryVector && queryVector.length > 0 ? this.applyStructuredFilters(
|
|
6710
|
+
await this.vectorDB.query(queryVector, retrievalLimit, ns, baseFilter),
|
|
6711
|
+
baseFilter
|
|
6712
|
+
) : [];
|
|
6713
|
+
const resolvedSources = [];
|
|
6714
|
+
for (const source of sources) {
|
|
6715
|
+
const parentId = (_f = source.metadata) == null ? void 0 : _f.parent_id;
|
|
6716
|
+
if (parentId) {
|
|
6717
|
+
console.log(`[Pipeline] Multi-Vector: Found child chunk. Parent ID: ${parentId}`);
|
|
6718
|
+
resolvedSources.push(source);
|
|
6719
|
+
} else {
|
|
6720
|
+
resolvedSources.push(source);
|
|
6721
|
+
}
|
|
6722
|
+
}
|
|
6723
|
+
return { sources: resolvedSources, graphData };
|
|
5418
6724
|
}
|
|
5419
6725
|
/** Rewrite the user query for better retrieval performance. */
|
|
5420
6726
|
async rewriteQuery(question, history) {
|
|
@@ -5906,204 +7212,7 @@ var DocumentParser = class {
|
|
|
5906
7212
|
// src/server.ts
|
|
5907
7213
|
init_BaseVectorProvider();
|
|
5908
7214
|
init_PineconeProvider();
|
|
5909
|
-
|
|
5910
|
-
// src/providers/vectordb/PostgreSQLProvider.ts
|
|
5911
|
-
var import_pg2 = require("pg");
|
|
5912
|
-
init_BaseVectorProvider();
|
|
5913
|
-
var PostgreSQLProvider = class extends BaseVectorProvider {
|
|
5914
|
-
constructor(config) {
|
|
5915
|
-
var _a;
|
|
5916
|
-
super(config);
|
|
5917
|
-
this.tableName = this.indexName.replace(/[^a-z0-9_]/gi, "_");
|
|
5918
|
-
const opts = config.options;
|
|
5919
|
-
if (!opts.connectionString) throw new Error("[PostgreSQLProvider] options.connectionString is required");
|
|
5920
|
-
this.connectionString = opts.connectionString;
|
|
5921
|
-
this.dimensions = (_a = opts.dimensions) != null ? _a : 1536;
|
|
5922
|
-
}
|
|
5923
|
-
static getValidator() {
|
|
5924
|
-
return {
|
|
5925
|
-
validate(config) {
|
|
5926
|
-
const errors = [];
|
|
5927
|
-
const opts = config.options || {};
|
|
5928
|
-
if (!opts.connectionString) {
|
|
5929
|
-
errors.push({
|
|
5930
|
-
field: "vectorDb.options.connectionString",
|
|
5931
|
-
message: "PostgreSQL connection string is required",
|
|
5932
|
-
suggestion: "Set PGVECTOR_CONNECTION_STRING environment variable",
|
|
5933
|
-
severity: "error"
|
|
5934
|
-
});
|
|
5935
|
-
}
|
|
5936
|
-
if (opts.tables && typeof opts.tables !== "string" && !Array.isArray(opts.tables)) {
|
|
5937
|
-
errors.push({
|
|
5938
|
-
field: "vectorDb.options.tables",
|
|
5939
|
-
message: "PostgreSQL tables must be a string or a string array",
|
|
5940
|
-
severity: "error"
|
|
5941
|
-
});
|
|
5942
|
-
}
|
|
5943
|
-
return errors;
|
|
5944
|
-
}
|
|
5945
|
-
};
|
|
5946
|
-
}
|
|
5947
|
-
static getHealthChecker() {
|
|
5948
|
-
return {
|
|
5949
|
-
async check(config) {
|
|
5950
|
-
const opts = config.options || {};
|
|
5951
|
-
const timestamp = Date.now();
|
|
5952
|
-
try {
|
|
5953
|
-
const { Client } = await import("pg");
|
|
5954
|
-
const client = new Client({ connectionString: opts.connectionString });
|
|
5955
|
-
await client.connect();
|
|
5956
|
-
const result = await client.query(`
|
|
5957
|
-
SELECT EXISTS(SELECT 1 FROM pg_extension WHERE extname = 'vector');
|
|
5958
|
-
`);
|
|
5959
|
-
const hasVector = result.rows[0].exists;
|
|
5960
|
-
await client.end();
|
|
5961
|
-
return {
|
|
5962
|
-
healthy: true,
|
|
5963
|
-
provider: "postgresql",
|
|
5964
|
-
capabilities: { pgvectorInstalled: hasVector },
|
|
5965
|
-
timestamp
|
|
5966
|
-
};
|
|
5967
|
-
} catch (error) {
|
|
5968
|
-
return {
|
|
5969
|
-
healthy: false,
|
|
5970
|
-
provider: "postgresql",
|
|
5971
|
-
error: `Connection failed: ${error instanceof Error ? error.message : String(error)}`,
|
|
5972
|
-
timestamp
|
|
5973
|
-
};
|
|
5974
|
-
}
|
|
5975
|
-
}
|
|
5976
|
-
};
|
|
5977
|
-
}
|
|
5978
|
-
async initialize() {
|
|
5979
|
-
this.pool = new import_pg2.Pool({ connectionString: this.connectionString });
|
|
5980
|
-
const client = await this.pool.connect();
|
|
5981
|
-
try {
|
|
5982
|
-
await client.query("CREATE EXTENSION IF NOT EXISTS vector");
|
|
5983
|
-
await client.query(`
|
|
5984
|
-
CREATE TABLE IF NOT EXISTS ${this.tableName} (
|
|
5985
|
-
id TEXT PRIMARY KEY,
|
|
5986
|
-
namespace TEXT NOT NULL DEFAULT '',
|
|
5987
|
-
content TEXT NOT NULL,
|
|
5988
|
-
metadata JSONB,
|
|
5989
|
-
embedding VECTOR(${this.dimensions})
|
|
5990
|
-
)
|
|
5991
|
-
`);
|
|
5992
|
-
await client.query(`
|
|
5993
|
-
CREATE INDEX IF NOT EXISTS ${this.tableName}_embedding_idx
|
|
5994
|
-
ON ${this.tableName}
|
|
5995
|
-
USING hnsw (embedding vector_cosine_ops)
|
|
5996
|
-
`);
|
|
5997
|
-
} finally {
|
|
5998
|
-
client.release();
|
|
5999
|
-
}
|
|
6000
|
-
}
|
|
6001
|
-
async upsert(doc, namespace = "") {
|
|
6002
|
-
var _a;
|
|
6003
|
-
const vectorLiteral = `[${doc.vector.join(",")}]`;
|
|
6004
|
-
await this.pool.query(
|
|
6005
|
-
`INSERT INTO ${this.tableName} (id, namespace, content, metadata, embedding)
|
|
6006
|
-
VALUES ($1, $2, $3, $4, $5::vector)
|
|
6007
|
-
ON CONFLICT (id) DO UPDATE
|
|
6008
|
-
SET namespace = EXCLUDED.namespace,
|
|
6009
|
-
content = EXCLUDED.content,
|
|
6010
|
-
metadata = EXCLUDED.metadata,
|
|
6011
|
-
embedding = EXCLUDED.embedding`,
|
|
6012
|
-
[doc.id, namespace, doc.content, JSON.stringify((_a = doc.metadata) != null ? _a : {}), vectorLiteral]
|
|
6013
|
-
);
|
|
6014
|
-
}
|
|
6015
|
-
async batchUpsert(docs, namespace = "") {
|
|
6016
|
-
if (docs.length === 0) return;
|
|
6017
|
-
const client = await this.pool.connect();
|
|
6018
|
-
try {
|
|
6019
|
-
await client.query("BEGIN");
|
|
6020
|
-
const BATCH_SIZE = 50;
|
|
6021
|
-
for (let i = 0; i < docs.length; i += BATCH_SIZE) {
|
|
6022
|
-
const batch = docs.slice(i, i + BATCH_SIZE);
|
|
6023
|
-
const values = [];
|
|
6024
|
-
const valuePlaceholders = batch.map((doc, idx) => {
|
|
6025
|
-
var _a;
|
|
6026
|
-
const offset = idx * 5;
|
|
6027
|
-
values.push(doc.id, namespace, doc.content, JSON.stringify((_a = doc.metadata) != null ? _a : {}), `[${doc.vector.join(",")}]`);
|
|
6028
|
-
return `($${offset + 1}, $${offset + 2}, $${offset + 3}, $${offset + 4}, $${offset + 5}::vector)`;
|
|
6029
|
-
}).join(", ");
|
|
6030
|
-
const query = `
|
|
6031
|
-
INSERT INTO ${this.tableName} (id, namespace, content, metadata, embedding)
|
|
6032
|
-
VALUES ${valuePlaceholders}
|
|
6033
|
-
ON CONFLICT (id) DO UPDATE
|
|
6034
|
-
SET namespace = EXCLUDED.namespace,
|
|
6035
|
-
content = EXCLUDED.content,
|
|
6036
|
-
metadata = EXCLUDED.metadata,
|
|
6037
|
-
embedding = EXCLUDED.embedding
|
|
6038
|
-
`;
|
|
6039
|
-
await client.query(query, values);
|
|
6040
|
-
}
|
|
6041
|
-
await client.query("COMMIT");
|
|
6042
|
-
} catch (error) {
|
|
6043
|
-
await client.query("ROLLBACK");
|
|
6044
|
-
throw error;
|
|
6045
|
-
} finally {
|
|
6046
|
-
client.release();
|
|
6047
|
-
}
|
|
6048
|
-
}
|
|
6049
|
-
async query(vector, topK, namespace, filter) {
|
|
6050
|
-
const vectorLiteral = `[${vector.join(",")}]`;
|
|
6051
|
-
let whereClause = namespace ? `WHERE namespace = $3` : "";
|
|
6052
|
-
const params = [vectorLiteral, topK];
|
|
6053
|
-
if (namespace) params.push(namespace);
|
|
6054
|
-
const publicFilter = this.sanitizeFilter(filter);
|
|
6055
|
-
if (Object.keys(publicFilter).length > 0) {
|
|
6056
|
-
const filterConditions = Object.entries(publicFilter).map(([key, val]) => {
|
|
6057
|
-
const paramIdx = params.length + 1;
|
|
6058
|
-
params.push(JSON.stringify(val));
|
|
6059
|
-
return `metadata->>'${key}' = $${paramIdx}`;
|
|
6060
|
-
}).join(" AND ");
|
|
6061
|
-
whereClause = whereClause ? `${whereClause} AND ${filterConditions}` : `WHERE ${filterConditions}`;
|
|
6062
|
-
}
|
|
6063
|
-
const client = await this.pool.connect();
|
|
6064
|
-
try {
|
|
6065
|
-
const efSearch = this.config.options.efSearch || Math.max(topK * 10, 40);
|
|
6066
|
-
await client.query(`SET LOCAL hnsw.ef_search = ${efSearch}`);
|
|
6067
|
-
const result = await client.query(
|
|
6068
|
-
`SELECT id, content, metadata, 1 - (embedding <=> $1::vector) AS score
|
|
6069
|
-
FROM ${this.tableName}
|
|
6070
|
-
${whereClause}
|
|
6071
|
-
ORDER BY embedding <=> $1::vector
|
|
6072
|
-
LIMIT $2`,
|
|
6073
|
-
params
|
|
6074
|
-
);
|
|
6075
|
-
return result.rows.map((row) => ({
|
|
6076
|
-
id: String(row["id"]),
|
|
6077
|
-
score: parseFloat(String(row["score"])),
|
|
6078
|
-
content: String(row["content"]),
|
|
6079
|
-
metadata: row["metadata"]
|
|
6080
|
-
}));
|
|
6081
|
-
} finally {
|
|
6082
|
-
client.release();
|
|
6083
|
-
}
|
|
6084
|
-
}
|
|
6085
|
-
async delete(id, namespace) {
|
|
6086
|
-
const where = namespace ? "WHERE id = $1 AND namespace = $2" : "WHERE id = $1";
|
|
6087
|
-
const params = namespace ? [id, namespace] : [id];
|
|
6088
|
-
await this.pool.query(`DELETE FROM ${this.tableName} ${where}`, params);
|
|
6089
|
-
}
|
|
6090
|
-
async deleteNamespace(namespace) {
|
|
6091
|
-
await this.pool.query(`DELETE FROM ${this.tableName} WHERE namespace = $1`, [namespace]);
|
|
6092
|
-
}
|
|
6093
|
-
async ping() {
|
|
6094
|
-
try {
|
|
6095
|
-
await this.pool.query("SELECT 1");
|
|
6096
|
-
return true;
|
|
6097
|
-
} catch (e) {
|
|
6098
|
-
return false;
|
|
6099
|
-
}
|
|
6100
|
-
}
|
|
6101
|
-
async disconnect() {
|
|
6102
|
-
await this.pool.end();
|
|
6103
|
-
}
|
|
6104
|
-
};
|
|
6105
|
-
|
|
6106
|
-
// src/server.ts
|
|
7215
|
+
init_PostgreSQLProvider();
|
|
6107
7216
|
init_MultiTablePostgresProvider();
|
|
6108
7217
|
init_MongoDBProvider();
|
|
6109
7218
|
init_MilvusProvider();
|
|
@@ -6189,15 +7298,24 @@ function createStreamHandler(configOrPlugin) {
|
|
|
6189
7298
|
});
|
|
6190
7299
|
}
|
|
6191
7300
|
const encoder = new TextEncoder();
|
|
7301
|
+
let isActive = true;
|
|
6192
7302
|
const stream = new ReadableStream({
|
|
6193
7303
|
async start(controller) {
|
|
6194
|
-
var _a
|
|
6195
|
-
const enqueue = (text) =>
|
|
7304
|
+
var _a;
|
|
7305
|
+
const enqueue = (text) => {
|
|
7306
|
+
if (!isActive) return;
|
|
7307
|
+
try {
|
|
7308
|
+
controller.enqueue(encoder.encode(text));
|
|
7309
|
+
} catch (err) {
|
|
7310
|
+
console.warn("[createStreamHandler] Failed to enqueue (stream already closed):", err);
|
|
7311
|
+
}
|
|
7312
|
+
};
|
|
6196
7313
|
try {
|
|
6197
7314
|
const pipelineStream = plugin.chatStream(message, history, namespace);
|
|
6198
7315
|
try {
|
|
6199
7316
|
for (var iter = __forAwait(pipelineStream), more, temp, error; more = !(temp = await iter.next()).done; more = false) {
|
|
6200
7317
|
const chunk = temp.value;
|
|
7318
|
+
if (!isActive) break;
|
|
6201
7319
|
if (typeof chunk === "string") {
|
|
6202
7320
|
enqueue(sseTextFrame(chunk));
|
|
6203
7321
|
} else {
|
|
@@ -6209,8 +7327,7 @@ function createStreamHandler(configOrPlugin) {
|
|
|
6209
7327
|
}
|
|
6210
7328
|
if (sources.length > 0) {
|
|
6211
7329
|
try {
|
|
6212
|
-
const
|
|
6213
|
-
const uiTransformation = (_b = responseChunk == null ? void 0 : responseChunk.ui_transformation) != null ? _b : llmProvider ? await UITransformer.analyzeAndDecide(message, sources, llmProvider) : UITransformer.transform(message, sources, plugin.getConfig());
|
|
7330
|
+
const uiTransformation = (_a = responseChunk == null ? void 0 : responseChunk.ui_transformation) != null ? _a : UITransformer.transform(message, sources, plugin.getConfig());
|
|
6214
7331
|
if (uiTransformation) {
|
|
6215
7332
|
enqueue(sseUIFrame(uiTransformation));
|
|
6216
7333
|
}
|
|
@@ -6236,12 +7353,27 @@ function createStreamHandler(configOrPlugin) {
|
|
|
6236
7353
|
}
|
|
6237
7354
|
}
|
|
6238
7355
|
} catch (streamError) {
|
|
6239
|
-
|
|
6240
|
-
|
|
6241
|
-
|
|
7356
|
+
if (isActive) {
|
|
7357
|
+
const errorMessage = streamError instanceof Error ? streamError.message : String(streamError);
|
|
7358
|
+
console.error("[createStreamHandler] Stream error:", streamError);
|
|
7359
|
+
try {
|
|
7360
|
+
enqueue(sseErrorFrame(errorMessage));
|
|
7361
|
+
} catch (e) {
|
|
7362
|
+
}
|
|
7363
|
+
}
|
|
6242
7364
|
} finally {
|
|
6243
|
-
|
|
7365
|
+
if (isActive) {
|
|
7366
|
+
isActive = false;
|
|
7367
|
+
try {
|
|
7368
|
+
controller.close();
|
|
7369
|
+
} catch (e) {
|
|
7370
|
+
}
|
|
7371
|
+
}
|
|
6244
7372
|
}
|
|
7373
|
+
},
|
|
7374
|
+
cancel(reason) {
|
|
7375
|
+
isActive = false;
|
|
7376
|
+
console.log("[createStreamHandler] Stream connection closed by client:", reason);
|
|
6245
7377
|
}
|
|
6246
7378
|
});
|
|
6247
7379
|
return new Response(stream, { headers: SSE_HEADERS });
|
|
@@ -6302,6 +7434,7 @@ function createUploadHandler(configOrPlugin) {
|
|
|
6302
7434
|
if (parsed.data && parsed.data.length > 0) {
|
|
6303
7435
|
let i = 0;
|
|
6304
7436
|
let lastRowData = null;
|
|
7437
|
+
const csvCols = Object.keys(parsed.data[0]);
|
|
6305
7438
|
for (const row of parsed.data) {
|
|
6306
7439
|
i++;
|
|
6307
7440
|
const rowData = row;
|
|
@@ -6323,12 +7456,14 @@ function createUploadHandler(configOrPlugin) {
|
|
|
6323
7456
|
documents.push({
|
|
6324
7457
|
docId: `${file.name}-row-${i}`,
|
|
6325
7458
|
content: contentParts.join(", "),
|
|
6326
|
-
metadata: __spreadValues(__spreadValues({
|
|
7459
|
+
metadata: __spreadValues(__spreadProps(__spreadValues({
|
|
6327
7460
|
fileName: file.name,
|
|
6328
7461
|
fileSize: file.size,
|
|
6329
7462
|
fileType: file.type,
|
|
6330
7463
|
uploadedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
6331
|
-
}, dimension ? { dimension } : {}),
|
|
7464
|
+
}, dimension ? { dimension } : {}), {
|
|
7465
|
+
csvHeaders: csvCols
|
|
7466
|
+
}), rowData)
|
|
6332
7467
|
});
|
|
6333
7468
|
}
|
|
6334
7469
|
}
|