@retrivora-ai/rag-engine 1.9.0 → 1.9.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{ILLMProvider-BOJFz3Na.d.mts → ILLMProvider-Bw2A28nU.d.mts} +12 -0
- package/dist/{ILLMProvider-BOJFz3Na.d.ts → ILLMProvider-Bw2A28nU.d.ts} +12 -0
- package/dist/handlers/index.d.mts +2 -2
- package/dist/handlers/index.d.ts +2 -2
- package/dist/handlers/index.js +1874 -542
- package/dist/handlers/index.mjs +1873 -541
- package/dist/{index-D3V9Et2M.d.mts → index-B70ZLkfG.d.mts} +1 -1
- package/dist/{index-BwpcaziY.d.ts → index-DVu-mkAM.d.ts} +1 -1
- package/dist/index.css +83 -0
- package/dist/index.d.mts +2 -2
- package/dist/index.d.ts +2 -2
- package/dist/index.js +330 -106
- package/dist/index.mjs +333 -107
- package/dist/server.d.mts +32 -5
- package/dist/server.d.ts +32 -5
- package/dist/server.js +1871 -736
- package/dist/server.mjs +1870 -735
- package/package.json +1 -1
- package/src/components/ChatWindow.tsx +24 -14
- package/src/components/MarkdownComponents.tsx +3 -3
- package/src/components/MessageBubble.tsx +89 -7
- package/src/components/ProductCard.tsx +29 -2
- package/src/components/UIDispatcher.tsx +1 -0
- package/src/components/VisualizationRenderer.tsx +143 -11
- package/src/config/EmbeddingStrategy.ts +5 -4
- package/src/config/RagConfig.ts +10 -0
- package/src/config/serverConfig.ts +16 -1
- package/src/core/LLMRouter.ts +79 -0
- package/src/core/Pipeline.ts +295 -51
- package/src/core/ProviderRegistry.ts +6 -0
- package/src/core/QueryProcessor.ts +108 -9
- package/src/handlers/index.ts +37 -11
- package/src/hooks/useRagChat.ts +77 -17
- package/src/llm/providers/UniversalLLMAdapter.ts +110 -13
- package/src/providers/vectordb/ChromaDBProvider.ts +13 -2
- package/src/providers/vectordb/MilvusProvider.ts +18 -2
- package/src/providers/vectordb/MultiTablePostgresProvider.ts +48 -16
- package/src/providers/vectordb/PostgreSQLProvider.ts +1 -1
- package/src/providers/vectordb/QdrantProvider.ts +1 -1
- package/src/providers/vectordb/RedisProvider.ts +3 -4
- package/src/providers/vectordb/WeaviateProvider.ts +41 -3
- package/src/types/chat.ts +2 -0
- package/src/types/index.ts +26 -0
- package/src/utils/ProductExtractor.ts +5 -3
- package/src/utils/SchemaMapper.ts +6 -4
- package/src/utils/UITransformer.ts +1350 -490
- package/src/utils/synonyms.ts +6 -4
package/dist/server.mjs
CHANGED
|
@@ -313,17 +313,300 @@ var init_PineconeProvider = __esm({
|
|
|
313
313
|
}
|
|
314
314
|
});
|
|
315
315
|
|
|
316
|
+
// src/providers/vectordb/PostgreSQLProvider.ts
|
|
317
|
+
var PostgreSQLProvider_exports = {};
|
|
318
|
+
__export(PostgreSQLProvider_exports, {
|
|
319
|
+
PostgreSQLProvider: () => PostgreSQLProvider
|
|
320
|
+
});
|
|
321
|
+
import { Pool } from "pg";
|
|
322
|
+
var PostgreSQLProvider;
|
|
323
|
+
var init_PostgreSQLProvider = __esm({
|
|
324
|
+
"src/providers/vectordb/PostgreSQLProvider.ts"() {
|
|
325
|
+
"use strict";
|
|
326
|
+
init_BaseVectorProvider();
|
|
327
|
+
PostgreSQLProvider = class extends BaseVectorProvider {
|
|
328
|
+
constructor(config) {
|
|
329
|
+
var _a;
|
|
330
|
+
super(config);
|
|
331
|
+
this.tableName = this.indexName.replace(/[^a-z0-9_]/gi, "_");
|
|
332
|
+
const opts = config.options;
|
|
333
|
+
if (!opts.connectionString) throw new Error("[PostgreSQLProvider] options.connectionString is required");
|
|
334
|
+
this.connectionString = opts.connectionString;
|
|
335
|
+
this.dimensions = (_a = opts.dimensions) != null ? _a : 1536;
|
|
336
|
+
}
|
|
337
|
+
static getValidator() {
|
|
338
|
+
return {
|
|
339
|
+
validate(config) {
|
|
340
|
+
const errors = [];
|
|
341
|
+
const opts = config.options || {};
|
|
342
|
+
if (!opts.connectionString) {
|
|
343
|
+
errors.push({
|
|
344
|
+
field: "vectorDb.options.connectionString",
|
|
345
|
+
message: "PostgreSQL connection string is required",
|
|
346
|
+
suggestion: "Set PGVECTOR_CONNECTION_STRING environment variable",
|
|
347
|
+
severity: "error"
|
|
348
|
+
});
|
|
349
|
+
}
|
|
350
|
+
if (opts.tables && typeof opts.tables !== "string" && !Array.isArray(opts.tables)) {
|
|
351
|
+
errors.push({
|
|
352
|
+
field: "vectorDb.options.tables",
|
|
353
|
+
message: "PostgreSQL tables must be a string or a string array",
|
|
354
|
+
severity: "error"
|
|
355
|
+
});
|
|
356
|
+
}
|
|
357
|
+
return errors;
|
|
358
|
+
}
|
|
359
|
+
};
|
|
360
|
+
}
|
|
361
|
+
static getHealthChecker() {
|
|
362
|
+
return {
|
|
363
|
+
async check(config) {
|
|
364
|
+
const opts = config.options || {};
|
|
365
|
+
const timestamp = Date.now();
|
|
366
|
+
try {
|
|
367
|
+
const { Client } = await import("pg");
|
|
368
|
+
const client = new Client({ connectionString: opts.connectionString });
|
|
369
|
+
await client.connect();
|
|
370
|
+
const result = await client.query(`
|
|
371
|
+
SELECT EXISTS(SELECT 1 FROM pg_extension WHERE extname = 'vector');
|
|
372
|
+
`);
|
|
373
|
+
const hasVector = result.rows[0].exists;
|
|
374
|
+
await client.end();
|
|
375
|
+
return {
|
|
376
|
+
healthy: true,
|
|
377
|
+
provider: "postgresql",
|
|
378
|
+
capabilities: { pgvectorInstalled: hasVector },
|
|
379
|
+
timestamp
|
|
380
|
+
};
|
|
381
|
+
} catch (error) {
|
|
382
|
+
return {
|
|
383
|
+
healthy: false,
|
|
384
|
+
provider: "postgresql",
|
|
385
|
+
error: `Connection failed: ${error instanceof Error ? error.message : String(error)}`,
|
|
386
|
+
timestamp
|
|
387
|
+
};
|
|
388
|
+
}
|
|
389
|
+
}
|
|
390
|
+
};
|
|
391
|
+
}
|
|
392
|
+
async initialize() {
|
|
393
|
+
this.pool = new Pool({ connectionString: this.connectionString });
|
|
394
|
+
const client = await this.pool.connect();
|
|
395
|
+
try {
|
|
396
|
+
await client.query("CREATE EXTENSION IF NOT EXISTS vector");
|
|
397
|
+
await client.query(`
|
|
398
|
+
CREATE TABLE IF NOT EXISTS ${this.tableName} (
|
|
399
|
+
id TEXT PRIMARY KEY,
|
|
400
|
+
namespace TEXT NOT NULL DEFAULT '',
|
|
401
|
+
content TEXT NOT NULL,
|
|
402
|
+
metadata JSONB,
|
|
403
|
+
embedding VECTOR(${this.dimensions})
|
|
404
|
+
)
|
|
405
|
+
`);
|
|
406
|
+
await client.query(`
|
|
407
|
+
CREATE INDEX IF NOT EXISTS ${this.tableName}_embedding_idx
|
|
408
|
+
ON ${this.tableName}
|
|
409
|
+
USING hnsw (embedding vector_cosine_ops)
|
|
410
|
+
`);
|
|
411
|
+
} finally {
|
|
412
|
+
client.release();
|
|
413
|
+
}
|
|
414
|
+
}
|
|
415
|
+
async upsert(doc, namespace = "") {
|
|
416
|
+
var _a;
|
|
417
|
+
const vectorLiteral = `[${doc.vector.join(",")}]`;
|
|
418
|
+
await this.pool.query(
|
|
419
|
+
`INSERT INTO ${this.tableName} (id, namespace, content, metadata, embedding)
|
|
420
|
+
VALUES ($1, $2, $3, $4, $5::vector)
|
|
421
|
+
ON CONFLICT (id) DO UPDATE
|
|
422
|
+
SET namespace = EXCLUDED.namespace,
|
|
423
|
+
content = EXCLUDED.content,
|
|
424
|
+
metadata = EXCLUDED.metadata,
|
|
425
|
+
embedding = EXCLUDED.embedding`,
|
|
426
|
+
[doc.id, namespace, doc.content, JSON.stringify((_a = doc.metadata) != null ? _a : {}), vectorLiteral]
|
|
427
|
+
);
|
|
428
|
+
}
|
|
429
|
+
async batchUpsert(docs, namespace = "") {
|
|
430
|
+
if (docs.length === 0) return;
|
|
431
|
+
const client = await this.pool.connect();
|
|
432
|
+
try {
|
|
433
|
+
await client.query("BEGIN");
|
|
434
|
+
const BATCH_SIZE = 50;
|
|
435
|
+
for (let i = 0; i < docs.length; i += BATCH_SIZE) {
|
|
436
|
+
const batch = docs.slice(i, i + BATCH_SIZE);
|
|
437
|
+
const values = [];
|
|
438
|
+
const valuePlaceholders = batch.map((doc, idx) => {
|
|
439
|
+
var _a;
|
|
440
|
+
const offset = idx * 5;
|
|
441
|
+
values.push(doc.id, namespace, doc.content, JSON.stringify((_a = doc.metadata) != null ? _a : {}), `[${doc.vector.join(",")}]`);
|
|
442
|
+
return `($${offset + 1}, $${offset + 2}, $${offset + 3}, $${offset + 4}, $${offset + 5}::vector)`;
|
|
443
|
+
}).join(", ");
|
|
444
|
+
const query = `
|
|
445
|
+
INSERT INTO ${this.tableName} (id, namespace, content, metadata, embedding)
|
|
446
|
+
VALUES ${valuePlaceholders}
|
|
447
|
+
ON CONFLICT (id) DO UPDATE
|
|
448
|
+
SET namespace = EXCLUDED.namespace,
|
|
449
|
+
content = EXCLUDED.content,
|
|
450
|
+
metadata = EXCLUDED.metadata,
|
|
451
|
+
embedding = EXCLUDED.embedding
|
|
452
|
+
`;
|
|
453
|
+
await client.query(query, values);
|
|
454
|
+
}
|
|
455
|
+
await client.query("COMMIT");
|
|
456
|
+
} catch (error) {
|
|
457
|
+
await client.query("ROLLBACK");
|
|
458
|
+
throw error;
|
|
459
|
+
} finally {
|
|
460
|
+
client.release();
|
|
461
|
+
}
|
|
462
|
+
}
|
|
463
|
+
async query(vector, topK, namespace, filter) {
|
|
464
|
+
const vectorLiteral = `[${vector.join(",")}]`;
|
|
465
|
+
let whereClause = namespace ? `WHERE namespace = $3` : "";
|
|
466
|
+
const params = [vectorLiteral, topK];
|
|
467
|
+
if (namespace) params.push(namespace);
|
|
468
|
+
const publicFilter = this.sanitizeFilter(filter);
|
|
469
|
+
if (Object.keys(publicFilter).length > 0) {
|
|
470
|
+
const filterConditions = Object.entries(publicFilter).map(([key, val]) => {
|
|
471
|
+
const paramIdx = params.length + 1;
|
|
472
|
+
params.push(String(val));
|
|
473
|
+
return `metadata->>'${key}' = $${paramIdx}`;
|
|
474
|
+
}).join(" AND ");
|
|
475
|
+
whereClause = whereClause ? `${whereClause} AND ${filterConditions}` : `WHERE ${filterConditions}`;
|
|
476
|
+
}
|
|
477
|
+
const client = await this.pool.connect();
|
|
478
|
+
try {
|
|
479
|
+
const efSearch = this.config.options.efSearch || Math.max(topK * 10, 40);
|
|
480
|
+
await client.query(`SET LOCAL hnsw.ef_search = ${efSearch}`);
|
|
481
|
+
const result = await client.query(
|
|
482
|
+
`SELECT id, content, metadata, 1 - (embedding <=> $1::vector) AS score
|
|
483
|
+
FROM ${this.tableName}
|
|
484
|
+
${whereClause}
|
|
485
|
+
ORDER BY embedding <=> $1::vector
|
|
486
|
+
LIMIT $2`,
|
|
487
|
+
params
|
|
488
|
+
);
|
|
489
|
+
return result.rows.map((row) => ({
|
|
490
|
+
id: String(row["id"]),
|
|
491
|
+
score: parseFloat(String(row["score"])),
|
|
492
|
+
content: String(row["content"]),
|
|
493
|
+
metadata: row["metadata"]
|
|
494
|
+
}));
|
|
495
|
+
} finally {
|
|
496
|
+
client.release();
|
|
497
|
+
}
|
|
498
|
+
}
|
|
499
|
+
async delete(id, namespace) {
|
|
500
|
+
const where = namespace ? "WHERE id = $1 AND namespace = $2" : "WHERE id = $1";
|
|
501
|
+
const params = namespace ? [id, namespace] : [id];
|
|
502
|
+
await this.pool.query(`DELETE FROM ${this.tableName} ${where}`, params);
|
|
503
|
+
}
|
|
504
|
+
async deleteNamespace(namespace) {
|
|
505
|
+
await this.pool.query(`DELETE FROM ${this.tableName} WHERE namespace = $1`, [namespace]);
|
|
506
|
+
}
|
|
507
|
+
async ping() {
|
|
508
|
+
try {
|
|
509
|
+
await this.pool.query("SELECT 1");
|
|
510
|
+
return true;
|
|
511
|
+
} catch (e) {
|
|
512
|
+
return false;
|
|
513
|
+
}
|
|
514
|
+
}
|
|
515
|
+
async disconnect() {
|
|
516
|
+
await this.pool.end();
|
|
517
|
+
}
|
|
518
|
+
};
|
|
519
|
+
}
|
|
520
|
+
});
|
|
521
|
+
|
|
522
|
+
// src/utils/synonyms.ts
|
|
523
|
+
function resolveMetadataValue(meta, uiKey) {
|
|
524
|
+
var _a;
|
|
525
|
+
const synonyms = (_a = FIELD_SYNONYMS[uiKey]) != null ? _a : [];
|
|
526
|
+
const keys = Object.keys(meta);
|
|
527
|
+
const systemKeys = ["namespace", "filename", "filesize", "filetype", "docid", "uploadedat", "dimension", "chunkindex"];
|
|
528
|
+
let match = keys.find((k) => k.toLowerCase() === uiKey.toLowerCase());
|
|
529
|
+
if (match !== void 0) return meta[match];
|
|
530
|
+
match = keys.find((k) => synonyms.map((s) => s.toLowerCase()).includes(k.toLowerCase()));
|
|
531
|
+
if (match !== void 0) return meta[match];
|
|
532
|
+
const isBlacklisted = (kl) => {
|
|
533
|
+
return systemKeys.includes(kl) || kl.startsWith("option1") || kl.startsWith("option2") || kl.startsWith("option3");
|
|
534
|
+
};
|
|
535
|
+
match = keys.find((k) => {
|
|
536
|
+
const kl = k.toLowerCase();
|
|
537
|
+
if (isBlacklisted(kl)) return false;
|
|
538
|
+
return kl.includes(uiKey.toLowerCase());
|
|
539
|
+
});
|
|
540
|
+
if (match !== void 0) return meta[match];
|
|
541
|
+
match = keys.find((k) => {
|
|
542
|
+
const kl = k.toLowerCase();
|
|
543
|
+
if (isBlacklisted(kl)) return false;
|
|
544
|
+
return synonyms.some((sk) => kl.includes(sk.toLowerCase()) || sk.toLowerCase().includes(kl));
|
|
545
|
+
});
|
|
546
|
+
return match !== void 0 ? meta[match] : void 0;
|
|
547
|
+
}
|
|
548
|
+
var FIELD_SYNONYMS;
|
|
549
|
+
var init_synonyms = __esm({
|
|
550
|
+
"src/utils/synonyms.ts"() {
|
|
551
|
+
"use strict";
|
|
552
|
+
FIELD_SYNONYMS = {
|
|
553
|
+
name: ["product", "item", "title", "label", "heading", "subject", "product name", "item name"],
|
|
554
|
+
price: ["cost", "amount", "msrp", "price", "rate", "value", "price_usd", "variant price"],
|
|
555
|
+
brand: ["manufacturer", "vendor", "make", "company", "brand_name", "supplier"],
|
|
556
|
+
image: [
|
|
557
|
+
"imageUrl",
|
|
558
|
+
"thumbnail",
|
|
559
|
+
"img",
|
|
560
|
+
"url",
|
|
561
|
+
"photo",
|
|
562
|
+
"picture",
|
|
563
|
+
"media",
|
|
564
|
+
"image_url",
|
|
565
|
+
"main_image",
|
|
566
|
+
"product_image",
|
|
567
|
+
"thumb",
|
|
568
|
+
"image src",
|
|
569
|
+
"variant image"
|
|
570
|
+
],
|
|
571
|
+
stock: [
|
|
572
|
+
"inventory",
|
|
573
|
+
"quantity",
|
|
574
|
+
"count",
|
|
575
|
+
"availability",
|
|
576
|
+
"stock_level",
|
|
577
|
+
"inStock",
|
|
578
|
+
"is_available",
|
|
579
|
+
"in stock",
|
|
580
|
+
"status",
|
|
581
|
+
"variant inventory qty"
|
|
582
|
+
],
|
|
583
|
+
category: [
|
|
584
|
+
"product_category",
|
|
585
|
+
"product category",
|
|
586
|
+
"category_name",
|
|
587
|
+
"category name",
|
|
588
|
+
"department",
|
|
589
|
+
"collection",
|
|
590
|
+
"type"
|
|
591
|
+
],
|
|
592
|
+
description: ["summary", "content", "body", "text", "info", "details", "body (html)", "seo description"],
|
|
593
|
+
link: ["url", "href", "product_url", "page_url", "link"]
|
|
594
|
+
};
|
|
595
|
+
}
|
|
596
|
+
});
|
|
597
|
+
|
|
316
598
|
// src/providers/vectordb/MultiTablePostgresProvider.ts
|
|
317
599
|
var MultiTablePostgresProvider_exports = {};
|
|
318
600
|
__export(MultiTablePostgresProvider_exports, {
|
|
319
601
|
MultiTablePostgresProvider: () => MultiTablePostgresProvider
|
|
320
602
|
});
|
|
321
|
-
import { Pool } from "pg";
|
|
603
|
+
import { Pool as Pool2 } from "pg";
|
|
322
604
|
var MultiTablePostgresProvider;
|
|
323
605
|
var init_MultiTablePostgresProvider = __esm({
|
|
324
606
|
"src/providers/vectordb/MultiTablePostgresProvider.ts"() {
|
|
325
607
|
"use strict";
|
|
326
608
|
init_BaseVectorProvider();
|
|
609
|
+
init_synonyms();
|
|
327
610
|
MultiTablePostgresProvider = class extends BaseVectorProvider {
|
|
328
611
|
constructor(config) {
|
|
329
612
|
var _a, _b, _c;
|
|
@@ -340,7 +623,7 @@ var init_MultiTablePostgresProvider = __esm({
|
|
|
340
623
|
this.uploadTable = opts.uploadTable || "document_chunks";
|
|
341
624
|
}
|
|
342
625
|
async initialize() {
|
|
343
|
-
this.pool = new
|
|
626
|
+
this.pool = new Pool2({ connectionString: this.connectionString });
|
|
344
627
|
const client = await this.pool.connect();
|
|
345
628
|
try {
|
|
346
629
|
await client.query("CREATE EXTENSION IF NOT EXISTS vector");
|
|
@@ -404,8 +687,13 @@ var init_MultiTablePostgresProvider = __esm({
|
|
|
404
687
|
}
|
|
405
688
|
tableName = tableName.replace(/[^a-z0-9_]/gi, "_").toLowerCase() || this.uploadTable;
|
|
406
689
|
const firstMeta = fileDocs[0].metadata || {};
|
|
407
|
-
|
|
408
|
-
|
|
690
|
+
let csvHeaders = [];
|
|
691
|
+
if (Array.isArray(firstMeta.csvHeaders)) {
|
|
692
|
+
csvHeaders = firstMeta.csvHeaders;
|
|
693
|
+
} else {
|
|
694
|
+
const systemKeys = ["fileName", "fileSize", "fileType", "uploadedAt", "dimension", "chunkIndex", "id", "namespace", "content", "metadata", "embedding", "docId", "docid", "chunkId", "chunkid", "csvHeaders", "csvheaders"];
|
|
695
|
+
csvHeaders = Object.keys(firstMeta).filter((k) => !systemKeys.includes(k) && !systemKeys.includes(k.toLowerCase()));
|
|
696
|
+
}
|
|
409
697
|
const columnDefs = csvHeaders.map((h) => `"${h}" TEXT`).join(",\n ");
|
|
410
698
|
const createTableSql = `
|
|
411
699
|
CREATE TABLE IF NOT EXISTS "${tableName}" (
|
|
@@ -484,26 +772,41 @@ var init_MultiTablePostgresProvider = __esm({
|
|
|
484
772
|
const allResults = [];
|
|
485
773
|
console.log(`[MultiTablePostgresProvider] --- Starting Multi-Table Search ---`);
|
|
486
774
|
const queryText = _filter == null ? void 0 : _filter.queryText;
|
|
487
|
-
const entityHints = Array.isArray(_filter == null ? void 0 : _filter.
|
|
488
|
-
(hint) => typeof hint === "object" && hint !== null && typeof hint.value === "string"
|
|
489
|
-
).map((hint) => hint.value.trim().toLowerCase()).filter(Boolean) : [];
|
|
775
|
+
const entityHints = Array.isArray(_filter == null ? void 0 : _filter.keywords) ? _filter.keywords.map((k) => k.trim().toLowerCase()).filter(Boolean) : [];
|
|
490
776
|
console.log(`[MultiTablePostgresProvider] queryText: "${queryText}"`);
|
|
491
777
|
console.log(`[MultiTablePostgresProvider] entityHints: [${entityHints.join(", ")}]`);
|
|
492
778
|
const getDynamicKeywordQuery = () => {
|
|
493
779
|
if (entityHints.length > 0) {
|
|
494
|
-
return entityHints.map((h) => h.
|
|
780
|
+
return entityHints.map((h) => h.replace(/\s+/g, " & ")).join(" & ");
|
|
495
781
|
}
|
|
496
782
|
if (queryText) {
|
|
497
|
-
return queryText.toLowerCase().replace(/\b(show|me|the|product[s]?|item[s]?|card[s]?|of|category|find|list|browse|what|are|display|all|under|for|in|with|about)\b/g, "").trim().replace(/\s+/g, " & ");
|
|
783
|
+
return queryText.toLowerCase().replace(/\b(show|me|the|product[s]?|item[s]?|card[s]?|organization[s]?|comp(?:any|anies)|of|category|find|list|browse|what|are|display|all|whose|that|which|have|has|having|greater|than|more|less|equal|equals|above|below|over|under|at|least|most|for|in|with|about|get|details|information|info)\b/g, "").replace(/\b\d+(?:\.\d+)?\b/g, "").trim().replace(/\s+/g, " & ");
|
|
498
784
|
}
|
|
499
785
|
return "";
|
|
500
786
|
};
|
|
501
787
|
const dynamicKeywordQuery = getDynamicKeywordQuery();
|
|
788
|
+
const tableLimit = Math.max(topK, 50);
|
|
789
|
+
const metadataFilters = _filter == null ? void 0 : _filter.metadata;
|
|
502
790
|
const queryPromises = this.tables.map(async (table) => {
|
|
503
791
|
try {
|
|
504
792
|
let sqlQuery = "";
|
|
505
793
|
let params = [];
|
|
506
|
-
|
|
794
|
+
let whereClause = "";
|
|
795
|
+
const filterParams = [];
|
|
796
|
+
if (metadataFilters && Object.keys(metadataFilters).length > 0) {
|
|
797
|
+
const conditions = Object.entries(metadataFilters).map(([key, val]) => {
|
|
798
|
+
var _a3;
|
|
799
|
+
filterParams.push(String(val));
|
|
800
|
+
const baseOffset = queryText && dynamicKeywordQuery ? 2 : 1;
|
|
801
|
+
const paramIdx = baseOffset + filterParams.length;
|
|
802
|
+
const synonyms = (_a3 = FIELD_SYNONYMS[key]) != null ? _a3 : [];
|
|
803
|
+
const keysToCheck = [key, ...synonyms];
|
|
804
|
+
const coalesceExprs = keysToCheck.map((k) => `LOWER(metadata->>'${k}')`);
|
|
805
|
+
return `COALESCE(${coalesceExprs.join(", ")}) = LOWER($${paramIdx})`;
|
|
806
|
+
});
|
|
807
|
+
whereClause = `WHERE ${conditions.join(" AND ")}`;
|
|
808
|
+
}
|
|
809
|
+
if (queryText && dynamicKeywordQuery) {
|
|
507
810
|
const hasEntityHints = entityHints.length > 0;
|
|
508
811
|
const exactNameScoreExpr = hasEntityHints ? `+ (
|
|
509
812
|
SELECT COALESCE(MAX(
|
|
@@ -519,19 +822,21 @@ var init_MultiTablePostgresProvider = __esm({
|
|
|
519
822
|
COALESCE(ts_rank(to_tsvector('english', (to_jsonb(t) - 'embedding')::text), to_tsquery('english', $2)), 0) AS keyword_score,
|
|
520
823
|
((1 - (embedding <=> $1::vector)) + (COALESCE(ts_rank(to_tsvector('english', (to_jsonb(t) - 'embedding')::text), to_tsquery('english', $2)), 0) * 2.0) ${exactNameScoreExpr}) AS hybrid_score
|
|
521
824
|
FROM "${table}" t
|
|
825
|
+
${whereClause}
|
|
522
826
|
ORDER BY hybrid_score DESC
|
|
523
|
-
LIMIT
|
|
827
|
+
LIMIT ${tableLimit}
|
|
524
828
|
`;
|
|
525
|
-
params = [vectorLiteral, dynamicKeywordQuery];
|
|
829
|
+
params = [vectorLiteral, dynamicKeywordQuery, ...filterParams];
|
|
526
830
|
} else {
|
|
527
831
|
sqlQuery = `
|
|
528
832
|
SELECT *,
|
|
529
|
-
|
|
833
|
+
(1 - (embedding <=> $1::vector)) AS hybrid_score
|
|
530
834
|
FROM "${table}" t
|
|
835
|
+
${whereClause}
|
|
531
836
|
ORDER BY hybrid_score DESC
|
|
532
|
-
LIMIT
|
|
837
|
+
LIMIT ${tableLimit}
|
|
533
838
|
`;
|
|
534
|
-
params = [vectorLiteral];
|
|
839
|
+
params = [vectorLiteral, ...filterParams];
|
|
535
840
|
}
|
|
536
841
|
const result = await this.pool.query(sqlQuery, params);
|
|
537
842
|
if (result.rowCount && result.rowCount > 0) {
|
|
@@ -854,13 +1159,25 @@ var init_MilvusProvider = __esm({
|
|
|
854
1159
|
};
|
|
855
1160
|
await this.http.post("/v1/vector/upsert", payload);
|
|
856
1161
|
}
|
|
857
|
-
// eslint-disable-next-line @typescript-eslint/no-unused-vars
|
|
858
1162
|
async query(vector, topK, namespace, _filter) {
|
|
1163
|
+
const sanitizedFilter = this.sanitizeFilter(_filter);
|
|
1164
|
+
const filterParts = [];
|
|
1165
|
+
if (namespace) {
|
|
1166
|
+
filterParts.push(`namespace == "${namespace}"`);
|
|
1167
|
+
}
|
|
1168
|
+
for (const [key, value] of Object.entries(sanitizedFilter)) {
|
|
1169
|
+
if (key === "queryText" || key === "keywords") continue;
|
|
1170
|
+
if (typeof value === "string") {
|
|
1171
|
+
filterParts.push(`metadata["${key}"] == "${value.replace(/"/g, '\\"')}"`);
|
|
1172
|
+
} else if (typeof value === "number") {
|
|
1173
|
+
filterParts.push(`metadata["${key}"] == ${value}`);
|
|
1174
|
+
}
|
|
1175
|
+
}
|
|
859
1176
|
const payload = {
|
|
860
1177
|
collectionName: this.indexName,
|
|
861
1178
|
vector,
|
|
862
1179
|
limit: topK,
|
|
863
|
-
filter:
|
|
1180
|
+
filter: filterParts.length > 0 ? filterParts.join(" && ") : void 0,
|
|
864
1181
|
outputFields: ["content", "metadata"],
|
|
865
1182
|
searchParams: {
|
|
866
1183
|
nprobe: this.config.options.nprobe || 16,
|
|
@@ -1059,6 +1376,7 @@ var init_QdrantProvider = __esm({
|
|
|
1059
1376
|
await this.http.put(`/collections/${this.indexName}/points`, payload);
|
|
1060
1377
|
}
|
|
1061
1378
|
async query(vector, topK, namespace, _filter) {
|
|
1379
|
+
var _a;
|
|
1062
1380
|
const must = [];
|
|
1063
1381
|
if (namespace) {
|
|
1064
1382
|
must.push({ key: "namespace", match: { value: namespace } });
|
|
@@ -1080,7 +1398,7 @@ var init_QdrantProvider = __esm({
|
|
|
1080
1398
|
limit: topK,
|
|
1081
1399
|
with_payload: true,
|
|
1082
1400
|
params: {
|
|
1083
|
-
hnsw_ef: this.config.options.efSearch || Math.max(topK * 20, 128),
|
|
1401
|
+
hnsw_ef: ((_a = this.config.options) == null ? void 0 : _a.efSearch) || Math.max(topK * 20, 128),
|
|
1084
1402
|
exact: false
|
|
1085
1403
|
},
|
|
1086
1404
|
filter: must.length > 0 ? { must } : void 0
|
|
@@ -1205,12 +1523,20 @@ var init_ChromaDBProvider = __esm({
|
|
|
1205
1523
|
};
|
|
1206
1524
|
await this.http.post(`/api/v1/collections/${this.collectionId}/add`, payload);
|
|
1207
1525
|
}
|
|
1208
|
-
// eslint-disable-next-line @typescript-eslint/no-unused-vars
|
|
1209
1526
|
async query(vector, topK, namespace, _filter) {
|
|
1527
|
+
const sanitizedFilter = this.sanitizeFilter(_filter);
|
|
1528
|
+
const whereClauses = [];
|
|
1529
|
+
if (namespace) whereClauses.push({ namespace: { $eq: namespace } });
|
|
1530
|
+
Object.entries(sanitizedFilter).forEach(([key, value]) => {
|
|
1531
|
+
if (key === "namespace") return;
|
|
1532
|
+
if (typeof value === "string" || typeof value === "number" || typeof value === "boolean") {
|
|
1533
|
+
whereClauses.push({ [key]: { $eq: value } });
|
|
1534
|
+
}
|
|
1535
|
+
});
|
|
1210
1536
|
const payload = {
|
|
1211
1537
|
query_embeddings: [vector],
|
|
1212
1538
|
n_results: topK,
|
|
1213
|
-
where:
|
|
1539
|
+
where: whereClauses.length > 1 ? { $and: whereClauses } : whereClauses[0]
|
|
1214
1540
|
};
|
|
1215
1541
|
const { data } = await this.http.post(`/api/v1/collections/${this.collectionId}/query`, payload);
|
|
1216
1542
|
const matches = [];
|
|
@@ -1315,15 +1641,15 @@ var init_RedisProvider = __esm({
|
|
|
1315
1641
|
console.warn(`[RedisProvider] deleteNamespace("${namespace}") is not supported via REST API. Use Redis CLI: SCAN + DEL`);
|
|
1316
1642
|
}
|
|
1317
1643
|
/**
|
|
1318
|
-
*
|
|
1319
|
-
* Returns
|
|
1644
|
+
* Check reachability via a PING command.
|
|
1645
|
+
* Returns false on connection failure so health checks surface real problems.
|
|
1320
1646
|
*/
|
|
1321
1647
|
async ping() {
|
|
1322
1648
|
try {
|
|
1323
1649
|
await this.http.post("/", ["PING"]);
|
|
1324
1650
|
return true;
|
|
1325
1651
|
} catch (e) {
|
|
1326
|
-
return
|
|
1652
|
+
return false;
|
|
1327
1653
|
}
|
|
1328
1654
|
}
|
|
1329
1655
|
async disconnect() {
|
|
@@ -1363,15 +1689,17 @@ var init_WeaviateProvider = __esm({
|
|
|
1363
1689
|
await this.ping();
|
|
1364
1690
|
}
|
|
1365
1691
|
async upsert(doc, namespace) {
|
|
1692
|
+
const primitiveMetadata = this.extractPrimitiveMetadata(doc.metadata);
|
|
1366
1693
|
const payload = {
|
|
1367
1694
|
class: this.indexName,
|
|
1368
1695
|
id: doc.id,
|
|
1369
1696
|
vector: doc.vector,
|
|
1370
|
-
properties: {
|
|
1697
|
+
properties: __spreadProps(__spreadValues({
|
|
1371
1698
|
content: doc.content,
|
|
1372
|
-
metadata: JSON.stringify(doc.metadata || {})
|
|
1699
|
+
metadata: JSON.stringify(doc.metadata || {})
|
|
1700
|
+
}, primitiveMetadata), {
|
|
1373
1701
|
namespace: namespace || ""
|
|
1374
|
-
}
|
|
1702
|
+
})
|
|
1375
1703
|
};
|
|
1376
1704
|
await this.http.post("/v1/objects", payload);
|
|
1377
1705
|
}
|
|
@@ -1381,20 +1709,22 @@ var init_WeaviateProvider = __esm({
|
|
|
1381
1709
|
class: this.indexName,
|
|
1382
1710
|
id: doc.id,
|
|
1383
1711
|
vector: doc.vector,
|
|
1384
|
-
properties: {
|
|
1712
|
+
properties: __spreadProps(__spreadValues({
|
|
1385
1713
|
content: doc.content,
|
|
1386
|
-
metadata: JSON.stringify(doc.metadata || {})
|
|
1714
|
+
metadata: JSON.stringify(doc.metadata || {})
|
|
1715
|
+
}, this.extractPrimitiveMetadata(doc.metadata)), {
|
|
1387
1716
|
namespace: namespace || ""
|
|
1388
|
-
}
|
|
1717
|
+
})
|
|
1389
1718
|
}))
|
|
1390
1719
|
};
|
|
1391
1720
|
await this.http.post("/v1/batch/objects", payload);
|
|
1392
1721
|
}
|
|
1393
1722
|
async query(vector, topK, namespace, _filter) {
|
|
1394
1723
|
var _a, _b;
|
|
1724
|
+
const queryText = _filter == null ? void 0 : _filter.queryText;
|
|
1395
1725
|
const sanitizedFilter = this.sanitizeFilter(_filter);
|
|
1396
|
-
const queryText = sanitizedFilter.queryText;
|
|
1397
1726
|
const searchParams = queryText ? `hybrid: { query: ${JSON.stringify(queryText)}, alpha: 0.5 }` : `nearVector: { vector: ${JSON.stringify(vector)} }`;
|
|
1727
|
+
const where = this.buildWhereFilter(namespace, sanitizedFilter);
|
|
1398
1728
|
const graphqlQuery = {
|
|
1399
1729
|
query: `
|
|
1400
1730
|
{
|
|
@@ -1402,7 +1732,7 @@ var init_WeaviateProvider = __esm({
|
|
|
1402
1732
|
${this.indexName}(
|
|
1403
1733
|
${searchParams}
|
|
1404
1734
|
limit: ${topK}
|
|
1405
|
-
${
|
|
1735
|
+
${where ? `where: ${where}` : ""}
|
|
1406
1736
|
) {
|
|
1407
1737
|
content
|
|
1408
1738
|
metadata
|
|
@@ -1446,6 +1776,34 @@ var init_WeaviateProvider = __esm({
|
|
|
1446
1776
|
}
|
|
1447
1777
|
async disconnect() {
|
|
1448
1778
|
}
|
|
1779
|
+
extractPrimitiveMetadata(metadata) {
|
|
1780
|
+
const result = {};
|
|
1781
|
+
Object.entries(metadata || {}).forEach(([key, value]) => {
|
|
1782
|
+
if (typeof value === "string" || typeof value === "number" || typeof value === "boolean") {
|
|
1783
|
+
result[key] = value;
|
|
1784
|
+
}
|
|
1785
|
+
});
|
|
1786
|
+
return result;
|
|
1787
|
+
}
|
|
1788
|
+
buildWhereFilter(namespace, filter) {
|
|
1789
|
+
const operands = [];
|
|
1790
|
+
if (namespace) operands.push(this.weaviateOperand("namespace", namespace));
|
|
1791
|
+
Object.entries(filter || {}).forEach(([key, value]) => {
|
|
1792
|
+
if (key === "namespace") return;
|
|
1793
|
+
if (typeof value === "string" || typeof value === "number" || typeof value === "boolean") {
|
|
1794
|
+
operands.push(this.weaviateOperand(key, value));
|
|
1795
|
+
}
|
|
1796
|
+
});
|
|
1797
|
+
if (operands.length === 0) return void 0;
|
|
1798
|
+
if (operands.length === 1) return operands[0];
|
|
1799
|
+
return `{ operator: And, operands: [${operands.join(", ")}] }`;
|
|
1800
|
+
}
|
|
1801
|
+
weaviateOperand(key, value) {
|
|
1802
|
+
const path = `path: [${JSON.stringify(key)}], operator: Equal`;
|
|
1803
|
+
if (typeof value === "number") return `{ ${path}, valueNumber: ${value} }`;
|
|
1804
|
+
if (typeof value === "boolean") return `{ ${path}, valueBoolean: ${value} }`;
|
|
1805
|
+
return `{ ${path}, valueString: ${JSON.stringify(value)} }`;
|
|
1806
|
+
}
|
|
1449
1807
|
};
|
|
1450
1808
|
}
|
|
1451
1809
|
});
|
|
@@ -1751,7 +2109,7 @@ function getRagConfig(baseConfig, env = process.env) {
|
|
|
1751
2109
|
return getEnvConfig(env, baseConfig);
|
|
1752
2110
|
}
|
|
1753
2111
|
function getEnvConfig(env = process.env, base) {
|
|
1754
|
-
var _a, _b, _c, _d, _e, _f, _g, _h, _i, _j, _k, _l, _m, _n, _o, _p, _q, _r, _s, _t, _u, _v, _w, _x, _y, _z, _A, _B, _C, _D, _E, _F, _G, _H, _I, _J, _K, _L, _M, _N, _O, _P, _Q, _R, _S, _T, _U, _V, _W, _X, _Y, _Z, __, _$, _aa, _ba, _ca, _da, _ea, _fa, _ga, _ha, _ia, _ja, _ka, _la, _ma, _na, _oa, _pa, _qa, _ra, _sa, _ta, _ua, _va, _wa, _xa, _ya, _za, _Aa, _Ba, _Ca, _Da, _Ea, _Fa;
|
|
2112
|
+
var _a, _b, _c, _d, _e, _f, _g, _h, _i, _j, _k, _l, _m, _n, _o, _p, _q, _r, _s, _t, _u, _v, _w, _x, _y, _z, _A, _B, _C, _D, _E, _F, _G, _H, _I, _J, _K, _L, _M, _N, _O, _P, _Q, _R, _S, _T, _U, _V, _W, _X, _Y, _Z, __, _$, _aa, _ba, _ca, _da, _ea, _fa, _ga, _ha, _ia, _ja, _ka, _la, _ma, _na, _oa, _pa, _qa, _ra, _sa, _ta, _ua, _va, _wa, _xa, _ya, _za, _Aa, _Ba, _Ca, _Da, _Ea, _Fa, _Ga;
|
|
1755
2113
|
const projectId = (_c = (_b = (_a = readString(env, "RAG_PROJECT_ID")) != null ? _a : readString(env, "NEXT_PUBLIC_PROJECT_ID")) != null ? _b : base == null ? void 0 : base.projectId) != null ? _c : "__default__";
|
|
1756
2114
|
const vectorProvider = readEnum(env, "VECTOR_DB_PROVIDER", "pinecone", VECTOR_DB_PROVIDERS);
|
|
1757
2115
|
const llmProvider = readEnum(env, "LLM_PROVIDER", "openai", LLM_PROVIDERS);
|
|
@@ -1805,11 +2163,22 @@ function getEnvConfig(env = process.env, base) {
|
|
|
1805
2163
|
};
|
|
1806
2164
|
const embeddingApiKeyByProvider = {
|
|
1807
2165
|
openai: readString(env, "OPENAI_API_KEY"),
|
|
2166
|
+
anthropic: readString(env, "ANTHROPIC_API_KEY"),
|
|
2167
|
+
// Anthropic needs a separate embedding provider; key kept for completeness
|
|
1808
2168
|
gemini: readString(env, "GEMINI_API_KEY"),
|
|
1809
2169
|
ollama: void 0,
|
|
1810
2170
|
universal_rest: (_ea = readString(env, "EMBEDDING_API_KEY")) != null ? _ea : readString(env, "OPENAI_API_KEY"),
|
|
1811
2171
|
custom: (_fa = readString(env, "EMBEDDING_API_KEY")) != null ? _fa : readString(env, "OPENAI_API_KEY")
|
|
1812
2172
|
};
|
|
2173
|
+
const DEFAULT_MODEL_BY_PROVIDER = {
|
|
2174
|
+
openai: "gpt-4o",
|
|
2175
|
+
anthropic: "claude-3-5-sonnet-20241022",
|
|
2176
|
+
gemini: "gemini-2.0-flash",
|
|
2177
|
+
ollama: "llama3",
|
|
2178
|
+
universal_rest: "default",
|
|
2179
|
+
rest: "default",
|
|
2180
|
+
custom: "default"
|
|
2181
|
+
};
|
|
1813
2182
|
return {
|
|
1814
2183
|
projectId,
|
|
1815
2184
|
vectorDb: {
|
|
@@ -1819,8 +2188,8 @@ function getEnvConfig(env = process.env, base) {
|
|
|
1819
2188
|
},
|
|
1820
2189
|
llm: {
|
|
1821
2190
|
provider: llmProvider,
|
|
1822
|
-
model: (_ia = readString(env, "LLM_MODEL")) != null ? _ia : "gpt-4o",
|
|
1823
|
-
apiKey: (
|
|
2191
|
+
model: (_ja = (_ia = readString(env, "LLM_MODEL")) != null ? _ia : DEFAULT_MODEL_BY_PROVIDER[llmProvider]) != null ? _ja : "gpt-4o",
|
|
2192
|
+
apiKey: (_ka = llmApiKeyByProvider[llmProvider]) != null ? _ka : "",
|
|
1824
2193
|
baseUrl: readString(env, "LLM_BASE_URL"),
|
|
1825
2194
|
systemPrompt: readString(env, "LLM_SYSTEM_PROMPT"),
|
|
1826
2195
|
maxTokens: readNumber(env, "LLM_MAX_TOKENS", 4096),
|
|
@@ -1831,7 +2200,7 @@ function getEnvConfig(env = process.env, base) {
|
|
|
1831
2200
|
},
|
|
1832
2201
|
embedding: {
|
|
1833
2202
|
provider: embeddingProvider,
|
|
1834
|
-
model: (
|
|
2203
|
+
model: (_la = readString(env, "EMBEDDING_MODEL")) != null ? _la : "text-embedding-3-small",
|
|
1835
2204
|
apiKey: embeddingApiKeyByProvider[embeddingProvider],
|
|
1836
2205
|
baseUrl: readString(env, "EMBEDDING_BASE_URL"),
|
|
1837
2206
|
dimensions: embeddingDimensions,
|
|
@@ -1842,17 +2211,17 @@ function getEnvConfig(env = process.env, base) {
|
|
|
1842
2211
|
}
|
|
1843
2212
|
},
|
|
1844
2213
|
ui: {
|
|
1845
|
-
title: (
|
|
1846
|
-
subtitle: (
|
|
1847
|
-
primaryColor: (
|
|
1848
|
-
accentColor: (
|
|
1849
|
-
logoUrl: (
|
|
1850
|
-
placeholder: (
|
|
1851
|
-
showSources: ((
|
|
1852
|
-
welcomeMessage: (
|
|
1853
|
-
visualStyle: (
|
|
1854
|
-
borderRadius: (
|
|
1855
|
-
allowUpload: ((
|
|
2214
|
+
title: (_na = (_ma = readString(env, "NEXT_PUBLIC_UI_TITLE")) != null ? _ma : readString(env, "UI_TITLE")) != null ? _na : "AI Assistant",
|
|
2215
|
+
subtitle: (_pa = (_oa = readString(env, "NEXT_PUBLIC_UI_SUBTITLE")) != null ? _oa : readString(env, "UI_SUBTITLE")) != null ? _pa : "Powered by RAG",
|
|
2216
|
+
primaryColor: (_ra = (_qa = readString(env, "NEXT_PUBLIC_PRIMARY_COLOR")) != null ? _qa : readString(env, "UI_PRIMARY_COLOR")) != null ? _ra : "#10b981",
|
|
2217
|
+
accentColor: (_ta = (_sa = readString(env, "NEXT_PUBLIC_ACCENT_COLOR")) != null ? _sa : readString(env, "UI_ACCENT_COLOR")) != null ? _ta : "#3b82f6",
|
|
2218
|
+
logoUrl: (_ua = readString(env, "NEXT_PUBLIC_LOGO_URL")) != null ? _ua : readString(env, "UI_LOGO_URL"),
|
|
2219
|
+
placeholder: (_wa = (_va = readString(env, "NEXT_PUBLIC_PLACEHOLDER")) != null ? _va : readString(env, "UI_PLACEHOLDER")) != null ? _wa : "Ask me anything\u2026",
|
|
2220
|
+
showSources: ((_ya = (_xa = readString(env, "NEXT_PUBLIC_SHOW_SOURCES")) != null ? _xa : readString(env, "UI_SHOW_SOURCES")) != null ? _ya : "true") !== "false",
|
|
2221
|
+
welcomeMessage: (_Aa = (_za = readString(env, "NEXT_PUBLIC_WELCOME_MESSAGE")) != null ? _za : readString(env, "UI_WELCOME_MESSAGE")) != null ? _Aa : "Hello! I'm your AI assistant. Ask me anything about your documents.",
|
|
2222
|
+
visualStyle: (_Ca = (_Ba = readString(env, "NEXT_PUBLIC_UI_VISUAL_STYLE")) != null ? _Ba : readString(env, "UI_VISUAL_STYLE")) != null ? _Ca : "glass",
|
|
2223
|
+
borderRadius: (_Ea = (_Da = readString(env, "NEXT_PUBLIC_UI_BORDER_RADIUS")) != null ? _Da : readString(env, "UI_BORDER_RADIUS")) != null ? _Ea : "xl",
|
|
2224
|
+
allowUpload: ((_Ga = (_Fa = readString(env, "NEXT_PUBLIC_ALLOW_UPLOAD")) != null ? _Fa : readString(env, "UI_ALLOW_UPLOAD")) != null ? _Ga : "false") === "true"
|
|
1856
2225
|
},
|
|
1857
2226
|
rag: {
|
|
1858
2227
|
topK: readNumber(env, "RAG_TOP_K", 5),
|
|
@@ -2753,7 +3122,7 @@ var VECTOR_PROFILES = {
|
|
|
2753
3122
|
// src/llm/providers/UniversalLLMAdapter.ts
|
|
2754
3123
|
var UniversalLLMAdapter = class {
|
|
2755
3124
|
constructor(config) {
|
|
2756
|
-
var _a, _b, _c, _d, _e, _f, _g;
|
|
3125
|
+
var _a, _b, _c, _d, _e, _f, _g, _h;
|
|
2757
3126
|
this.model = config.model;
|
|
2758
3127
|
const llmConfig = config;
|
|
2759
3128
|
const options = (_a = llmConfig.options) != null ? _a : {};
|
|
@@ -2767,16 +3136,18 @@ var UniversalLLMAdapter = class {
|
|
|
2767
3136
|
this.systemPrompt = (_c = llmConfig.systemPrompt) != null ? _c : "You are a helpful AI assistant. Use the provided context to answer the user.";
|
|
2768
3137
|
this.maxTokens = (_d = llmConfig.maxTokens) != null ? _d : 1024;
|
|
2769
3138
|
this.temperature = (_e = llmConfig.temperature) != null ? _e : 0;
|
|
2770
|
-
|
|
2771
|
-
|
|
3139
|
+
this.apiKey = config.apiKey;
|
|
3140
|
+
this.baseUrl = (_g = (_f = llmConfig.baseUrl) != null ? _f : this.opts.baseUrl) != null ? _g : "";
|
|
3141
|
+
if (!this.baseUrl) {
|
|
2772
3142
|
throw new Error("[UniversalLLMAdapter] baseUrl is required in config or config.options");
|
|
2773
3143
|
}
|
|
3144
|
+
this.resolvedHeaders = __spreadValues(__spreadValues({
|
|
3145
|
+
"Content-Type": "application/json"
|
|
3146
|
+
}, config.apiKey ? { Authorization: `Bearer ${config.apiKey}` } : {}), this.opts.headers || {});
|
|
2774
3147
|
this.http = axios2.create({
|
|
2775
|
-
baseURL: baseUrl,
|
|
2776
|
-
headers:
|
|
2777
|
-
|
|
2778
|
-
}, config.apiKey ? { Authorization: `Bearer ${config.apiKey}` } : {}), this.opts.headers || {}),
|
|
2779
|
-
timeout: (_g = this.opts.timeout) != null ? _g : 6e4
|
|
3148
|
+
baseURL: this.baseUrl,
|
|
3149
|
+
headers: this.resolvedHeaders,
|
|
3150
|
+
timeout: (_h = this.opts.timeout) != null ? _h : 6e4
|
|
2780
3151
|
});
|
|
2781
3152
|
}
|
|
2782
3153
|
async chat(messages, context) {
|
|
@@ -2813,6 +3184,92 @@ ${context != null ? context : "None"}` },
|
|
|
2813
3184
|
}
|
|
2814
3185
|
return String(result);
|
|
2815
3186
|
}
|
|
3187
|
+
/**
|
|
3188
|
+
* Streaming chat using native fetch + ReadableStream.
|
|
3189
|
+
* Parses OpenAI-compatible SSE frames: `data: {...}\n\n`
|
|
3190
|
+
* Works with vLLM, LMStudio, Together AI, Fireworks, and any OpenAI-compatible API.
|
|
3191
|
+
*/
|
|
3192
|
+
chatStream(messages, context) {
|
|
3193
|
+
return __asyncGenerator(this, null, function* () {
|
|
3194
|
+
var _a, _b, _c;
|
|
3195
|
+
const path = (_a = this.opts.chatPath) != null ? _a : "/chat/completions";
|
|
3196
|
+
const url = `${this.baseUrl.replace(/\/$/, "")}${path}`;
|
|
3197
|
+
const extractPath = ((_b = this.opts.responseExtractPath) != null ? _b : "choices[0].message.content").replace("message.content", "delta.content");
|
|
3198
|
+
const formattedMessages = [
|
|
3199
|
+
{ role: "system", content: `${this.systemPrompt}
|
|
3200
|
+
|
|
3201
|
+
Context:
|
|
3202
|
+
${context != null ? context : "None"}` },
|
|
3203
|
+
...messages
|
|
3204
|
+
];
|
|
3205
|
+
let payload;
|
|
3206
|
+
if (this.opts.chatPayloadTemplate) {
|
|
3207
|
+
payload = buildPayload(this.opts.chatPayloadTemplate, {
|
|
3208
|
+
model: this.model,
|
|
3209
|
+
messages: formattedMessages,
|
|
3210
|
+
maxTokens: this.maxTokens,
|
|
3211
|
+
temperature: this.temperature
|
|
3212
|
+
});
|
|
3213
|
+
if (typeof payload === "object" && payload !== null) {
|
|
3214
|
+
payload.stream = true;
|
|
3215
|
+
}
|
|
3216
|
+
} else {
|
|
3217
|
+
payload = {
|
|
3218
|
+
model: this.model,
|
|
3219
|
+
messages: formattedMessages,
|
|
3220
|
+
max_tokens: this.maxTokens,
|
|
3221
|
+
temperature: this.temperature,
|
|
3222
|
+
stream: true
|
|
3223
|
+
};
|
|
3224
|
+
}
|
|
3225
|
+
const response = yield new __await(fetch(url, {
|
|
3226
|
+
method: "POST",
|
|
3227
|
+
headers: this.resolvedHeaders,
|
|
3228
|
+
body: JSON.stringify(payload)
|
|
3229
|
+
}));
|
|
3230
|
+
if (!response.ok) {
|
|
3231
|
+
const errorText = yield new __await(response.text().catch(() => response.statusText));
|
|
3232
|
+
throw new Error(`[UniversalLLMAdapter] Streaming request failed (${response.status}): ${errorText}`);
|
|
3233
|
+
}
|
|
3234
|
+
if (!response.body) {
|
|
3235
|
+
throw new Error("[UniversalLLMAdapter] Response body is null \u2014 server did not send a streaming response.");
|
|
3236
|
+
}
|
|
3237
|
+
const reader = response.body.getReader();
|
|
3238
|
+
const decoder = new TextDecoder("utf-8");
|
|
3239
|
+
let buffer = "";
|
|
3240
|
+
try {
|
|
3241
|
+
while (true) {
|
|
3242
|
+
const { done, value } = yield new __await(reader.read());
|
|
3243
|
+
if (done) break;
|
|
3244
|
+
buffer += decoder.decode(value, { stream: true });
|
|
3245
|
+
const lines = buffer.split("\n");
|
|
3246
|
+
buffer = (_c = lines.pop()) != null ? _c : "";
|
|
3247
|
+
for (const line of lines) {
|
|
3248
|
+
const trimmed = line.trim();
|
|
3249
|
+
if (!trimmed || trimmed === "data: [DONE]") continue;
|
|
3250
|
+
if (!trimmed.startsWith("data:")) continue;
|
|
3251
|
+
try {
|
|
3252
|
+
const json = JSON.parse(trimmed.slice(5).trim());
|
|
3253
|
+
const text = resolvePath(json, extractPath);
|
|
3254
|
+
if (text && typeof text === "string") yield text;
|
|
3255
|
+
} catch (e) {
|
|
3256
|
+
}
|
|
3257
|
+
}
|
|
3258
|
+
}
|
|
3259
|
+
if (buffer.trim() && buffer.trim() !== "data: [DONE]") {
|
|
3260
|
+
const jsonStr = buffer.replace(/^data:\s*/, "").trim();
|
|
3261
|
+
try {
|
|
3262
|
+
const json = JSON.parse(jsonStr);
|
|
3263
|
+
const text = resolvePath(json, extractPath);
|
|
3264
|
+
if (text && typeof text === "string") yield text;
|
|
3265
|
+
} catch (e) {
|
|
3266
|
+
}
|
|
3267
|
+
}
|
|
3268
|
+
} finally {
|
|
3269
|
+
reader.releaseLock();
|
|
3270
|
+
}
|
|
3271
|
+
});
|
|
3272
|
+
}
|
|
2816
3273
|
async embed(text) {
|
|
2817
3274
|
var _a, _b;
|
|
2818
3275
|
const path = (_a = this.opts.embedPath) != null ? _a : "/embeddings";
|
|
@@ -3009,6 +3466,7 @@ var ProviderRegistry = class {
|
|
|
3009
3466
|
return null;
|
|
3010
3467
|
}
|
|
3011
3468
|
static async loadVectorProviderClass(provider) {
|
|
3469
|
+
var _a;
|
|
3012
3470
|
if (this.vectorProviders[provider]) return this.vectorProviders[provider];
|
|
3013
3471
|
switch (provider) {
|
|
3014
3472
|
case "pinecone": {
|
|
@@ -3017,6 +3475,11 @@ var ProviderRegistry = class {
|
|
|
3017
3475
|
}
|
|
3018
3476
|
case "pgvector":
|
|
3019
3477
|
case "postgresql": {
|
|
3478
|
+
const postgresMode = ((_a = process.env.POSTGRES_MODE) != null ? _a : "multi").toLowerCase();
|
|
3479
|
+
if (postgresMode === "single") {
|
|
3480
|
+
const { PostgreSQLProvider: PostgreSQLProvider2 } = await Promise.resolve().then(() => (init_PostgreSQLProvider(), PostgreSQLProvider_exports));
|
|
3481
|
+
return PostgreSQLProvider2;
|
|
3482
|
+
}
|
|
3020
3483
|
const { MultiTablePostgresProvider: MultiTablePostgresProvider2 } = await Promise.resolve().then(() => (init_MultiTablePostgresProvider(), MultiTablePostgresProvider_exports));
|
|
3021
3484
|
return MultiTablePostgresProvider2;
|
|
3022
3485
|
}
|
|
@@ -3998,7 +4461,8 @@ var QueryProcessor = class {
|
|
|
3998
4461
|
const fieldValuePatterns = [
|
|
3999
4462
|
new RegExp(`\\b${fieldPattern}\\s*(?:=|:)\\s*["']?${valuePattern}["']?(?=[?.!,]|$)`, "gi"),
|
|
4000
4463
|
new RegExp(`\\b${fieldPattern}\\s+(?:is|are|was|were|equals?|equal to|named|called)\\s+["']?${valuePattern}["']?(?=[?.!,]|$)`, "gi"),
|
|
4001
|
-
new RegExp(`\\bwith\\s+${fieldPattern}\\s+["']?${valuePattern}["']?(?=[?.!,]|$)`, "gi")
|
|
4464
|
+
new RegExp(`\\bwith\\s+${fieldPattern}\\s+["']?${valuePattern}["']?(?=[?.!,]|$)`, "gi"),
|
|
4465
|
+
new RegExp(`\\b(?:under|in|for|from|of)\\s+${fieldPattern}\\s+["']?${valuePattern}["']?(?=[?.!,]|$)`, "gi")
|
|
4002
4466
|
];
|
|
4003
4467
|
for (const pattern of fieldValuePatterns) {
|
|
4004
4468
|
for (const match of question.matchAll(pattern)) {
|
|
@@ -4016,6 +4480,76 @@ var QueryProcessor = class {
|
|
|
4016
4480
|
}
|
|
4017
4481
|
return [...hints.values()];
|
|
4018
4482
|
}
|
|
4483
|
+
static extractNumericPredicates(question, validFields = []) {
|
|
4484
|
+
const predicates = [];
|
|
4485
|
+
const seen = /* @__PURE__ */ new Set();
|
|
4486
|
+
const comparatorPattern = [
|
|
4487
|
+
"greater than or equal to",
|
|
4488
|
+
"more than or equal to",
|
|
4489
|
+
"less than or equal to",
|
|
4490
|
+
"greater than",
|
|
4491
|
+
"more than",
|
|
4492
|
+
"less than",
|
|
4493
|
+
"equal to",
|
|
4494
|
+
"at least",
|
|
4495
|
+
"at most",
|
|
4496
|
+
"above",
|
|
4497
|
+
"over",
|
|
4498
|
+
"below",
|
|
4499
|
+
"under",
|
|
4500
|
+
"equals?",
|
|
4501
|
+
">=",
|
|
4502
|
+
"<=",
|
|
4503
|
+
">",
|
|
4504
|
+
"<",
|
|
4505
|
+
"="
|
|
4506
|
+
].join("|");
|
|
4507
|
+
const addPredicate = (rawField, rawOperator, rawValue) => {
|
|
4508
|
+
const value = Number(rawValue.replace(/,/g, ""));
|
|
4509
|
+
if (!Number.isFinite(value)) return;
|
|
4510
|
+
const operator = this.normalizeNumericOperator(rawOperator);
|
|
4511
|
+
const field = rawField ? this.normalizePredicateField(rawField, validFields) : void 0;
|
|
4512
|
+
const key = `${field != null ? field : "*"}::${operator}::${value}`;
|
|
4513
|
+
if (seen.has(key)) return;
|
|
4514
|
+
seen.add(key);
|
|
4515
|
+
predicates.push(__spreadProps(__spreadValues({}, field ? { field } : {}), { operator, value }));
|
|
4516
|
+
};
|
|
4517
|
+
const scopedPatterns = [
|
|
4518
|
+
new RegExp(`\\b(?:whose|with|having|where|that\\s+have|which\\s+have)\\s+([a-zA-Z][a-zA-Z0-9_\\s\\-/]{1,80}?)\\s+(?:is|are|was|were)?\\s*(?:${comparatorPattern})\\s+([\\d,]+(?:\\.\\d+)?)`, "gi"),
|
|
4519
|
+
new RegExp(`\\b([a-zA-Z][a-zA-Z0-9_\\s\\-/]{1,80}?)\\s+(?:is|are|was|were)?\\s*(?:${comparatorPattern})\\s+([\\d,]+(?:\\.\\d+)?)`, "gi")
|
|
4520
|
+
];
|
|
4521
|
+
for (const pattern of scopedPatterns) {
|
|
4522
|
+
for (const match of question.matchAll(pattern)) {
|
|
4523
|
+
const full = match[0];
|
|
4524
|
+
const operatorMatch = full.match(new RegExp(`(${comparatorPattern})`, "i"));
|
|
4525
|
+
if (!operatorMatch) continue;
|
|
4526
|
+
addPredicate(match[1], operatorMatch[1], match[2]);
|
|
4527
|
+
}
|
|
4528
|
+
}
|
|
4529
|
+
for (const match of question.matchAll(/\b([a-zA-Z][a-zA-Z0-9_\s\-/]{1,80}?)\s*(>=|<=|>|<|=)\s*([\d,]+(?:\.\d+)?)/g)) {
|
|
4530
|
+
addPredicate(match[1], match[2], match[3]);
|
|
4531
|
+
}
|
|
4532
|
+
return predicates;
|
|
4533
|
+
}
|
|
4534
|
+
static normalizeNumericOperator(operator) {
|
|
4535
|
+
const op = operator.toLowerCase().trim();
|
|
4536
|
+
if (op === ">" || /\b(greater than|more than|above|over)\b/.test(op)) return "gt";
|
|
4537
|
+
if (op === ">=" || /\b(greater than or equal to|more than or equal to|at least)\b/.test(op)) return "gte";
|
|
4538
|
+
if (op === "<" || /\b(less than|below|under)\b/.test(op)) return "lt";
|
|
4539
|
+
if (op === "<=" || /\b(less than or equal to|at most)\b/.test(op)) return "lte";
|
|
4540
|
+
return "eq";
|
|
4541
|
+
}
|
|
4542
|
+
static normalizePredicateField(field, validFields) {
|
|
4543
|
+
const cleaned = this.normalizeHintValue(field).replace(/\b(?:provide|show|get|give|list|all|the|a|an|of|organizations?|companies?|records?|items?|whose|with|having|where|that|which|have|has|is|are|was|were)\b/gi, " ").replace(/\s+/g, " ").trim();
|
|
4544
|
+
if (validFields.length === 0) return cleaned;
|
|
4545
|
+
const comparable = (value) => value.toLowerCase().replace(/[^a-z0-9]/g, "");
|
|
4546
|
+
const cleanedComparable = comparable(cleaned);
|
|
4547
|
+
const matchedField = validFields.find((fieldName) => {
|
|
4548
|
+
const candidate = comparable(fieldName);
|
|
4549
|
+
return candidate === cleanedComparable || candidate.includes(cleanedComparable) || cleanedComparable.includes(candidate);
|
|
4550
|
+
});
|
|
4551
|
+
return matchedField != null ? matchedField : cleaned;
|
|
4552
|
+
}
|
|
4019
4553
|
/**
|
|
4020
4554
|
* Constructs a QueryFilter object from extracted hints.
|
|
4021
4555
|
*
|
|
@@ -4036,6 +4570,10 @@ var QueryProcessor = class {
|
|
|
4036
4570
|
}
|
|
4037
4571
|
if (Object.keys(filter.metadata || {}).length === 0) delete filter.metadata;
|
|
4038
4572
|
if (filter.keywords && filter.keywords.length === 0) delete filter.keywords;
|
|
4573
|
+
const numericPredicates = this.extractNumericPredicates(question);
|
|
4574
|
+
if (numericPredicates.length > 0) {
|
|
4575
|
+
filter.__numericPredicates = numericPredicates;
|
|
4576
|
+
}
|
|
4039
4577
|
return filter;
|
|
4040
4578
|
}
|
|
4041
4579
|
/**
|
|
@@ -4103,102 +4641,360 @@ Return ONLY 'vector', 'graph', or 'both'. No explanation.`;
|
|
|
4103
4641
|
}
|
|
4104
4642
|
};
|
|
4105
4643
|
|
|
4106
|
-
// src/
|
|
4107
|
-
var
|
|
4108
|
-
|
|
4109
|
-
|
|
4110
|
-
|
|
4111
|
-
|
|
4112
|
-
|
|
4113
|
-
|
|
4114
|
-
|
|
4115
|
-
|
|
4116
|
-
|
|
4117
|
-
|
|
4118
|
-
|
|
4119
|
-
|
|
4120
|
-
|
|
4121
|
-
|
|
4122
|
-
|
|
4123
|
-
|
|
4124
|
-
|
|
4125
|
-
|
|
4126
|
-
|
|
4127
|
-
|
|
4128
|
-
|
|
4129
|
-
|
|
4130
|
-
|
|
4131
|
-
"
|
|
4132
|
-
|
|
4133
|
-
"
|
|
4134
|
-
|
|
4135
|
-
|
|
4136
|
-
|
|
4644
|
+
// src/core/LLMRouter.ts
|
|
4645
|
+
var FAST_MODEL_DEFAULTS = {
|
|
4646
|
+
openai: "gpt-4o-mini",
|
|
4647
|
+
gemini: "gemini-2.0-flash",
|
|
4648
|
+
anthropic: "claude-3-haiku-20240307",
|
|
4649
|
+
ollama: "",
|
|
4650
|
+
// Ollama has no universal lightweight default — reuse main model
|
|
4651
|
+
rest: "",
|
|
4652
|
+
universal_rest: "",
|
|
4653
|
+
custom: ""
|
|
4654
|
+
};
|
|
4655
|
+
var LLMRouter = class {
|
|
4656
|
+
constructor(config) {
|
|
4657
|
+
this.config = config;
|
|
4658
|
+
this.models = /* @__PURE__ */ new Map();
|
|
4659
|
+
}
|
|
4660
|
+
/**
|
|
4661
|
+
* Initialize all LLM roles.
|
|
4662
|
+
*
|
|
4663
|
+
* @param prebuiltDefault - optional pre-built provider (from EmbeddingStrategyResolver).
|
|
4664
|
+
* When provided it is used directly as the 'default' role without re-constructing.
|
|
4665
|
+
*/
|
|
4666
|
+
async initialize(prebuiltDefault) {
|
|
4667
|
+
var _a;
|
|
4668
|
+
const defaultModel = prebuiltDefault != null ? prebuiltDefault : LLMFactory.create(this.config.llm, this.config.embedding);
|
|
4669
|
+
this.models.set("default", defaultModel);
|
|
4670
|
+
const envFastModel = process.env.FAST_LLM_MODEL;
|
|
4671
|
+
const providerFastDefault = (_a = FAST_MODEL_DEFAULTS[this.config.llm.provider]) != null ? _a : "";
|
|
4672
|
+
const fastModelName = envFastModel || providerFastDefault;
|
|
4673
|
+
if (fastModelName && fastModelName !== this.config.llm.model) {
|
|
4674
|
+
console.log(`[LLMRouter] Fast role \u2192 provider="${this.config.llm.provider}" model="${fastModelName}"`);
|
|
4675
|
+
const fastConfig = __spreadProps(__spreadValues({}, this.config.llm), {
|
|
4676
|
+
model: fastModelName
|
|
4677
|
+
});
|
|
4678
|
+
this.models.set("fast", LLMFactory.create(fastConfig, this.config.embedding));
|
|
4679
|
+
} else {
|
|
4680
|
+
console.log(`[LLMRouter] Fast role \u2192 reusing default model (no lightweight alternative configured).`);
|
|
4681
|
+
this.models.set("fast", defaultModel);
|
|
4682
|
+
}
|
|
4683
|
+
this.models.set("powerful", defaultModel);
|
|
4684
|
+
}
|
|
4685
|
+
/**
|
|
4686
|
+
* Retrieve a model provider by its task role.
|
|
4687
|
+
* Falls back to 'default' if the requested role is not registered.
|
|
4688
|
+
*/
|
|
4689
|
+
get(role) {
|
|
4690
|
+
var _a;
|
|
4691
|
+
const provider = (_a = this.models.get(role)) != null ? _a : this.models.get("default");
|
|
4692
|
+
if (!provider) {
|
|
4693
|
+
throw new Error(`[LLMRouter] No provider registered for role: "${role}". Did you call initialize()?`);
|
|
4694
|
+
}
|
|
4695
|
+
return provider;
|
|
4696
|
+
}
|
|
4137
4697
|
};
|
|
4138
|
-
function resolveMetadataValue(meta, uiKey) {
|
|
4139
|
-
var _a;
|
|
4140
|
-
const synonyms = (_a = FIELD_SYNONYMS[uiKey]) != null ? _a : [];
|
|
4141
|
-
const keys = Object.keys(meta);
|
|
4142
|
-
const systemKeys = ["namespace", "filename", "filesize", "filetype", "docid", "uploadedat", "dimension", "chunkindex"];
|
|
4143
|
-
let match = keys.find((k) => k.toLowerCase() === uiKey.toLowerCase());
|
|
4144
|
-
if (match !== void 0) return meta[match];
|
|
4145
|
-
match = keys.find((k) => synonyms.map((s) => s.toLowerCase()).includes(k.toLowerCase()));
|
|
4146
|
-
if (match !== void 0) return meta[match];
|
|
4147
|
-
const isBlacklisted = (kl) => {
|
|
4148
|
-
return systemKeys.includes(kl) || kl.startsWith("option1") || kl.startsWith("option2") || kl.startsWith("option3");
|
|
4149
|
-
};
|
|
4150
|
-
match = keys.find((k) => {
|
|
4151
|
-
const kl = k.toLowerCase();
|
|
4152
|
-
if (isBlacklisted(kl)) return false;
|
|
4153
|
-
return kl.includes(uiKey.toLowerCase());
|
|
4154
|
-
});
|
|
4155
|
-
if (match !== void 0) return meta[match];
|
|
4156
|
-
match = keys.find((k) => {
|
|
4157
|
-
const kl = k.toLowerCase();
|
|
4158
|
-
if (isBlacklisted(kl)) return false;
|
|
4159
|
-
return synonyms.some((sk) => kl.includes(sk.toLowerCase()) || sk.toLowerCase().includes(kl));
|
|
4160
|
-
});
|
|
4161
|
-
return match !== void 0 ? meta[match] : void 0;
|
|
4162
|
-
}
|
|
4163
4698
|
|
|
4164
4699
|
// src/utils/UITransformer.ts
|
|
4700
|
+
init_synonyms();
|
|
4165
4701
|
var UITransformer = class {
|
|
4702
|
+
// ─── Public Entry Points ─────────────────────────────────────────────────
|
|
4166
4703
|
/**
|
|
4167
|
-
*
|
|
4168
|
-
*
|
|
4704
|
+
* Heuristic-only transform (no LLM required).
|
|
4705
|
+
* Uses the lightweight heuristic intent detector as a fallback.
|
|
4706
|
+
* Prefer `analyzeAndDecide()` in production.
|
|
4169
4707
|
*/
|
|
4170
|
-
static transform(userQuery, retrievedData, config, trainedSchema) {
|
|
4708
|
+
static transform(userQuery, retrievedData, config, trainedSchema, intent) {
|
|
4709
|
+
var _a, _b, _c;
|
|
4171
4710
|
if (!retrievedData || retrievedData.length === 0) {
|
|
4172
4711
|
return this.createTextResponse("No data available", "No relevant data found for your query.");
|
|
4173
4712
|
}
|
|
4174
|
-
const
|
|
4175
|
-
const filteredData =
|
|
4176
|
-
const
|
|
4713
|
+
const resolvedIntent = intent != null ? intent : this.detectIntentHeuristic(userQuery);
|
|
4714
|
+
const filteredData = resolvedIntent.filterInStockOnly ? retrievedData.filter((item) => this.determineStockStatus(item)) : retrievedData;
|
|
4715
|
+
const profile = this.profileData(filteredData);
|
|
4177
4716
|
const hasProducts = filteredData.some((item) => this.isProductData(item));
|
|
4178
|
-
const
|
|
4179
|
-
|
|
4180
|
-
|
|
4181
|
-
return this.transformToLineChart(filteredData);
|
|
4717
|
+
const wantsPieLikeChart = ["pie_chart", "donut_chart"].includes(resolvedIntent.recommendedChart);
|
|
4718
|
+
if (resolvedIntent.visualizationHint === "trend" && profile.dateFields.length > 0 && profile.numericFields.length > 0) {
|
|
4719
|
+
return this.transformToLineChart(profile);
|
|
4182
4720
|
}
|
|
4183
|
-
if (
|
|
4184
|
-
|
|
4721
|
+
if (wantsPieLikeChart || ["composition", "category_breakdown"].includes(resolvedIntent.visualizationHint)) {
|
|
4722
|
+
const pieChart = this.transformToPieChart(filteredData, profile, userQuery);
|
|
4723
|
+
if (pieChart) return pieChart;
|
|
4185
4724
|
}
|
|
4186
|
-
if (
|
|
4187
|
-
return this.
|
|
4725
|
+
if (["comparison", "ranking"].includes(resolvedIntent.visualizationHint)) {
|
|
4726
|
+
return this.transformToBarChart(filteredData, profile, userQuery, resolvedIntent.visualizationHint === "ranking");
|
|
4188
4727
|
}
|
|
4189
|
-
if (
|
|
4190
|
-
return this.
|
|
4728
|
+
if (resolvedIntent.visualizationHint === "distribution") {
|
|
4729
|
+
return (_a = this.transformToHistogram(profile, userQuery)) != null ? _a : this.transformToBarChart(filteredData, profile, userQuery);
|
|
4730
|
+
}
|
|
4731
|
+
if (resolvedIntent.visualizationHint === "correlation") {
|
|
4732
|
+
return (_b = this.transformToScatterPlot(profile, userQuery)) != null ? _b : this.transformToBarChart(filteredData, profile, userQuery);
|
|
4733
|
+
}
|
|
4734
|
+
if (resolvedIntent.visualizationHint === "geographic") {
|
|
4735
|
+
return this.transformToBarChart(filteredData, profile, userQuery);
|
|
4736
|
+
}
|
|
4737
|
+
if (this.isStructuredListQuery(userQuery)) {
|
|
4738
|
+
return this.hasMultipleFields(filteredData) ? this.transformToTable(filteredData, userQuery) : this.transformToText(filteredData);
|
|
4739
|
+
}
|
|
4740
|
+
if (resolvedIntent.visualizationHint === "kpi") {
|
|
4741
|
+
return (_c = this.transformToMetricCard(profile, userQuery)) != null ? _c : this.transformToText(filteredData);
|
|
4191
4742
|
}
|
|
4192
|
-
if (
|
|
4193
|
-
return this.transformToTable(filteredData);
|
|
4743
|
+
if (["tabular", "table"].includes(resolvedIntent.visualizationHint) || resolvedIntent.wantsExplicitTable) {
|
|
4744
|
+
return this.hasMultipleFields(filteredData) ? this.transformToTable(filteredData, userQuery) : this.transformToText(filteredData);
|
|
4194
4745
|
}
|
|
4746
|
+
if ((hasProducts || this.isProductQuery(userQuery)) && resolvedIntent.visualizationHint === "product_browse") {
|
|
4747
|
+
return this.transformToProductCarousel(filteredData, config, trainedSchema);
|
|
4748
|
+
}
|
|
4749
|
+
const automatic = this.chooseAutomaticVisualization(filteredData, profile, userQuery);
|
|
4750
|
+
if (automatic) return automatic;
|
|
4195
4751
|
return this.transformToText(filteredData);
|
|
4196
4752
|
}
|
|
4197
4753
|
/**
|
|
4198
|
-
*
|
|
4754
|
+
* LLM-driven entry point (recommended for production).
|
|
4755
|
+
*
|
|
4756
|
+
* Step 1 — Detect intent via a dedicated, lightweight LLM call.
|
|
4757
|
+
* Step 2 — Pass the intent + data to the visualization-selection prompt.
|
|
4758
|
+
* Step 3 — Fall back to the heuristic `transform()` if either LLM call fails.
|
|
4199
4759
|
*/
|
|
4760
|
+
static async analyzeAndDecide(query, sources, llm) {
|
|
4761
|
+
let intent;
|
|
4762
|
+
try {
|
|
4763
|
+
intent = await this.detectIntent(query, llm);
|
|
4764
|
+
console.debug("[UITransformer] Detected intent:", intent);
|
|
4765
|
+
} catch (err) {
|
|
4766
|
+
console.warn("[UITransformer] Intent detection failed; using heuristic.", err);
|
|
4767
|
+
intent = this.detectIntentHeuristic(query);
|
|
4768
|
+
}
|
|
4769
|
+
if (this.isProductQuery(query) && ["text", "product_browse"].includes(intent.visualizationHint)) {
|
|
4770
|
+
return this.transform(
|
|
4771
|
+
query,
|
|
4772
|
+
sources,
|
|
4773
|
+
void 0,
|
|
4774
|
+
void 0,
|
|
4775
|
+
__spreadProps(__spreadValues({}, intent), { visualizationHint: "product_browse", recommendedChart: "text" })
|
|
4776
|
+
);
|
|
4777
|
+
}
|
|
4778
|
+
try {
|
|
4779
|
+
const context = this.buildContextSummary(sources);
|
|
4780
|
+
const systemPrompt = this.buildVisualizationSystemPrompt();
|
|
4781
|
+
const userPrompt = [
|
|
4782
|
+
`USER QUESTION: ${query}`,
|
|
4783
|
+
"",
|
|
4784
|
+
`DETECTED INTENT (JSON): ${JSON.stringify(intent)}`,
|
|
4785
|
+
"",
|
|
4786
|
+
"RETRIEVED DATA (JSON):",
|
|
4787
|
+
context
|
|
4788
|
+
].join("\n");
|
|
4789
|
+
const rawResponse = await llm.chat(
|
|
4790
|
+
[{ role: "user", content: userPrompt }],
|
|
4791
|
+
"",
|
|
4792
|
+
{ systemPrompt, temperature: 0 }
|
|
4793
|
+
);
|
|
4794
|
+
const parsed = this.parseTransformationResponse(rawResponse);
|
|
4795
|
+
if (parsed) {
|
|
4796
|
+
const intentAllowsTable = intent.wantsExplicitTable || ["tabular", "table", "geographic"].includes(intent.visualizationHint);
|
|
4797
|
+
const intentWantsPieLikeChart = ["pie_chart", "donut_chart"].includes(intent.recommendedChart) || ["composition", "category_breakdown"].includes(intent.visualizationHint);
|
|
4798
|
+
if (parsed.type === "table" && !intentAllowsTable) {
|
|
4799
|
+
console.debug("[UITransformer] LLM chose table but intent says no. Falling back to text.");
|
|
4800
|
+
return this.transform(query, sources, void 0, void 0, intent);
|
|
4801
|
+
}
|
|
4802
|
+
if (intentWantsPieLikeChart && parsed.type !== "pie_chart" && this.detectCategories(sources).length > 1) {
|
|
4803
|
+
console.debug("[UITransformer] LLM ignored pie/composition intent. Using deterministic pie chart.");
|
|
4804
|
+
return this.transform(query, sources, void 0, void 0, intent);
|
|
4805
|
+
}
|
|
4806
|
+
console.debug("[UITransformer] LLM chose visualization type:", parsed.type);
|
|
4807
|
+
return parsed;
|
|
4808
|
+
}
|
|
4809
|
+
console.warn("[UITransformer] LLM returned unparseable response; falling back to heuristic.");
|
|
4810
|
+
} catch (err) {
|
|
4811
|
+
console.warn("[UITransformer] analyzeAndDecide LLM call failed; falling back to heuristic.", err);
|
|
4812
|
+
}
|
|
4813
|
+
return this.transform(query, sources, void 0, void 0, intent);
|
|
4814
|
+
}
|
|
4815
|
+
// ─── Dynamic Intent Detection ─────────────────────────────────────────────
|
|
4816
|
+
/**
|
|
4817
|
+
* Calls the LLM with a compact, focused prompt to extract a structured
|
|
4818
|
+
* `QueryIntent` from the user's query.
|
|
4819
|
+
*
|
|
4820
|
+
* Keeping this as a *separate* call from visualization selection means:
|
|
4821
|
+
* - The prompt is shorter and more reliable.
|
|
4822
|
+
* - The intent object can be reused across both the heuristic and LLM paths.
|
|
4823
|
+
* - It is easy to unit-test intent detection in isolation.
|
|
4824
|
+
*/
|
|
4825
|
+
static async detectIntent(query, llm) {
|
|
4826
|
+
const systemPrompt = `You are an intent classifier for a product-search RAG system.
|
|
4827
|
+
Given a user query, return ONLY a valid JSON object with this exact shape \u2014 no prose, no markdown:
|
|
4828
|
+
|
|
4829
|
+
{
|
|
4830
|
+
"visualizationHint": "trend" | "comparison" | "distribution" | "composition" | "correlation" | "ranking" | "kpi" | "tabular" | "geographic" | "product_browse" | "table" | "text",
|
|
4831
|
+
"recommendedChart": "line_chart" | "bar_chart" | "histogram" | "pie_chart" | "donut_chart" | "scatter_plot" | "horizontal_bar" | "metric_card" | "table" | "geo_map" | "text",
|
|
4832
|
+
"filterInStockOnly": boolean,
|
|
4833
|
+
"wantsExplicitTable": boolean,
|
|
4834
|
+
"isTemporal": boolean,
|
|
4835
|
+
"isComparison": boolean,
|
|
4836
|
+
"language": "<BCP-47 language tag, e.g. en, de, hi, ja>",
|
|
4837
|
+
"reasoning": "<one short sentence \u2014 why you chose these values>"
|
|
4838
|
+
}
|
|
4839
|
+
|
|
4840
|
+
RULES:
|
|
4841
|
+
- visualizationHint and recommendedChart
|
|
4842
|
+
"trend" \u2192 keywords: trend, growth, over time; recommendedChart "line_chart"
|
|
4843
|
+
"comparison" \u2192 keywords: compare, versus, top; recommendedChart "bar_chart"
|
|
4844
|
+
"distribution" \u2192 keywords: distribution, spread; recommendedChart "histogram"
|
|
4845
|
+
"composition" \u2192 keywords: share, percentage, breakup, breakdown; recommendedChart "pie_chart" or "donut_chart"
|
|
4846
|
+
"correlation" \u2192 keywords: relation, correlation; recommendedChart "scatter_plot"
|
|
4847
|
+
"ranking" \u2192 keywords: highest, lowest, top 10, ranking; recommendedChart "horizontal_bar"
|
|
4848
|
+
"kpi" \u2192 keywords: total, average, count; recommendedChart "metric_card"
|
|
4849
|
+
"tabular" \u2192 keywords: detailed records, table, grid, spreadsheet; recommendedChart "table"
|
|
4850
|
+
"geographic" \u2192 keywords: region, country, map; recommendedChart "geo_map"
|
|
4851
|
+
"product_browse" \u2192 user is browsing, searching, describing, viewing details for, or asking about one or more products without analytical visualization intent
|
|
4852
|
+
"table" \u2192 legacy alias for "tabular"; use only if the query literally says table
|
|
4853
|
+
"text" \u2192 conversational, factual, or other intent
|
|
4854
|
+
- If the user explicitly asks for a chart type, honor that chart type in recommendedChart.
|
|
4855
|
+
- If a query says "distribution ... in a pie chart", use visualizationHint "composition" and recommendedChart "pie_chart".
|
|
4856
|
+
- filterInStockOnly: true only if user mentions stock, availability, in stock, etc.
|
|
4857
|
+
- wantsExplicitTable: true if the user asks for a table, list, grid, spreadsheet, or detailed records.
|
|
4858
|
+
- isTemporal: true if time words appear (trend, historical, over time, last year, monthly, etc.)
|
|
4859
|
+
- isComparison: true if user compares, ranks, or contrasts entities or categories.
|
|
4860
|
+
- language: detect from the query text itself; default "en" if uncertain.`;
|
|
4861
|
+
const rawResponse = await llm.chat(
|
|
4862
|
+
[{ role: "user", content: `QUERY: ${query}` }],
|
|
4863
|
+
"",
|
|
4864
|
+
{ systemPrompt, temperature: 0 }
|
|
4865
|
+
);
|
|
4866
|
+
const parsed = this.parseIntentResponse(rawResponse);
|
|
4867
|
+
if (!parsed) {
|
|
4868
|
+
throw new Error(`Could not parse intent JSON from LLM response: ${rawResponse}`);
|
|
4869
|
+
}
|
|
4870
|
+
return parsed;
|
|
4871
|
+
}
|
|
4872
|
+
/**
|
|
4873
|
+
* Parse and validate the raw LLM response into a `QueryIntent`.
|
|
4874
|
+
*/
|
|
4875
|
+
static parseIntentResponse(raw) {
|
|
4876
|
+
const jsonStr = this.extractJsonCandidate(raw);
|
|
4877
|
+
if (!jsonStr) return null;
|
|
4878
|
+
try {
|
|
4879
|
+
const obj = JSON.parse(jsonStr);
|
|
4880
|
+
const validHints = [
|
|
4881
|
+
"trend",
|
|
4882
|
+
"comparison",
|
|
4883
|
+
"distribution",
|
|
4884
|
+
"composition",
|
|
4885
|
+
"correlation",
|
|
4886
|
+
"ranking",
|
|
4887
|
+
"kpi",
|
|
4888
|
+
"tabular",
|
|
4889
|
+
"geographic",
|
|
4890
|
+
"category_breakdown",
|
|
4891
|
+
"product_browse",
|
|
4892
|
+
"table",
|
|
4893
|
+
"text"
|
|
4894
|
+
];
|
|
4895
|
+
const validCharts = [
|
|
4896
|
+
"line_chart",
|
|
4897
|
+
"bar_chart",
|
|
4898
|
+
"histogram",
|
|
4899
|
+
"pie_chart",
|
|
4900
|
+
"donut_chart",
|
|
4901
|
+
"scatter_plot",
|
|
4902
|
+
"horizontal_bar",
|
|
4903
|
+
"metric_card",
|
|
4904
|
+
"table",
|
|
4905
|
+
"geo_map",
|
|
4906
|
+
"text"
|
|
4907
|
+
];
|
|
4908
|
+
const hint = obj.visualizationHint;
|
|
4909
|
+
if (!hint || !validHints.includes(hint)) return null;
|
|
4910
|
+
const normalizedHint = hint === "category_breakdown" ? "composition" : hint;
|
|
4911
|
+
const recommendedChart = typeof obj.recommendedChart === "string" && validCharts.includes(obj.recommendedChart) ? obj.recommendedChart : this.getRecommendedChartForIntent(normalizedHint);
|
|
4912
|
+
return {
|
|
4913
|
+
visualizationHint: normalizedHint,
|
|
4914
|
+
recommendedChart,
|
|
4915
|
+
filterInStockOnly: Boolean(obj.filterInStockOnly),
|
|
4916
|
+
wantsExplicitTable: Boolean(obj.wantsExplicitTable),
|
|
4917
|
+
isTemporal: Boolean(obj.isTemporal),
|
|
4918
|
+
isComparison: Boolean(obj.isComparison),
|
|
4919
|
+
language: typeof obj.language === "string" && obj.language ? obj.language : "en",
|
|
4920
|
+
reasoning: typeof obj.reasoning === "string" ? obj.reasoning : void 0
|
|
4921
|
+
};
|
|
4922
|
+
} catch (e) {
|
|
4923
|
+
return null;
|
|
4924
|
+
}
|
|
4925
|
+
}
|
|
4926
|
+
/**
|
|
4927
|
+
* Heuristic intent detector — used when the LLM is unavailable.
|
|
4928
|
+
* Intentionally minimal: it should only catch the most obvious signals.
|
|
4929
|
+
* The LLM path handles everything subtle.
|
|
4930
|
+
*/
|
|
4931
|
+
static detectIntentHeuristic(query) {
|
|
4932
|
+
const q = query.toLowerCase();
|
|
4933
|
+
const isTemporal = /\b(trend|trends|over time|historical|history|growth|decline|monthly|yearly|weekly|daily|last (year|month|week)|timeline|forecast)\b/.test(q);
|
|
4934
|
+
const isRanking = /\b(highest|lowest|top\s*\d+|bottom\s*\d+|rank(?:ing)?|best|worst|leading)\b/.test(q);
|
|
4935
|
+
const isComparison = /\b(compare|comparison|vs\.?|versus|against|differ|difference|contrast|top)\b/.test(q) || isRanking;
|
|
4936
|
+
const isDistribution = /\b(distribution|spread|histogram|frequency|variance|range)\b/.test(q);
|
|
4937
|
+
const wantsPieLikeChart = /\b(pie|donut|doughnut)(?:\s+chart)?\b/.test(q);
|
|
4938
|
+
const isComposition = wantsPieLikeChart || /\b(share|percentage|percent|breakup|breakdown|composition|split|segmentation|by category|by type|proportion)\b/.test(q);
|
|
4939
|
+
const isCorrelation = /\b(relation|relationship|correlation|correlate|scatter|association|impact of|depend(?:s|ence)? on)\b/.test(q);
|
|
4940
|
+
const isKpi = /\b(total|average|avg|count|sum|median|minimum|maximum|metric|kpi|how many|number of)\b/.test(q);
|
|
4941
|
+
const isGeographic = /\b(region|country|countries|state|city|location|map|geo|geographic|territory)\b/.test(q);
|
|
4942
|
+
const filterInStockOnly = /\b(in[- ]?stock|available|availability|inventory|stock status)\b/.test(q);
|
|
4943
|
+
const wantsExplicitTable = /\b(table|spreadsheet|grid|detailed records|record details|list all|compare all)\b/.test(q);
|
|
4944
|
+
let visualizationHint = "text";
|
|
4945
|
+
if (wantsExplicitTable) visualizationHint = "tabular";
|
|
4946
|
+
else if (isTemporal) visualizationHint = "trend";
|
|
4947
|
+
else if (wantsPieLikeChart) visualizationHint = "composition";
|
|
4948
|
+
else if (isRanking) visualizationHint = "ranking";
|
|
4949
|
+
else if (isComparison) visualizationHint = "comparison";
|
|
4950
|
+
else if (isDistribution) visualizationHint = "distribution";
|
|
4951
|
+
else if (isComposition) visualizationHint = "composition";
|
|
4952
|
+
else if (isCorrelation) visualizationHint = "correlation";
|
|
4953
|
+
else if (isGeographic) visualizationHint = "geographic";
|
|
4954
|
+
else if (isKpi) visualizationHint = "kpi";
|
|
4955
|
+
else if (this.isProductQuery(query)) visualizationHint = "product_browse";
|
|
4956
|
+
return {
|
|
4957
|
+
visualizationHint,
|
|
4958
|
+
recommendedChart: this.getRecommendedChartForIntent(visualizationHint),
|
|
4959
|
+
filterInStockOnly,
|
|
4960
|
+
wantsExplicitTable,
|
|
4961
|
+
isTemporal,
|
|
4962
|
+
isComparison,
|
|
4963
|
+
language: "en"
|
|
4964
|
+
// heuristic cannot reliably detect language
|
|
4965
|
+
};
|
|
4966
|
+
}
|
|
4967
|
+
static getRecommendedChartForIntent(intent) {
|
|
4968
|
+
switch (intent) {
|
|
4969
|
+
case "trend":
|
|
4970
|
+
return "line_chart";
|
|
4971
|
+
case "comparison":
|
|
4972
|
+
return "bar_chart";
|
|
4973
|
+
case "distribution":
|
|
4974
|
+
return "histogram";
|
|
4975
|
+
case "composition":
|
|
4976
|
+
case "category_breakdown":
|
|
4977
|
+
return "pie_chart";
|
|
4978
|
+
case "correlation":
|
|
4979
|
+
return "scatter_plot";
|
|
4980
|
+
case "ranking":
|
|
4981
|
+
return "horizontal_bar";
|
|
4982
|
+
case "kpi":
|
|
4983
|
+
return "metric_card";
|
|
4984
|
+
case "tabular":
|
|
4985
|
+
case "table":
|
|
4986
|
+
return "table";
|
|
4987
|
+
case "geographic":
|
|
4988
|
+
return "geo_map";
|
|
4989
|
+
case "product_browse":
|
|
4990
|
+
case "text":
|
|
4991
|
+
default:
|
|
4992
|
+
return "text";
|
|
4993
|
+
}
|
|
4994
|
+
}
|
|
4995
|
+
// ─── Transform Helpers ────────────────────────────────────────────────────
|
|
4200
4996
|
static transformToProductCarousel(data, config, trainedSchema) {
|
|
4201
|
-
const products = data.
|
|
4997
|
+
const products = data.map((item) => this.extractProductInfo(item, config, trainedSchema)).filter((p) => p !== null).slice(0, 15);
|
|
4202
4998
|
return {
|
|
4203
4999
|
type: "product_carousel",
|
|
4204
5000
|
title: "Recommended Products",
|
|
@@ -4206,55 +5002,154 @@ var UITransformer = class {
|
|
|
4206
5002
|
data: products
|
|
4207
5003
|
};
|
|
4208
5004
|
}
|
|
4209
|
-
|
|
4210
|
-
|
|
4211
|
-
|
|
4212
|
-
|
|
4213
|
-
|
|
4214
|
-
|
|
5005
|
+
static transformToPieChart(data, profile, query = "") {
|
|
5006
|
+
var _a;
|
|
5007
|
+
const dimension = profile ? this.selectDimensionField(profile, query) : void 0;
|
|
5008
|
+
const categories = dimension ? Array.from(new Set(profile.records.map((record) => {
|
|
5009
|
+
var _a2;
|
|
5010
|
+
return String((_a2 = record.fields[dimension.key]) != null ? _a2 : "");
|
|
5011
|
+
}).filter(Boolean))) : this.detectCategories(data);
|
|
5012
|
+
if (categories.length === 0) return null;
|
|
5013
|
+
const categoryData = dimension && profile ? this.aggregateProfileByDimension(profile, dimension.key) : this.aggregateByCategory(data, categories);
|
|
4215
5014
|
const pieData = Object.entries(categoryData).map(([label, count]) => {
|
|
4216
5015
|
const { inStockCount, outOfStockCount } = this.calculateStockCounts(label, data);
|
|
4217
|
-
return {
|
|
4218
|
-
label,
|
|
4219
|
-
value: count,
|
|
4220
|
-
inStockCount,
|
|
4221
|
-
outOfStockCount
|
|
4222
|
-
};
|
|
5016
|
+
return { label, value: count, inStockCount, outOfStockCount };
|
|
4223
5017
|
});
|
|
4224
5018
|
return {
|
|
4225
5019
|
type: "pie_chart",
|
|
4226
|
-
title:
|
|
4227
|
-
description: `Showing breakdown across ${
|
|
5020
|
+
title: `Distribution by ${(_a = dimension == null ? void 0 : dimension.label) != null ? _a : "Category"}`,
|
|
5021
|
+
description: `Showing breakdown across ${pieData.length} categories`,
|
|
4228
5022
|
data: pieData
|
|
4229
5023
|
};
|
|
4230
5024
|
}
|
|
4231
|
-
|
|
4232
|
-
|
|
4233
|
-
|
|
4234
|
-
|
|
4235
|
-
|
|
4236
|
-
|
|
4237
|
-
timestamp:
|
|
4238
|
-
value:
|
|
4239
|
-
|
|
4240
|
-
})
|
|
5025
|
+
static transformToLineChart(profile) {
|
|
5026
|
+
const dateField = profile.dateFields[0];
|
|
5027
|
+
const valueField = profile.numericFields[0];
|
|
5028
|
+
const buckets = /* @__PURE__ */ new Map();
|
|
5029
|
+
profile.records.forEach((record) => {
|
|
5030
|
+
var _a, _b, _c;
|
|
5031
|
+
const timestamp = String((_a = record.fields[dateField.key]) != null ? _a : "");
|
|
5032
|
+
const value = (_b = this.toFiniteNumber(record.fields[valueField.key])) != null ? _b : 0;
|
|
5033
|
+
buckets.set(timestamp, ((_c = buckets.get(timestamp)) != null ? _c : 0) + value);
|
|
5034
|
+
});
|
|
5035
|
+
const lineData = Array.from(buckets.entries()).sort(([a], [b]) => Date.parse(a) - Date.parse(b)).slice(0, 24).map(([timestamp, value]) => ({ timestamp, value, label: timestamp }));
|
|
4241
5036
|
return {
|
|
4242
5037
|
type: "line_chart",
|
|
4243
|
-
title:
|
|
5038
|
+
title: `${valueField.label} Over Time`,
|
|
4244
5039
|
description: `Showing ${lineData.length} data points`,
|
|
4245
5040
|
data: lineData
|
|
4246
5041
|
};
|
|
4247
5042
|
}
|
|
4248
|
-
|
|
4249
|
-
|
|
4250
|
-
|
|
4251
|
-
|
|
4252
|
-
const
|
|
4253
|
-
const
|
|
4254
|
-
const
|
|
4255
|
-
|
|
4256
|
-
|
|
5043
|
+
static transformToBarChart(data, profile, query = "", horizontal = false) {
|
|
5044
|
+
var _a;
|
|
5045
|
+
const dimension = profile ? this.selectDimensionField(profile, query) : void 0;
|
|
5046
|
+
const measure = profile ? this.selectNumericField(profile, query) : void 0;
|
|
5047
|
+
const aggregate = dimension && profile ? this.aggregateProfileByDimension(profile, dimension.key, measure == null ? void 0 : measure.key) : this.aggregateByCategory(data, this.detectCategories(data));
|
|
5048
|
+
const barData = Object.entries(aggregate).map(([category, value]) => ({ category, value: Number(value) })).sort((a, b) => horizontal ? b.value - a.value : 0).slice(0, 12);
|
|
5049
|
+
const fallbackData = barData.length > 0 ? barData : data.slice(0, 12).map((item, index) => {
|
|
5050
|
+
var _a2, _b, _c, _d, _e;
|
|
5051
|
+
const meta = item.metadata || {};
|
|
5052
|
+
const label = String(
|
|
5053
|
+
(_c = (_b = (_a2 = this.getDynamicVal(meta, "name")) != null ? _a2 : meta.label) != null ? _b : meta.title) != null ? _c : `Result ${index + 1}`
|
|
5054
|
+
);
|
|
5055
|
+
const value = (_e = (_d = this.extractNumericValue(meta)) != null ? _d : item.score) != null ? _e : 0;
|
|
5056
|
+
return { category: label, value: Number(value) };
|
|
5057
|
+
});
|
|
5058
|
+
return {
|
|
5059
|
+
type: horizontal ? "horizontal_bar" : "bar_chart",
|
|
5060
|
+
title: dimension ? `${(_a = measure == null ? void 0 : measure.label) != null ? _a : "Count"} by ${dimension.label}` : "Comparison",
|
|
5061
|
+
description: `Showing ${fallbackData.length} comparable values`,
|
|
5062
|
+
data: fallbackData
|
|
5063
|
+
};
|
|
5064
|
+
}
|
|
5065
|
+
static transformToHistogram(profile, query = "") {
|
|
5066
|
+
const field = this.selectNumericField(profile, query);
|
|
5067
|
+
if (!field) return null;
|
|
5068
|
+
const values = profile.records.map((record) => this.toFiniteNumber(record.fields[field.key])).filter((value) => value !== null).sort((a, b) => a - b);
|
|
5069
|
+
if (values.length === 0) return null;
|
|
5070
|
+
const min = values[0];
|
|
5071
|
+
const max = values[values.length - 1];
|
|
5072
|
+
const bucketCount = Math.min(10, Math.max(3, Math.ceil(Math.sqrt(values.length))));
|
|
5073
|
+
const width = max === min ? 1 : (max - min) / bucketCount;
|
|
5074
|
+
const buckets = Array.from({ length: bucketCount }, (_, index) => {
|
|
5075
|
+
const start = min + index * width;
|
|
5076
|
+
const end = index === bucketCount - 1 ? max : start + width;
|
|
5077
|
+
return { category: `${this.formatNumber(start)}-${this.formatNumber(end)}`, value: 0 };
|
|
5078
|
+
});
|
|
5079
|
+
values.forEach((value) => {
|
|
5080
|
+
const bucketIndex = max === min ? 0 : Math.min(bucketCount - 1, Math.floor((value - min) / width));
|
|
5081
|
+
buckets[bucketIndex].value += 1;
|
|
5082
|
+
});
|
|
5083
|
+
return {
|
|
5084
|
+
type: "histogram",
|
|
5085
|
+
title: `${field.label} Distribution`,
|
|
5086
|
+
description: `Showing ${values.length} values across ${bucketCount} buckets`,
|
|
5087
|
+
data: buckets
|
|
5088
|
+
};
|
|
5089
|
+
}
|
|
5090
|
+
static transformToScatterPlot(profile, query = "") {
|
|
5091
|
+
const fields = this.rankFieldsByQuery(profile.numericFields, query).slice(0, 2);
|
|
5092
|
+
if (fields.length < 2) return null;
|
|
5093
|
+
const [xField, yField] = fields;
|
|
5094
|
+
const points = profile.records.map((record) => {
|
|
5095
|
+
var _a;
|
|
5096
|
+
const x = this.toFiniteNumber(record.fields[xField.key]);
|
|
5097
|
+
const y = this.toFiniteNumber(record.fields[yField.key]);
|
|
5098
|
+
if (x === null || y === null) return null;
|
|
5099
|
+
return { x, y, label: String((_a = this.getRecordLabel(record)) != null ? _a : record.id) };
|
|
5100
|
+
}).filter((point) => point !== null).slice(0, 100);
|
|
5101
|
+
if (points.length === 0) return null;
|
|
5102
|
+
return {
|
|
5103
|
+
type: "scatter_plot",
|
|
5104
|
+
title: `${xField.label} vs ${yField.label}`,
|
|
5105
|
+
description: `Showing ${points.length} paired values`,
|
|
5106
|
+
data: points
|
|
5107
|
+
};
|
|
5108
|
+
}
|
|
5109
|
+
static transformToMetricCard(profile, query = "") {
|
|
5110
|
+
const operation = this.detectAggregationOperation(query);
|
|
5111
|
+
const numericField = this.selectNumericField(profile, query);
|
|
5112
|
+
const values = numericField ? profile.records.map((record) => this.toFiniteNumber(record.fields[numericField.key])).filter((value2) => value2 !== null) : [];
|
|
5113
|
+
const value = operation === "count" || values.length === 0 ? profile.records.length : this.calculateAggregate(values, operation);
|
|
5114
|
+
const metric = {
|
|
5115
|
+
label: numericField ? `${operation} ${numericField.label}` : "Count",
|
|
5116
|
+
value,
|
|
5117
|
+
operation
|
|
4257
5118
|
};
|
|
5119
|
+
return {
|
|
5120
|
+
type: "metric_card",
|
|
5121
|
+
title: metric.label,
|
|
5122
|
+
description: `Calculated from ${profile.records.length} result${profile.records.length === 1 ? "" : "s"}`,
|
|
5123
|
+
data: metric
|
|
5124
|
+
};
|
|
5125
|
+
}
|
|
5126
|
+
static transformToRadarChart(data) {
|
|
5127
|
+
const attributeMap = {};
|
|
5128
|
+
data.forEach((item) => {
|
|
5129
|
+
var _a, _b, _c;
|
|
5130
|
+
const meta = item.metadata || {};
|
|
5131
|
+
const seriesName = String((_c = (_b = (_a = meta.name) != null ? _a : meta.product) != null ? _b : item.id) != null ? _c : "Item");
|
|
5132
|
+
Object.entries(meta).forEach(([key, val]) => {
|
|
5133
|
+
if (typeof val === "number" && !["price", "id"].includes(key.toLowerCase())) {
|
|
5134
|
+
if (!attributeMap[key]) attributeMap[key] = {};
|
|
5135
|
+
attributeMap[key][seriesName] = val;
|
|
5136
|
+
}
|
|
5137
|
+
});
|
|
5138
|
+
});
|
|
5139
|
+
const radarData = Object.entries(attributeMap).map(([attribute, series]) => __spreadValues({
|
|
5140
|
+
attribute
|
|
5141
|
+
}, series));
|
|
5142
|
+
return {
|
|
5143
|
+
type: "radar_chart",
|
|
5144
|
+
title: "Product Comparison",
|
|
5145
|
+
description: `Comparing ${data.length} items across ${radarData.length} attributes`,
|
|
5146
|
+
data: radarData.length > 0 ? radarData : data.map((d) => ({ attribute: d.content.substring(0, 40) }))
|
|
5147
|
+
};
|
|
5148
|
+
}
|
|
5149
|
+
static transformToTable(data, query = "") {
|
|
5150
|
+
const columns = this.extractTableColumns(data, query);
|
|
5151
|
+
const rows = data.map((item) => this.extractTableRow(item, columns));
|
|
5152
|
+
const tableData = { columns, rows };
|
|
4258
5153
|
return {
|
|
4259
5154
|
type: "table",
|
|
4260
5155
|
title: "Detailed Results",
|
|
@@ -4262,28 +5157,17 @@ var UITransformer = class {
|
|
|
4262
5157
|
data: tableData
|
|
4263
5158
|
};
|
|
4264
5159
|
}
|
|
4265
|
-
/**
|
|
4266
|
-
* Transform data to text format (fallback)
|
|
4267
|
-
*/
|
|
4268
5160
|
static transformToText(data) {
|
|
4269
|
-
const textContent = data.map((item) => item.content).join("\n\n");
|
|
4270
5161
|
return this.createTextResponse(
|
|
4271
5162
|
"Search Results",
|
|
4272
|
-
|
|
5163
|
+
data.map((item) => item.content).join("\n\n"),
|
|
4273
5164
|
`Found ${data.length} relevant results`
|
|
4274
5165
|
);
|
|
4275
5166
|
}
|
|
4276
|
-
/**
|
|
4277
|
-
* Helper: Create text response
|
|
4278
|
-
*/
|
|
4279
5167
|
static createTextResponse(title, content, description) {
|
|
4280
|
-
return {
|
|
4281
|
-
type: "text",
|
|
4282
|
-
title,
|
|
4283
|
-
description,
|
|
4284
|
-
data: { content }
|
|
4285
|
-
};
|
|
5168
|
+
return { type: "text", title, description, data: { content } };
|
|
4286
5169
|
}
|
|
5170
|
+
// ─── LLM Response Parsing ─────────────────────────────────────────────────
|
|
4287
5171
|
static parseTransformationResponse(raw) {
|
|
4288
5172
|
const payloadText = this.extractJsonCandidate(raw);
|
|
4289
5173
|
if (!payloadText) return null;
|
|
@@ -4320,9 +5204,7 @@ var UITransformer = class {
|
|
|
4320
5204
|
if (char === "{") depth += 1;
|
|
4321
5205
|
if (char === "}") {
|
|
4322
5206
|
depth -= 1;
|
|
4323
|
-
if (depth === 0)
|
|
4324
|
-
return cleaned.slice(start, i + 1);
|
|
4325
|
-
}
|
|
5207
|
+
if (depth === 0) return cleaned.slice(start, i + 1);
|
|
4326
5208
|
}
|
|
4327
5209
|
}
|
|
4328
5210
|
return null;
|
|
@@ -4330,19 +5212,16 @@ var UITransformer = class {
|
|
|
4330
5212
|
static normalizeTransformation(payload) {
|
|
4331
5213
|
var _a, _b, _c, _d, _e, _f, _g, _h, _i, _j;
|
|
4332
5214
|
if (!payload || typeof payload !== "object") return null;
|
|
4333
|
-
const
|
|
4334
|
-
const type = this.normalizeVisualizationType(
|
|
5215
|
+
const p = payload;
|
|
5216
|
+
const type = this.normalizeVisualizationType(
|
|
5217
|
+
String((_c = (_b = (_a = p.type) != null ? _a : p.view) != null ? _b : p.chartType) != null ? _c : "")
|
|
5218
|
+
);
|
|
4335
5219
|
if (!type) return null;
|
|
4336
|
-
const title = String((_e = (_d =
|
|
4337
|
-
const description =
|
|
4338
|
-
const rawData = (_j = (_i = (_h = (_g = (_f =
|
|
5220
|
+
const title = String((_e = (_d = p.title) != null ? _d : p.heading) != null ? _e : "Visualization");
|
|
5221
|
+
const description = p.description ? String(p.description) : void 0;
|
|
5222
|
+
const rawData = (_j = (_i = (_h = (_g = (_f = p.data) != null ? _f : p.table) != null ? _g : p.rows) != null ? _h : p.items) != null ? _i : p.content) != null ? _j : null;
|
|
4339
5223
|
const data = type === "text" && typeof rawData === "string" ? { content: rawData } : rawData;
|
|
4340
|
-
const transformation = {
|
|
4341
|
-
type,
|
|
4342
|
-
title,
|
|
4343
|
-
description,
|
|
4344
|
-
data
|
|
4345
|
-
};
|
|
5224
|
+
const transformation = { type, title, description, data };
|
|
4346
5225
|
return this.validateTransformation(transformation) ? transformation : null;
|
|
4347
5226
|
}
|
|
4348
5227
|
static normalizeVisualizationType(type) {
|
|
@@ -4352,10 +5231,23 @@ var UITransformer = class {
|
|
|
4352
5231
|
pie_chart: "pie_chart",
|
|
4353
5232
|
bar: "bar_chart",
|
|
4354
5233
|
bar_chart: "bar_chart",
|
|
5234
|
+
histogram: "histogram",
|
|
5235
|
+
horizontal_bar: "horizontal_bar",
|
|
5236
|
+
horizontalbar: "horizontal_bar",
|
|
4355
5237
|
line: "line_chart",
|
|
4356
5238
|
line_chart: "line_chart",
|
|
5239
|
+
scatter: "scatter_plot",
|
|
5240
|
+
scatter_plot: "scatter_plot",
|
|
5241
|
+
scatterplot: "scatter_plot",
|
|
4357
5242
|
radar: "radar_chart",
|
|
4358
5243
|
radar_chart: "radar_chart",
|
|
5244
|
+
metric: "metric_card",
|
|
5245
|
+
metric_card: "metric_card",
|
|
5246
|
+
card: "metric_card",
|
|
5247
|
+
kpi: "metric_card",
|
|
5248
|
+
geo: "geo_map",
|
|
5249
|
+
geo_map: "geo_map",
|
|
5250
|
+
map: "geo_map",
|
|
4359
5251
|
table: "table",
|
|
4360
5252
|
text: "text",
|
|
4361
5253
|
product_carousel: "product_carousel",
|
|
@@ -4363,21 +5255,31 @@ var UITransformer = class {
|
|
|
4363
5255
|
};
|
|
4364
5256
|
return (_a = mapping[type.toLowerCase()]) != null ? _a : null;
|
|
4365
5257
|
}
|
|
4366
|
-
static validateTransformation(
|
|
4367
|
-
const { type, data } =
|
|
5258
|
+
static validateTransformation(t) {
|
|
5259
|
+
const { type, data } = t;
|
|
4368
5260
|
switch (type) {
|
|
4369
5261
|
case "pie_chart":
|
|
4370
5262
|
case "bar_chart":
|
|
5263
|
+
case "histogram":
|
|
5264
|
+
case "horizontal_bar":
|
|
4371
5265
|
return Array.isArray(data) && data.every(
|
|
4372
|
-
(
|
|
5266
|
+
(i) => i !== null && typeof i === "object" && (typeof i.value === "number" || !Number.isNaN(Number(i.value)))
|
|
5267
|
+
);
|
|
5268
|
+
case "scatter_plot":
|
|
5269
|
+
return Array.isArray(data) && data.every(
|
|
5270
|
+
(i) => i !== null && typeof i === "object" && (typeof i.x === "number" || !Number.isNaN(Number(i.x))) && (typeof i.y === "number" || !Number.isNaN(Number(i.y)))
|
|
4373
5271
|
);
|
|
4374
5272
|
case "line_chart":
|
|
4375
5273
|
return Array.isArray(data) && data.every(
|
|
4376
|
-
(
|
|
5274
|
+
(i) => i !== null && typeof i === "object" && "timestamp" in i && (typeof i.value === "number" || !Number.isNaN(Number(i.value)))
|
|
4377
5275
|
);
|
|
5276
|
+
case "metric_card":
|
|
5277
|
+
return typeof data === "object" && data !== null && (typeof data.value === "number" || !Number.isNaN(Number(data.value)));
|
|
5278
|
+
case "geo_map":
|
|
5279
|
+
return Array.isArray(data) || typeof data === "object" && data !== null;
|
|
4378
5280
|
case "radar_chart":
|
|
4379
5281
|
return Array.isArray(data) && data.every(
|
|
4380
|
-
(
|
|
5282
|
+
(i) => i !== null && typeof i === "object" && "attribute" in i
|
|
4381
5283
|
);
|
|
4382
5284
|
case "table":
|
|
4383
5285
|
return typeof data === "object" && data !== null && Array.isArray(data.columns) && Array.isArray(data.rows);
|
|
@@ -4386,15 +5288,27 @@ var UITransformer = class {
|
|
|
4386
5288
|
case "product_carousel":
|
|
4387
5289
|
case "carousel":
|
|
4388
5290
|
return Array.isArray(data) && data.every(
|
|
4389
|
-
(
|
|
5291
|
+
(i) => i !== null && typeof i === "object" && ("id" in i || "name" in i)
|
|
4390
5292
|
);
|
|
4391
5293
|
default:
|
|
4392
5294
|
return false;
|
|
4393
5295
|
}
|
|
4394
5296
|
}
|
|
4395
|
-
|
|
4396
|
-
|
|
4397
|
-
|
|
5297
|
+
// ─── Data Inspection Helpers ──────────────────────────────────────────────
|
|
5298
|
+
static isStructuredListQuery(query) {
|
|
5299
|
+
const q = query.toLowerCase();
|
|
5300
|
+
const asksForRecords = /\b(list|provide|show|give|get|find|which|whose|records?|details?)\b/.test(q);
|
|
5301
|
+
const hasNumericPredicate = /\b(greater than|more than|above|over|less than|below|under|at least|at most|minimum|maximum|>=|<=|>|<|equals?|equal to)\b\s*\$?\d/i.test(q) || /\b\d+\b\s*(?:or more|or less|and above|and below)\b/i.test(q);
|
|
5302
|
+
const hasEntityTerms = /\b(organization|organisations?|organizations?|company|companies|employee|employees|staff|headcount|workforce)\b/.test(q);
|
|
5303
|
+
return asksForRecords && (hasNumericPredicate || hasEntityTerms);
|
|
5304
|
+
}
|
|
5305
|
+
static isProductQuery(query) {
|
|
5306
|
+
const q = query.toLowerCase();
|
|
5307
|
+
const productTerms = /\b(product|products|item|items|sku|catalog|catalogue|price|prices|brand|model|stock|inventory|in stock|out of stock|buy|shop)\b/.test(q);
|
|
5308
|
+
const productAction = /\b(recommend|suggest|describe|description|detail|details|about|show|find|search|browse|list)\b/.test(q);
|
|
5309
|
+
const nonProductEntityTerms = /\b(organization|organisations?|organizations?|company|companies|employee|employees|staff|headcount|workforce)\b/.test(q);
|
|
5310
|
+
return productTerms && productAction && !nonProductEntityTerms;
|
|
5311
|
+
}
|
|
4398
5312
|
static isProductData(item) {
|
|
4399
5313
|
const content = (item.content || "").toLowerCase();
|
|
4400
5314
|
const productKeywords = [
|
|
@@ -4420,360 +5334,582 @@ var UITransformer = class {
|
|
|
4420
5334
|
"fragrance"
|
|
4421
5335
|
];
|
|
4422
5336
|
const hasKeywords = productKeywords.some((kw) => content.includes(kw));
|
|
4423
|
-
const
|
|
4424
|
-
|
|
4425
|
-
|
|
4426
|
-
|
|
5337
|
+
const metadata = item.metadata || {};
|
|
5338
|
+
const hasMetadataKey = Object.keys(metadata).some((k) => {
|
|
5339
|
+
const val = metadata[k];
|
|
5340
|
+
return ["price", "product", "sku", "brand", "model", "cost", "item"].includes(k.toLowerCase()) && val !== null && val !== void 0 && val !== "";
|
|
5341
|
+
});
|
|
5342
|
+
const hasPricePattern = /\$\s*\d+\.\d{2}/.test(content);
|
|
4427
5343
|
return hasKeywords || hasMetadataKey || hasPricePattern;
|
|
4428
5344
|
}
|
|
4429
|
-
/**
|
|
4430
|
-
* Helper: Check if data contains time series
|
|
4431
|
-
*/
|
|
4432
5345
|
static isTimeSeriesData(item) {
|
|
4433
|
-
const content = (item.content || "").toLowerCase();
|
|
4434
|
-
const timeKeywords = ["trend", "historical", "growth", "decline", "change", "increase", "decrease"];
|
|
4435
|
-
const hasTimeKeyword = timeKeywords.some((kw) => content.includes(kw));
|
|
4436
5346
|
const metadata = item.metadata || {};
|
|
4437
5347
|
const maybeDateKeys = Object.keys(metadata).filter(
|
|
4438
5348
|
(k) => ["date", "timestamp", "time", "period"].includes(k.toLowerCase())
|
|
4439
5349
|
);
|
|
4440
|
-
|
|
5350
|
+
return maybeDateKeys.some((key) => {
|
|
4441
5351
|
const value = metadata[key];
|
|
4442
|
-
if (typeof value === "string")
|
|
4443
|
-
return !Number.isNaN(Date.parse(value)) && value.trim().length > 0;
|
|
4444
|
-
}
|
|
5352
|
+
if (typeof value === "string") return !Number.isNaN(Date.parse(value)) && value.trim().length > 0;
|
|
4445
5353
|
return typeof value === "number";
|
|
4446
5354
|
});
|
|
4447
|
-
return hasTimeKeyword || hasValidDateValue;
|
|
4448
5355
|
}
|
|
4449
|
-
static
|
|
4450
|
-
|
|
4451
|
-
|
|
4452
|
-
|
|
4453
|
-
const normalized = query.toLowerCase();
|
|
4454
|
-
const chartKeywords = [
|
|
4455
|
-
"distribution",
|
|
4456
|
-
"breakdown",
|
|
4457
|
-
"by category",
|
|
4458
|
-
"by type",
|
|
4459
|
-
"compare",
|
|
4460
|
-
"share",
|
|
4461
|
-
"percentage",
|
|
4462
|
-
"segmentation",
|
|
4463
|
-
"split",
|
|
4464
|
-
"category breakdown",
|
|
4465
|
-
"category distribution"
|
|
4466
|
-
];
|
|
4467
|
-
return chartKeywords.some((keyword) => normalized.includes(keyword));
|
|
5356
|
+
static hasMultipleFields(data) {
|
|
5357
|
+
const fieldCount = /* @__PURE__ */ new Set();
|
|
5358
|
+
data.forEach((item) => Object.keys(item.metadata || {}).forEach((k) => fieldCount.add(k)));
|
|
5359
|
+
return fieldCount.size > 2;
|
|
4468
5360
|
}
|
|
4469
|
-
static
|
|
4470
|
-
const
|
|
4471
|
-
|
|
4472
|
-
|
|
4473
|
-
|
|
4474
|
-
|
|
4475
|
-
|
|
4476
|
-
|
|
4477
|
-
|
|
4478
|
-
|
|
4479
|
-
|
|
4480
|
-
|
|
4481
|
-
|
|
4482
|
-
|
|
4483
|
-
|
|
4484
|
-
|
|
4485
|
-
|
|
4486
|
-
|
|
4487
|
-
|
|
4488
|
-
|
|
5361
|
+
static profileData(data) {
|
|
5362
|
+
const records = data.map((item) => {
|
|
5363
|
+
const fields2 = {};
|
|
5364
|
+
Object.entries(item.metadata || {}).forEach(([key, value]) => {
|
|
5365
|
+
const primitive = this.toPrimitive(value);
|
|
5366
|
+
if (primitive !== null) fields2[key] = primitive;
|
|
5367
|
+
});
|
|
5368
|
+
if (!fields2.content && item.content) fields2.content = item.content.substring(0, 500);
|
|
5369
|
+
return {
|
|
5370
|
+
id: item.id,
|
|
5371
|
+
content: item.content,
|
|
5372
|
+
score: item.score,
|
|
5373
|
+
fields: fields2,
|
|
5374
|
+
source: item
|
|
5375
|
+
};
|
|
5376
|
+
});
|
|
5377
|
+
const keys = Array.from(new Set(records.flatMap((record) => Object.keys(record.fields))));
|
|
5378
|
+
const fields = keys.map((key) => {
|
|
5379
|
+
const values = records.map((record) => record.fields[key]).filter((value) => value !== void 0 && value !== null && String(value).trim() !== "");
|
|
5380
|
+
const uniqueCount = new Set(values.map((value) => String(value).toLowerCase())).size;
|
|
5381
|
+
return {
|
|
5382
|
+
key,
|
|
5383
|
+
label: this.humanizeFieldName(key),
|
|
5384
|
+
kind: this.inferFieldKind(key, values, records.length, uniqueCount),
|
|
5385
|
+
values,
|
|
5386
|
+
uniqueCount
|
|
5387
|
+
};
|
|
5388
|
+
});
|
|
5389
|
+
return {
|
|
5390
|
+
records,
|
|
5391
|
+
fields,
|
|
5392
|
+
numericFields: fields.filter((field) => field.kind === "number"),
|
|
5393
|
+
dateFields: fields.filter((field) => field.kind === "date"),
|
|
5394
|
+
categoricalFields: fields.filter((field) => field.kind === "category"),
|
|
5395
|
+
booleanFields: fields.filter((field) => field.kind === "boolean")
|
|
5396
|
+
};
|
|
4489
5397
|
}
|
|
4490
|
-
static
|
|
4491
|
-
|
|
4492
|
-
|
|
5398
|
+
static toPrimitive(value) {
|
|
5399
|
+
if (typeof value === "string" || typeof value === "number" || typeof value === "boolean") return value;
|
|
5400
|
+
if (value === null || value === void 0) return null;
|
|
5401
|
+
if (value instanceof Date) return value.toISOString();
|
|
5402
|
+
return null;
|
|
4493
5403
|
}
|
|
4494
|
-
|
|
4495
|
-
|
|
4496
|
-
|
|
4497
|
-
|
|
4498
|
-
|
|
4499
|
-
if (!meta) return void 0;
|
|
4500
|
-
const mapping = (_a = config == null ? void 0 : config.rag) == null ? void 0 : _a.uiMapping;
|
|
4501
|
-
if (mapping && mapping[uiKey]) {
|
|
4502
|
-
const mappedKey = mapping[uiKey];
|
|
4503
|
-
if (meta[mappedKey] !== void 0) return meta[mappedKey];
|
|
5404
|
+
static inferFieldKind(key, values, rowCount, uniqueCount) {
|
|
5405
|
+
const lowerKey = key.toLowerCase();
|
|
5406
|
+
if (values.length === 0) return "text";
|
|
5407
|
+
if (values.every((value) => typeof value === "boolean" || /^(true|false|yes|no|in_stock|out_of_stock|available|unavailable)$/i.test(String(value)))) {
|
|
5408
|
+
return "boolean";
|
|
4504
5409
|
}
|
|
4505
|
-
if (
|
|
4506
|
-
|
|
4507
|
-
if (trainedKey && meta[trainedKey] !== void 0) return meta[trainedKey];
|
|
5410
|
+
if (/(date|time|timestamp|created|updated|month|year|period)/i.test(lowerKey) && values.some((value) => this.isDateLike(value))) {
|
|
5411
|
+
return "date";
|
|
4508
5412
|
}
|
|
4509
|
-
|
|
5413
|
+
const numericCount = values.filter((value) => this.toFiniteNumber(value) !== null).length;
|
|
5414
|
+
if (numericCount / values.length >= 0.85 && !/(id|sku|ean|upc|zip|postal|phone|code)/i.test(lowerKey)) {
|
|
5415
|
+
return "number";
|
|
5416
|
+
}
|
|
5417
|
+
if (uniqueCount <= Math.max(20, Math.ceil(rowCount * 0.7)) && !/(id|sku|ean|upc|description|content|url|image|thumbnail)/i.test(lowerKey)) {
|
|
5418
|
+
return "category";
|
|
5419
|
+
}
|
|
5420
|
+
return "text";
|
|
4510
5421
|
}
|
|
4511
|
-
static
|
|
4512
|
-
|
|
4513
|
-
const
|
|
4514
|
-
|
|
4515
|
-
|
|
4516
|
-
|
|
4517
|
-
|
|
4518
|
-
|
|
4519
|
-
|
|
4520
|
-
|
|
4521
|
-
|
|
4522
|
-
|
|
4523
|
-
|
|
4524
|
-
|
|
4525
|
-
|
|
4526
|
-
|
|
4527
|
-
|
|
4528
|
-
return
|
|
4529
|
-
|
|
4530
|
-
|
|
4531
|
-
|
|
4532
|
-
image: typeof imageValue === "string" ? imageValue : void 0,
|
|
4533
|
-
brand: brand ? String(brand) : void 0,
|
|
4534
|
-
description: item.content,
|
|
4535
|
-
inStock: this.determineStockStatus(item)
|
|
4536
|
-
};
|
|
5422
|
+
static isDateLike(value) {
|
|
5423
|
+
if (typeof value === "number") return value > 1900 && value < 3e3;
|
|
5424
|
+
const text = String(value).trim();
|
|
5425
|
+
return text.length > 3 && !Number.isNaN(Date.parse(text));
|
|
5426
|
+
}
|
|
5427
|
+
static chooseAutomaticVisualization(data, profile, query) {
|
|
5428
|
+
if (profile.records.length === 0) return null;
|
|
5429
|
+
if (profile.dateFields.length > 0 && profile.numericFields.length > 0) {
|
|
5430
|
+
return this.transformToLineChart(profile);
|
|
5431
|
+
}
|
|
5432
|
+
if (profile.categoricalFields.length > 0 && profile.numericFields.length > 0) {
|
|
5433
|
+
return this.transformToBarChart(data, profile, query);
|
|
5434
|
+
}
|
|
5435
|
+
if (profile.categoricalFields.length > 0) {
|
|
5436
|
+
return this.transformToPieChart(data, profile, query);
|
|
5437
|
+
}
|
|
5438
|
+
if (profile.numericFields.length >= 2) {
|
|
5439
|
+
return this.transformToScatterPlot(profile, query);
|
|
5440
|
+
}
|
|
5441
|
+
if (profile.numericFields.length === 1) {
|
|
5442
|
+
return this.transformToMetricCard(profile, query);
|
|
4537
5443
|
}
|
|
4538
5444
|
return null;
|
|
4539
5445
|
}
|
|
4540
|
-
|
|
4541
|
-
|
|
4542
|
-
|
|
5446
|
+
static selectDimensionField(profile, query) {
|
|
5447
|
+
var _a, _b;
|
|
5448
|
+
const productCategory = profile.categoricalFields.find(
|
|
5449
|
+
(field) => /category|department|collection|type|group|segment|region|country|state|city/i.test(field.key)
|
|
5450
|
+
);
|
|
5451
|
+
const ranked = this.rankFieldsByQuery(profile.categoricalFields, query);
|
|
5452
|
+
return (_b = (_a = ranked[0]) != null ? _a : productCategory) != null ? _b : profile.categoricalFields[0];
|
|
5453
|
+
}
|
|
5454
|
+
static selectNumericField(profile, query) {
|
|
5455
|
+
var _a;
|
|
5456
|
+
return (_a = this.rankFieldsByQuery(profile.numericFields, query)[0]) != null ? _a : profile.numericFields[0];
|
|
5457
|
+
}
|
|
5458
|
+
static rankFieldsByQuery(fields, query) {
|
|
5459
|
+
const q = query.toLowerCase();
|
|
5460
|
+
return [...fields].sort((a, b) => this.fieldScore(b, q) - this.fieldScore(a, q));
|
|
5461
|
+
}
|
|
5462
|
+
static fieldScore(field, query) {
|
|
5463
|
+
const key = field.key.toLowerCase();
|
|
5464
|
+
const label = field.label.toLowerCase();
|
|
5465
|
+
let score = 0;
|
|
5466
|
+
if (query.includes(key)) score += 4;
|
|
5467
|
+
if (query.includes(label)) score += 4;
|
|
5468
|
+
key.split(/[_\s-]+/).forEach((part) => {
|
|
5469
|
+
if (part && query.includes(part)) score += 1;
|
|
5470
|
+
});
|
|
5471
|
+
if (/category|department|collection|type|group|segment/.test(key)) score += 2;
|
|
5472
|
+
if (/count|total|quantity|stock|inventory|sales|revenue|amount|price|value|score/.test(key)) score += 2;
|
|
5473
|
+
if (/id|sku|ean|upc|code/.test(key)) score -= 5;
|
|
5474
|
+
return score;
|
|
5475
|
+
}
|
|
5476
|
+
static normalizeComparableField(field) {
|
|
5477
|
+
return field.toLowerCase().replace(/&/g, "and").replace(/[^a-z0-9]+/g, "").trim();
|
|
5478
|
+
}
|
|
5479
|
+
static fieldTokens(field) {
|
|
5480
|
+
return field.toLowerCase().replace(/[_-]+/g, " ").split(/\s+/).map((token) => token.replace(/[^a-z0-9]/g, "")).filter(Boolean);
|
|
5481
|
+
}
|
|
5482
|
+
static aggregateProfileByDimension(profile, dimensionKey, measureKey) {
|
|
5483
|
+
const result = {};
|
|
5484
|
+
profile.records.forEach((record) => {
|
|
5485
|
+
var _a, _b, _c;
|
|
5486
|
+
const category = String((_a = record.fields[dimensionKey]) != null ? _a : "Other").trim() || "Other";
|
|
5487
|
+
const value = measureKey ? (_b = this.toFiniteNumber(record.fields[measureKey])) != null ? _b : 0 : 1;
|
|
5488
|
+
result[category] = ((_c = result[category]) != null ? _c : 0) + value;
|
|
5489
|
+
});
|
|
5490
|
+
return result;
|
|
5491
|
+
}
|
|
5492
|
+
static getRecordLabel(record) {
|
|
5493
|
+
const labelKeys = ["name", "title", "label", "product", "item", "brand"];
|
|
5494
|
+
const key = Object.keys(record.fields).find(
|
|
5495
|
+
(fieldKey) => labelKeys.some((labelKey) => fieldKey.toLowerCase().includes(labelKey))
|
|
5496
|
+
);
|
|
5497
|
+
return key ? record.fields[key] : void 0;
|
|
5498
|
+
}
|
|
5499
|
+
static detectAggregationOperation(query) {
|
|
5500
|
+
const q = query.toLowerCase();
|
|
5501
|
+
if (/\b(avg|average|mean)\b/.test(q)) return "average";
|
|
5502
|
+
if (/\b(count|how many|number of)\b/.test(q)) return "count";
|
|
5503
|
+
if (/\b(min|minimum|lowest)\b/.test(q)) return "min";
|
|
5504
|
+
if (/\b(max|maximum|highest)\b/.test(q)) return "max";
|
|
5505
|
+
if (/\bmedian\b/.test(q)) return "median";
|
|
5506
|
+
return "sum";
|
|
5507
|
+
}
|
|
5508
|
+
static calculateAggregate(values, operation) {
|
|
5509
|
+
if (values.length === 0) return 0;
|
|
5510
|
+
switch (operation) {
|
|
5511
|
+
case "average":
|
|
5512
|
+
return values.reduce((sum, value) => sum + value, 0) / values.length;
|
|
5513
|
+
case "count":
|
|
5514
|
+
return values.length;
|
|
5515
|
+
case "min":
|
|
5516
|
+
return Math.min(...values);
|
|
5517
|
+
case "max":
|
|
5518
|
+
return Math.max(...values);
|
|
5519
|
+
case "median": {
|
|
5520
|
+
const sorted = [...values].sort((a, b) => a - b);
|
|
5521
|
+
const middle = Math.floor(sorted.length / 2);
|
|
5522
|
+
return sorted.length % 2 === 0 ? (sorted[middle - 1] + sorted[middle]) / 2 : sorted[middle];
|
|
5523
|
+
}
|
|
5524
|
+
case "sum":
|
|
5525
|
+
default:
|
|
5526
|
+
return values.reduce((sum, value) => sum + value, 0);
|
|
5527
|
+
}
|
|
5528
|
+
}
|
|
5529
|
+
static humanizeFieldName(key) {
|
|
5530
|
+
return key.replace(/[_-]+/g, " ").replace(/([a-z])([A-Z])/g, "$1 $2").replace(/\s+/g, " ").trim().replace(/\b\w/g, (char) => char.toUpperCase());
|
|
5531
|
+
}
|
|
5532
|
+
static formatNumber(value) {
|
|
5533
|
+
return Number.isInteger(value) ? String(value) : value.toFixed(1);
|
|
5534
|
+
}
|
|
4543
5535
|
static detectCategories(data) {
|
|
4544
5536
|
const categories = /* @__PURE__ */ new Set();
|
|
4545
5537
|
data.forEach((item) => {
|
|
4546
|
-
const
|
|
4547
|
-
if (
|
|
4548
|
-
categories.add(String(meta.category));
|
|
4549
|
-
}
|
|
4550
|
-
if (meta.type) {
|
|
4551
|
-
categories.add(String(meta.type));
|
|
4552
|
-
}
|
|
4553
|
-
if (meta.tag) {
|
|
4554
|
-
const tags = Array.isArray(meta.tag) ? meta.tag : [meta.tag];
|
|
4555
|
-
tags.forEach((t) => categories.add(String(t)));
|
|
4556
|
-
}
|
|
4557
|
-
const contentCategories = Array.from(new Set(
|
|
4558
|
-
Array.from(item.content.matchAll(/^\s*([^:\n]+):\s*(?:•|\*|-)*/gm)).map((match) => match[1].trim()).filter(Boolean)
|
|
4559
|
-
));
|
|
4560
|
-
contentCategories.forEach((category) => categories.add(category));
|
|
4561
|
-
const categoryMatch = item.content.match(/(?:Category|Type|Class):\s*([^\n]+)/i);
|
|
4562
|
-
if (categoryMatch) {
|
|
4563
|
-
categories.add(categoryMatch[1].trim());
|
|
4564
|
-
}
|
|
5538
|
+
const category = this.getProductCategory(item);
|
|
5539
|
+
if (category) categories.add(category);
|
|
4565
5540
|
});
|
|
4566
5541
|
return Array.from(categories);
|
|
4567
5542
|
}
|
|
4568
|
-
|
|
4569
|
-
|
|
4570
|
-
|
|
5543
|
+
static getProductCategory(item) {
|
|
5544
|
+
const meta = item.metadata || {};
|
|
5545
|
+
const metadataCategory = resolveMetadataValue(meta, "category");
|
|
5546
|
+
if (this.isUsableCategory(metadataCategory)) return String(metadataCategory).trim();
|
|
5547
|
+
const content = item.content || "";
|
|
5548
|
+
const explicitCategoryMatch = content.match(
|
|
5549
|
+
/(?:^|\n)\s*(?:product\s*)?(?:category|department|collection)\s*[:\-–]\s*([^\n]+)/i
|
|
5550
|
+
);
|
|
5551
|
+
if (explicitCategoryMatch && this.isUsableCategory(explicitCategoryMatch[1])) {
|
|
5552
|
+
return explicitCategoryMatch[1].trim();
|
|
5553
|
+
}
|
|
5554
|
+
return null;
|
|
5555
|
+
}
|
|
5556
|
+
static isUsableCategory(value) {
|
|
5557
|
+
if (value === null || value === void 0) return false;
|
|
5558
|
+
const category = String(value).trim();
|
|
5559
|
+
if (!category) return false;
|
|
5560
|
+
return !/^(category|type|tag|price|cost|stock|availability|description|product|item|name|brand|sku|id|ean|currency|color|size|index|internal id|other|\[?type\]?|\[?products?_?\d*\]?)$/i.test(category);
|
|
5561
|
+
}
|
|
4571
5562
|
static aggregateByCategory(data, categories) {
|
|
4572
|
-
const result =
|
|
4573
|
-
categories.forEach((cat) => {
|
|
4574
|
-
result[cat] = 0;
|
|
4575
|
-
});
|
|
5563
|
+
const result = Object.fromEntries(categories.map((c) => [c, 0]));
|
|
4576
5564
|
data.forEach((item) => {
|
|
4577
|
-
|
|
4578
|
-
const
|
|
4579
|
-
if (Object.prototype.hasOwnProperty.call(result,
|
|
4580
|
-
|
|
4581
|
-
} else {
|
|
4582
|
-
result["Other"] = (result["Other"] || 0) + 1;
|
|
4583
|
-
}
|
|
5565
|
+
var _a;
|
|
5566
|
+
const cat = (_a = this.getProductCategory(item)) != null ? _a : "Other";
|
|
5567
|
+
if (Object.prototype.hasOwnProperty.call(result, cat)) result[cat]++;
|
|
5568
|
+
else result["Other"] = (result["Other"] || 0) + 1;
|
|
4584
5569
|
});
|
|
4585
5570
|
return result;
|
|
4586
5571
|
}
|
|
4587
|
-
/**
|
|
4588
|
-
* Helper: Extract time series data
|
|
4589
|
-
*/
|
|
4590
5572
|
static extractTimeSeriesData(data) {
|
|
4591
5573
|
return data.map((item) => {
|
|
5574
|
+
var _a, _b, _c, _d, _e;
|
|
4592
5575
|
const meta = item.metadata || {};
|
|
4593
5576
|
return {
|
|
4594
|
-
timestamp: meta.timestamp
|
|
4595
|
-
value: meta.value
|
|
4596
|
-
label: meta.label
|
|
5577
|
+
timestamp: (_b = (_a = meta.timestamp) != null ? _a : meta.date) != null ? _b : (/* @__PURE__ */ new Date()).toISOString(),
|
|
5578
|
+
value: (_d = (_c = meta.value) != null ? _c : item.score) != null ? _d : 0,
|
|
5579
|
+
label: (_e = meta.label) != null ? _e : item.content.substring(0, 50)
|
|
4597
5580
|
};
|
|
4598
5581
|
});
|
|
4599
5582
|
}
|
|
4600
|
-
|
|
4601
|
-
|
|
4602
|
-
|
|
4603
|
-
|
|
4604
|
-
|
|
4605
|
-
|
|
4606
|
-
|
|
4607
|
-
|
|
4608
|
-
|
|
4609
|
-
|
|
4610
|
-
|
|
4611
|
-
|
|
5583
|
+
static extractNumericValue(meta) {
|
|
5584
|
+
var _a;
|
|
5585
|
+
const preferredKeys = [
|
|
5586
|
+
"value",
|
|
5587
|
+
"count",
|
|
5588
|
+
"total",
|
|
5589
|
+
"average",
|
|
5590
|
+
"avg",
|
|
5591
|
+
"amount",
|
|
5592
|
+
"sales",
|
|
5593
|
+
"revenue",
|
|
5594
|
+
"score",
|
|
5595
|
+
"quantity",
|
|
5596
|
+
"price"
|
|
5597
|
+
];
|
|
5598
|
+
for (const key of preferredKeys) {
|
|
5599
|
+
const raw = (_a = resolveMetadataValue(meta, key)) != null ? _a : meta[key];
|
|
5600
|
+
const value = typeof raw === "number" ? raw : Number(String(raw != null ? raw : "").replace(/[$,% ,]/g, ""));
|
|
5601
|
+
if (Number.isFinite(value)) return value;
|
|
5602
|
+
}
|
|
5603
|
+
for (const value of Object.values(meta)) {
|
|
5604
|
+
const numeric = typeof value === "number" ? value : Number(String(value != null ? value : "").replace(/[$,% ,]/g, ""));
|
|
5605
|
+
if (Number.isFinite(numeric)) return numeric;
|
|
5606
|
+
}
|
|
5607
|
+
return null;
|
|
5608
|
+
}
|
|
5609
|
+
static extractTableColumns(data, query = "") {
|
|
5610
|
+
const q = query.toLowerCase();
|
|
5611
|
+
const availableFields = this.extractAvailableTableFields(data);
|
|
5612
|
+
if (/\b(organization|organisations?|organizations?|company|companies)\b/.test(q) && /\b(employee|employees|staff|headcount|workforce)\b/.test(q)) {
|
|
5613
|
+
const nameField = this.pickTableField(availableFields, [
|
|
5614
|
+
"company name",
|
|
5615
|
+
"organization name",
|
|
5616
|
+
"organisation name",
|
|
5617
|
+
"name",
|
|
5618
|
+
"company",
|
|
5619
|
+
"organization",
|
|
5620
|
+
"organisation"
|
|
5621
|
+
]);
|
|
5622
|
+
const employeeField = this.pickTableField(availableFields, [
|
|
5623
|
+
"number of employees",
|
|
5624
|
+
"employee count",
|
|
5625
|
+
"employees",
|
|
5626
|
+
"employee_count",
|
|
5627
|
+
"number_employees",
|
|
5628
|
+
"num employees",
|
|
5629
|
+
"staff count",
|
|
5630
|
+
"headcount",
|
|
5631
|
+
"workforce"
|
|
5632
|
+
]);
|
|
5633
|
+
const columns = [nameField, employeeField].filter((field) => Boolean(field));
|
|
5634
|
+
if (columns.length > 0) return columns;
|
|
5635
|
+
}
|
|
5636
|
+
const requestedFields = availableFields.map((field) => ({
|
|
5637
|
+
field,
|
|
5638
|
+
score: this.tableFieldQueryScore(field, q)
|
|
5639
|
+
})).filter((item) => item.score > 0).sort((a, b) => b.score - a.score).map((item) => item.field);
|
|
5640
|
+
if (requestedFields.length > 0) {
|
|
5641
|
+
return requestedFields.slice(0, Math.min(6, requestedFields.length));
|
|
5642
|
+
}
|
|
5643
|
+
const metadataFields = Array.from(new Set(
|
|
5644
|
+
data.flatMap((item) => Object.keys(item.metadata || {}).map((k) => this.humanizeFieldName(k)))
|
|
5645
|
+
));
|
|
5646
|
+
return metadataFields.length > 0 ? metadataFields : ["Content"];
|
|
4612
5647
|
}
|
|
4613
|
-
/**
|
|
4614
|
-
* Helper: Extract table row
|
|
4615
|
-
*/
|
|
4616
5648
|
static extractTableRow(item, columns) {
|
|
4617
|
-
|
|
4618
|
-
|
|
4619
|
-
|
|
4620
|
-
|
|
5649
|
+
return columns.map((col) => this.resolveTableCellValue(item, col));
|
|
5650
|
+
}
|
|
5651
|
+
static extractAvailableTableFields(data) {
|
|
5652
|
+
const fields = /* @__PURE__ */ new Map();
|
|
5653
|
+
const addField = (field) => {
|
|
5654
|
+
const clean = this.humanizeFieldName(field);
|
|
5655
|
+
if (!clean || /^(content|\[?type\]?)$/i.test(clean)) return;
|
|
5656
|
+
const normalized = this.normalizeComparableField(clean);
|
|
5657
|
+
if (!fields.has(normalized)) fields.set(normalized, clean);
|
|
5658
|
+
};
|
|
5659
|
+
data.forEach((item) => {
|
|
5660
|
+
Object.keys(item.metadata || {}).forEach(addField);
|
|
5661
|
+
for (const match of (item.content || "").matchAll(/(?:^|\n)\s*([A-Za-z][A-Za-z0-9_ /-]{1,50})\s*[:\-–]\s*([^\n]+)/g)) {
|
|
5662
|
+
addField(match[1]);
|
|
4621
5663
|
}
|
|
4622
|
-
const metaKey = col.charAt(0).toLowerCase() + col.slice(1);
|
|
4623
|
-
const value = meta[metaKey];
|
|
4624
|
-
return value !== void 0 ? String(value) : "";
|
|
4625
5664
|
});
|
|
5665
|
+
return Array.from(fields.values());
|
|
5666
|
+
}
|
|
5667
|
+
static pickTableField(fields, aliases) {
|
|
5668
|
+
const normalizedAliases = aliases.map((alias) => this.normalizeComparableField(alias));
|
|
5669
|
+
for (const alias of normalizedAliases) {
|
|
5670
|
+
const exact = fields.find((field) => this.normalizeComparableField(field) === alias);
|
|
5671
|
+
if (exact) return exact;
|
|
5672
|
+
}
|
|
5673
|
+
for (const alias of normalizedAliases) {
|
|
5674
|
+
const fuzzy = fields.find((field) => {
|
|
5675
|
+
const normalizedField = this.normalizeComparableField(field);
|
|
5676
|
+
if (/(id|code|uuid)$/.test(normalizedField) && !/(id|code|uuid)$/.test(alias)) return false;
|
|
5677
|
+
return normalizedField.includes(alias) || alias.includes(normalizedField);
|
|
5678
|
+
});
|
|
5679
|
+
if (fuzzy) return fuzzy;
|
|
5680
|
+
}
|
|
5681
|
+
return void 0;
|
|
5682
|
+
}
|
|
5683
|
+
static tableFieldQueryScore(field, query) {
|
|
5684
|
+
const normalizedField = this.normalizeComparableField(field);
|
|
5685
|
+
if (!normalizedField) return 0;
|
|
5686
|
+
const fieldTokens = this.fieldTokens(field);
|
|
5687
|
+
return fieldTokens.reduce((score, token) => {
|
|
5688
|
+
if (token.length < 3) return score;
|
|
5689
|
+
return query.includes(token) ? score + 1 : score;
|
|
5690
|
+
}, query.includes(normalizedField) ? 3 : 0);
|
|
5691
|
+
}
|
|
5692
|
+
static resolveTableCellValue(item, column) {
|
|
5693
|
+
if (column === "Content") return item.content.substring(0, 100);
|
|
5694
|
+
const meta = item.metadata || {};
|
|
5695
|
+
const normalizedColumn = this.normalizeComparableField(column);
|
|
5696
|
+
const exactMetadata = Object.entries(meta).find(
|
|
5697
|
+
([key]) => this.normalizeComparableField(key) === normalizedColumn || this.normalizeComparableField(this.humanizeFieldName(key)) === normalizedColumn
|
|
5698
|
+
);
|
|
5699
|
+
if (exactMetadata && exactMetadata[1] !== void 0 && exactMetadata[1] !== null) {
|
|
5700
|
+
return this.toDisplayValue(exactMetadata[1]);
|
|
5701
|
+
}
|
|
5702
|
+
const aliasValue = this.resolveAliasedTableCell(meta, column);
|
|
5703
|
+
if (aliasValue !== void 0) return this.toDisplayValue(aliasValue);
|
|
5704
|
+
const contentValue = this.extractContentFieldValue(item.content, column);
|
|
5705
|
+
if (contentValue !== null) return contentValue;
|
|
5706
|
+
const aliasContentValue = this.extractAliasedContentFieldValue(item.content, column);
|
|
5707
|
+
return aliasContentValue != null ? aliasContentValue : "";
|
|
5708
|
+
}
|
|
5709
|
+
static resolveAliasedTableCell(meta, column) {
|
|
5710
|
+
const normalizedColumn = this.normalizeComparableField(column);
|
|
5711
|
+
const aliases = normalizedColumn.includes("employee") ? ["number of employees", "employee count", "employees", "headcount", "staff count", "workforce"] : normalizedColumn.includes("name") ? ["company name", "organization name", "organisation name", "name", "company", "organization", "organisation"] : [];
|
|
5712
|
+
for (const alias of aliases) {
|
|
5713
|
+
const normalizedAlias = this.normalizeComparableField(alias);
|
|
5714
|
+
const match = Object.entries(meta).find(([key]) => {
|
|
5715
|
+
const normalizedKey = this.normalizeComparableField(key);
|
|
5716
|
+
return normalizedKey === normalizedAlias || normalizedKey.includes(normalizedAlias) || normalizedAlias.includes(normalizedKey);
|
|
5717
|
+
});
|
|
5718
|
+
if (match && match[1] !== void 0 && match[1] !== null) return match[1];
|
|
5719
|
+
}
|
|
5720
|
+
return void 0;
|
|
5721
|
+
}
|
|
5722
|
+
static extractContentFieldValue(content, column) {
|
|
5723
|
+
const escaped = column.replace(/[.*+?^${}()|[\]\\]/g, "\\$&").replace(/\s+/g, "[\\s_ -]+");
|
|
5724
|
+
const pattern = new RegExp(`(?:^|\\n)\\s*${escaped}\\s*[:\\-\u2013]\\s*([^\\n]+)`, "i");
|
|
5725
|
+
const match = content.match(pattern);
|
|
5726
|
+
return match ? match[1].trim() : null;
|
|
5727
|
+
}
|
|
5728
|
+
static extractAliasedContentFieldValue(content, column) {
|
|
5729
|
+
const normalizedColumn = this.normalizeComparableField(column);
|
|
5730
|
+
const aliases = normalizedColumn.includes("employee") ? ["Number Of Employees", "Number of Employees", "Employee Count", "Employees", "Headcount", "Staff Count", "Workforce"] : normalizedColumn.includes("name") ? ["Company Name", "Organization Name", "Organisation Name", "Name", "Company", "Organization", "Organisation"] : [];
|
|
5731
|
+
for (const alias of aliases) {
|
|
5732
|
+
const value = this.extractContentFieldValue(content, alias);
|
|
5733
|
+
if (value !== null) return value;
|
|
5734
|
+
}
|
|
5735
|
+
return null;
|
|
5736
|
+
}
|
|
5737
|
+
static toDisplayValue(value) {
|
|
5738
|
+
if (typeof value === "string" || typeof value === "number" || typeof value === "boolean") return value;
|
|
5739
|
+
return String(value != null ? value : "");
|
|
4626
5740
|
}
|
|
4627
|
-
/**
|
|
4628
|
-
* Helper: Calculate stock counts by category
|
|
4629
|
-
*/
|
|
4630
5741
|
static calculateStockCounts(category, data) {
|
|
4631
5742
|
let inStock = 0;
|
|
4632
5743
|
let outOfStock = 0;
|
|
4633
5744
|
data.forEach((d) => {
|
|
4634
|
-
|
|
4635
|
-
const
|
|
4636
|
-
if (
|
|
4637
|
-
|
|
4638
|
-
|
|
4639
|
-
|
|
4640
|
-
outOfStock++;
|
|
4641
|
-
}
|
|
5745
|
+
var _a, _b;
|
|
5746
|
+
const cat = (_a = this.getProductCategory(d)) != null ? _a : "Other";
|
|
5747
|
+
if (cat === category) {
|
|
5748
|
+
const quantity = (_b = this.extractStockQuantity(d)) != null ? _b : 1;
|
|
5749
|
+
if (this.determineStockStatus(d)) inStock += quantity;
|
|
5750
|
+
else outOfStock += quantity;
|
|
4642
5751
|
}
|
|
4643
5752
|
});
|
|
4644
5753
|
return { inStockCount: inStock, outOfStockCount: outOfStock };
|
|
4645
5754
|
}
|
|
4646
|
-
/**
|
|
4647
|
-
* Helper: Determine if item is in stock
|
|
4648
|
-
*/
|
|
4649
5755
|
static determineStockStatus(item) {
|
|
4650
5756
|
const meta = item.metadata || {};
|
|
4651
|
-
|
|
4652
|
-
|
|
4653
|
-
|
|
4654
|
-
|
|
4655
|
-
|
|
4656
|
-
|
|
4657
|
-
|
|
4658
|
-
|
|
4659
|
-
|
|
5757
|
+
const stockValue = resolveMetadataValue(meta, "stock");
|
|
5758
|
+
if (stockValue !== void 0) {
|
|
5759
|
+
const normalized = String(stockValue).toLowerCase();
|
|
5760
|
+
if (/out[_\s-]?of[_\s-]?stock|unavailable|false|no|sold out|0/.test(normalized)) return false;
|
|
5761
|
+
if (/in[_\s-]?stock|available|true|yes/.test(normalized)) return true;
|
|
5762
|
+
const numeric = this.toFiniteNumber(stockValue);
|
|
5763
|
+
if (numeric !== null) return numeric > 0;
|
|
5764
|
+
}
|
|
5765
|
+
if (meta.inStock !== void 0) return Boolean(meta.inStock);
|
|
5766
|
+
if (meta.stock !== void 0) return Boolean(meta.stock);
|
|
5767
|
+
if (meta.available !== void 0) return Boolean(meta.available);
|
|
4660
5768
|
const content = (item.content || "").toLowerCase();
|
|
4661
|
-
if (
|
|
4662
|
-
|
|
5769
|
+
if (/out[_\s-]?of[_\s-]?stock|unavailable|sold out/.test(content)) return false;
|
|
5770
|
+
if (/in[_\s-]?stock|available/.test(content)) return true;
|
|
5771
|
+
return true;
|
|
5772
|
+
}
|
|
5773
|
+
static extractStockQuantity(item) {
|
|
5774
|
+
const meta = item.metadata || {};
|
|
5775
|
+
const stockValue = resolveMetadataValue(meta, "stock");
|
|
5776
|
+
const numericStock = this.toFiniteNumber(stockValue);
|
|
5777
|
+
if (numericStock !== null) return numericStock;
|
|
5778
|
+
const content = item.content || "";
|
|
5779
|
+
const stockMatch = content.match(/\b(?:stock|inventory|quantity|count)\s*[:\-–]?\s*([\d,]+)/i);
|
|
5780
|
+
if (!stockMatch) return null;
|
|
5781
|
+
return this.toFiniteNumber(stockMatch[1]);
|
|
5782
|
+
}
|
|
5783
|
+
static toFiniteNumber(value) {
|
|
5784
|
+
if (typeof value === "number") return Number.isFinite(value) ? value : null;
|
|
5785
|
+
const numeric = Number(String(value != null ? value : "").replace(/[$,% ,]/g, ""));
|
|
5786
|
+
return Number.isFinite(numeric) ? numeric : null;
|
|
5787
|
+
}
|
|
5788
|
+
// ─── Product Extraction ───────────────────────────────────────────────────
|
|
5789
|
+
static getDynamicVal(meta, uiKey, config, trainedSchema) {
|
|
5790
|
+
var _a;
|
|
5791
|
+
if (!meta) return void 0;
|
|
5792
|
+
const mapping = (_a = config == null ? void 0 : config.rag) == null ? void 0 : _a.uiMapping;
|
|
5793
|
+
if ((mapping == null ? void 0 : mapping[uiKey]) && meta[mapping[uiKey]] !== void 0) return meta[mapping[uiKey]];
|
|
5794
|
+
if (trainedSchema && typeof trainedSchema === "object") {
|
|
5795
|
+
const trainedKey = trainedSchema[uiKey];
|
|
5796
|
+
if (trainedKey && meta[trainedKey] !== void 0) return meta[trainedKey];
|
|
4663
5797
|
}
|
|
4664
|
-
|
|
4665
|
-
|
|
5798
|
+
return resolveMetadataValue(meta, uiKey);
|
|
5799
|
+
}
|
|
5800
|
+
static extractProductInfo(item, config, trainedSchema) {
|
|
5801
|
+
var _a;
|
|
5802
|
+
const meta = item.metadata || {};
|
|
5803
|
+
const name = this.getDynamicVal(meta, "name", config, trainedSchema);
|
|
5804
|
+
const price = this.getDynamicVal(meta, "price", config, trainedSchema);
|
|
5805
|
+
const brand = this.getDynamicVal(meta, "brand", config, trainedSchema);
|
|
5806
|
+
const description = this.cleanProductDescription(
|
|
5807
|
+
(_a = this.extractProductDescriptionFromContent(item.content)) != null ? _a : this.getProductDescriptionValue(meta, config, trainedSchema)
|
|
5808
|
+
);
|
|
5809
|
+
if (name || this.isProductData(item)) {
|
|
5810
|
+
let finalName = name ? String(name) : void 0;
|
|
5811
|
+
if (!finalName) {
|
|
5812
|
+
const nameMatch = item.content.match(/(?:Product|Item|Name):\s*([^\n]+)/i);
|
|
5813
|
+
finalName = nameMatch ? nameMatch[1].trim() : item.content.split("\n")[0].substring(0, 60);
|
|
5814
|
+
}
|
|
5815
|
+
let finalPrice = typeof price === "number" || typeof price === "string" ? price : void 0;
|
|
5816
|
+
if (!finalPrice) {
|
|
5817
|
+
const priceMatch = item.content.match(/(?:Price|Cost):\s*\$?([\d,.]+)/i) || item.content.match(/\$\s*([\d,.]+)/);
|
|
5818
|
+
if (priceMatch) finalPrice = priceMatch[1].replace(/,/g, "");
|
|
5819
|
+
}
|
|
5820
|
+
const imageValue = this.getDynamicVal(meta, "image", config, trainedSchema);
|
|
5821
|
+
return {
|
|
5822
|
+
id: item.id,
|
|
5823
|
+
name: finalName,
|
|
5824
|
+
price: finalPrice,
|
|
5825
|
+
image: typeof imageValue === "string" ? imageValue : void 0,
|
|
5826
|
+
brand: brand ? String(brand) : void 0,
|
|
5827
|
+
description,
|
|
5828
|
+
inStock: this.determineStockStatus(item)
|
|
5829
|
+
};
|
|
4666
5830
|
}
|
|
4667
|
-
return
|
|
5831
|
+
return null;
|
|
4668
5832
|
}
|
|
4669
|
-
|
|
4670
|
-
|
|
4671
|
-
|
|
4672
|
-
|
|
4673
|
-
|
|
4674
|
-
|
|
4675
|
-
|
|
4676
|
-
|
|
4677
|
-
|
|
4678
|
-
|
|
4679
|
-
return fieldCount.size > 2;
|
|
5833
|
+
static extractProductDescriptionFromContent(content) {
|
|
5834
|
+
const bodyMatch = content.match(
|
|
5835
|
+
/\bBody\s*\(HTML\)\s*:\s*([\s\S]*?)(?=\s*[,.]?\s+\b(?:Handle|Title|Vendor|Type|Tags|Published|Option\d*\s+Name|Option\d*\s+Value|Option|Variant|Image|SKU|Price|Status|Category)\b\s*:|$)/i
|
|
5836
|
+
);
|
|
5837
|
+
if (bodyMatch == null ? void 0 : bodyMatch[1]) return bodyMatch[1];
|
|
5838
|
+
const descriptionMatch = content.match(
|
|
5839
|
+
/\b(?:Description|Summary|Details|Body|Content)\s*:\s*([\s\S]*?)(?=\s*[,.]?\s+\b(?:Handle|Title|Vendor|Type|Tags|Published|Option\d*\s+Name|Option\d*\s+Value|Option|Variant|Image|SKU|Price|Status|Category)\b\s*:|$)/i
|
|
5840
|
+
);
|
|
5841
|
+
if (descriptionMatch == null ? void 0 : descriptionMatch[1]) return descriptionMatch[1];
|
|
5842
|
+
return null;
|
|
4680
5843
|
}
|
|
4681
|
-
|
|
4682
|
-
|
|
4683
|
-
|
|
4684
|
-
|
|
4685
|
-
|
|
4686
|
-
|
|
4687
|
-
|
|
4688
|
-
|
|
4689
|
-
|
|
4690
|
-
|
|
4691
|
-
|
|
4692
|
-
|
|
4693
|
-
|
|
4694
|
-
|
|
4695
|
-
|
|
4696
|
-
* @param sources - vector DB matches returned by RAG retrieval
|
|
4697
|
-
* @param llm - any ILLMProvider instance (OpenAI, Anthropic, Ollama, Gemini, REST…)
|
|
4698
|
-
* @returns - a validated UITransformationResponse (type + title + description + data)
|
|
4699
|
-
*/
|
|
4700
|
-
static async analyzeAndDecide(query, sources, llm) {
|
|
4701
|
-
try {
|
|
4702
|
-
const context = this.buildContextSummary(sources);
|
|
4703
|
-
const systemPrompt = this.buildVisualizationSystemPrompt();
|
|
4704
|
-
const userPrompt = [
|
|
4705
|
-
`USER QUESTION: ${query}`,
|
|
4706
|
-
"",
|
|
4707
|
-
"RETRIEVED DATA (JSON):",
|
|
4708
|
-
context
|
|
4709
|
-
].join("\n");
|
|
4710
|
-
const rawResponse = await llm.chat(
|
|
4711
|
-
[{ role: "user", content: userPrompt }],
|
|
4712
|
-
"",
|
|
4713
|
-
{ systemPrompt, temperature: 0 }
|
|
5844
|
+
static getProductDescriptionValue(meta, config, trainedSchema) {
|
|
5845
|
+
const mapped = this.getDynamicVal(meta, "description", config, trainedSchema);
|
|
5846
|
+
if (mapped !== void 0 && mapped !== meta.content && mapped !== meta.text) return mapped;
|
|
5847
|
+
const preferredKeys = [
|
|
5848
|
+
"body_html",
|
|
5849
|
+
"body html",
|
|
5850
|
+
"bodyHtml",
|
|
5851
|
+
"description",
|
|
5852
|
+
"summary",
|
|
5853
|
+
"details",
|
|
5854
|
+
"body"
|
|
5855
|
+
];
|
|
5856
|
+
for (const key of preferredKeys) {
|
|
5857
|
+
const match = Object.keys(meta).find(
|
|
5858
|
+
(candidate) => this.normalizeComparableField(candidate) === this.normalizeComparableField(key)
|
|
4714
5859
|
);
|
|
4715
|
-
|
|
4716
|
-
if (parsed) {
|
|
4717
|
-
console.debug("[UITransformer] LLM chose visualization type:", parsed.type);
|
|
4718
|
-
return parsed;
|
|
4719
|
-
}
|
|
4720
|
-
console.warn("[UITransformer] LLM returned unparseable response; falling back to heuristic.");
|
|
4721
|
-
} catch (err) {
|
|
4722
|
-
console.warn("[UITransformer] analyzeAndDecide LLM call failed; falling back to heuristic.", err);
|
|
5860
|
+
if (match && meta[match] !== void 0 && meta[match] !== null) return meta[match];
|
|
4723
5861
|
}
|
|
4724
|
-
return
|
|
5862
|
+
return void 0;
|
|
4725
5863
|
}
|
|
4726
|
-
|
|
4727
|
-
|
|
4728
|
-
|
|
5864
|
+
static cleanProductDescription(raw) {
|
|
5865
|
+
if (raw === null || raw === void 0) return void 0;
|
|
5866
|
+
const extracted = this.extractProductDescriptionFromContent(String(raw));
|
|
5867
|
+
if (extracted && extracted !== String(raw)) return this.cleanProductDescription(extracted);
|
|
5868
|
+
const text = String(raw).replace(/<[^>]*>/g, " ").replace(/ /gi, " ").replace(/&/gi, "&").replace(/"/gi, '"').replace(/'/gi, "'").replace(/\[[^\]]*TYPE[^\]]*\]/gi, " ").replace(/\b(?:Handle|Title|Vendor|Type|Tags|Published|Option\d*\s+Name|Option\d*\s+Value|Option|Variant|Image|SKU|Price|Status|Category)\s*:\s*[^,.\n:]+[,.]?/gi, " ").replace(/\bBody\s*\(HTML\)\s*:\s*/gi, " ").replace(/\b(?:Description|Summary|Details|Body|Content)\s*:\s*/gi, " ").replace(/\s+/g, " ").trim();
|
|
5869
|
+
return text || void 0;
|
|
5870
|
+
}
|
|
5871
|
+
// ─── Visualization Prompt ─────────────────────────────────────────────────
|
|
4729
5872
|
static buildVisualizationSystemPrompt() {
|
|
4730
5873
|
return `You are a data visualization expert embedded in a RAG chat system.
|
|
4731
|
-
You will receive a user question and structured data
|
|
4732
|
-
Your ONLY job is to
|
|
4733
|
-
|
|
5874
|
+
You will receive a user question, a pre-computed INTENT object, and structured data from a vector database.
|
|
5875
|
+
Your ONLY job is to return a single JSON object describing how to visualize the data.
|
|
5876
|
+
The INTENT object is authoritative \u2014 honor it. For example:
|
|
5877
|
+
- If intent.wantsExplicitTable is false, never return type "table".
|
|
5878
|
+
- If intent.isTemporal is true, prefer line_chart.
|
|
5879
|
+
- If intent.visualizationHint is "comparison" or "ranking", prefer bar_chart.
|
|
5880
|
+
- If intent.visualizationHint is "composition", prefer pie_chart.
|
|
5881
|
+
- If intent.recommendedChart is unsupported by the response schema, use the nearest supported type.
|
|
5882
|
+
- Always respond in the language specified by intent.language.
|
|
4734
5883
|
|
|
4735
|
-
Return ONLY a valid JSON object \u2014 no markdown
|
|
5884
|
+
Return ONLY a valid JSON object \u2014 no markdown fences, no prose.
|
|
4736
5885
|
|
|
4737
|
-
The JSON must have this exact shape:
|
|
4738
5886
|
{
|
|
4739
|
-
"type": "bar_chart" | "line_chart" | "pie_chart" | "radar_chart" | "table" | "text",
|
|
4740
|
-
"title": "
|
|
4741
|
-
"description": "
|
|
4742
|
-
"data": <structured data
|
|
5887
|
+
"type": "bar_chart" | "horizontal_bar" | "line_chart" | "pie_chart" | "histogram" | "scatter_plot" | "radar_chart" | "metric_card" | "geo_map" | "table" | "text",
|
|
5888
|
+
"title": "concise descriptive title",
|
|
5889
|
+
"description": "one sentence describing the visualization",
|
|
5890
|
+
"data": <structured data per type below>
|
|
4743
5891
|
}
|
|
4744
5892
|
|
|
4745
|
-
DATA
|
|
4746
|
-
- bar_chart:
|
|
4747
|
-
-
|
|
4748
|
-
-
|
|
4749
|
-
-
|
|
4750
|
-
|
|
4751
|
-
|
|
4752
|
-
|
|
4753
|
-
|
|
4754
|
-
|
|
4755
|
-
|
|
4756
|
-
-
|
|
4757
|
-
- text: { "content": "<prose answer>" }
|
|
4758
|
-
|
|
4759
|
-
DECISION RULES (follow strictly):
|
|
4760
|
-
1. bar_chart \u2192 comparing quantities across categories (e.g. sales by region, price by product). Use this when there is only ONE value per category.
|
|
4761
|
-
2. line_chart \u2192 trends or changes over time (dates, months, years, sequential events)
|
|
4762
|
-
3. pie_chart \u2192 proportional breakdown or percentage distribution
|
|
4763
|
-
4. radar_chart \u2192 comparing 2 or more products across MULTIPLE attributes (e.g. comparing features, ratings, or characteristics of 2 specific products). If the user asks to "compare" products and the data contains multiple dimensions or ratings for them, you MUST use this.
|
|
4764
|
-
5. table \u2192 multi-field structured records where each item has \u2265 3 attributes
|
|
4765
|
-
6. text \u2192 conversational or free-form answers where no chart adds value
|
|
5893
|
+
DATA SHAPES:
|
|
5894
|
+
- bar_chart: [{ "category": string, "value": number }]
|
|
5895
|
+
- horizontal_bar: [{ "category": string, "value": number }]
|
|
5896
|
+
- histogram: [{ "category": string, "value": number }]
|
|
5897
|
+
- line_chart: [{ "timestamp": string, "value": number, "label": string }]
|
|
5898
|
+
- pie_chart: [{ "label": string, "value": number }]
|
|
5899
|
+
- scatter_plot:[{ "x": number, "y": number, "label": string }]
|
|
5900
|
+
- radar_chart: [{ "attribute": string, "<series1>": number, "<series2>": number, ... }]
|
|
5901
|
+
- metric_card: { "label": string, "value": number, "operation": "sum"|"average"|"count"|"min"|"max"|"median" }
|
|
5902
|
+
- geo_map: [{ "category": string, "value": number }]
|
|
5903
|
+
- table: { "columns": string[], "rows": (string|number)[][] }
|
|
5904
|
+
- text: { "content": "A concise plain-language answer." }
|
|
4766
5905
|
|
|
4767
|
-
|
|
4768
|
-
|
|
4769
|
-
|
|
4770
|
-
|
|
4771
|
-
|
|
5906
|
+
RULES:
|
|
5907
|
+
1. Aggregate values \u2014 never pass raw nested objects in chart arrays.
|
|
5908
|
+
2. All "value" fields must be numbers.
|
|
5909
|
+
3. Cap bar/line/pie at 12 data points.
|
|
5910
|
+
4. Use dollar signs only for monetary prices, not for stock quantities or years.
|
|
5911
|
+
5. If no relevant data is found, return text: "I cannot answer this question based on the available information."`;
|
|
4772
5912
|
}
|
|
4773
|
-
/**
|
|
4774
|
-
* Serialize retrieved vector matches into a compact JSON context string.
|
|
4775
|
-
* Limits the total character count to avoid exceeding LLM context windows.
|
|
4776
|
-
*/
|
|
4777
5913
|
static buildContextSummary(sources, maxChars = 6e3) {
|
|
4778
5914
|
const items = sources.map((s, i) => {
|
|
4779
5915
|
var _a, _b, _c, _d;
|
|
@@ -4817,14 +5953,16 @@ Given these metadata keys from a database: [${keys.join(", ")}]
|
|
|
4817
5953
|
Identify which keys best correspond to these standard UI properties:
|
|
4818
5954
|
${propertyList}
|
|
4819
5955
|
|
|
4820
|
-
Return ONLY a JSON object where the keys are the UI properties and the values are the matching database keys.
|
|
5956
|
+
Return ONLY a valid JSON object where the keys are the UI properties and the values are the EXACT matching database keys from the list above.
|
|
4821
5957
|
If no good match is found for a property, omit it.
|
|
4822
5958
|
|
|
4823
5959
|
Example:
|
|
4824
5960
|
{
|
|
4825
|
-
"name": "
|
|
4826
|
-
"price": "
|
|
4827
|
-
"brand": "
|
|
5961
|
+
"name": "Title",
|
|
5962
|
+
"price": "Variant Price",
|
|
5963
|
+
"brand": "Vendor",
|
|
5964
|
+
"image": "Image Src",
|
|
5965
|
+
"stock": "Variant Inventory Qty"
|
|
4828
5966
|
}
|
|
4829
5967
|
`;
|
|
4830
5968
|
try {
|
|
@@ -4994,27 +6132,38 @@ var Pipeline = class {
|
|
|
4994
6132
|
async initialize() {
|
|
4995
6133
|
var _a;
|
|
4996
6134
|
if (this.initialised) return;
|
|
4997
|
-
const chartInstruction = `You are a helpful
|
|
6135
|
+
const chartInstruction = `You are a helpful assistant. Use the provided context to answer questions accurately.
|
|
4998
6136
|
|
|
4999
|
-
###
|
|
6137
|
+
### CRITICAL RULES:
|
|
6138
|
+
- ONLY answer the user's specific question. Do NOT suggest or recommend unrelated products unless specifically asked.
|
|
5000
6139
|
- NEVER generate markdown tables. If you do, the UI will break.
|
|
5001
6140
|
- NEVER generate HTML tags like <figure>, <tbody>, <tr>, etc.
|
|
5002
6141
|
- NEVER generate text-based charts or graphs.
|
|
6142
|
+
- NEVER say you cannot display, render, draw, or create a chart when the user asks for a visualization. The UI handles visual rendering separately.
|
|
5003
6143
|
- ONLY use plain text and bullet points.
|
|
6144
|
+
- NEVER use the plus sign (+) as a separator between names, categories, or products. Use commas or bullet points.
|
|
6145
|
+
- ONLY answer the question if the sources contain relevant information to answer it, else say that you cannot answer the question.
|
|
6146
|
+
- If answer cannot be found in the sources, say that you cannot answer the question and do not hallucinate.
|
|
6147
|
+
- Do not use information from previous turns to answer the question.
|
|
6148
|
+
- You CAN use numbers for years and counts (e.g., 2006, 5800). But NEVER put a dollar sign ($) before non-monetary numbers like Stock quantities, EAN numbers, or years. Only use dollar signs for actual monetary prices.
|
|
5004
6149
|
|
|
5005
6150
|
### PRODUCT DISPLAY:
|
|
5006
|
-
-
|
|
6151
|
+
- Recommended Products should only be displayed when the user explicitly asks for products.
|
|
6152
|
+
- When recommending products (ONLY when asked), simply list their names, prices, and features in a friendly, conversational manner.
|
|
5007
6153
|
- The UI will automatically detect these products and show high-quality product cards in a carousel below your message.
|
|
6154
|
+
- For product descriptions, summarize customer-facing details only. Do NOT list internal catalog fields such as Handle, Body (HTML), Vendor, Type, Tags, Published, Option, Variant, SKU, or raw metadata labels.
|
|
5008
6155
|
- Do NOT try to format product lists as tables.
|
|
5009
6156
|
`;
|
|
5010
6157
|
this.config.llm.systemPrompt = chartInstruction;
|
|
5011
6158
|
this.vectorDB = await ProviderRegistry.createVectorProvider(this.config.vectorDb);
|
|
5012
|
-
|
|
6159
|
+
this.llmRouter = new LLMRouter(this.config);
|
|
6160
|
+
const { llmProvider: resolvedLLM, embeddingProvider } = await EmbeddingStrategyResolver.resolve(
|
|
5013
6161
|
this.config.llm,
|
|
5014
6162
|
this.config.embedding
|
|
5015
6163
|
);
|
|
5016
|
-
this.llmProvider = llmProvider;
|
|
5017
6164
|
this.embeddingProvider = embeddingProvider;
|
|
6165
|
+
await this.llmRouter.initialize(resolvedLLM);
|
|
6166
|
+
this.llmProvider = this.llmRouter.get("default");
|
|
5018
6167
|
if (this.config.graphDb) {
|
|
5019
6168
|
this.graphDB = await ProviderRegistry.createGraphProvider(this.config.graphDb);
|
|
5020
6169
|
await this.graphDB.initialize();
|
|
@@ -5104,7 +6253,10 @@ var Pipeline = class {
|
|
|
5104
6253
|
upsertBatchOptions
|
|
5105
6254
|
);
|
|
5106
6255
|
if (upsertResult.errors.length > 0) {
|
|
5107
|
-
console.warn(`[Pipeline] Ingestion: ${upsertResult.errors.length} batches failed
|
|
6256
|
+
console.warn(`[Pipeline] Ingestion: ${upsertResult.errors.length} batches failed. Error details:`);
|
|
6257
|
+
upsertResult.errors.forEach((err, idx) => {
|
|
6258
|
+
console.warn(` Batch ${idx + 1} Error: ${err.error.message || String(err.error)}`);
|
|
6259
|
+
});
|
|
5108
6260
|
}
|
|
5109
6261
|
return upsertResult.totalProcessed;
|
|
5110
6262
|
}
|
|
@@ -5171,10 +6323,16 @@ var Pipeline = class {
|
|
|
5171
6323
|
/**
|
|
5172
6324
|
* High-performance streaming RAG flow.
|
|
5173
6325
|
* Yields text chunks first, then the retrieval metadata + observability trace at the end.
|
|
6326
|
+
*
|
|
6327
|
+
* Latency optimizations:
|
|
6328
|
+
* - Strategy classification runs in parallel with query embedding (saves ~400ms)
|
|
6329
|
+
* - Hallucination scoring is fire-and-forget (doesn't block metadata yield)
|
|
6330
|
+
* - UITransformation is computed after text streaming and emitted with metadata
|
|
6331
|
+
* - SchemaMapper.train runs while answer generation streams
|
|
5174
6332
|
*/
|
|
5175
6333
|
askStream(_0) {
|
|
5176
6334
|
return __asyncGenerator(this, arguments, function* (question, history = [], namespace) {
|
|
5177
|
-
var _a, _b, _c, _d, _e, _f, _g, _h, _i;
|
|
6335
|
+
var _a, _b, _c, _d, _e, _f, _g, _h, _i, _j, _k, _l, _m;
|
|
5178
6336
|
yield new __await(this.initialize());
|
|
5179
6337
|
const ns = namespace != null ? namespace : this.config.projectId;
|
|
5180
6338
|
const topK = (_b = (_a = this.config.rag) == null ? void 0 : _a.topK) != null ? _b : 5;
|
|
@@ -5190,27 +6348,61 @@ var Pipeline = class {
|
|
|
5190
6348
|
}
|
|
5191
6349
|
const hints = QueryProcessor.extractQueryFieldHints(question, (_f = this.config.rag) == null ? void 0 : _f.filterableFields);
|
|
5192
6350
|
const filter = QueryProcessor.buildQueryFilter(question, hints);
|
|
6351
|
+
const numericPredicates = QueryProcessor.extractNumericPredicates(question, (_g = this.config.rag) == null ? void 0 : _g.filterableFields);
|
|
6352
|
+
if (numericPredicates.length > 0) {
|
|
6353
|
+
filter.__numericPredicates = numericPredicates;
|
|
6354
|
+
}
|
|
5193
6355
|
const embedStart = performance.now();
|
|
5194
|
-
const
|
|
5195
|
-
|
|
5196
|
-
|
|
5197
|
-
|
|
5198
|
-
|
|
6356
|
+
const cacheKey = `${ns}::${searchQuery}`;
|
|
6357
|
+
const cachedVector = this.embeddingCache.get(cacheKey);
|
|
6358
|
+
const [strategyResult, embeddedVector] = yield new __await(Promise.all([
|
|
6359
|
+
QueryProcessor.determineRetrievalStrategy(
|
|
6360
|
+
searchQuery,
|
|
6361
|
+
this.llmRouter.get("fast"),
|
|
6362
|
+
(_h = this.config.rag) == null ? void 0 : _h.graphKeywords,
|
|
6363
|
+
(_i = this.config.rag) == null ? void 0 : _i.vectorKeywords
|
|
6364
|
+
),
|
|
6365
|
+
// Embed immediately regardless of strategy — costs nothing if cached.
|
|
6366
|
+
// If strategy turns out to be 'graph'-only we just won't use the vector.
|
|
6367
|
+
cachedVector ? Promise.resolve(cachedVector) : this.embeddingProvider.embed(searchQuery, { taskType: "query" })
|
|
6368
|
+
]));
|
|
6369
|
+
const queryVector = embeddedVector;
|
|
6370
|
+
if (!cachedVector && queryVector.length > 0) {
|
|
6371
|
+
this.embeddingCache.set(cacheKey, queryVector);
|
|
6372
|
+
}
|
|
6373
|
+
const graphData = (strategyResult === "graph" || strategyResult === "both") && this.graphDB && ((_j = this.config.rag) == null ? void 0 : _j.useGraphRetrieval) ? yield new __await(this.graphDB.query(searchQuery)) : void 0;
|
|
6374
|
+
const hasMetadataFilter = filter.metadata && Object.keys(filter.metadata).length > 0;
|
|
6375
|
+
const hasNumericPredicates = Array.isArray(filter.__numericPredicates) && filter.__numericPredicates.length > 0;
|
|
6376
|
+
const wantsExhaustiveList = hasNumericPredicates || hasMetadataFilter || /\b(list|all|show|provide|give|get|browse|find|search|display)\b/i.test(question);
|
|
6377
|
+
const retrievalLimit = wantsExhaustiveList ? Math.max(topK * 20, 100) : topK * 2;
|
|
6378
|
+
const rawSources = (strategyResult === "vector" || strategyResult === "both") && queryVector && queryVector.length > 0 ? yield new __await(this.vectorDB.query(queryVector, retrievalLimit, ns, filter)) : [];
|
|
5199
6379
|
const retrieveEnd = performance.now();
|
|
5200
6380
|
const embedMs = retrieveEnd - embedStart;
|
|
5201
6381
|
const retrieveMs = retrieveEnd - embedStart;
|
|
5202
6382
|
const rerankStart = performance.now();
|
|
5203
|
-
|
|
5204
|
-
|
|
5205
|
-
|
|
5206
|
-
|
|
5207
|
-
|
|
6383
|
+
const structuredSources = this.applyStructuredFilters(rawSources, filter);
|
|
6384
|
+
let fullSources = hasNumericPredicates ? structuredSources : structuredSources.filter((m) => m.score >= scoreThreshold);
|
|
6385
|
+
const rerankLimit = wantsExhaustiveList ? retrievalLimit : topK;
|
|
6386
|
+
if (!hasNumericPredicates && ((_k = this.config.rag) == null ? void 0 : _k.useReranking)) {
|
|
6387
|
+
fullSources = yield new __await(this.reranker.rerank(fullSources, question, rerankLimit));
|
|
6388
|
+
} else if (!wantsExhaustiveList) {
|
|
6389
|
+
fullSources = fullSources.slice(0, topK);
|
|
5208
6390
|
}
|
|
5209
6391
|
const rerankMs = performance.now() - rerankStart;
|
|
5210
|
-
let context =
|
|
6392
|
+
let context = fullSources.length ? fullSources.map((m, i) => `[Source ${i + 1}]
|
|
5211
6393
|
${m.content}`).join("\n\n---\n\n") : "No relevant context found.";
|
|
6394
|
+
let displayCount = 15;
|
|
6395
|
+
if (hasMetadataFilter) {
|
|
6396
|
+
displayCount = fullSources.length;
|
|
6397
|
+
} else {
|
|
6398
|
+
const highlyRelevant = fullSources.filter((m) => m.score >= 0.4);
|
|
6399
|
+
displayCount = Math.max(highlyRelevant.length, topK);
|
|
6400
|
+
if (displayCount > 15) {
|
|
6401
|
+
displayCount = 15;
|
|
6402
|
+
}
|
|
6403
|
+
}
|
|
6404
|
+
const sources = [...fullSources].sort((a, b) => b.score - a.score).slice(0, displayCount);
|
|
5212
6405
|
if (graphData && graphData.nodes.length > 0) {
|
|
5213
|
-
console.log(`[Graph Retrieval] Found ${graphData.nodes.length} relevant entities.`);
|
|
5214
6406
|
const graphContext = graphData.nodes.map(
|
|
5215
6407
|
(n) => `Entity: ${n.label} (${n.id})${n.properties ? " - " + JSON.stringify(n.properties) : ""}`
|
|
5216
6408
|
).join("\n");
|
|
@@ -5221,20 +6413,29 @@ VECTOR CONTEXT:
|
|
|
5221
6413
|
${context}`;
|
|
5222
6414
|
}
|
|
5223
6415
|
const allMetadataKeys = Array.from(new Set(sources.flatMap((s) => Object.keys(s.metadata || {}))));
|
|
5224
|
-
const
|
|
5225
|
-
const
|
|
6416
|
+
const trainedSchemaPromise = allMetadataKeys.length > 0 ? SchemaMapper.train(this.llmProvider, ns, allMetadataKeys).catch(() => void 0) : Promise.resolve(void 0);
|
|
6417
|
+
const uiTransformationPromise = trainedSchemaPromise.then(
|
|
6418
|
+
(trainedSchema) => this.generateUiTransformation(
|
|
6419
|
+
question,
|
|
6420
|
+
sources,
|
|
6421
|
+
trainedSchema,
|
|
6422
|
+
hasNumericPredicates
|
|
6423
|
+
)
|
|
6424
|
+
).catch((uiError) => {
|
|
6425
|
+
console.warn("[Pipeline] UI transformation failed concurrently:", uiError);
|
|
6426
|
+
return UITransformer.transform(question, sources, this.config);
|
|
6427
|
+
});
|
|
6428
|
+
const restrictionSuffix = "\n\n(IMPORTANT: Format your response beautifully using rich Markdown! Use bolding for emphasis, headings (##) for structure, bullet points for lists, and proper line breaks between paragraphs to make it highly readable. However, NEVER generate Markdown tables, HTML figures, or text charts (the UI renders requested charts separately, so NEVER say you cannot create, display, draw, render, or provide a chart/visualization). If listing products, use simple markdown bullet points or comma-separated names. NEVER use plus signs (+) to separate product names, category names, or list items. For product description/detail questions, provide customer-facing prose only and do NOT list internal catalog fields such as Handle, Body (HTML), Vendor, Type, Tags, Published, Option, Variant, SKU, or raw metadata labels. You CAN use numbers for years/counts like 2006 or 5800, but NEVER put a dollar sign ($) before them.)";
|
|
5226
6429
|
const hardenedHistory = [...history];
|
|
5227
6430
|
const userQuestion = { role: "user", content: question + restrictionSuffix };
|
|
5228
6431
|
const messages = [...hardenedHistory, userQuestion];
|
|
5229
|
-
const systemPrompt = (
|
|
6432
|
+
const systemPrompt = (_l = this.config.llm.systemPrompt) != null ? _l : "";
|
|
5230
6433
|
const userPrompt = messages.map((m) => `${m.role}: ${m.content}`).join("\n");
|
|
5231
6434
|
let fullReply = "";
|
|
5232
6435
|
const generateStart = performance.now();
|
|
5233
6436
|
if (this.llmProvider.chatStream) {
|
|
5234
6437
|
const stream = this.llmProvider.chatStream(messages, context);
|
|
5235
|
-
if (!stream) {
|
|
5236
|
-
throw new Error(`[Pipeline] ${this.config.llm.provider} chatStream returned undefined`);
|
|
5237
|
-
}
|
|
6438
|
+
if (!stream) throw new Error(`[Pipeline] ${this.config.llm.provider} chatStream returned undefined`);
|
|
5238
6439
|
try {
|
|
5239
6440
|
for (var iter = __forAwait(stream), more, temp, error; more = !(temp = yield new __await(iter.next())).done; more = false) {
|
|
5240
6441
|
const chunk = temp.value;
|
|
@@ -5261,7 +6462,7 @@ ${context}`;
|
|
|
5261
6462
|
const latency = {
|
|
5262
6463
|
embedMs: Math.round(embedMs),
|
|
5263
6464
|
retrieveMs: Math.round(retrieveMs),
|
|
5264
|
-
rerankMs: ((
|
|
6465
|
+
rerankMs: ((_m = this.config.rag) == null ? void 0 : _m.useReranking) ? Math.round(rerankMs) : void 0,
|
|
5265
6466
|
generateMs: Math.round(generateMs),
|
|
5266
6467
|
totalMs: Math.round(totalMs)
|
|
5267
6468
|
};
|
|
@@ -5274,9 +6475,6 @@ ${context}`;
|
|
|
5274
6475
|
totalTokens: promptTokens + completionTokens,
|
|
5275
6476
|
estimatedCostUsd: estimateCostUsd(promptTokens, completionTokens, this.config.llm.model)
|
|
5276
6477
|
};
|
|
5277
|
-
const hallucinationResult = yield new __await(scoreHallucination(this.llmProvider, fullReply, context).catch(() => void 0));
|
|
5278
|
-
const trainedSchema = yield new __await(trainingPromise);
|
|
5279
|
-
const uiTransformation = yield new __await(this.generateUiTransformation(question, sources, trainedSchema));
|
|
5280
6478
|
const trace = {
|
|
5281
6479
|
requestId,
|
|
5282
6480
|
query: question,
|
|
@@ -5295,10 +6493,9 @@ ${context}`;
|
|
|
5295
6493
|
}),
|
|
5296
6494
|
latency,
|
|
5297
6495
|
tokens,
|
|
5298
|
-
hallucinationScore: hallucinationResult == null ? void 0 : hallucinationResult.score,
|
|
5299
|
-
hallucinationReason: hallucinationResult == null ? void 0 : hallucinationResult.reason,
|
|
5300
6496
|
timestamp: (/* @__PURE__ */ new Date()).toISOString()
|
|
5301
6497
|
};
|
|
6498
|
+
const uiTransformation = yield new __await(uiTransformationPromise);
|
|
5302
6499
|
yield {
|
|
5303
6500
|
reply: "",
|
|
5304
6501
|
sources,
|
|
@@ -5306,6 +6503,13 @@ ${context}`;
|
|
|
5306
6503
|
ui_transformation: uiTransformation,
|
|
5307
6504
|
trace
|
|
5308
6505
|
};
|
|
6506
|
+
scoreHallucination(this.llmProvider, fullReply, context).then((hallucinationResult) => {
|
|
6507
|
+
if (hallucinationResult) {
|
|
6508
|
+
trace.hallucinationScore = hallucinationResult.score;
|
|
6509
|
+
trace.hallucinationReason = hallucinationResult.reason;
|
|
6510
|
+
}
|
|
6511
|
+
}).catch(() => {
|
|
6512
|
+
});
|
|
5309
6513
|
} catch (error2) {
|
|
5310
6514
|
throw new Error(`[Pipeline] Stream failed: ${error2 instanceof Error ? error2.message : String(error2)}`);
|
|
5311
6515
|
}
|
|
@@ -5315,24 +6519,107 @@ ${context}`;
|
|
|
5315
6519
|
* Universal retrieval method combining all enabled providers.
|
|
5316
6520
|
* Uses an LRU-bounded embedding cache to avoid re-embedding the same query.
|
|
5317
6521
|
*/
|
|
5318
|
-
async generateUiTransformation(question, sources, trainedSchema) {
|
|
6522
|
+
async generateUiTransformation(question, sources, trainedSchema, forceDeterministic = false) {
|
|
5319
6523
|
if (!sources || sources.length === 0) {
|
|
5320
6524
|
return UITransformer.transform(question, sources, this.config, trainedSchema);
|
|
5321
6525
|
}
|
|
6526
|
+
if (forceDeterministic) {
|
|
6527
|
+
return UITransformer.transform(question, sources, this.config, trainedSchema);
|
|
6528
|
+
}
|
|
5322
6529
|
try {
|
|
5323
|
-
return await UITransformer.analyzeAndDecide(question, sources, this.
|
|
6530
|
+
return await UITransformer.analyzeAndDecide(question, sources, this.llmRouter.get("fast"));
|
|
5324
6531
|
} catch (err) {
|
|
5325
6532
|
console.warn("[Pipeline] generateUiTransformation failed, using heuristic fallback:", err);
|
|
5326
6533
|
return UITransformer.transform(question, sources, this.config, trainedSchema);
|
|
5327
6534
|
}
|
|
5328
6535
|
}
|
|
6536
|
+
applyStructuredFilters(sources, filter) {
|
|
6537
|
+
const predicates = Array.isArray(filter.__numericPredicates) ? filter.__numericPredicates : [];
|
|
6538
|
+
if (predicates.length === 0) return sources;
|
|
6539
|
+
return sources.filter((source) => predicates.every((predicate) => {
|
|
6540
|
+
const value = this.resolveNumericPredicateValue(source, predicate);
|
|
6541
|
+
return value !== null && this.matchesNumericPredicate(value, predicate);
|
|
6542
|
+
})).sort((a, b) => {
|
|
6543
|
+
var _a, _b;
|
|
6544
|
+
const primary = predicates[0];
|
|
6545
|
+
const aValue = (_a = this.resolveNumericPredicateValue(a, primary)) != null ? _a : 0;
|
|
6546
|
+
const bValue = (_b = this.resolveNumericPredicateValue(b, primary)) != null ? _b : 0;
|
|
6547
|
+
return primary.operator === "lt" || primary.operator === "lte" ? aValue - bValue : bValue - aValue;
|
|
6548
|
+
});
|
|
6549
|
+
}
|
|
6550
|
+
resolveNumericPredicateValue(source, predicate) {
|
|
6551
|
+
const meta = source.metadata || {};
|
|
6552
|
+
const field = predicate.field;
|
|
6553
|
+
const entries = Object.entries(meta).filter(([, value]) => value !== null && value !== void 0 && typeof value !== "object");
|
|
6554
|
+
if (field) {
|
|
6555
|
+
const normalizedField = this.normalizeComparableField(field);
|
|
6556
|
+
const exact = entries.find(([key]) => this.normalizeComparableField(key) === normalizedField);
|
|
6557
|
+
const fuzzy = exact != null ? exact : entries.map(([key, value]) => ({ key, value, score: this.fieldSimilarityScore(key, field) })).filter((candidate) => candidate.score > 0).sort((a, b) => b.score - a.score)[0];
|
|
6558
|
+
if (fuzzy) {
|
|
6559
|
+
const value = this.toFiniteNumber(Array.isArray(fuzzy) ? fuzzy[1] : fuzzy.value);
|
|
6560
|
+
if (value !== null) return value;
|
|
6561
|
+
}
|
|
6562
|
+
const contentValue = this.extractNumericValueFromContent(source.content, field);
|
|
6563
|
+
if (contentValue !== null) return contentValue;
|
|
6564
|
+
}
|
|
6565
|
+
for (const [key, value] of entries) {
|
|
6566
|
+
if (/(id|sku|ean|upc|phone|zip|postal|code)/i.test(key)) continue;
|
|
6567
|
+
const numeric = this.toFiniteNumber(value);
|
|
6568
|
+
if (numeric !== null) return numeric;
|
|
6569
|
+
}
|
|
6570
|
+
return null;
|
|
6571
|
+
}
|
|
6572
|
+
extractNumericValueFromContent(content, field) {
|
|
6573
|
+
const escapedWords = field.split(/\s+|_+|-+/).filter(Boolean).map((word) => word.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"));
|
|
6574
|
+
if (escapedWords.length === 0) return null;
|
|
6575
|
+
const pattern = new RegExp(`(?:${escapedWords.join("[\\\\s_:-]*")})\\s*[:\\-\u2013]?\\s*([\\d,]+(?:\\.\\d+)?)`, "i");
|
|
6576
|
+
const match = content.match(pattern);
|
|
6577
|
+
return match ? this.toFiniteNumber(match[1]) : null;
|
|
6578
|
+
}
|
|
6579
|
+
matchesNumericPredicate(value, predicate) {
|
|
6580
|
+
switch (predicate.operator) {
|
|
6581
|
+
case "gt":
|
|
6582
|
+
return value > predicate.value;
|
|
6583
|
+
case "gte":
|
|
6584
|
+
return value >= predicate.value;
|
|
6585
|
+
case "lt":
|
|
6586
|
+
return value < predicate.value;
|
|
6587
|
+
case "lte":
|
|
6588
|
+
return value <= predicate.value;
|
|
6589
|
+
case "eq":
|
|
6590
|
+
default:
|
|
6591
|
+
return value === predicate.value;
|
|
6592
|
+
}
|
|
6593
|
+
}
|
|
6594
|
+
normalizeComparableField(value) {
|
|
6595
|
+
return value.toLowerCase().replace(/[^a-z0-9]/g, "");
|
|
6596
|
+
}
|
|
6597
|
+
fieldSimilarityScore(candidate, requested) {
|
|
6598
|
+
const normalizedCandidate = this.normalizeComparableField(candidate);
|
|
6599
|
+
const normalizedRequested = this.normalizeComparableField(requested);
|
|
6600
|
+
if (normalizedCandidate === normalizedRequested) return 10;
|
|
6601
|
+
if (normalizedCandidate.includes(normalizedRequested) || normalizedRequested.includes(normalizedCandidate)) return 8;
|
|
6602
|
+
const candidateTokens = this.fieldTokens(candidate);
|
|
6603
|
+
const requestedTokens = this.fieldTokens(requested);
|
|
6604
|
+
const overlap = requestedTokens.filter((token) => candidateTokens.includes(token)).length;
|
|
6605
|
+
if (overlap === 0) return 0;
|
|
6606
|
+
return overlap / Math.max(requestedTokens.length, candidateTokens.length);
|
|
6607
|
+
}
|
|
6608
|
+
fieldTokens(value) {
|
|
6609
|
+
return value.toLowerCase().split(/[^a-z0-9]+/).map((token) => token.replace(/ies$/, "y").replace(/s$/, "")).filter((token) => token.length > 1);
|
|
6610
|
+
}
|
|
6611
|
+
toFiniteNumber(value) {
|
|
6612
|
+
if (typeof value === "number") return Number.isFinite(value) ? value : null;
|
|
6613
|
+
const numeric = Number(String(value != null ? value : "").replace(/[$,% ,]/g, ""));
|
|
6614
|
+
return Number.isFinite(numeric) ? numeric : null;
|
|
6615
|
+
}
|
|
5329
6616
|
async retrieve(query, options) {
|
|
5330
|
-
var _a, _b, _c;
|
|
6617
|
+
var _a, _b, _c, _d, _e, _f;
|
|
5331
6618
|
const ns = (_a = options.namespace) != null ? _a : this.config.projectId;
|
|
5332
6619
|
const topK = (_b = options.topK) != null ? _b : 5;
|
|
5333
6620
|
const cacheKey = `${ns}::${query}`;
|
|
5334
6621
|
let queryVector = this.embeddingCache.get(cacheKey);
|
|
5335
|
-
const strategy = await QueryProcessor.determineRetrievalStrategy(query, this.
|
|
6622
|
+
const strategy = await QueryProcessor.determineRetrievalStrategy(query, this.llmRouter.get("fast"));
|
|
5336
6623
|
console.debug(`[Pipeline] Determined retrieval strategy: ${strategy}`);
|
|
5337
6624
|
const [retrievedVector, graphData] = await Promise.all([
|
|
5338
6625
|
// Only embed if we need vector search (strategy is 'vector' or 'both')
|
|
@@ -5344,8 +6631,27 @@ ${context}`;
|
|
|
5344
6631
|
this.embeddingCache.set(cacheKey, retrievedVector);
|
|
5345
6632
|
queryVector = retrievedVector;
|
|
5346
6633
|
}
|
|
5347
|
-
const
|
|
5348
|
-
|
|
6634
|
+
const baseFilter = __spreadProps(__spreadValues({}, (_d = options.filter) != null ? _d : {}), { queryText: query });
|
|
6635
|
+
const numericPredicates = QueryProcessor.extractNumericPredicates(query, (_e = this.config.rag) == null ? void 0 : _e.filterableFields);
|
|
6636
|
+
if (numericPredicates.length > 0) {
|
|
6637
|
+
baseFilter.__numericPredicates = numericPredicates;
|
|
6638
|
+
}
|
|
6639
|
+
const retrievalLimit = numericPredicates.length > 0 ? Math.max(topK * 20, 100) : topK;
|
|
6640
|
+
const sources = (strategy === "vector" || strategy === "both") && queryVector && queryVector.length > 0 ? this.applyStructuredFilters(
|
|
6641
|
+
await this.vectorDB.query(queryVector, retrievalLimit, ns, baseFilter),
|
|
6642
|
+
baseFilter
|
|
6643
|
+
) : [];
|
|
6644
|
+
const resolvedSources = [];
|
|
6645
|
+
for (const source of sources) {
|
|
6646
|
+
const parentId = (_f = source.metadata) == null ? void 0 : _f.parent_id;
|
|
6647
|
+
if (parentId) {
|
|
6648
|
+
console.log(`[Pipeline] Multi-Vector: Found child chunk. Parent ID: ${parentId}`);
|
|
6649
|
+
resolvedSources.push(source);
|
|
6650
|
+
} else {
|
|
6651
|
+
resolvedSources.push(source);
|
|
6652
|
+
}
|
|
6653
|
+
}
|
|
6654
|
+
return { sources: resolvedSources, graphData };
|
|
5349
6655
|
}
|
|
5350
6656
|
/** Rewrite the user query for better retrieval performance. */
|
|
5351
6657
|
async rewriteQuery(question, history) {
|
|
@@ -5837,204 +7143,7 @@ var DocumentParser = class {
|
|
|
5837
7143
|
// src/server.ts
|
|
5838
7144
|
init_BaseVectorProvider();
|
|
5839
7145
|
init_PineconeProvider();
|
|
5840
|
-
|
|
5841
|
-
// src/providers/vectordb/PostgreSQLProvider.ts
|
|
5842
|
-
init_BaseVectorProvider();
|
|
5843
|
-
import { Pool as Pool2 } from "pg";
|
|
5844
|
-
var PostgreSQLProvider = class extends BaseVectorProvider {
|
|
5845
|
-
constructor(config) {
|
|
5846
|
-
var _a;
|
|
5847
|
-
super(config);
|
|
5848
|
-
this.tableName = this.indexName.replace(/[^a-z0-9_]/gi, "_");
|
|
5849
|
-
const opts = config.options;
|
|
5850
|
-
if (!opts.connectionString) throw new Error("[PostgreSQLProvider] options.connectionString is required");
|
|
5851
|
-
this.connectionString = opts.connectionString;
|
|
5852
|
-
this.dimensions = (_a = opts.dimensions) != null ? _a : 1536;
|
|
5853
|
-
}
|
|
5854
|
-
static getValidator() {
|
|
5855
|
-
return {
|
|
5856
|
-
validate(config) {
|
|
5857
|
-
const errors = [];
|
|
5858
|
-
const opts = config.options || {};
|
|
5859
|
-
if (!opts.connectionString) {
|
|
5860
|
-
errors.push({
|
|
5861
|
-
field: "vectorDb.options.connectionString",
|
|
5862
|
-
message: "PostgreSQL connection string is required",
|
|
5863
|
-
suggestion: "Set PGVECTOR_CONNECTION_STRING environment variable",
|
|
5864
|
-
severity: "error"
|
|
5865
|
-
});
|
|
5866
|
-
}
|
|
5867
|
-
if (opts.tables && typeof opts.tables !== "string" && !Array.isArray(opts.tables)) {
|
|
5868
|
-
errors.push({
|
|
5869
|
-
field: "vectorDb.options.tables",
|
|
5870
|
-
message: "PostgreSQL tables must be a string or a string array",
|
|
5871
|
-
severity: "error"
|
|
5872
|
-
});
|
|
5873
|
-
}
|
|
5874
|
-
return errors;
|
|
5875
|
-
}
|
|
5876
|
-
};
|
|
5877
|
-
}
|
|
5878
|
-
static getHealthChecker() {
|
|
5879
|
-
return {
|
|
5880
|
-
async check(config) {
|
|
5881
|
-
const opts = config.options || {};
|
|
5882
|
-
const timestamp = Date.now();
|
|
5883
|
-
try {
|
|
5884
|
-
const { Client } = await import("pg");
|
|
5885
|
-
const client = new Client({ connectionString: opts.connectionString });
|
|
5886
|
-
await client.connect();
|
|
5887
|
-
const result = await client.query(`
|
|
5888
|
-
SELECT EXISTS(SELECT 1 FROM pg_extension WHERE extname = 'vector');
|
|
5889
|
-
`);
|
|
5890
|
-
const hasVector = result.rows[0].exists;
|
|
5891
|
-
await client.end();
|
|
5892
|
-
return {
|
|
5893
|
-
healthy: true,
|
|
5894
|
-
provider: "postgresql",
|
|
5895
|
-
capabilities: { pgvectorInstalled: hasVector },
|
|
5896
|
-
timestamp
|
|
5897
|
-
};
|
|
5898
|
-
} catch (error) {
|
|
5899
|
-
return {
|
|
5900
|
-
healthy: false,
|
|
5901
|
-
provider: "postgresql",
|
|
5902
|
-
error: `Connection failed: ${error instanceof Error ? error.message : String(error)}`,
|
|
5903
|
-
timestamp
|
|
5904
|
-
};
|
|
5905
|
-
}
|
|
5906
|
-
}
|
|
5907
|
-
};
|
|
5908
|
-
}
|
|
5909
|
-
async initialize() {
|
|
5910
|
-
this.pool = new Pool2({ connectionString: this.connectionString });
|
|
5911
|
-
const client = await this.pool.connect();
|
|
5912
|
-
try {
|
|
5913
|
-
await client.query("CREATE EXTENSION IF NOT EXISTS vector");
|
|
5914
|
-
await client.query(`
|
|
5915
|
-
CREATE TABLE IF NOT EXISTS ${this.tableName} (
|
|
5916
|
-
id TEXT PRIMARY KEY,
|
|
5917
|
-
namespace TEXT NOT NULL DEFAULT '',
|
|
5918
|
-
content TEXT NOT NULL,
|
|
5919
|
-
metadata JSONB,
|
|
5920
|
-
embedding VECTOR(${this.dimensions})
|
|
5921
|
-
)
|
|
5922
|
-
`);
|
|
5923
|
-
await client.query(`
|
|
5924
|
-
CREATE INDEX IF NOT EXISTS ${this.tableName}_embedding_idx
|
|
5925
|
-
ON ${this.tableName}
|
|
5926
|
-
USING hnsw (embedding vector_cosine_ops)
|
|
5927
|
-
`);
|
|
5928
|
-
} finally {
|
|
5929
|
-
client.release();
|
|
5930
|
-
}
|
|
5931
|
-
}
|
|
5932
|
-
async upsert(doc, namespace = "") {
|
|
5933
|
-
var _a;
|
|
5934
|
-
const vectorLiteral = `[${doc.vector.join(",")}]`;
|
|
5935
|
-
await this.pool.query(
|
|
5936
|
-
`INSERT INTO ${this.tableName} (id, namespace, content, metadata, embedding)
|
|
5937
|
-
VALUES ($1, $2, $3, $4, $5::vector)
|
|
5938
|
-
ON CONFLICT (id) DO UPDATE
|
|
5939
|
-
SET namespace = EXCLUDED.namespace,
|
|
5940
|
-
content = EXCLUDED.content,
|
|
5941
|
-
metadata = EXCLUDED.metadata,
|
|
5942
|
-
embedding = EXCLUDED.embedding`,
|
|
5943
|
-
[doc.id, namespace, doc.content, JSON.stringify((_a = doc.metadata) != null ? _a : {}), vectorLiteral]
|
|
5944
|
-
);
|
|
5945
|
-
}
|
|
5946
|
-
async batchUpsert(docs, namespace = "") {
|
|
5947
|
-
if (docs.length === 0) return;
|
|
5948
|
-
const client = await this.pool.connect();
|
|
5949
|
-
try {
|
|
5950
|
-
await client.query("BEGIN");
|
|
5951
|
-
const BATCH_SIZE = 50;
|
|
5952
|
-
for (let i = 0; i < docs.length; i += BATCH_SIZE) {
|
|
5953
|
-
const batch = docs.slice(i, i + BATCH_SIZE);
|
|
5954
|
-
const values = [];
|
|
5955
|
-
const valuePlaceholders = batch.map((doc, idx) => {
|
|
5956
|
-
var _a;
|
|
5957
|
-
const offset = idx * 5;
|
|
5958
|
-
values.push(doc.id, namespace, doc.content, JSON.stringify((_a = doc.metadata) != null ? _a : {}), `[${doc.vector.join(",")}]`);
|
|
5959
|
-
return `($${offset + 1}, $${offset + 2}, $${offset + 3}, $${offset + 4}, $${offset + 5}::vector)`;
|
|
5960
|
-
}).join(", ");
|
|
5961
|
-
const query = `
|
|
5962
|
-
INSERT INTO ${this.tableName} (id, namespace, content, metadata, embedding)
|
|
5963
|
-
VALUES ${valuePlaceholders}
|
|
5964
|
-
ON CONFLICT (id) DO UPDATE
|
|
5965
|
-
SET namespace = EXCLUDED.namespace,
|
|
5966
|
-
content = EXCLUDED.content,
|
|
5967
|
-
metadata = EXCLUDED.metadata,
|
|
5968
|
-
embedding = EXCLUDED.embedding
|
|
5969
|
-
`;
|
|
5970
|
-
await client.query(query, values);
|
|
5971
|
-
}
|
|
5972
|
-
await client.query("COMMIT");
|
|
5973
|
-
} catch (error) {
|
|
5974
|
-
await client.query("ROLLBACK");
|
|
5975
|
-
throw error;
|
|
5976
|
-
} finally {
|
|
5977
|
-
client.release();
|
|
5978
|
-
}
|
|
5979
|
-
}
|
|
5980
|
-
async query(vector, topK, namespace, filter) {
|
|
5981
|
-
const vectorLiteral = `[${vector.join(",")}]`;
|
|
5982
|
-
let whereClause = namespace ? `WHERE namespace = $3` : "";
|
|
5983
|
-
const params = [vectorLiteral, topK];
|
|
5984
|
-
if (namespace) params.push(namespace);
|
|
5985
|
-
const publicFilter = this.sanitizeFilter(filter);
|
|
5986
|
-
if (Object.keys(publicFilter).length > 0) {
|
|
5987
|
-
const filterConditions = Object.entries(publicFilter).map(([key, val]) => {
|
|
5988
|
-
const paramIdx = params.length + 1;
|
|
5989
|
-
params.push(JSON.stringify(val));
|
|
5990
|
-
return `metadata->>'${key}' = $${paramIdx}`;
|
|
5991
|
-
}).join(" AND ");
|
|
5992
|
-
whereClause = whereClause ? `${whereClause} AND ${filterConditions}` : `WHERE ${filterConditions}`;
|
|
5993
|
-
}
|
|
5994
|
-
const client = await this.pool.connect();
|
|
5995
|
-
try {
|
|
5996
|
-
const efSearch = this.config.options.efSearch || Math.max(topK * 10, 40);
|
|
5997
|
-
await client.query(`SET LOCAL hnsw.ef_search = ${efSearch}`);
|
|
5998
|
-
const result = await client.query(
|
|
5999
|
-
`SELECT id, content, metadata, 1 - (embedding <=> $1::vector) AS score
|
|
6000
|
-
FROM ${this.tableName}
|
|
6001
|
-
${whereClause}
|
|
6002
|
-
ORDER BY embedding <=> $1::vector
|
|
6003
|
-
LIMIT $2`,
|
|
6004
|
-
params
|
|
6005
|
-
);
|
|
6006
|
-
return result.rows.map((row) => ({
|
|
6007
|
-
id: String(row["id"]),
|
|
6008
|
-
score: parseFloat(String(row["score"])),
|
|
6009
|
-
content: String(row["content"]),
|
|
6010
|
-
metadata: row["metadata"]
|
|
6011
|
-
}));
|
|
6012
|
-
} finally {
|
|
6013
|
-
client.release();
|
|
6014
|
-
}
|
|
6015
|
-
}
|
|
6016
|
-
async delete(id, namespace) {
|
|
6017
|
-
const where = namespace ? "WHERE id = $1 AND namespace = $2" : "WHERE id = $1";
|
|
6018
|
-
const params = namespace ? [id, namespace] : [id];
|
|
6019
|
-
await this.pool.query(`DELETE FROM ${this.tableName} ${where}`, params);
|
|
6020
|
-
}
|
|
6021
|
-
async deleteNamespace(namespace) {
|
|
6022
|
-
await this.pool.query(`DELETE FROM ${this.tableName} WHERE namespace = $1`, [namespace]);
|
|
6023
|
-
}
|
|
6024
|
-
async ping() {
|
|
6025
|
-
try {
|
|
6026
|
-
await this.pool.query("SELECT 1");
|
|
6027
|
-
return true;
|
|
6028
|
-
} catch (e) {
|
|
6029
|
-
return false;
|
|
6030
|
-
}
|
|
6031
|
-
}
|
|
6032
|
-
async disconnect() {
|
|
6033
|
-
await this.pool.end();
|
|
6034
|
-
}
|
|
6035
|
-
};
|
|
6036
|
-
|
|
6037
|
-
// src/server.ts
|
|
7146
|
+
init_PostgreSQLProvider();
|
|
6038
7147
|
init_MultiTablePostgresProvider();
|
|
6039
7148
|
init_MongoDBProvider();
|
|
6040
7149
|
init_MilvusProvider();
|
|
@@ -6120,15 +7229,24 @@ function createStreamHandler(configOrPlugin) {
|
|
|
6120
7229
|
});
|
|
6121
7230
|
}
|
|
6122
7231
|
const encoder = new TextEncoder();
|
|
7232
|
+
let isActive = true;
|
|
6123
7233
|
const stream = new ReadableStream({
|
|
6124
7234
|
async start(controller) {
|
|
6125
|
-
var _a
|
|
6126
|
-
const enqueue = (text) =>
|
|
7235
|
+
var _a;
|
|
7236
|
+
const enqueue = (text) => {
|
|
7237
|
+
if (!isActive) return;
|
|
7238
|
+
try {
|
|
7239
|
+
controller.enqueue(encoder.encode(text));
|
|
7240
|
+
} catch (err) {
|
|
7241
|
+
console.warn("[createStreamHandler] Failed to enqueue (stream already closed):", err);
|
|
7242
|
+
}
|
|
7243
|
+
};
|
|
6127
7244
|
try {
|
|
6128
7245
|
const pipelineStream = plugin.chatStream(message, history, namespace);
|
|
6129
7246
|
try {
|
|
6130
7247
|
for (var iter = __forAwait(pipelineStream), more, temp, error; more = !(temp = await iter.next()).done; more = false) {
|
|
6131
7248
|
const chunk = temp.value;
|
|
7249
|
+
if (!isActive) break;
|
|
6132
7250
|
if (typeof chunk === "string") {
|
|
6133
7251
|
enqueue(sseTextFrame(chunk));
|
|
6134
7252
|
} else {
|
|
@@ -6140,8 +7258,7 @@ function createStreamHandler(configOrPlugin) {
|
|
|
6140
7258
|
}
|
|
6141
7259
|
if (sources.length > 0) {
|
|
6142
7260
|
try {
|
|
6143
|
-
const
|
|
6144
|
-
const uiTransformation = (_b = responseChunk == null ? void 0 : responseChunk.ui_transformation) != null ? _b : llmProvider ? await UITransformer.analyzeAndDecide(message, sources, llmProvider) : UITransformer.transform(message, sources, plugin.getConfig());
|
|
7261
|
+
const uiTransformation = (_a = responseChunk == null ? void 0 : responseChunk.ui_transformation) != null ? _a : UITransformer.transform(message, sources, plugin.getConfig());
|
|
6145
7262
|
if (uiTransformation) {
|
|
6146
7263
|
enqueue(sseUIFrame(uiTransformation));
|
|
6147
7264
|
}
|
|
@@ -6167,12 +7284,27 @@ function createStreamHandler(configOrPlugin) {
|
|
|
6167
7284
|
}
|
|
6168
7285
|
}
|
|
6169
7286
|
} catch (streamError) {
|
|
6170
|
-
|
|
6171
|
-
|
|
6172
|
-
|
|
7287
|
+
if (isActive) {
|
|
7288
|
+
const errorMessage = streamError instanceof Error ? streamError.message : String(streamError);
|
|
7289
|
+
console.error("[createStreamHandler] Stream error:", streamError);
|
|
7290
|
+
try {
|
|
7291
|
+
enqueue(sseErrorFrame(errorMessage));
|
|
7292
|
+
} catch (e) {
|
|
7293
|
+
}
|
|
7294
|
+
}
|
|
6173
7295
|
} finally {
|
|
6174
|
-
|
|
7296
|
+
if (isActive) {
|
|
7297
|
+
isActive = false;
|
|
7298
|
+
try {
|
|
7299
|
+
controller.close();
|
|
7300
|
+
} catch (e) {
|
|
7301
|
+
}
|
|
7302
|
+
}
|
|
6175
7303
|
}
|
|
7304
|
+
},
|
|
7305
|
+
cancel(reason) {
|
|
7306
|
+
isActive = false;
|
|
7307
|
+
console.log("[createStreamHandler] Stream connection closed by client:", reason);
|
|
6176
7308
|
}
|
|
6177
7309
|
});
|
|
6178
7310
|
return new Response(stream, { headers: SSE_HEADERS });
|
|
@@ -6233,6 +7365,7 @@ function createUploadHandler(configOrPlugin) {
|
|
|
6233
7365
|
if (parsed.data && parsed.data.length > 0) {
|
|
6234
7366
|
let i = 0;
|
|
6235
7367
|
let lastRowData = null;
|
|
7368
|
+
const csvCols = Object.keys(parsed.data[0]);
|
|
6236
7369
|
for (const row of parsed.data) {
|
|
6237
7370
|
i++;
|
|
6238
7371
|
const rowData = row;
|
|
@@ -6254,12 +7387,14 @@ function createUploadHandler(configOrPlugin) {
|
|
|
6254
7387
|
documents.push({
|
|
6255
7388
|
docId: `${file.name}-row-${i}`,
|
|
6256
7389
|
content: contentParts.join(", "),
|
|
6257
|
-
metadata: __spreadValues(__spreadValues({
|
|
7390
|
+
metadata: __spreadValues(__spreadProps(__spreadValues({
|
|
6258
7391
|
fileName: file.name,
|
|
6259
7392
|
fileSize: file.size,
|
|
6260
7393
|
fileType: file.type,
|
|
6261
7394
|
uploadedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
6262
|
-
}, dimension ? { dimension } : {}),
|
|
7395
|
+
}, dimension ? { dimension } : {}), {
|
|
7396
|
+
csvHeaders: csvCols
|
|
7397
|
+
}), rowData)
|
|
6263
7398
|
});
|
|
6264
7399
|
}
|
|
6265
7400
|
}
|