@retrivora-ai/rag-engine 1.8.9 → 1.9.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{ILLMProvider-CbUtvWAW.d.mts → ILLMProvider-BQyKZLnd.d.mts} +10 -2
- package/dist/{ILLMProvider-CbUtvWAW.d.ts → ILLMProvider-BQyKZLnd.d.ts} +10 -2
- package/dist/handlers/index.d.mts +2 -2
- package/dist/handlers/index.d.ts +2 -2
- package/dist/handlers/index.js +1726 -522
- package/dist/handlers/index.mjs +1725 -521
- package/dist/{index-DgzcnqZs.d.ts → index-A0GqPsaG.d.ts} +1 -1
- package/dist/{index-c_y_5qdw.d.mts → index-CDftK3qs.d.mts} +1 -1
- package/dist/index.css +26 -0
- package/dist/index.d.mts +2 -2
- package/dist/index.d.ts +2 -2
- package/dist/index.js +246 -76
- package/dist/index.mjs +248 -76
- package/dist/server.d.mts +32 -5
- package/dist/server.d.ts +32 -5
- package/dist/server.js +1735 -733
- package/dist/server.mjs +1733 -731
- package/package.json +1 -1
- package/src/components/MarkdownComponents.tsx +3 -3
- package/src/components/MessageBubble.tsx +49 -2
- package/src/components/ProductCard.tsx +33 -6
- package/src/components/UIDispatcher.tsx +1 -0
- package/src/components/VisualizationRenderer.tsx +143 -11
- package/src/config/EmbeddingStrategy.ts +5 -4
- package/src/config/RagConfig.ts +10 -2
- package/src/config/serverConfig.ts +16 -1
- package/src/core/LLMRouter.ts +79 -0
- package/src/core/Pipeline.ts +262 -45
- package/src/core/ProviderRegistry.ts +6 -0
- package/src/core/QueryProcessor.ts +98 -0
- package/src/handlers/index.ts +8 -5
- package/src/hooks/useRagChat.ts +53 -17
- package/src/llm/providers/UniversalLLMAdapter.ts +110 -13
- package/src/providers/vectordb/ChromaDBProvider.ts +13 -6
- package/src/providers/vectordb/MilvusProvider.ts +18 -2
- package/src/providers/vectordb/MultiTablePostgresProvider.ts +16 -29
- package/src/providers/vectordb/PostgreSQLProvider.ts +4 -15
- package/src/providers/vectordb/QdrantProvider.ts +2 -5
- package/src/providers/vectordb/RedisProvider.ts +3 -4
- package/src/providers/vectordb/WeaviateProvider.ts +41 -3
- package/src/types/index.ts +26 -0
- package/src/utils/ProductExtractor.ts +5 -3
- package/src/utils/UITransformer.ts +1345 -534
- package/src/utils/synonyms.ts +2 -0
package/dist/server.js
CHANGED
|
@@ -334,16 +334,222 @@ var init_PineconeProvider = __esm({
|
|
|
334
334
|
}
|
|
335
335
|
});
|
|
336
336
|
|
|
337
|
+
// src/providers/vectordb/PostgreSQLProvider.ts
|
|
338
|
+
var PostgreSQLProvider_exports = {};
|
|
339
|
+
__export(PostgreSQLProvider_exports, {
|
|
340
|
+
PostgreSQLProvider: () => PostgreSQLProvider
|
|
341
|
+
});
|
|
342
|
+
var import_pg, PostgreSQLProvider;
|
|
343
|
+
var init_PostgreSQLProvider = __esm({
|
|
344
|
+
"src/providers/vectordb/PostgreSQLProvider.ts"() {
|
|
345
|
+
"use strict";
|
|
346
|
+
import_pg = require("pg");
|
|
347
|
+
init_BaseVectorProvider();
|
|
348
|
+
PostgreSQLProvider = class extends BaseVectorProvider {
|
|
349
|
+
constructor(config) {
|
|
350
|
+
var _a;
|
|
351
|
+
super(config);
|
|
352
|
+
this.tableName = this.indexName.replace(/[^a-z0-9_]/gi, "_");
|
|
353
|
+
const opts = config.options;
|
|
354
|
+
if (!opts.connectionString) throw new Error("[PostgreSQLProvider] options.connectionString is required");
|
|
355
|
+
this.connectionString = opts.connectionString;
|
|
356
|
+
this.dimensions = (_a = opts.dimensions) != null ? _a : 1536;
|
|
357
|
+
}
|
|
358
|
+
static getValidator() {
|
|
359
|
+
return {
|
|
360
|
+
validate(config) {
|
|
361
|
+
const errors = [];
|
|
362
|
+
const opts = config.options || {};
|
|
363
|
+
if (!opts.connectionString) {
|
|
364
|
+
errors.push({
|
|
365
|
+
field: "vectorDb.options.connectionString",
|
|
366
|
+
message: "PostgreSQL connection string is required",
|
|
367
|
+
suggestion: "Set PGVECTOR_CONNECTION_STRING environment variable",
|
|
368
|
+
severity: "error"
|
|
369
|
+
});
|
|
370
|
+
}
|
|
371
|
+
if (opts.tables && typeof opts.tables !== "string" && !Array.isArray(opts.tables)) {
|
|
372
|
+
errors.push({
|
|
373
|
+
field: "vectorDb.options.tables",
|
|
374
|
+
message: "PostgreSQL tables must be a string or a string array",
|
|
375
|
+
severity: "error"
|
|
376
|
+
});
|
|
377
|
+
}
|
|
378
|
+
return errors;
|
|
379
|
+
}
|
|
380
|
+
};
|
|
381
|
+
}
|
|
382
|
+
static getHealthChecker() {
|
|
383
|
+
return {
|
|
384
|
+
async check(config) {
|
|
385
|
+
const opts = config.options || {};
|
|
386
|
+
const timestamp = Date.now();
|
|
387
|
+
try {
|
|
388
|
+
const { Client } = await import("pg");
|
|
389
|
+
const client = new Client({ connectionString: opts.connectionString });
|
|
390
|
+
await client.connect();
|
|
391
|
+
const result = await client.query(`
|
|
392
|
+
SELECT EXISTS(SELECT 1 FROM pg_extension WHERE extname = 'vector');
|
|
393
|
+
`);
|
|
394
|
+
const hasVector = result.rows[0].exists;
|
|
395
|
+
await client.end();
|
|
396
|
+
return {
|
|
397
|
+
healthy: true,
|
|
398
|
+
provider: "postgresql",
|
|
399
|
+
capabilities: { pgvectorInstalled: hasVector },
|
|
400
|
+
timestamp
|
|
401
|
+
};
|
|
402
|
+
} catch (error) {
|
|
403
|
+
return {
|
|
404
|
+
healthy: false,
|
|
405
|
+
provider: "postgresql",
|
|
406
|
+
error: `Connection failed: ${error instanceof Error ? error.message : String(error)}`,
|
|
407
|
+
timestamp
|
|
408
|
+
};
|
|
409
|
+
}
|
|
410
|
+
}
|
|
411
|
+
};
|
|
412
|
+
}
|
|
413
|
+
async initialize() {
|
|
414
|
+
this.pool = new import_pg.Pool({ connectionString: this.connectionString });
|
|
415
|
+
const client = await this.pool.connect();
|
|
416
|
+
try {
|
|
417
|
+
await client.query("CREATE EXTENSION IF NOT EXISTS vector");
|
|
418
|
+
await client.query(`
|
|
419
|
+
CREATE TABLE IF NOT EXISTS ${this.tableName} (
|
|
420
|
+
id TEXT PRIMARY KEY,
|
|
421
|
+
namespace TEXT NOT NULL DEFAULT '',
|
|
422
|
+
content TEXT NOT NULL,
|
|
423
|
+
metadata JSONB,
|
|
424
|
+
embedding VECTOR(${this.dimensions})
|
|
425
|
+
)
|
|
426
|
+
`);
|
|
427
|
+
await client.query(`
|
|
428
|
+
CREATE INDEX IF NOT EXISTS ${this.tableName}_embedding_idx
|
|
429
|
+
ON ${this.tableName}
|
|
430
|
+
USING hnsw (embedding vector_cosine_ops)
|
|
431
|
+
`);
|
|
432
|
+
} finally {
|
|
433
|
+
client.release();
|
|
434
|
+
}
|
|
435
|
+
}
|
|
436
|
+
async upsert(doc, namespace = "") {
|
|
437
|
+
var _a;
|
|
438
|
+
const vectorLiteral = `[${doc.vector.join(",")}]`;
|
|
439
|
+
await this.pool.query(
|
|
440
|
+
`INSERT INTO ${this.tableName} (id, namespace, content, metadata, embedding)
|
|
441
|
+
VALUES ($1, $2, $3, $4, $5::vector)
|
|
442
|
+
ON CONFLICT (id) DO UPDATE
|
|
443
|
+
SET namespace = EXCLUDED.namespace,
|
|
444
|
+
content = EXCLUDED.content,
|
|
445
|
+
metadata = EXCLUDED.metadata,
|
|
446
|
+
embedding = EXCLUDED.embedding`,
|
|
447
|
+
[doc.id, namespace, doc.content, JSON.stringify((_a = doc.metadata) != null ? _a : {}), vectorLiteral]
|
|
448
|
+
);
|
|
449
|
+
}
|
|
450
|
+
async batchUpsert(docs, namespace = "") {
|
|
451
|
+
if (docs.length === 0) return;
|
|
452
|
+
const client = await this.pool.connect();
|
|
453
|
+
try {
|
|
454
|
+
await client.query("BEGIN");
|
|
455
|
+
const BATCH_SIZE = 50;
|
|
456
|
+
for (let i = 0; i < docs.length; i += BATCH_SIZE) {
|
|
457
|
+
const batch = docs.slice(i, i + BATCH_SIZE);
|
|
458
|
+
const values = [];
|
|
459
|
+
const valuePlaceholders = batch.map((doc, idx) => {
|
|
460
|
+
var _a;
|
|
461
|
+
const offset = idx * 5;
|
|
462
|
+
values.push(doc.id, namespace, doc.content, JSON.stringify((_a = doc.metadata) != null ? _a : {}), `[${doc.vector.join(",")}]`);
|
|
463
|
+
return `($${offset + 1}, $${offset + 2}, $${offset + 3}, $${offset + 4}, $${offset + 5}::vector)`;
|
|
464
|
+
}).join(", ");
|
|
465
|
+
const query = `
|
|
466
|
+
INSERT INTO ${this.tableName} (id, namespace, content, metadata, embedding)
|
|
467
|
+
VALUES ${valuePlaceholders}
|
|
468
|
+
ON CONFLICT (id) DO UPDATE
|
|
469
|
+
SET namespace = EXCLUDED.namespace,
|
|
470
|
+
content = EXCLUDED.content,
|
|
471
|
+
metadata = EXCLUDED.metadata,
|
|
472
|
+
embedding = EXCLUDED.embedding
|
|
473
|
+
`;
|
|
474
|
+
await client.query(query, values);
|
|
475
|
+
}
|
|
476
|
+
await client.query("COMMIT");
|
|
477
|
+
} catch (error) {
|
|
478
|
+
await client.query("ROLLBACK");
|
|
479
|
+
throw error;
|
|
480
|
+
} finally {
|
|
481
|
+
client.release();
|
|
482
|
+
}
|
|
483
|
+
}
|
|
484
|
+
async query(vector, topK, namespace, filter) {
|
|
485
|
+
const vectorLiteral = `[${vector.join(",")}]`;
|
|
486
|
+
let whereClause = namespace ? `WHERE namespace = $3` : "";
|
|
487
|
+
const params = [vectorLiteral, topK];
|
|
488
|
+
if (namespace) params.push(namespace);
|
|
489
|
+
const publicFilter = this.sanitizeFilter(filter);
|
|
490
|
+
if (Object.keys(publicFilter).length > 0) {
|
|
491
|
+
const filterConditions = Object.entries(publicFilter).map(([key, val]) => {
|
|
492
|
+
const paramIdx = params.length + 1;
|
|
493
|
+
params.push(String(val));
|
|
494
|
+
return `metadata->>'${key}' = $${paramIdx}`;
|
|
495
|
+
}).join(" AND ");
|
|
496
|
+
whereClause = whereClause ? `${whereClause} AND ${filterConditions}` : `WHERE ${filterConditions}`;
|
|
497
|
+
}
|
|
498
|
+
const client = await this.pool.connect();
|
|
499
|
+
try {
|
|
500
|
+
const efSearch = this.config.options.efSearch || Math.max(topK * 10, 40);
|
|
501
|
+
await client.query(`SET LOCAL hnsw.ef_search = ${efSearch}`);
|
|
502
|
+
const result = await client.query(
|
|
503
|
+
`SELECT id, content, metadata, 1 - (embedding <=> $1::vector) AS score
|
|
504
|
+
FROM ${this.tableName}
|
|
505
|
+
${whereClause}
|
|
506
|
+
ORDER BY embedding <=> $1::vector
|
|
507
|
+
LIMIT $2`,
|
|
508
|
+
params
|
|
509
|
+
);
|
|
510
|
+
return result.rows.map((row) => ({
|
|
511
|
+
id: String(row["id"]),
|
|
512
|
+
score: parseFloat(String(row["score"])),
|
|
513
|
+
content: String(row["content"]),
|
|
514
|
+
metadata: row["metadata"]
|
|
515
|
+
}));
|
|
516
|
+
} finally {
|
|
517
|
+
client.release();
|
|
518
|
+
}
|
|
519
|
+
}
|
|
520
|
+
async delete(id, namespace) {
|
|
521
|
+
const where = namespace ? "WHERE id = $1 AND namespace = $2" : "WHERE id = $1";
|
|
522
|
+
const params = namespace ? [id, namespace] : [id];
|
|
523
|
+
await this.pool.query(`DELETE FROM ${this.tableName} ${where}`, params);
|
|
524
|
+
}
|
|
525
|
+
async deleteNamespace(namespace) {
|
|
526
|
+
await this.pool.query(`DELETE FROM ${this.tableName} WHERE namespace = $1`, [namespace]);
|
|
527
|
+
}
|
|
528
|
+
async ping() {
|
|
529
|
+
try {
|
|
530
|
+
await this.pool.query("SELECT 1");
|
|
531
|
+
return true;
|
|
532
|
+
} catch (e) {
|
|
533
|
+
return false;
|
|
534
|
+
}
|
|
535
|
+
}
|
|
536
|
+
async disconnect() {
|
|
537
|
+
await this.pool.end();
|
|
538
|
+
}
|
|
539
|
+
};
|
|
540
|
+
}
|
|
541
|
+
});
|
|
542
|
+
|
|
337
543
|
// src/providers/vectordb/MultiTablePostgresProvider.ts
|
|
338
544
|
var MultiTablePostgresProvider_exports = {};
|
|
339
545
|
__export(MultiTablePostgresProvider_exports, {
|
|
340
546
|
MultiTablePostgresProvider: () => MultiTablePostgresProvider
|
|
341
547
|
});
|
|
342
|
-
var
|
|
548
|
+
var import_pg2, MultiTablePostgresProvider;
|
|
343
549
|
var init_MultiTablePostgresProvider = __esm({
|
|
344
550
|
"src/providers/vectordb/MultiTablePostgresProvider.ts"() {
|
|
345
551
|
"use strict";
|
|
346
|
-
|
|
552
|
+
import_pg2 = require("pg");
|
|
347
553
|
init_BaseVectorProvider();
|
|
348
554
|
MultiTablePostgresProvider = class extends BaseVectorProvider {
|
|
349
555
|
constructor(config) {
|
|
@@ -361,7 +567,7 @@ var init_MultiTablePostgresProvider = __esm({
|
|
|
361
567
|
this.uploadTable = opts.uploadTable || "document_chunks";
|
|
362
568
|
}
|
|
363
569
|
async initialize() {
|
|
364
|
-
this.pool = new
|
|
570
|
+
this.pool = new import_pg2.Pool({ connectionString: this.connectionString });
|
|
365
571
|
const client = await this.pool.connect();
|
|
366
572
|
try {
|
|
367
573
|
await client.query("CREATE EXTENSION IF NOT EXISTS vector");
|
|
@@ -374,12 +580,10 @@ var init_MultiTablePostgresProvider = __esm({
|
|
|
374
580
|
embedding VECTOR(${this.dimensions})
|
|
375
581
|
)
|
|
376
582
|
`);
|
|
377
|
-
const metric = this.config.distanceMetric || "cosine";
|
|
378
|
-
const indexOps = metric === "euclidean" ? "vector_l2_ops" : metric === "dotproduct" ? "vector_ip_ops" : "vector_cosine_ops";
|
|
379
583
|
await client.query(`
|
|
380
584
|
CREATE INDEX IF NOT EXISTS ${this.uploadTable}_embedding_idx
|
|
381
585
|
ON ${this.uploadTable}
|
|
382
|
-
USING hnsw (embedding
|
|
586
|
+
USING hnsw (embedding vector_cosine_ops)
|
|
383
587
|
`);
|
|
384
588
|
const res = await client.query(`
|
|
385
589
|
SELECT table_name
|
|
@@ -441,12 +645,10 @@ var init_MultiTablePostgresProvider = __esm({
|
|
|
441
645
|
)
|
|
442
646
|
`;
|
|
443
647
|
await client.query(createTableSql);
|
|
444
|
-
const metric = this.config.distanceMetric || "cosine";
|
|
445
|
-
const indexOps = metric === "euclidean" ? "vector_l2_ops" : metric === "dotproduct" ? "vector_ip_ops" : "vector_cosine_ops";
|
|
446
648
|
await client.query(`
|
|
447
649
|
CREATE INDEX IF NOT EXISTS "${tableName}_embedding_idx"
|
|
448
650
|
ON "${tableName}"
|
|
449
|
-
USING hnsw (embedding
|
|
651
|
+
USING hnsw (embedding vector_cosine_ops)
|
|
450
652
|
`);
|
|
451
653
|
if (!this.tables.includes(tableName)) {
|
|
452
654
|
this.tables.push(tableName);
|
|
@@ -509,28 +711,25 @@ var init_MultiTablePostgresProvider = __esm({
|
|
|
509
711
|
const allResults = [];
|
|
510
712
|
console.log(`[MultiTablePostgresProvider] --- Starting Multi-Table Search ---`);
|
|
511
713
|
const queryText = _filter == null ? void 0 : _filter.queryText;
|
|
512
|
-
const entityHints = Array.isArray(_filter == null ? void 0 : _filter.
|
|
513
|
-
(hint) => typeof hint === "object" && hint !== null && typeof hint.value === "string"
|
|
514
|
-
).map((hint) => hint.value.trim().toLowerCase()).filter(Boolean) : [];
|
|
714
|
+
const entityHints = Array.isArray(_filter == null ? void 0 : _filter.keywords) ? _filter.keywords.map((k) => k.trim().toLowerCase()).filter(Boolean) : [];
|
|
515
715
|
console.log(`[MultiTablePostgresProvider] queryText: "${queryText}"`);
|
|
516
716
|
console.log(`[MultiTablePostgresProvider] entityHints: [${entityHints.join(", ")}]`);
|
|
517
717
|
const getDynamicKeywordQuery = () => {
|
|
518
718
|
if (entityHints.length > 0) {
|
|
519
|
-
return entityHints.map((h) => h.
|
|
719
|
+
return entityHints.map((h) => h.replace(/\s+/g, " & ")).join(" & ");
|
|
520
720
|
}
|
|
521
721
|
if (queryText) {
|
|
522
|
-
return queryText.toLowerCase().replace(/\b(show|me|the|product[s]?|item[s]?|card[s]?|of|category|find|list|browse|what|are|display|all|under|for|in|with|about)\b/g, "").trim().replace(/\s+/g, " & ");
|
|
722
|
+
return queryText.toLowerCase().replace(/\b(show|me|the|product[s]?|item[s]?|card[s]?|organization[s]?|comp(?:any|anies)|of|category|find|list|browse|what|are|display|all|whose|that|which|have|has|having|greater|than|more|less|equal|equals|above|below|over|under|at|least|most|for|in|with|about|get|details|information|info)\b/g, "").replace(/\b\d+(?:\.\d+)?\b/g, "").trim().replace(/\s+/g, " & ");
|
|
523
723
|
}
|
|
524
724
|
return "";
|
|
525
725
|
};
|
|
526
726
|
const dynamicKeywordQuery = getDynamicKeywordQuery();
|
|
727
|
+
const tableLimit = Math.max(topK, 50);
|
|
527
728
|
const queryPromises = this.tables.map(async (table) => {
|
|
528
729
|
try {
|
|
529
730
|
let sqlQuery = "";
|
|
530
731
|
let params = [];
|
|
531
|
-
|
|
532
|
-
const scoreExpr = metric === "euclidean" ? `1 / (1 + (embedding <-> $1::vector))` : metric === "dotproduct" ? `(embedding <#> $1::vector) * -1` : `1 - (embedding <=> $1::vector)`;
|
|
533
|
-
if (queryText) {
|
|
732
|
+
if (queryText && dynamicKeywordQuery) {
|
|
534
733
|
const hasEntityHints = entityHints.length > 0;
|
|
535
734
|
const exactNameScoreExpr = hasEntityHints ? `+ (
|
|
536
735
|
SELECT COALESCE(MAX(
|
|
@@ -542,21 +741,21 @@ var init_MultiTablePostgresProvider = __esm({
|
|
|
542
741
|
) * 3.0` : "";
|
|
543
742
|
sqlQuery = `
|
|
544
743
|
SELECT *,
|
|
545
|
-
($
|
|
744
|
+
(1 - (embedding <=> $1::vector)) AS vector_score,
|
|
546
745
|
COALESCE(ts_rank(to_tsvector('english', (to_jsonb(t) - 'embedding')::text), to_tsquery('english', $2)), 0) AS keyword_score,
|
|
547
|
-
(($
|
|
746
|
+
((1 - (embedding <=> $1::vector)) + (COALESCE(ts_rank(to_tsvector('english', (to_jsonb(t) - 'embedding')::text), to_tsquery('english', $2)), 0) * 2.0) ${exactNameScoreExpr}) AS hybrid_score
|
|
548
747
|
FROM "${table}" t
|
|
549
748
|
ORDER BY hybrid_score DESC
|
|
550
|
-
LIMIT
|
|
749
|
+
LIMIT ${tableLimit}
|
|
551
750
|
`;
|
|
552
751
|
params = [vectorLiteral, dynamicKeywordQuery];
|
|
553
752
|
} else {
|
|
554
753
|
sqlQuery = `
|
|
555
754
|
SELECT *,
|
|
556
|
-
|
|
755
|
+
(1 - (embedding <=> $1::vector)) AS hybrid_score
|
|
557
756
|
FROM "${table}" t
|
|
558
757
|
ORDER BY hybrid_score DESC
|
|
559
|
-
LIMIT
|
|
758
|
+
LIMIT ${tableLimit}
|
|
560
759
|
`;
|
|
561
760
|
params = [vectorLiteral];
|
|
562
761
|
}
|
|
@@ -881,13 +1080,25 @@ var init_MilvusProvider = __esm({
|
|
|
881
1080
|
};
|
|
882
1081
|
await this.http.post("/v1/vector/upsert", payload);
|
|
883
1082
|
}
|
|
884
|
-
// eslint-disable-next-line @typescript-eslint/no-unused-vars
|
|
885
1083
|
async query(vector, topK, namespace, _filter) {
|
|
1084
|
+
const sanitizedFilter = this.sanitizeFilter(_filter);
|
|
1085
|
+
const filterParts = [];
|
|
1086
|
+
if (namespace) {
|
|
1087
|
+
filterParts.push(`namespace == "${namespace}"`);
|
|
1088
|
+
}
|
|
1089
|
+
for (const [key, value] of Object.entries(sanitizedFilter)) {
|
|
1090
|
+
if (key === "queryText" || key === "keywords") continue;
|
|
1091
|
+
if (typeof value === "string") {
|
|
1092
|
+
filterParts.push(`metadata["${key}"] == "${value.replace(/"/g, '\\"')}"`);
|
|
1093
|
+
} else if (typeof value === "number") {
|
|
1094
|
+
filterParts.push(`metadata["${key}"] == ${value}`);
|
|
1095
|
+
}
|
|
1096
|
+
}
|
|
886
1097
|
const payload = {
|
|
887
1098
|
collectionName: this.indexName,
|
|
888
1099
|
vector,
|
|
889
1100
|
limit: topK,
|
|
890
|
-
filter:
|
|
1101
|
+
filter: filterParts.length > 0 ? filterParts.join(" && ") : void 0,
|
|
891
1102
|
outputFields: ["content", "metadata"],
|
|
892
1103
|
searchParams: {
|
|
893
1104
|
nprobe: this.config.options.nprobe || 16,
|
|
@@ -1027,13 +1238,11 @@ var init_QdrantProvider = __esm({
|
|
|
1027
1238
|
if (import_axios4.default.isAxiosError(err) && ((_a = err.response) == null ? void 0 : _a.status) === 404) {
|
|
1028
1239
|
const opts = this.config.options;
|
|
1029
1240
|
const dimensionsForCreate = opts.dimensions || 1536;
|
|
1030
|
-
const metric = this.config.distanceMetric || "cosine";
|
|
1031
|
-
const distance = metric === "euclidean" ? "Euclid" : metric === "dotproduct" ? "Dot" : "Cosine";
|
|
1032
1241
|
console.log(`[QdrantProvider] \u23F3 Creating collection "${this.indexName}" (dimensions: ${dimensionsForCreate})...`);
|
|
1033
1242
|
await this.http.put(`/collections/${this.indexName}`, {
|
|
1034
1243
|
vectors: {
|
|
1035
1244
|
size: dimensionsForCreate,
|
|
1036
|
-
distance
|
|
1245
|
+
distance: "Cosine"
|
|
1037
1246
|
}
|
|
1038
1247
|
});
|
|
1039
1248
|
console.log(`[QdrantProvider] \u2705 Created collection "${this.indexName}"`);
|
|
@@ -1088,6 +1297,7 @@ var init_QdrantProvider = __esm({
|
|
|
1088
1297
|
await this.http.put(`/collections/${this.indexName}/points`, payload);
|
|
1089
1298
|
}
|
|
1090
1299
|
async query(vector, topK, namespace, _filter) {
|
|
1300
|
+
var _a;
|
|
1091
1301
|
const must = [];
|
|
1092
1302
|
if (namespace) {
|
|
1093
1303
|
must.push({ key: "namespace", match: { value: namespace } });
|
|
@@ -1109,7 +1319,7 @@ var init_QdrantProvider = __esm({
|
|
|
1109
1319
|
limit: topK,
|
|
1110
1320
|
with_payload: true,
|
|
1111
1321
|
params: {
|
|
1112
|
-
hnsw_ef: this.config.options.efSearch || Math.max(topK * 20, 128),
|
|
1322
|
+
hnsw_ef: ((_a = this.config.options) == null ? void 0 : _a.efSearch) || Math.max(topK * 20, 128),
|
|
1113
1323
|
exact: false
|
|
1114
1324
|
},
|
|
1115
1325
|
filter: must.length > 0 ? { must } : void 0
|
|
@@ -1211,12 +1421,9 @@ var init_ChromaDBProvider = __esm({
|
|
|
1211
1421
|
console.log(`[ChromaDBProvider] \u2705 Collection "${this.indexName}" found (id: ${this.collectionId})`);
|
|
1212
1422
|
} catch (err) {
|
|
1213
1423
|
if (import_axios5.default.isAxiosError(err) && ((_a = err.response) == null ? void 0 : _a.status) === 404) {
|
|
1214
|
-
const metric = this.config.distanceMetric || "cosine";
|
|
1215
|
-
const space = metric === "euclidean" ? "l2" : metric === "dotproduct" ? "ip" : "cosine";
|
|
1216
1424
|
console.log(`[ChromaDBProvider] \u23F3 Collection "${this.indexName}" not found. Creating...`);
|
|
1217
1425
|
const { data } = await this.http.post("/api/v1/collections", {
|
|
1218
|
-
name: this.indexName
|
|
1219
|
-
metadata: { "hnsw:space": space }
|
|
1426
|
+
name: this.indexName
|
|
1220
1427
|
});
|
|
1221
1428
|
this.collectionId = data.id;
|
|
1222
1429
|
console.log(`[ChromaDBProvider] \u2705 Created collection "${this.indexName}" (id: ${this.collectionId})`);
|
|
@@ -1237,12 +1444,20 @@ var init_ChromaDBProvider = __esm({
|
|
|
1237
1444
|
};
|
|
1238
1445
|
await this.http.post(`/api/v1/collections/${this.collectionId}/add`, payload);
|
|
1239
1446
|
}
|
|
1240
|
-
// eslint-disable-next-line @typescript-eslint/no-unused-vars
|
|
1241
1447
|
async query(vector, topK, namespace, _filter) {
|
|
1448
|
+
const sanitizedFilter = this.sanitizeFilter(_filter);
|
|
1449
|
+
const whereClauses = [];
|
|
1450
|
+
if (namespace) whereClauses.push({ namespace: { $eq: namespace } });
|
|
1451
|
+
Object.entries(sanitizedFilter).forEach(([key, value]) => {
|
|
1452
|
+
if (key === "namespace") return;
|
|
1453
|
+
if (typeof value === "string" || typeof value === "number" || typeof value === "boolean") {
|
|
1454
|
+
whereClauses.push({ [key]: { $eq: value } });
|
|
1455
|
+
}
|
|
1456
|
+
});
|
|
1242
1457
|
const payload = {
|
|
1243
1458
|
query_embeddings: [vector],
|
|
1244
1459
|
n_results: topK,
|
|
1245
|
-
where:
|
|
1460
|
+
where: whereClauses.length > 1 ? { $and: whereClauses } : whereClauses[0]
|
|
1246
1461
|
};
|
|
1247
1462
|
const { data } = await this.http.post(`/api/v1/collections/${this.collectionId}/query`, payload);
|
|
1248
1463
|
const matches = [];
|
|
@@ -1347,15 +1562,15 @@ var init_RedisProvider = __esm({
|
|
|
1347
1562
|
console.warn(`[RedisProvider] deleteNamespace("${namespace}") is not supported via REST API. Use Redis CLI: SCAN + DEL`);
|
|
1348
1563
|
}
|
|
1349
1564
|
/**
|
|
1350
|
-
*
|
|
1351
|
-
* Returns
|
|
1565
|
+
* Check reachability via a PING command.
|
|
1566
|
+
* Returns false on connection failure so health checks surface real problems.
|
|
1352
1567
|
*/
|
|
1353
1568
|
async ping() {
|
|
1354
1569
|
try {
|
|
1355
1570
|
await this.http.post("/", ["PING"]);
|
|
1356
1571
|
return true;
|
|
1357
1572
|
} catch (e) {
|
|
1358
|
-
return
|
|
1573
|
+
return false;
|
|
1359
1574
|
}
|
|
1360
1575
|
}
|
|
1361
1576
|
async disconnect() {
|
|
@@ -1395,15 +1610,17 @@ var init_WeaviateProvider = __esm({
|
|
|
1395
1610
|
await this.ping();
|
|
1396
1611
|
}
|
|
1397
1612
|
async upsert(doc, namespace) {
|
|
1613
|
+
const primitiveMetadata = this.extractPrimitiveMetadata(doc.metadata);
|
|
1398
1614
|
const payload = {
|
|
1399
1615
|
class: this.indexName,
|
|
1400
1616
|
id: doc.id,
|
|
1401
1617
|
vector: doc.vector,
|
|
1402
|
-
properties: {
|
|
1618
|
+
properties: __spreadProps(__spreadValues({
|
|
1403
1619
|
content: doc.content,
|
|
1404
|
-
metadata: JSON.stringify(doc.metadata || {})
|
|
1620
|
+
metadata: JSON.stringify(doc.metadata || {})
|
|
1621
|
+
}, primitiveMetadata), {
|
|
1405
1622
|
namespace: namespace || ""
|
|
1406
|
-
}
|
|
1623
|
+
})
|
|
1407
1624
|
};
|
|
1408
1625
|
await this.http.post("/v1/objects", payload);
|
|
1409
1626
|
}
|
|
@@ -1413,20 +1630,22 @@ var init_WeaviateProvider = __esm({
|
|
|
1413
1630
|
class: this.indexName,
|
|
1414
1631
|
id: doc.id,
|
|
1415
1632
|
vector: doc.vector,
|
|
1416
|
-
properties: {
|
|
1633
|
+
properties: __spreadProps(__spreadValues({
|
|
1417
1634
|
content: doc.content,
|
|
1418
|
-
metadata: JSON.stringify(doc.metadata || {})
|
|
1635
|
+
metadata: JSON.stringify(doc.metadata || {})
|
|
1636
|
+
}, this.extractPrimitiveMetadata(doc.metadata)), {
|
|
1419
1637
|
namespace: namespace || ""
|
|
1420
|
-
}
|
|
1638
|
+
})
|
|
1421
1639
|
}))
|
|
1422
1640
|
};
|
|
1423
1641
|
await this.http.post("/v1/batch/objects", payload);
|
|
1424
1642
|
}
|
|
1425
1643
|
async query(vector, topK, namespace, _filter) {
|
|
1426
1644
|
var _a, _b;
|
|
1645
|
+
const queryText = _filter == null ? void 0 : _filter.queryText;
|
|
1427
1646
|
const sanitizedFilter = this.sanitizeFilter(_filter);
|
|
1428
|
-
const queryText = sanitizedFilter.queryText;
|
|
1429
1647
|
const searchParams = queryText ? `hybrid: { query: ${JSON.stringify(queryText)}, alpha: 0.5 }` : `nearVector: { vector: ${JSON.stringify(vector)} }`;
|
|
1648
|
+
const where = this.buildWhereFilter(namespace, sanitizedFilter);
|
|
1430
1649
|
const graphqlQuery = {
|
|
1431
1650
|
query: `
|
|
1432
1651
|
{
|
|
@@ -1434,7 +1653,7 @@ var init_WeaviateProvider = __esm({
|
|
|
1434
1653
|
${this.indexName}(
|
|
1435
1654
|
${searchParams}
|
|
1436
1655
|
limit: ${topK}
|
|
1437
|
-
${
|
|
1656
|
+
${where ? `where: ${where}` : ""}
|
|
1438
1657
|
) {
|
|
1439
1658
|
content
|
|
1440
1659
|
metadata
|
|
@@ -1478,6 +1697,34 @@ var init_WeaviateProvider = __esm({
|
|
|
1478
1697
|
}
|
|
1479
1698
|
async disconnect() {
|
|
1480
1699
|
}
|
|
1700
|
+
extractPrimitiveMetadata(metadata) {
|
|
1701
|
+
const result = {};
|
|
1702
|
+
Object.entries(metadata || {}).forEach(([key, value]) => {
|
|
1703
|
+
if (typeof value === "string" || typeof value === "number" || typeof value === "boolean") {
|
|
1704
|
+
result[key] = value;
|
|
1705
|
+
}
|
|
1706
|
+
});
|
|
1707
|
+
return result;
|
|
1708
|
+
}
|
|
1709
|
+
buildWhereFilter(namespace, filter) {
|
|
1710
|
+
const operands = [];
|
|
1711
|
+
if (namespace) operands.push(this.weaviateOperand("namespace", namespace));
|
|
1712
|
+
Object.entries(filter || {}).forEach(([key, value]) => {
|
|
1713
|
+
if (key === "namespace") return;
|
|
1714
|
+
if (typeof value === "string" || typeof value === "number" || typeof value === "boolean") {
|
|
1715
|
+
operands.push(this.weaviateOperand(key, value));
|
|
1716
|
+
}
|
|
1717
|
+
});
|
|
1718
|
+
if (operands.length === 0) return void 0;
|
|
1719
|
+
if (operands.length === 1) return operands[0];
|
|
1720
|
+
return `{ operator: And, operands: [${operands.join(", ")}] }`;
|
|
1721
|
+
}
|
|
1722
|
+
weaviateOperand(key, value) {
|
|
1723
|
+
const path = `path: [${JSON.stringify(key)}], operator: Equal`;
|
|
1724
|
+
if (typeof value === "number") return `{ ${path}, valueNumber: ${value} }`;
|
|
1725
|
+
if (typeof value === "boolean") return `{ ${path}, valueBoolean: ${value} }`;
|
|
1726
|
+
return `{ ${path}, valueString: ${JSON.stringify(value)} }`;
|
|
1727
|
+
}
|
|
1481
1728
|
};
|
|
1482
1729
|
}
|
|
1483
1730
|
});
|
|
@@ -1831,7 +2078,7 @@ function getRagConfig(baseConfig, env = process.env) {
|
|
|
1831
2078
|
return getEnvConfig(env, baseConfig);
|
|
1832
2079
|
}
|
|
1833
2080
|
function getEnvConfig(env = process.env, base) {
|
|
1834
|
-
var _a, _b, _c, _d, _e, _f, _g, _h, _i, _j, _k, _l, _m, _n, _o, _p, _q, _r, _s, _t, _u, _v, _w, _x, _y, _z, _A, _B, _C, _D, _E, _F, _G, _H, _I, _J, _K, _L, _M, _N, _O, _P, _Q, _R, _S, _T, _U, _V, _W, _X, _Y, _Z, __, _$, _aa, _ba, _ca, _da, _ea, _fa, _ga, _ha, _ia, _ja, _ka, _la, _ma, _na, _oa, _pa, _qa, _ra, _sa, _ta, _ua, _va, _wa, _xa, _ya, _za, _Aa, _Ba, _Ca, _Da, _Ea, _Fa;
|
|
2081
|
+
var _a, _b, _c, _d, _e, _f, _g, _h, _i, _j, _k, _l, _m, _n, _o, _p, _q, _r, _s, _t, _u, _v, _w, _x, _y, _z, _A, _B, _C, _D, _E, _F, _G, _H, _I, _J, _K, _L, _M, _N, _O, _P, _Q, _R, _S, _T, _U, _V, _W, _X, _Y, _Z, __, _$, _aa, _ba, _ca, _da, _ea, _fa, _ga, _ha, _ia, _ja, _ka, _la, _ma, _na, _oa, _pa, _qa, _ra, _sa, _ta, _ua, _va, _wa, _xa, _ya, _za, _Aa, _Ba, _Ca, _Da, _Ea, _Fa, _Ga;
|
|
1835
2082
|
const projectId = (_c = (_b = (_a = readString(env, "RAG_PROJECT_ID")) != null ? _a : readString(env, "NEXT_PUBLIC_PROJECT_ID")) != null ? _b : base == null ? void 0 : base.projectId) != null ? _c : "__default__";
|
|
1836
2083
|
const vectorProvider = readEnum(env, "VECTOR_DB_PROVIDER", "pinecone", VECTOR_DB_PROVIDERS);
|
|
1837
2084
|
const llmProvider = readEnum(env, "LLM_PROVIDER", "openai", LLM_PROVIDERS);
|
|
@@ -1885,11 +2132,22 @@ function getEnvConfig(env = process.env, base) {
|
|
|
1885
2132
|
};
|
|
1886
2133
|
const embeddingApiKeyByProvider = {
|
|
1887
2134
|
openai: readString(env, "OPENAI_API_KEY"),
|
|
2135
|
+
anthropic: readString(env, "ANTHROPIC_API_KEY"),
|
|
2136
|
+
// Anthropic needs a separate embedding provider; key kept for completeness
|
|
1888
2137
|
gemini: readString(env, "GEMINI_API_KEY"),
|
|
1889
2138
|
ollama: void 0,
|
|
1890
2139
|
universal_rest: (_ea = readString(env, "EMBEDDING_API_KEY")) != null ? _ea : readString(env, "OPENAI_API_KEY"),
|
|
1891
2140
|
custom: (_fa = readString(env, "EMBEDDING_API_KEY")) != null ? _fa : readString(env, "OPENAI_API_KEY")
|
|
1892
2141
|
};
|
|
2142
|
+
const DEFAULT_MODEL_BY_PROVIDER = {
|
|
2143
|
+
openai: "gpt-4o",
|
|
2144
|
+
anthropic: "claude-3-5-sonnet-20241022",
|
|
2145
|
+
gemini: "gemini-2.0-flash",
|
|
2146
|
+
ollama: "llama3",
|
|
2147
|
+
universal_rest: "default",
|
|
2148
|
+
rest: "default",
|
|
2149
|
+
custom: "default"
|
|
2150
|
+
};
|
|
1893
2151
|
return {
|
|
1894
2152
|
projectId,
|
|
1895
2153
|
vectorDb: {
|
|
@@ -1899,8 +2157,8 @@ function getEnvConfig(env = process.env, base) {
|
|
|
1899
2157
|
},
|
|
1900
2158
|
llm: {
|
|
1901
2159
|
provider: llmProvider,
|
|
1902
|
-
model: (_ia = readString(env, "LLM_MODEL")) != null ? _ia : "gpt-4o",
|
|
1903
|
-
apiKey: (
|
|
2160
|
+
model: (_ja = (_ia = readString(env, "LLM_MODEL")) != null ? _ia : DEFAULT_MODEL_BY_PROVIDER[llmProvider]) != null ? _ja : "gpt-4o",
|
|
2161
|
+
apiKey: (_ka = llmApiKeyByProvider[llmProvider]) != null ? _ka : "",
|
|
1904
2162
|
baseUrl: readString(env, "LLM_BASE_URL"),
|
|
1905
2163
|
systemPrompt: readString(env, "LLM_SYSTEM_PROMPT"),
|
|
1906
2164
|
maxTokens: readNumber(env, "LLM_MAX_TOKENS", 4096),
|
|
@@ -1911,7 +2169,7 @@ function getEnvConfig(env = process.env, base) {
|
|
|
1911
2169
|
},
|
|
1912
2170
|
embedding: {
|
|
1913
2171
|
provider: embeddingProvider,
|
|
1914
|
-
model: (
|
|
2172
|
+
model: (_la = readString(env, "EMBEDDING_MODEL")) != null ? _la : "text-embedding-3-small",
|
|
1915
2173
|
apiKey: embeddingApiKeyByProvider[embeddingProvider],
|
|
1916
2174
|
baseUrl: readString(env, "EMBEDDING_BASE_URL"),
|
|
1917
2175
|
dimensions: embeddingDimensions,
|
|
@@ -1922,17 +2180,17 @@ function getEnvConfig(env = process.env, base) {
|
|
|
1922
2180
|
}
|
|
1923
2181
|
},
|
|
1924
2182
|
ui: {
|
|
1925
|
-
title: (
|
|
1926
|
-
subtitle: (
|
|
1927
|
-
primaryColor: (
|
|
1928
|
-
accentColor: (
|
|
1929
|
-
logoUrl: (
|
|
1930
|
-
placeholder: (
|
|
1931
|
-
showSources: ((
|
|
1932
|
-
welcomeMessage: (
|
|
1933
|
-
visualStyle: (
|
|
1934
|
-
borderRadius: (
|
|
1935
|
-
allowUpload: ((
|
|
2183
|
+
title: (_na = (_ma = readString(env, "NEXT_PUBLIC_UI_TITLE")) != null ? _ma : readString(env, "UI_TITLE")) != null ? _na : "AI Assistant",
|
|
2184
|
+
subtitle: (_pa = (_oa = readString(env, "NEXT_PUBLIC_UI_SUBTITLE")) != null ? _oa : readString(env, "UI_SUBTITLE")) != null ? _pa : "Powered by RAG",
|
|
2185
|
+
primaryColor: (_ra = (_qa = readString(env, "NEXT_PUBLIC_PRIMARY_COLOR")) != null ? _qa : readString(env, "UI_PRIMARY_COLOR")) != null ? _ra : "#10b981",
|
|
2186
|
+
accentColor: (_ta = (_sa = readString(env, "NEXT_PUBLIC_ACCENT_COLOR")) != null ? _sa : readString(env, "UI_ACCENT_COLOR")) != null ? _ta : "#3b82f6",
|
|
2187
|
+
logoUrl: (_ua = readString(env, "NEXT_PUBLIC_LOGO_URL")) != null ? _ua : readString(env, "UI_LOGO_URL"),
|
|
2188
|
+
placeholder: (_wa = (_va = readString(env, "NEXT_PUBLIC_PLACEHOLDER")) != null ? _va : readString(env, "UI_PLACEHOLDER")) != null ? _wa : "Ask me anything\u2026",
|
|
2189
|
+
showSources: ((_ya = (_xa = readString(env, "NEXT_PUBLIC_SHOW_SOURCES")) != null ? _xa : readString(env, "UI_SHOW_SOURCES")) != null ? _ya : "true") !== "false",
|
|
2190
|
+
welcomeMessage: (_Aa = (_za = readString(env, "NEXT_PUBLIC_WELCOME_MESSAGE")) != null ? _za : readString(env, "UI_WELCOME_MESSAGE")) != null ? _Aa : "Hello! I'm your AI assistant. Ask me anything about your documents.",
|
|
2191
|
+
visualStyle: (_Ca = (_Ba = readString(env, "NEXT_PUBLIC_UI_VISUAL_STYLE")) != null ? _Ba : readString(env, "UI_VISUAL_STYLE")) != null ? _Ca : "glass",
|
|
2192
|
+
borderRadius: (_Ea = (_Da = readString(env, "NEXT_PUBLIC_UI_BORDER_RADIUS")) != null ? _Da : readString(env, "UI_BORDER_RADIUS")) != null ? _Ea : "xl",
|
|
2193
|
+
allowUpload: ((_Ga = (_Fa = readString(env, "NEXT_PUBLIC_ALLOW_UPLOAD")) != null ? _Fa : readString(env, "UI_ALLOW_UPLOAD")) != null ? _Ga : "false") === "true"
|
|
1936
2194
|
},
|
|
1937
2195
|
rag: {
|
|
1938
2196
|
topK: readNumber(env, "RAG_TOP_K", 5),
|
|
@@ -2833,7 +3091,7 @@ var VECTOR_PROFILES = {
|
|
|
2833
3091
|
// src/llm/providers/UniversalLLMAdapter.ts
|
|
2834
3092
|
var UniversalLLMAdapter = class {
|
|
2835
3093
|
constructor(config) {
|
|
2836
|
-
var _a, _b, _c, _d, _e, _f, _g;
|
|
3094
|
+
var _a, _b, _c, _d, _e, _f, _g, _h;
|
|
2837
3095
|
this.model = config.model;
|
|
2838
3096
|
const llmConfig = config;
|
|
2839
3097
|
const options = (_a = llmConfig.options) != null ? _a : {};
|
|
@@ -2847,16 +3105,18 @@ var UniversalLLMAdapter = class {
|
|
|
2847
3105
|
this.systemPrompt = (_c = llmConfig.systemPrompt) != null ? _c : "You are a helpful AI assistant. Use the provided context to answer the user.";
|
|
2848
3106
|
this.maxTokens = (_d = llmConfig.maxTokens) != null ? _d : 1024;
|
|
2849
3107
|
this.temperature = (_e = llmConfig.temperature) != null ? _e : 0;
|
|
2850
|
-
|
|
2851
|
-
|
|
3108
|
+
this.apiKey = config.apiKey;
|
|
3109
|
+
this.baseUrl = (_g = (_f = llmConfig.baseUrl) != null ? _f : this.opts.baseUrl) != null ? _g : "";
|
|
3110
|
+
if (!this.baseUrl) {
|
|
2852
3111
|
throw new Error("[UniversalLLMAdapter] baseUrl is required in config or config.options");
|
|
2853
3112
|
}
|
|
3113
|
+
this.resolvedHeaders = __spreadValues(__spreadValues({
|
|
3114
|
+
"Content-Type": "application/json"
|
|
3115
|
+
}, config.apiKey ? { Authorization: `Bearer ${config.apiKey}` } : {}), this.opts.headers || {});
|
|
2854
3116
|
this.http = import_axios2.default.create({
|
|
2855
|
-
baseURL: baseUrl,
|
|
2856
|
-
headers:
|
|
2857
|
-
|
|
2858
|
-
}, config.apiKey ? { Authorization: `Bearer ${config.apiKey}` } : {}), this.opts.headers || {}),
|
|
2859
|
-
timeout: (_g = this.opts.timeout) != null ? _g : 6e4
|
|
3117
|
+
baseURL: this.baseUrl,
|
|
3118
|
+
headers: this.resolvedHeaders,
|
|
3119
|
+
timeout: (_h = this.opts.timeout) != null ? _h : 6e4
|
|
2860
3120
|
});
|
|
2861
3121
|
}
|
|
2862
3122
|
async chat(messages, context) {
|
|
@@ -2893,6 +3153,92 @@ ${context != null ? context : "None"}` },
|
|
|
2893
3153
|
}
|
|
2894
3154
|
return String(result);
|
|
2895
3155
|
}
|
|
3156
|
+
/**
|
|
3157
|
+
* Streaming chat using native fetch + ReadableStream.
|
|
3158
|
+
* Parses OpenAI-compatible SSE frames: `data: {...}\n\n`
|
|
3159
|
+
* Works with vLLM, LMStudio, Together AI, Fireworks, and any OpenAI-compatible API.
|
|
3160
|
+
*/
|
|
3161
|
+
chatStream(messages, context) {
|
|
3162
|
+
return __asyncGenerator(this, null, function* () {
|
|
3163
|
+
var _a, _b, _c;
|
|
3164
|
+
const path = (_a = this.opts.chatPath) != null ? _a : "/chat/completions";
|
|
3165
|
+
const url = `${this.baseUrl.replace(/\/$/, "")}${path}`;
|
|
3166
|
+
const extractPath = ((_b = this.opts.responseExtractPath) != null ? _b : "choices[0].message.content").replace("message.content", "delta.content");
|
|
3167
|
+
const formattedMessages = [
|
|
3168
|
+
{ role: "system", content: `${this.systemPrompt}
|
|
3169
|
+
|
|
3170
|
+
Context:
|
|
3171
|
+
${context != null ? context : "None"}` },
|
|
3172
|
+
...messages
|
|
3173
|
+
];
|
|
3174
|
+
let payload;
|
|
3175
|
+
if (this.opts.chatPayloadTemplate) {
|
|
3176
|
+
payload = buildPayload(this.opts.chatPayloadTemplate, {
|
|
3177
|
+
model: this.model,
|
|
3178
|
+
messages: formattedMessages,
|
|
3179
|
+
maxTokens: this.maxTokens,
|
|
3180
|
+
temperature: this.temperature
|
|
3181
|
+
});
|
|
3182
|
+
if (typeof payload === "object" && payload !== null) {
|
|
3183
|
+
payload.stream = true;
|
|
3184
|
+
}
|
|
3185
|
+
} else {
|
|
3186
|
+
payload = {
|
|
3187
|
+
model: this.model,
|
|
3188
|
+
messages: formattedMessages,
|
|
3189
|
+
max_tokens: this.maxTokens,
|
|
3190
|
+
temperature: this.temperature,
|
|
3191
|
+
stream: true
|
|
3192
|
+
};
|
|
3193
|
+
}
|
|
3194
|
+
const response = yield new __await(fetch(url, {
|
|
3195
|
+
method: "POST",
|
|
3196
|
+
headers: this.resolvedHeaders,
|
|
3197
|
+
body: JSON.stringify(payload)
|
|
3198
|
+
}));
|
|
3199
|
+
if (!response.ok) {
|
|
3200
|
+
const errorText = yield new __await(response.text().catch(() => response.statusText));
|
|
3201
|
+
throw new Error(`[UniversalLLMAdapter] Streaming request failed (${response.status}): ${errorText}`);
|
|
3202
|
+
}
|
|
3203
|
+
if (!response.body) {
|
|
3204
|
+
throw new Error("[UniversalLLMAdapter] Response body is null \u2014 server did not send a streaming response.");
|
|
3205
|
+
}
|
|
3206
|
+
const reader = response.body.getReader();
|
|
3207
|
+
const decoder = new TextDecoder("utf-8");
|
|
3208
|
+
let buffer = "";
|
|
3209
|
+
try {
|
|
3210
|
+
while (true) {
|
|
3211
|
+
const { done, value } = yield new __await(reader.read());
|
|
3212
|
+
if (done) break;
|
|
3213
|
+
buffer += decoder.decode(value, { stream: true });
|
|
3214
|
+
const lines = buffer.split("\n");
|
|
3215
|
+
buffer = (_c = lines.pop()) != null ? _c : "";
|
|
3216
|
+
for (const line of lines) {
|
|
3217
|
+
const trimmed = line.trim();
|
|
3218
|
+
if (!trimmed || trimmed === "data: [DONE]") continue;
|
|
3219
|
+
if (!trimmed.startsWith("data:")) continue;
|
|
3220
|
+
try {
|
|
3221
|
+
const json = JSON.parse(trimmed.slice(5).trim());
|
|
3222
|
+
const text = resolvePath(json, extractPath);
|
|
3223
|
+
if (text && typeof text === "string") yield text;
|
|
3224
|
+
} catch (e) {
|
|
3225
|
+
}
|
|
3226
|
+
}
|
|
3227
|
+
}
|
|
3228
|
+
if (buffer.trim() && buffer.trim() !== "data: [DONE]") {
|
|
3229
|
+
const jsonStr = buffer.replace(/^data:\s*/, "").trim();
|
|
3230
|
+
try {
|
|
3231
|
+
const json = JSON.parse(jsonStr);
|
|
3232
|
+
const text = resolvePath(json, extractPath);
|
|
3233
|
+
if (text && typeof text === "string") yield text;
|
|
3234
|
+
} catch (e) {
|
|
3235
|
+
}
|
|
3236
|
+
}
|
|
3237
|
+
} finally {
|
|
3238
|
+
reader.releaseLock();
|
|
3239
|
+
}
|
|
3240
|
+
});
|
|
3241
|
+
}
|
|
2896
3242
|
async embed(text) {
|
|
2897
3243
|
var _a, _b;
|
|
2898
3244
|
const path = (_a = this.opts.embedPath) != null ? _a : "/embeddings";
|
|
@@ -3089,6 +3435,7 @@ var ProviderRegistry = class {
|
|
|
3089
3435
|
return null;
|
|
3090
3436
|
}
|
|
3091
3437
|
static async loadVectorProviderClass(provider) {
|
|
3438
|
+
var _a;
|
|
3092
3439
|
if (this.vectorProviders[provider]) return this.vectorProviders[provider];
|
|
3093
3440
|
switch (provider) {
|
|
3094
3441
|
case "pinecone": {
|
|
@@ -3097,6 +3444,11 @@ var ProviderRegistry = class {
|
|
|
3097
3444
|
}
|
|
3098
3445
|
case "pgvector":
|
|
3099
3446
|
case "postgresql": {
|
|
3447
|
+
const postgresMode = ((_a = process.env.POSTGRES_MODE) != null ? _a : "multi").toLowerCase();
|
|
3448
|
+
if (postgresMode === "single") {
|
|
3449
|
+
const { PostgreSQLProvider: PostgreSQLProvider2 } = await Promise.resolve().then(() => (init_PostgreSQLProvider(), PostgreSQLProvider_exports));
|
|
3450
|
+
return PostgreSQLProvider2;
|
|
3451
|
+
}
|
|
3100
3452
|
const { MultiTablePostgresProvider: MultiTablePostgresProvider2 } = await Promise.resolve().then(() => (init_MultiTablePostgresProvider(), MultiTablePostgresProvider_exports));
|
|
3101
3453
|
return MultiTablePostgresProvider2;
|
|
3102
3454
|
}
|
|
@@ -4096,6 +4448,76 @@ var QueryProcessor = class {
|
|
|
4096
4448
|
}
|
|
4097
4449
|
return [...hints.values()];
|
|
4098
4450
|
}
|
|
4451
|
+
static extractNumericPredicates(question, validFields = []) {
|
|
4452
|
+
const predicates = [];
|
|
4453
|
+
const seen = /* @__PURE__ */ new Set();
|
|
4454
|
+
const comparatorPattern = [
|
|
4455
|
+
"greater than or equal to",
|
|
4456
|
+
"more than or equal to",
|
|
4457
|
+
"less than or equal to",
|
|
4458
|
+
"greater than",
|
|
4459
|
+
"more than",
|
|
4460
|
+
"less than",
|
|
4461
|
+
"equal to",
|
|
4462
|
+
"at least",
|
|
4463
|
+
"at most",
|
|
4464
|
+
"above",
|
|
4465
|
+
"over",
|
|
4466
|
+
"below",
|
|
4467
|
+
"under",
|
|
4468
|
+
"equals?",
|
|
4469
|
+
">=",
|
|
4470
|
+
"<=",
|
|
4471
|
+
">",
|
|
4472
|
+
"<",
|
|
4473
|
+
"="
|
|
4474
|
+
].join("|");
|
|
4475
|
+
const addPredicate = (rawField, rawOperator, rawValue) => {
|
|
4476
|
+
const value = Number(rawValue.replace(/,/g, ""));
|
|
4477
|
+
if (!Number.isFinite(value)) return;
|
|
4478
|
+
const operator = this.normalizeNumericOperator(rawOperator);
|
|
4479
|
+
const field = rawField ? this.normalizePredicateField(rawField, validFields) : void 0;
|
|
4480
|
+
const key = `${field != null ? field : "*"}::${operator}::${value}`;
|
|
4481
|
+
if (seen.has(key)) return;
|
|
4482
|
+
seen.add(key);
|
|
4483
|
+
predicates.push(__spreadProps(__spreadValues({}, field ? { field } : {}), { operator, value }));
|
|
4484
|
+
};
|
|
4485
|
+
const scopedPatterns = [
|
|
4486
|
+
new RegExp(`\\b(?:whose|with|having|where|that\\s+have|which\\s+have)\\s+([a-zA-Z][a-zA-Z0-9_\\s\\-/]{1,80}?)\\s+(?:is|are|was|were)?\\s*(?:${comparatorPattern})\\s+([\\d,]+(?:\\.\\d+)?)`, "gi"),
|
|
4487
|
+
new RegExp(`\\b([a-zA-Z][a-zA-Z0-9_\\s\\-/]{1,80}?)\\s+(?:is|are|was|were)?\\s*(?:${comparatorPattern})\\s+([\\d,]+(?:\\.\\d+)?)`, "gi")
|
|
4488
|
+
];
|
|
4489
|
+
for (const pattern of scopedPatterns) {
|
|
4490
|
+
for (const match of question.matchAll(pattern)) {
|
|
4491
|
+
const full = match[0];
|
|
4492
|
+
const operatorMatch = full.match(new RegExp(`(${comparatorPattern})`, "i"));
|
|
4493
|
+
if (!operatorMatch) continue;
|
|
4494
|
+
addPredicate(match[1], operatorMatch[1], match[2]);
|
|
4495
|
+
}
|
|
4496
|
+
}
|
|
4497
|
+
for (const match of question.matchAll(/\b([a-zA-Z][a-zA-Z0-9_\s\-/]{1,80}?)\s*(>=|<=|>|<|=)\s*([\d,]+(?:\.\d+)?)/g)) {
|
|
4498
|
+
addPredicate(match[1], match[2], match[3]);
|
|
4499
|
+
}
|
|
4500
|
+
return predicates;
|
|
4501
|
+
}
|
|
4502
|
+
static normalizeNumericOperator(operator) {
|
|
4503
|
+
const op = operator.toLowerCase().trim();
|
|
4504
|
+
if (op === ">" || /\b(greater than|more than|above|over)\b/.test(op)) return "gt";
|
|
4505
|
+
if (op === ">=" || /\b(greater than or equal to|more than or equal to|at least)\b/.test(op)) return "gte";
|
|
4506
|
+
if (op === "<" || /\b(less than|below|under)\b/.test(op)) return "lt";
|
|
4507
|
+
if (op === "<=" || /\b(less than or equal to|at most)\b/.test(op)) return "lte";
|
|
4508
|
+
return "eq";
|
|
4509
|
+
}
|
|
4510
|
+
static normalizePredicateField(field, validFields) {
|
|
4511
|
+
const cleaned = this.normalizeHintValue(field).replace(/\b(?:provide|show|get|give|list|all|the|a|an|of|organizations?|companies?|records?|items?|whose|with|having|where|that|which|have|has|is|are|was|were)\b/gi, " ").replace(/\s+/g, " ").trim();
|
|
4512
|
+
if (validFields.length === 0) return cleaned;
|
|
4513
|
+
const comparable = (value) => value.toLowerCase().replace(/[^a-z0-9]/g, "");
|
|
4514
|
+
const cleanedComparable = comparable(cleaned);
|
|
4515
|
+
const matchedField = validFields.find((fieldName) => {
|
|
4516
|
+
const candidate = comparable(fieldName);
|
|
4517
|
+
return candidate === cleanedComparable || candidate.includes(cleanedComparable) || cleanedComparable.includes(candidate);
|
|
4518
|
+
});
|
|
4519
|
+
return matchedField != null ? matchedField : cleaned;
|
|
4520
|
+
}
|
|
4099
4521
|
/**
|
|
4100
4522
|
* Constructs a QueryFilter object from extracted hints.
|
|
4101
4523
|
*
|
|
@@ -4116,6 +4538,10 @@ var QueryProcessor = class {
|
|
|
4116
4538
|
}
|
|
4117
4539
|
if (Object.keys(filter.metadata || {}).length === 0) delete filter.metadata;
|
|
4118
4540
|
if (filter.keywords && filter.keywords.length === 0) delete filter.keywords;
|
|
4541
|
+
const numericPredicates = this.extractNumericPredicates(question);
|
|
4542
|
+
if (numericPredicates.length > 0) {
|
|
4543
|
+
filter.__numericPredicates = numericPredicates;
|
|
4544
|
+
}
|
|
4119
4545
|
return filter;
|
|
4120
4546
|
}
|
|
4121
4547
|
/**
|
|
@@ -4183,6 +4609,61 @@ Return ONLY 'vector', 'graph', or 'both'. No explanation.`;
|
|
|
4183
4609
|
}
|
|
4184
4610
|
};
|
|
4185
4611
|
|
|
4612
|
+
// src/core/LLMRouter.ts
|
|
4613
|
+
var FAST_MODEL_DEFAULTS = {
|
|
4614
|
+
openai: "gpt-4o-mini",
|
|
4615
|
+
gemini: "gemini-2.0-flash",
|
|
4616
|
+
anthropic: "claude-3-haiku-20240307",
|
|
4617
|
+
ollama: "",
|
|
4618
|
+
// Ollama has no universal lightweight default — reuse main model
|
|
4619
|
+
rest: "",
|
|
4620
|
+
universal_rest: "",
|
|
4621
|
+
custom: ""
|
|
4622
|
+
};
|
|
4623
|
+
var LLMRouter = class {
|
|
4624
|
+
constructor(config) {
|
|
4625
|
+
this.config = config;
|
|
4626
|
+
this.models = /* @__PURE__ */ new Map();
|
|
4627
|
+
}
|
|
4628
|
+
/**
|
|
4629
|
+
* Initialize all LLM roles.
|
|
4630
|
+
*
|
|
4631
|
+
* @param prebuiltDefault - optional pre-built provider (from EmbeddingStrategyResolver).
|
|
4632
|
+
* When provided it is used directly as the 'default' role without re-constructing.
|
|
4633
|
+
*/
|
|
4634
|
+
async initialize(prebuiltDefault) {
|
|
4635
|
+
var _a;
|
|
4636
|
+
const defaultModel = prebuiltDefault != null ? prebuiltDefault : LLMFactory.create(this.config.llm, this.config.embedding);
|
|
4637
|
+
this.models.set("default", defaultModel);
|
|
4638
|
+
const envFastModel = process.env.FAST_LLM_MODEL;
|
|
4639
|
+
const providerFastDefault = (_a = FAST_MODEL_DEFAULTS[this.config.llm.provider]) != null ? _a : "";
|
|
4640
|
+
const fastModelName = envFastModel || providerFastDefault;
|
|
4641
|
+
if (fastModelName && fastModelName !== this.config.llm.model) {
|
|
4642
|
+
console.log(`[LLMRouter] Fast role \u2192 provider="${this.config.llm.provider}" model="${fastModelName}"`);
|
|
4643
|
+
const fastConfig = __spreadProps(__spreadValues({}, this.config.llm), {
|
|
4644
|
+
model: fastModelName
|
|
4645
|
+
});
|
|
4646
|
+
this.models.set("fast", LLMFactory.create(fastConfig, this.config.embedding));
|
|
4647
|
+
} else {
|
|
4648
|
+
console.log(`[LLMRouter] Fast role \u2192 reusing default model (no lightweight alternative configured).`);
|
|
4649
|
+
this.models.set("fast", defaultModel);
|
|
4650
|
+
}
|
|
4651
|
+
this.models.set("powerful", defaultModel);
|
|
4652
|
+
}
|
|
4653
|
+
/**
|
|
4654
|
+
* Retrieve a model provider by its task role.
|
|
4655
|
+
* Falls back to 'default' if the requested role is not registered.
|
|
4656
|
+
*/
|
|
4657
|
+
get(role) {
|
|
4658
|
+
var _a;
|
|
4659
|
+
const provider = (_a = this.models.get(role)) != null ? _a : this.models.get("default");
|
|
4660
|
+
if (!provider) {
|
|
4661
|
+
throw new Error(`[LLMRouter] No provider registered for role: "${role}". Did you call initialize()?`);
|
|
4662
|
+
}
|
|
4663
|
+
return provider;
|
|
4664
|
+
}
|
|
4665
|
+
};
|
|
4666
|
+
|
|
4186
4667
|
// src/utils/synonyms.ts
|
|
4187
4668
|
var FIELD_SYNONYMS = {
|
|
4188
4669
|
name: ["product", "item", "title", "label", "heading", "subject", "product name", "item name"],
|
|
@@ -4212,6 +4693,14 @@ var FIELD_SYNONYMS = {
|
|
|
4212
4693
|
"in stock",
|
|
4213
4694
|
"status"
|
|
4214
4695
|
],
|
|
4696
|
+
category: [
|
|
4697
|
+
"product_category",
|
|
4698
|
+
"product category",
|
|
4699
|
+
"category_name",
|
|
4700
|
+
"category name",
|
|
4701
|
+
"department",
|
|
4702
|
+
"collection"
|
|
4703
|
+
],
|
|
4215
4704
|
description: ["summary", "content", "body", "text", "info", "details"],
|
|
4216
4705
|
link: ["url", "href", "product_url", "page_url", "link"]
|
|
4217
4706
|
};
|
|
@@ -4243,101 +4732,457 @@ function resolveMetadataValue(meta, uiKey) {
|
|
|
4243
4732
|
|
|
4244
4733
|
// src/utils/UITransformer.ts
|
|
4245
4734
|
var UITransformer = class {
|
|
4735
|
+
// ─── Public Entry Points ─────────────────────────────────────────────────
|
|
4246
4736
|
/**
|
|
4247
|
-
*
|
|
4248
|
-
*
|
|
4737
|
+
* Heuristic-only transform (no LLM required).
|
|
4738
|
+
* Uses the lightweight heuristic intent detector as a fallback.
|
|
4739
|
+
* Prefer `analyzeAndDecide()` in production.
|
|
4249
4740
|
*/
|
|
4250
|
-
static transform(userQuery, retrievedData, config, trainedSchema) {
|
|
4741
|
+
static transform(userQuery, retrievedData, config, trainedSchema, intent) {
|
|
4742
|
+
var _a, _b, _c;
|
|
4251
4743
|
if (!retrievedData || retrievedData.length === 0) {
|
|
4252
4744
|
return this.createTextResponse("No data available", "No relevant data found for your query.");
|
|
4253
4745
|
}
|
|
4254
|
-
const
|
|
4255
|
-
const filteredData =
|
|
4256
|
-
const
|
|
4257
|
-
const categories = this.detectCategories(filteredData, groupByField);
|
|
4746
|
+
const resolvedIntent = intent != null ? intent : this.detectIntentHeuristic(userQuery);
|
|
4747
|
+
const filteredData = resolvedIntent.filterInStockOnly ? retrievedData.filter((item) => this.determineStockStatus(item)) : retrievedData;
|
|
4748
|
+
const profile = this.profileData(filteredData);
|
|
4258
4749
|
const hasProducts = filteredData.some((item) => this.isProductData(item));
|
|
4259
|
-
const
|
|
4260
|
-
|
|
4261
|
-
|
|
4262
|
-
return this.transformToLineChart(filteredData);
|
|
4750
|
+
const wantsPieLikeChart = ["pie_chart", "donut_chart"].includes(resolvedIntent.recommendedChart);
|
|
4751
|
+
if (resolvedIntent.visualizationHint === "trend" && profile.dateFields.length > 0 && profile.numericFields.length > 0) {
|
|
4752
|
+
return this.transformToLineChart(profile);
|
|
4263
4753
|
}
|
|
4264
|
-
|
|
4265
|
-
|
|
4266
|
-
|
|
4754
|
+
if (wantsPieLikeChart || ["composition", "category_breakdown"].includes(resolvedIntent.visualizationHint)) {
|
|
4755
|
+
const pieChart = this.transformToPieChart(filteredData, profile, userQuery);
|
|
4756
|
+
if (pieChart) return pieChart;
|
|
4267
4757
|
}
|
|
4268
|
-
if (
|
|
4269
|
-
return this.
|
|
4758
|
+
if (["comparison", "ranking"].includes(resolvedIntent.visualizationHint)) {
|
|
4759
|
+
return this.transformToBarChart(filteredData, profile, userQuery, resolvedIntent.visualizationHint === "ranking");
|
|
4270
4760
|
}
|
|
4271
|
-
if (
|
|
4272
|
-
return this.
|
|
4761
|
+
if (resolvedIntent.visualizationHint === "distribution") {
|
|
4762
|
+
return (_a = this.transformToHistogram(profile, userQuery)) != null ? _a : this.transformToBarChart(filteredData, profile, userQuery);
|
|
4763
|
+
}
|
|
4764
|
+
if (resolvedIntent.visualizationHint === "correlation") {
|
|
4765
|
+
return (_b = this.transformToScatterPlot(profile, userQuery)) != null ? _b : this.transformToBarChart(filteredData, profile, userQuery);
|
|
4766
|
+
}
|
|
4767
|
+
if (resolvedIntent.visualizationHint === "geographic") {
|
|
4768
|
+
return this.transformToBarChart(filteredData, profile, userQuery);
|
|
4769
|
+
}
|
|
4770
|
+
if (this.isStructuredListQuery(userQuery)) {
|
|
4771
|
+
return this.hasMultipleFields(filteredData) ? this.transformToTable(filteredData, userQuery) : this.transformToText(filteredData);
|
|
4273
4772
|
}
|
|
4274
|
-
if (
|
|
4275
|
-
return this.
|
|
4773
|
+
if (resolvedIntent.visualizationHint === "kpi") {
|
|
4774
|
+
return (_c = this.transformToMetricCard(profile, userQuery)) != null ? _c : this.transformToText(filteredData);
|
|
4276
4775
|
}
|
|
4776
|
+
if (["tabular", "table"].includes(resolvedIntent.visualizationHint) || resolvedIntent.wantsExplicitTable) {
|
|
4777
|
+
return this.hasMultipleFields(filteredData) ? this.transformToTable(filteredData, userQuery) : this.transformToText(filteredData);
|
|
4778
|
+
}
|
|
4779
|
+
if ((hasProducts || this.isProductQuery(userQuery)) && resolvedIntent.visualizationHint === "product_browse") {
|
|
4780
|
+
return this.transformToProductCarousel(filteredData, config, trainedSchema);
|
|
4781
|
+
}
|
|
4782
|
+
const automatic = this.chooseAutomaticVisualization(filteredData, profile, userQuery);
|
|
4783
|
+
if (automatic) return automatic;
|
|
4277
4784
|
return this.transformToText(filteredData);
|
|
4278
4785
|
}
|
|
4279
4786
|
/**
|
|
4280
|
-
*
|
|
4281
|
-
|
|
4282
|
-
|
|
4283
|
-
|
|
4284
|
-
|
|
4285
|
-
type: "product_carousel",
|
|
4286
|
-
title: "Recommended Products",
|
|
4287
|
-
description: `Found ${products.length} relevant products`,
|
|
4288
|
-
data: products
|
|
4289
|
-
};
|
|
4290
|
-
}
|
|
4291
|
-
/**
|
|
4292
|
-
* Transform data to pie chart format
|
|
4787
|
+
* LLM-driven entry point (recommended for production).
|
|
4788
|
+
*
|
|
4789
|
+
* Step 1 — Detect intent via a dedicated, lightweight LLM call.
|
|
4790
|
+
* Step 2 — Pass the intent + data to the visualization-selection prompt.
|
|
4791
|
+
* Step 3 — Fall back to the heuristic `transform()` if either LLM call fails.
|
|
4293
4792
|
*/
|
|
4294
|
-
static
|
|
4295
|
-
let
|
|
4296
|
-
|
|
4297
|
-
|
|
4298
|
-
|
|
4299
|
-
|
|
4300
|
-
|
|
4301
|
-
|
|
4302
|
-
|
|
4303
|
-
|
|
4304
|
-
|
|
4305
|
-
|
|
4306
|
-
|
|
4307
|
-
|
|
4308
|
-
|
|
4309
|
-
|
|
4310
|
-
|
|
4311
|
-
|
|
4312
|
-
|
|
4793
|
+
static async analyzeAndDecide(query, sources, llm) {
|
|
4794
|
+
let intent;
|
|
4795
|
+
try {
|
|
4796
|
+
intent = await this.detectIntent(query, llm);
|
|
4797
|
+
console.debug("[UITransformer] Detected intent:", intent);
|
|
4798
|
+
} catch (err) {
|
|
4799
|
+
console.warn("[UITransformer] Intent detection failed; using heuristic.", err);
|
|
4800
|
+
intent = this.detectIntentHeuristic(query);
|
|
4801
|
+
}
|
|
4802
|
+
if (this.isProductQuery(query) && ["text", "product_browse"].includes(intent.visualizationHint)) {
|
|
4803
|
+
return this.transform(
|
|
4804
|
+
query,
|
|
4805
|
+
sources,
|
|
4806
|
+
void 0,
|
|
4807
|
+
void 0,
|
|
4808
|
+
__spreadProps(__spreadValues({}, intent), { visualizationHint: "product_browse", recommendedChart: "text" })
|
|
4809
|
+
);
|
|
4810
|
+
}
|
|
4811
|
+
try {
|
|
4812
|
+
const context = this.buildContextSummary(sources);
|
|
4813
|
+
const systemPrompt = this.buildVisualizationSystemPrompt();
|
|
4814
|
+
const userPrompt = [
|
|
4815
|
+
`USER QUESTION: ${query}`,
|
|
4816
|
+
"",
|
|
4817
|
+
`DETECTED INTENT (JSON): ${JSON.stringify(intent)}`,
|
|
4818
|
+
"",
|
|
4819
|
+
"RETRIEVED DATA (JSON):",
|
|
4820
|
+
context
|
|
4821
|
+
].join("\n");
|
|
4822
|
+
const rawResponse = await llm.chat(
|
|
4823
|
+
[{ role: "user", content: userPrompt }],
|
|
4824
|
+
"",
|
|
4825
|
+
{ systemPrompt, temperature: 0 }
|
|
4826
|
+
);
|
|
4827
|
+
const parsed = this.parseTransformationResponse(rawResponse);
|
|
4828
|
+
if (parsed) {
|
|
4829
|
+
const intentAllowsTable = intent.wantsExplicitTable || ["tabular", "table", "geographic"].includes(intent.visualizationHint);
|
|
4830
|
+
const intentWantsPieLikeChart = ["pie_chart", "donut_chart"].includes(intent.recommendedChart) || ["composition", "category_breakdown"].includes(intent.visualizationHint);
|
|
4831
|
+
if (parsed.type === "table" && !intentAllowsTable) {
|
|
4832
|
+
console.debug("[UITransformer] LLM chose table but intent says no. Falling back to text.");
|
|
4833
|
+
return this.transform(query, sources, void 0, void 0, intent);
|
|
4834
|
+
}
|
|
4835
|
+
if (intentWantsPieLikeChart && parsed.type !== "pie_chart" && this.detectCategories(sources).length > 1) {
|
|
4836
|
+
console.debug("[UITransformer] LLM ignored pie/composition intent. Using deterministic pie chart.");
|
|
4837
|
+
return this.transform(query, sources, void 0, void 0, intent);
|
|
4838
|
+
}
|
|
4839
|
+
console.debug("[UITransformer] LLM chose visualization type:", parsed.type);
|
|
4840
|
+
return parsed;
|
|
4841
|
+
}
|
|
4842
|
+
console.warn("[UITransformer] LLM returned unparseable response; falling back to heuristic.");
|
|
4843
|
+
} catch (err) {
|
|
4844
|
+
console.warn("[UITransformer] analyzeAndDecide LLM call failed; falling back to heuristic.", err);
|
|
4845
|
+
}
|
|
4846
|
+
return this.transform(query, sources, void 0, void 0, intent);
|
|
4313
4847
|
}
|
|
4848
|
+
// ─── Dynamic Intent Detection ─────────────────────────────────────────────
|
|
4314
4849
|
/**
|
|
4315
|
-
*
|
|
4850
|
+
* Calls the LLM with a compact, focused prompt to extract a structured
|
|
4851
|
+
* `QueryIntent` from the user's query.
|
|
4852
|
+
*
|
|
4853
|
+
* Keeping this as a *separate* call from visualization selection means:
|
|
4854
|
+
* - The prompt is shorter and more reliable.
|
|
4855
|
+
* - The intent object can be reused across both the heuristic and LLM paths.
|
|
4856
|
+
* - It is easy to unit-test intent detection in isolation.
|
|
4316
4857
|
*/
|
|
4317
|
-
static
|
|
4318
|
-
const
|
|
4319
|
-
|
|
4320
|
-
|
|
4321
|
-
|
|
4322
|
-
|
|
4323
|
-
|
|
4858
|
+
static async detectIntent(query, llm) {
|
|
4859
|
+
const systemPrompt = `You are an intent classifier for a product-search RAG system.
|
|
4860
|
+
Given a user query, return ONLY a valid JSON object with this exact shape \u2014 no prose, no markdown:
|
|
4861
|
+
|
|
4862
|
+
{
|
|
4863
|
+
"visualizationHint": "trend" | "comparison" | "distribution" | "composition" | "correlation" | "ranking" | "kpi" | "tabular" | "geographic" | "product_browse" | "table" | "text",
|
|
4864
|
+
"recommendedChart": "line_chart" | "bar_chart" | "histogram" | "pie_chart" | "donut_chart" | "scatter_plot" | "horizontal_bar" | "metric_card" | "table" | "geo_map" | "text",
|
|
4865
|
+
"filterInStockOnly": boolean,
|
|
4866
|
+
"wantsExplicitTable": boolean,
|
|
4867
|
+
"isTemporal": boolean,
|
|
4868
|
+
"isComparison": boolean,
|
|
4869
|
+
"language": "<BCP-47 language tag, e.g. en, de, hi, ja>",
|
|
4870
|
+
"reasoning": "<one short sentence \u2014 why you chose these values>"
|
|
4871
|
+
}
|
|
4872
|
+
|
|
4873
|
+
RULES:
|
|
4874
|
+
- visualizationHint and recommendedChart
|
|
4875
|
+
"trend" \u2192 keywords: trend, growth, over time; recommendedChart "line_chart"
|
|
4876
|
+
"comparison" \u2192 keywords: compare, versus, top; recommendedChart "bar_chart"
|
|
4877
|
+
"distribution" \u2192 keywords: distribution, spread; recommendedChart "histogram"
|
|
4878
|
+
"composition" \u2192 keywords: share, percentage, breakup, breakdown; recommendedChart "pie_chart" or "donut_chart"
|
|
4879
|
+
"correlation" \u2192 keywords: relation, correlation; recommendedChart "scatter_plot"
|
|
4880
|
+
"ranking" \u2192 keywords: highest, lowest, top 10, ranking; recommendedChart "horizontal_bar"
|
|
4881
|
+
"kpi" \u2192 keywords: total, average, count; recommendedChart "metric_card"
|
|
4882
|
+
"tabular" \u2192 keywords: detailed records, table, grid, spreadsheet; recommendedChart "table"
|
|
4883
|
+
"geographic" \u2192 keywords: region, country, map; recommendedChart "geo_map"
|
|
4884
|
+
"product_browse" \u2192 user is browsing, searching, describing, viewing details for, or asking about one or more products without analytical visualization intent
|
|
4885
|
+
"table" \u2192 legacy alias for "tabular"; use only if the query literally says table
|
|
4886
|
+
"text" \u2192 conversational, factual, or other intent
|
|
4887
|
+
- If the user explicitly asks for a chart type, honor that chart type in recommendedChart.
|
|
4888
|
+
- If a query says "distribution ... in a pie chart", use visualizationHint "composition" and recommendedChart "pie_chart".
|
|
4889
|
+
- filterInStockOnly: true only if user mentions stock, availability, in stock, etc.
|
|
4890
|
+
- wantsExplicitTable: true if the user asks for a table, list, grid, spreadsheet, or detailed records.
|
|
4891
|
+
- isTemporal: true if time words appear (trend, historical, over time, last year, monthly, etc.)
|
|
4892
|
+
- isComparison: true if user compares, ranks, or contrasts entities or categories.
|
|
4893
|
+
- language: detect from the query text itself; default "en" if uncertain.`;
|
|
4894
|
+
const rawResponse = await llm.chat(
|
|
4895
|
+
[{ role: "user", content: `QUERY: ${query}` }],
|
|
4896
|
+
"",
|
|
4897
|
+
{ systemPrompt, temperature: 0 }
|
|
4898
|
+
);
|
|
4899
|
+
const parsed = this.parseIntentResponse(rawResponse);
|
|
4900
|
+
if (!parsed) {
|
|
4901
|
+
throw new Error(`Could not parse intent JSON from LLM response: ${rawResponse}`);
|
|
4902
|
+
}
|
|
4903
|
+
return parsed;
|
|
4904
|
+
}
|
|
4905
|
+
/**
|
|
4906
|
+
* Parse and validate the raw LLM response into a `QueryIntent`.
|
|
4907
|
+
*/
|
|
4908
|
+
static parseIntentResponse(raw) {
|
|
4909
|
+
const jsonStr = this.extractJsonCandidate(raw);
|
|
4910
|
+
if (!jsonStr) return null;
|
|
4911
|
+
try {
|
|
4912
|
+
const obj = JSON.parse(jsonStr);
|
|
4913
|
+
const validHints = [
|
|
4914
|
+
"trend",
|
|
4915
|
+
"comparison",
|
|
4916
|
+
"distribution",
|
|
4917
|
+
"composition",
|
|
4918
|
+
"correlation",
|
|
4919
|
+
"ranking",
|
|
4920
|
+
"kpi",
|
|
4921
|
+
"tabular",
|
|
4922
|
+
"geographic",
|
|
4923
|
+
"category_breakdown",
|
|
4924
|
+
"product_browse",
|
|
4925
|
+
"table",
|
|
4926
|
+
"text"
|
|
4927
|
+
];
|
|
4928
|
+
const validCharts = [
|
|
4929
|
+
"line_chart",
|
|
4930
|
+
"bar_chart",
|
|
4931
|
+
"histogram",
|
|
4932
|
+
"pie_chart",
|
|
4933
|
+
"donut_chart",
|
|
4934
|
+
"scatter_plot",
|
|
4935
|
+
"horizontal_bar",
|
|
4936
|
+
"metric_card",
|
|
4937
|
+
"table",
|
|
4938
|
+
"geo_map",
|
|
4939
|
+
"text"
|
|
4940
|
+
];
|
|
4941
|
+
const hint = obj.visualizationHint;
|
|
4942
|
+
if (!hint || !validHints.includes(hint)) return null;
|
|
4943
|
+
const normalizedHint = hint === "category_breakdown" ? "composition" : hint;
|
|
4944
|
+
const recommendedChart = typeof obj.recommendedChart === "string" && validCharts.includes(obj.recommendedChart) ? obj.recommendedChart : this.getRecommendedChartForIntent(normalizedHint);
|
|
4945
|
+
return {
|
|
4946
|
+
visualizationHint: normalizedHint,
|
|
4947
|
+
recommendedChart,
|
|
4948
|
+
filterInStockOnly: Boolean(obj.filterInStockOnly),
|
|
4949
|
+
wantsExplicitTable: Boolean(obj.wantsExplicitTable),
|
|
4950
|
+
isTemporal: Boolean(obj.isTemporal),
|
|
4951
|
+
isComparison: Boolean(obj.isComparison),
|
|
4952
|
+
language: typeof obj.language === "string" && obj.language ? obj.language : "en",
|
|
4953
|
+
reasoning: typeof obj.reasoning === "string" ? obj.reasoning : void 0
|
|
4954
|
+
};
|
|
4955
|
+
} catch (e) {
|
|
4956
|
+
return null;
|
|
4957
|
+
}
|
|
4958
|
+
}
|
|
4959
|
+
/**
|
|
4960
|
+
* Heuristic intent detector — used when the LLM is unavailable.
|
|
4961
|
+
* Intentionally minimal: it should only catch the most obvious signals.
|
|
4962
|
+
* The LLM path handles everything subtle.
|
|
4963
|
+
*/
|
|
4964
|
+
static detectIntentHeuristic(query) {
|
|
4965
|
+
const q = query.toLowerCase();
|
|
4966
|
+
const isTemporal = /\b(trend|trends|over time|historical|history|growth|decline|monthly|yearly|weekly|daily|last (year|month|week)|timeline|forecast)\b/.test(q);
|
|
4967
|
+
const isRanking = /\b(highest|lowest|top\s*\d+|bottom\s*\d+|rank(?:ing)?|best|worst|leading)\b/.test(q);
|
|
4968
|
+
const isComparison = /\b(compare|comparison|vs\.?|versus|against|differ|difference|contrast|top)\b/.test(q) || isRanking;
|
|
4969
|
+
const isDistribution = /\b(distribution|spread|histogram|frequency|variance|range)\b/.test(q);
|
|
4970
|
+
const wantsPieLikeChart = /\b(pie|donut|doughnut)(?:\s+chart)?\b/.test(q);
|
|
4971
|
+
const isComposition = wantsPieLikeChart || /\b(share|percentage|percent|breakup|breakdown|composition|split|segmentation|by category|by type|proportion)\b/.test(q);
|
|
4972
|
+
const isCorrelation = /\b(relation|relationship|correlation|correlate|scatter|association|impact of|depend(?:s|ence)? on)\b/.test(q);
|
|
4973
|
+
const isKpi = /\b(total|average|avg|count|sum|median|minimum|maximum|metric|kpi|how many|number of)\b/.test(q);
|
|
4974
|
+
const isGeographic = /\b(region|country|countries|state|city|location|map|geo|geographic|territory)\b/.test(q);
|
|
4975
|
+
const filterInStockOnly = /\b(in[- ]?stock|available|availability|inventory|stock status)\b/.test(q);
|
|
4976
|
+
const wantsExplicitTable = /\b(table|spreadsheet|grid|detailed records|record details|list all|compare all)\b/.test(q);
|
|
4977
|
+
let visualizationHint = "text";
|
|
4978
|
+
if (wantsExplicitTable) visualizationHint = "tabular";
|
|
4979
|
+
else if (isTemporal) visualizationHint = "trend";
|
|
4980
|
+
else if (wantsPieLikeChart) visualizationHint = "composition";
|
|
4981
|
+
else if (isRanking) visualizationHint = "ranking";
|
|
4982
|
+
else if (isComparison) visualizationHint = "comparison";
|
|
4983
|
+
else if (isDistribution) visualizationHint = "distribution";
|
|
4984
|
+
else if (isComposition) visualizationHint = "composition";
|
|
4985
|
+
else if (isCorrelation) visualizationHint = "correlation";
|
|
4986
|
+
else if (isGeographic) visualizationHint = "geographic";
|
|
4987
|
+
else if (isKpi) visualizationHint = "kpi";
|
|
4988
|
+
else if (this.isProductQuery(query)) visualizationHint = "product_browse";
|
|
4989
|
+
return {
|
|
4990
|
+
visualizationHint,
|
|
4991
|
+
recommendedChart: this.getRecommendedChartForIntent(visualizationHint),
|
|
4992
|
+
filterInStockOnly,
|
|
4993
|
+
wantsExplicitTable,
|
|
4994
|
+
isTemporal,
|
|
4995
|
+
isComparison,
|
|
4996
|
+
language: "en"
|
|
4997
|
+
// heuristic cannot reliably detect language
|
|
4998
|
+
};
|
|
4999
|
+
}
|
|
5000
|
+
static getRecommendedChartForIntent(intent) {
|
|
5001
|
+
switch (intent) {
|
|
5002
|
+
case "trend":
|
|
5003
|
+
return "line_chart";
|
|
5004
|
+
case "comparison":
|
|
5005
|
+
return "bar_chart";
|
|
5006
|
+
case "distribution":
|
|
5007
|
+
return "histogram";
|
|
5008
|
+
case "composition":
|
|
5009
|
+
case "category_breakdown":
|
|
5010
|
+
return "pie_chart";
|
|
5011
|
+
case "correlation":
|
|
5012
|
+
return "scatter_plot";
|
|
5013
|
+
case "ranking":
|
|
5014
|
+
return "horizontal_bar";
|
|
5015
|
+
case "kpi":
|
|
5016
|
+
return "metric_card";
|
|
5017
|
+
case "tabular":
|
|
5018
|
+
case "table":
|
|
5019
|
+
return "table";
|
|
5020
|
+
case "geographic":
|
|
5021
|
+
return "geo_map";
|
|
5022
|
+
case "product_browse":
|
|
5023
|
+
case "text":
|
|
5024
|
+
default:
|
|
5025
|
+
return "text";
|
|
5026
|
+
}
|
|
5027
|
+
}
|
|
5028
|
+
// ─── Transform Helpers ────────────────────────────────────────────────────
|
|
5029
|
+
static transformToProductCarousel(data, config, trainedSchema) {
|
|
5030
|
+
const products = data.map((item) => this.extractProductInfo(item, config, trainedSchema)).filter((p) => p !== null);
|
|
5031
|
+
return {
|
|
5032
|
+
type: "product_carousel",
|
|
5033
|
+
title: "Recommended Products",
|
|
5034
|
+
description: `Found ${products.length} relevant products`,
|
|
5035
|
+
data: products
|
|
5036
|
+
};
|
|
5037
|
+
}
|
|
5038
|
+
static transformToPieChart(data, profile, query = "") {
|
|
5039
|
+
var _a;
|
|
5040
|
+
const dimension = profile ? this.selectDimensionField(profile, query) : void 0;
|
|
5041
|
+
const categories = dimension ? Array.from(new Set(profile.records.map((record) => {
|
|
5042
|
+
var _a2;
|
|
5043
|
+
return String((_a2 = record.fields[dimension.key]) != null ? _a2 : "");
|
|
5044
|
+
}).filter(Boolean))) : this.detectCategories(data);
|
|
5045
|
+
if (categories.length === 0) return null;
|
|
5046
|
+
const categoryData = dimension && profile ? this.aggregateProfileByDimension(profile, dimension.key) : this.aggregateByCategory(data, categories);
|
|
5047
|
+
const pieData = Object.entries(categoryData).map(([label, count]) => {
|
|
5048
|
+
const { inStockCount, outOfStockCount } = this.calculateStockCounts(label, data);
|
|
5049
|
+
return { label, value: count, inStockCount, outOfStockCount };
|
|
5050
|
+
});
|
|
5051
|
+
return {
|
|
5052
|
+
type: "pie_chart",
|
|
5053
|
+
title: `Distribution by ${(_a = dimension == null ? void 0 : dimension.label) != null ? _a : "Category"}`,
|
|
5054
|
+
description: `Showing breakdown across ${pieData.length} categories`,
|
|
5055
|
+
data: pieData
|
|
5056
|
+
};
|
|
5057
|
+
}
|
|
5058
|
+
static transformToLineChart(profile) {
|
|
5059
|
+
const dateField = profile.dateFields[0];
|
|
5060
|
+
const valueField = profile.numericFields[0];
|
|
5061
|
+
const buckets = /* @__PURE__ */ new Map();
|
|
5062
|
+
profile.records.forEach((record) => {
|
|
5063
|
+
var _a, _b, _c;
|
|
5064
|
+
const timestamp = String((_a = record.fields[dateField.key]) != null ? _a : "");
|
|
5065
|
+
const value = (_b = this.toFiniteNumber(record.fields[valueField.key])) != null ? _b : 0;
|
|
5066
|
+
buckets.set(timestamp, ((_c = buckets.get(timestamp)) != null ? _c : 0) + value);
|
|
5067
|
+
});
|
|
5068
|
+
const lineData = Array.from(buckets.entries()).sort(([a], [b]) => Date.parse(a) - Date.parse(b)).slice(0, 24).map(([timestamp, value]) => ({ timestamp, value, label: timestamp }));
|
|
4324
5069
|
return {
|
|
4325
5070
|
type: "line_chart",
|
|
4326
|
-
title:
|
|
5071
|
+
title: `${valueField.label} Over Time`,
|
|
4327
5072
|
description: `Showing ${lineData.length} data points`,
|
|
4328
5073
|
data: lineData
|
|
4329
5074
|
};
|
|
4330
5075
|
}
|
|
4331
|
-
|
|
4332
|
-
|
|
4333
|
-
|
|
4334
|
-
|
|
4335
|
-
const
|
|
4336
|
-
const
|
|
4337
|
-
const
|
|
4338
|
-
|
|
4339
|
-
|
|
5076
|
+
static transformToBarChart(data, profile, query = "", horizontal = false) {
|
|
5077
|
+
var _a;
|
|
5078
|
+
const dimension = profile ? this.selectDimensionField(profile, query) : void 0;
|
|
5079
|
+
const measure = profile ? this.selectNumericField(profile, query) : void 0;
|
|
5080
|
+
const aggregate = dimension && profile ? this.aggregateProfileByDimension(profile, dimension.key, measure == null ? void 0 : measure.key) : this.aggregateByCategory(data, this.detectCategories(data));
|
|
5081
|
+
const barData = Object.entries(aggregate).map(([category, value]) => ({ category, value: Number(value) })).sort((a, b) => horizontal ? b.value - a.value : 0).slice(0, 12);
|
|
5082
|
+
const fallbackData = barData.length > 0 ? barData : data.slice(0, 12).map((item, index) => {
|
|
5083
|
+
var _a2, _b, _c, _d, _e;
|
|
5084
|
+
const meta = item.metadata || {};
|
|
5085
|
+
const label = String(
|
|
5086
|
+
(_c = (_b = (_a2 = this.getDynamicVal(meta, "name")) != null ? _a2 : meta.label) != null ? _b : meta.title) != null ? _c : `Result ${index + 1}`
|
|
5087
|
+
);
|
|
5088
|
+
const value = (_e = (_d = this.extractNumericValue(meta)) != null ? _d : item.score) != null ? _e : 0;
|
|
5089
|
+
return { category: label, value: Number(value) };
|
|
5090
|
+
});
|
|
5091
|
+
return {
|
|
5092
|
+
type: horizontal ? "horizontal_bar" : "bar_chart",
|
|
5093
|
+
title: dimension ? `${(_a = measure == null ? void 0 : measure.label) != null ? _a : "Count"} by ${dimension.label}` : "Comparison",
|
|
5094
|
+
description: `Showing ${fallbackData.length} comparable values`,
|
|
5095
|
+
data: fallbackData
|
|
5096
|
+
};
|
|
5097
|
+
}
|
|
5098
|
+
static transformToHistogram(profile, query = "") {
|
|
5099
|
+
const field = this.selectNumericField(profile, query);
|
|
5100
|
+
if (!field) return null;
|
|
5101
|
+
const values = profile.records.map((record) => this.toFiniteNumber(record.fields[field.key])).filter((value) => value !== null).sort((a, b) => a - b);
|
|
5102
|
+
if (values.length === 0) return null;
|
|
5103
|
+
const min = values[0];
|
|
5104
|
+
const max = values[values.length - 1];
|
|
5105
|
+
const bucketCount = Math.min(10, Math.max(3, Math.ceil(Math.sqrt(values.length))));
|
|
5106
|
+
const width = max === min ? 1 : (max - min) / bucketCount;
|
|
5107
|
+
const buckets = Array.from({ length: bucketCount }, (_, index) => {
|
|
5108
|
+
const start = min + index * width;
|
|
5109
|
+
const end = index === bucketCount - 1 ? max : start + width;
|
|
5110
|
+
return { category: `${this.formatNumber(start)}-${this.formatNumber(end)}`, value: 0 };
|
|
5111
|
+
});
|
|
5112
|
+
values.forEach((value) => {
|
|
5113
|
+
const bucketIndex = max === min ? 0 : Math.min(bucketCount - 1, Math.floor((value - min) / width));
|
|
5114
|
+
buckets[bucketIndex].value += 1;
|
|
5115
|
+
});
|
|
5116
|
+
return {
|
|
5117
|
+
type: "histogram",
|
|
5118
|
+
title: `${field.label} Distribution`,
|
|
5119
|
+
description: `Showing ${values.length} values across ${bucketCount} buckets`,
|
|
5120
|
+
data: buckets
|
|
4340
5121
|
};
|
|
5122
|
+
}
|
|
5123
|
+
static transformToScatterPlot(profile, query = "") {
|
|
5124
|
+
const fields = this.rankFieldsByQuery(profile.numericFields, query).slice(0, 2);
|
|
5125
|
+
if (fields.length < 2) return null;
|
|
5126
|
+
const [xField, yField] = fields;
|
|
5127
|
+
const points = profile.records.map((record) => {
|
|
5128
|
+
var _a;
|
|
5129
|
+
const x = this.toFiniteNumber(record.fields[xField.key]);
|
|
5130
|
+
const y = this.toFiniteNumber(record.fields[yField.key]);
|
|
5131
|
+
if (x === null || y === null) return null;
|
|
5132
|
+
return { x, y, label: String((_a = this.getRecordLabel(record)) != null ? _a : record.id) };
|
|
5133
|
+
}).filter((point) => point !== null).slice(0, 100);
|
|
5134
|
+
if (points.length === 0) return null;
|
|
5135
|
+
return {
|
|
5136
|
+
type: "scatter_plot",
|
|
5137
|
+
title: `${xField.label} vs ${yField.label}`,
|
|
5138
|
+
description: `Showing ${points.length} paired values`,
|
|
5139
|
+
data: points
|
|
5140
|
+
};
|
|
5141
|
+
}
|
|
5142
|
+
static transformToMetricCard(profile, query = "") {
|
|
5143
|
+
const operation = this.detectAggregationOperation(query);
|
|
5144
|
+
const numericField = this.selectNumericField(profile, query);
|
|
5145
|
+
const values = numericField ? profile.records.map((record) => this.toFiniteNumber(record.fields[numericField.key])).filter((value2) => value2 !== null) : [];
|
|
5146
|
+
const value = operation === "count" || values.length === 0 ? profile.records.length : this.calculateAggregate(values, operation);
|
|
5147
|
+
const metric = {
|
|
5148
|
+
label: numericField ? `${operation} ${numericField.label}` : "Count",
|
|
5149
|
+
value,
|
|
5150
|
+
operation
|
|
5151
|
+
};
|
|
5152
|
+
return {
|
|
5153
|
+
type: "metric_card",
|
|
5154
|
+
title: metric.label,
|
|
5155
|
+
description: `Calculated from ${profile.records.length} result${profile.records.length === 1 ? "" : "s"}`,
|
|
5156
|
+
data: metric
|
|
5157
|
+
};
|
|
5158
|
+
}
|
|
5159
|
+
static transformToRadarChart(data) {
|
|
5160
|
+
const attributeMap = {};
|
|
5161
|
+
data.forEach((item) => {
|
|
5162
|
+
var _a, _b, _c;
|
|
5163
|
+
const meta = item.metadata || {};
|
|
5164
|
+
const seriesName = String((_c = (_b = (_a = meta.name) != null ? _a : meta.product) != null ? _b : item.id) != null ? _c : "Item");
|
|
5165
|
+
Object.entries(meta).forEach(([key, val]) => {
|
|
5166
|
+
if (typeof val === "number" && !["price", "id"].includes(key.toLowerCase())) {
|
|
5167
|
+
if (!attributeMap[key]) attributeMap[key] = {};
|
|
5168
|
+
attributeMap[key][seriesName] = val;
|
|
5169
|
+
}
|
|
5170
|
+
});
|
|
5171
|
+
});
|
|
5172
|
+
const radarData = Object.entries(attributeMap).map(([attribute, series]) => __spreadValues({
|
|
5173
|
+
attribute
|
|
5174
|
+
}, series));
|
|
5175
|
+
return {
|
|
5176
|
+
type: "radar_chart",
|
|
5177
|
+
title: "Product Comparison",
|
|
5178
|
+
description: `Comparing ${data.length} items across ${radarData.length} attributes`,
|
|
5179
|
+
data: radarData.length > 0 ? radarData : data.map((d) => ({ attribute: d.content.substring(0, 40) }))
|
|
5180
|
+
};
|
|
5181
|
+
}
|
|
5182
|
+
static transformToTable(data, query = "") {
|
|
5183
|
+
const columns = this.extractTableColumns(data, query);
|
|
5184
|
+
const rows = data.map((item) => this.extractTableRow(item, columns));
|
|
5185
|
+
const tableData = { columns, rows };
|
|
4341
5186
|
return {
|
|
4342
5187
|
type: "table",
|
|
4343
5188
|
title: "Detailed Results",
|
|
@@ -4345,28 +5190,17 @@ var UITransformer = class {
|
|
|
4345
5190
|
data: tableData
|
|
4346
5191
|
};
|
|
4347
5192
|
}
|
|
4348
|
-
/**
|
|
4349
|
-
* Transform data to text format (fallback)
|
|
4350
|
-
*/
|
|
4351
5193
|
static transformToText(data) {
|
|
4352
|
-
const textContent = data.map((item) => item.content).join("\n\n");
|
|
4353
5194
|
return this.createTextResponse(
|
|
4354
5195
|
"Search Results",
|
|
4355
|
-
|
|
5196
|
+
data.map((item) => item.content).join("\n\n"),
|
|
4356
5197
|
`Found ${data.length} relevant results`
|
|
4357
5198
|
);
|
|
4358
5199
|
}
|
|
4359
|
-
/**
|
|
4360
|
-
* Helper: Create text response
|
|
4361
|
-
*/
|
|
4362
5200
|
static createTextResponse(title, content, description) {
|
|
4363
|
-
return {
|
|
4364
|
-
type: "text",
|
|
4365
|
-
title,
|
|
4366
|
-
description,
|
|
4367
|
-
data: { content }
|
|
4368
|
-
};
|
|
5201
|
+
return { type: "text", title, description, data: { content } };
|
|
4369
5202
|
}
|
|
5203
|
+
// ─── LLM Response Parsing ─────────────────────────────────────────────────
|
|
4370
5204
|
static parseTransformationResponse(raw) {
|
|
4371
5205
|
const payloadText = this.extractJsonCandidate(raw);
|
|
4372
5206
|
if (!payloadText) return null;
|
|
@@ -4403,9 +5237,7 @@ var UITransformer = class {
|
|
|
4403
5237
|
if (char === "{") depth += 1;
|
|
4404
5238
|
if (char === "}") {
|
|
4405
5239
|
depth -= 1;
|
|
4406
|
-
if (depth === 0)
|
|
4407
|
-
return cleaned.slice(start, i + 1);
|
|
4408
|
-
}
|
|
5240
|
+
if (depth === 0) return cleaned.slice(start, i + 1);
|
|
4409
5241
|
}
|
|
4410
5242
|
}
|
|
4411
5243
|
return null;
|
|
@@ -4413,19 +5245,16 @@ var UITransformer = class {
|
|
|
4413
5245
|
static normalizeTransformation(payload) {
|
|
4414
5246
|
var _a, _b, _c, _d, _e, _f, _g, _h, _i, _j;
|
|
4415
5247
|
if (!payload || typeof payload !== "object") return null;
|
|
4416
|
-
const
|
|
4417
|
-
const type = this.normalizeVisualizationType(
|
|
5248
|
+
const p = payload;
|
|
5249
|
+
const type = this.normalizeVisualizationType(
|
|
5250
|
+
String((_c = (_b = (_a = p.type) != null ? _a : p.view) != null ? _b : p.chartType) != null ? _c : "")
|
|
5251
|
+
);
|
|
4418
5252
|
if (!type) return null;
|
|
4419
|
-
const title = String((_e = (_d =
|
|
4420
|
-
const description =
|
|
4421
|
-
const rawData = (_j = (_i = (_h = (_g = (_f =
|
|
5253
|
+
const title = String((_e = (_d = p.title) != null ? _d : p.heading) != null ? _e : "Visualization");
|
|
5254
|
+
const description = p.description ? String(p.description) : void 0;
|
|
5255
|
+
const rawData = (_j = (_i = (_h = (_g = (_f = p.data) != null ? _f : p.table) != null ? _g : p.rows) != null ? _h : p.items) != null ? _i : p.content) != null ? _j : null;
|
|
4422
5256
|
const data = type === "text" && typeof rawData === "string" ? { content: rawData } : rawData;
|
|
4423
|
-
const transformation = {
|
|
4424
|
-
type,
|
|
4425
|
-
title,
|
|
4426
|
-
description,
|
|
4427
|
-
data
|
|
4428
|
-
};
|
|
5257
|
+
const transformation = { type, title, description, data };
|
|
4429
5258
|
return this.validateTransformation(transformation) ? transformation : null;
|
|
4430
5259
|
}
|
|
4431
5260
|
static normalizeVisualizationType(type) {
|
|
@@ -4435,10 +5264,23 @@ var UITransformer = class {
|
|
|
4435
5264
|
pie_chart: "pie_chart",
|
|
4436
5265
|
bar: "bar_chart",
|
|
4437
5266
|
bar_chart: "bar_chart",
|
|
5267
|
+
histogram: "histogram",
|
|
5268
|
+
horizontal_bar: "horizontal_bar",
|
|
5269
|
+
horizontalbar: "horizontal_bar",
|
|
4438
5270
|
line: "line_chart",
|
|
4439
5271
|
line_chart: "line_chart",
|
|
5272
|
+
scatter: "scatter_plot",
|
|
5273
|
+
scatter_plot: "scatter_plot",
|
|
5274
|
+
scatterplot: "scatter_plot",
|
|
4440
5275
|
radar: "radar_chart",
|
|
4441
5276
|
radar_chart: "radar_chart",
|
|
5277
|
+
metric: "metric_card",
|
|
5278
|
+
metric_card: "metric_card",
|
|
5279
|
+
card: "metric_card",
|
|
5280
|
+
kpi: "metric_card",
|
|
5281
|
+
geo: "geo_map",
|
|
5282
|
+
geo_map: "geo_map",
|
|
5283
|
+
map: "geo_map",
|
|
4442
5284
|
table: "table",
|
|
4443
5285
|
text: "text",
|
|
4444
5286
|
product_carousel: "product_carousel",
|
|
@@ -4446,21 +5288,31 @@ var UITransformer = class {
|
|
|
4446
5288
|
};
|
|
4447
5289
|
return (_a = mapping[type.toLowerCase()]) != null ? _a : null;
|
|
4448
5290
|
}
|
|
4449
|
-
static validateTransformation(
|
|
4450
|
-
const { type, data } =
|
|
5291
|
+
static validateTransformation(t) {
|
|
5292
|
+
const { type, data } = t;
|
|
4451
5293
|
switch (type) {
|
|
4452
5294
|
case "pie_chart":
|
|
4453
5295
|
case "bar_chart":
|
|
5296
|
+
case "histogram":
|
|
5297
|
+
case "horizontal_bar":
|
|
5298
|
+
return Array.isArray(data) && data.every(
|
|
5299
|
+
(i) => i !== null && typeof i === "object" && (typeof i.value === "number" || !Number.isNaN(Number(i.value)))
|
|
5300
|
+
);
|
|
5301
|
+
case "scatter_plot":
|
|
4454
5302
|
return Array.isArray(data) && data.every(
|
|
4455
|
-
(
|
|
5303
|
+
(i) => i !== null && typeof i === "object" && (typeof i.x === "number" || !Number.isNaN(Number(i.x))) && (typeof i.y === "number" || !Number.isNaN(Number(i.y)))
|
|
4456
5304
|
);
|
|
4457
5305
|
case "line_chart":
|
|
4458
5306
|
return Array.isArray(data) && data.every(
|
|
4459
|
-
(
|
|
5307
|
+
(i) => i !== null && typeof i === "object" && "timestamp" in i && (typeof i.value === "number" || !Number.isNaN(Number(i.value)))
|
|
4460
5308
|
);
|
|
5309
|
+
case "metric_card":
|
|
5310
|
+
return typeof data === "object" && data !== null && (typeof data.value === "number" || !Number.isNaN(Number(data.value)));
|
|
5311
|
+
case "geo_map":
|
|
5312
|
+
return Array.isArray(data) || typeof data === "object" && data !== null;
|
|
4461
5313
|
case "radar_chart":
|
|
4462
5314
|
return Array.isArray(data) && data.every(
|
|
4463
|
-
(
|
|
5315
|
+
(i) => i !== null && typeof i === "object" && "attribute" in i
|
|
4464
5316
|
);
|
|
4465
5317
|
case "table":
|
|
4466
5318
|
return typeof data === "object" && data !== null && Array.isArray(data.columns) && Array.isArray(data.rows);
|
|
@@ -4469,15 +5321,27 @@ var UITransformer = class {
|
|
|
4469
5321
|
case "product_carousel":
|
|
4470
5322
|
case "carousel":
|
|
4471
5323
|
return Array.isArray(data) && data.every(
|
|
4472
|
-
(
|
|
5324
|
+
(i) => i !== null && typeof i === "object" && ("id" in i || "name" in i)
|
|
4473
5325
|
);
|
|
4474
5326
|
default:
|
|
4475
5327
|
return false;
|
|
4476
5328
|
}
|
|
4477
5329
|
}
|
|
4478
|
-
|
|
4479
|
-
|
|
4480
|
-
|
|
5330
|
+
// ─── Data Inspection Helpers ──────────────────────────────────────────────
|
|
5331
|
+
static isStructuredListQuery(query) {
|
|
5332
|
+
const q = query.toLowerCase();
|
|
5333
|
+
const asksForRecords = /\b(list|provide|show|give|get|find|which|whose|records?|details?)\b/.test(q);
|
|
5334
|
+
const hasNumericPredicate = /\b(greater than|more than|above|over|less than|below|under|at least|at most|minimum|maximum|>=|<=|>|<|equals?|equal to)\b\s*\$?\d/i.test(q) || /\b\d+\b\s*(?:or more|or less|and above|and below)\b/i.test(q);
|
|
5335
|
+
const hasEntityTerms = /\b(organization|organisations?|organizations?|company|companies|employee|employees|staff|headcount|workforce)\b/.test(q);
|
|
5336
|
+
return asksForRecords && (hasNumericPredicate || hasEntityTerms);
|
|
5337
|
+
}
|
|
5338
|
+
static isProductQuery(query) {
|
|
5339
|
+
const q = query.toLowerCase();
|
|
5340
|
+
const productTerms = /\b(product|products|item|items|sku|catalog|catalogue|price|prices|brand|model|stock|inventory|in stock|out of stock|buy|shop)\b/.test(q);
|
|
5341
|
+
const productAction = /\b(recommend|suggest|describe|description|detail|details|about|show|find|search|browse|list)\b/.test(q);
|
|
5342
|
+
const nonProductEntityTerms = /\b(organization|organisations?|organizations?|company|companies|employee|employees|staff|headcount|workforce)\b/.test(q);
|
|
5343
|
+
return productTerms && productAction && !nonProductEntityTerms;
|
|
5344
|
+
}
|
|
4481
5345
|
static isProductData(item) {
|
|
4482
5346
|
const content = (item.content || "").toLowerCase();
|
|
4483
5347
|
const productKeywords = [
|
|
@@ -4503,394 +5367,582 @@ var UITransformer = class {
|
|
|
4503
5367
|
"fragrance"
|
|
4504
5368
|
];
|
|
4505
5369
|
const hasKeywords = productKeywords.some((kw) => content.includes(kw));
|
|
4506
|
-
const
|
|
4507
|
-
|
|
4508
|
-
|
|
4509
|
-
|
|
5370
|
+
const metadata = item.metadata || {};
|
|
5371
|
+
const hasMetadataKey = Object.keys(metadata).some((k) => {
|
|
5372
|
+
const val = metadata[k];
|
|
5373
|
+
return ["price", "product", "sku", "brand", "model", "cost", "item"].includes(k.toLowerCase()) && val !== null && val !== void 0 && val !== "";
|
|
5374
|
+
});
|
|
5375
|
+
const hasPricePattern = /\$\s*\d+\.\d{2}/.test(content);
|
|
4510
5376
|
return hasKeywords || hasMetadataKey || hasPricePattern;
|
|
4511
5377
|
}
|
|
4512
|
-
/**
|
|
4513
|
-
* Helper: Check if data contains time series
|
|
4514
|
-
*/
|
|
4515
5378
|
static isTimeSeriesData(item) {
|
|
4516
|
-
const content = (item.content || "").toLowerCase();
|
|
4517
|
-
const timeKeywords = ["trend", "historical", "growth", "decline", "change", "increase", "decrease"];
|
|
4518
|
-
const hasTimeKeyword = timeKeywords.some((kw) => content.includes(kw));
|
|
4519
5379
|
const metadata = item.metadata || {};
|
|
4520
5380
|
const maybeDateKeys = Object.keys(metadata).filter(
|
|
4521
5381
|
(k) => ["date", "timestamp", "time", "period"].includes(k.toLowerCase())
|
|
4522
5382
|
);
|
|
4523
|
-
|
|
5383
|
+
return maybeDateKeys.some((key) => {
|
|
4524
5384
|
const value = metadata[key];
|
|
4525
|
-
if (typeof value === "string")
|
|
4526
|
-
return !Number.isNaN(Date.parse(value)) && value.trim().length > 0;
|
|
4527
|
-
}
|
|
5385
|
+
if (typeof value === "string") return !Number.isNaN(Date.parse(value)) && value.trim().length > 0;
|
|
4528
5386
|
return typeof value === "number";
|
|
4529
5387
|
});
|
|
4530
|
-
return hasTimeKeyword || hasValidDateValue;
|
|
4531
|
-
}
|
|
4532
|
-
static isExplicitChartQuery(query) {
|
|
4533
|
-
const normalized = query.toLowerCase();
|
|
4534
|
-
return normalized.includes("pie chart") || normalized.includes("piechart") || normalized.includes("bar chart") || normalized.includes("barchart") || normalized.includes("radar chart");
|
|
4535
5388
|
}
|
|
4536
|
-
static
|
|
4537
|
-
const
|
|
4538
|
-
|
|
4539
|
-
|
|
4540
|
-
"breakdown",
|
|
4541
|
-
"by category",
|
|
4542
|
-
"by type",
|
|
4543
|
-
"compare",
|
|
4544
|
-
"share",
|
|
4545
|
-
"percentage",
|
|
4546
|
-
"segmentation",
|
|
4547
|
-
"split",
|
|
4548
|
-
"category breakdown",
|
|
4549
|
-
"category distribution",
|
|
4550
|
-
"pie chart",
|
|
4551
|
-
"piechart",
|
|
4552
|
-
"bar chart",
|
|
4553
|
-
"barchart"
|
|
4554
|
-
];
|
|
4555
|
-
return chartKeywords.some((keyword) => normalized.includes(keyword));
|
|
5389
|
+
static hasMultipleFields(data) {
|
|
5390
|
+
const fieldCount = /* @__PURE__ */ new Set();
|
|
5391
|
+
data.forEach((item) => Object.keys(item.metadata || {}).forEach((k) => fieldCount.add(k)));
|
|
5392
|
+
return fieldCount.size > 2;
|
|
4556
5393
|
}
|
|
4557
|
-
static
|
|
4558
|
-
const
|
|
4559
|
-
|
|
4560
|
-
|
|
4561
|
-
|
|
4562
|
-
|
|
4563
|
-
|
|
4564
|
-
|
|
4565
|
-
|
|
4566
|
-
|
|
4567
|
-
|
|
4568
|
-
|
|
4569
|
-
|
|
4570
|
-
|
|
4571
|
-
|
|
4572
|
-
|
|
4573
|
-
|
|
4574
|
-
|
|
4575
|
-
|
|
4576
|
-
|
|
5394
|
+
static profileData(data) {
|
|
5395
|
+
const records = data.map((item) => {
|
|
5396
|
+
const fields2 = {};
|
|
5397
|
+
Object.entries(item.metadata || {}).forEach(([key, value]) => {
|
|
5398
|
+
const primitive = this.toPrimitive(value);
|
|
5399
|
+
if (primitive !== null) fields2[key] = primitive;
|
|
5400
|
+
});
|
|
5401
|
+
if (!fields2.content && item.content) fields2.content = item.content.substring(0, 500);
|
|
5402
|
+
return {
|
|
5403
|
+
id: item.id,
|
|
5404
|
+
content: item.content,
|
|
5405
|
+
score: item.score,
|
|
5406
|
+
fields: fields2,
|
|
5407
|
+
source: item
|
|
5408
|
+
};
|
|
5409
|
+
});
|
|
5410
|
+
const keys = Array.from(new Set(records.flatMap((record) => Object.keys(record.fields))));
|
|
5411
|
+
const fields = keys.map((key) => {
|
|
5412
|
+
const values = records.map((record) => record.fields[key]).filter((value) => value !== void 0 && value !== null && String(value).trim() !== "");
|
|
5413
|
+
const uniqueCount = new Set(values.map((value) => String(value).toLowerCase())).size;
|
|
5414
|
+
return {
|
|
5415
|
+
key,
|
|
5416
|
+
label: this.humanizeFieldName(key),
|
|
5417
|
+
kind: this.inferFieldKind(key, values, records.length, uniqueCount),
|
|
5418
|
+
values,
|
|
5419
|
+
uniqueCount
|
|
5420
|
+
};
|
|
5421
|
+
});
|
|
5422
|
+
return {
|
|
5423
|
+
records,
|
|
5424
|
+
fields,
|
|
5425
|
+
numericFields: fields.filter((field) => field.kind === "number"),
|
|
5426
|
+
dateFields: fields.filter((field) => field.kind === "date"),
|
|
5427
|
+
categoricalFields: fields.filter((field) => field.kind === "category"),
|
|
5428
|
+
booleanFields: fields.filter((field) => field.kind === "boolean")
|
|
5429
|
+
};
|
|
4577
5430
|
}
|
|
4578
|
-
static
|
|
4579
|
-
|
|
4580
|
-
|
|
5431
|
+
static toPrimitive(value) {
|
|
5432
|
+
if (typeof value === "string" || typeof value === "number" || typeof value === "boolean") return value;
|
|
5433
|
+
if (value === null || value === void 0) return null;
|
|
5434
|
+
if (value instanceof Date) return value.toISOString();
|
|
5435
|
+
return null;
|
|
4581
5436
|
}
|
|
4582
|
-
|
|
4583
|
-
|
|
4584
|
-
|
|
4585
|
-
|
|
4586
|
-
|
|
4587
|
-
if (!meta) return void 0;
|
|
4588
|
-
const mapping = (_a = config == null ? void 0 : config.rag) == null ? void 0 : _a.uiMapping;
|
|
4589
|
-
if (mapping && mapping[uiKey]) {
|
|
4590
|
-
const mappedKey = mapping[uiKey];
|
|
4591
|
-
if (meta[mappedKey] !== void 0) return meta[mappedKey];
|
|
5437
|
+
static inferFieldKind(key, values, rowCount, uniqueCount) {
|
|
5438
|
+
const lowerKey = key.toLowerCase();
|
|
5439
|
+
if (values.length === 0) return "text";
|
|
5440
|
+
if (values.every((value) => typeof value === "boolean" || /^(true|false|yes|no|in_stock|out_of_stock|available|unavailable)$/i.test(String(value)))) {
|
|
5441
|
+
return "boolean";
|
|
4592
5442
|
}
|
|
4593
|
-
if (
|
|
4594
|
-
|
|
4595
|
-
if (trainedKey && meta[trainedKey] !== void 0) return meta[trainedKey];
|
|
5443
|
+
if (/(date|time|timestamp|created|updated|month|year|period)/i.test(lowerKey) && values.some((value) => this.isDateLike(value))) {
|
|
5444
|
+
return "date";
|
|
4596
5445
|
}
|
|
4597
|
-
|
|
5446
|
+
const numericCount = values.filter((value) => this.toFiniteNumber(value) !== null).length;
|
|
5447
|
+
if (numericCount / values.length >= 0.85 && !/(id|sku|ean|upc|zip|postal|phone|code)/i.test(lowerKey)) {
|
|
5448
|
+
return "number";
|
|
5449
|
+
}
|
|
5450
|
+
if (uniqueCount <= Math.max(20, Math.ceil(rowCount * 0.7)) && !/(id|sku|ean|upc|description|content|url|image|thumbnail)/i.test(lowerKey)) {
|
|
5451
|
+
return "category";
|
|
5452
|
+
}
|
|
5453
|
+
return "text";
|
|
4598
5454
|
}
|
|
4599
|
-
static
|
|
4600
|
-
|
|
4601
|
-
const
|
|
4602
|
-
|
|
4603
|
-
|
|
4604
|
-
|
|
4605
|
-
|
|
4606
|
-
|
|
4607
|
-
|
|
4608
|
-
|
|
4609
|
-
|
|
4610
|
-
|
|
4611
|
-
|
|
4612
|
-
|
|
4613
|
-
|
|
4614
|
-
|
|
4615
|
-
|
|
4616
|
-
return
|
|
4617
|
-
|
|
4618
|
-
|
|
4619
|
-
|
|
4620
|
-
image: typeof imageValue === "string" ? imageValue : void 0,
|
|
4621
|
-
brand: brand ? String(brand) : void 0,
|
|
4622
|
-
description: item.content,
|
|
4623
|
-
inStock: this.determineStockStatus(item)
|
|
4624
|
-
};
|
|
5455
|
+
static isDateLike(value) {
|
|
5456
|
+
if (typeof value === "number") return value > 1900 && value < 3e3;
|
|
5457
|
+
const text = String(value).trim();
|
|
5458
|
+
return text.length > 3 && !Number.isNaN(Date.parse(text));
|
|
5459
|
+
}
|
|
5460
|
+
static chooseAutomaticVisualization(data, profile, query) {
|
|
5461
|
+
if (profile.records.length === 0) return null;
|
|
5462
|
+
if (profile.dateFields.length > 0 && profile.numericFields.length > 0) {
|
|
5463
|
+
return this.transformToLineChart(profile);
|
|
5464
|
+
}
|
|
5465
|
+
if (profile.categoricalFields.length > 0 && profile.numericFields.length > 0) {
|
|
5466
|
+
return this.transformToBarChart(data, profile, query);
|
|
5467
|
+
}
|
|
5468
|
+
if (profile.categoricalFields.length > 0) {
|
|
5469
|
+
return this.transformToPieChart(data, profile, query);
|
|
5470
|
+
}
|
|
5471
|
+
if (profile.numericFields.length >= 2) {
|
|
5472
|
+
return this.transformToScatterPlot(profile, query);
|
|
5473
|
+
}
|
|
5474
|
+
if (profile.numericFields.length === 1) {
|
|
5475
|
+
return this.transformToMetricCard(profile, query);
|
|
4625
5476
|
}
|
|
4626
5477
|
return null;
|
|
4627
5478
|
}
|
|
4628
|
-
|
|
4629
|
-
|
|
4630
|
-
|
|
4631
|
-
|
|
4632
|
-
|
|
4633
|
-
const
|
|
4634
|
-
|
|
4635
|
-
|
|
4636
|
-
|
|
4637
|
-
|
|
4638
|
-
|
|
4639
|
-
|
|
4640
|
-
|
|
4641
|
-
|
|
4642
|
-
|
|
4643
|
-
|
|
4644
|
-
|
|
4645
|
-
|
|
4646
|
-
|
|
4647
|
-
|
|
5479
|
+
static selectDimensionField(profile, query) {
|
|
5480
|
+
var _a, _b;
|
|
5481
|
+
const productCategory = profile.categoricalFields.find(
|
|
5482
|
+
(field) => /category|department|collection|type|group|segment|region|country|state|city/i.test(field.key)
|
|
5483
|
+
);
|
|
5484
|
+
const ranked = this.rankFieldsByQuery(profile.categoricalFields, query);
|
|
5485
|
+
return (_b = (_a = ranked[0]) != null ? _a : productCategory) != null ? _b : profile.categoricalFields[0];
|
|
5486
|
+
}
|
|
5487
|
+
static selectNumericField(profile, query) {
|
|
5488
|
+
var _a;
|
|
5489
|
+
return (_a = this.rankFieldsByQuery(profile.numericFields, query)[0]) != null ? _a : profile.numericFields[0];
|
|
5490
|
+
}
|
|
5491
|
+
static rankFieldsByQuery(fields, query) {
|
|
5492
|
+
const q = query.toLowerCase();
|
|
5493
|
+
return [...fields].sort((a, b) => this.fieldScore(b, q) - this.fieldScore(a, q));
|
|
5494
|
+
}
|
|
5495
|
+
static fieldScore(field, query) {
|
|
5496
|
+
const key = field.key.toLowerCase();
|
|
5497
|
+
const label = field.label.toLowerCase();
|
|
5498
|
+
let score = 0;
|
|
5499
|
+
if (query.includes(key)) score += 4;
|
|
5500
|
+
if (query.includes(label)) score += 4;
|
|
5501
|
+
key.split(/[_\s-]+/).forEach((part) => {
|
|
5502
|
+
if (part && query.includes(part)) score += 1;
|
|
5503
|
+
});
|
|
5504
|
+
if (/category|department|collection|type|group|segment/.test(key)) score += 2;
|
|
5505
|
+
if (/count|total|quantity|stock|inventory|sales|revenue|amount|price|value|score/.test(key)) score += 2;
|
|
5506
|
+
if (/id|sku|ean|upc|code/.test(key)) score -= 5;
|
|
5507
|
+
return score;
|
|
5508
|
+
}
|
|
5509
|
+
static normalizeComparableField(field) {
|
|
5510
|
+
return field.toLowerCase().replace(/&/g, "and").replace(/[^a-z0-9]+/g, "").trim();
|
|
5511
|
+
}
|
|
5512
|
+
static fieldTokens(field) {
|
|
5513
|
+
return field.toLowerCase().replace(/[_-]+/g, " ").split(/\s+/).map((token) => token.replace(/[^a-z0-9]/g, "")).filter(Boolean);
|
|
5514
|
+
}
|
|
5515
|
+
static aggregateProfileByDimension(profile, dimensionKey, measureKey) {
|
|
5516
|
+
const result = {};
|
|
5517
|
+
profile.records.forEach((record) => {
|
|
5518
|
+
var _a, _b, _c;
|
|
5519
|
+
const category = String((_a = record.fields[dimensionKey]) != null ? _a : "Other").trim() || "Other";
|
|
5520
|
+
const value = measureKey ? (_b = this.toFiniteNumber(record.fields[measureKey])) != null ? _b : 0 : 1;
|
|
5521
|
+
result[category] = ((_c = result[category]) != null ? _c : 0) + value;
|
|
5522
|
+
});
|
|
5523
|
+
return result;
|
|
5524
|
+
}
|
|
5525
|
+
static getRecordLabel(record) {
|
|
5526
|
+
const labelKeys = ["name", "title", "label", "product", "item", "brand"];
|
|
5527
|
+
const key = Object.keys(record.fields).find(
|
|
5528
|
+
(fieldKey) => labelKeys.some((labelKey) => fieldKey.toLowerCase().includes(labelKey))
|
|
5529
|
+
);
|
|
5530
|
+
return key ? record.fields[key] : void 0;
|
|
5531
|
+
}
|
|
5532
|
+
static detectAggregationOperation(query) {
|
|
5533
|
+
const q = query.toLowerCase();
|
|
5534
|
+
if (/\b(avg|average|mean)\b/.test(q)) return "average";
|
|
5535
|
+
if (/\b(count|how many|number of)\b/.test(q)) return "count";
|
|
5536
|
+
if (/\b(min|minimum|lowest)\b/.test(q)) return "min";
|
|
5537
|
+
if (/\b(max|maximum|highest)\b/.test(q)) return "max";
|
|
5538
|
+
if (/\bmedian\b/.test(q)) return "median";
|
|
5539
|
+
return "sum";
|
|
5540
|
+
}
|
|
5541
|
+
static calculateAggregate(values, operation) {
|
|
5542
|
+
if (values.length === 0) return 0;
|
|
5543
|
+
switch (operation) {
|
|
5544
|
+
case "average":
|
|
5545
|
+
return values.reduce((sum, value) => sum + value, 0) / values.length;
|
|
5546
|
+
case "count":
|
|
5547
|
+
return values.length;
|
|
5548
|
+
case "min":
|
|
5549
|
+
return Math.min(...values);
|
|
5550
|
+
case "max":
|
|
5551
|
+
return Math.max(...values);
|
|
5552
|
+
case "median": {
|
|
5553
|
+
const sorted = [...values].sort((a, b) => a - b);
|
|
5554
|
+
const middle = Math.floor(sorted.length / 2);
|
|
5555
|
+
return sorted.length % 2 === 0 ? (sorted[middle - 1] + sorted[middle]) / 2 : sorted[middle];
|
|
5556
|
+
}
|
|
5557
|
+
case "sum":
|
|
5558
|
+
default:
|
|
5559
|
+
return values.reduce((sum, value) => sum + value, 0);
|
|
4648
5560
|
}
|
|
4649
|
-
return targetField;
|
|
4650
5561
|
}
|
|
4651
|
-
|
|
4652
|
-
|
|
4653
|
-
|
|
4654
|
-
static
|
|
5562
|
+
static humanizeFieldName(key) {
|
|
5563
|
+
return key.replace(/[_-]+/g, " ").replace(/([a-z])([A-Z])/g, "$1 $2").replace(/\s+/g, " ").trim().replace(/\b\w/g, (char) => char.toUpperCase());
|
|
5564
|
+
}
|
|
5565
|
+
static formatNumber(value) {
|
|
5566
|
+
return Number.isInteger(value) ? String(value) : value.toFixed(1);
|
|
5567
|
+
}
|
|
5568
|
+
static detectCategories(data) {
|
|
4655
5569
|
const categories = /* @__PURE__ */ new Set();
|
|
4656
5570
|
data.forEach((item) => {
|
|
4657
|
-
const
|
|
4658
|
-
if (
|
|
4659
|
-
const val = meta[groupByField];
|
|
4660
|
-
if (Array.isArray(val)) val.forEach((v) => categories.add(String(v)));
|
|
4661
|
-
else categories.add(String(val));
|
|
4662
|
-
} else if (groupByField === "category") {
|
|
4663
|
-
if (meta.type) categories.add(String(meta.type));
|
|
4664
|
-
if (meta.tag) {
|
|
4665
|
-
const tags = Array.isArray(meta.tag) ? meta.tag : [meta.tag];
|
|
4666
|
-
tags.forEach((t) => categories.add(String(t)));
|
|
4667
|
-
}
|
|
4668
|
-
const categoryMatch = item.content.match(/(?:Category|Type|Class):\s*([^\n]+)/i);
|
|
4669
|
-
if (categoryMatch) categories.add(categoryMatch[1].trim());
|
|
4670
|
-
}
|
|
5571
|
+
const category = this.getProductCategory(item);
|
|
5572
|
+
if (category) categories.add(category);
|
|
4671
5573
|
});
|
|
4672
5574
|
return Array.from(categories);
|
|
4673
5575
|
}
|
|
4674
|
-
|
|
4675
|
-
|
|
4676
|
-
|
|
4677
|
-
|
|
4678
|
-
const
|
|
4679
|
-
|
|
4680
|
-
|
|
4681
|
-
|
|
5576
|
+
static getProductCategory(item) {
|
|
5577
|
+
const meta = item.metadata || {};
|
|
5578
|
+
const metadataCategory = resolveMetadataValue(meta, "category");
|
|
5579
|
+
if (this.isUsableCategory(metadataCategory)) return String(metadataCategory).trim();
|
|
5580
|
+
const content = item.content || "";
|
|
5581
|
+
const explicitCategoryMatch = content.match(
|
|
5582
|
+
/(?:^|\n)\s*(?:product\s*)?(?:category|department|collection)\s*[:\-–]\s*([^\n]+)/i
|
|
5583
|
+
);
|
|
5584
|
+
if (explicitCategoryMatch && this.isUsableCategory(explicitCategoryMatch[1])) {
|
|
5585
|
+
return explicitCategoryMatch[1].trim();
|
|
5586
|
+
}
|
|
5587
|
+
return null;
|
|
5588
|
+
}
|
|
5589
|
+
static isUsableCategory(value) {
|
|
5590
|
+
if (value === null || value === void 0) return false;
|
|
5591
|
+
const category = String(value).trim();
|
|
5592
|
+
if (!category) return false;
|
|
5593
|
+
return !/^(category|type|tag|price|cost|stock|availability|description|product|item|name|brand|sku|id|ean|currency|color|size|index|internal id|other|\[?type\]?|\[?products?_?\d*\]?)$/i.test(category);
|
|
5594
|
+
}
|
|
5595
|
+
static aggregateByCategory(data, categories) {
|
|
5596
|
+
const result = Object.fromEntries(categories.map((c) => [c, 0]));
|
|
4682
5597
|
data.forEach((item) => {
|
|
4683
|
-
|
|
4684
|
-
|
|
4685
|
-
if (
|
|
4686
|
-
|
|
4687
|
-
} else if (groupByField === "category" && meta.type) {
|
|
4688
|
-
itemCategory = String(meta.type);
|
|
4689
|
-
}
|
|
4690
|
-
if (Object.prototype.hasOwnProperty.call(result, itemCategory)) {
|
|
4691
|
-
result[itemCategory]++;
|
|
4692
|
-
} else {
|
|
4693
|
-
result["Other"] = (result["Other"] || 0) + 1;
|
|
4694
|
-
}
|
|
5598
|
+
var _a;
|
|
5599
|
+
const cat = (_a = this.getProductCategory(item)) != null ? _a : "Other";
|
|
5600
|
+
if (Object.prototype.hasOwnProperty.call(result, cat)) result[cat]++;
|
|
5601
|
+
else result["Other"] = (result["Other"] || 0) + 1;
|
|
4695
5602
|
});
|
|
4696
5603
|
return result;
|
|
4697
5604
|
}
|
|
4698
|
-
/**
|
|
4699
|
-
* Helper: Extract time series data
|
|
4700
|
-
*/
|
|
4701
5605
|
static extractTimeSeriesData(data) {
|
|
4702
5606
|
return data.map((item) => {
|
|
5607
|
+
var _a, _b, _c, _d, _e;
|
|
4703
5608
|
const meta = item.metadata || {};
|
|
4704
5609
|
return {
|
|
4705
|
-
timestamp: meta.timestamp
|
|
4706
|
-
value: meta.value
|
|
4707
|
-
label: meta.label
|
|
5610
|
+
timestamp: (_b = (_a = meta.timestamp) != null ? _a : meta.date) != null ? _b : (/* @__PURE__ */ new Date()).toISOString(),
|
|
5611
|
+
value: (_d = (_c = meta.value) != null ? _c : item.score) != null ? _d : 0,
|
|
5612
|
+
label: (_e = meta.label) != null ? _e : item.content.substring(0, 50)
|
|
4708
5613
|
};
|
|
4709
5614
|
});
|
|
4710
5615
|
}
|
|
4711
|
-
|
|
4712
|
-
|
|
4713
|
-
|
|
4714
|
-
|
|
4715
|
-
|
|
4716
|
-
|
|
4717
|
-
|
|
4718
|
-
|
|
4719
|
-
|
|
4720
|
-
|
|
4721
|
-
|
|
4722
|
-
|
|
5616
|
+
static extractNumericValue(meta) {
|
|
5617
|
+
var _a;
|
|
5618
|
+
const preferredKeys = [
|
|
5619
|
+
"value",
|
|
5620
|
+
"count",
|
|
5621
|
+
"total",
|
|
5622
|
+
"average",
|
|
5623
|
+
"avg",
|
|
5624
|
+
"amount",
|
|
5625
|
+
"sales",
|
|
5626
|
+
"revenue",
|
|
5627
|
+
"score",
|
|
5628
|
+
"quantity",
|
|
5629
|
+
"price"
|
|
5630
|
+
];
|
|
5631
|
+
for (const key of preferredKeys) {
|
|
5632
|
+
const raw = (_a = resolveMetadataValue(meta, key)) != null ? _a : meta[key];
|
|
5633
|
+
const value = typeof raw === "number" ? raw : Number(String(raw != null ? raw : "").replace(/[$,% ,]/g, ""));
|
|
5634
|
+
if (Number.isFinite(value)) return value;
|
|
5635
|
+
}
|
|
5636
|
+
for (const value of Object.values(meta)) {
|
|
5637
|
+
const numeric = typeof value === "number" ? value : Number(String(value != null ? value : "").replace(/[$,% ,]/g, ""));
|
|
5638
|
+
if (Number.isFinite(numeric)) return numeric;
|
|
5639
|
+
}
|
|
5640
|
+
return null;
|
|
5641
|
+
}
|
|
5642
|
+
static extractTableColumns(data, query = "") {
|
|
5643
|
+
const q = query.toLowerCase();
|
|
5644
|
+
const availableFields = this.extractAvailableTableFields(data);
|
|
5645
|
+
if (/\b(organization|organisations?|organizations?|company|companies)\b/.test(q) && /\b(employee|employees|staff|headcount|workforce)\b/.test(q)) {
|
|
5646
|
+
const nameField = this.pickTableField(availableFields, [
|
|
5647
|
+
"company name",
|
|
5648
|
+
"organization name",
|
|
5649
|
+
"organisation name",
|
|
5650
|
+
"name",
|
|
5651
|
+
"company",
|
|
5652
|
+
"organization",
|
|
5653
|
+
"organisation"
|
|
5654
|
+
]);
|
|
5655
|
+
const employeeField = this.pickTableField(availableFields, [
|
|
5656
|
+
"number of employees",
|
|
5657
|
+
"employee count",
|
|
5658
|
+
"employees",
|
|
5659
|
+
"employee_count",
|
|
5660
|
+
"number_employees",
|
|
5661
|
+
"num employees",
|
|
5662
|
+
"staff count",
|
|
5663
|
+
"headcount",
|
|
5664
|
+
"workforce"
|
|
5665
|
+
]);
|
|
5666
|
+
const columns = [nameField, employeeField].filter((field) => Boolean(field));
|
|
5667
|
+
if (columns.length > 0) return columns;
|
|
5668
|
+
}
|
|
5669
|
+
const requestedFields = availableFields.map((field) => ({
|
|
5670
|
+
field,
|
|
5671
|
+
score: this.tableFieldQueryScore(field, q)
|
|
5672
|
+
})).filter((item) => item.score > 0).sort((a, b) => b.score - a.score).map((item) => item.field);
|
|
5673
|
+
if (requestedFields.length > 0) {
|
|
5674
|
+
return requestedFields.slice(0, Math.min(6, requestedFields.length));
|
|
5675
|
+
}
|
|
5676
|
+
const metadataFields = Array.from(new Set(
|
|
5677
|
+
data.flatMap((item) => Object.keys(item.metadata || {}).map((k) => this.humanizeFieldName(k)))
|
|
5678
|
+
));
|
|
5679
|
+
return metadataFields.length > 0 ? metadataFields : ["Content"];
|
|
4723
5680
|
}
|
|
4724
|
-
/**
|
|
4725
|
-
* Helper: Extract table row
|
|
4726
|
-
*/
|
|
4727
5681
|
static extractTableRow(item, columns) {
|
|
4728
|
-
|
|
4729
|
-
|
|
4730
|
-
|
|
4731
|
-
|
|
5682
|
+
return columns.map((col) => this.resolveTableCellValue(item, col));
|
|
5683
|
+
}
|
|
5684
|
+
static extractAvailableTableFields(data) {
|
|
5685
|
+
const fields = /* @__PURE__ */ new Map();
|
|
5686
|
+
const addField = (field) => {
|
|
5687
|
+
const clean = this.humanizeFieldName(field);
|
|
5688
|
+
if (!clean || /^(content|\[?type\]?)$/i.test(clean)) return;
|
|
5689
|
+
const normalized = this.normalizeComparableField(clean);
|
|
5690
|
+
if (!fields.has(normalized)) fields.set(normalized, clean);
|
|
5691
|
+
};
|
|
5692
|
+
data.forEach((item) => {
|
|
5693
|
+
Object.keys(item.metadata || {}).forEach(addField);
|
|
5694
|
+
for (const match of (item.content || "").matchAll(/(?:^|\n)\s*([A-Za-z][A-Za-z0-9_ /-]{1,50})\s*[:\-–]\s*([^\n]+)/g)) {
|
|
5695
|
+
addField(match[1]);
|
|
4732
5696
|
}
|
|
4733
|
-
const metaKey = col.charAt(0).toLowerCase() + col.slice(1);
|
|
4734
|
-
const value = meta[metaKey];
|
|
4735
|
-
return value !== void 0 ? String(value) : "";
|
|
4736
5697
|
});
|
|
5698
|
+
return Array.from(fields.values());
|
|
5699
|
+
}
|
|
5700
|
+
static pickTableField(fields, aliases) {
|
|
5701
|
+
const normalizedAliases = aliases.map((alias) => this.normalizeComparableField(alias));
|
|
5702
|
+
for (const alias of normalizedAliases) {
|
|
5703
|
+
const exact = fields.find((field) => this.normalizeComparableField(field) === alias);
|
|
5704
|
+
if (exact) return exact;
|
|
5705
|
+
}
|
|
5706
|
+
for (const alias of normalizedAliases) {
|
|
5707
|
+
const fuzzy = fields.find((field) => {
|
|
5708
|
+
const normalizedField = this.normalizeComparableField(field);
|
|
5709
|
+
if (/(id|code|uuid)$/.test(normalizedField) && !/(id|code|uuid)$/.test(alias)) return false;
|
|
5710
|
+
return normalizedField.includes(alias) || alias.includes(normalizedField);
|
|
5711
|
+
});
|
|
5712
|
+
if (fuzzy) return fuzzy;
|
|
5713
|
+
}
|
|
5714
|
+
return void 0;
|
|
4737
5715
|
}
|
|
4738
|
-
|
|
4739
|
-
|
|
4740
|
-
|
|
4741
|
-
|
|
5716
|
+
static tableFieldQueryScore(field, query) {
|
|
5717
|
+
const normalizedField = this.normalizeComparableField(field);
|
|
5718
|
+
if (!normalizedField) return 0;
|
|
5719
|
+
const fieldTokens = this.fieldTokens(field);
|
|
5720
|
+
return fieldTokens.reduce((score, token) => {
|
|
5721
|
+
if (token.length < 3) return score;
|
|
5722
|
+
return query.includes(token) ? score + 1 : score;
|
|
5723
|
+
}, query.includes(normalizedField) ? 3 : 0);
|
|
5724
|
+
}
|
|
5725
|
+
static resolveTableCellValue(item, column) {
|
|
5726
|
+
if (column === "Content") return item.content.substring(0, 100);
|
|
5727
|
+
const meta = item.metadata || {};
|
|
5728
|
+
const normalizedColumn = this.normalizeComparableField(column);
|
|
5729
|
+
const exactMetadata = Object.entries(meta).find(
|
|
5730
|
+
([key]) => this.normalizeComparableField(key) === normalizedColumn || this.normalizeComparableField(this.humanizeFieldName(key)) === normalizedColumn
|
|
5731
|
+
);
|
|
5732
|
+
if (exactMetadata && exactMetadata[1] !== void 0 && exactMetadata[1] !== null) {
|
|
5733
|
+
return this.toDisplayValue(exactMetadata[1]);
|
|
5734
|
+
}
|
|
5735
|
+
const aliasValue = this.resolveAliasedTableCell(meta, column);
|
|
5736
|
+
if (aliasValue !== void 0) return this.toDisplayValue(aliasValue);
|
|
5737
|
+
const contentValue = this.extractContentFieldValue(item.content, column);
|
|
5738
|
+
if (contentValue !== null) return contentValue;
|
|
5739
|
+
const aliasContentValue = this.extractAliasedContentFieldValue(item.content, column);
|
|
5740
|
+
return aliasContentValue != null ? aliasContentValue : "";
|
|
5741
|
+
}
|
|
5742
|
+
static resolveAliasedTableCell(meta, column) {
|
|
5743
|
+
const normalizedColumn = this.normalizeComparableField(column);
|
|
5744
|
+
const aliases = normalizedColumn.includes("employee") ? ["number of employees", "employee count", "employees", "headcount", "staff count", "workforce"] : normalizedColumn.includes("name") ? ["company name", "organization name", "organisation name", "name", "company", "organization", "organisation"] : [];
|
|
5745
|
+
for (const alias of aliases) {
|
|
5746
|
+
const normalizedAlias = this.normalizeComparableField(alias);
|
|
5747
|
+
const match = Object.entries(meta).find(([key]) => {
|
|
5748
|
+
const normalizedKey = this.normalizeComparableField(key);
|
|
5749
|
+
return normalizedKey === normalizedAlias || normalizedKey.includes(normalizedAlias) || normalizedAlias.includes(normalizedKey);
|
|
5750
|
+
});
|
|
5751
|
+
if (match && match[1] !== void 0 && match[1] !== null) return match[1];
|
|
5752
|
+
}
|
|
5753
|
+
return void 0;
|
|
5754
|
+
}
|
|
5755
|
+
static extractContentFieldValue(content, column) {
|
|
5756
|
+
const escaped = column.replace(/[.*+?^${}()|[\]\\]/g, "\\$&").replace(/\s+/g, "[\\s_ -]+");
|
|
5757
|
+
const pattern = new RegExp(`(?:^|\\n)\\s*${escaped}\\s*[:\\-\u2013]\\s*([^\\n]+)`, "i");
|
|
5758
|
+
const match = content.match(pattern);
|
|
5759
|
+
return match ? match[1].trim() : null;
|
|
5760
|
+
}
|
|
5761
|
+
static extractAliasedContentFieldValue(content, column) {
|
|
5762
|
+
const normalizedColumn = this.normalizeComparableField(column);
|
|
5763
|
+
const aliases = normalizedColumn.includes("employee") ? ["Number Of Employees", "Number of Employees", "Employee Count", "Employees", "Headcount", "Staff Count", "Workforce"] : normalizedColumn.includes("name") ? ["Company Name", "Organization Name", "Organisation Name", "Name", "Company", "Organization", "Organisation"] : [];
|
|
5764
|
+
for (const alias of aliases) {
|
|
5765
|
+
const value = this.extractContentFieldValue(content, alias);
|
|
5766
|
+
if (value !== null) return value;
|
|
5767
|
+
}
|
|
5768
|
+
return null;
|
|
5769
|
+
}
|
|
5770
|
+
static toDisplayValue(value) {
|
|
5771
|
+
if (typeof value === "string" || typeof value === "number" || typeof value === "boolean") return value;
|
|
5772
|
+
return String(value != null ? value : "");
|
|
5773
|
+
}
|
|
5774
|
+
static calculateStockCounts(category, data) {
|
|
4742
5775
|
let inStock = 0;
|
|
4743
5776
|
let outOfStock = 0;
|
|
4744
5777
|
data.forEach((d) => {
|
|
4745
|
-
|
|
4746
|
-
|
|
4747
|
-
if (
|
|
4748
|
-
|
|
4749
|
-
|
|
4750
|
-
|
|
4751
|
-
}
|
|
4752
|
-
if (itemCategory === category) {
|
|
4753
|
-
if (this.determineStockStatus(d)) {
|
|
4754
|
-
inStock++;
|
|
4755
|
-
} else {
|
|
4756
|
-
outOfStock++;
|
|
4757
|
-
}
|
|
5778
|
+
var _a, _b;
|
|
5779
|
+
const cat = (_a = this.getProductCategory(d)) != null ? _a : "Other";
|
|
5780
|
+
if (cat === category) {
|
|
5781
|
+
const quantity = (_b = this.extractStockQuantity(d)) != null ? _b : 1;
|
|
5782
|
+
if (this.determineStockStatus(d)) inStock += quantity;
|
|
5783
|
+
else outOfStock += quantity;
|
|
4758
5784
|
}
|
|
4759
5785
|
});
|
|
4760
5786
|
return { inStockCount: inStock, outOfStockCount: outOfStock };
|
|
4761
5787
|
}
|
|
4762
|
-
/**
|
|
4763
|
-
* Helper: Determine if item is in stock
|
|
4764
|
-
*/
|
|
4765
5788
|
static determineStockStatus(item) {
|
|
4766
5789
|
const meta = item.metadata || {};
|
|
4767
|
-
|
|
4768
|
-
|
|
4769
|
-
|
|
4770
|
-
|
|
4771
|
-
|
|
4772
|
-
|
|
4773
|
-
|
|
4774
|
-
|
|
4775
|
-
|
|
5790
|
+
const stockValue = resolveMetadataValue(meta, "stock");
|
|
5791
|
+
if (stockValue !== void 0) {
|
|
5792
|
+
const normalized = String(stockValue).toLowerCase();
|
|
5793
|
+
if (/out[_\s-]?of[_\s-]?stock|unavailable|false|no|sold out|0/.test(normalized)) return false;
|
|
5794
|
+
if (/in[_\s-]?stock|available|true|yes/.test(normalized)) return true;
|
|
5795
|
+
const numeric = this.toFiniteNumber(stockValue);
|
|
5796
|
+
if (numeric !== null) return numeric > 0;
|
|
5797
|
+
}
|
|
5798
|
+
if (meta.inStock !== void 0) return Boolean(meta.inStock);
|
|
5799
|
+
if (meta.stock !== void 0) return Boolean(meta.stock);
|
|
5800
|
+
if (meta.available !== void 0) return Boolean(meta.available);
|
|
4776
5801
|
const content = (item.content || "").toLowerCase();
|
|
4777
|
-
if (
|
|
4778
|
-
|
|
5802
|
+
if (/out[_\s-]?of[_\s-]?stock|unavailable|sold out/.test(content)) return false;
|
|
5803
|
+
if (/in[_\s-]?stock|available/.test(content)) return true;
|
|
5804
|
+
return true;
|
|
5805
|
+
}
|
|
5806
|
+
static extractStockQuantity(item) {
|
|
5807
|
+
const meta = item.metadata || {};
|
|
5808
|
+
const stockValue = resolveMetadataValue(meta, "stock");
|
|
5809
|
+
const numericStock = this.toFiniteNumber(stockValue);
|
|
5810
|
+
if (numericStock !== null) return numericStock;
|
|
5811
|
+
const content = item.content || "";
|
|
5812
|
+
const stockMatch = content.match(/\b(?:stock|inventory|quantity|count)\s*[:\-–]?\s*([\d,]+)/i);
|
|
5813
|
+
if (!stockMatch) return null;
|
|
5814
|
+
return this.toFiniteNumber(stockMatch[1]);
|
|
5815
|
+
}
|
|
5816
|
+
static toFiniteNumber(value) {
|
|
5817
|
+
if (typeof value === "number") return Number.isFinite(value) ? value : null;
|
|
5818
|
+
const numeric = Number(String(value != null ? value : "").replace(/[$,% ,]/g, ""));
|
|
5819
|
+
return Number.isFinite(numeric) ? numeric : null;
|
|
5820
|
+
}
|
|
5821
|
+
// ─── Product Extraction ───────────────────────────────────────────────────
|
|
5822
|
+
static getDynamicVal(meta, uiKey, config, trainedSchema) {
|
|
5823
|
+
var _a;
|
|
5824
|
+
if (!meta) return void 0;
|
|
5825
|
+
const mapping = (_a = config == null ? void 0 : config.rag) == null ? void 0 : _a.uiMapping;
|
|
5826
|
+
if ((mapping == null ? void 0 : mapping[uiKey]) && meta[mapping[uiKey]] !== void 0) return meta[mapping[uiKey]];
|
|
5827
|
+
if (trainedSchema && typeof trainedSchema === "object") {
|
|
5828
|
+
const trainedKey = trainedSchema[uiKey];
|
|
5829
|
+
if (trainedKey && meta[trainedKey] !== void 0) return meta[trainedKey];
|
|
4779
5830
|
}
|
|
4780
|
-
|
|
4781
|
-
|
|
5831
|
+
return resolveMetadataValue(meta, uiKey);
|
|
5832
|
+
}
|
|
5833
|
+
static extractProductInfo(item, config, trainedSchema) {
|
|
5834
|
+
var _a;
|
|
5835
|
+
const meta = item.metadata || {};
|
|
5836
|
+
const name = this.getDynamicVal(meta, "name", config, trainedSchema);
|
|
5837
|
+
const price = this.getDynamicVal(meta, "price", config, trainedSchema);
|
|
5838
|
+
const brand = this.getDynamicVal(meta, "brand", config, trainedSchema);
|
|
5839
|
+
const description = this.cleanProductDescription(
|
|
5840
|
+
(_a = this.extractProductDescriptionFromContent(item.content)) != null ? _a : this.getProductDescriptionValue(meta, config, trainedSchema)
|
|
5841
|
+
);
|
|
5842
|
+
if (name || this.isProductData(item)) {
|
|
5843
|
+
let finalName = name ? String(name) : void 0;
|
|
5844
|
+
if (!finalName) {
|
|
5845
|
+
const nameMatch = item.content.match(/(?:Product|Item|Name):\s*([^\n]+)/i);
|
|
5846
|
+
finalName = nameMatch ? nameMatch[1].trim() : item.content.split("\n")[0].substring(0, 60);
|
|
5847
|
+
}
|
|
5848
|
+
let finalPrice = typeof price === "number" || typeof price === "string" ? price : void 0;
|
|
5849
|
+
if (!finalPrice) {
|
|
5850
|
+
const priceMatch = item.content.match(/(?:Price|Cost):\s*\$?([\d,.]+)/i) || item.content.match(/\$\s*([\d,.]+)/);
|
|
5851
|
+
if (priceMatch) finalPrice = priceMatch[1].replace(/,/g, "");
|
|
5852
|
+
}
|
|
5853
|
+
const imageValue = this.getDynamicVal(meta, "image", config, trainedSchema);
|
|
5854
|
+
return {
|
|
5855
|
+
id: item.id,
|
|
5856
|
+
name: finalName,
|
|
5857
|
+
price: finalPrice,
|
|
5858
|
+
image: typeof imageValue === "string" ? imageValue : void 0,
|
|
5859
|
+
brand: brand ? String(brand) : void 0,
|
|
5860
|
+
description,
|
|
5861
|
+
inStock: this.determineStockStatus(item)
|
|
5862
|
+
};
|
|
4782
5863
|
}
|
|
4783
|
-
return
|
|
5864
|
+
return null;
|
|
4784
5865
|
}
|
|
4785
|
-
|
|
4786
|
-
|
|
4787
|
-
|
|
4788
|
-
|
|
4789
|
-
|
|
4790
|
-
|
|
4791
|
-
|
|
4792
|
-
|
|
4793
|
-
|
|
4794
|
-
|
|
4795
|
-
return fieldCount.size > 2;
|
|
5866
|
+
static extractProductDescriptionFromContent(content) {
|
|
5867
|
+
const bodyMatch = content.match(
|
|
5868
|
+
/\bBody\s*\(HTML\)\s*:\s*([\s\S]*?)(?=\s*[,.]?\s+\b(?:Handle|Title|Vendor|Type|Tags|Published|Option\d*\s+Name|Option\d*\s+Value|Option|Variant|Image|SKU|Price|Status|Category)\b\s*:|$)/i
|
|
5869
|
+
);
|
|
5870
|
+
if (bodyMatch == null ? void 0 : bodyMatch[1]) return bodyMatch[1];
|
|
5871
|
+
const descriptionMatch = content.match(
|
|
5872
|
+
/\b(?:Description|Summary|Details|Body|Content)\s*:\s*([\s\S]*?)(?=\s*[,.]?\s+\b(?:Handle|Title|Vendor|Type|Tags|Published|Option\d*\s+Name|Option\d*\s+Value|Option|Variant|Image|SKU|Price|Status|Category)\b\s*:|$)/i
|
|
5873
|
+
);
|
|
5874
|
+
if (descriptionMatch == null ? void 0 : descriptionMatch[1]) return descriptionMatch[1];
|
|
5875
|
+
return null;
|
|
4796
5876
|
}
|
|
4797
|
-
|
|
4798
|
-
|
|
4799
|
-
|
|
4800
|
-
|
|
4801
|
-
|
|
4802
|
-
|
|
4803
|
-
|
|
4804
|
-
|
|
4805
|
-
|
|
4806
|
-
|
|
4807
|
-
|
|
4808
|
-
|
|
4809
|
-
|
|
4810
|
-
|
|
4811
|
-
|
|
4812
|
-
* @param sources - vector DB matches returned by RAG retrieval
|
|
4813
|
-
* @param llm - any ILLMProvider instance (OpenAI, Anthropic, Ollama, Gemini, REST…)
|
|
4814
|
-
* @returns - a validated UITransformationResponse (type + title + description + data)
|
|
4815
|
-
*/
|
|
4816
|
-
static async analyzeAndDecide(query, sources, llm) {
|
|
4817
|
-
try {
|
|
4818
|
-
const context = this.buildContextSummary(sources);
|
|
4819
|
-
const systemPrompt = this.buildVisualizationSystemPrompt();
|
|
4820
|
-
const userPrompt = [
|
|
4821
|
-
`USER QUESTION: ${query}`,
|
|
4822
|
-
"",
|
|
4823
|
-
"RETRIEVED DATA (JSON):",
|
|
4824
|
-
context
|
|
4825
|
-
].join("\n");
|
|
4826
|
-
const rawResponse = await llm.chat(
|
|
4827
|
-
[{ role: "user", content: userPrompt }],
|
|
4828
|
-
"",
|
|
4829
|
-
{ systemPrompt, temperature: 0 }
|
|
5877
|
+
static getProductDescriptionValue(meta, config, trainedSchema) {
|
|
5878
|
+
const mapped = this.getDynamicVal(meta, "description", config, trainedSchema);
|
|
5879
|
+
if (mapped !== void 0 && mapped !== meta.content && mapped !== meta.text) return mapped;
|
|
5880
|
+
const preferredKeys = [
|
|
5881
|
+
"body_html",
|
|
5882
|
+
"body html",
|
|
5883
|
+
"bodyHtml",
|
|
5884
|
+
"description",
|
|
5885
|
+
"summary",
|
|
5886
|
+
"details",
|
|
5887
|
+
"body"
|
|
5888
|
+
];
|
|
5889
|
+
for (const key of preferredKeys) {
|
|
5890
|
+
const match = Object.keys(meta).find(
|
|
5891
|
+
(candidate) => this.normalizeComparableField(candidate) === this.normalizeComparableField(key)
|
|
4830
5892
|
);
|
|
4831
|
-
|
|
4832
|
-
if (parsed) {
|
|
4833
|
-
console.debug("[UITransformer] LLM chose visualization type:", parsed.type);
|
|
4834
|
-
return parsed;
|
|
4835
|
-
}
|
|
4836
|
-
console.warn("[UITransformer] LLM returned unparseable response; falling back to heuristic.");
|
|
4837
|
-
} catch (err) {
|
|
4838
|
-
console.warn("[UITransformer] analyzeAndDecide LLM call failed; falling back to heuristic.", err);
|
|
5893
|
+
if (match && meta[match] !== void 0 && meta[match] !== null) return meta[match];
|
|
4839
5894
|
}
|
|
4840
|
-
return
|
|
5895
|
+
return void 0;
|
|
4841
5896
|
}
|
|
4842
|
-
|
|
4843
|
-
|
|
4844
|
-
|
|
5897
|
+
static cleanProductDescription(raw) {
|
|
5898
|
+
if (raw === null || raw === void 0) return void 0;
|
|
5899
|
+
const extracted = this.extractProductDescriptionFromContent(String(raw));
|
|
5900
|
+
if (extracted && extracted !== String(raw)) return this.cleanProductDescription(extracted);
|
|
5901
|
+
const text = String(raw).replace(/<[^>]*>/g, " ").replace(/ /gi, " ").replace(/&/gi, "&").replace(/"/gi, '"').replace(/'/gi, "'").replace(/\[[^\]]*TYPE[^\]]*\]/gi, " ").replace(/\b(?:Handle|Title|Vendor|Type|Tags|Published|Option\d*\s+Name|Option\d*\s+Value|Option|Variant|Image|SKU|Price|Status|Category)\s*:\s*[^,.\n:]+[,.]?/gi, " ").replace(/\bBody\s*\(HTML\)\s*:\s*/gi, " ").replace(/\b(?:Description|Summary|Details|Body|Content)\s*:\s*/gi, " ").replace(/\s+/g, " ").trim();
|
|
5902
|
+
return text || void 0;
|
|
5903
|
+
}
|
|
5904
|
+
// ─── Visualization Prompt ─────────────────────────────────────────────────
|
|
4845
5905
|
static buildVisualizationSystemPrompt() {
|
|
4846
5906
|
return `You are a data visualization expert embedded in a RAG chat system.
|
|
4847
|
-
You will receive a user question and structured data
|
|
4848
|
-
Your ONLY job is to
|
|
4849
|
-
|
|
5907
|
+
You will receive a user question, a pre-computed INTENT object, and structured data from a vector database.
|
|
5908
|
+
Your ONLY job is to return a single JSON object describing how to visualize the data.
|
|
5909
|
+
The INTENT object is authoritative \u2014 honor it. For example:
|
|
5910
|
+
- If intent.wantsExplicitTable is false, never return type "table".
|
|
5911
|
+
- If intent.isTemporal is true, prefer line_chart.
|
|
5912
|
+
- If intent.visualizationHint is "comparison" or "ranking", prefer bar_chart.
|
|
5913
|
+
- If intent.visualizationHint is "composition", prefer pie_chart.
|
|
5914
|
+
- If intent.recommendedChart is unsupported by the response schema, use the nearest supported type.
|
|
5915
|
+
- Always respond in the language specified by intent.language.
|
|
4850
5916
|
|
|
4851
|
-
Return ONLY a valid JSON object \u2014 no markdown
|
|
5917
|
+
Return ONLY a valid JSON object \u2014 no markdown fences, no prose.
|
|
4852
5918
|
|
|
4853
|
-
The JSON must have this exact shape:
|
|
4854
5919
|
{
|
|
4855
|
-
"type": "bar_chart" | "line_chart" | "pie_chart" | "radar_chart" | "table" | "text",
|
|
4856
|
-
"title": "
|
|
4857
|
-
"description": "
|
|
4858
|
-
"data": <structured data
|
|
5920
|
+
"type": "bar_chart" | "horizontal_bar" | "line_chart" | "pie_chart" | "histogram" | "scatter_plot" | "radar_chart" | "metric_card" | "geo_map" | "table" | "text",
|
|
5921
|
+
"title": "concise descriptive title",
|
|
5922
|
+
"description": "one sentence describing the visualization",
|
|
5923
|
+
"data": <structured data per type below>
|
|
4859
5924
|
}
|
|
4860
5925
|
|
|
4861
|
-
DATA
|
|
4862
|
-
- bar_chart:
|
|
4863
|
-
-
|
|
4864
|
-
-
|
|
4865
|
-
-
|
|
4866
|
-
|
|
4867
|
-
|
|
4868
|
-
|
|
4869
|
-
|
|
4870
|
-
|
|
4871
|
-
|
|
4872
|
-
-
|
|
4873
|
-
- text: { "content": "<prose answer>" }
|
|
4874
|
-
|
|
4875
|
-
DECISION RULES (follow strictly):
|
|
4876
|
-
1. IF the user explicitly asks for a "pie chart", "bar chart", or "table", you MUST output that exact type. Do not deviate from their request.
|
|
4877
|
-
2. bar_chart \u2192 comparing quantities across categories (e.g. sales by region, price by product). Use this when there is only ONE value per category.
|
|
4878
|
-
3. line_chart \u2192 trends or changes over time (dates, months, years, sequential events)
|
|
4879
|
-
4. pie_chart \u2192 proportional breakdown or percentage distribution
|
|
4880
|
-
5. radar_chart \u2192 comparing 2 or more products across MULTIPLE attributes.
|
|
4881
|
-
6. table \u2192 multi-field structured records where each item has \u2265 3 attributes
|
|
4882
|
-
7. text \u2192 conversational or free-form answers where no chart adds value
|
|
5926
|
+
DATA SHAPES:
|
|
5927
|
+
- bar_chart: [{ "category": string, "value": number }]
|
|
5928
|
+
- horizontal_bar: [{ "category": string, "value": number }]
|
|
5929
|
+
- histogram: [{ "category": string, "value": number }]
|
|
5930
|
+
- line_chart: [{ "timestamp": string, "value": number, "label": string }]
|
|
5931
|
+
- pie_chart: [{ "label": string, "value": number }]
|
|
5932
|
+
- scatter_plot:[{ "x": number, "y": number, "label": string }]
|
|
5933
|
+
- radar_chart: [{ "attribute": string, "<series1>": number, "<series2>": number, ... }]
|
|
5934
|
+
- metric_card: { "label": string, "value": number, "operation": "sum"|"average"|"count"|"min"|"max"|"median" }
|
|
5935
|
+
- geo_map: [{ "category": string, "value": number }]
|
|
5936
|
+
- table: { "columns": string[], "rows": (string|number)[][] }
|
|
5937
|
+
- text: { "content": "A concise plain-language answer." }
|
|
4883
5938
|
|
|
4884
|
-
|
|
4885
|
-
|
|
4886
|
-
|
|
4887
|
-
|
|
4888
|
-
|
|
5939
|
+
RULES:
|
|
5940
|
+
1. Aggregate values \u2014 never pass raw nested objects in chart arrays.
|
|
5941
|
+
2. All "value" fields must be numbers.
|
|
5942
|
+
3. Cap bar/line/pie at 12 data points.
|
|
5943
|
+
4. Use dollar signs only for monetary prices, not for stock quantities or years.
|
|
5944
|
+
5. If no relevant data is found, return text: "I cannot answer this question based on the available information."`;
|
|
4889
5945
|
}
|
|
4890
|
-
/**
|
|
4891
|
-
* Serialize retrieved vector matches into a compact JSON context string.
|
|
4892
|
-
* Limits the total character count to avoid exceeding LLM context windows.
|
|
4893
|
-
*/
|
|
4894
5946
|
static buildContextSummary(sources, maxChars = 6e3) {
|
|
4895
5947
|
const items = sources.map((s, i) => {
|
|
4896
5948
|
var _a, _b, _c, _d;
|
|
@@ -5111,27 +6163,38 @@ var Pipeline = class {
|
|
|
5111
6163
|
async initialize() {
|
|
5112
6164
|
var _a;
|
|
5113
6165
|
if (this.initialised) return;
|
|
5114
|
-
const chartInstruction = `You are a helpful
|
|
6166
|
+
const chartInstruction = `You are a helpful assistant. Use the provided context to answer questions accurately.
|
|
5115
6167
|
|
|
5116
|
-
###
|
|
6168
|
+
### CRITICAL RULES:
|
|
6169
|
+
- ONLY answer the user's specific question. Do NOT suggest or recommend unrelated products unless specifically asked.
|
|
5117
6170
|
- NEVER generate markdown tables. If you do, the UI will break.
|
|
5118
6171
|
- NEVER generate HTML tags like <figure>, <tbody>, <tr>, etc.
|
|
5119
6172
|
- NEVER generate text-based charts or graphs.
|
|
6173
|
+
- NEVER say you cannot display, render, draw, or create a chart when the user asks for a visualization. The UI handles visual rendering separately.
|
|
5120
6174
|
- ONLY use plain text and bullet points.
|
|
6175
|
+
- NEVER use the plus sign (+) as a separator between names, categories, or products. Use commas or bullet points.
|
|
6176
|
+
- ONLY answer the question if the sources contain relevant information to answer it, else say that you cannot answer the question.
|
|
6177
|
+
- If answer cannot be found in the sources, say that you cannot answer the question and do not hallucinate.
|
|
6178
|
+
- Do not use information from previous turns to answer the question.
|
|
6179
|
+
- You CAN use numbers for years and counts (e.g., 2006, 5800). But NEVER put a dollar sign ($) before non-monetary numbers like Stock quantities, EAN numbers, or years. Only use dollar signs for actual monetary prices.
|
|
5121
6180
|
|
|
5122
6181
|
### PRODUCT DISPLAY:
|
|
5123
|
-
-
|
|
6182
|
+
- Recommended Products should only be displayed when the user explicitly asks for products.
|
|
6183
|
+
- When recommending products (ONLY when asked), simply list their names, prices, and features in a friendly, conversational manner.
|
|
5124
6184
|
- The UI will automatically detect these products and show high-quality product cards in a carousel below your message.
|
|
6185
|
+
- For product descriptions, summarize customer-facing details only. Do NOT list internal catalog fields such as Handle, Body (HTML), Vendor, Type, Tags, Published, Option, Variant, SKU, or raw metadata labels.
|
|
5125
6186
|
- Do NOT try to format product lists as tables.
|
|
5126
6187
|
`;
|
|
5127
6188
|
this.config.llm.systemPrompt = chartInstruction;
|
|
5128
6189
|
this.vectorDB = await ProviderRegistry.createVectorProvider(this.config.vectorDb);
|
|
5129
|
-
|
|
6190
|
+
this.llmRouter = new LLMRouter(this.config);
|
|
6191
|
+
const { llmProvider: resolvedLLM, embeddingProvider } = await EmbeddingStrategyResolver.resolve(
|
|
5130
6192
|
this.config.llm,
|
|
5131
6193
|
this.config.embedding
|
|
5132
6194
|
);
|
|
5133
|
-
this.llmProvider = llmProvider;
|
|
5134
6195
|
this.embeddingProvider = embeddingProvider;
|
|
6196
|
+
await this.llmRouter.initialize(resolvedLLM);
|
|
6197
|
+
this.llmProvider = this.llmRouter.get("default");
|
|
5135
6198
|
if (this.config.graphDb) {
|
|
5136
6199
|
this.graphDB = await ProviderRegistry.createGraphProvider(this.config.graphDb);
|
|
5137
6200
|
await this.graphDB.initialize();
|
|
@@ -5288,10 +6351,16 @@ var Pipeline = class {
|
|
|
5288
6351
|
/**
|
|
5289
6352
|
* High-performance streaming RAG flow.
|
|
5290
6353
|
* Yields text chunks first, then the retrieval metadata + observability trace at the end.
|
|
6354
|
+
*
|
|
6355
|
+
* Latency optimizations:
|
|
6356
|
+
* - Strategy classification runs in parallel with query embedding (saves ~400ms)
|
|
6357
|
+
* - Hallucination scoring is fire-and-forget (doesn't block metadata yield)
|
|
6358
|
+
* - UITransformation is computed after text streaming and emitted with metadata
|
|
6359
|
+
* - SchemaMapper.train runs while answer generation streams
|
|
5291
6360
|
*/
|
|
5292
6361
|
askStream(_0) {
|
|
5293
6362
|
return __asyncGenerator(this, arguments, function* (question, history = [], namespace) {
|
|
5294
|
-
var _a, _b, _c, _d, _e, _f, _g, _h, _i;
|
|
6363
|
+
var _a, _b, _c, _d, _e, _f, _g, _h, _i, _j, _k, _l, _m;
|
|
5295
6364
|
yield new __await(this.initialize());
|
|
5296
6365
|
const ns = namespace != null ? namespace : this.config.projectId;
|
|
5297
6366
|
const topK = (_b = (_a = this.config.rag) == null ? void 0 : _a.topK) != null ? _b : 5;
|
|
@@ -5307,27 +6376,48 @@ var Pipeline = class {
|
|
|
5307
6376
|
}
|
|
5308
6377
|
const hints = QueryProcessor.extractQueryFieldHints(question, (_f = this.config.rag) == null ? void 0 : _f.filterableFields);
|
|
5309
6378
|
const filter = QueryProcessor.buildQueryFilter(question, hints);
|
|
6379
|
+
const numericPredicates = QueryProcessor.extractNumericPredicates(question, (_g = this.config.rag) == null ? void 0 : _g.filterableFields);
|
|
6380
|
+
if (numericPredicates.length > 0) {
|
|
6381
|
+
filter.__numericPredicates = numericPredicates;
|
|
6382
|
+
}
|
|
5310
6383
|
const embedStart = performance.now();
|
|
5311
|
-
const
|
|
5312
|
-
|
|
5313
|
-
|
|
5314
|
-
|
|
5315
|
-
|
|
6384
|
+
const cacheKey = `${ns}::${searchQuery}`;
|
|
6385
|
+
const cachedVector = this.embeddingCache.get(cacheKey);
|
|
6386
|
+
const [strategyResult, embeddedVector] = yield new __await(Promise.all([
|
|
6387
|
+
QueryProcessor.determineRetrievalStrategy(
|
|
6388
|
+
searchQuery,
|
|
6389
|
+
this.llmRouter.get("fast"),
|
|
6390
|
+
(_h = this.config.rag) == null ? void 0 : _h.graphKeywords,
|
|
6391
|
+
(_i = this.config.rag) == null ? void 0 : _i.vectorKeywords
|
|
6392
|
+
),
|
|
6393
|
+
// Embed immediately regardless of strategy — costs nothing if cached.
|
|
6394
|
+
// If strategy turns out to be 'graph'-only we just won't use the vector.
|
|
6395
|
+
cachedVector ? Promise.resolve(cachedVector) : this.embeddingProvider.embed(searchQuery, { taskType: "query" })
|
|
6396
|
+
]));
|
|
6397
|
+
const queryVector = embeddedVector;
|
|
6398
|
+
if (!cachedVector && queryVector.length > 0) {
|
|
6399
|
+
this.embeddingCache.set(cacheKey, queryVector);
|
|
6400
|
+
}
|
|
6401
|
+
const graphData = (strategyResult === "graph" || strategyResult === "both") && this.graphDB && ((_j = this.config.rag) == null ? void 0 : _j.useGraphRetrieval) ? yield new __await(this.graphDB.query(searchQuery)) : void 0;
|
|
6402
|
+
const hasNumericPredicates = Array.isArray(filter.__numericPredicates) && filter.__numericPredicates.length > 0;
|
|
6403
|
+
const wantsExhaustiveList = hasNumericPredicates || /\b(list|all|show|provide|give|get)\b/i.test(question);
|
|
6404
|
+
const retrievalLimit = wantsExhaustiveList ? Math.max(topK * 20, 100) : topK * 2;
|
|
6405
|
+
const rawSources = (strategyResult === "vector" || strategyResult === "both") && queryVector && queryVector.length > 0 ? yield new __await(this.vectorDB.query(queryVector, retrievalLimit, ns, filter)) : [];
|
|
5316
6406
|
const retrieveEnd = performance.now();
|
|
5317
6407
|
const embedMs = retrieveEnd - embedStart;
|
|
5318
6408
|
const retrieveMs = retrieveEnd - embedStart;
|
|
5319
6409
|
const rerankStart = performance.now();
|
|
5320
|
-
|
|
5321
|
-
|
|
6410
|
+
const structuredSources = this.applyStructuredFilters(rawSources, filter);
|
|
6411
|
+
let sources = hasNumericPredicates ? structuredSources : structuredSources.filter((m) => m.score >= scoreThreshold);
|
|
6412
|
+
if (!hasNumericPredicates && ((_k = this.config.rag) == null ? void 0 : _k.useReranking)) {
|
|
5322
6413
|
sources = yield new __await(this.reranker.rerank(sources, question, topK));
|
|
5323
|
-
} else {
|
|
6414
|
+
} else if (!wantsExhaustiveList) {
|
|
5324
6415
|
sources = sources.slice(0, topK);
|
|
5325
6416
|
}
|
|
5326
6417
|
const rerankMs = performance.now() - rerankStart;
|
|
5327
6418
|
let context = sources.length ? sources.map((m, i) => `[Source ${i + 1}]
|
|
5328
6419
|
${m.content}`).join("\n\n---\n\n") : "No relevant context found.";
|
|
5329
6420
|
if (graphData && graphData.nodes.length > 0) {
|
|
5330
|
-
console.log(`[Graph Retrieval] Found ${graphData.nodes.length} relevant entities.`);
|
|
5331
6421
|
const graphContext = graphData.nodes.map(
|
|
5332
6422
|
(n) => `Entity: ${n.label} (${n.id})${n.properties ? " - " + JSON.stringify(n.properties) : ""}`
|
|
5333
6423
|
).join("\n");
|
|
@@ -5338,20 +6428,18 @@ VECTOR CONTEXT:
|
|
|
5338
6428
|
${context}`;
|
|
5339
6429
|
}
|
|
5340
6430
|
const allMetadataKeys = Array.from(new Set(sources.flatMap((s) => Object.keys(s.metadata || {}))));
|
|
5341
|
-
const
|
|
5342
|
-
const restrictionSuffix = "\n\n(IMPORTANT: Use plain text only. NEVER generate tables, HTML figures, or text charts. If listing products, use simple bullet points.)";
|
|
6431
|
+
const trainedSchemaPromise = allMetadataKeys.length > 0 ? SchemaMapper.train(this.llmProvider, ns, allMetadataKeys).catch(() => void 0) : Promise.resolve(void 0);
|
|
6432
|
+
const restrictionSuffix = "\n\n(IMPORTANT: Use plain text only. NEVER generate tables, HTML figures, or text charts. The UI renders requested charts separately, so NEVER say you cannot create, display, draw, render, or provide a chart/visualization. If listing products, use simple bullet points or comma-separated names. NEVER use plus signs (+) to separate product names, category names, or list items. For product description/detail questions, provide customer-facing prose only and do NOT list internal catalog fields such as Handle, Body (HTML), Vendor, Type, Tags, Published, Option, Variant, SKU, or raw metadata labels. You CAN use numbers for years/counts like 2006 or 5800, but NEVER put a dollar sign ($) before them.)";
|
|
5343
6433
|
const hardenedHistory = [...history];
|
|
5344
6434
|
const userQuestion = { role: "user", content: question + restrictionSuffix };
|
|
5345
6435
|
const messages = [...hardenedHistory, userQuestion];
|
|
5346
|
-
const systemPrompt = (
|
|
6436
|
+
const systemPrompt = (_l = this.config.llm.systemPrompt) != null ? _l : "";
|
|
5347
6437
|
const userPrompt = messages.map((m) => `${m.role}: ${m.content}`).join("\n");
|
|
5348
6438
|
let fullReply = "";
|
|
5349
6439
|
const generateStart = performance.now();
|
|
5350
6440
|
if (this.llmProvider.chatStream) {
|
|
5351
6441
|
const stream = this.llmProvider.chatStream(messages, context);
|
|
5352
|
-
if (!stream) {
|
|
5353
|
-
throw new Error(`[Pipeline] ${this.config.llm.provider} chatStream returned undefined`);
|
|
5354
|
-
}
|
|
6442
|
+
if (!stream) throw new Error(`[Pipeline] ${this.config.llm.provider} chatStream returned undefined`);
|
|
5355
6443
|
try {
|
|
5356
6444
|
for (var iter = __forAwait(stream), more, temp, error; more = !(temp = yield new __await(iter.next())).done; more = false) {
|
|
5357
6445
|
const chunk = temp.value;
|
|
@@ -5378,7 +6466,7 @@ ${context}`;
|
|
|
5378
6466
|
const latency = {
|
|
5379
6467
|
embedMs: Math.round(embedMs),
|
|
5380
6468
|
retrieveMs: Math.round(retrieveMs),
|
|
5381
|
-
rerankMs: ((
|
|
6469
|
+
rerankMs: ((_m = this.config.rag) == null ? void 0 : _m.useReranking) ? Math.round(rerankMs) : void 0,
|
|
5382
6470
|
generateMs: Math.round(generateMs),
|
|
5383
6471
|
totalMs: Math.round(totalMs)
|
|
5384
6472
|
};
|
|
@@ -5391,9 +6479,6 @@ ${context}`;
|
|
|
5391
6479
|
totalTokens: promptTokens + completionTokens,
|
|
5392
6480
|
estimatedCostUsd: estimateCostUsd(promptTokens, completionTokens, this.config.llm.model)
|
|
5393
6481
|
};
|
|
5394
|
-
const hallucinationResult = yield new __await(scoreHallucination(this.llmProvider, fullReply, context).catch(() => void 0));
|
|
5395
|
-
const trainedSchema = yield new __await(trainingPromise);
|
|
5396
|
-
const uiTransformation = yield new __await(this.generateUiTransformation(question, sources, trainedSchema));
|
|
5397
6482
|
const trace = {
|
|
5398
6483
|
requestId,
|
|
5399
6484
|
query: question,
|
|
@@ -5412,10 +6497,21 @@ ${context}`;
|
|
|
5412
6497
|
}),
|
|
5413
6498
|
latency,
|
|
5414
6499
|
tokens,
|
|
5415
|
-
hallucinationScore: hallucinationResult == null ? void 0 : hallucinationResult.score,
|
|
5416
|
-
hallucinationReason: hallucinationResult == null ? void 0 : hallucinationResult.reason,
|
|
5417
6500
|
timestamp: (/* @__PURE__ */ new Date()).toISOString()
|
|
5418
6501
|
};
|
|
6502
|
+
let uiTransformation;
|
|
6503
|
+
try {
|
|
6504
|
+
const trainedSchema = yield new __await(trainedSchemaPromise);
|
|
6505
|
+
uiTransformation = yield new __await(this.generateUiTransformation(
|
|
6506
|
+
question,
|
|
6507
|
+
sources,
|
|
6508
|
+
trainedSchema,
|
|
6509
|
+
hasNumericPredicates
|
|
6510
|
+
));
|
|
6511
|
+
} catch (uiError) {
|
|
6512
|
+
console.warn("[Pipeline] UI transformation failed before metadata yield:", uiError);
|
|
6513
|
+
uiTransformation = UITransformer.transform(question, sources, this.config);
|
|
6514
|
+
}
|
|
5419
6515
|
yield {
|
|
5420
6516
|
reply: "",
|
|
5421
6517
|
sources,
|
|
@@ -5423,6 +6519,13 @@ ${context}`;
|
|
|
5423
6519
|
ui_transformation: uiTransformation,
|
|
5424
6520
|
trace
|
|
5425
6521
|
};
|
|
6522
|
+
scoreHallucination(this.llmProvider, fullReply, context).then((hallucinationResult) => {
|
|
6523
|
+
if (hallucinationResult) {
|
|
6524
|
+
trace.hallucinationScore = hallucinationResult.score;
|
|
6525
|
+
trace.hallucinationReason = hallucinationResult.reason;
|
|
6526
|
+
}
|
|
6527
|
+
}).catch(() => {
|
|
6528
|
+
});
|
|
5426
6529
|
} catch (error2) {
|
|
5427
6530
|
throw new Error(`[Pipeline] Stream failed: ${error2 instanceof Error ? error2.message : String(error2)}`);
|
|
5428
6531
|
}
|
|
@@ -5432,24 +6535,107 @@ ${context}`;
|
|
|
5432
6535
|
* Universal retrieval method combining all enabled providers.
|
|
5433
6536
|
* Uses an LRU-bounded embedding cache to avoid re-embedding the same query.
|
|
5434
6537
|
*/
|
|
5435
|
-
async generateUiTransformation(question, sources, trainedSchema) {
|
|
6538
|
+
async generateUiTransformation(question, sources, trainedSchema, forceDeterministic = false) {
|
|
5436
6539
|
if (!sources || sources.length === 0) {
|
|
5437
6540
|
return UITransformer.transform(question, sources, this.config, trainedSchema);
|
|
5438
6541
|
}
|
|
6542
|
+
if (forceDeterministic) {
|
|
6543
|
+
return UITransformer.transform(question, sources, this.config, trainedSchema);
|
|
6544
|
+
}
|
|
5439
6545
|
try {
|
|
5440
|
-
return await UITransformer.analyzeAndDecide(question, sources, this.
|
|
6546
|
+
return await UITransformer.analyzeAndDecide(question, sources, this.llmRouter.get("fast"));
|
|
5441
6547
|
} catch (err) {
|
|
5442
6548
|
console.warn("[Pipeline] generateUiTransformation failed, using heuristic fallback:", err);
|
|
5443
6549
|
return UITransformer.transform(question, sources, this.config, trainedSchema);
|
|
5444
6550
|
}
|
|
5445
6551
|
}
|
|
6552
|
+
applyStructuredFilters(sources, filter) {
|
|
6553
|
+
const predicates = Array.isArray(filter.__numericPredicates) ? filter.__numericPredicates : [];
|
|
6554
|
+
if (predicates.length === 0) return sources;
|
|
6555
|
+
return sources.filter((source) => predicates.every((predicate) => {
|
|
6556
|
+
const value = this.resolveNumericPredicateValue(source, predicate);
|
|
6557
|
+
return value !== null && this.matchesNumericPredicate(value, predicate);
|
|
6558
|
+
})).sort((a, b) => {
|
|
6559
|
+
var _a, _b;
|
|
6560
|
+
const primary = predicates[0];
|
|
6561
|
+
const aValue = (_a = this.resolveNumericPredicateValue(a, primary)) != null ? _a : 0;
|
|
6562
|
+
const bValue = (_b = this.resolveNumericPredicateValue(b, primary)) != null ? _b : 0;
|
|
6563
|
+
return primary.operator === "lt" || primary.operator === "lte" ? aValue - bValue : bValue - aValue;
|
|
6564
|
+
});
|
|
6565
|
+
}
|
|
6566
|
+
resolveNumericPredicateValue(source, predicate) {
|
|
6567
|
+
const meta = source.metadata || {};
|
|
6568
|
+
const field = predicate.field;
|
|
6569
|
+
const entries = Object.entries(meta).filter(([, value]) => value !== null && value !== void 0 && typeof value !== "object");
|
|
6570
|
+
if (field) {
|
|
6571
|
+
const normalizedField = this.normalizeComparableField(field);
|
|
6572
|
+
const exact = entries.find(([key]) => this.normalizeComparableField(key) === normalizedField);
|
|
6573
|
+
const fuzzy = exact != null ? exact : entries.map(([key, value]) => ({ key, value, score: this.fieldSimilarityScore(key, field) })).filter((candidate) => candidate.score > 0).sort((a, b) => b.score - a.score)[0];
|
|
6574
|
+
if (fuzzy) {
|
|
6575
|
+
const value = this.toFiniteNumber(Array.isArray(fuzzy) ? fuzzy[1] : fuzzy.value);
|
|
6576
|
+
if (value !== null) return value;
|
|
6577
|
+
}
|
|
6578
|
+
const contentValue = this.extractNumericValueFromContent(source.content, field);
|
|
6579
|
+
if (contentValue !== null) return contentValue;
|
|
6580
|
+
}
|
|
6581
|
+
for (const [key, value] of entries) {
|
|
6582
|
+
if (/(id|sku|ean|upc|phone|zip|postal|code)/i.test(key)) continue;
|
|
6583
|
+
const numeric = this.toFiniteNumber(value);
|
|
6584
|
+
if (numeric !== null) return numeric;
|
|
6585
|
+
}
|
|
6586
|
+
return null;
|
|
6587
|
+
}
|
|
6588
|
+
extractNumericValueFromContent(content, field) {
|
|
6589
|
+
const escapedWords = field.split(/\s+|_+|-+/).filter(Boolean).map((word) => word.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"));
|
|
6590
|
+
if (escapedWords.length === 0) return null;
|
|
6591
|
+
const pattern = new RegExp(`(?:${escapedWords.join("[\\\\s_:-]*")})\\s*[:\\-\u2013]?\\s*([\\d,]+(?:\\.\\d+)?)`, "i");
|
|
6592
|
+
const match = content.match(pattern);
|
|
6593
|
+
return match ? this.toFiniteNumber(match[1]) : null;
|
|
6594
|
+
}
|
|
6595
|
+
matchesNumericPredicate(value, predicate) {
|
|
6596
|
+
switch (predicate.operator) {
|
|
6597
|
+
case "gt":
|
|
6598
|
+
return value > predicate.value;
|
|
6599
|
+
case "gte":
|
|
6600
|
+
return value >= predicate.value;
|
|
6601
|
+
case "lt":
|
|
6602
|
+
return value < predicate.value;
|
|
6603
|
+
case "lte":
|
|
6604
|
+
return value <= predicate.value;
|
|
6605
|
+
case "eq":
|
|
6606
|
+
default:
|
|
6607
|
+
return value === predicate.value;
|
|
6608
|
+
}
|
|
6609
|
+
}
|
|
6610
|
+
normalizeComparableField(value) {
|
|
6611
|
+
return value.toLowerCase().replace(/[^a-z0-9]/g, "");
|
|
6612
|
+
}
|
|
6613
|
+
fieldSimilarityScore(candidate, requested) {
|
|
6614
|
+
const normalizedCandidate = this.normalizeComparableField(candidate);
|
|
6615
|
+
const normalizedRequested = this.normalizeComparableField(requested);
|
|
6616
|
+
if (normalizedCandidate === normalizedRequested) return 10;
|
|
6617
|
+
if (normalizedCandidate.includes(normalizedRequested) || normalizedRequested.includes(normalizedCandidate)) return 8;
|
|
6618
|
+
const candidateTokens = this.fieldTokens(candidate);
|
|
6619
|
+
const requestedTokens = this.fieldTokens(requested);
|
|
6620
|
+
const overlap = requestedTokens.filter((token) => candidateTokens.includes(token)).length;
|
|
6621
|
+
if (overlap === 0) return 0;
|
|
6622
|
+
return overlap / Math.max(requestedTokens.length, candidateTokens.length);
|
|
6623
|
+
}
|
|
6624
|
+
fieldTokens(value) {
|
|
6625
|
+
return value.toLowerCase().split(/[^a-z0-9]+/).map((token) => token.replace(/ies$/, "y").replace(/s$/, "")).filter((token) => token.length > 1);
|
|
6626
|
+
}
|
|
6627
|
+
toFiniteNumber(value) {
|
|
6628
|
+
if (typeof value === "number") return Number.isFinite(value) ? value : null;
|
|
6629
|
+
const numeric = Number(String(value != null ? value : "").replace(/[$,% ,]/g, ""));
|
|
6630
|
+
return Number.isFinite(numeric) ? numeric : null;
|
|
6631
|
+
}
|
|
5446
6632
|
async retrieve(query, options) {
|
|
5447
|
-
var _a, _b, _c;
|
|
6633
|
+
var _a, _b, _c, _d, _e, _f;
|
|
5448
6634
|
const ns = (_a = options.namespace) != null ? _a : this.config.projectId;
|
|
5449
6635
|
const topK = (_b = options.topK) != null ? _b : 5;
|
|
5450
6636
|
const cacheKey = `${ns}::${query}`;
|
|
5451
6637
|
let queryVector = this.embeddingCache.get(cacheKey);
|
|
5452
|
-
const strategy = await QueryProcessor.determineRetrievalStrategy(query, this.
|
|
6638
|
+
const strategy = await QueryProcessor.determineRetrievalStrategy(query, this.llmRouter.get("fast"));
|
|
5453
6639
|
console.debug(`[Pipeline] Determined retrieval strategy: ${strategy}`);
|
|
5454
6640
|
const [retrievedVector, graphData] = await Promise.all([
|
|
5455
6641
|
// Only embed if we need vector search (strategy is 'vector' or 'both')
|
|
@@ -5461,8 +6647,27 @@ ${context}`;
|
|
|
5461
6647
|
this.embeddingCache.set(cacheKey, retrievedVector);
|
|
5462
6648
|
queryVector = retrievedVector;
|
|
5463
6649
|
}
|
|
5464
|
-
const
|
|
5465
|
-
|
|
6650
|
+
const baseFilter = __spreadProps(__spreadValues({}, (_d = options.filter) != null ? _d : {}), { queryText: query });
|
|
6651
|
+
const numericPredicates = QueryProcessor.extractNumericPredicates(query, (_e = this.config.rag) == null ? void 0 : _e.filterableFields);
|
|
6652
|
+
if (numericPredicates.length > 0) {
|
|
6653
|
+
baseFilter.__numericPredicates = numericPredicates;
|
|
6654
|
+
}
|
|
6655
|
+
const retrievalLimit = numericPredicates.length > 0 ? Math.max(topK * 20, 100) : topK;
|
|
6656
|
+
const sources = (strategy === "vector" || strategy === "both") && queryVector && queryVector.length > 0 ? this.applyStructuredFilters(
|
|
6657
|
+
await this.vectorDB.query(queryVector, retrievalLimit, ns, baseFilter),
|
|
6658
|
+
baseFilter
|
|
6659
|
+
) : [];
|
|
6660
|
+
const resolvedSources = [];
|
|
6661
|
+
for (const source of sources) {
|
|
6662
|
+
const parentId = (_f = source.metadata) == null ? void 0 : _f.parent_id;
|
|
6663
|
+
if (parentId) {
|
|
6664
|
+
console.log(`[Pipeline] Multi-Vector: Found child chunk. Parent ID: ${parentId}`);
|
|
6665
|
+
resolvedSources.push(source);
|
|
6666
|
+
} else {
|
|
6667
|
+
resolvedSources.push(source);
|
|
6668
|
+
}
|
|
6669
|
+
}
|
|
6670
|
+
return { sources: resolvedSources, graphData };
|
|
5466
6671
|
}
|
|
5467
6672
|
/** Rewrite the user query for better retrieval performance. */
|
|
5468
6673
|
async rewriteQuery(question, history) {
|
|
@@ -5954,209 +7159,7 @@ var DocumentParser = class {
|
|
|
5954
7159
|
// src/server.ts
|
|
5955
7160
|
init_BaseVectorProvider();
|
|
5956
7161
|
init_PineconeProvider();
|
|
5957
|
-
|
|
5958
|
-
// src/providers/vectordb/PostgreSQLProvider.ts
|
|
5959
|
-
var import_pg2 = require("pg");
|
|
5960
|
-
init_BaseVectorProvider();
|
|
5961
|
-
var PostgreSQLProvider = class extends BaseVectorProvider {
|
|
5962
|
-
constructor(config) {
|
|
5963
|
-
var _a;
|
|
5964
|
-
super(config);
|
|
5965
|
-
this.tableName = this.indexName.replace(/[^a-z0-9_]/gi, "_");
|
|
5966
|
-
const opts = config.options;
|
|
5967
|
-
if (!opts.connectionString) throw new Error("[PostgreSQLProvider] options.connectionString is required");
|
|
5968
|
-
this.connectionString = opts.connectionString;
|
|
5969
|
-
this.dimensions = (_a = opts.dimensions) != null ? _a : 1536;
|
|
5970
|
-
}
|
|
5971
|
-
static getValidator() {
|
|
5972
|
-
return {
|
|
5973
|
-
validate(config) {
|
|
5974
|
-
const errors = [];
|
|
5975
|
-
const opts = config.options || {};
|
|
5976
|
-
if (!opts.connectionString) {
|
|
5977
|
-
errors.push({
|
|
5978
|
-
field: "vectorDb.options.connectionString",
|
|
5979
|
-
message: "PostgreSQL connection string is required",
|
|
5980
|
-
suggestion: "Set PGVECTOR_CONNECTION_STRING environment variable",
|
|
5981
|
-
severity: "error"
|
|
5982
|
-
});
|
|
5983
|
-
}
|
|
5984
|
-
if (opts.tables && typeof opts.tables !== "string" && !Array.isArray(opts.tables)) {
|
|
5985
|
-
errors.push({
|
|
5986
|
-
field: "vectorDb.options.tables",
|
|
5987
|
-
message: "PostgreSQL tables must be a string or a string array",
|
|
5988
|
-
severity: "error"
|
|
5989
|
-
});
|
|
5990
|
-
}
|
|
5991
|
-
return errors;
|
|
5992
|
-
}
|
|
5993
|
-
};
|
|
5994
|
-
}
|
|
5995
|
-
static getHealthChecker() {
|
|
5996
|
-
return {
|
|
5997
|
-
async check(config) {
|
|
5998
|
-
const opts = config.options || {};
|
|
5999
|
-
const timestamp = Date.now();
|
|
6000
|
-
try {
|
|
6001
|
-
const { Client } = await import("pg");
|
|
6002
|
-
const client = new Client({ connectionString: opts.connectionString });
|
|
6003
|
-
await client.connect();
|
|
6004
|
-
const result = await client.query(`
|
|
6005
|
-
SELECT EXISTS(SELECT 1 FROM pg_extension WHERE extname = 'vector');
|
|
6006
|
-
`);
|
|
6007
|
-
const hasVector = result.rows[0].exists;
|
|
6008
|
-
await client.end();
|
|
6009
|
-
return {
|
|
6010
|
-
healthy: true,
|
|
6011
|
-
provider: "postgresql",
|
|
6012
|
-
capabilities: { pgvectorInstalled: hasVector },
|
|
6013
|
-
timestamp
|
|
6014
|
-
};
|
|
6015
|
-
} catch (error) {
|
|
6016
|
-
return {
|
|
6017
|
-
healthy: false,
|
|
6018
|
-
provider: "postgresql",
|
|
6019
|
-
error: `Connection failed: ${error instanceof Error ? error.message : String(error)}`,
|
|
6020
|
-
timestamp
|
|
6021
|
-
};
|
|
6022
|
-
}
|
|
6023
|
-
}
|
|
6024
|
-
};
|
|
6025
|
-
}
|
|
6026
|
-
async initialize() {
|
|
6027
|
-
this.pool = new import_pg2.Pool({ connectionString: this.connectionString });
|
|
6028
|
-
const client = await this.pool.connect();
|
|
6029
|
-
try {
|
|
6030
|
-
await client.query("CREATE EXTENSION IF NOT EXISTS vector");
|
|
6031
|
-
await client.query(`
|
|
6032
|
-
CREATE TABLE IF NOT EXISTS ${this.tableName} (
|
|
6033
|
-
id TEXT PRIMARY KEY,
|
|
6034
|
-
namespace TEXT NOT NULL DEFAULT '',
|
|
6035
|
-
content TEXT NOT NULL,
|
|
6036
|
-
metadata JSONB,
|
|
6037
|
-
embedding VECTOR(${this.dimensions})
|
|
6038
|
-
)
|
|
6039
|
-
`);
|
|
6040
|
-
const metric = this.config.distanceMetric || "cosine";
|
|
6041
|
-
const indexOps = metric === "euclidean" ? "vector_l2_ops" : metric === "dotproduct" ? "vector_ip_ops" : "vector_cosine_ops";
|
|
6042
|
-
await client.query(`
|
|
6043
|
-
CREATE INDEX IF NOT EXISTS ${this.tableName}_embedding_idx
|
|
6044
|
-
ON ${this.tableName}
|
|
6045
|
-
USING hnsw (embedding ${indexOps})
|
|
6046
|
-
`);
|
|
6047
|
-
} finally {
|
|
6048
|
-
client.release();
|
|
6049
|
-
}
|
|
6050
|
-
}
|
|
6051
|
-
async upsert(doc, namespace = "") {
|
|
6052
|
-
var _a;
|
|
6053
|
-
const vectorLiteral = `[${doc.vector.join(",")}]`;
|
|
6054
|
-
await this.pool.query(
|
|
6055
|
-
`INSERT INTO ${this.tableName} (id, namespace, content, metadata, embedding)
|
|
6056
|
-
VALUES ($1, $2, $3, $4, $5::vector)
|
|
6057
|
-
ON CONFLICT (id) DO UPDATE
|
|
6058
|
-
SET namespace = EXCLUDED.namespace,
|
|
6059
|
-
content = EXCLUDED.content,
|
|
6060
|
-
metadata = EXCLUDED.metadata,
|
|
6061
|
-
embedding = EXCLUDED.embedding`,
|
|
6062
|
-
[doc.id, namespace, doc.content, JSON.stringify((_a = doc.metadata) != null ? _a : {}), vectorLiteral]
|
|
6063
|
-
);
|
|
6064
|
-
}
|
|
6065
|
-
async batchUpsert(docs, namespace = "") {
|
|
6066
|
-
if (docs.length === 0) return;
|
|
6067
|
-
const client = await this.pool.connect();
|
|
6068
|
-
try {
|
|
6069
|
-
await client.query("BEGIN");
|
|
6070
|
-
const BATCH_SIZE = 50;
|
|
6071
|
-
for (let i = 0; i < docs.length; i += BATCH_SIZE) {
|
|
6072
|
-
const batch = docs.slice(i, i + BATCH_SIZE);
|
|
6073
|
-
const values = [];
|
|
6074
|
-
const valuePlaceholders = batch.map((doc, idx) => {
|
|
6075
|
-
var _a;
|
|
6076
|
-
const offset = idx * 5;
|
|
6077
|
-
values.push(doc.id, namespace, doc.content, JSON.stringify((_a = doc.metadata) != null ? _a : {}), `[${doc.vector.join(",")}]`);
|
|
6078
|
-
return `($${offset + 1}, $${offset + 2}, $${offset + 3}, $${offset + 4}, $${offset + 5}::vector)`;
|
|
6079
|
-
}).join(", ");
|
|
6080
|
-
const query = `
|
|
6081
|
-
INSERT INTO ${this.tableName} (id, namespace, content, metadata, embedding)
|
|
6082
|
-
VALUES ${valuePlaceholders}
|
|
6083
|
-
ON CONFLICT (id) DO UPDATE
|
|
6084
|
-
SET namespace = EXCLUDED.namespace,
|
|
6085
|
-
content = EXCLUDED.content,
|
|
6086
|
-
metadata = EXCLUDED.metadata,
|
|
6087
|
-
embedding = EXCLUDED.embedding
|
|
6088
|
-
`;
|
|
6089
|
-
await client.query(query, values);
|
|
6090
|
-
}
|
|
6091
|
-
await client.query("COMMIT");
|
|
6092
|
-
} catch (error) {
|
|
6093
|
-
await client.query("ROLLBACK");
|
|
6094
|
-
throw error;
|
|
6095
|
-
} finally {
|
|
6096
|
-
client.release();
|
|
6097
|
-
}
|
|
6098
|
-
}
|
|
6099
|
-
async query(vector, topK, namespace, filter) {
|
|
6100
|
-
const vectorLiteral = `[${vector.join(",")}]`;
|
|
6101
|
-
let whereClause = namespace ? `WHERE namespace = $3` : "";
|
|
6102
|
-
const params = [vectorLiteral, topK];
|
|
6103
|
-
if (namespace) params.push(namespace);
|
|
6104
|
-
const publicFilter = this.sanitizeFilter(filter);
|
|
6105
|
-
if (Object.keys(publicFilter).length > 0) {
|
|
6106
|
-
const filterConditions = Object.entries(publicFilter).map(([key, val]) => {
|
|
6107
|
-
const paramIdx = params.length + 1;
|
|
6108
|
-
params.push(JSON.stringify(val));
|
|
6109
|
-
return `metadata->>'${key}' = $${paramIdx}`;
|
|
6110
|
-
}).join(" AND ");
|
|
6111
|
-
whereClause = whereClause ? `${whereClause} AND ${filterConditions}` : `WHERE ${filterConditions}`;
|
|
6112
|
-
}
|
|
6113
|
-
const client = await this.pool.connect();
|
|
6114
|
-
try {
|
|
6115
|
-
const efSearch = this.config.options.efSearch || Math.max(topK * 10, 40);
|
|
6116
|
-
await client.query(`SET LOCAL hnsw.ef_search = ${efSearch}`);
|
|
6117
|
-
const metric = this.config.distanceMetric || "cosine";
|
|
6118
|
-
const queryOp = metric === "euclidean" ? "<->" : metric === "dotproduct" ? "<#>" : "<=>";
|
|
6119
|
-
const scoreExpr = metric === "euclidean" ? `1 / (1 + (embedding <-> $1::vector))` : metric === "dotproduct" ? `(embedding <#> $1::vector) * -1` : `1 - (embedding <=> $1::vector)`;
|
|
6120
|
-
const result = await client.query(
|
|
6121
|
-
`SELECT id, content, metadata, ${scoreExpr} AS score
|
|
6122
|
-
FROM ${this.tableName}
|
|
6123
|
-
${whereClause}
|
|
6124
|
-
ORDER BY embedding ${queryOp} $1::vector
|
|
6125
|
-
LIMIT $2`,
|
|
6126
|
-
params
|
|
6127
|
-
);
|
|
6128
|
-
return result.rows.map((row) => ({
|
|
6129
|
-
id: String(row["id"]),
|
|
6130
|
-
score: parseFloat(String(row["score"])),
|
|
6131
|
-
content: String(row["content"]),
|
|
6132
|
-
metadata: row["metadata"]
|
|
6133
|
-
}));
|
|
6134
|
-
} finally {
|
|
6135
|
-
client.release();
|
|
6136
|
-
}
|
|
6137
|
-
}
|
|
6138
|
-
async delete(id, namespace) {
|
|
6139
|
-
const where = namespace ? "WHERE id = $1 AND namespace = $2" : "WHERE id = $1";
|
|
6140
|
-
const params = namespace ? [id, namespace] : [id];
|
|
6141
|
-
await this.pool.query(`DELETE FROM ${this.tableName} ${where}`, params);
|
|
6142
|
-
}
|
|
6143
|
-
async deleteNamespace(namespace) {
|
|
6144
|
-
await this.pool.query(`DELETE FROM ${this.tableName} WHERE namespace = $1`, [namespace]);
|
|
6145
|
-
}
|
|
6146
|
-
async ping() {
|
|
6147
|
-
try {
|
|
6148
|
-
await this.pool.query("SELECT 1");
|
|
6149
|
-
return true;
|
|
6150
|
-
} catch (e) {
|
|
6151
|
-
return false;
|
|
6152
|
-
}
|
|
6153
|
-
}
|
|
6154
|
-
async disconnect() {
|
|
6155
|
-
await this.pool.end();
|
|
6156
|
-
}
|
|
6157
|
-
};
|
|
6158
|
-
|
|
6159
|
-
// src/server.ts
|
|
7162
|
+
init_PostgreSQLProvider();
|
|
6160
7163
|
init_MultiTablePostgresProvider();
|
|
6161
7164
|
init_MongoDBProvider();
|
|
6162
7165
|
init_MilvusProvider();
|
|
@@ -6244,7 +7247,7 @@ function createStreamHandler(configOrPlugin) {
|
|
|
6244
7247
|
const encoder = new TextEncoder();
|
|
6245
7248
|
const stream = new ReadableStream({
|
|
6246
7249
|
async start(controller) {
|
|
6247
|
-
var _a
|
|
7250
|
+
var _a;
|
|
6248
7251
|
const enqueue = (text) => controller.enqueue(encoder.encode(text));
|
|
6249
7252
|
try {
|
|
6250
7253
|
const pipelineStream = plugin.chatStream(message, history, namespace);
|
|
@@ -6262,8 +7265,7 @@ function createStreamHandler(configOrPlugin) {
|
|
|
6262
7265
|
}
|
|
6263
7266
|
if (sources.length > 0) {
|
|
6264
7267
|
try {
|
|
6265
|
-
const
|
|
6266
|
-
const uiTransformation = (_b = responseChunk == null ? void 0 : responseChunk.ui_transformation) != null ? _b : llmProvider ? await UITransformer.analyzeAndDecide(message, sources, llmProvider) : UITransformer.transform(message, sources, plugin.getConfig());
|
|
7268
|
+
const uiTransformation = (_a = responseChunk == null ? void 0 : responseChunk.ui_transformation) != null ? _a : UITransformer.transform(message, sources, plugin.getConfig());
|
|
6267
7269
|
if (uiTransformation) {
|
|
6268
7270
|
enqueue(sseUIFrame(uiTransformation));
|
|
6269
7271
|
}
|