@retrivora-ai/rag-engine 1.8.9 → 1.9.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{ILLMProvider-CbUtvWAW.d.mts → ILLMProvider-BQyKZLnd.d.mts} +10 -2
- package/dist/{ILLMProvider-CbUtvWAW.d.ts → ILLMProvider-BQyKZLnd.d.ts} +10 -2
- package/dist/handlers/index.d.mts +2 -2
- package/dist/handlers/index.d.ts +2 -2
- package/dist/handlers/index.js +1726 -522
- package/dist/handlers/index.mjs +1725 -521
- package/dist/{index-DgzcnqZs.d.ts → index-A0GqPsaG.d.ts} +1 -1
- package/dist/{index-c_y_5qdw.d.mts → index-CDftK3qs.d.mts} +1 -1
- package/dist/index.css +26 -0
- package/dist/index.d.mts +2 -2
- package/dist/index.d.ts +2 -2
- package/dist/index.js +246 -76
- package/dist/index.mjs +248 -76
- package/dist/server.d.mts +32 -5
- package/dist/server.d.ts +32 -5
- package/dist/server.js +1735 -733
- package/dist/server.mjs +1733 -731
- package/package.json +1 -1
- package/src/components/MarkdownComponents.tsx +3 -3
- package/src/components/MessageBubble.tsx +49 -2
- package/src/components/ProductCard.tsx +33 -6
- package/src/components/UIDispatcher.tsx +1 -0
- package/src/components/VisualizationRenderer.tsx +143 -11
- package/src/config/EmbeddingStrategy.ts +5 -4
- package/src/config/RagConfig.ts +10 -2
- package/src/config/serverConfig.ts +16 -1
- package/src/core/LLMRouter.ts +79 -0
- package/src/core/Pipeline.ts +262 -45
- package/src/core/ProviderRegistry.ts +6 -0
- package/src/core/QueryProcessor.ts +98 -0
- package/src/handlers/index.ts +8 -5
- package/src/hooks/useRagChat.ts +53 -17
- package/src/llm/providers/UniversalLLMAdapter.ts +110 -13
- package/src/providers/vectordb/ChromaDBProvider.ts +13 -6
- package/src/providers/vectordb/MilvusProvider.ts +18 -2
- package/src/providers/vectordb/MultiTablePostgresProvider.ts +16 -29
- package/src/providers/vectordb/PostgreSQLProvider.ts +4 -15
- package/src/providers/vectordb/QdrantProvider.ts +2 -5
- package/src/providers/vectordb/RedisProvider.ts +3 -4
- package/src/providers/vectordb/WeaviateProvider.ts +41 -3
- package/src/types/index.ts +26 -0
- package/src/utils/ProductExtractor.ts +5 -3
- package/src/utils/UITransformer.ts +1345 -534
- package/src/utils/synonyms.ts +2 -0
package/dist/server.mjs
CHANGED
|
@@ -313,12 +313,218 @@ var init_PineconeProvider = __esm({
|
|
|
313
313
|
}
|
|
314
314
|
});
|
|
315
315
|
|
|
316
|
+
// src/providers/vectordb/PostgreSQLProvider.ts
|
|
317
|
+
var PostgreSQLProvider_exports = {};
|
|
318
|
+
__export(PostgreSQLProvider_exports, {
|
|
319
|
+
PostgreSQLProvider: () => PostgreSQLProvider
|
|
320
|
+
});
|
|
321
|
+
import { Pool } from "pg";
|
|
322
|
+
var PostgreSQLProvider;
|
|
323
|
+
var init_PostgreSQLProvider = __esm({
|
|
324
|
+
"src/providers/vectordb/PostgreSQLProvider.ts"() {
|
|
325
|
+
"use strict";
|
|
326
|
+
init_BaseVectorProvider();
|
|
327
|
+
PostgreSQLProvider = class extends BaseVectorProvider {
|
|
328
|
+
constructor(config) {
|
|
329
|
+
var _a;
|
|
330
|
+
super(config);
|
|
331
|
+
this.tableName = this.indexName.replace(/[^a-z0-9_]/gi, "_");
|
|
332
|
+
const opts = config.options;
|
|
333
|
+
if (!opts.connectionString) throw new Error("[PostgreSQLProvider] options.connectionString is required");
|
|
334
|
+
this.connectionString = opts.connectionString;
|
|
335
|
+
this.dimensions = (_a = opts.dimensions) != null ? _a : 1536;
|
|
336
|
+
}
|
|
337
|
+
static getValidator() {
|
|
338
|
+
return {
|
|
339
|
+
validate(config) {
|
|
340
|
+
const errors = [];
|
|
341
|
+
const opts = config.options || {};
|
|
342
|
+
if (!opts.connectionString) {
|
|
343
|
+
errors.push({
|
|
344
|
+
field: "vectorDb.options.connectionString",
|
|
345
|
+
message: "PostgreSQL connection string is required",
|
|
346
|
+
suggestion: "Set PGVECTOR_CONNECTION_STRING environment variable",
|
|
347
|
+
severity: "error"
|
|
348
|
+
});
|
|
349
|
+
}
|
|
350
|
+
if (opts.tables && typeof opts.tables !== "string" && !Array.isArray(opts.tables)) {
|
|
351
|
+
errors.push({
|
|
352
|
+
field: "vectorDb.options.tables",
|
|
353
|
+
message: "PostgreSQL tables must be a string or a string array",
|
|
354
|
+
severity: "error"
|
|
355
|
+
});
|
|
356
|
+
}
|
|
357
|
+
return errors;
|
|
358
|
+
}
|
|
359
|
+
};
|
|
360
|
+
}
|
|
361
|
+
static getHealthChecker() {
|
|
362
|
+
return {
|
|
363
|
+
async check(config) {
|
|
364
|
+
const opts = config.options || {};
|
|
365
|
+
const timestamp = Date.now();
|
|
366
|
+
try {
|
|
367
|
+
const { Client } = await import("pg");
|
|
368
|
+
const client = new Client({ connectionString: opts.connectionString });
|
|
369
|
+
await client.connect();
|
|
370
|
+
const result = await client.query(`
|
|
371
|
+
SELECT EXISTS(SELECT 1 FROM pg_extension WHERE extname = 'vector');
|
|
372
|
+
`);
|
|
373
|
+
const hasVector = result.rows[0].exists;
|
|
374
|
+
await client.end();
|
|
375
|
+
return {
|
|
376
|
+
healthy: true,
|
|
377
|
+
provider: "postgresql",
|
|
378
|
+
capabilities: { pgvectorInstalled: hasVector },
|
|
379
|
+
timestamp
|
|
380
|
+
};
|
|
381
|
+
} catch (error) {
|
|
382
|
+
return {
|
|
383
|
+
healthy: false,
|
|
384
|
+
provider: "postgresql",
|
|
385
|
+
error: `Connection failed: ${error instanceof Error ? error.message : String(error)}`,
|
|
386
|
+
timestamp
|
|
387
|
+
};
|
|
388
|
+
}
|
|
389
|
+
}
|
|
390
|
+
};
|
|
391
|
+
}
|
|
392
|
+
async initialize() {
|
|
393
|
+
this.pool = new Pool({ connectionString: this.connectionString });
|
|
394
|
+
const client = await this.pool.connect();
|
|
395
|
+
try {
|
|
396
|
+
await client.query("CREATE EXTENSION IF NOT EXISTS vector");
|
|
397
|
+
await client.query(`
|
|
398
|
+
CREATE TABLE IF NOT EXISTS ${this.tableName} (
|
|
399
|
+
id TEXT PRIMARY KEY,
|
|
400
|
+
namespace TEXT NOT NULL DEFAULT '',
|
|
401
|
+
content TEXT NOT NULL,
|
|
402
|
+
metadata JSONB,
|
|
403
|
+
embedding VECTOR(${this.dimensions})
|
|
404
|
+
)
|
|
405
|
+
`);
|
|
406
|
+
await client.query(`
|
|
407
|
+
CREATE INDEX IF NOT EXISTS ${this.tableName}_embedding_idx
|
|
408
|
+
ON ${this.tableName}
|
|
409
|
+
USING hnsw (embedding vector_cosine_ops)
|
|
410
|
+
`);
|
|
411
|
+
} finally {
|
|
412
|
+
client.release();
|
|
413
|
+
}
|
|
414
|
+
}
|
|
415
|
+
async upsert(doc, namespace = "") {
|
|
416
|
+
var _a;
|
|
417
|
+
const vectorLiteral = `[${doc.vector.join(",")}]`;
|
|
418
|
+
await this.pool.query(
|
|
419
|
+
`INSERT INTO ${this.tableName} (id, namespace, content, metadata, embedding)
|
|
420
|
+
VALUES ($1, $2, $3, $4, $5::vector)
|
|
421
|
+
ON CONFLICT (id) DO UPDATE
|
|
422
|
+
SET namespace = EXCLUDED.namespace,
|
|
423
|
+
content = EXCLUDED.content,
|
|
424
|
+
metadata = EXCLUDED.metadata,
|
|
425
|
+
embedding = EXCLUDED.embedding`,
|
|
426
|
+
[doc.id, namespace, doc.content, JSON.stringify((_a = doc.metadata) != null ? _a : {}), vectorLiteral]
|
|
427
|
+
);
|
|
428
|
+
}
|
|
429
|
+
async batchUpsert(docs, namespace = "") {
|
|
430
|
+
if (docs.length === 0) return;
|
|
431
|
+
const client = await this.pool.connect();
|
|
432
|
+
try {
|
|
433
|
+
await client.query("BEGIN");
|
|
434
|
+
const BATCH_SIZE = 50;
|
|
435
|
+
for (let i = 0; i < docs.length; i += BATCH_SIZE) {
|
|
436
|
+
const batch = docs.slice(i, i + BATCH_SIZE);
|
|
437
|
+
const values = [];
|
|
438
|
+
const valuePlaceholders = batch.map((doc, idx) => {
|
|
439
|
+
var _a;
|
|
440
|
+
const offset = idx * 5;
|
|
441
|
+
values.push(doc.id, namespace, doc.content, JSON.stringify((_a = doc.metadata) != null ? _a : {}), `[${doc.vector.join(",")}]`);
|
|
442
|
+
return `($${offset + 1}, $${offset + 2}, $${offset + 3}, $${offset + 4}, $${offset + 5}::vector)`;
|
|
443
|
+
}).join(", ");
|
|
444
|
+
const query = `
|
|
445
|
+
INSERT INTO ${this.tableName} (id, namespace, content, metadata, embedding)
|
|
446
|
+
VALUES ${valuePlaceholders}
|
|
447
|
+
ON CONFLICT (id) DO UPDATE
|
|
448
|
+
SET namespace = EXCLUDED.namespace,
|
|
449
|
+
content = EXCLUDED.content,
|
|
450
|
+
metadata = EXCLUDED.metadata,
|
|
451
|
+
embedding = EXCLUDED.embedding
|
|
452
|
+
`;
|
|
453
|
+
await client.query(query, values);
|
|
454
|
+
}
|
|
455
|
+
await client.query("COMMIT");
|
|
456
|
+
} catch (error) {
|
|
457
|
+
await client.query("ROLLBACK");
|
|
458
|
+
throw error;
|
|
459
|
+
} finally {
|
|
460
|
+
client.release();
|
|
461
|
+
}
|
|
462
|
+
}
|
|
463
|
+
async query(vector, topK, namespace, filter) {
|
|
464
|
+
const vectorLiteral = `[${vector.join(",")}]`;
|
|
465
|
+
let whereClause = namespace ? `WHERE namespace = $3` : "";
|
|
466
|
+
const params = [vectorLiteral, topK];
|
|
467
|
+
if (namespace) params.push(namespace);
|
|
468
|
+
const publicFilter = this.sanitizeFilter(filter);
|
|
469
|
+
if (Object.keys(publicFilter).length > 0) {
|
|
470
|
+
const filterConditions = Object.entries(publicFilter).map(([key, val]) => {
|
|
471
|
+
const paramIdx = params.length + 1;
|
|
472
|
+
params.push(String(val));
|
|
473
|
+
return `metadata->>'${key}' = $${paramIdx}`;
|
|
474
|
+
}).join(" AND ");
|
|
475
|
+
whereClause = whereClause ? `${whereClause} AND ${filterConditions}` : `WHERE ${filterConditions}`;
|
|
476
|
+
}
|
|
477
|
+
const client = await this.pool.connect();
|
|
478
|
+
try {
|
|
479
|
+
const efSearch = this.config.options.efSearch || Math.max(topK * 10, 40);
|
|
480
|
+
await client.query(`SET LOCAL hnsw.ef_search = ${efSearch}`);
|
|
481
|
+
const result = await client.query(
|
|
482
|
+
`SELECT id, content, metadata, 1 - (embedding <=> $1::vector) AS score
|
|
483
|
+
FROM ${this.tableName}
|
|
484
|
+
${whereClause}
|
|
485
|
+
ORDER BY embedding <=> $1::vector
|
|
486
|
+
LIMIT $2`,
|
|
487
|
+
params
|
|
488
|
+
);
|
|
489
|
+
return result.rows.map((row) => ({
|
|
490
|
+
id: String(row["id"]),
|
|
491
|
+
score: parseFloat(String(row["score"])),
|
|
492
|
+
content: String(row["content"]),
|
|
493
|
+
metadata: row["metadata"]
|
|
494
|
+
}));
|
|
495
|
+
} finally {
|
|
496
|
+
client.release();
|
|
497
|
+
}
|
|
498
|
+
}
|
|
499
|
+
async delete(id, namespace) {
|
|
500
|
+
const where = namespace ? "WHERE id = $1 AND namespace = $2" : "WHERE id = $1";
|
|
501
|
+
const params = namespace ? [id, namespace] : [id];
|
|
502
|
+
await this.pool.query(`DELETE FROM ${this.tableName} ${where}`, params);
|
|
503
|
+
}
|
|
504
|
+
async deleteNamespace(namespace) {
|
|
505
|
+
await this.pool.query(`DELETE FROM ${this.tableName} WHERE namespace = $1`, [namespace]);
|
|
506
|
+
}
|
|
507
|
+
async ping() {
|
|
508
|
+
try {
|
|
509
|
+
await this.pool.query("SELECT 1");
|
|
510
|
+
return true;
|
|
511
|
+
} catch (e) {
|
|
512
|
+
return false;
|
|
513
|
+
}
|
|
514
|
+
}
|
|
515
|
+
async disconnect() {
|
|
516
|
+
await this.pool.end();
|
|
517
|
+
}
|
|
518
|
+
};
|
|
519
|
+
}
|
|
520
|
+
});
|
|
521
|
+
|
|
316
522
|
// src/providers/vectordb/MultiTablePostgresProvider.ts
|
|
317
523
|
var MultiTablePostgresProvider_exports = {};
|
|
318
524
|
__export(MultiTablePostgresProvider_exports, {
|
|
319
525
|
MultiTablePostgresProvider: () => MultiTablePostgresProvider
|
|
320
526
|
});
|
|
321
|
-
import { Pool } from "pg";
|
|
527
|
+
import { Pool as Pool2 } from "pg";
|
|
322
528
|
var MultiTablePostgresProvider;
|
|
323
529
|
var init_MultiTablePostgresProvider = __esm({
|
|
324
530
|
"src/providers/vectordb/MultiTablePostgresProvider.ts"() {
|
|
@@ -340,7 +546,7 @@ var init_MultiTablePostgresProvider = __esm({
|
|
|
340
546
|
this.uploadTable = opts.uploadTable || "document_chunks";
|
|
341
547
|
}
|
|
342
548
|
async initialize() {
|
|
343
|
-
this.pool = new
|
|
549
|
+
this.pool = new Pool2({ connectionString: this.connectionString });
|
|
344
550
|
const client = await this.pool.connect();
|
|
345
551
|
try {
|
|
346
552
|
await client.query("CREATE EXTENSION IF NOT EXISTS vector");
|
|
@@ -353,12 +559,10 @@ var init_MultiTablePostgresProvider = __esm({
|
|
|
353
559
|
embedding VECTOR(${this.dimensions})
|
|
354
560
|
)
|
|
355
561
|
`);
|
|
356
|
-
const metric = this.config.distanceMetric || "cosine";
|
|
357
|
-
const indexOps = metric === "euclidean" ? "vector_l2_ops" : metric === "dotproduct" ? "vector_ip_ops" : "vector_cosine_ops";
|
|
358
562
|
await client.query(`
|
|
359
563
|
CREATE INDEX IF NOT EXISTS ${this.uploadTable}_embedding_idx
|
|
360
564
|
ON ${this.uploadTable}
|
|
361
|
-
USING hnsw (embedding
|
|
565
|
+
USING hnsw (embedding vector_cosine_ops)
|
|
362
566
|
`);
|
|
363
567
|
const res = await client.query(`
|
|
364
568
|
SELECT table_name
|
|
@@ -420,12 +624,10 @@ var init_MultiTablePostgresProvider = __esm({
|
|
|
420
624
|
)
|
|
421
625
|
`;
|
|
422
626
|
await client.query(createTableSql);
|
|
423
|
-
const metric = this.config.distanceMetric || "cosine";
|
|
424
|
-
const indexOps = metric === "euclidean" ? "vector_l2_ops" : metric === "dotproduct" ? "vector_ip_ops" : "vector_cosine_ops";
|
|
425
627
|
await client.query(`
|
|
426
628
|
CREATE INDEX IF NOT EXISTS "${tableName}_embedding_idx"
|
|
427
629
|
ON "${tableName}"
|
|
428
|
-
USING hnsw (embedding
|
|
630
|
+
USING hnsw (embedding vector_cosine_ops)
|
|
429
631
|
`);
|
|
430
632
|
if (!this.tables.includes(tableName)) {
|
|
431
633
|
this.tables.push(tableName);
|
|
@@ -488,28 +690,25 @@ var init_MultiTablePostgresProvider = __esm({
|
|
|
488
690
|
const allResults = [];
|
|
489
691
|
console.log(`[MultiTablePostgresProvider] --- Starting Multi-Table Search ---`);
|
|
490
692
|
const queryText = _filter == null ? void 0 : _filter.queryText;
|
|
491
|
-
const entityHints = Array.isArray(_filter == null ? void 0 : _filter.
|
|
492
|
-
(hint) => typeof hint === "object" && hint !== null && typeof hint.value === "string"
|
|
493
|
-
).map((hint) => hint.value.trim().toLowerCase()).filter(Boolean) : [];
|
|
693
|
+
const entityHints = Array.isArray(_filter == null ? void 0 : _filter.keywords) ? _filter.keywords.map((k) => k.trim().toLowerCase()).filter(Boolean) : [];
|
|
494
694
|
console.log(`[MultiTablePostgresProvider] queryText: "${queryText}"`);
|
|
495
695
|
console.log(`[MultiTablePostgresProvider] entityHints: [${entityHints.join(", ")}]`);
|
|
496
696
|
const getDynamicKeywordQuery = () => {
|
|
497
697
|
if (entityHints.length > 0) {
|
|
498
|
-
return entityHints.map((h) => h.
|
|
698
|
+
return entityHints.map((h) => h.replace(/\s+/g, " & ")).join(" & ");
|
|
499
699
|
}
|
|
500
700
|
if (queryText) {
|
|
501
|
-
return queryText.toLowerCase().replace(/\b(show|me|the|product[s]?|item[s]?|card[s]?|of|category|find|list|browse|what|are|display|all|under|for|in|with|about)\b/g, "").trim().replace(/\s+/g, " & ");
|
|
701
|
+
return queryText.toLowerCase().replace(/\b(show|me|the|product[s]?|item[s]?|card[s]?|organization[s]?|comp(?:any|anies)|of|category|find|list|browse|what|are|display|all|whose|that|which|have|has|having|greater|than|more|less|equal|equals|above|below|over|under|at|least|most|for|in|with|about|get|details|information|info)\b/g, "").replace(/\b\d+(?:\.\d+)?\b/g, "").trim().replace(/\s+/g, " & ");
|
|
502
702
|
}
|
|
503
703
|
return "";
|
|
504
704
|
};
|
|
505
705
|
const dynamicKeywordQuery = getDynamicKeywordQuery();
|
|
706
|
+
const tableLimit = Math.max(topK, 50);
|
|
506
707
|
const queryPromises = this.tables.map(async (table) => {
|
|
507
708
|
try {
|
|
508
709
|
let sqlQuery = "";
|
|
509
710
|
let params = [];
|
|
510
|
-
|
|
511
|
-
const scoreExpr = metric === "euclidean" ? `1 / (1 + (embedding <-> $1::vector))` : metric === "dotproduct" ? `(embedding <#> $1::vector) * -1` : `1 - (embedding <=> $1::vector)`;
|
|
512
|
-
if (queryText) {
|
|
711
|
+
if (queryText && dynamicKeywordQuery) {
|
|
513
712
|
const hasEntityHints = entityHints.length > 0;
|
|
514
713
|
const exactNameScoreExpr = hasEntityHints ? `+ (
|
|
515
714
|
SELECT COALESCE(MAX(
|
|
@@ -521,21 +720,21 @@ var init_MultiTablePostgresProvider = __esm({
|
|
|
521
720
|
) * 3.0` : "";
|
|
522
721
|
sqlQuery = `
|
|
523
722
|
SELECT *,
|
|
524
|
-
($
|
|
723
|
+
(1 - (embedding <=> $1::vector)) AS vector_score,
|
|
525
724
|
COALESCE(ts_rank(to_tsvector('english', (to_jsonb(t) - 'embedding')::text), to_tsquery('english', $2)), 0) AS keyword_score,
|
|
526
|
-
(($
|
|
725
|
+
((1 - (embedding <=> $1::vector)) + (COALESCE(ts_rank(to_tsvector('english', (to_jsonb(t) - 'embedding')::text), to_tsquery('english', $2)), 0) * 2.0) ${exactNameScoreExpr}) AS hybrid_score
|
|
527
726
|
FROM "${table}" t
|
|
528
727
|
ORDER BY hybrid_score DESC
|
|
529
|
-
LIMIT
|
|
728
|
+
LIMIT ${tableLimit}
|
|
530
729
|
`;
|
|
531
730
|
params = [vectorLiteral, dynamicKeywordQuery];
|
|
532
731
|
} else {
|
|
533
732
|
sqlQuery = `
|
|
534
733
|
SELECT *,
|
|
535
|
-
|
|
734
|
+
(1 - (embedding <=> $1::vector)) AS hybrid_score
|
|
536
735
|
FROM "${table}" t
|
|
537
736
|
ORDER BY hybrid_score DESC
|
|
538
|
-
LIMIT
|
|
737
|
+
LIMIT ${tableLimit}
|
|
539
738
|
`;
|
|
540
739
|
params = [vectorLiteral];
|
|
541
740
|
}
|
|
@@ -860,13 +1059,25 @@ var init_MilvusProvider = __esm({
|
|
|
860
1059
|
};
|
|
861
1060
|
await this.http.post("/v1/vector/upsert", payload);
|
|
862
1061
|
}
|
|
863
|
-
// eslint-disable-next-line @typescript-eslint/no-unused-vars
|
|
864
1062
|
async query(vector, topK, namespace, _filter) {
|
|
1063
|
+
const sanitizedFilter = this.sanitizeFilter(_filter);
|
|
1064
|
+
const filterParts = [];
|
|
1065
|
+
if (namespace) {
|
|
1066
|
+
filterParts.push(`namespace == "${namespace}"`);
|
|
1067
|
+
}
|
|
1068
|
+
for (const [key, value] of Object.entries(sanitizedFilter)) {
|
|
1069
|
+
if (key === "queryText" || key === "keywords") continue;
|
|
1070
|
+
if (typeof value === "string") {
|
|
1071
|
+
filterParts.push(`metadata["${key}"] == "${value.replace(/"/g, '\\"')}"`);
|
|
1072
|
+
} else if (typeof value === "number") {
|
|
1073
|
+
filterParts.push(`metadata["${key}"] == ${value}`);
|
|
1074
|
+
}
|
|
1075
|
+
}
|
|
865
1076
|
const payload = {
|
|
866
1077
|
collectionName: this.indexName,
|
|
867
1078
|
vector,
|
|
868
1079
|
limit: topK,
|
|
869
|
-
filter:
|
|
1080
|
+
filter: filterParts.length > 0 ? filterParts.join(" && ") : void 0,
|
|
870
1081
|
outputFields: ["content", "metadata"],
|
|
871
1082
|
searchParams: {
|
|
872
1083
|
nprobe: this.config.options.nprobe || 16,
|
|
@@ -1006,13 +1217,11 @@ var init_QdrantProvider = __esm({
|
|
|
1006
1217
|
if (axios4.isAxiosError(err) && ((_a = err.response) == null ? void 0 : _a.status) === 404) {
|
|
1007
1218
|
const opts = this.config.options;
|
|
1008
1219
|
const dimensionsForCreate = opts.dimensions || 1536;
|
|
1009
|
-
const metric = this.config.distanceMetric || "cosine";
|
|
1010
|
-
const distance = metric === "euclidean" ? "Euclid" : metric === "dotproduct" ? "Dot" : "Cosine";
|
|
1011
1220
|
console.log(`[QdrantProvider] \u23F3 Creating collection "${this.indexName}" (dimensions: ${dimensionsForCreate})...`);
|
|
1012
1221
|
await this.http.put(`/collections/${this.indexName}`, {
|
|
1013
1222
|
vectors: {
|
|
1014
1223
|
size: dimensionsForCreate,
|
|
1015
|
-
distance
|
|
1224
|
+
distance: "Cosine"
|
|
1016
1225
|
}
|
|
1017
1226
|
});
|
|
1018
1227
|
console.log(`[QdrantProvider] \u2705 Created collection "${this.indexName}"`);
|
|
@@ -1067,6 +1276,7 @@ var init_QdrantProvider = __esm({
|
|
|
1067
1276
|
await this.http.put(`/collections/${this.indexName}/points`, payload);
|
|
1068
1277
|
}
|
|
1069
1278
|
async query(vector, topK, namespace, _filter) {
|
|
1279
|
+
var _a;
|
|
1070
1280
|
const must = [];
|
|
1071
1281
|
if (namespace) {
|
|
1072
1282
|
must.push({ key: "namespace", match: { value: namespace } });
|
|
@@ -1088,7 +1298,7 @@ var init_QdrantProvider = __esm({
|
|
|
1088
1298
|
limit: topK,
|
|
1089
1299
|
with_payload: true,
|
|
1090
1300
|
params: {
|
|
1091
|
-
hnsw_ef: this.config.options.efSearch || Math.max(topK * 20, 128),
|
|
1301
|
+
hnsw_ef: ((_a = this.config.options) == null ? void 0 : _a.efSearch) || Math.max(topK * 20, 128),
|
|
1092
1302
|
exact: false
|
|
1093
1303
|
},
|
|
1094
1304
|
filter: must.length > 0 ? { must } : void 0
|
|
@@ -1190,12 +1400,9 @@ var init_ChromaDBProvider = __esm({
|
|
|
1190
1400
|
console.log(`[ChromaDBProvider] \u2705 Collection "${this.indexName}" found (id: ${this.collectionId})`);
|
|
1191
1401
|
} catch (err) {
|
|
1192
1402
|
if (axios5.isAxiosError(err) && ((_a = err.response) == null ? void 0 : _a.status) === 404) {
|
|
1193
|
-
const metric = this.config.distanceMetric || "cosine";
|
|
1194
|
-
const space = metric === "euclidean" ? "l2" : metric === "dotproduct" ? "ip" : "cosine";
|
|
1195
1403
|
console.log(`[ChromaDBProvider] \u23F3 Collection "${this.indexName}" not found. Creating...`);
|
|
1196
1404
|
const { data } = await this.http.post("/api/v1/collections", {
|
|
1197
|
-
name: this.indexName
|
|
1198
|
-
metadata: { "hnsw:space": space }
|
|
1405
|
+
name: this.indexName
|
|
1199
1406
|
});
|
|
1200
1407
|
this.collectionId = data.id;
|
|
1201
1408
|
console.log(`[ChromaDBProvider] \u2705 Created collection "${this.indexName}" (id: ${this.collectionId})`);
|
|
@@ -1216,12 +1423,20 @@ var init_ChromaDBProvider = __esm({
|
|
|
1216
1423
|
};
|
|
1217
1424
|
await this.http.post(`/api/v1/collections/${this.collectionId}/add`, payload);
|
|
1218
1425
|
}
|
|
1219
|
-
// eslint-disable-next-line @typescript-eslint/no-unused-vars
|
|
1220
1426
|
async query(vector, topK, namespace, _filter) {
|
|
1427
|
+
const sanitizedFilter = this.sanitizeFilter(_filter);
|
|
1428
|
+
const whereClauses = [];
|
|
1429
|
+
if (namespace) whereClauses.push({ namespace: { $eq: namespace } });
|
|
1430
|
+
Object.entries(sanitizedFilter).forEach(([key, value]) => {
|
|
1431
|
+
if (key === "namespace") return;
|
|
1432
|
+
if (typeof value === "string" || typeof value === "number" || typeof value === "boolean") {
|
|
1433
|
+
whereClauses.push({ [key]: { $eq: value } });
|
|
1434
|
+
}
|
|
1435
|
+
});
|
|
1221
1436
|
const payload = {
|
|
1222
1437
|
query_embeddings: [vector],
|
|
1223
1438
|
n_results: topK,
|
|
1224
|
-
where:
|
|
1439
|
+
where: whereClauses.length > 1 ? { $and: whereClauses } : whereClauses[0]
|
|
1225
1440
|
};
|
|
1226
1441
|
const { data } = await this.http.post(`/api/v1/collections/${this.collectionId}/query`, payload);
|
|
1227
1442
|
const matches = [];
|
|
@@ -1326,15 +1541,15 @@ var init_RedisProvider = __esm({
|
|
|
1326
1541
|
console.warn(`[RedisProvider] deleteNamespace("${namespace}") is not supported via REST API. Use Redis CLI: SCAN + DEL`);
|
|
1327
1542
|
}
|
|
1328
1543
|
/**
|
|
1329
|
-
*
|
|
1330
|
-
* Returns
|
|
1544
|
+
* Check reachability via a PING command.
|
|
1545
|
+
* Returns false on connection failure so health checks surface real problems.
|
|
1331
1546
|
*/
|
|
1332
1547
|
async ping() {
|
|
1333
1548
|
try {
|
|
1334
1549
|
await this.http.post("/", ["PING"]);
|
|
1335
1550
|
return true;
|
|
1336
1551
|
} catch (e) {
|
|
1337
|
-
return
|
|
1552
|
+
return false;
|
|
1338
1553
|
}
|
|
1339
1554
|
}
|
|
1340
1555
|
async disconnect() {
|
|
@@ -1374,15 +1589,17 @@ var init_WeaviateProvider = __esm({
|
|
|
1374
1589
|
await this.ping();
|
|
1375
1590
|
}
|
|
1376
1591
|
async upsert(doc, namespace) {
|
|
1592
|
+
const primitiveMetadata = this.extractPrimitiveMetadata(doc.metadata);
|
|
1377
1593
|
const payload = {
|
|
1378
1594
|
class: this.indexName,
|
|
1379
1595
|
id: doc.id,
|
|
1380
1596
|
vector: doc.vector,
|
|
1381
|
-
properties: {
|
|
1597
|
+
properties: __spreadProps(__spreadValues({
|
|
1382
1598
|
content: doc.content,
|
|
1383
|
-
metadata: JSON.stringify(doc.metadata || {})
|
|
1599
|
+
metadata: JSON.stringify(doc.metadata || {})
|
|
1600
|
+
}, primitiveMetadata), {
|
|
1384
1601
|
namespace: namespace || ""
|
|
1385
|
-
}
|
|
1602
|
+
})
|
|
1386
1603
|
};
|
|
1387
1604
|
await this.http.post("/v1/objects", payload);
|
|
1388
1605
|
}
|
|
@@ -1392,20 +1609,22 @@ var init_WeaviateProvider = __esm({
|
|
|
1392
1609
|
class: this.indexName,
|
|
1393
1610
|
id: doc.id,
|
|
1394
1611
|
vector: doc.vector,
|
|
1395
|
-
properties: {
|
|
1612
|
+
properties: __spreadProps(__spreadValues({
|
|
1396
1613
|
content: doc.content,
|
|
1397
|
-
metadata: JSON.stringify(doc.metadata || {})
|
|
1614
|
+
metadata: JSON.stringify(doc.metadata || {})
|
|
1615
|
+
}, this.extractPrimitiveMetadata(doc.metadata)), {
|
|
1398
1616
|
namespace: namespace || ""
|
|
1399
|
-
}
|
|
1617
|
+
})
|
|
1400
1618
|
}))
|
|
1401
1619
|
};
|
|
1402
1620
|
await this.http.post("/v1/batch/objects", payload);
|
|
1403
1621
|
}
|
|
1404
1622
|
async query(vector, topK, namespace, _filter) {
|
|
1405
1623
|
var _a, _b;
|
|
1624
|
+
const queryText = _filter == null ? void 0 : _filter.queryText;
|
|
1406
1625
|
const sanitizedFilter = this.sanitizeFilter(_filter);
|
|
1407
|
-
const queryText = sanitizedFilter.queryText;
|
|
1408
1626
|
const searchParams = queryText ? `hybrid: { query: ${JSON.stringify(queryText)}, alpha: 0.5 }` : `nearVector: { vector: ${JSON.stringify(vector)} }`;
|
|
1627
|
+
const where = this.buildWhereFilter(namespace, sanitizedFilter);
|
|
1409
1628
|
const graphqlQuery = {
|
|
1410
1629
|
query: `
|
|
1411
1630
|
{
|
|
@@ -1413,7 +1632,7 @@ var init_WeaviateProvider = __esm({
|
|
|
1413
1632
|
${this.indexName}(
|
|
1414
1633
|
${searchParams}
|
|
1415
1634
|
limit: ${topK}
|
|
1416
|
-
${
|
|
1635
|
+
${where ? `where: ${where}` : ""}
|
|
1417
1636
|
) {
|
|
1418
1637
|
content
|
|
1419
1638
|
metadata
|
|
@@ -1457,6 +1676,34 @@ var init_WeaviateProvider = __esm({
|
|
|
1457
1676
|
}
|
|
1458
1677
|
async disconnect() {
|
|
1459
1678
|
}
|
|
1679
|
+
extractPrimitiveMetadata(metadata) {
|
|
1680
|
+
const result = {};
|
|
1681
|
+
Object.entries(metadata || {}).forEach(([key, value]) => {
|
|
1682
|
+
if (typeof value === "string" || typeof value === "number" || typeof value === "boolean") {
|
|
1683
|
+
result[key] = value;
|
|
1684
|
+
}
|
|
1685
|
+
});
|
|
1686
|
+
return result;
|
|
1687
|
+
}
|
|
1688
|
+
buildWhereFilter(namespace, filter) {
|
|
1689
|
+
const operands = [];
|
|
1690
|
+
if (namespace) operands.push(this.weaviateOperand("namespace", namespace));
|
|
1691
|
+
Object.entries(filter || {}).forEach(([key, value]) => {
|
|
1692
|
+
if (key === "namespace") return;
|
|
1693
|
+
if (typeof value === "string" || typeof value === "number" || typeof value === "boolean") {
|
|
1694
|
+
operands.push(this.weaviateOperand(key, value));
|
|
1695
|
+
}
|
|
1696
|
+
});
|
|
1697
|
+
if (operands.length === 0) return void 0;
|
|
1698
|
+
if (operands.length === 1) return operands[0];
|
|
1699
|
+
return `{ operator: And, operands: [${operands.join(", ")}] }`;
|
|
1700
|
+
}
|
|
1701
|
+
weaviateOperand(key, value) {
|
|
1702
|
+
const path = `path: [${JSON.stringify(key)}], operator: Equal`;
|
|
1703
|
+
if (typeof value === "number") return `{ ${path}, valueNumber: ${value} }`;
|
|
1704
|
+
if (typeof value === "boolean") return `{ ${path}, valueBoolean: ${value} }`;
|
|
1705
|
+
return `{ ${path}, valueString: ${JSON.stringify(value)} }`;
|
|
1706
|
+
}
|
|
1460
1707
|
};
|
|
1461
1708
|
}
|
|
1462
1709
|
});
|
|
@@ -1762,7 +2009,7 @@ function getRagConfig(baseConfig, env = process.env) {
|
|
|
1762
2009
|
return getEnvConfig(env, baseConfig);
|
|
1763
2010
|
}
|
|
1764
2011
|
function getEnvConfig(env = process.env, base) {
|
|
1765
|
-
var _a, _b, _c, _d, _e, _f, _g, _h, _i, _j, _k, _l, _m, _n, _o, _p, _q, _r, _s, _t, _u, _v, _w, _x, _y, _z, _A, _B, _C, _D, _E, _F, _G, _H, _I, _J, _K, _L, _M, _N, _O, _P, _Q, _R, _S, _T, _U, _V, _W, _X, _Y, _Z, __, _$, _aa, _ba, _ca, _da, _ea, _fa, _ga, _ha, _ia, _ja, _ka, _la, _ma, _na, _oa, _pa, _qa, _ra, _sa, _ta, _ua, _va, _wa, _xa, _ya, _za, _Aa, _Ba, _Ca, _Da, _Ea, _Fa;
|
|
2012
|
+
var _a, _b, _c, _d, _e, _f, _g, _h, _i, _j, _k, _l, _m, _n, _o, _p, _q, _r, _s, _t, _u, _v, _w, _x, _y, _z, _A, _B, _C, _D, _E, _F, _G, _H, _I, _J, _K, _L, _M, _N, _O, _P, _Q, _R, _S, _T, _U, _V, _W, _X, _Y, _Z, __, _$, _aa, _ba, _ca, _da, _ea, _fa, _ga, _ha, _ia, _ja, _ka, _la, _ma, _na, _oa, _pa, _qa, _ra, _sa, _ta, _ua, _va, _wa, _xa, _ya, _za, _Aa, _Ba, _Ca, _Da, _Ea, _Fa, _Ga;
|
|
1766
2013
|
const projectId = (_c = (_b = (_a = readString(env, "RAG_PROJECT_ID")) != null ? _a : readString(env, "NEXT_PUBLIC_PROJECT_ID")) != null ? _b : base == null ? void 0 : base.projectId) != null ? _c : "__default__";
|
|
1767
2014
|
const vectorProvider = readEnum(env, "VECTOR_DB_PROVIDER", "pinecone", VECTOR_DB_PROVIDERS);
|
|
1768
2015
|
const llmProvider = readEnum(env, "LLM_PROVIDER", "openai", LLM_PROVIDERS);
|
|
@@ -1816,11 +2063,22 @@ function getEnvConfig(env = process.env, base) {
|
|
|
1816
2063
|
};
|
|
1817
2064
|
const embeddingApiKeyByProvider = {
|
|
1818
2065
|
openai: readString(env, "OPENAI_API_KEY"),
|
|
2066
|
+
anthropic: readString(env, "ANTHROPIC_API_KEY"),
|
|
2067
|
+
// Anthropic needs a separate embedding provider; key kept for completeness
|
|
1819
2068
|
gemini: readString(env, "GEMINI_API_KEY"),
|
|
1820
2069
|
ollama: void 0,
|
|
1821
2070
|
universal_rest: (_ea = readString(env, "EMBEDDING_API_KEY")) != null ? _ea : readString(env, "OPENAI_API_KEY"),
|
|
1822
2071
|
custom: (_fa = readString(env, "EMBEDDING_API_KEY")) != null ? _fa : readString(env, "OPENAI_API_KEY")
|
|
1823
2072
|
};
|
|
2073
|
+
const DEFAULT_MODEL_BY_PROVIDER = {
|
|
2074
|
+
openai: "gpt-4o",
|
|
2075
|
+
anthropic: "claude-3-5-sonnet-20241022",
|
|
2076
|
+
gemini: "gemini-2.0-flash",
|
|
2077
|
+
ollama: "llama3",
|
|
2078
|
+
universal_rest: "default",
|
|
2079
|
+
rest: "default",
|
|
2080
|
+
custom: "default"
|
|
2081
|
+
};
|
|
1824
2082
|
return {
|
|
1825
2083
|
projectId,
|
|
1826
2084
|
vectorDb: {
|
|
@@ -1830,8 +2088,8 @@ function getEnvConfig(env = process.env, base) {
|
|
|
1830
2088
|
},
|
|
1831
2089
|
llm: {
|
|
1832
2090
|
provider: llmProvider,
|
|
1833
|
-
model: (_ia = readString(env, "LLM_MODEL")) != null ? _ia : "gpt-4o",
|
|
1834
|
-
apiKey: (
|
|
2091
|
+
model: (_ja = (_ia = readString(env, "LLM_MODEL")) != null ? _ia : DEFAULT_MODEL_BY_PROVIDER[llmProvider]) != null ? _ja : "gpt-4o",
|
|
2092
|
+
apiKey: (_ka = llmApiKeyByProvider[llmProvider]) != null ? _ka : "",
|
|
1835
2093
|
baseUrl: readString(env, "LLM_BASE_URL"),
|
|
1836
2094
|
systemPrompt: readString(env, "LLM_SYSTEM_PROMPT"),
|
|
1837
2095
|
maxTokens: readNumber(env, "LLM_MAX_TOKENS", 4096),
|
|
@@ -1842,7 +2100,7 @@ function getEnvConfig(env = process.env, base) {
|
|
|
1842
2100
|
},
|
|
1843
2101
|
embedding: {
|
|
1844
2102
|
provider: embeddingProvider,
|
|
1845
|
-
model: (
|
|
2103
|
+
model: (_la = readString(env, "EMBEDDING_MODEL")) != null ? _la : "text-embedding-3-small",
|
|
1846
2104
|
apiKey: embeddingApiKeyByProvider[embeddingProvider],
|
|
1847
2105
|
baseUrl: readString(env, "EMBEDDING_BASE_URL"),
|
|
1848
2106
|
dimensions: embeddingDimensions,
|
|
@@ -1853,17 +2111,17 @@ function getEnvConfig(env = process.env, base) {
|
|
|
1853
2111
|
}
|
|
1854
2112
|
},
|
|
1855
2113
|
ui: {
|
|
1856
|
-
title: (
|
|
1857
|
-
subtitle: (
|
|
1858
|
-
primaryColor: (
|
|
1859
|
-
accentColor: (
|
|
1860
|
-
logoUrl: (
|
|
1861
|
-
placeholder: (
|
|
1862
|
-
showSources: ((
|
|
1863
|
-
welcomeMessage: (
|
|
1864
|
-
visualStyle: (
|
|
1865
|
-
borderRadius: (
|
|
1866
|
-
allowUpload: ((
|
|
2114
|
+
title: (_na = (_ma = readString(env, "NEXT_PUBLIC_UI_TITLE")) != null ? _ma : readString(env, "UI_TITLE")) != null ? _na : "AI Assistant",
|
|
2115
|
+
subtitle: (_pa = (_oa = readString(env, "NEXT_PUBLIC_UI_SUBTITLE")) != null ? _oa : readString(env, "UI_SUBTITLE")) != null ? _pa : "Powered by RAG",
|
|
2116
|
+
primaryColor: (_ra = (_qa = readString(env, "NEXT_PUBLIC_PRIMARY_COLOR")) != null ? _qa : readString(env, "UI_PRIMARY_COLOR")) != null ? _ra : "#10b981",
|
|
2117
|
+
accentColor: (_ta = (_sa = readString(env, "NEXT_PUBLIC_ACCENT_COLOR")) != null ? _sa : readString(env, "UI_ACCENT_COLOR")) != null ? _ta : "#3b82f6",
|
|
2118
|
+
logoUrl: (_ua = readString(env, "NEXT_PUBLIC_LOGO_URL")) != null ? _ua : readString(env, "UI_LOGO_URL"),
|
|
2119
|
+
placeholder: (_wa = (_va = readString(env, "NEXT_PUBLIC_PLACEHOLDER")) != null ? _va : readString(env, "UI_PLACEHOLDER")) != null ? _wa : "Ask me anything\u2026",
|
|
2120
|
+
showSources: ((_ya = (_xa = readString(env, "NEXT_PUBLIC_SHOW_SOURCES")) != null ? _xa : readString(env, "UI_SHOW_SOURCES")) != null ? _ya : "true") !== "false",
|
|
2121
|
+
welcomeMessage: (_Aa = (_za = readString(env, "NEXT_PUBLIC_WELCOME_MESSAGE")) != null ? _za : readString(env, "UI_WELCOME_MESSAGE")) != null ? _Aa : "Hello! I'm your AI assistant. Ask me anything about your documents.",
|
|
2122
|
+
visualStyle: (_Ca = (_Ba = readString(env, "NEXT_PUBLIC_UI_VISUAL_STYLE")) != null ? _Ba : readString(env, "UI_VISUAL_STYLE")) != null ? _Ca : "glass",
|
|
2123
|
+
borderRadius: (_Ea = (_Da = readString(env, "NEXT_PUBLIC_UI_BORDER_RADIUS")) != null ? _Da : readString(env, "UI_BORDER_RADIUS")) != null ? _Ea : "xl",
|
|
2124
|
+
allowUpload: ((_Ga = (_Fa = readString(env, "NEXT_PUBLIC_ALLOW_UPLOAD")) != null ? _Fa : readString(env, "UI_ALLOW_UPLOAD")) != null ? _Ga : "false") === "true"
|
|
1867
2125
|
},
|
|
1868
2126
|
rag: {
|
|
1869
2127
|
topK: readNumber(env, "RAG_TOP_K", 5),
|
|
@@ -2764,7 +3022,7 @@ var VECTOR_PROFILES = {
|
|
|
2764
3022
|
// src/llm/providers/UniversalLLMAdapter.ts
|
|
2765
3023
|
var UniversalLLMAdapter = class {
|
|
2766
3024
|
constructor(config) {
|
|
2767
|
-
var _a, _b, _c, _d, _e, _f, _g;
|
|
3025
|
+
var _a, _b, _c, _d, _e, _f, _g, _h;
|
|
2768
3026
|
this.model = config.model;
|
|
2769
3027
|
const llmConfig = config;
|
|
2770
3028
|
const options = (_a = llmConfig.options) != null ? _a : {};
|
|
@@ -2778,16 +3036,18 @@ var UniversalLLMAdapter = class {
|
|
|
2778
3036
|
this.systemPrompt = (_c = llmConfig.systemPrompt) != null ? _c : "You are a helpful AI assistant. Use the provided context to answer the user.";
|
|
2779
3037
|
this.maxTokens = (_d = llmConfig.maxTokens) != null ? _d : 1024;
|
|
2780
3038
|
this.temperature = (_e = llmConfig.temperature) != null ? _e : 0;
|
|
2781
|
-
|
|
2782
|
-
|
|
3039
|
+
this.apiKey = config.apiKey;
|
|
3040
|
+
this.baseUrl = (_g = (_f = llmConfig.baseUrl) != null ? _f : this.opts.baseUrl) != null ? _g : "";
|
|
3041
|
+
if (!this.baseUrl) {
|
|
2783
3042
|
throw new Error("[UniversalLLMAdapter] baseUrl is required in config or config.options");
|
|
2784
3043
|
}
|
|
3044
|
+
this.resolvedHeaders = __spreadValues(__spreadValues({
|
|
3045
|
+
"Content-Type": "application/json"
|
|
3046
|
+
}, config.apiKey ? { Authorization: `Bearer ${config.apiKey}` } : {}), this.opts.headers || {});
|
|
2785
3047
|
this.http = axios2.create({
|
|
2786
|
-
baseURL: baseUrl,
|
|
2787
|
-
headers:
|
|
2788
|
-
|
|
2789
|
-
}, config.apiKey ? { Authorization: `Bearer ${config.apiKey}` } : {}), this.opts.headers || {}),
|
|
2790
|
-
timeout: (_g = this.opts.timeout) != null ? _g : 6e4
|
|
3048
|
+
baseURL: this.baseUrl,
|
|
3049
|
+
headers: this.resolvedHeaders,
|
|
3050
|
+
timeout: (_h = this.opts.timeout) != null ? _h : 6e4
|
|
2791
3051
|
});
|
|
2792
3052
|
}
|
|
2793
3053
|
async chat(messages, context) {
|
|
@@ -2824,6 +3084,92 @@ ${context != null ? context : "None"}` },
|
|
|
2824
3084
|
}
|
|
2825
3085
|
return String(result);
|
|
2826
3086
|
}
|
|
3087
|
+
/**
|
|
3088
|
+
* Streaming chat using native fetch + ReadableStream.
|
|
3089
|
+
* Parses OpenAI-compatible SSE frames: `data: {...}\n\n`
|
|
3090
|
+
* Works with vLLM, LMStudio, Together AI, Fireworks, and any OpenAI-compatible API.
|
|
3091
|
+
*/
|
|
3092
|
+
chatStream(messages, context) {
|
|
3093
|
+
return __asyncGenerator(this, null, function* () {
|
|
3094
|
+
var _a, _b, _c;
|
|
3095
|
+
const path = (_a = this.opts.chatPath) != null ? _a : "/chat/completions";
|
|
3096
|
+
const url = `${this.baseUrl.replace(/\/$/, "")}${path}`;
|
|
3097
|
+
const extractPath = ((_b = this.opts.responseExtractPath) != null ? _b : "choices[0].message.content").replace("message.content", "delta.content");
|
|
3098
|
+
const formattedMessages = [
|
|
3099
|
+
{ role: "system", content: `${this.systemPrompt}
|
|
3100
|
+
|
|
3101
|
+
Context:
|
|
3102
|
+
${context != null ? context : "None"}` },
|
|
3103
|
+
...messages
|
|
3104
|
+
];
|
|
3105
|
+
let payload;
|
|
3106
|
+
if (this.opts.chatPayloadTemplate) {
|
|
3107
|
+
payload = buildPayload(this.opts.chatPayloadTemplate, {
|
|
3108
|
+
model: this.model,
|
|
3109
|
+
messages: formattedMessages,
|
|
3110
|
+
maxTokens: this.maxTokens,
|
|
3111
|
+
temperature: this.temperature
|
|
3112
|
+
});
|
|
3113
|
+
if (typeof payload === "object" && payload !== null) {
|
|
3114
|
+
payload.stream = true;
|
|
3115
|
+
}
|
|
3116
|
+
} else {
|
|
3117
|
+
payload = {
|
|
3118
|
+
model: this.model,
|
|
3119
|
+
messages: formattedMessages,
|
|
3120
|
+
max_tokens: this.maxTokens,
|
|
3121
|
+
temperature: this.temperature,
|
|
3122
|
+
stream: true
|
|
3123
|
+
};
|
|
3124
|
+
}
|
|
3125
|
+
const response = yield new __await(fetch(url, {
|
|
3126
|
+
method: "POST",
|
|
3127
|
+
headers: this.resolvedHeaders,
|
|
3128
|
+
body: JSON.stringify(payload)
|
|
3129
|
+
}));
|
|
3130
|
+
if (!response.ok) {
|
|
3131
|
+
const errorText = yield new __await(response.text().catch(() => response.statusText));
|
|
3132
|
+
throw new Error(`[UniversalLLMAdapter] Streaming request failed (${response.status}): ${errorText}`);
|
|
3133
|
+
}
|
|
3134
|
+
if (!response.body) {
|
|
3135
|
+
throw new Error("[UniversalLLMAdapter] Response body is null \u2014 server did not send a streaming response.");
|
|
3136
|
+
}
|
|
3137
|
+
const reader = response.body.getReader();
|
|
3138
|
+
const decoder = new TextDecoder("utf-8");
|
|
3139
|
+
let buffer = "";
|
|
3140
|
+
try {
|
|
3141
|
+
while (true) {
|
|
3142
|
+
const { done, value } = yield new __await(reader.read());
|
|
3143
|
+
if (done) break;
|
|
3144
|
+
buffer += decoder.decode(value, { stream: true });
|
|
3145
|
+
const lines = buffer.split("\n");
|
|
3146
|
+
buffer = (_c = lines.pop()) != null ? _c : "";
|
|
3147
|
+
for (const line of lines) {
|
|
3148
|
+
const trimmed = line.trim();
|
|
3149
|
+
if (!trimmed || trimmed === "data: [DONE]") continue;
|
|
3150
|
+
if (!trimmed.startsWith("data:")) continue;
|
|
3151
|
+
try {
|
|
3152
|
+
const json = JSON.parse(trimmed.slice(5).trim());
|
|
3153
|
+
const text = resolvePath(json, extractPath);
|
|
3154
|
+
if (text && typeof text === "string") yield text;
|
|
3155
|
+
} catch (e) {
|
|
3156
|
+
}
|
|
3157
|
+
}
|
|
3158
|
+
}
|
|
3159
|
+
if (buffer.trim() && buffer.trim() !== "data: [DONE]") {
|
|
3160
|
+
const jsonStr = buffer.replace(/^data:\s*/, "").trim();
|
|
3161
|
+
try {
|
|
3162
|
+
const json = JSON.parse(jsonStr);
|
|
3163
|
+
const text = resolvePath(json, extractPath);
|
|
3164
|
+
if (text && typeof text === "string") yield text;
|
|
3165
|
+
} catch (e) {
|
|
3166
|
+
}
|
|
3167
|
+
}
|
|
3168
|
+
} finally {
|
|
3169
|
+
reader.releaseLock();
|
|
3170
|
+
}
|
|
3171
|
+
});
|
|
3172
|
+
}
|
|
2827
3173
|
async embed(text) {
|
|
2828
3174
|
var _a, _b;
|
|
2829
3175
|
const path = (_a = this.opts.embedPath) != null ? _a : "/embeddings";
|
|
@@ -3020,6 +3366,7 @@ var ProviderRegistry = class {
|
|
|
3020
3366
|
return null;
|
|
3021
3367
|
}
|
|
3022
3368
|
static async loadVectorProviderClass(provider) {
|
|
3369
|
+
var _a;
|
|
3023
3370
|
if (this.vectorProviders[provider]) return this.vectorProviders[provider];
|
|
3024
3371
|
switch (provider) {
|
|
3025
3372
|
case "pinecone": {
|
|
@@ -3028,6 +3375,11 @@ var ProviderRegistry = class {
|
|
|
3028
3375
|
}
|
|
3029
3376
|
case "pgvector":
|
|
3030
3377
|
case "postgresql": {
|
|
3378
|
+
const postgresMode = ((_a = process.env.POSTGRES_MODE) != null ? _a : "multi").toLowerCase();
|
|
3379
|
+
if (postgresMode === "single") {
|
|
3380
|
+
const { PostgreSQLProvider: PostgreSQLProvider2 } = await Promise.resolve().then(() => (init_PostgreSQLProvider(), PostgreSQLProvider_exports));
|
|
3381
|
+
return PostgreSQLProvider2;
|
|
3382
|
+
}
|
|
3031
3383
|
const { MultiTablePostgresProvider: MultiTablePostgresProvider2 } = await Promise.resolve().then(() => (init_MultiTablePostgresProvider(), MultiTablePostgresProvider_exports));
|
|
3032
3384
|
return MultiTablePostgresProvider2;
|
|
3033
3385
|
}
|
|
@@ -4027,6 +4379,76 @@ var QueryProcessor = class {
|
|
|
4027
4379
|
}
|
|
4028
4380
|
return [...hints.values()];
|
|
4029
4381
|
}
|
|
4382
|
+
static extractNumericPredicates(question, validFields = []) {
|
|
4383
|
+
const predicates = [];
|
|
4384
|
+
const seen = /* @__PURE__ */ new Set();
|
|
4385
|
+
const comparatorPattern = [
|
|
4386
|
+
"greater than or equal to",
|
|
4387
|
+
"more than or equal to",
|
|
4388
|
+
"less than or equal to",
|
|
4389
|
+
"greater than",
|
|
4390
|
+
"more than",
|
|
4391
|
+
"less than",
|
|
4392
|
+
"equal to",
|
|
4393
|
+
"at least",
|
|
4394
|
+
"at most",
|
|
4395
|
+
"above",
|
|
4396
|
+
"over",
|
|
4397
|
+
"below",
|
|
4398
|
+
"under",
|
|
4399
|
+
"equals?",
|
|
4400
|
+
">=",
|
|
4401
|
+
"<=",
|
|
4402
|
+
">",
|
|
4403
|
+
"<",
|
|
4404
|
+
"="
|
|
4405
|
+
].join("|");
|
|
4406
|
+
const addPredicate = (rawField, rawOperator, rawValue) => {
|
|
4407
|
+
const value = Number(rawValue.replace(/,/g, ""));
|
|
4408
|
+
if (!Number.isFinite(value)) return;
|
|
4409
|
+
const operator = this.normalizeNumericOperator(rawOperator);
|
|
4410
|
+
const field = rawField ? this.normalizePredicateField(rawField, validFields) : void 0;
|
|
4411
|
+
const key = `${field != null ? field : "*"}::${operator}::${value}`;
|
|
4412
|
+
if (seen.has(key)) return;
|
|
4413
|
+
seen.add(key);
|
|
4414
|
+
predicates.push(__spreadProps(__spreadValues({}, field ? { field } : {}), { operator, value }));
|
|
4415
|
+
};
|
|
4416
|
+
const scopedPatterns = [
|
|
4417
|
+
new RegExp(`\\b(?:whose|with|having|where|that\\s+have|which\\s+have)\\s+([a-zA-Z][a-zA-Z0-9_\\s\\-/]{1,80}?)\\s+(?:is|are|was|were)?\\s*(?:${comparatorPattern})\\s+([\\d,]+(?:\\.\\d+)?)`, "gi"),
|
|
4418
|
+
new RegExp(`\\b([a-zA-Z][a-zA-Z0-9_\\s\\-/]{1,80}?)\\s+(?:is|are|was|were)?\\s*(?:${comparatorPattern})\\s+([\\d,]+(?:\\.\\d+)?)`, "gi")
|
|
4419
|
+
];
|
|
4420
|
+
for (const pattern of scopedPatterns) {
|
|
4421
|
+
for (const match of question.matchAll(pattern)) {
|
|
4422
|
+
const full = match[0];
|
|
4423
|
+
const operatorMatch = full.match(new RegExp(`(${comparatorPattern})`, "i"));
|
|
4424
|
+
if (!operatorMatch) continue;
|
|
4425
|
+
addPredicate(match[1], operatorMatch[1], match[2]);
|
|
4426
|
+
}
|
|
4427
|
+
}
|
|
4428
|
+
for (const match of question.matchAll(/\b([a-zA-Z][a-zA-Z0-9_\s\-/]{1,80}?)\s*(>=|<=|>|<|=)\s*([\d,]+(?:\.\d+)?)/g)) {
|
|
4429
|
+
addPredicate(match[1], match[2], match[3]);
|
|
4430
|
+
}
|
|
4431
|
+
return predicates;
|
|
4432
|
+
}
|
|
4433
|
+
static normalizeNumericOperator(operator) {
|
|
4434
|
+
const op = operator.toLowerCase().trim();
|
|
4435
|
+
if (op === ">" || /\b(greater than|more than|above|over)\b/.test(op)) return "gt";
|
|
4436
|
+
if (op === ">=" || /\b(greater than or equal to|more than or equal to|at least)\b/.test(op)) return "gte";
|
|
4437
|
+
if (op === "<" || /\b(less than|below|under)\b/.test(op)) return "lt";
|
|
4438
|
+
if (op === "<=" || /\b(less than or equal to|at most)\b/.test(op)) return "lte";
|
|
4439
|
+
return "eq";
|
|
4440
|
+
}
|
|
4441
|
+
static normalizePredicateField(field, validFields) {
|
|
4442
|
+
const cleaned = this.normalizeHintValue(field).replace(/\b(?:provide|show|get|give|list|all|the|a|an|of|organizations?|companies?|records?|items?|whose|with|having|where|that|which|have|has|is|are|was|were)\b/gi, " ").replace(/\s+/g, " ").trim();
|
|
4443
|
+
if (validFields.length === 0) return cleaned;
|
|
4444
|
+
const comparable = (value) => value.toLowerCase().replace(/[^a-z0-9]/g, "");
|
|
4445
|
+
const cleanedComparable = comparable(cleaned);
|
|
4446
|
+
const matchedField = validFields.find((fieldName) => {
|
|
4447
|
+
const candidate = comparable(fieldName);
|
|
4448
|
+
return candidate === cleanedComparable || candidate.includes(cleanedComparable) || cleanedComparable.includes(candidate);
|
|
4449
|
+
});
|
|
4450
|
+
return matchedField != null ? matchedField : cleaned;
|
|
4451
|
+
}
|
|
4030
4452
|
/**
|
|
4031
4453
|
* Constructs a QueryFilter object from extracted hints.
|
|
4032
4454
|
*
|
|
@@ -4047,6 +4469,10 @@ var QueryProcessor = class {
|
|
|
4047
4469
|
}
|
|
4048
4470
|
if (Object.keys(filter.metadata || {}).length === 0) delete filter.metadata;
|
|
4049
4471
|
if (filter.keywords && filter.keywords.length === 0) delete filter.keywords;
|
|
4472
|
+
const numericPredicates = this.extractNumericPredicates(question);
|
|
4473
|
+
if (numericPredicates.length > 0) {
|
|
4474
|
+
filter.__numericPredicates = numericPredicates;
|
|
4475
|
+
}
|
|
4050
4476
|
return filter;
|
|
4051
4477
|
}
|
|
4052
4478
|
/**
|
|
@@ -4114,6 +4540,61 @@ Return ONLY 'vector', 'graph', or 'both'. No explanation.`;
|
|
|
4114
4540
|
}
|
|
4115
4541
|
};
|
|
4116
4542
|
|
|
4543
|
+
// src/core/LLMRouter.ts
|
|
4544
|
+
var FAST_MODEL_DEFAULTS = {
|
|
4545
|
+
openai: "gpt-4o-mini",
|
|
4546
|
+
gemini: "gemini-2.0-flash",
|
|
4547
|
+
anthropic: "claude-3-haiku-20240307",
|
|
4548
|
+
ollama: "",
|
|
4549
|
+
// Ollama has no universal lightweight default — reuse main model
|
|
4550
|
+
rest: "",
|
|
4551
|
+
universal_rest: "",
|
|
4552
|
+
custom: ""
|
|
4553
|
+
};
|
|
4554
|
+
var LLMRouter = class {
|
|
4555
|
+
constructor(config) {
|
|
4556
|
+
this.config = config;
|
|
4557
|
+
this.models = /* @__PURE__ */ new Map();
|
|
4558
|
+
}
|
|
4559
|
+
/**
|
|
4560
|
+
* Initialize all LLM roles.
|
|
4561
|
+
*
|
|
4562
|
+
* @param prebuiltDefault - optional pre-built provider (from EmbeddingStrategyResolver).
|
|
4563
|
+
* When provided it is used directly as the 'default' role without re-constructing.
|
|
4564
|
+
*/
|
|
4565
|
+
async initialize(prebuiltDefault) {
|
|
4566
|
+
var _a;
|
|
4567
|
+
const defaultModel = prebuiltDefault != null ? prebuiltDefault : LLMFactory.create(this.config.llm, this.config.embedding);
|
|
4568
|
+
this.models.set("default", defaultModel);
|
|
4569
|
+
const envFastModel = process.env.FAST_LLM_MODEL;
|
|
4570
|
+
const providerFastDefault = (_a = FAST_MODEL_DEFAULTS[this.config.llm.provider]) != null ? _a : "";
|
|
4571
|
+
const fastModelName = envFastModel || providerFastDefault;
|
|
4572
|
+
if (fastModelName && fastModelName !== this.config.llm.model) {
|
|
4573
|
+
console.log(`[LLMRouter] Fast role \u2192 provider="${this.config.llm.provider}" model="${fastModelName}"`);
|
|
4574
|
+
const fastConfig = __spreadProps(__spreadValues({}, this.config.llm), {
|
|
4575
|
+
model: fastModelName
|
|
4576
|
+
});
|
|
4577
|
+
this.models.set("fast", LLMFactory.create(fastConfig, this.config.embedding));
|
|
4578
|
+
} else {
|
|
4579
|
+
console.log(`[LLMRouter] Fast role \u2192 reusing default model (no lightweight alternative configured).`);
|
|
4580
|
+
this.models.set("fast", defaultModel);
|
|
4581
|
+
}
|
|
4582
|
+
this.models.set("powerful", defaultModel);
|
|
4583
|
+
}
|
|
4584
|
+
/**
|
|
4585
|
+
* Retrieve a model provider by its task role.
|
|
4586
|
+
* Falls back to 'default' if the requested role is not registered.
|
|
4587
|
+
*/
|
|
4588
|
+
get(role) {
|
|
4589
|
+
var _a;
|
|
4590
|
+
const provider = (_a = this.models.get(role)) != null ? _a : this.models.get("default");
|
|
4591
|
+
if (!provider) {
|
|
4592
|
+
throw new Error(`[LLMRouter] No provider registered for role: "${role}". Did you call initialize()?`);
|
|
4593
|
+
}
|
|
4594
|
+
return provider;
|
|
4595
|
+
}
|
|
4596
|
+
};
|
|
4597
|
+
|
|
4117
4598
|
// src/utils/synonyms.ts
|
|
4118
4599
|
var FIELD_SYNONYMS = {
|
|
4119
4600
|
name: ["product", "item", "title", "label", "heading", "subject", "product name", "item name"],
|
|
@@ -4143,6 +4624,14 @@ var FIELD_SYNONYMS = {
|
|
|
4143
4624
|
"in stock",
|
|
4144
4625
|
"status"
|
|
4145
4626
|
],
|
|
4627
|
+
category: [
|
|
4628
|
+
"product_category",
|
|
4629
|
+
"product category",
|
|
4630
|
+
"category_name",
|
|
4631
|
+
"category name",
|
|
4632
|
+
"department",
|
|
4633
|
+
"collection"
|
|
4634
|
+
],
|
|
4146
4635
|
description: ["summary", "content", "body", "text", "info", "details"],
|
|
4147
4636
|
link: ["url", "href", "product_url", "page_url", "link"]
|
|
4148
4637
|
};
|
|
@@ -4174,101 +4663,457 @@ function resolveMetadataValue(meta, uiKey) {
|
|
|
4174
4663
|
|
|
4175
4664
|
// src/utils/UITransformer.ts
|
|
4176
4665
|
var UITransformer = class {
|
|
4666
|
+
// ─── Public Entry Points ─────────────────────────────────────────────────
|
|
4177
4667
|
/**
|
|
4178
|
-
*
|
|
4179
|
-
*
|
|
4668
|
+
* Heuristic-only transform (no LLM required).
|
|
4669
|
+
* Uses the lightweight heuristic intent detector as a fallback.
|
|
4670
|
+
* Prefer `analyzeAndDecide()` in production.
|
|
4180
4671
|
*/
|
|
4181
|
-
static transform(userQuery, retrievedData, config, trainedSchema) {
|
|
4672
|
+
static transform(userQuery, retrievedData, config, trainedSchema, intent) {
|
|
4673
|
+
var _a, _b, _c;
|
|
4182
4674
|
if (!retrievedData || retrievedData.length === 0) {
|
|
4183
4675
|
return this.createTextResponse("No data available", "No relevant data found for your query.");
|
|
4184
4676
|
}
|
|
4185
|
-
const
|
|
4186
|
-
const filteredData =
|
|
4187
|
-
const
|
|
4188
|
-
const categories = this.detectCategories(filteredData, groupByField);
|
|
4677
|
+
const resolvedIntent = intent != null ? intent : this.detectIntentHeuristic(userQuery);
|
|
4678
|
+
const filteredData = resolvedIntent.filterInStockOnly ? retrievedData.filter((item) => this.determineStockStatus(item)) : retrievedData;
|
|
4679
|
+
const profile = this.profileData(filteredData);
|
|
4189
4680
|
const hasProducts = filteredData.some((item) => this.isProductData(item));
|
|
4190
|
-
const
|
|
4191
|
-
|
|
4192
|
-
|
|
4193
|
-
return this.transformToLineChart(filteredData);
|
|
4681
|
+
const wantsPieLikeChart = ["pie_chart", "donut_chart"].includes(resolvedIntent.recommendedChart);
|
|
4682
|
+
if (resolvedIntent.visualizationHint === "trend" && profile.dateFields.length > 0 && profile.numericFields.length > 0) {
|
|
4683
|
+
return this.transformToLineChart(profile);
|
|
4194
4684
|
}
|
|
4195
|
-
|
|
4196
|
-
|
|
4197
|
-
|
|
4685
|
+
if (wantsPieLikeChart || ["composition", "category_breakdown"].includes(resolvedIntent.visualizationHint)) {
|
|
4686
|
+
const pieChart = this.transformToPieChart(filteredData, profile, userQuery);
|
|
4687
|
+
if (pieChart) return pieChart;
|
|
4198
4688
|
}
|
|
4199
|
-
if (
|
|
4200
|
-
return this.
|
|
4689
|
+
if (["comparison", "ranking"].includes(resolvedIntent.visualizationHint)) {
|
|
4690
|
+
return this.transformToBarChart(filteredData, profile, userQuery, resolvedIntent.visualizationHint === "ranking");
|
|
4201
4691
|
}
|
|
4202
|
-
if (
|
|
4203
|
-
return this.
|
|
4692
|
+
if (resolvedIntent.visualizationHint === "distribution") {
|
|
4693
|
+
return (_a = this.transformToHistogram(profile, userQuery)) != null ? _a : this.transformToBarChart(filteredData, profile, userQuery);
|
|
4694
|
+
}
|
|
4695
|
+
if (resolvedIntent.visualizationHint === "correlation") {
|
|
4696
|
+
return (_b = this.transformToScatterPlot(profile, userQuery)) != null ? _b : this.transformToBarChart(filteredData, profile, userQuery);
|
|
4697
|
+
}
|
|
4698
|
+
if (resolvedIntent.visualizationHint === "geographic") {
|
|
4699
|
+
return this.transformToBarChart(filteredData, profile, userQuery);
|
|
4204
4700
|
}
|
|
4205
|
-
if (this.
|
|
4206
|
-
return this.transformToTable(filteredData);
|
|
4701
|
+
if (this.isStructuredListQuery(userQuery)) {
|
|
4702
|
+
return this.hasMultipleFields(filteredData) ? this.transformToTable(filteredData, userQuery) : this.transformToText(filteredData);
|
|
4703
|
+
}
|
|
4704
|
+
if (resolvedIntent.visualizationHint === "kpi") {
|
|
4705
|
+
return (_c = this.transformToMetricCard(profile, userQuery)) != null ? _c : this.transformToText(filteredData);
|
|
4706
|
+
}
|
|
4707
|
+
if (["tabular", "table"].includes(resolvedIntent.visualizationHint) || resolvedIntent.wantsExplicitTable) {
|
|
4708
|
+
return this.hasMultipleFields(filteredData) ? this.transformToTable(filteredData, userQuery) : this.transformToText(filteredData);
|
|
4709
|
+
}
|
|
4710
|
+
if ((hasProducts || this.isProductQuery(userQuery)) && resolvedIntent.visualizationHint === "product_browse") {
|
|
4711
|
+
return this.transformToProductCarousel(filteredData, config, trainedSchema);
|
|
4207
4712
|
}
|
|
4713
|
+
const automatic = this.chooseAutomaticVisualization(filteredData, profile, userQuery);
|
|
4714
|
+
if (automatic) return automatic;
|
|
4208
4715
|
return this.transformToText(filteredData);
|
|
4209
4716
|
}
|
|
4210
4717
|
/**
|
|
4211
|
-
*
|
|
4212
|
-
|
|
4213
|
-
|
|
4214
|
-
|
|
4215
|
-
|
|
4216
|
-
type: "product_carousel",
|
|
4217
|
-
title: "Recommended Products",
|
|
4218
|
-
description: `Found ${products.length} relevant products`,
|
|
4219
|
-
data: products
|
|
4220
|
-
};
|
|
4221
|
-
}
|
|
4222
|
-
/**
|
|
4223
|
-
* Transform data to pie chart format
|
|
4718
|
+
* LLM-driven entry point (recommended for production).
|
|
4719
|
+
*
|
|
4720
|
+
* Step 1 — Detect intent via a dedicated, lightweight LLM call.
|
|
4721
|
+
* Step 2 — Pass the intent + data to the visualization-selection prompt.
|
|
4722
|
+
* Step 3 — Fall back to the heuristic `transform()` if either LLM call fails.
|
|
4224
4723
|
*/
|
|
4225
|
-
static
|
|
4226
|
-
let
|
|
4227
|
-
|
|
4228
|
-
|
|
4229
|
-
|
|
4230
|
-
|
|
4231
|
-
|
|
4232
|
-
|
|
4233
|
-
|
|
4234
|
-
|
|
4235
|
-
|
|
4236
|
-
|
|
4237
|
-
|
|
4238
|
-
|
|
4724
|
+
static async analyzeAndDecide(query, sources, llm) {
|
|
4725
|
+
let intent;
|
|
4726
|
+
try {
|
|
4727
|
+
intent = await this.detectIntent(query, llm);
|
|
4728
|
+
console.debug("[UITransformer] Detected intent:", intent);
|
|
4729
|
+
} catch (err) {
|
|
4730
|
+
console.warn("[UITransformer] Intent detection failed; using heuristic.", err);
|
|
4731
|
+
intent = this.detectIntentHeuristic(query);
|
|
4732
|
+
}
|
|
4733
|
+
if (this.isProductQuery(query) && ["text", "product_browse"].includes(intent.visualizationHint)) {
|
|
4734
|
+
return this.transform(
|
|
4735
|
+
query,
|
|
4736
|
+
sources,
|
|
4737
|
+
void 0,
|
|
4738
|
+
void 0,
|
|
4739
|
+
__spreadProps(__spreadValues({}, intent), { visualizationHint: "product_browse", recommendedChart: "text" })
|
|
4740
|
+
);
|
|
4741
|
+
}
|
|
4742
|
+
try {
|
|
4743
|
+
const context = this.buildContextSummary(sources);
|
|
4744
|
+
const systemPrompt = this.buildVisualizationSystemPrompt();
|
|
4745
|
+
const userPrompt = [
|
|
4746
|
+
`USER QUESTION: ${query}`,
|
|
4747
|
+
"",
|
|
4748
|
+
`DETECTED INTENT (JSON): ${JSON.stringify(intent)}`,
|
|
4749
|
+
"",
|
|
4750
|
+
"RETRIEVED DATA (JSON):",
|
|
4751
|
+
context
|
|
4752
|
+
].join("\n");
|
|
4753
|
+
const rawResponse = await llm.chat(
|
|
4754
|
+
[{ role: "user", content: userPrompt }],
|
|
4755
|
+
"",
|
|
4756
|
+
{ systemPrompt, temperature: 0 }
|
|
4757
|
+
);
|
|
4758
|
+
const parsed = this.parseTransformationResponse(rawResponse);
|
|
4759
|
+
if (parsed) {
|
|
4760
|
+
const intentAllowsTable = intent.wantsExplicitTable || ["tabular", "table", "geographic"].includes(intent.visualizationHint);
|
|
4761
|
+
const intentWantsPieLikeChart = ["pie_chart", "donut_chart"].includes(intent.recommendedChart) || ["composition", "category_breakdown"].includes(intent.visualizationHint);
|
|
4762
|
+
if (parsed.type === "table" && !intentAllowsTable) {
|
|
4763
|
+
console.debug("[UITransformer] LLM chose table but intent says no. Falling back to text.");
|
|
4764
|
+
return this.transform(query, sources, void 0, void 0, intent);
|
|
4765
|
+
}
|
|
4766
|
+
if (intentWantsPieLikeChart && parsed.type !== "pie_chart" && this.detectCategories(sources).length > 1) {
|
|
4767
|
+
console.debug("[UITransformer] LLM ignored pie/composition intent. Using deterministic pie chart.");
|
|
4768
|
+
return this.transform(query, sources, void 0, void 0, intent);
|
|
4769
|
+
}
|
|
4770
|
+
console.debug("[UITransformer] LLM chose visualization type:", parsed.type);
|
|
4771
|
+
return parsed;
|
|
4772
|
+
}
|
|
4773
|
+
console.warn("[UITransformer] LLM returned unparseable response; falling back to heuristic.");
|
|
4774
|
+
} catch (err) {
|
|
4775
|
+
console.warn("[UITransformer] analyzeAndDecide LLM call failed; falling back to heuristic.", err);
|
|
4776
|
+
}
|
|
4777
|
+
return this.transform(query, sources, void 0, void 0, intent);
|
|
4778
|
+
}
|
|
4779
|
+
// ─── Dynamic Intent Detection ─────────────────────────────────────────────
|
|
4780
|
+
/**
|
|
4781
|
+
* Calls the LLM with a compact, focused prompt to extract a structured
|
|
4782
|
+
* `QueryIntent` from the user's query.
|
|
4783
|
+
*
|
|
4784
|
+
* Keeping this as a *separate* call from visualization selection means:
|
|
4785
|
+
* - The prompt is shorter and more reliable.
|
|
4786
|
+
* - The intent object can be reused across both the heuristic and LLM paths.
|
|
4787
|
+
* - It is easy to unit-test intent detection in isolation.
|
|
4788
|
+
*/
|
|
4789
|
+
static async detectIntent(query, llm) {
|
|
4790
|
+
const systemPrompt = `You are an intent classifier for a product-search RAG system.
|
|
4791
|
+
Given a user query, return ONLY a valid JSON object with this exact shape \u2014 no prose, no markdown:
|
|
4792
|
+
|
|
4793
|
+
{
|
|
4794
|
+
"visualizationHint": "trend" | "comparison" | "distribution" | "composition" | "correlation" | "ranking" | "kpi" | "tabular" | "geographic" | "product_browse" | "table" | "text",
|
|
4795
|
+
"recommendedChart": "line_chart" | "bar_chart" | "histogram" | "pie_chart" | "donut_chart" | "scatter_plot" | "horizontal_bar" | "metric_card" | "table" | "geo_map" | "text",
|
|
4796
|
+
"filterInStockOnly": boolean,
|
|
4797
|
+
"wantsExplicitTable": boolean,
|
|
4798
|
+
"isTemporal": boolean,
|
|
4799
|
+
"isComparison": boolean,
|
|
4800
|
+
"language": "<BCP-47 language tag, e.g. en, de, hi, ja>",
|
|
4801
|
+
"reasoning": "<one short sentence \u2014 why you chose these values>"
|
|
4802
|
+
}
|
|
4803
|
+
|
|
4804
|
+
RULES:
|
|
4805
|
+
- visualizationHint and recommendedChart
|
|
4806
|
+
"trend" \u2192 keywords: trend, growth, over time; recommendedChart "line_chart"
|
|
4807
|
+
"comparison" \u2192 keywords: compare, versus, top; recommendedChart "bar_chart"
|
|
4808
|
+
"distribution" \u2192 keywords: distribution, spread; recommendedChart "histogram"
|
|
4809
|
+
"composition" \u2192 keywords: share, percentage, breakup, breakdown; recommendedChart "pie_chart" or "donut_chart"
|
|
4810
|
+
"correlation" \u2192 keywords: relation, correlation; recommendedChart "scatter_plot"
|
|
4811
|
+
"ranking" \u2192 keywords: highest, lowest, top 10, ranking; recommendedChart "horizontal_bar"
|
|
4812
|
+
"kpi" \u2192 keywords: total, average, count; recommendedChart "metric_card"
|
|
4813
|
+
"tabular" \u2192 keywords: detailed records, table, grid, spreadsheet; recommendedChart "table"
|
|
4814
|
+
"geographic" \u2192 keywords: region, country, map; recommendedChart "geo_map"
|
|
4815
|
+
"product_browse" \u2192 user is browsing, searching, describing, viewing details for, or asking about one or more products without analytical visualization intent
|
|
4816
|
+
"table" \u2192 legacy alias for "tabular"; use only if the query literally says table
|
|
4817
|
+
"text" \u2192 conversational, factual, or other intent
|
|
4818
|
+
- If the user explicitly asks for a chart type, honor that chart type in recommendedChart.
|
|
4819
|
+
- If a query says "distribution ... in a pie chart", use visualizationHint "composition" and recommendedChart "pie_chart".
|
|
4820
|
+
- filterInStockOnly: true only if user mentions stock, availability, in stock, etc.
|
|
4821
|
+
- wantsExplicitTable: true if the user asks for a table, list, grid, spreadsheet, or detailed records.
|
|
4822
|
+
- isTemporal: true if time words appear (trend, historical, over time, last year, monthly, etc.)
|
|
4823
|
+
- isComparison: true if user compares, ranks, or contrasts entities or categories.
|
|
4824
|
+
- language: detect from the query text itself; default "en" if uncertain.`;
|
|
4825
|
+
const rawResponse = await llm.chat(
|
|
4826
|
+
[{ role: "user", content: `QUERY: ${query}` }],
|
|
4827
|
+
"",
|
|
4828
|
+
{ systemPrompt, temperature: 0 }
|
|
4829
|
+
);
|
|
4830
|
+
const parsed = this.parseIntentResponse(rawResponse);
|
|
4831
|
+
if (!parsed) {
|
|
4832
|
+
throw new Error(`Could not parse intent JSON from LLM response: ${rawResponse}`);
|
|
4833
|
+
}
|
|
4834
|
+
return parsed;
|
|
4835
|
+
}
|
|
4836
|
+
/**
|
|
4837
|
+
* Parse and validate the raw LLM response into a `QueryIntent`.
|
|
4838
|
+
*/
|
|
4839
|
+
static parseIntentResponse(raw) {
|
|
4840
|
+
const jsonStr = this.extractJsonCandidate(raw);
|
|
4841
|
+
if (!jsonStr) return null;
|
|
4842
|
+
try {
|
|
4843
|
+
const obj = JSON.parse(jsonStr);
|
|
4844
|
+
const validHints = [
|
|
4845
|
+
"trend",
|
|
4846
|
+
"comparison",
|
|
4847
|
+
"distribution",
|
|
4848
|
+
"composition",
|
|
4849
|
+
"correlation",
|
|
4850
|
+
"ranking",
|
|
4851
|
+
"kpi",
|
|
4852
|
+
"tabular",
|
|
4853
|
+
"geographic",
|
|
4854
|
+
"category_breakdown",
|
|
4855
|
+
"product_browse",
|
|
4856
|
+
"table",
|
|
4857
|
+
"text"
|
|
4858
|
+
];
|
|
4859
|
+
const validCharts = [
|
|
4860
|
+
"line_chart",
|
|
4861
|
+
"bar_chart",
|
|
4862
|
+
"histogram",
|
|
4863
|
+
"pie_chart",
|
|
4864
|
+
"donut_chart",
|
|
4865
|
+
"scatter_plot",
|
|
4866
|
+
"horizontal_bar",
|
|
4867
|
+
"metric_card",
|
|
4868
|
+
"table",
|
|
4869
|
+
"geo_map",
|
|
4870
|
+
"text"
|
|
4871
|
+
];
|
|
4872
|
+
const hint = obj.visualizationHint;
|
|
4873
|
+
if (!hint || !validHints.includes(hint)) return null;
|
|
4874
|
+
const normalizedHint = hint === "category_breakdown" ? "composition" : hint;
|
|
4875
|
+
const recommendedChart = typeof obj.recommendedChart === "string" && validCharts.includes(obj.recommendedChart) ? obj.recommendedChart : this.getRecommendedChartForIntent(normalizedHint);
|
|
4876
|
+
return {
|
|
4877
|
+
visualizationHint: normalizedHint,
|
|
4878
|
+
recommendedChart,
|
|
4879
|
+
filterInStockOnly: Boolean(obj.filterInStockOnly),
|
|
4880
|
+
wantsExplicitTable: Boolean(obj.wantsExplicitTable),
|
|
4881
|
+
isTemporal: Boolean(obj.isTemporal),
|
|
4882
|
+
isComparison: Boolean(obj.isComparison),
|
|
4883
|
+
language: typeof obj.language === "string" && obj.language ? obj.language : "en",
|
|
4884
|
+
reasoning: typeof obj.reasoning === "string" ? obj.reasoning : void 0
|
|
4885
|
+
};
|
|
4886
|
+
} catch (e) {
|
|
4887
|
+
return null;
|
|
4888
|
+
}
|
|
4889
|
+
}
|
|
4890
|
+
/**
|
|
4891
|
+
* Heuristic intent detector — used when the LLM is unavailable.
|
|
4892
|
+
* Intentionally minimal: it should only catch the most obvious signals.
|
|
4893
|
+
* The LLM path handles everything subtle.
|
|
4894
|
+
*/
|
|
4895
|
+
static detectIntentHeuristic(query) {
|
|
4896
|
+
const q = query.toLowerCase();
|
|
4897
|
+
const isTemporal = /\b(trend|trends|over time|historical|history|growth|decline|monthly|yearly|weekly|daily|last (year|month|week)|timeline|forecast)\b/.test(q);
|
|
4898
|
+
const isRanking = /\b(highest|lowest|top\s*\d+|bottom\s*\d+|rank(?:ing)?|best|worst|leading)\b/.test(q);
|
|
4899
|
+
const isComparison = /\b(compare|comparison|vs\.?|versus|against|differ|difference|contrast|top)\b/.test(q) || isRanking;
|
|
4900
|
+
const isDistribution = /\b(distribution|spread|histogram|frequency|variance|range)\b/.test(q);
|
|
4901
|
+
const wantsPieLikeChart = /\b(pie|donut|doughnut)(?:\s+chart)?\b/.test(q);
|
|
4902
|
+
const isComposition = wantsPieLikeChart || /\b(share|percentage|percent|breakup|breakdown|composition|split|segmentation|by category|by type|proportion)\b/.test(q);
|
|
4903
|
+
const isCorrelation = /\b(relation|relationship|correlation|correlate|scatter|association|impact of|depend(?:s|ence)? on)\b/.test(q);
|
|
4904
|
+
const isKpi = /\b(total|average|avg|count|sum|median|minimum|maximum|metric|kpi|how many|number of)\b/.test(q);
|
|
4905
|
+
const isGeographic = /\b(region|country|countries|state|city|location|map|geo|geographic|territory)\b/.test(q);
|
|
4906
|
+
const filterInStockOnly = /\b(in[- ]?stock|available|availability|inventory|stock status)\b/.test(q);
|
|
4907
|
+
const wantsExplicitTable = /\b(table|spreadsheet|grid|detailed records|record details|list all|compare all)\b/.test(q);
|
|
4908
|
+
let visualizationHint = "text";
|
|
4909
|
+
if (wantsExplicitTable) visualizationHint = "tabular";
|
|
4910
|
+
else if (isTemporal) visualizationHint = "trend";
|
|
4911
|
+
else if (wantsPieLikeChart) visualizationHint = "composition";
|
|
4912
|
+
else if (isRanking) visualizationHint = "ranking";
|
|
4913
|
+
else if (isComparison) visualizationHint = "comparison";
|
|
4914
|
+
else if (isDistribution) visualizationHint = "distribution";
|
|
4915
|
+
else if (isComposition) visualizationHint = "composition";
|
|
4916
|
+
else if (isCorrelation) visualizationHint = "correlation";
|
|
4917
|
+
else if (isGeographic) visualizationHint = "geographic";
|
|
4918
|
+
else if (isKpi) visualizationHint = "kpi";
|
|
4919
|
+
else if (this.isProductQuery(query)) visualizationHint = "product_browse";
|
|
4920
|
+
return {
|
|
4921
|
+
visualizationHint,
|
|
4922
|
+
recommendedChart: this.getRecommendedChartForIntent(visualizationHint),
|
|
4923
|
+
filterInStockOnly,
|
|
4924
|
+
wantsExplicitTable,
|
|
4925
|
+
isTemporal,
|
|
4926
|
+
isComparison,
|
|
4927
|
+
language: "en"
|
|
4928
|
+
// heuristic cannot reliably detect language
|
|
4929
|
+
};
|
|
4930
|
+
}
|
|
4931
|
+
static getRecommendedChartForIntent(intent) {
|
|
4932
|
+
switch (intent) {
|
|
4933
|
+
case "trend":
|
|
4934
|
+
return "line_chart";
|
|
4935
|
+
case "comparison":
|
|
4936
|
+
return "bar_chart";
|
|
4937
|
+
case "distribution":
|
|
4938
|
+
return "histogram";
|
|
4939
|
+
case "composition":
|
|
4940
|
+
case "category_breakdown":
|
|
4941
|
+
return "pie_chart";
|
|
4942
|
+
case "correlation":
|
|
4943
|
+
return "scatter_plot";
|
|
4944
|
+
case "ranking":
|
|
4945
|
+
return "horizontal_bar";
|
|
4946
|
+
case "kpi":
|
|
4947
|
+
return "metric_card";
|
|
4948
|
+
case "tabular":
|
|
4949
|
+
case "table":
|
|
4950
|
+
return "table";
|
|
4951
|
+
case "geographic":
|
|
4952
|
+
return "geo_map";
|
|
4953
|
+
case "product_browse":
|
|
4954
|
+
case "text":
|
|
4955
|
+
default:
|
|
4956
|
+
return "text";
|
|
4957
|
+
}
|
|
4958
|
+
}
|
|
4959
|
+
// ─── Transform Helpers ────────────────────────────────────────────────────
|
|
4960
|
+
static transformToProductCarousel(data, config, trainedSchema) {
|
|
4961
|
+
const products = data.map((item) => this.extractProductInfo(item, config, trainedSchema)).filter((p) => p !== null);
|
|
4962
|
+
return {
|
|
4963
|
+
type: "product_carousel",
|
|
4964
|
+
title: "Recommended Products",
|
|
4965
|
+
description: `Found ${products.length} relevant products`,
|
|
4966
|
+
data: products
|
|
4967
|
+
};
|
|
4968
|
+
}
|
|
4969
|
+
static transformToPieChart(data, profile, query = "") {
|
|
4970
|
+
var _a;
|
|
4971
|
+
const dimension = profile ? this.selectDimensionField(profile, query) : void 0;
|
|
4972
|
+
const categories = dimension ? Array.from(new Set(profile.records.map((record) => {
|
|
4973
|
+
var _a2;
|
|
4974
|
+
return String((_a2 = record.fields[dimension.key]) != null ? _a2 : "");
|
|
4975
|
+
}).filter(Boolean))) : this.detectCategories(data);
|
|
4976
|
+
if (categories.length === 0) return null;
|
|
4977
|
+
const categoryData = dimension && profile ? this.aggregateProfileByDimension(profile, dimension.key) : this.aggregateByCategory(data, categories);
|
|
4978
|
+
const pieData = Object.entries(categoryData).map(([label, count]) => {
|
|
4979
|
+
const { inStockCount, outOfStockCount } = this.calculateStockCounts(label, data);
|
|
4980
|
+
return { label, value: count, inStockCount, outOfStockCount };
|
|
4981
|
+
});
|
|
4982
|
+
return {
|
|
4239
4983
|
type: "pie_chart",
|
|
4240
|
-
title: `Distribution by ${
|
|
4241
|
-
description: `Showing breakdown across ${
|
|
4984
|
+
title: `Distribution by ${(_a = dimension == null ? void 0 : dimension.label) != null ? _a : "Category"}`,
|
|
4985
|
+
description: `Showing breakdown across ${pieData.length} categories`,
|
|
4242
4986
|
data: pieData
|
|
4243
4987
|
};
|
|
4244
4988
|
}
|
|
4245
|
-
|
|
4246
|
-
|
|
4247
|
-
|
|
4248
|
-
|
|
4249
|
-
|
|
4250
|
-
|
|
4251
|
-
timestamp:
|
|
4252
|
-
value:
|
|
4253
|
-
|
|
4254
|
-
})
|
|
4989
|
+
static transformToLineChart(profile) {
|
|
4990
|
+
const dateField = profile.dateFields[0];
|
|
4991
|
+
const valueField = profile.numericFields[0];
|
|
4992
|
+
const buckets = /* @__PURE__ */ new Map();
|
|
4993
|
+
profile.records.forEach((record) => {
|
|
4994
|
+
var _a, _b, _c;
|
|
4995
|
+
const timestamp = String((_a = record.fields[dateField.key]) != null ? _a : "");
|
|
4996
|
+
const value = (_b = this.toFiniteNumber(record.fields[valueField.key])) != null ? _b : 0;
|
|
4997
|
+
buckets.set(timestamp, ((_c = buckets.get(timestamp)) != null ? _c : 0) + value);
|
|
4998
|
+
});
|
|
4999
|
+
const lineData = Array.from(buckets.entries()).sort(([a], [b]) => Date.parse(a) - Date.parse(b)).slice(0, 24).map(([timestamp, value]) => ({ timestamp, value, label: timestamp }));
|
|
4255
5000
|
return {
|
|
4256
5001
|
type: "line_chart",
|
|
4257
|
-
title:
|
|
5002
|
+
title: `${valueField.label} Over Time`,
|
|
4258
5003
|
description: `Showing ${lineData.length} data points`,
|
|
4259
5004
|
data: lineData
|
|
4260
5005
|
};
|
|
4261
5006
|
}
|
|
4262
|
-
|
|
4263
|
-
|
|
4264
|
-
|
|
4265
|
-
|
|
4266
|
-
const
|
|
4267
|
-
const
|
|
4268
|
-
const
|
|
4269
|
-
|
|
4270
|
-
|
|
5007
|
+
static transformToBarChart(data, profile, query = "", horizontal = false) {
|
|
5008
|
+
var _a;
|
|
5009
|
+
const dimension = profile ? this.selectDimensionField(profile, query) : void 0;
|
|
5010
|
+
const measure = profile ? this.selectNumericField(profile, query) : void 0;
|
|
5011
|
+
const aggregate = dimension && profile ? this.aggregateProfileByDimension(profile, dimension.key, measure == null ? void 0 : measure.key) : this.aggregateByCategory(data, this.detectCategories(data));
|
|
5012
|
+
const barData = Object.entries(aggregate).map(([category, value]) => ({ category, value: Number(value) })).sort((a, b) => horizontal ? b.value - a.value : 0).slice(0, 12);
|
|
5013
|
+
const fallbackData = barData.length > 0 ? barData : data.slice(0, 12).map((item, index) => {
|
|
5014
|
+
var _a2, _b, _c, _d, _e;
|
|
5015
|
+
const meta = item.metadata || {};
|
|
5016
|
+
const label = String(
|
|
5017
|
+
(_c = (_b = (_a2 = this.getDynamicVal(meta, "name")) != null ? _a2 : meta.label) != null ? _b : meta.title) != null ? _c : `Result ${index + 1}`
|
|
5018
|
+
);
|
|
5019
|
+
const value = (_e = (_d = this.extractNumericValue(meta)) != null ? _d : item.score) != null ? _e : 0;
|
|
5020
|
+
return { category: label, value: Number(value) };
|
|
5021
|
+
});
|
|
5022
|
+
return {
|
|
5023
|
+
type: horizontal ? "horizontal_bar" : "bar_chart",
|
|
5024
|
+
title: dimension ? `${(_a = measure == null ? void 0 : measure.label) != null ? _a : "Count"} by ${dimension.label}` : "Comparison",
|
|
5025
|
+
description: `Showing ${fallbackData.length} comparable values`,
|
|
5026
|
+
data: fallbackData
|
|
4271
5027
|
};
|
|
5028
|
+
}
|
|
5029
|
+
static transformToHistogram(profile, query = "") {
|
|
5030
|
+
const field = this.selectNumericField(profile, query);
|
|
5031
|
+
if (!field) return null;
|
|
5032
|
+
const values = profile.records.map((record) => this.toFiniteNumber(record.fields[field.key])).filter((value) => value !== null).sort((a, b) => a - b);
|
|
5033
|
+
if (values.length === 0) return null;
|
|
5034
|
+
const min = values[0];
|
|
5035
|
+
const max = values[values.length - 1];
|
|
5036
|
+
const bucketCount = Math.min(10, Math.max(3, Math.ceil(Math.sqrt(values.length))));
|
|
5037
|
+
const width = max === min ? 1 : (max - min) / bucketCount;
|
|
5038
|
+
const buckets = Array.from({ length: bucketCount }, (_, index) => {
|
|
5039
|
+
const start = min + index * width;
|
|
5040
|
+
const end = index === bucketCount - 1 ? max : start + width;
|
|
5041
|
+
return { category: `${this.formatNumber(start)}-${this.formatNumber(end)}`, value: 0 };
|
|
5042
|
+
});
|
|
5043
|
+
values.forEach((value) => {
|
|
5044
|
+
const bucketIndex = max === min ? 0 : Math.min(bucketCount - 1, Math.floor((value - min) / width));
|
|
5045
|
+
buckets[bucketIndex].value += 1;
|
|
5046
|
+
});
|
|
5047
|
+
return {
|
|
5048
|
+
type: "histogram",
|
|
5049
|
+
title: `${field.label} Distribution`,
|
|
5050
|
+
description: `Showing ${values.length} values across ${bucketCount} buckets`,
|
|
5051
|
+
data: buckets
|
|
5052
|
+
};
|
|
5053
|
+
}
|
|
5054
|
+
static transformToScatterPlot(profile, query = "") {
|
|
5055
|
+
const fields = this.rankFieldsByQuery(profile.numericFields, query).slice(0, 2);
|
|
5056
|
+
if (fields.length < 2) return null;
|
|
5057
|
+
const [xField, yField] = fields;
|
|
5058
|
+
const points = profile.records.map((record) => {
|
|
5059
|
+
var _a;
|
|
5060
|
+
const x = this.toFiniteNumber(record.fields[xField.key]);
|
|
5061
|
+
const y = this.toFiniteNumber(record.fields[yField.key]);
|
|
5062
|
+
if (x === null || y === null) return null;
|
|
5063
|
+
return { x, y, label: String((_a = this.getRecordLabel(record)) != null ? _a : record.id) };
|
|
5064
|
+
}).filter((point) => point !== null).slice(0, 100);
|
|
5065
|
+
if (points.length === 0) return null;
|
|
5066
|
+
return {
|
|
5067
|
+
type: "scatter_plot",
|
|
5068
|
+
title: `${xField.label} vs ${yField.label}`,
|
|
5069
|
+
description: `Showing ${points.length} paired values`,
|
|
5070
|
+
data: points
|
|
5071
|
+
};
|
|
5072
|
+
}
|
|
5073
|
+
static transformToMetricCard(profile, query = "") {
|
|
5074
|
+
const operation = this.detectAggregationOperation(query);
|
|
5075
|
+
const numericField = this.selectNumericField(profile, query);
|
|
5076
|
+
const values = numericField ? profile.records.map((record) => this.toFiniteNumber(record.fields[numericField.key])).filter((value2) => value2 !== null) : [];
|
|
5077
|
+
const value = operation === "count" || values.length === 0 ? profile.records.length : this.calculateAggregate(values, operation);
|
|
5078
|
+
const metric = {
|
|
5079
|
+
label: numericField ? `${operation} ${numericField.label}` : "Count",
|
|
5080
|
+
value,
|
|
5081
|
+
operation
|
|
5082
|
+
};
|
|
5083
|
+
return {
|
|
5084
|
+
type: "metric_card",
|
|
5085
|
+
title: metric.label,
|
|
5086
|
+
description: `Calculated from ${profile.records.length} result${profile.records.length === 1 ? "" : "s"}`,
|
|
5087
|
+
data: metric
|
|
5088
|
+
};
|
|
5089
|
+
}
|
|
5090
|
+
static transformToRadarChart(data) {
|
|
5091
|
+
const attributeMap = {};
|
|
5092
|
+
data.forEach((item) => {
|
|
5093
|
+
var _a, _b, _c;
|
|
5094
|
+
const meta = item.metadata || {};
|
|
5095
|
+
const seriesName = String((_c = (_b = (_a = meta.name) != null ? _a : meta.product) != null ? _b : item.id) != null ? _c : "Item");
|
|
5096
|
+
Object.entries(meta).forEach(([key, val]) => {
|
|
5097
|
+
if (typeof val === "number" && !["price", "id"].includes(key.toLowerCase())) {
|
|
5098
|
+
if (!attributeMap[key]) attributeMap[key] = {};
|
|
5099
|
+
attributeMap[key][seriesName] = val;
|
|
5100
|
+
}
|
|
5101
|
+
});
|
|
5102
|
+
});
|
|
5103
|
+
const radarData = Object.entries(attributeMap).map(([attribute, series]) => __spreadValues({
|
|
5104
|
+
attribute
|
|
5105
|
+
}, series));
|
|
5106
|
+
return {
|
|
5107
|
+
type: "radar_chart",
|
|
5108
|
+
title: "Product Comparison",
|
|
5109
|
+
description: `Comparing ${data.length} items across ${radarData.length} attributes`,
|
|
5110
|
+
data: radarData.length > 0 ? radarData : data.map((d) => ({ attribute: d.content.substring(0, 40) }))
|
|
5111
|
+
};
|
|
5112
|
+
}
|
|
5113
|
+
static transformToTable(data, query = "") {
|
|
5114
|
+
const columns = this.extractTableColumns(data, query);
|
|
5115
|
+
const rows = data.map((item) => this.extractTableRow(item, columns));
|
|
5116
|
+
const tableData = { columns, rows };
|
|
4272
5117
|
return {
|
|
4273
5118
|
type: "table",
|
|
4274
5119
|
title: "Detailed Results",
|
|
@@ -4276,28 +5121,17 @@ var UITransformer = class {
|
|
|
4276
5121
|
data: tableData
|
|
4277
5122
|
};
|
|
4278
5123
|
}
|
|
4279
|
-
/**
|
|
4280
|
-
* Transform data to text format (fallback)
|
|
4281
|
-
*/
|
|
4282
5124
|
static transformToText(data) {
|
|
4283
|
-
const textContent = data.map((item) => item.content).join("\n\n");
|
|
4284
5125
|
return this.createTextResponse(
|
|
4285
5126
|
"Search Results",
|
|
4286
|
-
|
|
5127
|
+
data.map((item) => item.content).join("\n\n"),
|
|
4287
5128
|
`Found ${data.length} relevant results`
|
|
4288
5129
|
);
|
|
4289
5130
|
}
|
|
4290
|
-
/**
|
|
4291
|
-
* Helper: Create text response
|
|
4292
|
-
*/
|
|
4293
5131
|
static createTextResponse(title, content, description) {
|
|
4294
|
-
return {
|
|
4295
|
-
type: "text",
|
|
4296
|
-
title,
|
|
4297
|
-
description,
|
|
4298
|
-
data: { content }
|
|
4299
|
-
};
|
|
5132
|
+
return { type: "text", title, description, data: { content } };
|
|
4300
5133
|
}
|
|
5134
|
+
// ─── LLM Response Parsing ─────────────────────────────────────────────────
|
|
4301
5135
|
static parseTransformationResponse(raw) {
|
|
4302
5136
|
const payloadText = this.extractJsonCandidate(raw);
|
|
4303
5137
|
if (!payloadText) return null;
|
|
@@ -4334,9 +5168,7 @@ var UITransformer = class {
|
|
|
4334
5168
|
if (char === "{") depth += 1;
|
|
4335
5169
|
if (char === "}") {
|
|
4336
5170
|
depth -= 1;
|
|
4337
|
-
if (depth === 0)
|
|
4338
|
-
return cleaned.slice(start, i + 1);
|
|
4339
|
-
}
|
|
5171
|
+
if (depth === 0) return cleaned.slice(start, i + 1);
|
|
4340
5172
|
}
|
|
4341
5173
|
}
|
|
4342
5174
|
return null;
|
|
@@ -4344,19 +5176,16 @@ var UITransformer = class {
|
|
|
4344
5176
|
static normalizeTransformation(payload) {
|
|
4345
5177
|
var _a, _b, _c, _d, _e, _f, _g, _h, _i, _j;
|
|
4346
5178
|
if (!payload || typeof payload !== "object") return null;
|
|
4347
|
-
const
|
|
4348
|
-
const type = this.normalizeVisualizationType(
|
|
5179
|
+
const p = payload;
|
|
5180
|
+
const type = this.normalizeVisualizationType(
|
|
5181
|
+
String((_c = (_b = (_a = p.type) != null ? _a : p.view) != null ? _b : p.chartType) != null ? _c : "")
|
|
5182
|
+
);
|
|
4349
5183
|
if (!type) return null;
|
|
4350
|
-
const title = String((_e = (_d =
|
|
4351
|
-
const description =
|
|
4352
|
-
const rawData = (_j = (_i = (_h = (_g = (_f =
|
|
5184
|
+
const title = String((_e = (_d = p.title) != null ? _d : p.heading) != null ? _e : "Visualization");
|
|
5185
|
+
const description = p.description ? String(p.description) : void 0;
|
|
5186
|
+
const rawData = (_j = (_i = (_h = (_g = (_f = p.data) != null ? _f : p.table) != null ? _g : p.rows) != null ? _h : p.items) != null ? _i : p.content) != null ? _j : null;
|
|
4353
5187
|
const data = type === "text" && typeof rawData === "string" ? { content: rawData } : rawData;
|
|
4354
|
-
const transformation = {
|
|
4355
|
-
type,
|
|
4356
|
-
title,
|
|
4357
|
-
description,
|
|
4358
|
-
data
|
|
4359
|
-
};
|
|
5188
|
+
const transformation = { type, title, description, data };
|
|
4360
5189
|
return this.validateTransformation(transformation) ? transformation : null;
|
|
4361
5190
|
}
|
|
4362
5191
|
static normalizeVisualizationType(type) {
|
|
@@ -4366,10 +5195,23 @@ var UITransformer = class {
|
|
|
4366
5195
|
pie_chart: "pie_chart",
|
|
4367
5196
|
bar: "bar_chart",
|
|
4368
5197
|
bar_chart: "bar_chart",
|
|
5198
|
+
histogram: "histogram",
|
|
5199
|
+
horizontal_bar: "horizontal_bar",
|
|
5200
|
+
horizontalbar: "horizontal_bar",
|
|
4369
5201
|
line: "line_chart",
|
|
4370
5202
|
line_chart: "line_chart",
|
|
5203
|
+
scatter: "scatter_plot",
|
|
5204
|
+
scatter_plot: "scatter_plot",
|
|
5205
|
+
scatterplot: "scatter_plot",
|
|
4371
5206
|
radar: "radar_chart",
|
|
4372
5207
|
radar_chart: "radar_chart",
|
|
5208
|
+
metric: "metric_card",
|
|
5209
|
+
metric_card: "metric_card",
|
|
5210
|
+
card: "metric_card",
|
|
5211
|
+
kpi: "metric_card",
|
|
5212
|
+
geo: "geo_map",
|
|
5213
|
+
geo_map: "geo_map",
|
|
5214
|
+
map: "geo_map",
|
|
4373
5215
|
table: "table",
|
|
4374
5216
|
text: "text",
|
|
4375
5217
|
product_carousel: "product_carousel",
|
|
@@ -4377,21 +5219,31 @@ var UITransformer = class {
|
|
|
4377
5219
|
};
|
|
4378
5220
|
return (_a = mapping[type.toLowerCase()]) != null ? _a : null;
|
|
4379
5221
|
}
|
|
4380
|
-
static validateTransformation(
|
|
4381
|
-
const { type, data } =
|
|
5222
|
+
static validateTransformation(t) {
|
|
5223
|
+
const { type, data } = t;
|
|
4382
5224
|
switch (type) {
|
|
4383
5225
|
case "pie_chart":
|
|
4384
5226
|
case "bar_chart":
|
|
5227
|
+
case "histogram":
|
|
5228
|
+
case "horizontal_bar":
|
|
4385
5229
|
return Array.isArray(data) && data.every(
|
|
4386
|
-
(
|
|
5230
|
+
(i) => i !== null && typeof i === "object" && (typeof i.value === "number" || !Number.isNaN(Number(i.value)))
|
|
5231
|
+
);
|
|
5232
|
+
case "scatter_plot":
|
|
5233
|
+
return Array.isArray(data) && data.every(
|
|
5234
|
+
(i) => i !== null && typeof i === "object" && (typeof i.x === "number" || !Number.isNaN(Number(i.x))) && (typeof i.y === "number" || !Number.isNaN(Number(i.y)))
|
|
4387
5235
|
);
|
|
4388
5236
|
case "line_chart":
|
|
4389
5237
|
return Array.isArray(data) && data.every(
|
|
4390
|
-
(
|
|
5238
|
+
(i) => i !== null && typeof i === "object" && "timestamp" in i && (typeof i.value === "number" || !Number.isNaN(Number(i.value)))
|
|
4391
5239
|
);
|
|
5240
|
+
case "metric_card":
|
|
5241
|
+
return typeof data === "object" && data !== null && (typeof data.value === "number" || !Number.isNaN(Number(data.value)));
|
|
5242
|
+
case "geo_map":
|
|
5243
|
+
return Array.isArray(data) || typeof data === "object" && data !== null;
|
|
4392
5244
|
case "radar_chart":
|
|
4393
5245
|
return Array.isArray(data) && data.every(
|
|
4394
|
-
(
|
|
5246
|
+
(i) => i !== null && typeof i === "object" && "attribute" in i
|
|
4395
5247
|
);
|
|
4396
5248
|
case "table":
|
|
4397
5249
|
return typeof data === "object" && data !== null && Array.isArray(data.columns) && Array.isArray(data.rows);
|
|
@@ -4400,15 +5252,27 @@ var UITransformer = class {
|
|
|
4400
5252
|
case "product_carousel":
|
|
4401
5253
|
case "carousel":
|
|
4402
5254
|
return Array.isArray(data) && data.every(
|
|
4403
|
-
(
|
|
5255
|
+
(i) => i !== null && typeof i === "object" && ("id" in i || "name" in i)
|
|
4404
5256
|
);
|
|
4405
5257
|
default:
|
|
4406
5258
|
return false;
|
|
4407
5259
|
}
|
|
4408
5260
|
}
|
|
4409
|
-
|
|
4410
|
-
|
|
4411
|
-
|
|
5261
|
+
// ─── Data Inspection Helpers ──────────────────────────────────────────────
|
|
5262
|
+
static isStructuredListQuery(query) {
|
|
5263
|
+
const q = query.toLowerCase();
|
|
5264
|
+
const asksForRecords = /\b(list|provide|show|give|get|find|which|whose|records?|details?)\b/.test(q);
|
|
5265
|
+
const hasNumericPredicate = /\b(greater than|more than|above|over|less than|below|under|at least|at most|minimum|maximum|>=|<=|>|<|equals?|equal to)\b\s*\$?\d/i.test(q) || /\b\d+\b\s*(?:or more|or less|and above|and below)\b/i.test(q);
|
|
5266
|
+
const hasEntityTerms = /\b(organization|organisations?|organizations?|company|companies|employee|employees|staff|headcount|workforce)\b/.test(q);
|
|
5267
|
+
return asksForRecords && (hasNumericPredicate || hasEntityTerms);
|
|
5268
|
+
}
|
|
5269
|
+
static isProductQuery(query) {
|
|
5270
|
+
const q = query.toLowerCase();
|
|
5271
|
+
const productTerms = /\b(product|products|item|items|sku|catalog|catalogue|price|prices|brand|model|stock|inventory|in stock|out of stock|buy|shop)\b/.test(q);
|
|
5272
|
+
const productAction = /\b(recommend|suggest|describe|description|detail|details|about|show|find|search|browse|list)\b/.test(q);
|
|
5273
|
+
const nonProductEntityTerms = /\b(organization|organisations?|organizations?|company|companies|employee|employees|staff|headcount|workforce)\b/.test(q);
|
|
5274
|
+
return productTerms && productAction && !nonProductEntityTerms;
|
|
5275
|
+
}
|
|
4412
5276
|
static isProductData(item) {
|
|
4413
5277
|
const content = (item.content || "").toLowerCase();
|
|
4414
5278
|
const productKeywords = [
|
|
@@ -4434,394 +5298,582 @@ var UITransformer = class {
|
|
|
4434
5298
|
"fragrance"
|
|
4435
5299
|
];
|
|
4436
5300
|
const hasKeywords = productKeywords.some((kw) => content.includes(kw));
|
|
4437
|
-
const
|
|
4438
|
-
|
|
4439
|
-
|
|
4440
|
-
|
|
5301
|
+
const metadata = item.metadata || {};
|
|
5302
|
+
const hasMetadataKey = Object.keys(metadata).some((k) => {
|
|
5303
|
+
const val = metadata[k];
|
|
5304
|
+
return ["price", "product", "sku", "brand", "model", "cost", "item"].includes(k.toLowerCase()) && val !== null && val !== void 0 && val !== "";
|
|
5305
|
+
});
|
|
5306
|
+
const hasPricePattern = /\$\s*\d+\.\d{2}/.test(content);
|
|
4441
5307
|
return hasKeywords || hasMetadataKey || hasPricePattern;
|
|
4442
5308
|
}
|
|
4443
|
-
/**
|
|
4444
|
-
* Helper: Check if data contains time series
|
|
4445
|
-
*/
|
|
4446
5309
|
static isTimeSeriesData(item) {
|
|
4447
|
-
const content = (item.content || "").toLowerCase();
|
|
4448
|
-
const timeKeywords = ["trend", "historical", "growth", "decline", "change", "increase", "decrease"];
|
|
4449
|
-
const hasTimeKeyword = timeKeywords.some((kw) => content.includes(kw));
|
|
4450
5310
|
const metadata = item.metadata || {};
|
|
4451
5311
|
const maybeDateKeys = Object.keys(metadata).filter(
|
|
4452
5312
|
(k) => ["date", "timestamp", "time", "period"].includes(k.toLowerCase())
|
|
4453
5313
|
);
|
|
4454
|
-
|
|
5314
|
+
return maybeDateKeys.some((key) => {
|
|
4455
5315
|
const value = metadata[key];
|
|
4456
|
-
if (typeof value === "string")
|
|
4457
|
-
return !Number.isNaN(Date.parse(value)) && value.trim().length > 0;
|
|
4458
|
-
}
|
|
5316
|
+
if (typeof value === "string") return !Number.isNaN(Date.parse(value)) && value.trim().length > 0;
|
|
4459
5317
|
return typeof value === "number";
|
|
4460
5318
|
});
|
|
4461
|
-
return hasTimeKeyword || hasValidDateValue;
|
|
4462
|
-
}
|
|
4463
|
-
static isExplicitChartQuery(query) {
|
|
4464
|
-
const normalized = query.toLowerCase();
|
|
4465
|
-
return normalized.includes("pie chart") || normalized.includes("piechart") || normalized.includes("bar chart") || normalized.includes("barchart") || normalized.includes("radar chart");
|
|
4466
5319
|
}
|
|
4467
|
-
static
|
|
4468
|
-
const
|
|
4469
|
-
|
|
4470
|
-
|
|
4471
|
-
"breakdown",
|
|
4472
|
-
"by category",
|
|
4473
|
-
"by type",
|
|
4474
|
-
"compare",
|
|
4475
|
-
"share",
|
|
4476
|
-
"percentage",
|
|
4477
|
-
"segmentation",
|
|
4478
|
-
"split",
|
|
4479
|
-
"category breakdown",
|
|
4480
|
-
"category distribution",
|
|
4481
|
-
"pie chart",
|
|
4482
|
-
"piechart",
|
|
4483
|
-
"bar chart",
|
|
4484
|
-
"barchart"
|
|
4485
|
-
];
|
|
4486
|
-
return chartKeywords.some((keyword) => normalized.includes(keyword));
|
|
5320
|
+
static hasMultipleFields(data) {
|
|
5321
|
+
const fieldCount = /* @__PURE__ */ new Set();
|
|
5322
|
+
data.forEach((item) => Object.keys(item.metadata || {}).forEach((k) => fieldCount.add(k)));
|
|
5323
|
+
return fieldCount.size > 2;
|
|
4487
5324
|
}
|
|
4488
|
-
static
|
|
4489
|
-
const
|
|
4490
|
-
|
|
4491
|
-
|
|
4492
|
-
|
|
4493
|
-
|
|
4494
|
-
|
|
4495
|
-
|
|
4496
|
-
|
|
4497
|
-
|
|
4498
|
-
|
|
4499
|
-
|
|
4500
|
-
|
|
4501
|
-
|
|
4502
|
-
|
|
4503
|
-
|
|
4504
|
-
|
|
4505
|
-
|
|
4506
|
-
|
|
4507
|
-
|
|
5325
|
+
static profileData(data) {
|
|
5326
|
+
const records = data.map((item) => {
|
|
5327
|
+
const fields2 = {};
|
|
5328
|
+
Object.entries(item.metadata || {}).forEach(([key, value]) => {
|
|
5329
|
+
const primitive = this.toPrimitive(value);
|
|
5330
|
+
if (primitive !== null) fields2[key] = primitive;
|
|
5331
|
+
});
|
|
5332
|
+
if (!fields2.content && item.content) fields2.content = item.content.substring(0, 500);
|
|
5333
|
+
return {
|
|
5334
|
+
id: item.id,
|
|
5335
|
+
content: item.content,
|
|
5336
|
+
score: item.score,
|
|
5337
|
+
fields: fields2,
|
|
5338
|
+
source: item
|
|
5339
|
+
};
|
|
5340
|
+
});
|
|
5341
|
+
const keys = Array.from(new Set(records.flatMap((record) => Object.keys(record.fields))));
|
|
5342
|
+
const fields = keys.map((key) => {
|
|
5343
|
+
const values = records.map((record) => record.fields[key]).filter((value) => value !== void 0 && value !== null && String(value).trim() !== "");
|
|
5344
|
+
const uniqueCount = new Set(values.map((value) => String(value).toLowerCase())).size;
|
|
5345
|
+
return {
|
|
5346
|
+
key,
|
|
5347
|
+
label: this.humanizeFieldName(key),
|
|
5348
|
+
kind: this.inferFieldKind(key, values, records.length, uniqueCount),
|
|
5349
|
+
values,
|
|
5350
|
+
uniqueCount
|
|
5351
|
+
};
|
|
5352
|
+
});
|
|
5353
|
+
return {
|
|
5354
|
+
records,
|
|
5355
|
+
fields,
|
|
5356
|
+
numericFields: fields.filter((field) => field.kind === "number"),
|
|
5357
|
+
dateFields: fields.filter((field) => field.kind === "date"),
|
|
5358
|
+
categoricalFields: fields.filter((field) => field.kind === "category"),
|
|
5359
|
+
booleanFields: fields.filter((field) => field.kind === "boolean")
|
|
5360
|
+
};
|
|
4508
5361
|
}
|
|
4509
|
-
static
|
|
4510
|
-
|
|
4511
|
-
|
|
5362
|
+
static toPrimitive(value) {
|
|
5363
|
+
if (typeof value === "string" || typeof value === "number" || typeof value === "boolean") return value;
|
|
5364
|
+
if (value === null || value === void 0) return null;
|
|
5365
|
+
if (value instanceof Date) return value.toISOString();
|
|
5366
|
+
return null;
|
|
4512
5367
|
}
|
|
4513
|
-
|
|
4514
|
-
|
|
4515
|
-
|
|
4516
|
-
|
|
4517
|
-
|
|
4518
|
-
if (!meta) return void 0;
|
|
4519
|
-
const mapping = (_a = config == null ? void 0 : config.rag) == null ? void 0 : _a.uiMapping;
|
|
4520
|
-
if (mapping && mapping[uiKey]) {
|
|
4521
|
-
const mappedKey = mapping[uiKey];
|
|
4522
|
-
if (meta[mappedKey] !== void 0) return meta[mappedKey];
|
|
5368
|
+
static inferFieldKind(key, values, rowCount, uniqueCount) {
|
|
5369
|
+
const lowerKey = key.toLowerCase();
|
|
5370
|
+
if (values.length === 0) return "text";
|
|
5371
|
+
if (values.every((value) => typeof value === "boolean" || /^(true|false|yes|no|in_stock|out_of_stock|available|unavailable)$/i.test(String(value)))) {
|
|
5372
|
+
return "boolean";
|
|
4523
5373
|
}
|
|
4524
|
-
if (
|
|
4525
|
-
|
|
4526
|
-
if (trainedKey && meta[trainedKey] !== void 0) return meta[trainedKey];
|
|
5374
|
+
if (/(date|time|timestamp|created|updated|month|year|period)/i.test(lowerKey) && values.some((value) => this.isDateLike(value))) {
|
|
5375
|
+
return "date";
|
|
4527
5376
|
}
|
|
4528
|
-
|
|
5377
|
+
const numericCount = values.filter((value) => this.toFiniteNumber(value) !== null).length;
|
|
5378
|
+
if (numericCount / values.length >= 0.85 && !/(id|sku|ean|upc|zip|postal|phone|code)/i.test(lowerKey)) {
|
|
5379
|
+
return "number";
|
|
5380
|
+
}
|
|
5381
|
+
if (uniqueCount <= Math.max(20, Math.ceil(rowCount * 0.7)) && !/(id|sku|ean|upc|description|content|url|image|thumbnail)/i.test(lowerKey)) {
|
|
5382
|
+
return "category";
|
|
5383
|
+
}
|
|
5384
|
+
return "text";
|
|
4529
5385
|
}
|
|
4530
|
-
static
|
|
4531
|
-
|
|
4532
|
-
const
|
|
4533
|
-
|
|
4534
|
-
|
|
4535
|
-
|
|
4536
|
-
|
|
4537
|
-
|
|
4538
|
-
|
|
4539
|
-
|
|
4540
|
-
|
|
4541
|
-
|
|
4542
|
-
|
|
4543
|
-
|
|
4544
|
-
|
|
4545
|
-
|
|
4546
|
-
|
|
4547
|
-
return
|
|
4548
|
-
|
|
4549
|
-
|
|
4550
|
-
|
|
4551
|
-
image: typeof imageValue === "string" ? imageValue : void 0,
|
|
4552
|
-
brand: brand ? String(brand) : void 0,
|
|
4553
|
-
description: item.content,
|
|
4554
|
-
inStock: this.determineStockStatus(item)
|
|
4555
|
-
};
|
|
5386
|
+
static isDateLike(value) {
|
|
5387
|
+
if (typeof value === "number") return value > 1900 && value < 3e3;
|
|
5388
|
+
const text = String(value).trim();
|
|
5389
|
+
return text.length > 3 && !Number.isNaN(Date.parse(text));
|
|
5390
|
+
}
|
|
5391
|
+
static chooseAutomaticVisualization(data, profile, query) {
|
|
5392
|
+
if (profile.records.length === 0) return null;
|
|
5393
|
+
if (profile.dateFields.length > 0 && profile.numericFields.length > 0) {
|
|
5394
|
+
return this.transformToLineChart(profile);
|
|
5395
|
+
}
|
|
5396
|
+
if (profile.categoricalFields.length > 0 && profile.numericFields.length > 0) {
|
|
5397
|
+
return this.transformToBarChart(data, profile, query);
|
|
5398
|
+
}
|
|
5399
|
+
if (profile.categoricalFields.length > 0) {
|
|
5400
|
+
return this.transformToPieChart(data, profile, query);
|
|
5401
|
+
}
|
|
5402
|
+
if (profile.numericFields.length >= 2) {
|
|
5403
|
+
return this.transformToScatterPlot(profile, query);
|
|
5404
|
+
}
|
|
5405
|
+
if (profile.numericFields.length === 1) {
|
|
5406
|
+
return this.transformToMetricCard(profile, query);
|
|
4556
5407
|
}
|
|
4557
5408
|
return null;
|
|
4558
5409
|
}
|
|
4559
|
-
|
|
4560
|
-
|
|
4561
|
-
|
|
4562
|
-
|
|
4563
|
-
|
|
4564
|
-
const
|
|
4565
|
-
|
|
4566
|
-
|
|
4567
|
-
|
|
4568
|
-
|
|
4569
|
-
|
|
4570
|
-
|
|
4571
|
-
|
|
4572
|
-
|
|
4573
|
-
|
|
4574
|
-
|
|
4575
|
-
|
|
4576
|
-
|
|
4577
|
-
|
|
4578
|
-
|
|
5410
|
+
static selectDimensionField(profile, query) {
|
|
5411
|
+
var _a, _b;
|
|
5412
|
+
const productCategory = profile.categoricalFields.find(
|
|
5413
|
+
(field) => /category|department|collection|type|group|segment|region|country|state|city/i.test(field.key)
|
|
5414
|
+
);
|
|
5415
|
+
const ranked = this.rankFieldsByQuery(profile.categoricalFields, query);
|
|
5416
|
+
return (_b = (_a = ranked[0]) != null ? _a : productCategory) != null ? _b : profile.categoricalFields[0];
|
|
5417
|
+
}
|
|
5418
|
+
static selectNumericField(profile, query) {
|
|
5419
|
+
var _a;
|
|
5420
|
+
return (_a = this.rankFieldsByQuery(profile.numericFields, query)[0]) != null ? _a : profile.numericFields[0];
|
|
5421
|
+
}
|
|
5422
|
+
static rankFieldsByQuery(fields, query) {
|
|
5423
|
+
const q = query.toLowerCase();
|
|
5424
|
+
return [...fields].sort((a, b) => this.fieldScore(b, q) - this.fieldScore(a, q));
|
|
5425
|
+
}
|
|
5426
|
+
static fieldScore(field, query) {
|
|
5427
|
+
const key = field.key.toLowerCase();
|
|
5428
|
+
const label = field.label.toLowerCase();
|
|
5429
|
+
let score = 0;
|
|
5430
|
+
if (query.includes(key)) score += 4;
|
|
5431
|
+
if (query.includes(label)) score += 4;
|
|
5432
|
+
key.split(/[_\s-]+/).forEach((part) => {
|
|
5433
|
+
if (part && query.includes(part)) score += 1;
|
|
5434
|
+
});
|
|
5435
|
+
if (/category|department|collection|type|group|segment/.test(key)) score += 2;
|
|
5436
|
+
if (/count|total|quantity|stock|inventory|sales|revenue|amount|price|value|score/.test(key)) score += 2;
|
|
5437
|
+
if (/id|sku|ean|upc|code/.test(key)) score -= 5;
|
|
5438
|
+
return score;
|
|
5439
|
+
}
|
|
5440
|
+
static normalizeComparableField(field) {
|
|
5441
|
+
return field.toLowerCase().replace(/&/g, "and").replace(/[^a-z0-9]+/g, "").trim();
|
|
5442
|
+
}
|
|
5443
|
+
static fieldTokens(field) {
|
|
5444
|
+
return field.toLowerCase().replace(/[_-]+/g, " ").split(/\s+/).map((token) => token.replace(/[^a-z0-9]/g, "")).filter(Boolean);
|
|
5445
|
+
}
|
|
5446
|
+
static aggregateProfileByDimension(profile, dimensionKey, measureKey) {
|
|
5447
|
+
const result = {};
|
|
5448
|
+
profile.records.forEach((record) => {
|
|
5449
|
+
var _a, _b, _c;
|
|
5450
|
+
const category = String((_a = record.fields[dimensionKey]) != null ? _a : "Other").trim() || "Other";
|
|
5451
|
+
const value = measureKey ? (_b = this.toFiniteNumber(record.fields[measureKey])) != null ? _b : 0 : 1;
|
|
5452
|
+
result[category] = ((_c = result[category]) != null ? _c : 0) + value;
|
|
5453
|
+
});
|
|
5454
|
+
return result;
|
|
5455
|
+
}
|
|
5456
|
+
static getRecordLabel(record) {
|
|
5457
|
+
const labelKeys = ["name", "title", "label", "product", "item", "brand"];
|
|
5458
|
+
const key = Object.keys(record.fields).find(
|
|
5459
|
+
(fieldKey) => labelKeys.some((labelKey) => fieldKey.toLowerCase().includes(labelKey))
|
|
5460
|
+
);
|
|
5461
|
+
return key ? record.fields[key] : void 0;
|
|
5462
|
+
}
|
|
5463
|
+
static detectAggregationOperation(query) {
|
|
5464
|
+
const q = query.toLowerCase();
|
|
5465
|
+
if (/\b(avg|average|mean)\b/.test(q)) return "average";
|
|
5466
|
+
if (/\b(count|how many|number of)\b/.test(q)) return "count";
|
|
5467
|
+
if (/\b(min|minimum|lowest)\b/.test(q)) return "min";
|
|
5468
|
+
if (/\b(max|maximum|highest)\b/.test(q)) return "max";
|
|
5469
|
+
if (/\bmedian\b/.test(q)) return "median";
|
|
5470
|
+
return "sum";
|
|
5471
|
+
}
|
|
5472
|
+
static calculateAggregate(values, operation) {
|
|
5473
|
+
if (values.length === 0) return 0;
|
|
5474
|
+
switch (operation) {
|
|
5475
|
+
case "average":
|
|
5476
|
+
return values.reduce((sum, value) => sum + value, 0) / values.length;
|
|
5477
|
+
case "count":
|
|
5478
|
+
return values.length;
|
|
5479
|
+
case "min":
|
|
5480
|
+
return Math.min(...values);
|
|
5481
|
+
case "max":
|
|
5482
|
+
return Math.max(...values);
|
|
5483
|
+
case "median": {
|
|
5484
|
+
const sorted = [...values].sort((a, b) => a - b);
|
|
5485
|
+
const middle = Math.floor(sorted.length / 2);
|
|
5486
|
+
return sorted.length % 2 === 0 ? (sorted[middle - 1] + sorted[middle]) / 2 : sorted[middle];
|
|
5487
|
+
}
|
|
5488
|
+
case "sum":
|
|
5489
|
+
default:
|
|
5490
|
+
return values.reduce((sum, value) => sum + value, 0);
|
|
4579
5491
|
}
|
|
4580
|
-
return targetField;
|
|
4581
5492
|
}
|
|
4582
|
-
|
|
4583
|
-
|
|
4584
|
-
|
|
4585
|
-
static
|
|
5493
|
+
static humanizeFieldName(key) {
|
|
5494
|
+
return key.replace(/[_-]+/g, " ").replace(/([a-z])([A-Z])/g, "$1 $2").replace(/\s+/g, " ").trim().replace(/\b\w/g, (char) => char.toUpperCase());
|
|
5495
|
+
}
|
|
5496
|
+
static formatNumber(value) {
|
|
5497
|
+
return Number.isInteger(value) ? String(value) : value.toFixed(1);
|
|
5498
|
+
}
|
|
5499
|
+
static detectCategories(data) {
|
|
4586
5500
|
const categories = /* @__PURE__ */ new Set();
|
|
4587
5501
|
data.forEach((item) => {
|
|
4588
|
-
const
|
|
4589
|
-
if (
|
|
4590
|
-
const val = meta[groupByField];
|
|
4591
|
-
if (Array.isArray(val)) val.forEach((v) => categories.add(String(v)));
|
|
4592
|
-
else categories.add(String(val));
|
|
4593
|
-
} else if (groupByField === "category") {
|
|
4594
|
-
if (meta.type) categories.add(String(meta.type));
|
|
4595
|
-
if (meta.tag) {
|
|
4596
|
-
const tags = Array.isArray(meta.tag) ? meta.tag : [meta.tag];
|
|
4597
|
-
tags.forEach((t) => categories.add(String(t)));
|
|
4598
|
-
}
|
|
4599
|
-
const categoryMatch = item.content.match(/(?:Category|Type|Class):\s*([^\n]+)/i);
|
|
4600
|
-
if (categoryMatch) categories.add(categoryMatch[1].trim());
|
|
4601
|
-
}
|
|
5502
|
+
const category = this.getProductCategory(item);
|
|
5503
|
+
if (category) categories.add(category);
|
|
4602
5504
|
});
|
|
4603
5505
|
return Array.from(categories);
|
|
4604
5506
|
}
|
|
4605
|
-
|
|
4606
|
-
|
|
4607
|
-
|
|
4608
|
-
|
|
4609
|
-
const
|
|
4610
|
-
|
|
4611
|
-
|
|
4612
|
-
|
|
5507
|
+
static getProductCategory(item) {
|
|
5508
|
+
const meta = item.metadata || {};
|
|
5509
|
+
const metadataCategory = resolveMetadataValue(meta, "category");
|
|
5510
|
+
if (this.isUsableCategory(metadataCategory)) return String(metadataCategory).trim();
|
|
5511
|
+
const content = item.content || "";
|
|
5512
|
+
const explicitCategoryMatch = content.match(
|
|
5513
|
+
/(?:^|\n)\s*(?:product\s*)?(?:category|department|collection)\s*[:\-–]\s*([^\n]+)/i
|
|
5514
|
+
);
|
|
5515
|
+
if (explicitCategoryMatch && this.isUsableCategory(explicitCategoryMatch[1])) {
|
|
5516
|
+
return explicitCategoryMatch[1].trim();
|
|
5517
|
+
}
|
|
5518
|
+
return null;
|
|
5519
|
+
}
|
|
5520
|
+
static isUsableCategory(value) {
|
|
5521
|
+
if (value === null || value === void 0) return false;
|
|
5522
|
+
const category = String(value).trim();
|
|
5523
|
+
if (!category) return false;
|
|
5524
|
+
return !/^(category|type|tag|price|cost|stock|availability|description|product|item|name|brand|sku|id|ean|currency|color|size|index|internal id|other|\[?type\]?|\[?products?_?\d*\]?)$/i.test(category);
|
|
5525
|
+
}
|
|
5526
|
+
static aggregateByCategory(data, categories) {
|
|
5527
|
+
const result = Object.fromEntries(categories.map((c) => [c, 0]));
|
|
4613
5528
|
data.forEach((item) => {
|
|
4614
|
-
|
|
4615
|
-
|
|
4616
|
-
if (
|
|
4617
|
-
|
|
4618
|
-
} else if (groupByField === "category" && meta.type) {
|
|
4619
|
-
itemCategory = String(meta.type);
|
|
4620
|
-
}
|
|
4621
|
-
if (Object.prototype.hasOwnProperty.call(result, itemCategory)) {
|
|
4622
|
-
result[itemCategory]++;
|
|
4623
|
-
} else {
|
|
4624
|
-
result["Other"] = (result["Other"] || 0) + 1;
|
|
4625
|
-
}
|
|
5529
|
+
var _a;
|
|
5530
|
+
const cat = (_a = this.getProductCategory(item)) != null ? _a : "Other";
|
|
5531
|
+
if (Object.prototype.hasOwnProperty.call(result, cat)) result[cat]++;
|
|
5532
|
+
else result["Other"] = (result["Other"] || 0) + 1;
|
|
4626
5533
|
});
|
|
4627
5534
|
return result;
|
|
4628
5535
|
}
|
|
4629
|
-
/**
|
|
4630
|
-
* Helper: Extract time series data
|
|
4631
|
-
*/
|
|
4632
5536
|
static extractTimeSeriesData(data) {
|
|
4633
5537
|
return data.map((item) => {
|
|
5538
|
+
var _a, _b, _c, _d, _e;
|
|
4634
5539
|
const meta = item.metadata || {};
|
|
4635
5540
|
return {
|
|
4636
|
-
timestamp: meta.timestamp
|
|
4637
|
-
value: meta.value
|
|
4638
|
-
label: meta.label
|
|
5541
|
+
timestamp: (_b = (_a = meta.timestamp) != null ? _a : meta.date) != null ? _b : (/* @__PURE__ */ new Date()).toISOString(),
|
|
5542
|
+
value: (_d = (_c = meta.value) != null ? _c : item.score) != null ? _d : 0,
|
|
5543
|
+
label: (_e = meta.label) != null ? _e : item.content.substring(0, 50)
|
|
4639
5544
|
};
|
|
4640
5545
|
});
|
|
4641
5546
|
}
|
|
4642
|
-
|
|
4643
|
-
|
|
4644
|
-
|
|
4645
|
-
|
|
4646
|
-
|
|
4647
|
-
|
|
4648
|
-
|
|
4649
|
-
|
|
4650
|
-
|
|
4651
|
-
|
|
4652
|
-
|
|
4653
|
-
|
|
5547
|
+
static extractNumericValue(meta) {
|
|
5548
|
+
var _a;
|
|
5549
|
+
const preferredKeys = [
|
|
5550
|
+
"value",
|
|
5551
|
+
"count",
|
|
5552
|
+
"total",
|
|
5553
|
+
"average",
|
|
5554
|
+
"avg",
|
|
5555
|
+
"amount",
|
|
5556
|
+
"sales",
|
|
5557
|
+
"revenue",
|
|
5558
|
+
"score",
|
|
5559
|
+
"quantity",
|
|
5560
|
+
"price"
|
|
5561
|
+
];
|
|
5562
|
+
for (const key of preferredKeys) {
|
|
5563
|
+
const raw = (_a = resolveMetadataValue(meta, key)) != null ? _a : meta[key];
|
|
5564
|
+
const value = typeof raw === "number" ? raw : Number(String(raw != null ? raw : "").replace(/[$,% ,]/g, ""));
|
|
5565
|
+
if (Number.isFinite(value)) return value;
|
|
5566
|
+
}
|
|
5567
|
+
for (const value of Object.values(meta)) {
|
|
5568
|
+
const numeric = typeof value === "number" ? value : Number(String(value != null ? value : "").replace(/[$,% ,]/g, ""));
|
|
5569
|
+
if (Number.isFinite(numeric)) return numeric;
|
|
5570
|
+
}
|
|
5571
|
+
return null;
|
|
5572
|
+
}
|
|
5573
|
+
static extractTableColumns(data, query = "") {
|
|
5574
|
+
const q = query.toLowerCase();
|
|
5575
|
+
const availableFields = this.extractAvailableTableFields(data);
|
|
5576
|
+
if (/\b(organization|organisations?|organizations?|company|companies)\b/.test(q) && /\b(employee|employees|staff|headcount|workforce)\b/.test(q)) {
|
|
5577
|
+
const nameField = this.pickTableField(availableFields, [
|
|
5578
|
+
"company name",
|
|
5579
|
+
"organization name",
|
|
5580
|
+
"organisation name",
|
|
5581
|
+
"name",
|
|
5582
|
+
"company",
|
|
5583
|
+
"organization",
|
|
5584
|
+
"organisation"
|
|
5585
|
+
]);
|
|
5586
|
+
const employeeField = this.pickTableField(availableFields, [
|
|
5587
|
+
"number of employees",
|
|
5588
|
+
"employee count",
|
|
5589
|
+
"employees",
|
|
5590
|
+
"employee_count",
|
|
5591
|
+
"number_employees",
|
|
5592
|
+
"num employees",
|
|
5593
|
+
"staff count",
|
|
5594
|
+
"headcount",
|
|
5595
|
+
"workforce"
|
|
5596
|
+
]);
|
|
5597
|
+
const columns = [nameField, employeeField].filter((field) => Boolean(field));
|
|
5598
|
+
if (columns.length > 0) return columns;
|
|
5599
|
+
}
|
|
5600
|
+
const requestedFields = availableFields.map((field) => ({
|
|
5601
|
+
field,
|
|
5602
|
+
score: this.tableFieldQueryScore(field, q)
|
|
5603
|
+
})).filter((item) => item.score > 0).sort((a, b) => b.score - a.score).map((item) => item.field);
|
|
5604
|
+
if (requestedFields.length > 0) {
|
|
5605
|
+
return requestedFields.slice(0, Math.min(6, requestedFields.length));
|
|
5606
|
+
}
|
|
5607
|
+
const metadataFields = Array.from(new Set(
|
|
5608
|
+
data.flatMap((item) => Object.keys(item.metadata || {}).map((k) => this.humanizeFieldName(k)))
|
|
5609
|
+
));
|
|
5610
|
+
return metadataFields.length > 0 ? metadataFields : ["Content"];
|
|
4654
5611
|
}
|
|
4655
|
-
/**
|
|
4656
|
-
* Helper: Extract table row
|
|
4657
|
-
*/
|
|
4658
5612
|
static extractTableRow(item, columns) {
|
|
4659
|
-
|
|
4660
|
-
|
|
4661
|
-
|
|
4662
|
-
|
|
5613
|
+
return columns.map((col) => this.resolveTableCellValue(item, col));
|
|
5614
|
+
}
|
|
5615
|
+
static extractAvailableTableFields(data) {
|
|
5616
|
+
const fields = /* @__PURE__ */ new Map();
|
|
5617
|
+
const addField = (field) => {
|
|
5618
|
+
const clean = this.humanizeFieldName(field);
|
|
5619
|
+
if (!clean || /^(content|\[?type\]?)$/i.test(clean)) return;
|
|
5620
|
+
const normalized = this.normalizeComparableField(clean);
|
|
5621
|
+
if (!fields.has(normalized)) fields.set(normalized, clean);
|
|
5622
|
+
};
|
|
5623
|
+
data.forEach((item) => {
|
|
5624
|
+
Object.keys(item.metadata || {}).forEach(addField);
|
|
5625
|
+
for (const match of (item.content || "").matchAll(/(?:^|\n)\s*([A-Za-z][A-Za-z0-9_ /-]{1,50})\s*[:\-–]\s*([^\n]+)/g)) {
|
|
5626
|
+
addField(match[1]);
|
|
4663
5627
|
}
|
|
4664
|
-
const metaKey = col.charAt(0).toLowerCase() + col.slice(1);
|
|
4665
|
-
const value = meta[metaKey];
|
|
4666
|
-
return value !== void 0 ? String(value) : "";
|
|
4667
5628
|
});
|
|
5629
|
+
return Array.from(fields.values());
|
|
5630
|
+
}
|
|
5631
|
+
static pickTableField(fields, aliases) {
|
|
5632
|
+
const normalizedAliases = aliases.map((alias) => this.normalizeComparableField(alias));
|
|
5633
|
+
for (const alias of normalizedAliases) {
|
|
5634
|
+
const exact = fields.find((field) => this.normalizeComparableField(field) === alias);
|
|
5635
|
+
if (exact) return exact;
|
|
5636
|
+
}
|
|
5637
|
+
for (const alias of normalizedAliases) {
|
|
5638
|
+
const fuzzy = fields.find((field) => {
|
|
5639
|
+
const normalizedField = this.normalizeComparableField(field);
|
|
5640
|
+
if (/(id|code|uuid)$/.test(normalizedField) && !/(id|code|uuid)$/.test(alias)) return false;
|
|
5641
|
+
return normalizedField.includes(alias) || alias.includes(normalizedField);
|
|
5642
|
+
});
|
|
5643
|
+
if (fuzzy) return fuzzy;
|
|
5644
|
+
}
|
|
5645
|
+
return void 0;
|
|
4668
5646
|
}
|
|
4669
|
-
|
|
4670
|
-
|
|
4671
|
-
|
|
4672
|
-
|
|
5647
|
+
static tableFieldQueryScore(field, query) {
|
|
5648
|
+
const normalizedField = this.normalizeComparableField(field);
|
|
5649
|
+
if (!normalizedField) return 0;
|
|
5650
|
+
const fieldTokens = this.fieldTokens(field);
|
|
5651
|
+
return fieldTokens.reduce((score, token) => {
|
|
5652
|
+
if (token.length < 3) return score;
|
|
5653
|
+
return query.includes(token) ? score + 1 : score;
|
|
5654
|
+
}, query.includes(normalizedField) ? 3 : 0);
|
|
5655
|
+
}
|
|
5656
|
+
static resolveTableCellValue(item, column) {
|
|
5657
|
+
if (column === "Content") return item.content.substring(0, 100);
|
|
5658
|
+
const meta = item.metadata || {};
|
|
5659
|
+
const normalizedColumn = this.normalizeComparableField(column);
|
|
5660
|
+
const exactMetadata = Object.entries(meta).find(
|
|
5661
|
+
([key]) => this.normalizeComparableField(key) === normalizedColumn || this.normalizeComparableField(this.humanizeFieldName(key)) === normalizedColumn
|
|
5662
|
+
);
|
|
5663
|
+
if (exactMetadata && exactMetadata[1] !== void 0 && exactMetadata[1] !== null) {
|
|
5664
|
+
return this.toDisplayValue(exactMetadata[1]);
|
|
5665
|
+
}
|
|
5666
|
+
const aliasValue = this.resolveAliasedTableCell(meta, column);
|
|
5667
|
+
if (aliasValue !== void 0) return this.toDisplayValue(aliasValue);
|
|
5668
|
+
const contentValue = this.extractContentFieldValue(item.content, column);
|
|
5669
|
+
if (contentValue !== null) return contentValue;
|
|
5670
|
+
const aliasContentValue = this.extractAliasedContentFieldValue(item.content, column);
|
|
5671
|
+
return aliasContentValue != null ? aliasContentValue : "";
|
|
5672
|
+
}
|
|
5673
|
+
static resolveAliasedTableCell(meta, column) {
|
|
5674
|
+
const normalizedColumn = this.normalizeComparableField(column);
|
|
5675
|
+
const aliases = normalizedColumn.includes("employee") ? ["number of employees", "employee count", "employees", "headcount", "staff count", "workforce"] : normalizedColumn.includes("name") ? ["company name", "organization name", "organisation name", "name", "company", "organization", "organisation"] : [];
|
|
5676
|
+
for (const alias of aliases) {
|
|
5677
|
+
const normalizedAlias = this.normalizeComparableField(alias);
|
|
5678
|
+
const match = Object.entries(meta).find(([key]) => {
|
|
5679
|
+
const normalizedKey = this.normalizeComparableField(key);
|
|
5680
|
+
return normalizedKey === normalizedAlias || normalizedKey.includes(normalizedAlias) || normalizedAlias.includes(normalizedKey);
|
|
5681
|
+
});
|
|
5682
|
+
if (match && match[1] !== void 0 && match[1] !== null) return match[1];
|
|
5683
|
+
}
|
|
5684
|
+
return void 0;
|
|
5685
|
+
}
|
|
5686
|
+
static extractContentFieldValue(content, column) {
|
|
5687
|
+
const escaped = column.replace(/[.*+?^${}()|[\]\\]/g, "\\$&").replace(/\s+/g, "[\\s_ -]+");
|
|
5688
|
+
const pattern = new RegExp(`(?:^|\\n)\\s*${escaped}\\s*[:\\-\u2013]\\s*([^\\n]+)`, "i");
|
|
5689
|
+
const match = content.match(pattern);
|
|
5690
|
+
return match ? match[1].trim() : null;
|
|
5691
|
+
}
|
|
5692
|
+
static extractAliasedContentFieldValue(content, column) {
|
|
5693
|
+
const normalizedColumn = this.normalizeComparableField(column);
|
|
5694
|
+
const aliases = normalizedColumn.includes("employee") ? ["Number Of Employees", "Number of Employees", "Employee Count", "Employees", "Headcount", "Staff Count", "Workforce"] : normalizedColumn.includes("name") ? ["Company Name", "Organization Name", "Organisation Name", "Name", "Company", "Organization", "Organisation"] : [];
|
|
5695
|
+
for (const alias of aliases) {
|
|
5696
|
+
const value = this.extractContentFieldValue(content, alias);
|
|
5697
|
+
if (value !== null) return value;
|
|
5698
|
+
}
|
|
5699
|
+
return null;
|
|
5700
|
+
}
|
|
5701
|
+
static toDisplayValue(value) {
|
|
5702
|
+
if (typeof value === "string" || typeof value === "number" || typeof value === "boolean") return value;
|
|
5703
|
+
return String(value != null ? value : "");
|
|
5704
|
+
}
|
|
5705
|
+
static calculateStockCounts(category, data) {
|
|
4673
5706
|
let inStock = 0;
|
|
4674
5707
|
let outOfStock = 0;
|
|
4675
5708
|
data.forEach((d) => {
|
|
4676
|
-
|
|
4677
|
-
|
|
4678
|
-
if (
|
|
4679
|
-
|
|
4680
|
-
|
|
4681
|
-
|
|
4682
|
-
}
|
|
4683
|
-
if (itemCategory === category) {
|
|
4684
|
-
if (this.determineStockStatus(d)) {
|
|
4685
|
-
inStock++;
|
|
4686
|
-
} else {
|
|
4687
|
-
outOfStock++;
|
|
4688
|
-
}
|
|
5709
|
+
var _a, _b;
|
|
5710
|
+
const cat = (_a = this.getProductCategory(d)) != null ? _a : "Other";
|
|
5711
|
+
if (cat === category) {
|
|
5712
|
+
const quantity = (_b = this.extractStockQuantity(d)) != null ? _b : 1;
|
|
5713
|
+
if (this.determineStockStatus(d)) inStock += quantity;
|
|
5714
|
+
else outOfStock += quantity;
|
|
4689
5715
|
}
|
|
4690
5716
|
});
|
|
4691
5717
|
return { inStockCount: inStock, outOfStockCount: outOfStock };
|
|
4692
5718
|
}
|
|
4693
|
-
/**
|
|
4694
|
-
* Helper: Determine if item is in stock
|
|
4695
|
-
*/
|
|
4696
5719
|
static determineStockStatus(item) {
|
|
4697
5720
|
const meta = item.metadata || {};
|
|
4698
|
-
|
|
4699
|
-
|
|
4700
|
-
|
|
4701
|
-
|
|
4702
|
-
|
|
4703
|
-
|
|
4704
|
-
|
|
4705
|
-
|
|
4706
|
-
|
|
5721
|
+
const stockValue = resolveMetadataValue(meta, "stock");
|
|
5722
|
+
if (stockValue !== void 0) {
|
|
5723
|
+
const normalized = String(stockValue).toLowerCase();
|
|
5724
|
+
if (/out[_\s-]?of[_\s-]?stock|unavailable|false|no|sold out|0/.test(normalized)) return false;
|
|
5725
|
+
if (/in[_\s-]?stock|available|true|yes/.test(normalized)) return true;
|
|
5726
|
+
const numeric = this.toFiniteNumber(stockValue);
|
|
5727
|
+
if (numeric !== null) return numeric > 0;
|
|
5728
|
+
}
|
|
5729
|
+
if (meta.inStock !== void 0) return Boolean(meta.inStock);
|
|
5730
|
+
if (meta.stock !== void 0) return Boolean(meta.stock);
|
|
5731
|
+
if (meta.available !== void 0) return Boolean(meta.available);
|
|
4707
5732
|
const content = (item.content || "").toLowerCase();
|
|
4708
|
-
if (
|
|
4709
|
-
|
|
5733
|
+
if (/out[_\s-]?of[_\s-]?stock|unavailable|sold out/.test(content)) return false;
|
|
5734
|
+
if (/in[_\s-]?stock|available/.test(content)) return true;
|
|
5735
|
+
return true;
|
|
5736
|
+
}
|
|
5737
|
+
static extractStockQuantity(item) {
|
|
5738
|
+
const meta = item.metadata || {};
|
|
5739
|
+
const stockValue = resolveMetadataValue(meta, "stock");
|
|
5740
|
+
const numericStock = this.toFiniteNumber(stockValue);
|
|
5741
|
+
if (numericStock !== null) return numericStock;
|
|
5742
|
+
const content = item.content || "";
|
|
5743
|
+
const stockMatch = content.match(/\b(?:stock|inventory|quantity|count)\s*[:\-–]?\s*([\d,]+)/i);
|
|
5744
|
+
if (!stockMatch) return null;
|
|
5745
|
+
return this.toFiniteNumber(stockMatch[1]);
|
|
5746
|
+
}
|
|
5747
|
+
static toFiniteNumber(value) {
|
|
5748
|
+
if (typeof value === "number") return Number.isFinite(value) ? value : null;
|
|
5749
|
+
const numeric = Number(String(value != null ? value : "").replace(/[$,% ,]/g, ""));
|
|
5750
|
+
return Number.isFinite(numeric) ? numeric : null;
|
|
5751
|
+
}
|
|
5752
|
+
// ─── Product Extraction ───────────────────────────────────────────────────
|
|
5753
|
+
static getDynamicVal(meta, uiKey, config, trainedSchema) {
|
|
5754
|
+
var _a;
|
|
5755
|
+
if (!meta) return void 0;
|
|
5756
|
+
const mapping = (_a = config == null ? void 0 : config.rag) == null ? void 0 : _a.uiMapping;
|
|
5757
|
+
if ((mapping == null ? void 0 : mapping[uiKey]) && meta[mapping[uiKey]] !== void 0) return meta[mapping[uiKey]];
|
|
5758
|
+
if (trainedSchema && typeof trainedSchema === "object") {
|
|
5759
|
+
const trainedKey = trainedSchema[uiKey];
|
|
5760
|
+
if (trainedKey && meta[trainedKey] !== void 0) return meta[trainedKey];
|
|
4710
5761
|
}
|
|
4711
|
-
|
|
4712
|
-
|
|
5762
|
+
return resolveMetadataValue(meta, uiKey);
|
|
5763
|
+
}
|
|
5764
|
+
static extractProductInfo(item, config, trainedSchema) {
|
|
5765
|
+
var _a;
|
|
5766
|
+
const meta = item.metadata || {};
|
|
5767
|
+
const name = this.getDynamicVal(meta, "name", config, trainedSchema);
|
|
5768
|
+
const price = this.getDynamicVal(meta, "price", config, trainedSchema);
|
|
5769
|
+
const brand = this.getDynamicVal(meta, "brand", config, trainedSchema);
|
|
5770
|
+
const description = this.cleanProductDescription(
|
|
5771
|
+
(_a = this.extractProductDescriptionFromContent(item.content)) != null ? _a : this.getProductDescriptionValue(meta, config, trainedSchema)
|
|
5772
|
+
);
|
|
5773
|
+
if (name || this.isProductData(item)) {
|
|
5774
|
+
let finalName = name ? String(name) : void 0;
|
|
5775
|
+
if (!finalName) {
|
|
5776
|
+
const nameMatch = item.content.match(/(?:Product|Item|Name):\s*([^\n]+)/i);
|
|
5777
|
+
finalName = nameMatch ? nameMatch[1].trim() : item.content.split("\n")[0].substring(0, 60);
|
|
5778
|
+
}
|
|
5779
|
+
let finalPrice = typeof price === "number" || typeof price === "string" ? price : void 0;
|
|
5780
|
+
if (!finalPrice) {
|
|
5781
|
+
const priceMatch = item.content.match(/(?:Price|Cost):\s*\$?([\d,.]+)/i) || item.content.match(/\$\s*([\d,.]+)/);
|
|
5782
|
+
if (priceMatch) finalPrice = priceMatch[1].replace(/,/g, "");
|
|
5783
|
+
}
|
|
5784
|
+
const imageValue = this.getDynamicVal(meta, "image", config, trainedSchema);
|
|
5785
|
+
return {
|
|
5786
|
+
id: item.id,
|
|
5787
|
+
name: finalName,
|
|
5788
|
+
price: finalPrice,
|
|
5789
|
+
image: typeof imageValue === "string" ? imageValue : void 0,
|
|
5790
|
+
brand: brand ? String(brand) : void 0,
|
|
5791
|
+
description,
|
|
5792
|
+
inStock: this.determineStockStatus(item)
|
|
5793
|
+
};
|
|
4713
5794
|
}
|
|
4714
|
-
return
|
|
5795
|
+
return null;
|
|
4715
5796
|
}
|
|
4716
|
-
|
|
4717
|
-
|
|
4718
|
-
|
|
4719
|
-
|
|
4720
|
-
|
|
4721
|
-
|
|
4722
|
-
|
|
4723
|
-
|
|
4724
|
-
|
|
4725
|
-
|
|
4726
|
-
return fieldCount.size > 2;
|
|
5797
|
+
static extractProductDescriptionFromContent(content) {
|
|
5798
|
+
const bodyMatch = content.match(
|
|
5799
|
+
/\bBody\s*\(HTML\)\s*:\s*([\s\S]*?)(?=\s*[,.]?\s+\b(?:Handle|Title|Vendor|Type|Tags|Published|Option\d*\s+Name|Option\d*\s+Value|Option|Variant|Image|SKU|Price|Status|Category)\b\s*:|$)/i
|
|
5800
|
+
);
|
|
5801
|
+
if (bodyMatch == null ? void 0 : bodyMatch[1]) return bodyMatch[1];
|
|
5802
|
+
const descriptionMatch = content.match(
|
|
5803
|
+
/\b(?:Description|Summary|Details|Body|Content)\s*:\s*([\s\S]*?)(?=\s*[,.]?\s+\b(?:Handle|Title|Vendor|Type|Tags|Published|Option\d*\s+Name|Option\d*\s+Value|Option|Variant|Image|SKU|Price|Status|Category)\b\s*:|$)/i
|
|
5804
|
+
);
|
|
5805
|
+
if (descriptionMatch == null ? void 0 : descriptionMatch[1]) return descriptionMatch[1];
|
|
5806
|
+
return null;
|
|
4727
5807
|
}
|
|
4728
|
-
|
|
4729
|
-
|
|
4730
|
-
|
|
4731
|
-
|
|
4732
|
-
|
|
4733
|
-
|
|
4734
|
-
|
|
4735
|
-
|
|
4736
|
-
|
|
4737
|
-
|
|
4738
|
-
|
|
4739
|
-
|
|
4740
|
-
|
|
4741
|
-
|
|
4742
|
-
|
|
4743
|
-
* @param sources - vector DB matches returned by RAG retrieval
|
|
4744
|
-
* @param llm - any ILLMProvider instance (OpenAI, Anthropic, Ollama, Gemini, REST…)
|
|
4745
|
-
* @returns - a validated UITransformationResponse (type + title + description + data)
|
|
4746
|
-
*/
|
|
4747
|
-
static async analyzeAndDecide(query, sources, llm) {
|
|
4748
|
-
try {
|
|
4749
|
-
const context = this.buildContextSummary(sources);
|
|
4750
|
-
const systemPrompt = this.buildVisualizationSystemPrompt();
|
|
4751
|
-
const userPrompt = [
|
|
4752
|
-
`USER QUESTION: ${query}`,
|
|
4753
|
-
"",
|
|
4754
|
-
"RETRIEVED DATA (JSON):",
|
|
4755
|
-
context
|
|
4756
|
-
].join("\n");
|
|
4757
|
-
const rawResponse = await llm.chat(
|
|
4758
|
-
[{ role: "user", content: userPrompt }],
|
|
4759
|
-
"",
|
|
4760
|
-
{ systemPrompt, temperature: 0 }
|
|
5808
|
+
static getProductDescriptionValue(meta, config, trainedSchema) {
|
|
5809
|
+
const mapped = this.getDynamicVal(meta, "description", config, trainedSchema);
|
|
5810
|
+
if (mapped !== void 0 && mapped !== meta.content && mapped !== meta.text) return mapped;
|
|
5811
|
+
const preferredKeys = [
|
|
5812
|
+
"body_html",
|
|
5813
|
+
"body html",
|
|
5814
|
+
"bodyHtml",
|
|
5815
|
+
"description",
|
|
5816
|
+
"summary",
|
|
5817
|
+
"details",
|
|
5818
|
+
"body"
|
|
5819
|
+
];
|
|
5820
|
+
for (const key of preferredKeys) {
|
|
5821
|
+
const match = Object.keys(meta).find(
|
|
5822
|
+
(candidate) => this.normalizeComparableField(candidate) === this.normalizeComparableField(key)
|
|
4761
5823
|
);
|
|
4762
|
-
|
|
4763
|
-
if (parsed) {
|
|
4764
|
-
console.debug("[UITransformer] LLM chose visualization type:", parsed.type);
|
|
4765
|
-
return parsed;
|
|
4766
|
-
}
|
|
4767
|
-
console.warn("[UITransformer] LLM returned unparseable response; falling back to heuristic.");
|
|
4768
|
-
} catch (err) {
|
|
4769
|
-
console.warn("[UITransformer] analyzeAndDecide LLM call failed; falling back to heuristic.", err);
|
|
5824
|
+
if (match && meta[match] !== void 0 && meta[match] !== null) return meta[match];
|
|
4770
5825
|
}
|
|
4771
|
-
return
|
|
5826
|
+
return void 0;
|
|
4772
5827
|
}
|
|
4773
|
-
|
|
4774
|
-
|
|
4775
|
-
|
|
5828
|
+
static cleanProductDescription(raw) {
|
|
5829
|
+
if (raw === null || raw === void 0) return void 0;
|
|
5830
|
+
const extracted = this.extractProductDescriptionFromContent(String(raw));
|
|
5831
|
+
if (extracted && extracted !== String(raw)) return this.cleanProductDescription(extracted);
|
|
5832
|
+
const text = String(raw).replace(/<[^>]*>/g, " ").replace(/ /gi, " ").replace(/&/gi, "&").replace(/"/gi, '"').replace(/'/gi, "'").replace(/\[[^\]]*TYPE[^\]]*\]/gi, " ").replace(/\b(?:Handle|Title|Vendor|Type|Tags|Published|Option\d*\s+Name|Option\d*\s+Value|Option|Variant|Image|SKU|Price|Status|Category)\s*:\s*[^,.\n:]+[,.]?/gi, " ").replace(/\bBody\s*\(HTML\)\s*:\s*/gi, " ").replace(/\b(?:Description|Summary|Details|Body|Content)\s*:\s*/gi, " ").replace(/\s+/g, " ").trim();
|
|
5833
|
+
return text || void 0;
|
|
5834
|
+
}
|
|
5835
|
+
// ─── Visualization Prompt ─────────────────────────────────────────────────
|
|
4776
5836
|
static buildVisualizationSystemPrompt() {
|
|
4777
5837
|
return `You are a data visualization expert embedded in a RAG chat system.
|
|
4778
|
-
You will receive a user question and structured data
|
|
4779
|
-
Your ONLY job is to
|
|
4780
|
-
|
|
5838
|
+
You will receive a user question, a pre-computed INTENT object, and structured data from a vector database.
|
|
5839
|
+
Your ONLY job is to return a single JSON object describing how to visualize the data.
|
|
5840
|
+
The INTENT object is authoritative \u2014 honor it. For example:
|
|
5841
|
+
- If intent.wantsExplicitTable is false, never return type "table".
|
|
5842
|
+
- If intent.isTemporal is true, prefer line_chart.
|
|
5843
|
+
- If intent.visualizationHint is "comparison" or "ranking", prefer bar_chart.
|
|
5844
|
+
- If intent.visualizationHint is "composition", prefer pie_chart.
|
|
5845
|
+
- If intent.recommendedChart is unsupported by the response schema, use the nearest supported type.
|
|
5846
|
+
- Always respond in the language specified by intent.language.
|
|
4781
5847
|
|
|
4782
|
-
Return ONLY a valid JSON object \u2014 no markdown
|
|
5848
|
+
Return ONLY a valid JSON object \u2014 no markdown fences, no prose.
|
|
4783
5849
|
|
|
4784
|
-
The JSON must have this exact shape:
|
|
4785
5850
|
{
|
|
4786
|
-
"type": "bar_chart" | "line_chart" | "pie_chart" | "radar_chart" | "table" | "text",
|
|
4787
|
-
"title": "
|
|
4788
|
-
"description": "
|
|
4789
|
-
"data": <structured data
|
|
5851
|
+
"type": "bar_chart" | "horizontal_bar" | "line_chart" | "pie_chart" | "histogram" | "scatter_plot" | "radar_chart" | "metric_card" | "geo_map" | "table" | "text",
|
|
5852
|
+
"title": "concise descriptive title",
|
|
5853
|
+
"description": "one sentence describing the visualization",
|
|
5854
|
+
"data": <structured data per type below>
|
|
4790
5855
|
}
|
|
4791
5856
|
|
|
4792
|
-
DATA
|
|
4793
|
-
- bar_chart:
|
|
4794
|
-
-
|
|
4795
|
-
-
|
|
4796
|
-
-
|
|
4797
|
-
|
|
4798
|
-
|
|
4799
|
-
|
|
4800
|
-
|
|
4801
|
-
|
|
4802
|
-
|
|
4803
|
-
-
|
|
4804
|
-
- text: { "content": "<prose answer>" }
|
|
5857
|
+
DATA SHAPES:
|
|
5858
|
+
- bar_chart: [{ "category": string, "value": number }]
|
|
5859
|
+
- horizontal_bar: [{ "category": string, "value": number }]
|
|
5860
|
+
- histogram: [{ "category": string, "value": number }]
|
|
5861
|
+
- line_chart: [{ "timestamp": string, "value": number, "label": string }]
|
|
5862
|
+
- pie_chart: [{ "label": string, "value": number }]
|
|
5863
|
+
- scatter_plot:[{ "x": number, "y": number, "label": string }]
|
|
5864
|
+
- radar_chart: [{ "attribute": string, "<series1>": number, "<series2>": number, ... }]
|
|
5865
|
+
- metric_card: { "label": string, "value": number, "operation": "sum"|"average"|"count"|"min"|"max"|"median" }
|
|
5866
|
+
- geo_map: [{ "category": string, "value": number }]
|
|
5867
|
+
- table: { "columns": string[], "rows": (string|number)[][] }
|
|
5868
|
+
- text: { "content": "A concise plain-language answer." }
|
|
4805
5869
|
|
|
4806
|
-
|
|
4807
|
-
1.
|
|
4808
|
-
2.
|
|
4809
|
-
3.
|
|
4810
|
-
4.
|
|
4811
|
-
5.
|
|
4812
|
-
6. table \u2192 multi-field structured records where each item has \u2265 3 attributes
|
|
4813
|
-
7. text \u2192 conversational or free-form answers where no chart adds value
|
|
4814
|
-
|
|
4815
|
-
IMPORTANT:
|
|
4816
|
-
- Aggregate numeric values from the raw data \u2014 do not pass raw object arrays as data.
|
|
4817
|
-
- Ensure all "value" fields are numbers, not strings.
|
|
4818
|
-
- For bar/line/pie, keep at most 12 data points for readability.
|
|
4819
|
-
- Never include nested objects or arrays inside bar_chart / line_chart / pie_chart data items.`;
|
|
5870
|
+
RULES:
|
|
5871
|
+
1. Aggregate values \u2014 never pass raw nested objects in chart arrays.
|
|
5872
|
+
2. All "value" fields must be numbers.
|
|
5873
|
+
3. Cap bar/line/pie at 12 data points.
|
|
5874
|
+
4. Use dollar signs only for monetary prices, not for stock quantities or years.
|
|
5875
|
+
5. If no relevant data is found, return text: "I cannot answer this question based on the available information."`;
|
|
4820
5876
|
}
|
|
4821
|
-
/**
|
|
4822
|
-
* Serialize retrieved vector matches into a compact JSON context string.
|
|
4823
|
-
* Limits the total character count to avoid exceeding LLM context windows.
|
|
4824
|
-
*/
|
|
4825
5877
|
static buildContextSummary(sources, maxChars = 6e3) {
|
|
4826
5878
|
const items = sources.map((s, i) => {
|
|
4827
5879
|
var _a, _b, _c, _d;
|
|
@@ -5042,27 +6094,38 @@ var Pipeline = class {
|
|
|
5042
6094
|
async initialize() {
|
|
5043
6095
|
var _a;
|
|
5044
6096
|
if (this.initialised) return;
|
|
5045
|
-
const chartInstruction = `You are a helpful
|
|
6097
|
+
const chartInstruction = `You are a helpful assistant. Use the provided context to answer questions accurately.
|
|
5046
6098
|
|
|
5047
|
-
###
|
|
6099
|
+
### CRITICAL RULES:
|
|
6100
|
+
- ONLY answer the user's specific question. Do NOT suggest or recommend unrelated products unless specifically asked.
|
|
5048
6101
|
- NEVER generate markdown tables. If you do, the UI will break.
|
|
5049
6102
|
- NEVER generate HTML tags like <figure>, <tbody>, <tr>, etc.
|
|
5050
6103
|
- NEVER generate text-based charts or graphs.
|
|
6104
|
+
- NEVER say you cannot display, render, draw, or create a chart when the user asks for a visualization. The UI handles visual rendering separately.
|
|
5051
6105
|
- ONLY use plain text and bullet points.
|
|
6106
|
+
- NEVER use the plus sign (+) as a separator between names, categories, or products. Use commas or bullet points.
|
|
6107
|
+
- ONLY answer the question if the sources contain relevant information to answer it, else say that you cannot answer the question.
|
|
6108
|
+
- If answer cannot be found in the sources, say that you cannot answer the question and do not hallucinate.
|
|
6109
|
+
- Do not use information from previous turns to answer the question.
|
|
6110
|
+
- You CAN use numbers for years and counts (e.g., 2006, 5800). But NEVER put a dollar sign ($) before non-monetary numbers like Stock quantities, EAN numbers, or years. Only use dollar signs for actual monetary prices.
|
|
5052
6111
|
|
|
5053
6112
|
### PRODUCT DISPLAY:
|
|
5054
|
-
-
|
|
6113
|
+
- Recommended Products should only be displayed when the user explicitly asks for products.
|
|
6114
|
+
- When recommending products (ONLY when asked), simply list their names, prices, and features in a friendly, conversational manner.
|
|
5055
6115
|
- The UI will automatically detect these products and show high-quality product cards in a carousel below your message.
|
|
6116
|
+
- For product descriptions, summarize customer-facing details only. Do NOT list internal catalog fields such as Handle, Body (HTML), Vendor, Type, Tags, Published, Option, Variant, SKU, or raw metadata labels.
|
|
5056
6117
|
- Do NOT try to format product lists as tables.
|
|
5057
6118
|
`;
|
|
5058
6119
|
this.config.llm.systemPrompt = chartInstruction;
|
|
5059
6120
|
this.vectorDB = await ProviderRegistry.createVectorProvider(this.config.vectorDb);
|
|
5060
|
-
|
|
6121
|
+
this.llmRouter = new LLMRouter(this.config);
|
|
6122
|
+
const { llmProvider: resolvedLLM, embeddingProvider } = await EmbeddingStrategyResolver.resolve(
|
|
5061
6123
|
this.config.llm,
|
|
5062
6124
|
this.config.embedding
|
|
5063
6125
|
);
|
|
5064
|
-
this.llmProvider = llmProvider;
|
|
5065
6126
|
this.embeddingProvider = embeddingProvider;
|
|
6127
|
+
await this.llmRouter.initialize(resolvedLLM);
|
|
6128
|
+
this.llmProvider = this.llmRouter.get("default");
|
|
5066
6129
|
if (this.config.graphDb) {
|
|
5067
6130
|
this.graphDB = await ProviderRegistry.createGraphProvider(this.config.graphDb);
|
|
5068
6131
|
await this.graphDB.initialize();
|
|
@@ -5219,10 +6282,16 @@ var Pipeline = class {
|
|
|
5219
6282
|
/**
|
|
5220
6283
|
* High-performance streaming RAG flow.
|
|
5221
6284
|
* Yields text chunks first, then the retrieval metadata + observability trace at the end.
|
|
6285
|
+
*
|
|
6286
|
+
* Latency optimizations:
|
|
6287
|
+
* - Strategy classification runs in parallel with query embedding (saves ~400ms)
|
|
6288
|
+
* - Hallucination scoring is fire-and-forget (doesn't block metadata yield)
|
|
6289
|
+
* - UITransformation is computed after text streaming and emitted with metadata
|
|
6290
|
+
* - SchemaMapper.train runs while answer generation streams
|
|
5222
6291
|
*/
|
|
5223
6292
|
askStream(_0) {
|
|
5224
6293
|
return __asyncGenerator(this, arguments, function* (question, history = [], namespace) {
|
|
5225
|
-
var _a, _b, _c, _d, _e, _f, _g, _h, _i;
|
|
6294
|
+
var _a, _b, _c, _d, _e, _f, _g, _h, _i, _j, _k, _l, _m;
|
|
5226
6295
|
yield new __await(this.initialize());
|
|
5227
6296
|
const ns = namespace != null ? namespace : this.config.projectId;
|
|
5228
6297
|
const topK = (_b = (_a = this.config.rag) == null ? void 0 : _a.topK) != null ? _b : 5;
|
|
@@ -5238,27 +6307,48 @@ var Pipeline = class {
|
|
|
5238
6307
|
}
|
|
5239
6308
|
const hints = QueryProcessor.extractQueryFieldHints(question, (_f = this.config.rag) == null ? void 0 : _f.filterableFields);
|
|
5240
6309
|
const filter = QueryProcessor.buildQueryFilter(question, hints);
|
|
6310
|
+
const numericPredicates = QueryProcessor.extractNumericPredicates(question, (_g = this.config.rag) == null ? void 0 : _g.filterableFields);
|
|
6311
|
+
if (numericPredicates.length > 0) {
|
|
6312
|
+
filter.__numericPredicates = numericPredicates;
|
|
6313
|
+
}
|
|
5241
6314
|
const embedStart = performance.now();
|
|
5242
|
-
const
|
|
5243
|
-
|
|
5244
|
-
|
|
5245
|
-
|
|
5246
|
-
|
|
6315
|
+
const cacheKey = `${ns}::${searchQuery}`;
|
|
6316
|
+
const cachedVector = this.embeddingCache.get(cacheKey);
|
|
6317
|
+
const [strategyResult, embeddedVector] = yield new __await(Promise.all([
|
|
6318
|
+
QueryProcessor.determineRetrievalStrategy(
|
|
6319
|
+
searchQuery,
|
|
6320
|
+
this.llmRouter.get("fast"),
|
|
6321
|
+
(_h = this.config.rag) == null ? void 0 : _h.graphKeywords,
|
|
6322
|
+
(_i = this.config.rag) == null ? void 0 : _i.vectorKeywords
|
|
6323
|
+
),
|
|
6324
|
+
// Embed immediately regardless of strategy — costs nothing if cached.
|
|
6325
|
+
// If strategy turns out to be 'graph'-only we just won't use the vector.
|
|
6326
|
+
cachedVector ? Promise.resolve(cachedVector) : this.embeddingProvider.embed(searchQuery, { taskType: "query" })
|
|
6327
|
+
]));
|
|
6328
|
+
const queryVector = embeddedVector;
|
|
6329
|
+
if (!cachedVector && queryVector.length > 0) {
|
|
6330
|
+
this.embeddingCache.set(cacheKey, queryVector);
|
|
6331
|
+
}
|
|
6332
|
+
const graphData = (strategyResult === "graph" || strategyResult === "both") && this.graphDB && ((_j = this.config.rag) == null ? void 0 : _j.useGraphRetrieval) ? yield new __await(this.graphDB.query(searchQuery)) : void 0;
|
|
6333
|
+
const hasNumericPredicates = Array.isArray(filter.__numericPredicates) && filter.__numericPredicates.length > 0;
|
|
6334
|
+
const wantsExhaustiveList = hasNumericPredicates || /\b(list|all|show|provide|give|get)\b/i.test(question);
|
|
6335
|
+
const retrievalLimit = wantsExhaustiveList ? Math.max(topK * 20, 100) : topK * 2;
|
|
6336
|
+
const rawSources = (strategyResult === "vector" || strategyResult === "both") && queryVector && queryVector.length > 0 ? yield new __await(this.vectorDB.query(queryVector, retrievalLimit, ns, filter)) : [];
|
|
5247
6337
|
const retrieveEnd = performance.now();
|
|
5248
6338
|
const embedMs = retrieveEnd - embedStart;
|
|
5249
6339
|
const retrieveMs = retrieveEnd - embedStart;
|
|
5250
6340
|
const rerankStart = performance.now();
|
|
5251
|
-
|
|
5252
|
-
|
|
6341
|
+
const structuredSources = this.applyStructuredFilters(rawSources, filter);
|
|
6342
|
+
let sources = hasNumericPredicates ? structuredSources : structuredSources.filter((m) => m.score >= scoreThreshold);
|
|
6343
|
+
if (!hasNumericPredicates && ((_k = this.config.rag) == null ? void 0 : _k.useReranking)) {
|
|
5253
6344
|
sources = yield new __await(this.reranker.rerank(sources, question, topK));
|
|
5254
|
-
} else {
|
|
6345
|
+
} else if (!wantsExhaustiveList) {
|
|
5255
6346
|
sources = sources.slice(0, topK);
|
|
5256
6347
|
}
|
|
5257
6348
|
const rerankMs = performance.now() - rerankStart;
|
|
5258
6349
|
let context = sources.length ? sources.map((m, i) => `[Source ${i + 1}]
|
|
5259
6350
|
${m.content}`).join("\n\n---\n\n") : "No relevant context found.";
|
|
5260
6351
|
if (graphData && graphData.nodes.length > 0) {
|
|
5261
|
-
console.log(`[Graph Retrieval] Found ${graphData.nodes.length} relevant entities.`);
|
|
5262
6352
|
const graphContext = graphData.nodes.map(
|
|
5263
6353
|
(n) => `Entity: ${n.label} (${n.id})${n.properties ? " - " + JSON.stringify(n.properties) : ""}`
|
|
5264
6354
|
).join("\n");
|
|
@@ -5269,20 +6359,18 @@ VECTOR CONTEXT:
|
|
|
5269
6359
|
${context}`;
|
|
5270
6360
|
}
|
|
5271
6361
|
const allMetadataKeys = Array.from(new Set(sources.flatMap((s) => Object.keys(s.metadata || {}))));
|
|
5272
|
-
const
|
|
5273
|
-
const restrictionSuffix = "\n\n(IMPORTANT: Use plain text only. NEVER generate tables, HTML figures, or text charts. If listing products, use simple bullet points.)";
|
|
6362
|
+
const trainedSchemaPromise = allMetadataKeys.length > 0 ? SchemaMapper.train(this.llmProvider, ns, allMetadataKeys).catch(() => void 0) : Promise.resolve(void 0);
|
|
6363
|
+
const restrictionSuffix = "\n\n(IMPORTANT: Use plain text only. NEVER generate tables, HTML figures, or text charts. The UI renders requested charts separately, so NEVER say you cannot create, display, draw, render, or provide a chart/visualization. If listing products, use simple bullet points or comma-separated names. NEVER use plus signs (+) to separate product names, category names, or list items. For product description/detail questions, provide customer-facing prose only and do NOT list internal catalog fields such as Handle, Body (HTML), Vendor, Type, Tags, Published, Option, Variant, SKU, or raw metadata labels. You CAN use numbers for years/counts like 2006 or 5800, but NEVER put a dollar sign ($) before them.)";
|
|
5274
6364
|
const hardenedHistory = [...history];
|
|
5275
6365
|
const userQuestion = { role: "user", content: question + restrictionSuffix };
|
|
5276
6366
|
const messages = [...hardenedHistory, userQuestion];
|
|
5277
|
-
const systemPrompt = (
|
|
6367
|
+
const systemPrompt = (_l = this.config.llm.systemPrompt) != null ? _l : "";
|
|
5278
6368
|
const userPrompt = messages.map((m) => `${m.role}: ${m.content}`).join("\n");
|
|
5279
6369
|
let fullReply = "";
|
|
5280
6370
|
const generateStart = performance.now();
|
|
5281
6371
|
if (this.llmProvider.chatStream) {
|
|
5282
6372
|
const stream = this.llmProvider.chatStream(messages, context);
|
|
5283
|
-
if (!stream) {
|
|
5284
|
-
throw new Error(`[Pipeline] ${this.config.llm.provider} chatStream returned undefined`);
|
|
5285
|
-
}
|
|
6373
|
+
if (!stream) throw new Error(`[Pipeline] ${this.config.llm.provider} chatStream returned undefined`);
|
|
5286
6374
|
try {
|
|
5287
6375
|
for (var iter = __forAwait(stream), more, temp, error; more = !(temp = yield new __await(iter.next())).done; more = false) {
|
|
5288
6376
|
const chunk = temp.value;
|
|
@@ -5309,7 +6397,7 @@ ${context}`;
|
|
|
5309
6397
|
const latency = {
|
|
5310
6398
|
embedMs: Math.round(embedMs),
|
|
5311
6399
|
retrieveMs: Math.round(retrieveMs),
|
|
5312
|
-
rerankMs: ((
|
|
6400
|
+
rerankMs: ((_m = this.config.rag) == null ? void 0 : _m.useReranking) ? Math.round(rerankMs) : void 0,
|
|
5313
6401
|
generateMs: Math.round(generateMs),
|
|
5314
6402
|
totalMs: Math.round(totalMs)
|
|
5315
6403
|
};
|
|
@@ -5322,9 +6410,6 @@ ${context}`;
|
|
|
5322
6410
|
totalTokens: promptTokens + completionTokens,
|
|
5323
6411
|
estimatedCostUsd: estimateCostUsd(promptTokens, completionTokens, this.config.llm.model)
|
|
5324
6412
|
};
|
|
5325
|
-
const hallucinationResult = yield new __await(scoreHallucination(this.llmProvider, fullReply, context).catch(() => void 0));
|
|
5326
|
-
const trainedSchema = yield new __await(trainingPromise);
|
|
5327
|
-
const uiTransformation = yield new __await(this.generateUiTransformation(question, sources, trainedSchema));
|
|
5328
6413
|
const trace = {
|
|
5329
6414
|
requestId,
|
|
5330
6415
|
query: question,
|
|
@@ -5343,10 +6428,21 @@ ${context}`;
|
|
|
5343
6428
|
}),
|
|
5344
6429
|
latency,
|
|
5345
6430
|
tokens,
|
|
5346
|
-
hallucinationScore: hallucinationResult == null ? void 0 : hallucinationResult.score,
|
|
5347
|
-
hallucinationReason: hallucinationResult == null ? void 0 : hallucinationResult.reason,
|
|
5348
6431
|
timestamp: (/* @__PURE__ */ new Date()).toISOString()
|
|
5349
6432
|
};
|
|
6433
|
+
let uiTransformation;
|
|
6434
|
+
try {
|
|
6435
|
+
const trainedSchema = yield new __await(trainedSchemaPromise);
|
|
6436
|
+
uiTransformation = yield new __await(this.generateUiTransformation(
|
|
6437
|
+
question,
|
|
6438
|
+
sources,
|
|
6439
|
+
trainedSchema,
|
|
6440
|
+
hasNumericPredicates
|
|
6441
|
+
));
|
|
6442
|
+
} catch (uiError) {
|
|
6443
|
+
console.warn("[Pipeline] UI transformation failed before metadata yield:", uiError);
|
|
6444
|
+
uiTransformation = UITransformer.transform(question, sources, this.config);
|
|
6445
|
+
}
|
|
5350
6446
|
yield {
|
|
5351
6447
|
reply: "",
|
|
5352
6448
|
sources,
|
|
@@ -5354,6 +6450,13 @@ ${context}`;
|
|
|
5354
6450
|
ui_transformation: uiTransformation,
|
|
5355
6451
|
trace
|
|
5356
6452
|
};
|
|
6453
|
+
scoreHallucination(this.llmProvider, fullReply, context).then((hallucinationResult) => {
|
|
6454
|
+
if (hallucinationResult) {
|
|
6455
|
+
trace.hallucinationScore = hallucinationResult.score;
|
|
6456
|
+
trace.hallucinationReason = hallucinationResult.reason;
|
|
6457
|
+
}
|
|
6458
|
+
}).catch(() => {
|
|
6459
|
+
});
|
|
5357
6460
|
} catch (error2) {
|
|
5358
6461
|
throw new Error(`[Pipeline] Stream failed: ${error2 instanceof Error ? error2.message : String(error2)}`);
|
|
5359
6462
|
}
|
|
@@ -5363,24 +6466,107 @@ ${context}`;
|
|
|
5363
6466
|
* Universal retrieval method combining all enabled providers.
|
|
5364
6467
|
* Uses an LRU-bounded embedding cache to avoid re-embedding the same query.
|
|
5365
6468
|
*/
|
|
5366
|
-
async generateUiTransformation(question, sources, trainedSchema) {
|
|
6469
|
+
async generateUiTransformation(question, sources, trainedSchema, forceDeterministic = false) {
|
|
5367
6470
|
if (!sources || sources.length === 0) {
|
|
5368
6471
|
return UITransformer.transform(question, sources, this.config, trainedSchema);
|
|
5369
6472
|
}
|
|
6473
|
+
if (forceDeterministic) {
|
|
6474
|
+
return UITransformer.transform(question, sources, this.config, trainedSchema);
|
|
6475
|
+
}
|
|
5370
6476
|
try {
|
|
5371
|
-
return await UITransformer.analyzeAndDecide(question, sources, this.
|
|
6477
|
+
return await UITransformer.analyzeAndDecide(question, sources, this.llmRouter.get("fast"));
|
|
5372
6478
|
} catch (err) {
|
|
5373
6479
|
console.warn("[Pipeline] generateUiTransformation failed, using heuristic fallback:", err);
|
|
5374
6480
|
return UITransformer.transform(question, sources, this.config, trainedSchema);
|
|
5375
6481
|
}
|
|
5376
6482
|
}
|
|
6483
|
+
applyStructuredFilters(sources, filter) {
|
|
6484
|
+
const predicates = Array.isArray(filter.__numericPredicates) ? filter.__numericPredicates : [];
|
|
6485
|
+
if (predicates.length === 0) return sources;
|
|
6486
|
+
return sources.filter((source) => predicates.every((predicate) => {
|
|
6487
|
+
const value = this.resolveNumericPredicateValue(source, predicate);
|
|
6488
|
+
return value !== null && this.matchesNumericPredicate(value, predicate);
|
|
6489
|
+
})).sort((a, b) => {
|
|
6490
|
+
var _a, _b;
|
|
6491
|
+
const primary = predicates[0];
|
|
6492
|
+
const aValue = (_a = this.resolveNumericPredicateValue(a, primary)) != null ? _a : 0;
|
|
6493
|
+
const bValue = (_b = this.resolveNumericPredicateValue(b, primary)) != null ? _b : 0;
|
|
6494
|
+
return primary.operator === "lt" || primary.operator === "lte" ? aValue - bValue : bValue - aValue;
|
|
6495
|
+
});
|
|
6496
|
+
}
|
|
6497
|
+
resolveNumericPredicateValue(source, predicate) {
|
|
6498
|
+
const meta = source.metadata || {};
|
|
6499
|
+
const field = predicate.field;
|
|
6500
|
+
const entries = Object.entries(meta).filter(([, value]) => value !== null && value !== void 0 && typeof value !== "object");
|
|
6501
|
+
if (field) {
|
|
6502
|
+
const normalizedField = this.normalizeComparableField(field);
|
|
6503
|
+
const exact = entries.find(([key]) => this.normalizeComparableField(key) === normalizedField);
|
|
6504
|
+
const fuzzy = exact != null ? exact : entries.map(([key, value]) => ({ key, value, score: this.fieldSimilarityScore(key, field) })).filter((candidate) => candidate.score > 0).sort((a, b) => b.score - a.score)[0];
|
|
6505
|
+
if (fuzzy) {
|
|
6506
|
+
const value = this.toFiniteNumber(Array.isArray(fuzzy) ? fuzzy[1] : fuzzy.value);
|
|
6507
|
+
if (value !== null) return value;
|
|
6508
|
+
}
|
|
6509
|
+
const contentValue = this.extractNumericValueFromContent(source.content, field);
|
|
6510
|
+
if (contentValue !== null) return contentValue;
|
|
6511
|
+
}
|
|
6512
|
+
for (const [key, value] of entries) {
|
|
6513
|
+
if (/(id|sku|ean|upc|phone|zip|postal|code)/i.test(key)) continue;
|
|
6514
|
+
const numeric = this.toFiniteNumber(value);
|
|
6515
|
+
if (numeric !== null) return numeric;
|
|
6516
|
+
}
|
|
6517
|
+
return null;
|
|
6518
|
+
}
|
|
6519
|
+
extractNumericValueFromContent(content, field) {
|
|
6520
|
+
const escapedWords = field.split(/\s+|_+|-+/).filter(Boolean).map((word) => word.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"));
|
|
6521
|
+
if (escapedWords.length === 0) return null;
|
|
6522
|
+
const pattern = new RegExp(`(?:${escapedWords.join("[\\\\s_:-]*")})\\s*[:\\-\u2013]?\\s*([\\d,]+(?:\\.\\d+)?)`, "i");
|
|
6523
|
+
const match = content.match(pattern);
|
|
6524
|
+
return match ? this.toFiniteNumber(match[1]) : null;
|
|
6525
|
+
}
|
|
6526
|
+
matchesNumericPredicate(value, predicate) {
|
|
6527
|
+
switch (predicate.operator) {
|
|
6528
|
+
case "gt":
|
|
6529
|
+
return value > predicate.value;
|
|
6530
|
+
case "gte":
|
|
6531
|
+
return value >= predicate.value;
|
|
6532
|
+
case "lt":
|
|
6533
|
+
return value < predicate.value;
|
|
6534
|
+
case "lte":
|
|
6535
|
+
return value <= predicate.value;
|
|
6536
|
+
case "eq":
|
|
6537
|
+
default:
|
|
6538
|
+
return value === predicate.value;
|
|
6539
|
+
}
|
|
6540
|
+
}
|
|
6541
|
+
normalizeComparableField(value) {
|
|
6542
|
+
return value.toLowerCase().replace(/[^a-z0-9]/g, "");
|
|
6543
|
+
}
|
|
6544
|
+
fieldSimilarityScore(candidate, requested) {
|
|
6545
|
+
const normalizedCandidate = this.normalizeComparableField(candidate);
|
|
6546
|
+
const normalizedRequested = this.normalizeComparableField(requested);
|
|
6547
|
+
if (normalizedCandidate === normalizedRequested) return 10;
|
|
6548
|
+
if (normalizedCandidate.includes(normalizedRequested) || normalizedRequested.includes(normalizedCandidate)) return 8;
|
|
6549
|
+
const candidateTokens = this.fieldTokens(candidate);
|
|
6550
|
+
const requestedTokens = this.fieldTokens(requested);
|
|
6551
|
+
const overlap = requestedTokens.filter((token) => candidateTokens.includes(token)).length;
|
|
6552
|
+
if (overlap === 0) return 0;
|
|
6553
|
+
return overlap / Math.max(requestedTokens.length, candidateTokens.length);
|
|
6554
|
+
}
|
|
6555
|
+
fieldTokens(value) {
|
|
6556
|
+
return value.toLowerCase().split(/[^a-z0-9]+/).map((token) => token.replace(/ies$/, "y").replace(/s$/, "")).filter((token) => token.length > 1);
|
|
6557
|
+
}
|
|
6558
|
+
toFiniteNumber(value) {
|
|
6559
|
+
if (typeof value === "number") return Number.isFinite(value) ? value : null;
|
|
6560
|
+
const numeric = Number(String(value != null ? value : "").replace(/[$,% ,]/g, ""));
|
|
6561
|
+
return Number.isFinite(numeric) ? numeric : null;
|
|
6562
|
+
}
|
|
5377
6563
|
async retrieve(query, options) {
|
|
5378
|
-
var _a, _b, _c;
|
|
6564
|
+
var _a, _b, _c, _d, _e, _f;
|
|
5379
6565
|
const ns = (_a = options.namespace) != null ? _a : this.config.projectId;
|
|
5380
6566
|
const topK = (_b = options.topK) != null ? _b : 5;
|
|
5381
6567
|
const cacheKey = `${ns}::${query}`;
|
|
5382
6568
|
let queryVector = this.embeddingCache.get(cacheKey);
|
|
5383
|
-
const strategy = await QueryProcessor.determineRetrievalStrategy(query, this.
|
|
6569
|
+
const strategy = await QueryProcessor.determineRetrievalStrategy(query, this.llmRouter.get("fast"));
|
|
5384
6570
|
console.debug(`[Pipeline] Determined retrieval strategy: ${strategy}`);
|
|
5385
6571
|
const [retrievedVector, graphData] = await Promise.all([
|
|
5386
6572
|
// Only embed if we need vector search (strategy is 'vector' or 'both')
|
|
@@ -5392,8 +6578,27 @@ ${context}`;
|
|
|
5392
6578
|
this.embeddingCache.set(cacheKey, retrievedVector);
|
|
5393
6579
|
queryVector = retrievedVector;
|
|
5394
6580
|
}
|
|
5395
|
-
const
|
|
5396
|
-
|
|
6581
|
+
const baseFilter = __spreadProps(__spreadValues({}, (_d = options.filter) != null ? _d : {}), { queryText: query });
|
|
6582
|
+
const numericPredicates = QueryProcessor.extractNumericPredicates(query, (_e = this.config.rag) == null ? void 0 : _e.filterableFields);
|
|
6583
|
+
if (numericPredicates.length > 0) {
|
|
6584
|
+
baseFilter.__numericPredicates = numericPredicates;
|
|
6585
|
+
}
|
|
6586
|
+
const retrievalLimit = numericPredicates.length > 0 ? Math.max(topK * 20, 100) : topK;
|
|
6587
|
+
const sources = (strategy === "vector" || strategy === "both") && queryVector && queryVector.length > 0 ? this.applyStructuredFilters(
|
|
6588
|
+
await this.vectorDB.query(queryVector, retrievalLimit, ns, baseFilter),
|
|
6589
|
+
baseFilter
|
|
6590
|
+
) : [];
|
|
6591
|
+
const resolvedSources = [];
|
|
6592
|
+
for (const source of sources) {
|
|
6593
|
+
const parentId = (_f = source.metadata) == null ? void 0 : _f.parent_id;
|
|
6594
|
+
if (parentId) {
|
|
6595
|
+
console.log(`[Pipeline] Multi-Vector: Found child chunk. Parent ID: ${parentId}`);
|
|
6596
|
+
resolvedSources.push(source);
|
|
6597
|
+
} else {
|
|
6598
|
+
resolvedSources.push(source);
|
|
6599
|
+
}
|
|
6600
|
+
}
|
|
6601
|
+
return { sources: resolvedSources, graphData };
|
|
5397
6602
|
}
|
|
5398
6603
|
/** Rewrite the user query for better retrieval performance. */
|
|
5399
6604
|
async rewriteQuery(question, history) {
|
|
@@ -5885,209 +7090,7 @@ var DocumentParser = class {
|
|
|
5885
7090
|
// src/server.ts
|
|
5886
7091
|
init_BaseVectorProvider();
|
|
5887
7092
|
init_PineconeProvider();
|
|
5888
|
-
|
|
5889
|
-
// src/providers/vectordb/PostgreSQLProvider.ts
|
|
5890
|
-
init_BaseVectorProvider();
|
|
5891
|
-
import { Pool as Pool2 } from "pg";
|
|
5892
|
-
var PostgreSQLProvider = class extends BaseVectorProvider {
|
|
5893
|
-
constructor(config) {
|
|
5894
|
-
var _a;
|
|
5895
|
-
super(config);
|
|
5896
|
-
this.tableName = this.indexName.replace(/[^a-z0-9_]/gi, "_");
|
|
5897
|
-
const opts = config.options;
|
|
5898
|
-
if (!opts.connectionString) throw new Error("[PostgreSQLProvider] options.connectionString is required");
|
|
5899
|
-
this.connectionString = opts.connectionString;
|
|
5900
|
-
this.dimensions = (_a = opts.dimensions) != null ? _a : 1536;
|
|
5901
|
-
}
|
|
5902
|
-
static getValidator() {
|
|
5903
|
-
return {
|
|
5904
|
-
validate(config) {
|
|
5905
|
-
const errors = [];
|
|
5906
|
-
const opts = config.options || {};
|
|
5907
|
-
if (!opts.connectionString) {
|
|
5908
|
-
errors.push({
|
|
5909
|
-
field: "vectorDb.options.connectionString",
|
|
5910
|
-
message: "PostgreSQL connection string is required",
|
|
5911
|
-
suggestion: "Set PGVECTOR_CONNECTION_STRING environment variable",
|
|
5912
|
-
severity: "error"
|
|
5913
|
-
});
|
|
5914
|
-
}
|
|
5915
|
-
if (opts.tables && typeof opts.tables !== "string" && !Array.isArray(opts.tables)) {
|
|
5916
|
-
errors.push({
|
|
5917
|
-
field: "vectorDb.options.tables",
|
|
5918
|
-
message: "PostgreSQL tables must be a string or a string array",
|
|
5919
|
-
severity: "error"
|
|
5920
|
-
});
|
|
5921
|
-
}
|
|
5922
|
-
return errors;
|
|
5923
|
-
}
|
|
5924
|
-
};
|
|
5925
|
-
}
|
|
5926
|
-
static getHealthChecker() {
|
|
5927
|
-
return {
|
|
5928
|
-
async check(config) {
|
|
5929
|
-
const opts = config.options || {};
|
|
5930
|
-
const timestamp = Date.now();
|
|
5931
|
-
try {
|
|
5932
|
-
const { Client } = await import("pg");
|
|
5933
|
-
const client = new Client({ connectionString: opts.connectionString });
|
|
5934
|
-
await client.connect();
|
|
5935
|
-
const result = await client.query(`
|
|
5936
|
-
SELECT EXISTS(SELECT 1 FROM pg_extension WHERE extname = 'vector');
|
|
5937
|
-
`);
|
|
5938
|
-
const hasVector = result.rows[0].exists;
|
|
5939
|
-
await client.end();
|
|
5940
|
-
return {
|
|
5941
|
-
healthy: true,
|
|
5942
|
-
provider: "postgresql",
|
|
5943
|
-
capabilities: { pgvectorInstalled: hasVector },
|
|
5944
|
-
timestamp
|
|
5945
|
-
};
|
|
5946
|
-
} catch (error) {
|
|
5947
|
-
return {
|
|
5948
|
-
healthy: false,
|
|
5949
|
-
provider: "postgresql",
|
|
5950
|
-
error: `Connection failed: ${error instanceof Error ? error.message : String(error)}`,
|
|
5951
|
-
timestamp
|
|
5952
|
-
};
|
|
5953
|
-
}
|
|
5954
|
-
}
|
|
5955
|
-
};
|
|
5956
|
-
}
|
|
5957
|
-
async initialize() {
|
|
5958
|
-
this.pool = new Pool2({ connectionString: this.connectionString });
|
|
5959
|
-
const client = await this.pool.connect();
|
|
5960
|
-
try {
|
|
5961
|
-
await client.query("CREATE EXTENSION IF NOT EXISTS vector");
|
|
5962
|
-
await client.query(`
|
|
5963
|
-
CREATE TABLE IF NOT EXISTS ${this.tableName} (
|
|
5964
|
-
id TEXT PRIMARY KEY,
|
|
5965
|
-
namespace TEXT NOT NULL DEFAULT '',
|
|
5966
|
-
content TEXT NOT NULL,
|
|
5967
|
-
metadata JSONB,
|
|
5968
|
-
embedding VECTOR(${this.dimensions})
|
|
5969
|
-
)
|
|
5970
|
-
`);
|
|
5971
|
-
const metric = this.config.distanceMetric || "cosine";
|
|
5972
|
-
const indexOps = metric === "euclidean" ? "vector_l2_ops" : metric === "dotproduct" ? "vector_ip_ops" : "vector_cosine_ops";
|
|
5973
|
-
await client.query(`
|
|
5974
|
-
CREATE INDEX IF NOT EXISTS ${this.tableName}_embedding_idx
|
|
5975
|
-
ON ${this.tableName}
|
|
5976
|
-
USING hnsw (embedding ${indexOps})
|
|
5977
|
-
`);
|
|
5978
|
-
} finally {
|
|
5979
|
-
client.release();
|
|
5980
|
-
}
|
|
5981
|
-
}
|
|
5982
|
-
async upsert(doc, namespace = "") {
|
|
5983
|
-
var _a;
|
|
5984
|
-
const vectorLiteral = `[${doc.vector.join(",")}]`;
|
|
5985
|
-
await this.pool.query(
|
|
5986
|
-
`INSERT INTO ${this.tableName} (id, namespace, content, metadata, embedding)
|
|
5987
|
-
VALUES ($1, $2, $3, $4, $5::vector)
|
|
5988
|
-
ON CONFLICT (id) DO UPDATE
|
|
5989
|
-
SET namespace = EXCLUDED.namespace,
|
|
5990
|
-
content = EXCLUDED.content,
|
|
5991
|
-
metadata = EXCLUDED.metadata,
|
|
5992
|
-
embedding = EXCLUDED.embedding`,
|
|
5993
|
-
[doc.id, namespace, doc.content, JSON.stringify((_a = doc.metadata) != null ? _a : {}), vectorLiteral]
|
|
5994
|
-
);
|
|
5995
|
-
}
|
|
5996
|
-
async batchUpsert(docs, namespace = "") {
|
|
5997
|
-
if (docs.length === 0) return;
|
|
5998
|
-
const client = await this.pool.connect();
|
|
5999
|
-
try {
|
|
6000
|
-
await client.query("BEGIN");
|
|
6001
|
-
const BATCH_SIZE = 50;
|
|
6002
|
-
for (let i = 0; i < docs.length; i += BATCH_SIZE) {
|
|
6003
|
-
const batch = docs.slice(i, i + BATCH_SIZE);
|
|
6004
|
-
const values = [];
|
|
6005
|
-
const valuePlaceholders = batch.map((doc, idx) => {
|
|
6006
|
-
var _a;
|
|
6007
|
-
const offset = idx * 5;
|
|
6008
|
-
values.push(doc.id, namespace, doc.content, JSON.stringify((_a = doc.metadata) != null ? _a : {}), `[${doc.vector.join(",")}]`);
|
|
6009
|
-
return `($${offset + 1}, $${offset + 2}, $${offset + 3}, $${offset + 4}, $${offset + 5}::vector)`;
|
|
6010
|
-
}).join(", ");
|
|
6011
|
-
const query = `
|
|
6012
|
-
INSERT INTO ${this.tableName} (id, namespace, content, metadata, embedding)
|
|
6013
|
-
VALUES ${valuePlaceholders}
|
|
6014
|
-
ON CONFLICT (id) DO UPDATE
|
|
6015
|
-
SET namespace = EXCLUDED.namespace,
|
|
6016
|
-
content = EXCLUDED.content,
|
|
6017
|
-
metadata = EXCLUDED.metadata,
|
|
6018
|
-
embedding = EXCLUDED.embedding
|
|
6019
|
-
`;
|
|
6020
|
-
await client.query(query, values);
|
|
6021
|
-
}
|
|
6022
|
-
await client.query("COMMIT");
|
|
6023
|
-
} catch (error) {
|
|
6024
|
-
await client.query("ROLLBACK");
|
|
6025
|
-
throw error;
|
|
6026
|
-
} finally {
|
|
6027
|
-
client.release();
|
|
6028
|
-
}
|
|
6029
|
-
}
|
|
6030
|
-
async query(vector, topK, namespace, filter) {
|
|
6031
|
-
const vectorLiteral = `[${vector.join(",")}]`;
|
|
6032
|
-
let whereClause = namespace ? `WHERE namespace = $3` : "";
|
|
6033
|
-
const params = [vectorLiteral, topK];
|
|
6034
|
-
if (namespace) params.push(namespace);
|
|
6035
|
-
const publicFilter = this.sanitizeFilter(filter);
|
|
6036
|
-
if (Object.keys(publicFilter).length > 0) {
|
|
6037
|
-
const filterConditions = Object.entries(publicFilter).map(([key, val]) => {
|
|
6038
|
-
const paramIdx = params.length + 1;
|
|
6039
|
-
params.push(JSON.stringify(val));
|
|
6040
|
-
return `metadata->>'${key}' = $${paramIdx}`;
|
|
6041
|
-
}).join(" AND ");
|
|
6042
|
-
whereClause = whereClause ? `${whereClause} AND ${filterConditions}` : `WHERE ${filterConditions}`;
|
|
6043
|
-
}
|
|
6044
|
-
const client = await this.pool.connect();
|
|
6045
|
-
try {
|
|
6046
|
-
const efSearch = this.config.options.efSearch || Math.max(topK * 10, 40);
|
|
6047
|
-
await client.query(`SET LOCAL hnsw.ef_search = ${efSearch}`);
|
|
6048
|
-
const metric = this.config.distanceMetric || "cosine";
|
|
6049
|
-
const queryOp = metric === "euclidean" ? "<->" : metric === "dotproduct" ? "<#>" : "<=>";
|
|
6050
|
-
const scoreExpr = metric === "euclidean" ? `1 / (1 + (embedding <-> $1::vector))` : metric === "dotproduct" ? `(embedding <#> $1::vector) * -1` : `1 - (embedding <=> $1::vector)`;
|
|
6051
|
-
const result = await client.query(
|
|
6052
|
-
`SELECT id, content, metadata, ${scoreExpr} AS score
|
|
6053
|
-
FROM ${this.tableName}
|
|
6054
|
-
${whereClause}
|
|
6055
|
-
ORDER BY embedding ${queryOp} $1::vector
|
|
6056
|
-
LIMIT $2`,
|
|
6057
|
-
params
|
|
6058
|
-
);
|
|
6059
|
-
return result.rows.map((row) => ({
|
|
6060
|
-
id: String(row["id"]),
|
|
6061
|
-
score: parseFloat(String(row["score"])),
|
|
6062
|
-
content: String(row["content"]),
|
|
6063
|
-
metadata: row["metadata"]
|
|
6064
|
-
}));
|
|
6065
|
-
} finally {
|
|
6066
|
-
client.release();
|
|
6067
|
-
}
|
|
6068
|
-
}
|
|
6069
|
-
async delete(id, namespace) {
|
|
6070
|
-
const where = namespace ? "WHERE id = $1 AND namespace = $2" : "WHERE id = $1";
|
|
6071
|
-
const params = namespace ? [id, namespace] : [id];
|
|
6072
|
-
await this.pool.query(`DELETE FROM ${this.tableName} ${where}`, params);
|
|
6073
|
-
}
|
|
6074
|
-
async deleteNamespace(namespace) {
|
|
6075
|
-
await this.pool.query(`DELETE FROM ${this.tableName} WHERE namespace = $1`, [namespace]);
|
|
6076
|
-
}
|
|
6077
|
-
async ping() {
|
|
6078
|
-
try {
|
|
6079
|
-
await this.pool.query("SELECT 1");
|
|
6080
|
-
return true;
|
|
6081
|
-
} catch (e) {
|
|
6082
|
-
return false;
|
|
6083
|
-
}
|
|
6084
|
-
}
|
|
6085
|
-
async disconnect() {
|
|
6086
|
-
await this.pool.end();
|
|
6087
|
-
}
|
|
6088
|
-
};
|
|
6089
|
-
|
|
6090
|
-
// src/server.ts
|
|
7093
|
+
init_PostgreSQLProvider();
|
|
6091
7094
|
init_MultiTablePostgresProvider();
|
|
6092
7095
|
init_MongoDBProvider();
|
|
6093
7096
|
init_MilvusProvider();
|
|
@@ -6175,7 +7178,7 @@ function createStreamHandler(configOrPlugin) {
|
|
|
6175
7178
|
const encoder = new TextEncoder();
|
|
6176
7179
|
const stream = new ReadableStream({
|
|
6177
7180
|
async start(controller) {
|
|
6178
|
-
var _a
|
|
7181
|
+
var _a;
|
|
6179
7182
|
const enqueue = (text) => controller.enqueue(encoder.encode(text));
|
|
6180
7183
|
try {
|
|
6181
7184
|
const pipelineStream = plugin.chatStream(message, history, namespace);
|
|
@@ -6193,8 +7196,7 @@ function createStreamHandler(configOrPlugin) {
|
|
|
6193
7196
|
}
|
|
6194
7197
|
if (sources.length > 0) {
|
|
6195
7198
|
try {
|
|
6196
|
-
const
|
|
6197
|
-
const uiTransformation = (_b = responseChunk == null ? void 0 : responseChunk.ui_transformation) != null ? _b : llmProvider ? await UITransformer.analyzeAndDecide(message, sources, llmProvider) : UITransformer.transform(message, sources, plugin.getConfig());
|
|
7199
|
+
const uiTransformation = (_a = responseChunk == null ? void 0 : responseChunk.ui_transformation) != null ? _a : UITransformer.transform(message, sources, plugin.getConfig());
|
|
6198
7200
|
if (uiTransformation) {
|
|
6199
7201
|
enqueue(sseUIFrame(uiTransformation));
|
|
6200
7202
|
}
|