@retrivora-ai/rag-engine 2.5.8 → 2.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{ILLMProvider-B2N3lMBc.d.mts → ILLMProvider-DUCL1r75.d.mts} +10 -0
- package/dist/{ILLMProvider-B2N3lMBc.d.ts → ILLMProvider-DUCL1r75.d.ts} +10 -0
- package/dist/handlers/index.d.mts +2 -2
- package/dist/handlers/index.d.ts +2 -2
- package/dist/handlers/index.js +969 -104
- package/dist/handlers/index.mjs +969 -104
- package/dist/{index-CAGGPyI7.d.mts → index-BVSHblli.d.mts} +1 -1
- package/dist/{index-BxzDv2jd.d.ts → index-CsBcqng1.d.ts} +1 -1
- package/dist/{index-37V810a-.d.mts → index-D6Oq2zyc.d.mts} +1 -1
- package/dist/{index-DdQxoIpu.d.ts → index-DGW0NNmR.d.ts} +1 -1
- package/dist/index.d.mts +4 -4
- package/dist/index.d.ts +4 -4
- package/dist/index.js +4 -4
- package/dist/index.mjs +4 -4
- package/dist/server.d.mts +271 -9
- package/dist/server.d.ts +271 -9
- package/dist/server.js +983 -104
- package/dist/server.mjs +976 -104
- package/package.json +1 -1
package/dist/server.mjs
CHANGED
|
@@ -326,7 +326,7 @@ var init_package = __esm({
|
|
|
326
326
|
"package.json"() {
|
|
327
327
|
package_default = {
|
|
328
328
|
name: "@retrivora-ai/rag-engine",
|
|
329
|
-
version: "2.
|
|
329
|
+
version: "2.6.0",
|
|
330
330
|
description: "Retrivora AI is a plug-and-play AI engine for RAG chat experiences \u2014 generic vector DB + LLM provider, embeddable or standalone.",
|
|
331
331
|
author: "Abhinav Alkuchi",
|
|
332
332
|
license: "UNLICENSED",
|
|
@@ -2795,7 +2795,7 @@ var _LicenseVerifier = class _LicenseVerifier {
|
|
|
2795
2795
|
*/
|
|
2796
2796
|
static async generateMachineFingerprint() {
|
|
2797
2797
|
if (typeof window !== "undefined" && typeof navigator !== "undefined") {
|
|
2798
|
-
return
|
|
2798
|
+
return "browser-client";
|
|
2799
2799
|
}
|
|
2800
2800
|
try {
|
|
2801
2801
|
const os2 = await import("os");
|
|
@@ -2867,7 +2867,7 @@ var _LicenseVerifier = class _LicenseVerifier {
|
|
|
2867
2867
|
try {
|
|
2868
2868
|
response = await fetch(endpoint, {
|
|
2869
2869
|
method: "POST",
|
|
2870
|
-
signal: AbortSignal.timeout(
|
|
2870
|
+
signal: AbortSignal.timeout(6e3),
|
|
2871
2871
|
headers: {
|
|
2872
2872
|
"Content-Type": "application/json",
|
|
2873
2873
|
"x-license-key": licenseKey
|
|
@@ -5645,27 +5645,384 @@ Text to extract from:
|
|
|
5645
5645
|
}
|
|
5646
5646
|
};
|
|
5647
5647
|
|
|
5648
|
+
// src/rag/HybridSearch.ts
|
|
5649
|
+
var HybridSearch = class {
|
|
5650
|
+
/**
|
|
5651
|
+
* Tokenize text into normalized alphanumeric keywords, keeping exact codes, numbers, and identifiers.
|
|
5652
|
+
*/
|
|
5653
|
+
static tokenize(text) {
|
|
5654
|
+
if (!text) return [];
|
|
5655
|
+
return text.toLowerCase().replace(/[^\w\s\-_#./:]/g, " ").split(/\s+/).map((t) => t.trim()).filter((t) => t.length > 1 && !this.stopWords.has(t));
|
|
5656
|
+
}
|
|
5657
|
+
/**
|
|
5658
|
+
* Calculate Okapi BM25 sparse scores for an array of document chunks given a query.
|
|
5659
|
+
*/
|
|
5660
|
+
static scoreBM25(documents, query, k1 = 1.5, b = 0.75) {
|
|
5661
|
+
const scores = /* @__PURE__ */ new Map();
|
|
5662
|
+
const queryTokens = this.tokenize(query);
|
|
5663
|
+
if (queryTokens.length === 0 || documents.length === 0) {
|
|
5664
|
+
documents.forEach((d) => scores.set(d.id, 0));
|
|
5665
|
+
return scores;
|
|
5666
|
+
}
|
|
5667
|
+
const docTokens = documents.map((d) => ({
|
|
5668
|
+
id: d.id,
|
|
5669
|
+
tokens: this.tokenize(d.content)
|
|
5670
|
+
}));
|
|
5671
|
+
const N = documents.length;
|
|
5672
|
+
const avgdl = docTokens.reduce((acc, d) => acc + d.tokens.length, 0) / (N || 1);
|
|
5673
|
+
const dfMap = /* @__PURE__ */ new Map();
|
|
5674
|
+
for (const qt of queryTokens) {
|
|
5675
|
+
let count = 0;
|
|
5676
|
+
for (const dt of docTokens) {
|
|
5677
|
+
if (dt.tokens.includes(qt)) count++;
|
|
5678
|
+
}
|
|
5679
|
+
dfMap.set(qt, count);
|
|
5680
|
+
}
|
|
5681
|
+
for (const dt of docTokens) {
|
|
5682
|
+
let score = 0;
|
|
5683
|
+
const docLen = dt.tokens.length;
|
|
5684
|
+
const tfMap = /* @__PURE__ */ new Map();
|
|
5685
|
+
for (const t of dt.tokens) {
|
|
5686
|
+
tfMap.set(t, (tfMap.get(t) || 0) + 1);
|
|
5687
|
+
}
|
|
5688
|
+
for (const qt of queryTokens) {
|
|
5689
|
+
const tf = tfMap.get(qt) || 0;
|
|
5690
|
+
if (tf === 0) continue;
|
|
5691
|
+
const df = dfMap.get(qt) || 0;
|
|
5692
|
+
const idf = Math.log((N - df + 0.5) / (df + 0.5) + 1);
|
|
5693
|
+
const numerator = tf * (k1 + 1);
|
|
5694
|
+
const denominator = tf + k1 * (1 - b + b * (docLen / (avgdl || 1)));
|
|
5695
|
+
score += idf * (numerator / (denominator || 1));
|
|
5696
|
+
}
|
|
5697
|
+
scores.set(dt.id, score);
|
|
5698
|
+
}
|
|
5699
|
+
return scores;
|
|
5700
|
+
}
|
|
5701
|
+
/**
|
|
5702
|
+
* Fuse Dense Vector Matches with Sparse BM25 Scores using Reciprocal Rank Fusion (RRF).
|
|
5703
|
+
*/
|
|
5704
|
+
static fuse(denseMatches, query, options = {}) {
|
|
5705
|
+
var _a2, _b, _c;
|
|
5706
|
+
if (denseMatches.length === 0) return [];
|
|
5707
|
+
const denseWeight = (_a2 = options.denseWeight) != null ? _a2 : 0.6;
|
|
5708
|
+
const sparseWeight = (_b = options.sparseWeight) != null ? _b : 0.4;
|
|
5709
|
+
const k = (_c = options.rrfK) != null ? _c : 60;
|
|
5710
|
+
const sortedDense = [...denseMatches].sort((a, b) => b.score - a.score);
|
|
5711
|
+
const denseRankMap = /* @__PURE__ */ new Map();
|
|
5712
|
+
sortedDense.forEach((match, idx) => {
|
|
5713
|
+
denseRankMap.set(match.id, idx + 1);
|
|
5714
|
+
});
|
|
5715
|
+
const docItems = denseMatches.map((m) => ({ id: m.id, content: m.content }));
|
|
5716
|
+
const bm25ScoreMap = this.scoreBM25(docItems, query, options.bm25K1, options.bm25B);
|
|
5717
|
+
const sortedSparse = [...denseMatches].sort((a, b) => {
|
|
5718
|
+
const scoreA = bm25ScoreMap.get(a.id) || 0;
|
|
5719
|
+
const scoreB = bm25ScoreMap.get(b.id) || 0;
|
|
5720
|
+
return scoreB - scoreA;
|
|
5721
|
+
});
|
|
5722
|
+
const sparseRankMap = /* @__PURE__ */ new Map();
|
|
5723
|
+
sortedSparse.forEach((match, idx) => {
|
|
5724
|
+
sparseRankMap.set(match.id, idx + 1);
|
|
5725
|
+
});
|
|
5726
|
+
const fusedMatches = denseMatches.map((match) => {
|
|
5727
|
+
var _a3;
|
|
5728
|
+
const denseRank = denseRankMap.get(match.id) || denseMatches.length;
|
|
5729
|
+
const sparseRank = sparseRankMap.get(match.id) || denseMatches.length;
|
|
5730
|
+
const denseRrf = denseWeight / (k + denseRank);
|
|
5731
|
+
const sparseRrf = sparseWeight / (k + sparseRank);
|
|
5732
|
+
const combinedRrf = denseRrf + sparseRrf;
|
|
5733
|
+
const bm25Raw = bm25ScoreMap.get(match.id) || 0;
|
|
5734
|
+
return __spreadProps(__spreadValues({}, match), {
|
|
5735
|
+
rrfScore: combinedRrf,
|
|
5736
|
+
bm25Score: bm25Raw,
|
|
5737
|
+
rerankScore: ((_a3 = match.rerankScore) != null ? _a3 : match.score) * 0.5 + combinedRrf * 50
|
|
5738
|
+
});
|
|
5739
|
+
});
|
|
5740
|
+
return fusedMatches.sort((a, b) => {
|
|
5741
|
+
var _a3, _b2;
|
|
5742
|
+
return ((_a3 = b.rerankScore) != null ? _a3 : b.score) - ((_b2 = a.rerankScore) != null ? _b2 : a.score);
|
|
5743
|
+
});
|
|
5744
|
+
}
|
|
5745
|
+
};
|
|
5746
|
+
HybridSearch.stopWords = /* @__PURE__ */ new Set([
|
|
5747
|
+
"a",
|
|
5748
|
+
"about",
|
|
5749
|
+
"above",
|
|
5750
|
+
"after",
|
|
5751
|
+
"again",
|
|
5752
|
+
"against",
|
|
5753
|
+
"all",
|
|
5754
|
+
"am",
|
|
5755
|
+
"an",
|
|
5756
|
+
"and",
|
|
5757
|
+
"any",
|
|
5758
|
+
"are",
|
|
5759
|
+
"aren't",
|
|
5760
|
+
"as",
|
|
5761
|
+
"at",
|
|
5762
|
+
"be",
|
|
5763
|
+
"because",
|
|
5764
|
+
"been",
|
|
5765
|
+
"before",
|
|
5766
|
+
"being",
|
|
5767
|
+
"below",
|
|
5768
|
+
"between",
|
|
5769
|
+
"both",
|
|
5770
|
+
"but",
|
|
5771
|
+
"by",
|
|
5772
|
+
"can't",
|
|
5773
|
+
"cannot",
|
|
5774
|
+
"could",
|
|
5775
|
+
"couldn't",
|
|
5776
|
+
"did",
|
|
5777
|
+
"didn't",
|
|
5778
|
+
"do",
|
|
5779
|
+
"does",
|
|
5780
|
+
"doesn't",
|
|
5781
|
+
"doing",
|
|
5782
|
+
"don't",
|
|
5783
|
+
"down",
|
|
5784
|
+
"during",
|
|
5785
|
+
"each",
|
|
5786
|
+
"few",
|
|
5787
|
+
"for",
|
|
5788
|
+
"from",
|
|
5789
|
+
"further",
|
|
5790
|
+
"had",
|
|
5791
|
+
"hadn't",
|
|
5792
|
+
"has",
|
|
5793
|
+
"hasn't",
|
|
5794
|
+
"have",
|
|
5795
|
+
"haven't",
|
|
5796
|
+
"having",
|
|
5797
|
+
"he",
|
|
5798
|
+
"he'd",
|
|
5799
|
+
"he'll",
|
|
5800
|
+
"he's",
|
|
5801
|
+
"her",
|
|
5802
|
+
"here",
|
|
5803
|
+
"here's",
|
|
5804
|
+
"hers",
|
|
5805
|
+
"herself",
|
|
5806
|
+
"him",
|
|
5807
|
+
"himself",
|
|
5808
|
+
"his",
|
|
5809
|
+
"how",
|
|
5810
|
+
"how's",
|
|
5811
|
+
"i",
|
|
5812
|
+
"i'd",
|
|
5813
|
+
"i'll",
|
|
5814
|
+
"i'm",
|
|
5815
|
+
"i've",
|
|
5816
|
+
"if",
|
|
5817
|
+
"in",
|
|
5818
|
+
"into",
|
|
5819
|
+
"is",
|
|
5820
|
+
"isn't",
|
|
5821
|
+
"it",
|
|
5822
|
+
"it's",
|
|
5823
|
+
"its",
|
|
5824
|
+
"itself",
|
|
5825
|
+
"let's",
|
|
5826
|
+
"me",
|
|
5827
|
+
"more",
|
|
5828
|
+
"most",
|
|
5829
|
+
"mustn't",
|
|
5830
|
+
"my",
|
|
5831
|
+
"myself",
|
|
5832
|
+
"no",
|
|
5833
|
+
"nor",
|
|
5834
|
+
"not",
|
|
5835
|
+
"of",
|
|
5836
|
+
"off",
|
|
5837
|
+
"on",
|
|
5838
|
+
"once",
|
|
5839
|
+
"only",
|
|
5840
|
+
"or",
|
|
5841
|
+
"other",
|
|
5842
|
+
"ought",
|
|
5843
|
+
"our",
|
|
5844
|
+
"ours",
|
|
5845
|
+
"ourselves",
|
|
5846
|
+
"out",
|
|
5847
|
+
"over",
|
|
5848
|
+
"own",
|
|
5849
|
+
"same",
|
|
5850
|
+
"shan't",
|
|
5851
|
+
"she",
|
|
5852
|
+
"she'd",
|
|
5853
|
+
"she'll",
|
|
5854
|
+
"she's",
|
|
5855
|
+
"should",
|
|
5856
|
+
"shouldn't",
|
|
5857
|
+
"so",
|
|
5858
|
+
"some",
|
|
5859
|
+
"such",
|
|
5860
|
+
"than",
|
|
5861
|
+
"that",
|
|
5862
|
+
"that's",
|
|
5863
|
+
"the",
|
|
5864
|
+
"their",
|
|
5865
|
+
"theirs",
|
|
5866
|
+
"them",
|
|
5867
|
+
"themselves",
|
|
5868
|
+
"then",
|
|
5869
|
+
"there",
|
|
5870
|
+
"there's",
|
|
5871
|
+
"these",
|
|
5872
|
+
"they",
|
|
5873
|
+
"they'd",
|
|
5874
|
+
"they'll",
|
|
5875
|
+
"they're",
|
|
5876
|
+
"they've",
|
|
5877
|
+
"this",
|
|
5878
|
+
"those",
|
|
5879
|
+
"through",
|
|
5880
|
+
"to",
|
|
5881
|
+
"too",
|
|
5882
|
+
"under",
|
|
5883
|
+
"until",
|
|
5884
|
+
"up",
|
|
5885
|
+
"very",
|
|
5886
|
+
"was",
|
|
5887
|
+
"wasn't",
|
|
5888
|
+
"we",
|
|
5889
|
+
"we'd",
|
|
5890
|
+
"we'll",
|
|
5891
|
+
"we're",
|
|
5892
|
+
"we've",
|
|
5893
|
+
"were",
|
|
5894
|
+
"weren't",
|
|
5895
|
+
"what",
|
|
5896
|
+
"what's",
|
|
5897
|
+
"when",
|
|
5898
|
+
"when's",
|
|
5899
|
+
"where",
|
|
5900
|
+
"where's",
|
|
5901
|
+
"which",
|
|
5902
|
+
"while",
|
|
5903
|
+
"who",
|
|
5904
|
+
"who's",
|
|
5905
|
+
"whom",
|
|
5906
|
+
"why",
|
|
5907
|
+
"why's",
|
|
5908
|
+
"with",
|
|
5909
|
+
"won't",
|
|
5910
|
+
"would",
|
|
5911
|
+
"wouldn't",
|
|
5912
|
+
"you",
|
|
5913
|
+
"you'd",
|
|
5914
|
+
"you'll",
|
|
5915
|
+
"you're",
|
|
5916
|
+
"you've",
|
|
5917
|
+
"your",
|
|
5918
|
+
"yours",
|
|
5919
|
+
"yourself",
|
|
5920
|
+
"yourselves"
|
|
5921
|
+
]);
|
|
5922
|
+
|
|
5648
5923
|
// src/rag/Reranker.ts
|
|
5649
5924
|
var MAX_RERANK_CONTENT_CHARS = 64e3;
|
|
5650
5925
|
var Reranker = class {
|
|
5651
5926
|
/**
|
|
5652
|
-
* Re-ranks matches based on a secondary relevance score using
|
|
5653
|
-
*
|
|
5654
|
-
* Fix RAG-2: keyword re-ranking no longer mutates the original `score`
|
|
5655
|
-
* field (which consumers interpret as the raw vector similarity). A new
|
|
5656
|
-
* `rerankScore` field is added so both the original similarity and the
|
|
5657
|
-
* boosted rank signal are available to callers.
|
|
5927
|
+
* Re-ranks matches based on a secondary relevance score using Hybrid BM25, Cohere, Jina or LLM.
|
|
5658
5928
|
*/
|
|
5659
|
-
async rerank(matches, query, limit = 5, llm) {
|
|
5660
|
-
|
|
5661
|
-
|
|
5929
|
+
async rerank(matches, query, limit = 5, llm, options = {}) {
|
|
5930
|
+
var _a2, _b;
|
|
5931
|
+
if (!matches || matches.length <= 1) {
|
|
5932
|
+
return matches ? matches.slice(0, limit) : [];
|
|
5662
5933
|
}
|
|
5663
|
-
|
|
5664
|
-
|
|
5665
|
-
|
|
5666
|
-
|
|
5667
|
-
|
|
5934
|
+
const cohereKey = options.cohereApiKey || (typeof process !== "undefined" ? (_a2 = process.env) == null ? void 0 : _a2.COHERE_API_KEY : void 0);
|
|
5935
|
+
const jinaKey = options.jinaApiKey || (typeof process !== "undefined" ? (_b = process.env) == null ? void 0 : _b.JINA_API_KEY : void 0);
|
|
5936
|
+
if ((options.provider === "cohere" || cohereKey) && cohereKey) {
|
|
5937
|
+
try {
|
|
5938
|
+
const cohereResult = await this.cohereRerank(matches, query, limit, cohereKey, options.cohereModel);
|
|
5939
|
+
if (cohereResult.length > 0) return cohereResult;
|
|
5940
|
+
} catch (err) {
|
|
5941
|
+
console.warn("[Reranker] Cohere reranking failed, falling back to HybridSearch:", err);
|
|
5942
|
+
}
|
|
5668
5943
|
}
|
|
5944
|
+
if ((options.provider === "jina" || jinaKey) && jinaKey) {
|
|
5945
|
+
try {
|
|
5946
|
+
const jinaResult = await this.jinaRerank(matches, query, limit, jinaKey, options.jinaModel);
|
|
5947
|
+
if (jinaResult.length > 0) return jinaResult;
|
|
5948
|
+
} catch (err) {
|
|
5949
|
+
console.warn("[Reranker] Jina reranking failed, falling back to HybridSearch:", err);
|
|
5950
|
+
}
|
|
5951
|
+
}
|
|
5952
|
+
if (!options.provider || options.provider === "hybrid") {
|
|
5953
|
+
const fused = HybridSearch.fuse(matches, query);
|
|
5954
|
+
if (fused.length > 0) {
|
|
5955
|
+
return fused.slice(0, limit);
|
|
5956
|
+
}
|
|
5957
|
+
}
|
|
5958
|
+
if (llm && (options.provider === "llm" || !options.provider)) {
|
|
5959
|
+
try {
|
|
5960
|
+
return await this.llmRerank(matches, query, limit, llm);
|
|
5961
|
+
} catch (error) {
|
|
5962
|
+
console.warn("[Reranker] LLM re-ranking failed, falling back to keyword sorting:", error);
|
|
5963
|
+
}
|
|
5964
|
+
}
|
|
5965
|
+
return this.keywordRerank(matches, query, limit);
|
|
5966
|
+
}
|
|
5967
|
+
/**
|
|
5968
|
+
* Cohere Rerank v3.5 API integration.
|
|
5969
|
+
*/
|
|
5970
|
+
async cohereRerank(matches, query, limit, apiKey, model = "rerank-v3.5") {
|
|
5971
|
+
const documents = matches.map((m) => m.content || "");
|
|
5972
|
+
const response = await fetch("https://api.cohere.com/v1/rerank", {
|
|
5973
|
+
method: "POST",
|
|
5974
|
+
headers: {
|
|
5975
|
+
"Authorization": `Bearer ${apiKey}`,
|
|
5976
|
+
"Content-Type": "application/json"
|
|
5977
|
+
},
|
|
5978
|
+
body: JSON.stringify({
|
|
5979
|
+
model,
|
|
5980
|
+
query,
|
|
5981
|
+
documents,
|
|
5982
|
+
top_n: limit
|
|
5983
|
+
})
|
|
5984
|
+
});
|
|
5985
|
+
if (!response.ok) {
|
|
5986
|
+
throw new Error(`Cohere API returned status ${response.status}`);
|
|
5987
|
+
}
|
|
5988
|
+
const data = await response.json();
|
|
5989
|
+
if (!data.results || !Array.isArray(data.results)) return [];
|
|
5990
|
+
return data.results.map((res) => {
|
|
5991
|
+
const match = matches[res.index];
|
|
5992
|
+
return __spreadProps(__spreadValues({}, match), {
|
|
5993
|
+
rerankScore: res.relevance_score
|
|
5994
|
+
});
|
|
5995
|
+
});
|
|
5996
|
+
}
|
|
5997
|
+
/**
|
|
5998
|
+
* Jina AI Reranker v2 API integration.
|
|
5999
|
+
*/
|
|
6000
|
+
async jinaRerank(matches, query, limit, apiKey, model = "jina-reranker-v2-base-multilingual") {
|
|
6001
|
+
const documents = matches.map((m) => m.content || "");
|
|
6002
|
+
const response = await fetch("https://api.jina.ai/v1/rerank", {
|
|
6003
|
+
method: "POST",
|
|
6004
|
+
headers: {
|
|
6005
|
+
"Authorization": `Bearer ${apiKey}`,
|
|
6006
|
+
"Content-Type": "application/json"
|
|
6007
|
+
},
|
|
6008
|
+
body: JSON.stringify({
|
|
6009
|
+
model,
|
|
6010
|
+
query,
|
|
6011
|
+
documents,
|
|
6012
|
+
top_n: limit
|
|
6013
|
+
})
|
|
6014
|
+
});
|
|
6015
|
+
if (!response.ok) {
|
|
6016
|
+
throw new Error(`Jina API returned status ${response.status}`);
|
|
6017
|
+
}
|
|
6018
|
+
const data = await response.json();
|
|
6019
|
+
if (!data.results || !Array.isArray(data.results)) return [];
|
|
6020
|
+
return data.results.map((res) => {
|
|
6021
|
+
const match = matches[res.index];
|
|
6022
|
+
return __spreadProps(__spreadValues({}, match), {
|
|
6023
|
+
rerankScore: res.relevance_score
|
|
6024
|
+
});
|
|
6025
|
+
});
|
|
5669
6026
|
}
|
|
5670
6027
|
/**
|
|
5671
6028
|
* Keyword-based re-ranking fallback.
|
|
@@ -5751,6 +6108,176 @@ ${docList}` }],
|
|
|
5751
6108
|
}
|
|
5752
6109
|
};
|
|
5753
6110
|
|
|
6111
|
+
// src/rag/RAGEvaluator.ts
|
|
6112
|
+
var RAGEvaluator = class {
|
|
6113
|
+
/**
|
|
6114
|
+
* Evaluate a RAG interaction across the full RAG Triad.
|
|
6115
|
+
*/
|
|
6116
|
+
static async evaluate(query, context, answer, llm) {
|
|
6117
|
+
if (!llm || !context || !answer) {
|
|
6118
|
+
return this.deterministicHeuristic(query, context, answer);
|
|
6119
|
+
}
|
|
6120
|
+
try {
|
|
6121
|
+
const prompt = `You are an automated RAG Triad Evaluator. Evaluate the quality of the following RAG system interaction:
|
|
6122
|
+
|
|
6123
|
+
USER QUERY:
|
|
6124
|
+
${query}
|
|
6125
|
+
|
|
6126
|
+
RETRIEVED CONTEXT:
|
|
6127
|
+
${context.slice(0, 3e3)}
|
|
6128
|
+
|
|
6129
|
+
GENERATED ANSWER:
|
|
6130
|
+
${answer}
|
|
6131
|
+
|
|
6132
|
+
Evaluate these 3 metrics on a scale of 0.0 to 1.0:
|
|
6133
|
+
1. "contextRelevance": Are the retrieved documents relevant to the query? (1.0 = highly relevant, 0.0 = irrelevant)
|
|
6134
|
+
2. "faithfulness": Is every claim in the generated answer supported by the retrieved context? (1.0 = 100% grounded/no hallucination, 0.0 = completely fabricated)
|
|
6135
|
+
3. "answerRelevance": Does the answer directly address the user's question? (1.0 = direct answer, 0.0 = evasive/unrelated)
|
|
6136
|
+
|
|
6137
|
+
Return ONLY valid JSON with no markdown fences:
|
|
6138
|
+
{"contextRelevance": 0.95, "faithfulness": 0.90, "answerRelevance": 0.92, "reason": "Answer is fully supported by the document."}`;
|
|
6139
|
+
const raw = await llm.chat(
|
|
6140
|
+
[{ role: "user", content: prompt }],
|
|
6141
|
+
"",
|
|
6142
|
+
{
|
|
6143
|
+
systemPrompt: "You are a strict AI safety and hallucination evaluator. Output only valid JSON.",
|
|
6144
|
+
temperature: 0,
|
|
6145
|
+
maxTokens: 150
|
|
6146
|
+
}
|
|
6147
|
+
);
|
|
6148
|
+
const cleaned = raw.replace(/```json|```/g, "").trim();
|
|
6149
|
+
const parsed = JSON.parse(cleaned);
|
|
6150
|
+
const cr = typeof parsed.contextRelevance === "number" ? Math.max(0, Math.min(1, parsed.contextRelevance)) : 0.85;
|
|
6151
|
+
const f = typeof parsed.faithfulness === "number" ? Math.max(0, Math.min(1, parsed.faithfulness)) : 0.9;
|
|
6152
|
+
const ar = typeof parsed.answerRelevance === "number" ? Math.max(0, Math.min(1, parsed.answerRelevance)) : 0.88;
|
|
6153
|
+
const overall = Number((cr * 0.3 + f * 0.4 + ar * 0.3).toFixed(2));
|
|
6154
|
+
return {
|
|
6155
|
+
contextRelevance: cr,
|
|
6156
|
+
faithfulness: f,
|
|
6157
|
+
answerRelevance: ar,
|
|
6158
|
+
overallQuality: overall,
|
|
6159
|
+
isHallucinated: f < 0.6,
|
|
6160
|
+
reason: parsed.reason || "Evaluated successfully."
|
|
6161
|
+
};
|
|
6162
|
+
} catch (err) {
|
|
6163
|
+
return this.deterministicHeuristic(query, context, answer);
|
|
6164
|
+
}
|
|
6165
|
+
}
|
|
6166
|
+
/**
|
|
6167
|
+
* Deterministic fast keyword-overlap heuristic when LLM evaluation is not invoked.
|
|
6168
|
+
*/
|
|
6169
|
+
static deterministicHeuristic(query, context, answer) {
|
|
6170
|
+
if (!context || !answer) {
|
|
6171
|
+
return { contextRelevance: 0.5, faithfulness: 0.8, answerRelevance: 0.7, overallQuality: 0.67, isHallucinated: false };
|
|
6172
|
+
}
|
|
6173
|
+
const queryWords = query.toLowerCase().split(/\s+/).filter((w) => w.length > 2);
|
|
6174
|
+
const contextLower = context.toLowerCase();
|
|
6175
|
+
const answerLower = answer.toLowerCase();
|
|
6176
|
+
let queryContextHits = 0;
|
|
6177
|
+
queryWords.forEach((w) => {
|
|
6178
|
+
if (contextLower.includes(w)) queryContextHits++;
|
|
6179
|
+
});
|
|
6180
|
+
const contextRelevance = queryWords.length > 0 ? Math.min(1, queryContextHits / queryWords.length + 0.3) : 0.8;
|
|
6181
|
+
const numbersInAnswer = answer.match(/\b\d+(?:\.\d+)?%?\b/g) || [];
|
|
6182
|
+
let numberHits = 0;
|
|
6183
|
+
numbersInAnswer.forEach((num) => {
|
|
6184
|
+
if (context.includes(num)) numberHits++;
|
|
6185
|
+
});
|
|
6186
|
+
const faithfulness = numbersInAnswer.length > 0 ? Math.min(1, numberHits / numbersInAnswer.length + 0.2) : 0.9;
|
|
6187
|
+
const answerRelevance = 0.85;
|
|
6188
|
+
const overallQuality = Number((contextRelevance * 0.3 + faithfulness * 0.4 + answerRelevance * 0.3).toFixed(2));
|
|
6189
|
+
return {
|
|
6190
|
+
contextRelevance: Number(contextRelevance.toFixed(2)),
|
|
6191
|
+
faithfulness: Number(faithfulness.toFixed(2)),
|
|
6192
|
+
answerRelevance,
|
|
6193
|
+
overallQuality,
|
|
6194
|
+
isHallucinated: faithfulness < 0.5,
|
|
6195
|
+
reason: "Computed via deterministic grounding heuristic."
|
|
6196
|
+
};
|
|
6197
|
+
}
|
|
6198
|
+
};
|
|
6199
|
+
|
|
6200
|
+
// src/security/PIIMasker.ts
|
|
6201
|
+
var PIIMasker = class {
|
|
6202
|
+
/**
|
|
6203
|
+
* Redact sensitive information from text.
|
|
6204
|
+
*/
|
|
6205
|
+
static mask(text, options = {}) {
|
|
6206
|
+
if (!text || typeof text !== "string") return text;
|
|
6207
|
+
let sanitized = text;
|
|
6208
|
+
if (options.maskApiKeys !== false) {
|
|
6209
|
+
sanitized = sanitized.replace(this.API_KEY_REGEX, "[REDACTED_API_KEY]");
|
|
6210
|
+
}
|
|
6211
|
+
if (options.maskCreditCards !== false) {
|
|
6212
|
+
sanitized = sanitized.replace(this.CREDIT_CARD_REGEX, (match) => {
|
|
6213
|
+
const digits = match.replace(/[- ]/g, "");
|
|
6214
|
+
if (digits.length >= 13 && digits.length <= 19) {
|
|
6215
|
+
return "[REDACTED_CREDIT_CARD]";
|
|
6216
|
+
}
|
|
6217
|
+
return match;
|
|
6218
|
+
});
|
|
6219
|
+
}
|
|
6220
|
+
if (options.maskSSN !== false) {
|
|
6221
|
+
sanitized = sanitized.replace(this.SSN_REGEX, "[REDACTED_SSN]");
|
|
6222
|
+
}
|
|
6223
|
+
if (options.maskEmails !== false) {
|
|
6224
|
+
sanitized = sanitized.replace(this.EMAIL_REGEX, "[REDACTED_EMAIL]");
|
|
6225
|
+
}
|
|
6226
|
+
if (options.maskPhones !== false) {
|
|
6227
|
+
sanitized = sanitized.replace(this.PHONE_REGEX, "[REDACTED_PHONE]");
|
|
6228
|
+
}
|
|
6229
|
+
if (options.maskIPs === true) {
|
|
6230
|
+
sanitized = sanitized.replace(this.IP_REGEX, "[REDACTED_IP]");
|
|
6231
|
+
}
|
|
6232
|
+
return sanitized;
|
|
6233
|
+
}
|
|
6234
|
+
};
|
|
6235
|
+
PIIMasker.SSN_REGEX = /\b(?!000|666|9\d{2})\d{3}[- ]?(?!00)\d{2}[- ]?(?!0000)\d{4}\b/g;
|
|
6236
|
+
PIIMasker.CREDIT_CARD_REGEX = /\b(?:\d{4}[- ]?){3}\d{4}\b|\b\d{15,16}\b/g;
|
|
6237
|
+
PIIMasker.EMAIL_REGEX = /\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Z|a-z]{2,}\b/g;
|
|
6238
|
+
PIIMasker.PHONE_REGEX = /\b(?:\+?\d{1,3}[-.\s]?)?\(?\d{3}\)?[-.\s]?\d{3}[-.\s]?\d{4}\b/g;
|
|
6239
|
+
PIIMasker.API_KEY_REGEX = /\b(?:sk-[a-zA-Z0-9]{20,}|AKIA[0-9A-Z]{16}|rtv_[a-zA-Z0-9_-]{20,}|ghp_[a-zA-Z0-9]{36}|bearer\s+[a-zA-Z0-9._-]{20,})\b/gi;
|
|
6240
|
+
PIIMasker.IP_REGEX = /\b(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\b/g;
|
|
6241
|
+
|
|
6242
|
+
// src/security/PromptGuard.ts
|
|
6243
|
+
var PromptGuard = class {
|
|
6244
|
+
/**
|
|
6245
|
+
* Scan and evaluate user query or document text for prompt injection vectors.
|
|
6246
|
+
*/
|
|
6247
|
+
static evaluate(text) {
|
|
6248
|
+
if (!text || typeof text !== "string") {
|
|
6249
|
+
return { isSafe: true, threatLevel: "none", detectedPatterns: [], sanitizedText: text };
|
|
6250
|
+
}
|
|
6251
|
+
const detected = [];
|
|
6252
|
+
let maxSeverity = "none";
|
|
6253
|
+
for (const item of this.INJECTION_PATTERNS) {
|
|
6254
|
+
if (item.pattern.test(text)) {
|
|
6255
|
+
detected.push(item.name);
|
|
6256
|
+
if (item.severity === "high") {
|
|
6257
|
+
maxSeverity = "high";
|
|
6258
|
+
} else if (item.severity === "medium" && maxSeverity !== "high") {
|
|
6259
|
+
maxSeverity = "medium";
|
|
6260
|
+
}
|
|
6261
|
+
}
|
|
6262
|
+
}
|
|
6263
|
+
const sanitized = text.replace(/<\/?(?:system|instruction|admin|prompt_override)>/gi, "");
|
|
6264
|
+
return {
|
|
6265
|
+
isSafe: maxSeverity !== "high",
|
|
6266
|
+
threatLevel: maxSeverity,
|
|
6267
|
+
detectedPatterns: detected,
|
|
6268
|
+
sanitizedText: sanitized
|
|
6269
|
+
};
|
|
6270
|
+
}
|
|
6271
|
+
};
|
|
6272
|
+
PromptGuard.INJECTION_PATTERNS = [
|
|
6273
|
+
{ pattern: /ignore\s+(?:all\s+)?(?:previous|prior|above)\s+(?:instructions|prompts|rules)/i, name: "override_instructions", severity: "high" },
|
|
6274
|
+
{ pattern: /(?:show|repeat|print|reveal|expose|output)\s+(?:the\s+)?(?:system\s+prompt|initial\s+instructions|system\s+message)/i, name: "system_prompt_leak", severity: "high" },
|
|
6275
|
+
{ pattern: /you\s+are\s+now\s+(?:in\s+developer\s+mode|unrestricted|DAN|jailbroken|godmode)/i, name: "jailbreak_persona", severity: "high" },
|
|
6276
|
+
{ pattern: /<\/?(?:system|instruction|admin|prompt_override)>/i, name: "tag_injection", severity: "medium" },
|
|
6277
|
+
{ pattern: /base64\s*decode\s*and\s*execute/i, name: "obfuscated_payload", severity: "high" },
|
|
6278
|
+
{ pattern: /do\s+anything\s+now/i, name: "dan_variant", severity: "high" }
|
|
6279
|
+
];
|
|
6280
|
+
|
|
5754
6281
|
// src/rag/LlamaIndexIngestor.ts
|
|
5755
6282
|
var LlamaIndexIngestor = class {
|
|
5756
6283
|
/**
|
|
@@ -9706,7 +10233,7 @@ ${m.content}`).join("\n\n---\n\n");
|
|
|
9706
10233
|
*/
|
|
9707
10234
|
async processEmbeddings(chunks) {
|
|
9708
10235
|
if (chunks.length === 0) return [];
|
|
9709
|
-
const texts = chunks.map((c) => c.content);
|
|
10236
|
+
const texts = chunks.map((c) => PIIMasker.mask(c.content));
|
|
9710
10237
|
if (typeof this.embeddingProvider.batchEmbed === "function") {
|
|
9711
10238
|
try {
|
|
9712
10239
|
const batchSize = 50;
|
|
@@ -9912,10 +10439,12 @@ ${m.content}`).join("\n\n---\n\n");
|
|
|
9912
10439
|
const requestId = `req_${Date.now()}_${Math.random().toString(36).slice(2, 8)}`;
|
|
9913
10440
|
const requestStart = performance.now();
|
|
9914
10441
|
try {
|
|
9915
|
-
|
|
10442
|
+
const guardCheck = PromptGuard.evaluate(question);
|
|
10443
|
+
let effectiveQuestion = guardCheck.sanitizedText;
|
|
10444
|
+
effectiveQuestion = PIIMasker.mask(effectiveQuestion);
|
|
9916
10445
|
let promptOptResult;
|
|
9917
10446
|
if (((_e = this.config.rag) == null ? void 0 : _e.optimizePrompt) !== false) {
|
|
9918
|
-
promptOptResult = PromptOptimizer.optimize(
|
|
10447
|
+
promptOptResult = PromptOptimizer.optimize(effectiveQuestion, history);
|
|
9919
10448
|
effectiveQuestion = promptOptResult.optimizedPrompt;
|
|
9920
10449
|
}
|
|
9921
10450
|
let searchQuery = effectiveQuestion;
|
|
@@ -10200,6 +10729,7 @@ ${context}`;
|
|
|
10200
10729
|
uiTransformationPromise,
|
|
10201
10730
|
awaitHallucination ? hallucinationScoringPromise : Promise.resolve(void 0)
|
|
10202
10731
|
]));
|
|
10732
|
+
const triadScore = yield new __await(RAGEvaluator.evaluate(effectiveQuestion, context, fullReply));
|
|
10203
10733
|
const buildTrace = (hScore) => __spreadValues({
|
|
10204
10734
|
requestId,
|
|
10205
10735
|
query: question,
|
|
@@ -10218,7 +10748,13 @@ ${context}`;
|
|
|
10218
10748
|
}),
|
|
10219
10749
|
latency,
|
|
10220
10750
|
tokens,
|
|
10221
|
-
timestamp: (/* @__PURE__ */ new Date()).toISOString()
|
|
10751
|
+
timestamp: (/* @__PURE__ */ new Date()).toISOString(),
|
|
10752
|
+
ragTriad: {
|
|
10753
|
+
contextRelevance: triadScore.contextRelevance,
|
|
10754
|
+
faithfulness: triadScore.faithfulness,
|
|
10755
|
+
answerRelevance: triadScore.answerRelevance,
|
|
10756
|
+
overallQuality: triadScore.overallQuality
|
|
10757
|
+
}
|
|
10222
10758
|
}, hScore ? {
|
|
10223
10759
|
hallucinationScore: hScore.score,
|
|
10224
10760
|
hallucinationReason: hScore.reason
|
|
@@ -11089,6 +11625,374 @@ function createFromPreset(presetName) {
|
|
|
11089
11625
|
return new ConfigBuilder().vectorDb(preset.vectorDb, { apiKey: process.env[`${preset.vectorDb.toUpperCase()}_API_KEY`] }).llm(preset.llm, void 0, process.env[`${preset.llm.toUpperCase()}_API_KEY`]).embedding(preset.embedding, void 0, process.env[`${preset.embedding.toUpperCase()}_API_KEY`]);
|
|
11090
11626
|
}
|
|
11091
11627
|
|
|
11628
|
+
// src/utils/UniversalTableParser.ts
|
|
11629
|
+
var UniversalTableParser = class {
|
|
11630
|
+
/**
|
|
11631
|
+
* Parse a raw 2D grid of strings with complete matrix intelligence.
|
|
11632
|
+
*/
|
|
11633
|
+
static parseGrid(grid, sheetName = "Data") {
|
|
11634
|
+
if (!grid || grid.length === 0) return "";
|
|
11635
|
+
const cleanGrid = grid.map((row) => Array.isArray(row) ? row.map((c) => c !== null && c !== void 0 ? String(c).trim() : "") : []).filter((row) => row.some((cell) => cell.length > 0));
|
|
11636
|
+
if (cleanGrid.length === 0) return "";
|
|
11637
|
+
const { headerRows, dataRows } = this.splitHeaderAndDataRows(cleanGrid);
|
|
11638
|
+
const fusedHeaders = this.fuseHeaders(headerRows);
|
|
11639
|
+
const matrixInfo = this.analyzeMatrixStructure(fusedHeaders, dataRows);
|
|
11640
|
+
const sections = [`--- Sheet: ${sheetName} ---`];
|
|
11641
|
+
const markdownTable = this.generateMarkdownTable(fusedHeaders, dataRows);
|
|
11642
|
+
if (markdownTable) {
|
|
11643
|
+
sections.push(`
|
|
11644
|
+
## Table: ${sheetName}
|
|
11645
|
+
${markdownTable}`);
|
|
11646
|
+
}
|
|
11647
|
+
if (matrixInfo.isComparisonMatrix && matrixInfo.entityIndices.length > 0) {
|
|
11648
|
+
const entityProfiles = this.generateEntityProfiles(fusedHeaders, dataRows, matrixInfo);
|
|
11649
|
+
if (entityProfiles) {
|
|
11650
|
+
sections.push(`
|
|
11651
|
+
## Entity Comparison Profiles
|
|
11652
|
+
${entityProfiles}`);
|
|
11653
|
+
}
|
|
11654
|
+
const criteriaComparisons = this.generateCriteriaComparisons(fusedHeaders, dataRows, matrixInfo);
|
|
11655
|
+
if (criteriaComparisons) {
|
|
11656
|
+
sections.push(`
|
|
11657
|
+
## Criteria Breakdown & Comparisons
|
|
11658
|
+
${criteriaComparisons}`);
|
|
11659
|
+
}
|
|
11660
|
+
} else {
|
|
11661
|
+
const contextualRows = this.generateContextualRows(fusedHeaders, dataRows);
|
|
11662
|
+
if (contextualRows) {
|
|
11663
|
+
sections.push(`
|
|
11664
|
+
## Structured Records
|
|
11665
|
+
${contextualRows}`);
|
|
11666
|
+
}
|
|
11667
|
+
}
|
|
11668
|
+
return sections.join("\n\n");
|
|
11669
|
+
}
|
|
11670
|
+
/**
|
|
11671
|
+
* Parse a raw CSV / TSV text string into tri-perspective contextual format.
|
|
11672
|
+
*/
|
|
11673
|
+
static parseDelimitedText(text, title = "Table", delimiter) {
|
|
11674
|
+
if (!text || !text.trim()) return "";
|
|
11675
|
+
if (!delimiter) {
|
|
11676
|
+
const firstLine = text.split(/\r?\n/)[0] || "";
|
|
11677
|
+
const commaCount = (firstLine.match(/,/g) || []).length;
|
|
11678
|
+
const tabCount = (firstLine.match(/\t/g) || []).length;
|
|
11679
|
+
const semiCount = (firstLine.match(/;/g) || []).length;
|
|
11680
|
+
const pipeCount = (firstLine.match(/\|/g) || []).length;
|
|
11681
|
+
if (tabCount > commaCount && tabCount > semiCount) delimiter = " ";
|
|
11682
|
+
else if (semiCount > commaCount && semiCount > tabCount) delimiter = ";";
|
|
11683
|
+
else if (pipeCount > commaCount && pipeCount > semiCount) delimiter = "|";
|
|
11684
|
+
else delimiter = ",";
|
|
11685
|
+
}
|
|
11686
|
+
const lines = text.split(/\r?\n/);
|
|
11687
|
+
const grid = [];
|
|
11688
|
+
for (const line of lines) {
|
|
11689
|
+
if (!line.trim()) continue;
|
|
11690
|
+
const row = [];
|
|
11691
|
+
let inQuote = false;
|
|
11692
|
+
let current = "";
|
|
11693
|
+
for (let i = 0; i < line.length; i++) {
|
|
11694
|
+
const char = line[i];
|
|
11695
|
+
if (char === '"') {
|
|
11696
|
+
if (inQuote && line[i + 1] === '"') {
|
|
11697
|
+
current += '"';
|
|
11698
|
+
i++;
|
|
11699
|
+
} else {
|
|
11700
|
+
inQuote = !inQuote;
|
|
11701
|
+
}
|
|
11702
|
+
} else if (char === delimiter && !inQuote) {
|
|
11703
|
+
row.push(current.trim());
|
|
11704
|
+
current = "";
|
|
11705
|
+
} else {
|
|
11706
|
+
current += char;
|
|
11707
|
+
}
|
|
11708
|
+
}
|
|
11709
|
+
row.push(current.trim());
|
|
11710
|
+
grid.push(row);
|
|
11711
|
+
}
|
|
11712
|
+
return this.parseGrid(grid, title);
|
|
11713
|
+
}
|
|
11714
|
+
/**
|
|
11715
|
+
* Parse a SheetJS sheet object with complete merged cell and matrix intelligence.
|
|
11716
|
+
*/
|
|
11717
|
+
static parseSheet(sheet, sheetName, XLSX) {
|
|
11718
|
+
if (!sheet) return "";
|
|
11719
|
+
const grid = this.extractAndUnrollGrid(sheet, XLSX);
|
|
11720
|
+
if (!grid || grid.length === 0) {
|
|
11721
|
+
const csv = XLSX.utils.sheet_to_csv(sheet, { blankrows: false });
|
|
11722
|
+
return csv && csv.trim() ? `--- Sheet: ${sheetName} ---
|
|
11723
|
+
${csv.trim()}` : "";
|
|
11724
|
+
}
|
|
11725
|
+
return this.parseGrid(grid, sheetName);
|
|
11726
|
+
}
|
|
11727
|
+
/**
|
|
11728
|
+
* Unrolls SheetJS merged cell ranges (`!merges`) across the 2D grid
|
|
11729
|
+
* so no child cell is left blank.
|
|
11730
|
+
*/
|
|
11731
|
+
static extractAndUnrollGrid(sheet, XLSX) {
|
|
11732
|
+
var _a2;
|
|
11733
|
+
const rawGrid = XLSX.utils.sheet_to_json(sheet, { header: 1, blankrows: false, defval: "" });
|
|
11734
|
+
if (!rawGrid || rawGrid.length === 0) return [];
|
|
11735
|
+
const grid = rawGrid.map(
|
|
11736
|
+
(row) => (Array.isArray(row) ? row : []).map((cell) => cell !== null && cell !== void 0 ? String(cell).trim() : "")
|
|
11737
|
+
);
|
|
11738
|
+
const merges = sheet["!merges"] || [];
|
|
11739
|
+
for (const merge of merges) {
|
|
11740
|
+
const { s, e } = merge;
|
|
11741
|
+
const originVal = ((_a2 = grid[s.r]) == null ? void 0 : _a2[s.c]) || "";
|
|
11742
|
+
if (!originVal) continue;
|
|
11743
|
+
for (let r = s.r; r <= e.r; r++) {
|
|
11744
|
+
if (!grid[r]) grid[r] = [];
|
|
11745
|
+
for (let c = s.c; c <= e.c; c++) {
|
|
11746
|
+
if (!grid[r][c] || grid[r][c] === "") {
|
|
11747
|
+
grid[r][c] = originVal;
|
|
11748
|
+
}
|
|
11749
|
+
}
|
|
11750
|
+
}
|
|
11751
|
+
}
|
|
11752
|
+
return grid.filter((row) => row.some((cell) => cell.length > 0));
|
|
11753
|
+
}
|
|
11754
|
+
/**
|
|
11755
|
+
* Intelligently identify where header rows end and data rows begin.
|
|
11756
|
+
*/
|
|
11757
|
+
static splitHeaderAndDataRows(grid) {
|
|
11758
|
+
if (grid.length <= 1) {
|
|
11759
|
+
return { headerRows: grid, dataRows: [] };
|
|
11760
|
+
}
|
|
11761
|
+
let headerRowCount = 1;
|
|
11762
|
+
for (let r = 1; r < Math.min(4, grid.length); r++) {
|
|
11763
|
+
const row = grid[r];
|
|
11764
|
+
const nonEmptyCells = row.filter(Boolean);
|
|
11765
|
+
if (nonEmptyCells.length === 0) continue;
|
|
11766
|
+
const isDataRow = nonEmptyCells.some(
|
|
11767
|
+
(c) => /^-?\d+(?:\.\d+)?%?$/.test(c) || /^\$?[\d,.]+(?:\s*(?:mil|k|m|b|usd|eur))?$/i.test(c) || /^(yes|no|true|false|pass|fail|compliant|non-compliant)$/i.test(c)
|
|
11768
|
+
);
|
|
11769
|
+
const hasSubheaderLabels = nonEmptyCells.some(
|
|
11770
|
+
(c) => /\b(bidder|vendor|supplier|score|weight|criteria|item|specification|technical|commercial|rating|status|cost|price|qty|unit)\b/i.test(c)
|
|
11771
|
+
);
|
|
11772
|
+
if (!isDataRow && hasSubheaderLabels) {
|
|
11773
|
+
headerRowCount = r + 1;
|
|
11774
|
+
} else if (!isDataRow && r === 1 && nonEmptyCells.length >= 2) {
|
|
11775
|
+
headerRowCount = 2;
|
|
11776
|
+
} else {
|
|
11777
|
+
break;
|
|
11778
|
+
}
|
|
11779
|
+
}
|
|
11780
|
+
const headerRows = grid.slice(0, headerRowCount);
|
|
11781
|
+
const dataRows = grid.slice(headerRowCount);
|
|
11782
|
+
return { headerRows, dataRows };
|
|
11783
|
+
}
|
|
11784
|
+
/**
|
|
11785
|
+
* Fuses multi-tier header rows into composite paths (e.g. `Commercial > Payment Terms > Bidder A`).
|
|
11786
|
+
*/
|
|
11787
|
+
static fuseHeaders(headerRows) {
|
|
11788
|
+
var _a2, _b;
|
|
11789
|
+
if (headerRows.length === 0) return [];
|
|
11790
|
+
if (headerRows.length === 1) {
|
|
11791
|
+
return headerRows[0].map((h, i) => h || `Column_${i + 1}`);
|
|
11792
|
+
}
|
|
11793
|
+
const maxCols = Math.max(...headerRows.map((r) => r.length));
|
|
11794
|
+
const fused = [];
|
|
11795
|
+
for (let c = 0; c < maxCols; c++) {
|
|
11796
|
+
const components = [];
|
|
11797
|
+
for (let r = 0; r < headerRows.length; r++) {
|
|
11798
|
+
const val = (_b = (_a2 = headerRows[r]) == null ? void 0 : _a2[c]) == null ? void 0 : _b.trim();
|
|
11799
|
+
if (val && !components.includes(val)) {
|
|
11800
|
+
components.push(val);
|
|
11801
|
+
}
|
|
11802
|
+
}
|
|
11803
|
+
fused.push(components.join(" > ") || `Column_${c + 1}`);
|
|
11804
|
+
}
|
|
11805
|
+
return fused;
|
|
11806
|
+
}
|
|
11807
|
+
/**
|
|
11808
|
+
* Analyzes whether the table is a comparison matrix (columns = entities, rows = criteria).
|
|
11809
|
+
*/
|
|
11810
|
+
static analyzeMatrixStructure(headers, dataRows) {
|
|
11811
|
+
const entityPattern = /\b(bidder|vendor|supplier|candidate|option|company|contractor|partner|proposal|firm|provider|solution|tier|package)\b/i;
|
|
11812
|
+
const entityIndices = [];
|
|
11813
|
+
const entityNames = [];
|
|
11814
|
+
let criteriaColumnIndex = 0;
|
|
11815
|
+
headers.forEach((h, idx) => {
|
|
11816
|
+
if (entityPattern.test(h) || /\b(bidder\s*\d+|vendor\s*[a-z0-9]+|option\s*[a-z0-9]+)\b/i.test(h)) {
|
|
11817
|
+
entityIndices.push(idx);
|
|
11818
|
+
const parts = h.split(" > ");
|
|
11819
|
+
entityNames.push(parts[parts.length - 1]);
|
|
11820
|
+
} else if (/\b(criteria|criterion|description|specification|category|requirement|item|feature)\b/i.test(h)) {
|
|
11821
|
+
criteriaColumnIndex = idx;
|
|
11822
|
+
}
|
|
11823
|
+
});
|
|
11824
|
+
const isComparisonMatrix = entityIndices.length >= 2 || headers.length >= 3 && entityIndices.length >= 1;
|
|
11825
|
+
return {
|
|
11826
|
+
isComparisonMatrix,
|
|
11827
|
+
entityIndices: entityIndices.length > 0 ? entityIndices : headers.slice(1).map((_, i) => i + 1),
|
|
11828
|
+
entityNames: entityNames.length > 0 ? entityNames : headers.slice(1),
|
|
11829
|
+
criteriaColumnIndex
|
|
11830
|
+
};
|
|
11831
|
+
}
|
|
11832
|
+
/**
|
|
11833
|
+
* Perspective A: Clean Markdown Table
|
|
11834
|
+
*/
|
|
11835
|
+
static generateMarkdownTable(headers, dataRows) {
|
|
11836
|
+
if (headers.length === 0 || dataRows.length === 0) return "";
|
|
11837
|
+
const cleanHeaders = headers.map((h) => h.replace(/\|/g, "-"));
|
|
11838
|
+
const lines = [];
|
|
11839
|
+
lines.push(`| ${cleanHeaders.join(" | ")} |`);
|
|
11840
|
+
lines.push(`| ${cleanHeaders.map(() => "---").join(" | ")} |`);
|
|
11841
|
+
for (const row of dataRows) {
|
|
11842
|
+
const paddedRow = headers.map((_, i) => (row[i] || "").replace(/\|/g, "-"));
|
|
11843
|
+
if (paddedRow.some((cell) => cell.trim().length > 0)) {
|
|
11844
|
+
lines.push(`| ${paddedRow.join(" | ")} |`);
|
|
11845
|
+
}
|
|
11846
|
+
}
|
|
11847
|
+
return lines.join("\n");
|
|
11848
|
+
}
|
|
11849
|
+
/**
|
|
11850
|
+
* Perspective B: Entity-Centric Profiles (e.g. Bidder 1, Bidder 2 full summary)
|
|
11851
|
+
*/
|
|
11852
|
+
static generateEntityProfiles(headers, dataRows, matrixInfo) {
|
|
11853
|
+
const profiles = [];
|
|
11854
|
+
matrixInfo.entityIndices.forEach((colIdx, i) => {
|
|
11855
|
+
const entityName = matrixInfo.entityNames[i] || headers[colIdx] || `Entity ${i + 1}`;
|
|
11856
|
+
const lines = [`### Profile & Evaluation: ${entityName}`];
|
|
11857
|
+
for (const row of dataRows) {
|
|
11858
|
+
const criterion = row[matrixInfo.criteriaColumnIndex] || row[0] || "Specification";
|
|
11859
|
+
const val = row[colIdx];
|
|
11860
|
+
if (val !== void 0 && val !== null && String(val).trim() !== "") {
|
|
11861
|
+
lines.push(`- **${criterion}**: ${String(val).trim()}`);
|
|
11862
|
+
}
|
|
11863
|
+
}
|
|
11864
|
+
if (lines.length > 1) {
|
|
11865
|
+
profiles.push(lines.join("\n"));
|
|
11866
|
+
}
|
|
11867
|
+
});
|
|
11868
|
+
return profiles.join("\n\n");
|
|
11869
|
+
}
|
|
11870
|
+
/**
|
|
11871
|
+
* Perspective C: Criteria Cross-Comparisons (e.g. Commercial Bid comparison across all Bidders)
|
|
11872
|
+
*/
|
|
11873
|
+
static generateCriteriaComparisons(headers, dataRows, matrixInfo) {
|
|
11874
|
+
const comparisons = [];
|
|
11875
|
+
for (const row of dataRows) {
|
|
11876
|
+
const criterion = row[matrixInfo.criteriaColumnIndex] || row[0] || "Metric";
|
|
11877
|
+
const entityPairs = [];
|
|
11878
|
+
matrixInfo.entityIndices.forEach((colIdx, i) => {
|
|
11879
|
+
const entityName = matrixInfo.entityNames[i] || headers[colIdx] || `Entity ${i + 1}`;
|
|
11880
|
+
const val = row[colIdx];
|
|
11881
|
+
if (val !== void 0 && val !== null && String(val).trim() !== "") {
|
|
11882
|
+
entityPairs.push(`${entityName}: ${String(val).trim()}`);
|
|
11883
|
+
}
|
|
11884
|
+
});
|
|
11885
|
+
if (entityPairs.length > 0) {
|
|
11886
|
+
comparisons.push(`- **${criterion}** $\\rightarrow$ ${entityPairs.join(" | ")}`);
|
|
11887
|
+
}
|
|
11888
|
+
}
|
|
11889
|
+
return comparisons.join("\n");
|
|
11890
|
+
}
|
|
11891
|
+
/**
|
|
11892
|
+
* Standard contextual key-value rows for general tabular records.
|
|
11893
|
+
*/
|
|
11894
|
+
static generateContextualRows(headers, dataRows) {
|
|
11895
|
+
const lines = [];
|
|
11896
|
+
for (const row of dataRows) {
|
|
11897
|
+
const pairs = [];
|
|
11898
|
+
headers.forEach((h, i) => {
|
|
11899
|
+
const val = row[i];
|
|
11900
|
+
if (val !== void 0 && val !== null && String(val).trim() !== "") {
|
|
11901
|
+
pairs.push(`${h}: ${String(val).trim()}`);
|
|
11902
|
+
}
|
|
11903
|
+
});
|
|
11904
|
+
if (pairs.length > 0) {
|
|
11905
|
+
lines.push(pairs.join(" | "));
|
|
11906
|
+
}
|
|
11907
|
+
}
|
|
11908
|
+
return lines.join("\n");
|
|
11909
|
+
}
|
|
11910
|
+
};
|
|
11911
|
+
|
|
11912
|
+
// src/utils/VisionExtractor.ts
|
|
11913
|
+
var VisionExtractor = class {
|
|
11914
|
+
/**
|
|
11915
|
+
* Check if a buffer represents an image format.
|
|
11916
|
+
*/
|
|
11917
|
+
static isImageBuffer(buffer, fileName = "", mimeType = "") {
|
|
11918
|
+
var _a2;
|
|
11919
|
+
const ext = ((_a2 = fileName.split(".").pop()) == null ? void 0 : _a2.toLowerCase()) || "";
|
|
11920
|
+
if (["png", "jpg", "jpeg", "webp", "bmp", "tiff", "gif"].includes(ext)) {
|
|
11921
|
+
return true;
|
|
11922
|
+
}
|
|
11923
|
+
if (mimeType.startsWith("image/")) {
|
|
11924
|
+
return true;
|
|
11925
|
+
}
|
|
11926
|
+
if (buffer && buffer.length >= 4) {
|
|
11927
|
+
if (buffer[0] === 137 && buffer[1] === 80 && buffer[2] === 78 && buffer[3] === 71) return true;
|
|
11928
|
+
if (buffer[0] === 255 && buffer[1] === 216 && buffer[2] === 255) return true;
|
|
11929
|
+
if (buffer[0] === 82 && buffer[1] === 73 && buffer[2] === 70 && buffer[3] === 70) return true;
|
|
11930
|
+
}
|
|
11931
|
+
return false;
|
|
11932
|
+
}
|
|
11933
|
+
/**
|
|
11934
|
+
* Extract rich textual content, tables, and diagram descriptions from an image buffer using Vision LLM.
|
|
11935
|
+
*/
|
|
11936
|
+
static async extractFromImage(buffer, fileName = "document.png", options = {}) {
|
|
11937
|
+
var _a2, _b, _c, _d, _e, _f, _g2, _h, _i;
|
|
11938
|
+
const base64Image = buffer.toString("base64");
|
|
11939
|
+
const ext = ((_a2 = fileName.split(".").pop()) == null ? void 0 : _a2.toLowerCase()) || "png";
|
|
11940
|
+
const mime = ext === "jpg" || ext === "jpeg" ? "image/jpeg" : ext === "webp" ? "image/webp" : "image/png";
|
|
11941
|
+
const apiKey = options.apiKey || (typeof process !== "undefined" ? ((_b = process.env) == null ? void 0 : _b.OPENAI_API_KEY) || ((_c = process.env) == null ? void 0 : _c.GEMINI_API_KEY) || ((_d = process.env) == null ? void 0 : _d.ANTHROPIC_API_KEY) : void 0);
|
|
11942
|
+
if (!apiKey) {
|
|
11943
|
+
return `[Scanned Document / Image: ${fileName} - Size: ${(buffer.length / 1024).toFixed(1)} KB]`;
|
|
11944
|
+
}
|
|
11945
|
+
try {
|
|
11946
|
+
const response = await fetch("https://api.openai.com/v1/chat/completions", {
|
|
11947
|
+
method: "POST",
|
|
11948
|
+
headers: {
|
|
11949
|
+
"Authorization": `Bearer ${apiKey}`,
|
|
11950
|
+
"Content-Type": "application/json"
|
|
11951
|
+
},
|
|
11952
|
+
body: JSON.stringify({
|
|
11953
|
+
model: options.model || "gpt-4o-mini",
|
|
11954
|
+
messages: [
|
|
11955
|
+
{
|
|
11956
|
+
role: "system",
|
|
11957
|
+
content: "You are an expert Document OCR & Data Extraction engine. Transcribe all text, numbers, headings, tables, and diagram structures in this image into clean, structured GitHub Markdown. For tables, use standard markdown table syntax. Preserve exact numbers, dates, and column headers. Do not summarize \u2014 transcribe faithfully."
|
|
11958
|
+
},
|
|
11959
|
+
{
|
|
11960
|
+
role: "user",
|
|
11961
|
+
content: [
|
|
11962
|
+
{
|
|
11963
|
+
type: "text",
|
|
11964
|
+
text: `Please transcribe the content of this document/image (${fileName}):`
|
|
11965
|
+
},
|
|
11966
|
+
{
|
|
11967
|
+
type: "image_url",
|
|
11968
|
+
image_url: {
|
|
11969
|
+
url: `data:${mime};base64,${base64Image}`,
|
|
11970
|
+
detail: "high"
|
|
11971
|
+
}
|
|
11972
|
+
}
|
|
11973
|
+
]
|
|
11974
|
+
}
|
|
11975
|
+
],
|
|
11976
|
+
max_tokens: options.maxTokens || 2048,
|
|
11977
|
+
temperature: (_e = options.temperature) != null ? _e : 0.1
|
|
11978
|
+
})
|
|
11979
|
+
});
|
|
11980
|
+
if (response.ok) {
|
|
11981
|
+
const data = await response.json();
|
|
11982
|
+
const text = (_i = (_h = (_g2 = (_f = data.choices) == null ? void 0 : _f[0]) == null ? void 0 : _g2.message) == null ? void 0 : _h.content) == null ? void 0 : _i.trim();
|
|
11983
|
+
if (text) {
|
|
11984
|
+
return `--- Document / Image OCR: ${fileName} ---
|
|
11985
|
+
|
|
11986
|
+
${text}`;
|
|
11987
|
+
}
|
|
11988
|
+
}
|
|
11989
|
+
} catch (err) {
|
|
11990
|
+
console.warn("[VisionExtractor] OpenAI Vision OCR failed:", err);
|
|
11991
|
+
}
|
|
11992
|
+
return `[Scanned Image Document: ${fileName}]`;
|
|
11993
|
+
}
|
|
11994
|
+
};
|
|
11995
|
+
|
|
11092
11996
|
// src/utils/DocumentParser.ts
|
|
11093
11997
|
var DocumentParser = class {
|
|
11094
11998
|
/**
|
|
@@ -11112,52 +12016,11 @@ var DocumentParser = class {
|
|
|
11112
12016
|
return "text";
|
|
11113
12017
|
}
|
|
11114
12018
|
/**
|
|
11115
|
-
* Convert spreadsheet sheets into high-fidelity, context-preserved tabular text.
|
|
11116
|
-
*
|
|
12019
|
+
* Convert spreadsheet sheets into high-fidelity, context-preserved tabular text via UniversalTableParser.
|
|
12020
|
+
* Unrolls merged cells, fuses multi-level headers, and generates tri-perspective comparison profiles.
|
|
11117
12021
|
*/
|
|
11118
12022
|
static formatSheetToContextualText(sheet, sheetName, XLSX) {
|
|
11119
|
-
|
|
11120
|
-
if (!jsonRows || jsonRows.length === 0) {
|
|
11121
|
-
const csv = XLSX.utils.sheet_to_csv(sheet, { blankrows: false });
|
|
11122
|
-
return csv && csv.trim() ? `--- Sheet: ${sheetName} ---
|
|
11123
|
-
${csv.trim()}` : "";
|
|
11124
|
-
}
|
|
11125
|
-
const lines = [`--- Sheet: ${sheetName} ---`];
|
|
11126
|
-
let currentHeader = [];
|
|
11127
|
-
for (let r = 0; r < jsonRows.length; r++) {
|
|
11128
|
-
const row = jsonRows[r];
|
|
11129
|
-
if (!row || row.length === 0) continue;
|
|
11130
|
-
const cells = row.map((c) => c !== null && c !== void 0 ? String(c).trim() : "").filter(Boolean);
|
|
11131
|
-
if (cells.length === 0) continue;
|
|
11132
|
-
if (cells.length === 1 || cells.length <= 2 && cells[0].toUpperCase().includes("STEP")) {
|
|
11133
|
-
lines.push(`
|
|
11134
|
-
## ${cells.join(" ")}`);
|
|
11135
|
-
continue;
|
|
11136
|
-
}
|
|
11137
|
-
const isHeaderCandidate = cells.every((c) => isNaN(Number(c)) && !/^\$?[\d,.]+/.test(c));
|
|
11138
|
-
if (isHeaderCandidate && cells.length >= 2) {
|
|
11139
|
-
currentHeader = row.map((c) => c !== null && c !== void 0 ? String(c).trim() : "");
|
|
11140
|
-
lines.push(`| ${cells.join(" | ")} |`);
|
|
11141
|
-
lines.push(`| ${cells.map(() => "---").join(" | ")} |`);
|
|
11142
|
-
continue;
|
|
11143
|
-
}
|
|
11144
|
-
if (currentHeader.length > 0 && cells.length > 1) {
|
|
11145
|
-
const fieldPairs = [];
|
|
11146
|
-
for (let c = 0; c < row.length; c++) {
|
|
11147
|
-
const h = currentHeader[c];
|
|
11148
|
-
const val = row[c];
|
|
11149
|
-
if (h && val !== null && val !== void 0 && String(val).trim() !== "") {
|
|
11150
|
-
fieldPairs.push(`${h}: ${String(val).trim()}`);
|
|
11151
|
-
}
|
|
11152
|
-
}
|
|
11153
|
-
if (fieldPairs.length > 0) {
|
|
11154
|
-
lines.push(fieldPairs.join(" | "));
|
|
11155
|
-
continue;
|
|
11156
|
-
}
|
|
11157
|
-
}
|
|
11158
|
-
lines.push(cells.join(", "));
|
|
11159
|
-
}
|
|
11160
|
-
return lines.join("\n");
|
|
12023
|
+
return UniversalTableParser.parseSheet(sheet, sheetName, XLSX);
|
|
11161
12024
|
}
|
|
11162
12025
|
/**
|
|
11163
12026
|
* Extract text from a File or Buffer based on its type.
|
|
@@ -11181,7 +12044,9 @@ ${csv.trim()}` : "";
|
|
|
11181
12044
|
return this.readAsText(file);
|
|
11182
12045
|
}
|
|
11183
12046
|
if (extension === "csv" || extension === "tsv" || mimeType === "text/csv" || mimeType === "text/tab-separated-values") {
|
|
11184
|
-
|
|
12047
|
+
const text = await this.readAsText(file);
|
|
12048
|
+
const parsed = UniversalTableParser.parseDelimitedText(text, fileName);
|
|
12049
|
+
return parsed || text;
|
|
11185
12050
|
}
|
|
11186
12051
|
if (extension === "pptx" || extension === "ppt" || extension === "odp" || mimeType.includes("presentation") || mimeType.includes("powerpoint")) {
|
|
11187
12052
|
try {
|
|
@@ -11288,16 +12153,22 @@ ${csv.trim()}`);
|
|
|
11288
12153
|
return this.parseLegacyExcelOrFallback(buffer, fileName);
|
|
11289
12154
|
}
|
|
11290
12155
|
if (extension === "pdf" || mimeType === "application/pdf") {
|
|
12156
|
+
const buffer = Buffer.isBuffer(file) ? file : Buffer.from(await file.arrayBuffer());
|
|
11291
12157
|
try {
|
|
11292
12158
|
const pdfModule = await import("pdf-parse");
|
|
11293
|
-
const buffer = Buffer.isBuffer(file) ? file : Buffer.from(await file.arrayBuffer());
|
|
11294
12159
|
const pdf = pdfModule.default || pdfModule;
|
|
11295
12160
|
const data = await pdf(buffer);
|
|
11296
|
-
|
|
12161
|
+
if (data.text && data.text.trim().length > 30) {
|
|
12162
|
+
return data.text;
|
|
12163
|
+
}
|
|
11297
12164
|
} catch (err) {
|
|
11298
|
-
console.warn(
|
|
11299
|
-
return `[PDF Parsing Error for ${fileName}]`;
|
|
12165
|
+
console.warn("[DocumentParser] pdf-parse failed, attempting Vision OCR fallback:", err);
|
|
11300
12166
|
}
|
|
12167
|
+
return VisionExtractor.extractFromImage(buffer, fileName);
|
|
12168
|
+
}
|
|
12169
|
+
if (["png", "jpg", "jpeg", "webp", "bmp", "tiff", "gif"].includes(extension) || mimeType.startsWith("image/")) {
|
|
12170
|
+
const buffer = Buffer.isBuffer(file) ? file : Buffer.from(await file.arrayBuffer());
|
|
12171
|
+
return VisionExtractor.extractFromImage(buffer, fileName);
|
|
11301
12172
|
}
|
|
11302
12173
|
if (extension === "docx" || extension === "doc" || extension === "odt" || extension === "rtf" || mimeType === "application/vnd.openxmlformats-officedocument.wordprocessingml.document" || mimeType === "application/msword" || mimeType === "application/rtf") {
|
|
11303
12174
|
try {
|
|
@@ -11655,44 +12526,38 @@ var _DatabaseStorage = class _DatabaseStorage {
|
|
|
11655
12526
|
*
|
|
11656
12527
|
* In non-production environments this is a complete no-op so local
|
|
11657
12528
|
* development works without a license key.
|
|
12529
|
+
/**
|
|
12530
|
+
* Checks if the caller holds a valid paid-tier license in production.
|
|
12531
|
+
* Returns true if allowed, false if not supported on current tier.
|
|
11658
12532
|
*/
|
|
11659
|
-
async
|
|
11660
|
-
if (process.env.NODE_ENV !== "production") return;
|
|
11661
|
-
if (
|
|
11662
|
-
if (
|
|
11663
|
-
|
|
11664
|
-
"[Retrivora SDK] Chat History and Feedback features require a paid license key in production."
|
|
11665
|
-
);
|
|
12533
|
+
async isPaidSubscription() {
|
|
12534
|
+
if (process.env.NODE_ENV !== "production") return true;
|
|
12535
|
+
if (!this.config.licenseKey) return false;
|
|
12536
|
+
if (Date.now() - this.subscriptionCheckedAt < _DatabaseStorage.SUBSCRIPTION_CACHE_TTL_MS) {
|
|
12537
|
+
return true;
|
|
11666
12538
|
}
|
|
11667
12539
|
try {
|
|
11668
12540
|
const payload = await LicenseVerifier.verifyAsync(this.config.licenseKey, this.config.projectId);
|
|
11669
12541
|
const tier = (payload.tier || "").toLowerCase();
|
|
11670
|
-
const PAID_TIERS = /* @__PURE__ */ new Set(["premium", "enterprise", "growth", "pro", "developer_pro"]);
|
|
11671
|
-
|
|
11672
|
-
|
|
11673
|
-
|
|
11674
|
-
);
|
|
12542
|
+
const PAID_TIERS = /* @__PURE__ */ new Set(["premium", "enterprise", "growth", "pro", "developer_pro", "free_trial"]);
|
|
12543
|
+
const isAllowed = PAID_TIERS.has(tier) || tier.includes("pro") || tier.includes("enterprise");
|
|
12544
|
+
if (isAllowed) {
|
|
12545
|
+
this.subscriptionCheckedAt = Date.now();
|
|
11675
12546
|
}
|
|
11676
|
-
|
|
11677
|
-
} catch (
|
|
11678
|
-
|
|
11679
|
-
`[Retrivora SDK] Subscription check failed: ${err instanceof Error ? err.message : String(err)}`
|
|
11680
|
-
);
|
|
12547
|
+
return isAllowed;
|
|
12548
|
+
} catch (e) {
|
|
12549
|
+
return false;
|
|
11681
12550
|
}
|
|
11682
12551
|
}
|
|
11683
|
-
async
|
|
12552
|
+
async isHistoryAvailable() {
|
|
11684
12553
|
var _a2, _b;
|
|
11685
|
-
|
|
11686
|
-
|
|
11687
|
-
throw new Error("[Retrivora SDK] Chat History is disabled in RagConfig.");
|
|
11688
|
-
}
|
|
12554
|
+
if (((_b = (_a2 = this.config.rag) == null ? void 0 : _a2.history) == null ? void 0 : _b.enabled) === false) return false;
|
|
12555
|
+
return this.isPaidSubscription();
|
|
11689
12556
|
}
|
|
11690
|
-
async
|
|
12557
|
+
async isFeedbackAvailable() {
|
|
11691
12558
|
var _a2, _b;
|
|
11692
|
-
|
|
11693
|
-
|
|
11694
|
-
throw new Error("[Retrivora SDK] User Feedback is disabled in RagConfig.");
|
|
11695
|
-
}
|
|
12559
|
+
if (((_b = (_a2 = this.config.rag) == null ? void 0 : _a2.feedback) == null ? void 0 : _b.enabled) === false) return false;
|
|
12560
|
+
return this.isPaidSubscription();
|
|
11696
12561
|
}
|
|
11697
12562
|
/**
|
|
11698
12563
|
* Returns (or lazily creates) the PostgreSQL connection pool for this project.
|
|
@@ -11806,7 +12671,7 @@ var _DatabaseStorage = class _DatabaseStorage {
|
|
|
11806
12671
|
}
|
|
11807
12672
|
// ─── Message History Operations ──────────────────────────────────────────
|
|
11808
12673
|
async saveMessage(sessionId, message) {
|
|
11809
|
-
await this.
|
|
12674
|
+
if (!await this.isHistoryAvailable()) return;
|
|
11810
12675
|
const createdAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
11811
12676
|
const msgId = message.id || `msg_${Date.now()}_${Math.random().toString(36).slice(2, 6)}`;
|
|
11812
12677
|
if (this.provider === "postgresql" || this.provider === "pgvector") {
|
|
@@ -11869,7 +12734,7 @@ var _DatabaseStorage = class _DatabaseStorage {
|
|
|
11869
12734
|
}
|
|
11870
12735
|
}
|
|
11871
12736
|
async getHistory(sessionId, projectIds) {
|
|
11872
|
-
await this.
|
|
12737
|
+
if (!await this.isHistoryAvailable()) return [];
|
|
11873
12738
|
if (projectIds && projectIds.length > 0 && !this.sessionInScope(sessionId, projectIds)) {
|
|
11874
12739
|
return [];
|
|
11875
12740
|
}
|
|
@@ -11903,7 +12768,7 @@ var _DatabaseStorage = class _DatabaseStorage {
|
|
|
11903
12768
|
}
|
|
11904
12769
|
}
|
|
11905
12770
|
async clearHistory(sessionId, projectIds) {
|
|
11906
|
-
await this.
|
|
12771
|
+
if (!await this.isHistoryAvailable()) return;
|
|
11907
12772
|
if (projectIds && projectIds.length > 0 && !this.sessionInScope(sessionId, projectIds)) {
|
|
11908
12773
|
return;
|
|
11909
12774
|
}
|
|
@@ -11943,7 +12808,7 @@ var _DatabaseStorage = class _DatabaseStorage {
|
|
|
11943
12808
|
});
|
|
11944
12809
|
}
|
|
11945
12810
|
async listSessions(projectIds) {
|
|
11946
|
-
await this.
|
|
12811
|
+
if (!await this.isHistoryAvailable()) return [];
|
|
11947
12812
|
let raw = [];
|
|
11948
12813
|
if (this.provider === "postgresql" || this.provider === "pgvector") {
|
|
11949
12814
|
const pool = await this.getPGPool();
|
|
@@ -11962,7 +12827,7 @@ var _DatabaseStorage = class _DatabaseStorage {
|
|
|
11962
12827
|
}
|
|
11963
12828
|
// ─── Feedback Operations ──────────────────────────────────────────────────
|
|
11964
12829
|
async saveFeedback(feedback) {
|
|
11965
|
-
await this.
|
|
12830
|
+
if (!await this.isFeedbackAvailable()) return;
|
|
11966
12831
|
const id = `fb_${Date.now()}_${Math.random().toString(36).slice(2, 6)}`;
|
|
11967
12832
|
const createdAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
11968
12833
|
if (this.provider === "postgresql" || this.provider === "pgvector") {
|
|
@@ -12012,7 +12877,7 @@ var _DatabaseStorage = class _DatabaseStorage {
|
|
|
12012
12877
|
}
|
|
12013
12878
|
}
|
|
12014
12879
|
async getFeedback(messageId, projectIds) {
|
|
12015
|
-
await this.
|
|
12880
|
+
if (!await this.isFeedbackAvailable()) return null;
|
|
12016
12881
|
let item = null;
|
|
12017
12882
|
if (this.provider === "postgresql" || this.provider === "pgvector") {
|
|
12018
12883
|
const pool = await this.getPGPool();
|
|
@@ -12048,7 +12913,7 @@ var _DatabaseStorage = class _DatabaseStorage {
|
|
|
12048
12913
|
return item;
|
|
12049
12914
|
}
|
|
12050
12915
|
async listFeedback(projectIds) {
|
|
12051
|
-
await this.
|
|
12916
|
+
if (!await this.isFeedbackAvailable()) return [];
|
|
12052
12917
|
let raw = [];
|
|
12053
12918
|
if (this.provider === "postgresql" || this.provider === "pgvector") {
|
|
12054
12919
|
const pool = await this.getPGPool();
|
|
@@ -14004,6 +14869,7 @@ export {
|
|
|
14004
14869
|
EmbeddingStrategyResolver,
|
|
14005
14870
|
EventType,
|
|
14006
14871
|
GroqProvider,
|
|
14872
|
+
HybridSearch,
|
|
14007
14873
|
IntentClassifier,
|
|
14008
14874
|
LLMFactory,
|
|
14009
14875
|
LLM_PROFILES,
|
|
@@ -14018,19 +14884,23 @@ export {
|
|
|
14018
14884
|
MultiTablePostgresProvider,
|
|
14019
14885
|
OllamaProvider,
|
|
14020
14886
|
OpenAIProvider,
|
|
14887
|
+
PIIMasker,
|
|
14021
14888
|
PRESETS,
|
|
14022
14889
|
PineconeProvider,
|
|
14023
14890
|
Pipeline,
|
|
14024
14891
|
PostgreSQLProvider,
|
|
14892
|
+
PromptGuard,
|
|
14025
14893
|
PromptOptimizer,
|
|
14026
14894
|
ProviderHealthCheck,
|
|
14027
14895
|
ProviderNotFoundException,
|
|
14028
14896
|
ProviderRegistry,
|
|
14029
14897
|
QdrantProvider,
|
|
14030
14898
|
QwenProvider,
|
|
14899
|
+
RAGEvaluator,
|
|
14031
14900
|
RateLimitException,
|
|
14032
14901
|
RedisProvider,
|
|
14033
14902
|
RendererRegistry,
|
|
14903
|
+
Reranker,
|
|
14034
14904
|
RetrievalException,
|
|
14035
14905
|
Retrivora,
|
|
14036
14906
|
RetrivoraError,
|
|
@@ -14048,9 +14918,11 @@ export {
|
|
|
14048
14918
|
TelemetryService,
|
|
14049
14919
|
TextRendererStrategy,
|
|
14050
14920
|
UniversalLLMAdapter,
|
|
14921
|
+
UniversalTableParser,
|
|
14051
14922
|
UniversalVectorProvider,
|
|
14052
14923
|
VECTOR_PROFILES,
|
|
14053
14924
|
VectorPlugin,
|
|
14925
|
+
VisionExtractor,
|
|
14054
14926
|
VisualizationDecisionEngine,
|
|
14055
14927
|
WeaviateProvider,
|
|
14056
14928
|
createChatHandler,
|