@retrivora-ai/rag-engine 2.5.9 → 2.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{ILLMProvider-B2N3lMBc.d.mts → ILLMProvider-DUCL1r75.d.mts} +10 -0
- package/dist/{ILLMProvider-B2N3lMBc.d.ts → ILLMProvider-DUCL1r75.d.ts} +10 -0
- package/dist/handlers/index.d.mts +2 -2
- package/dist/handlers/index.d.ts +2 -2
- package/dist/handlers/index.js +967 -102
- package/dist/handlers/index.mjs +967 -102
- package/dist/{index-CAGGPyI7.d.mts → index-BVSHblli.d.mts} +1 -1
- package/dist/{index-BxzDv2jd.d.ts → index-CsBcqng1.d.ts} +1 -1
- package/dist/{index-37V810a-.d.mts → index-D6Oq2zyc.d.mts} +1 -1
- package/dist/{index-DdQxoIpu.d.ts → index-DGW0NNmR.d.ts} +1 -1
- package/dist/index.d.mts +4 -4
- package/dist/index.d.ts +4 -4
- package/dist/index.js +1 -1
- package/dist/index.mjs +1 -1
- package/dist/server.d.mts +271 -9
- package/dist/server.d.ts +271 -9
- package/dist/server.js +981 -102
- package/dist/server.mjs +974 -102
- package/package.json +1 -1
package/dist/server.js
CHANGED
|
@@ -332,7 +332,7 @@ var init_package = __esm({
|
|
|
332
332
|
"package.json"() {
|
|
333
333
|
package_default = {
|
|
334
334
|
name: "@retrivora-ai/rag-engine",
|
|
335
|
-
version: "2.
|
|
335
|
+
version: "2.6.0",
|
|
336
336
|
description: "Retrivora AI is a plug-and-play AI engine for RAG chat experiences \u2014 generic vector DB + LLM provider, embeddable or standalone.",
|
|
337
337
|
author: "Abhinav Alkuchi",
|
|
338
338
|
license: "UNLICENSED",
|
|
@@ -2729,6 +2729,7 @@ __export(server_exports, {
|
|
|
2729
2729
|
EmbeddingStrategyResolver: () => EmbeddingStrategyResolver,
|
|
2730
2730
|
EventType: () => EventType,
|
|
2731
2731
|
GroqProvider: () => GroqProvider,
|
|
2732
|
+
HybridSearch: () => HybridSearch,
|
|
2732
2733
|
IntentClassifier: () => IntentClassifier,
|
|
2733
2734
|
LLMFactory: () => LLMFactory,
|
|
2734
2735
|
LLM_PROFILES: () => LLM_PROFILES,
|
|
@@ -2743,19 +2744,23 @@ __export(server_exports, {
|
|
|
2743
2744
|
MultiTablePostgresProvider: () => MultiTablePostgresProvider,
|
|
2744
2745
|
OllamaProvider: () => OllamaProvider,
|
|
2745
2746
|
OpenAIProvider: () => OpenAIProvider,
|
|
2747
|
+
PIIMasker: () => PIIMasker,
|
|
2746
2748
|
PRESETS: () => PRESETS,
|
|
2747
2749
|
PineconeProvider: () => PineconeProvider,
|
|
2748
2750
|
Pipeline: () => Pipeline,
|
|
2749
2751
|
PostgreSQLProvider: () => PostgreSQLProvider,
|
|
2752
|
+
PromptGuard: () => PromptGuard,
|
|
2750
2753
|
PromptOptimizer: () => PromptOptimizer,
|
|
2751
2754
|
ProviderHealthCheck: () => ProviderHealthCheck,
|
|
2752
2755
|
ProviderNotFoundException: () => ProviderNotFoundException,
|
|
2753
2756
|
ProviderRegistry: () => ProviderRegistry,
|
|
2754
2757
|
QdrantProvider: () => QdrantProvider,
|
|
2755
2758
|
QwenProvider: () => QwenProvider,
|
|
2759
|
+
RAGEvaluator: () => RAGEvaluator,
|
|
2756
2760
|
RateLimitException: () => RateLimitException,
|
|
2757
2761
|
RedisProvider: () => RedisProvider,
|
|
2758
2762
|
RendererRegistry: () => RendererRegistry,
|
|
2763
|
+
Reranker: () => Reranker,
|
|
2759
2764
|
RetrievalException: () => RetrievalException,
|
|
2760
2765
|
Retrivora: () => Retrivora,
|
|
2761
2766
|
RetrivoraError: () => RetrivoraError,
|
|
@@ -2773,9 +2778,11 @@ __export(server_exports, {
|
|
|
2773
2778
|
TelemetryService: () => TelemetryService,
|
|
2774
2779
|
TextRendererStrategy: () => TextRendererStrategy,
|
|
2775
2780
|
UniversalLLMAdapter: () => UniversalLLMAdapter,
|
|
2781
|
+
UniversalTableParser: () => UniversalTableParser,
|
|
2776
2782
|
UniversalVectorProvider: () => UniversalVectorProvider,
|
|
2777
2783
|
VECTOR_PROFILES: () => VECTOR_PROFILES,
|
|
2778
2784
|
VectorPlugin: () => VectorPlugin,
|
|
2785
|
+
VisionExtractor: () => VisionExtractor,
|
|
2779
2786
|
VisualizationDecisionEngine: () => VisualizationDecisionEngine,
|
|
2780
2787
|
WeaviateProvider: () => WeaviateProvider,
|
|
2781
2788
|
createChatHandler: () => createChatHandler,
|
|
@@ -5752,27 +5759,384 @@ Text to extract from:
|
|
|
5752
5759
|
}
|
|
5753
5760
|
};
|
|
5754
5761
|
|
|
5762
|
+
// src/rag/HybridSearch.ts
|
|
5763
|
+
var HybridSearch = class {
|
|
5764
|
+
/**
|
|
5765
|
+
* Tokenize text into normalized alphanumeric keywords, keeping exact codes, numbers, and identifiers.
|
|
5766
|
+
*/
|
|
5767
|
+
static tokenize(text) {
|
|
5768
|
+
if (!text) return [];
|
|
5769
|
+
return text.toLowerCase().replace(/[^\w\s\-_#./:]/g, " ").split(/\s+/).map((t) => t.trim()).filter((t) => t.length > 1 && !this.stopWords.has(t));
|
|
5770
|
+
}
|
|
5771
|
+
/**
|
|
5772
|
+
* Calculate Okapi BM25 sparse scores for an array of document chunks given a query.
|
|
5773
|
+
*/
|
|
5774
|
+
static scoreBM25(documents, query, k1 = 1.5, b = 0.75) {
|
|
5775
|
+
const scores = /* @__PURE__ */ new Map();
|
|
5776
|
+
const queryTokens = this.tokenize(query);
|
|
5777
|
+
if (queryTokens.length === 0 || documents.length === 0) {
|
|
5778
|
+
documents.forEach((d) => scores.set(d.id, 0));
|
|
5779
|
+
return scores;
|
|
5780
|
+
}
|
|
5781
|
+
const docTokens = documents.map((d) => ({
|
|
5782
|
+
id: d.id,
|
|
5783
|
+
tokens: this.tokenize(d.content)
|
|
5784
|
+
}));
|
|
5785
|
+
const N = documents.length;
|
|
5786
|
+
const avgdl = docTokens.reduce((acc, d) => acc + d.tokens.length, 0) / (N || 1);
|
|
5787
|
+
const dfMap = /* @__PURE__ */ new Map();
|
|
5788
|
+
for (const qt of queryTokens) {
|
|
5789
|
+
let count = 0;
|
|
5790
|
+
for (const dt of docTokens) {
|
|
5791
|
+
if (dt.tokens.includes(qt)) count++;
|
|
5792
|
+
}
|
|
5793
|
+
dfMap.set(qt, count);
|
|
5794
|
+
}
|
|
5795
|
+
for (const dt of docTokens) {
|
|
5796
|
+
let score = 0;
|
|
5797
|
+
const docLen = dt.tokens.length;
|
|
5798
|
+
const tfMap = /* @__PURE__ */ new Map();
|
|
5799
|
+
for (const t of dt.tokens) {
|
|
5800
|
+
tfMap.set(t, (tfMap.get(t) || 0) + 1);
|
|
5801
|
+
}
|
|
5802
|
+
for (const qt of queryTokens) {
|
|
5803
|
+
const tf = tfMap.get(qt) || 0;
|
|
5804
|
+
if (tf === 0) continue;
|
|
5805
|
+
const df = dfMap.get(qt) || 0;
|
|
5806
|
+
const idf = Math.log((N - df + 0.5) / (df + 0.5) + 1);
|
|
5807
|
+
const numerator = tf * (k1 + 1);
|
|
5808
|
+
const denominator = tf + k1 * (1 - b + b * (docLen / (avgdl || 1)));
|
|
5809
|
+
score += idf * (numerator / (denominator || 1));
|
|
5810
|
+
}
|
|
5811
|
+
scores.set(dt.id, score);
|
|
5812
|
+
}
|
|
5813
|
+
return scores;
|
|
5814
|
+
}
|
|
5815
|
+
/**
|
|
5816
|
+
* Fuse Dense Vector Matches with Sparse BM25 Scores using Reciprocal Rank Fusion (RRF).
|
|
5817
|
+
*/
|
|
5818
|
+
static fuse(denseMatches, query, options = {}) {
|
|
5819
|
+
var _a2, _b, _c;
|
|
5820
|
+
if (denseMatches.length === 0) return [];
|
|
5821
|
+
const denseWeight = (_a2 = options.denseWeight) != null ? _a2 : 0.6;
|
|
5822
|
+
const sparseWeight = (_b = options.sparseWeight) != null ? _b : 0.4;
|
|
5823
|
+
const k = (_c = options.rrfK) != null ? _c : 60;
|
|
5824
|
+
const sortedDense = [...denseMatches].sort((a, b) => b.score - a.score);
|
|
5825
|
+
const denseRankMap = /* @__PURE__ */ new Map();
|
|
5826
|
+
sortedDense.forEach((match, idx) => {
|
|
5827
|
+
denseRankMap.set(match.id, idx + 1);
|
|
5828
|
+
});
|
|
5829
|
+
const docItems = denseMatches.map((m) => ({ id: m.id, content: m.content }));
|
|
5830
|
+
const bm25ScoreMap = this.scoreBM25(docItems, query, options.bm25K1, options.bm25B);
|
|
5831
|
+
const sortedSparse = [...denseMatches].sort((a, b) => {
|
|
5832
|
+
const scoreA = bm25ScoreMap.get(a.id) || 0;
|
|
5833
|
+
const scoreB = bm25ScoreMap.get(b.id) || 0;
|
|
5834
|
+
return scoreB - scoreA;
|
|
5835
|
+
});
|
|
5836
|
+
const sparseRankMap = /* @__PURE__ */ new Map();
|
|
5837
|
+
sortedSparse.forEach((match, idx) => {
|
|
5838
|
+
sparseRankMap.set(match.id, idx + 1);
|
|
5839
|
+
});
|
|
5840
|
+
const fusedMatches = denseMatches.map((match) => {
|
|
5841
|
+
var _a3;
|
|
5842
|
+
const denseRank = denseRankMap.get(match.id) || denseMatches.length;
|
|
5843
|
+
const sparseRank = sparseRankMap.get(match.id) || denseMatches.length;
|
|
5844
|
+
const denseRrf = denseWeight / (k + denseRank);
|
|
5845
|
+
const sparseRrf = sparseWeight / (k + sparseRank);
|
|
5846
|
+
const combinedRrf = denseRrf + sparseRrf;
|
|
5847
|
+
const bm25Raw = bm25ScoreMap.get(match.id) || 0;
|
|
5848
|
+
return __spreadProps(__spreadValues({}, match), {
|
|
5849
|
+
rrfScore: combinedRrf,
|
|
5850
|
+
bm25Score: bm25Raw,
|
|
5851
|
+
rerankScore: ((_a3 = match.rerankScore) != null ? _a3 : match.score) * 0.5 + combinedRrf * 50
|
|
5852
|
+
});
|
|
5853
|
+
});
|
|
5854
|
+
return fusedMatches.sort((a, b) => {
|
|
5855
|
+
var _a3, _b2;
|
|
5856
|
+
return ((_a3 = b.rerankScore) != null ? _a3 : b.score) - ((_b2 = a.rerankScore) != null ? _b2 : a.score);
|
|
5857
|
+
});
|
|
5858
|
+
}
|
|
5859
|
+
};
|
|
5860
|
+
HybridSearch.stopWords = /* @__PURE__ */ new Set([
|
|
5861
|
+
"a",
|
|
5862
|
+
"about",
|
|
5863
|
+
"above",
|
|
5864
|
+
"after",
|
|
5865
|
+
"again",
|
|
5866
|
+
"against",
|
|
5867
|
+
"all",
|
|
5868
|
+
"am",
|
|
5869
|
+
"an",
|
|
5870
|
+
"and",
|
|
5871
|
+
"any",
|
|
5872
|
+
"are",
|
|
5873
|
+
"aren't",
|
|
5874
|
+
"as",
|
|
5875
|
+
"at",
|
|
5876
|
+
"be",
|
|
5877
|
+
"because",
|
|
5878
|
+
"been",
|
|
5879
|
+
"before",
|
|
5880
|
+
"being",
|
|
5881
|
+
"below",
|
|
5882
|
+
"between",
|
|
5883
|
+
"both",
|
|
5884
|
+
"but",
|
|
5885
|
+
"by",
|
|
5886
|
+
"can't",
|
|
5887
|
+
"cannot",
|
|
5888
|
+
"could",
|
|
5889
|
+
"couldn't",
|
|
5890
|
+
"did",
|
|
5891
|
+
"didn't",
|
|
5892
|
+
"do",
|
|
5893
|
+
"does",
|
|
5894
|
+
"doesn't",
|
|
5895
|
+
"doing",
|
|
5896
|
+
"don't",
|
|
5897
|
+
"down",
|
|
5898
|
+
"during",
|
|
5899
|
+
"each",
|
|
5900
|
+
"few",
|
|
5901
|
+
"for",
|
|
5902
|
+
"from",
|
|
5903
|
+
"further",
|
|
5904
|
+
"had",
|
|
5905
|
+
"hadn't",
|
|
5906
|
+
"has",
|
|
5907
|
+
"hasn't",
|
|
5908
|
+
"have",
|
|
5909
|
+
"haven't",
|
|
5910
|
+
"having",
|
|
5911
|
+
"he",
|
|
5912
|
+
"he'd",
|
|
5913
|
+
"he'll",
|
|
5914
|
+
"he's",
|
|
5915
|
+
"her",
|
|
5916
|
+
"here",
|
|
5917
|
+
"here's",
|
|
5918
|
+
"hers",
|
|
5919
|
+
"herself",
|
|
5920
|
+
"him",
|
|
5921
|
+
"himself",
|
|
5922
|
+
"his",
|
|
5923
|
+
"how",
|
|
5924
|
+
"how's",
|
|
5925
|
+
"i",
|
|
5926
|
+
"i'd",
|
|
5927
|
+
"i'll",
|
|
5928
|
+
"i'm",
|
|
5929
|
+
"i've",
|
|
5930
|
+
"if",
|
|
5931
|
+
"in",
|
|
5932
|
+
"into",
|
|
5933
|
+
"is",
|
|
5934
|
+
"isn't",
|
|
5935
|
+
"it",
|
|
5936
|
+
"it's",
|
|
5937
|
+
"its",
|
|
5938
|
+
"itself",
|
|
5939
|
+
"let's",
|
|
5940
|
+
"me",
|
|
5941
|
+
"more",
|
|
5942
|
+
"most",
|
|
5943
|
+
"mustn't",
|
|
5944
|
+
"my",
|
|
5945
|
+
"myself",
|
|
5946
|
+
"no",
|
|
5947
|
+
"nor",
|
|
5948
|
+
"not",
|
|
5949
|
+
"of",
|
|
5950
|
+
"off",
|
|
5951
|
+
"on",
|
|
5952
|
+
"once",
|
|
5953
|
+
"only",
|
|
5954
|
+
"or",
|
|
5955
|
+
"other",
|
|
5956
|
+
"ought",
|
|
5957
|
+
"our",
|
|
5958
|
+
"ours",
|
|
5959
|
+
"ourselves",
|
|
5960
|
+
"out",
|
|
5961
|
+
"over",
|
|
5962
|
+
"own",
|
|
5963
|
+
"same",
|
|
5964
|
+
"shan't",
|
|
5965
|
+
"she",
|
|
5966
|
+
"she'd",
|
|
5967
|
+
"she'll",
|
|
5968
|
+
"she's",
|
|
5969
|
+
"should",
|
|
5970
|
+
"shouldn't",
|
|
5971
|
+
"so",
|
|
5972
|
+
"some",
|
|
5973
|
+
"such",
|
|
5974
|
+
"than",
|
|
5975
|
+
"that",
|
|
5976
|
+
"that's",
|
|
5977
|
+
"the",
|
|
5978
|
+
"their",
|
|
5979
|
+
"theirs",
|
|
5980
|
+
"them",
|
|
5981
|
+
"themselves",
|
|
5982
|
+
"then",
|
|
5983
|
+
"there",
|
|
5984
|
+
"there's",
|
|
5985
|
+
"these",
|
|
5986
|
+
"they",
|
|
5987
|
+
"they'd",
|
|
5988
|
+
"they'll",
|
|
5989
|
+
"they're",
|
|
5990
|
+
"they've",
|
|
5991
|
+
"this",
|
|
5992
|
+
"those",
|
|
5993
|
+
"through",
|
|
5994
|
+
"to",
|
|
5995
|
+
"too",
|
|
5996
|
+
"under",
|
|
5997
|
+
"until",
|
|
5998
|
+
"up",
|
|
5999
|
+
"very",
|
|
6000
|
+
"was",
|
|
6001
|
+
"wasn't",
|
|
6002
|
+
"we",
|
|
6003
|
+
"we'd",
|
|
6004
|
+
"we'll",
|
|
6005
|
+
"we're",
|
|
6006
|
+
"we've",
|
|
6007
|
+
"were",
|
|
6008
|
+
"weren't",
|
|
6009
|
+
"what",
|
|
6010
|
+
"what's",
|
|
6011
|
+
"when",
|
|
6012
|
+
"when's",
|
|
6013
|
+
"where",
|
|
6014
|
+
"where's",
|
|
6015
|
+
"which",
|
|
6016
|
+
"while",
|
|
6017
|
+
"who",
|
|
6018
|
+
"who's",
|
|
6019
|
+
"whom",
|
|
6020
|
+
"why",
|
|
6021
|
+
"why's",
|
|
6022
|
+
"with",
|
|
6023
|
+
"won't",
|
|
6024
|
+
"would",
|
|
6025
|
+
"wouldn't",
|
|
6026
|
+
"you",
|
|
6027
|
+
"you'd",
|
|
6028
|
+
"you'll",
|
|
6029
|
+
"you're",
|
|
6030
|
+
"you've",
|
|
6031
|
+
"your",
|
|
6032
|
+
"yours",
|
|
6033
|
+
"yourself",
|
|
6034
|
+
"yourselves"
|
|
6035
|
+
]);
|
|
6036
|
+
|
|
5755
6037
|
// src/rag/Reranker.ts
|
|
5756
6038
|
var MAX_RERANK_CONTENT_CHARS = 64e3;
|
|
5757
6039
|
var Reranker = class {
|
|
5758
6040
|
/**
|
|
5759
|
-
* Re-ranks matches based on a secondary relevance score using
|
|
5760
|
-
*
|
|
5761
|
-
* Fix RAG-2: keyword re-ranking no longer mutates the original `score`
|
|
5762
|
-
* field (which consumers interpret as the raw vector similarity). A new
|
|
5763
|
-
* `rerankScore` field is added so both the original similarity and the
|
|
5764
|
-
* boosted rank signal are available to callers.
|
|
6041
|
+
* Re-ranks matches based on a secondary relevance score using Hybrid BM25, Cohere, Jina or LLM.
|
|
5765
6042
|
*/
|
|
5766
|
-
async rerank(matches, query, limit = 5, llm) {
|
|
5767
|
-
|
|
5768
|
-
|
|
6043
|
+
async rerank(matches, query, limit = 5, llm, options = {}) {
|
|
6044
|
+
var _a2, _b;
|
|
6045
|
+
if (!matches || matches.length <= 1) {
|
|
6046
|
+
return matches ? matches.slice(0, limit) : [];
|
|
5769
6047
|
}
|
|
5770
|
-
|
|
5771
|
-
|
|
5772
|
-
|
|
5773
|
-
|
|
5774
|
-
|
|
6048
|
+
const cohereKey = options.cohereApiKey || (typeof process !== "undefined" ? (_a2 = process.env) == null ? void 0 : _a2.COHERE_API_KEY : void 0);
|
|
6049
|
+
const jinaKey = options.jinaApiKey || (typeof process !== "undefined" ? (_b = process.env) == null ? void 0 : _b.JINA_API_KEY : void 0);
|
|
6050
|
+
if ((options.provider === "cohere" || cohereKey) && cohereKey) {
|
|
6051
|
+
try {
|
|
6052
|
+
const cohereResult = await this.cohereRerank(matches, query, limit, cohereKey, options.cohereModel);
|
|
6053
|
+
if (cohereResult.length > 0) return cohereResult;
|
|
6054
|
+
} catch (err) {
|
|
6055
|
+
console.warn("[Reranker] Cohere reranking failed, falling back to HybridSearch:", err);
|
|
6056
|
+
}
|
|
5775
6057
|
}
|
|
6058
|
+
if ((options.provider === "jina" || jinaKey) && jinaKey) {
|
|
6059
|
+
try {
|
|
6060
|
+
const jinaResult = await this.jinaRerank(matches, query, limit, jinaKey, options.jinaModel);
|
|
6061
|
+
if (jinaResult.length > 0) return jinaResult;
|
|
6062
|
+
} catch (err) {
|
|
6063
|
+
console.warn("[Reranker] Jina reranking failed, falling back to HybridSearch:", err);
|
|
6064
|
+
}
|
|
6065
|
+
}
|
|
6066
|
+
if (!options.provider || options.provider === "hybrid") {
|
|
6067
|
+
const fused = HybridSearch.fuse(matches, query);
|
|
6068
|
+
if (fused.length > 0) {
|
|
6069
|
+
return fused.slice(0, limit);
|
|
6070
|
+
}
|
|
6071
|
+
}
|
|
6072
|
+
if (llm && (options.provider === "llm" || !options.provider)) {
|
|
6073
|
+
try {
|
|
6074
|
+
return await this.llmRerank(matches, query, limit, llm);
|
|
6075
|
+
} catch (error) {
|
|
6076
|
+
console.warn("[Reranker] LLM re-ranking failed, falling back to keyword sorting:", error);
|
|
6077
|
+
}
|
|
6078
|
+
}
|
|
6079
|
+
return this.keywordRerank(matches, query, limit);
|
|
6080
|
+
}
|
|
6081
|
+
/**
|
|
6082
|
+
* Cohere Rerank v3.5 API integration.
|
|
6083
|
+
*/
|
|
6084
|
+
async cohereRerank(matches, query, limit, apiKey, model = "rerank-v3.5") {
|
|
6085
|
+
const documents = matches.map((m) => m.content || "");
|
|
6086
|
+
const response = await fetch("https://api.cohere.com/v1/rerank", {
|
|
6087
|
+
method: "POST",
|
|
6088
|
+
headers: {
|
|
6089
|
+
"Authorization": `Bearer ${apiKey}`,
|
|
6090
|
+
"Content-Type": "application/json"
|
|
6091
|
+
},
|
|
6092
|
+
body: JSON.stringify({
|
|
6093
|
+
model,
|
|
6094
|
+
query,
|
|
6095
|
+
documents,
|
|
6096
|
+
top_n: limit
|
|
6097
|
+
})
|
|
6098
|
+
});
|
|
6099
|
+
if (!response.ok) {
|
|
6100
|
+
throw new Error(`Cohere API returned status ${response.status}`);
|
|
6101
|
+
}
|
|
6102
|
+
const data = await response.json();
|
|
6103
|
+
if (!data.results || !Array.isArray(data.results)) return [];
|
|
6104
|
+
return data.results.map((res) => {
|
|
6105
|
+
const match = matches[res.index];
|
|
6106
|
+
return __spreadProps(__spreadValues({}, match), {
|
|
6107
|
+
rerankScore: res.relevance_score
|
|
6108
|
+
});
|
|
6109
|
+
});
|
|
6110
|
+
}
|
|
6111
|
+
/**
|
|
6112
|
+
* Jina AI Reranker v2 API integration.
|
|
6113
|
+
*/
|
|
6114
|
+
async jinaRerank(matches, query, limit, apiKey, model = "jina-reranker-v2-base-multilingual") {
|
|
6115
|
+
const documents = matches.map((m) => m.content || "");
|
|
6116
|
+
const response = await fetch("https://api.jina.ai/v1/rerank", {
|
|
6117
|
+
method: "POST",
|
|
6118
|
+
headers: {
|
|
6119
|
+
"Authorization": `Bearer ${apiKey}`,
|
|
6120
|
+
"Content-Type": "application/json"
|
|
6121
|
+
},
|
|
6122
|
+
body: JSON.stringify({
|
|
6123
|
+
model,
|
|
6124
|
+
query,
|
|
6125
|
+
documents,
|
|
6126
|
+
top_n: limit
|
|
6127
|
+
})
|
|
6128
|
+
});
|
|
6129
|
+
if (!response.ok) {
|
|
6130
|
+
throw new Error(`Jina API returned status ${response.status}`);
|
|
6131
|
+
}
|
|
6132
|
+
const data = await response.json();
|
|
6133
|
+
if (!data.results || !Array.isArray(data.results)) return [];
|
|
6134
|
+
return data.results.map((res) => {
|
|
6135
|
+
const match = matches[res.index];
|
|
6136
|
+
return __spreadProps(__spreadValues({}, match), {
|
|
6137
|
+
rerankScore: res.relevance_score
|
|
6138
|
+
});
|
|
6139
|
+
});
|
|
5776
6140
|
}
|
|
5777
6141
|
/**
|
|
5778
6142
|
* Keyword-based re-ranking fallback.
|
|
@@ -5858,6 +6222,176 @@ ${docList}` }],
|
|
|
5858
6222
|
}
|
|
5859
6223
|
};
|
|
5860
6224
|
|
|
6225
|
+
// src/rag/RAGEvaluator.ts
|
|
6226
|
+
var RAGEvaluator = class {
|
|
6227
|
+
/**
|
|
6228
|
+
* Evaluate a RAG interaction across the full RAG Triad.
|
|
6229
|
+
*/
|
|
6230
|
+
static async evaluate(query, context, answer, llm) {
|
|
6231
|
+
if (!llm || !context || !answer) {
|
|
6232
|
+
return this.deterministicHeuristic(query, context, answer);
|
|
6233
|
+
}
|
|
6234
|
+
try {
|
|
6235
|
+
const prompt = `You are an automated RAG Triad Evaluator. Evaluate the quality of the following RAG system interaction:
|
|
6236
|
+
|
|
6237
|
+
USER QUERY:
|
|
6238
|
+
${query}
|
|
6239
|
+
|
|
6240
|
+
RETRIEVED CONTEXT:
|
|
6241
|
+
${context.slice(0, 3e3)}
|
|
6242
|
+
|
|
6243
|
+
GENERATED ANSWER:
|
|
6244
|
+
${answer}
|
|
6245
|
+
|
|
6246
|
+
Evaluate these 3 metrics on a scale of 0.0 to 1.0:
|
|
6247
|
+
1. "contextRelevance": Are the retrieved documents relevant to the query? (1.0 = highly relevant, 0.0 = irrelevant)
|
|
6248
|
+
2. "faithfulness": Is every claim in the generated answer supported by the retrieved context? (1.0 = 100% grounded/no hallucination, 0.0 = completely fabricated)
|
|
6249
|
+
3. "answerRelevance": Does the answer directly address the user's question? (1.0 = direct answer, 0.0 = evasive/unrelated)
|
|
6250
|
+
|
|
6251
|
+
Return ONLY valid JSON with no markdown fences:
|
|
6252
|
+
{"contextRelevance": 0.95, "faithfulness": 0.90, "answerRelevance": 0.92, "reason": "Answer is fully supported by the document."}`;
|
|
6253
|
+
const raw = await llm.chat(
|
|
6254
|
+
[{ role: "user", content: prompt }],
|
|
6255
|
+
"",
|
|
6256
|
+
{
|
|
6257
|
+
systemPrompt: "You are a strict AI safety and hallucination evaluator. Output only valid JSON.",
|
|
6258
|
+
temperature: 0,
|
|
6259
|
+
maxTokens: 150
|
|
6260
|
+
}
|
|
6261
|
+
);
|
|
6262
|
+
const cleaned = raw.replace(/```json|```/g, "").trim();
|
|
6263
|
+
const parsed = JSON.parse(cleaned);
|
|
6264
|
+
const cr = typeof parsed.contextRelevance === "number" ? Math.max(0, Math.min(1, parsed.contextRelevance)) : 0.85;
|
|
6265
|
+
const f = typeof parsed.faithfulness === "number" ? Math.max(0, Math.min(1, parsed.faithfulness)) : 0.9;
|
|
6266
|
+
const ar = typeof parsed.answerRelevance === "number" ? Math.max(0, Math.min(1, parsed.answerRelevance)) : 0.88;
|
|
6267
|
+
const overall = Number((cr * 0.3 + f * 0.4 + ar * 0.3).toFixed(2));
|
|
6268
|
+
return {
|
|
6269
|
+
contextRelevance: cr,
|
|
6270
|
+
faithfulness: f,
|
|
6271
|
+
answerRelevance: ar,
|
|
6272
|
+
overallQuality: overall,
|
|
6273
|
+
isHallucinated: f < 0.6,
|
|
6274
|
+
reason: parsed.reason || "Evaluated successfully."
|
|
6275
|
+
};
|
|
6276
|
+
} catch (err) {
|
|
6277
|
+
return this.deterministicHeuristic(query, context, answer);
|
|
6278
|
+
}
|
|
6279
|
+
}
|
|
6280
|
+
/**
|
|
6281
|
+
* Deterministic fast keyword-overlap heuristic when LLM evaluation is not invoked.
|
|
6282
|
+
*/
|
|
6283
|
+
static deterministicHeuristic(query, context, answer) {
|
|
6284
|
+
if (!context || !answer) {
|
|
6285
|
+
return { contextRelevance: 0.5, faithfulness: 0.8, answerRelevance: 0.7, overallQuality: 0.67, isHallucinated: false };
|
|
6286
|
+
}
|
|
6287
|
+
const queryWords = query.toLowerCase().split(/\s+/).filter((w) => w.length > 2);
|
|
6288
|
+
const contextLower = context.toLowerCase();
|
|
6289
|
+
const answerLower = answer.toLowerCase();
|
|
6290
|
+
let queryContextHits = 0;
|
|
6291
|
+
queryWords.forEach((w) => {
|
|
6292
|
+
if (contextLower.includes(w)) queryContextHits++;
|
|
6293
|
+
});
|
|
6294
|
+
const contextRelevance = queryWords.length > 0 ? Math.min(1, queryContextHits / queryWords.length + 0.3) : 0.8;
|
|
6295
|
+
const numbersInAnswer = answer.match(/\b\d+(?:\.\d+)?%?\b/g) || [];
|
|
6296
|
+
let numberHits = 0;
|
|
6297
|
+
numbersInAnswer.forEach((num) => {
|
|
6298
|
+
if (context.includes(num)) numberHits++;
|
|
6299
|
+
});
|
|
6300
|
+
const faithfulness = numbersInAnswer.length > 0 ? Math.min(1, numberHits / numbersInAnswer.length + 0.2) : 0.9;
|
|
6301
|
+
const answerRelevance = 0.85;
|
|
6302
|
+
const overallQuality = Number((contextRelevance * 0.3 + faithfulness * 0.4 + answerRelevance * 0.3).toFixed(2));
|
|
6303
|
+
return {
|
|
6304
|
+
contextRelevance: Number(contextRelevance.toFixed(2)),
|
|
6305
|
+
faithfulness: Number(faithfulness.toFixed(2)),
|
|
6306
|
+
answerRelevance,
|
|
6307
|
+
overallQuality,
|
|
6308
|
+
isHallucinated: faithfulness < 0.5,
|
|
6309
|
+
reason: "Computed via deterministic grounding heuristic."
|
|
6310
|
+
};
|
|
6311
|
+
}
|
|
6312
|
+
};
|
|
6313
|
+
|
|
6314
|
+
// src/security/PIIMasker.ts
|
|
6315
|
+
var PIIMasker = class {
|
|
6316
|
+
/**
|
|
6317
|
+
* Redact sensitive information from text.
|
|
6318
|
+
*/
|
|
6319
|
+
static mask(text, options = {}) {
|
|
6320
|
+
if (!text || typeof text !== "string") return text;
|
|
6321
|
+
let sanitized = text;
|
|
6322
|
+
if (options.maskApiKeys !== false) {
|
|
6323
|
+
sanitized = sanitized.replace(this.API_KEY_REGEX, "[REDACTED_API_KEY]");
|
|
6324
|
+
}
|
|
6325
|
+
if (options.maskCreditCards !== false) {
|
|
6326
|
+
sanitized = sanitized.replace(this.CREDIT_CARD_REGEX, (match) => {
|
|
6327
|
+
const digits = match.replace(/[- ]/g, "");
|
|
6328
|
+
if (digits.length >= 13 && digits.length <= 19) {
|
|
6329
|
+
return "[REDACTED_CREDIT_CARD]";
|
|
6330
|
+
}
|
|
6331
|
+
return match;
|
|
6332
|
+
});
|
|
6333
|
+
}
|
|
6334
|
+
if (options.maskSSN !== false) {
|
|
6335
|
+
sanitized = sanitized.replace(this.SSN_REGEX, "[REDACTED_SSN]");
|
|
6336
|
+
}
|
|
6337
|
+
if (options.maskEmails !== false) {
|
|
6338
|
+
sanitized = sanitized.replace(this.EMAIL_REGEX, "[REDACTED_EMAIL]");
|
|
6339
|
+
}
|
|
6340
|
+
if (options.maskPhones !== false) {
|
|
6341
|
+
sanitized = sanitized.replace(this.PHONE_REGEX, "[REDACTED_PHONE]");
|
|
6342
|
+
}
|
|
6343
|
+
if (options.maskIPs === true) {
|
|
6344
|
+
sanitized = sanitized.replace(this.IP_REGEX, "[REDACTED_IP]");
|
|
6345
|
+
}
|
|
6346
|
+
return sanitized;
|
|
6347
|
+
}
|
|
6348
|
+
};
|
|
6349
|
+
PIIMasker.SSN_REGEX = /\b(?!000|666|9\d{2})\d{3}[- ]?(?!00)\d{2}[- ]?(?!0000)\d{4}\b/g;
|
|
6350
|
+
PIIMasker.CREDIT_CARD_REGEX = /\b(?:\d{4}[- ]?){3}\d{4}\b|\b\d{15,16}\b/g;
|
|
6351
|
+
PIIMasker.EMAIL_REGEX = /\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Z|a-z]{2,}\b/g;
|
|
6352
|
+
PIIMasker.PHONE_REGEX = /\b(?:\+?\d{1,3}[-.\s]?)?\(?\d{3}\)?[-.\s]?\d{3}[-.\s]?\d{4}\b/g;
|
|
6353
|
+
PIIMasker.API_KEY_REGEX = /\b(?:sk-[a-zA-Z0-9]{20,}|AKIA[0-9A-Z]{16}|rtv_[a-zA-Z0-9_-]{20,}|ghp_[a-zA-Z0-9]{36}|bearer\s+[a-zA-Z0-9._-]{20,})\b/gi;
|
|
6354
|
+
PIIMasker.IP_REGEX = /\b(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\b/g;
|
|
6355
|
+
|
|
6356
|
+
// src/security/PromptGuard.ts
|
|
6357
|
+
var PromptGuard = class {
|
|
6358
|
+
/**
|
|
6359
|
+
* Scan and evaluate user query or document text for prompt injection vectors.
|
|
6360
|
+
*/
|
|
6361
|
+
static evaluate(text) {
|
|
6362
|
+
if (!text || typeof text !== "string") {
|
|
6363
|
+
return { isSafe: true, threatLevel: "none", detectedPatterns: [], sanitizedText: text };
|
|
6364
|
+
}
|
|
6365
|
+
const detected = [];
|
|
6366
|
+
let maxSeverity = "none";
|
|
6367
|
+
for (const item of this.INJECTION_PATTERNS) {
|
|
6368
|
+
if (item.pattern.test(text)) {
|
|
6369
|
+
detected.push(item.name);
|
|
6370
|
+
if (item.severity === "high") {
|
|
6371
|
+
maxSeverity = "high";
|
|
6372
|
+
} else if (item.severity === "medium" && maxSeverity !== "high") {
|
|
6373
|
+
maxSeverity = "medium";
|
|
6374
|
+
}
|
|
6375
|
+
}
|
|
6376
|
+
}
|
|
6377
|
+
const sanitized = text.replace(/<\/?(?:system|instruction|admin|prompt_override)>/gi, "");
|
|
6378
|
+
return {
|
|
6379
|
+
isSafe: maxSeverity !== "high",
|
|
6380
|
+
threatLevel: maxSeverity,
|
|
6381
|
+
detectedPatterns: detected,
|
|
6382
|
+
sanitizedText: sanitized
|
|
6383
|
+
};
|
|
6384
|
+
}
|
|
6385
|
+
};
|
|
6386
|
+
PromptGuard.INJECTION_PATTERNS = [
|
|
6387
|
+
{ pattern: /ignore\s+(?:all\s+)?(?:previous|prior|above)\s+(?:instructions|prompts|rules)/i, name: "override_instructions", severity: "high" },
|
|
6388
|
+
{ pattern: /(?:show|repeat|print|reveal|expose|output)\s+(?:the\s+)?(?:system\s+prompt|initial\s+instructions|system\s+message)/i, name: "system_prompt_leak", severity: "high" },
|
|
6389
|
+
{ pattern: /you\s+are\s+now\s+(?:in\s+developer\s+mode|unrestricted|DAN|jailbroken|godmode)/i, name: "jailbreak_persona", severity: "high" },
|
|
6390
|
+
{ pattern: /<\/?(?:system|instruction|admin|prompt_override)>/i, name: "tag_injection", severity: "medium" },
|
|
6391
|
+
{ pattern: /base64\s*decode\s*and\s*execute/i, name: "obfuscated_payload", severity: "high" },
|
|
6392
|
+
{ pattern: /do\s+anything\s+now/i, name: "dan_variant", severity: "high" }
|
|
6393
|
+
];
|
|
6394
|
+
|
|
5861
6395
|
// src/rag/LlamaIndexIngestor.ts
|
|
5862
6396
|
var LlamaIndexIngestor = class {
|
|
5863
6397
|
/**
|
|
@@ -9813,7 +10347,7 @@ ${m.content}`).join("\n\n---\n\n");
|
|
|
9813
10347
|
*/
|
|
9814
10348
|
async processEmbeddings(chunks) {
|
|
9815
10349
|
if (chunks.length === 0) return [];
|
|
9816
|
-
const texts = chunks.map((c) => c.content);
|
|
10350
|
+
const texts = chunks.map((c) => PIIMasker.mask(c.content));
|
|
9817
10351
|
if (typeof this.embeddingProvider.batchEmbed === "function") {
|
|
9818
10352
|
try {
|
|
9819
10353
|
const batchSize = 50;
|
|
@@ -10019,10 +10553,12 @@ ${m.content}`).join("\n\n---\n\n");
|
|
|
10019
10553
|
const requestId = `req_${Date.now()}_${Math.random().toString(36).slice(2, 8)}`;
|
|
10020
10554
|
const requestStart = performance.now();
|
|
10021
10555
|
try {
|
|
10022
|
-
|
|
10556
|
+
const guardCheck = PromptGuard.evaluate(question);
|
|
10557
|
+
let effectiveQuestion = guardCheck.sanitizedText;
|
|
10558
|
+
effectiveQuestion = PIIMasker.mask(effectiveQuestion);
|
|
10023
10559
|
let promptOptResult;
|
|
10024
10560
|
if (((_e = this.config.rag) == null ? void 0 : _e.optimizePrompt) !== false) {
|
|
10025
|
-
promptOptResult = PromptOptimizer.optimize(
|
|
10561
|
+
promptOptResult = PromptOptimizer.optimize(effectiveQuestion, history);
|
|
10026
10562
|
effectiveQuestion = promptOptResult.optimizedPrompt;
|
|
10027
10563
|
}
|
|
10028
10564
|
let searchQuery = effectiveQuestion;
|
|
@@ -10307,6 +10843,7 @@ ${context}`;
|
|
|
10307
10843
|
uiTransformationPromise,
|
|
10308
10844
|
awaitHallucination ? hallucinationScoringPromise : Promise.resolve(void 0)
|
|
10309
10845
|
]));
|
|
10846
|
+
const triadScore = yield new __await(RAGEvaluator.evaluate(effectiveQuestion, context, fullReply));
|
|
10310
10847
|
const buildTrace = (hScore) => __spreadValues({
|
|
10311
10848
|
requestId,
|
|
10312
10849
|
query: question,
|
|
@@ -10325,7 +10862,13 @@ ${context}`;
|
|
|
10325
10862
|
}),
|
|
10326
10863
|
latency,
|
|
10327
10864
|
tokens,
|
|
10328
|
-
timestamp: (/* @__PURE__ */ new Date()).toISOString()
|
|
10865
|
+
timestamp: (/* @__PURE__ */ new Date()).toISOString(),
|
|
10866
|
+
ragTriad: {
|
|
10867
|
+
contextRelevance: triadScore.contextRelevance,
|
|
10868
|
+
faithfulness: triadScore.faithfulness,
|
|
10869
|
+
answerRelevance: triadScore.answerRelevance,
|
|
10870
|
+
overallQuality: triadScore.overallQuality
|
|
10871
|
+
}
|
|
10329
10872
|
}, hScore ? {
|
|
10330
10873
|
hallucinationScore: hScore.score,
|
|
10331
10874
|
hallucinationReason: hScore.reason
|
|
@@ -11195,6 +11738,374 @@ function createFromPreset(presetName) {
|
|
|
11195
11738
|
return new ConfigBuilder().vectorDb(preset.vectorDb, { apiKey: process.env[`${preset.vectorDb.toUpperCase()}_API_KEY`] }).llm(preset.llm, void 0, process.env[`${preset.llm.toUpperCase()}_API_KEY`]).embedding(preset.embedding, void 0, process.env[`${preset.embedding.toUpperCase()}_API_KEY`]);
|
|
11196
11739
|
}
|
|
11197
11740
|
|
|
11741
|
+
// src/utils/UniversalTableParser.ts
|
|
11742
|
+
var UniversalTableParser = class {
|
|
11743
|
+
/**
|
|
11744
|
+
* Parse a raw 2D grid of strings with complete matrix intelligence.
|
|
11745
|
+
*/
|
|
11746
|
+
static parseGrid(grid, sheetName = "Data") {
|
|
11747
|
+
if (!grid || grid.length === 0) return "";
|
|
11748
|
+
const cleanGrid = grid.map((row) => Array.isArray(row) ? row.map((c) => c !== null && c !== void 0 ? String(c).trim() : "") : []).filter((row) => row.some((cell) => cell.length > 0));
|
|
11749
|
+
if (cleanGrid.length === 0) return "";
|
|
11750
|
+
const { headerRows, dataRows } = this.splitHeaderAndDataRows(cleanGrid);
|
|
11751
|
+
const fusedHeaders = this.fuseHeaders(headerRows);
|
|
11752
|
+
const matrixInfo = this.analyzeMatrixStructure(fusedHeaders, dataRows);
|
|
11753
|
+
const sections = [`--- Sheet: ${sheetName} ---`];
|
|
11754
|
+
const markdownTable = this.generateMarkdownTable(fusedHeaders, dataRows);
|
|
11755
|
+
if (markdownTable) {
|
|
11756
|
+
sections.push(`
|
|
11757
|
+
## Table: ${sheetName}
|
|
11758
|
+
${markdownTable}`);
|
|
11759
|
+
}
|
|
11760
|
+
if (matrixInfo.isComparisonMatrix && matrixInfo.entityIndices.length > 0) {
|
|
11761
|
+
const entityProfiles = this.generateEntityProfiles(fusedHeaders, dataRows, matrixInfo);
|
|
11762
|
+
if (entityProfiles) {
|
|
11763
|
+
sections.push(`
|
|
11764
|
+
## Entity Comparison Profiles
|
|
11765
|
+
${entityProfiles}`);
|
|
11766
|
+
}
|
|
11767
|
+
const criteriaComparisons = this.generateCriteriaComparisons(fusedHeaders, dataRows, matrixInfo);
|
|
11768
|
+
if (criteriaComparisons) {
|
|
11769
|
+
sections.push(`
|
|
11770
|
+
## Criteria Breakdown & Comparisons
|
|
11771
|
+
${criteriaComparisons}`);
|
|
11772
|
+
}
|
|
11773
|
+
} else {
|
|
11774
|
+
const contextualRows = this.generateContextualRows(fusedHeaders, dataRows);
|
|
11775
|
+
if (contextualRows) {
|
|
11776
|
+
sections.push(`
|
|
11777
|
+
## Structured Records
|
|
11778
|
+
${contextualRows}`);
|
|
11779
|
+
}
|
|
11780
|
+
}
|
|
11781
|
+
return sections.join("\n\n");
|
|
11782
|
+
}
|
|
11783
|
+
/**
|
|
11784
|
+
* Parse a raw CSV / TSV text string into tri-perspective contextual format.
|
|
11785
|
+
*/
|
|
11786
|
+
static parseDelimitedText(text, title = "Table", delimiter) {
|
|
11787
|
+
if (!text || !text.trim()) return "";
|
|
11788
|
+
if (!delimiter) {
|
|
11789
|
+
const firstLine = text.split(/\r?\n/)[0] || "";
|
|
11790
|
+
const commaCount = (firstLine.match(/,/g) || []).length;
|
|
11791
|
+
const tabCount = (firstLine.match(/\t/g) || []).length;
|
|
11792
|
+
const semiCount = (firstLine.match(/;/g) || []).length;
|
|
11793
|
+
const pipeCount = (firstLine.match(/\|/g) || []).length;
|
|
11794
|
+
if (tabCount > commaCount && tabCount > semiCount) delimiter = " ";
|
|
11795
|
+
else if (semiCount > commaCount && semiCount > tabCount) delimiter = ";";
|
|
11796
|
+
else if (pipeCount > commaCount && pipeCount > semiCount) delimiter = "|";
|
|
11797
|
+
else delimiter = ",";
|
|
11798
|
+
}
|
|
11799
|
+
const lines = text.split(/\r?\n/);
|
|
11800
|
+
const grid = [];
|
|
11801
|
+
for (const line of lines) {
|
|
11802
|
+
if (!line.trim()) continue;
|
|
11803
|
+
const row = [];
|
|
11804
|
+
let inQuote = false;
|
|
11805
|
+
let current = "";
|
|
11806
|
+
for (let i = 0; i < line.length; i++) {
|
|
11807
|
+
const char = line[i];
|
|
11808
|
+
if (char === '"') {
|
|
11809
|
+
if (inQuote && line[i + 1] === '"') {
|
|
11810
|
+
current += '"';
|
|
11811
|
+
i++;
|
|
11812
|
+
} else {
|
|
11813
|
+
inQuote = !inQuote;
|
|
11814
|
+
}
|
|
11815
|
+
} else if (char === delimiter && !inQuote) {
|
|
11816
|
+
row.push(current.trim());
|
|
11817
|
+
current = "";
|
|
11818
|
+
} else {
|
|
11819
|
+
current += char;
|
|
11820
|
+
}
|
|
11821
|
+
}
|
|
11822
|
+
row.push(current.trim());
|
|
11823
|
+
grid.push(row);
|
|
11824
|
+
}
|
|
11825
|
+
return this.parseGrid(grid, title);
|
|
11826
|
+
}
|
|
11827
|
+
/**
|
|
11828
|
+
* Parse a SheetJS sheet object with complete merged cell and matrix intelligence.
|
|
11829
|
+
*/
|
|
11830
|
+
static parseSheet(sheet, sheetName, XLSX) {
|
|
11831
|
+
if (!sheet) return "";
|
|
11832
|
+
const grid = this.extractAndUnrollGrid(sheet, XLSX);
|
|
11833
|
+
if (!grid || grid.length === 0) {
|
|
11834
|
+
const csv = XLSX.utils.sheet_to_csv(sheet, { blankrows: false });
|
|
11835
|
+
return csv && csv.trim() ? `--- Sheet: ${sheetName} ---
|
|
11836
|
+
${csv.trim()}` : "";
|
|
11837
|
+
}
|
|
11838
|
+
return this.parseGrid(grid, sheetName);
|
|
11839
|
+
}
|
|
11840
|
+
/**
|
|
11841
|
+
* Unrolls SheetJS merged cell ranges (`!merges`) across the 2D grid
|
|
11842
|
+
* so no child cell is left blank.
|
|
11843
|
+
*/
|
|
11844
|
+
static extractAndUnrollGrid(sheet, XLSX) {
|
|
11845
|
+
var _a2;
|
|
11846
|
+
const rawGrid = XLSX.utils.sheet_to_json(sheet, { header: 1, blankrows: false, defval: "" });
|
|
11847
|
+
if (!rawGrid || rawGrid.length === 0) return [];
|
|
11848
|
+
const grid = rawGrid.map(
|
|
11849
|
+
(row) => (Array.isArray(row) ? row : []).map((cell) => cell !== null && cell !== void 0 ? String(cell).trim() : "")
|
|
11850
|
+
);
|
|
11851
|
+
const merges = sheet["!merges"] || [];
|
|
11852
|
+
for (const merge of merges) {
|
|
11853
|
+
const { s, e } = merge;
|
|
11854
|
+
const originVal = ((_a2 = grid[s.r]) == null ? void 0 : _a2[s.c]) || "";
|
|
11855
|
+
if (!originVal) continue;
|
|
11856
|
+
for (let r = s.r; r <= e.r; r++) {
|
|
11857
|
+
if (!grid[r]) grid[r] = [];
|
|
11858
|
+
for (let c = s.c; c <= e.c; c++) {
|
|
11859
|
+
if (!grid[r][c] || grid[r][c] === "") {
|
|
11860
|
+
grid[r][c] = originVal;
|
|
11861
|
+
}
|
|
11862
|
+
}
|
|
11863
|
+
}
|
|
11864
|
+
}
|
|
11865
|
+
return grid.filter((row) => row.some((cell) => cell.length > 0));
|
|
11866
|
+
}
|
|
11867
|
+
/**
|
|
11868
|
+
* Intelligently identify where header rows end and data rows begin.
|
|
11869
|
+
*/
|
|
11870
|
+
static splitHeaderAndDataRows(grid) {
|
|
11871
|
+
if (grid.length <= 1) {
|
|
11872
|
+
return { headerRows: grid, dataRows: [] };
|
|
11873
|
+
}
|
|
11874
|
+
let headerRowCount = 1;
|
|
11875
|
+
for (let r = 1; r < Math.min(4, grid.length); r++) {
|
|
11876
|
+
const row = grid[r];
|
|
11877
|
+
const nonEmptyCells = row.filter(Boolean);
|
|
11878
|
+
if (nonEmptyCells.length === 0) continue;
|
|
11879
|
+
const isDataRow = nonEmptyCells.some(
|
|
11880
|
+
(c) => /^-?\d+(?:\.\d+)?%?$/.test(c) || /^\$?[\d,.]+(?:\s*(?:mil|k|m|b|usd|eur))?$/i.test(c) || /^(yes|no|true|false|pass|fail|compliant|non-compliant)$/i.test(c)
|
|
11881
|
+
);
|
|
11882
|
+
const hasSubheaderLabels = nonEmptyCells.some(
|
|
11883
|
+
(c) => /\b(bidder|vendor|supplier|score|weight|criteria|item|specification|technical|commercial|rating|status|cost|price|qty|unit)\b/i.test(c)
|
|
11884
|
+
);
|
|
11885
|
+
if (!isDataRow && hasSubheaderLabels) {
|
|
11886
|
+
headerRowCount = r + 1;
|
|
11887
|
+
} else if (!isDataRow && r === 1 && nonEmptyCells.length >= 2) {
|
|
11888
|
+
headerRowCount = 2;
|
|
11889
|
+
} else {
|
|
11890
|
+
break;
|
|
11891
|
+
}
|
|
11892
|
+
}
|
|
11893
|
+
const headerRows = grid.slice(0, headerRowCount);
|
|
11894
|
+
const dataRows = grid.slice(headerRowCount);
|
|
11895
|
+
return { headerRows, dataRows };
|
|
11896
|
+
}
|
|
11897
|
+
/**
|
|
11898
|
+
* Fuses multi-tier header rows into composite paths (e.g. `Commercial > Payment Terms > Bidder A`).
|
|
11899
|
+
*/
|
|
11900
|
+
static fuseHeaders(headerRows) {
|
|
11901
|
+
var _a2, _b;
|
|
11902
|
+
if (headerRows.length === 0) return [];
|
|
11903
|
+
if (headerRows.length === 1) {
|
|
11904
|
+
return headerRows[0].map((h, i) => h || `Column_${i + 1}`);
|
|
11905
|
+
}
|
|
11906
|
+
const maxCols = Math.max(...headerRows.map((r) => r.length));
|
|
11907
|
+
const fused = [];
|
|
11908
|
+
for (let c = 0; c < maxCols; c++) {
|
|
11909
|
+
const components = [];
|
|
11910
|
+
for (let r = 0; r < headerRows.length; r++) {
|
|
11911
|
+
const val = (_b = (_a2 = headerRows[r]) == null ? void 0 : _a2[c]) == null ? void 0 : _b.trim();
|
|
11912
|
+
if (val && !components.includes(val)) {
|
|
11913
|
+
components.push(val);
|
|
11914
|
+
}
|
|
11915
|
+
}
|
|
11916
|
+
fused.push(components.join(" > ") || `Column_${c + 1}`);
|
|
11917
|
+
}
|
|
11918
|
+
return fused;
|
|
11919
|
+
}
|
|
11920
|
+
/**
|
|
11921
|
+
* Analyzes whether the table is a comparison matrix (columns = entities, rows = criteria).
|
|
11922
|
+
*/
|
|
11923
|
+
static analyzeMatrixStructure(headers, dataRows) {
|
|
11924
|
+
const entityPattern = /\b(bidder|vendor|supplier|candidate|option|company|contractor|partner|proposal|firm|provider|solution|tier|package)\b/i;
|
|
11925
|
+
const entityIndices = [];
|
|
11926
|
+
const entityNames = [];
|
|
11927
|
+
let criteriaColumnIndex = 0;
|
|
11928
|
+
headers.forEach((h, idx) => {
|
|
11929
|
+
if (entityPattern.test(h) || /\b(bidder\s*\d+|vendor\s*[a-z0-9]+|option\s*[a-z0-9]+)\b/i.test(h)) {
|
|
11930
|
+
entityIndices.push(idx);
|
|
11931
|
+
const parts = h.split(" > ");
|
|
11932
|
+
entityNames.push(parts[parts.length - 1]);
|
|
11933
|
+
} else if (/\b(criteria|criterion|description|specification|category|requirement|item|feature)\b/i.test(h)) {
|
|
11934
|
+
criteriaColumnIndex = idx;
|
|
11935
|
+
}
|
|
11936
|
+
});
|
|
11937
|
+
const isComparisonMatrix = entityIndices.length >= 2 || headers.length >= 3 && entityIndices.length >= 1;
|
|
11938
|
+
return {
|
|
11939
|
+
isComparisonMatrix,
|
|
11940
|
+
entityIndices: entityIndices.length > 0 ? entityIndices : headers.slice(1).map((_, i) => i + 1),
|
|
11941
|
+
entityNames: entityNames.length > 0 ? entityNames : headers.slice(1),
|
|
11942
|
+
criteriaColumnIndex
|
|
11943
|
+
};
|
|
11944
|
+
}
|
|
11945
|
+
/**
|
|
11946
|
+
* Perspective A: Clean Markdown Table
|
|
11947
|
+
*/
|
|
11948
|
+
static generateMarkdownTable(headers, dataRows) {
|
|
11949
|
+
if (headers.length === 0 || dataRows.length === 0) return "";
|
|
11950
|
+
const cleanHeaders = headers.map((h) => h.replace(/\|/g, "-"));
|
|
11951
|
+
const lines = [];
|
|
11952
|
+
lines.push(`| ${cleanHeaders.join(" | ")} |`);
|
|
11953
|
+
lines.push(`| ${cleanHeaders.map(() => "---").join(" | ")} |`);
|
|
11954
|
+
for (const row of dataRows) {
|
|
11955
|
+
const paddedRow = headers.map((_, i) => (row[i] || "").replace(/\|/g, "-"));
|
|
11956
|
+
if (paddedRow.some((cell) => cell.trim().length > 0)) {
|
|
11957
|
+
lines.push(`| ${paddedRow.join(" | ")} |`);
|
|
11958
|
+
}
|
|
11959
|
+
}
|
|
11960
|
+
return lines.join("\n");
|
|
11961
|
+
}
|
|
11962
|
+
/**
|
|
11963
|
+
* Perspective B: Entity-Centric Profiles (e.g. Bidder 1, Bidder 2 full summary)
|
|
11964
|
+
*/
|
|
11965
|
+
static generateEntityProfiles(headers, dataRows, matrixInfo) {
|
|
11966
|
+
const profiles = [];
|
|
11967
|
+
matrixInfo.entityIndices.forEach((colIdx, i) => {
|
|
11968
|
+
const entityName = matrixInfo.entityNames[i] || headers[colIdx] || `Entity ${i + 1}`;
|
|
11969
|
+
const lines = [`### Profile & Evaluation: ${entityName}`];
|
|
11970
|
+
for (const row of dataRows) {
|
|
11971
|
+
const criterion = row[matrixInfo.criteriaColumnIndex] || row[0] || "Specification";
|
|
11972
|
+
const val = row[colIdx];
|
|
11973
|
+
if (val !== void 0 && val !== null && String(val).trim() !== "") {
|
|
11974
|
+
lines.push(`- **${criterion}**: ${String(val).trim()}`);
|
|
11975
|
+
}
|
|
11976
|
+
}
|
|
11977
|
+
if (lines.length > 1) {
|
|
11978
|
+
profiles.push(lines.join("\n"));
|
|
11979
|
+
}
|
|
11980
|
+
});
|
|
11981
|
+
return profiles.join("\n\n");
|
|
11982
|
+
}
|
|
11983
|
+
/**
|
|
11984
|
+
* Perspective C: Criteria Cross-Comparisons (e.g. Commercial Bid comparison across all Bidders)
|
|
11985
|
+
*/
|
|
11986
|
+
static generateCriteriaComparisons(headers, dataRows, matrixInfo) {
|
|
11987
|
+
const comparisons = [];
|
|
11988
|
+
for (const row of dataRows) {
|
|
11989
|
+
const criterion = row[matrixInfo.criteriaColumnIndex] || row[0] || "Metric";
|
|
11990
|
+
const entityPairs = [];
|
|
11991
|
+
matrixInfo.entityIndices.forEach((colIdx, i) => {
|
|
11992
|
+
const entityName = matrixInfo.entityNames[i] || headers[colIdx] || `Entity ${i + 1}`;
|
|
11993
|
+
const val = row[colIdx];
|
|
11994
|
+
if (val !== void 0 && val !== null && String(val).trim() !== "") {
|
|
11995
|
+
entityPairs.push(`${entityName}: ${String(val).trim()}`);
|
|
11996
|
+
}
|
|
11997
|
+
});
|
|
11998
|
+
if (entityPairs.length > 0) {
|
|
11999
|
+
comparisons.push(`- **${criterion}** $\\rightarrow$ ${entityPairs.join(" | ")}`);
|
|
12000
|
+
}
|
|
12001
|
+
}
|
|
12002
|
+
return comparisons.join("\n");
|
|
12003
|
+
}
|
|
12004
|
+
/**
|
|
12005
|
+
* Standard contextual key-value rows for general tabular records.
|
|
12006
|
+
*/
|
|
12007
|
+
static generateContextualRows(headers, dataRows) {
|
|
12008
|
+
const lines = [];
|
|
12009
|
+
for (const row of dataRows) {
|
|
12010
|
+
const pairs = [];
|
|
12011
|
+
headers.forEach((h, i) => {
|
|
12012
|
+
const val = row[i];
|
|
12013
|
+
if (val !== void 0 && val !== null && String(val).trim() !== "") {
|
|
12014
|
+
pairs.push(`${h}: ${String(val).trim()}`);
|
|
12015
|
+
}
|
|
12016
|
+
});
|
|
12017
|
+
if (pairs.length > 0) {
|
|
12018
|
+
lines.push(pairs.join(" | "));
|
|
12019
|
+
}
|
|
12020
|
+
}
|
|
12021
|
+
return lines.join("\n");
|
|
12022
|
+
}
|
|
12023
|
+
};
|
|
12024
|
+
|
|
12025
|
+
// src/utils/VisionExtractor.ts
|
|
12026
|
+
var VisionExtractor = class {
|
|
12027
|
+
/**
|
|
12028
|
+
* Check if a buffer represents an image format.
|
|
12029
|
+
*/
|
|
12030
|
+
static isImageBuffer(buffer, fileName = "", mimeType = "") {
|
|
12031
|
+
var _a2;
|
|
12032
|
+
const ext = ((_a2 = fileName.split(".").pop()) == null ? void 0 : _a2.toLowerCase()) || "";
|
|
12033
|
+
if (["png", "jpg", "jpeg", "webp", "bmp", "tiff", "gif"].includes(ext)) {
|
|
12034
|
+
return true;
|
|
12035
|
+
}
|
|
12036
|
+
if (mimeType.startsWith("image/")) {
|
|
12037
|
+
return true;
|
|
12038
|
+
}
|
|
12039
|
+
if (buffer && buffer.length >= 4) {
|
|
12040
|
+
if (buffer[0] === 137 && buffer[1] === 80 && buffer[2] === 78 && buffer[3] === 71) return true;
|
|
12041
|
+
if (buffer[0] === 255 && buffer[1] === 216 && buffer[2] === 255) return true;
|
|
12042
|
+
if (buffer[0] === 82 && buffer[1] === 73 && buffer[2] === 70 && buffer[3] === 70) return true;
|
|
12043
|
+
}
|
|
12044
|
+
return false;
|
|
12045
|
+
}
|
|
12046
|
+
/**
|
|
12047
|
+
* Extract rich textual content, tables, and diagram descriptions from an image buffer using Vision LLM.
|
|
12048
|
+
*/
|
|
12049
|
+
static async extractFromImage(buffer, fileName = "document.png", options = {}) {
|
|
12050
|
+
var _a2, _b, _c, _d, _e, _f, _g2, _h, _i;
|
|
12051
|
+
const base64Image = buffer.toString("base64");
|
|
12052
|
+
const ext = ((_a2 = fileName.split(".").pop()) == null ? void 0 : _a2.toLowerCase()) || "png";
|
|
12053
|
+
const mime = ext === "jpg" || ext === "jpeg" ? "image/jpeg" : ext === "webp" ? "image/webp" : "image/png";
|
|
12054
|
+
const apiKey = options.apiKey || (typeof process !== "undefined" ? ((_b = process.env) == null ? void 0 : _b.OPENAI_API_KEY) || ((_c = process.env) == null ? void 0 : _c.GEMINI_API_KEY) || ((_d = process.env) == null ? void 0 : _d.ANTHROPIC_API_KEY) : void 0);
|
|
12055
|
+
if (!apiKey) {
|
|
12056
|
+
return `[Scanned Document / Image: ${fileName} - Size: ${(buffer.length / 1024).toFixed(1)} KB]`;
|
|
12057
|
+
}
|
|
12058
|
+
try {
|
|
12059
|
+
const response = await fetch("https://api.openai.com/v1/chat/completions", {
|
|
12060
|
+
method: "POST",
|
|
12061
|
+
headers: {
|
|
12062
|
+
"Authorization": `Bearer ${apiKey}`,
|
|
12063
|
+
"Content-Type": "application/json"
|
|
12064
|
+
},
|
|
12065
|
+
body: JSON.stringify({
|
|
12066
|
+
model: options.model || "gpt-4o-mini",
|
|
12067
|
+
messages: [
|
|
12068
|
+
{
|
|
12069
|
+
role: "system",
|
|
12070
|
+
content: "You are an expert Document OCR & Data Extraction engine. Transcribe all text, numbers, headings, tables, and diagram structures in this image into clean, structured GitHub Markdown. For tables, use standard markdown table syntax. Preserve exact numbers, dates, and column headers. Do not summarize \u2014 transcribe faithfully."
|
|
12071
|
+
},
|
|
12072
|
+
{
|
|
12073
|
+
role: "user",
|
|
12074
|
+
content: [
|
|
12075
|
+
{
|
|
12076
|
+
type: "text",
|
|
12077
|
+
text: `Please transcribe the content of this document/image (${fileName}):`
|
|
12078
|
+
},
|
|
12079
|
+
{
|
|
12080
|
+
type: "image_url",
|
|
12081
|
+
image_url: {
|
|
12082
|
+
url: `data:${mime};base64,${base64Image}`,
|
|
12083
|
+
detail: "high"
|
|
12084
|
+
}
|
|
12085
|
+
}
|
|
12086
|
+
]
|
|
12087
|
+
}
|
|
12088
|
+
],
|
|
12089
|
+
max_tokens: options.maxTokens || 2048,
|
|
12090
|
+
temperature: (_e = options.temperature) != null ? _e : 0.1
|
|
12091
|
+
})
|
|
12092
|
+
});
|
|
12093
|
+
if (response.ok) {
|
|
12094
|
+
const data = await response.json();
|
|
12095
|
+
const text = (_i = (_h = (_g2 = (_f = data.choices) == null ? void 0 : _f[0]) == null ? void 0 : _g2.message) == null ? void 0 : _h.content) == null ? void 0 : _i.trim();
|
|
12096
|
+
if (text) {
|
|
12097
|
+
return `--- Document / Image OCR: ${fileName} ---
|
|
12098
|
+
|
|
12099
|
+
${text}`;
|
|
12100
|
+
}
|
|
12101
|
+
}
|
|
12102
|
+
} catch (err) {
|
|
12103
|
+
console.warn("[VisionExtractor] OpenAI Vision OCR failed:", err);
|
|
12104
|
+
}
|
|
12105
|
+
return `[Scanned Image Document: ${fileName}]`;
|
|
12106
|
+
}
|
|
12107
|
+
};
|
|
12108
|
+
|
|
11198
12109
|
// src/utils/DocumentParser.ts
|
|
11199
12110
|
var DocumentParser = class {
|
|
11200
12111
|
/**
|
|
@@ -11218,52 +12129,11 @@ var DocumentParser = class {
|
|
|
11218
12129
|
return "text";
|
|
11219
12130
|
}
|
|
11220
12131
|
/**
|
|
11221
|
-
* Convert spreadsheet sheets into high-fidelity, context-preserved tabular text.
|
|
11222
|
-
*
|
|
12132
|
+
* Convert spreadsheet sheets into high-fidelity, context-preserved tabular text via UniversalTableParser.
|
|
12133
|
+
* Unrolls merged cells, fuses multi-level headers, and generates tri-perspective comparison profiles.
|
|
11223
12134
|
*/
|
|
11224
12135
|
static formatSheetToContextualText(sheet, sheetName, XLSX) {
|
|
11225
|
-
|
|
11226
|
-
if (!jsonRows || jsonRows.length === 0) {
|
|
11227
|
-
const csv = XLSX.utils.sheet_to_csv(sheet, { blankrows: false });
|
|
11228
|
-
return csv && csv.trim() ? `--- Sheet: ${sheetName} ---
|
|
11229
|
-
${csv.trim()}` : "";
|
|
11230
|
-
}
|
|
11231
|
-
const lines = [`--- Sheet: ${sheetName} ---`];
|
|
11232
|
-
let currentHeader = [];
|
|
11233
|
-
for (let r = 0; r < jsonRows.length; r++) {
|
|
11234
|
-
const row = jsonRows[r];
|
|
11235
|
-
if (!row || row.length === 0) continue;
|
|
11236
|
-
const cells = row.map((c) => c !== null && c !== void 0 ? String(c).trim() : "").filter(Boolean);
|
|
11237
|
-
if (cells.length === 0) continue;
|
|
11238
|
-
if (cells.length === 1 || cells.length <= 2 && cells[0].toUpperCase().includes("STEP")) {
|
|
11239
|
-
lines.push(`
|
|
11240
|
-
## ${cells.join(" ")}`);
|
|
11241
|
-
continue;
|
|
11242
|
-
}
|
|
11243
|
-
const isHeaderCandidate = cells.every((c) => isNaN(Number(c)) && !/^\$?[\d,.]+/.test(c));
|
|
11244
|
-
if (isHeaderCandidate && cells.length >= 2) {
|
|
11245
|
-
currentHeader = row.map((c) => c !== null && c !== void 0 ? String(c).trim() : "");
|
|
11246
|
-
lines.push(`| ${cells.join(" | ")} |`);
|
|
11247
|
-
lines.push(`| ${cells.map(() => "---").join(" | ")} |`);
|
|
11248
|
-
continue;
|
|
11249
|
-
}
|
|
11250
|
-
if (currentHeader.length > 0 && cells.length > 1) {
|
|
11251
|
-
const fieldPairs = [];
|
|
11252
|
-
for (let c = 0; c < row.length; c++) {
|
|
11253
|
-
const h = currentHeader[c];
|
|
11254
|
-
const val = row[c];
|
|
11255
|
-
if (h && val !== null && val !== void 0 && String(val).trim() !== "") {
|
|
11256
|
-
fieldPairs.push(`${h}: ${String(val).trim()}`);
|
|
11257
|
-
}
|
|
11258
|
-
}
|
|
11259
|
-
if (fieldPairs.length > 0) {
|
|
11260
|
-
lines.push(fieldPairs.join(" | "));
|
|
11261
|
-
continue;
|
|
11262
|
-
}
|
|
11263
|
-
}
|
|
11264
|
-
lines.push(cells.join(", "));
|
|
11265
|
-
}
|
|
11266
|
-
return lines.join("\n");
|
|
12136
|
+
return UniversalTableParser.parseSheet(sheet, sheetName, XLSX);
|
|
11267
12137
|
}
|
|
11268
12138
|
/**
|
|
11269
12139
|
* Extract text from a File or Buffer based on its type.
|
|
@@ -11287,7 +12157,9 @@ ${csv.trim()}` : "";
|
|
|
11287
12157
|
return this.readAsText(file);
|
|
11288
12158
|
}
|
|
11289
12159
|
if (extension === "csv" || extension === "tsv" || mimeType === "text/csv" || mimeType === "text/tab-separated-values") {
|
|
11290
|
-
|
|
12160
|
+
const text = await this.readAsText(file);
|
|
12161
|
+
const parsed = UniversalTableParser.parseDelimitedText(text, fileName);
|
|
12162
|
+
return parsed || text;
|
|
11291
12163
|
}
|
|
11292
12164
|
if (extension === "pptx" || extension === "ppt" || extension === "odp" || mimeType.includes("presentation") || mimeType.includes("powerpoint")) {
|
|
11293
12165
|
try {
|
|
@@ -11394,16 +12266,22 @@ ${csv.trim()}`);
|
|
|
11394
12266
|
return this.parseLegacyExcelOrFallback(buffer, fileName);
|
|
11395
12267
|
}
|
|
11396
12268
|
if (extension === "pdf" || mimeType === "application/pdf") {
|
|
12269
|
+
const buffer = Buffer.isBuffer(file) ? file : Buffer.from(await file.arrayBuffer());
|
|
11397
12270
|
try {
|
|
11398
12271
|
const pdfModule = await import("pdf-parse");
|
|
11399
|
-
const buffer = Buffer.isBuffer(file) ? file : Buffer.from(await file.arrayBuffer());
|
|
11400
12272
|
const pdf = pdfModule.default || pdfModule;
|
|
11401
12273
|
const data = await pdf(buffer);
|
|
11402
|
-
|
|
12274
|
+
if (data.text && data.text.trim().length > 30) {
|
|
12275
|
+
return data.text;
|
|
12276
|
+
}
|
|
11403
12277
|
} catch (err) {
|
|
11404
|
-
console.warn(
|
|
11405
|
-
return `[PDF Parsing Error for ${fileName}]`;
|
|
12278
|
+
console.warn("[DocumentParser] pdf-parse failed, attempting Vision OCR fallback:", err);
|
|
11406
12279
|
}
|
|
12280
|
+
return VisionExtractor.extractFromImage(buffer, fileName);
|
|
12281
|
+
}
|
|
12282
|
+
if (["png", "jpg", "jpeg", "webp", "bmp", "tiff", "gif"].includes(extension) || mimeType.startsWith("image/")) {
|
|
12283
|
+
const buffer = Buffer.isBuffer(file) ? file : Buffer.from(await file.arrayBuffer());
|
|
12284
|
+
return VisionExtractor.extractFromImage(buffer, fileName);
|
|
11407
12285
|
}
|
|
11408
12286
|
if (extension === "docx" || extension === "doc" || extension === "odt" || extension === "rtf" || mimeType === "application/vnd.openxmlformats-officedocument.wordprocessingml.document" || mimeType === "application/msword" || mimeType === "application/rtf") {
|
|
11409
12287
|
try {
|
|
@@ -11761,44 +12639,38 @@ var _DatabaseStorage = class _DatabaseStorage {
|
|
|
11761
12639
|
*
|
|
11762
12640
|
* In non-production environments this is a complete no-op so local
|
|
11763
12641
|
* development works without a license key.
|
|
12642
|
+
/**
|
|
12643
|
+
* Checks if the caller holds a valid paid-tier license in production.
|
|
12644
|
+
* Returns true if allowed, false if not supported on current tier.
|
|
11764
12645
|
*/
|
|
11765
|
-
async
|
|
11766
|
-
if (process.env.NODE_ENV !== "production") return;
|
|
11767
|
-
if (
|
|
11768
|
-
if (
|
|
11769
|
-
|
|
11770
|
-
"[Retrivora SDK] Chat History and Feedback features require a paid license key in production."
|
|
11771
|
-
);
|
|
12646
|
+
async isPaidSubscription() {
|
|
12647
|
+
if (process.env.NODE_ENV !== "production") return true;
|
|
12648
|
+
if (!this.config.licenseKey) return false;
|
|
12649
|
+
if (Date.now() - this.subscriptionCheckedAt < _DatabaseStorage.SUBSCRIPTION_CACHE_TTL_MS) {
|
|
12650
|
+
return true;
|
|
11772
12651
|
}
|
|
11773
12652
|
try {
|
|
11774
12653
|
const payload = await LicenseVerifier.verifyAsync(this.config.licenseKey, this.config.projectId);
|
|
11775
12654
|
const tier = (payload.tier || "").toLowerCase();
|
|
11776
|
-
const PAID_TIERS = /* @__PURE__ */ new Set(["premium", "enterprise", "growth", "pro", "developer_pro"]);
|
|
11777
|
-
|
|
11778
|
-
|
|
11779
|
-
|
|
11780
|
-
);
|
|
12655
|
+
const PAID_TIERS = /* @__PURE__ */ new Set(["premium", "enterprise", "growth", "pro", "developer_pro", "free_trial"]);
|
|
12656
|
+
const isAllowed = PAID_TIERS.has(tier) || tier.includes("pro") || tier.includes("enterprise");
|
|
12657
|
+
if (isAllowed) {
|
|
12658
|
+
this.subscriptionCheckedAt = Date.now();
|
|
11781
12659
|
}
|
|
11782
|
-
|
|
11783
|
-
} catch (
|
|
11784
|
-
|
|
11785
|
-
`[Retrivora SDK] Subscription check failed: ${err instanceof Error ? err.message : String(err)}`
|
|
11786
|
-
);
|
|
12660
|
+
return isAllowed;
|
|
12661
|
+
} catch (e) {
|
|
12662
|
+
return false;
|
|
11787
12663
|
}
|
|
11788
12664
|
}
|
|
11789
|
-
async
|
|
12665
|
+
async isHistoryAvailable() {
|
|
11790
12666
|
var _a2, _b;
|
|
11791
|
-
|
|
11792
|
-
|
|
11793
|
-
throw new Error("[Retrivora SDK] Chat History is disabled in RagConfig.");
|
|
11794
|
-
}
|
|
12667
|
+
if (((_b = (_a2 = this.config.rag) == null ? void 0 : _a2.history) == null ? void 0 : _b.enabled) === false) return false;
|
|
12668
|
+
return this.isPaidSubscription();
|
|
11795
12669
|
}
|
|
11796
|
-
async
|
|
12670
|
+
async isFeedbackAvailable() {
|
|
11797
12671
|
var _a2, _b;
|
|
11798
|
-
|
|
11799
|
-
|
|
11800
|
-
throw new Error("[Retrivora SDK] User Feedback is disabled in RagConfig.");
|
|
11801
|
-
}
|
|
12672
|
+
if (((_b = (_a2 = this.config.rag) == null ? void 0 : _a2.feedback) == null ? void 0 : _b.enabled) === false) return false;
|
|
12673
|
+
return this.isPaidSubscription();
|
|
11802
12674
|
}
|
|
11803
12675
|
/**
|
|
11804
12676
|
* Returns (or lazily creates) the PostgreSQL connection pool for this project.
|
|
@@ -11912,7 +12784,7 @@ var _DatabaseStorage = class _DatabaseStorage {
|
|
|
11912
12784
|
}
|
|
11913
12785
|
// ─── Message History Operations ──────────────────────────────────────────
|
|
11914
12786
|
async saveMessage(sessionId, message) {
|
|
11915
|
-
await this.
|
|
12787
|
+
if (!await this.isHistoryAvailable()) return;
|
|
11916
12788
|
const createdAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
11917
12789
|
const msgId = message.id || `msg_${Date.now()}_${Math.random().toString(36).slice(2, 6)}`;
|
|
11918
12790
|
if (this.provider === "postgresql" || this.provider === "pgvector") {
|
|
@@ -11975,7 +12847,7 @@ var _DatabaseStorage = class _DatabaseStorage {
|
|
|
11975
12847
|
}
|
|
11976
12848
|
}
|
|
11977
12849
|
async getHistory(sessionId, projectIds) {
|
|
11978
|
-
await this.
|
|
12850
|
+
if (!await this.isHistoryAvailable()) return [];
|
|
11979
12851
|
if (projectIds && projectIds.length > 0 && !this.sessionInScope(sessionId, projectIds)) {
|
|
11980
12852
|
return [];
|
|
11981
12853
|
}
|
|
@@ -12009,7 +12881,7 @@ var _DatabaseStorage = class _DatabaseStorage {
|
|
|
12009
12881
|
}
|
|
12010
12882
|
}
|
|
12011
12883
|
async clearHistory(sessionId, projectIds) {
|
|
12012
|
-
await this.
|
|
12884
|
+
if (!await this.isHistoryAvailable()) return;
|
|
12013
12885
|
if (projectIds && projectIds.length > 0 && !this.sessionInScope(sessionId, projectIds)) {
|
|
12014
12886
|
return;
|
|
12015
12887
|
}
|
|
@@ -12049,7 +12921,7 @@ var _DatabaseStorage = class _DatabaseStorage {
|
|
|
12049
12921
|
});
|
|
12050
12922
|
}
|
|
12051
12923
|
async listSessions(projectIds) {
|
|
12052
|
-
await this.
|
|
12924
|
+
if (!await this.isHistoryAvailable()) return [];
|
|
12053
12925
|
let raw = [];
|
|
12054
12926
|
if (this.provider === "postgresql" || this.provider === "pgvector") {
|
|
12055
12927
|
const pool = await this.getPGPool();
|
|
@@ -12068,7 +12940,7 @@ var _DatabaseStorage = class _DatabaseStorage {
|
|
|
12068
12940
|
}
|
|
12069
12941
|
// ─── Feedback Operations ──────────────────────────────────────────────────
|
|
12070
12942
|
async saveFeedback(feedback) {
|
|
12071
|
-
await this.
|
|
12943
|
+
if (!await this.isFeedbackAvailable()) return;
|
|
12072
12944
|
const id = `fb_${Date.now()}_${Math.random().toString(36).slice(2, 6)}`;
|
|
12073
12945
|
const createdAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
12074
12946
|
if (this.provider === "postgresql" || this.provider === "pgvector") {
|
|
@@ -12118,7 +12990,7 @@ var _DatabaseStorage = class _DatabaseStorage {
|
|
|
12118
12990
|
}
|
|
12119
12991
|
}
|
|
12120
12992
|
async getFeedback(messageId, projectIds) {
|
|
12121
|
-
await this.
|
|
12993
|
+
if (!await this.isFeedbackAvailable()) return null;
|
|
12122
12994
|
let item = null;
|
|
12123
12995
|
if (this.provider === "postgresql" || this.provider === "pgvector") {
|
|
12124
12996
|
const pool = await this.getPGPool();
|
|
@@ -12154,7 +13026,7 @@ var _DatabaseStorage = class _DatabaseStorage {
|
|
|
12154
13026
|
return item;
|
|
12155
13027
|
}
|
|
12156
13028
|
async listFeedback(projectIds) {
|
|
12157
|
-
await this.
|
|
13029
|
+
if (!await this.isFeedbackAvailable()) return [];
|
|
12158
13030
|
let raw = [];
|
|
12159
13031
|
if (this.provider === "postgresql" || this.provider === "pgvector") {
|
|
12160
13032
|
const pool = await this.getPGPool();
|
|
@@ -14111,6 +14983,7 @@ async function createRetrivora(options) {
|
|
|
14111
14983
|
EmbeddingStrategyResolver,
|
|
14112
14984
|
EventType,
|
|
14113
14985
|
GroqProvider,
|
|
14986
|
+
HybridSearch,
|
|
14114
14987
|
IntentClassifier,
|
|
14115
14988
|
LLMFactory,
|
|
14116
14989
|
LLM_PROFILES,
|
|
@@ -14125,19 +14998,23 @@ async function createRetrivora(options) {
|
|
|
14125
14998
|
MultiTablePostgresProvider,
|
|
14126
14999
|
OllamaProvider,
|
|
14127
15000
|
OpenAIProvider,
|
|
15001
|
+
PIIMasker,
|
|
14128
15002
|
PRESETS,
|
|
14129
15003
|
PineconeProvider,
|
|
14130
15004
|
Pipeline,
|
|
14131
15005
|
PostgreSQLProvider,
|
|
15006
|
+
PromptGuard,
|
|
14132
15007
|
PromptOptimizer,
|
|
14133
15008
|
ProviderHealthCheck,
|
|
14134
15009
|
ProviderNotFoundException,
|
|
14135
15010
|
ProviderRegistry,
|
|
14136
15011
|
QdrantProvider,
|
|
14137
15012
|
QwenProvider,
|
|
15013
|
+
RAGEvaluator,
|
|
14138
15014
|
RateLimitException,
|
|
14139
15015
|
RedisProvider,
|
|
14140
15016
|
RendererRegistry,
|
|
15017
|
+
Reranker,
|
|
14141
15018
|
RetrievalException,
|
|
14142
15019
|
Retrivora,
|
|
14143
15020
|
RetrivoraError,
|
|
@@ -14155,9 +15032,11 @@ async function createRetrivora(options) {
|
|
|
14155
15032
|
TelemetryService,
|
|
14156
15033
|
TextRendererStrategy,
|
|
14157
15034
|
UniversalLLMAdapter,
|
|
15035
|
+
UniversalTableParser,
|
|
14158
15036
|
UniversalVectorProvider,
|
|
14159
15037
|
VECTOR_PROFILES,
|
|
14160
15038
|
VectorPlugin,
|
|
15039
|
+
VisionExtractor,
|
|
14161
15040
|
VisualizationDecisionEngine,
|
|
14162
15041
|
WeaviateProvider,
|
|
14163
15042
|
createChatHandler,
|