@retrivora-ai/rag-engine 2.5.9 → 2.6.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +8 -6
- package/dist/{ILLMProvider-B2N3lMBc.d.mts → ILLMProvider-CzXcTA6d.d.mts} +31 -3
- package/dist/{ILLMProvider-B2N3lMBc.d.ts → ILLMProvider-CzXcTA6d.d.ts} +31 -3
- package/dist/handlers/index.d.mts +2 -2
- package/dist/handlers/index.d.ts +2 -2
- package/dist/handlers/index.js +9815 -8601
- package/dist/handlers/index.mjs +9818 -8612
- package/dist/{index-CAGGPyI7.d.mts → index-BST9Gzsv.d.mts} +8 -7
- package/dist/{index-BxzDv2jd.d.ts → index-CFl4KrKq.d.ts} +58 -46
- package/dist/{index-37V810a-.d.mts → index-D7FuQRSb.d.mts} +58 -46
- package/dist/{index-DdQxoIpu.d.ts → index-qLvtMUr-.d.ts} +8 -7
- package/dist/index.css +23 -0
- package/dist/index.d.mts +4 -4
- package/dist/index.d.ts +4 -4
- package/dist/index.js +267 -187
- package/dist/index.min.css +1 -1
- package/dist/index.mjs +268 -193
- package/dist/server.d.mts +322 -14
- package/dist/server.d.ts +322 -14
- package/dist/server.js +1925 -688
- package/dist/server.mjs +1912 -693
- package/package.json +1 -1
package/dist/server.js
CHANGED
|
@@ -332,7 +332,7 @@ var init_package = __esm({
|
|
|
332
332
|
"package.json"() {
|
|
333
333
|
package_default = {
|
|
334
334
|
name: "@retrivora-ai/rag-engine",
|
|
335
|
-
version: "2.
|
|
335
|
+
version: "2.6.2",
|
|
336
336
|
description: "Retrivora AI is a plug-and-play AI engine for RAG chat experiences \u2014 generic vector DB + LLM provider, embeddable or standalone.",
|
|
337
337
|
author: "Abhinav Alkuchi",
|
|
338
338
|
license: "UNLICENSED",
|
|
@@ -511,7 +511,7 @@ var init_ConfigFetcher = __esm({
|
|
|
511
511
|
init_ProjectSettings();
|
|
512
512
|
init_version();
|
|
513
513
|
CONTROL_PLANE_URL = "https://www.retrivora.com";
|
|
514
|
-
LLM_PROXY_BASE_URL = "https://
|
|
514
|
+
LLM_PROXY_BASE_URL = "https://llm.retrivora.com/api/v1";
|
|
515
515
|
ConfigFetcher = class {
|
|
516
516
|
// 5-minute in-memory session cache
|
|
517
517
|
/**
|
|
@@ -2729,9 +2729,11 @@ __export(server_exports, {
|
|
|
2729
2729
|
EmbeddingStrategyResolver: () => EmbeddingStrategyResolver,
|
|
2730
2730
|
EventType: () => EventType,
|
|
2731
2731
|
GroqProvider: () => GroqProvider,
|
|
2732
|
+
HybridSearch: () => HybridSearch,
|
|
2732
2733
|
IntentClassifier: () => IntentClassifier,
|
|
2733
2734
|
LLMFactory: () => LLMFactory,
|
|
2734
2735
|
LLM_PROFILES: () => LLM_PROFILES,
|
|
2736
|
+
LangChainAgent: () => LangChainAgent,
|
|
2735
2737
|
LicenseConfigResolver: () => LicenseConfigResolver,
|
|
2736
2738
|
LicenseValidationError: () => LicenseValidationError,
|
|
2737
2739
|
LicenseValidator: () => LicenseValidator,
|
|
@@ -2743,19 +2745,23 @@ __export(server_exports, {
|
|
|
2743
2745
|
MultiTablePostgresProvider: () => MultiTablePostgresProvider,
|
|
2744
2746
|
OllamaProvider: () => OllamaProvider,
|
|
2745
2747
|
OpenAIProvider: () => OpenAIProvider,
|
|
2748
|
+
PIIMasker: () => PIIMasker,
|
|
2746
2749
|
PRESETS: () => PRESETS,
|
|
2747
2750
|
PineconeProvider: () => PineconeProvider,
|
|
2748
2751
|
Pipeline: () => Pipeline,
|
|
2749
2752
|
PostgreSQLProvider: () => PostgreSQLProvider,
|
|
2753
|
+
PromptGuard: () => PromptGuard,
|
|
2750
2754
|
PromptOptimizer: () => PromptOptimizer,
|
|
2751
2755
|
ProviderHealthCheck: () => ProviderHealthCheck,
|
|
2752
2756
|
ProviderNotFoundException: () => ProviderNotFoundException,
|
|
2753
2757
|
ProviderRegistry: () => ProviderRegistry,
|
|
2754
2758
|
QdrantProvider: () => QdrantProvider,
|
|
2755
2759
|
QwenProvider: () => QwenProvider,
|
|
2760
|
+
RAGEvaluator: () => RAGEvaluator,
|
|
2756
2761
|
RateLimitException: () => RateLimitException,
|
|
2757
2762
|
RedisProvider: () => RedisProvider,
|
|
2758
2763
|
RendererRegistry: () => RendererRegistry,
|
|
2764
|
+
Reranker: () => Reranker,
|
|
2759
2765
|
RetrievalException: () => RetrievalException,
|
|
2760
2766
|
Retrivora: () => Retrivora,
|
|
2761
2767
|
RetrivoraError: () => RetrivoraError,
|
|
@@ -2773,9 +2779,11 @@ __export(server_exports, {
|
|
|
2773
2779
|
TelemetryService: () => TelemetryService,
|
|
2774
2780
|
TextRendererStrategy: () => TextRendererStrategy,
|
|
2775
2781
|
UniversalLLMAdapter: () => UniversalLLMAdapter,
|
|
2782
|
+
UniversalTableParser: () => UniversalTableParser,
|
|
2776
2783
|
UniversalVectorProvider: () => UniversalVectorProvider,
|
|
2777
2784
|
VECTOR_PROFILES: () => VECTOR_PROFILES,
|
|
2778
2785
|
VectorPlugin: () => VectorPlugin,
|
|
2786
|
+
VisionExtractor: () => VisionExtractor,
|
|
2779
2787
|
VisualizationDecisionEngine: () => VisualizationDecisionEngine,
|
|
2780
2788
|
WeaviateProvider: () => WeaviateProvider,
|
|
2781
2789
|
createChatHandler: () => createChatHandler,
|
|
@@ -2785,10 +2793,12 @@ __export(server_exports, {
|
|
|
2785
2793
|
createHealthHandler: () => createHealthHandler,
|
|
2786
2794
|
createHistoryHandler: () => createHistoryHandler,
|
|
2787
2795
|
createIngestHandler: () => createIngestHandler,
|
|
2796
|
+
createLicenseHandler: () => createLicenseHandler,
|
|
2788
2797
|
createRagHandler: () => createRagHandler,
|
|
2789
2798
|
createRetrivora: () => createRetrivora,
|
|
2790
2799
|
createSessionsHandler: () => createSessionsHandler,
|
|
2791
2800
|
createStreamHandler: () => createStreamHandler,
|
|
2801
|
+
createSuggestionsHandler: () => createSuggestionsHandler,
|
|
2792
2802
|
createUploadHandler: () => createUploadHandler,
|
|
2793
2803
|
decideVisualization: () => decideVisualization,
|
|
2794
2804
|
getRagConfig: () => getRagConfig,
|
|
@@ -2802,7 +2812,9 @@ __export(server_exports, {
|
|
|
2802
2812
|
sseErrorFrame: () => sseErrorFrame,
|
|
2803
2813
|
sseFrame: () => sseFrame,
|
|
2804
2814
|
sseMetaFrame: () => sseMetaFrame,
|
|
2815
|
+
sseObservabilityFrame: () => sseObservabilityFrame,
|
|
2805
2816
|
sseTextFrame: () => sseTextFrame,
|
|
2817
|
+
sseUIFrame: () => sseUIFrame,
|
|
2806
2818
|
wrapError: () => wrapError
|
|
2807
2819
|
});
|
|
2808
2820
|
module.exports = __toCommonJS(server_exports);
|
|
@@ -2868,13 +2880,14 @@ function wrapError(err, defaultCode, defaultMessage) {
|
|
|
2868
2880
|
const message = (error == null ? void 0 : error.message) || defaultMessage || String(err);
|
|
2869
2881
|
const status = (error == null ? void 0 : error.status) || (error == null ? void 0 : error.statusCode) || ((_a2 = error == null ? void 0 : error.response) == null ? void 0 : _a2.status);
|
|
2870
2882
|
const code = error == null ? void 0 : error.code;
|
|
2871
|
-
if (status === 429 || /rate[- ]?limit/i.test(message) || code === "RATE_LIMIT_EXCEEDED") {
|
|
2883
|
+
if (status === 429 || /rate[- ]?limit|token limit|too many requests|tokens per minute/i.test(message) || code === "RATE_LIMIT_EXCEEDED") {
|
|
2872
2884
|
return new RateLimitException(message, err);
|
|
2873
2885
|
}
|
|
2874
|
-
if (
|
|
2875
|
-
|
|
2876
|
-
|
|
2877
|
-
|
|
2886
|
+
if (code === "LICENSE_DELETED" || code === "LICENSE_INACTIVE" || code === "LICENSE_REVOKED" || /license key has been terminated|license is revoked|license is suspended|license is inactive/i.test(message)) {
|
|
2887
|
+
return new AuthenticationException("Your Retrivora license key has been terminated, suspended, or revoked. Access denied.", err);
|
|
2888
|
+
}
|
|
2889
|
+
if (status === 401 || status === 403 || code === "INVALID_API_KEY" || code === "AUTHENTICATION_ERROR" || /unauthorized|forbidden|invalid api key/i.test(message)) {
|
|
2890
|
+
return new AuthenticationException(message, err);
|
|
2878
2891
|
}
|
|
2879
2892
|
switch (defaultCode) {
|
|
2880
2893
|
case "PROVIDER_NOT_FOUND":
|
|
@@ -2895,6 +2908,20 @@ function wrapError(err, defaultCode, defaultMessage) {
|
|
|
2895
2908
|
}
|
|
2896
2909
|
|
|
2897
2910
|
// src/core/LicenseVerifier.ts
|
|
2911
|
+
function decodeBase64UrlToUtf8(b64url) {
|
|
2912
|
+
const normalized = b64url.replace(/-/g, "+").replace(/_/g, "/");
|
|
2913
|
+
const padded = normalized + "=".repeat((4 - normalized.length % 4) % 4);
|
|
2914
|
+
if (typeof Buffer !== "undefined" && typeof Buffer.from === "function") {
|
|
2915
|
+
return Buffer.from(padded, "base64").toString("utf8");
|
|
2916
|
+
}
|
|
2917
|
+
if (typeof atob === "function") {
|
|
2918
|
+
const binary = atob(padded);
|
|
2919
|
+
const bytes = new Uint8Array(binary.length);
|
|
2920
|
+
for (let i = 0; i < binary.length; i++) bytes[i] = binary.charCodeAt(i);
|
|
2921
|
+
return new TextDecoder().decode(bytes);
|
|
2922
|
+
}
|
|
2923
|
+
throw new Error("Base64 decoding is not supported in this environment.");
|
|
2924
|
+
}
|
|
2898
2925
|
var _LicenseVerifier = class _LicenseVerifier {
|
|
2899
2926
|
// Cache verified license for 5 minutes
|
|
2900
2927
|
/**
|
|
@@ -3026,6 +3053,9 @@ var _LicenseVerifier = class _LicenseVerifier {
|
|
|
3026
3053
|
minimumSupportedVersion: data.minimumSupportedVersion,
|
|
3027
3054
|
forceUpgrade: data.forceUpgrade
|
|
3028
3055
|
};
|
|
3056
|
+
if (payload.expiresAt && Math.floor(Date.now() / 1e3) > payload.expiresAt) {
|
|
3057
|
+
throw new Error("License key has expired.");
|
|
3058
|
+
}
|
|
3029
3059
|
if (isProduction && provider) {
|
|
3030
3060
|
const normalizedProvider = provider.toLowerCase();
|
|
3031
3061
|
if (tier === "hobby") {
|
|
@@ -3064,63 +3094,67 @@ var _LicenseVerifier = class _LicenseVerifier {
|
|
|
3064
3094
|
}
|
|
3065
3095
|
}
|
|
3066
3096
|
static async verifyIP(licenseKey) {
|
|
3097
|
+
var _a2, _b;
|
|
3067
3098
|
if (!licenseKey) return;
|
|
3068
3099
|
try {
|
|
3069
3100
|
const parts = licenseKey.split(".");
|
|
3070
3101
|
if (parts.length !== 3) return;
|
|
3071
3102
|
const [, payloadB64] = parts;
|
|
3072
3103
|
const payload = JSON.parse(
|
|
3073
|
-
|
|
3104
|
+
decodeBase64UrlToUtf8(payloadB64)
|
|
3074
3105
|
);
|
|
3075
3106
|
const tier = (payload.tier || "").toLowerCase();
|
|
3076
|
-
if (tier === "enterprise" || tier === "custom") {
|
|
3107
|
+
if (tier === "enterprise" || tier === "custom" || !payload.ipv4 && !payload.ipv6) {
|
|
3077
3108
|
return;
|
|
3078
3109
|
}
|
|
3079
|
-
|
|
3080
|
-
|
|
3081
|
-
|
|
3082
|
-
|
|
3083
|
-
|
|
3084
|
-
|
|
3085
|
-
|
|
3086
|
-
|
|
3110
|
+
let currentIpv4 = "";
|
|
3111
|
+
let currentIpv6 = "";
|
|
3112
|
+
const now = Date.now();
|
|
3113
|
+
if (this.cachedIps && now - this.cachedIps.fetchedAt < 10 * 60 * 1e3) {
|
|
3114
|
+
currentIpv4 = this.cachedIps.ipv4;
|
|
3115
|
+
currentIpv6 = this.cachedIps.ipv6;
|
|
3116
|
+
} else {
|
|
3117
|
+
const [v4Result, v6Result] = await Promise.allSettled([
|
|
3118
|
+
fetch("https://api4.ipify.org?format=json", { signal: AbortSignal.timeout(2e3) }).then((r) => r.json()),
|
|
3119
|
+
fetch("https://api6.ipify.org?format=json", { signal: AbortSignal.timeout(2e3) }).then((r) => r.json())
|
|
3120
|
+
]);
|
|
3121
|
+
if (v4Result.status === "fulfilled" && ((_a2 = v4Result.value) == null ? void 0 : _a2.ip)) {
|
|
3122
|
+
currentIpv4 = v4Result.value.ip;
|
|
3087
3123
|
}
|
|
3088
|
-
|
|
3089
|
-
|
|
3090
|
-
const data = await res.json();
|
|
3091
|
-
currentIpv6 = data.ip;
|
|
3092
|
-
} catch (e) {
|
|
3124
|
+
if (v6Result.status === "fulfilled" && ((_b = v6Result.value) == null ? void 0 : _b.ip)) {
|
|
3125
|
+
currentIpv6 = v6Result.value.ip;
|
|
3093
3126
|
}
|
|
3094
|
-
|
|
3095
|
-
|
|
3096
|
-
|
|
3097
|
-
|
|
3098
|
-
|
|
3099
|
-
|
|
3100
|
-
|
|
3101
|
-
|
|
3102
|
-
|
|
3127
|
+
this.cachedIps = { ipv4: currentIpv4, ipv6: currentIpv6, fetchedAt: now };
|
|
3128
|
+
}
|
|
3129
|
+
const localIps = [];
|
|
3130
|
+
try {
|
|
3131
|
+
const osModule = await import("os");
|
|
3132
|
+
const interfaces = osModule.networkInterfaces();
|
|
3133
|
+
for (const name of Object.keys(interfaces)) {
|
|
3134
|
+
for (const iface of interfaces[name] || []) {
|
|
3135
|
+
if (!iface.internal) {
|
|
3136
|
+
localIps.push(iface.address);
|
|
3103
3137
|
}
|
|
3104
3138
|
}
|
|
3105
|
-
} catch (e) {
|
|
3106
3139
|
}
|
|
3107
|
-
|
|
3108
|
-
|
|
3109
|
-
|
|
3110
|
-
|
|
3111
|
-
|
|
3112
|
-
|
|
3113
|
-
);
|
|
3114
|
-
}
|
|
3115
|
-
} else if (payload.ipv4 && !isIpv4Match) {
|
|
3116
|
-
throw new Error(
|
|
3117
|
-
`License key access restricted. This license key was created for a different system (authorized IPv4: ${payload.ipv4}) and cannot be used on this machine.`
|
|
3118
|
-
);
|
|
3119
|
-
} else if (payload.ipv6 && !isIpv6Match) {
|
|
3140
|
+
} catch (e) {
|
|
3141
|
+
}
|
|
3142
|
+
const isIpv4Match = payload.ipv4 && (currentIpv4 === payload.ipv4 || localIps.includes(payload.ipv4));
|
|
3143
|
+
const isIpv6Match = payload.ipv6 && (currentIpv6 === payload.ipv6 || localIps.includes(payload.ipv6));
|
|
3144
|
+
if (payload.ipv4 && payload.ipv6) {
|
|
3145
|
+
if (!isIpv4Match && !isIpv6Match) {
|
|
3120
3146
|
throw new Error(
|
|
3121
|
-
`License key access restricted. This license key was created for a different system (authorized IPv6: ${payload.ipv6}) and cannot be used on this machine.`
|
|
3147
|
+
`License key access restricted. This license key was created for a different system (authorized IPv4: ${payload.ipv4}, IPv6: ${payload.ipv6}) and cannot be used on this machine.`
|
|
3122
3148
|
);
|
|
3123
3149
|
}
|
|
3150
|
+
} else if (payload.ipv4 && !isIpv4Match) {
|
|
3151
|
+
throw new Error(
|
|
3152
|
+
`License key access restricted. This license key was created for a different system (authorized IPv4: ${payload.ipv4}) and cannot be used on this machine.`
|
|
3153
|
+
);
|
|
3154
|
+
} else if (payload.ipv6 && !isIpv6Match) {
|
|
3155
|
+
throw new Error(
|
|
3156
|
+
`License key access restricted. This license key was created for a different system (authorized IPv6: ${payload.ipv6}) and cannot be used on this machine.`
|
|
3157
|
+
);
|
|
3124
3158
|
}
|
|
3125
3159
|
} catch (err) {
|
|
3126
3160
|
if (err.message.includes("License key access restricted")) {
|
|
@@ -3135,7 +3169,7 @@ var _LicenseVerifier = class _LicenseVerifier {
|
|
|
3135
3169
|
const rawToken = sanitizedKey.replace(/^rtv_/i, "").trim();
|
|
3136
3170
|
const parts = rawToken.split(".");
|
|
3137
3171
|
if (parts.length === 3) {
|
|
3138
|
-
const decodedPayload = JSON.parse(
|
|
3172
|
+
const decodedPayload = JSON.parse(decodeBase64UrlToUtf8(parts[1]));
|
|
3139
3173
|
if (decodedPayload.tier) return decodedPayload.tier;
|
|
3140
3174
|
}
|
|
3141
3175
|
} catch (e) {
|
|
@@ -3146,6 +3180,7 @@ var _LicenseVerifier = class _LicenseVerifier {
|
|
|
3146
3180
|
// In-memory cache to save network overhead on hot paths.
|
|
3147
3181
|
_LicenseVerifier.cache = /* @__PURE__ */ new Map();
|
|
3148
3182
|
_LicenseVerifier.CACHE_TTL_MS = 5 * 60 * 1e3;
|
|
3183
|
+
_LicenseVerifier.cachedIps = null;
|
|
3149
3184
|
var LicenseVerifier = _LicenseVerifier;
|
|
3150
3185
|
|
|
3151
3186
|
// src/config/constants.ts
|
|
@@ -4401,7 +4436,7 @@ var GroqProvider = class {
|
|
|
4401
4436
|
};
|
|
4402
4437
|
}
|
|
4403
4438
|
async chat(messages, context, options) {
|
|
4404
|
-
var _a2, _b, _c, _d, _e, _f, _g2, _h, _i;
|
|
4439
|
+
var _a2, _b, _c, _d, _e, _f, _g2, _h, _i, _j, _k, _l;
|
|
4405
4440
|
const basePrompt = (_b = (_a2 = options == null ? void 0 : options.systemPrompt) != null ? _a2 : this.llmConfig.systemPrompt) != null ? _b : "You are a helpful assistant. Answer questions based on the provided context.";
|
|
4406
4441
|
const systemMessage = {
|
|
4407
4442
|
role: "system",
|
|
@@ -4415,18 +4450,46 @@ var GroqProvider = class {
|
|
|
4415
4450
|
systemMessage,
|
|
4416
4451
|
...safeMessages
|
|
4417
4452
|
];
|
|
4418
|
-
const
|
|
4419
|
-
|
|
4420
|
-
|
|
4421
|
-
|
|
4422
|
-
|
|
4423
|
-
|
|
4424
|
-
|
|
4425
|
-
|
|
4453
|
+
const model = this.llmConfig.model === "qwen/qwen3.6-27b" || this.llmConfig.model === "qwen-3.6-27b" ? "qwen/qwen3.8-27b" : this.llmConfig.model;
|
|
4454
|
+
try {
|
|
4455
|
+
const completion = await this.client.chat.completions.create({
|
|
4456
|
+
model,
|
|
4457
|
+
messages: formattedMessages,
|
|
4458
|
+
max_tokens: Math.min((_d = (_c = options == null ? void 0 : options.maxTokens) != null ? _c : this.llmConfig.maxTokens) != null ? _d : 800, 800),
|
|
4459
|
+
temperature: (_f = (_e = options == null ? void 0 : options.temperature) != null ? _e : this.llmConfig.temperature) != null ? _f : 0.7,
|
|
4460
|
+
stop: options == null ? void 0 : options.stop
|
|
4461
|
+
});
|
|
4462
|
+
return (_i = (_h = (_g2 = completion.choices[0]) == null ? void 0 : _g2.message) == null ? void 0 : _h.content) != null ? _i : "";
|
|
4463
|
+
} catch (err) {
|
|
4464
|
+
console.warn("[GroqProvider] Groq chat request failed, falling back strictly to Hugging Face Qwen:", (err == null ? void 0 : err.message) || err);
|
|
4465
|
+
const hfToken = process.env.HF_TOKEN || process.env.HUGGINGFACE_API_KEY || "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";
|
|
4466
|
+
try {
|
|
4467
|
+
const hfRes = await fetch("https://router.huggingface.co/v1/chat/completions", {
|
|
4468
|
+
method: "POST",
|
|
4469
|
+
headers: {
|
|
4470
|
+
"Content-Type": "application/json",
|
|
4471
|
+
"Authorization": `Bearer ${hfToken}`
|
|
4472
|
+
},
|
|
4473
|
+
body: JSON.stringify({
|
|
4474
|
+
model: "Qwen/Qwen2.5-Coder-32B-Instruct",
|
|
4475
|
+
messages: formattedMessages,
|
|
4476
|
+
max_tokens: 1024,
|
|
4477
|
+
temperature: 0.7
|
|
4478
|
+
})
|
|
4479
|
+
});
|
|
4480
|
+
if (hfRes.ok) {
|
|
4481
|
+
const data = await hfRes.json();
|
|
4482
|
+
const content = (_l = (_k = (_j = data == null ? void 0 : data.choices) == null ? void 0 : _j[0]) == null ? void 0 : _k.message) == null ? void 0 : _l.content;
|
|
4483
|
+
if (content !== void 0) return content;
|
|
4484
|
+
}
|
|
4485
|
+
} catch (e) {
|
|
4486
|
+
}
|
|
4487
|
+
throw err;
|
|
4488
|
+
}
|
|
4426
4489
|
}
|
|
4427
4490
|
chatStream(messages, context, options) {
|
|
4428
4491
|
return __asyncGenerator(this, null, function* () {
|
|
4429
|
-
var _a2, _b, _c, _d, _e, _f, _g2, _h;
|
|
4492
|
+
var _a2, _b, _c, _d, _e, _f, _g2, _h, _i, _j, _k, _l;
|
|
4430
4493
|
const basePrompt = (_b = (_a2 = options == null ? void 0 : options.systemPrompt) != null ? _a2 : this.llmConfig.systemPrompt) != null ? _b : "You are a helpful assistant. Answer questions based on the provided context.";
|
|
4431
4494
|
const systemMessage = {
|
|
4432
4495
|
role: "system",
|
|
@@ -4440,32 +4503,81 @@ var GroqProvider = class {
|
|
|
4440
4503
|
systemMessage,
|
|
4441
4504
|
...safeStreamMessages
|
|
4442
4505
|
];
|
|
4443
|
-
const
|
|
4444
|
-
model: this.llmConfig.model,
|
|
4445
|
-
messages: formattedMessages,
|
|
4446
|
-
max_tokens: (_d = (_c = options == null ? void 0 : options.maxTokens) != null ? _c : this.llmConfig.maxTokens) != null ? _d : 1024,
|
|
4447
|
-
temperature: (_f = (_e = options == null ? void 0 : options.temperature) != null ? _e : this.llmConfig.temperature) != null ? _f : 0.7,
|
|
4448
|
-
stop: options == null ? void 0 : options.stop,
|
|
4449
|
-
stream: true
|
|
4450
|
-
}));
|
|
4451
|
-
if (!stream) {
|
|
4452
|
-
throw new Error("[GroqProvider] completions.create stream is undefined");
|
|
4453
|
-
}
|
|
4506
|
+
const model = this.llmConfig.model === "qwen/qwen3.6-27b" || this.llmConfig.model === "qwen-3.6-27b" ? "qwen/qwen3.8-27b" : this.llmConfig.model;
|
|
4454
4507
|
try {
|
|
4455
|
-
|
|
4456
|
-
|
|
4457
|
-
|
|
4458
|
-
|
|
4508
|
+
const stream = yield new __await(this.client.chat.completions.create({
|
|
4509
|
+
model,
|
|
4510
|
+
messages: formattedMessages,
|
|
4511
|
+
max_tokens: Math.min((_d = (_c = options == null ? void 0 : options.maxTokens) != null ? _c : this.llmConfig.maxTokens) != null ? _d : 800, 800),
|
|
4512
|
+
temperature: (_f = (_e = options == null ? void 0 : options.temperature) != null ? _e : this.llmConfig.temperature) != null ? _f : 0.7,
|
|
4513
|
+
stop: options == null ? void 0 : options.stop,
|
|
4514
|
+
stream: true
|
|
4515
|
+
}));
|
|
4516
|
+
if (!stream) {
|
|
4517
|
+
throw new Error("[GroqProvider] completions.create stream is undefined");
|
|
4459
4518
|
}
|
|
4460
|
-
} catch (temp) {
|
|
4461
|
-
error = [temp];
|
|
4462
|
-
} finally {
|
|
4463
4519
|
try {
|
|
4464
|
-
|
|
4520
|
+
for (var iter = __forAwait(stream), more, temp, error; more = !(temp = yield new __await(iter.next())).done; more = false) {
|
|
4521
|
+
const chunk = temp.value;
|
|
4522
|
+
const content = ((_h = (_g2 = chunk.choices[0]) == null ? void 0 : _g2.delta) == null ? void 0 : _h.content) || "";
|
|
4523
|
+
if (content) yield content;
|
|
4524
|
+
}
|
|
4525
|
+
} catch (temp) {
|
|
4526
|
+
error = [temp];
|
|
4465
4527
|
} finally {
|
|
4466
|
-
|
|
4467
|
-
|
|
4528
|
+
try {
|
|
4529
|
+
more && (temp = iter.return) && (yield new __await(temp.call(iter)));
|
|
4530
|
+
} finally {
|
|
4531
|
+
if (error)
|
|
4532
|
+
throw error[0];
|
|
4533
|
+
}
|
|
4534
|
+
}
|
|
4535
|
+
} catch (err) {
|
|
4536
|
+
console.warn("[GroqProvider] Groq chat stream failed, falling back strictly to Hugging Face Qwen stream:", (err == null ? void 0 : err.message) || err);
|
|
4537
|
+
const hfToken = process.env.HF_TOKEN || process.env.HUGGINGFACE_API_KEY || "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";
|
|
4538
|
+
try {
|
|
4539
|
+
const hfRes = yield new __await(fetch("https://router.huggingface.co/v1/chat/completions", {
|
|
4540
|
+
method: "POST",
|
|
4541
|
+
headers: {
|
|
4542
|
+
"Content-Type": "application/json",
|
|
4543
|
+
"Authorization": `Bearer ${hfToken}`
|
|
4544
|
+
},
|
|
4545
|
+
body: JSON.stringify({
|
|
4546
|
+
model: "Qwen/Qwen2.5-Coder-32B-Instruct",
|
|
4547
|
+
messages: formattedMessages,
|
|
4548
|
+
max_tokens: 1024,
|
|
4549
|
+
temperature: 0.7,
|
|
4550
|
+
stream: true
|
|
4551
|
+
})
|
|
4552
|
+
}));
|
|
4553
|
+
if (hfRes.ok && hfRes.body) {
|
|
4554
|
+
const reader = hfRes.body.getReader();
|
|
4555
|
+
const decoder = new TextDecoder("utf-8");
|
|
4556
|
+
let buffer = "";
|
|
4557
|
+
while (true) {
|
|
4558
|
+
const { done, value } = yield new __await(reader.read());
|
|
4559
|
+
if (done) break;
|
|
4560
|
+
buffer += decoder.decode(value, { stream: true });
|
|
4561
|
+
const lines = buffer.split("\n");
|
|
4562
|
+
buffer = (_i = lines.pop()) != null ? _i : "";
|
|
4563
|
+
for (const line of lines) {
|
|
4564
|
+
const trimmed = line.trim();
|
|
4565
|
+
if (!trimmed || trimmed === "data: [DONE]") continue;
|
|
4566
|
+
if (!trimmed.startsWith("data:")) continue;
|
|
4567
|
+
try {
|
|
4568
|
+
const json = JSON.parse(trimmed.slice(5).trim());
|
|
4569
|
+
const content = ((_l = (_k = (_j = json == null ? void 0 : json.choices) == null ? void 0 : _j[0]) == null ? void 0 : _k.delta) == null ? void 0 : _l.content) || "";
|
|
4570
|
+
if (content) yield content;
|
|
4571
|
+
} catch (e) {
|
|
4572
|
+
}
|
|
4573
|
+
}
|
|
4574
|
+
}
|
|
4575
|
+
return;
|
|
4576
|
+
}
|
|
4577
|
+
} catch (hfErr) {
|
|
4578
|
+
console.warn("[GroqProvider] Hugging Face Qwen fallback stream failed:", hfErr);
|
|
4468
4579
|
}
|
|
4580
|
+
throw err;
|
|
4469
4581
|
}
|
|
4470
4582
|
});
|
|
4471
4583
|
}
|
|
@@ -4915,6 +5027,7 @@ ${context != null ? context : "None"}` },
|
|
|
4915
5027
|
} catch (fetchErr) {
|
|
4916
5028
|
console.warn(`[UniversalLLMAdapter] Remote HTTP stream fetch warning: ${fetchErr == null ? void 0 : fetchErr.message}. Attempting in-process fallback...`);
|
|
4917
5029
|
}
|
|
5030
|
+
let lastDispatchErr = null;
|
|
4918
5031
|
if (!streamBody) {
|
|
4919
5032
|
const _g2 = globalThis;
|
|
4920
5033
|
const dispatch = _g2.__retrivoraDispatchChat;
|
|
@@ -4937,12 +5050,16 @@ ${context != null ? context : "None"}` },
|
|
|
4937
5050
|
}
|
|
4938
5051
|
}
|
|
4939
5052
|
} catch (dispatchErr) {
|
|
4940
|
-
|
|
5053
|
+
lastDispatchErr = dispatchErr;
|
|
5054
|
+
console.warn("[UniversalLLMAdapter] In-process dispatch error:", (dispatchErr == null ? void 0 : dispatchErr.message) || dispatchErr);
|
|
4941
5055
|
}
|
|
4942
5056
|
}
|
|
4943
5057
|
}
|
|
4944
5058
|
if (!streamBody) {
|
|
4945
|
-
|
|
5059
|
+
if (lastDispatchErr) {
|
|
5060
|
+
throw lastDispatchErr;
|
|
5061
|
+
}
|
|
5062
|
+
throw new Error(`[UniversalLLMAdapter] Streaming request failed: LLM provider unavailable.`);
|
|
4946
5063
|
}
|
|
4947
5064
|
const reader = streamBody.getReader();
|
|
4948
5065
|
const decoder = new TextDecoder("utf-8");
|
|
@@ -4981,73 +5098,75 @@ ${context != null ? context : "None"}` },
|
|
|
4981
5098
|
});
|
|
4982
5099
|
}
|
|
4983
5100
|
async embed(text) {
|
|
4984
|
-
var _a2, _b, _c, _d, _e, _f,
|
|
4985
|
-
const
|
|
5101
|
+
var _a2, _b, _c, _d, _e, _f, _g3, _h;
|
|
5102
|
+
const _g2 = globalThis;
|
|
5103
|
+
const dispatchEmbed = _g2.__retrivoraDispatchEmbedding;
|
|
5104
|
+
if (typeof dispatchEmbed === "function") {
|
|
5105
|
+
try {
|
|
5106
|
+
const res = await dispatchEmbed({ input: text, model: this.model }, this.apiKey);
|
|
5107
|
+
if ((_b = (_a2 = res == null ? void 0 : res.data) == null ? void 0 : _a2[0]) == null ? void 0 : _b.embedding) {
|
|
5108
|
+
return res.data[0].embedding;
|
|
5109
|
+
}
|
|
5110
|
+
} catch (dispatchErr) {
|
|
5111
|
+
}
|
|
5112
|
+
}
|
|
5113
|
+
const path2 = (_c = this.opts.embedPath) != null ? _c : "/embeddings";
|
|
4986
5114
|
const payload = this.opts.embedPayloadTemplate ? buildPayload(this.opts.embedPayloadTemplate, { input: text, model: this.model }) : { input: text, model: this.model };
|
|
4987
5115
|
try {
|
|
4988
5116
|
const { data } = await this.http.post(path2, payload);
|
|
4989
|
-
const extractPath = (
|
|
5117
|
+
const extractPath = (_d = this.opts.embedExtractPath) != null ? _d : "data[0].embedding";
|
|
4990
5118
|
const vector = resolvePath(data, extractPath);
|
|
4991
5119
|
if (Array.isArray(vector)) {
|
|
4992
5120
|
return vector;
|
|
4993
5121
|
}
|
|
4994
|
-
console.warn(
|
|
4995
|
-
`[UniversalLLMAdapter] Embedding API returned unexpected format for path '${extractPath}'. Response:`,
|
|
4996
|
-
data
|
|
4997
|
-
);
|
|
4998
5122
|
} catch (err) {
|
|
4999
|
-
console.warn(
|
|
5000
|
-
`[UniversalLLMAdapter] HTTP embedding request failed: ${(err == null ? void 0 : err.message) || err}`,
|
|
5001
|
-
{ path: path2, model: this.model, response: (_c = err == null ? void 0 : err.response) == null ? void 0 : _c.data }
|
|
5002
|
-
);
|
|
5003
|
-
const _g3 = globalThis;
|
|
5004
|
-
const dispatchEmbed = _g3.__retrivoraDispatchEmbedding;
|
|
5005
5123
|
if (typeof dispatchEmbed === "function") {
|
|
5006
5124
|
try {
|
|
5007
5125
|
const res = await dispatchEmbed({ input: text, model: this.model }, this.apiKey);
|
|
5008
|
-
if ((
|
|
5126
|
+
if ((_f = (_e = res == null ? void 0 : res.data) == null ? void 0 : _e[0]) == null ? void 0 : _f.embedding) {
|
|
5009
5127
|
return res.data[0].embedding;
|
|
5010
5128
|
}
|
|
5011
5129
|
} catch (dispatchErr) {
|
|
5012
5130
|
console.warn("[UniversalLLMAdapter] In-process dispatch embedding failed:", dispatchErr);
|
|
5013
5131
|
}
|
|
5014
5132
|
}
|
|
5015
|
-
}
|
|
5016
|
-
try {
|
|
5017
|
-
const { data } = await this.http.post(path2, payload);
|
|
5018
|
-
const extractPath = (_f = this.opts.embedExtractPath) != null ? _f : "data[0].embedding";
|
|
5019
|
-
const vector = resolvePath(data, extractPath);
|
|
5020
|
-
if (Array.isArray(vector)) {
|
|
5021
|
-
return vector;
|
|
5022
|
-
}
|
|
5023
|
-
console.error(
|
|
5024
|
-
`[UniversalLLMAdapter] Retry embedding failed: unexpected response format for path '${extractPath}'. Response:`,
|
|
5025
|
-
data
|
|
5026
|
-
);
|
|
5027
|
-
throw new Error(
|
|
5028
|
-
`[UniversalLLMAdapter] Could not extract embedding vector. Expected array at '${extractPath}', got: ${typeof vector}. Response keys: ${Object.keys(data || {})}.`
|
|
5029
|
-
);
|
|
5030
|
-
} catch (err) {
|
|
5031
5133
|
console.error(
|
|
5032
|
-
`[UniversalLLMAdapter]
|
|
5033
|
-
{ error: err == null ? void 0 : err.message, statusCode: (
|
|
5134
|
+
`[UniversalLLMAdapter] Embedding failed. Path: ${path2}, Model: ${this.model}`,
|
|
5135
|
+
{ error: err == null ? void 0 : err.message, statusCode: (_g3 = err == null ? void 0 : err.response) == null ? void 0 : _g3.status, response: (_h = err == null ? void 0 : err.response) == null ? void 0 : _h.data }
|
|
5034
5136
|
);
|
|
5035
5137
|
throw new Error(
|
|
5036
|
-
`[UniversalLLMAdapter] Failed to generate embedding
|
|
5138
|
+
`[UniversalLLMAdapter] Failed to generate embedding: ${(err == null ? void 0 : err.message) || err}`
|
|
5037
5139
|
);
|
|
5038
5140
|
}
|
|
5141
|
+
throw new Error(`[UniversalLLMAdapter] Failed to extract embedding vector.`);
|
|
5039
5142
|
}
|
|
5040
5143
|
async batchEmbed(texts) {
|
|
5041
|
-
|
|
5042
|
-
|
|
5144
|
+
var _a2;
|
|
5145
|
+
if (!texts || texts.length === 0) return [];
|
|
5146
|
+
const _g2 = globalThis;
|
|
5147
|
+
const dispatchEmbed = _g2.__retrivoraDispatchEmbedding;
|
|
5148
|
+
if (typeof dispatchEmbed === "function") {
|
|
5043
5149
|
try {
|
|
5044
|
-
const
|
|
5045
|
-
|
|
5046
|
-
|
|
5047
|
-
|
|
5048
|
-
|
|
5150
|
+
const res = await dispatchEmbed({ input: texts, model: this.model }, this.apiKey);
|
|
5151
|
+
if ((res == null ? void 0 : res.data) && Array.isArray(res.data) && res.data.length === texts.length) {
|
|
5152
|
+
return res.data.map((d) => d.embedding);
|
|
5153
|
+
}
|
|
5154
|
+
} catch (e) {
|
|
5049
5155
|
}
|
|
5050
5156
|
}
|
|
5157
|
+
const path2 = (_a2 = this.opts.embedPath) != null ? _a2 : "/embeddings";
|
|
5158
|
+
try {
|
|
5159
|
+
const { data } = await this.http.post(path2, { input: texts, model: this.model });
|
|
5160
|
+
if ((data == null ? void 0 : data.data) && Array.isArray(data.data) && data.data.length === texts.length) {
|
|
5161
|
+
return data.data.map((d) => d.embedding);
|
|
5162
|
+
}
|
|
5163
|
+
} catch (e) {
|
|
5164
|
+
}
|
|
5165
|
+
const results = [];
|
|
5166
|
+
for (const text of texts) {
|
|
5167
|
+
const vector = await this.embed(text);
|
|
5168
|
+
results.push(vector);
|
|
5169
|
+
}
|
|
5051
5170
|
return results;
|
|
5052
5171
|
}
|
|
5053
5172
|
async ping() {
|
|
@@ -5580,9 +5699,6 @@ var ConfigValidator = class {
|
|
|
5580
5699
|
}
|
|
5581
5700
|
};
|
|
5582
5701
|
|
|
5583
|
-
// src/core/Pipeline.ts
|
|
5584
|
-
init_version();
|
|
5585
|
-
|
|
5586
5702
|
// src/rag/DocumentChunker.ts
|
|
5587
5703
|
var DocumentChunker = class {
|
|
5588
5704
|
constructor(chunkSize = 1e3, chunkOverlap = 200, separators = ["\n# ", "\n## ", "\n### ", "\n#### ", "\n\n", "\n", " ", ""]) {
|
|
@@ -5752,27 +5868,384 @@ Text to extract from:
|
|
|
5752
5868
|
}
|
|
5753
5869
|
};
|
|
5754
5870
|
|
|
5871
|
+
// src/rag/HybridSearch.ts
|
|
5872
|
+
var HybridSearch = class {
|
|
5873
|
+
/**
|
|
5874
|
+
* Tokenize text into normalized alphanumeric keywords, keeping exact codes, numbers, and identifiers.
|
|
5875
|
+
*/
|
|
5876
|
+
static tokenize(text) {
|
|
5877
|
+
if (!text) return [];
|
|
5878
|
+
return text.toLowerCase().replace(/[^\w\s\-_#./:]/g, " ").split(/\s+/).map((t) => t.trim()).filter((t) => t.length > 1 && !this.stopWords.has(t));
|
|
5879
|
+
}
|
|
5880
|
+
/**
|
|
5881
|
+
* Calculate Okapi BM25 sparse scores for an array of document chunks given a query.
|
|
5882
|
+
*/
|
|
5883
|
+
static scoreBM25(documents, query, k1 = 1.5, b = 0.75) {
|
|
5884
|
+
const scores = /* @__PURE__ */ new Map();
|
|
5885
|
+
const queryTokens = this.tokenize(query);
|
|
5886
|
+
if (queryTokens.length === 0 || documents.length === 0) {
|
|
5887
|
+
documents.forEach((d) => scores.set(d.id, 0));
|
|
5888
|
+
return scores;
|
|
5889
|
+
}
|
|
5890
|
+
const docTokens = documents.map((d) => ({
|
|
5891
|
+
id: d.id,
|
|
5892
|
+
tokens: this.tokenize(d.content)
|
|
5893
|
+
}));
|
|
5894
|
+
const N = documents.length;
|
|
5895
|
+
const avgdl = docTokens.reduce((acc, d) => acc + d.tokens.length, 0) / (N || 1);
|
|
5896
|
+
const dfMap = /* @__PURE__ */ new Map();
|
|
5897
|
+
for (const qt of queryTokens) {
|
|
5898
|
+
let count = 0;
|
|
5899
|
+
for (const dt of docTokens) {
|
|
5900
|
+
if (dt.tokens.includes(qt)) count++;
|
|
5901
|
+
}
|
|
5902
|
+
dfMap.set(qt, count);
|
|
5903
|
+
}
|
|
5904
|
+
for (const dt of docTokens) {
|
|
5905
|
+
let score = 0;
|
|
5906
|
+
const docLen = dt.tokens.length;
|
|
5907
|
+
const tfMap = /* @__PURE__ */ new Map();
|
|
5908
|
+
for (const t of dt.tokens) {
|
|
5909
|
+
tfMap.set(t, (tfMap.get(t) || 0) + 1);
|
|
5910
|
+
}
|
|
5911
|
+
for (const qt of queryTokens) {
|
|
5912
|
+
const tf = tfMap.get(qt) || 0;
|
|
5913
|
+
if (tf === 0) continue;
|
|
5914
|
+
const df = dfMap.get(qt) || 0;
|
|
5915
|
+
const idf = Math.log((N - df + 0.5) / (df + 0.5) + 1);
|
|
5916
|
+
const numerator = tf * (k1 + 1);
|
|
5917
|
+
const denominator = tf + k1 * (1 - b + b * (docLen / (avgdl || 1)));
|
|
5918
|
+
score += idf * (numerator / (denominator || 1));
|
|
5919
|
+
}
|
|
5920
|
+
scores.set(dt.id, score);
|
|
5921
|
+
}
|
|
5922
|
+
return scores;
|
|
5923
|
+
}
|
|
5924
|
+
/**
|
|
5925
|
+
* Fuse Dense Vector Matches with Sparse BM25 Scores using Reciprocal Rank Fusion (RRF).
|
|
5926
|
+
*/
|
|
5927
|
+
static fuse(denseMatches, query, options = {}) {
|
|
5928
|
+
var _a2, _b, _c;
|
|
5929
|
+
if (denseMatches.length === 0) return [];
|
|
5930
|
+
const denseWeight = (_a2 = options.denseWeight) != null ? _a2 : 0.6;
|
|
5931
|
+
const sparseWeight = (_b = options.sparseWeight) != null ? _b : 0.4;
|
|
5932
|
+
const k = (_c = options.rrfK) != null ? _c : 60;
|
|
5933
|
+
const sortedDense = [...denseMatches].sort((a, b) => b.score - a.score);
|
|
5934
|
+
const denseRankMap = /* @__PURE__ */ new Map();
|
|
5935
|
+
sortedDense.forEach((match, idx) => {
|
|
5936
|
+
denseRankMap.set(match.id, idx + 1);
|
|
5937
|
+
});
|
|
5938
|
+
const docItems = denseMatches.map((m) => ({ id: m.id, content: m.content }));
|
|
5939
|
+
const bm25ScoreMap = this.scoreBM25(docItems, query, options.bm25K1, options.bm25B);
|
|
5940
|
+
const sortedSparse = [...denseMatches].sort((a, b) => {
|
|
5941
|
+
const scoreA = bm25ScoreMap.get(a.id) || 0;
|
|
5942
|
+
const scoreB = bm25ScoreMap.get(b.id) || 0;
|
|
5943
|
+
return scoreB - scoreA;
|
|
5944
|
+
});
|
|
5945
|
+
const sparseRankMap = /* @__PURE__ */ new Map();
|
|
5946
|
+
sortedSparse.forEach((match, idx) => {
|
|
5947
|
+
sparseRankMap.set(match.id, idx + 1);
|
|
5948
|
+
});
|
|
5949
|
+
const fusedMatches = denseMatches.map((match) => {
|
|
5950
|
+
var _a3;
|
|
5951
|
+
const denseRank = denseRankMap.get(match.id) || denseMatches.length;
|
|
5952
|
+
const sparseRank = sparseRankMap.get(match.id) || denseMatches.length;
|
|
5953
|
+
const denseRrf = denseWeight / (k + denseRank);
|
|
5954
|
+
const sparseRrf = sparseWeight / (k + sparseRank);
|
|
5955
|
+
const combinedRrf = denseRrf + sparseRrf;
|
|
5956
|
+
const bm25Raw = bm25ScoreMap.get(match.id) || 0;
|
|
5957
|
+
return __spreadProps(__spreadValues({}, match), {
|
|
5958
|
+
rrfScore: combinedRrf,
|
|
5959
|
+
bm25Score: bm25Raw,
|
|
5960
|
+
rerankScore: ((_a3 = match.rerankScore) != null ? _a3 : match.score) * 0.5 + combinedRrf * 50
|
|
5961
|
+
});
|
|
5962
|
+
});
|
|
5963
|
+
return fusedMatches.sort((a, b) => {
|
|
5964
|
+
var _a3, _b2;
|
|
5965
|
+
return ((_a3 = b.rerankScore) != null ? _a3 : b.score) - ((_b2 = a.rerankScore) != null ? _b2 : a.score);
|
|
5966
|
+
});
|
|
5967
|
+
}
|
|
5968
|
+
};
|
|
5969
|
+
HybridSearch.stopWords = /* @__PURE__ */ new Set([
|
|
5970
|
+
"a",
|
|
5971
|
+
"about",
|
|
5972
|
+
"above",
|
|
5973
|
+
"after",
|
|
5974
|
+
"again",
|
|
5975
|
+
"against",
|
|
5976
|
+
"all",
|
|
5977
|
+
"am",
|
|
5978
|
+
"an",
|
|
5979
|
+
"and",
|
|
5980
|
+
"any",
|
|
5981
|
+
"are",
|
|
5982
|
+
"aren't",
|
|
5983
|
+
"as",
|
|
5984
|
+
"at",
|
|
5985
|
+
"be",
|
|
5986
|
+
"because",
|
|
5987
|
+
"been",
|
|
5988
|
+
"before",
|
|
5989
|
+
"being",
|
|
5990
|
+
"below",
|
|
5991
|
+
"between",
|
|
5992
|
+
"both",
|
|
5993
|
+
"but",
|
|
5994
|
+
"by",
|
|
5995
|
+
"can't",
|
|
5996
|
+
"cannot",
|
|
5997
|
+
"could",
|
|
5998
|
+
"couldn't",
|
|
5999
|
+
"did",
|
|
6000
|
+
"didn't",
|
|
6001
|
+
"do",
|
|
6002
|
+
"does",
|
|
6003
|
+
"doesn't",
|
|
6004
|
+
"doing",
|
|
6005
|
+
"don't",
|
|
6006
|
+
"down",
|
|
6007
|
+
"during",
|
|
6008
|
+
"each",
|
|
6009
|
+
"few",
|
|
6010
|
+
"for",
|
|
6011
|
+
"from",
|
|
6012
|
+
"further",
|
|
6013
|
+
"had",
|
|
6014
|
+
"hadn't",
|
|
6015
|
+
"has",
|
|
6016
|
+
"hasn't",
|
|
6017
|
+
"have",
|
|
6018
|
+
"haven't",
|
|
6019
|
+
"having",
|
|
6020
|
+
"he",
|
|
6021
|
+
"he'd",
|
|
6022
|
+
"he'll",
|
|
6023
|
+
"he's",
|
|
6024
|
+
"her",
|
|
6025
|
+
"here",
|
|
6026
|
+
"here's",
|
|
6027
|
+
"hers",
|
|
6028
|
+
"herself",
|
|
6029
|
+
"him",
|
|
6030
|
+
"himself",
|
|
6031
|
+
"his",
|
|
6032
|
+
"how",
|
|
6033
|
+
"how's",
|
|
6034
|
+
"i",
|
|
6035
|
+
"i'd",
|
|
6036
|
+
"i'll",
|
|
6037
|
+
"i'm",
|
|
6038
|
+
"i've",
|
|
6039
|
+
"if",
|
|
6040
|
+
"in",
|
|
6041
|
+
"into",
|
|
6042
|
+
"is",
|
|
6043
|
+
"isn't",
|
|
6044
|
+
"it",
|
|
6045
|
+
"it's",
|
|
6046
|
+
"its",
|
|
6047
|
+
"itself",
|
|
6048
|
+
"let's",
|
|
6049
|
+
"me",
|
|
6050
|
+
"more",
|
|
6051
|
+
"most",
|
|
6052
|
+
"mustn't",
|
|
6053
|
+
"my",
|
|
6054
|
+
"myself",
|
|
6055
|
+
"no",
|
|
6056
|
+
"nor",
|
|
6057
|
+
"not",
|
|
6058
|
+
"of",
|
|
6059
|
+
"off",
|
|
6060
|
+
"on",
|
|
6061
|
+
"once",
|
|
6062
|
+
"only",
|
|
6063
|
+
"or",
|
|
6064
|
+
"other",
|
|
6065
|
+
"ought",
|
|
6066
|
+
"our",
|
|
6067
|
+
"ours",
|
|
6068
|
+
"ourselves",
|
|
6069
|
+
"out",
|
|
6070
|
+
"over",
|
|
6071
|
+
"own",
|
|
6072
|
+
"same",
|
|
6073
|
+
"shan't",
|
|
6074
|
+
"she",
|
|
6075
|
+
"she'd",
|
|
6076
|
+
"she'll",
|
|
6077
|
+
"she's",
|
|
6078
|
+
"should",
|
|
6079
|
+
"shouldn't",
|
|
6080
|
+
"so",
|
|
6081
|
+
"some",
|
|
6082
|
+
"such",
|
|
6083
|
+
"than",
|
|
6084
|
+
"that",
|
|
6085
|
+
"that's",
|
|
6086
|
+
"the",
|
|
6087
|
+
"their",
|
|
6088
|
+
"theirs",
|
|
6089
|
+
"them",
|
|
6090
|
+
"themselves",
|
|
6091
|
+
"then",
|
|
6092
|
+
"there",
|
|
6093
|
+
"there's",
|
|
6094
|
+
"these",
|
|
6095
|
+
"they",
|
|
6096
|
+
"they'd",
|
|
6097
|
+
"they'll",
|
|
6098
|
+
"they're",
|
|
6099
|
+
"they've",
|
|
6100
|
+
"this",
|
|
6101
|
+
"those",
|
|
6102
|
+
"through",
|
|
6103
|
+
"to",
|
|
6104
|
+
"too",
|
|
6105
|
+
"under",
|
|
6106
|
+
"until",
|
|
6107
|
+
"up",
|
|
6108
|
+
"very",
|
|
6109
|
+
"was",
|
|
6110
|
+
"wasn't",
|
|
6111
|
+
"we",
|
|
6112
|
+
"we'd",
|
|
6113
|
+
"we'll",
|
|
6114
|
+
"we're",
|
|
6115
|
+
"we've",
|
|
6116
|
+
"were",
|
|
6117
|
+
"weren't",
|
|
6118
|
+
"what",
|
|
6119
|
+
"what's",
|
|
6120
|
+
"when",
|
|
6121
|
+
"when's",
|
|
6122
|
+
"where",
|
|
6123
|
+
"where's",
|
|
6124
|
+
"which",
|
|
6125
|
+
"while",
|
|
6126
|
+
"who",
|
|
6127
|
+
"who's",
|
|
6128
|
+
"whom",
|
|
6129
|
+
"why",
|
|
6130
|
+
"why's",
|
|
6131
|
+
"with",
|
|
6132
|
+
"won't",
|
|
6133
|
+
"would",
|
|
6134
|
+
"wouldn't",
|
|
6135
|
+
"you",
|
|
6136
|
+
"you'd",
|
|
6137
|
+
"you'll",
|
|
6138
|
+
"you're",
|
|
6139
|
+
"you've",
|
|
6140
|
+
"your",
|
|
6141
|
+
"yours",
|
|
6142
|
+
"yourself",
|
|
6143
|
+
"yourselves"
|
|
6144
|
+
]);
|
|
6145
|
+
|
|
5755
6146
|
// src/rag/Reranker.ts
|
|
5756
6147
|
var MAX_RERANK_CONTENT_CHARS = 64e3;
|
|
5757
6148
|
var Reranker = class {
|
|
5758
6149
|
/**
|
|
5759
|
-
* Re-ranks matches based on a secondary relevance score using
|
|
5760
|
-
*
|
|
5761
|
-
* Fix RAG-2: keyword re-ranking no longer mutates the original `score`
|
|
5762
|
-
* field (which consumers interpret as the raw vector similarity). A new
|
|
5763
|
-
* `rerankScore` field is added so both the original similarity and the
|
|
5764
|
-
* boosted rank signal are available to callers.
|
|
6150
|
+
* Re-ranks matches based on a secondary relevance score using Hybrid BM25, Cohere, Jina or LLM.
|
|
5765
6151
|
*/
|
|
5766
|
-
async rerank(matches, query, limit = 5, llm) {
|
|
5767
|
-
|
|
5768
|
-
|
|
6152
|
+
async rerank(matches, query, limit = 5, llm, options = {}) {
|
|
6153
|
+
var _a2, _b;
|
|
6154
|
+
if (!matches || matches.length <= 1) {
|
|
6155
|
+
return matches ? matches.slice(0, limit) : [];
|
|
5769
6156
|
}
|
|
5770
|
-
|
|
5771
|
-
|
|
5772
|
-
|
|
5773
|
-
|
|
5774
|
-
|
|
6157
|
+
const cohereKey = options.cohereApiKey || (typeof process !== "undefined" ? (_a2 = process.env) == null ? void 0 : _a2.COHERE_API_KEY : void 0);
|
|
6158
|
+
const jinaKey = options.jinaApiKey || (typeof process !== "undefined" ? (_b = process.env) == null ? void 0 : _b.JINA_API_KEY : void 0);
|
|
6159
|
+
if ((options.provider === "cohere" || cohereKey) && cohereKey) {
|
|
6160
|
+
try {
|
|
6161
|
+
const cohereResult = await this.cohereRerank(matches, query, limit, cohereKey, options.cohereModel);
|
|
6162
|
+
if (cohereResult.length > 0) return cohereResult;
|
|
6163
|
+
} catch (err) {
|
|
6164
|
+
console.warn("[Reranker] Cohere reranking failed, falling back to HybridSearch:", err);
|
|
6165
|
+
}
|
|
6166
|
+
}
|
|
6167
|
+
if ((options.provider === "jina" || jinaKey) && jinaKey) {
|
|
6168
|
+
try {
|
|
6169
|
+
const jinaResult = await this.jinaRerank(matches, query, limit, jinaKey, options.jinaModel);
|
|
6170
|
+
if (jinaResult.length > 0) return jinaResult;
|
|
6171
|
+
} catch (err) {
|
|
6172
|
+
console.warn("[Reranker] Jina reranking failed, falling back to HybridSearch:", err);
|
|
6173
|
+
}
|
|
6174
|
+
}
|
|
6175
|
+
if (!options.provider || options.provider === "hybrid") {
|
|
6176
|
+
const fused = HybridSearch.fuse(matches, query);
|
|
6177
|
+
if (fused.length > 0) {
|
|
6178
|
+
return fused.slice(0, limit);
|
|
6179
|
+
}
|
|
6180
|
+
}
|
|
6181
|
+
if (llm && (options.provider === "llm" || !options.provider)) {
|
|
6182
|
+
try {
|
|
6183
|
+
return await this.llmRerank(matches, query, limit, llm);
|
|
6184
|
+
} catch (error) {
|
|
6185
|
+
console.warn("[Reranker] LLM re-ranking failed, falling back to keyword sorting:", error);
|
|
6186
|
+
}
|
|
5775
6187
|
}
|
|
6188
|
+
return this.keywordRerank(matches, query, limit);
|
|
6189
|
+
}
|
|
6190
|
+
/**
|
|
6191
|
+
* Cohere Rerank v3.5 API integration.
|
|
6192
|
+
*/
|
|
6193
|
+
async cohereRerank(matches, query, limit, apiKey, model = "rerank-v3.5") {
|
|
6194
|
+
const documents = matches.map((m) => m.content || "");
|
|
6195
|
+
const response = await fetch("https://api.cohere.com/v1/rerank", {
|
|
6196
|
+
method: "POST",
|
|
6197
|
+
headers: {
|
|
6198
|
+
"Authorization": `Bearer ${apiKey}`,
|
|
6199
|
+
"Content-Type": "application/json"
|
|
6200
|
+
},
|
|
6201
|
+
body: JSON.stringify({
|
|
6202
|
+
model,
|
|
6203
|
+
query,
|
|
6204
|
+
documents,
|
|
6205
|
+
top_n: limit
|
|
6206
|
+
})
|
|
6207
|
+
});
|
|
6208
|
+
if (!response.ok) {
|
|
6209
|
+
throw new Error(`Cohere API returned status ${response.status}`);
|
|
6210
|
+
}
|
|
6211
|
+
const data = await response.json();
|
|
6212
|
+
if (!data.results || !Array.isArray(data.results)) return [];
|
|
6213
|
+
return data.results.map((res) => {
|
|
6214
|
+
const match = matches[res.index];
|
|
6215
|
+
return __spreadProps(__spreadValues({}, match), {
|
|
6216
|
+
rerankScore: res.relevance_score
|
|
6217
|
+
});
|
|
6218
|
+
});
|
|
6219
|
+
}
|
|
6220
|
+
/**
|
|
6221
|
+
* Jina AI Reranker v2 API integration.
|
|
6222
|
+
*/
|
|
6223
|
+
async jinaRerank(matches, query, limit, apiKey, model = "jina-reranker-v2-base-multilingual") {
|
|
6224
|
+
const documents = matches.map((m) => m.content || "");
|
|
6225
|
+
const response = await fetch("https://api.jina.ai/v1/rerank", {
|
|
6226
|
+
method: "POST",
|
|
6227
|
+
headers: {
|
|
6228
|
+
"Authorization": `Bearer ${apiKey}`,
|
|
6229
|
+
"Content-Type": "application/json"
|
|
6230
|
+
},
|
|
6231
|
+
body: JSON.stringify({
|
|
6232
|
+
model,
|
|
6233
|
+
query,
|
|
6234
|
+
documents,
|
|
6235
|
+
top_n: limit
|
|
6236
|
+
})
|
|
6237
|
+
});
|
|
6238
|
+
if (!response.ok) {
|
|
6239
|
+
throw new Error(`Jina API returned status ${response.status}`);
|
|
6240
|
+
}
|
|
6241
|
+
const data = await response.json();
|
|
6242
|
+
if (!data.results || !Array.isArray(data.results)) return [];
|
|
6243
|
+
return data.results.map((res) => {
|
|
6244
|
+
const match = matches[res.index];
|
|
6245
|
+
return __spreadProps(__spreadValues({}, match), {
|
|
6246
|
+
rerankScore: res.relevance_score
|
|
6247
|
+
});
|
|
6248
|
+
});
|
|
5776
6249
|
}
|
|
5777
6250
|
/**
|
|
5778
6251
|
* Keyword-based re-ranking fallback.
|
|
@@ -5858,47 +6331,217 @@ ${docList}` }],
|
|
|
5858
6331
|
}
|
|
5859
6332
|
};
|
|
5860
6333
|
|
|
5861
|
-
// src/rag/
|
|
5862
|
-
var
|
|
5863
|
-
/**
|
|
5864
|
-
* Chunks document content using LlamaIndex SentenceSplitter.
|
|
5865
|
-
* This respects sentence and paragraph boundaries much more effectively
|
|
5866
|
-
* than standard character-count splitting.
|
|
5867
|
-
*/
|
|
5868
|
-
async chunk(text, options = {}) {
|
|
5869
|
-
var _a2, _b;
|
|
5870
|
-
try {
|
|
5871
|
-
const { Document, SentenceSplitter, MetadataMode } = await import(`${"llamaindex"}`);
|
|
5872
|
-
const splitter = new SentenceSplitter({
|
|
5873
|
-
chunkSize: (_a2 = options.chunkSize) != null ? _a2 : 1e3,
|
|
5874
|
-
chunkOverlap: (_b = options.chunkOverlap) != null ? _b : 200
|
|
5875
|
-
});
|
|
5876
|
-
const doc = new Document({ text, metadata: options.metadata || {} });
|
|
5877
|
-
const nodes = splitter.getNodesFromDocuments([doc]);
|
|
5878
|
-
return nodes.map((node, index) => ({
|
|
5879
|
-
id: `${options.docId || "doc"}_node_${index}`,
|
|
5880
|
-
content: node.getContent(MetadataMode.ALL),
|
|
5881
|
-
metadata: __spreadProps(__spreadValues(__spreadValues({}, options.metadata), node.metadata), {
|
|
5882
|
-
nodeId: node.id_,
|
|
5883
|
-
chunkIndex: index
|
|
5884
|
-
})
|
|
5885
|
-
}));
|
|
5886
|
-
} catch (error) {
|
|
5887
|
-
console.warn("[LlamaIndexIngestor] LlamaIndex package not found or failed. Falling back to default chunker.");
|
|
5888
|
-
throw error;
|
|
5889
|
-
}
|
|
5890
|
-
}
|
|
6334
|
+
// src/rag/RAGEvaluator.ts
|
|
6335
|
+
var RAGEvaluator = class {
|
|
5891
6336
|
/**
|
|
5892
|
-
*
|
|
6337
|
+
* Evaluate a RAG interaction across the full RAG Triad.
|
|
5893
6338
|
*/
|
|
5894
|
-
async
|
|
5895
|
-
|
|
5896
|
-
|
|
5897
|
-
const chunks = await this.chunk(doc.content, __spreadProps(__spreadValues({}, options), { docId: doc.docId, metadata: doc.metadata }));
|
|
5898
|
-
allChunks.push(...chunks);
|
|
6339
|
+
static async evaluate(query, context, answer, llm) {
|
|
6340
|
+
if (!llm || !context || !answer) {
|
|
6341
|
+
return this.deterministicHeuristic(query, context, answer);
|
|
5899
6342
|
}
|
|
5900
|
-
|
|
5901
|
-
|
|
6343
|
+
try {
|
|
6344
|
+
const prompt = `You are an automated RAG Triad Evaluator. Evaluate the quality of the following RAG system interaction:
|
|
6345
|
+
|
|
6346
|
+
USER QUERY:
|
|
6347
|
+
${query}
|
|
6348
|
+
|
|
6349
|
+
RETRIEVED CONTEXT:
|
|
6350
|
+
${context.slice(0, 3e3)}
|
|
6351
|
+
|
|
6352
|
+
GENERATED ANSWER:
|
|
6353
|
+
${answer}
|
|
6354
|
+
|
|
6355
|
+
Evaluate these 3 metrics on a scale of 0.0 to 1.0:
|
|
6356
|
+
1. "contextRelevance": Are the retrieved documents relevant to the query? (1.0 = highly relevant, 0.0 = irrelevant)
|
|
6357
|
+
2. "faithfulness": Is every claim in the generated answer supported by the retrieved context? (1.0 = 100% grounded/no hallucination, 0.0 = completely fabricated)
|
|
6358
|
+
3. "answerRelevance": Does the answer directly address the user's question? (1.0 = direct answer, 0.0 = evasive/unrelated)
|
|
6359
|
+
|
|
6360
|
+
Return ONLY valid JSON with no markdown fences:
|
|
6361
|
+
{"contextRelevance": 0.95, "faithfulness": 0.90, "answerRelevance": 0.92, "reason": "Answer is fully supported by the document."}`;
|
|
6362
|
+
const raw = await llm.chat(
|
|
6363
|
+
[{ role: "user", content: prompt }],
|
|
6364
|
+
"",
|
|
6365
|
+
{
|
|
6366
|
+
systemPrompt: "You are a strict AI safety and hallucination evaluator. Output only valid JSON.",
|
|
6367
|
+
temperature: 0,
|
|
6368
|
+
maxTokens: 150
|
|
6369
|
+
}
|
|
6370
|
+
);
|
|
6371
|
+
const cleaned = raw.replace(/```json|```/g, "").trim();
|
|
6372
|
+
const parsed = JSON.parse(cleaned);
|
|
6373
|
+
const cr = typeof parsed.contextRelevance === "number" ? Math.max(0, Math.min(1, parsed.contextRelevance)) : 0.85;
|
|
6374
|
+
const f = typeof parsed.faithfulness === "number" ? Math.max(0, Math.min(1, parsed.faithfulness)) : 0.9;
|
|
6375
|
+
const ar = typeof parsed.answerRelevance === "number" ? Math.max(0, Math.min(1, parsed.answerRelevance)) : 0.88;
|
|
6376
|
+
const overall = Number((cr * 0.3 + f * 0.4 + ar * 0.3).toFixed(2));
|
|
6377
|
+
return {
|
|
6378
|
+
contextRelevance: cr,
|
|
6379
|
+
faithfulness: f,
|
|
6380
|
+
answerRelevance: ar,
|
|
6381
|
+
overallQuality: overall,
|
|
6382
|
+
isHallucinated: f < 0.6,
|
|
6383
|
+
reason: parsed.reason || "Evaluated successfully."
|
|
6384
|
+
};
|
|
6385
|
+
} catch (err) {
|
|
6386
|
+
return this.deterministicHeuristic(query, context, answer);
|
|
6387
|
+
}
|
|
6388
|
+
}
|
|
6389
|
+
/**
|
|
6390
|
+
* Deterministic fast keyword-overlap heuristic when LLM evaluation is not invoked.
|
|
6391
|
+
*/
|
|
6392
|
+
static deterministicHeuristic(query, context, answer) {
|
|
6393
|
+
if (!context || !answer) {
|
|
6394
|
+
return { contextRelevance: 0.5, faithfulness: 0.8, answerRelevance: 0.7, overallQuality: 0.67, isHallucinated: false };
|
|
6395
|
+
}
|
|
6396
|
+
const queryWords = query.toLowerCase().split(/\s+/).filter((w) => w.length > 2);
|
|
6397
|
+
const contextLower = context.toLowerCase();
|
|
6398
|
+
const answerLower = answer.toLowerCase();
|
|
6399
|
+
let queryContextHits = 0;
|
|
6400
|
+
queryWords.forEach((w) => {
|
|
6401
|
+
if (contextLower.includes(w)) queryContextHits++;
|
|
6402
|
+
});
|
|
6403
|
+
const contextRelevance = queryWords.length > 0 ? Math.min(1, queryContextHits / queryWords.length + 0.3) : 0.8;
|
|
6404
|
+
const numbersInAnswer = answer.match(/\b\d+(?:\.\d+)?%?\b/g) || [];
|
|
6405
|
+
let numberHits = 0;
|
|
6406
|
+
numbersInAnswer.forEach((num) => {
|
|
6407
|
+
if (context.includes(num)) numberHits++;
|
|
6408
|
+
});
|
|
6409
|
+
const faithfulness = numbersInAnswer.length > 0 ? Math.min(1, numberHits / numbersInAnswer.length + 0.2) : 0.9;
|
|
6410
|
+
const answerRelevance = 0.85;
|
|
6411
|
+
const overallQuality = Number((contextRelevance * 0.3 + faithfulness * 0.4 + answerRelevance * 0.3).toFixed(2));
|
|
6412
|
+
return {
|
|
6413
|
+
contextRelevance: Number(contextRelevance.toFixed(2)),
|
|
6414
|
+
faithfulness: Number(faithfulness.toFixed(2)),
|
|
6415
|
+
answerRelevance,
|
|
6416
|
+
overallQuality,
|
|
6417
|
+
isHallucinated: faithfulness < 0.5,
|
|
6418
|
+
reason: "Computed via deterministic grounding heuristic."
|
|
6419
|
+
};
|
|
6420
|
+
}
|
|
6421
|
+
};
|
|
6422
|
+
|
|
6423
|
+
// src/security/PIIMasker.ts
|
|
6424
|
+
var PIIMasker = class {
|
|
6425
|
+
/**
|
|
6426
|
+
* Redact sensitive information from text.
|
|
6427
|
+
*/
|
|
6428
|
+
static mask(text, options = {}) {
|
|
6429
|
+
if (!text || typeof text !== "string") return text;
|
|
6430
|
+
let sanitized = text;
|
|
6431
|
+
if (options.maskApiKeys !== false) {
|
|
6432
|
+
sanitized = sanitized.replace(this.API_KEY_REGEX, "[REDACTED_API_KEY]");
|
|
6433
|
+
}
|
|
6434
|
+
if (options.maskCreditCards !== false) {
|
|
6435
|
+
sanitized = sanitized.replace(this.CREDIT_CARD_REGEX, (match) => {
|
|
6436
|
+
const digits = match.replace(/[- ]/g, "");
|
|
6437
|
+
if (digits.length >= 13 && digits.length <= 19) {
|
|
6438
|
+
return "[REDACTED_CREDIT_CARD]";
|
|
6439
|
+
}
|
|
6440
|
+
return match;
|
|
6441
|
+
});
|
|
6442
|
+
}
|
|
6443
|
+
if (options.maskSSN !== false) {
|
|
6444
|
+
sanitized = sanitized.replace(this.SSN_REGEX, "[REDACTED_SSN]");
|
|
6445
|
+
}
|
|
6446
|
+
if (options.maskEmails !== false) {
|
|
6447
|
+
sanitized = sanitized.replace(this.EMAIL_REGEX, "[REDACTED_EMAIL]");
|
|
6448
|
+
}
|
|
6449
|
+
if (options.maskPhones !== false) {
|
|
6450
|
+
sanitized = sanitized.replace(this.PHONE_REGEX, "[REDACTED_PHONE]");
|
|
6451
|
+
}
|
|
6452
|
+
if (options.maskIPs === true) {
|
|
6453
|
+
sanitized = sanitized.replace(this.IP_REGEX, "[REDACTED_IP]");
|
|
6454
|
+
}
|
|
6455
|
+
return sanitized;
|
|
6456
|
+
}
|
|
6457
|
+
};
|
|
6458
|
+
PIIMasker.SSN_REGEX = /\b(?!000|666|9\d{2})\d{3}[- ]?(?!00)\d{2}[- ]?(?!0000)\d{4}\b/g;
|
|
6459
|
+
PIIMasker.CREDIT_CARD_REGEX = /\b(?:\d{4}[- ]?){3}\d{4}\b|\b\d{15,16}\b/g;
|
|
6460
|
+
PIIMasker.EMAIL_REGEX = /\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Z|a-z]{2,}\b/g;
|
|
6461
|
+
PIIMasker.PHONE_REGEX = /\b(?:\+?\d{1,3}[-.\s]?)?\(?\d{3}\)?[-.\s]?\d{3}[-.\s]?\d{4}\b/g;
|
|
6462
|
+
PIIMasker.API_KEY_REGEX = /\b(?:sk-[a-zA-Z0-9]{20,}|AKIA[0-9A-Z]{16}|rtv_[a-zA-Z0-9_-]{20,}|ghp_[a-zA-Z0-9]{36}|bearer\s+[a-zA-Z0-9._-]{20,})\b/gi;
|
|
6463
|
+
PIIMasker.IP_REGEX = /\b(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\b/g;
|
|
6464
|
+
|
|
6465
|
+
// src/security/PromptGuard.ts
|
|
6466
|
+
var PromptGuard = class {
|
|
6467
|
+
/**
|
|
6468
|
+
* Scan and evaluate user query or document text for prompt injection vectors.
|
|
6469
|
+
*/
|
|
6470
|
+
static evaluate(text) {
|
|
6471
|
+
if (!text || typeof text !== "string") {
|
|
6472
|
+
return { isSafe: true, threatLevel: "none", detectedPatterns: [], sanitizedText: text };
|
|
6473
|
+
}
|
|
6474
|
+
const detected = [];
|
|
6475
|
+
let maxSeverity = "none";
|
|
6476
|
+
for (const item of this.INJECTION_PATTERNS) {
|
|
6477
|
+
if (item.pattern.test(text)) {
|
|
6478
|
+
detected.push(item.name);
|
|
6479
|
+
if (item.severity === "high") {
|
|
6480
|
+
maxSeverity = "high";
|
|
6481
|
+
} else if (item.severity === "medium" && maxSeverity !== "high") {
|
|
6482
|
+
maxSeverity = "medium";
|
|
6483
|
+
}
|
|
6484
|
+
}
|
|
6485
|
+
}
|
|
6486
|
+
const sanitized = text.replace(/<\/?(?:system|instruction|admin|prompt_override)>/gi, "");
|
|
6487
|
+
return {
|
|
6488
|
+
isSafe: maxSeverity !== "high",
|
|
6489
|
+
threatLevel: maxSeverity,
|
|
6490
|
+
detectedPatterns: detected,
|
|
6491
|
+
sanitizedText: sanitized
|
|
6492
|
+
};
|
|
6493
|
+
}
|
|
6494
|
+
};
|
|
6495
|
+
PromptGuard.INJECTION_PATTERNS = [
|
|
6496
|
+
{ pattern: /ignore\s+(?:all\s+)?(?:previous|prior|above)\s+(?:instructions|prompts|rules)/i, name: "override_instructions", severity: "high" },
|
|
6497
|
+
{ pattern: /(?:show|repeat|print|reveal|expose|output)\s+(?:the\s+)?(?:system\s+prompt|initial\s+instructions|system\s+message)/i, name: "system_prompt_leak", severity: "high" },
|
|
6498
|
+
{ pattern: /you\s+are\s+now\s+(?:in\s+developer\s+mode|unrestricted|DAN|jailbroken|godmode)/i, name: "jailbreak_persona", severity: "high" },
|
|
6499
|
+
{ pattern: /<\/?(?:system|instruction|admin|prompt_override)>/i, name: "tag_injection", severity: "medium" },
|
|
6500
|
+
{ pattern: /base64\s*decode\s*and\s*execute/i, name: "obfuscated_payload", severity: "high" },
|
|
6501
|
+
{ pattern: /do\s+anything\s+now/i, name: "dan_variant", severity: "high" }
|
|
6502
|
+
];
|
|
6503
|
+
|
|
6504
|
+
// src/rag/LlamaIndexIngestor.ts
|
|
6505
|
+
var LlamaIndexIngestor = class {
|
|
6506
|
+
/**
|
|
6507
|
+
* Chunks document content using LlamaIndex SentenceSplitter.
|
|
6508
|
+
* This respects sentence and paragraph boundaries much more effectively
|
|
6509
|
+
* than standard character-count splitting.
|
|
6510
|
+
*/
|
|
6511
|
+
async chunk(text, options = {}) {
|
|
6512
|
+
var _a2, _b;
|
|
6513
|
+
try {
|
|
6514
|
+
const { Document, SentenceSplitter, MetadataMode } = await import(`${"llamaindex"}`);
|
|
6515
|
+
const splitter = new SentenceSplitter({
|
|
6516
|
+
chunkSize: (_a2 = options.chunkSize) != null ? _a2 : 1e3,
|
|
6517
|
+
chunkOverlap: (_b = options.chunkOverlap) != null ? _b : 200
|
|
6518
|
+
});
|
|
6519
|
+
const doc = new Document({ text, metadata: options.metadata || {} });
|
|
6520
|
+
const nodes = splitter.getNodesFromDocuments([doc]);
|
|
6521
|
+
return nodes.map((node, index) => ({
|
|
6522
|
+
id: `${options.docId || "doc"}_node_${index}`,
|
|
6523
|
+
content: node.getContent(MetadataMode.ALL),
|
|
6524
|
+
metadata: __spreadProps(__spreadValues(__spreadValues({}, options.metadata), node.metadata), {
|
|
6525
|
+
nodeId: node.id_,
|
|
6526
|
+
chunkIndex: index
|
|
6527
|
+
})
|
|
6528
|
+
}));
|
|
6529
|
+
} catch (error) {
|
|
6530
|
+
console.warn("[LlamaIndexIngestor] LlamaIndex package not found or failed. Falling back to default chunker.");
|
|
6531
|
+
throw error;
|
|
6532
|
+
}
|
|
6533
|
+
}
|
|
6534
|
+
/**
|
|
6535
|
+
* Batch processing for multiple documents.
|
|
6536
|
+
*/
|
|
6537
|
+
async chunkMany(documents, options = {}) {
|
|
6538
|
+
const allChunks = [];
|
|
6539
|
+
for (const doc of documents) {
|
|
6540
|
+
const chunks = await this.chunk(doc.content, __spreadProps(__spreadValues({}, options), { docId: doc.docId, metadata: doc.metadata }));
|
|
6541
|
+
allChunks.push(...chunks);
|
|
6542
|
+
}
|
|
6543
|
+
return allChunks;
|
|
6544
|
+
}
|
|
5902
6545
|
};
|
|
5903
6546
|
|
|
5904
6547
|
// src/core/LangChainAgent.ts
|
|
@@ -5998,6 +6641,73 @@ ${error instanceof Error ? error.message : String(error)}`
|
|
|
5998
6641
|
}
|
|
5999
6642
|
return String((_c = (_b = response == null ? void 0 : response.output) != null ? _b : response) != null ? _c : "");
|
|
6000
6643
|
}
|
|
6644
|
+
/**
|
|
6645
|
+
* Run the agentic flow in streaming mode, emitting granular SSE event frames.
|
|
6646
|
+
* Leverages LangChain/LangGraph `streamEvents` (v2 API) to stream:
|
|
6647
|
+
* - Token-by-token LLM output (`type: "text"`)
|
|
6648
|
+
* - Tool invocation / thinking steps (`type: "thinking"`)
|
|
6649
|
+
* - Tool execution metadata & search sources
|
|
6650
|
+
*/
|
|
6651
|
+
streamRun(_0) {
|
|
6652
|
+
return __asyncGenerator(this, arguments, function* (input, chatHistory = []) {
|
|
6653
|
+
var _a2;
|
|
6654
|
+
if (!this.agent) {
|
|
6655
|
+
throw new Error("[LangChainAgent] Agent not initialized. Call initialize() first.");
|
|
6656
|
+
}
|
|
6657
|
+
const { HumanMessage, AIMessage } = yield new __await(import("@langchain/core/messages"));
|
|
6658
|
+
const historyMessages = chatHistory.map(
|
|
6659
|
+
(m) => m.role === "user" ? new HumanMessage(m.content) : new AIMessage(m.content)
|
|
6660
|
+
);
|
|
6661
|
+
const inputMessages = [...historyMessages, new HumanMessage(input)];
|
|
6662
|
+
if (typeof this.agent.streamEvents === "function") {
|
|
6663
|
+
const eventStream = this.agent.streamEvents(
|
|
6664
|
+
{ messages: inputMessages },
|
|
6665
|
+
{ version: "v2" }
|
|
6666
|
+
);
|
|
6667
|
+
try {
|
|
6668
|
+
for (var iter = __forAwait(eventStream), more, temp, error; more = !(temp = yield new __await(iter.next())).done; more = false) {
|
|
6669
|
+
const event = temp.value;
|
|
6670
|
+
if (event.event === "on_tool_start") {
|
|
6671
|
+
const toolName = event.name || "document_search";
|
|
6672
|
+
yield {
|
|
6673
|
+
type: "thinking",
|
|
6674
|
+
text: `Analyzing repository with tool "${toolName}"...
|
|
6675
|
+
`
|
|
6676
|
+
};
|
|
6677
|
+
} else if (event.event === "on_tool_end") {
|
|
6678
|
+
const toolName = event.name || "document_search";
|
|
6679
|
+
yield {
|
|
6680
|
+
type: "thinking",
|
|
6681
|
+
text: `Tool "${toolName}" completed successfully.
|
|
6682
|
+
`
|
|
6683
|
+
};
|
|
6684
|
+
} else if (event.event === "on_chat_model_stream") {
|
|
6685
|
+
const chunk = (_a2 = event.data) == null ? void 0 : _a2.chunk;
|
|
6686
|
+
if (chunk == null ? void 0 : chunk.content) {
|
|
6687
|
+
const tokenText = typeof chunk.content === "string" ? chunk.content : JSON.stringify(chunk.content);
|
|
6688
|
+
yield {
|
|
6689
|
+
type: "text",
|
|
6690
|
+
text: tokenText
|
|
6691
|
+
};
|
|
6692
|
+
}
|
|
6693
|
+
}
|
|
6694
|
+
}
|
|
6695
|
+
} catch (temp) {
|
|
6696
|
+
error = [temp];
|
|
6697
|
+
} finally {
|
|
6698
|
+
try {
|
|
6699
|
+
more && (temp = iter.return) && (yield new __await(temp.call(iter)));
|
|
6700
|
+
} finally {
|
|
6701
|
+
if (error)
|
|
6702
|
+
throw error[0];
|
|
6703
|
+
}
|
|
6704
|
+
}
|
|
6705
|
+
} else {
|
|
6706
|
+
const result = yield new __await(this.run(input, chatHistory));
|
|
6707
|
+
yield { type: "text", text: result };
|
|
6708
|
+
}
|
|
6709
|
+
});
|
|
6710
|
+
}
|
|
6001
6711
|
};
|
|
6002
6712
|
|
|
6003
6713
|
// src/rag/PromptOptimizer.ts
|
|
@@ -6905,25 +7615,43 @@ ${errorMessages}`
|
|
|
6905
7615
|
* Processes up to `concurrency` items at the same time.
|
|
6906
7616
|
*/
|
|
6907
7617
|
static async processConcurrent(items, processor, concurrency = 5, options) {
|
|
7618
|
+
var _a2, _b, _c, _d;
|
|
6908
7619
|
const { throwOnPartialFailure = false } = options != null ? options : {};
|
|
6909
7620
|
const results = [];
|
|
6910
7621
|
const errors = [];
|
|
7622
|
+
const maxRetries = (_a2 = options == null ? void 0 : options.maxRetries) != null ? _a2 : 3;
|
|
7623
|
+
const initialDelayMs = (_b = options == null ? void 0 : options.initialDelayMs) != null ? _b : 100;
|
|
7624
|
+
const maxDelayMs = (_c = options == null ? void 0 : options.maxDelayMs) != null ? _c : 1e4;
|
|
7625
|
+
const backoffMultiplier = (_d = options == null ? void 0 : options.backoffMultiplier) != null ? _d : 2;
|
|
6911
7626
|
for (let i = 0; i < items.length; i += concurrency) {
|
|
6912
7627
|
const chunk = items.slice(i, i + concurrency).map((item, idx) => ({
|
|
6913
7628
|
item,
|
|
6914
7629
|
originalIndex: i + idx
|
|
6915
7630
|
}));
|
|
6916
7631
|
const chunkPromises = chunk.map(async ({ item, originalIndex }) => {
|
|
6917
|
-
|
|
6918
|
-
|
|
6919
|
-
|
|
6920
|
-
|
|
6921
|
-
|
|
6922
|
-
|
|
6923
|
-
|
|
6924
|
-
|
|
6925
|
-
|
|
7632
|
+
let lastErr;
|
|
7633
|
+
for (let attempt = 0; attempt <= maxRetries; attempt++) {
|
|
7634
|
+
try {
|
|
7635
|
+
const result = await _BatchProcessor.executeWithCircuitBreaker(
|
|
7636
|
+
() => processor(item),
|
|
7637
|
+
options == null ? void 0 : options.circuitBreaker
|
|
7638
|
+
);
|
|
7639
|
+
return { success: true, result, index: originalIndex };
|
|
7640
|
+
} catch (err) {
|
|
7641
|
+
lastErr = err instanceof Error ? err : new Error(String(err));
|
|
7642
|
+
if (!isTransientError(err) || attempt === maxRetries) {
|
|
7643
|
+
break;
|
|
7644
|
+
}
|
|
7645
|
+
const delay = calculateBackoffDelay(
|
|
7646
|
+
attempt,
|
|
7647
|
+
initialDelayMs,
|
|
7648
|
+
maxDelayMs,
|
|
7649
|
+
backoffMultiplier
|
|
7650
|
+
);
|
|
7651
|
+
await sleep(delay);
|
|
7652
|
+
}
|
|
6926
7653
|
}
|
|
7654
|
+
return { success: false, error: lastErr || new Error("Unknown error"), index: originalIndex };
|
|
6927
7655
|
});
|
|
6928
7656
|
const chunkResults = await Promise.all(chunkPromises);
|
|
6929
7657
|
for (const { success, result, error, index } of chunkResults) {
|
|
@@ -7396,14 +8124,14 @@ var IntentClassifier = class {
|
|
|
7396
8124
|
static classify(context) {
|
|
7397
8125
|
const q = (context.userQuery || "").toLowerCase().trim();
|
|
7398
8126
|
const signals = this.extractDataSignals(context);
|
|
7399
|
-
if (this.
|
|
7400
|
-
return { intent: "
|
|
8127
|
+
if (this.isProductQuery(q) || signals.isProductLike && (this.isRecommendationQuery(q) || this.isDiscoveryQuery(q))) {
|
|
8128
|
+
return { intent: "product_search", signals };
|
|
7401
8129
|
}
|
|
7402
8130
|
if (this.isComparisonQuery(q)) {
|
|
7403
8131
|
return { intent: "comparison", signals };
|
|
7404
8132
|
}
|
|
7405
|
-
if (this.
|
|
7406
|
-
return { intent: "
|
|
8133
|
+
if (this.isInformationLookup(q)) {
|
|
8134
|
+
return { intent: "information_lookup", signals };
|
|
7407
8135
|
}
|
|
7408
8136
|
if (this.isTrendQuery(q) || signals.hasDateFields && signals.hasNumericFields && /\b(growth|trend|over time|monthly|yearly|weekly)\b/.test(q)) {
|
|
7409
8137
|
return { intent: "trend_analysis", signals };
|
|
@@ -7420,7 +8148,8 @@ var IntentClassifier = class {
|
|
|
7420
8148
|
return { intent: "information_lookup", signals };
|
|
7421
8149
|
}
|
|
7422
8150
|
/**
|
|
7423
|
-
* Extract key data structure signals from retrieved documents.
|
|
8151
|
+
* Extract key data structure signals dynamically from retrieved documents.
|
|
8152
|
+
* Inspects both structured metadata and parsed chunk text.
|
|
7424
8153
|
*/
|
|
7425
8154
|
static extractDataSignals(context) {
|
|
7426
8155
|
const docs = context.retrievedDocuments || [];
|
|
@@ -7453,18 +8182,21 @@ var IntentClassifier = class {
|
|
|
7453
8182
|
let categoricalFieldCount = 0;
|
|
7454
8183
|
if (docs.length > 0) {
|
|
7455
8184
|
const keys = /* @__PURE__ */ new Set();
|
|
7456
|
-
|
|
7457
|
-
|
|
7458
|
-
|
|
7459
|
-
let
|
|
8185
|
+
const numericKeys = /* @__PURE__ */ new Set();
|
|
8186
|
+
const dateKeys = /* @__PURE__ */ new Set();
|
|
8187
|
+
const catKeys = /* @__PURE__ */ new Set();
|
|
8188
|
+
let productDocCount = 0;
|
|
7460
8189
|
docs.forEach((doc) => {
|
|
8190
|
+
if (!doc) return;
|
|
7461
8191
|
const meta = doc.metadata || {};
|
|
8192
|
+
const content = doc.content || "";
|
|
8193
|
+
let hasDocProductSignal = false;
|
|
7462
8194
|
Object.entries(meta).forEach(([key, val]) => {
|
|
7463
8195
|
if (IGNORED_SYSTEM_KEYS.has(key.toLowerCase())) return;
|
|
7464
8196
|
keys.add(key);
|
|
7465
8197
|
const kLower = key.toLowerCase();
|
|
7466
|
-
if (["price", "image", "brand", "in_stock", "rating", "sku", "product"].some((p) => kLower.includes(p))) {
|
|
7467
|
-
|
|
8198
|
+
if (["price", "image", "brand", "in_stock", "rating", "sku", "product", "item", "cost", "msrp", "category", "vendor", "inventory", "stock"].some((p) => kLower.includes(p))) {
|
|
8199
|
+
hasDocProductSignal = true;
|
|
7468
8200
|
}
|
|
7469
8201
|
if (typeof val === "number") {
|
|
7470
8202
|
numericKeys.add(key);
|
|
@@ -7476,13 +8208,23 @@ var IntentClassifier = class {
|
|
|
7476
8208
|
}
|
|
7477
8209
|
}
|
|
7478
8210
|
});
|
|
8211
|
+
if (!hasDocProductSignal && content) {
|
|
8212
|
+
const hasContentName = /(?:^|\n)\s*(?:product\s*name|item\s*name|product|item|title|name)\s*[:\-–]\s*[^\n,]+/i.test(content);
|
|
8213
|
+
const hasContentAttr = /(?:^|\n)\s*(?:price|cost|msrp|brand|vendor|sku|category|stock|inventory|rating|image)\s*[:\-–]/i.test(content) || /\$\s*[\d,.]+/.test(content);
|
|
8214
|
+
if (hasContentName && hasContentAttr) {
|
|
8215
|
+
hasDocProductSignal = true;
|
|
8216
|
+
}
|
|
8217
|
+
}
|
|
8218
|
+
if (hasDocProductSignal) {
|
|
8219
|
+
productDocCount++;
|
|
8220
|
+
}
|
|
7479
8221
|
});
|
|
7480
8222
|
hasNumericFields = numericKeys.size > 0;
|
|
7481
8223
|
hasDateFields = dateKeys.size > 0;
|
|
7482
8224
|
hasCategoricalFields = catKeys.size > 0;
|
|
7483
8225
|
numericFieldCount = numericKeys.size;
|
|
7484
8226
|
categoricalFieldCount = catKeys.size;
|
|
7485
|
-
if (
|
|
8227
|
+
if (productDocCount > 0 && (productDocCount / docs.length >= 0.3 || productDocCount >= 2)) {
|
|
7486
8228
|
isProductLike = true;
|
|
7487
8229
|
}
|
|
7488
8230
|
}
|
|
@@ -7497,8 +8239,15 @@ var IntentClassifier = class {
|
|
|
7497
8239
|
};
|
|
7498
8240
|
}
|
|
7499
8241
|
// ─── Intent Heuristic Checkers ─────────────────────────────────────────────
|
|
8242
|
+
static isDiscoveryQuery(query) {
|
|
8243
|
+
const q = query.toLowerCase().trim();
|
|
8244
|
+
return /\b(show|find|list|browse|search|explore|get|buy|shop|shopping|purchase|options|selection|picks|available|what do you have|what is in stock|looking for|under \$|cheap|affordable|best|top)\b/i.test(q) && !/\b(chart|graph|plot|sales over time|monthly trend)\b/i.test(q);
|
|
8245
|
+
}
|
|
7500
8246
|
static isInformationLookup(query) {
|
|
7501
8247
|
const q = query.toLowerCase().trim();
|
|
8248
|
+
if (/\b(show|find|list|browse|recommend|suggest|available|in stock|buy|shop|catalog|inventory|what do you have|looking for)\b/i.test(q)) {
|
|
8249
|
+
return false;
|
|
8250
|
+
}
|
|
7502
8251
|
if (/^(what|who|explain|explore|define|where|when|why|how|how does|can|could|does|do|is|are|has|have|will|should|would|meaning of|tell me about|show details of|overview of|summarize)\s+/i.test(q)) {
|
|
7503
8252
|
if (!/\b(chart|graph|plot|visualize|pie chart|bar chart|line chart|scatter plot|draw a)\b/i.test(q)) {
|
|
7504
8253
|
return true;
|
|
@@ -7513,7 +8262,11 @@ var IntentClassifier = class {
|
|
|
7513
8262
|
return /\b(compare|comparison|vs\.?|versus|difference between|differ|contrast|against)\b/i.test(query);
|
|
7514
8263
|
}
|
|
7515
8264
|
static isProductQuery(query) {
|
|
7516
|
-
|
|
8265
|
+
const q = query.toLowerCase().trim();
|
|
8266
|
+
if (/\b(price per share|share price|total cost of the deal|debt|equity|ratio|revenue|ebit|ebitda|fcfe|fcff|wacc)\b/i.test(q)) {
|
|
8267
|
+
return false;
|
|
8268
|
+
}
|
|
8269
|
+
return /\b(best|recommend|top|buy|shopping|shop|under \$|cheap|affordable|products?|items?|catalog|inventory|in stock|show available|what do you have|options for)\b/i.test(q) && !/\b(compare|chart|sales|revenue|trend)\b/i.test(q);
|
|
7517
8270
|
}
|
|
7518
8271
|
static isRecommendationQuery(query) {
|
|
7519
8272
|
return /\b(recommend|suggest|what should i (buy|get|choose)|best options|top picks)\b/i.test(query);
|
|
@@ -7578,14 +8331,23 @@ var Rule3ProductDiscoveryRule = class {
|
|
|
7578
8331
|
}
|
|
7579
8332
|
evaluate(context, intent) {
|
|
7580
8333
|
const q = (context.userQuery || "").toLowerCase().trim();
|
|
7581
|
-
const
|
|
7582
|
-
|
|
7583
|
-
|
|
7584
|
-
|
|
8334
|
+
const isExplicitChart = /\b(chart|graph|plot|sales over time|monthly growth|revenue trend|pie chart|bar chart|line chart|histogram)\b/i.test(q);
|
|
8335
|
+
if (isExplicitChart) {
|
|
8336
|
+
return null;
|
|
8337
|
+
}
|
|
8338
|
+
const isFinancialReport = /\b(price per share|share price|debt to equity|ebitda|fcfe|fcff|wacc|beta|balance sheet|cash flow statement)\b/i.test(q);
|
|
8339
|
+
if (isFinancialReport) {
|
|
8340
|
+
return null;
|
|
8341
|
+
}
|
|
8342
|
+
const signals = IntentClassifier.extractDataSignals(context);
|
|
8343
|
+
const hasDiscoveryAction = intent === "product_search" || intent === "recommendation" || /\b(recommend|recommendation|suggest|show|find|list|browse|search|explore|get|buy|shop|shopping|purchase|options|selection|picks|available|what do you have|what is in stock|looking for|under \$|cheap|affordable|best|top|products?|items?|catalog|inventory)\b/i.test(q);
|
|
8344
|
+
const isAnalyticalRanking = /\b(top performing|top selling|best performing|performing|sales revenue|revenue ranking)\b/i.test(q);
|
|
8345
|
+
const isGenericDocumentQuery = /\b(documents? in system|all documents|system documents|overall documents)\b/i.test(q);
|
|
8346
|
+
if (!isAnalyticalRanking && !isGenericDocumentQuery && (signals.isProductLike && hasDiscoveryAction || intent === "product_search" && !isAnalyticalRanking)) {
|
|
7585
8347
|
return {
|
|
7586
8348
|
renderType: "carousel",
|
|
7587
|
-
confidence: 0.
|
|
7588
|
-
reason: "Rule 3:
|
|
8349
|
+
confidence: 0.92,
|
|
8350
|
+
reason: "Rule 3: Dynamically identified product catalog entities and discovery intent."
|
|
7589
8351
|
};
|
|
7590
8352
|
}
|
|
7591
8353
|
return null;
|
|
@@ -8056,11 +8818,11 @@ var UITransformer = class _UITransformer {
|
|
|
8056
8818
|
if (["tabular", "table"].includes(resolvedIntent.visualizationHint) || resolvedIntent.wantsExplicitTable) {
|
|
8057
8819
|
return this.hasMultipleFields(filteredData) ? this.transformToTable(filteredData, userQuery) : this.transformToText(filteredData);
|
|
8058
8820
|
}
|
|
8059
|
-
if (
|
|
8060
|
-
|
|
8061
|
-
|
|
8821
|
+
if (hasProducts || this.isProductQuery(userQuery) || resolvedIntent.visualizationHint === "product_browse") {
|
|
8822
|
+
const carousel = this.transformToProductCarousel(filteredData, config, trainedSchema);
|
|
8823
|
+
if (Array.isArray(carousel.data) && carousel.data.length > 0) {
|
|
8824
|
+
return carousel;
|
|
8062
8825
|
}
|
|
8063
|
-
return this.transformToProductCarousel(filteredData, config, trainedSchema);
|
|
8064
8826
|
}
|
|
8065
8827
|
const automatic = this.chooseAutomaticVisualization(filteredData, profile, userQuery);
|
|
8066
8828
|
if (automatic) return automatic;
|
|
@@ -8697,15 +9459,17 @@ ${schemaProfileText}` : ""}`;
|
|
|
8697
9459
|
return _UITransformer._productQueryTest(query);
|
|
8698
9460
|
}
|
|
8699
9461
|
static _productQueryTest(query) {
|
|
8700
|
-
const q = query.toLowerCase();
|
|
9462
|
+
const q = query.toLowerCase().trim();
|
|
8701
9463
|
const financialTerms = /\b(price per share|share price|total cost of the deal|cost of the deal|debt|equity|ratio|revenue|ebit|ebitda|tax|depreciation|interest rate|cashflow|cash flow|fcfe|fcff|wacc|beta|working capital|cogs|capital spending|valuation|leverage)\b/;
|
|
8702
9464
|
if (financialTerms.test(q)) {
|
|
8703
9465
|
return false;
|
|
8704
9466
|
}
|
|
8705
|
-
const productTerms = /\b(product|products|item|items|sku|catalog|catalogue|brand|model|stock|inventory|in stock|out of stock|buy|shop)\b/.test(q);
|
|
8706
|
-
const productAction = /\b(recommend|suggest|describe|description|detail|details|about|show|find|search|browse|list|explore|exploring)\b/.test(q);
|
|
8707
9467
|
const nonProductEntityTerms = /\b(organization|organisations?|organizations?|company|companies|employee|employees|staff|headcount|workforce)\b/.test(q);
|
|
8708
|
-
|
|
9468
|
+
if (nonProductEntityTerms) {
|
|
9469
|
+
return false;
|
|
9470
|
+
}
|
|
9471
|
+
const genericDiscoveryPatterns = /\b(product|products|item|items|sku|catalog|catalogue|brand|model|stock|inventory|in stock|out of stock|buy|shop|shopping|purchase|order|deals|recommend|recommendation|suggestions|show available|what do you have|available|looking for|options for|under \$|cheap|affordable|best|top|find me|browse)\b/i;
|
|
9472
|
+
return genericDiscoveryPatterns.test(q);
|
|
8709
9473
|
}
|
|
8710
9474
|
static isProductData(item) {
|
|
8711
9475
|
if (!item) return false;
|
|
@@ -8722,10 +9486,17 @@ ${schemaProfileText}` : ""}`;
|
|
|
8722
9486
|
if (financialTerms.test(content)) {
|
|
8723
9487
|
return false;
|
|
8724
9488
|
}
|
|
8725
|
-
const hasSku = Boolean(resolveMetadataValue(metadata, "sku") || resolveMetadataValue(metadata, "product_id"));
|
|
8726
|
-
const hasProductIdentity = Boolean(resolveMetadataValue(metadata, "product_name") || resolveMetadataValue(metadata, "brand") && resolveMetadataValue(metadata, "name"));
|
|
8727
|
-
const hasPrice = Boolean(resolveMetadataValue(metadata, "retail_price") || resolveMetadataValue(metadata, "unit_price") || resolveMetadataValue(metadata, "msrp"));
|
|
8728
|
-
|
|
9489
|
+
const hasSku = Boolean(resolveMetadataValue(metadata, "sku") || resolveMetadataValue(metadata, "product_id") || resolveMetadataValue(metadata, "item_id"));
|
|
9490
|
+
const hasProductIdentity = Boolean(resolveMetadataValue(metadata, "product_name") || resolveMetadataValue(metadata, "item_name") || resolveMetadataValue(metadata, "brand") && resolveMetadataValue(metadata, "name"));
|
|
9491
|
+
const hasPrice = Boolean(resolveMetadataValue(metadata, "price") || resolveMetadataValue(metadata, "retail_price") || resolveMetadataValue(metadata, "unit_price") || resolveMetadataValue(metadata, "msrp"));
|
|
9492
|
+
if (hasSku || hasProductIdentity && hasPrice || hasProductIdentity) {
|
|
9493
|
+
return true;
|
|
9494
|
+
}
|
|
9495
|
+
const hasContentName = /(?:^|\n)\s*(?:product\s*name|item\s*name|product|item|title|name)\s*[:\-–]\s*[^\n,]+/i.test(item.content || "");
|
|
9496
|
+
const hasContentPrice = /(?:^|\n)\s*(?:price|retail\s*price|unit\s*price|cost|msrp)\s*[:\-–]?\s*\$?\s*[\d,.]+/i.test(item.content || "");
|
|
9497
|
+
const hasContentCategory = /(?:^|\n)\s*(?:category|department|collection)\s*[:\-–]\s*[^\n,]+/i.test(item.content || "");
|
|
9498
|
+
const hasContentBrand = /(?:^|\n)\s*(?:brand|manufacturer|vendor)\s*[:\-–]\s*[^\n,]+/i.test(item.content || "");
|
|
9499
|
+
return hasContentName && (hasContentPrice || hasContentCategory || hasContentBrand);
|
|
8729
9500
|
}
|
|
8730
9501
|
static isTimeSeriesData(item) {
|
|
8731
9502
|
const metadata = item.metadata || {};
|
|
@@ -9215,64 +9986,46 @@ ${schemaProfileText}` : ""}`;
|
|
|
9215
9986
|
return resolveMetadataValue(meta, uiKey);
|
|
9216
9987
|
}
|
|
9217
9988
|
static extractProductInfo(item, config, trainedSchema) {
|
|
9218
|
-
var _a2, _b;
|
|
9989
|
+
var _a2, _b, _c;
|
|
9219
9990
|
if (!item) return null;
|
|
9220
9991
|
const meta = item.metadata || {};
|
|
9221
9992
|
const content = item.content || "";
|
|
9222
|
-
|
|
9223
|
-
|
|
9224
|
-
|
|
9993
|
+
let name = this.getDynamicVal(meta, "name", config, trainedSchema);
|
|
9994
|
+
let price = this.getDynamicVal(meta, "price", config, trainedSchema);
|
|
9995
|
+
let brand = this.getDynamicVal(meta, "brand", config, trainedSchema);
|
|
9225
9996
|
const description = this.cleanProductDescription(
|
|
9226
9997
|
(_a2 = this.extractProductDescriptionFromContent(content)) != null ? _a2 : this.getProductDescriptionValue(meta, config, trainedSchema)
|
|
9227
9998
|
);
|
|
9228
|
-
if (name
|
|
9229
|
-
|
|
9230
|
-
if (
|
|
9231
|
-
|
|
9232
|
-
finalName = nameMatch ? nameMatch[1].trim() : (_b = content.split("\n")[0]) == null ? void 0 : _b.substring(0, 60);
|
|
9233
|
-
}
|
|
9234
|
-
if (!finalName) {
|
|
9235
|
-
console.warn(
|
|
9236
|
-
`[UITransformer] Unable to read product name from metadata or content for item ${item.id}. Defaulting to 'Unnamed Product'.`,
|
|
9237
|
-
{ metadataKeys: Object.keys(meta), contentLength: content.length }
|
|
9238
|
-
);
|
|
9239
|
-
}
|
|
9240
|
-
let finalPrice = typeof price === "number" || typeof price === "string" ? price : void 0;
|
|
9241
|
-
if (!finalPrice && content) {
|
|
9242
|
-
const priceMatch = content.match(/(?:Price|Cost):\s*\$?([\d,.]+)/i) || content.match(/\$\s*([\d,.]+)/);
|
|
9243
|
-
if (priceMatch) finalPrice = priceMatch[1].replace(/,/g, "");
|
|
9244
|
-
}
|
|
9245
|
-
if (finalPrice === void 0) {
|
|
9246
|
-
console.warn(
|
|
9247
|
-
`[UITransformer] Unable to read product price from metadata or content for item ${item.id}.`,
|
|
9248
|
-
{ metadataKeys: Object.keys(meta), contentSnippet: content.substring(0, 100) }
|
|
9249
|
-
);
|
|
9999
|
+
if (!name && content) {
|
|
10000
|
+
const nameMatch = content.match(/(?:^|\n)\s*(?:product\s*name|item\s*name|product|item|title|name)\s*[:\-–]\s*([^\n,]+)/i) || content.match(/(?:Product|Item|Name):\s*([^\n]+)/i);
|
|
10001
|
+
if (nameMatch && nameMatch[1]) {
|
|
10002
|
+
name = nameMatch[1].trim();
|
|
9250
10003
|
}
|
|
9251
|
-
|
|
9252
|
-
|
|
9253
|
-
|
|
9254
|
-
|
|
9255
|
-
);
|
|
10004
|
+
}
|
|
10005
|
+
if (!brand && content) {
|
|
10006
|
+
const brandMatch = content.match(/(?:^|\n)\s*(?:brand|manufacturer|vendor)\s*[:\-–]\s*([^\n,]+)/i);
|
|
10007
|
+
if (brandMatch && brandMatch[1]) {
|
|
10008
|
+
brand = brandMatch[1].trim();
|
|
9256
10009
|
}
|
|
9257
|
-
|
|
9258
|
-
|
|
9259
|
-
|
|
9260
|
-
|
|
10010
|
+
}
|
|
10011
|
+
if (price === void 0 && content) {
|
|
10012
|
+
const priceMatch = content.match(/(?:^|\n)\s*(?:price|retail\s*price|unit\s*price|cost|msrp)\s*[:\-–]?\s*\$?\s*([\d,.]+)/i) || content.match(/(?:Price|Cost):\s*\$?([\d,.]+)/i) || content.match(/\$\s*([\d,.]+)/);
|
|
10013
|
+
if (priceMatch && priceMatch[1]) {
|
|
10014
|
+
price = priceMatch[1].replace(/,/g, "");
|
|
9261
10015
|
}
|
|
10016
|
+
}
|
|
10017
|
+
if (name || this.isProductData(item)) {
|
|
10018
|
+
const finalName = name ? String(name) : ((_b = content.split("\n")[0]) == null ? void 0 : _b.substring(0, 60)) || "Unnamed Product";
|
|
10019
|
+
const imageValue = this.getDynamicVal(meta, "image", config, trainedSchema) || ((_c = content.match(/(?:^|\n)\s*(?:image|image\s*url|photo|thumbnail)\s*[:\-–]\s*(https?:\/\/[^\s\n,]+)/i)) == null ? void 0 : _c[1]);
|
|
9262
10020
|
return {
|
|
9263
10021
|
id: item.id,
|
|
9264
|
-
name: finalName
|
|
9265
|
-
price:
|
|
10022
|
+
name: finalName,
|
|
10023
|
+
price: typeof price === "number" || typeof price === "string" ? price : void 0,
|
|
9266
10024
|
image: typeof imageValue === "string" ? imageValue : void 0,
|
|
9267
10025
|
brand: brand ? String(brand) : void 0,
|
|
9268
10026
|
description,
|
|
9269
10027
|
inStock: this.determineStockStatus(item)
|
|
9270
10028
|
};
|
|
9271
|
-
} else {
|
|
9272
|
-
console.warn(
|
|
9273
|
-
`[UITransformer] Item ${item.id} skipped: unable to read name or identify as product data from metadata/content.`,
|
|
9274
|
-
{ metadataKeys: Object.keys(meta), contentSnippet: content.substring(0, 100) }
|
|
9275
|
-
);
|
|
9276
10029
|
}
|
|
9277
10030
|
return null;
|
|
9278
10031
|
}
|
|
@@ -9647,6 +10400,17 @@ var Pipeline = class {
|
|
|
9647
10400
|
- If answer cannot be found in the sources, say that you cannot answer the question and do not hallucinate.
|
|
9648
10401
|
- Do not use information from previous turns to answer the question.
|
|
9649
10402
|
- You CAN use numbers for years and counts (e.g., 2006, 5800). But NEVER put a dollar sign ($) before non-monetary numbers like Stock quantities, EAN numbers, or years. Only use dollar signs for actual monetary prices.
|
|
10403
|
+
- Be concise, direct, and focused. Deliver clean, high-quality answers within token limits without unnecessary filler.
|
|
10404
|
+
|
|
10405
|
+
### CATEGORY & DOMAIN SYNONYM MATCHING:
|
|
10406
|
+
- Intelligently understand common category synonyms and related terms from the context:
|
|
10407
|
+
- If the user asks about "makeup", "cosmetics", "skincare", "toiletries", "personal care", or "beauty products", map these to the "Beauty" category in the catalog and present the matching products.
|
|
10408
|
+
- If the user asks about "clothes", "apparel", "wear", "shirts", "garments", or "fashion", map these to the "Clothing" category in the catalog.
|
|
10409
|
+
- If the user asks about "electronics", "gadgets", "tech", "devices", or "appliances", map these to the "Electronics" category.
|
|
10410
|
+
- If the user asks about "food", "snacks", "groceries", "beverages", or "drinks", map these to the "Grocery" category.
|
|
10411
|
+
- If the user asks about "office items", "stationery", "work supplies", or "desk equipment", map these to the "Office Supplies" category.
|
|
10412
|
+
- If the user asks about "toys", "games", "play items", or "hobbies", map these to the "Toys & Games" category.
|
|
10413
|
+
- Do NOT reject a query or say no items exist simply because the user used a natural synonym (e.g., "makeup") instead of the exact catalog column label (e.g., "Beauty"). Identify the relevant items from the context and present them.
|
|
9650
10414
|
|
|
9651
10415
|
### PRODUCT DISPLAY:
|
|
9652
10416
|
- Recommended Products should only be displayed when the user explicitly asks for products.
|
|
@@ -9813,16 +10577,26 @@ ${m.content}`).join("\n\n---\n\n");
|
|
|
9813
10577
|
*/
|
|
9814
10578
|
async processEmbeddings(chunks) {
|
|
9815
10579
|
if (chunks.length === 0) return [];
|
|
9816
|
-
const texts = chunks.map((c) => c.content);
|
|
10580
|
+
const texts = chunks.map((c) => PIIMasker.mask(c.content));
|
|
9817
10581
|
if (typeof this.embeddingProvider.batchEmbed === "function") {
|
|
9818
10582
|
try {
|
|
9819
|
-
const batchSize =
|
|
9820
|
-
const
|
|
10583
|
+
const batchSize = 100;
|
|
10584
|
+
const slices = [];
|
|
9821
10585
|
for (let i = 0; i < texts.length; i += batchSize) {
|
|
9822
|
-
|
|
9823
|
-
const batchVectors = await this.embeddingProvider.batchEmbed(slice, { taskType: "document" });
|
|
9824
|
-
allVectors.push(...batchVectors);
|
|
10586
|
+
slices.push(texts.slice(i, i + batchSize));
|
|
9825
10587
|
}
|
|
10588
|
+
const batchEmbedFn = this.embeddingProvider.batchEmbed.bind(this.embeddingProvider);
|
|
10589
|
+
const concurrentBatches = await BatchProcessor.processConcurrent(
|
|
10590
|
+
slices,
|
|
10591
|
+
(slice) => batchEmbedFn(slice, { taskType: "document" }),
|
|
10592
|
+
5,
|
|
10593
|
+
{
|
|
10594
|
+
maxRetries: 3,
|
|
10595
|
+
initialDelayMs: 100,
|
|
10596
|
+
throwOnPartialFailure: false
|
|
10597
|
+
}
|
|
10598
|
+
);
|
|
10599
|
+
const allVectors = concurrentBatches.results.flat();
|
|
9826
10600
|
if (allVectors.length === chunks.length) {
|
|
9827
10601
|
return allVectors;
|
|
9828
10602
|
}
|
|
@@ -9835,7 +10609,7 @@ ${m.content}`).join("\n\n---\n\n");
|
|
|
9835
10609
|
const concurrentResult = await BatchProcessor.processConcurrent(
|
|
9836
10610
|
texts,
|
|
9837
10611
|
(text) => this.embeddingProvider.embed(text, { taskType: "document" }),
|
|
9838
|
-
|
|
10612
|
+
20,
|
|
9839
10613
|
{
|
|
9840
10614
|
maxRetries: 3,
|
|
9841
10615
|
initialDelayMs: 100,
|
|
@@ -9853,7 +10627,7 @@ ${m.content}`).join("\n\n---\n\n");
|
|
|
9853
10627
|
/** Step 3: Upsert chunks to vector database with retry logic. */
|
|
9854
10628
|
async processUpserts(upsertDocs, namespace) {
|
|
9855
10629
|
const upsertBatchOptions = {
|
|
9856
|
-
batchSize:
|
|
10630
|
+
batchSize: 250,
|
|
9857
10631
|
maxRetries: 3,
|
|
9858
10632
|
initialDelayMs: 100
|
|
9859
10633
|
};
|
|
@@ -10011,7 +10785,7 @@ ${m.content}`).join("\n\n---\n\n");
|
|
|
10011
10785
|
*/
|
|
10012
10786
|
askStreamInternal(_0) {
|
|
10013
10787
|
return __asyncGenerator(this, arguments, function* (question, history = [], namespace) {
|
|
10014
|
-
var _a2, _b, _c, _d, _e, _f, _g2, _h, _i, _j, _k, _l, _m, _n, _o, _p, _q, _r, _s, _t, _u, _v, _w, _x
|
|
10788
|
+
var _a2, _b, _c, _d, _e, _f, _g2, _h, _i, _j, _k, _l, _m, _n, _o, _p, _q, _r, _s, _t, _u, _v, _w, _x;
|
|
10015
10789
|
yield new __await(this.initialize());
|
|
10016
10790
|
const ns = formatNamespace(namespace != null ? namespace : this.config.projectId);
|
|
10017
10791
|
const topK = (_b = (_a2 = this.config.rag) == null ? void 0 : _a2.topK) != null ? _b : 5;
|
|
@@ -10019,10 +10793,12 @@ ${m.content}`).join("\n\n---\n\n");
|
|
|
10019
10793
|
const requestId = `req_${Date.now()}_${Math.random().toString(36).slice(2, 8)}`;
|
|
10020
10794
|
const requestStart = performance.now();
|
|
10021
10795
|
try {
|
|
10022
|
-
|
|
10796
|
+
const guardCheck = PromptGuard.evaluate(question);
|
|
10797
|
+
let effectiveQuestion = guardCheck.sanitizedText;
|
|
10798
|
+
effectiveQuestion = PIIMasker.mask(effectiveQuestion);
|
|
10023
10799
|
let promptOptResult;
|
|
10024
10800
|
if (((_e = this.config.rag) == null ? void 0 : _e.optimizePrompt) !== false) {
|
|
10025
|
-
promptOptResult = PromptOptimizer.optimize(
|
|
10801
|
+
promptOptResult = PromptOptimizer.optimize(effectiveQuestion, history);
|
|
10026
10802
|
effectiveQuestion = promptOptResult.optimizedPrompt;
|
|
10027
10803
|
}
|
|
10028
10804
|
let searchQuery = effectiveQuestion;
|
|
@@ -10080,7 +10856,17 @@ ${m.content}`).join("\n\n---\n\n");
|
|
|
10080
10856
|
}
|
|
10081
10857
|
const rerankMs = performance.now() - rerankStart;
|
|
10082
10858
|
const totalCount = fullSources.length;
|
|
10083
|
-
const
|
|
10859
|
+
const rawPromptSources = fullSources.slice(0, 4);
|
|
10860
|
+
let accumulatedChars = 0;
|
|
10861
|
+
const promptSources = [];
|
|
10862
|
+
for (const s of rawPromptSources) {
|
|
10863
|
+
const textLen = ((s == null ? void 0 : s.content) || "").length;
|
|
10864
|
+
if (accumulatedChars + textLen > 3500 && promptSources.length >= 2) {
|
|
10865
|
+
break;
|
|
10866
|
+
}
|
|
10867
|
+
promptSources.push(s);
|
|
10868
|
+
accumulatedChars += textLen;
|
|
10869
|
+
}
|
|
10084
10870
|
let context = promptSources.length ? `[SEARCH SUMMARY: Found ${totalCount} matching record(s) out of ${rawSources.length} total resources searched in database. Context includes top ${promptSources.length} relevant records]
|
|
10085
10871
|
|
|
10086
10872
|
` + promptSources.map((m, i) => {
|
|
@@ -10158,7 +10944,7 @@ ${context}`;
|
|
|
10158
10944
|
const isNativeThinking = isClaude37 && (((_q = this.config.llm.options) == null ? void 0 : _q.thinking) === true || ((_r = this.config.llm.options) == null ? void 0 : _r.thinking) === "enabled" || ((_s = this.config.llm.options) == null ? void 0 : _s.thinking) !== false);
|
|
10159
10945
|
const modelNameLower = this.config.llm.model.toLowerCase();
|
|
10160
10946
|
const isReasoningModel = modelNameLower.includes("-r1") || modelNameLower.includes("deepseek-r1") || modelNameLower.includes("reasoning") || modelNameLower.includes("thinking");
|
|
10161
|
-
const isSimulatedThinking = !isNativeThinking && (((_t = this.config.llm.options) == null ? void 0 : _t.thinking) === true || ((_u = this.config.llm.options) == null ? void 0 : _u.thinking) === "enabled"
|
|
10947
|
+
const isSimulatedThinking = !isNativeThinking && (((_t = this.config.llm.options) == null ? void 0 : _t.thinking) === true || ((_u = this.config.llm.options) == null ? void 0 : _u.thinking) === "enabled");
|
|
10162
10948
|
let finalRestrictionSuffix = restrictionSuffix;
|
|
10163
10949
|
if (isSimulatedThinking) {
|
|
10164
10950
|
finalRestrictionSuffix += "\n\n(IMPORTANT: You must think step-by-step before answering. Write your thinking process inside a `<think>...</think>` block. Write the `<think>` block first, then follow it with your final response. Keep the thinking process thorough and detailed. Example: `<think>Thinking details here...</think>Final answer text here.`)";
|
|
@@ -10169,7 +10955,7 @@ ${context}`;
|
|
|
10169
10955
|
}));
|
|
10170
10956
|
const userQuestion = { role: "user", content: effectiveQuestion + finalRestrictionSuffix };
|
|
10171
10957
|
const messages = [...hardenedHistory, userQuestion];
|
|
10172
|
-
const systemPrompt = (
|
|
10958
|
+
const systemPrompt = (_v = this.config.llm.systemPrompt) != null ? _v : "";
|
|
10173
10959
|
const userPrompt = messages.map((m) => `${m.role}: ${m.content}`).join("\n");
|
|
10174
10960
|
let fullReply = "";
|
|
10175
10961
|
let thinkingText = "";
|
|
@@ -10191,57 +10977,54 @@ ${context}`;
|
|
|
10191
10977
|
yield { type: "thinking", text: thinkingDelta };
|
|
10192
10978
|
continue;
|
|
10193
10979
|
}
|
|
10194
|
-
|
|
10195
|
-
|
|
10196
|
-
|
|
10197
|
-
|
|
10198
|
-
|
|
10199
|
-
|
|
10200
|
-
|
|
10201
|
-
|
|
10202
|
-
|
|
10203
|
-
yield before;
|
|
10204
|
-
}
|
|
10205
|
-
inThinking = true;
|
|
10206
|
-
thinkingStartMs = performance.now();
|
|
10207
|
-
textBuffer = textBuffer.slice(thinkIndex + "<think>".length);
|
|
10208
|
-
} else {
|
|
10209
|
-
const maxSafeLength = textBuffer.length - "<think>".length;
|
|
10210
|
-
if (maxSafeLength > 0) {
|
|
10211
|
-
const safeText = textBuffer.slice(0, maxSafeLength);
|
|
10212
|
-
fullReply += safeText;
|
|
10213
|
-
yield safeText;
|
|
10214
|
-
textBuffer = textBuffer.slice(maxSafeLength);
|
|
10215
|
-
}
|
|
10216
|
-
break;
|
|
10980
|
+
textBuffer += chunk;
|
|
10981
|
+
while (textBuffer.length > 0) {
|
|
10982
|
+
if (!inThinking) {
|
|
10983
|
+
const thinkIndex = textBuffer.indexOf("<think>");
|
|
10984
|
+
if (thinkIndex !== -1) {
|
|
10985
|
+
const before = textBuffer.slice(0, thinkIndex);
|
|
10986
|
+
if (before) {
|
|
10987
|
+
fullReply += before;
|
|
10988
|
+
yield before;
|
|
10217
10989
|
}
|
|
10990
|
+
inThinking = true;
|
|
10991
|
+
thinkingStartMs = performance.now();
|
|
10992
|
+
textBuffer = textBuffer.slice(thinkIndex + "<think>".length);
|
|
10218
10993
|
} else {
|
|
10219
|
-
const
|
|
10220
|
-
|
|
10221
|
-
|
|
10222
|
-
|
|
10223
|
-
|
|
10224
|
-
|
|
10225
|
-
|
|
10226
|
-
|
|
10227
|
-
|
|
10228
|
-
|
|
10229
|
-
|
|
10230
|
-
|
|
10231
|
-
|
|
10232
|
-
|
|
10233
|
-
|
|
10234
|
-
|
|
10235
|
-
|
|
10236
|
-
|
|
10237
|
-
|
|
10238
|
-
|
|
10994
|
+
const partialMatch = textBuffer.match(/<t?(?:h(?:i(?:n(?:k)?)?)?)?$/);
|
|
10995
|
+
const safeLen = partialMatch ? textBuffer.length - partialMatch[0].length : textBuffer.length;
|
|
10996
|
+
if (safeLen > 0) {
|
|
10997
|
+
const safeText = textBuffer.slice(0, safeLen);
|
|
10998
|
+
fullReply += safeText;
|
|
10999
|
+
yield safeText;
|
|
11000
|
+
textBuffer = textBuffer.slice(safeLen);
|
|
11001
|
+
}
|
|
11002
|
+
break;
|
|
11003
|
+
}
|
|
11004
|
+
} else {
|
|
11005
|
+
const endThinkIndex = textBuffer.indexOf("</think>");
|
|
11006
|
+
if (endThinkIndex !== -1) {
|
|
11007
|
+
const thinkingDelta = textBuffer.slice(0, endThinkIndex);
|
|
11008
|
+
if (thinkingDelta) {
|
|
11009
|
+
thinkingText += thinkingDelta;
|
|
11010
|
+
thinkingDurationMs = performance.now() - thinkingStartMs;
|
|
11011
|
+
yield { type: "thinking", text: thinkingDelta };
|
|
11012
|
+
}
|
|
11013
|
+
inThinking = false;
|
|
11014
|
+
textBuffer = textBuffer.slice(endThinkIndex + "</think>".length);
|
|
11015
|
+
} else {
|
|
11016
|
+
const partialMatch = textBuffer.match(/<\/?t?(?:h(?:i(?:n(?:k)?)?)?)?$/);
|
|
11017
|
+
const safeLen = partialMatch ? textBuffer.length - partialMatch[0].length : textBuffer.length;
|
|
11018
|
+
if (safeLen > 0) {
|
|
11019
|
+
const thinkingDelta = textBuffer.slice(0, safeLen);
|
|
11020
|
+
thinkingText += thinkingDelta;
|
|
11021
|
+
thinkingDurationMs = performance.now() - thinkingStartMs;
|
|
11022
|
+
yield { type: "thinking", text: thinkingDelta };
|
|
11023
|
+
textBuffer = textBuffer.slice(safeLen);
|
|
10239
11024
|
}
|
|
11025
|
+
break;
|
|
10240
11026
|
}
|
|
10241
11027
|
}
|
|
10242
|
-
} else {
|
|
10243
|
-
fullReply += chunk;
|
|
10244
|
-
yield chunk;
|
|
10245
11028
|
}
|
|
10246
11029
|
}
|
|
10247
11030
|
} catch (temp) {
|
|
@@ -10254,7 +11037,7 @@ ${context}`;
|
|
|
10254
11037
|
throw error[0];
|
|
10255
11038
|
}
|
|
10256
11039
|
}
|
|
10257
|
-
if (
|
|
11040
|
+
if (textBuffer.length > 0) {
|
|
10258
11041
|
if (inThinking) {
|
|
10259
11042
|
thinkingText += textBuffer;
|
|
10260
11043
|
yield { type: "thinking", text: textBuffer };
|
|
@@ -10265,22 +11048,21 @@ ${context}`;
|
|
|
10265
11048
|
}
|
|
10266
11049
|
} else {
|
|
10267
11050
|
const reply = yield new __await(this.llmProvider.chat(messages, context));
|
|
10268
|
-
|
|
10269
|
-
|
|
10270
|
-
|
|
10271
|
-
|
|
10272
|
-
|
|
10273
|
-
|
|
10274
|
-
|
|
10275
|
-
|
|
10276
|
-
|
|
10277
|
-
}
|
|
11051
|
+
const thinkStart = reply.indexOf("<think>");
|
|
11052
|
+
const thinkEnd = reply.indexOf("</think>");
|
|
11053
|
+
if (thinkStart !== -1 && thinkEnd !== -1 && thinkEnd > thinkStart) {
|
|
11054
|
+
thinkingText = reply.slice(thinkStart + "<think>".length, thinkEnd);
|
|
11055
|
+
fullReply = reply.slice(0, thinkStart) + reply.slice(thinkEnd + "</think>".length);
|
|
11056
|
+
thinkingDurationMs = 100;
|
|
11057
|
+
} else if (thinkStart !== -1) {
|
|
11058
|
+
thinkingText = reply.slice(thinkStart + "<think>".length);
|
|
11059
|
+
fullReply = reply.slice(0, thinkStart);
|
|
10278
11060
|
} else {
|
|
10279
11061
|
fullReply = reply;
|
|
10280
11062
|
}
|
|
10281
11063
|
yield fullReply;
|
|
10282
11064
|
}
|
|
10283
|
-
const runHallucination = ((
|
|
11065
|
+
const runHallucination = ((_w = this.config.llm.options) == null ? void 0 : _w.hallucinationScoring) === true;
|
|
10284
11066
|
if (runHallucination) {
|
|
10285
11067
|
hallucinationScoringPromise = scoreHallucination(this.llmProvider, fullReply, context).catch(() => void 0);
|
|
10286
11068
|
}
|
|
@@ -10302,89 +11084,45 @@ ${context}`;
|
|
|
10302
11084
|
totalTokens: promptTokens + completionTokens,
|
|
10303
11085
|
estimatedCostUsd: estimateCostUsd(promptTokens, completionTokens, this.config.llm.model)
|
|
10304
11086
|
};
|
|
10305
|
-
const awaitHallucination = ((
|
|
11087
|
+
const awaitHallucination = ((_x = this.config.llm.options) == null ? void 0 : _x.awaitHallucination) === true;
|
|
10306
11088
|
const [uiTransformation, hallucinationResult] = yield new __await(Promise.all([
|
|
10307
11089
|
uiTransformationPromise,
|
|
10308
11090
|
awaitHallucination ? hallucinationScoringPromise : Promise.resolve(void 0)
|
|
10309
11091
|
]));
|
|
10310
|
-
|
|
10311
|
-
|
|
10312
|
-
|
|
10313
|
-
|
|
10314
|
-
|
|
10315
|
-
|
|
10316
|
-
|
|
10317
|
-
|
|
10318
|
-
|
|
10319
|
-
|
|
10320
|
-
|
|
10321
|
-
|
|
10322
|
-
|
|
10323
|
-
|
|
10324
|
-
|
|
10325
|
-
|
|
10326
|
-
|
|
10327
|
-
|
|
10328
|
-
|
|
10329
|
-
|
|
10330
|
-
|
|
10331
|
-
|
|
10332
|
-
|
|
11092
|
+
RAGEvaluator.evaluate(effectiveQuestion, context, fullReply).catch(() => {
|
|
11093
|
+
});
|
|
11094
|
+
const buildTrace = (hScore) => {
|
|
11095
|
+
var _a3, _b2;
|
|
11096
|
+
return __spreadValues({
|
|
11097
|
+
requestId,
|
|
11098
|
+
query: question,
|
|
11099
|
+
rewrittenQuery: rewrittenQuery || ((promptOptResult == null ? void 0 : promptOptResult.wasModified) ? promptOptResult.optimizedPrompt : void 0),
|
|
11100
|
+
// systemPrompt / userPrompt intentionally excluded:
|
|
11101
|
+
// they may contain business logic or post-masking user PII and were
|
|
11102
|
+
// being emitted to the client browser via SSE (security risk).
|
|
11103
|
+
chunks: (sources || []).filter(Boolean).map((s) => {
|
|
11104
|
+
var _a4, _b3;
|
|
11105
|
+
return {
|
|
11106
|
+
id: s.id,
|
|
11107
|
+
score: s.score,
|
|
11108
|
+
content: (_a4 = s == null ? void 0 : s.content) != null ? _a4 : "",
|
|
11109
|
+
metadata: (_b3 = s == null ? void 0 : s.metadata) != null ? _b3 : {},
|
|
11110
|
+
namespace: ns
|
|
11111
|
+
};
|
|
11112
|
+
}),
|
|
11113
|
+
latency,
|
|
11114
|
+
tokens,
|
|
11115
|
+
// Always populate model/provider on the trace (fixes gap where telemetry
|
|
11116
|
+
// was reading finalTrace?.model with a fallback — now it's always set).
|
|
11117
|
+
model: (_a3 = this.config.llm) == null ? void 0 : _a3.model,
|
|
11118
|
+
provider: (_b2 = this.config.llm) == null ? void 0 : _b2.provider,
|
|
11119
|
+
timestamp: (/* @__PURE__ */ new Date()).toISOString()
|
|
11120
|
+
}, hScore ? {
|
|
11121
|
+
hallucinationScore: hScore.score,
|
|
11122
|
+
hallucinationReason: hScore.reason
|
|
11123
|
+
} : {});
|
|
11124
|
+
};
|
|
10333
11125
|
const trace = buildTrace(hallucinationResult);
|
|
10334
|
-
const isTelemetryActive = (_A = (_z = this.config.telemetry) == null ? void 0 : _z.enabled) != null ? _A : Boolean(this.config.licenseKey);
|
|
10335
|
-
if (isTelemetryActive) {
|
|
10336
|
-
const defaultUrl = "https://www.retrivora.com/api/telemetry";
|
|
10337
|
-
const telemetryUrl = ((_B = this.config.telemetry) == null ? void 0 : _B.url) || process.env.TELEMETRY_URL || process.env.NEXT_PUBLIC_TELEMETRY_URL || defaultUrl;
|
|
10338
|
-
const absoluteUrl = telemetryUrl.startsWith("http") ? telemetryUrl : "https://www.retrivora.com" + (telemetryUrl.startsWith("/") ? telemetryUrl : "/" + telemetryUrl);
|
|
10339
|
-
(async () => {
|
|
10340
|
-
var _a3, _b2, _c2, _d2, _e2;
|
|
10341
|
-
try {
|
|
10342
|
-
let finalTrace = trace;
|
|
10343
|
-
if (!awaitHallucination && runHallucination) {
|
|
10344
|
-
const backgroundScoreResult = await hallucinationScoringPromise;
|
|
10345
|
-
if (backgroundScoreResult) {
|
|
10346
|
-
finalTrace = buildTrace(backgroundScoreResult);
|
|
10347
|
-
}
|
|
10348
|
-
}
|
|
10349
|
-
const modelName = (finalTrace == null ? void 0 : finalTrace.model) || ((_a3 = this.config.llm) == null ? void 0 : _a3.model) || "llama-3.1-8b-instant";
|
|
10350
|
-
const providerName = (finalTrace == null ? void 0 : finalTrace.provider) || ((_b2 = this.config.llm) == null ? void 0 : _b2.provider) || "groq";
|
|
10351
|
-
const tokenCount = Number(((_c2 = finalTrace == null ? void 0 : finalTrace.tokens) == null ? void 0 : _c2.totalTokens) || 0);
|
|
10352
|
-
const costEst = Number(((_d2 = finalTrace == null ? void 0 : finalTrace.tokens) == null ? void 0 : _d2.estimatedCostUsd) || 0);
|
|
10353
|
-
const latencyDuration = Number(((_e2 = finalTrace == null ? void 0 : finalTrace.latency) == null ? void 0 : _e2.totalMs) || 0);
|
|
10354
|
-
if (typeof window === "undefined") {
|
|
10355
|
-
Logger.info(`[Retrivora Pipeline Telemetry] \u{1F4CA} Dispatching RAG telemetry -> Project: "${ns}", Tokens: ${tokenCount}, Latency: ${latencyDuration}ms`);
|
|
10356
|
-
}
|
|
10357
|
-
await fetch(absoluteUrl, {
|
|
10358
|
-
method: "POST",
|
|
10359
|
-
headers: {
|
|
10360
|
-
"Content-Type": "application/json"
|
|
10361
|
-
},
|
|
10362
|
-
body: JSON.stringify({
|
|
10363
|
-
trace: finalTrace,
|
|
10364
|
-
licenseKey: this.config.licenseKey,
|
|
10365
|
-
projectId: ns,
|
|
10366
|
-
organization: process.env.RETRIVORA_ORGANIZATION || "default-org",
|
|
10367
|
-
sdkVersion: SDK_VERSION,
|
|
10368
|
-
model: modelName,
|
|
10369
|
-
provider: providerName,
|
|
10370
|
-
tokens: tokenCount,
|
|
10371
|
-
cost: costEst,
|
|
10372
|
-
costUsd: costEst,
|
|
10373
|
-
latencyMs: latencyDuration,
|
|
10374
|
-
status: "success",
|
|
10375
|
-
action: "RAG_QUERY",
|
|
10376
|
-
feature: "RAG",
|
|
10377
|
-
timestamp: (/* @__PURE__ */ new Date()).toISOString(),
|
|
10378
|
-
details: `Org: default-org | Model: ${providerName}/${modelName} | Tokens: ${tokenCount} | Latency: ${latencyDuration}ms | Cost: $${costEst} | SDK: v${SDK_VERSION} | Feature: RAG`
|
|
10379
|
-
})
|
|
10380
|
-
}).catch((err) => {
|
|
10381
|
-
Logger.warn(`[Pipeline Telemetry] Failed to send trace async to ${absoluteUrl}`, { error: err.message });
|
|
10382
|
-
});
|
|
10383
|
-
} catch (err) {
|
|
10384
|
-
Logger.warn(`[Pipeline Telemetry] Failed to send trace async to ${absoluteUrl}:`, { error: err.message });
|
|
10385
|
-
}
|
|
10386
|
-
})();
|
|
10387
|
-
}
|
|
10388
11126
|
yield {
|
|
10389
11127
|
reply: "",
|
|
10390
11128
|
sources,
|
|
@@ -11195,6 +11933,374 @@ function createFromPreset(presetName) {
|
|
|
11195
11933
|
return new ConfigBuilder().vectorDb(preset.vectorDb, { apiKey: process.env[`${preset.vectorDb.toUpperCase()}_API_KEY`] }).llm(preset.llm, void 0, process.env[`${preset.llm.toUpperCase()}_API_KEY`]).embedding(preset.embedding, void 0, process.env[`${preset.embedding.toUpperCase()}_API_KEY`]);
|
|
11196
11934
|
}
|
|
11197
11935
|
|
|
11936
|
+
// src/utils/UniversalTableParser.ts
|
|
11937
|
+
var UniversalTableParser = class {
|
|
11938
|
+
/**
|
|
11939
|
+
* Parse a raw 2D grid of strings with complete matrix intelligence.
|
|
11940
|
+
*/
|
|
11941
|
+
static parseGrid(grid, sheetName = "Data") {
|
|
11942
|
+
if (!grid || grid.length === 0) return "";
|
|
11943
|
+
const cleanGrid = grid.map((row) => Array.isArray(row) ? row.map((c) => c !== null && c !== void 0 ? String(c).trim() : "") : []).filter((row) => row.some((cell) => cell.length > 0));
|
|
11944
|
+
if (cleanGrid.length === 0) return "";
|
|
11945
|
+
const { headerRows, dataRows } = this.splitHeaderAndDataRows(cleanGrid);
|
|
11946
|
+
const fusedHeaders = this.fuseHeaders(headerRows);
|
|
11947
|
+
const matrixInfo = this.analyzeMatrixStructure(fusedHeaders, dataRows);
|
|
11948
|
+
const sections = [`--- Sheet: ${sheetName} ---`];
|
|
11949
|
+
const markdownTable = this.generateMarkdownTable(fusedHeaders, dataRows);
|
|
11950
|
+
if (markdownTable) {
|
|
11951
|
+
sections.push(`
|
|
11952
|
+
## Table: ${sheetName}
|
|
11953
|
+
${markdownTable}`);
|
|
11954
|
+
}
|
|
11955
|
+
if (matrixInfo.isComparisonMatrix && matrixInfo.entityIndices.length > 0) {
|
|
11956
|
+
const entityProfiles = this.generateEntityProfiles(fusedHeaders, dataRows, matrixInfo);
|
|
11957
|
+
if (entityProfiles) {
|
|
11958
|
+
sections.push(`
|
|
11959
|
+
## Entity Comparison Profiles
|
|
11960
|
+
${entityProfiles}`);
|
|
11961
|
+
}
|
|
11962
|
+
const criteriaComparisons = this.generateCriteriaComparisons(fusedHeaders, dataRows, matrixInfo);
|
|
11963
|
+
if (criteriaComparisons) {
|
|
11964
|
+
sections.push(`
|
|
11965
|
+
## Criteria Breakdown & Comparisons
|
|
11966
|
+
${criteriaComparisons}`);
|
|
11967
|
+
}
|
|
11968
|
+
} else {
|
|
11969
|
+
const contextualRows = this.generateContextualRows(fusedHeaders, dataRows);
|
|
11970
|
+
if (contextualRows) {
|
|
11971
|
+
sections.push(`
|
|
11972
|
+
## Structured Records
|
|
11973
|
+
${contextualRows}`);
|
|
11974
|
+
}
|
|
11975
|
+
}
|
|
11976
|
+
return sections.join("\n\n");
|
|
11977
|
+
}
|
|
11978
|
+
/**
|
|
11979
|
+
* Parse a raw CSV / TSV text string into tri-perspective contextual format.
|
|
11980
|
+
*/
|
|
11981
|
+
static parseDelimitedText(text, title = "Table", delimiter) {
|
|
11982
|
+
if (!text || !text.trim()) return "";
|
|
11983
|
+
if (!delimiter) {
|
|
11984
|
+
const firstLine = text.split(/\r?\n/)[0] || "";
|
|
11985
|
+
const commaCount = (firstLine.match(/,/g) || []).length;
|
|
11986
|
+
const tabCount = (firstLine.match(/\t/g) || []).length;
|
|
11987
|
+
const semiCount = (firstLine.match(/;/g) || []).length;
|
|
11988
|
+
const pipeCount = (firstLine.match(/\|/g) || []).length;
|
|
11989
|
+
if (tabCount > commaCount && tabCount > semiCount) delimiter = " ";
|
|
11990
|
+
else if (semiCount > commaCount && semiCount > tabCount) delimiter = ";";
|
|
11991
|
+
else if (pipeCount > commaCount && pipeCount > semiCount) delimiter = "|";
|
|
11992
|
+
else delimiter = ",";
|
|
11993
|
+
}
|
|
11994
|
+
const lines = text.split(/\r?\n/);
|
|
11995
|
+
const grid = [];
|
|
11996
|
+
for (const line of lines) {
|
|
11997
|
+
if (!line.trim()) continue;
|
|
11998
|
+
const row = [];
|
|
11999
|
+
let inQuote = false;
|
|
12000
|
+
let current = "";
|
|
12001
|
+
for (let i = 0; i < line.length; i++) {
|
|
12002
|
+
const char = line[i];
|
|
12003
|
+
if (char === '"') {
|
|
12004
|
+
if (inQuote && line[i + 1] === '"') {
|
|
12005
|
+
current += '"';
|
|
12006
|
+
i++;
|
|
12007
|
+
} else {
|
|
12008
|
+
inQuote = !inQuote;
|
|
12009
|
+
}
|
|
12010
|
+
} else if (char === delimiter && !inQuote) {
|
|
12011
|
+
row.push(current.trim());
|
|
12012
|
+
current = "";
|
|
12013
|
+
} else {
|
|
12014
|
+
current += char;
|
|
12015
|
+
}
|
|
12016
|
+
}
|
|
12017
|
+
row.push(current.trim());
|
|
12018
|
+
grid.push(row);
|
|
12019
|
+
}
|
|
12020
|
+
return this.parseGrid(grid, title);
|
|
12021
|
+
}
|
|
12022
|
+
/**
|
|
12023
|
+
* Parse a SheetJS sheet object with complete merged cell and matrix intelligence.
|
|
12024
|
+
*/
|
|
12025
|
+
static parseSheet(sheet, sheetName, XLSX) {
|
|
12026
|
+
if (!sheet) return "";
|
|
12027
|
+
const grid = this.extractAndUnrollGrid(sheet, XLSX);
|
|
12028
|
+
if (!grid || grid.length === 0) {
|
|
12029
|
+
const csv = XLSX.utils.sheet_to_csv(sheet, { blankrows: false });
|
|
12030
|
+
return csv && csv.trim() ? `--- Sheet: ${sheetName} ---
|
|
12031
|
+
${csv.trim()}` : "";
|
|
12032
|
+
}
|
|
12033
|
+
return this.parseGrid(grid, sheetName);
|
|
12034
|
+
}
|
|
12035
|
+
/**
|
|
12036
|
+
* Unrolls SheetJS merged cell ranges (`!merges`) across the 2D grid
|
|
12037
|
+
* so no child cell is left blank.
|
|
12038
|
+
*/
|
|
12039
|
+
static extractAndUnrollGrid(sheet, XLSX) {
|
|
12040
|
+
var _a2;
|
|
12041
|
+
const rawGrid = XLSX.utils.sheet_to_json(sheet, { header: 1, blankrows: false, defval: "" });
|
|
12042
|
+
if (!rawGrid || rawGrid.length === 0) return [];
|
|
12043
|
+
const grid = rawGrid.map(
|
|
12044
|
+
(row) => (Array.isArray(row) ? row : []).map((cell) => cell !== null && cell !== void 0 ? String(cell).trim() : "")
|
|
12045
|
+
);
|
|
12046
|
+
const merges = sheet["!merges"] || [];
|
|
12047
|
+
for (const merge of merges) {
|
|
12048
|
+
const { s, e } = merge;
|
|
12049
|
+
const originVal = ((_a2 = grid[s.r]) == null ? void 0 : _a2[s.c]) || "";
|
|
12050
|
+
if (!originVal) continue;
|
|
12051
|
+
for (let r = s.r; r <= e.r; r++) {
|
|
12052
|
+
if (!grid[r]) grid[r] = [];
|
|
12053
|
+
for (let c = s.c; c <= e.c; c++) {
|
|
12054
|
+
if (!grid[r][c] || grid[r][c] === "") {
|
|
12055
|
+
grid[r][c] = originVal;
|
|
12056
|
+
}
|
|
12057
|
+
}
|
|
12058
|
+
}
|
|
12059
|
+
}
|
|
12060
|
+
return grid.filter((row) => row.some((cell) => cell.length > 0));
|
|
12061
|
+
}
|
|
12062
|
+
/**
|
|
12063
|
+
* Intelligently identify where header rows end and data rows begin.
|
|
12064
|
+
*/
|
|
12065
|
+
static splitHeaderAndDataRows(grid) {
|
|
12066
|
+
if (grid.length <= 1) {
|
|
12067
|
+
return { headerRows: grid, dataRows: [] };
|
|
12068
|
+
}
|
|
12069
|
+
let headerRowCount = 1;
|
|
12070
|
+
for (let r = 1; r < Math.min(4, grid.length); r++) {
|
|
12071
|
+
const row = grid[r];
|
|
12072
|
+
const nonEmptyCells = row.filter(Boolean);
|
|
12073
|
+
if (nonEmptyCells.length === 0) continue;
|
|
12074
|
+
const isDataRow = nonEmptyCells.some(
|
|
12075
|
+
(c) => /^-?\d+(?:\.\d+)?%?$/.test(c) || /^\$?[\d,.]+(?:\s*(?:mil|k|m|b|usd|eur))?$/i.test(c) || /^(yes|no|true|false|pass|fail|compliant|non-compliant)$/i.test(c)
|
|
12076
|
+
);
|
|
12077
|
+
const hasSubheaderLabels = nonEmptyCells.some(
|
|
12078
|
+
(c) => /\b(bidder|vendor|supplier|score|weight|criteria|item|specification|technical|commercial|rating|status|cost|price|qty|unit)\b/i.test(c)
|
|
12079
|
+
);
|
|
12080
|
+
if (!isDataRow && hasSubheaderLabels) {
|
|
12081
|
+
headerRowCount = r + 1;
|
|
12082
|
+
} else if (!isDataRow && r === 1 && nonEmptyCells.length >= 2) {
|
|
12083
|
+
headerRowCount = 2;
|
|
12084
|
+
} else {
|
|
12085
|
+
break;
|
|
12086
|
+
}
|
|
12087
|
+
}
|
|
12088
|
+
const headerRows = grid.slice(0, headerRowCount);
|
|
12089
|
+
const dataRows = grid.slice(headerRowCount);
|
|
12090
|
+
return { headerRows, dataRows };
|
|
12091
|
+
}
|
|
12092
|
+
/**
|
|
12093
|
+
* Fuses multi-tier header rows into composite paths (e.g. `Commercial > Payment Terms > Bidder A`).
|
|
12094
|
+
*/
|
|
12095
|
+
static fuseHeaders(headerRows) {
|
|
12096
|
+
var _a2, _b;
|
|
12097
|
+
if (headerRows.length === 0) return [];
|
|
12098
|
+
if (headerRows.length === 1) {
|
|
12099
|
+
return headerRows[0].map((h, i) => h || `Column_${i + 1}`);
|
|
12100
|
+
}
|
|
12101
|
+
const maxCols = Math.max(...headerRows.map((r) => r.length));
|
|
12102
|
+
const fused = [];
|
|
12103
|
+
for (let c = 0; c < maxCols; c++) {
|
|
12104
|
+
const components = [];
|
|
12105
|
+
for (let r = 0; r < headerRows.length; r++) {
|
|
12106
|
+
const val = (_b = (_a2 = headerRows[r]) == null ? void 0 : _a2[c]) == null ? void 0 : _b.trim();
|
|
12107
|
+
if (val && !components.includes(val)) {
|
|
12108
|
+
components.push(val);
|
|
12109
|
+
}
|
|
12110
|
+
}
|
|
12111
|
+
fused.push(components.join(" > ") || `Column_${c + 1}`);
|
|
12112
|
+
}
|
|
12113
|
+
return fused;
|
|
12114
|
+
}
|
|
12115
|
+
/**
|
|
12116
|
+
* Analyzes whether the table is a comparison matrix (columns = entities, rows = criteria).
|
|
12117
|
+
*/
|
|
12118
|
+
static analyzeMatrixStructure(headers, dataRows) {
|
|
12119
|
+
const entityPattern = /\b(bidder|vendor|supplier|candidate|option|company|contractor|partner|proposal|firm|provider|solution|tier|package)\b/i;
|
|
12120
|
+
const entityIndices = [];
|
|
12121
|
+
const entityNames = [];
|
|
12122
|
+
let criteriaColumnIndex = 0;
|
|
12123
|
+
headers.forEach((h, idx) => {
|
|
12124
|
+
if (entityPattern.test(h) || /\b(bidder\s*\d+|vendor\s*[a-z0-9]+|option\s*[a-z0-9]+)\b/i.test(h)) {
|
|
12125
|
+
entityIndices.push(idx);
|
|
12126
|
+
const parts = h.split(" > ");
|
|
12127
|
+
entityNames.push(parts[parts.length - 1]);
|
|
12128
|
+
} else if (/\b(criteria|criterion|description|specification|category|requirement|item|feature)\b/i.test(h)) {
|
|
12129
|
+
criteriaColumnIndex = idx;
|
|
12130
|
+
}
|
|
12131
|
+
});
|
|
12132
|
+
const isComparisonMatrix = entityIndices.length >= 2 || headers.length >= 3 && entityIndices.length >= 1;
|
|
12133
|
+
return {
|
|
12134
|
+
isComparisonMatrix,
|
|
12135
|
+
entityIndices: entityIndices.length > 0 ? entityIndices : headers.slice(1).map((_, i) => i + 1),
|
|
12136
|
+
entityNames: entityNames.length > 0 ? entityNames : headers.slice(1),
|
|
12137
|
+
criteriaColumnIndex
|
|
12138
|
+
};
|
|
12139
|
+
}
|
|
12140
|
+
/**
|
|
12141
|
+
* Perspective A: Clean Markdown Table
|
|
12142
|
+
*/
|
|
12143
|
+
static generateMarkdownTable(headers, dataRows) {
|
|
12144
|
+
if (headers.length === 0 || dataRows.length === 0) return "";
|
|
12145
|
+
const cleanHeaders = headers.map((h) => h.replace(/\|/g, "-"));
|
|
12146
|
+
const lines = [];
|
|
12147
|
+
lines.push(`| ${cleanHeaders.join(" | ")} |`);
|
|
12148
|
+
lines.push(`| ${cleanHeaders.map(() => "---").join(" | ")} |`);
|
|
12149
|
+
for (const row of dataRows) {
|
|
12150
|
+
const paddedRow = headers.map((_, i) => (row[i] || "").replace(/\|/g, "-"));
|
|
12151
|
+
if (paddedRow.some((cell) => cell.trim().length > 0)) {
|
|
12152
|
+
lines.push(`| ${paddedRow.join(" | ")} |`);
|
|
12153
|
+
}
|
|
12154
|
+
}
|
|
12155
|
+
return lines.join("\n");
|
|
12156
|
+
}
|
|
12157
|
+
/**
|
|
12158
|
+
* Perspective B: Entity-Centric Profiles (e.g. Bidder 1, Bidder 2 full summary)
|
|
12159
|
+
*/
|
|
12160
|
+
static generateEntityProfiles(headers, dataRows, matrixInfo) {
|
|
12161
|
+
const profiles = [];
|
|
12162
|
+
matrixInfo.entityIndices.forEach((colIdx, i) => {
|
|
12163
|
+
const entityName = matrixInfo.entityNames[i] || headers[colIdx] || `Entity ${i + 1}`;
|
|
12164
|
+
const lines = [`### Profile & Evaluation: ${entityName}`];
|
|
12165
|
+
for (const row of dataRows) {
|
|
12166
|
+
const criterion = row[matrixInfo.criteriaColumnIndex] || row[0] || "Specification";
|
|
12167
|
+
const val = row[colIdx];
|
|
12168
|
+
if (val !== void 0 && val !== null && String(val).trim() !== "") {
|
|
12169
|
+
lines.push(`- **${criterion}**: ${String(val).trim()}`);
|
|
12170
|
+
}
|
|
12171
|
+
}
|
|
12172
|
+
if (lines.length > 1) {
|
|
12173
|
+
profiles.push(lines.join("\n"));
|
|
12174
|
+
}
|
|
12175
|
+
});
|
|
12176
|
+
return profiles.join("\n\n");
|
|
12177
|
+
}
|
|
12178
|
+
/**
|
|
12179
|
+
* Perspective C: Criteria Cross-Comparisons (e.g. Commercial Bid comparison across all Bidders)
|
|
12180
|
+
*/
|
|
12181
|
+
static generateCriteriaComparisons(headers, dataRows, matrixInfo) {
|
|
12182
|
+
const comparisons = [];
|
|
12183
|
+
for (const row of dataRows) {
|
|
12184
|
+
const criterion = row[matrixInfo.criteriaColumnIndex] || row[0] || "Metric";
|
|
12185
|
+
const entityPairs = [];
|
|
12186
|
+
matrixInfo.entityIndices.forEach((colIdx, i) => {
|
|
12187
|
+
const entityName = matrixInfo.entityNames[i] || headers[colIdx] || `Entity ${i + 1}`;
|
|
12188
|
+
const val = row[colIdx];
|
|
12189
|
+
if (val !== void 0 && val !== null && String(val).trim() !== "") {
|
|
12190
|
+
entityPairs.push(`${entityName}: ${String(val).trim()}`);
|
|
12191
|
+
}
|
|
12192
|
+
});
|
|
12193
|
+
if (entityPairs.length > 0) {
|
|
12194
|
+
comparisons.push(`- **${criterion}** $\\rightarrow$ ${entityPairs.join(" | ")}`);
|
|
12195
|
+
}
|
|
12196
|
+
}
|
|
12197
|
+
return comparisons.join("\n");
|
|
12198
|
+
}
|
|
12199
|
+
/**
|
|
12200
|
+
* Standard contextual key-value rows for general tabular records.
|
|
12201
|
+
*/
|
|
12202
|
+
static generateContextualRows(headers, dataRows) {
|
|
12203
|
+
const lines = [];
|
|
12204
|
+
for (const row of dataRows) {
|
|
12205
|
+
const pairs = [];
|
|
12206
|
+
headers.forEach((h, i) => {
|
|
12207
|
+
const val = row[i];
|
|
12208
|
+
if (val !== void 0 && val !== null && String(val).trim() !== "") {
|
|
12209
|
+
pairs.push(`${h}: ${String(val).trim()}`);
|
|
12210
|
+
}
|
|
12211
|
+
});
|
|
12212
|
+
if (pairs.length > 0) {
|
|
12213
|
+
lines.push(pairs.join(" | "));
|
|
12214
|
+
}
|
|
12215
|
+
}
|
|
12216
|
+
return lines.join("\n");
|
|
12217
|
+
}
|
|
12218
|
+
};
|
|
12219
|
+
|
|
12220
|
+
// src/utils/VisionExtractor.ts
|
|
12221
|
+
var VisionExtractor = class {
|
|
12222
|
+
/**
|
|
12223
|
+
* Check if a buffer represents an image format.
|
|
12224
|
+
*/
|
|
12225
|
+
static isImageBuffer(buffer, fileName = "", mimeType = "") {
|
|
12226
|
+
var _a2;
|
|
12227
|
+
const ext = ((_a2 = fileName.split(".").pop()) == null ? void 0 : _a2.toLowerCase()) || "";
|
|
12228
|
+
if (["png", "jpg", "jpeg", "webp", "bmp", "tiff", "gif"].includes(ext)) {
|
|
12229
|
+
return true;
|
|
12230
|
+
}
|
|
12231
|
+
if (mimeType.startsWith("image/")) {
|
|
12232
|
+
return true;
|
|
12233
|
+
}
|
|
12234
|
+
if (buffer && buffer.length >= 4) {
|
|
12235
|
+
if (buffer[0] === 137 && buffer[1] === 80 && buffer[2] === 78 && buffer[3] === 71) return true;
|
|
12236
|
+
if (buffer[0] === 255 && buffer[1] === 216 && buffer[2] === 255) return true;
|
|
12237
|
+
if (buffer[0] === 82 && buffer[1] === 73 && buffer[2] === 70 && buffer[3] === 70) return true;
|
|
12238
|
+
}
|
|
12239
|
+
return false;
|
|
12240
|
+
}
|
|
12241
|
+
/**
|
|
12242
|
+
* Extract rich textual content, tables, and diagram descriptions from an image buffer using Vision LLM.
|
|
12243
|
+
*/
|
|
12244
|
+
static async extractFromImage(buffer, fileName = "document.png", options = {}) {
|
|
12245
|
+
var _a2, _b, _c, _d, _e, _f, _g2, _h, _i;
|
|
12246
|
+
const base64Image = buffer.toString("base64");
|
|
12247
|
+
const ext = ((_a2 = fileName.split(".").pop()) == null ? void 0 : _a2.toLowerCase()) || "png";
|
|
12248
|
+
const mime = ext === "jpg" || ext === "jpeg" ? "image/jpeg" : ext === "webp" ? "image/webp" : "image/png";
|
|
12249
|
+
const apiKey = options.apiKey || (typeof process !== "undefined" ? ((_b = process.env) == null ? void 0 : _b.OPENAI_API_KEY) || ((_c = process.env) == null ? void 0 : _c.GEMINI_API_KEY) || ((_d = process.env) == null ? void 0 : _d.ANTHROPIC_API_KEY) : void 0);
|
|
12250
|
+
if (!apiKey) {
|
|
12251
|
+
return `[Scanned Document / Image: ${fileName} - Size: ${(buffer.length / 1024).toFixed(1)} KB]`;
|
|
12252
|
+
}
|
|
12253
|
+
try {
|
|
12254
|
+
const response = await fetch("https://api.openai.com/v1/chat/completions", {
|
|
12255
|
+
method: "POST",
|
|
12256
|
+
headers: {
|
|
12257
|
+
"Authorization": `Bearer ${apiKey}`,
|
|
12258
|
+
"Content-Type": "application/json"
|
|
12259
|
+
},
|
|
12260
|
+
body: JSON.stringify({
|
|
12261
|
+
model: options.model || "gpt-4o-mini",
|
|
12262
|
+
messages: [
|
|
12263
|
+
{
|
|
12264
|
+
role: "system",
|
|
12265
|
+
content: "You are an expert Document OCR & Data Extraction engine. Transcribe all text, numbers, headings, tables, and diagram structures in this image into clean, structured GitHub Markdown. For tables, use standard markdown table syntax. Preserve exact numbers, dates, and column headers. Do not summarize \u2014 transcribe faithfully."
|
|
12266
|
+
},
|
|
12267
|
+
{
|
|
12268
|
+
role: "user",
|
|
12269
|
+
content: [
|
|
12270
|
+
{
|
|
12271
|
+
type: "text",
|
|
12272
|
+
text: `Please transcribe the content of this document/image (${fileName}):`
|
|
12273
|
+
},
|
|
12274
|
+
{
|
|
12275
|
+
type: "image_url",
|
|
12276
|
+
image_url: {
|
|
12277
|
+
url: `data:${mime};base64,${base64Image}`,
|
|
12278
|
+
detail: "high"
|
|
12279
|
+
}
|
|
12280
|
+
}
|
|
12281
|
+
]
|
|
12282
|
+
}
|
|
12283
|
+
],
|
|
12284
|
+
max_tokens: options.maxTokens || 2048,
|
|
12285
|
+
temperature: (_e = options.temperature) != null ? _e : 0.1
|
|
12286
|
+
})
|
|
12287
|
+
});
|
|
12288
|
+
if (response.ok) {
|
|
12289
|
+
const data = await response.json();
|
|
12290
|
+
const text = (_i = (_h = (_g2 = (_f = data.choices) == null ? void 0 : _f[0]) == null ? void 0 : _g2.message) == null ? void 0 : _h.content) == null ? void 0 : _i.trim();
|
|
12291
|
+
if (text) {
|
|
12292
|
+
return `--- Document / Image OCR: ${fileName} ---
|
|
12293
|
+
|
|
12294
|
+
${text}`;
|
|
12295
|
+
}
|
|
12296
|
+
}
|
|
12297
|
+
} catch (err) {
|
|
12298
|
+
console.warn("[VisionExtractor] OpenAI Vision OCR failed:", err);
|
|
12299
|
+
}
|
|
12300
|
+
return `[Scanned Image Document: ${fileName}]`;
|
|
12301
|
+
}
|
|
12302
|
+
};
|
|
12303
|
+
|
|
11198
12304
|
// src/utils/DocumentParser.ts
|
|
11199
12305
|
var DocumentParser = class {
|
|
11200
12306
|
/**
|
|
@@ -11218,52 +12324,11 @@ var DocumentParser = class {
|
|
|
11218
12324
|
return "text";
|
|
11219
12325
|
}
|
|
11220
12326
|
/**
|
|
11221
|
-
* Convert spreadsheet sheets into high-fidelity, context-preserved tabular text.
|
|
11222
|
-
*
|
|
12327
|
+
* Convert spreadsheet sheets into high-fidelity, context-preserved tabular text via UniversalTableParser.
|
|
12328
|
+
* Unrolls merged cells, fuses multi-level headers, and generates tri-perspective comparison profiles.
|
|
11223
12329
|
*/
|
|
11224
12330
|
static formatSheetToContextualText(sheet, sheetName, XLSX) {
|
|
11225
|
-
|
|
11226
|
-
if (!jsonRows || jsonRows.length === 0) {
|
|
11227
|
-
const csv = XLSX.utils.sheet_to_csv(sheet, { blankrows: false });
|
|
11228
|
-
return csv && csv.trim() ? `--- Sheet: ${sheetName} ---
|
|
11229
|
-
${csv.trim()}` : "";
|
|
11230
|
-
}
|
|
11231
|
-
const lines = [`--- Sheet: ${sheetName} ---`];
|
|
11232
|
-
let currentHeader = [];
|
|
11233
|
-
for (let r = 0; r < jsonRows.length; r++) {
|
|
11234
|
-
const row = jsonRows[r];
|
|
11235
|
-
if (!row || row.length === 0) continue;
|
|
11236
|
-
const cells = row.map((c) => c !== null && c !== void 0 ? String(c).trim() : "").filter(Boolean);
|
|
11237
|
-
if (cells.length === 0) continue;
|
|
11238
|
-
if (cells.length === 1 || cells.length <= 2 && cells[0].toUpperCase().includes("STEP")) {
|
|
11239
|
-
lines.push(`
|
|
11240
|
-
## ${cells.join(" ")}`);
|
|
11241
|
-
continue;
|
|
11242
|
-
}
|
|
11243
|
-
const isHeaderCandidate = cells.every((c) => isNaN(Number(c)) && !/^\$?[\d,.]+/.test(c));
|
|
11244
|
-
if (isHeaderCandidate && cells.length >= 2) {
|
|
11245
|
-
currentHeader = row.map((c) => c !== null && c !== void 0 ? String(c).trim() : "");
|
|
11246
|
-
lines.push(`| ${cells.join(" | ")} |`);
|
|
11247
|
-
lines.push(`| ${cells.map(() => "---").join(" | ")} |`);
|
|
11248
|
-
continue;
|
|
11249
|
-
}
|
|
11250
|
-
if (currentHeader.length > 0 && cells.length > 1) {
|
|
11251
|
-
const fieldPairs = [];
|
|
11252
|
-
for (let c = 0; c < row.length; c++) {
|
|
11253
|
-
const h = currentHeader[c];
|
|
11254
|
-
const val = row[c];
|
|
11255
|
-
if (h && val !== null && val !== void 0 && String(val).trim() !== "") {
|
|
11256
|
-
fieldPairs.push(`${h}: ${String(val).trim()}`);
|
|
11257
|
-
}
|
|
11258
|
-
}
|
|
11259
|
-
if (fieldPairs.length > 0) {
|
|
11260
|
-
lines.push(fieldPairs.join(" | "));
|
|
11261
|
-
continue;
|
|
11262
|
-
}
|
|
11263
|
-
}
|
|
11264
|
-
lines.push(cells.join(", "));
|
|
11265
|
-
}
|
|
11266
|
-
return lines.join("\n");
|
|
12331
|
+
return UniversalTableParser.parseSheet(sheet, sheetName, XLSX);
|
|
11267
12332
|
}
|
|
11268
12333
|
/**
|
|
11269
12334
|
* Extract text from a File or Buffer based on its type.
|
|
@@ -11287,7 +12352,9 @@ ${csv.trim()}` : "";
|
|
|
11287
12352
|
return this.readAsText(file);
|
|
11288
12353
|
}
|
|
11289
12354
|
if (extension === "csv" || extension === "tsv" || mimeType === "text/csv" || mimeType === "text/tab-separated-values") {
|
|
11290
|
-
|
|
12355
|
+
const text = await this.readAsText(file);
|
|
12356
|
+
const parsed = UniversalTableParser.parseDelimitedText(text, fileName);
|
|
12357
|
+
return parsed || text;
|
|
11291
12358
|
}
|
|
11292
12359
|
if (extension === "pptx" || extension === "ppt" || extension === "odp" || mimeType.includes("presentation") || mimeType.includes("powerpoint")) {
|
|
11293
12360
|
try {
|
|
@@ -11394,16 +12461,22 @@ ${csv.trim()}`);
|
|
|
11394
12461
|
return this.parseLegacyExcelOrFallback(buffer, fileName);
|
|
11395
12462
|
}
|
|
11396
12463
|
if (extension === "pdf" || mimeType === "application/pdf") {
|
|
12464
|
+
const buffer = Buffer.isBuffer(file) ? file : Buffer.from(await file.arrayBuffer());
|
|
11397
12465
|
try {
|
|
11398
12466
|
const pdfModule = await import("pdf-parse");
|
|
11399
|
-
const buffer = Buffer.isBuffer(file) ? file : Buffer.from(await file.arrayBuffer());
|
|
11400
12467
|
const pdf = pdfModule.default || pdfModule;
|
|
11401
12468
|
const data = await pdf(buffer);
|
|
11402
|
-
|
|
12469
|
+
if (data.text && data.text.trim().length > 30) {
|
|
12470
|
+
return data.text;
|
|
12471
|
+
}
|
|
11403
12472
|
} catch (err) {
|
|
11404
|
-
console.warn(
|
|
11405
|
-
return `[PDF Parsing Error for ${fileName}]`;
|
|
12473
|
+
console.warn("[DocumentParser] pdf-parse failed, attempting Vision OCR fallback:", err);
|
|
11406
12474
|
}
|
|
12475
|
+
return VisionExtractor.extractFromImage(buffer, fileName);
|
|
12476
|
+
}
|
|
12477
|
+
if (["png", "jpg", "jpeg", "webp", "bmp", "tiff", "gif"].includes(extension) || mimeType.startsWith("image/")) {
|
|
12478
|
+
const buffer = Buffer.isBuffer(file) ? file : Buffer.from(await file.arrayBuffer());
|
|
12479
|
+
return VisionExtractor.extractFromImage(buffer, fileName);
|
|
11407
12480
|
}
|
|
11408
12481
|
if (extension === "docx" || extension === "doc" || extension === "odt" || extension === "rtf" || mimeType === "application/vnd.openxmlformats-officedocument.wordprocessingml.document" || mimeType === "application/msword" || mimeType === "application/rtf") {
|
|
11409
12482
|
try {
|
|
@@ -11707,8 +12780,8 @@ init_RedisProvider();
|
|
|
11707
12780
|
init_WeaviateProvider();
|
|
11708
12781
|
init_UniversalVectorProvider();
|
|
11709
12782
|
|
|
11710
|
-
// src/handlers/
|
|
11711
|
-
var
|
|
12783
|
+
// src/handlers/chatHandler.ts
|
|
12784
|
+
var import_server3 = require("next/server");
|
|
11712
12785
|
|
|
11713
12786
|
// src/core/DatabaseStorage.ts
|
|
11714
12787
|
var fs = __toESM(require("fs"));
|
|
@@ -11761,44 +12834,38 @@ var _DatabaseStorage = class _DatabaseStorage {
|
|
|
11761
12834
|
*
|
|
11762
12835
|
* In non-production environments this is a complete no-op so local
|
|
11763
12836
|
* development works without a license key.
|
|
12837
|
+
/**
|
|
12838
|
+
* Checks if the caller holds a valid paid-tier license in production.
|
|
12839
|
+
* Returns true if allowed, false if not supported on current tier.
|
|
11764
12840
|
*/
|
|
11765
|
-
async
|
|
11766
|
-
if (process.env.NODE_ENV !== "production") return;
|
|
11767
|
-
if (
|
|
11768
|
-
if (
|
|
11769
|
-
|
|
11770
|
-
"[Retrivora SDK] Chat History and Feedback features require a paid license key in production."
|
|
11771
|
-
);
|
|
12841
|
+
async isPaidSubscription() {
|
|
12842
|
+
if (process.env.NODE_ENV !== "production") return true;
|
|
12843
|
+
if (!this.config.licenseKey) return false;
|
|
12844
|
+
if (Date.now() - this.subscriptionCheckedAt < _DatabaseStorage.SUBSCRIPTION_CACHE_TTL_MS) {
|
|
12845
|
+
return true;
|
|
11772
12846
|
}
|
|
11773
12847
|
try {
|
|
11774
12848
|
const payload = await LicenseVerifier.verifyAsync(this.config.licenseKey, this.config.projectId);
|
|
11775
12849
|
const tier = (payload.tier || "").toLowerCase();
|
|
11776
|
-
const PAID_TIERS = /* @__PURE__ */ new Set(["premium", "enterprise", "growth", "pro", "developer_pro"]);
|
|
11777
|
-
|
|
11778
|
-
|
|
11779
|
-
|
|
11780
|
-
);
|
|
12850
|
+
const PAID_TIERS = /* @__PURE__ */ new Set(["premium", "enterprise", "growth", "pro", "developer_pro", "free_trial"]);
|
|
12851
|
+
const isAllowed = PAID_TIERS.has(tier) || tier.includes("pro") || tier.includes("enterprise");
|
|
12852
|
+
if (isAllowed) {
|
|
12853
|
+
this.subscriptionCheckedAt = Date.now();
|
|
11781
12854
|
}
|
|
11782
|
-
|
|
11783
|
-
} catch (
|
|
11784
|
-
|
|
11785
|
-
`[Retrivora SDK] Subscription check failed: ${err instanceof Error ? err.message : String(err)}`
|
|
11786
|
-
);
|
|
12855
|
+
return isAllowed;
|
|
12856
|
+
} catch (e) {
|
|
12857
|
+
return false;
|
|
11787
12858
|
}
|
|
11788
12859
|
}
|
|
11789
|
-
async
|
|
12860
|
+
async isHistoryAvailable() {
|
|
11790
12861
|
var _a2, _b;
|
|
11791
|
-
|
|
11792
|
-
|
|
11793
|
-
throw new Error("[Retrivora SDK] Chat History is disabled in RagConfig.");
|
|
11794
|
-
}
|
|
12862
|
+
if (((_b = (_a2 = this.config.rag) == null ? void 0 : _a2.history) == null ? void 0 : _b.enabled) === false) return false;
|
|
12863
|
+
return this.isPaidSubscription();
|
|
11795
12864
|
}
|
|
11796
|
-
async
|
|
12865
|
+
async isFeedbackAvailable() {
|
|
11797
12866
|
var _a2, _b;
|
|
11798
|
-
|
|
11799
|
-
|
|
11800
|
-
throw new Error("[Retrivora SDK] User Feedback is disabled in RagConfig.");
|
|
11801
|
-
}
|
|
12867
|
+
if (((_b = (_a2 = this.config.rag) == null ? void 0 : _a2.feedback) == null ? void 0 : _b.enabled) === false) return false;
|
|
12868
|
+
return this.isPaidSubscription();
|
|
11802
12869
|
}
|
|
11803
12870
|
/**
|
|
11804
12871
|
* Returns (or lazily creates) the PostgreSQL connection pool for this project.
|
|
@@ -11912,7 +12979,7 @@ var _DatabaseStorage = class _DatabaseStorage {
|
|
|
11912
12979
|
}
|
|
11913
12980
|
// ─── Message History Operations ──────────────────────────────────────────
|
|
11914
12981
|
async saveMessage(sessionId, message) {
|
|
11915
|
-
await this.
|
|
12982
|
+
if (!await this.isHistoryAvailable()) return;
|
|
11916
12983
|
const createdAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
11917
12984
|
const msgId = message.id || `msg_${Date.now()}_${Math.random().toString(36).slice(2, 6)}`;
|
|
11918
12985
|
if (this.provider === "postgresql" || this.provider === "pgvector") {
|
|
@@ -11975,7 +13042,7 @@ var _DatabaseStorage = class _DatabaseStorage {
|
|
|
11975
13042
|
}
|
|
11976
13043
|
}
|
|
11977
13044
|
async getHistory(sessionId, projectIds) {
|
|
11978
|
-
await this.
|
|
13045
|
+
if (!await this.isHistoryAvailable()) return [];
|
|
11979
13046
|
if (projectIds && projectIds.length > 0 && !this.sessionInScope(sessionId, projectIds)) {
|
|
11980
13047
|
return [];
|
|
11981
13048
|
}
|
|
@@ -12009,7 +13076,7 @@ var _DatabaseStorage = class _DatabaseStorage {
|
|
|
12009
13076
|
}
|
|
12010
13077
|
}
|
|
12011
13078
|
async clearHistory(sessionId, projectIds) {
|
|
12012
|
-
await this.
|
|
13079
|
+
if (!await this.isHistoryAvailable()) return;
|
|
12013
13080
|
if (projectIds && projectIds.length > 0 && !this.sessionInScope(sessionId, projectIds)) {
|
|
12014
13081
|
return;
|
|
12015
13082
|
}
|
|
@@ -12049,7 +13116,7 @@ var _DatabaseStorage = class _DatabaseStorage {
|
|
|
12049
13116
|
});
|
|
12050
13117
|
}
|
|
12051
13118
|
async listSessions(projectIds) {
|
|
12052
|
-
await this.
|
|
13119
|
+
if (!await this.isHistoryAvailable()) return [];
|
|
12053
13120
|
let raw = [];
|
|
12054
13121
|
if (this.provider === "postgresql" || this.provider === "pgvector") {
|
|
12055
13122
|
const pool = await this.getPGPool();
|
|
@@ -12068,7 +13135,7 @@ var _DatabaseStorage = class _DatabaseStorage {
|
|
|
12068
13135
|
}
|
|
12069
13136
|
// ─── Feedback Operations ──────────────────────────────────────────────────
|
|
12070
13137
|
async saveFeedback(feedback) {
|
|
12071
|
-
await this.
|
|
13138
|
+
if (!await this.isFeedbackAvailable()) return;
|
|
12072
13139
|
const id = `fb_${Date.now()}_${Math.random().toString(36).slice(2, 6)}`;
|
|
12073
13140
|
const createdAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
12074
13141
|
if (this.provider === "postgresql" || this.provider === "pgvector") {
|
|
@@ -12118,7 +13185,7 @@ var _DatabaseStorage = class _DatabaseStorage {
|
|
|
12118
13185
|
}
|
|
12119
13186
|
}
|
|
12120
13187
|
async getFeedback(messageId, projectIds) {
|
|
12121
|
-
await this.
|
|
13188
|
+
if (!await this.isFeedbackAvailable()) return null;
|
|
12122
13189
|
let item = null;
|
|
12123
13190
|
if (this.provider === "postgresql" || this.provider === "pgvector") {
|
|
12124
13191
|
const pool = await this.getPGPool();
|
|
@@ -12154,7 +13221,7 @@ var _DatabaseStorage = class _DatabaseStorage {
|
|
|
12154
13221
|
return item;
|
|
12155
13222
|
}
|
|
12156
13223
|
async listFeedback(projectIds) {
|
|
12157
|
-
await this.
|
|
13224
|
+
if (!await this.isFeedbackAvailable()) return [];
|
|
12158
13225
|
let raw = [];
|
|
12159
13226
|
if (this.provider === "postgresql" || this.provider === "pgvector") {
|
|
12160
13227
|
const pool = await this.getPGPool();
|
|
@@ -12198,9 +13265,8 @@ var _DatabaseStorage = class _DatabaseStorage {
|
|
|
12198
13265
|
_DatabaseStorage.SUBSCRIPTION_CACHE_TTL_MS = 5 * 60 * 1e3;
|
|
12199
13266
|
var DatabaseStorage = _DatabaseStorage;
|
|
12200
13267
|
|
|
12201
|
-
// src/handlers/
|
|
12202
|
-
|
|
12203
|
-
var import_crypto2 = require("crypto");
|
|
13268
|
+
// src/handlers/internal/auth.ts
|
|
13269
|
+
var import_server = require("next/server");
|
|
12204
13270
|
async function checkAuth(req, onAuthorize) {
|
|
12205
13271
|
if (!onAuthorize) return null;
|
|
12206
13272
|
try {
|
|
@@ -12217,6 +13283,8 @@ async function checkAuth(req, onAuthorize) {
|
|
|
12217
13283
|
return import_server.NextResponse.json({ error: msg }, { status: 401 });
|
|
12218
13284
|
}
|
|
12219
13285
|
}
|
|
13286
|
+
|
|
13287
|
+
// src/handlers/internal/rateLimit.ts
|
|
12220
13288
|
var RateLimiter = class {
|
|
12221
13289
|
constructor(windowMs = 6e4, max = 30) {
|
|
12222
13290
|
this.hits = /* @__PURE__ */ new Map();
|
|
@@ -12278,6 +13346,9 @@ function checkRateLimit(req) {
|
|
|
12278
13346
|
}
|
|
12279
13347
|
return null;
|
|
12280
13348
|
}
|
|
13349
|
+
|
|
13350
|
+
// src/handlers/internal/license.ts
|
|
13351
|
+
var import_server2 = require("next/server");
|
|
12281
13352
|
var FREE_TIER_NAMES = /* @__PURE__ */ new Set(["hobby", "free", "free_trial", "trial"]);
|
|
12282
13353
|
function isFreeTier(tier) {
|
|
12283
13354
|
if (!tier) return true;
|
|
@@ -12322,7 +13393,7 @@ async function enforceLicense(req, config, bodyLicenseKey) {
|
|
|
12322
13393
|
} catch (err) {
|
|
12323
13394
|
return {
|
|
12324
13395
|
ok: false,
|
|
12325
|
-
response:
|
|
13396
|
+
response: import_server2.NextResponse.json(
|
|
12326
13397
|
{ error: `License validation failed: ${err.message}` },
|
|
12327
13398
|
{ status: 401 }
|
|
12328
13399
|
)
|
|
@@ -12380,6 +13451,8 @@ function resolveLicensedNamespace(payload, requestedNamespace) {
|
|
|
12380
13451
|
const defaultNamespace = ((_a2 = payload.projectId) == null ? void 0 : _a2.startsWith("retrivora-")) ? payload.projectId : `retrivora-${payload.projectId || "default"}`;
|
|
12381
13452
|
return { ok: true, namespace: requested || defaultNamespace };
|
|
12382
13453
|
}
|
|
13454
|
+
|
|
13455
|
+
// src/handlers/internal/sanitize.ts
|
|
12383
13456
|
var MAX_MESSAGE_LENGTH = 8e3;
|
|
12384
13457
|
function sanitizeInput(raw) {
|
|
12385
13458
|
const stripped = raw.replace(/[\u0000-\u0008\u000B\u000C\u000E-\u001F\u007F-\u009F]/g, "");
|
|
@@ -12392,6 +13465,9 @@ function sanitizeInput(raw) {
|
|
|
12392
13465
|
}
|
|
12393
13466
|
return { ok: true, value: trimmed };
|
|
12394
13467
|
}
|
|
13468
|
+
|
|
13469
|
+
// src/handlers/internal/plugin.ts
|
|
13470
|
+
var import_crypto2 = require("crypto");
|
|
12395
13471
|
var _pluginCache = /* @__PURE__ */ new Map();
|
|
12396
13472
|
var PLUGIN_CACHE_TTL_MS = 10 * 60 * 1e3;
|
|
12397
13473
|
function configHash(config) {
|
|
@@ -12406,6 +13482,9 @@ function getOrCreatePlugin(configOrPlugin) {
|
|
|
12406
13482
|
_pluginCache.set(key, { plugin, createdAt: Date.now() });
|
|
12407
13483
|
return plugin;
|
|
12408
13484
|
}
|
|
13485
|
+
|
|
13486
|
+
// src/handlers/internal/telemetry.ts
|
|
13487
|
+
init_version();
|
|
12409
13488
|
var TELEMETRY_ALLOWED_HOSTS = /* @__PURE__ */ new Set([
|
|
12410
13489
|
"retrivora.com",
|
|
12411
13490
|
"www.retrivora.com"
|
|
@@ -12426,8 +13505,20 @@ function sanitizeTelemetryUrl(raw) {
|
|
|
12426
13505
|
return null;
|
|
12427
13506
|
}
|
|
12428
13507
|
}
|
|
12429
|
-
|
|
12430
|
-
|
|
13508
|
+
function classifyError(message) {
|
|
13509
|
+
const m = (message || "").toLowerCase();
|
|
13510
|
+
if (/license|revoked|terminated|suspended/.test(m)) return { errorCode: "LICENSE_ERROR", errorType: "LICENSE_ERROR" };
|
|
13511
|
+
if (/rate.?limit|429/.test(m)) return { errorCode: "RATE_LIMIT_ERROR", errorType: "RATE_LIMIT_ERROR" };
|
|
13512
|
+
if (/unauthorized|401|403/.test(m)) return { errorCode: "AUTH_ERROR", errorType: "AUTH_ERROR" };
|
|
13513
|
+
if (/embedding|embed.?fail/.test(m)) return { errorCode: "EMBEDDING_ERROR", errorType: "EMBEDDING_ERROR" };
|
|
13514
|
+
if (/vector|pinecone|qdrant|pgvector|chroma|milvus|mongodb/.test(m)) return { errorCode: "VECTOR_DB_ERROR", errorType: "VECTOR_DB_ERROR" };
|
|
13515
|
+
if (/llm|chat|generation|openai|groq|anthropic|gemini|mistral/.test(m)) return { errorCode: "LLM_ERROR", errorType: "LLM_ERROR" };
|
|
13516
|
+
if (/fetch|network|timeout|econnrefused|socket/.test(m)) return { errorCode: "NETWORK_ERROR", errorType: "NETWORK_ERROR" };
|
|
13517
|
+
if (/stream/.test(m)) return { errorCode: "STREAMING_ERROR", errorType: "STREAMING_ERROR" };
|
|
13518
|
+
return { errorCode: "INTERNAL_ERROR", errorType: "INTERNAL_ERROR" };
|
|
13519
|
+
}
|
|
13520
|
+
async function reportTelemetry(req, plugin, action, status, details, trace, extra) {
|
|
13521
|
+
var _a2, _b, _c, _d, _e, _f, _g2, _h, _i, _j, _k, _l, _m, _n, _o, _p;
|
|
12431
13522
|
try {
|
|
12432
13523
|
const config = plugin.getConfig();
|
|
12433
13524
|
const licenseKey = config.licenseKey || process.env.RETRIVORA_LICENSE_KEY || process.env.RAG_LICENSE_KEY;
|
|
@@ -12438,13 +13529,14 @@ async function reportTelemetry(req, plugin, action, status, details, trace) {
|
|
|
12438
13529
|
if (!telemetryUrl) return;
|
|
12439
13530
|
const userAgent = req.headers.get("user-agent") || `Retrivora-SDK/${SDK_VERSION}`;
|
|
12440
13531
|
const projectId = config.projectId || "default";
|
|
12441
|
-
const model = (trace == null ? void 0 : trace.model) || ((_a2 = config.llm) == null ? void 0 : _a2.model) ||
|
|
12442
|
-
const provider = (trace == null ? void 0 : trace.provider) || ((
|
|
12443
|
-
const tokens = Number(((
|
|
12444
|
-
const costUsd = Number(((
|
|
12445
|
-
const latencyMs = Number(((
|
|
13532
|
+
const model = (trace == null ? void 0 : trace.model) || ((_a2 = config.llm) == null ? void 0 : _a2.model) || "llama-3.1-8b-instant";
|
|
13533
|
+
const provider = (trace == null ? void 0 : trace.provider) || ((_b = config.llm) == null ? void 0 : _b.provider) || "groq";
|
|
13534
|
+
const tokens = Number(((_c = trace == null ? void 0 : trace.tokens) == null ? void 0 : _c.totalTokens) || (trace == null ? void 0 : trace.totalTokens) || 0);
|
|
13535
|
+
const costUsd = Number(((_d = trace == null ? void 0 : trace.tokens) == null ? void 0 : _d.estimatedCostUsd) || (trace == null ? void 0 : trace.costUsd) || 0);
|
|
13536
|
+
const latencyMs = Number(((_e = trace == null ? void 0 : trace.latency) == null ? void 0 : _e.totalMs) || (trace == null ? void 0 : trace.latencyMs) || 0);
|
|
13537
|
+
const sanitizedTrace = trace ? __spreadProps(__spreadValues({}, trace), { systemPrompt: void 0, userPrompt: void 0 }) : void 0;
|
|
12446
13538
|
const payload = {
|
|
12447
|
-
trace,
|
|
13539
|
+
trace: sanitizedTrace,
|
|
12448
13540
|
projectId,
|
|
12449
13541
|
organization: process.env.RETRIVORA_ORGANIZATION || "default-org",
|
|
12450
13542
|
sdkVersion: SDK_VERSION,
|
|
@@ -12459,7 +13551,17 @@ async function reportTelemetry(req, plugin, action, status, details, trace) {
|
|
|
12459
13551
|
feature: action,
|
|
12460
13552
|
userAgent,
|
|
12461
13553
|
timestamp: (/* @__PURE__ */ new Date()).toISOString(),
|
|
12462
|
-
details
|
|
13554
|
+
details,
|
|
13555
|
+
// ── Structured observability fields ──────────────────────────────────
|
|
13556
|
+
requestId: (_f = trace == null ? void 0 : trace.requestId) != null ? _f : null,
|
|
13557
|
+
embedMs: (_h = (_g2 = trace == null ? void 0 : trace.latency) == null ? void 0 : _g2.embedMs) != null ? _h : null,
|
|
13558
|
+
retrieveMs: (_j = (_i = trace == null ? void 0 : trace.latency) == null ? void 0 : _i.retrieveMs) != null ? _j : null,
|
|
13559
|
+
generateMs: (_l = (_k = trace == null ? void 0 : trace.latency) == null ? void 0 : _k.generateMs) != null ? _l : null,
|
|
13560
|
+
ttftMs: (_m = extra == null ? void 0 : extra.ttftMs) != null ? _m : null,
|
|
13561
|
+
errorCode: (_n = extra == null ? void 0 : extra.errorCode) != null ? _n : null,
|
|
13562
|
+
errorType: (_o = extra == null ? void 0 : extra.errorType) != null ? _o : null,
|
|
13563
|
+
isStreaming: (_p = extra == null ? void 0 : extra.isStreaming) != null ? _p : false,
|
|
13564
|
+
environment: process.env.NODE_ENV || "development"
|
|
12463
13565
|
};
|
|
12464
13566
|
await fetch(telemetryUrl, {
|
|
12465
13567
|
method: "POST",
|
|
@@ -12475,43 +13577,8 @@ async function reportTelemetry(req, plugin, action, status, details, trace) {
|
|
|
12475
13577
|
} catch (e) {
|
|
12476
13578
|
}
|
|
12477
13579
|
}
|
|
12478
|
-
function sseFrame(payload) {
|
|
12479
|
-
return `data: ${JSON.stringify(payload)}
|
|
12480
|
-
|
|
12481
|
-
`;
|
|
12482
|
-
}
|
|
12483
|
-
function sseTextFrame(text) {
|
|
12484
|
-
return `data: ${JSON.stringify({ type: "text", text })}
|
|
12485
|
-
|
|
12486
|
-
`;
|
|
12487
|
-
}
|
|
12488
|
-
function sseMetaFrame(meta) {
|
|
12489
|
-
return `data: ${JSON.stringify(__spreadValues({ type: "metadata" }, meta != null ? meta : {}))}
|
|
12490
|
-
|
|
12491
|
-
`;
|
|
12492
|
-
}
|
|
12493
|
-
function sseUIFrame(uiTransformation) {
|
|
12494
|
-
return `data: ${JSON.stringify({ type: "ui_transformation", data: uiTransformation })}
|
|
12495
|
-
|
|
12496
|
-
`;
|
|
12497
|
-
}
|
|
12498
|
-
function sseObservabilityFrame(trace) {
|
|
12499
|
-
return `data: ${JSON.stringify({ type: "observability", data: trace })}
|
|
12500
|
-
|
|
12501
|
-
`;
|
|
12502
|
-
}
|
|
12503
|
-
function sseErrorFrame(message) {
|
|
12504
|
-
return `data: ${JSON.stringify({ type: "error", error: message })}
|
|
12505
13580
|
|
|
12506
|
-
|
|
12507
|
-
}
|
|
12508
|
-
var SSE_HEADERS = {
|
|
12509
|
-
"Content-Type": "text/event-stream; charset=utf-8",
|
|
12510
|
-
"Cache-Control": "no-cache, no-transform",
|
|
12511
|
-
Connection: "keep-alive",
|
|
12512
|
-
"X-Accel-Buffering": "no"
|
|
12513
|
-
// Disable Nginx buffering for streaming
|
|
12514
|
-
};
|
|
13581
|
+
// src/handlers/chatHandler.ts
|
|
12515
13582
|
function createChatHandler(configOrPlugin, options) {
|
|
12516
13583
|
const plugin = getOrCreatePlugin(configOrPlugin);
|
|
12517
13584
|
const storage = new DatabaseStorage(plugin.getConfig());
|
|
@@ -12563,7 +13630,7 @@ function createChatHandler(configOrPlugin, options) {
|
|
|
12563
13630
|
if (!namespaceCheck.ok) return namespaceCheck.response;
|
|
12564
13631
|
const sanitized = sanitizeInput(rawMessage != null ? rawMessage : "");
|
|
12565
13632
|
if (!sanitized.ok) {
|
|
12566
|
-
return
|
|
13633
|
+
return import_server3.NextResponse.json({ error: { code: "INVALID_INPUT", message: sanitized.error } }, { status: 400 });
|
|
12567
13634
|
}
|
|
12568
13635
|
const message = sanitized.value;
|
|
12569
13636
|
const userMsgId = userMessageId || `user_${Date.now()}_${Math.random().toString(36).slice(2, 6)}`;
|
|
@@ -12582,18 +13649,68 @@ function createChatHandler(configOrPlugin, options) {
|
|
|
12582
13649
|
uiTransformation: result.ui_transformation,
|
|
12583
13650
|
trace: result.trace
|
|
12584
13651
|
}).catch((err) => console.warn("[createChatHandler] Failed to save assistant message:", err));
|
|
12585
|
-
await reportTelemetry(
|
|
12586
|
-
|
|
13652
|
+
await reportTelemetry(
|
|
13653
|
+
req,
|
|
13654
|
+
plugin,
|
|
13655
|
+
"RAG_QUERY",
|
|
13656
|
+
"success",
|
|
13657
|
+
`Tokens: ${((_d = (_c = result.trace) == null ? void 0 : _c.tokens) == null ? void 0 : _d.totalTokens) || 0}, Latency: ${((_f = (_e = result.trace) == null ? void 0 : _e.latency) == null ? void 0 : _f.totalMs) || 0}ms`,
|
|
13658
|
+
result.trace,
|
|
13659
|
+
{ isStreaming: false }
|
|
13660
|
+
);
|
|
13661
|
+
return import_server3.NextResponse.json(__spreadProps(__spreadValues({}, result), {
|
|
12587
13662
|
messageId: assistantMsgId,
|
|
12588
13663
|
userMessageId: userMsgId
|
|
12589
13664
|
}));
|
|
12590
13665
|
} catch (err) {
|
|
12591
13666
|
const message = err instanceof Error ? err.message : "Internal server error";
|
|
12592
|
-
|
|
12593
|
-
|
|
13667
|
+
const { errorCode, errorType } = classifyError(message);
|
|
13668
|
+
await reportTelemetry(req, plugin, "RAG_QUERY", "error", message, void 0, { isStreaming: false, errorCode, errorType });
|
|
13669
|
+
return import_server3.NextResponse.json({ error: message }, { status: 500 });
|
|
12594
13670
|
}
|
|
12595
13671
|
};
|
|
12596
13672
|
}
|
|
13673
|
+
|
|
13674
|
+
// src/handlers/internal/sse.ts
|
|
13675
|
+
function sseFrame(payload) {
|
|
13676
|
+
return `data: ${JSON.stringify(payload)}
|
|
13677
|
+
|
|
13678
|
+
`;
|
|
13679
|
+
}
|
|
13680
|
+
function sseTextFrame(text) {
|
|
13681
|
+
return `data: ${JSON.stringify({ type: "text", text })}
|
|
13682
|
+
|
|
13683
|
+
`;
|
|
13684
|
+
}
|
|
13685
|
+
function sseMetaFrame(meta) {
|
|
13686
|
+
return `data: ${JSON.stringify(__spreadValues({ type: "metadata" }, meta != null ? meta : {}))}
|
|
13687
|
+
|
|
13688
|
+
`;
|
|
13689
|
+
}
|
|
13690
|
+
function sseUIFrame(uiTransformation) {
|
|
13691
|
+
return `data: ${JSON.stringify({ type: "ui_transformation", data: uiTransformation })}
|
|
13692
|
+
|
|
13693
|
+
`;
|
|
13694
|
+
}
|
|
13695
|
+
function sseObservabilityFrame(trace) {
|
|
13696
|
+
return `data: ${JSON.stringify({ type: "observability", data: trace })}
|
|
13697
|
+
|
|
13698
|
+
`;
|
|
13699
|
+
}
|
|
13700
|
+
function sseErrorFrame(message) {
|
|
13701
|
+
return `data: ${JSON.stringify({ type: "error", error: message })}
|
|
13702
|
+
|
|
13703
|
+
`;
|
|
13704
|
+
}
|
|
13705
|
+
var SSE_HEADERS = {
|
|
13706
|
+
"Content-Type": "text/event-stream; charset=utf-8",
|
|
13707
|
+
"Cache-Control": "no-cache, no-transform",
|
|
13708
|
+
Connection: "keep-alive",
|
|
13709
|
+
"X-Accel-Buffering": "no"
|
|
13710
|
+
// Disable Nginx buffering for streaming
|
|
13711
|
+
};
|
|
13712
|
+
|
|
13713
|
+
// src/handlers/streamHandler.ts
|
|
12597
13714
|
function createStreamHandler(configOrPlugin, options) {
|
|
12598
13715
|
const plugin = getOrCreatePlugin(configOrPlugin);
|
|
12599
13716
|
const storage = new DatabaseStorage(plugin.getConfig());
|
|
@@ -12668,6 +13785,7 @@ function createStreamHandler(configOrPlugin, options) {
|
|
|
12668
13785
|
const assistantMsgId = messageId || `assistant_${Date.now()}_${Math.random().toString(36).slice(2, 6)}`;
|
|
12669
13786
|
const encoder = new TextEncoder();
|
|
12670
13787
|
let isActive = true;
|
|
13788
|
+
let streamTelemetrySent = false;
|
|
12671
13789
|
const stream = new ReadableStream({
|
|
12672
13790
|
async start(controller) {
|
|
12673
13791
|
var _a3, _b2, _c, _d, _e;
|
|
@@ -12682,11 +13800,16 @@ function createStreamHandler(configOrPlugin, options) {
|
|
|
12682
13800
|
try {
|
|
12683
13801
|
const pipelineStream = plugin.chatStream(message, history, namespaceCheck.namespace);
|
|
12684
13802
|
let fullReply = "";
|
|
13803
|
+
let ttftMs;
|
|
13804
|
+
const handlerRequestStartMs = Date.now();
|
|
12685
13805
|
try {
|
|
12686
13806
|
for (var iter = __forAwait(pipelineStream), more, temp, error; more = !(temp = await iter.next()).done; more = false) {
|
|
12687
13807
|
const chunk = temp.value;
|
|
12688
13808
|
if (!isActive) break;
|
|
12689
13809
|
if (typeof chunk === "string") {
|
|
13810
|
+
if (ttftMs === void 0) {
|
|
13811
|
+
ttftMs = Date.now() - handlerRequestStartMs;
|
|
13812
|
+
}
|
|
12690
13813
|
fullReply += chunk;
|
|
12691
13814
|
enqueue(sseTextFrame(chunk));
|
|
12692
13815
|
} else if (chunk && typeof chunk === "object" && "type" in chunk && chunk.type === "thinking") {
|
|
@@ -12727,7 +13850,16 @@ function createStreamHandler(configOrPlugin, options) {
|
|
|
12727
13850
|
uiTransformation,
|
|
12728
13851
|
trace: responseChunk == null ? void 0 : responseChunk.trace
|
|
12729
13852
|
}).catch((err) => console.warn("[createStreamHandler] Failed to save assistant message:", err));
|
|
12730
|
-
|
|
13853
|
+
streamTelemetrySent = true;
|
|
13854
|
+
await reportTelemetry(
|
|
13855
|
+
req,
|
|
13856
|
+
plugin,
|
|
13857
|
+
"RAG_QUERY",
|
|
13858
|
+
"success",
|
|
13859
|
+
`Tokens: ${((_c = (_b2 = responseChunk == null ? void 0 : responseChunk.trace) == null ? void 0 : _b2.tokens) == null ? void 0 : _c.totalTokens) || 0}, Latency: ${((_e = (_d = responseChunk == null ? void 0 : responseChunk.trace) == null ? void 0 : _d.latency) == null ? void 0 : _e.totalMs) || 0}ms`,
|
|
13860
|
+
responseChunk == null ? void 0 : responseChunk.trace,
|
|
13861
|
+
{ ttftMs, isStreaming: true }
|
|
13862
|
+
);
|
|
12731
13863
|
}
|
|
12732
13864
|
}
|
|
12733
13865
|
} catch (temp) {
|
|
@@ -12743,11 +13875,21 @@ function createStreamHandler(configOrPlugin, options) {
|
|
|
12743
13875
|
} catch (streamError) {
|
|
12744
13876
|
if (isActive) {
|
|
12745
13877
|
let errorMessage = streamError instanceof Error ? streamError.message : String(streamError);
|
|
12746
|
-
if (errorMessage.includes("
|
|
13878
|
+
if (errorMessage.includes("LICENSE_REVOKED") || errorMessage.toLowerCase().includes("key has been revoked") || errorMessage.toLowerCase().includes("key has been terminated") || errorMessage.toLowerCase().includes("key has been suspended")) {
|
|
12747
13879
|
errorMessage = "Your Retrivora license key has been terminated, suspended, or revoked. Access denied.";
|
|
12748
13880
|
}
|
|
12749
13881
|
console.error("[createStreamHandler] Stream error:", streamError);
|
|
12750
|
-
|
|
13882
|
+
const { errorCode, errorType } = classifyError(errorMessage);
|
|
13883
|
+
streamTelemetrySent = true;
|
|
13884
|
+
await reportTelemetry(
|
|
13885
|
+
req,
|
|
13886
|
+
plugin,
|
|
13887
|
+
"RAG_QUERY",
|
|
13888
|
+
"error",
|
|
13889
|
+
errorMessage,
|
|
13890
|
+
void 0,
|
|
13891
|
+
{ isStreaming: true, errorCode, errorType }
|
|
13892
|
+
);
|
|
12751
13893
|
try {
|
|
12752
13894
|
enqueue(sseErrorFrame(errorMessage));
|
|
12753
13895
|
} catch (e) {
|
|
@@ -12756,6 +13898,10 @@ function createStreamHandler(configOrPlugin, options) {
|
|
|
12756
13898
|
} finally {
|
|
12757
13899
|
if (isActive) {
|
|
12758
13900
|
isActive = false;
|
|
13901
|
+
try {
|
|
13902
|
+
enqueue(sseFrame({ type: "done" }));
|
|
13903
|
+
} catch (e) {
|
|
13904
|
+
}
|
|
12759
13905
|
try {
|
|
12760
13906
|
controller.close();
|
|
12761
13907
|
} catch (e) {
|
|
@@ -12765,11 +13911,26 @@ function createStreamHandler(configOrPlugin, options) {
|
|
|
12765
13911
|
},
|
|
12766
13912
|
cancel(reason) {
|
|
12767
13913
|
isActive = false;
|
|
13914
|
+
if (!streamTelemetrySent) {
|
|
13915
|
+
reportTelemetry(
|
|
13916
|
+
req,
|
|
13917
|
+
plugin,
|
|
13918
|
+
"RAG_QUERY_ABORTED",
|
|
13919
|
+
"error",
|
|
13920
|
+
"Client disconnected mid-stream",
|
|
13921
|
+
void 0,
|
|
13922
|
+
{ isStreaming: true, errorCode: "CLIENT_DISCONNECT", errorType: "STREAMING_ERROR" }
|
|
13923
|
+
).catch(() => {
|
|
13924
|
+
});
|
|
13925
|
+
}
|
|
12768
13926
|
}
|
|
12769
13927
|
});
|
|
12770
13928
|
return new Response(stream, { headers: __spreadProps(__spreadValues({}, SSE_HEADERS), { "x-message-id": assistantMsgId, "x-user-message-id": userMsgId }) });
|
|
12771
13929
|
};
|
|
12772
13930
|
}
|
|
13931
|
+
|
|
13932
|
+
// src/handlers/ingestHandler.ts
|
|
13933
|
+
var import_server4 = require("next/server");
|
|
12773
13934
|
function createIngestHandler(configOrPlugin, options) {
|
|
12774
13935
|
const plugin = getOrCreatePlugin(configOrPlugin);
|
|
12775
13936
|
const onAuthorize = options == null ? void 0 : options.onAuthorize;
|
|
@@ -12783,7 +13944,7 @@ function createIngestHandler(configOrPlugin, options) {
|
|
|
12783
13944
|
if (!licenseCheck.ok) return licenseCheck.response;
|
|
12784
13945
|
const { documents, namespace } = body;
|
|
12785
13946
|
if (!Array.isArray(documents) || documents.length === 0) {
|
|
12786
|
-
return
|
|
13947
|
+
return import_server4.NextResponse.json({ error: "documents array is required" }, { status: 400 });
|
|
12787
13948
|
}
|
|
12788
13949
|
const namespaceCheck = resolveLicensedNamespace(licenseCheck.payload, namespace);
|
|
12789
13950
|
if (!namespaceCheck.ok) return namespaceCheck.response;
|
|
@@ -12810,14 +13971,17 @@ function createIngestHandler(configOrPlugin, options) {
|
|
|
12810
13971
|
}
|
|
12811
13972
|
const results = await plugin.ingest(documents, namespaceCheck.namespace);
|
|
12812
13973
|
await reportTelemetry(req, plugin, "DOCUMENT_INGESTION", "success", `Ingested ${documents.length} document(s)`);
|
|
12813
|
-
return
|
|
13974
|
+
return import_server4.NextResponse.json({ results });
|
|
12814
13975
|
} catch (err) {
|
|
12815
13976
|
const message = err instanceof Error ? err.message : "Internal server error";
|
|
12816
13977
|
await reportTelemetry(req, plugin, "DOCUMENT_INGESTION", "error", message);
|
|
12817
|
-
return
|
|
13978
|
+
return import_server4.NextResponse.json({ error: message }, { status: 500 });
|
|
12818
13979
|
}
|
|
12819
13980
|
};
|
|
12820
13981
|
}
|
|
13982
|
+
|
|
13983
|
+
// src/handlers/healthHandler.ts
|
|
13984
|
+
var import_server5 = require("next/server");
|
|
12821
13985
|
function createHealthHandler(configOrPlugin, options) {
|
|
12822
13986
|
const plugin = getOrCreatePlugin(configOrPlugin);
|
|
12823
13987
|
const onAuthorize = options == null ? void 0 : options.onAuthorize;
|
|
@@ -12829,17 +13993,21 @@ function createHealthHandler(configOrPlugin, options) {
|
|
|
12829
13993
|
try {
|
|
12830
13994
|
const health = await plugin.checkHealth();
|
|
12831
13995
|
const status = health.allHealthy ? "ok" : "degraded";
|
|
12832
|
-
return
|
|
13996
|
+
return import_server5.NextResponse.json(__spreadProps(__spreadValues({
|
|
12833
13997
|
status
|
|
12834
13998
|
}, health), {
|
|
12835
13999
|
timestamp: (/* @__PURE__ */ new Date()).toISOString()
|
|
12836
14000
|
}));
|
|
12837
14001
|
} catch (err) {
|
|
12838
14002
|
const message = err instanceof Error ? err.message : "Unknown error";
|
|
12839
|
-
return
|
|
14003
|
+
return import_server5.NextResponse.json({ status: "error", error: message }, { status: 500 });
|
|
12840
14004
|
}
|
|
12841
14005
|
};
|
|
12842
14006
|
}
|
|
14007
|
+
|
|
14008
|
+
// src/handlers/licenseHandler.ts
|
|
14009
|
+
var import_server6 = require("next/server");
|
|
14010
|
+
init_version();
|
|
12843
14011
|
function createLicenseHandler(configOrPlugin, options) {
|
|
12844
14012
|
const plugin = getOrCreatePlugin(configOrPlugin);
|
|
12845
14013
|
const onAuthorize = options == null ? void 0 : options.onAuthorize;
|
|
@@ -12865,7 +14033,7 @@ function createLicenseHandler(configOrPlugin, options) {
|
|
|
12865
14033
|
const licenseKey = (rawKey || "").trim().replace(/^["']|["']$/g, "").trim();
|
|
12866
14034
|
const projectId = (body == null ? void 0 : body.projectId) || config.projectId || "";
|
|
12867
14035
|
const payload = await LicenseVerifier.verifyAsync(licenseKey, projectId);
|
|
12868
|
-
return
|
|
14036
|
+
return import_server6.NextResponse.json({
|
|
12869
14037
|
valid: true,
|
|
12870
14038
|
licenseStatus: "ACTIVE",
|
|
12871
14039
|
minimumSupportedVersion: "2.1.0",
|
|
@@ -12875,7 +14043,7 @@ function createLicenseHandler(configOrPlugin, options) {
|
|
|
12875
14043
|
message: "License key is active and valid."
|
|
12876
14044
|
});
|
|
12877
14045
|
} catch (err) {
|
|
12878
|
-
return
|
|
14046
|
+
return import_server6.NextResponse.json(
|
|
12879
14047
|
{
|
|
12880
14048
|
valid: false,
|
|
12881
14049
|
licenseStatus: "REVOKED",
|
|
@@ -12890,6 +14058,9 @@ function createLicenseHandler(configOrPlugin, options) {
|
|
|
12890
14058
|
}
|
|
12891
14059
|
};
|
|
12892
14060
|
}
|
|
14061
|
+
|
|
14062
|
+
// src/handlers/uploadHandler.ts
|
|
14063
|
+
var import_server7 = require("next/server");
|
|
12893
14064
|
function createUploadHandler(configOrPlugin, options) {
|
|
12894
14065
|
const plugin = getOrCreatePlugin(configOrPlugin);
|
|
12895
14066
|
const onAuthorize = options == null ? void 0 : options.onAuthorize;
|
|
@@ -12908,7 +14079,7 @@ function createUploadHandler(configOrPlugin, options) {
|
|
|
12908
14079
|
const namespaceCheck = resolveLicensedNamespace(licenseCheck.payload, namespace);
|
|
12909
14080
|
if (!namespaceCheck.ok) return namespaceCheck.response;
|
|
12910
14081
|
if (!files || files.length === 0) {
|
|
12911
|
-
return
|
|
14082
|
+
return import_server7.NextResponse.json({ error: "No files provided" }, { status: 400 });
|
|
12912
14083
|
}
|
|
12913
14084
|
if (isFreeTier(licenseCheck.payload.tier)) {
|
|
12914
14085
|
const totalUploadBytes = files.reduce((sum, f) => sum + (f.size || 0), 0);
|
|
@@ -12930,137 +14101,174 @@ function createUploadHandler(configOrPlugin, options) {
|
|
|
12930
14101
|
);
|
|
12931
14102
|
}
|
|
12932
14103
|
}
|
|
12933
|
-
const
|
|
12934
|
-
|
|
12935
|
-
|
|
12936
|
-
|
|
12937
|
-
|
|
12938
|
-
const
|
|
12939
|
-
|
|
12940
|
-
|
|
12941
|
-
|
|
12942
|
-
|
|
12943
|
-
|
|
12944
|
-
|
|
12945
|
-
|
|
12946
|
-
|
|
12947
|
-
const
|
|
12948
|
-
|
|
12949
|
-
|
|
12950
|
-
|
|
12951
|
-
|
|
12952
|
-
rowData[key]
|
|
14104
|
+
const fileDocsArrays = await Promise.all(
|
|
14105
|
+
files.map(async (file) => {
|
|
14106
|
+
const fileDocs = [];
|
|
14107
|
+
const buffer = Buffer.isBuffer(file) ? file : Buffer.from(await file.arrayBuffer());
|
|
14108
|
+
const isExcel = file.name.toLowerCase().endsWith(".xlsx") || file.name.toLowerCase().endsWith(".xls") || file.type.includes("spreadsheet") || file.type.includes("excel");
|
|
14109
|
+
const SPREADSHEET_CHUNK_SIZE = 30;
|
|
14110
|
+
if (file.name.toLowerCase().endsWith(".csv") || file.name.toLowerCase().endsWith(".tsv") || file.type === "text/csv" || file.type.includes("tab-separated")) {
|
|
14111
|
+
const text = buffer.toString("utf-8");
|
|
14112
|
+
const Papa = await import("papaparse");
|
|
14113
|
+
const parsed = Papa.default.parse(text, { header: true, skipEmptyLines: true });
|
|
14114
|
+
if (parsed.data && parsed.data.length > 0) {
|
|
14115
|
+
const csvCols = Object.keys(parsed.data[0]);
|
|
14116
|
+
const rawRows = [];
|
|
14117
|
+
let lastRowData = null;
|
|
14118
|
+
for (const row of parsed.data) {
|
|
14119
|
+
const rowData = __spreadValues({}, row);
|
|
14120
|
+
const groupingKey = Object.keys(rowData)[0];
|
|
14121
|
+
if (lastRowData && groupingKey && rowData[groupingKey] && rowData[groupingKey] === lastRowData[groupingKey]) {
|
|
14122
|
+
for (const key of Object.keys(rowData)) {
|
|
14123
|
+
if (!rowData[key] && lastRowData[key]) {
|
|
14124
|
+
rowData[key] = lastRowData[key];
|
|
14125
|
+
}
|
|
12953
14126
|
}
|
|
12954
14127
|
}
|
|
14128
|
+
lastRowData = __spreadValues({}, rowData);
|
|
14129
|
+
rawRows.push(rowData);
|
|
12955
14130
|
}
|
|
12956
|
-
|
|
12957
|
-
|
|
12958
|
-
|
|
12959
|
-
|
|
12960
|
-
|
|
14131
|
+
for (let i = 0; i < rawRows.length; i += SPREADSHEET_CHUNK_SIZE) {
|
|
14132
|
+
const slice = rawRows.slice(i, i + SPREADSHEET_CHUNK_SIZE);
|
|
14133
|
+
const startRow = i + 1;
|
|
14134
|
+
const endRow = i + slice.length;
|
|
14135
|
+
const chunkIndex = Math.floor(i / SPREADSHEET_CHUNK_SIZE) + 1;
|
|
14136
|
+
const lines = [
|
|
14137
|
+
`File: ${file.name} (Rows ${startRow}-${endRow} of ${rawRows.length})`,
|
|
14138
|
+
`| ${csvCols.join(" | ")} |`,
|
|
14139
|
+
`| ${csvCols.map(() => "---").join(" | ")} |`
|
|
14140
|
+
];
|
|
14141
|
+
for (const r of slice) {
|
|
14142
|
+
const rowValues = csvCols.map((col) => {
|
|
14143
|
+
const val = r[col];
|
|
14144
|
+
if (val === void 0 || val === null) return "";
|
|
14145
|
+
return String(val).replace(/\|/g, "\\|").replace(/\r?\n/g, " ").trim();
|
|
14146
|
+
});
|
|
14147
|
+
lines.push(`| ${rowValues.join(" | ")} |`);
|
|
12961
14148
|
}
|
|
14149
|
+
fileDocs.push({
|
|
14150
|
+
docId: `${file.name}-chunk-${chunkIndex}`,
|
|
14151
|
+
content: lines.join("\n"),
|
|
14152
|
+
metadata: __spreadValues({
|
|
14153
|
+
fileName: file.name,
|
|
14154
|
+
fileSize: file.size,
|
|
14155
|
+
fileType: file.type || "text/csv",
|
|
14156
|
+
uploadedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
14157
|
+
startRow,
|
|
14158
|
+
endRow,
|
|
14159
|
+
rowCount: slice.length,
|
|
14160
|
+
headers: csvCols
|
|
14161
|
+
}, dimension ? { dimension } : {})
|
|
14162
|
+
});
|
|
12962
14163
|
}
|
|
12963
|
-
documents.push({
|
|
12964
|
-
docId: `${file.name}-row-${i}`,
|
|
12965
|
-
content: contentParts.join(", "),
|
|
12966
|
-
metadata: __spreadValues(__spreadProps(__spreadValues({
|
|
12967
|
-
fileName: file.name,
|
|
12968
|
-
fileSize: file.size,
|
|
12969
|
-
fileType: file.type || "text/csv",
|
|
12970
|
-
uploadedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
12971
|
-
}, dimension ? { dimension } : {}), {
|
|
12972
|
-
csvHeaders: csvCols
|
|
12973
|
-
}), rowData)
|
|
12974
|
-
});
|
|
12975
14164
|
}
|
|
12976
|
-
}
|
|
12977
|
-
|
|
12978
|
-
|
|
12979
|
-
|
|
12980
|
-
|
|
12981
|
-
|
|
12982
|
-
|
|
12983
|
-
|
|
12984
|
-
|
|
12985
|
-
|
|
12986
|
-
|
|
12987
|
-
|
|
12988
|
-
|
|
12989
|
-
|
|
12990
|
-
|
|
12991
|
-
|
|
12992
|
-
|
|
12993
|
-
|
|
12994
|
-
|
|
12995
|
-
|
|
12996
|
-
|
|
12997
|
-
|
|
12998
|
-
|
|
12999
|
-
|
|
13000
|
-
|
|
13001
|
-
|
|
13002
|
-
|
|
13003
|
-
|
|
13004
|
-
|
|
13005
|
-
|
|
13006
|
-
|
|
13007
|
-
|
|
13008
|
-
|
|
13009
|
-
|
|
13010
|
-
|
|
13011
|
-
|
|
13012
|
-
|
|
13013
|
-
|
|
13014
|
-
uploadedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
13015
|
-
}, dimension ? { dimension } : {}), {
|
|
13016
|
-
headers
|
|
13017
|
-
}), rowData)
|
|
14165
|
+
} else if (isExcel) {
|
|
14166
|
+
let excelLoaded = false;
|
|
14167
|
+
try {
|
|
14168
|
+
const XLSX = await import("xlsx");
|
|
14169
|
+
const workbook = XLSX.read(buffer, { type: "buffer", cellDates: true });
|
|
14170
|
+
for (const sheetName of workbook.SheetNames) {
|
|
14171
|
+
const sheet = workbook.Sheets[sheetName];
|
|
14172
|
+
if (!sheet) continue;
|
|
14173
|
+
const rawGrid = XLSX.utils.sheet_to_json(sheet, { header: 1, defval: "", blankrows: false });
|
|
14174
|
+
if (!rawGrid || rawGrid.length === 0) continue;
|
|
14175
|
+
let headerRowIndex = 0;
|
|
14176
|
+
while (headerRowIndex < rawGrid.length && (!rawGrid[headerRowIndex] || !rawGrid[headerRowIndex].some((c) => String(c != null ? c : "").trim().length > 0))) {
|
|
14177
|
+
headerRowIndex++;
|
|
14178
|
+
}
|
|
14179
|
+
if (headerRowIndex >= rawGrid.length) continue;
|
|
14180
|
+
const rawHeaders = rawGrid[headerRowIndex].map((h, idx) => {
|
|
14181
|
+
const s = String(h != null ? h : "").trim();
|
|
14182
|
+
return s || `Column_${idx + 1}`;
|
|
14183
|
+
});
|
|
14184
|
+
const dataRows = rawGrid.slice(headerRowIndex + 1).filter(
|
|
14185
|
+
(row) => row && row.some((c) => String(c != null ? c : "").trim().length > 0)
|
|
14186
|
+
);
|
|
14187
|
+
if (dataRows.length === 0) continue;
|
|
14188
|
+
for (let i = 0; i < dataRows.length; i += SPREADSHEET_CHUNK_SIZE) {
|
|
14189
|
+
const slice = dataRows.slice(i, i + SPREADSHEET_CHUNK_SIZE);
|
|
14190
|
+
const startRow = headerRowIndex + 1 + i + 1;
|
|
14191
|
+
const endRow = startRow + slice.length - 1;
|
|
14192
|
+
const chunkIndex = Math.floor(i / SPREADSHEET_CHUNK_SIZE) + 1;
|
|
14193
|
+
const lines = [
|
|
14194
|
+
`File: ${file.name} | Sheet: ${sheetName} (Rows ${startRow}-${endRow} of ${dataRows.length})`,
|
|
14195
|
+
`| ${rawHeaders.join(" | ")} |`,
|
|
14196
|
+
`| ${rawHeaders.map(() => "---").join(" | ")} |`
|
|
14197
|
+
];
|
|
14198
|
+
for (const r of slice) {
|
|
14199
|
+
const rowValues = rawHeaders.map((_, colIdx) => {
|
|
14200
|
+
const val = r[colIdx];
|
|
14201
|
+
if (val === void 0 || val === null) return "";
|
|
14202
|
+
return String(val).replace(/\|/g, "\\|").replace(/\r?\n/g, " ").trim();
|
|
13018
14203
|
});
|
|
14204
|
+
lines.push(`| ${rowValues.join(" | ")} |`);
|
|
13019
14205
|
}
|
|
14206
|
+
fileDocs.push({
|
|
14207
|
+
docId: `${file.name}-${sheetName}-chunk-${chunkIndex}`,
|
|
14208
|
+
content: lines.join("\n"),
|
|
14209
|
+
metadata: __spreadValues({
|
|
14210
|
+
fileName: file.name,
|
|
14211
|
+
sheetName,
|
|
14212
|
+
fileSize: file.size,
|
|
14213
|
+
fileType: file.type || "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
|
|
14214
|
+
uploadedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
14215
|
+
startRow,
|
|
14216
|
+
endRow,
|
|
14217
|
+
rowCount: slice.length,
|
|
14218
|
+
headers: rawHeaders
|
|
14219
|
+
}, dimension ? { dimension } : {})
|
|
14220
|
+
});
|
|
14221
|
+
}
|
|
14222
|
+
}
|
|
14223
|
+
excelLoaded = fileDocs.length > 0;
|
|
14224
|
+
} catch (xlsxErr) {
|
|
14225
|
+
excelLoaded = false;
|
|
14226
|
+
}
|
|
14227
|
+
if (!excelLoaded) {
|
|
14228
|
+
const content = await DocumentParser.parse(buffer, file.name, file.type);
|
|
14229
|
+
if (content && content.trim()) {
|
|
14230
|
+
fileDocs.push({
|
|
14231
|
+
docId: file.name,
|
|
14232
|
+
content,
|
|
14233
|
+
metadata: __spreadValues({
|
|
14234
|
+
fileName: file.name,
|
|
14235
|
+
fileSize: file.size,
|
|
14236
|
+
fileType: file.type || "application/vnd.ms-excel",
|
|
14237
|
+
uploadedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
14238
|
+
}, dimension ? { dimension } : {})
|
|
13020
14239
|
});
|
|
13021
|
-
}
|
|
13022
|
-
excelLoaded = documents.length > 0;
|
|
14240
|
+
}
|
|
13023
14241
|
}
|
|
13024
|
-
}
|
|
13025
|
-
excelLoaded = false;
|
|
13026
|
-
}
|
|
13027
|
-
if (!excelLoaded) {
|
|
14242
|
+
} else {
|
|
13028
14243
|
const content = await DocumentParser.parse(buffer, file.name, file.type);
|
|
13029
|
-
|
|
14244
|
+
fileDocs.push({
|
|
13030
14245
|
docId: file.name,
|
|
13031
14246
|
content,
|
|
13032
14247
|
metadata: __spreadValues({
|
|
13033
14248
|
fileName: file.name,
|
|
13034
14249
|
fileSize: file.size,
|
|
13035
|
-
fileType: file.type
|
|
14250
|
+
fileType: file.type,
|
|
13036
14251
|
uploadedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
13037
14252
|
}, dimension ? { dimension } : {})
|
|
13038
14253
|
});
|
|
13039
14254
|
}
|
|
13040
|
-
|
|
13041
|
-
|
|
13042
|
-
|
|
13043
|
-
|
|
13044
|
-
content,
|
|
13045
|
-
metadata: __spreadValues({
|
|
13046
|
-
fileName: file.name,
|
|
13047
|
-
fileSize: file.size,
|
|
13048
|
-
fileType: file.type,
|
|
13049
|
-
uploadedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
13050
|
-
}, dimension ? { dimension } : {})
|
|
13051
|
-
});
|
|
13052
|
-
}
|
|
13053
|
-
}
|
|
14255
|
+
return fileDocs;
|
|
14256
|
+
})
|
|
14257
|
+
);
|
|
14258
|
+
const documents = fileDocsArrays.flat();
|
|
13054
14259
|
const results = await plugin.ingest(documents, namespaceCheck.namespace);
|
|
13055
14260
|
await reportTelemetry(req, plugin, "DOCUMENT_INGESTION", "success", `Uploaded & ingested ${files.length} file(s)`);
|
|
13056
|
-
return
|
|
14261
|
+
return import_server7.NextResponse.json({ message: "Upload successful", results });
|
|
13057
14262
|
} catch (err) {
|
|
13058
14263
|
const message = err instanceof Error ? err.message : "Upload failed";
|
|
13059
14264
|
await reportTelemetry(req, plugin, "DOCUMENT_INGESTION", "error", message);
|
|
13060
|
-
return
|
|
14265
|
+
return import_server7.NextResponse.json({ error: message }, { status: 500 });
|
|
13061
14266
|
}
|
|
13062
14267
|
};
|
|
13063
14268
|
}
|
|
14269
|
+
|
|
14270
|
+
// src/handlers/suggestionsHandler.ts
|
|
14271
|
+
var import_server8 = require("next/server");
|
|
13064
14272
|
function createSuggestionsHandler(configOrPlugin, options) {
|
|
13065
14273
|
const plugin = getOrCreatePlugin(configOrPlugin);
|
|
13066
14274
|
const onAuthorize = options == null ? void 0 : options.onAuthorize;
|
|
@@ -13091,20 +14299,23 @@ function createSuggestionsHandler(configOrPlugin, options) {
|
|
|
13091
14299
|
namespace = url.searchParams.get("namespace") || void 0;
|
|
13092
14300
|
}
|
|
13093
14301
|
if (!query || typeof query !== "string" || !query.trim()) {
|
|
13094
|
-
return
|
|
14302
|
+
return import_server8.NextResponse.json({ suggestions: [] });
|
|
13095
14303
|
}
|
|
13096
14304
|
const namespaceCheck = resolveLicensedNamespace(effectiveLicensePayload, namespace);
|
|
13097
14305
|
if (!namespaceCheck.ok) return namespaceCheck.response;
|
|
13098
14306
|
const suggestions = await plugin.getSuggestions(query, namespaceCheck.namespace);
|
|
13099
14307
|
await reportTelemetry(req, plugin, "AUTO_SUGGESTIONS", "success", `Fetched ${suggestions.length} suggestions for: ${query.slice(0, 30)}`);
|
|
13100
|
-
return
|
|
14308
|
+
return import_server8.NextResponse.json({ suggestions });
|
|
13101
14309
|
} catch (err) {
|
|
13102
14310
|
const message = err instanceof Error ? err.message : "Internal server error";
|
|
13103
14311
|
await reportTelemetry(req, plugin, "AUTO_SUGGESTIONS", "error", message);
|
|
13104
|
-
return
|
|
14312
|
+
return import_server8.NextResponse.json({ error: message }, { status: 500 });
|
|
13105
14313
|
}
|
|
13106
14314
|
};
|
|
13107
14315
|
}
|
|
14316
|
+
|
|
14317
|
+
// src/handlers/historyHandler.ts
|
|
14318
|
+
var import_server9 = require("next/server");
|
|
13108
14319
|
function createHistoryHandler(configOrPlugin, options) {
|
|
13109
14320
|
const plugin = getOrCreatePlugin(configOrPlugin);
|
|
13110
14321
|
const storage = new DatabaseStorage(plugin.getConfig());
|
|
@@ -13119,25 +14330,28 @@ function createHistoryHandler(configOrPlugin, options) {
|
|
|
13119
14330
|
if (req.method === "GET") {
|
|
13120
14331
|
try {
|
|
13121
14332
|
const history = await storage.getHistory(sessionId, scope);
|
|
13122
|
-
return
|
|
14333
|
+
return import_server9.NextResponse.json({ history });
|
|
13123
14334
|
} catch (err) {
|
|
13124
14335
|
const message = err instanceof Error ? err.message : "Failed to fetch history";
|
|
13125
|
-
return
|
|
14336
|
+
return import_server9.NextResponse.json({ error: message }, { status: 500 });
|
|
13126
14337
|
}
|
|
13127
14338
|
} else if (req.method === "POST") {
|
|
13128
14339
|
try {
|
|
13129
14340
|
const body = await req.json().catch(() => ({}));
|
|
13130
14341
|
const sid = body.sessionId || sessionId;
|
|
13131
14342
|
await storage.clearHistory(sid, scope);
|
|
13132
|
-
return
|
|
14343
|
+
return import_server9.NextResponse.json({ success: true });
|
|
13133
14344
|
} catch (err) {
|
|
13134
14345
|
const message = err instanceof Error ? err.message : "Failed to clear history";
|
|
13135
|
-
return
|
|
14346
|
+
return import_server9.NextResponse.json({ error: message }, { status: 500 });
|
|
13136
14347
|
}
|
|
13137
14348
|
}
|
|
13138
|
-
return
|
|
14349
|
+
return import_server9.NextResponse.json({ error: "Method Not Allowed" }, { status: 405 });
|
|
13139
14350
|
};
|
|
13140
14351
|
}
|
|
14352
|
+
|
|
14353
|
+
// src/handlers/feedbackHandler.ts
|
|
14354
|
+
var import_server10 = require("next/server");
|
|
13141
14355
|
function createFeedbackHandler(configOrPlugin, options) {
|
|
13142
14356
|
const plugin = getOrCreatePlugin(configOrPlugin);
|
|
13143
14357
|
const storage = new DatabaseStorage(plugin.getConfig());
|
|
@@ -13153,45 +14367,48 @@ function createFeedbackHandler(configOrPlugin, options) {
|
|
|
13153
14367
|
const messageId = url.searchParams.get("messageId");
|
|
13154
14368
|
if (!messageId) {
|
|
13155
14369
|
const feedbackList = await storage.listFeedback(scope);
|
|
13156
|
-
return
|
|
14370
|
+
return import_server10.NextResponse.json({ feedback: feedbackList });
|
|
13157
14371
|
}
|
|
13158
14372
|
const feedback = await storage.getFeedback(messageId, scope);
|
|
13159
|
-
return
|
|
14373
|
+
return import_server10.NextResponse.json({ feedback });
|
|
13160
14374
|
} catch (err) {
|
|
13161
14375
|
const message = err instanceof Error ? err.message : "Failed to fetch feedback";
|
|
13162
|
-
return
|
|
14376
|
+
return import_server10.NextResponse.json({ error: message }, { status: 500 });
|
|
13163
14377
|
}
|
|
13164
14378
|
} else if (req.method === "POST") {
|
|
13165
14379
|
try {
|
|
13166
14380
|
const body = await req.json();
|
|
13167
14381
|
const { messageId, sessionId = "default", rating, comment } = body;
|
|
13168
14382
|
if (!messageId) {
|
|
13169
|
-
return
|
|
14383
|
+
return import_server10.NextResponse.json({ error: "messageId is required" }, { status: 400 });
|
|
13170
14384
|
}
|
|
13171
14385
|
if (!rating) {
|
|
13172
|
-
return
|
|
14386
|
+
return import_server10.NextResponse.json({ error: "rating is required" }, { status: 400 });
|
|
13173
14387
|
}
|
|
13174
14388
|
if (scope && scope.length > 0) {
|
|
13175
14389
|
const matches = scope.some(
|
|
13176
14390
|
(pid) => pid && (sessionId === pid || sessionId.startsWith(pid) || sessionId.includes(pid))
|
|
13177
14391
|
);
|
|
13178
14392
|
if (!matches) {
|
|
13179
|
-
return
|
|
14393
|
+
return import_server10.NextResponse.json(
|
|
13180
14394
|
{ error: "sessionId is outside the authorized project scope" },
|
|
13181
14395
|
{ status: 403 }
|
|
13182
14396
|
);
|
|
13183
14397
|
}
|
|
13184
14398
|
}
|
|
13185
14399
|
await storage.saveFeedback({ messageId, sessionId, rating, comment });
|
|
13186
|
-
return
|
|
14400
|
+
return import_server10.NextResponse.json({ success: true });
|
|
13187
14401
|
} catch (err) {
|
|
13188
14402
|
const message = err instanceof Error ? err.message : "Failed to save feedback";
|
|
13189
|
-
return
|
|
14403
|
+
return import_server10.NextResponse.json({ error: message }, { status: 500 });
|
|
13190
14404
|
}
|
|
13191
14405
|
}
|
|
13192
|
-
return
|
|
14406
|
+
return import_server10.NextResponse.json({ error: "Method Not Allowed" }, { status: 405 });
|
|
13193
14407
|
};
|
|
13194
14408
|
}
|
|
14409
|
+
|
|
14410
|
+
// src/handlers/sessionsHandler.ts
|
|
14411
|
+
var import_server11 = require("next/server");
|
|
13195
14412
|
function createSessionsHandler(configOrPlugin, options) {
|
|
13196
14413
|
const plugin = getOrCreatePlugin(configOrPlugin);
|
|
13197
14414
|
const storage = new DatabaseStorage(plugin.getConfig());
|
|
@@ -13205,13 +14422,16 @@ function createSessionsHandler(configOrPlugin, options) {
|
|
|
13205
14422
|
try {
|
|
13206
14423
|
const scope = req && onResolveScope ? await onResolveScope(req) : void 0;
|
|
13207
14424
|
const sessions = await storage.listSessions(scope);
|
|
13208
|
-
return
|
|
14425
|
+
return import_server11.NextResponse.json({ sessions });
|
|
13209
14426
|
} catch (err) {
|
|
13210
14427
|
const message = err instanceof Error ? err.message : "Failed to list sessions";
|
|
13211
|
-
return
|
|
14428
|
+
return import_server11.NextResponse.json({ error: message }, { status: 500 });
|
|
13212
14429
|
}
|
|
13213
14430
|
};
|
|
13214
14431
|
}
|
|
14432
|
+
|
|
14433
|
+
// src/handlers/ragHandler.ts
|
|
14434
|
+
var import_server12 = require("next/server");
|
|
13215
14435
|
function createRagHandler(configOrPlugin, options) {
|
|
13216
14436
|
const plugin = getOrCreatePlugin(configOrPlugin);
|
|
13217
14437
|
const onAuthorize = options == null ? void 0 : options.onAuthorize;
|
|
@@ -13247,7 +14467,7 @@ function createRagHandler(configOrPlugin, options) {
|
|
|
13247
14467
|
case "v1/license/validate":
|
|
13248
14468
|
return licenseHandler(req);
|
|
13249
14469
|
default:
|
|
13250
|
-
return
|
|
14470
|
+
return import_server12.NextResponse.json({ error: `Not Found: POST segment "${segment}" not supported.` }, { status: 404 });
|
|
13251
14471
|
}
|
|
13252
14472
|
}
|
|
13253
14473
|
async function routeGetRequest(req, segment) {
|
|
@@ -13263,7 +14483,7 @@ function createRagHandler(configOrPlugin, options) {
|
|
|
13263
14483
|
case "sessions":
|
|
13264
14484
|
return sessionsHandler(req);
|
|
13265
14485
|
default:
|
|
13266
|
-
return
|
|
14486
|
+
return import_server12.NextResponse.json({ error: `Method Not Allowed: GET segment "${segment}" not supported.` }, { status: 405 });
|
|
13267
14487
|
}
|
|
13268
14488
|
}
|
|
13269
14489
|
async function getSegment(req, context) {
|
|
@@ -13298,7 +14518,7 @@ function createRagHandler(configOrPlugin, options) {
|
|
|
13298
14518
|
return await routeGetRequest(req, segment);
|
|
13299
14519
|
} catch (err) {
|
|
13300
14520
|
const msg = err instanceof Error ? err.message : "GET Routing failed";
|
|
13301
|
-
return
|
|
14521
|
+
return import_server12.NextResponse.json({ error: msg }, { status: 500 });
|
|
13302
14522
|
}
|
|
13303
14523
|
},
|
|
13304
14524
|
POST: async (req, context) => {
|
|
@@ -13307,7 +14527,7 @@ function createRagHandler(configOrPlugin, options) {
|
|
|
13307
14527
|
return await routePostRequest(req, segment);
|
|
13308
14528
|
} catch (err) {
|
|
13309
14529
|
const msg = err instanceof Error ? err.message : "POST Routing failed";
|
|
13310
|
-
return
|
|
14530
|
+
return import_server12.NextResponse.json({ error: msg }, { status: 500 });
|
|
13311
14531
|
}
|
|
13312
14532
|
}
|
|
13313
14533
|
};
|
|
@@ -13356,11 +14576,11 @@ var _LicenseValidator = class _LicenseValidator {
|
|
|
13356
14576
|
* Primary validation entry-point for UI / hook consumers.
|
|
13357
14577
|
*
|
|
13358
14578
|
* Resolves the license key, consults the local success cache, performs a
|
|
13359
|
-
* server POST for authoritative status, and falls back to
|
|
13360
|
-
*
|
|
14579
|
+
* server POST for authoritative status, and falls back to `LicenseVerifier.verifyAsync()`
|
|
14580
|
+
* if the server can't be reached.
|
|
13361
14581
|
*
|
|
13362
14582
|
* @throws LicenseValidationError - When no key is provided OR both the
|
|
13363
|
-
* server call AND the
|
|
14583
|
+
* server call AND the fallback report an invalid/expired key.
|
|
13364
14584
|
* @throws SDKVersionUnsupportedError - When server reports forceUpgrade=true.
|
|
13365
14585
|
* @returns A `LicenseValidationResponse` that the UI can use to enable or
|
|
13366
14586
|
* lock the chat widget.
|
|
@@ -13409,6 +14629,8 @@ var _LicenseValidator = class _LicenseValidator {
|
|
|
13409
14629
|
this.cache.delete(effectiveLicenseKey);
|
|
13410
14630
|
}
|
|
13411
14631
|
}
|
|
14632
|
+
const existing = this.inflight.get(effectiveLicenseKey);
|
|
14633
|
+
if (existing) return existing;
|
|
13412
14634
|
const inflightPromise = this._doNetworkValidate(
|
|
13413
14635
|
effectiveLicenseKey,
|
|
13414
14636
|
effectiveProjectId,
|
|
@@ -13677,8 +14899,8 @@ var LicenseConfigResolver = class {
|
|
|
13677
14899
|
};
|
|
13678
14900
|
}
|
|
13679
14901
|
};
|
|
13680
|
-
LicenseConfigResolver.RETRIVORA_CONFIG_URL = "https://
|
|
13681
|
-
LicenseConfigResolver.RETRIVORA_CREDENTIALS_URL = "https://
|
|
14902
|
+
LicenseConfigResolver.RETRIVORA_CONFIG_URL = "https://llm.retrivora.com/api/v1/config";
|
|
14903
|
+
LicenseConfigResolver.RETRIVORA_CREDENTIALS_URL = "https://llm.retrivora.com/api/v1/credentials";
|
|
13682
14904
|
LicenseConfigResolver.CACHE_TTL_MS = 36e5;
|
|
13683
14905
|
// 1 hour
|
|
13684
14906
|
LicenseConfigResolver.configCache = /* @__PURE__ */ new Map();
|
|
@@ -13842,6 +15064,9 @@ var _TelemetryService = class _TelemetryService {
|
|
|
13842
15064
|
console.warn("[TelemetryService] Auto-flush error:", err.message);
|
|
13843
15065
|
});
|
|
13844
15066
|
}, this.flushIntervalMs);
|
|
15067
|
+
if (this.flushTimer && typeof this.flushTimer.unref === "function") {
|
|
15068
|
+
this.flushTimer.unref();
|
|
15069
|
+
}
|
|
13845
15070
|
}
|
|
13846
15071
|
/**
|
|
13847
15072
|
* Stop telemetry service and flush remaining events
|
|
@@ -14111,9 +15336,11 @@ async function createRetrivora(options) {
|
|
|
14111
15336
|
EmbeddingStrategyResolver,
|
|
14112
15337
|
EventType,
|
|
14113
15338
|
GroqProvider,
|
|
15339
|
+
HybridSearch,
|
|
14114
15340
|
IntentClassifier,
|
|
14115
15341
|
LLMFactory,
|
|
14116
15342
|
LLM_PROFILES,
|
|
15343
|
+
LangChainAgent,
|
|
14117
15344
|
LicenseConfigResolver,
|
|
14118
15345
|
LicenseValidationError,
|
|
14119
15346
|
LicenseValidator,
|
|
@@ -14125,19 +15352,23 @@ async function createRetrivora(options) {
|
|
|
14125
15352
|
MultiTablePostgresProvider,
|
|
14126
15353
|
OllamaProvider,
|
|
14127
15354
|
OpenAIProvider,
|
|
15355
|
+
PIIMasker,
|
|
14128
15356
|
PRESETS,
|
|
14129
15357
|
PineconeProvider,
|
|
14130
15358
|
Pipeline,
|
|
14131
15359
|
PostgreSQLProvider,
|
|
15360
|
+
PromptGuard,
|
|
14132
15361
|
PromptOptimizer,
|
|
14133
15362
|
ProviderHealthCheck,
|
|
14134
15363
|
ProviderNotFoundException,
|
|
14135
15364
|
ProviderRegistry,
|
|
14136
15365
|
QdrantProvider,
|
|
14137
15366
|
QwenProvider,
|
|
15367
|
+
RAGEvaluator,
|
|
14138
15368
|
RateLimitException,
|
|
14139
15369
|
RedisProvider,
|
|
14140
15370
|
RendererRegistry,
|
|
15371
|
+
Reranker,
|
|
14141
15372
|
RetrievalException,
|
|
14142
15373
|
Retrivora,
|
|
14143
15374
|
RetrivoraError,
|
|
@@ -14155,9 +15386,11 @@ async function createRetrivora(options) {
|
|
|
14155
15386
|
TelemetryService,
|
|
14156
15387
|
TextRendererStrategy,
|
|
14157
15388
|
UniversalLLMAdapter,
|
|
15389
|
+
UniversalTableParser,
|
|
14158
15390
|
UniversalVectorProvider,
|
|
14159
15391
|
VECTOR_PROFILES,
|
|
14160
15392
|
VectorPlugin,
|
|
15393
|
+
VisionExtractor,
|
|
14161
15394
|
VisualizationDecisionEngine,
|
|
14162
15395
|
WeaviateProvider,
|
|
14163
15396
|
createChatHandler,
|
|
@@ -14167,10 +15400,12 @@ async function createRetrivora(options) {
|
|
|
14167
15400
|
createHealthHandler,
|
|
14168
15401
|
createHistoryHandler,
|
|
14169
15402
|
createIngestHandler,
|
|
15403
|
+
createLicenseHandler,
|
|
14170
15404
|
createRagHandler,
|
|
14171
15405
|
createRetrivora,
|
|
14172
15406
|
createSessionsHandler,
|
|
14173
15407
|
createStreamHandler,
|
|
15408
|
+
createSuggestionsHandler,
|
|
14174
15409
|
createUploadHandler,
|
|
14175
15410
|
decideVisualization,
|
|
14176
15411
|
getRagConfig,
|
|
@@ -14184,6 +15419,8 @@ async function createRetrivora(options) {
|
|
|
14184
15419
|
sseErrorFrame,
|
|
14185
15420
|
sseFrame,
|
|
14186
15421
|
sseMetaFrame,
|
|
15422
|
+
sseObservabilityFrame,
|
|
14187
15423
|
sseTextFrame,
|
|
15424
|
+
sseUIFrame,
|
|
14188
15425
|
wrapError
|
|
14189
15426
|
});
|