@retrivora-ai/rag-engine 2.5.9 → 2.6.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +8 -6
- package/dist/{ILLMProvider-B2N3lMBc.d.mts → ILLMProvider-CzXcTA6d.d.mts} +31 -3
- package/dist/{ILLMProvider-B2N3lMBc.d.ts → ILLMProvider-CzXcTA6d.d.ts} +31 -3
- package/dist/handlers/index.d.mts +2 -2
- package/dist/handlers/index.d.ts +2 -2
- package/dist/handlers/index.js +9815 -8601
- package/dist/handlers/index.mjs +9818 -8612
- package/dist/{index-CAGGPyI7.d.mts → index-BST9Gzsv.d.mts} +8 -7
- package/dist/{index-BxzDv2jd.d.ts → index-CFl4KrKq.d.ts} +58 -46
- package/dist/{index-37V810a-.d.mts → index-D7FuQRSb.d.mts} +58 -46
- package/dist/{index-DdQxoIpu.d.ts → index-qLvtMUr-.d.ts} +8 -7
- package/dist/index.css +23 -0
- package/dist/index.d.mts +4 -4
- package/dist/index.d.ts +4 -4
- package/dist/index.js +267 -187
- package/dist/index.min.css +1 -1
- package/dist/index.mjs +268 -193
- package/dist/server.d.mts +322 -14
- package/dist/server.d.ts +322 -14
- package/dist/server.js +1925 -688
- package/dist/server.mjs +1912 -693
- package/package.json +1 -1
package/dist/server.mjs
CHANGED
|
@@ -20,12 +20,6 @@ var __spreadValues = (a, b) => {
|
|
|
20
20
|
return a;
|
|
21
21
|
};
|
|
22
22
|
var __spreadProps = (a, b) => __defProps(a, __getOwnPropDescs(b));
|
|
23
|
-
var __require = /* @__PURE__ */ ((x) => typeof require !== "undefined" ? require : typeof Proxy !== "undefined" ? new Proxy(x, {
|
|
24
|
-
get: (a, b) => (typeof require !== "undefined" ? require : a)[b]
|
|
25
|
-
}) : x)(function(x) {
|
|
26
|
-
if (typeof require !== "undefined") return require.apply(this, arguments);
|
|
27
|
-
throw Error('Dynamic require of "' + x + '" is not supported');
|
|
28
|
-
});
|
|
29
23
|
var __objRest = (source, exclude) => {
|
|
30
24
|
var target = {};
|
|
31
25
|
for (var prop in source)
|
|
@@ -326,7 +320,7 @@ var init_package = __esm({
|
|
|
326
320
|
"package.json"() {
|
|
327
321
|
package_default = {
|
|
328
322
|
name: "@retrivora-ai/rag-engine",
|
|
329
|
-
version: "2.
|
|
323
|
+
version: "2.6.2",
|
|
330
324
|
description: "Retrivora AI is a plug-and-play AI engine for RAG chat experiences \u2014 generic vector DB + LLM provider, embeddable or standalone.",
|
|
331
325
|
author: "Abhinav Alkuchi",
|
|
332
326
|
license: "UNLICENSED",
|
|
@@ -505,7 +499,7 @@ var init_ConfigFetcher = __esm({
|
|
|
505
499
|
init_ProjectSettings();
|
|
506
500
|
init_version();
|
|
507
501
|
CONTROL_PLANE_URL = "https://www.retrivora.com";
|
|
508
|
-
LLM_PROXY_BASE_URL = "https://
|
|
502
|
+
LLM_PROXY_BASE_URL = "https://llm.retrivora.com/api/v1";
|
|
509
503
|
ConfigFetcher = class {
|
|
510
504
|
// 5-minute in-memory session cache
|
|
511
505
|
/**
|
|
@@ -2761,13 +2755,14 @@ function wrapError(err, defaultCode, defaultMessage) {
|
|
|
2761
2755
|
const message = (error == null ? void 0 : error.message) || defaultMessage || String(err);
|
|
2762
2756
|
const status = (error == null ? void 0 : error.status) || (error == null ? void 0 : error.statusCode) || ((_a2 = error == null ? void 0 : error.response) == null ? void 0 : _a2.status);
|
|
2763
2757
|
const code = error == null ? void 0 : error.code;
|
|
2764
|
-
if (status === 429 || /rate[- ]?limit/i.test(message) || code === "RATE_LIMIT_EXCEEDED") {
|
|
2758
|
+
if (status === 429 || /rate[- ]?limit|token limit|too many requests|tokens per minute/i.test(message) || code === "RATE_LIMIT_EXCEEDED") {
|
|
2765
2759
|
return new RateLimitException(message, err);
|
|
2766
2760
|
}
|
|
2767
|
-
if (
|
|
2768
|
-
|
|
2769
|
-
|
|
2770
|
-
|
|
2761
|
+
if (code === "LICENSE_DELETED" || code === "LICENSE_INACTIVE" || code === "LICENSE_REVOKED" || /license key has been terminated|license is revoked|license is suspended|license is inactive/i.test(message)) {
|
|
2762
|
+
return new AuthenticationException("Your Retrivora license key has been terminated, suspended, or revoked. Access denied.", err);
|
|
2763
|
+
}
|
|
2764
|
+
if (status === 401 || status === 403 || code === "INVALID_API_KEY" || code === "AUTHENTICATION_ERROR" || /unauthorized|forbidden|invalid api key/i.test(message)) {
|
|
2765
|
+
return new AuthenticationException(message, err);
|
|
2771
2766
|
}
|
|
2772
2767
|
switch (defaultCode) {
|
|
2773
2768
|
case "PROVIDER_NOT_FOUND":
|
|
@@ -2788,6 +2783,20 @@ function wrapError(err, defaultCode, defaultMessage) {
|
|
|
2788
2783
|
}
|
|
2789
2784
|
|
|
2790
2785
|
// src/core/LicenseVerifier.ts
|
|
2786
|
+
function decodeBase64UrlToUtf8(b64url) {
|
|
2787
|
+
const normalized = b64url.replace(/-/g, "+").replace(/_/g, "/");
|
|
2788
|
+
const padded = normalized + "=".repeat((4 - normalized.length % 4) % 4);
|
|
2789
|
+
if (typeof Buffer !== "undefined" && typeof Buffer.from === "function") {
|
|
2790
|
+
return Buffer.from(padded, "base64").toString("utf8");
|
|
2791
|
+
}
|
|
2792
|
+
if (typeof atob === "function") {
|
|
2793
|
+
const binary = atob(padded);
|
|
2794
|
+
const bytes = new Uint8Array(binary.length);
|
|
2795
|
+
for (let i = 0; i < binary.length; i++) bytes[i] = binary.charCodeAt(i);
|
|
2796
|
+
return new TextDecoder().decode(bytes);
|
|
2797
|
+
}
|
|
2798
|
+
throw new Error("Base64 decoding is not supported in this environment.");
|
|
2799
|
+
}
|
|
2791
2800
|
var _LicenseVerifier = class _LicenseVerifier {
|
|
2792
2801
|
// Cache verified license for 5 minutes
|
|
2793
2802
|
/**
|
|
@@ -2919,6 +2928,9 @@ var _LicenseVerifier = class _LicenseVerifier {
|
|
|
2919
2928
|
minimumSupportedVersion: data.minimumSupportedVersion,
|
|
2920
2929
|
forceUpgrade: data.forceUpgrade
|
|
2921
2930
|
};
|
|
2931
|
+
if (payload.expiresAt && Math.floor(Date.now() / 1e3) > payload.expiresAt) {
|
|
2932
|
+
throw new Error("License key has expired.");
|
|
2933
|
+
}
|
|
2922
2934
|
if (isProduction && provider) {
|
|
2923
2935
|
const normalizedProvider = provider.toLowerCase();
|
|
2924
2936
|
if (tier === "hobby") {
|
|
@@ -2957,63 +2969,67 @@ var _LicenseVerifier = class _LicenseVerifier {
|
|
|
2957
2969
|
}
|
|
2958
2970
|
}
|
|
2959
2971
|
static async verifyIP(licenseKey) {
|
|
2972
|
+
var _a2, _b;
|
|
2960
2973
|
if (!licenseKey) return;
|
|
2961
2974
|
try {
|
|
2962
2975
|
const parts = licenseKey.split(".");
|
|
2963
2976
|
if (parts.length !== 3) return;
|
|
2964
2977
|
const [, payloadB64] = parts;
|
|
2965
2978
|
const payload = JSON.parse(
|
|
2966
|
-
|
|
2979
|
+
decodeBase64UrlToUtf8(payloadB64)
|
|
2967
2980
|
);
|
|
2968
2981
|
const tier = (payload.tier || "").toLowerCase();
|
|
2969
|
-
if (tier === "enterprise" || tier === "custom") {
|
|
2982
|
+
if (tier === "enterprise" || tier === "custom" || !payload.ipv4 && !payload.ipv6) {
|
|
2970
2983
|
return;
|
|
2971
2984
|
}
|
|
2972
|
-
|
|
2973
|
-
|
|
2974
|
-
|
|
2975
|
-
|
|
2976
|
-
|
|
2977
|
-
|
|
2978
|
-
|
|
2979
|
-
|
|
2985
|
+
let currentIpv4 = "";
|
|
2986
|
+
let currentIpv6 = "";
|
|
2987
|
+
const now = Date.now();
|
|
2988
|
+
if (this.cachedIps && now - this.cachedIps.fetchedAt < 10 * 60 * 1e3) {
|
|
2989
|
+
currentIpv4 = this.cachedIps.ipv4;
|
|
2990
|
+
currentIpv6 = this.cachedIps.ipv6;
|
|
2991
|
+
} else {
|
|
2992
|
+
const [v4Result, v6Result] = await Promise.allSettled([
|
|
2993
|
+
fetch("https://api4.ipify.org?format=json", { signal: AbortSignal.timeout(2e3) }).then((r) => r.json()),
|
|
2994
|
+
fetch("https://api6.ipify.org?format=json", { signal: AbortSignal.timeout(2e3) }).then((r) => r.json())
|
|
2995
|
+
]);
|
|
2996
|
+
if (v4Result.status === "fulfilled" && ((_a2 = v4Result.value) == null ? void 0 : _a2.ip)) {
|
|
2997
|
+
currentIpv4 = v4Result.value.ip;
|
|
2980
2998
|
}
|
|
2981
|
-
|
|
2982
|
-
|
|
2983
|
-
const data = await res.json();
|
|
2984
|
-
currentIpv6 = data.ip;
|
|
2985
|
-
} catch (e) {
|
|
2999
|
+
if (v6Result.status === "fulfilled" && ((_b = v6Result.value) == null ? void 0 : _b.ip)) {
|
|
3000
|
+
currentIpv6 = v6Result.value.ip;
|
|
2986
3001
|
}
|
|
2987
|
-
|
|
2988
|
-
|
|
2989
|
-
|
|
2990
|
-
|
|
2991
|
-
|
|
2992
|
-
|
|
2993
|
-
|
|
2994
|
-
|
|
2995
|
-
|
|
3002
|
+
this.cachedIps = { ipv4: currentIpv4, ipv6: currentIpv6, fetchedAt: now };
|
|
3003
|
+
}
|
|
3004
|
+
const localIps = [];
|
|
3005
|
+
try {
|
|
3006
|
+
const osModule = await import("os");
|
|
3007
|
+
const interfaces = osModule.networkInterfaces();
|
|
3008
|
+
for (const name of Object.keys(interfaces)) {
|
|
3009
|
+
for (const iface of interfaces[name] || []) {
|
|
3010
|
+
if (!iface.internal) {
|
|
3011
|
+
localIps.push(iface.address);
|
|
2996
3012
|
}
|
|
2997
3013
|
}
|
|
2998
|
-
} catch (e) {
|
|
2999
3014
|
}
|
|
3000
|
-
|
|
3001
|
-
|
|
3002
|
-
|
|
3003
|
-
|
|
3004
|
-
|
|
3005
|
-
|
|
3006
|
-
);
|
|
3007
|
-
}
|
|
3008
|
-
} else if (payload.ipv4 && !isIpv4Match) {
|
|
3009
|
-
throw new Error(
|
|
3010
|
-
`License key access restricted. This license key was created for a different system (authorized IPv4: ${payload.ipv4}) and cannot be used on this machine.`
|
|
3011
|
-
);
|
|
3012
|
-
} else if (payload.ipv6 && !isIpv6Match) {
|
|
3015
|
+
} catch (e) {
|
|
3016
|
+
}
|
|
3017
|
+
const isIpv4Match = payload.ipv4 && (currentIpv4 === payload.ipv4 || localIps.includes(payload.ipv4));
|
|
3018
|
+
const isIpv6Match = payload.ipv6 && (currentIpv6 === payload.ipv6 || localIps.includes(payload.ipv6));
|
|
3019
|
+
if (payload.ipv4 && payload.ipv6) {
|
|
3020
|
+
if (!isIpv4Match && !isIpv6Match) {
|
|
3013
3021
|
throw new Error(
|
|
3014
|
-
`License key access restricted. This license key was created for a different system (authorized IPv6: ${payload.ipv6}) and cannot be used on this machine.`
|
|
3022
|
+
`License key access restricted. This license key was created for a different system (authorized IPv4: ${payload.ipv4}, IPv6: ${payload.ipv6}) and cannot be used on this machine.`
|
|
3015
3023
|
);
|
|
3016
3024
|
}
|
|
3025
|
+
} else if (payload.ipv4 && !isIpv4Match) {
|
|
3026
|
+
throw new Error(
|
|
3027
|
+
`License key access restricted. This license key was created for a different system (authorized IPv4: ${payload.ipv4}) and cannot be used on this machine.`
|
|
3028
|
+
);
|
|
3029
|
+
} else if (payload.ipv6 && !isIpv6Match) {
|
|
3030
|
+
throw new Error(
|
|
3031
|
+
`License key access restricted. This license key was created for a different system (authorized IPv6: ${payload.ipv6}) and cannot be used on this machine.`
|
|
3032
|
+
);
|
|
3017
3033
|
}
|
|
3018
3034
|
} catch (err) {
|
|
3019
3035
|
if (err.message.includes("License key access restricted")) {
|
|
@@ -3028,7 +3044,7 @@ var _LicenseVerifier = class _LicenseVerifier {
|
|
|
3028
3044
|
const rawToken = sanitizedKey.replace(/^rtv_/i, "").trim();
|
|
3029
3045
|
const parts = rawToken.split(".");
|
|
3030
3046
|
if (parts.length === 3) {
|
|
3031
|
-
const decodedPayload = JSON.parse(
|
|
3047
|
+
const decodedPayload = JSON.parse(decodeBase64UrlToUtf8(parts[1]));
|
|
3032
3048
|
if (decodedPayload.tier) return decodedPayload.tier;
|
|
3033
3049
|
}
|
|
3034
3050
|
} catch (e) {
|
|
@@ -3039,6 +3055,7 @@ var _LicenseVerifier = class _LicenseVerifier {
|
|
|
3039
3055
|
// In-memory cache to save network overhead on hot paths.
|
|
3040
3056
|
_LicenseVerifier.cache = /* @__PURE__ */ new Map();
|
|
3041
3057
|
_LicenseVerifier.CACHE_TTL_MS = 5 * 60 * 1e3;
|
|
3058
|
+
_LicenseVerifier.cachedIps = null;
|
|
3042
3059
|
var LicenseVerifier = _LicenseVerifier;
|
|
3043
3060
|
|
|
3044
3061
|
// src/config/constants.ts
|
|
@@ -4294,7 +4311,7 @@ var GroqProvider = class {
|
|
|
4294
4311
|
};
|
|
4295
4312
|
}
|
|
4296
4313
|
async chat(messages, context, options) {
|
|
4297
|
-
var _a2, _b, _c, _d, _e, _f, _g2, _h, _i;
|
|
4314
|
+
var _a2, _b, _c, _d, _e, _f, _g2, _h, _i, _j, _k, _l;
|
|
4298
4315
|
const basePrompt = (_b = (_a2 = options == null ? void 0 : options.systemPrompt) != null ? _a2 : this.llmConfig.systemPrompt) != null ? _b : "You are a helpful assistant. Answer questions based on the provided context.";
|
|
4299
4316
|
const systemMessage = {
|
|
4300
4317
|
role: "system",
|
|
@@ -4308,18 +4325,46 @@ var GroqProvider = class {
|
|
|
4308
4325
|
systemMessage,
|
|
4309
4326
|
...safeMessages
|
|
4310
4327
|
];
|
|
4311
|
-
const
|
|
4312
|
-
|
|
4313
|
-
|
|
4314
|
-
|
|
4315
|
-
|
|
4316
|
-
|
|
4317
|
-
|
|
4318
|
-
|
|
4328
|
+
const model = this.llmConfig.model === "qwen/qwen3.6-27b" || this.llmConfig.model === "qwen-3.6-27b" ? "qwen/qwen3.8-27b" : this.llmConfig.model;
|
|
4329
|
+
try {
|
|
4330
|
+
const completion = await this.client.chat.completions.create({
|
|
4331
|
+
model,
|
|
4332
|
+
messages: formattedMessages,
|
|
4333
|
+
max_tokens: Math.min((_d = (_c = options == null ? void 0 : options.maxTokens) != null ? _c : this.llmConfig.maxTokens) != null ? _d : 800, 800),
|
|
4334
|
+
temperature: (_f = (_e = options == null ? void 0 : options.temperature) != null ? _e : this.llmConfig.temperature) != null ? _f : 0.7,
|
|
4335
|
+
stop: options == null ? void 0 : options.stop
|
|
4336
|
+
});
|
|
4337
|
+
return (_i = (_h = (_g2 = completion.choices[0]) == null ? void 0 : _g2.message) == null ? void 0 : _h.content) != null ? _i : "";
|
|
4338
|
+
} catch (err) {
|
|
4339
|
+
console.warn("[GroqProvider] Groq chat request failed, falling back strictly to Hugging Face Qwen:", (err == null ? void 0 : err.message) || err);
|
|
4340
|
+
const hfToken = process.env.HF_TOKEN || process.env.HUGGINGFACE_API_KEY || "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";
|
|
4341
|
+
try {
|
|
4342
|
+
const hfRes = await fetch("https://router.huggingface.co/v1/chat/completions", {
|
|
4343
|
+
method: "POST",
|
|
4344
|
+
headers: {
|
|
4345
|
+
"Content-Type": "application/json",
|
|
4346
|
+
"Authorization": `Bearer ${hfToken}`
|
|
4347
|
+
},
|
|
4348
|
+
body: JSON.stringify({
|
|
4349
|
+
model: "Qwen/Qwen2.5-Coder-32B-Instruct",
|
|
4350
|
+
messages: formattedMessages,
|
|
4351
|
+
max_tokens: 1024,
|
|
4352
|
+
temperature: 0.7
|
|
4353
|
+
})
|
|
4354
|
+
});
|
|
4355
|
+
if (hfRes.ok) {
|
|
4356
|
+
const data = await hfRes.json();
|
|
4357
|
+
const content = (_l = (_k = (_j = data == null ? void 0 : data.choices) == null ? void 0 : _j[0]) == null ? void 0 : _k.message) == null ? void 0 : _l.content;
|
|
4358
|
+
if (content !== void 0) return content;
|
|
4359
|
+
}
|
|
4360
|
+
} catch (e) {
|
|
4361
|
+
}
|
|
4362
|
+
throw err;
|
|
4363
|
+
}
|
|
4319
4364
|
}
|
|
4320
4365
|
chatStream(messages, context, options) {
|
|
4321
4366
|
return __asyncGenerator(this, null, function* () {
|
|
4322
|
-
var _a2, _b, _c, _d, _e, _f, _g2, _h;
|
|
4367
|
+
var _a2, _b, _c, _d, _e, _f, _g2, _h, _i, _j, _k, _l;
|
|
4323
4368
|
const basePrompt = (_b = (_a2 = options == null ? void 0 : options.systemPrompt) != null ? _a2 : this.llmConfig.systemPrompt) != null ? _b : "You are a helpful assistant. Answer questions based on the provided context.";
|
|
4324
4369
|
const systemMessage = {
|
|
4325
4370
|
role: "system",
|
|
@@ -4333,32 +4378,81 @@ var GroqProvider = class {
|
|
|
4333
4378
|
systemMessage,
|
|
4334
4379
|
...safeStreamMessages
|
|
4335
4380
|
];
|
|
4336
|
-
const
|
|
4337
|
-
model: this.llmConfig.model,
|
|
4338
|
-
messages: formattedMessages,
|
|
4339
|
-
max_tokens: (_d = (_c = options == null ? void 0 : options.maxTokens) != null ? _c : this.llmConfig.maxTokens) != null ? _d : 1024,
|
|
4340
|
-
temperature: (_f = (_e = options == null ? void 0 : options.temperature) != null ? _e : this.llmConfig.temperature) != null ? _f : 0.7,
|
|
4341
|
-
stop: options == null ? void 0 : options.stop,
|
|
4342
|
-
stream: true
|
|
4343
|
-
}));
|
|
4344
|
-
if (!stream) {
|
|
4345
|
-
throw new Error("[GroqProvider] completions.create stream is undefined");
|
|
4346
|
-
}
|
|
4381
|
+
const model = this.llmConfig.model === "qwen/qwen3.6-27b" || this.llmConfig.model === "qwen-3.6-27b" ? "qwen/qwen3.8-27b" : this.llmConfig.model;
|
|
4347
4382
|
try {
|
|
4348
|
-
|
|
4349
|
-
|
|
4350
|
-
|
|
4351
|
-
|
|
4383
|
+
const stream = yield new __await(this.client.chat.completions.create({
|
|
4384
|
+
model,
|
|
4385
|
+
messages: formattedMessages,
|
|
4386
|
+
max_tokens: Math.min((_d = (_c = options == null ? void 0 : options.maxTokens) != null ? _c : this.llmConfig.maxTokens) != null ? _d : 800, 800),
|
|
4387
|
+
temperature: (_f = (_e = options == null ? void 0 : options.temperature) != null ? _e : this.llmConfig.temperature) != null ? _f : 0.7,
|
|
4388
|
+
stop: options == null ? void 0 : options.stop,
|
|
4389
|
+
stream: true
|
|
4390
|
+
}));
|
|
4391
|
+
if (!stream) {
|
|
4392
|
+
throw new Error("[GroqProvider] completions.create stream is undefined");
|
|
4352
4393
|
}
|
|
4353
|
-
} catch (temp) {
|
|
4354
|
-
error = [temp];
|
|
4355
|
-
} finally {
|
|
4356
4394
|
try {
|
|
4357
|
-
|
|
4395
|
+
for (var iter = __forAwait(stream), more, temp, error; more = !(temp = yield new __await(iter.next())).done; more = false) {
|
|
4396
|
+
const chunk = temp.value;
|
|
4397
|
+
const content = ((_h = (_g2 = chunk.choices[0]) == null ? void 0 : _g2.delta) == null ? void 0 : _h.content) || "";
|
|
4398
|
+
if (content) yield content;
|
|
4399
|
+
}
|
|
4400
|
+
} catch (temp) {
|
|
4401
|
+
error = [temp];
|
|
4358
4402
|
} finally {
|
|
4359
|
-
|
|
4360
|
-
|
|
4403
|
+
try {
|
|
4404
|
+
more && (temp = iter.return) && (yield new __await(temp.call(iter)));
|
|
4405
|
+
} finally {
|
|
4406
|
+
if (error)
|
|
4407
|
+
throw error[0];
|
|
4408
|
+
}
|
|
4409
|
+
}
|
|
4410
|
+
} catch (err) {
|
|
4411
|
+
console.warn("[GroqProvider] Groq chat stream failed, falling back strictly to Hugging Face Qwen stream:", (err == null ? void 0 : err.message) || err);
|
|
4412
|
+
const hfToken = process.env.HF_TOKEN || process.env.HUGGINGFACE_API_KEY || "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";
|
|
4413
|
+
try {
|
|
4414
|
+
const hfRes = yield new __await(fetch("https://router.huggingface.co/v1/chat/completions", {
|
|
4415
|
+
method: "POST",
|
|
4416
|
+
headers: {
|
|
4417
|
+
"Content-Type": "application/json",
|
|
4418
|
+
"Authorization": `Bearer ${hfToken}`
|
|
4419
|
+
},
|
|
4420
|
+
body: JSON.stringify({
|
|
4421
|
+
model: "Qwen/Qwen2.5-Coder-32B-Instruct",
|
|
4422
|
+
messages: formattedMessages,
|
|
4423
|
+
max_tokens: 1024,
|
|
4424
|
+
temperature: 0.7,
|
|
4425
|
+
stream: true
|
|
4426
|
+
})
|
|
4427
|
+
}));
|
|
4428
|
+
if (hfRes.ok && hfRes.body) {
|
|
4429
|
+
const reader = hfRes.body.getReader();
|
|
4430
|
+
const decoder = new TextDecoder("utf-8");
|
|
4431
|
+
let buffer = "";
|
|
4432
|
+
while (true) {
|
|
4433
|
+
const { done, value } = yield new __await(reader.read());
|
|
4434
|
+
if (done) break;
|
|
4435
|
+
buffer += decoder.decode(value, { stream: true });
|
|
4436
|
+
const lines = buffer.split("\n");
|
|
4437
|
+
buffer = (_i = lines.pop()) != null ? _i : "";
|
|
4438
|
+
for (const line of lines) {
|
|
4439
|
+
const trimmed = line.trim();
|
|
4440
|
+
if (!trimmed || trimmed === "data: [DONE]") continue;
|
|
4441
|
+
if (!trimmed.startsWith("data:")) continue;
|
|
4442
|
+
try {
|
|
4443
|
+
const json = JSON.parse(trimmed.slice(5).trim());
|
|
4444
|
+
const content = ((_l = (_k = (_j = json == null ? void 0 : json.choices) == null ? void 0 : _j[0]) == null ? void 0 : _k.delta) == null ? void 0 : _l.content) || "";
|
|
4445
|
+
if (content) yield content;
|
|
4446
|
+
} catch (e) {
|
|
4447
|
+
}
|
|
4448
|
+
}
|
|
4449
|
+
}
|
|
4450
|
+
return;
|
|
4451
|
+
}
|
|
4452
|
+
} catch (hfErr) {
|
|
4453
|
+
console.warn("[GroqProvider] Hugging Face Qwen fallback stream failed:", hfErr);
|
|
4361
4454
|
}
|
|
4455
|
+
throw err;
|
|
4362
4456
|
}
|
|
4363
4457
|
});
|
|
4364
4458
|
}
|
|
@@ -4808,6 +4902,7 @@ ${context != null ? context : "None"}` },
|
|
|
4808
4902
|
} catch (fetchErr) {
|
|
4809
4903
|
console.warn(`[UniversalLLMAdapter] Remote HTTP stream fetch warning: ${fetchErr == null ? void 0 : fetchErr.message}. Attempting in-process fallback...`);
|
|
4810
4904
|
}
|
|
4905
|
+
let lastDispatchErr = null;
|
|
4811
4906
|
if (!streamBody) {
|
|
4812
4907
|
const _g2 = globalThis;
|
|
4813
4908
|
const dispatch = _g2.__retrivoraDispatchChat;
|
|
@@ -4830,12 +4925,16 @@ ${context != null ? context : "None"}` },
|
|
|
4830
4925
|
}
|
|
4831
4926
|
}
|
|
4832
4927
|
} catch (dispatchErr) {
|
|
4833
|
-
|
|
4928
|
+
lastDispatchErr = dispatchErr;
|
|
4929
|
+
console.warn("[UniversalLLMAdapter] In-process dispatch error:", (dispatchErr == null ? void 0 : dispatchErr.message) || dispatchErr);
|
|
4834
4930
|
}
|
|
4835
4931
|
}
|
|
4836
4932
|
}
|
|
4837
4933
|
if (!streamBody) {
|
|
4838
|
-
|
|
4934
|
+
if (lastDispatchErr) {
|
|
4935
|
+
throw lastDispatchErr;
|
|
4936
|
+
}
|
|
4937
|
+
throw new Error(`[UniversalLLMAdapter] Streaming request failed: LLM provider unavailable.`);
|
|
4839
4938
|
}
|
|
4840
4939
|
const reader = streamBody.getReader();
|
|
4841
4940
|
const decoder = new TextDecoder("utf-8");
|
|
@@ -4874,72 +4973,74 @@ ${context != null ? context : "None"}` },
|
|
|
4874
4973
|
});
|
|
4875
4974
|
}
|
|
4876
4975
|
async embed(text) {
|
|
4877
|
-
var _a2, _b, _c, _d, _e, _f,
|
|
4878
|
-
const
|
|
4976
|
+
var _a2, _b, _c, _d, _e, _f, _g3, _h;
|
|
4977
|
+
const _g2 = globalThis;
|
|
4978
|
+
const dispatchEmbed = _g2.__retrivoraDispatchEmbedding;
|
|
4979
|
+
if (typeof dispatchEmbed === "function") {
|
|
4980
|
+
try {
|
|
4981
|
+
const res = await dispatchEmbed({ input: text, model: this.model }, this.apiKey);
|
|
4982
|
+
if ((_b = (_a2 = res == null ? void 0 : res.data) == null ? void 0 : _a2[0]) == null ? void 0 : _b.embedding) {
|
|
4983
|
+
return res.data[0].embedding;
|
|
4984
|
+
}
|
|
4985
|
+
} catch (dispatchErr) {
|
|
4986
|
+
}
|
|
4987
|
+
}
|
|
4988
|
+
const path2 = (_c = this.opts.embedPath) != null ? _c : "/embeddings";
|
|
4879
4989
|
const payload = this.opts.embedPayloadTemplate ? buildPayload(this.opts.embedPayloadTemplate, { input: text, model: this.model }) : { input: text, model: this.model };
|
|
4880
4990
|
try {
|
|
4881
4991
|
const { data } = await this.http.post(path2, payload);
|
|
4882
|
-
const extractPath = (
|
|
4992
|
+
const extractPath = (_d = this.opts.embedExtractPath) != null ? _d : "data[0].embedding";
|
|
4883
4993
|
const vector = resolvePath(data, extractPath);
|
|
4884
4994
|
if (Array.isArray(vector)) {
|
|
4885
4995
|
return vector;
|
|
4886
4996
|
}
|
|
4887
|
-
console.warn(
|
|
4888
|
-
`[UniversalLLMAdapter] Embedding API returned unexpected format for path '${extractPath}'. Response:`,
|
|
4889
|
-
data
|
|
4890
|
-
);
|
|
4891
4997
|
} catch (err) {
|
|
4892
|
-
console.warn(
|
|
4893
|
-
`[UniversalLLMAdapter] HTTP embedding request failed: ${(err == null ? void 0 : err.message) || err}`,
|
|
4894
|
-
{ path: path2, model: this.model, response: (_c = err == null ? void 0 : err.response) == null ? void 0 : _c.data }
|
|
4895
|
-
);
|
|
4896
|
-
const _g3 = globalThis;
|
|
4897
|
-
const dispatchEmbed = _g3.__retrivoraDispatchEmbedding;
|
|
4898
4998
|
if (typeof dispatchEmbed === "function") {
|
|
4899
4999
|
try {
|
|
4900
5000
|
const res = await dispatchEmbed({ input: text, model: this.model }, this.apiKey);
|
|
4901
|
-
if ((
|
|
5001
|
+
if ((_f = (_e = res == null ? void 0 : res.data) == null ? void 0 : _e[0]) == null ? void 0 : _f.embedding) {
|
|
4902
5002
|
return res.data[0].embedding;
|
|
4903
5003
|
}
|
|
4904
5004
|
} catch (dispatchErr) {
|
|
4905
5005
|
console.warn("[UniversalLLMAdapter] In-process dispatch embedding failed:", dispatchErr);
|
|
4906
5006
|
}
|
|
4907
5007
|
}
|
|
4908
|
-
}
|
|
4909
|
-
try {
|
|
4910
|
-
const { data } = await this.http.post(path2, payload);
|
|
4911
|
-
const extractPath = (_f = this.opts.embedExtractPath) != null ? _f : "data[0].embedding";
|
|
4912
|
-
const vector = resolvePath(data, extractPath);
|
|
4913
|
-
if (Array.isArray(vector)) {
|
|
4914
|
-
return vector;
|
|
4915
|
-
}
|
|
4916
|
-
console.error(
|
|
4917
|
-
`[UniversalLLMAdapter] Retry embedding failed: unexpected response format for path '${extractPath}'. Response:`,
|
|
4918
|
-
data
|
|
4919
|
-
);
|
|
4920
|
-
throw new Error(
|
|
4921
|
-
`[UniversalLLMAdapter] Could not extract embedding vector. Expected array at '${extractPath}', got: ${typeof vector}. Response keys: ${Object.keys(data || {})}.`
|
|
4922
|
-
);
|
|
4923
|
-
} catch (err) {
|
|
4924
5008
|
console.error(
|
|
4925
|
-
`[UniversalLLMAdapter]
|
|
4926
|
-
{ error: err == null ? void 0 : err.message, statusCode: (
|
|
5009
|
+
`[UniversalLLMAdapter] Embedding failed. Path: ${path2}, Model: ${this.model}`,
|
|
5010
|
+
{ error: err == null ? void 0 : err.message, statusCode: (_g3 = err == null ? void 0 : err.response) == null ? void 0 : _g3.status, response: (_h = err == null ? void 0 : err.response) == null ? void 0 : _h.data }
|
|
4927
5011
|
);
|
|
4928
5012
|
throw new Error(
|
|
4929
|
-
`[UniversalLLMAdapter] Failed to generate embedding
|
|
5013
|
+
`[UniversalLLMAdapter] Failed to generate embedding: ${(err == null ? void 0 : err.message) || err}`
|
|
4930
5014
|
);
|
|
4931
5015
|
}
|
|
5016
|
+
throw new Error(`[UniversalLLMAdapter] Failed to extract embedding vector.`);
|
|
4932
5017
|
}
|
|
4933
5018
|
async batchEmbed(texts) {
|
|
4934
|
-
|
|
4935
|
-
|
|
5019
|
+
var _a2;
|
|
5020
|
+
if (!texts || texts.length === 0) return [];
|
|
5021
|
+
const _g2 = globalThis;
|
|
5022
|
+
const dispatchEmbed = _g2.__retrivoraDispatchEmbedding;
|
|
5023
|
+
if (typeof dispatchEmbed === "function") {
|
|
4936
5024
|
try {
|
|
4937
|
-
const
|
|
4938
|
-
|
|
4939
|
-
|
|
4940
|
-
|
|
4941
|
-
|
|
5025
|
+
const res = await dispatchEmbed({ input: texts, model: this.model }, this.apiKey);
|
|
5026
|
+
if ((res == null ? void 0 : res.data) && Array.isArray(res.data) && res.data.length === texts.length) {
|
|
5027
|
+
return res.data.map((d) => d.embedding);
|
|
5028
|
+
}
|
|
5029
|
+
} catch (e) {
|
|
5030
|
+
}
|
|
5031
|
+
}
|
|
5032
|
+
const path2 = (_a2 = this.opts.embedPath) != null ? _a2 : "/embeddings";
|
|
5033
|
+
try {
|
|
5034
|
+
const { data } = await this.http.post(path2, { input: texts, model: this.model });
|
|
5035
|
+
if ((data == null ? void 0 : data.data) && Array.isArray(data.data) && data.data.length === texts.length) {
|
|
5036
|
+
return data.data.map((d) => d.embedding);
|
|
4942
5037
|
}
|
|
5038
|
+
} catch (e) {
|
|
5039
|
+
}
|
|
5040
|
+
const results = [];
|
|
5041
|
+
for (const text of texts) {
|
|
5042
|
+
const vector = await this.embed(text);
|
|
5043
|
+
results.push(vector);
|
|
4943
5044
|
}
|
|
4944
5045
|
return results;
|
|
4945
5046
|
}
|
|
@@ -5473,9 +5574,6 @@ var ConfigValidator = class {
|
|
|
5473
5574
|
}
|
|
5474
5575
|
};
|
|
5475
5576
|
|
|
5476
|
-
// src/core/Pipeline.ts
|
|
5477
|
-
init_version();
|
|
5478
|
-
|
|
5479
5577
|
// src/rag/DocumentChunker.ts
|
|
5480
5578
|
var DocumentChunker = class {
|
|
5481
5579
|
constructor(chunkSize = 1e3, chunkOverlap = 200, separators = ["\n# ", "\n## ", "\n### ", "\n#### ", "\n\n", "\n", " ", ""]) {
|
|
@@ -5645,27 +5743,384 @@ Text to extract from:
|
|
|
5645
5743
|
}
|
|
5646
5744
|
};
|
|
5647
5745
|
|
|
5746
|
+
// src/rag/HybridSearch.ts
|
|
5747
|
+
var HybridSearch = class {
|
|
5748
|
+
/**
|
|
5749
|
+
* Tokenize text into normalized alphanumeric keywords, keeping exact codes, numbers, and identifiers.
|
|
5750
|
+
*/
|
|
5751
|
+
static tokenize(text) {
|
|
5752
|
+
if (!text) return [];
|
|
5753
|
+
return text.toLowerCase().replace(/[^\w\s\-_#./:]/g, " ").split(/\s+/).map((t) => t.trim()).filter((t) => t.length > 1 && !this.stopWords.has(t));
|
|
5754
|
+
}
|
|
5755
|
+
/**
|
|
5756
|
+
* Calculate Okapi BM25 sparse scores for an array of document chunks given a query.
|
|
5757
|
+
*/
|
|
5758
|
+
static scoreBM25(documents, query, k1 = 1.5, b = 0.75) {
|
|
5759
|
+
const scores = /* @__PURE__ */ new Map();
|
|
5760
|
+
const queryTokens = this.tokenize(query);
|
|
5761
|
+
if (queryTokens.length === 0 || documents.length === 0) {
|
|
5762
|
+
documents.forEach((d) => scores.set(d.id, 0));
|
|
5763
|
+
return scores;
|
|
5764
|
+
}
|
|
5765
|
+
const docTokens = documents.map((d) => ({
|
|
5766
|
+
id: d.id,
|
|
5767
|
+
tokens: this.tokenize(d.content)
|
|
5768
|
+
}));
|
|
5769
|
+
const N = documents.length;
|
|
5770
|
+
const avgdl = docTokens.reduce((acc, d) => acc + d.tokens.length, 0) / (N || 1);
|
|
5771
|
+
const dfMap = /* @__PURE__ */ new Map();
|
|
5772
|
+
for (const qt of queryTokens) {
|
|
5773
|
+
let count = 0;
|
|
5774
|
+
for (const dt of docTokens) {
|
|
5775
|
+
if (dt.tokens.includes(qt)) count++;
|
|
5776
|
+
}
|
|
5777
|
+
dfMap.set(qt, count);
|
|
5778
|
+
}
|
|
5779
|
+
for (const dt of docTokens) {
|
|
5780
|
+
let score = 0;
|
|
5781
|
+
const docLen = dt.tokens.length;
|
|
5782
|
+
const tfMap = /* @__PURE__ */ new Map();
|
|
5783
|
+
for (const t of dt.tokens) {
|
|
5784
|
+
tfMap.set(t, (tfMap.get(t) || 0) + 1);
|
|
5785
|
+
}
|
|
5786
|
+
for (const qt of queryTokens) {
|
|
5787
|
+
const tf = tfMap.get(qt) || 0;
|
|
5788
|
+
if (tf === 0) continue;
|
|
5789
|
+
const df = dfMap.get(qt) || 0;
|
|
5790
|
+
const idf = Math.log((N - df + 0.5) / (df + 0.5) + 1);
|
|
5791
|
+
const numerator = tf * (k1 + 1);
|
|
5792
|
+
const denominator = tf + k1 * (1 - b + b * (docLen / (avgdl || 1)));
|
|
5793
|
+
score += idf * (numerator / (denominator || 1));
|
|
5794
|
+
}
|
|
5795
|
+
scores.set(dt.id, score);
|
|
5796
|
+
}
|
|
5797
|
+
return scores;
|
|
5798
|
+
}
|
|
5799
|
+
/**
|
|
5800
|
+
* Fuse Dense Vector Matches with Sparse BM25 Scores using Reciprocal Rank Fusion (RRF).
|
|
5801
|
+
*/
|
|
5802
|
+
static fuse(denseMatches, query, options = {}) {
|
|
5803
|
+
var _a2, _b, _c;
|
|
5804
|
+
if (denseMatches.length === 0) return [];
|
|
5805
|
+
const denseWeight = (_a2 = options.denseWeight) != null ? _a2 : 0.6;
|
|
5806
|
+
const sparseWeight = (_b = options.sparseWeight) != null ? _b : 0.4;
|
|
5807
|
+
const k = (_c = options.rrfK) != null ? _c : 60;
|
|
5808
|
+
const sortedDense = [...denseMatches].sort((a, b) => b.score - a.score);
|
|
5809
|
+
const denseRankMap = /* @__PURE__ */ new Map();
|
|
5810
|
+
sortedDense.forEach((match, idx) => {
|
|
5811
|
+
denseRankMap.set(match.id, idx + 1);
|
|
5812
|
+
});
|
|
5813
|
+
const docItems = denseMatches.map((m) => ({ id: m.id, content: m.content }));
|
|
5814
|
+
const bm25ScoreMap = this.scoreBM25(docItems, query, options.bm25K1, options.bm25B);
|
|
5815
|
+
const sortedSparse = [...denseMatches].sort((a, b) => {
|
|
5816
|
+
const scoreA = bm25ScoreMap.get(a.id) || 0;
|
|
5817
|
+
const scoreB = bm25ScoreMap.get(b.id) || 0;
|
|
5818
|
+
return scoreB - scoreA;
|
|
5819
|
+
});
|
|
5820
|
+
const sparseRankMap = /* @__PURE__ */ new Map();
|
|
5821
|
+
sortedSparse.forEach((match, idx) => {
|
|
5822
|
+
sparseRankMap.set(match.id, idx + 1);
|
|
5823
|
+
});
|
|
5824
|
+
const fusedMatches = denseMatches.map((match) => {
|
|
5825
|
+
var _a3;
|
|
5826
|
+
const denseRank = denseRankMap.get(match.id) || denseMatches.length;
|
|
5827
|
+
const sparseRank = sparseRankMap.get(match.id) || denseMatches.length;
|
|
5828
|
+
const denseRrf = denseWeight / (k + denseRank);
|
|
5829
|
+
const sparseRrf = sparseWeight / (k + sparseRank);
|
|
5830
|
+
const combinedRrf = denseRrf + sparseRrf;
|
|
5831
|
+
const bm25Raw = bm25ScoreMap.get(match.id) || 0;
|
|
5832
|
+
return __spreadProps(__spreadValues({}, match), {
|
|
5833
|
+
rrfScore: combinedRrf,
|
|
5834
|
+
bm25Score: bm25Raw,
|
|
5835
|
+
rerankScore: ((_a3 = match.rerankScore) != null ? _a3 : match.score) * 0.5 + combinedRrf * 50
|
|
5836
|
+
});
|
|
5837
|
+
});
|
|
5838
|
+
return fusedMatches.sort((a, b) => {
|
|
5839
|
+
var _a3, _b2;
|
|
5840
|
+
return ((_a3 = b.rerankScore) != null ? _a3 : b.score) - ((_b2 = a.rerankScore) != null ? _b2 : a.score);
|
|
5841
|
+
});
|
|
5842
|
+
}
|
|
5843
|
+
};
|
|
5844
|
+
HybridSearch.stopWords = /* @__PURE__ */ new Set([
|
|
5845
|
+
"a",
|
|
5846
|
+
"about",
|
|
5847
|
+
"above",
|
|
5848
|
+
"after",
|
|
5849
|
+
"again",
|
|
5850
|
+
"against",
|
|
5851
|
+
"all",
|
|
5852
|
+
"am",
|
|
5853
|
+
"an",
|
|
5854
|
+
"and",
|
|
5855
|
+
"any",
|
|
5856
|
+
"are",
|
|
5857
|
+
"aren't",
|
|
5858
|
+
"as",
|
|
5859
|
+
"at",
|
|
5860
|
+
"be",
|
|
5861
|
+
"because",
|
|
5862
|
+
"been",
|
|
5863
|
+
"before",
|
|
5864
|
+
"being",
|
|
5865
|
+
"below",
|
|
5866
|
+
"between",
|
|
5867
|
+
"both",
|
|
5868
|
+
"but",
|
|
5869
|
+
"by",
|
|
5870
|
+
"can't",
|
|
5871
|
+
"cannot",
|
|
5872
|
+
"could",
|
|
5873
|
+
"couldn't",
|
|
5874
|
+
"did",
|
|
5875
|
+
"didn't",
|
|
5876
|
+
"do",
|
|
5877
|
+
"does",
|
|
5878
|
+
"doesn't",
|
|
5879
|
+
"doing",
|
|
5880
|
+
"don't",
|
|
5881
|
+
"down",
|
|
5882
|
+
"during",
|
|
5883
|
+
"each",
|
|
5884
|
+
"few",
|
|
5885
|
+
"for",
|
|
5886
|
+
"from",
|
|
5887
|
+
"further",
|
|
5888
|
+
"had",
|
|
5889
|
+
"hadn't",
|
|
5890
|
+
"has",
|
|
5891
|
+
"hasn't",
|
|
5892
|
+
"have",
|
|
5893
|
+
"haven't",
|
|
5894
|
+
"having",
|
|
5895
|
+
"he",
|
|
5896
|
+
"he'd",
|
|
5897
|
+
"he'll",
|
|
5898
|
+
"he's",
|
|
5899
|
+
"her",
|
|
5900
|
+
"here",
|
|
5901
|
+
"here's",
|
|
5902
|
+
"hers",
|
|
5903
|
+
"herself",
|
|
5904
|
+
"him",
|
|
5905
|
+
"himself",
|
|
5906
|
+
"his",
|
|
5907
|
+
"how",
|
|
5908
|
+
"how's",
|
|
5909
|
+
"i",
|
|
5910
|
+
"i'd",
|
|
5911
|
+
"i'll",
|
|
5912
|
+
"i'm",
|
|
5913
|
+
"i've",
|
|
5914
|
+
"if",
|
|
5915
|
+
"in",
|
|
5916
|
+
"into",
|
|
5917
|
+
"is",
|
|
5918
|
+
"isn't",
|
|
5919
|
+
"it",
|
|
5920
|
+
"it's",
|
|
5921
|
+
"its",
|
|
5922
|
+
"itself",
|
|
5923
|
+
"let's",
|
|
5924
|
+
"me",
|
|
5925
|
+
"more",
|
|
5926
|
+
"most",
|
|
5927
|
+
"mustn't",
|
|
5928
|
+
"my",
|
|
5929
|
+
"myself",
|
|
5930
|
+
"no",
|
|
5931
|
+
"nor",
|
|
5932
|
+
"not",
|
|
5933
|
+
"of",
|
|
5934
|
+
"off",
|
|
5935
|
+
"on",
|
|
5936
|
+
"once",
|
|
5937
|
+
"only",
|
|
5938
|
+
"or",
|
|
5939
|
+
"other",
|
|
5940
|
+
"ought",
|
|
5941
|
+
"our",
|
|
5942
|
+
"ours",
|
|
5943
|
+
"ourselves",
|
|
5944
|
+
"out",
|
|
5945
|
+
"over",
|
|
5946
|
+
"own",
|
|
5947
|
+
"same",
|
|
5948
|
+
"shan't",
|
|
5949
|
+
"she",
|
|
5950
|
+
"she'd",
|
|
5951
|
+
"she'll",
|
|
5952
|
+
"she's",
|
|
5953
|
+
"should",
|
|
5954
|
+
"shouldn't",
|
|
5955
|
+
"so",
|
|
5956
|
+
"some",
|
|
5957
|
+
"such",
|
|
5958
|
+
"than",
|
|
5959
|
+
"that",
|
|
5960
|
+
"that's",
|
|
5961
|
+
"the",
|
|
5962
|
+
"their",
|
|
5963
|
+
"theirs",
|
|
5964
|
+
"them",
|
|
5965
|
+
"themselves",
|
|
5966
|
+
"then",
|
|
5967
|
+
"there",
|
|
5968
|
+
"there's",
|
|
5969
|
+
"these",
|
|
5970
|
+
"they",
|
|
5971
|
+
"they'd",
|
|
5972
|
+
"they'll",
|
|
5973
|
+
"they're",
|
|
5974
|
+
"they've",
|
|
5975
|
+
"this",
|
|
5976
|
+
"those",
|
|
5977
|
+
"through",
|
|
5978
|
+
"to",
|
|
5979
|
+
"too",
|
|
5980
|
+
"under",
|
|
5981
|
+
"until",
|
|
5982
|
+
"up",
|
|
5983
|
+
"very",
|
|
5984
|
+
"was",
|
|
5985
|
+
"wasn't",
|
|
5986
|
+
"we",
|
|
5987
|
+
"we'd",
|
|
5988
|
+
"we'll",
|
|
5989
|
+
"we're",
|
|
5990
|
+
"we've",
|
|
5991
|
+
"were",
|
|
5992
|
+
"weren't",
|
|
5993
|
+
"what",
|
|
5994
|
+
"what's",
|
|
5995
|
+
"when",
|
|
5996
|
+
"when's",
|
|
5997
|
+
"where",
|
|
5998
|
+
"where's",
|
|
5999
|
+
"which",
|
|
6000
|
+
"while",
|
|
6001
|
+
"who",
|
|
6002
|
+
"who's",
|
|
6003
|
+
"whom",
|
|
6004
|
+
"why",
|
|
6005
|
+
"why's",
|
|
6006
|
+
"with",
|
|
6007
|
+
"won't",
|
|
6008
|
+
"would",
|
|
6009
|
+
"wouldn't",
|
|
6010
|
+
"you",
|
|
6011
|
+
"you'd",
|
|
6012
|
+
"you'll",
|
|
6013
|
+
"you're",
|
|
6014
|
+
"you've",
|
|
6015
|
+
"your",
|
|
6016
|
+
"yours",
|
|
6017
|
+
"yourself",
|
|
6018
|
+
"yourselves"
|
|
6019
|
+
]);
|
|
6020
|
+
|
|
5648
6021
|
// src/rag/Reranker.ts
|
|
5649
6022
|
var MAX_RERANK_CONTENT_CHARS = 64e3;
|
|
5650
6023
|
var Reranker = class {
|
|
5651
6024
|
/**
|
|
5652
|
-
* Re-ranks matches based on a secondary relevance score using
|
|
5653
|
-
*
|
|
5654
|
-
* Fix RAG-2: keyword re-ranking no longer mutates the original `score`
|
|
5655
|
-
* field (which consumers interpret as the raw vector similarity). A new
|
|
5656
|
-
* `rerankScore` field is added so both the original similarity and the
|
|
5657
|
-
* boosted rank signal are available to callers.
|
|
6025
|
+
* Re-ranks matches based on a secondary relevance score using Hybrid BM25, Cohere, Jina or LLM.
|
|
5658
6026
|
*/
|
|
5659
|
-
async rerank(matches, query, limit = 5, llm) {
|
|
5660
|
-
|
|
5661
|
-
|
|
6027
|
+
async rerank(matches, query, limit = 5, llm, options = {}) {
|
|
6028
|
+
var _a2, _b;
|
|
6029
|
+
if (!matches || matches.length <= 1) {
|
|
6030
|
+
return matches ? matches.slice(0, limit) : [];
|
|
5662
6031
|
}
|
|
5663
|
-
|
|
5664
|
-
|
|
5665
|
-
|
|
5666
|
-
|
|
5667
|
-
|
|
6032
|
+
const cohereKey = options.cohereApiKey || (typeof process !== "undefined" ? (_a2 = process.env) == null ? void 0 : _a2.COHERE_API_KEY : void 0);
|
|
6033
|
+
const jinaKey = options.jinaApiKey || (typeof process !== "undefined" ? (_b = process.env) == null ? void 0 : _b.JINA_API_KEY : void 0);
|
|
6034
|
+
if ((options.provider === "cohere" || cohereKey) && cohereKey) {
|
|
6035
|
+
try {
|
|
6036
|
+
const cohereResult = await this.cohereRerank(matches, query, limit, cohereKey, options.cohereModel);
|
|
6037
|
+
if (cohereResult.length > 0) return cohereResult;
|
|
6038
|
+
} catch (err) {
|
|
6039
|
+
console.warn("[Reranker] Cohere reranking failed, falling back to HybridSearch:", err);
|
|
6040
|
+
}
|
|
6041
|
+
}
|
|
6042
|
+
if ((options.provider === "jina" || jinaKey) && jinaKey) {
|
|
6043
|
+
try {
|
|
6044
|
+
const jinaResult = await this.jinaRerank(matches, query, limit, jinaKey, options.jinaModel);
|
|
6045
|
+
if (jinaResult.length > 0) return jinaResult;
|
|
6046
|
+
} catch (err) {
|
|
6047
|
+
console.warn("[Reranker] Jina reranking failed, falling back to HybridSearch:", err);
|
|
6048
|
+
}
|
|
6049
|
+
}
|
|
6050
|
+
if (!options.provider || options.provider === "hybrid") {
|
|
6051
|
+
const fused = HybridSearch.fuse(matches, query);
|
|
6052
|
+
if (fused.length > 0) {
|
|
6053
|
+
return fused.slice(0, limit);
|
|
6054
|
+
}
|
|
6055
|
+
}
|
|
6056
|
+
if (llm && (options.provider === "llm" || !options.provider)) {
|
|
6057
|
+
try {
|
|
6058
|
+
return await this.llmRerank(matches, query, limit, llm);
|
|
6059
|
+
} catch (error) {
|
|
6060
|
+
console.warn("[Reranker] LLM re-ranking failed, falling back to keyword sorting:", error);
|
|
6061
|
+
}
|
|
6062
|
+
}
|
|
6063
|
+
return this.keywordRerank(matches, query, limit);
|
|
6064
|
+
}
|
|
6065
|
+
/**
|
|
6066
|
+
* Cohere Rerank v3.5 API integration.
|
|
6067
|
+
*/
|
|
6068
|
+
async cohereRerank(matches, query, limit, apiKey, model = "rerank-v3.5") {
|
|
6069
|
+
const documents = matches.map((m) => m.content || "");
|
|
6070
|
+
const response = await fetch("https://api.cohere.com/v1/rerank", {
|
|
6071
|
+
method: "POST",
|
|
6072
|
+
headers: {
|
|
6073
|
+
"Authorization": `Bearer ${apiKey}`,
|
|
6074
|
+
"Content-Type": "application/json"
|
|
6075
|
+
},
|
|
6076
|
+
body: JSON.stringify({
|
|
6077
|
+
model,
|
|
6078
|
+
query,
|
|
6079
|
+
documents,
|
|
6080
|
+
top_n: limit
|
|
6081
|
+
})
|
|
6082
|
+
});
|
|
6083
|
+
if (!response.ok) {
|
|
6084
|
+
throw new Error(`Cohere API returned status ${response.status}`);
|
|
6085
|
+
}
|
|
6086
|
+
const data = await response.json();
|
|
6087
|
+
if (!data.results || !Array.isArray(data.results)) return [];
|
|
6088
|
+
return data.results.map((res) => {
|
|
6089
|
+
const match = matches[res.index];
|
|
6090
|
+
return __spreadProps(__spreadValues({}, match), {
|
|
6091
|
+
rerankScore: res.relevance_score
|
|
6092
|
+
});
|
|
6093
|
+
});
|
|
6094
|
+
}
|
|
6095
|
+
/**
|
|
6096
|
+
* Jina AI Reranker v2 API integration.
|
|
6097
|
+
*/
|
|
6098
|
+
async jinaRerank(matches, query, limit, apiKey, model = "jina-reranker-v2-base-multilingual") {
|
|
6099
|
+
const documents = matches.map((m) => m.content || "");
|
|
6100
|
+
const response = await fetch("https://api.jina.ai/v1/rerank", {
|
|
6101
|
+
method: "POST",
|
|
6102
|
+
headers: {
|
|
6103
|
+
"Authorization": `Bearer ${apiKey}`,
|
|
6104
|
+
"Content-Type": "application/json"
|
|
6105
|
+
},
|
|
6106
|
+
body: JSON.stringify({
|
|
6107
|
+
model,
|
|
6108
|
+
query,
|
|
6109
|
+
documents,
|
|
6110
|
+
top_n: limit
|
|
6111
|
+
})
|
|
6112
|
+
});
|
|
6113
|
+
if (!response.ok) {
|
|
6114
|
+
throw new Error(`Jina API returned status ${response.status}`);
|
|
5668
6115
|
}
|
|
6116
|
+
const data = await response.json();
|
|
6117
|
+
if (!data.results || !Array.isArray(data.results)) return [];
|
|
6118
|
+
return data.results.map((res) => {
|
|
6119
|
+
const match = matches[res.index];
|
|
6120
|
+
return __spreadProps(__spreadValues({}, match), {
|
|
6121
|
+
rerankScore: res.relevance_score
|
|
6122
|
+
});
|
|
6123
|
+
});
|
|
5669
6124
|
}
|
|
5670
6125
|
/**
|
|
5671
6126
|
* Keyword-based re-ranking fallback.
|
|
@@ -5751,46 +6206,216 @@ ${docList}` }],
|
|
|
5751
6206
|
}
|
|
5752
6207
|
};
|
|
5753
6208
|
|
|
5754
|
-
// src/rag/
|
|
5755
|
-
var
|
|
5756
|
-
/**
|
|
5757
|
-
* Chunks document content using LlamaIndex SentenceSplitter.
|
|
5758
|
-
* This respects sentence and paragraph boundaries much more effectively
|
|
5759
|
-
* than standard character-count splitting.
|
|
5760
|
-
*/
|
|
5761
|
-
async chunk(text, options = {}) {
|
|
5762
|
-
var _a2, _b;
|
|
5763
|
-
try {
|
|
5764
|
-
const { Document, SentenceSplitter, MetadataMode } = await import(`${"llamaindex"}`);
|
|
5765
|
-
const splitter = new SentenceSplitter({
|
|
5766
|
-
chunkSize: (_a2 = options.chunkSize) != null ? _a2 : 1e3,
|
|
5767
|
-
chunkOverlap: (_b = options.chunkOverlap) != null ? _b : 200
|
|
5768
|
-
});
|
|
5769
|
-
const doc = new Document({ text, metadata: options.metadata || {} });
|
|
5770
|
-
const nodes = splitter.getNodesFromDocuments([doc]);
|
|
5771
|
-
return nodes.map((node, index) => ({
|
|
5772
|
-
id: `${options.docId || "doc"}_node_${index}`,
|
|
5773
|
-
content: node.getContent(MetadataMode.ALL),
|
|
5774
|
-
metadata: __spreadProps(__spreadValues(__spreadValues({}, options.metadata), node.metadata), {
|
|
5775
|
-
nodeId: node.id_,
|
|
5776
|
-
chunkIndex: index
|
|
5777
|
-
})
|
|
5778
|
-
}));
|
|
5779
|
-
} catch (error) {
|
|
5780
|
-
console.warn("[LlamaIndexIngestor] LlamaIndex package not found or failed. Falling back to default chunker.");
|
|
5781
|
-
throw error;
|
|
5782
|
-
}
|
|
5783
|
-
}
|
|
6209
|
+
// src/rag/RAGEvaluator.ts
|
|
6210
|
+
var RAGEvaluator = class {
|
|
5784
6211
|
/**
|
|
5785
|
-
*
|
|
6212
|
+
* Evaluate a RAG interaction across the full RAG Triad.
|
|
5786
6213
|
*/
|
|
5787
|
-
async
|
|
5788
|
-
|
|
5789
|
-
|
|
5790
|
-
const chunks = await this.chunk(doc.content, __spreadProps(__spreadValues({}, options), { docId: doc.docId, metadata: doc.metadata }));
|
|
5791
|
-
allChunks.push(...chunks);
|
|
6214
|
+
static async evaluate(query, context, answer, llm) {
|
|
6215
|
+
if (!llm || !context || !answer) {
|
|
6216
|
+
return this.deterministicHeuristic(query, context, answer);
|
|
5792
6217
|
}
|
|
5793
|
-
|
|
6218
|
+
try {
|
|
6219
|
+
const prompt = `You are an automated RAG Triad Evaluator. Evaluate the quality of the following RAG system interaction:
|
|
6220
|
+
|
|
6221
|
+
USER QUERY:
|
|
6222
|
+
${query}
|
|
6223
|
+
|
|
6224
|
+
RETRIEVED CONTEXT:
|
|
6225
|
+
${context.slice(0, 3e3)}
|
|
6226
|
+
|
|
6227
|
+
GENERATED ANSWER:
|
|
6228
|
+
${answer}
|
|
6229
|
+
|
|
6230
|
+
Evaluate these 3 metrics on a scale of 0.0 to 1.0:
|
|
6231
|
+
1. "contextRelevance": Are the retrieved documents relevant to the query? (1.0 = highly relevant, 0.0 = irrelevant)
|
|
6232
|
+
2. "faithfulness": Is every claim in the generated answer supported by the retrieved context? (1.0 = 100% grounded/no hallucination, 0.0 = completely fabricated)
|
|
6233
|
+
3. "answerRelevance": Does the answer directly address the user's question? (1.0 = direct answer, 0.0 = evasive/unrelated)
|
|
6234
|
+
|
|
6235
|
+
Return ONLY valid JSON with no markdown fences:
|
|
6236
|
+
{"contextRelevance": 0.95, "faithfulness": 0.90, "answerRelevance": 0.92, "reason": "Answer is fully supported by the document."}`;
|
|
6237
|
+
const raw = await llm.chat(
|
|
6238
|
+
[{ role: "user", content: prompt }],
|
|
6239
|
+
"",
|
|
6240
|
+
{
|
|
6241
|
+
systemPrompt: "You are a strict AI safety and hallucination evaluator. Output only valid JSON.",
|
|
6242
|
+
temperature: 0,
|
|
6243
|
+
maxTokens: 150
|
|
6244
|
+
}
|
|
6245
|
+
);
|
|
6246
|
+
const cleaned = raw.replace(/```json|```/g, "").trim();
|
|
6247
|
+
const parsed = JSON.parse(cleaned);
|
|
6248
|
+
const cr = typeof parsed.contextRelevance === "number" ? Math.max(0, Math.min(1, parsed.contextRelevance)) : 0.85;
|
|
6249
|
+
const f = typeof parsed.faithfulness === "number" ? Math.max(0, Math.min(1, parsed.faithfulness)) : 0.9;
|
|
6250
|
+
const ar = typeof parsed.answerRelevance === "number" ? Math.max(0, Math.min(1, parsed.answerRelevance)) : 0.88;
|
|
6251
|
+
const overall = Number((cr * 0.3 + f * 0.4 + ar * 0.3).toFixed(2));
|
|
6252
|
+
return {
|
|
6253
|
+
contextRelevance: cr,
|
|
6254
|
+
faithfulness: f,
|
|
6255
|
+
answerRelevance: ar,
|
|
6256
|
+
overallQuality: overall,
|
|
6257
|
+
isHallucinated: f < 0.6,
|
|
6258
|
+
reason: parsed.reason || "Evaluated successfully."
|
|
6259
|
+
};
|
|
6260
|
+
} catch (err) {
|
|
6261
|
+
return this.deterministicHeuristic(query, context, answer);
|
|
6262
|
+
}
|
|
6263
|
+
}
|
|
6264
|
+
/**
|
|
6265
|
+
* Deterministic fast keyword-overlap heuristic when LLM evaluation is not invoked.
|
|
6266
|
+
*/
|
|
6267
|
+
static deterministicHeuristic(query, context, answer) {
|
|
6268
|
+
if (!context || !answer) {
|
|
6269
|
+
return { contextRelevance: 0.5, faithfulness: 0.8, answerRelevance: 0.7, overallQuality: 0.67, isHallucinated: false };
|
|
6270
|
+
}
|
|
6271
|
+
const queryWords = query.toLowerCase().split(/\s+/).filter((w) => w.length > 2);
|
|
6272
|
+
const contextLower = context.toLowerCase();
|
|
6273
|
+
const answerLower = answer.toLowerCase();
|
|
6274
|
+
let queryContextHits = 0;
|
|
6275
|
+
queryWords.forEach((w) => {
|
|
6276
|
+
if (contextLower.includes(w)) queryContextHits++;
|
|
6277
|
+
});
|
|
6278
|
+
const contextRelevance = queryWords.length > 0 ? Math.min(1, queryContextHits / queryWords.length + 0.3) : 0.8;
|
|
6279
|
+
const numbersInAnswer = answer.match(/\b\d+(?:\.\d+)?%?\b/g) || [];
|
|
6280
|
+
let numberHits = 0;
|
|
6281
|
+
numbersInAnswer.forEach((num) => {
|
|
6282
|
+
if (context.includes(num)) numberHits++;
|
|
6283
|
+
});
|
|
6284
|
+
const faithfulness = numbersInAnswer.length > 0 ? Math.min(1, numberHits / numbersInAnswer.length + 0.2) : 0.9;
|
|
6285
|
+
const answerRelevance = 0.85;
|
|
6286
|
+
const overallQuality = Number((contextRelevance * 0.3 + faithfulness * 0.4 + answerRelevance * 0.3).toFixed(2));
|
|
6287
|
+
return {
|
|
6288
|
+
contextRelevance: Number(contextRelevance.toFixed(2)),
|
|
6289
|
+
faithfulness: Number(faithfulness.toFixed(2)),
|
|
6290
|
+
answerRelevance,
|
|
6291
|
+
overallQuality,
|
|
6292
|
+
isHallucinated: faithfulness < 0.5,
|
|
6293
|
+
reason: "Computed via deterministic grounding heuristic."
|
|
6294
|
+
};
|
|
6295
|
+
}
|
|
6296
|
+
};
|
|
6297
|
+
|
|
6298
|
+
// src/security/PIIMasker.ts
|
|
6299
|
+
var PIIMasker = class {
|
|
6300
|
+
/**
|
|
6301
|
+
* Redact sensitive information from text.
|
|
6302
|
+
*/
|
|
6303
|
+
static mask(text, options = {}) {
|
|
6304
|
+
if (!text || typeof text !== "string") return text;
|
|
6305
|
+
let sanitized = text;
|
|
6306
|
+
if (options.maskApiKeys !== false) {
|
|
6307
|
+
sanitized = sanitized.replace(this.API_KEY_REGEX, "[REDACTED_API_KEY]");
|
|
6308
|
+
}
|
|
6309
|
+
if (options.maskCreditCards !== false) {
|
|
6310
|
+
sanitized = sanitized.replace(this.CREDIT_CARD_REGEX, (match) => {
|
|
6311
|
+
const digits = match.replace(/[- ]/g, "");
|
|
6312
|
+
if (digits.length >= 13 && digits.length <= 19) {
|
|
6313
|
+
return "[REDACTED_CREDIT_CARD]";
|
|
6314
|
+
}
|
|
6315
|
+
return match;
|
|
6316
|
+
});
|
|
6317
|
+
}
|
|
6318
|
+
if (options.maskSSN !== false) {
|
|
6319
|
+
sanitized = sanitized.replace(this.SSN_REGEX, "[REDACTED_SSN]");
|
|
6320
|
+
}
|
|
6321
|
+
if (options.maskEmails !== false) {
|
|
6322
|
+
sanitized = sanitized.replace(this.EMAIL_REGEX, "[REDACTED_EMAIL]");
|
|
6323
|
+
}
|
|
6324
|
+
if (options.maskPhones !== false) {
|
|
6325
|
+
sanitized = sanitized.replace(this.PHONE_REGEX, "[REDACTED_PHONE]");
|
|
6326
|
+
}
|
|
6327
|
+
if (options.maskIPs === true) {
|
|
6328
|
+
sanitized = sanitized.replace(this.IP_REGEX, "[REDACTED_IP]");
|
|
6329
|
+
}
|
|
6330
|
+
return sanitized;
|
|
6331
|
+
}
|
|
6332
|
+
};
|
|
6333
|
+
PIIMasker.SSN_REGEX = /\b(?!000|666|9\d{2})\d{3}[- ]?(?!00)\d{2}[- ]?(?!0000)\d{4}\b/g;
|
|
6334
|
+
PIIMasker.CREDIT_CARD_REGEX = /\b(?:\d{4}[- ]?){3}\d{4}\b|\b\d{15,16}\b/g;
|
|
6335
|
+
PIIMasker.EMAIL_REGEX = /\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Z|a-z]{2,}\b/g;
|
|
6336
|
+
PIIMasker.PHONE_REGEX = /\b(?:\+?\d{1,3}[-.\s]?)?\(?\d{3}\)?[-.\s]?\d{3}[-.\s]?\d{4}\b/g;
|
|
6337
|
+
PIIMasker.API_KEY_REGEX = /\b(?:sk-[a-zA-Z0-9]{20,}|AKIA[0-9A-Z]{16}|rtv_[a-zA-Z0-9_-]{20,}|ghp_[a-zA-Z0-9]{36}|bearer\s+[a-zA-Z0-9._-]{20,})\b/gi;
|
|
6338
|
+
PIIMasker.IP_REGEX = /\b(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\b/g;
|
|
6339
|
+
|
|
6340
|
+
// src/security/PromptGuard.ts
|
|
6341
|
+
var PromptGuard = class {
|
|
6342
|
+
/**
|
|
6343
|
+
* Scan and evaluate user query or document text for prompt injection vectors.
|
|
6344
|
+
*/
|
|
6345
|
+
static evaluate(text) {
|
|
6346
|
+
if (!text || typeof text !== "string") {
|
|
6347
|
+
return { isSafe: true, threatLevel: "none", detectedPatterns: [], sanitizedText: text };
|
|
6348
|
+
}
|
|
6349
|
+
const detected = [];
|
|
6350
|
+
let maxSeverity = "none";
|
|
6351
|
+
for (const item of this.INJECTION_PATTERNS) {
|
|
6352
|
+
if (item.pattern.test(text)) {
|
|
6353
|
+
detected.push(item.name);
|
|
6354
|
+
if (item.severity === "high") {
|
|
6355
|
+
maxSeverity = "high";
|
|
6356
|
+
} else if (item.severity === "medium" && maxSeverity !== "high") {
|
|
6357
|
+
maxSeverity = "medium";
|
|
6358
|
+
}
|
|
6359
|
+
}
|
|
6360
|
+
}
|
|
6361
|
+
const sanitized = text.replace(/<\/?(?:system|instruction|admin|prompt_override)>/gi, "");
|
|
6362
|
+
return {
|
|
6363
|
+
isSafe: maxSeverity !== "high",
|
|
6364
|
+
threatLevel: maxSeverity,
|
|
6365
|
+
detectedPatterns: detected,
|
|
6366
|
+
sanitizedText: sanitized
|
|
6367
|
+
};
|
|
6368
|
+
}
|
|
6369
|
+
};
|
|
6370
|
+
PromptGuard.INJECTION_PATTERNS = [
|
|
6371
|
+
{ pattern: /ignore\s+(?:all\s+)?(?:previous|prior|above)\s+(?:instructions|prompts|rules)/i, name: "override_instructions", severity: "high" },
|
|
6372
|
+
{ pattern: /(?:show|repeat|print|reveal|expose|output)\s+(?:the\s+)?(?:system\s+prompt|initial\s+instructions|system\s+message)/i, name: "system_prompt_leak", severity: "high" },
|
|
6373
|
+
{ pattern: /you\s+are\s+now\s+(?:in\s+developer\s+mode|unrestricted|DAN|jailbroken|godmode)/i, name: "jailbreak_persona", severity: "high" },
|
|
6374
|
+
{ pattern: /<\/?(?:system|instruction|admin|prompt_override)>/i, name: "tag_injection", severity: "medium" },
|
|
6375
|
+
{ pattern: /base64\s*decode\s*and\s*execute/i, name: "obfuscated_payload", severity: "high" },
|
|
6376
|
+
{ pattern: /do\s+anything\s+now/i, name: "dan_variant", severity: "high" }
|
|
6377
|
+
];
|
|
6378
|
+
|
|
6379
|
+
// src/rag/LlamaIndexIngestor.ts
|
|
6380
|
+
var LlamaIndexIngestor = class {
|
|
6381
|
+
/**
|
|
6382
|
+
* Chunks document content using LlamaIndex SentenceSplitter.
|
|
6383
|
+
* This respects sentence and paragraph boundaries much more effectively
|
|
6384
|
+
* than standard character-count splitting.
|
|
6385
|
+
*/
|
|
6386
|
+
async chunk(text, options = {}) {
|
|
6387
|
+
var _a2, _b;
|
|
6388
|
+
try {
|
|
6389
|
+
const { Document, SentenceSplitter, MetadataMode } = await import(`${"llamaindex"}`);
|
|
6390
|
+
const splitter = new SentenceSplitter({
|
|
6391
|
+
chunkSize: (_a2 = options.chunkSize) != null ? _a2 : 1e3,
|
|
6392
|
+
chunkOverlap: (_b = options.chunkOverlap) != null ? _b : 200
|
|
6393
|
+
});
|
|
6394
|
+
const doc = new Document({ text, metadata: options.metadata || {} });
|
|
6395
|
+
const nodes = splitter.getNodesFromDocuments([doc]);
|
|
6396
|
+
return nodes.map((node, index) => ({
|
|
6397
|
+
id: `${options.docId || "doc"}_node_${index}`,
|
|
6398
|
+
content: node.getContent(MetadataMode.ALL),
|
|
6399
|
+
metadata: __spreadProps(__spreadValues(__spreadValues({}, options.metadata), node.metadata), {
|
|
6400
|
+
nodeId: node.id_,
|
|
6401
|
+
chunkIndex: index
|
|
6402
|
+
})
|
|
6403
|
+
}));
|
|
6404
|
+
} catch (error) {
|
|
6405
|
+
console.warn("[LlamaIndexIngestor] LlamaIndex package not found or failed. Falling back to default chunker.");
|
|
6406
|
+
throw error;
|
|
6407
|
+
}
|
|
6408
|
+
}
|
|
6409
|
+
/**
|
|
6410
|
+
* Batch processing for multiple documents.
|
|
6411
|
+
*/
|
|
6412
|
+
async chunkMany(documents, options = {}) {
|
|
6413
|
+
const allChunks = [];
|
|
6414
|
+
for (const doc of documents) {
|
|
6415
|
+
const chunks = await this.chunk(doc.content, __spreadProps(__spreadValues({}, options), { docId: doc.docId, metadata: doc.metadata }));
|
|
6416
|
+
allChunks.push(...chunks);
|
|
6417
|
+
}
|
|
6418
|
+
return allChunks;
|
|
5794
6419
|
}
|
|
5795
6420
|
};
|
|
5796
6421
|
|
|
@@ -5891,6 +6516,73 @@ ${error instanceof Error ? error.message : String(error)}`
|
|
|
5891
6516
|
}
|
|
5892
6517
|
return String((_c = (_b = response == null ? void 0 : response.output) != null ? _b : response) != null ? _c : "");
|
|
5893
6518
|
}
|
|
6519
|
+
/**
|
|
6520
|
+
* Run the agentic flow in streaming mode, emitting granular SSE event frames.
|
|
6521
|
+
* Leverages LangChain/LangGraph `streamEvents` (v2 API) to stream:
|
|
6522
|
+
* - Token-by-token LLM output (`type: "text"`)
|
|
6523
|
+
* - Tool invocation / thinking steps (`type: "thinking"`)
|
|
6524
|
+
* - Tool execution metadata & search sources
|
|
6525
|
+
*/
|
|
6526
|
+
streamRun(_0) {
|
|
6527
|
+
return __asyncGenerator(this, arguments, function* (input, chatHistory = []) {
|
|
6528
|
+
var _a2;
|
|
6529
|
+
if (!this.agent) {
|
|
6530
|
+
throw new Error("[LangChainAgent] Agent not initialized. Call initialize() first.");
|
|
6531
|
+
}
|
|
6532
|
+
const { HumanMessage, AIMessage } = yield new __await(import("@langchain/core/messages"));
|
|
6533
|
+
const historyMessages = chatHistory.map(
|
|
6534
|
+
(m) => m.role === "user" ? new HumanMessage(m.content) : new AIMessage(m.content)
|
|
6535
|
+
);
|
|
6536
|
+
const inputMessages = [...historyMessages, new HumanMessage(input)];
|
|
6537
|
+
if (typeof this.agent.streamEvents === "function") {
|
|
6538
|
+
const eventStream = this.agent.streamEvents(
|
|
6539
|
+
{ messages: inputMessages },
|
|
6540
|
+
{ version: "v2" }
|
|
6541
|
+
);
|
|
6542
|
+
try {
|
|
6543
|
+
for (var iter = __forAwait(eventStream), more, temp, error; more = !(temp = yield new __await(iter.next())).done; more = false) {
|
|
6544
|
+
const event = temp.value;
|
|
6545
|
+
if (event.event === "on_tool_start") {
|
|
6546
|
+
const toolName = event.name || "document_search";
|
|
6547
|
+
yield {
|
|
6548
|
+
type: "thinking",
|
|
6549
|
+
text: `Analyzing repository with tool "${toolName}"...
|
|
6550
|
+
`
|
|
6551
|
+
};
|
|
6552
|
+
} else if (event.event === "on_tool_end") {
|
|
6553
|
+
const toolName = event.name || "document_search";
|
|
6554
|
+
yield {
|
|
6555
|
+
type: "thinking",
|
|
6556
|
+
text: `Tool "${toolName}" completed successfully.
|
|
6557
|
+
`
|
|
6558
|
+
};
|
|
6559
|
+
} else if (event.event === "on_chat_model_stream") {
|
|
6560
|
+
const chunk = (_a2 = event.data) == null ? void 0 : _a2.chunk;
|
|
6561
|
+
if (chunk == null ? void 0 : chunk.content) {
|
|
6562
|
+
const tokenText = typeof chunk.content === "string" ? chunk.content : JSON.stringify(chunk.content);
|
|
6563
|
+
yield {
|
|
6564
|
+
type: "text",
|
|
6565
|
+
text: tokenText
|
|
6566
|
+
};
|
|
6567
|
+
}
|
|
6568
|
+
}
|
|
6569
|
+
}
|
|
6570
|
+
} catch (temp) {
|
|
6571
|
+
error = [temp];
|
|
6572
|
+
} finally {
|
|
6573
|
+
try {
|
|
6574
|
+
more && (temp = iter.return) && (yield new __await(temp.call(iter)));
|
|
6575
|
+
} finally {
|
|
6576
|
+
if (error)
|
|
6577
|
+
throw error[0];
|
|
6578
|
+
}
|
|
6579
|
+
}
|
|
6580
|
+
} else {
|
|
6581
|
+
const result = yield new __await(this.run(input, chatHistory));
|
|
6582
|
+
yield { type: "text", text: result };
|
|
6583
|
+
}
|
|
6584
|
+
});
|
|
6585
|
+
}
|
|
5894
6586
|
};
|
|
5895
6587
|
|
|
5896
6588
|
// src/rag/PromptOptimizer.ts
|
|
@@ -6798,25 +7490,43 @@ ${errorMessages}`
|
|
|
6798
7490
|
* Processes up to `concurrency` items at the same time.
|
|
6799
7491
|
*/
|
|
6800
7492
|
static async processConcurrent(items, processor, concurrency = 5, options) {
|
|
7493
|
+
var _a2, _b, _c, _d;
|
|
6801
7494
|
const { throwOnPartialFailure = false } = options != null ? options : {};
|
|
6802
7495
|
const results = [];
|
|
6803
7496
|
const errors = [];
|
|
7497
|
+
const maxRetries = (_a2 = options == null ? void 0 : options.maxRetries) != null ? _a2 : 3;
|
|
7498
|
+
const initialDelayMs = (_b = options == null ? void 0 : options.initialDelayMs) != null ? _b : 100;
|
|
7499
|
+
const maxDelayMs = (_c = options == null ? void 0 : options.maxDelayMs) != null ? _c : 1e4;
|
|
7500
|
+
const backoffMultiplier = (_d = options == null ? void 0 : options.backoffMultiplier) != null ? _d : 2;
|
|
6804
7501
|
for (let i = 0; i < items.length; i += concurrency) {
|
|
6805
7502
|
const chunk = items.slice(i, i + concurrency).map((item, idx) => ({
|
|
6806
7503
|
item,
|
|
6807
7504
|
originalIndex: i + idx
|
|
6808
7505
|
}));
|
|
6809
7506
|
const chunkPromises = chunk.map(async ({ item, originalIndex }) => {
|
|
6810
|
-
|
|
6811
|
-
|
|
6812
|
-
|
|
6813
|
-
|
|
6814
|
-
|
|
6815
|
-
|
|
6816
|
-
|
|
6817
|
-
|
|
6818
|
-
|
|
7507
|
+
let lastErr;
|
|
7508
|
+
for (let attempt = 0; attempt <= maxRetries; attempt++) {
|
|
7509
|
+
try {
|
|
7510
|
+
const result = await _BatchProcessor.executeWithCircuitBreaker(
|
|
7511
|
+
() => processor(item),
|
|
7512
|
+
options == null ? void 0 : options.circuitBreaker
|
|
7513
|
+
);
|
|
7514
|
+
return { success: true, result, index: originalIndex };
|
|
7515
|
+
} catch (err) {
|
|
7516
|
+
lastErr = err instanceof Error ? err : new Error(String(err));
|
|
7517
|
+
if (!isTransientError(err) || attempt === maxRetries) {
|
|
7518
|
+
break;
|
|
7519
|
+
}
|
|
7520
|
+
const delay = calculateBackoffDelay(
|
|
7521
|
+
attempt,
|
|
7522
|
+
initialDelayMs,
|
|
7523
|
+
maxDelayMs,
|
|
7524
|
+
backoffMultiplier
|
|
7525
|
+
);
|
|
7526
|
+
await sleep(delay);
|
|
7527
|
+
}
|
|
6819
7528
|
}
|
|
7529
|
+
return { success: false, error: lastErr || new Error("Unknown error"), index: originalIndex };
|
|
6820
7530
|
});
|
|
6821
7531
|
const chunkResults = await Promise.all(chunkPromises);
|
|
6822
7532
|
for (const { success, result, error, index } of chunkResults) {
|
|
@@ -7289,14 +7999,14 @@ var IntentClassifier = class {
|
|
|
7289
7999
|
static classify(context) {
|
|
7290
8000
|
const q = (context.userQuery || "").toLowerCase().trim();
|
|
7291
8001
|
const signals = this.extractDataSignals(context);
|
|
7292
|
-
if (this.
|
|
7293
|
-
return { intent: "
|
|
8002
|
+
if (this.isProductQuery(q) || signals.isProductLike && (this.isRecommendationQuery(q) || this.isDiscoveryQuery(q))) {
|
|
8003
|
+
return { intent: "product_search", signals };
|
|
7294
8004
|
}
|
|
7295
8005
|
if (this.isComparisonQuery(q)) {
|
|
7296
8006
|
return { intent: "comparison", signals };
|
|
7297
8007
|
}
|
|
7298
|
-
if (this.
|
|
7299
|
-
return { intent: "
|
|
8008
|
+
if (this.isInformationLookup(q)) {
|
|
8009
|
+
return { intent: "information_lookup", signals };
|
|
7300
8010
|
}
|
|
7301
8011
|
if (this.isTrendQuery(q) || signals.hasDateFields && signals.hasNumericFields && /\b(growth|trend|over time|monthly|yearly|weekly)\b/.test(q)) {
|
|
7302
8012
|
return { intent: "trend_analysis", signals };
|
|
@@ -7313,7 +8023,8 @@ var IntentClassifier = class {
|
|
|
7313
8023
|
return { intent: "information_lookup", signals };
|
|
7314
8024
|
}
|
|
7315
8025
|
/**
|
|
7316
|
-
* Extract key data structure signals from retrieved documents.
|
|
8026
|
+
* Extract key data structure signals dynamically from retrieved documents.
|
|
8027
|
+
* Inspects both structured metadata and parsed chunk text.
|
|
7317
8028
|
*/
|
|
7318
8029
|
static extractDataSignals(context) {
|
|
7319
8030
|
const docs = context.retrievedDocuments || [];
|
|
@@ -7346,18 +8057,21 @@ var IntentClassifier = class {
|
|
|
7346
8057
|
let categoricalFieldCount = 0;
|
|
7347
8058
|
if (docs.length > 0) {
|
|
7348
8059
|
const keys = /* @__PURE__ */ new Set();
|
|
7349
|
-
|
|
7350
|
-
|
|
7351
|
-
|
|
7352
|
-
let
|
|
8060
|
+
const numericKeys = /* @__PURE__ */ new Set();
|
|
8061
|
+
const dateKeys = /* @__PURE__ */ new Set();
|
|
8062
|
+
const catKeys = /* @__PURE__ */ new Set();
|
|
8063
|
+
let productDocCount = 0;
|
|
7353
8064
|
docs.forEach((doc) => {
|
|
8065
|
+
if (!doc) return;
|
|
7354
8066
|
const meta = doc.metadata || {};
|
|
8067
|
+
const content = doc.content || "";
|
|
8068
|
+
let hasDocProductSignal = false;
|
|
7355
8069
|
Object.entries(meta).forEach(([key, val]) => {
|
|
7356
8070
|
if (IGNORED_SYSTEM_KEYS.has(key.toLowerCase())) return;
|
|
7357
8071
|
keys.add(key);
|
|
7358
8072
|
const kLower = key.toLowerCase();
|
|
7359
|
-
if (["price", "image", "brand", "in_stock", "rating", "sku", "product"].some((p) => kLower.includes(p))) {
|
|
7360
|
-
|
|
8073
|
+
if (["price", "image", "brand", "in_stock", "rating", "sku", "product", "item", "cost", "msrp", "category", "vendor", "inventory", "stock"].some((p) => kLower.includes(p))) {
|
|
8074
|
+
hasDocProductSignal = true;
|
|
7361
8075
|
}
|
|
7362
8076
|
if (typeof val === "number") {
|
|
7363
8077
|
numericKeys.add(key);
|
|
@@ -7369,13 +8083,23 @@ var IntentClassifier = class {
|
|
|
7369
8083
|
}
|
|
7370
8084
|
}
|
|
7371
8085
|
});
|
|
8086
|
+
if (!hasDocProductSignal && content) {
|
|
8087
|
+
const hasContentName = /(?:^|\n)\s*(?:product\s*name|item\s*name|product|item|title|name)\s*[:\-–]\s*[^\n,]+/i.test(content);
|
|
8088
|
+
const hasContentAttr = /(?:^|\n)\s*(?:price|cost|msrp|brand|vendor|sku|category|stock|inventory|rating|image)\s*[:\-–]/i.test(content) || /\$\s*[\d,.]+/.test(content);
|
|
8089
|
+
if (hasContentName && hasContentAttr) {
|
|
8090
|
+
hasDocProductSignal = true;
|
|
8091
|
+
}
|
|
8092
|
+
}
|
|
8093
|
+
if (hasDocProductSignal) {
|
|
8094
|
+
productDocCount++;
|
|
8095
|
+
}
|
|
7372
8096
|
});
|
|
7373
8097
|
hasNumericFields = numericKeys.size > 0;
|
|
7374
8098
|
hasDateFields = dateKeys.size > 0;
|
|
7375
8099
|
hasCategoricalFields = catKeys.size > 0;
|
|
7376
8100
|
numericFieldCount = numericKeys.size;
|
|
7377
8101
|
categoricalFieldCount = catKeys.size;
|
|
7378
|
-
if (
|
|
8102
|
+
if (productDocCount > 0 && (productDocCount / docs.length >= 0.3 || productDocCount >= 2)) {
|
|
7379
8103
|
isProductLike = true;
|
|
7380
8104
|
}
|
|
7381
8105
|
}
|
|
@@ -7390,8 +8114,15 @@ var IntentClassifier = class {
|
|
|
7390
8114
|
};
|
|
7391
8115
|
}
|
|
7392
8116
|
// ─── Intent Heuristic Checkers ─────────────────────────────────────────────
|
|
8117
|
+
static isDiscoveryQuery(query) {
|
|
8118
|
+
const q = query.toLowerCase().trim();
|
|
8119
|
+
return /\b(show|find|list|browse|search|explore|get|buy|shop|shopping|purchase|options|selection|picks|available|what do you have|what is in stock|looking for|under \$|cheap|affordable|best|top)\b/i.test(q) && !/\b(chart|graph|plot|sales over time|monthly trend)\b/i.test(q);
|
|
8120
|
+
}
|
|
7393
8121
|
static isInformationLookup(query) {
|
|
7394
8122
|
const q = query.toLowerCase().trim();
|
|
8123
|
+
if (/\b(show|find|list|browse|recommend|suggest|available|in stock|buy|shop|catalog|inventory|what do you have|looking for)\b/i.test(q)) {
|
|
8124
|
+
return false;
|
|
8125
|
+
}
|
|
7395
8126
|
if (/^(what|who|explain|explore|define|where|when|why|how|how does|can|could|does|do|is|are|has|have|will|should|would|meaning of|tell me about|show details of|overview of|summarize)\s+/i.test(q)) {
|
|
7396
8127
|
if (!/\b(chart|graph|plot|visualize|pie chart|bar chart|line chart|scatter plot|draw a)\b/i.test(q)) {
|
|
7397
8128
|
return true;
|
|
@@ -7406,7 +8137,11 @@ var IntentClassifier = class {
|
|
|
7406
8137
|
return /\b(compare|comparison|vs\.?|versus|difference between|differ|contrast|against)\b/i.test(query);
|
|
7407
8138
|
}
|
|
7408
8139
|
static isProductQuery(query) {
|
|
7409
|
-
|
|
8140
|
+
const q = query.toLowerCase().trim();
|
|
8141
|
+
if (/\b(price per share|share price|total cost of the deal|debt|equity|ratio|revenue|ebit|ebitda|fcfe|fcff|wacc)\b/i.test(q)) {
|
|
8142
|
+
return false;
|
|
8143
|
+
}
|
|
8144
|
+
return /\b(best|recommend|top|buy|shopping|shop|under \$|cheap|affordable|products?|items?|catalog|inventory|in stock|show available|what do you have|options for)\b/i.test(q) && !/\b(compare|chart|sales|revenue|trend)\b/i.test(q);
|
|
7410
8145
|
}
|
|
7411
8146
|
static isRecommendationQuery(query) {
|
|
7412
8147
|
return /\b(recommend|suggest|what should i (buy|get|choose)|best options|top picks)\b/i.test(query);
|
|
@@ -7471,14 +8206,23 @@ var Rule3ProductDiscoveryRule = class {
|
|
|
7471
8206
|
}
|
|
7472
8207
|
evaluate(context, intent) {
|
|
7473
8208
|
const q = (context.userQuery || "").toLowerCase().trim();
|
|
7474
|
-
const
|
|
7475
|
-
|
|
7476
|
-
|
|
7477
|
-
|
|
8209
|
+
const isExplicitChart = /\b(chart|graph|plot|sales over time|monthly growth|revenue trend|pie chart|bar chart|line chart|histogram)\b/i.test(q);
|
|
8210
|
+
if (isExplicitChart) {
|
|
8211
|
+
return null;
|
|
8212
|
+
}
|
|
8213
|
+
const isFinancialReport = /\b(price per share|share price|debt to equity|ebitda|fcfe|fcff|wacc|beta|balance sheet|cash flow statement)\b/i.test(q);
|
|
8214
|
+
if (isFinancialReport) {
|
|
8215
|
+
return null;
|
|
8216
|
+
}
|
|
8217
|
+
const signals = IntentClassifier.extractDataSignals(context);
|
|
8218
|
+
const hasDiscoveryAction = intent === "product_search" || intent === "recommendation" || /\b(recommend|recommendation|suggest|show|find|list|browse|search|explore|get|buy|shop|shopping|purchase|options|selection|picks|available|what do you have|what is in stock|looking for|under \$|cheap|affordable|best|top|products?|items?|catalog|inventory)\b/i.test(q);
|
|
8219
|
+
const isAnalyticalRanking = /\b(top performing|top selling|best performing|performing|sales revenue|revenue ranking)\b/i.test(q);
|
|
8220
|
+
const isGenericDocumentQuery = /\b(documents? in system|all documents|system documents|overall documents)\b/i.test(q);
|
|
8221
|
+
if (!isAnalyticalRanking && !isGenericDocumentQuery && (signals.isProductLike && hasDiscoveryAction || intent === "product_search" && !isAnalyticalRanking)) {
|
|
7478
8222
|
return {
|
|
7479
8223
|
renderType: "carousel",
|
|
7480
|
-
confidence: 0.
|
|
7481
|
-
reason: "Rule 3:
|
|
8224
|
+
confidence: 0.92,
|
|
8225
|
+
reason: "Rule 3: Dynamically identified product catalog entities and discovery intent."
|
|
7482
8226
|
};
|
|
7483
8227
|
}
|
|
7484
8228
|
return null;
|
|
@@ -7949,11 +8693,11 @@ var UITransformer = class _UITransformer {
|
|
|
7949
8693
|
if (["tabular", "table"].includes(resolvedIntent.visualizationHint) || resolvedIntent.wantsExplicitTable) {
|
|
7950
8694
|
return this.hasMultipleFields(filteredData) ? this.transformToTable(filteredData, userQuery) : this.transformToText(filteredData);
|
|
7951
8695
|
}
|
|
7952
|
-
if (
|
|
7953
|
-
|
|
7954
|
-
|
|
8696
|
+
if (hasProducts || this.isProductQuery(userQuery) || resolvedIntent.visualizationHint === "product_browse") {
|
|
8697
|
+
const carousel = this.transformToProductCarousel(filteredData, config, trainedSchema);
|
|
8698
|
+
if (Array.isArray(carousel.data) && carousel.data.length > 0) {
|
|
8699
|
+
return carousel;
|
|
7955
8700
|
}
|
|
7956
|
-
return this.transformToProductCarousel(filteredData, config, trainedSchema);
|
|
7957
8701
|
}
|
|
7958
8702
|
const automatic = this.chooseAutomaticVisualization(filteredData, profile, userQuery);
|
|
7959
8703
|
if (automatic) return automatic;
|
|
@@ -8590,15 +9334,17 @@ ${schemaProfileText}` : ""}`;
|
|
|
8590
9334
|
return _UITransformer._productQueryTest(query);
|
|
8591
9335
|
}
|
|
8592
9336
|
static _productQueryTest(query) {
|
|
8593
|
-
const q = query.toLowerCase();
|
|
9337
|
+
const q = query.toLowerCase().trim();
|
|
8594
9338
|
const financialTerms = /\b(price per share|share price|total cost of the deal|cost of the deal|debt|equity|ratio|revenue|ebit|ebitda|tax|depreciation|interest rate|cashflow|cash flow|fcfe|fcff|wacc|beta|working capital|cogs|capital spending|valuation|leverage)\b/;
|
|
8595
9339
|
if (financialTerms.test(q)) {
|
|
8596
9340
|
return false;
|
|
8597
9341
|
}
|
|
8598
|
-
const productTerms = /\b(product|products|item|items|sku|catalog|catalogue|brand|model|stock|inventory|in stock|out of stock|buy|shop)\b/.test(q);
|
|
8599
|
-
const productAction = /\b(recommend|suggest|describe|description|detail|details|about|show|find|search|browse|list|explore|exploring)\b/.test(q);
|
|
8600
9342
|
const nonProductEntityTerms = /\b(organization|organisations?|organizations?|company|companies|employee|employees|staff|headcount|workforce)\b/.test(q);
|
|
8601
|
-
|
|
9343
|
+
if (nonProductEntityTerms) {
|
|
9344
|
+
return false;
|
|
9345
|
+
}
|
|
9346
|
+
const genericDiscoveryPatterns = /\b(product|products|item|items|sku|catalog|catalogue|brand|model|stock|inventory|in stock|out of stock|buy|shop|shopping|purchase|order|deals|recommend|recommendation|suggestions|show available|what do you have|available|looking for|options for|under \$|cheap|affordable|best|top|find me|browse)\b/i;
|
|
9347
|
+
return genericDiscoveryPatterns.test(q);
|
|
8602
9348
|
}
|
|
8603
9349
|
static isProductData(item) {
|
|
8604
9350
|
if (!item) return false;
|
|
@@ -8615,10 +9361,17 @@ ${schemaProfileText}` : ""}`;
|
|
|
8615
9361
|
if (financialTerms.test(content)) {
|
|
8616
9362
|
return false;
|
|
8617
9363
|
}
|
|
8618
|
-
const hasSku = Boolean(resolveMetadataValue(metadata, "sku") || resolveMetadataValue(metadata, "product_id"));
|
|
8619
|
-
const hasProductIdentity = Boolean(resolveMetadataValue(metadata, "product_name") || resolveMetadataValue(metadata, "brand") && resolveMetadataValue(metadata, "name"));
|
|
8620
|
-
const hasPrice = Boolean(resolveMetadataValue(metadata, "retail_price") || resolveMetadataValue(metadata, "unit_price") || resolveMetadataValue(metadata, "msrp"));
|
|
8621
|
-
|
|
9364
|
+
const hasSku = Boolean(resolveMetadataValue(metadata, "sku") || resolveMetadataValue(metadata, "product_id") || resolveMetadataValue(metadata, "item_id"));
|
|
9365
|
+
const hasProductIdentity = Boolean(resolveMetadataValue(metadata, "product_name") || resolveMetadataValue(metadata, "item_name") || resolveMetadataValue(metadata, "brand") && resolveMetadataValue(metadata, "name"));
|
|
9366
|
+
const hasPrice = Boolean(resolveMetadataValue(metadata, "price") || resolveMetadataValue(metadata, "retail_price") || resolveMetadataValue(metadata, "unit_price") || resolveMetadataValue(metadata, "msrp"));
|
|
9367
|
+
if (hasSku || hasProductIdentity && hasPrice || hasProductIdentity) {
|
|
9368
|
+
return true;
|
|
9369
|
+
}
|
|
9370
|
+
const hasContentName = /(?:^|\n)\s*(?:product\s*name|item\s*name|product|item|title|name)\s*[:\-–]\s*[^\n,]+/i.test(item.content || "");
|
|
9371
|
+
const hasContentPrice = /(?:^|\n)\s*(?:price|retail\s*price|unit\s*price|cost|msrp)\s*[:\-–]?\s*\$?\s*[\d,.]+/i.test(item.content || "");
|
|
9372
|
+
const hasContentCategory = /(?:^|\n)\s*(?:category|department|collection)\s*[:\-–]\s*[^\n,]+/i.test(item.content || "");
|
|
9373
|
+
const hasContentBrand = /(?:^|\n)\s*(?:brand|manufacturer|vendor)\s*[:\-–]\s*[^\n,]+/i.test(item.content || "");
|
|
9374
|
+
return hasContentName && (hasContentPrice || hasContentCategory || hasContentBrand);
|
|
8622
9375
|
}
|
|
8623
9376
|
static isTimeSeriesData(item) {
|
|
8624
9377
|
const metadata = item.metadata || {};
|
|
@@ -9108,64 +9861,46 @@ ${schemaProfileText}` : ""}`;
|
|
|
9108
9861
|
return resolveMetadataValue(meta, uiKey);
|
|
9109
9862
|
}
|
|
9110
9863
|
static extractProductInfo(item, config, trainedSchema) {
|
|
9111
|
-
var _a2, _b;
|
|
9864
|
+
var _a2, _b, _c;
|
|
9112
9865
|
if (!item) return null;
|
|
9113
9866
|
const meta = item.metadata || {};
|
|
9114
9867
|
const content = item.content || "";
|
|
9115
|
-
|
|
9116
|
-
|
|
9117
|
-
|
|
9868
|
+
let name = this.getDynamicVal(meta, "name", config, trainedSchema);
|
|
9869
|
+
let price = this.getDynamicVal(meta, "price", config, trainedSchema);
|
|
9870
|
+
let brand = this.getDynamicVal(meta, "brand", config, trainedSchema);
|
|
9118
9871
|
const description = this.cleanProductDescription(
|
|
9119
9872
|
(_a2 = this.extractProductDescriptionFromContent(content)) != null ? _a2 : this.getProductDescriptionValue(meta, config, trainedSchema)
|
|
9120
9873
|
);
|
|
9121
|
-
if (name
|
|
9122
|
-
|
|
9123
|
-
if (
|
|
9124
|
-
|
|
9125
|
-
finalName = nameMatch ? nameMatch[1].trim() : (_b = content.split("\n")[0]) == null ? void 0 : _b.substring(0, 60);
|
|
9126
|
-
}
|
|
9127
|
-
if (!finalName) {
|
|
9128
|
-
console.warn(
|
|
9129
|
-
`[UITransformer] Unable to read product name from metadata or content for item ${item.id}. Defaulting to 'Unnamed Product'.`,
|
|
9130
|
-
{ metadataKeys: Object.keys(meta), contentLength: content.length }
|
|
9131
|
-
);
|
|
9874
|
+
if (!name && content) {
|
|
9875
|
+
const nameMatch = content.match(/(?:^|\n)\s*(?:product\s*name|item\s*name|product|item|title|name)\s*[:\-–]\s*([^\n,]+)/i) || content.match(/(?:Product|Item|Name):\s*([^\n]+)/i);
|
|
9876
|
+
if (nameMatch && nameMatch[1]) {
|
|
9877
|
+
name = nameMatch[1].trim();
|
|
9132
9878
|
}
|
|
9133
|
-
|
|
9134
|
-
|
|
9135
|
-
|
|
9136
|
-
|
|
9137
|
-
|
|
9138
|
-
if (finalPrice === void 0) {
|
|
9139
|
-
console.warn(
|
|
9140
|
-
`[UITransformer] Unable to read product price from metadata or content for item ${item.id}.`,
|
|
9141
|
-
{ metadataKeys: Object.keys(meta), contentSnippet: content.substring(0, 100) }
|
|
9142
|
-
);
|
|
9143
|
-
}
|
|
9144
|
-
const imageValue = this.getDynamicVal(meta, "image", config, trainedSchema);
|
|
9145
|
-
if (!imageValue) {
|
|
9146
|
-
console.debug(
|
|
9147
|
-
`[UITransformer] Unable to read product image URL from metadata for item ${item.id}.`
|
|
9148
|
-
);
|
|
9879
|
+
}
|
|
9880
|
+
if (!brand && content) {
|
|
9881
|
+
const brandMatch = content.match(/(?:^|\n)\s*(?:brand|manufacturer|vendor)\s*[:\-–]\s*([^\n,]+)/i);
|
|
9882
|
+
if (brandMatch && brandMatch[1]) {
|
|
9883
|
+
brand = brandMatch[1].trim();
|
|
9149
9884
|
}
|
|
9150
|
-
|
|
9151
|
-
|
|
9152
|
-
|
|
9153
|
-
|
|
9885
|
+
}
|
|
9886
|
+
if (price === void 0 && content) {
|
|
9887
|
+
const priceMatch = content.match(/(?:^|\n)\s*(?:price|retail\s*price|unit\s*price|cost|msrp)\s*[:\-–]?\s*\$?\s*([\d,.]+)/i) || content.match(/(?:Price|Cost):\s*\$?([\d,.]+)/i) || content.match(/\$\s*([\d,.]+)/);
|
|
9888
|
+
if (priceMatch && priceMatch[1]) {
|
|
9889
|
+
price = priceMatch[1].replace(/,/g, "");
|
|
9154
9890
|
}
|
|
9891
|
+
}
|
|
9892
|
+
if (name || this.isProductData(item)) {
|
|
9893
|
+
const finalName = name ? String(name) : ((_b = content.split("\n")[0]) == null ? void 0 : _b.substring(0, 60)) || "Unnamed Product";
|
|
9894
|
+
const imageValue = this.getDynamicVal(meta, "image", config, trainedSchema) || ((_c = content.match(/(?:^|\n)\s*(?:image|image\s*url|photo|thumbnail)\s*[:\-–]\s*(https?:\/\/[^\s\n,]+)/i)) == null ? void 0 : _c[1]);
|
|
9155
9895
|
return {
|
|
9156
9896
|
id: item.id,
|
|
9157
|
-
name: finalName
|
|
9158
|
-
price:
|
|
9897
|
+
name: finalName,
|
|
9898
|
+
price: typeof price === "number" || typeof price === "string" ? price : void 0,
|
|
9159
9899
|
image: typeof imageValue === "string" ? imageValue : void 0,
|
|
9160
9900
|
brand: brand ? String(brand) : void 0,
|
|
9161
9901
|
description,
|
|
9162
9902
|
inStock: this.determineStockStatus(item)
|
|
9163
9903
|
};
|
|
9164
|
-
} else {
|
|
9165
|
-
console.warn(
|
|
9166
|
-
`[UITransformer] Item ${item.id} skipped: unable to read name or identify as product data from metadata/content.`,
|
|
9167
|
-
{ metadataKeys: Object.keys(meta), contentSnippet: content.substring(0, 100) }
|
|
9168
|
-
);
|
|
9169
9904
|
}
|
|
9170
9905
|
return null;
|
|
9171
9906
|
}
|
|
@@ -9540,6 +10275,17 @@ var Pipeline = class {
|
|
|
9540
10275
|
- If answer cannot be found in the sources, say that you cannot answer the question and do not hallucinate.
|
|
9541
10276
|
- Do not use information from previous turns to answer the question.
|
|
9542
10277
|
- You CAN use numbers for years and counts (e.g., 2006, 5800). But NEVER put a dollar sign ($) before non-monetary numbers like Stock quantities, EAN numbers, or years. Only use dollar signs for actual monetary prices.
|
|
10278
|
+
- Be concise, direct, and focused. Deliver clean, high-quality answers within token limits without unnecessary filler.
|
|
10279
|
+
|
|
10280
|
+
### CATEGORY & DOMAIN SYNONYM MATCHING:
|
|
10281
|
+
- Intelligently understand common category synonyms and related terms from the context:
|
|
10282
|
+
- If the user asks about "makeup", "cosmetics", "skincare", "toiletries", "personal care", or "beauty products", map these to the "Beauty" category in the catalog and present the matching products.
|
|
10283
|
+
- If the user asks about "clothes", "apparel", "wear", "shirts", "garments", or "fashion", map these to the "Clothing" category in the catalog.
|
|
10284
|
+
- If the user asks about "electronics", "gadgets", "tech", "devices", or "appliances", map these to the "Electronics" category.
|
|
10285
|
+
- If the user asks about "food", "snacks", "groceries", "beverages", or "drinks", map these to the "Grocery" category.
|
|
10286
|
+
- If the user asks about "office items", "stationery", "work supplies", or "desk equipment", map these to the "Office Supplies" category.
|
|
10287
|
+
- If the user asks about "toys", "games", "play items", or "hobbies", map these to the "Toys & Games" category.
|
|
10288
|
+
- Do NOT reject a query or say no items exist simply because the user used a natural synonym (e.g., "makeup") instead of the exact catalog column label (e.g., "Beauty"). Identify the relevant items from the context and present them.
|
|
9543
10289
|
|
|
9544
10290
|
### PRODUCT DISPLAY:
|
|
9545
10291
|
- Recommended Products should only be displayed when the user explicitly asks for products.
|
|
@@ -9706,16 +10452,26 @@ ${m.content}`).join("\n\n---\n\n");
|
|
|
9706
10452
|
*/
|
|
9707
10453
|
async processEmbeddings(chunks) {
|
|
9708
10454
|
if (chunks.length === 0) return [];
|
|
9709
|
-
const texts = chunks.map((c) => c.content);
|
|
10455
|
+
const texts = chunks.map((c) => PIIMasker.mask(c.content));
|
|
9710
10456
|
if (typeof this.embeddingProvider.batchEmbed === "function") {
|
|
9711
10457
|
try {
|
|
9712
|
-
const batchSize =
|
|
9713
|
-
const
|
|
10458
|
+
const batchSize = 100;
|
|
10459
|
+
const slices = [];
|
|
9714
10460
|
for (let i = 0; i < texts.length; i += batchSize) {
|
|
9715
|
-
|
|
9716
|
-
const batchVectors = await this.embeddingProvider.batchEmbed(slice, { taskType: "document" });
|
|
9717
|
-
allVectors.push(...batchVectors);
|
|
10461
|
+
slices.push(texts.slice(i, i + batchSize));
|
|
9718
10462
|
}
|
|
10463
|
+
const batchEmbedFn = this.embeddingProvider.batchEmbed.bind(this.embeddingProvider);
|
|
10464
|
+
const concurrentBatches = await BatchProcessor.processConcurrent(
|
|
10465
|
+
slices,
|
|
10466
|
+
(slice) => batchEmbedFn(slice, { taskType: "document" }),
|
|
10467
|
+
5,
|
|
10468
|
+
{
|
|
10469
|
+
maxRetries: 3,
|
|
10470
|
+
initialDelayMs: 100,
|
|
10471
|
+
throwOnPartialFailure: false
|
|
10472
|
+
}
|
|
10473
|
+
);
|
|
10474
|
+
const allVectors = concurrentBatches.results.flat();
|
|
9719
10475
|
if (allVectors.length === chunks.length) {
|
|
9720
10476
|
return allVectors;
|
|
9721
10477
|
}
|
|
@@ -9728,7 +10484,7 @@ ${m.content}`).join("\n\n---\n\n");
|
|
|
9728
10484
|
const concurrentResult = await BatchProcessor.processConcurrent(
|
|
9729
10485
|
texts,
|
|
9730
10486
|
(text) => this.embeddingProvider.embed(text, { taskType: "document" }),
|
|
9731
|
-
|
|
10487
|
+
20,
|
|
9732
10488
|
{
|
|
9733
10489
|
maxRetries: 3,
|
|
9734
10490
|
initialDelayMs: 100,
|
|
@@ -9746,7 +10502,7 @@ ${m.content}`).join("\n\n---\n\n");
|
|
|
9746
10502
|
/** Step 3: Upsert chunks to vector database with retry logic. */
|
|
9747
10503
|
async processUpserts(upsertDocs, namespace) {
|
|
9748
10504
|
const upsertBatchOptions = {
|
|
9749
|
-
batchSize:
|
|
10505
|
+
batchSize: 250,
|
|
9750
10506
|
maxRetries: 3,
|
|
9751
10507
|
initialDelayMs: 100
|
|
9752
10508
|
};
|
|
@@ -9904,7 +10660,7 @@ ${m.content}`).join("\n\n---\n\n");
|
|
|
9904
10660
|
*/
|
|
9905
10661
|
askStreamInternal(_0) {
|
|
9906
10662
|
return __asyncGenerator(this, arguments, function* (question, history = [], namespace) {
|
|
9907
|
-
var _a2, _b, _c, _d, _e, _f, _g2, _h, _i, _j, _k, _l, _m, _n, _o, _p, _q, _r, _s, _t, _u, _v, _w, _x
|
|
10663
|
+
var _a2, _b, _c, _d, _e, _f, _g2, _h, _i, _j, _k, _l, _m, _n, _o, _p, _q, _r, _s, _t, _u, _v, _w, _x;
|
|
9908
10664
|
yield new __await(this.initialize());
|
|
9909
10665
|
const ns = formatNamespace(namespace != null ? namespace : this.config.projectId);
|
|
9910
10666
|
const topK = (_b = (_a2 = this.config.rag) == null ? void 0 : _a2.topK) != null ? _b : 5;
|
|
@@ -9912,10 +10668,12 @@ ${m.content}`).join("\n\n---\n\n");
|
|
|
9912
10668
|
const requestId = `req_${Date.now()}_${Math.random().toString(36).slice(2, 8)}`;
|
|
9913
10669
|
const requestStart = performance.now();
|
|
9914
10670
|
try {
|
|
9915
|
-
|
|
10671
|
+
const guardCheck = PromptGuard.evaluate(question);
|
|
10672
|
+
let effectiveQuestion = guardCheck.sanitizedText;
|
|
10673
|
+
effectiveQuestion = PIIMasker.mask(effectiveQuestion);
|
|
9916
10674
|
let promptOptResult;
|
|
9917
10675
|
if (((_e = this.config.rag) == null ? void 0 : _e.optimizePrompt) !== false) {
|
|
9918
|
-
promptOptResult = PromptOptimizer.optimize(
|
|
10676
|
+
promptOptResult = PromptOptimizer.optimize(effectiveQuestion, history);
|
|
9919
10677
|
effectiveQuestion = promptOptResult.optimizedPrompt;
|
|
9920
10678
|
}
|
|
9921
10679
|
let searchQuery = effectiveQuestion;
|
|
@@ -9973,7 +10731,17 @@ ${m.content}`).join("\n\n---\n\n");
|
|
|
9973
10731
|
}
|
|
9974
10732
|
const rerankMs = performance.now() - rerankStart;
|
|
9975
10733
|
const totalCount = fullSources.length;
|
|
9976
|
-
const
|
|
10734
|
+
const rawPromptSources = fullSources.slice(0, 4);
|
|
10735
|
+
let accumulatedChars = 0;
|
|
10736
|
+
const promptSources = [];
|
|
10737
|
+
for (const s of rawPromptSources) {
|
|
10738
|
+
const textLen = ((s == null ? void 0 : s.content) || "").length;
|
|
10739
|
+
if (accumulatedChars + textLen > 3500 && promptSources.length >= 2) {
|
|
10740
|
+
break;
|
|
10741
|
+
}
|
|
10742
|
+
promptSources.push(s);
|
|
10743
|
+
accumulatedChars += textLen;
|
|
10744
|
+
}
|
|
9977
10745
|
let context = promptSources.length ? `[SEARCH SUMMARY: Found ${totalCount} matching record(s) out of ${rawSources.length} total resources searched in database. Context includes top ${promptSources.length} relevant records]
|
|
9978
10746
|
|
|
9979
10747
|
` + promptSources.map((m, i) => {
|
|
@@ -10051,7 +10819,7 @@ ${context}`;
|
|
|
10051
10819
|
const isNativeThinking = isClaude37 && (((_q = this.config.llm.options) == null ? void 0 : _q.thinking) === true || ((_r = this.config.llm.options) == null ? void 0 : _r.thinking) === "enabled" || ((_s = this.config.llm.options) == null ? void 0 : _s.thinking) !== false);
|
|
10052
10820
|
const modelNameLower = this.config.llm.model.toLowerCase();
|
|
10053
10821
|
const isReasoningModel = modelNameLower.includes("-r1") || modelNameLower.includes("deepseek-r1") || modelNameLower.includes("reasoning") || modelNameLower.includes("thinking");
|
|
10054
|
-
const isSimulatedThinking = !isNativeThinking && (((_t = this.config.llm.options) == null ? void 0 : _t.thinking) === true || ((_u = this.config.llm.options) == null ? void 0 : _u.thinking) === "enabled"
|
|
10822
|
+
const isSimulatedThinking = !isNativeThinking && (((_t = this.config.llm.options) == null ? void 0 : _t.thinking) === true || ((_u = this.config.llm.options) == null ? void 0 : _u.thinking) === "enabled");
|
|
10055
10823
|
let finalRestrictionSuffix = restrictionSuffix;
|
|
10056
10824
|
if (isSimulatedThinking) {
|
|
10057
10825
|
finalRestrictionSuffix += "\n\n(IMPORTANT: You must think step-by-step before answering. Write your thinking process inside a `<think>...</think>` block. Write the `<think>` block first, then follow it with your final response. Keep the thinking process thorough and detailed. Example: `<think>Thinking details here...</think>Final answer text here.`)";
|
|
@@ -10062,7 +10830,7 @@ ${context}`;
|
|
|
10062
10830
|
}));
|
|
10063
10831
|
const userQuestion = { role: "user", content: effectiveQuestion + finalRestrictionSuffix };
|
|
10064
10832
|
const messages = [...hardenedHistory, userQuestion];
|
|
10065
|
-
const systemPrompt = (
|
|
10833
|
+
const systemPrompt = (_v = this.config.llm.systemPrompt) != null ? _v : "";
|
|
10066
10834
|
const userPrompt = messages.map((m) => `${m.role}: ${m.content}`).join("\n");
|
|
10067
10835
|
let fullReply = "";
|
|
10068
10836
|
let thinkingText = "";
|
|
@@ -10084,57 +10852,54 @@ ${context}`;
|
|
|
10084
10852
|
yield { type: "thinking", text: thinkingDelta };
|
|
10085
10853
|
continue;
|
|
10086
10854
|
}
|
|
10087
|
-
|
|
10088
|
-
|
|
10089
|
-
|
|
10090
|
-
|
|
10091
|
-
|
|
10092
|
-
|
|
10093
|
-
|
|
10094
|
-
|
|
10095
|
-
|
|
10096
|
-
yield before;
|
|
10097
|
-
}
|
|
10098
|
-
inThinking = true;
|
|
10099
|
-
thinkingStartMs = performance.now();
|
|
10100
|
-
textBuffer = textBuffer.slice(thinkIndex + "<think>".length);
|
|
10101
|
-
} else {
|
|
10102
|
-
const maxSafeLength = textBuffer.length - "<think>".length;
|
|
10103
|
-
if (maxSafeLength > 0) {
|
|
10104
|
-
const safeText = textBuffer.slice(0, maxSafeLength);
|
|
10105
|
-
fullReply += safeText;
|
|
10106
|
-
yield safeText;
|
|
10107
|
-
textBuffer = textBuffer.slice(maxSafeLength);
|
|
10108
|
-
}
|
|
10109
|
-
break;
|
|
10855
|
+
textBuffer += chunk;
|
|
10856
|
+
while (textBuffer.length > 0) {
|
|
10857
|
+
if (!inThinking) {
|
|
10858
|
+
const thinkIndex = textBuffer.indexOf("<think>");
|
|
10859
|
+
if (thinkIndex !== -1) {
|
|
10860
|
+
const before = textBuffer.slice(0, thinkIndex);
|
|
10861
|
+
if (before) {
|
|
10862
|
+
fullReply += before;
|
|
10863
|
+
yield before;
|
|
10110
10864
|
}
|
|
10865
|
+
inThinking = true;
|
|
10866
|
+
thinkingStartMs = performance.now();
|
|
10867
|
+
textBuffer = textBuffer.slice(thinkIndex + "<think>".length);
|
|
10111
10868
|
} else {
|
|
10112
|
-
const
|
|
10113
|
-
|
|
10114
|
-
|
|
10115
|
-
|
|
10116
|
-
|
|
10117
|
-
|
|
10118
|
-
|
|
10119
|
-
|
|
10120
|
-
|
|
10121
|
-
|
|
10122
|
-
|
|
10123
|
-
|
|
10124
|
-
|
|
10125
|
-
|
|
10126
|
-
|
|
10127
|
-
|
|
10128
|
-
|
|
10129
|
-
|
|
10130
|
-
|
|
10131
|
-
|
|
10869
|
+
const partialMatch = textBuffer.match(/<t?(?:h(?:i(?:n(?:k)?)?)?)?$/);
|
|
10870
|
+
const safeLen = partialMatch ? textBuffer.length - partialMatch[0].length : textBuffer.length;
|
|
10871
|
+
if (safeLen > 0) {
|
|
10872
|
+
const safeText = textBuffer.slice(0, safeLen);
|
|
10873
|
+
fullReply += safeText;
|
|
10874
|
+
yield safeText;
|
|
10875
|
+
textBuffer = textBuffer.slice(safeLen);
|
|
10876
|
+
}
|
|
10877
|
+
break;
|
|
10878
|
+
}
|
|
10879
|
+
} else {
|
|
10880
|
+
const endThinkIndex = textBuffer.indexOf("</think>");
|
|
10881
|
+
if (endThinkIndex !== -1) {
|
|
10882
|
+
const thinkingDelta = textBuffer.slice(0, endThinkIndex);
|
|
10883
|
+
if (thinkingDelta) {
|
|
10884
|
+
thinkingText += thinkingDelta;
|
|
10885
|
+
thinkingDurationMs = performance.now() - thinkingStartMs;
|
|
10886
|
+
yield { type: "thinking", text: thinkingDelta };
|
|
10887
|
+
}
|
|
10888
|
+
inThinking = false;
|
|
10889
|
+
textBuffer = textBuffer.slice(endThinkIndex + "</think>".length);
|
|
10890
|
+
} else {
|
|
10891
|
+
const partialMatch = textBuffer.match(/<\/?t?(?:h(?:i(?:n(?:k)?)?)?)?$/);
|
|
10892
|
+
const safeLen = partialMatch ? textBuffer.length - partialMatch[0].length : textBuffer.length;
|
|
10893
|
+
if (safeLen > 0) {
|
|
10894
|
+
const thinkingDelta = textBuffer.slice(0, safeLen);
|
|
10895
|
+
thinkingText += thinkingDelta;
|
|
10896
|
+
thinkingDurationMs = performance.now() - thinkingStartMs;
|
|
10897
|
+
yield { type: "thinking", text: thinkingDelta };
|
|
10898
|
+
textBuffer = textBuffer.slice(safeLen);
|
|
10132
10899
|
}
|
|
10900
|
+
break;
|
|
10133
10901
|
}
|
|
10134
10902
|
}
|
|
10135
|
-
} else {
|
|
10136
|
-
fullReply += chunk;
|
|
10137
|
-
yield chunk;
|
|
10138
10903
|
}
|
|
10139
10904
|
}
|
|
10140
10905
|
} catch (temp) {
|
|
@@ -10147,7 +10912,7 @@ ${context}`;
|
|
|
10147
10912
|
throw error[0];
|
|
10148
10913
|
}
|
|
10149
10914
|
}
|
|
10150
|
-
if (
|
|
10915
|
+
if (textBuffer.length > 0) {
|
|
10151
10916
|
if (inThinking) {
|
|
10152
10917
|
thinkingText += textBuffer;
|
|
10153
10918
|
yield { type: "thinking", text: textBuffer };
|
|
@@ -10158,22 +10923,21 @@ ${context}`;
|
|
|
10158
10923
|
}
|
|
10159
10924
|
} else {
|
|
10160
10925
|
const reply = yield new __await(this.llmProvider.chat(messages, context));
|
|
10161
|
-
|
|
10162
|
-
|
|
10163
|
-
|
|
10164
|
-
|
|
10165
|
-
|
|
10166
|
-
|
|
10167
|
-
|
|
10168
|
-
|
|
10169
|
-
|
|
10170
|
-
}
|
|
10926
|
+
const thinkStart = reply.indexOf("<think>");
|
|
10927
|
+
const thinkEnd = reply.indexOf("</think>");
|
|
10928
|
+
if (thinkStart !== -1 && thinkEnd !== -1 && thinkEnd > thinkStart) {
|
|
10929
|
+
thinkingText = reply.slice(thinkStart + "<think>".length, thinkEnd);
|
|
10930
|
+
fullReply = reply.slice(0, thinkStart) + reply.slice(thinkEnd + "</think>".length);
|
|
10931
|
+
thinkingDurationMs = 100;
|
|
10932
|
+
} else if (thinkStart !== -1) {
|
|
10933
|
+
thinkingText = reply.slice(thinkStart + "<think>".length);
|
|
10934
|
+
fullReply = reply.slice(0, thinkStart);
|
|
10171
10935
|
} else {
|
|
10172
10936
|
fullReply = reply;
|
|
10173
10937
|
}
|
|
10174
10938
|
yield fullReply;
|
|
10175
10939
|
}
|
|
10176
|
-
const runHallucination = ((
|
|
10940
|
+
const runHallucination = ((_w = this.config.llm.options) == null ? void 0 : _w.hallucinationScoring) === true;
|
|
10177
10941
|
if (runHallucination) {
|
|
10178
10942
|
hallucinationScoringPromise = scoreHallucination(this.llmProvider, fullReply, context).catch(() => void 0);
|
|
10179
10943
|
}
|
|
@@ -10195,89 +10959,45 @@ ${context}`;
|
|
|
10195
10959
|
totalTokens: promptTokens + completionTokens,
|
|
10196
10960
|
estimatedCostUsd: estimateCostUsd(promptTokens, completionTokens, this.config.llm.model)
|
|
10197
10961
|
};
|
|
10198
|
-
const awaitHallucination = ((
|
|
10962
|
+
const awaitHallucination = ((_x = this.config.llm.options) == null ? void 0 : _x.awaitHallucination) === true;
|
|
10199
10963
|
const [uiTransformation, hallucinationResult] = yield new __await(Promise.all([
|
|
10200
10964
|
uiTransformationPromise,
|
|
10201
10965
|
awaitHallucination ? hallucinationScoringPromise : Promise.resolve(void 0)
|
|
10202
10966
|
]));
|
|
10203
|
-
|
|
10204
|
-
|
|
10205
|
-
|
|
10206
|
-
|
|
10207
|
-
|
|
10208
|
-
|
|
10209
|
-
|
|
10210
|
-
|
|
10211
|
-
|
|
10212
|
-
|
|
10213
|
-
|
|
10214
|
-
|
|
10215
|
-
|
|
10216
|
-
|
|
10217
|
-
|
|
10218
|
-
|
|
10219
|
-
|
|
10220
|
-
|
|
10221
|
-
|
|
10222
|
-
|
|
10223
|
-
|
|
10224
|
-
|
|
10225
|
-
|
|
10967
|
+
RAGEvaluator.evaluate(effectiveQuestion, context, fullReply).catch(() => {
|
|
10968
|
+
});
|
|
10969
|
+
const buildTrace = (hScore) => {
|
|
10970
|
+
var _a3, _b2;
|
|
10971
|
+
return __spreadValues({
|
|
10972
|
+
requestId,
|
|
10973
|
+
query: question,
|
|
10974
|
+
rewrittenQuery: rewrittenQuery || ((promptOptResult == null ? void 0 : promptOptResult.wasModified) ? promptOptResult.optimizedPrompt : void 0),
|
|
10975
|
+
// systemPrompt / userPrompt intentionally excluded:
|
|
10976
|
+
// they may contain business logic or post-masking user PII and were
|
|
10977
|
+
// being emitted to the client browser via SSE (security risk).
|
|
10978
|
+
chunks: (sources || []).filter(Boolean).map((s) => {
|
|
10979
|
+
var _a4, _b3;
|
|
10980
|
+
return {
|
|
10981
|
+
id: s.id,
|
|
10982
|
+
score: s.score,
|
|
10983
|
+
content: (_a4 = s == null ? void 0 : s.content) != null ? _a4 : "",
|
|
10984
|
+
metadata: (_b3 = s == null ? void 0 : s.metadata) != null ? _b3 : {},
|
|
10985
|
+
namespace: ns
|
|
10986
|
+
};
|
|
10987
|
+
}),
|
|
10988
|
+
latency,
|
|
10989
|
+
tokens,
|
|
10990
|
+
// Always populate model/provider on the trace (fixes gap where telemetry
|
|
10991
|
+
// was reading finalTrace?.model with a fallback — now it's always set).
|
|
10992
|
+
model: (_a3 = this.config.llm) == null ? void 0 : _a3.model,
|
|
10993
|
+
provider: (_b2 = this.config.llm) == null ? void 0 : _b2.provider,
|
|
10994
|
+
timestamp: (/* @__PURE__ */ new Date()).toISOString()
|
|
10995
|
+
}, hScore ? {
|
|
10996
|
+
hallucinationScore: hScore.score,
|
|
10997
|
+
hallucinationReason: hScore.reason
|
|
10998
|
+
} : {});
|
|
10999
|
+
};
|
|
10226
11000
|
const trace = buildTrace(hallucinationResult);
|
|
10227
|
-
const isTelemetryActive = (_A = (_z = this.config.telemetry) == null ? void 0 : _z.enabled) != null ? _A : Boolean(this.config.licenseKey);
|
|
10228
|
-
if (isTelemetryActive) {
|
|
10229
|
-
const defaultUrl = "https://www.retrivora.com/api/telemetry";
|
|
10230
|
-
const telemetryUrl = ((_B = this.config.telemetry) == null ? void 0 : _B.url) || process.env.TELEMETRY_URL || process.env.NEXT_PUBLIC_TELEMETRY_URL || defaultUrl;
|
|
10231
|
-
const absoluteUrl = telemetryUrl.startsWith("http") ? telemetryUrl : "https://www.retrivora.com" + (telemetryUrl.startsWith("/") ? telemetryUrl : "/" + telemetryUrl);
|
|
10232
|
-
(async () => {
|
|
10233
|
-
var _a3, _b2, _c2, _d2, _e2;
|
|
10234
|
-
try {
|
|
10235
|
-
let finalTrace = trace;
|
|
10236
|
-
if (!awaitHallucination && runHallucination) {
|
|
10237
|
-
const backgroundScoreResult = await hallucinationScoringPromise;
|
|
10238
|
-
if (backgroundScoreResult) {
|
|
10239
|
-
finalTrace = buildTrace(backgroundScoreResult);
|
|
10240
|
-
}
|
|
10241
|
-
}
|
|
10242
|
-
const modelName = (finalTrace == null ? void 0 : finalTrace.model) || ((_a3 = this.config.llm) == null ? void 0 : _a3.model) || "llama-3.1-8b-instant";
|
|
10243
|
-
const providerName = (finalTrace == null ? void 0 : finalTrace.provider) || ((_b2 = this.config.llm) == null ? void 0 : _b2.provider) || "groq";
|
|
10244
|
-
const tokenCount = Number(((_c2 = finalTrace == null ? void 0 : finalTrace.tokens) == null ? void 0 : _c2.totalTokens) || 0);
|
|
10245
|
-
const costEst = Number(((_d2 = finalTrace == null ? void 0 : finalTrace.tokens) == null ? void 0 : _d2.estimatedCostUsd) || 0);
|
|
10246
|
-
const latencyDuration = Number(((_e2 = finalTrace == null ? void 0 : finalTrace.latency) == null ? void 0 : _e2.totalMs) || 0);
|
|
10247
|
-
if (typeof window === "undefined") {
|
|
10248
|
-
Logger.info(`[Retrivora Pipeline Telemetry] \u{1F4CA} Dispatching RAG telemetry -> Project: "${ns}", Tokens: ${tokenCount}, Latency: ${latencyDuration}ms`);
|
|
10249
|
-
}
|
|
10250
|
-
await fetch(absoluteUrl, {
|
|
10251
|
-
method: "POST",
|
|
10252
|
-
headers: {
|
|
10253
|
-
"Content-Type": "application/json"
|
|
10254
|
-
},
|
|
10255
|
-
body: JSON.stringify({
|
|
10256
|
-
trace: finalTrace,
|
|
10257
|
-
licenseKey: this.config.licenseKey,
|
|
10258
|
-
projectId: ns,
|
|
10259
|
-
organization: process.env.RETRIVORA_ORGANIZATION || "default-org",
|
|
10260
|
-
sdkVersion: SDK_VERSION,
|
|
10261
|
-
model: modelName,
|
|
10262
|
-
provider: providerName,
|
|
10263
|
-
tokens: tokenCount,
|
|
10264
|
-
cost: costEst,
|
|
10265
|
-
costUsd: costEst,
|
|
10266
|
-
latencyMs: latencyDuration,
|
|
10267
|
-
status: "success",
|
|
10268
|
-
action: "RAG_QUERY",
|
|
10269
|
-
feature: "RAG",
|
|
10270
|
-
timestamp: (/* @__PURE__ */ new Date()).toISOString(),
|
|
10271
|
-
details: `Org: default-org | Model: ${providerName}/${modelName} | Tokens: ${tokenCount} | Latency: ${latencyDuration}ms | Cost: $${costEst} | SDK: v${SDK_VERSION} | Feature: RAG`
|
|
10272
|
-
})
|
|
10273
|
-
}).catch((err) => {
|
|
10274
|
-
Logger.warn(`[Pipeline Telemetry] Failed to send trace async to ${absoluteUrl}`, { error: err.message });
|
|
10275
|
-
});
|
|
10276
|
-
} catch (err) {
|
|
10277
|
-
Logger.warn(`[Pipeline Telemetry] Failed to send trace async to ${absoluteUrl}:`, { error: err.message });
|
|
10278
|
-
}
|
|
10279
|
-
})();
|
|
10280
|
-
}
|
|
10281
11001
|
yield {
|
|
10282
11002
|
reply: "",
|
|
10283
11003
|
sources,
|
|
@@ -11089,6 +11809,374 @@ function createFromPreset(presetName) {
|
|
|
11089
11809
|
return new ConfigBuilder().vectorDb(preset.vectorDb, { apiKey: process.env[`${preset.vectorDb.toUpperCase()}_API_KEY`] }).llm(preset.llm, void 0, process.env[`${preset.llm.toUpperCase()}_API_KEY`]).embedding(preset.embedding, void 0, process.env[`${preset.embedding.toUpperCase()}_API_KEY`]);
|
|
11090
11810
|
}
|
|
11091
11811
|
|
|
11812
|
+
// src/utils/UniversalTableParser.ts
|
|
11813
|
+
var UniversalTableParser = class {
|
|
11814
|
+
/**
|
|
11815
|
+
* Parse a raw 2D grid of strings with complete matrix intelligence.
|
|
11816
|
+
*/
|
|
11817
|
+
static parseGrid(grid, sheetName = "Data") {
|
|
11818
|
+
if (!grid || grid.length === 0) return "";
|
|
11819
|
+
const cleanGrid = grid.map((row) => Array.isArray(row) ? row.map((c) => c !== null && c !== void 0 ? String(c).trim() : "") : []).filter((row) => row.some((cell) => cell.length > 0));
|
|
11820
|
+
if (cleanGrid.length === 0) return "";
|
|
11821
|
+
const { headerRows, dataRows } = this.splitHeaderAndDataRows(cleanGrid);
|
|
11822
|
+
const fusedHeaders = this.fuseHeaders(headerRows);
|
|
11823
|
+
const matrixInfo = this.analyzeMatrixStructure(fusedHeaders, dataRows);
|
|
11824
|
+
const sections = [`--- Sheet: ${sheetName} ---`];
|
|
11825
|
+
const markdownTable = this.generateMarkdownTable(fusedHeaders, dataRows);
|
|
11826
|
+
if (markdownTable) {
|
|
11827
|
+
sections.push(`
|
|
11828
|
+
## Table: ${sheetName}
|
|
11829
|
+
${markdownTable}`);
|
|
11830
|
+
}
|
|
11831
|
+
if (matrixInfo.isComparisonMatrix && matrixInfo.entityIndices.length > 0) {
|
|
11832
|
+
const entityProfiles = this.generateEntityProfiles(fusedHeaders, dataRows, matrixInfo);
|
|
11833
|
+
if (entityProfiles) {
|
|
11834
|
+
sections.push(`
|
|
11835
|
+
## Entity Comparison Profiles
|
|
11836
|
+
${entityProfiles}`);
|
|
11837
|
+
}
|
|
11838
|
+
const criteriaComparisons = this.generateCriteriaComparisons(fusedHeaders, dataRows, matrixInfo);
|
|
11839
|
+
if (criteriaComparisons) {
|
|
11840
|
+
sections.push(`
|
|
11841
|
+
## Criteria Breakdown & Comparisons
|
|
11842
|
+
${criteriaComparisons}`);
|
|
11843
|
+
}
|
|
11844
|
+
} else {
|
|
11845
|
+
const contextualRows = this.generateContextualRows(fusedHeaders, dataRows);
|
|
11846
|
+
if (contextualRows) {
|
|
11847
|
+
sections.push(`
|
|
11848
|
+
## Structured Records
|
|
11849
|
+
${contextualRows}`);
|
|
11850
|
+
}
|
|
11851
|
+
}
|
|
11852
|
+
return sections.join("\n\n");
|
|
11853
|
+
}
|
|
11854
|
+
/**
|
|
11855
|
+
* Parse a raw CSV / TSV text string into tri-perspective contextual format.
|
|
11856
|
+
*/
|
|
11857
|
+
static parseDelimitedText(text, title = "Table", delimiter) {
|
|
11858
|
+
if (!text || !text.trim()) return "";
|
|
11859
|
+
if (!delimiter) {
|
|
11860
|
+
const firstLine = text.split(/\r?\n/)[0] || "";
|
|
11861
|
+
const commaCount = (firstLine.match(/,/g) || []).length;
|
|
11862
|
+
const tabCount = (firstLine.match(/\t/g) || []).length;
|
|
11863
|
+
const semiCount = (firstLine.match(/;/g) || []).length;
|
|
11864
|
+
const pipeCount = (firstLine.match(/\|/g) || []).length;
|
|
11865
|
+
if (tabCount > commaCount && tabCount > semiCount) delimiter = " ";
|
|
11866
|
+
else if (semiCount > commaCount && semiCount > tabCount) delimiter = ";";
|
|
11867
|
+
else if (pipeCount > commaCount && pipeCount > semiCount) delimiter = "|";
|
|
11868
|
+
else delimiter = ",";
|
|
11869
|
+
}
|
|
11870
|
+
const lines = text.split(/\r?\n/);
|
|
11871
|
+
const grid = [];
|
|
11872
|
+
for (const line of lines) {
|
|
11873
|
+
if (!line.trim()) continue;
|
|
11874
|
+
const row = [];
|
|
11875
|
+
let inQuote = false;
|
|
11876
|
+
let current = "";
|
|
11877
|
+
for (let i = 0; i < line.length; i++) {
|
|
11878
|
+
const char = line[i];
|
|
11879
|
+
if (char === '"') {
|
|
11880
|
+
if (inQuote && line[i + 1] === '"') {
|
|
11881
|
+
current += '"';
|
|
11882
|
+
i++;
|
|
11883
|
+
} else {
|
|
11884
|
+
inQuote = !inQuote;
|
|
11885
|
+
}
|
|
11886
|
+
} else if (char === delimiter && !inQuote) {
|
|
11887
|
+
row.push(current.trim());
|
|
11888
|
+
current = "";
|
|
11889
|
+
} else {
|
|
11890
|
+
current += char;
|
|
11891
|
+
}
|
|
11892
|
+
}
|
|
11893
|
+
row.push(current.trim());
|
|
11894
|
+
grid.push(row);
|
|
11895
|
+
}
|
|
11896
|
+
return this.parseGrid(grid, title);
|
|
11897
|
+
}
|
|
11898
|
+
/**
|
|
11899
|
+
* Parse a SheetJS sheet object with complete merged cell and matrix intelligence.
|
|
11900
|
+
*/
|
|
11901
|
+
static parseSheet(sheet, sheetName, XLSX) {
|
|
11902
|
+
if (!sheet) return "";
|
|
11903
|
+
const grid = this.extractAndUnrollGrid(sheet, XLSX);
|
|
11904
|
+
if (!grid || grid.length === 0) {
|
|
11905
|
+
const csv = XLSX.utils.sheet_to_csv(sheet, { blankrows: false });
|
|
11906
|
+
return csv && csv.trim() ? `--- Sheet: ${sheetName} ---
|
|
11907
|
+
${csv.trim()}` : "";
|
|
11908
|
+
}
|
|
11909
|
+
return this.parseGrid(grid, sheetName);
|
|
11910
|
+
}
|
|
11911
|
+
/**
|
|
11912
|
+
* Unrolls SheetJS merged cell ranges (`!merges`) across the 2D grid
|
|
11913
|
+
* so no child cell is left blank.
|
|
11914
|
+
*/
|
|
11915
|
+
static extractAndUnrollGrid(sheet, XLSX) {
|
|
11916
|
+
var _a2;
|
|
11917
|
+
const rawGrid = XLSX.utils.sheet_to_json(sheet, { header: 1, blankrows: false, defval: "" });
|
|
11918
|
+
if (!rawGrid || rawGrid.length === 0) return [];
|
|
11919
|
+
const grid = rawGrid.map(
|
|
11920
|
+
(row) => (Array.isArray(row) ? row : []).map((cell) => cell !== null && cell !== void 0 ? String(cell).trim() : "")
|
|
11921
|
+
);
|
|
11922
|
+
const merges = sheet["!merges"] || [];
|
|
11923
|
+
for (const merge of merges) {
|
|
11924
|
+
const { s, e } = merge;
|
|
11925
|
+
const originVal = ((_a2 = grid[s.r]) == null ? void 0 : _a2[s.c]) || "";
|
|
11926
|
+
if (!originVal) continue;
|
|
11927
|
+
for (let r = s.r; r <= e.r; r++) {
|
|
11928
|
+
if (!grid[r]) grid[r] = [];
|
|
11929
|
+
for (let c = s.c; c <= e.c; c++) {
|
|
11930
|
+
if (!grid[r][c] || grid[r][c] === "") {
|
|
11931
|
+
grid[r][c] = originVal;
|
|
11932
|
+
}
|
|
11933
|
+
}
|
|
11934
|
+
}
|
|
11935
|
+
}
|
|
11936
|
+
return grid.filter((row) => row.some((cell) => cell.length > 0));
|
|
11937
|
+
}
|
|
11938
|
+
/**
|
|
11939
|
+
* Intelligently identify where header rows end and data rows begin.
|
|
11940
|
+
*/
|
|
11941
|
+
static splitHeaderAndDataRows(grid) {
|
|
11942
|
+
if (grid.length <= 1) {
|
|
11943
|
+
return { headerRows: grid, dataRows: [] };
|
|
11944
|
+
}
|
|
11945
|
+
let headerRowCount = 1;
|
|
11946
|
+
for (let r = 1; r < Math.min(4, grid.length); r++) {
|
|
11947
|
+
const row = grid[r];
|
|
11948
|
+
const nonEmptyCells = row.filter(Boolean);
|
|
11949
|
+
if (nonEmptyCells.length === 0) continue;
|
|
11950
|
+
const isDataRow = nonEmptyCells.some(
|
|
11951
|
+
(c) => /^-?\d+(?:\.\d+)?%?$/.test(c) || /^\$?[\d,.]+(?:\s*(?:mil|k|m|b|usd|eur))?$/i.test(c) || /^(yes|no|true|false|pass|fail|compliant|non-compliant)$/i.test(c)
|
|
11952
|
+
);
|
|
11953
|
+
const hasSubheaderLabels = nonEmptyCells.some(
|
|
11954
|
+
(c) => /\b(bidder|vendor|supplier|score|weight|criteria|item|specification|technical|commercial|rating|status|cost|price|qty|unit)\b/i.test(c)
|
|
11955
|
+
);
|
|
11956
|
+
if (!isDataRow && hasSubheaderLabels) {
|
|
11957
|
+
headerRowCount = r + 1;
|
|
11958
|
+
} else if (!isDataRow && r === 1 && nonEmptyCells.length >= 2) {
|
|
11959
|
+
headerRowCount = 2;
|
|
11960
|
+
} else {
|
|
11961
|
+
break;
|
|
11962
|
+
}
|
|
11963
|
+
}
|
|
11964
|
+
const headerRows = grid.slice(0, headerRowCount);
|
|
11965
|
+
const dataRows = grid.slice(headerRowCount);
|
|
11966
|
+
return { headerRows, dataRows };
|
|
11967
|
+
}
|
|
11968
|
+
/**
|
|
11969
|
+
* Fuses multi-tier header rows into composite paths (e.g. `Commercial > Payment Terms > Bidder A`).
|
|
11970
|
+
*/
|
|
11971
|
+
static fuseHeaders(headerRows) {
|
|
11972
|
+
var _a2, _b;
|
|
11973
|
+
if (headerRows.length === 0) return [];
|
|
11974
|
+
if (headerRows.length === 1) {
|
|
11975
|
+
return headerRows[0].map((h, i) => h || `Column_${i + 1}`);
|
|
11976
|
+
}
|
|
11977
|
+
const maxCols = Math.max(...headerRows.map((r) => r.length));
|
|
11978
|
+
const fused = [];
|
|
11979
|
+
for (let c = 0; c < maxCols; c++) {
|
|
11980
|
+
const components = [];
|
|
11981
|
+
for (let r = 0; r < headerRows.length; r++) {
|
|
11982
|
+
const val = (_b = (_a2 = headerRows[r]) == null ? void 0 : _a2[c]) == null ? void 0 : _b.trim();
|
|
11983
|
+
if (val && !components.includes(val)) {
|
|
11984
|
+
components.push(val);
|
|
11985
|
+
}
|
|
11986
|
+
}
|
|
11987
|
+
fused.push(components.join(" > ") || `Column_${c + 1}`);
|
|
11988
|
+
}
|
|
11989
|
+
return fused;
|
|
11990
|
+
}
|
|
11991
|
+
/**
|
|
11992
|
+
* Analyzes whether the table is a comparison matrix (columns = entities, rows = criteria).
|
|
11993
|
+
*/
|
|
11994
|
+
static analyzeMatrixStructure(headers, dataRows) {
|
|
11995
|
+
const entityPattern = /\b(bidder|vendor|supplier|candidate|option|company|contractor|partner|proposal|firm|provider|solution|tier|package)\b/i;
|
|
11996
|
+
const entityIndices = [];
|
|
11997
|
+
const entityNames = [];
|
|
11998
|
+
let criteriaColumnIndex = 0;
|
|
11999
|
+
headers.forEach((h, idx) => {
|
|
12000
|
+
if (entityPattern.test(h) || /\b(bidder\s*\d+|vendor\s*[a-z0-9]+|option\s*[a-z0-9]+)\b/i.test(h)) {
|
|
12001
|
+
entityIndices.push(idx);
|
|
12002
|
+
const parts = h.split(" > ");
|
|
12003
|
+
entityNames.push(parts[parts.length - 1]);
|
|
12004
|
+
} else if (/\b(criteria|criterion|description|specification|category|requirement|item|feature)\b/i.test(h)) {
|
|
12005
|
+
criteriaColumnIndex = idx;
|
|
12006
|
+
}
|
|
12007
|
+
});
|
|
12008
|
+
const isComparisonMatrix = entityIndices.length >= 2 || headers.length >= 3 && entityIndices.length >= 1;
|
|
12009
|
+
return {
|
|
12010
|
+
isComparisonMatrix,
|
|
12011
|
+
entityIndices: entityIndices.length > 0 ? entityIndices : headers.slice(1).map((_, i) => i + 1),
|
|
12012
|
+
entityNames: entityNames.length > 0 ? entityNames : headers.slice(1),
|
|
12013
|
+
criteriaColumnIndex
|
|
12014
|
+
};
|
|
12015
|
+
}
|
|
12016
|
+
/**
|
|
12017
|
+
* Perspective A: Clean Markdown Table
|
|
12018
|
+
*/
|
|
12019
|
+
static generateMarkdownTable(headers, dataRows) {
|
|
12020
|
+
if (headers.length === 0 || dataRows.length === 0) return "";
|
|
12021
|
+
const cleanHeaders = headers.map((h) => h.replace(/\|/g, "-"));
|
|
12022
|
+
const lines = [];
|
|
12023
|
+
lines.push(`| ${cleanHeaders.join(" | ")} |`);
|
|
12024
|
+
lines.push(`| ${cleanHeaders.map(() => "---").join(" | ")} |`);
|
|
12025
|
+
for (const row of dataRows) {
|
|
12026
|
+
const paddedRow = headers.map((_, i) => (row[i] || "").replace(/\|/g, "-"));
|
|
12027
|
+
if (paddedRow.some((cell) => cell.trim().length > 0)) {
|
|
12028
|
+
lines.push(`| ${paddedRow.join(" | ")} |`);
|
|
12029
|
+
}
|
|
12030
|
+
}
|
|
12031
|
+
return lines.join("\n");
|
|
12032
|
+
}
|
|
12033
|
+
/**
|
|
12034
|
+
* Perspective B: Entity-Centric Profiles (e.g. Bidder 1, Bidder 2 full summary)
|
|
12035
|
+
*/
|
|
12036
|
+
static generateEntityProfiles(headers, dataRows, matrixInfo) {
|
|
12037
|
+
const profiles = [];
|
|
12038
|
+
matrixInfo.entityIndices.forEach((colIdx, i) => {
|
|
12039
|
+
const entityName = matrixInfo.entityNames[i] || headers[colIdx] || `Entity ${i + 1}`;
|
|
12040
|
+
const lines = [`### Profile & Evaluation: ${entityName}`];
|
|
12041
|
+
for (const row of dataRows) {
|
|
12042
|
+
const criterion = row[matrixInfo.criteriaColumnIndex] || row[0] || "Specification";
|
|
12043
|
+
const val = row[colIdx];
|
|
12044
|
+
if (val !== void 0 && val !== null && String(val).trim() !== "") {
|
|
12045
|
+
lines.push(`- **${criterion}**: ${String(val).trim()}`);
|
|
12046
|
+
}
|
|
12047
|
+
}
|
|
12048
|
+
if (lines.length > 1) {
|
|
12049
|
+
profiles.push(lines.join("\n"));
|
|
12050
|
+
}
|
|
12051
|
+
});
|
|
12052
|
+
return profiles.join("\n\n");
|
|
12053
|
+
}
|
|
12054
|
+
/**
|
|
12055
|
+
* Perspective C: Criteria Cross-Comparisons (e.g. Commercial Bid comparison across all Bidders)
|
|
12056
|
+
*/
|
|
12057
|
+
static generateCriteriaComparisons(headers, dataRows, matrixInfo) {
|
|
12058
|
+
const comparisons = [];
|
|
12059
|
+
for (const row of dataRows) {
|
|
12060
|
+
const criterion = row[matrixInfo.criteriaColumnIndex] || row[0] || "Metric";
|
|
12061
|
+
const entityPairs = [];
|
|
12062
|
+
matrixInfo.entityIndices.forEach((colIdx, i) => {
|
|
12063
|
+
const entityName = matrixInfo.entityNames[i] || headers[colIdx] || `Entity ${i + 1}`;
|
|
12064
|
+
const val = row[colIdx];
|
|
12065
|
+
if (val !== void 0 && val !== null && String(val).trim() !== "") {
|
|
12066
|
+
entityPairs.push(`${entityName}: ${String(val).trim()}`);
|
|
12067
|
+
}
|
|
12068
|
+
});
|
|
12069
|
+
if (entityPairs.length > 0) {
|
|
12070
|
+
comparisons.push(`- **${criterion}** $\\rightarrow$ ${entityPairs.join(" | ")}`);
|
|
12071
|
+
}
|
|
12072
|
+
}
|
|
12073
|
+
return comparisons.join("\n");
|
|
12074
|
+
}
|
|
12075
|
+
/**
|
|
12076
|
+
* Standard contextual key-value rows for general tabular records.
|
|
12077
|
+
*/
|
|
12078
|
+
static generateContextualRows(headers, dataRows) {
|
|
12079
|
+
const lines = [];
|
|
12080
|
+
for (const row of dataRows) {
|
|
12081
|
+
const pairs = [];
|
|
12082
|
+
headers.forEach((h, i) => {
|
|
12083
|
+
const val = row[i];
|
|
12084
|
+
if (val !== void 0 && val !== null && String(val).trim() !== "") {
|
|
12085
|
+
pairs.push(`${h}: ${String(val).trim()}`);
|
|
12086
|
+
}
|
|
12087
|
+
});
|
|
12088
|
+
if (pairs.length > 0) {
|
|
12089
|
+
lines.push(pairs.join(" | "));
|
|
12090
|
+
}
|
|
12091
|
+
}
|
|
12092
|
+
return lines.join("\n");
|
|
12093
|
+
}
|
|
12094
|
+
};
|
|
12095
|
+
|
|
12096
|
+
// src/utils/VisionExtractor.ts
|
|
12097
|
+
var VisionExtractor = class {
|
|
12098
|
+
/**
|
|
12099
|
+
* Check if a buffer represents an image format.
|
|
12100
|
+
*/
|
|
12101
|
+
static isImageBuffer(buffer, fileName = "", mimeType = "") {
|
|
12102
|
+
var _a2;
|
|
12103
|
+
const ext = ((_a2 = fileName.split(".").pop()) == null ? void 0 : _a2.toLowerCase()) || "";
|
|
12104
|
+
if (["png", "jpg", "jpeg", "webp", "bmp", "tiff", "gif"].includes(ext)) {
|
|
12105
|
+
return true;
|
|
12106
|
+
}
|
|
12107
|
+
if (mimeType.startsWith("image/")) {
|
|
12108
|
+
return true;
|
|
12109
|
+
}
|
|
12110
|
+
if (buffer && buffer.length >= 4) {
|
|
12111
|
+
if (buffer[0] === 137 && buffer[1] === 80 && buffer[2] === 78 && buffer[3] === 71) return true;
|
|
12112
|
+
if (buffer[0] === 255 && buffer[1] === 216 && buffer[2] === 255) return true;
|
|
12113
|
+
if (buffer[0] === 82 && buffer[1] === 73 && buffer[2] === 70 && buffer[3] === 70) return true;
|
|
12114
|
+
}
|
|
12115
|
+
return false;
|
|
12116
|
+
}
|
|
12117
|
+
/**
|
|
12118
|
+
* Extract rich textual content, tables, and diagram descriptions from an image buffer using Vision LLM.
|
|
12119
|
+
*/
|
|
12120
|
+
static async extractFromImage(buffer, fileName = "document.png", options = {}) {
|
|
12121
|
+
var _a2, _b, _c, _d, _e, _f, _g2, _h, _i;
|
|
12122
|
+
const base64Image = buffer.toString("base64");
|
|
12123
|
+
const ext = ((_a2 = fileName.split(".").pop()) == null ? void 0 : _a2.toLowerCase()) || "png";
|
|
12124
|
+
const mime = ext === "jpg" || ext === "jpeg" ? "image/jpeg" : ext === "webp" ? "image/webp" : "image/png";
|
|
12125
|
+
const apiKey = options.apiKey || (typeof process !== "undefined" ? ((_b = process.env) == null ? void 0 : _b.OPENAI_API_KEY) || ((_c = process.env) == null ? void 0 : _c.GEMINI_API_KEY) || ((_d = process.env) == null ? void 0 : _d.ANTHROPIC_API_KEY) : void 0);
|
|
12126
|
+
if (!apiKey) {
|
|
12127
|
+
return `[Scanned Document / Image: ${fileName} - Size: ${(buffer.length / 1024).toFixed(1)} KB]`;
|
|
12128
|
+
}
|
|
12129
|
+
try {
|
|
12130
|
+
const response = await fetch("https://api.openai.com/v1/chat/completions", {
|
|
12131
|
+
method: "POST",
|
|
12132
|
+
headers: {
|
|
12133
|
+
"Authorization": `Bearer ${apiKey}`,
|
|
12134
|
+
"Content-Type": "application/json"
|
|
12135
|
+
},
|
|
12136
|
+
body: JSON.stringify({
|
|
12137
|
+
model: options.model || "gpt-4o-mini",
|
|
12138
|
+
messages: [
|
|
12139
|
+
{
|
|
12140
|
+
role: "system",
|
|
12141
|
+
content: "You are an expert Document OCR & Data Extraction engine. Transcribe all text, numbers, headings, tables, and diagram structures in this image into clean, structured GitHub Markdown. For tables, use standard markdown table syntax. Preserve exact numbers, dates, and column headers. Do not summarize \u2014 transcribe faithfully."
|
|
12142
|
+
},
|
|
12143
|
+
{
|
|
12144
|
+
role: "user",
|
|
12145
|
+
content: [
|
|
12146
|
+
{
|
|
12147
|
+
type: "text",
|
|
12148
|
+
text: `Please transcribe the content of this document/image (${fileName}):`
|
|
12149
|
+
},
|
|
12150
|
+
{
|
|
12151
|
+
type: "image_url",
|
|
12152
|
+
image_url: {
|
|
12153
|
+
url: `data:${mime};base64,${base64Image}`,
|
|
12154
|
+
detail: "high"
|
|
12155
|
+
}
|
|
12156
|
+
}
|
|
12157
|
+
]
|
|
12158
|
+
}
|
|
12159
|
+
],
|
|
12160
|
+
max_tokens: options.maxTokens || 2048,
|
|
12161
|
+
temperature: (_e = options.temperature) != null ? _e : 0.1
|
|
12162
|
+
})
|
|
12163
|
+
});
|
|
12164
|
+
if (response.ok) {
|
|
12165
|
+
const data = await response.json();
|
|
12166
|
+
const text = (_i = (_h = (_g2 = (_f = data.choices) == null ? void 0 : _f[0]) == null ? void 0 : _g2.message) == null ? void 0 : _h.content) == null ? void 0 : _i.trim();
|
|
12167
|
+
if (text) {
|
|
12168
|
+
return `--- Document / Image OCR: ${fileName} ---
|
|
12169
|
+
|
|
12170
|
+
${text}`;
|
|
12171
|
+
}
|
|
12172
|
+
}
|
|
12173
|
+
} catch (err) {
|
|
12174
|
+
console.warn("[VisionExtractor] OpenAI Vision OCR failed:", err);
|
|
12175
|
+
}
|
|
12176
|
+
return `[Scanned Image Document: ${fileName}]`;
|
|
12177
|
+
}
|
|
12178
|
+
};
|
|
12179
|
+
|
|
11092
12180
|
// src/utils/DocumentParser.ts
|
|
11093
12181
|
var DocumentParser = class {
|
|
11094
12182
|
/**
|
|
@@ -11112,52 +12200,11 @@ var DocumentParser = class {
|
|
|
11112
12200
|
return "text";
|
|
11113
12201
|
}
|
|
11114
12202
|
/**
|
|
11115
|
-
* Convert spreadsheet sheets into high-fidelity, context-preserved tabular text.
|
|
11116
|
-
*
|
|
12203
|
+
* Convert spreadsheet sheets into high-fidelity, context-preserved tabular text via UniversalTableParser.
|
|
12204
|
+
* Unrolls merged cells, fuses multi-level headers, and generates tri-perspective comparison profiles.
|
|
11117
12205
|
*/
|
|
11118
12206
|
static formatSheetToContextualText(sheet, sheetName, XLSX) {
|
|
11119
|
-
|
|
11120
|
-
if (!jsonRows || jsonRows.length === 0) {
|
|
11121
|
-
const csv = XLSX.utils.sheet_to_csv(sheet, { blankrows: false });
|
|
11122
|
-
return csv && csv.trim() ? `--- Sheet: ${sheetName} ---
|
|
11123
|
-
${csv.trim()}` : "";
|
|
11124
|
-
}
|
|
11125
|
-
const lines = [`--- Sheet: ${sheetName} ---`];
|
|
11126
|
-
let currentHeader = [];
|
|
11127
|
-
for (let r = 0; r < jsonRows.length; r++) {
|
|
11128
|
-
const row = jsonRows[r];
|
|
11129
|
-
if (!row || row.length === 0) continue;
|
|
11130
|
-
const cells = row.map((c) => c !== null && c !== void 0 ? String(c).trim() : "").filter(Boolean);
|
|
11131
|
-
if (cells.length === 0) continue;
|
|
11132
|
-
if (cells.length === 1 || cells.length <= 2 && cells[0].toUpperCase().includes("STEP")) {
|
|
11133
|
-
lines.push(`
|
|
11134
|
-
## ${cells.join(" ")}`);
|
|
11135
|
-
continue;
|
|
11136
|
-
}
|
|
11137
|
-
const isHeaderCandidate = cells.every((c) => isNaN(Number(c)) && !/^\$?[\d,.]+/.test(c));
|
|
11138
|
-
if (isHeaderCandidate && cells.length >= 2) {
|
|
11139
|
-
currentHeader = row.map((c) => c !== null && c !== void 0 ? String(c).trim() : "");
|
|
11140
|
-
lines.push(`| ${cells.join(" | ")} |`);
|
|
11141
|
-
lines.push(`| ${cells.map(() => "---").join(" | ")} |`);
|
|
11142
|
-
continue;
|
|
11143
|
-
}
|
|
11144
|
-
if (currentHeader.length > 0 && cells.length > 1) {
|
|
11145
|
-
const fieldPairs = [];
|
|
11146
|
-
for (let c = 0; c < row.length; c++) {
|
|
11147
|
-
const h = currentHeader[c];
|
|
11148
|
-
const val = row[c];
|
|
11149
|
-
if (h && val !== null && val !== void 0 && String(val).trim() !== "") {
|
|
11150
|
-
fieldPairs.push(`${h}: ${String(val).trim()}`);
|
|
11151
|
-
}
|
|
11152
|
-
}
|
|
11153
|
-
if (fieldPairs.length > 0) {
|
|
11154
|
-
lines.push(fieldPairs.join(" | "));
|
|
11155
|
-
continue;
|
|
11156
|
-
}
|
|
11157
|
-
}
|
|
11158
|
-
lines.push(cells.join(", "));
|
|
11159
|
-
}
|
|
11160
|
-
return lines.join("\n");
|
|
12207
|
+
return UniversalTableParser.parseSheet(sheet, sheetName, XLSX);
|
|
11161
12208
|
}
|
|
11162
12209
|
/**
|
|
11163
12210
|
* Extract text from a File or Buffer based on its type.
|
|
@@ -11181,7 +12228,9 @@ ${csv.trim()}` : "";
|
|
|
11181
12228
|
return this.readAsText(file);
|
|
11182
12229
|
}
|
|
11183
12230
|
if (extension === "csv" || extension === "tsv" || mimeType === "text/csv" || mimeType === "text/tab-separated-values") {
|
|
11184
|
-
|
|
12231
|
+
const text = await this.readAsText(file);
|
|
12232
|
+
const parsed = UniversalTableParser.parseDelimitedText(text, fileName);
|
|
12233
|
+
return parsed || text;
|
|
11185
12234
|
}
|
|
11186
12235
|
if (extension === "pptx" || extension === "ppt" || extension === "odp" || mimeType.includes("presentation") || mimeType.includes("powerpoint")) {
|
|
11187
12236
|
try {
|
|
@@ -11288,16 +12337,22 @@ ${csv.trim()}`);
|
|
|
11288
12337
|
return this.parseLegacyExcelOrFallback(buffer, fileName);
|
|
11289
12338
|
}
|
|
11290
12339
|
if (extension === "pdf" || mimeType === "application/pdf") {
|
|
12340
|
+
const buffer = Buffer.isBuffer(file) ? file : Buffer.from(await file.arrayBuffer());
|
|
11291
12341
|
try {
|
|
11292
12342
|
const pdfModule = await import("pdf-parse");
|
|
11293
|
-
const buffer = Buffer.isBuffer(file) ? file : Buffer.from(await file.arrayBuffer());
|
|
11294
12343
|
const pdf = pdfModule.default || pdfModule;
|
|
11295
12344
|
const data = await pdf(buffer);
|
|
11296
|
-
|
|
12345
|
+
if (data.text && data.text.trim().length > 30) {
|
|
12346
|
+
return data.text;
|
|
12347
|
+
}
|
|
11297
12348
|
} catch (err) {
|
|
11298
|
-
console.warn(
|
|
11299
|
-
return `[PDF Parsing Error for ${fileName}]`;
|
|
12349
|
+
console.warn("[DocumentParser] pdf-parse failed, attempting Vision OCR fallback:", err);
|
|
11300
12350
|
}
|
|
12351
|
+
return VisionExtractor.extractFromImage(buffer, fileName);
|
|
12352
|
+
}
|
|
12353
|
+
if (["png", "jpg", "jpeg", "webp", "bmp", "tiff", "gif"].includes(extension) || mimeType.startsWith("image/")) {
|
|
12354
|
+
const buffer = Buffer.isBuffer(file) ? file : Buffer.from(await file.arrayBuffer());
|
|
12355
|
+
return VisionExtractor.extractFromImage(buffer, fileName);
|
|
11301
12356
|
}
|
|
11302
12357
|
if (extension === "docx" || extension === "doc" || extension === "odt" || extension === "rtf" || mimeType === "application/vnd.openxmlformats-officedocument.wordprocessingml.document" || mimeType === "application/msword" || mimeType === "application/rtf") {
|
|
11303
12358
|
try {
|
|
@@ -11601,8 +12656,8 @@ init_RedisProvider();
|
|
|
11601
12656
|
init_WeaviateProvider();
|
|
11602
12657
|
init_UniversalVectorProvider();
|
|
11603
12658
|
|
|
11604
|
-
// src/handlers/
|
|
11605
|
-
import { NextResponse } from "next/server";
|
|
12659
|
+
// src/handlers/chatHandler.ts
|
|
12660
|
+
import { NextResponse as NextResponse3 } from "next/server";
|
|
11606
12661
|
|
|
11607
12662
|
// src/core/DatabaseStorage.ts
|
|
11608
12663
|
import * as fs from "fs";
|
|
@@ -11655,44 +12710,38 @@ var _DatabaseStorage = class _DatabaseStorage {
|
|
|
11655
12710
|
*
|
|
11656
12711
|
* In non-production environments this is a complete no-op so local
|
|
11657
12712
|
* development works without a license key.
|
|
12713
|
+
/**
|
|
12714
|
+
* Checks if the caller holds a valid paid-tier license in production.
|
|
12715
|
+
* Returns true if allowed, false if not supported on current tier.
|
|
11658
12716
|
*/
|
|
11659
|
-
async
|
|
11660
|
-
if (process.env.NODE_ENV !== "production") return;
|
|
11661
|
-
if (
|
|
11662
|
-
if (
|
|
11663
|
-
|
|
11664
|
-
"[Retrivora SDK] Chat History and Feedback features require a paid license key in production."
|
|
11665
|
-
);
|
|
12717
|
+
async isPaidSubscription() {
|
|
12718
|
+
if (process.env.NODE_ENV !== "production") return true;
|
|
12719
|
+
if (!this.config.licenseKey) return false;
|
|
12720
|
+
if (Date.now() - this.subscriptionCheckedAt < _DatabaseStorage.SUBSCRIPTION_CACHE_TTL_MS) {
|
|
12721
|
+
return true;
|
|
11666
12722
|
}
|
|
11667
12723
|
try {
|
|
11668
12724
|
const payload = await LicenseVerifier.verifyAsync(this.config.licenseKey, this.config.projectId);
|
|
11669
12725
|
const tier = (payload.tier || "").toLowerCase();
|
|
11670
|
-
const PAID_TIERS = /* @__PURE__ */ new Set(["premium", "enterprise", "growth", "pro", "developer_pro"]);
|
|
11671
|
-
|
|
11672
|
-
|
|
11673
|
-
|
|
11674
|
-
);
|
|
12726
|
+
const PAID_TIERS = /* @__PURE__ */ new Set(["premium", "enterprise", "growth", "pro", "developer_pro", "free_trial"]);
|
|
12727
|
+
const isAllowed = PAID_TIERS.has(tier) || tier.includes("pro") || tier.includes("enterprise");
|
|
12728
|
+
if (isAllowed) {
|
|
12729
|
+
this.subscriptionCheckedAt = Date.now();
|
|
11675
12730
|
}
|
|
11676
|
-
|
|
11677
|
-
} catch (
|
|
11678
|
-
|
|
11679
|
-
`[Retrivora SDK] Subscription check failed: ${err instanceof Error ? err.message : String(err)}`
|
|
11680
|
-
);
|
|
12731
|
+
return isAllowed;
|
|
12732
|
+
} catch (e) {
|
|
12733
|
+
return false;
|
|
11681
12734
|
}
|
|
11682
12735
|
}
|
|
11683
|
-
async
|
|
12736
|
+
async isHistoryAvailable() {
|
|
11684
12737
|
var _a2, _b;
|
|
11685
|
-
|
|
11686
|
-
|
|
11687
|
-
throw new Error("[Retrivora SDK] Chat History is disabled in RagConfig.");
|
|
11688
|
-
}
|
|
12738
|
+
if (((_b = (_a2 = this.config.rag) == null ? void 0 : _a2.history) == null ? void 0 : _b.enabled) === false) return false;
|
|
12739
|
+
return this.isPaidSubscription();
|
|
11689
12740
|
}
|
|
11690
|
-
async
|
|
12741
|
+
async isFeedbackAvailable() {
|
|
11691
12742
|
var _a2, _b;
|
|
11692
|
-
|
|
11693
|
-
|
|
11694
|
-
throw new Error("[Retrivora SDK] User Feedback is disabled in RagConfig.");
|
|
11695
|
-
}
|
|
12743
|
+
if (((_b = (_a2 = this.config.rag) == null ? void 0 : _a2.feedback) == null ? void 0 : _b.enabled) === false) return false;
|
|
12744
|
+
return this.isPaidSubscription();
|
|
11696
12745
|
}
|
|
11697
12746
|
/**
|
|
11698
12747
|
* Returns (or lazily creates) the PostgreSQL connection pool for this project.
|
|
@@ -11806,7 +12855,7 @@ var _DatabaseStorage = class _DatabaseStorage {
|
|
|
11806
12855
|
}
|
|
11807
12856
|
// ─── Message History Operations ──────────────────────────────────────────
|
|
11808
12857
|
async saveMessage(sessionId, message) {
|
|
11809
|
-
await this.
|
|
12858
|
+
if (!await this.isHistoryAvailable()) return;
|
|
11810
12859
|
const createdAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
11811
12860
|
const msgId = message.id || `msg_${Date.now()}_${Math.random().toString(36).slice(2, 6)}`;
|
|
11812
12861
|
if (this.provider === "postgresql" || this.provider === "pgvector") {
|
|
@@ -11869,7 +12918,7 @@ var _DatabaseStorage = class _DatabaseStorage {
|
|
|
11869
12918
|
}
|
|
11870
12919
|
}
|
|
11871
12920
|
async getHistory(sessionId, projectIds) {
|
|
11872
|
-
await this.
|
|
12921
|
+
if (!await this.isHistoryAvailable()) return [];
|
|
11873
12922
|
if (projectIds && projectIds.length > 0 && !this.sessionInScope(sessionId, projectIds)) {
|
|
11874
12923
|
return [];
|
|
11875
12924
|
}
|
|
@@ -11903,7 +12952,7 @@ var _DatabaseStorage = class _DatabaseStorage {
|
|
|
11903
12952
|
}
|
|
11904
12953
|
}
|
|
11905
12954
|
async clearHistory(sessionId, projectIds) {
|
|
11906
|
-
await this.
|
|
12955
|
+
if (!await this.isHistoryAvailable()) return;
|
|
11907
12956
|
if (projectIds && projectIds.length > 0 && !this.sessionInScope(sessionId, projectIds)) {
|
|
11908
12957
|
return;
|
|
11909
12958
|
}
|
|
@@ -11943,7 +12992,7 @@ var _DatabaseStorage = class _DatabaseStorage {
|
|
|
11943
12992
|
});
|
|
11944
12993
|
}
|
|
11945
12994
|
async listSessions(projectIds) {
|
|
11946
|
-
await this.
|
|
12995
|
+
if (!await this.isHistoryAvailable()) return [];
|
|
11947
12996
|
let raw = [];
|
|
11948
12997
|
if (this.provider === "postgresql" || this.provider === "pgvector") {
|
|
11949
12998
|
const pool = await this.getPGPool();
|
|
@@ -11962,7 +13011,7 @@ var _DatabaseStorage = class _DatabaseStorage {
|
|
|
11962
13011
|
}
|
|
11963
13012
|
// ─── Feedback Operations ──────────────────────────────────────────────────
|
|
11964
13013
|
async saveFeedback(feedback) {
|
|
11965
|
-
await this.
|
|
13014
|
+
if (!await this.isFeedbackAvailable()) return;
|
|
11966
13015
|
const id = `fb_${Date.now()}_${Math.random().toString(36).slice(2, 6)}`;
|
|
11967
13016
|
const createdAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
11968
13017
|
if (this.provider === "postgresql" || this.provider === "pgvector") {
|
|
@@ -12012,7 +13061,7 @@ var _DatabaseStorage = class _DatabaseStorage {
|
|
|
12012
13061
|
}
|
|
12013
13062
|
}
|
|
12014
13063
|
async getFeedback(messageId, projectIds) {
|
|
12015
|
-
await this.
|
|
13064
|
+
if (!await this.isFeedbackAvailable()) return null;
|
|
12016
13065
|
let item = null;
|
|
12017
13066
|
if (this.provider === "postgresql" || this.provider === "pgvector") {
|
|
12018
13067
|
const pool = await this.getPGPool();
|
|
@@ -12048,7 +13097,7 @@ var _DatabaseStorage = class _DatabaseStorage {
|
|
|
12048
13097
|
return item;
|
|
12049
13098
|
}
|
|
12050
13099
|
async listFeedback(projectIds) {
|
|
12051
|
-
await this.
|
|
13100
|
+
if (!await this.isFeedbackAvailable()) return [];
|
|
12052
13101
|
let raw = [];
|
|
12053
13102
|
if (this.provider === "postgresql" || this.provider === "pgvector") {
|
|
12054
13103
|
const pool = await this.getPGPool();
|
|
@@ -12092,9 +13141,8 @@ var _DatabaseStorage = class _DatabaseStorage {
|
|
|
12092
13141
|
_DatabaseStorage.SUBSCRIPTION_CACHE_TTL_MS = 5 * 60 * 1e3;
|
|
12093
13142
|
var DatabaseStorage = _DatabaseStorage;
|
|
12094
13143
|
|
|
12095
|
-
// src/handlers/
|
|
12096
|
-
|
|
12097
|
-
import { createHash } from "crypto";
|
|
13144
|
+
// src/handlers/internal/auth.ts
|
|
13145
|
+
import { NextResponse } from "next/server";
|
|
12098
13146
|
async function checkAuth(req, onAuthorize) {
|
|
12099
13147
|
if (!onAuthorize) return null;
|
|
12100
13148
|
try {
|
|
@@ -12111,6 +13159,8 @@ async function checkAuth(req, onAuthorize) {
|
|
|
12111
13159
|
return NextResponse.json({ error: msg }, { status: 401 });
|
|
12112
13160
|
}
|
|
12113
13161
|
}
|
|
13162
|
+
|
|
13163
|
+
// src/handlers/internal/rateLimit.ts
|
|
12114
13164
|
var RateLimiter = class {
|
|
12115
13165
|
constructor(windowMs = 6e4, max = 30) {
|
|
12116
13166
|
this.hits = /* @__PURE__ */ new Map();
|
|
@@ -12172,6 +13222,9 @@ function checkRateLimit(req) {
|
|
|
12172
13222
|
}
|
|
12173
13223
|
return null;
|
|
12174
13224
|
}
|
|
13225
|
+
|
|
13226
|
+
// src/handlers/internal/license.ts
|
|
13227
|
+
import { NextResponse as NextResponse2 } from "next/server";
|
|
12175
13228
|
var FREE_TIER_NAMES = /* @__PURE__ */ new Set(["hobby", "free", "free_trial", "trial"]);
|
|
12176
13229
|
function isFreeTier(tier) {
|
|
12177
13230
|
if (!tier) return true;
|
|
@@ -12216,7 +13269,7 @@ async function enforceLicense(req, config, bodyLicenseKey) {
|
|
|
12216
13269
|
} catch (err) {
|
|
12217
13270
|
return {
|
|
12218
13271
|
ok: false,
|
|
12219
|
-
response:
|
|
13272
|
+
response: NextResponse2.json(
|
|
12220
13273
|
{ error: `License validation failed: ${err.message}` },
|
|
12221
13274
|
{ status: 401 }
|
|
12222
13275
|
)
|
|
@@ -12274,6 +13327,8 @@ function resolveLicensedNamespace(payload, requestedNamespace) {
|
|
|
12274
13327
|
const defaultNamespace = ((_a2 = payload.projectId) == null ? void 0 : _a2.startsWith("retrivora-")) ? payload.projectId : `retrivora-${payload.projectId || "default"}`;
|
|
12275
13328
|
return { ok: true, namespace: requested || defaultNamespace };
|
|
12276
13329
|
}
|
|
13330
|
+
|
|
13331
|
+
// src/handlers/internal/sanitize.ts
|
|
12277
13332
|
var MAX_MESSAGE_LENGTH = 8e3;
|
|
12278
13333
|
function sanitizeInput(raw) {
|
|
12279
13334
|
const stripped = raw.replace(/[\u0000-\u0008\u000B\u000C\u000E-\u001F\u007F-\u009F]/g, "");
|
|
@@ -12286,6 +13341,9 @@ function sanitizeInput(raw) {
|
|
|
12286
13341
|
}
|
|
12287
13342
|
return { ok: true, value: trimmed };
|
|
12288
13343
|
}
|
|
13344
|
+
|
|
13345
|
+
// src/handlers/internal/plugin.ts
|
|
13346
|
+
import { createHash } from "crypto";
|
|
12289
13347
|
var _pluginCache = /* @__PURE__ */ new Map();
|
|
12290
13348
|
var PLUGIN_CACHE_TTL_MS = 10 * 60 * 1e3;
|
|
12291
13349
|
function configHash(config) {
|
|
@@ -12300,6 +13358,9 @@ function getOrCreatePlugin(configOrPlugin) {
|
|
|
12300
13358
|
_pluginCache.set(key, { plugin, createdAt: Date.now() });
|
|
12301
13359
|
return plugin;
|
|
12302
13360
|
}
|
|
13361
|
+
|
|
13362
|
+
// src/handlers/internal/telemetry.ts
|
|
13363
|
+
init_version();
|
|
12303
13364
|
var TELEMETRY_ALLOWED_HOSTS = /* @__PURE__ */ new Set([
|
|
12304
13365
|
"retrivora.com",
|
|
12305
13366
|
"www.retrivora.com"
|
|
@@ -12320,8 +13381,20 @@ function sanitizeTelemetryUrl(raw) {
|
|
|
12320
13381
|
return null;
|
|
12321
13382
|
}
|
|
12322
13383
|
}
|
|
12323
|
-
|
|
12324
|
-
|
|
13384
|
+
function classifyError(message) {
|
|
13385
|
+
const m = (message || "").toLowerCase();
|
|
13386
|
+
if (/license|revoked|terminated|suspended/.test(m)) return { errorCode: "LICENSE_ERROR", errorType: "LICENSE_ERROR" };
|
|
13387
|
+
if (/rate.?limit|429/.test(m)) return { errorCode: "RATE_LIMIT_ERROR", errorType: "RATE_LIMIT_ERROR" };
|
|
13388
|
+
if (/unauthorized|401|403/.test(m)) return { errorCode: "AUTH_ERROR", errorType: "AUTH_ERROR" };
|
|
13389
|
+
if (/embedding|embed.?fail/.test(m)) return { errorCode: "EMBEDDING_ERROR", errorType: "EMBEDDING_ERROR" };
|
|
13390
|
+
if (/vector|pinecone|qdrant|pgvector|chroma|milvus|mongodb/.test(m)) return { errorCode: "VECTOR_DB_ERROR", errorType: "VECTOR_DB_ERROR" };
|
|
13391
|
+
if (/llm|chat|generation|openai|groq|anthropic|gemini|mistral/.test(m)) return { errorCode: "LLM_ERROR", errorType: "LLM_ERROR" };
|
|
13392
|
+
if (/fetch|network|timeout|econnrefused|socket/.test(m)) return { errorCode: "NETWORK_ERROR", errorType: "NETWORK_ERROR" };
|
|
13393
|
+
if (/stream/.test(m)) return { errorCode: "STREAMING_ERROR", errorType: "STREAMING_ERROR" };
|
|
13394
|
+
return { errorCode: "INTERNAL_ERROR", errorType: "INTERNAL_ERROR" };
|
|
13395
|
+
}
|
|
13396
|
+
async function reportTelemetry(req, plugin, action, status, details, trace, extra) {
|
|
13397
|
+
var _a2, _b, _c, _d, _e, _f, _g2, _h, _i, _j, _k, _l, _m, _n, _o, _p;
|
|
12325
13398
|
try {
|
|
12326
13399
|
const config = plugin.getConfig();
|
|
12327
13400
|
const licenseKey = config.licenseKey || process.env.RETRIVORA_LICENSE_KEY || process.env.RAG_LICENSE_KEY;
|
|
@@ -12332,13 +13405,14 @@ async function reportTelemetry(req, plugin, action, status, details, trace) {
|
|
|
12332
13405
|
if (!telemetryUrl) return;
|
|
12333
13406
|
const userAgent = req.headers.get("user-agent") || `Retrivora-SDK/${SDK_VERSION}`;
|
|
12334
13407
|
const projectId = config.projectId || "default";
|
|
12335
|
-
const model = (trace == null ? void 0 : trace.model) || ((_a2 = config.llm) == null ? void 0 : _a2.model) ||
|
|
12336
|
-
const provider = (trace == null ? void 0 : trace.provider) || ((
|
|
12337
|
-
const tokens = Number(((
|
|
12338
|
-
const costUsd = Number(((
|
|
12339
|
-
const latencyMs = Number(((
|
|
13408
|
+
const model = (trace == null ? void 0 : trace.model) || ((_a2 = config.llm) == null ? void 0 : _a2.model) || "llama-3.1-8b-instant";
|
|
13409
|
+
const provider = (trace == null ? void 0 : trace.provider) || ((_b = config.llm) == null ? void 0 : _b.provider) || "groq";
|
|
13410
|
+
const tokens = Number(((_c = trace == null ? void 0 : trace.tokens) == null ? void 0 : _c.totalTokens) || (trace == null ? void 0 : trace.totalTokens) || 0);
|
|
13411
|
+
const costUsd = Number(((_d = trace == null ? void 0 : trace.tokens) == null ? void 0 : _d.estimatedCostUsd) || (trace == null ? void 0 : trace.costUsd) || 0);
|
|
13412
|
+
const latencyMs = Number(((_e = trace == null ? void 0 : trace.latency) == null ? void 0 : _e.totalMs) || (trace == null ? void 0 : trace.latencyMs) || 0);
|
|
13413
|
+
const sanitizedTrace = trace ? __spreadProps(__spreadValues({}, trace), { systemPrompt: void 0, userPrompt: void 0 }) : void 0;
|
|
12340
13414
|
const payload = {
|
|
12341
|
-
trace,
|
|
13415
|
+
trace: sanitizedTrace,
|
|
12342
13416
|
projectId,
|
|
12343
13417
|
organization: process.env.RETRIVORA_ORGANIZATION || "default-org",
|
|
12344
13418
|
sdkVersion: SDK_VERSION,
|
|
@@ -12353,7 +13427,17 @@ async function reportTelemetry(req, plugin, action, status, details, trace) {
|
|
|
12353
13427
|
feature: action,
|
|
12354
13428
|
userAgent,
|
|
12355
13429
|
timestamp: (/* @__PURE__ */ new Date()).toISOString(),
|
|
12356
|
-
details
|
|
13430
|
+
details,
|
|
13431
|
+
// ── Structured observability fields ──────────────────────────────────
|
|
13432
|
+
requestId: (_f = trace == null ? void 0 : trace.requestId) != null ? _f : null,
|
|
13433
|
+
embedMs: (_h = (_g2 = trace == null ? void 0 : trace.latency) == null ? void 0 : _g2.embedMs) != null ? _h : null,
|
|
13434
|
+
retrieveMs: (_j = (_i = trace == null ? void 0 : trace.latency) == null ? void 0 : _i.retrieveMs) != null ? _j : null,
|
|
13435
|
+
generateMs: (_l = (_k = trace == null ? void 0 : trace.latency) == null ? void 0 : _k.generateMs) != null ? _l : null,
|
|
13436
|
+
ttftMs: (_m = extra == null ? void 0 : extra.ttftMs) != null ? _m : null,
|
|
13437
|
+
errorCode: (_n = extra == null ? void 0 : extra.errorCode) != null ? _n : null,
|
|
13438
|
+
errorType: (_o = extra == null ? void 0 : extra.errorType) != null ? _o : null,
|
|
13439
|
+
isStreaming: (_p = extra == null ? void 0 : extra.isStreaming) != null ? _p : false,
|
|
13440
|
+
environment: process.env.NODE_ENV || "development"
|
|
12357
13441
|
};
|
|
12358
13442
|
await fetch(telemetryUrl, {
|
|
12359
13443
|
method: "POST",
|
|
@@ -12369,43 +13453,8 @@ async function reportTelemetry(req, plugin, action, status, details, trace) {
|
|
|
12369
13453
|
} catch (e) {
|
|
12370
13454
|
}
|
|
12371
13455
|
}
|
|
12372
|
-
function sseFrame(payload) {
|
|
12373
|
-
return `data: ${JSON.stringify(payload)}
|
|
12374
|
-
|
|
12375
|
-
`;
|
|
12376
|
-
}
|
|
12377
|
-
function sseTextFrame(text) {
|
|
12378
|
-
return `data: ${JSON.stringify({ type: "text", text })}
|
|
12379
|
-
|
|
12380
|
-
`;
|
|
12381
|
-
}
|
|
12382
|
-
function sseMetaFrame(meta) {
|
|
12383
|
-
return `data: ${JSON.stringify(__spreadValues({ type: "metadata" }, meta != null ? meta : {}))}
|
|
12384
|
-
|
|
12385
|
-
`;
|
|
12386
|
-
}
|
|
12387
|
-
function sseUIFrame(uiTransformation) {
|
|
12388
|
-
return `data: ${JSON.stringify({ type: "ui_transformation", data: uiTransformation })}
|
|
12389
|
-
|
|
12390
|
-
`;
|
|
12391
|
-
}
|
|
12392
|
-
function sseObservabilityFrame(trace) {
|
|
12393
|
-
return `data: ${JSON.stringify({ type: "observability", data: trace })}
|
|
12394
|
-
|
|
12395
|
-
`;
|
|
12396
|
-
}
|
|
12397
|
-
function sseErrorFrame(message) {
|
|
12398
|
-
return `data: ${JSON.stringify({ type: "error", error: message })}
|
|
12399
13456
|
|
|
12400
|
-
|
|
12401
|
-
}
|
|
12402
|
-
var SSE_HEADERS = {
|
|
12403
|
-
"Content-Type": "text/event-stream; charset=utf-8",
|
|
12404
|
-
"Cache-Control": "no-cache, no-transform",
|
|
12405
|
-
Connection: "keep-alive",
|
|
12406
|
-
"X-Accel-Buffering": "no"
|
|
12407
|
-
// Disable Nginx buffering for streaming
|
|
12408
|
-
};
|
|
13457
|
+
// src/handlers/chatHandler.ts
|
|
12409
13458
|
function createChatHandler(configOrPlugin, options) {
|
|
12410
13459
|
const plugin = getOrCreatePlugin(configOrPlugin);
|
|
12411
13460
|
const storage = new DatabaseStorage(plugin.getConfig());
|
|
@@ -12457,7 +13506,7 @@ function createChatHandler(configOrPlugin, options) {
|
|
|
12457
13506
|
if (!namespaceCheck.ok) return namespaceCheck.response;
|
|
12458
13507
|
const sanitized = sanitizeInput(rawMessage != null ? rawMessage : "");
|
|
12459
13508
|
if (!sanitized.ok) {
|
|
12460
|
-
return
|
|
13509
|
+
return NextResponse3.json({ error: { code: "INVALID_INPUT", message: sanitized.error } }, { status: 400 });
|
|
12461
13510
|
}
|
|
12462
13511
|
const message = sanitized.value;
|
|
12463
13512
|
const userMsgId = userMessageId || `user_${Date.now()}_${Math.random().toString(36).slice(2, 6)}`;
|
|
@@ -12476,18 +13525,68 @@ function createChatHandler(configOrPlugin, options) {
|
|
|
12476
13525
|
uiTransformation: result.ui_transformation,
|
|
12477
13526
|
trace: result.trace
|
|
12478
13527
|
}).catch((err) => console.warn("[createChatHandler] Failed to save assistant message:", err));
|
|
12479
|
-
await reportTelemetry(
|
|
12480
|
-
|
|
13528
|
+
await reportTelemetry(
|
|
13529
|
+
req,
|
|
13530
|
+
plugin,
|
|
13531
|
+
"RAG_QUERY",
|
|
13532
|
+
"success",
|
|
13533
|
+
`Tokens: ${((_d = (_c = result.trace) == null ? void 0 : _c.tokens) == null ? void 0 : _d.totalTokens) || 0}, Latency: ${((_f = (_e = result.trace) == null ? void 0 : _e.latency) == null ? void 0 : _f.totalMs) || 0}ms`,
|
|
13534
|
+
result.trace,
|
|
13535
|
+
{ isStreaming: false }
|
|
13536
|
+
);
|
|
13537
|
+
return NextResponse3.json(__spreadProps(__spreadValues({}, result), {
|
|
12481
13538
|
messageId: assistantMsgId,
|
|
12482
13539
|
userMessageId: userMsgId
|
|
12483
13540
|
}));
|
|
12484
13541
|
} catch (err) {
|
|
12485
13542
|
const message = err instanceof Error ? err.message : "Internal server error";
|
|
12486
|
-
|
|
12487
|
-
|
|
13543
|
+
const { errorCode, errorType } = classifyError(message);
|
|
13544
|
+
await reportTelemetry(req, plugin, "RAG_QUERY", "error", message, void 0, { isStreaming: false, errorCode, errorType });
|
|
13545
|
+
return NextResponse3.json({ error: message }, { status: 500 });
|
|
12488
13546
|
}
|
|
12489
13547
|
};
|
|
12490
13548
|
}
|
|
13549
|
+
|
|
13550
|
+
// src/handlers/internal/sse.ts
|
|
13551
|
+
function sseFrame(payload) {
|
|
13552
|
+
return `data: ${JSON.stringify(payload)}
|
|
13553
|
+
|
|
13554
|
+
`;
|
|
13555
|
+
}
|
|
13556
|
+
function sseTextFrame(text) {
|
|
13557
|
+
return `data: ${JSON.stringify({ type: "text", text })}
|
|
13558
|
+
|
|
13559
|
+
`;
|
|
13560
|
+
}
|
|
13561
|
+
function sseMetaFrame(meta) {
|
|
13562
|
+
return `data: ${JSON.stringify(__spreadValues({ type: "metadata" }, meta != null ? meta : {}))}
|
|
13563
|
+
|
|
13564
|
+
`;
|
|
13565
|
+
}
|
|
13566
|
+
function sseUIFrame(uiTransformation) {
|
|
13567
|
+
return `data: ${JSON.stringify({ type: "ui_transformation", data: uiTransformation })}
|
|
13568
|
+
|
|
13569
|
+
`;
|
|
13570
|
+
}
|
|
13571
|
+
function sseObservabilityFrame(trace) {
|
|
13572
|
+
return `data: ${JSON.stringify({ type: "observability", data: trace })}
|
|
13573
|
+
|
|
13574
|
+
`;
|
|
13575
|
+
}
|
|
13576
|
+
function sseErrorFrame(message) {
|
|
13577
|
+
return `data: ${JSON.stringify({ type: "error", error: message })}
|
|
13578
|
+
|
|
13579
|
+
`;
|
|
13580
|
+
}
|
|
13581
|
+
var SSE_HEADERS = {
|
|
13582
|
+
"Content-Type": "text/event-stream; charset=utf-8",
|
|
13583
|
+
"Cache-Control": "no-cache, no-transform",
|
|
13584
|
+
Connection: "keep-alive",
|
|
13585
|
+
"X-Accel-Buffering": "no"
|
|
13586
|
+
// Disable Nginx buffering for streaming
|
|
13587
|
+
};
|
|
13588
|
+
|
|
13589
|
+
// src/handlers/streamHandler.ts
|
|
12491
13590
|
function createStreamHandler(configOrPlugin, options) {
|
|
12492
13591
|
const plugin = getOrCreatePlugin(configOrPlugin);
|
|
12493
13592
|
const storage = new DatabaseStorage(plugin.getConfig());
|
|
@@ -12562,6 +13661,7 @@ function createStreamHandler(configOrPlugin, options) {
|
|
|
12562
13661
|
const assistantMsgId = messageId || `assistant_${Date.now()}_${Math.random().toString(36).slice(2, 6)}`;
|
|
12563
13662
|
const encoder = new TextEncoder();
|
|
12564
13663
|
let isActive = true;
|
|
13664
|
+
let streamTelemetrySent = false;
|
|
12565
13665
|
const stream = new ReadableStream({
|
|
12566
13666
|
async start(controller) {
|
|
12567
13667
|
var _a3, _b2, _c, _d, _e;
|
|
@@ -12576,11 +13676,16 @@ function createStreamHandler(configOrPlugin, options) {
|
|
|
12576
13676
|
try {
|
|
12577
13677
|
const pipelineStream = plugin.chatStream(message, history, namespaceCheck.namespace);
|
|
12578
13678
|
let fullReply = "";
|
|
13679
|
+
let ttftMs;
|
|
13680
|
+
const handlerRequestStartMs = Date.now();
|
|
12579
13681
|
try {
|
|
12580
13682
|
for (var iter = __forAwait(pipelineStream), more, temp, error; more = !(temp = await iter.next()).done; more = false) {
|
|
12581
13683
|
const chunk = temp.value;
|
|
12582
13684
|
if (!isActive) break;
|
|
12583
13685
|
if (typeof chunk === "string") {
|
|
13686
|
+
if (ttftMs === void 0) {
|
|
13687
|
+
ttftMs = Date.now() - handlerRequestStartMs;
|
|
13688
|
+
}
|
|
12584
13689
|
fullReply += chunk;
|
|
12585
13690
|
enqueue(sseTextFrame(chunk));
|
|
12586
13691
|
} else if (chunk && typeof chunk === "object" && "type" in chunk && chunk.type === "thinking") {
|
|
@@ -12621,7 +13726,16 @@ function createStreamHandler(configOrPlugin, options) {
|
|
|
12621
13726
|
uiTransformation,
|
|
12622
13727
|
trace: responseChunk == null ? void 0 : responseChunk.trace
|
|
12623
13728
|
}).catch((err) => console.warn("[createStreamHandler] Failed to save assistant message:", err));
|
|
12624
|
-
|
|
13729
|
+
streamTelemetrySent = true;
|
|
13730
|
+
await reportTelemetry(
|
|
13731
|
+
req,
|
|
13732
|
+
plugin,
|
|
13733
|
+
"RAG_QUERY",
|
|
13734
|
+
"success",
|
|
13735
|
+
`Tokens: ${((_c = (_b2 = responseChunk == null ? void 0 : responseChunk.trace) == null ? void 0 : _b2.tokens) == null ? void 0 : _c.totalTokens) || 0}, Latency: ${((_e = (_d = responseChunk == null ? void 0 : responseChunk.trace) == null ? void 0 : _d.latency) == null ? void 0 : _e.totalMs) || 0}ms`,
|
|
13736
|
+
responseChunk == null ? void 0 : responseChunk.trace,
|
|
13737
|
+
{ ttftMs, isStreaming: true }
|
|
13738
|
+
);
|
|
12625
13739
|
}
|
|
12626
13740
|
}
|
|
12627
13741
|
} catch (temp) {
|
|
@@ -12637,11 +13751,21 @@ function createStreamHandler(configOrPlugin, options) {
|
|
|
12637
13751
|
} catch (streamError) {
|
|
12638
13752
|
if (isActive) {
|
|
12639
13753
|
let errorMessage = streamError instanceof Error ? streamError.message : String(streamError);
|
|
12640
|
-
if (errorMessage.includes("
|
|
13754
|
+
if (errorMessage.includes("LICENSE_REVOKED") || errorMessage.toLowerCase().includes("key has been revoked") || errorMessage.toLowerCase().includes("key has been terminated") || errorMessage.toLowerCase().includes("key has been suspended")) {
|
|
12641
13755
|
errorMessage = "Your Retrivora license key has been terminated, suspended, or revoked. Access denied.";
|
|
12642
13756
|
}
|
|
12643
13757
|
console.error("[createStreamHandler] Stream error:", streamError);
|
|
12644
|
-
|
|
13758
|
+
const { errorCode, errorType } = classifyError(errorMessage);
|
|
13759
|
+
streamTelemetrySent = true;
|
|
13760
|
+
await reportTelemetry(
|
|
13761
|
+
req,
|
|
13762
|
+
plugin,
|
|
13763
|
+
"RAG_QUERY",
|
|
13764
|
+
"error",
|
|
13765
|
+
errorMessage,
|
|
13766
|
+
void 0,
|
|
13767
|
+
{ isStreaming: true, errorCode, errorType }
|
|
13768
|
+
);
|
|
12645
13769
|
try {
|
|
12646
13770
|
enqueue(sseErrorFrame(errorMessage));
|
|
12647
13771
|
} catch (e) {
|
|
@@ -12650,6 +13774,10 @@ function createStreamHandler(configOrPlugin, options) {
|
|
|
12650
13774
|
} finally {
|
|
12651
13775
|
if (isActive) {
|
|
12652
13776
|
isActive = false;
|
|
13777
|
+
try {
|
|
13778
|
+
enqueue(sseFrame({ type: "done" }));
|
|
13779
|
+
} catch (e) {
|
|
13780
|
+
}
|
|
12653
13781
|
try {
|
|
12654
13782
|
controller.close();
|
|
12655
13783
|
} catch (e) {
|
|
@@ -12659,11 +13787,26 @@ function createStreamHandler(configOrPlugin, options) {
|
|
|
12659
13787
|
},
|
|
12660
13788
|
cancel(reason) {
|
|
12661
13789
|
isActive = false;
|
|
13790
|
+
if (!streamTelemetrySent) {
|
|
13791
|
+
reportTelemetry(
|
|
13792
|
+
req,
|
|
13793
|
+
plugin,
|
|
13794
|
+
"RAG_QUERY_ABORTED",
|
|
13795
|
+
"error",
|
|
13796
|
+
"Client disconnected mid-stream",
|
|
13797
|
+
void 0,
|
|
13798
|
+
{ isStreaming: true, errorCode: "CLIENT_DISCONNECT", errorType: "STREAMING_ERROR" }
|
|
13799
|
+
).catch(() => {
|
|
13800
|
+
});
|
|
13801
|
+
}
|
|
12662
13802
|
}
|
|
12663
13803
|
});
|
|
12664
13804
|
return new Response(stream, { headers: __spreadProps(__spreadValues({}, SSE_HEADERS), { "x-message-id": assistantMsgId, "x-user-message-id": userMsgId }) });
|
|
12665
13805
|
};
|
|
12666
13806
|
}
|
|
13807
|
+
|
|
13808
|
+
// src/handlers/ingestHandler.ts
|
|
13809
|
+
import { NextResponse as NextResponse4 } from "next/server";
|
|
12667
13810
|
function createIngestHandler(configOrPlugin, options) {
|
|
12668
13811
|
const plugin = getOrCreatePlugin(configOrPlugin);
|
|
12669
13812
|
const onAuthorize = options == null ? void 0 : options.onAuthorize;
|
|
@@ -12677,7 +13820,7 @@ function createIngestHandler(configOrPlugin, options) {
|
|
|
12677
13820
|
if (!licenseCheck.ok) return licenseCheck.response;
|
|
12678
13821
|
const { documents, namespace } = body;
|
|
12679
13822
|
if (!Array.isArray(documents) || documents.length === 0) {
|
|
12680
|
-
return
|
|
13823
|
+
return NextResponse4.json({ error: "documents array is required" }, { status: 400 });
|
|
12681
13824
|
}
|
|
12682
13825
|
const namespaceCheck = resolveLicensedNamespace(licenseCheck.payload, namespace);
|
|
12683
13826
|
if (!namespaceCheck.ok) return namespaceCheck.response;
|
|
@@ -12704,14 +13847,17 @@ function createIngestHandler(configOrPlugin, options) {
|
|
|
12704
13847
|
}
|
|
12705
13848
|
const results = await plugin.ingest(documents, namespaceCheck.namespace);
|
|
12706
13849
|
await reportTelemetry(req, plugin, "DOCUMENT_INGESTION", "success", `Ingested ${documents.length} document(s)`);
|
|
12707
|
-
return
|
|
13850
|
+
return NextResponse4.json({ results });
|
|
12708
13851
|
} catch (err) {
|
|
12709
13852
|
const message = err instanceof Error ? err.message : "Internal server error";
|
|
12710
13853
|
await reportTelemetry(req, plugin, "DOCUMENT_INGESTION", "error", message);
|
|
12711
|
-
return
|
|
13854
|
+
return NextResponse4.json({ error: message }, { status: 500 });
|
|
12712
13855
|
}
|
|
12713
13856
|
};
|
|
12714
13857
|
}
|
|
13858
|
+
|
|
13859
|
+
// src/handlers/healthHandler.ts
|
|
13860
|
+
import { NextResponse as NextResponse5 } from "next/server";
|
|
12715
13861
|
function createHealthHandler(configOrPlugin, options) {
|
|
12716
13862
|
const plugin = getOrCreatePlugin(configOrPlugin);
|
|
12717
13863
|
const onAuthorize = options == null ? void 0 : options.onAuthorize;
|
|
@@ -12723,17 +13869,21 @@ function createHealthHandler(configOrPlugin, options) {
|
|
|
12723
13869
|
try {
|
|
12724
13870
|
const health = await plugin.checkHealth();
|
|
12725
13871
|
const status = health.allHealthy ? "ok" : "degraded";
|
|
12726
|
-
return
|
|
13872
|
+
return NextResponse5.json(__spreadProps(__spreadValues({
|
|
12727
13873
|
status
|
|
12728
13874
|
}, health), {
|
|
12729
13875
|
timestamp: (/* @__PURE__ */ new Date()).toISOString()
|
|
12730
13876
|
}));
|
|
12731
13877
|
} catch (err) {
|
|
12732
13878
|
const message = err instanceof Error ? err.message : "Unknown error";
|
|
12733
|
-
return
|
|
13879
|
+
return NextResponse5.json({ status: "error", error: message }, { status: 500 });
|
|
12734
13880
|
}
|
|
12735
13881
|
};
|
|
12736
13882
|
}
|
|
13883
|
+
|
|
13884
|
+
// src/handlers/licenseHandler.ts
|
|
13885
|
+
import { NextResponse as NextResponse6 } from "next/server";
|
|
13886
|
+
init_version();
|
|
12737
13887
|
function createLicenseHandler(configOrPlugin, options) {
|
|
12738
13888
|
const plugin = getOrCreatePlugin(configOrPlugin);
|
|
12739
13889
|
const onAuthorize = options == null ? void 0 : options.onAuthorize;
|
|
@@ -12759,7 +13909,7 @@ function createLicenseHandler(configOrPlugin, options) {
|
|
|
12759
13909
|
const licenseKey = (rawKey || "").trim().replace(/^["']|["']$/g, "").trim();
|
|
12760
13910
|
const projectId = (body == null ? void 0 : body.projectId) || config.projectId || "";
|
|
12761
13911
|
const payload = await LicenseVerifier.verifyAsync(licenseKey, projectId);
|
|
12762
|
-
return
|
|
13912
|
+
return NextResponse6.json({
|
|
12763
13913
|
valid: true,
|
|
12764
13914
|
licenseStatus: "ACTIVE",
|
|
12765
13915
|
minimumSupportedVersion: "2.1.0",
|
|
@@ -12769,7 +13919,7 @@ function createLicenseHandler(configOrPlugin, options) {
|
|
|
12769
13919
|
message: "License key is active and valid."
|
|
12770
13920
|
});
|
|
12771
13921
|
} catch (err) {
|
|
12772
|
-
return
|
|
13922
|
+
return NextResponse6.json(
|
|
12773
13923
|
{
|
|
12774
13924
|
valid: false,
|
|
12775
13925
|
licenseStatus: "REVOKED",
|
|
@@ -12784,6 +13934,9 @@ function createLicenseHandler(configOrPlugin, options) {
|
|
|
12784
13934
|
}
|
|
12785
13935
|
};
|
|
12786
13936
|
}
|
|
13937
|
+
|
|
13938
|
+
// src/handlers/uploadHandler.ts
|
|
13939
|
+
import { NextResponse as NextResponse7 } from "next/server";
|
|
12787
13940
|
function createUploadHandler(configOrPlugin, options) {
|
|
12788
13941
|
const plugin = getOrCreatePlugin(configOrPlugin);
|
|
12789
13942
|
const onAuthorize = options == null ? void 0 : options.onAuthorize;
|
|
@@ -12802,7 +13955,7 @@ function createUploadHandler(configOrPlugin, options) {
|
|
|
12802
13955
|
const namespaceCheck = resolveLicensedNamespace(licenseCheck.payload, namespace);
|
|
12803
13956
|
if (!namespaceCheck.ok) return namespaceCheck.response;
|
|
12804
13957
|
if (!files || files.length === 0) {
|
|
12805
|
-
return
|
|
13958
|
+
return NextResponse7.json({ error: "No files provided" }, { status: 400 });
|
|
12806
13959
|
}
|
|
12807
13960
|
if (isFreeTier(licenseCheck.payload.tier)) {
|
|
12808
13961
|
const totalUploadBytes = files.reduce((sum, f) => sum + (f.size || 0), 0);
|
|
@@ -12824,137 +13977,174 @@ function createUploadHandler(configOrPlugin, options) {
|
|
|
12824
13977
|
);
|
|
12825
13978
|
}
|
|
12826
13979
|
}
|
|
12827
|
-
const
|
|
12828
|
-
|
|
12829
|
-
|
|
12830
|
-
|
|
12831
|
-
|
|
12832
|
-
const
|
|
12833
|
-
|
|
12834
|
-
|
|
12835
|
-
|
|
12836
|
-
|
|
12837
|
-
|
|
12838
|
-
|
|
12839
|
-
|
|
12840
|
-
|
|
12841
|
-
const
|
|
12842
|
-
|
|
12843
|
-
|
|
12844
|
-
|
|
12845
|
-
|
|
12846
|
-
rowData[key]
|
|
13980
|
+
const fileDocsArrays = await Promise.all(
|
|
13981
|
+
files.map(async (file) => {
|
|
13982
|
+
const fileDocs = [];
|
|
13983
|
+
const buffer = Buffer.isBuffer(file) ? file : Buffer.from(await file.arrayBuffer());
|
|
13984
|
+
const isExcel = file.name.toLowerCase().endsWith(".xlsx") || file.name.toLowerCase().endsWith(".xls") || file.type.includes("spreadsheet") || file.type.includes("excel");
|
|
13985
|
+
const SPREADSHEET_CHUNK_SIZE = 30;
|
|
13986
|
+
if (file.name.toLowerCase().endsWith(".csv") || file.name.toLowerCase().endsWith(".tsv") || file.type === "text/csv" || file.type.includes("tab-separated")) {
|
|
13987
|
+
const text = buffer.toString("utf-8");
|
|
13988
|
+
const Papa = await import("papaparse");
|
|
13989
|
+
const parsed = Papa.default.parse(text, { header: true, skipEmptyLines: true });
|
|
13990
|
+
if (parsed.data && parsed.data.length > 0) {
|
|
13991
|
+
const csvCols = Object.keys(parsed.data[0]);
|
|
13992
|
+
const rawRows = [];
|
|
13993
|
+
let lastRowData = null;
|
|
13994
|
+
for (const row of parsed.data) {
|
|
13995
|
+
const rowData = __spreadValues({}, row);
|
|
13996
|
+
const groupingKey = Object.keys(rowData)[0];
|
|
13997
|
+
if (lastRowData && groupingKey && rowData[groupingKey] && rowData[groupingKey] === lastRowData[groupingKey]) {
|
|
13998
|
+
for (const key of Object.keys(rowData)) {
|
|
13999
|
+
if (!rowData[key] && lastRowData[key]) {
|
|
14000
|
+
rowData[key] = lastRowData[key];
|
|
14001
|
+
}
|
|
12847
14002
|
}
|
|
12848
14003
|
}
|
|
14004
|
+
lastRowData = __spreadValues({}, rowData);
|
|
14005
|
+
rawRows.push(rowData);
|
|
12849
14006
|
}
|
|
12850
|
-
|
|
12851
|
-
|
|
12852
|
-
|
|
12853
|
-
|
|
12854
|
-
|
|
14007
|
+
for (let i = 0; i < rawRows.length; i += SPREADSHEET_CHUNK_SIZE) {
|
|
14008
|
+
const slice = rawRows.slice(i, i + SPREADSHEET_CHUNK_SIZE);
|
|
14009
|
+
const startRow = i + 1;
|
|
14010
|
+
const endRow = i + slice.length;
|
|
14011
|
+
const chunkIndex = Math.floor(i / SPREADSHEET_CHUNK_SIZE) + 1;
|
|
14012
|
+
const lines = [
|
|
14013
|
+
`File: ${file.name} (Rows ${startRow}-${endRow} of ${rawRows.length})`,
|
|
14014
|
+
`| ${csvCols.join(" | ")} |`,
|
|
14015
|
+
`| ${csvCols.map(() => "---").join(" | ")} |`
|
|
14016
|
+
];
|
|
14017
|
+
for (const r of slice) {
|
|
14018
|
+
const rowValues = csvCols.map((col) => {
|
|
14019
|
+
const val = r[col];
|
|
14020
|
+
if (val === void 0 || val === null) return "";
|
|
14021
|
+
return String(val).replace(/\|/g, "\\|").replace(/\r?\n/g, " ").trim();
|
|
14022
|
+
});
|
|
14023
|
+
lines.push(`| ${rowValues.join(" | ")} |`);
|
|
12855
14024
|
}
|
|
14025
|
+
fileDocs.push({
|
|
14026
|
+
docId: `${file.name}-chunk-${chunkIndex}`,
|
|
14027
|
+
content: lines.join("\n"),
|
|
14028
|
+
metadata: __spreadValues({
|
|
14029
|
+
fileName: file.name,
|
|
14030
|
+
fileSize: file.size,
|
|
14031
|
+
fileType: file.type || "text/csv",
|
|
14032
|
+
uploadedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
14033
|
+
startRow,
|
|
14034
|
+
endRow,
|
|
14035
|
+
rowCount: slice.length,
|
|
14036
|
+
headers: csvCols
|
|
14037
|
+
}, dimension ? { dimension } : {})
|
|
14038
|
+
});
|
|
12856
14039
|
}
|
|
12857
|
-
documents.push({
|
|
12858
|
-
docId: `${file.name}-row-${i}`,
|
|
12859
|
-
content: contentParts.join(", "),
|
|
12860
|
-
metadata: __spreadValues(__spreadProps(__spreadValues({
|
|
12861
|
-
fileName: file.name,
|
|
12862
|
-
fileSize: file.size,
|
|
12863
|
-
fileType: file.type || "text/csv",
|
|
12864
|
-
uploadedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
12865
|
-
}, dimension ? { dimension } : {}), {
|
|
12866
|
-
csvHeaders: csvCols
|
|
12867
|
-
}), rowData)
|
|
12868
|
-
});
|
|
12869
14040
|
}
|
|
12870
|
-
}
|
|
12871
|
-
|
|
12872
|
-
|
|
12873
|
-
|
|
12874
|
-
|
|
12875
|
-
|
|
12876
|
-
|
|
12877
|
-
|
|
12878
|
-
|
|
12879
|
-
|
|
12880
|
-
|
|
12881
|
-
|
|
12882
|
-
|
|
12883
|
-
|
|
12884
|
-
|
|
12885
|
-
|
|
12886
|
-
|
|
12887
|
-
|
|
12888
|
-
|
|
12889
|
-
|
|
12890
|
-
|
|
12891
|
-
|
|
12892
|
-
|
|
12893
|
-
|
|
12894
|
-
|
|
12895
|
-
|
|
12896
|
-
|
|
12897
|
-
|
|
12898
|
-
|
|
12899
|
-
|
|
12900
|
-
|
|
12901
|
-
|
|
12902
|
-
|
|
12903
|
-
|
|
12904
|
-
|
|
12905
|
-
|
|
12906
|
-
|
|
12907
|
-
|
|
12908
|
-
uploadedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
12909
|
-
}, dimension ? { dimension } : {}), {
|
|
12910
|
-
headers
|
|
12911
|
-
}), rowData)
|
|
14041
|
+
} else if (isExcel) {
|
|
14042
|
+
let excelLoaded = false;
|
|
14043
|
+
try {
|
|
14044
|
+
const XLSX = await import("xlsx");
|
|
14045
|
+
const workbook = XLSX.read(buffer, { type: "buffer", cellDates: true });
|
|
14046
|
+
for (const sheetName of workbook.SheetNames) {
|
|
14047
|
+
const sheet = workbook.Sheets[sheetName];
|
|
14048
|
+
if (!sheet) continue;
|
|
14049
|
+
const rawGrid = XLSX.utils.sheet_to_json(sheet, { header: 1, defval: "", blankrows: false });
|
|
14050
|
+
if (!rawGrid || rawGrid.length === 0) continue;
|
|
14051
|
+
let headerRowIndex = 0;
|
|
14052
|
+
while (headerRowIndex < rawGrid.length && (!rawGrid[headerRowIndex] || !rawGrid[headerRowIndex].some((c) => String(c != null ? c : "").trim().length > 0))) {
|
|
14053
|
+
headerRowIndex++;
|
|
14054
|
+
}
|
|
14055
|
+
if (headerRowIndex >= rawGrid.length) continue;
|
|
14056
|
+
const rawHeaders = rawGrid[headerRowIndex].map((h, idx) => {
|
|
14057
|
+
const s = String(h != null ? h : "").trim();
|
|
14058
|
+
return s || `Column_${idx + 1}`;
|
|
14059
|
+
});
|
|
14060
|
+
const dataRows = rawGrid.slice(headerRowIndex + 1).filter(
|
|
14061
|
+
(row) => row && row.some((c) => String(c != null ? c : "").trim().length > 0)
|
|
14062
|
+
);
|
|
14063
|
+
if (dataRows.length === 0) continue;
|
|
14064
|
+
for (let i = 0; i < dataRows.length; i += SPREADSHEET_CHUNK_SIZE) {
|
|
14065
|
+
const slice = dataRows.slice(i, i + SPREADSHEET_CHUNK_SIZE);
|
|
14066
|
+
const startRow = headerRowIndex + 1 + i + 1;
|
|
14067
|
+
const endRow = startRow + slice.length - 1;
|
|
14068
|
+
const chunkIndex = Math.floor(i / SPREADSHEET_CHUNK_SIZE) + 1;
|
|
14069
|
+
const lines = [
|
|
14070
|
+
`File: ${file.name} | Sheet: ${sheetName} (Rows ${startRow}-${endRow} of ${dataRows.length})`,
|
|
14071
|
+
`| ${rawHeaders.join(" | ")} |`,
|
|
14072
|
+
`| ${rawHeaders.map(() => "---").join(" | ")} |`
|
|
14073
|
+
];
|
|
14074
|
+
for (const r of slice) {
|
|
14075
|
+
const rowValues = rawHeaders.map((_, colIdx) => {
|
|
14076
|
+
const val = r[colIdx];
|
|
14077
|
+
if (val === void 0 || val === null) return "";
|
|
14078
|
+
return String(val).replace(/\|/g, "\\|").replace(/\r?\n/g, " ").trim();
|
|
12912
14079
|
});
|
|
14080
|
+
lines.push(`| ${rowValues.join(" | ")} |`);
|
|
12913
14081
|
}
|
|
14082
|
+
fileDocs.push({
|
|
14083
|
+
docId: `${file.name}-${sheetName}-chunk-${chunkIndex}`,
|
|
14084
|
+
content: lines.join("\n"),
|
|
14085
|
+
metadata: __spreadValues({
|
|
14086
|
+
fileName: file.name,
|
|
14087
|
+
sheetName,
|
|
14088
|
+
fileSize: file.size,
|
|
14089
|
+
fileType: file.type || "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
|
|
14090
|
+
uploadedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
14091
|
+
startRow,
|
|
14092
|
+
endRow,
|
|
14093
|
+
rowCount: slice.length,
|
|
14094
|
+
headers: rawHeaders
|
|
14095
|
+
}, dimension ? { dimension } : {})
|
|
14096
|
+
});
|
|
14097
|
+
}
|
|
14098
|
+
}
|
|
14099
|
+
excelLoaded = fileDocs.length > 0;
|
|
14100
|
+
} catch (xlsxErr) {
|
|
14101
|
+
excelLoaded = false;
|
|
14102
|
+
}
|
|
14103
|
+
if (!excelLoaded) {
|
|
14104
|
+
const content = await DocumentParser.parse(buffer, file.name, file.type);
|
|
14105
|
+
if (content && content.trim()) {
|
|
14106
|
+
fileDocs.push({
|
|
14107
|
+
docId: file.name,
|
|
14108
|
+
content,
|
|
14109
|
+
metadata: __spreadValues({
|
|
14110
|
+
fileName: file.name,
|
|
14111
|
+
fileSize: file.size,
|
|
14112
|
+
fileType: file.type || "application/vnd.ms-excel",
|
|
14113
|
+
uploadedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
14114
|
+
}, dimension ? { dimension } : {})
|
|
12914
14115
|
});
|
|
12915
|
-
}
|
|
12916
|
-
excelLoaded = documents.length > 0;
|
|
14116
|
+
}
|
|
12917
14117
|
}
|
|
12918
|
-
}
|
|
12919
|
-
excelLoaded = false;
|
|
12920
|
-
}
|
|
12921
|
-
if (!excelLoaded) {
|
|
14118
|
+
} else {
|
|
12922
14119
|
const content = await DocumentParser.parse(buffer, file.name, file.type);
|
|
12923
|
-
|
|
14120
|
+
fileDocs.push({
|
|
12924
14121
|
docId: file.name,
|
|
12925
14122
|
content,
|
|
12926
14123
|
metadata: __spreadValues({
|
|
12927
14124
|
fileName: file.name,
|
|
12928
14125
|
fileSize: file.size,
|
|
12929
|
-
fileType: file.type
|
|
14126
|
+
fileType: file.type,
|
|
12930
14127
|
uploadedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
12931
14128
|
}, dimension ? { dimension } : {})
|
|
12932
14129
|
});
|
|
12933
14130
|
}
|
|
12934
|
-
|
|
12935
|
-
|
|
12936
|
-
|
|
12937
|
-
|
|
12938
|
-
content,
|
|
12939
|
-
metadata: __spreadValues({
|
|
12940
|
-
fileName: file.name,
|
|
12941
|
-
fileSize: file.size,
|
|
12942
|
-
fileType: file.type,
|
|
12943
|
-
uploadedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
12944
|
-
}, dimension ? { dimension } : {})
|
|
12945
|
-
});
|
|
12946
|
-
}
|
|
12947
|
-
}
|
|
14131
|
+
return fileDocs;
|
|
14132
|
+
})
|
|
14133
|
+
);
|
|
14134
|
+
const documents = fileDocsArrays.flat();
|
|
12948
14135
|
const results = await plugin.ingest(documents, namespaceCheck.namespace);
|
|
12949
14136
|
await reportTelemetry(req, plugin, "DOCUMENT_INGESTION", "success", `Uploaded & ingested ${files.length} file(s)`);
|
|
12950
|
-
return
|
|
14137
|
+
return NextResponse7.json({ message: "Upload successful", results });
|
|
12951
14138
|
} catch (err) {
|
|
12952
14139
|
const message = err instanceof Error ? err.message : "Upload failed";
|
|
12953
14140
|
await reportTelemetry(req, plugin, "DOCUMENT_INGESTION", "error", message);
|
|
12954
|
-
return
|
|
14141
|
+
return NextResponse7.json({ error: message }, { status: 500 });
|
|
12955
14142
|
}
|
|
12956
14143
|
};
|
|
12957
14144
|
}
|
|
14145
|
+
|
|
14146
|
+
// src/handlers/suggestionsHandler.ts
|
|
14147
|
+
import { NextResponse as NextResponse8 } from "next/server";
|
|
12958
14148
|
function createSuggestionsHandler(configOrPlugin, options) {
|
|
12959
14149
|
const plugin = getOrCreatePlugin(configOrPlugin);
|
|
12960
14150
|
const onAuthorize = options == null ? void 0 : options.onAuthorize;
|
|
@@ -12985,20 +14175,23 @@ function createSuggestionsHandler(configOrPlugin, options) {
|
|
|
12985
14175
|
namespace = url.searchParams.get("namespace") || void 0;
|
|
12986
14176
|
}
|
|
12987
14177
|
if (!query || typeof query !== "string" || !query.trim()) {
|
|
12988
|
-
return
|
|
14178
|
+
return NextResponse8.json({ suggestions: [] });
|
|
12989
14179
|
}
|
|
12990
14180
|
const namespaceCheck = resolveLicensedNamespace(effectiveLicensePayload, namespace);
|
|
12991
14181
|
if (!namespaceCheck.ok) return namespaceCheck.response;
|
|
12992
14182
|
const suggestions = await plugin.getSuggestions(query, namespaceCheck.namespace);
|
|
12993
14183
|
await reportTelemetry(req, plugin, "AUTO_SUGGESTIONS", "success", `Fetched ${suggestions.length} suggestions for: ${query.slice(0, 30)}`);
|
|
12994
|
-
return
|
|
14184
|
+
return NextResponse8.json({ suggestions });
|
|
12995
14185
|
} catch (err) {
|
|
12996
14186
|
const message = err instanceof Error ? err.message : "Internal server error";
|
|
12997
14187
|
await reportTelemetry(req, plugin, "AUTO_SUGGESTIONS", "error", message);
|
|
12998
|
-
return
|
|
14188
|
+
return NextResponse8.json({ error: message }, { status: 500 });
|
|
12999
14189
|
}
|
|
13000
14190
|
};
|
|
13001
14191
|
}
|
|
14192
|
+
|
|
14193
|
+
// src/handlers/historyHandler.ts
|
|
14194
|
+
import { NextResponse as NextResponse9 } from "next/server";
|
|
13002
14195
|
function createHistoryHandler(configOrPlugin, options) {
|
|
13003
14196
|
const plugin = getOrCreatePlugin(configOrPlugin);
|
|
13004
14197
|
const storage = new DatabaseStorage(plugin.getConfig());
|
|
@@ -13013,25 +14206,28 @@ function createHistoryHandler(configOrPlugin, options) {
|
|
|
13013
14206
|
if (req.method === "GET") {
|
|
13014
14207
|
try {
|
|
13015
14208
|
const history = await storage.getHistory(sessionId, scope);
|
|
13016
|
-
return
|
|
14209
|
+
return NextResponse9.json({ history });
|
|
13017
14210
|
} catch (err) {
|
|
13018
14211
|
const message = err instanceof Error ? err.message : "Failed to fetch history";
|
|
13019
|
-
return
|
|
14212
|
+
return NextResponse9.json({ error: message }, { status: 500 });
|
|
13020
14213
|
}
|
|
13021
14214
|
} else if (req.method === "POST") {
|
|
13022
14215
|
try {
|
|
13023
14216
|
const body = await req.json().catch(() => ({}));
|
|
13024
14217
|
const sid = body.sessionId || sessionId;
|
|
13025
14218
|
await storage.clearHistory(sid, scope);
|
|
13026
|
-
return
|
|
14219
|
+
return NextResponse9.json({ success: true });
|
|
13027
14220
|
} catch (err) {
|
|
13028
14221
|
const message = err instanceof Error ? err.message : "Failed to clear history";
|
|
13029
|
-
return
|
|
14222
|
+
return NextResponse9.json({ error: message }, { status: 500 });
|
|
13030
14223
|
}
|
|
13031
14224
|
}
|
|
13032
|
-
return
|
|
14225
|
+
return NextResponse9.json({ error: "Method Not Allowed" }, { status: 405 });
|
|
13033
14226
|
};
|
|
13034
14227
|
}
|
|
14228
|
+
|
|
14229
|
+
// src/handlers/feedbackHandler.ts
|
|
14230
|
+
import { NextResponse as NextResponse10 } from "next/server";
|
|
13035
14231
|
function createFeedbackHandler(configOrPlugin, options) {
|
|
13036
14232
|
const plugin = getOrCreatePlugin(configOrPlugin);
|
|
13037
14233
|
const storage = new DatabaseStorage(plugin.getConfig());
|
|
@@ -13047,45 +14243,48 @@ function createFeedbackHandler(configOrPlugin, options) {
|
|
|
13047
14243
|
const messageId = url.searchParams.get("messageId");
|
|
13048
14244
|
if (!messageId) {
|
|
13049
14245
|
const feedbackList = await storage.listFeedback(scope);
|
|
13050
|
-
return
|
|
14246
|
+
return NextResponse10.json({ feedback: feedbackList });
|
|
13051
14247
|
}
|
|
13052
14248
|
const feedback = await storage.getFeedback(messageId, scope);
|
|
13053
|
-
return
|
|
14249
|
+
return NextResponse10.json({ feedback });
|
|
13054
14250
|
} catch (err) {
|
|
13055
14251
|
const message = err instanceof Error ? err.message : "Failed to fetch feedback";
|
|
13056
|
-
return
|
|
14252
|
+
return NextResponse10.json({ error: message }, { status: 500 });
|
|
13057
14253
|
}
|
|
13058
14254
|
} else if (req.method === "POST") {
|
|
13059
14255
|
try {
|
|
13060
14256
|
const body = await req.json();
|
|
13061
14257
|
const { messageId, sessionId = "default", rating, comment } = body;
|
|
13062
14258
|
if (!messageId) {
|
|
13063
|
-
return
|
|
14259
|
+
return NextResponse10.json({ error: "messageId is required" }, { status: 400 });
|
|
13064
14260
|
}
|
|
13065
14261
|
if (!rating) {
|
|
13066
|
-
return
|
|
14262
|
+
return NextResponse10.json({ error: "rating is required" }, { status: 400 });
|
|
13067
14263
|
}
|
|
13068
14264
|
if (scope && scope.length > 0) {
|
|
13069
14265
|
const matches = scope.some(
|
|
13070
14266
|
(pid) => pid && (sessionId === pid || sessionId.startsWith(pid) || sessionId.includes(pid))
|
|
13071
14267
|
);
|
|
13072
14268
|
if (!matches) {
|
|
13073
|
-
return
|
|
14269
|
+
return NextResponse10.json(
|
|
13074
14270
|
{ error: "sessionId is outside the authorized project scope" },
|
|
13075
14271
|
{ status: 403 }
|
|
13076
14272
|
);
|
|
13077
14273
|
}
|
|
13078
14274
|
}
|
|
13079
14275
|
await storage.saveFeedback({ messageId, sessionId, rating, comment });
|
|
13080
|
-
return
|
|
14276
|
+
return NextResponse10.json({ success: true });
|
|
13081
14277
|
} catch (err) {
|
|
13082
14278
|
const message = err instanceof Error ? err.message : "Failed to save feedback";
|
|
13083
|
-
return
|
|
14279
|
+
return NextResponse10.json({ error: message }, { status: 500 });
|
|
13084
14280
|
}
|
|
13085
14281
|
}
|
|
13086
|
-
return
|
|
14282
|
+
return NextResponse10.json({ error: "Method Not Allowed" }, { status: 405 });
|
|
13087
14283
|
};
|
|
13088
14284
|
}
|
|
14285
|
+
|
|
14286
|
+
// src/handlers/sessionsHandler.ts
|
|
14287
|
+
import { NextResponse as NextResponse11 } from "next/server";
|
|
13089
14288
|
function createSessionsHandler(configOrPlugin, options) {
|
|
13090
14289
|
const plugin = getOrCreatePlugin(configOrPlugin);
|
|
13091
14290
|
const storage = new DatabaseStorage(plugin.getConfig());
|
|
@@ -13099,13 +14298,16 @@ function createSessionsHandler(configOrPlugin, options) {
|
|
|
13099
14298
|
try {
|
|
13100
14299
|
const scope = req && onResolveScope ? await onResolveScope(req) : void 0;
|
|
13101
14300
|
const sessions = await storage.listSessions(scope);
|
|
13102
|
-
return
|
|
14301
|
+
return NextResponse11.json({ sessions });
|
|
13103
14302
|
} catch (err) {
|
|
13104
14303
|
const message = err instanceof Error ? err.message : "Failed to list sessions";
|
|
13105
|
-
return
|
|
14304
|
+
return NextResponse11.json({ error: message }, { status: 500 });
|
|
13106
14305
|
}
|
|
13107
14306
|
};
|
|
13108
14307
|
}
|
|
14308
|
+
|
|
14309
|
+
// src/handlers/ragHandler.ts
|
|
14310
|
+
import { NextResponse as NextResponse12 } from "next/server";
|
|
13109
14311
|
function createRagHandler(configOrPlugin, options) {
|
|
13110
14312
|
const plugin = getOrCreatePlugin(configOrPlugin);
|
|
13111
14313
|
const onAuthorize = options == null ? void 0 : options.onAuthorize;
|
|
@@ -13141,7 +14343,7 @@ function createRagHandler(configOrPlugin, options) {
|
|
|
13141
14343
|
case "v1/license/validate":
|
|
13142
14344
|
return licenseHandler(req);
|
|
13143
14345
|
default:
|
|
13144
|
-
return
|
|
14346
|
+
return NextResponse12.json({ error: `Not Found: POST segment "${segment}" not supported.` }, { status: 404 });
|
|
13145
14347
|
}
|
|
13146
14348
|
}
|
|
13147
14349
|
async function routeGetRequest(req, segment) {
|
|
@@ -13157,7 +14359,7 @@ function createRagHandler(configOrPlugin, options) {
|
|
|
13157
14359
|
case "sessions":
|
|
13158
14360
|
return sessionsHandler(req);
|
|
13159
14361
|
default:
|
|
13160
|
-
return
|
|
14362
|
+
return NextResponse12.json({ error: `Method Not Allowed: GET segment "${segment}" not supported.` }, { status: 405 });
|
|
13161
14363
|
}
|
|
13162
14364
|
}
|
|
13163
14365
|
async function getSegment(req, context) {
|
|
@@ -13192,7 +14394,7 @@ function createRagHandler(configOrPlugin, options) {
|
|
|
13192
14394
|
return await routeGetRequest(req, segment);
|
|
13193
14395
|
} catch (err) {
|
|
13194
14396
|
const msg = err instanceof Error ? err.message : "GET Routing failed";
|
|
13195
|
-
return
|
|
14397
|
+
return NextResponse12.json({ error: msg }, { status: 500 });
|
|
13196
14398
|
}
|
|
13197
14399
|
},
|
|
13198
14400
|
POST: async (req, context) => {
|
|
@@ -13201,7 +14403,7 @@ function createRagHandler(configOrPlugin, options) {
|
|
|
13201
14403
|
return await routePostRequest(req, segment);
|
|
13202
14404
|
} catch (err) {
|
|
13203
14405
|
const msg = err instanceof Error ? err.message : "POST Routing failed";
|
|
13204
|
-
return
|
|
14406
|
+
return NextResponse12.json({ error: msg }, { status: 500 });
|
|
13205
14407
|
}
|
|
13206
14408
|
}
|
|
13207
14409
|
};
|
|
@@ -13250,11 +14452,11 @@ var _LicenseValidator = class _LicenseValidator {
|
|
|
13250
14452
|
* Primary validation entry-point for UI / hook consumers.
|
|
13251
14453
|
*
|
|
13252
14454
|
* Resolves the license key, consults the local success cache, performs a
|
|
13253
|
-
* server POST for authoritative status, and falls back to
|
|
13254
|
-
*
|
|
14455
|
+
* server POST for authoritative status, and falls back to `LicenseVerifier.verifyAsync()`
|
|
14456
|
+
* if the server can't be reached.
|
|
13255
14457
|
*
|
|
13256
14458
|
* @throws LicenseValidationError - When no key is provided OR both the
|
|
13257
|
-
* server call AND the
|
|
14459
|
+
* server call AND the fallback report an invalid/expired key.
|
|
13258
14460
|
* @throws SDKVersionUnsupportedError - When server reports forceUpgrade=true.
|
|
13259
14461
|
* @returns A `LicenseValidationResponse` that the UI can use to enable or
|
|
13260
14462
|
* lock the chat widget.
|
|
@@ -13303,6 +14505,8 @@ var _LicenseValidator = class _LicenseValidator {
|
|
|
13303
14505
|
this.cache.delete(effectiveLicenseKey);
|
|
13304
14506
|
}
|
|
13305
14507
|
}
|
|
14508
|
+
const existing = this.inflight.get(effectiveLicenseKey);
|
|
14509
|
+
if (existing) return existing;
|
|
13306
14510
|
const inflightPromise = this._doNetworkValidate(
|
|
13307
14511
|
effectiveLicenseKey,
|
|
13308
14512
|
effectiveProjectId,
|
|
@@ -13571,8 +14775,8 @@ var LicenseConfigResolver = class {
|
|
|
13571
14775
|
};
|
|
13572
14776
|
}
|
|
13573
14777
|
};
|
|
13574
|
-
LicenseConfigResolver.RETRIVORA_CONFIG_URL = "https://
|
|
13575
|
-
LicenseConfigResolver.RETRIVORA_CREDENTIALS_URL = "https://
|
|
14778
|
+
LicenseConfigResolver.RETRIVORA_CONFIG_URL = "https://llm.retrivora.com/api/v1/config";
|
|
14779
|
+
LicenseConfigResolver.RETRIVORA_CREDENTIALS_URL = "https://llm.retrivora.com/api/v1/credentials";
|
|
13576
14780
|
LicenseConfigResolver.CACHE_TTL_MS = 36e5;
|
|
13577
14781
|
// 1 hour
|
|
13578
14782
|
LicenseConfigResolver.configCache = /* @__PURE__ */ new Map();
|
|
@@ -13736,6 +14940,9 @@ var _TelemetryService = class _TelemetryService {
|
|
|
13736
14940
|
console.warn("[TelemetryService] Auto-flush error:", err.message);
|
|
13737
14941
|
});
|
|
13738
14942
|
}, this.flushIntervalMs);
|
|
14943
|
+
if (this.flushTimer && typeof this.flushTimer.unref === "function") {
|
|
14944
|
+
this.flushTimer.unref();
|
|
14945
|
+
}
|
|
13739
14946
|
}
|
|
13740
14947
|
/**
|
|
13741
14948
|
* Stop telemetry service and flush remaining events
|
|
@@ -14004,9 +15211,11 @@ export {
|
|
|
14004
15211
|
EmbeddingStrategyResolver,
|
|
14005
15212
|
EventType,
|
|
14006
15213
|
GroqProvider,
|
|
15214
|
+
HybridSearch,
|
|
14007
15215
|
IntentClassifier,
|
|
14008
15216
|
LLMFactory,
|
|
14009
15217
|
LLM_PROFILES,
|
|
15218
|
+
LangChainAgent,
|
|
14010
15219
|
LicenseConfigResolver,
|
|
14011
15220
|
LicenseValidationError,
|
|
14012
15221
|
LicenseValidator,
|
|
@@ -14018,19 +15227,23 @@ export {
|
|
|
14018
15227
|
MultiTablePostgresProvider,
|
|
14019
15228
|
OllamaProvider,
|
|
14020
15229
|
OpenAIProvider,
|
|
15230
|
+
PIIMasker,
|
|
14021
15231
|
PRESETS,
|
|
14022
15232
|
PineconeProvider,
|
|
14023
15233
|
Pipeline,
|
|
14024
15234
|
PostgreSQLProvider,
|
|
15235
|
+
PromptGuard,
|
|
14025
15236
|
PromptOptimizer,
|
|
14026
15237
|
ProviderHealthCheck,
|
|
14027
15238
|
ProviderNotFoundException,
|
|
14028
15239
|
ProviderRegistry,
|
|
14029
15240
|
QdrantProvider,
|
|
14030
15241
|
QwenProvider,
|
|
15242
|
+
RAGEvaluator,
|
|
14031
15243
|
RateLimitException,
|
|
14032
15244
|
RedisProvider,
|
|
14033
15245
|
RendererRegistry,
|
|
15246
|
+
Reranker,
|
|
14034
15247
|
RetrievalException,
|
|
14035
15248
|
Retrivora,
|
|
14036
15249
|
RetrivoraError,
|
|
@@ -14048,9 +15261,11 @@ export {
|
|
|
14048
15261
|
TelemetryService,
|
|
14049
15262
|
TextRendererStrategy,
|
|
14050
15263
|
UniversalLLMAdapter,
|
|
15264
|
+
UniversalTableParser,
|
|
14051
15265
|
UniversalVectorProvider,
|
|
14052
15266
|
VECTOR_PROFILES,
|
|
14053
15267
|
VectorPlugin,
|
|
15268
|
+
VisionExtractor,
|
|
14054
15269
|
VisualizationDecisionEngine,
|
|
14055
15270
|
WeaviateProvider,
|
|
14056
15271
|
createChatHandler,
|
|
@@ -14060,10 +15275,12 @@ export {
|
|
|
14060
15275
|
createHealthHandler,
|
|
14061
15276
|
createHistoryHandler,
|
|
14062
15277
|
createIngestHandler,
|
|
15278
|
+
createLicenseHandler,
|
|
14063
15279
|
createRagHandler,
|
|
14064
15280
|
createRetrivora,
|
|
14065
15281
|
createSessionsHandler,
|
|
14066
15282
|
createStreamHandler,
|
|
15283
|
+
createSuggestionsHandler,
|
|
14067
15284
|
createUploadHandler,
|
|
14068
15285
|
decideVisualization,
|
|
14069
15286
|
getRagConfig,
|
|
@@ -14077,6 +15294,8 @@ export {
|
|
|
14077
15294
|
sseErrorFrame,
|
|
14078
15295
|
sseFrame,
|
|
14079
15296
|
sseMetaFrame,
|
|
15297
|
+
sseObservabilityFrame,
|
|
14080
15298
|
sseTextFrame,
|
|
15299
|
+
sseUIFrame,
|
|
14081
15300
|
wrapError
|
|
14082
15301
|
};
|