@infuro/cms-core 1.0.20 → 1.0.22
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/admin.cjs +194 -75
- package/dist/admin.cjs.map +1 -1
- package/dist/admin.js +188 -70
- package/dist/admin.js.map +1 -1
- package/dist/api.cjs +3230 -259
- package/dist/api.cjs.map +1 -1
- package/dist/api.d.cts +1 -1
- package/dist/api.d.ts +1 -1
- package/dist/api.js +3232 -252
- package/dist/api.js.map +1 -1
- package/dist/{index-DGtM2Gsk.d.cts → index-BGAh4fPQ.d.cts} +18 -1
- package/dist/{index--GBYw5JE.d.ts → index-Cnwh7B3r.d.ts} +18 -1
- package/dist/index.cjs +753 -209
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +34 -9
- package/dist/index.d.ts +34 -9
- package/dist/index.js +753 -209
- package/dist/index.js.map +1 -1
- package/dist/migrations/1775300000000-LlmAgents.ts +57 -57
- package/dist/migrations/1775300000001-LlmAgentsValidationRulesText.ts +43 -43
- package/dist/migrations/1775300000002-SeedLlmAgentsPermissions.ts +33 -33
- package/dist/migrations/1775300000003-LlmAgentKnowledgeDocuments.ts +50 -50
- package/package.json +2 -1
package/dist/index.cjs
CHANGED
|
@@ -2613,6 +2613,17 @@ function localStoragePlugin(config = {}) {
|
|
|
2613
2613
|
}
|
|
2614
2614
|
|
|
2615
2615
|
// src/plugins/llm/llm-service.ts
|
|
2616
|
+
function parseOpenAiEmbeddingJson(data) {
|
|
2617
|
+
let embedding = data?.data?.[0]?.embedding;
|
|
2618
|
+
if (!Array.isArray(embedding) && Array.isArray(data?.embedding)) {
|
|
2619
|
+
embedding = data.embedding;
|
|
2620
|
+
}
|
|
2621
|
+
if (!Array.isArray(embedding) && Array.isArray(data?.data?.[0])) {
|
|
2622
|
+
const first = data.data[0];
|
|
2623
|
+
embedding = Array.isArray(first) ? first : first?.embedding;
|
|
2624
|
+
}
|
|
2625
|
+
return Array.isArray(embedding) ? embedding : null;
|
|
2626
|
+
}
|
|
2616
2627
|
function parseHfInferenceEmbeddingBody(data) {
|
|
2617
2628
|
if (data === null || data === void 0) return null;
|
|
2618
2629
|
if (Array.isArray(data)) {
|
|
@@ -2634,28 +2645,59 @@ function parseHfInferenceEmbeddingBody(data) {
|
|
|
2634
2645
|
}
|
|
2635
2646
|
return null;
|
|
2636
2647
|
}
|
|
2648
|
+
var FALLBACK_CHAT_MODEL_OPENAI_SHAPE = "openai/gpt-4o-mini";
|
|
2649
|
+
var FALLBACK_CHAT_MODEL_HUGGINGFACE = "meta-llama/Meta-Llama-3.1-8B-Instruct";
|
|
2650
|
+
function defaultChatModelForGateway(baseURL) {
|
|
2651
|
+
const u = baseURL.toLowerCase();
|
|
2652
|
+
if (u.includes("huggingface.co") || u.includes("hf.co") || u.includes("hf-inference")) {
|
|
2653
|
+
return FALLBACK_CHAT_MODEL_HUGGINGFACE;
|
|
2654
|
+
}
|
|
2655
|
+
return FALLBACK_CHAT_MODEL_OPENAI_SHAPE;
|
|
2656
|
+
}
|
|
2657
|
+
function formatLlmGatewayError(status, bodyText, defaultChatModel) {
|
|
2658
|
+
let hint = "";
|
|
2659
|
+
try {
|
|
2660
|
+
const j = JSON.parse(bodyText);
|
|
2661
|
+
const code = j?.error?.code;
|
|
2662
|
+
if (code === "model_not_supported" || code === "model_not_found" || /not supported by any provider|does not exist/i.test(j?.error?.message ?? "")) {
|
|
2663
|
+
hint = ` Hint: set LLM_CHAT_MODEL (or CMS) to a model your gateway lists. HF router: Hub ids (e.g. "${FALLBACK_CHAT_MODEL_HUGGINGFACE}"); OpenRouter: e.g. "${FALLBACK_CHAT_MODEL_OPENAI_SHAPE}".`;
|
|
2664
|
+
}
|
|
2665
|
+
} catch {
|
|
2666
|
+
}
|
|
2667
|
+
return `LLM gateway error: ${status} ${bodyText}${hint}`;
|
|
2668
|
+
}
|
|
2637
2669
|
var LlmService = class _LlmService {
|
|
2638
2670
|
static embedHttpErrorLogged = false;
|
|
2639
2671
|
static embedShapeErrorLogged = false;
|
|
2640
2672
|
static embedSuccessLogged = false;
|
|
2641
2673
|
static hfDimensionMismatchWarned = false;
|
|
2674
|
+
static hfInferenceSdkErrorLogged = false;
|
|
2642
2675
|
baseURL;
|
|
2643
2676
|
apiKey;
|
|
2644
2677
|
defaultEmbeddingModel;
|
|
2645
2678
|
embeddingProvider;
|
|
2646
2679
|
hfInferenceBaseURL;
|
|
2647
2680
|
embeddingApiKey;
|
|
2681
|
+
defaultChatModel;
|
|
2648
2682
|
constructor(baseURL, apiKey, defaultEmbeddingModel, embedOpts) {
|
|
2649
2683
|
this.baseURL = baseURL.replace(/\/$/, "");
|
|
2650
2684
|
this.apiKey = apiKey;
|
|
2651
2685
|
this.defaultEmbeddingModel = defaultEmbeddingModel;
|
|
2652
2686
|
const p = (embedOpts?.embeddingProvider ?? "openai").toLowerCase();
|
|
2653
2687
|
this.embeddingProvider = p === "huggingface" ? "huggingface" : "openai";
|
|
2654
|
-
this.hfInferenceBaseURL = (embedOpts?.hfInferenceBaseUrl ?? "https://
|
|
2688
|
+
this.hfInferenceBaseURL = (embedOpts?.hfInferenceBaseUrl ?? "https://router.huggingface.co/v1").replace(
|
|
2655
2689
|
/\/$/,
|
|
2656
2690
|
""
|
|
2657
2691
|
);
|
|
2658
2692
|
this.embeddingApiKey = (embedOpts?.embeddingApiKey ?? apiKey).trim();
|
|
2693
|
+
const fromOpts = embedOpts?.defaultChatModel?.trim();
|
|
2694
|
+
const fromEnv = typeof process !== "undefined" ? process.env.LLM_CHAT_MODEL?.trim() : "";
|
|
2695
|
+
const auto = defaultChatModelForGateway(this.baseURL);
|
|
2696
|
+
this.defaultChatModel = (fromOpts || fromEnv || auto).trim() || auto;
|
|
2697
|
+
}
|
|
2698
|
+
resolveChatModel(explicit) {
|
|
2699
|
+
const m = (explicit?.trim() || this.defaultChatModel).trim();
|
|
2700
|
+
return m || defaultChatModelForGateway(this.baseURL);
|
|
2659
2701
|
}
|
|
2660
2702
|
get base() {
|
|
2661
2703
|
return this.baseURL.replace(/\/$/, "");
|
|
@@ -2695,7 +2737,7 @@ var LlmService = class _LlmService {
|
|
|
2695
2737
|
}
|
|
2696
2738
|
logChatCompletionsRequest(kind, messages, options, stream) {
|
|
2697
2739
|
if (!this.llmDebugEnabled()) return;
|
|
2698
|
-
const model = options.model
|
|
2740
|
+
const model = this.resolveChatModel(options.model);
|
|
2699
2741
|
console.log(`
|
|
2700
2742
|
[LLM ${kind}] POST`, this.url);
|
|
2701
2743
|
console.log("[Request options]", {
|
|
@@ -2713,7 +2755,12 @@ var LlmService = class _LlmService {
|
|
|
2713
2755
|
return this.embedHuggingFaceInference(input, model2);
|
|
2714
2756
|
}
|
|
2715
2757
|
const model = options.model ?? this.defaultEmbeddingModel ?? "text-embedding-3-small";
|
|
2716
|
-
|
|
2758
|
+
return this.fetchOpenAiShapeEmbeddings(`${this.base}/v1/embeddings`, model, input, this.apiKey);
|
|
2759
|
+
}
|
|
2760
|
+
/**
|
|
2761
|
+
* POST `{ url }` with OpenAI-style `{ model, input }` and Bearer token; parses `{ data:[{embedding}] }`.
|
|
2762
|
+
*/
|
|
2763
|
+
async fetchOpenAiShapeEmbeddings(embedUrl, model, input, bearerToken) {
|
|
2717
2764
|
if (this.embedDebugEnabled()) {
|
|
2718
2765
|
console.info("[LLM embed] request", { url: embedUrl, model, inputChars: input.length });
|
|
2719
2766
|
}
|
|
@@ -2721,7 +2768,7 @@ var LlmService = class _LlmService {
|
|
|
2721
2768
|
method: "POST",
|
|
2722
2769
|
headers: {
|
|
2723
2770
|
"Content-Type": "application/json",
|
|
2724
|
-
Authorization: `Bearer ${
|
|
2771
|
+
Authorization: `Bearer ${bearerToken}`
|
|
2725
2772
|
},
|
|
2726
2773
|
body: JSON.stringify({
|
|
2727
2774
|
model,
|
|
@@ -2730,7 +2777,7 @@ var LlmService = class _LlmService {
|
|
|
2730
2777
|
});
|
|
2731
2778
|
if (!res.ok) {
|
|
2732
2779
|
const body = await res.text();
|
|
2733
|
-
const hint404 = res.status === 404 ? "This base URL does not expose OpenAI-compatible POST /v1/embeddings
|
|
2780
|
+
const hint404 = res.status === 404 ? "This base URL does not expose OpenAI-compatible POST /v1/embeddings. For Hugging Face embeddings, set EMBEDDING_PROVIDER=huggingface and an HF token (EMBEDDING_API_KEY); the server uses @huggingface/inference (router /v1/embeddings is chat-only and is not used)." : "Embeddings HTTP error \u2014 knowledge_base_chunks.embedding will stay NULL for failed calls.";
|
|
2734
2781
|
if (!_LlmService.embedHttpErrorLogged) {
|
|
2735
2782
|
_LlmService.embedHttpErrorLogged = true;
|
|
2736
2783
|
console.error("[LLM embed]", hint404, {
|
|
@@ -2754,15 +2801,8 @@ var LlmService = class _LlmService {
|
|
|
2754
2801
|
}
|
|
2755
2802
|
return [];
|
|
2756
2803
|
}
|
|
2757
|
-
|
|
2758
|
-
if (!
|
|
2759
|
-
embedding = data.embedding;
|
|
2760
|
-
}
|
|
2761
|
-
if (!Array.isArray(embedding) && Array.isArray(data?.data?.[0])) {
|
|
2762
|
-
const first = data.data[0];
|
|
2763
|
-
embedding = Array.isArray(first) ? first : first?.embedding;
|
|
2764
|
-
}
|
|
2765
|
-
if (!Array.isArray(embedding)) {
|
|
2804
|
+
const embedding = parseOpenAiEmbeddingJson(data);
|
|
2805
|
+
if (!embedding) {
|
|
2766
2806
|
if (!_LlmService.embedShapeErrorLogged) {
|
|
2767
2807
|
_LlmService.embedShapeErrorLogged = true;
|
|
2768
2808
|
console.error(
|
|
@@ -2784,96 +2824,201 @@ var LlmService = class _LlmService {
|
|
|
2784
2824
|
}
|
|
2785
2825
|
return embedding;
|
|
2786
2826
|
}
|
|
2787
|
-
|
|
2788
|
-
|
|
2789
|
-
|
|
2790
|
-
|
|
2791
|
-
|
|
2792
|
-
|
|
2793
|
-
|
|
2794
|
-
|
|
2795
|
-
|
|
2796
|
-
|
|
2797
|
-
|
|
2798
|
-
|
|
2799
|
-
let embedUrl = pipelineUrl;
|
|
2800
|
-
let res = await fetch(embedUrl, {
|
|
2801
|
-
method: "POST",
|
|
2802
|
-
headers: {
|
|
2803
|
-
"Content-Type": "application/json",
|
|
2804
|
-
Authorization: `Bearer ${this.embeddingApiKey}`
|
|
2805
|
-
},
|
|
2806
|
-
body: requestBody
|
|
2807
|
-
});
|
|
2808
|
-
let bodyText = await res.text();
|
|
2809
|
-
if (res.status === 404) {
|
|
2810
|
-
embedUrl = modelUrl;
|
|
2811
|
-
if (this.embedDebugEnabled()) {
|
|
2812
|
-
console.warn("[LLM embed HF] /pipeline route returned 404; retrying with /models.", {
|
|
2813
|
-
model,
|
|
2814
|
-
fallbackUrl: embedUrl
|
|
2815
|
-
});
|
|
2816
|
-
}
|
|
2817
|
-
res = await fetch(embedUrl, {
|
|
2818
|
-
method: "POST",
|
|
2819
|
-
headers: {
|
|
2820
|
-
"Content-Type": "application/json",
|
|
2821
|
-
Authorization: `Bearer ${this.embeddingApiKey}`
|
|
2822
|
-
},
|
|
2823
|
-
body: requestBody
|
|
2827
|
+
/**
|
|
2828
|
+
* Hugging Face–recommended path: Hub resolves a provider that supports `feature-extraction` for the model.
|
|
2829
|
+
*/
|
|
2830
|
+
async embedViaHuggingfaceInferenceSdk(input, model) {
|
|
2831
|
+
const token = this.embeddingApiKey.trim();
|
|
2832
|
+
if (!token) return null;
|
|
2833
|
+
try {
|
|
2834
|
+
const { InferenceClient } = await import("@huggingface/inference");
|
|
2835
|
+
const hf = new InferenceClient(token);
|
|
2836
|
+
const raw = await hf.featureExtraction({
|
|
2837
|
+
model,
|
|
2838
|
+
inputs: input
|
|
2824
2839
|
});
|
|
2825
|
-
|
|
2826
|
-
|
|
2827
|
-
|
|
2828
|
-
|
|
2829
|
-
|
|
2830
|
-
|
|
2831
|
-
|
|
2832
|
-
|
|
2833
|
-
|
|
2834
|
-
|
|
2835
|
-
|
|
2836
|
-
|
|
2837
|
-
|
|
2838
|
-
|
|
2840
|
+
const embedding = parseHfInferenceEmbeddingBody(raw);
|
|
2841
|
+
if (!embedding?.length) {
|
|
2842
|
+
if (this.embedDebugEnabled()) {
|
|
2843
|
+
console.warn("[LLM embed HF] Inference SDK returned no parseable vector", {
|
|
2844
|
+
model,
|
|
2845
|
+
shape: Array.isArray(raw) ? `array(len=${raw.length})` : typeof raw
|
|
2846
|
+
});
|
|
2847
|
+
}
|
|
2848
|
+
return null;
|
|
2849
|
+
}
|
|
2850
|
+
return embedding;
|
|
2851
|
+
} catch (err) {
|
|
2852
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
2853
|
+
if (!_LlmService.hfInferenceSdkErrorLogged) {
|
|
2854
|
+
_LlmService.hfInferenceSdkErrorLogged = true;
|
|
2855
|
+
console.warn(
|
|
2856
|
+
"[LLM embed HF] @huggingface/inference featureExtraction failed \u2014 trying legacy HTTP URLs.",
|
|
2857
|
+
{ model, message: message.slice(0, 500) }
|
|
2839
2858
|
);
|
|
2840
2859
|
} else if (this.embedDebugEnabled()) {
|
|
2841
|
-
console.warn("[LLM embed HF] repeat
|
|
2860
|
+
console.warn("[LLM embed HF] repeat Inference SDK failure", { message: message.slice(0, 200) });
|
|
2842
2861
|
}
|
|
2843
|
-
return
|
|
2862
|
+
return null;
|
|
2844
2863
|
}
|
|
2845
|
-
|
|
2846
|
-
|
|
2847
|
-
|
|
2848
|
-
|
|
2849
|
-
|
|
2850
|
-
_LlmService.embedShapeErrorLogged = true;
|
|
2851
|
-
console.error("[LLM embed HF] Response is not JSON \u2014 treating as no embedding.");
|
|
2852
|
-
}
|
|
2853
|
-
return [];
|
|
2864
|
+
}
|
|
2865
|
+
/** Base URL for legacy `…/pipeline/…` and `…/models/{model}` tries (HF router only). */
|
|
2866
|
+
hfLegacyInferenceBase(normalizedBase) {
|
|
2867
|
+
if (/router\.huggingface\.co|api-inference\.huggingface\.co/i.test(normalizedBase)) {
|
|
2868
|
+
return "https://router.huggingface.co/hf-inference";
|
|
2854
2869
|
}
|
|
2855
|
-
|
|
2856
|
-
|
|
2857
|
-
|
|
2858
|
-
|
|
2859
|
-
|
|
2860
|
-
|
|
2870
|
+
return normalizedBase;
|
|
2871
|
+
}
|
|
2872
|
+
/**
|
|
2873
|
+
* Hugging Face embeddings: `@huggingface/inference` (Hub-routed feature extraction), then
|
|
2874
|
+
* legacy direct `hf-inference` HTTP tries. Note: `https://router.huggingface.co/v1/embeddings` is not
|
|
2875
|
+
* offered for embeddings (OpenAI-compatible `/v1` on HF is chat-only).
|
|
2876
|
+
*/
|
|
2877
|
+
async embedHuggingFaceInference(input, model) {
|
|
2878
|
+
const normalizedBase = this.hfInferenceBaseURL.replace(/\/$/, "");
|
|
2879
|
+
const fromSdk = await this.embedViaHuggingfaceInferenceSdk(input, model);
|
|
2880
|
+
if (fromSdk?.length) {
|
|
2881
|
+
if (!_LlmService.embedSuccessLogged) {
|
|
2882
|
+
_LlmService.embedSuccessLogged = true;
|
|
2883
|
+
console.info("[LLM embed HF] first success", { dimension: fromSdk.length, model, via: "@huggingface/inference" });
|
|
2884
|
+
if (!_LlmService.hfDimensionMismatchWarned && fromSdk.length !== 1536) {
|
|
2885
|
+
_LlmService.hfDimensionMismatchWarned = true;
|
|
2886
|
+
console.warn(
|
|
2887
|
+
`[LLM embed HF] Embedding dimensions=${fromSdk.length}; ensure knowledge_base_chunks.embedding uses vector(${fromSdk.length}) (see migrations).`
|
|
2888
|
+
);
|
|
2889
|
+
}
|
|
2890
|
+
} else if (this.embedDebugEnabled()) {
|
|
2891
|
+
console.info("[LLM embed HF] ok", { dimension: fromSdk.length, via: "@huggingface/inference" });
|
|
2861
2892
|
}
|
|
2862
|
-
return
|
|
2893
|
+
return fromSdk;
|
|
2863
2894
|
}
|
|
2864
|
-
|
|
2865
|
-
|
|
2866
|
-
|
|
2867
|
-
|
|
2868
|
-
|
|
2869
|
-
|
|
2870
|
-
|
|
2871
|
-
|
|
2895
|
+
const legacyBase = this.hfLegacyInferenceBase(normalizedBase);
|
|
2896
|
+
const baseWithProvider = /\/hf-inference$/i.test(legacyBase) || !/router\.huggingface\.co/i.test(legacyBase) ? legacyBase : `${legacyBase.replace(/\/$/, "")}/hf-inference`;
|
|
2897
|
+
const bodyVariants = [
|
|
2898
|
+
{
|
|
2899
|
+
label: "inputs_parameters_mean_normalize",
|
|
2900
|
+
body: JSON.stringify({
|
|
2901
|
+
inputs: input,
|
|
2902
|
+
parameters: { pooling: "mean", normalize: true },
|
|
2903
|
+
options: { wait_for_model: true }
|
|
2904
|
+
})
|
|
2905
|
+
},
|
|
2906
|
+
{
|
|
2907
|
+
label: "inputs_string",
|
|
2908
|
+
body: JSON.stringify({ inputs: input, options: { wait_for_model: true } })
|
|
2909
|
+
},
|
|
2910
|
+
{
|
|
2911
|
+
label: "inputs_string_array",
|
|
2912
|
+
body: JSON.stringify({ inputs: [input], options: { wait_for_model: true } })
|
|
2913
|
+
},
|
|
2914
|
+
{
|
|
2915
|
+
label: "inputs_wrapped_sentences",
|
|
2916
|
+
body: JSON.stringify({ inputs: { sentences: [input] }, options: { wait_for_model: true } })
|
|
2917
|
+
},
|
|
2918
|
+
{
|
|
2919
|
+
label: "top_level_sentences",
|
|
2920
|
+
body: JSON.stringify({ sentences: [input], options: { wait_for_model: true } })
|
|
2872
2921
|
}
|
|
2922
|
+
];
|
|
2923
|
+
const headers = {
|
|
2924
|
+
"Content-Type": "application/json",
|
|
2925
|
+
Authorization: `Bearer ${this.embeddingApiKey}`
|
|
2926
|
+
};
|
|
2927
|
+
const candidateUrls = [];
|
|
2928
|
+
const pushUrl = (u) => {
|
|
2929
|
+
if (!candidateUrls.includes(u)) candidateUrls.push(u);
|
|
2930
|
+
};
|
|
2931
|
+
pushUrl(`${baseWithProvider}/pipeline/feature-extraction/${model}`);
|
|
2932
|
+
pushUrl(`${baseWithProvider}/models/${model}`);
|
|
2933
|
+
let lastUrl = "";
|
|
2934
|
+
let lastStatus = 0;
|
|
2935
|
+
let lastBodyPreview = "";
|
|
2936
|
+
let lastAttempt = "";
|
|
2937
|
+
for (const embedUrl of candidateUrls) {
|
|
2938
|
+
for (const { label, body } of bodyVariants) {
|
|
2939
|
+
if (this.embedDebugEnabled()) {
|
|
2940
|
+
console.info("[LLM embed HF] attempt", { url: embedUrl, model, inputChars: input.length, body: label });
|
|
2941
|
+
}
|
|
2942
|
+
const res = await fetch(embedUrl, {
|
|
2943
|
+
method: "POST",
|
|
2944
|
+
headers,
|
|
2945
|
+
body
|
|
2946
|
+
});
|
|
2947
|
+
const bodyText = await res.text();
|
|
2948
|
+
lastUrl = embedUrl;
|
|
2949
|
+
lastStatus = res.status;
|
|
2950
|
+
lastBodyPreview = bodyText.slice(0, 800);
|
|
2951
|
+
lastAttempt = `${embedUrl} (${label})`;
|
|
2952
|
+
if (!res.ok) {
|
|
2953
|
+
if (this.embedDebugEnabled()) {
|
|
2954
|
+
console.warn("[LLM embed HF] attempt failed, trying next if any", {
|
|
2955
|
+
url: embedUrl,
|
|
2956
|
+
body: label,
|
|
2957
|
+
status: res.status,
|
|
2958
|
+
bodyPreview: bodyText.slice(0, 200)
|
|
2959
|
+
});
|
|
2960
|
+
}
|
|
2961
|
+
continue;
|
|
2962
|
+
}
|
|
2963
|
+
let data;
|
|
2964
|
+
try {
|
|
2965
|
+
data = JSON.parse(bodyText);
|
|
2966
|
+
} catch {
|
|
2967
|
+
if (this.embedDebugEnabled()) {
|
|
2968
|
+
console.warn("[LLM embed HF] response not JSON, trying next if any", { url: embedUrl, body: label });
|
|
2969
|
+
}
|
|
2970
|
+
continue;
|
|
2971
|
+
}
|
|
2972
|
+
const embedding = parseHfInferenceEmbeddingBody(data);
|
|
2973
|
+
if (!embedding?.length) {
|
|
2974
|
+
if (this.embedDebugEnabled()) {
|
|
2975
|
+
const preview = typeof data === "object" && data !== null && !Array.isArray(data) ? Object.keys(data) : typeof data;
|
|
2976
|
+
console.warn("[LLM embed HF] no parseable embedding, trying next if any", {
|
|
2977
|
+
url: embedUrl,
|
|
2978
|
+
body: label,
|
|
2979
|
+
preview
|
|
2980
|
+
});
|
|
2981
|
+
}
|
|
2982
|
+
continue;
|
|
2983
|
+
}
|
|
2984
|
+
if (!_LlmService.embedSuccessLogged) {
|
|
2985
|
+
_LlmService.embedSuccessLogged = true;
|
|
2986
|
+
console.info("[LLM embed HF] first success", { dimension: embedding.length, model, url: embedUrl, body: label });
|
|
2987
|
+
if (!_LlmService.hfDimensionMismatchWarned && embedding.length !== 1536) {
|
|
2988
|
+
_LlmService.hfDimensionMismatchWarned = true;
|
|
2989
|
+
console.warn(
|
|
2990
|
+
`[LLM embed HF] Embedding dimensions=${embedding.length}; ensure knowledge_base_chunks.embedding uses vector(${embedding.length}) (see migrations).`
|
|
2991
|
+
);
|
|
2992
|
+
}
|
|
2993
|
+
} else if (this.embedDebugEnabled()) {
|
|
2994
|
+
console.info("[LLM embed HF] ok", { dimension: embedding.length, url: embedUrl, body: label });
|
|
2995
|
+
}
|
|
2996
|
+
return embedding;
|
|
2997
|
+
}
|
|
2998
|
+
}
|
|
2999
|
+
if (!_LlmService.embedHttpErrorLogged) {
|
|
3000
|
+
_LlmService.embedHttpErrorLogged = true;
|
|
3001
|
+
console.error(
|
|
3002
|
+
"[LLM embed HF] All Hugging Face embedding attempts failed \u2014 knowledge chunks stay without vectors.",
|
|
3003
|
+
{
|
|
3004
|
+
triedInferenceSdk: true,
|
|
3005
|
+
triedLegacyHttp: candidateUrls,
|
|
3006
|
+
lastAttempt,
|
|
3007
|
+
lastUrl,
|
|
3008
|
+
lastStatus,
|
|
3009
|
+
lastBodyPreview,
|
|
3010
|
+
hint: "Use an HF token with permission to call Inference Providers / Hub API mapping. Install ships @huggingface/inference; EMBEDDING_MODEL defaults to sentence-transformers/all-MiniLM-L6-v2. Router OpenAI /v1/embeddings is not used (chat-only)."
|
|
3011
|
+
}
|
|
3012
|
+
);
|
|
2873
3013
|
} else if (this.embedDebugEnabled()) {
|
|
2874
|
-
console.
|
|
3014
|
+
console.warn("[LLM embed HF] repeat failure (all attempts)", {
|
|
3015
|
+
lastAttempt,
|
|
3016
|
+
lastUrl,
|
|
3017
|
+
lastStatus,
|
|
3018
|
+
bodyPreview: lastBodyPreview.slice(0, 200)
|
|
3019
|
+
});
|
|
2875
3020
|
}
|
|
2876
|
-
return
|
|
3021
|
+
return [];
|
|
2877
3022
|
}
|
|
2878
3023
|
buildAgentMessages(options) {
|
|
2879
3024
|
const { systemPrompt, userPrompt, history = [], context } = options;
|
|
@@ -2917,7 +3062,7 @@ ${context}`;
|
|
|
2917
3062
|
Authorization: `Bearer ${this.apiKey}`
|
|
2918
3063
|
},
|
|
2919
3064
|
body: JSON.stringify({
|
|
2920
|
-
model: options.model
|
|
3065
|
+
model: this.resolveChatModel(options.model),
|
|
2921
3066
|
messages,
|
|
2922
3067
|
stream: false,
|
|
2923
3068
|
temperature: options.temperature,
|
|
@@ -2926,7 +3071,7 @@ ${context}`;
|
|
|
2926
3071
|
});
|
|
2927
3072
|
if (!res.ok) {
|
|
2928
3073
|
const text = await res.text();
|
|
2929
|
-
throw new Error(
|
|
3074
|
+
throw new Error(formatLlmGatewayError(res.status, text, this.defaultChatModel));
|
|
2930
3075
|
}
|
|
2931
3076
|
const data = await res.json();
|
|
2932
3077
|
const content = data.choices?.[0]?.message?.content ?? "";
|
|
@@ -2944,7 +3089,7 @@ ${context}`;
|
|
|
2944
3089
|
Authorization: `Bearer ${this.apiKey}`
|
|
2945
3090
|
},
|
|
2946
3091
|
body: JSON.stringify({
|
|
2947
|
-
model: options.model
|
|
3092
|
+
model: this.resolveChatModel(options.model),
|
|
2948
3093
|
messages,
|
|
2949
3094
|
stream: true,
|
|
2950
3095
|
temperature: options.temperature,
|
|
@@ -2953,7 +3098,7 @@ ${context}`;
|
|
|
2953
3098
|
});
|
|
2954
3099
|
if (!res.ok) {
|
|
2955
3100
|
const text = await res.text();
|
|
2956
|
-
throw new Error(
|
|
3101
|
+
throw new Error(formatLlmGatewayError(res.status, text, this.defaultChatModel));
|
|
2957
3102
|
}
|
|
2958
3103
|
const reader = res.body?.getReader();
|
|
2959
3104
|
if (!reader) return;
|
|
@@ -3006,6 +3151,7 @@ function llmPlugin(config = {}) {
|
|
|
3006
3151
|
return void 0;
|
|
3007
3152
|
}
|
|
3008
3153
|
const embeddingModel = config.embeddingModel ?? context.config.EMBEDDING_MODEL;
|
|
3154
|
+
const defaultChatModel = config.defaultChatModel?.trim() || (typeof context.config.LLM_CHAT_MODEL === "string" ? context.config.LLM_CHAT_MODEL.trim() : "") || void 0;
|
|
3009
3155
|
const embeddingProvider = normalizeEmbeddingProvider(
|
|
3010
3156
|
config.embeddingProvider ?? context.config.EMBEDDING_PROVIDER
|
|
3011
3157
|
);
|
|
@@ -3014,7 +3160,8 @@ function llmPlugin(config = {}) {
|
|
|
3014
3160
|
return new LlmService(baseURL.replace(/\/$/, ""), apiKey, embeddingModel, {
|
|
3015
3161
|
embeddingProvider,
|
|
3016
3162
|
hfInferenceBaseUrl,
|
|
3017
|
-
embeddingApiKey
|
|
3163
|
+
embeddingApiKey,
|
|
3164
|
+
defaultChatModel
|
|
3018
3165
|
});
|
|
3019
3166
|
}
|
|
3020
3167
|
};
|
|
@@ -5357,15 +5504,33 @@ function createChatHandlers(config) {
|
|
|
5357
5504
|
const email = body?.email?.trim();
|
|
5358
5505
|
if (!name || !email) return json({ error: "name and email required" }, { status: 400 });
|
|
5359
5506
|
const repo = contactRepo();
|
|
5360
|
-
|
|
5361
|
-
|
|
5362
|
-
|
|
5363
|
-
|
|
5507
|
+
const phone = body.phone?.trim() || null;
|
|
5508
|
+
const existing = await repo.findOne({ where: { email } });
|
|
5509
|
+
let contact;
|
|
5510
|
+
if (!existing) {
|
|
5511
|
+
contact = await repo.save(repo.create({ name, email, phone }));
|
|
5512
|
+
} else {
|
|
5513
|
+
const row = existing;
|
|
5514
|
+
if (row.deleted) {
|
|
5515
|
+
await repo.update(row.id, {
|
|
5516
|
+
deleted: false,
|
|
5517
|
+
deletedAt: null,
|
|
5518
|
+
deletedBy: null,
|
|
5519
|
+
name,
|
|
5520
|
+
phone
|
|
5521
|
+
});
|
|
5522
|
+
const refreshed = await repo.findOne({ where: { id: row.id } });
|
|
5523
|
+
if (!refreshed) return json({ error: "Failed to identify", detail: "contact missing after reactivate" }, { status: 500 });
|
|
5524
|
+
contact = refreshed;
|
|
5525
|
+
} else {
|
|
5526
|
+
contact = existing;
|
|
5527
|
+
}
|
|
5364
5528
|
}
|
|
5365
5529
|
const convRepoInst = convRepo();
|
|
5366
|
-
const
|
|
5530
|
+
const contactId = contact.id;
|
|
5531
|
+
const conv = await convRepoInst.save(convRepoInst.create({ contactId }));
|
|
5367
5532
|
return json({
|
|
5368
|
-
contactId
|
|
5533
|
+
contactId,
|
|
5369
5534
|
conversationId: conv.id
|
|
5370
5535
|
});
|
|
5371
5536
|
} catch (err) {
|
|
@@ -8535,6 +8700,13 @@ function getNextAuthOptions(config) {
|
|
|
8535
8700
|
|
|
8536
8701
|
// src/api/crud.ts
|
|
8537
8702
|
var import_typeorm46 = require("typeorm");
|
|
8703
|
+
var CRUD_LOG = "[cms-crud]";
|
|
8704
|
+
function logCrudClientError(op, detail) {
|
|
8705
|
+
console.warn(CRUD_LOG, op, detail);
|
|
8706
|
+
}
|
|
8707
|
+
function logCrudServerError(op, detail) {
|
|
8708
|
+
console.error(CRUD_LOG, op, detail);
|
|
8709
|
+
}
|
|
8538
8710
|
var DATE_COLUMN_TYPES = /* @__PURE__ */ new Set([
|
|
8539
8711
|
"date",
|
|
8540
8712
|
"datetime",
|
|
@@ -8646,6 +8818,13 @@ function createCrudHandler(dataSource, entityMap, options) {
|
|
|
8646
8818
|
if (authError) return authError;
|
|
8647
8819
|
const entity = entityMap[resource];
|
|
8648
8820
|
if (!resource || !entity) {
|
|
8821
|
+
logCrudClientError("GET list", {
|
|
8822
|
+
reason: "invalid_resource",
|
|
8823
|
+
resource,
|
|
8824
|
+
hasEntity: Boolean(entity),
|
|
8825
|
+
entityMapHasLlmAgents: Boolean(entityMap.llm_agents),
|
|
8826
|
+
entityMapKeyCount: Object.keys(entityMap).length
|
|
8827
|
+
});
|
|
8649
8828
|
return json({ error: "Invalid resource" }, { status: 400 });
|
|
8650
8829
|
}
|
|
8651
8830
|
const { searchParams } = new URL(req.url);
|
|
@@ -8853,12 +9032,22 @@ function createCrudHandler(dataSource, entityMap, options) {
|
|
|
8853
9032
|
}
|
|
8854
9033
|
}
|
|
8855
9034
|
where = mergeDeletedFalseWhere(repo, where);
|
|
8856
|
-
|
|
8857
|
-
|
|
8858
|
-
|
|
8859
|
-
|
|
8860
|
-
|
|
8861
|
-
|
|
9035
|
+
let data;
|
|
9036
|
+
let total;
|
|
9037
|
+
try {
|
|
9038
|
+
const r = await repo.findAndCount({
|
|
9039
|
+
skip,
|
|
9040
|
+
take: limit,
|
|
9041
|
+
order: { [sortField]: sortOrder },
|
|
9042
|
+
where
|
|
9043
|
+
});
|
|
9044
|
+
data = r[0];
|
|
9045
|
+
total = r[1];
|
|
9046
|
+
} catch (err) {
|
|
9047
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
9048
|
+
logCrudServerError("GET list query failed", { resource, sortField, sortOrder, message });
|
|
9049
|
+
throw err;
|
|
9050
|
+
}
|
|
8862
9051
|
return json({ total, page, limit, totalPages: Math.ceil(total / limit), data });
|
|
8863
9052
|
},
|
|
8864
9053
|
async POST(req, resource) {
|
|
@@ -8866,12 +9055,25 @@ function createCrudHandler(dataSource, entityMap, options) {
|
|
|
8866
9055
|
if (authError) return authError;
|
|
8867
9056
|
const entity = entityMap[resource];
|
|
8868
9057
|
if (!resource || !entity) {
|
|
9058
|
+
logCrudClientError("POST create", {
|
|
9059
|
+
reason: "invalid_resource",
|
|
9060
|
+
resource,
|
|
9061
|
+
hasEntity: Boolean(entity),
|
|
9062
|
+
entityMapHasLlmAgents: Boolean(entityMap.llm_agents)
|
|
9063
|
+
});
|
|
8869
9064
|
return json({ error: "Invalid resource" }, { status: 400 });
|
|
8870
9065
|
}
|
|
8871
|
-
const
|
|
8872
|
-
if (!
|
|
9066
|
+
const rawPostBody = await req.json();
|
|
9067
|
+
if (!rawPostBody || typeof rawPostBody !== "object" || Object.keys(rawPostBody).length === 0) {
|
|
9068
|
+
logCrudClientError("POST create", {
|
|
9069
|
+
reason: "invalid_request_payload",
|
|
9070
|
+
resource,
|
|
9071
|
+
rawType: rawPostBody == null ? "nullish" : typeof rawPostBody,
|
|
9072
|
+
keyCount: rawPostBody && typeof rawPostBody === "object" ? Object.keys(rawPostBody).length : 0
|
|
9073
|
+
});
|
|
8873
9074
|
return json({ error: "Invalid request payload" }, { status: 400 });
|
|
8874
9075
|
}
|
|
9076
|
+
const body = rawPostBody;
|
|
8875
9077
|
if (resource === "media") {
|
|
8876
9078
|
const b = body;
|
|
8877
9079
|
const kind = b.kind === "folder" ? "folder" : "file";
|
|
@@ -8905,8 +9107,24 @@ function createCrudHandler(dataSource, entityMap, options) {
|
|
|
8905
9107
|
}
|
|
8906
9108
|
}
|
|
8907
9109
|
const repo = dataSource.getRepository(entity);
|
|
8908
|
-
|
|
8909
|
-
|
|
9110
|
+
const persistBody = resource === "media" ? body : pickColumnUpdates(repo, body);
|
|
9111
|
+
if (resource !== "media" && Object.keys(persistBody).length === 0) {
|
|
9112
|
+
logCrudClientError("POST create", {
|
|
9113
|
+
reason: "no_scalar_columns_after_pick",
|
|
9114
|
+
resource,
|
|
9115
|
+
incomingKeys: Object.keys(body)
|
|
9116
|
+
});
|
|
9117
|
+
return json({ error: "Invalid request payload" }, { status: 400 });
|
|
9118
|
+
}
|
|
9119
|
+
sanitizeBodyForEntity(repo, persistBody);
|
|
9120
|
+
let created;
|
|
9121
|
+
try {
|
|
9122
|
+
created = await repo.save(repo.create(persistBody));
|
|
9123
|
+
} catch (err) {
|
|
9124
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
9125
|
+
logCrudServerError("POST create save failed", { resource, message, persistKeys: Object.keys(persistBody) });
|
|
9126
|
+
throw err;
|
|
9127
|
+
}
|
|
8910
9128
|
if (resource === "contacts") {
|
|
8911
9129
|
await syncContactRowToErp(created);
|
|
8912
9130
|
}
|
|
@@ -8921,6 +9139,7 @@ function createCrudHandler(dataSource, entityMap, options) {
|
|
|
8921
9139
|
if (authError) return authError;
|
|
8922
9140
|
const entity = entityMap[resource];
|
|
8923
9141
|
if (!resource || !entity) {
|
|
9142
|
+
logCrudClientError("GET_METADATA", { reason: "invalid_resource", resource });
|
|
8924
9143
|
return json({ error: "Invalid resource" }, { status: 400 });
|
|
8925
9144
|
}
|
|
8926
9145
|
const repo = dataSource.getRepository(entity);
|
|
@@ -8952,11 +9171,18 @@ function createCrudHandler(dataSource, entityMap, options) {
|
|
|
8952
9171
|
if (authError) return authError;
|
|
8953
9172
|
const entity = entityMap[resource];
|
|
8954
9173
|
if (!resource || !entity) {
|
|
9174
|
+
logCrudClientError("BULK_POST", { reason: "invalid_resource", resource });
|
|
8955
9175
|
return json({ error: "Invalid resource" }, { status: 400 });
|
|
8956
9176
|
}
|
|
8957
9177
|
const body = await req.json();
|
|
8958
9178
|
const { records, upsertKey = "id" } = body;
|
|
8959
9179
|
if (!Array.isArray(records) || records.length === 0) {
|
|
9180
|
+
logCrudClientError("BULK_POST", {
|
|
9181
|
+
reason: "records_required",
|
|
9182
|
+
resource,
|
|
9183
|
+
recordsIsArray: Array.isArray(records),
|
|
9184
|
+
recordCount: Array.isArray(records) ? records.length : 0
|
|
9185
|
+
});
|
|
8960
9186
|
return json({ error: "Records array is required" }, { status: 400 });
|
|
8961
9187
|
}
|
|
8962
9188
|
const repo = dataSource.getRepository(entity);
|
|
@@ -8975,6 +9201,7 @@ function createCrudHandler(dataSource, entityMap, options) {
|
|
|
8975
9201
|
});
|
|
8976
9202
|
} catch (error) {
|
|
8977
9203
|
const message = error instanceof Error ? error.message : "Bulk import failed";
|
|
9204
|
+
logCrudClientError("BULK_POST upsert failed", { resource, upsertKey, message });
|
|
8978
9205
|
return json({ error: message }, { status: 400 });
|
|
8979
9206
|
}
|
|
8980
9207
|
},
|
|
@@ -8983,6 +9210,7 @@ function createCrudHandler(dataSource, entityMap, options) {
|
|
|
8983
9210
|
if (authError) return authError;
|
|
8984
9211
|
const entity = entityMap[resource];
|
|
8985
9212
|
if (!resource || !entity) {
|
|
9213
|
+
logCrudClientError("GET_EXPORT", { reason: "invalid_resource", resource });
|
|
8986
9214
|
return json({ error: "Invalid resource" }, { status: 400 });
|
|
8987
9215
|
}
|
|
8988
9216
|
const { searchParams } = new URL(req.url);
|
|
@@ -9037,7 +9265,10 @@ function createCrudByIdHandler(dataSource, entityMap, options) {
|
|
|
9037
9265
|
const authError = await authz(req, resource, "read");
|
|
9038
9266
|
if (authError) return authError;
|
|
9039
9267
|
const entity = entityMap[resource];
|
|
9040
|
-
if (!entity)
|
|
9268
|
+
if (!entity) {
|
|
9269
|
+
logCrudClientError("GET by id", { reason: "invalid_resource", resource, id });
|
|
9270
|
+
return json({ error: "Invalid resource" }, { status: 400 });
|
|
9271
|
+
}
|
|
9041
9272
|
const repo = dataSource.getRepository(entity);
|
|
9042
9273
|
if (resource === "orders") {
|
|
9043
9274
|
const order = await repo.findOne({
|
|
@@ -9096,7 +9327,10 @@ function createCrudByIdHandler(dataSource, entityMap, options) {
|
|
|
9096
9327
|
const authError = await authz(req, resource, "update");
|
|
9097
9328
|
if (authError) return authError;
|
|
9098
9329
|
const entity = entityMap[resource];
|
|
9099
|
-
if (!entity)
|
|
9330
|
+
if (!entity) {
|
|
9331
|
+
logCrudClientError("PUT by id", { reason: "invalid_resource", resource, id });
|
|
9332
|
+
return json({ error: "Invalid resource" }, { status: 400 });
|
|
9333
|
+
}
|
|
9100
9334
|
const rawBody = await req.json();
|
|
9101
9335
|
const repo = dataSource.getRepository(entity);
|
|
9102
9336
|
const numericId = Number(id);
|
|
@@ -9209,7 +9443,10 @@ function createCrudByIdHandler(dataSource, entityMap, options) {
|
|
|
9209
9443
|
const authError = await authz(req, resource, "delete");
|
|
9210
9444
|
if (authError) return authError;
|
|
9211
9445
|
const entity = entityMap[resource];
|
|
9212
|
-
if (!entity)
|
|
9446
|
+
if (!entity) {
|
|
9447
|
+
logCrudClientError("DELETE by id", { reason: "invalid_resource", resource, id });
|
|
9448
|
+
return json({ error: "Invalid resource" }, { status: 400 });
|
|
9449
|
+
}
|
|
9213
9450
|
const repo = dataSource.getRepository(entity);
|
|
9214
9451
|
const numericId = Number(id);
|
|
9215
9452
|
if (entityHasSoftDelete(repo)) {
|
|
@@ -9379,6 +9616,9 @@ var EMBED_CONCURRENCY = 5;
|
|
|
9379
9616
|
var MAX_PDF_BYTES = 25 * 1024 * 1024;
|
|
9380
9617
|
var TEXT_FILE_TYPES = /* @__PURE__ */ new Set(["text/plain", "text/markdown", "application/json"]);
|
|
9381
9618
|
var KB_LOG = "[llm-agent-knowledge]";
|
|
9619
|
+
function logKbPipeline(step, meta) {
|
|
9620
|
+
console.info(`${KB_LOG} pipeline`, { step, ...meta });
|
|
9621
|
+
}
|
|
9382
9622
|
function llmEmbedDebug() {
|
|
9383
9623
|
const v = process.env.LLM_EMBED_DEBUG?.toLowerCase();
|
|
9384
9624
|
return v === "1" || v === "true" || v === "yes";
|
|
@@ -9393,11 +9633,29 @@ async function extractTextFromPdf(buffer) {
|
|
|
9393
9633
|
const data = await pdfParse(buffer);
|
|
9394
9634
|
return (data?.text ?? "").trim();
|
|
9395
9635
|
}
|
|
9396
|
-
async function
|
|
9636
|
+
async function loadChunksMissingEmbeddings(dataSource, documentId, slug) {
|
|
9637
|
+
const rows = await dataSource.query(
|
|
9638
|
+
`SELECT id, content FROM knowledge_base_chunks WHERE "documentId" = $1 AND embedding IS NULL ORDER BY "chunkIndex" ASC`,
|
|
9639
|
+
[documentId]
|
|
9640
|
+
);
|
|
9641
|
+
const list = rows ?? [];
|
|
9642
|
+
logKbPipeline("07_query_chunks_missing_embedding", {
|
|
9643
|
+
slug,
|
|
9644
|
+
documentId,
|
|
9645
|
+
missingEmbeddingCount: list.length
|
|
9646
|
+
});
|
|
9647
|
+
return list;
|
|
9648
|
+
}
|
|
9649
|
+
async function writeEmbeddingsConcurrent(dataSource, embed, chunks, concurrency, slug) {
|
|
9397
9650
|
let next = 0;
|
|
9398
9651
|
let written = 0;
|
|
9399
9652
|
let failed = 0;
|
|
9400
9653
|
let skippedEmpty = 0;
|
|
9654
|
+
logKbPipeline("09_embedding_workers_start", {
|
|
9655
|
+
slug,
|
|
9656
|
+
chunkCount: chunks.length,
|
|
9657
|
+
concurrency: Math.max(1, Math.min(concurrency, chunks.length))
|
|
9658
|
+
});
|
|
9401
9659
|
async function worker() {
|
|
9402
9660
|
for (; ; ) {
|
|
9403
9661
|
const i = next++;
|
|
@@ -9436,9 +9694,12 @@ async function writeEmbeddingsConcurrent(dataSource, embed, chunks, concurrency)
|
|
|
9436
9694
|
skippedEmpty
|
|
9437
9695
|
};
|
|
9438
9696
|
if (skippedEmpty > 0 && written === 0) {
|
|
9439
|
-
summary.hint = "embed() returned empty vectors \u2014 see [LLM embed] logs (
|
|
9697
|
+
summary.hint = "embed() returned empty vectors \u2014 see [LLM embed] / [LLM embed HF] logs (OpenAI gateway vs @huggingface/inference + legacy HTTP).";
|
|
9698
|
+
}
|
|
9699
|
+
logKbPipeline("10_embedding_workers_finished", { slug, ...summary });
|
|
9700
|
+
if (failed > 0 || skippedEmpty > 0 && written === 0) {
|
|
9701
|
+
console.error(`${KB_LOG} embedding pass finished with issues`, summary);
|
|
9440
9702
|
}
|
|
9441
|
-
console.error(`${KB_LOG} embedding pass finished`, summary);
|
|
9442
9703
|
return { written, failed };
|
|
9443
9704
|
}
|
|
9444
9705
|
function splitIntoChunks(text, maxLen) {
|
|
@@ -9474,6 +9735,58 @@ function createLlmAgentKnowledgeHandlers(config) {
|
|
|
9474
9735
|
}
|
|
9475
9736
|
return null;
|
|
9476
9737
|
}
|
|
9738
|
+
async function runEmbeddingPass(slug, savedChunks) {
|
|
9739
|
+
let embeddingsWritten = 0;
|
|
9740
|
+
let embeddingsFailed = 0;
|
|
9741
|
+
try {
|
|
9742
|
+
logKbPipeline("08_resolve_llm_plugin", { slug, chunkCount: savedChunks.length });
|
|
9743
|
+
const cms = await getCms();
|
|
9744
|
+
const llm = cms.getPlugin("llm");
|
|
9745
|
+
if (llm?.embed && savedChunks.length > 0) {
|
|
9746
|
+
logKbPipeline("08b_embed_fn_available", {
|
|
9747
|
+
slug,
|
|
9748
|
+
chunkCount: savedChunks.length,
|
|
9749
|
+
firstChunkId: savedChunks[0]?.id,
|
|
9750
|
+
lastChunkId: savedChunks[savedChunks.length - 1]?.id
|
|
9751
|
+
});
|
|
9752
|
+
const embedBound = (text) => llm.embed(text);
|
|
9753
|
+
const { written, failed } = await writeEmbeddingsConcurrent(
|
|
9754
|
+
dataSource,
|
|
9755
|
+
embedBound,
|
|
9756
|
+
savedChunks,
|
|
9757
|
+
EMBED_CONCURRENCY,
|
|
9758
|
+
slug
|
|
9759
|
+
);
|
|
9760
|
+
embeddingsWritten = written;
|
|
9761
|
+
embeddingsFailed = failed;
|
|
9762
|
+
} else {
|
|
9763
|
+
logKbPipeline("08c_embed_skipped", {
|
|
9764
|
+
slug,
|
|
9765
|
+
hasLlmPlugin: !!llm,
|
|
9766
|
+
hasEmbed: typeof llm?.embed === "function",
|
|
9767
|
+
chunkCount: savedChunks.length,
|
|
9768
|
+
reason: savedChunks.length === 0 ? "no_chunks" : !llm ? "no_llm_plugin" : "no_embed_method"
|
|
9769
|
+
});
|
|
9770
|
+
console.error(`${KB_LOG} embeddings skipped`, {
|
|
9771
|
+
slug,
|
|
9772
|
+
hasLlmPlugin: !!llm,
|
|
9773
|
+
hasEmbed: typeof llm?.embed === "function",
|
|
9774
|
+
chunkCount: savedChunks.length,
|
|
9775
|
+
hint: !llm || typeof llm.embed !== "function" ? "LLM plugin missing or no embed(); check LLM_GATEWAY_URL + LLM_API_KEY so llm plugin initializes." : void 0
|
|
9776
|
+
});
|
|
9777
|
+
}
|
|
9778
|
+
} catch (embErr) {
|
|
9779
|
+
const detail = embErr instanceof Error ? embErr.message : String(embErr);
|
|
9780
|
+
logKbPipeline("08d_embed_pass_exception", { slug, detail });
|
|
9781
|
+
console.error(`${KB_LOG} embedding step threw before/during batch`, { slug, detail, embErr });
|
|
9782
|
+
return {
|
|
9783
|
+
embeddingsWritten: 0,
|
|
9784
|
+
embeddingsFailed: savedChunks.length,
|
|
9785
|
+
embedError: detail
|
|
9786
|
+
};
|
|
9787
|
+
}
|
|
9788
|
+
return { embeddingsWritten, embeddingsFailed };
|
|
9789
|
+
}
|
|
9477
9790
|
return {
|
|
9478
9791
|
async list(req, slug) {
|
|
9479
9792
|
const denied = await gate(req, "read");
|
|
@@ -9502,8 +9815,11 @@ function createLlmAgentKnowledgeHandlers(config) {
|
|
|
9502
9815
|
const denied = await gate(req, "update");
|
|
9503
9816
|
if (denied) return denied;
|
|
9504
9817
|
try {
|
|
9818
|
+
const ct0 = req.headers.get("content-type") || "";
|
|
9819
|
+
logKbPipeline("01_request", { slug, contentType: ct0.slice(0, 80) });
|
|
9505
9820
|
const agent = await findAgentBySlug(dataSource, llmAgents, slug);
|
|
9506
9821
|
if (!agent) return json({ error: "Agent not found" }, { status: 404 });
|
|
9822
|
+
logKbPipeline("02_agent_loaded", { slug, agentId: agent.id });
|
|
9507
9823
|
let name = "";
|
|
9508
9824
|
let text = "";
|
|
9509
9825
|
let sourceUrl = null;
|
|
@@ -9515,6 +9831,14 @@ function createLlmAgentKnowledgeHandlers(config) {
|
|
|
9515
9831
|
name = (body?.name ?? "").trim();
|
|
9516
9832
|
text = (body?.text ?? "").trim();
|
|
9517
9833
|
sourceUrl = typeof body?.sourceUrl === "string" && body.sourceUrl.trim() ? body.sourceUrl.trim() : null;
|
|
9834
|
+
logKbPipeline("03_body_parsed", {
|
|
9835
|
+
slug,
|
|
9836
|
+
mode: "json",
|
|
9837
|
+
existingDocumentId,
|
|
9838
|
+
nameLen: name.length,
|
|
9839
|
+
textChars: text.length,
|
|
9840
|
+
hasSourceUrl: !!sourceUrl
|
|
9841
|
+
});
|
|
9518
9842
|
} else if (ct.includes("multipart/form-data")) {
|
|
9519
9843
|
const form = await req.formData();
|
|
9520
9844
|
name = form.get("name")?.trim() ?? "";
|
|
@@ -9524,9 +9848,17 @@ function createLlmAgentKnowledgeHandlers(config) {
|
|
|
9524
9848
|
const f = file;
|
|
9525
9849
|
const mime = (f.type || "").split(";")[0].trim().toLowerCase();
|
|
9526
9850
|
const buf = Buffer.from(await f.arrayBuffer());
|
|
9851
|
+
logKbPipeline("03_body_parsed", {
|
|
9852
|
+
slug,
|
|
9853
|
+
mode: "multipart",
|
|
9854
|
+
fileName: f.name || "(no name)",
|
|
9855
|
+
mime,
|
|
9856
|
+
fileBytes: buf.length
|
|
9857
|
+
});
|
|
9527
9858
|
if (TEXT_FILE_TYPES.has(mime)) {
|
|
9528
9859
|
const decoded = buf.toString("utf8");
|
|
9529
9860
|
if (!text) text = decoded;
|
|
9861
|
+
logKbPipeline("04_file_decoded_text", { slug, mime, textChars: text.length });
|
|
9530
9862
|
} else if (isPdfUpload(mime, f.name || "")) {
|
|
9531
9863
|
if (buf.length > MAX_PDF_BYTES) {
|
|
9532
9864
|
return json(
|
|
@@ -9535,8 +9867,10 @@ function createLlmAgentKnowledgeHandlers(config) {
|
|
|
9535
9867
|
);
|
|
9536
9868
|
}
|
|
9537
9869
|
try {
|
|
9870
|
+
logKbPipeline("04_pdf_extract_start", { slug, fileBytes: buf.length });
|
|
9538
9871
|
const extracted = await extractTextFromPdf(buf);
|
|
9539
9872
|
if (!text) text = extracted;
|
|
9873
|
+
logKbPipeline("04_pdf_extract_done", { slug, textChars: text.length });
|
|
9540
9874
|
} catch {
|
|
9541
9875
|
return json(
|
|
9542
9876
|
{ error: "Could not read PDF text (file may be encrypted, corrupt, or image-only)" },
|
|
@@ -9552,12 +9886,15 @@ function createLlmAgentKnowledgeHandlers(config) {
|
|
|
9552
9886
|
);
|
|
9553
9887
|
}
|
|
9554
9888
|
if (!name && f.name) name = f.name.replace(/\.[^/.]+$/, "") || f.name;
|
|
9889
|
+
} else {
|
|
9890
|
+
logKbPipeline("03_body_parsed", { slug, mode: "multipart", hasFile: false, textChars: text.length });
|
|
9555
9891
|
}
|
|
9556
9892
|
} else {
|
|
9557
9893
|
return json({ error: "Use application/json or multipart/form-data" }, { status: 400 });
|
|
9558
9894
|
}
|
|
9559
9895
|
const linkRepo = dataSource.getRepository(junction);
|
|
9560
9896
|
if (existingDocumentId != null) {
|
|
9897
|
+
logKbPipeline("05_branch", { slug, branch: "link_existing_document", documentId: existingDocumentId });
|
|
9561
9898
|
const docRepo2 = dataSource.getRepository(kbDoc);
|
|
9562
9899
|
const existing = await docRepo2.findOne({ where: { id: existingDocumentId } });
|
|
9563
9900
|
if (!existing) return json({ error: "documentId not found" }, { status: 404 });
|
|
@@ -9566,9 +9903,137 @@ function createLlmAgentKnowledgeHandlers(config) {
|
|
|
9566
9903
|
});
|
|
9567
9904
|
if (!dup2) {
|
|
9568
9905
|
await linkRepo.save(linkRepo.create({ agentId: agent.id, documentId: existingDocumentId }));
|
|
9906
|
+
logKbPipeline("06_junction_link_created", {
|
|
9907
|
+
slug,
|
|
9908
|
+
agentId: agent.id,
|
|
9909
|
+
documentId: existingDocumentId
|
|
9910
|
+
});
|
|
9911
|
+
} else {
|
|
9912
|
+
logKbPipeline("06_junction_link_exists", {
|
|
9913
|
+
slug,
|
|
9914
|
+
agentId: agent.id,
|
|
9915
|
+
documentId: existingDocumentId
|
|
9916
|
+
});
|
|
9917
|
+
}
|
|
9918
|
+
const chunkRepo2 = dataSource.getRepository(kbChunk);
|
|
9919
|
+
const docRow = existing;
|
|
9920
|
+
const chunkCount = await chunkRepo2.count({ where: { documentId: existingDocumentId } });
|
|
9921
|
+
logKbPipeline("06b_chunks_existing_count", {
|
|
9922
|
+
slug,
|
|
9923
|
+
documentId: existingDocumentId,
|
|
9924
|
+
chunkCount
|
|
9925
|
+
});
|
|
9926
|
+
let chunksToEmbed = [];
|
|
9927
|
+
let chunksCreated = 0;
|
|
9928
|
+
if (chunkCount === 0) {
|
|
9929
|
+
const text2 = (docRow.content ?? "").trim();
|
|
9930
|
+
if (!text2) {
|
|
9931
|
+
logKbPipeline("07_ingest_aborted_empty_document", { slug, documentId: existingDocumentId });
|
|
9932
|
+
return json({
|
|
9933
|
+
documentId: existingDocumentId,
|
|
9934
|
+
linked: true,
|
|
9935
|
+
created: false,
|
|
9936
|
+
chunkCount: 0,
|
|
9937
|
+
embeddingsWritten: 0,
|
|
9938
|
+
embeddingsFailed: 0,
|
|
9939
|
+
warning: "Document has no text content; add content then attach again to build chunks and embeddings."
|
|
9940
|
+
});
|
|
9941
|
+
}
|
|
9942
|
+
const parts2 = splitIntoChunks(text2, INGEST_CHUNK_CHARS);
|
|
9943
|
+
if (parts2.length > MAX_CHUNKS_PER_UPLOAD) {
|
|
9944
|
+
return json(
|
|
9945
|
+
{
|
|
9946
|
+
error: `Document is too large for one ingest (${parts2.length} chunks; max ${MAX_CHUNKS_PER_UPLOAD}). Split into smaller files.`
|
|
9947
|
+
},
|
|
9948
|
+
{ status: 413 }
|
|
9949
|
+
);
|
|
9950
|
+
}
|
|
9951
|
+
logKbPipeline("07_chunk_split", {
|
|
9952
|
+
slug,
|
|
9953
|
+
documentId: existingDocumentId,
|
|
9954
|
+
partCount: parts2.length,
|
|
9955
|
+
maxChunkChars: INGEST_CHUNK_CHARS,
|
|
9956
|
+
totalChars: text2.length
|
|
9957
|
+
});
|
|
9958
|
+
const now2 = /* @__PURE__ */ new Date();
|
|
9959
|
+
const chunkRows2 = parts2.map(
|
|
9960
|
+
(content, i) => chunkRepo2.create({
|
|
9961
|
+
documentId: existingDocumentId,
|
|
9962
|
+
content,
|
|
9963
|
+
chunkIndex: i,
|
|
9964
|
+
createdAt: now2
|
|
9965
|
+
})
|
|
9966
|
+
);
|
|
9967
|
+
const savedList2 = await chunkRepo2.save(chunkRows2);
|
|
9968
|
+
chunksCreated = savedList2.length;
|
|
9969
|
+
chunksToEmbed = savedList2.map((row, i) => ({
|
|
9970
|
+
id: row.id,
|
|
9971
|
+
content: parts2[i]
|
|
9972
|
+
}));
|
|
9973
|
+
logKbPipeline("07b_chunks_persisted", {
|
|
9974
|
+
slug,
|
|
9975
|
+
documentId: existingDocumentId,
|
|
9976
|
+
rowsInserted: chunksCreated
|
|
9977
|
+
});
|
|
9978
|
+
} else {
|
|
9979
|
+
chunksToEmbed = await loadChunksMissingEmbeddings(dataSource, existingDocumentId, slug);
|
|
9980
|
+
if (chunksToEmbed.length === 0) {
|
|
9981
|
+
logKbPipeline("08_skip_embed_all_chunks_have_vectors", {
|
|
9982
|
+
slug,
|
|
9983
|
+
documentId: existingDocumentId,
|
|
9984
|
+
existingChunkCount: chunkCount
|
|
9985
|
+
});
|
|
9986
|
+
}
|
|
9569
9987
|
}
|
|
9570
|
-
|
|
9988
|
+
const embedResult2 = chunksToEmbed.length > 0 ? await runEmbeddingPass(slug, chunksToEmbed) : { embeddingsWritten: 0, embeddingsFailed: 0 };
|
|
9989
|
+
if (embedResult2.embedError) {
|
|
9990
|
+
logKbPipeline("11_response", {
|
|
9991
|
+
slug,
|
|
9992
|
+
branch: "link",
|
|
9993
|
+
documentId: existingDocumentId,
|
|
9994
|
+
ok: false,
|
|
9995
|
+
embeddingError: true
|
|
9996
|
+
});
|
|
9997
|
+
return json(
|
|
9998
|
+
{
|
|
9999
|
+
documentId: existingDocumentId,
|
|
10000
|
+
linked: true,
|
|
10001
|
+
created: false,
|
|
10002
|
+
chunkCount: chunkCount + chunksCreated,
|
|
10003
|
+
chunksCreated,
|
|
10004
|
+
embeddingAttempted: true,
|
|
10005
|
+
embeddingsWritten: 0,
|
|
10006
|
+
embeddingsFailed: chunksToEmbed.length,
|
|
10007
|
+
warning: "Document linked; embedding step failed. Fix LLM/embed config and attach again (or unlink and re-attach) to retry NULL embeddings.",
|
|
10008
|
+
detail: embedResult2.embedError
|
|
10009
|
+
},
|
|
10010
|
+
{ status: 201 }
|
|
10011
|
+
);
|
|
10012
|
+
}
|
|
10013
|
+
logKbPipeline("11_response", {
|
|
10014
|
+
slug,
|
|
10015
|
+
branch: "link",
|
|
10016
|
+
documentId: existingDocumentId,
|
|
10017
|
+
ok: true,
|
|
10018
|
+
chunkCount: chunkCount + chunksCreated,
|
|
10019
|
+
chunksCreated,
|
|
10020
|
+
chunksQueuedForEmbedding: chunksToEmbed.length,
|
|
10021
|
+
embeddingsWritten: embedResult2.embeddingsWritten,
|
|
10022
|
+
embeddingsFailed: embedResult2.embeddingsFailed
|
|
10023
|
+
});
|
|
10024
|
+
return json({
|
|
10025
|
+
documentId: existingDocumentId,
|
|
10026
|
+
linked: true,
|
|
10027
|
+
created: false,
|
|
10028
|
+
chunkCount: chunkCount + chunksCreated,
|
|
10029
|
+
chunksCreated: chunksCreated || void 0,
|
|
10030
|
+
chunksQueuedForEmbedding: chunksToEmbed.length,
|
|
10031
|
+
embeddingAttempted: chunksToEmbed.length > 0,
|
|
10032
|
+
embeddingsWritten: embedResult2.embeddingsWritten,
|
|
10033
|
+
embeddingsFailed: embedResult2.embeddingsFailed
|
|
10034
|
+
});
|
|
9571
10035
|
}
|
|
10036
|
+
logKbPipeline("05_branch", { slug, branch: "new_upload_ingest" });
|
|
9572
10037
|
if (!text) return json({ error: "text or file with text content is required" }, { status: 400 });
|
|
9573
10038
|
if (!name) name = "Untitled";
|
|
9574
10039
|
const parts = splitIntoChunks(text, INGEST_CHUNK_CHARS);
|
|
@@ -9583,6 +10048,13 @@ function createLlmAgentKnowledgeHandlers(config) {
|
|
|
9583
10048
|
{ status: 413 }
|
|
9584
10049
|
);
|
|
9585
10050
|
}
|
|
10051
|
+
logKbPipeline("07_chunk_split", {
|
|
10052
|
+
slug,
|
|
10053
|
+
partCount: parts.length,
|
|
10054
|
+
maxChunkChars: INGEST_CHUNK_CHARS,
|
|
10055
|
+
totalChars: text.length,
|
|
10056
|
+
documentName: name
|
|
10057
|
+
});
|
|
9586
10058
|
const docRepo = dataSource.getRepository(kbDoc);
|
|
9587
10059
|
const chunkRepo = dataSource.getRepository(kbChunk);
|
|
9588
10060
|
const now = /* @__PURE__ */ new Date();
|
|
@@ -9590,6 +10062,13 @@ function createLlmAgentKnowledgeHandlers(config) {
|
|
|
9590
10062
|
docRepo.create({ name, content: text, sourceUrl, createdAt: now, updatedAt: now })
|
|
9591
10063
|
);
|
|
9592
10064
|
const docId = doc.id;
|
|
10065
|
+
logKbPipeline("06_knowledge_document_saved", {
|
|
10066
|
+
slug,
|
|
10067
|
+
documentId: docId,
|
|
10068
|
+
name,
|
|
10069
|
+
contentChars: text.length,
|
|
10070
|
+
hasSourceUrl: !!sourceUrl
|
|
10071
|
+
});
|
|
9593
10072
|
const chunkRows = parts.map(
|
|
9594
10073
|
(content, i) => chunkRepo.create({ documentId: docId, content, chunkIndex: i, createdAt: now })
|
|
9595
10074
|
);
|
|
@@ -9600,63 +10079,66 @@ function createLlmAgentKnowledgeHandlers(config) {
|
|
|
9600
10079
|
content: parts[i]
|
|
9601
10080
|
})
|
|
9602
10081
|
);
|
|
10082
|
+
logKbPipeline("07b_chunks_persisted", {
|
|
10083
|
+
slug,
|
|
10084
|
+
documentId: docId,
|
|
10085
|
+
rowsInserted: savedChunks.length,
|
|
10086
|
+
firstChunkId: savedChunks[0]?.id,
|
|
10087
|
+
lastChunkId: savedChunks[savedChunks.length - 1]?.id
|
|
10088
|
+
});
|
|
9603
10089
|
const dup = await linkRepo.findOne({ where: { agentId: agent.id, documentId: docId } });
|
|
9604
10090
|
if (!dup) {
|
|
9605
10091
|
await linkRepo.save(linkRepo.create({ agentId: agent.id, documentId: docId }));
|
|
10092
|
+
logKbPipeline("06c_junction_link_created", { slug, agentId: agent.id, documentId: docId });
|
|
10093
|
+
} else {
|
|
10094
|
+
logKbPipeline("06c_junction_link_exists", { slug, agentId: agent.id, documentId: docId });
|
|
9606
10095
|
}
|
|
9607
|
-
|
|
9608
|
-
|
|
9609
|
-
|
|
9610
|
-
|
|
9611
|
-
|
|
9612
|
-
|
|
9613
|
-
|
|
9614
|
-
|
|
9615
|
-
|
|
9616
|
-
dataSource,
|
|
9617
|
-
embedBound,
|
|
9618
|
-
savedChunks,
|
|
9619
|
-
EMBED_CONCURRENCY
|
|
9620
|
-
);
|
|
9621
|
-
embeddingsWritten = written;
|
|
9622
|
-
embeddingsFailed = failed;
|
|
9623
|
-
} else {
|
|
9624
|
-
console.error(`${KB_LOG} embeddings skipped`, {
|
|
9625
|
-
slug,
|
|
9626
|
-
hasLlmPlugin: !!llm,
|
|
9627
|
-
hasEmbed: typeof llm?.embed === "function",
|
|
9628
|
-
chunkCount: savedChunks.length,
|
|
9629
|
-
hint: !llm || typeof llm.embed !== "function" ? "LLM plugin missing or no embed(); check LLM_GATEWAY_URL + LLM_API_KEY so llm plugin initializes." : void 0
|
|
9630
|
-
});
|
|
9631
|
-
}
|
|
9632
|
-
} catch (embErr) {
|
|
9633
|
-
const detail = embErr instanceof Error ? embErr.message : String(embErr);
|
|
9634
|
-
console.error(`${KB_LOG} embedding step threw before/during batch`, { slug, detail, embErr });
|
|
10096
|
+
const embedResult = await runEmbeddingPass(slug, savedChunks);
|
|
10097
|
+
if (embedResult.embedError) {
|
|
10098
|
+
logKbPipeline("11_response", {
|
|
10099
|
+
slug,
|
|
10100
|
+
branch: "ingest",
|
|
10101
|
+
documentId: docId,
|
|
10102
|
+
ok: false,
|
|
10103
|
+
embeddingError: true
|
|
10104
|
+
});
|
|
9635
10105
|
return json(
|
|
9636
10106
|
{
|
|
9637
10107
|
documentId: docId,
|
|
9638
10108
|
chunkCount: savedChunks.length,
|
|
10109
|
+
embeddingAttempted: true,
|
|
9639
10110
|
created: true,
|
|
9640
10111
|
linked: true,
|
|
9641
10112
|
embeddingsWritten: 0,
|
|
9642
10113
|
embeddingsFailed: savedChunks.length,
|
|
9643
10114
|
warning: "Document saved and linked; embedding step failed.",
|
|
9644
|
-
detail
|
|
10115
|
+
detail: embedResult.embedError
|
|
9645
10116
|
},
|
|
9646
10117
|
{ status: 201 }
|
|
9647
10118
|
);
|
|
9648
10119
|
}
|
|
10120
|
+
logKbPipeline("11_response", {
|
|
10121
|
+
slug,
|
|
10122
|
+
branch: "ingest",
|
|
10123
|
+
documentId: docId,
|
|
10124
|
+
ok: true,
|
|
10125
|
+
chunkCount: savedChunks.length,
|
|
10126
|
+
embeddingsWritten: embedResult.embeddingsWritten,
|
|
10127
|
+
embeddingsFailed: embedResult.embeddingsFailed
|
|
10128
|
+
});
|
|
9649
10129
|
return json({
|
|
9650
10130
|
documentId: docId,
|
|
9651
10131
|
chunkCount: savedChunks.length,
|
|
10132
|
+
embeddingAttempted: savedChunks.length > 0,
|
|
9652
10133
|
created: true,
|
|
9653
10134
|
linked: true,
|
|
9654
|
-
embeddingsWritten,
|
|
9655
|
-
embeddingsFailed
|
|
10135
|
+
embeddingsWritten: embedResult.embeddingsWritten,
|
|
10136
|
+
embeddingsFailed: embedResult.embeddingsFailed
|
|
9656
10137
|
});
|
|
9657
10138
|
} catch (err) {
|
|
9658
10139
|
const msg = err instanceof Error ? err.message : "Failed to ingest knowledge";
|
|
9659
10140
|
const name = err instanceof Error ? err.name : "";
|
|
10141
|
+
logKbPipeline("99_pipeline_error", { slug, errorName: name, message: msg });
|
|
9660
10142
|
return json({ error: msg, errorName: name || void 0 }, { status: 500 });
|
|
9661
10143
|
}
|
|
9662
10144
|
},
|
|
@@ -9668,6 +10150,7 @@ function createLlmAgentKnowledgeHandlers(config) {
|
|
|
9668
10150
|
try {
|
|
9669
10151
|
const agent = await findAgentBySlug(dataSource, llmAgents, slug);
|
|
9670
10152
|
if (!agent) return json({ error: "Agent not found" }, { status: 404 });
|
|
10153
|
+
logKbPipeline("unlink_junction", { slug, agentId: agent.id, documentId });
|
|
9671
10154
|
const linkRepo = dataSource.getRepository(junction);
|
|
9672
10155
|
await linkRepo.delete({ agentId: agent.id, documentId });
|
|
9673
10156
|
return json({ ok: true });
|
|
@@ -9929,6 +10412,28 @@ function createAdminRolesHandlers(config) {
|
|
|
9929
10412
|
|
|
9930
10413
|
// src/api/cms-api-handler.ts
|
|
9931
10414
|
var KNOWLEDGE_SUFFIX = "knowledge";
|
|
10415
|
+
var CMS_API_LOG = "[cms-api]";
|
|
10416
|
+
function withLlmKnowledgeEntityFallbacks(base) {
|
|
10417
|
+
const keys = [
|
|
10418
|
+
"llm_agents",
|
|
10419
|
+
"llm_agent_knowledge_documents",
|
|
10420
|
+
"knowledge_base_documents",
|
|
10421
|
+
"knowledge_base_chunks"
|
|
10422
|
+
];
|
|
10423
|
+
const patch = {};
|
|
10424
|
+
for (const key of keys) {
|
|
10425
|
+
if (!(key in base) || base[key] == null) {
|
|
10426
|
+
const fallback = CMS_ENTITY_MAP[key];
|
|
10427
|
+
if (fallback) patch[key] = fallback;
|
|
10428
|
+
}
|
|
10429
|
+
}
|
|
10430
|
+
const merged = Object.keys(patch).length > 0 ? { ...base, ...patch } : base;
|
|
10431
|
+
if (!merged.llm_agents) {
|
|
10432
|
+
const agent = CMS_ENTITY_MAP.llm_agents ?? LlmAgent;
|
|
10433
|
+
return { ...merged, llm_agents: agent };
|
|
10434
|
+
}
|
|
10435
|
+
return merged;
|
|
10436
|
+
}
|
|
9932
10437
|
function matchLlmAgentKnowledgeRoute(path) {
|
|
9933
10438
|
const p = path[0] === "api" ? path.slice(1) : path;
|
|
9934
10439
|
if (p[0] !== "llm_agents" || p.length < 2) return null;
|
|
@@ -9966,9 +10471,9 @@ var DEFAULT_EXCLUDE = /* @__PURE__ */ new Set([
|
|
|
9966
10471
|
function createCmsApiHandler(config) {
|
|
9967
10472
|
const {
|
|
9968
10473
|
dataSource,
|
|
9969
|
-
entityMap,
|
|
10474
|
+
entityMap: rawEntityMap,
|
|
9970
10475
|
pathToModel = (s) => s,
|
|
9971
|
-
crudResources
|
|
10476
|
+
crudResources: crudResourcesOverride,
|
|
9972
10477
|
getCms,
|
|
9973
10478
|
userAuth: userAuthConfig,
|
|
9974
10479
|
dashboard,
|
|
@@ -9989,6 +10494,9 @@ function createCmsApiHandler(config) {
|
|
|
9989
10494
|
requireEntityPermission: userRequireEntityPermission,
|
|
9990
10495
|
getSessionUser
|
|
9991
10496
|
} = config;
|
|
10497
|
+
const entityMap = withLlmKnowledgeEntityFallbacks(rawEntityMap);
|
|
10498
|
+
const baseCrudResources = crudResourcesOverride ?? Object.keys(entityMap).filter((k) => !DEFAULT_EXCLUDE.has(k));
|
|
10499
|
+
const crudResources = entityMap.llm_agents && !baseCrudResources.includes("llm_agents") ? [...baseCrudResources, "llm_agents"] : baseCrudResources;
|
|
9992
10500
|
const requireEntityPermissionEffective = userRequireEntityPermission ?? (async (_req, entity, action) => config.json({ error: "Forbidden", reason: "entity_rbac_required", entity, action }, { status: 403 }));
|
|
9993
10501
|
const analytics = analyticsConfig ?? (getCms ? {
|
|
9994
10502
|
json: config.json,
|
|
@@ -10095,7 +10603,7 @@ function createCmsApiHandler(config) {
|
|
|
10095
10603
|
const chatHandlers = chatConfig ? createChatHandlers(chatConfig) : null;
|
|
10096
10604
|
const llmAgentKnowledgeMerged = llmAgentKnowledgeConfig ?? (chatConfig ? {
|
|
10097
10605
|
dataSource: chatConfig.dataSource,
|
|
10098
|
-
entityMap: chatConfig.entityMap,
|
|
10606
|
+
entityMap: withLlmKnowledgeEntityFallbacks(chatConfig.entityMap ?? rawEntityMap),
|
|
10099
10607
|
getCms: chatConfig.getCms,
|
|
10100
10608
|
json: chatConfig.json,
|
|
10101
10609
|
requireAuth: chatConfig.requireAuth
|
|
@@ -10109,7 +10617,9 @@ function createCmsApiHandler(config) {
|
|
|
10109
10617
|
return crudResources.includes(model) ? model : segment;
|
|
10110
10618
|
}
|
|
10111
10619
|
return {
|
|
10112
|
-
async handle(method,
|
|
10620
|
+
async handle(method, pathInput, req) {
|
|
10621
|
+
const m = typeof method === "string" ? method.toUpperCase() : "GET";
|
|
10622
|
+
const path = pathInput.length > 0 && pathInput[0] === "api" ? pathInput.slice(1) : pathInput;
|
|
10113
10623
|
async function analyticsGate() {
|
|
10114
10624
|
const a = await config.requireAuth(req);
|
|
10115
10625
|
if (a) return a;
|
|
@@ -10117,86 +10627,86 @@ function createCmsApiHandler(config) {
|
|
|
10117
10627
|
}
|
|
10118
10628
|
if (path[0] === "admin" && path[1] === "roles") {
|
|
10119
10629
|
if (!adminRoles) return config.json({ error: "Not found" }, { status: 404 });
|
|
10120
|
-
if (path.length === 2 &&
|
|
10121
|
-
if (path.length === 2 &&
|
|
10122
|
-
if (path.length === 3 &&
|
|
10123
|
-
if (path.length === 3 &&
|
|
10124
|
-
if (path.length === 4 && path[3] === "permissions" &&
|
|
10630
|
+
if (path.length === 2 && m === "GET") return adminRoles.list();
|
|
10631
|
+
if (path.length === 2 && m === "POST") return adminRoles.createGroup(req);
|
|
10632
|
+
if (path.length === 3 && m === "PATCH") return adminRoles.patchGroup(req, path[2]);
|
|
10633
|
+
if (path.length === 3 && m === "DELETE") return adminRoles.deleteGroup(path[2]);
|
|
10634
|
+
if (path.length === 4 && path[3] === "permissions" && m === "PUT") return adminRoles.putPermissions(req, path[2]);
|
|
10125
10635
|
return config.json({ error: "Not found" }, { status: 404 });
|
|
10126
10636
|
}
|
|
10127
|
-
if (path[0] === "dashboard" && path[1] === "stats" && path.length === 2 &&
|
|
10637
|
+
if (path[0] === "dashboard" && path[1] === "stats" && path.length === 2 && m === "GET" && dashboardGet) {
|
|
10128
10638
|
return dashboardGet(req);
|
|
10129
10639
|
}
|
|
10130
|
-
if (path[0] === "dashboard" && path[1] === "ecommerce" && path.length === 2 &&
|
|
10640
|
+
if (path[0] === "dashboard" && path[1] === "ecommerce" && path.length === 2 && m === "GET" && ecommerceAnalyticsGet) {
|
|
10131
10641
|
const g = await analyticsGate();
|
|
10132
10642
|
if (g) return g;
|
|
10133
10643
|
return ecommerceAnalyticsGet(req);
|
|
10134
10644
|
}
|
|
10135
10645
|
if (path[0] === "analytics" && analyticsHandlers) {
|
|
10136
|
-
if (path.length === 1 &&
|
|
10646
|
+
if (path.length === 1 && m === "GET") {
|
|
10137
10647
|
const g = await analyticsGate();
|
|
10138
10648
|
if (g) return g;
|
|
10139
10649
|
return analyticsHandlers.GET(req);
|
|
10140
10650
|
}
|
|
10141
|
-
if (path.length === 2 && path[1] === "property-id" &&
|
|
10651
|
+
if (path.length === 2 && path[1] === "property-id" && m === "GET") {
|
|
10142
10652
|
const g = await analyticsGate();
|
|
10143
10653
|
if (g) return g;
|
|
10144
10654
|
return analyticsHandlers.propertyId();
|
|
10145
10655
|
}
|
|
10146
|
-
if (path.length === 2 && path[1] === "permissions" &&
|
|
10656
|
+
if (path.length === 2 && path[1] === "permissions" && m === "GET") {
|
|
10147
10657
|
const g = await analyticsGate();
|
|
10148
10658
|
if (g) return g;
|
|
10149
10659
|
return analyticsHandlers.permissions();
|
|
10150
10660
|
}
|
|
10151
10661
|
}
|
|
10152
|
-
if (path[0] === "upload" && path.length === 1 &&
|
|
10153
|
-
if (path[0] === "media" && path[1] === "extract" && path.length === 3 &&
|
|
10662
|
+
if (path[0] === "upload" && path.length === 1 && m === "POST" && uploadPost) return uploadPost(req);
|
|
10663
|
+
if (path[0] === "media" && path[1] === "extract" && path.length === 3 && m === "POST" && zipExtractPost) {
|
|
10154
10664
|
return zipExtractPost(req, path[2]);
|
|
10155
10665
|
}
|
|
10156
|
-
if (path[0] === "blogs" && path[1] === "slug" && path.length === 3 &&
|
|
10666
|
+
if (path[0] === "blogs" && path[1] === "slug" && path.length === 3 && m === "GET" && blogBySlugGet) {
|
|
10157
10667
|
return blogBySlugGet(req, path[2]);
|
|
10158
10668
|
}
|
|
10159
|
-
if (path[0] === "forms" && path[1] === "slug" && path.length === 3 &&
|
|
10669
|
+
if (path[0] === "forms" && path[1] === "slug" && path.length === 3 && m === "GET" && formBySlugGet) {
|
|
10160
10670
|
return formBySlugGet(req, path[2]);
|
|
10161
10671
|
}
|
|
10162
10672
|
if (path[0] === "form-submissions") {
|
|
10163
10673
|
if (path.length === 1) {
|
|
10164
|
-
if (
|
|
10165
|
-
if (
|
|
10674
|
+
if (m === "GET" && formSubmissionList) return formSubmissionList(req);
|
|
10675
|
+
if (m === "POST" && formSubmissionPost) return formSubmissionPost(req);
|
|
10166
10676
|
}
|
|
10167
|
-
if (path.length === 2 &&
|
|
10677
|
+
if (path.length === 2 && m === "GET" && formSubmissionGetById) return formSubmissionGetById(req, path[1]);
|
|
10168
10678
|
}
|
|
10169
10679
|
if (path[0] === "forms" && formSaveHandlers) {
|
|
10170
|
-
if (path.length === 1 &&
|
|
10680
|
+
if (path.length === 1 && m === "POST") return formSaveHandlers.POST(req);
|
|
10171
10681
|
if (path.length === 2) {
|
|
10172
|
-
if (
|
|
10173
|
-
if (
|
|
10682
|
+
if (m === "GET") return formSaveHandlers.GET(req, path[1]);
|
|
10683
|
+
if (m === "PUT" || m === "PATCH") return formSaveHandlers.PUT(req, path[1]);
|
|
10174
10684
|
}
|
|
10175
10685
|
}
|
|
10176
10686
|
if (path[0] === "users" && usersHandlers) {
|
|
10177
10687
|
if (path.length === 1) {
|
|
10178
|
-
if (
|
|
10179
|
-
if (
|
|
10688
|
+
if (m === "GET") return usersHandlers.list(req);
|
|
10689
|
+
if (m === "POST") return usersHandlers.create(req);
|
|
10180
10690
|
}
|
|
10181
10691
|
if (path.length === 2) {
|
|
10182
|
-
if (path[1] === "avatar" &&
|
|
10183
|
-
if (path[1] === "profile" &&
|
|
10692
|
+
if (path[1] === "avatar" && m === "POST" && avatarPost) return avatarPost(req);
|
|
10693
|
+
if (path[1] === "profile" && m === "PUT" && profilePut) return profilePut(req);
|
|
10184
10694
|
const id = path[1];
|
|
10185
|
-
if (
|
|
10186
|
-
if (
|
|
10187
|
-
if (
|
|
10695
|
+
if (m === "GET") return usersHandlers.getById(req, id);
|
|
10696
|
+
if (m === "PUT" || m === "PATCH") return usersHandlers.update(req, id);
|
|
10697
|
+
if (m === "DELETE") return usersHandlers.delete(req, id);
|
|
10188
10698
|
}
|
|
10189
|
-
if (path.length === 3 && path[2] === "regenerate-invite" &&
|
|
10699
|
+
if (path.length === 3 && path[2] === "regenerate-invite" && m === "POST") {
|
|
10190
10700
|
return usersHandlers.regenerateInvite(req, path[1]);
|
|
10191
10701
|
}
|
|
10192
10702
|
}
|
|
10193
|
-
if (path[0] === "users" && path.length === 2 && userAuthRouter &&
|
|
10703
|
+
if (path[0] === "users" && path.length === 2 && userAuthRouter && m === "POST") {
|
|
10194
10704
|
return userAuthRouter.POST(req, path[1]);
|
|
10195
10705
|
}
|
|
10196
10706
|
if (path[0] === "settings" && path.length === 2 && settingsHandlers) {
|
|
10197
10707
|
const group = path[1];
|
|
10198
10708
|
const isPublic = settingsConfig?.publicGetGroups?.includes(group);
|
|
10199
|
-
if (
|
|
10709
|
+
if (m === "GET") {
|
|
10200
10710
|
if (!isPublic) {
|
|
10201
10711
|
const a = await config.requireAuth(req);
|
|
10202
10712
|
if (a) return a;
|
|
@@ -10205,15 +10715,38 @@ function createCmsApiHandler(config) {
|
|
|
10205
10715
|
}
|
|
10206
10716
|
return settingsHandlers.GET(req, group);
|
|
10207
10717
|
}
|
|
10208
|
-
if (
|
|
10718
|
+
if (m === "PUT") {
|
|
10209
10719
|
const pe = await requireEntityPermissionEffective(req, "settings", "update");
|
|
10210
10720
|
if (pe) return pe;
|
|
10211
10721
|
return settingsHandlers.PUT(req, group);
|
|
10212
10722
|
}
|
|
10213
10723
|
}
|
|
10214
10724
|
if (path[0] === "message-templates" && path[1] === "sms" && path.length === 2) {
|
|
10215
|
-
if (
|
|
10216
|
-
if (
|
|
10725
|
+
if (m === "GET") return smsMessageTemplateHandlers.GET(req);
|
|
10726
|
+
if (m === "PUT") return smsMessageTemplateHandlers.PUT(req);
|
|
10727
|
+
}
|
|
10728
|
+
if (path[0] === "llm_agents") {
|
|
10729
|
+
if (!entityMap.llm_agents) {
|
|
10730
|
+
console.error(CMS_API_LOG, "llm_agents route: entity missing after merge", {
|
|
10731
|
+
method: m,
|
|
10732
|
+
pathJoined: path.join("/"),
|
|
10733
|
+
entityMapKeyCount: Object.keys(entityMap).length
|
|
10734
|
+
});
|
|
10735
|
+
return config.json(
|
|
10736
|
+
{ error: "LlmAgent entity is not registered", hint: "Ensure migrations ran and entities include LlmAgent." },
|
|
10737
|
+
{ status: 503 }
|
|
10738
|
+
);
|
|
10739
|
+
}
|
|
10740
|
+
if (path.length === 1) {
|
|
10741
|
+
if (m === "GET") return crud.GET(req, "llm_agents");
|
|
10742
|
+
if (m === "POST") return crud.POST(req, "llm_agents");
|
|
10743
|
+
}
|
|
10744
|
+
if (path.length === 2 && !path[1]?.includes("knowledge")) {
|
|
10745
|
+
const id = path[1];
|
|
10746
|
+
if (m === "GET") return crudById.GET(req, "llm_agents", id);
|
|
10747
|
+
if (m === "PUT" || m === "PATCH") return crudById.PUT(req, "llm_agents", id);
|
|
10748
|
+
if (m === "DELETE") return crudById.DELETE(req, "llm_agents", id);
|
|
10749
|
+
}
|
|
10217
10750
|
}
|
|
10218
10751
|
{
|
|
10219
10752
|
const kbMatch = matchLlmAgentKnowledgeRoute(path);
|
|
@@ -10228,24 +10761,24 @@ function createCmsApiHandler(config) {
|
|
|
10228
10761
|
);
|
|
10229
10762
|
}
|
|
10230
10763
|
const { slug, documentId } = kbMatch;
|
|
10231
|
-
if (
|
|
10764
|
+
if (m === "DELETE" && documentId != null && documentId !== "") {
|
|
10232
10765
|
return llmAgentKnowledgeHandlers.unlink(req, slug, documentId);
|
|
10233
10766
|
}
|
|
10234
|
-
if (
|
|
10767
|
+
if (m === "GET" && (documentId == null || documentId === "")) {
|
|
10235
10768
|
return llmAgentKnowledgeHandlers.list(req, slug);
|
|
10236
10769
|
}
|
|
10237
|
-
if (
|
|
10770
|
+
if (m === "POST" && (documentId == null || documentId === "")) {
|
|
10238
10771
|
return llmAgentKnowledgeHandlers.post(req, slug);
|
|
10239
10772
|
}
|
|
10240
10773
|
}
|
|
10241
10774
|
}
|
|
10242
10775
|
if (path[0] === "chat" && chatHandlers) {
|
|
10243
|
-
if (path.length === 2 && path[1] === "config" &&
|
|
10244
|
-
if (path.length === 2 && path[1] === "identify" &&
|
|
10245
|
-
if (path.length === 4 && path[1] === "conversations" && path[3] === "messages" &&
|
|
10246
|
-
if (path.length === 2 && path[1] === "messages" &&
|
|
10776
|
+
if (path.length === 2 && path[1] === "config" && m === "GET") return chatHandlers.publicConfig(req);
|
|
10777
|
+
if (path.length === 2 && path[1] === "identify" && m === "POST") return chatHandlers.identify(req);
|
|
10778
|
+
if (path.length === 4 && path[1] === "conversations" && path[3] === "messages" && m === "GET") return chatHandlers.getMessages(req, path[2]);
|
|
10779
|
+
if (path.length === 2 && path[1] === "messages" && m === "POST") return chatHandlers.postMessage(req);
|
|
10247
10780
|
}
|
|
10248
|
-
if (path[0] === "orders" && path.length === 3 && path[2] === "invoice" &&
|
|
10781
|
+
if (path[0] === "orders" && path.length === 3 && path[2] === "invoice" && m === "GET" && getCms) {
|
|
10249
10782
|
const a = await config.requireAuth(req);
|
|
10250
10783
|
if (a) return a;
|
|
10251
10784
|
const pe = await requireEntityPermissionEffective(req, "orders", "read");
|
|
@@ -10259,17 +10792,17 @@ function createCmsApiHandler(config) {
|
|
|
10259
10792
|
if (path[0] === "orders" && path.length === 3 && path[2] === "repost-erp" && getCms) {
|
|
10260
10793
|
const a = await config.requireAuth(req);
|
|
10261
10794
|
if (a) return a;
|
|
10262
|
-
const pe = await requireEntityPermissionEffective(req, "orders",
|
|
10795
|
+
const pe = await requireEntityPermissionEffective(req, "orders", m === "GET" ? "read" : "update");
|
|
10263
10796
|
if (pe) return pe;
|
|
10264
10797
|
const oid = Number(path[1]);
|
|
10265
10798
|
if (!Number.isFinite(oid)) return config.json({ error: "Invalid id" }, { status: 400 });
|
|
10266
10799
|
const cms = await getCms();
|
|
10267
10800
|
const { isErpIntegrationEnabled: isErpIntegrationEnabled3 } = await Promise.resolve().then(() => (init_erp_config_enabled(), erp_config_enabled_exports));
|
|
10268
10801
|
const enabled = await isErpIntegrationEnabled3(cms, dataSource, entityMap);
|
|
10269
|
-
if (
|
|
10802
|
+
if (m === "GET") {
|
|
10270
10803
|
return config.json({ enabled });
|
|
10271
10804
|
}
|
|
10272
|
-
if (
|
|
10805
|
+
if (m === "POST") {
|
|
10273
10806
|
if (!enabled) return config.json({ error: "ERP integration is disabled" }, { status: 409 });
|
|
10274
10807
|
const { queueErpPaidOrderForOrderId: queueErpPaidOrderForOrderId2 } = await Promise.resolve().then(() => (init_paid_order_erp(), paid_order_erp_exports));
|
|
10275
10808
|
await queueErpPaidOrderForOrderId2(cms, dataSource, entityMap, oid);
|
|
@@ -10279,28 +10812,39 @@ function createCmsApiHandler(config) {
|
|
|
10279
10812
|
}
|
|
10280
10813
|
if (path.length === 0) return config.json({ error: "Not found" }, { status: 404 });
|
|
10281
10814
|
const resource = resolveResource(path[0]);
|
|
10282
|
-
if (!crudResources.includes(resource))
|
|
10815
|
+
if (!crudResources.includes(resource)) {
|
|
10816
|
+
console.warn(CMS_API_LOG, "generic CRUD gate: Invalid resource (not in crudResources)", {
|
|
10817
|
+
method: m,
|
|
10818
|
+
pathJoined: path.join("/"),
|
|
10819
|
+
pathSegments: path,
|
|
10820
|
+
resource,
|
|
10821
|
+
hasLlmAgentsEntity: Boolean(entityMap.llm_agents),
|
|
10822
|
+
crudResourcesHasResource: crudResources.includes(resource),
|
|
10823
|
+
crudResourcesLength: crudResources.length
|
|
10824
|
+
});
|
|
10825
|
+
return config.json({ error: "Invalid resource" }, { status: 400 });
|
|
10826
|
+
}
|
|
10283
10827
|
if (path.length === 2) {
|
|
10284
|
-
if (path[1] === "metadata" &&
|
|
10828
|
+
if (path[1] === "metadata" && m === "GET") {
|
|
10285
10829
|
return crud.GET_METADATA(req, resource);
|
|
10286
10830
|
}
|
|
10287
|
-
if (path[1] === "bulk" &&
|
|
10831
|
+
if (path[1] === "bulk" && m === "POST") {
|
|
10288
10832
|
return crud.BULK_POST(req, resource);
|
|
10289
10833
|
}
|
|
10290
|
-
if (path[1] === "export" &&
|
|
10834
|
+
if (path[1] === "export" && m === "GET") {
|
|
10291
10835
|
return crud.GET_EXPORT(req, resource);
|
|
10292
10836
|
}
|
|
10293
10837
|
}
|
|
10294
10838
|
if (path.length === 1) {
|
|
10295
|
-
if (
|
|
10296
|
-
if (
|
|
10839
|
+
if (m === "GET") return crud.GET(req, resource);
|
|
10840
|
+
if (m === "POST") return crud.POST(req, resource);
|
|
10297
10841
|
return config.json({ error: "Method not allowed" }, { status: 405 });
|
|
10298
10842
|
}
|
|
10299
10843
|
if (path.length === 2) {
|
|
10300
10844
|
const id = path[1];
|
|
10301
|
-
if (
|
|
10302
|
-
if (
|
|
10303
|
-
if (
|
|
10845
|
+
if (m === "GET") return crudById.GET(req, resource, id);
|
|
10846
|
+
if (m === "PUT" || m === "PATCH") return crudById.PUT(req, resource, id);
|
|
10847
|
+
if (m === "DELETE") return crudById.DELETE(req, resource, id);
|
|
10304
10848
|
return config.json({ error: "Method not allowed" }, { status: 405 });
|
|
10305
10849
|
}
|
|
10306
10850
|
return config.json({ error: "Not found" }, { status: 404 });
|