@infuro/cms-core 1.0.20 → 1.0.21
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/admin.cjs +194 -75
- package/dist/admin.cjs.map +1 -1
- package/dist/admin.js +188 -70
- package/dist/admin.js.map +1 -1
- package/dist/api.cjs +3212 -259
- package/dist/api.cjs.map +1 -1
- package/dist/api.d.cts +1 -1
- package/dist/api.d.ts +1 -1
- package/dist/api.js +3214 -252
- package/dist/api.js.map +1 -1
- package/dist/{index-DGtM2Gsk.d.cts → index-BGAh4fPQ.d.cts} +18 -1
- package/dist/{index--GBYw5JE.d.ts → index-Cnwh7B3r.d.ts} +18 -1
- package/dist/index.cjs +729 -203
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +34 -9
- package/dist/index.d.ts +34 -9
- package/dist/index.js +729 -203
- package/dist/index.js.map +1 -1
- package/dist/migrations/1775300000000-LlmAgents.ts +57 -57
- package/dist/migrations/1775300000001-LlmAgentsValidationRulesText.ts +43 -43
- package/dist/migrations/1775300000002-SeedLlmAgentsPermissions.ts +33 -33
- package/dist/migrations/1775300000003-LlmAgentKnowledgeDocuments.ts +50 -50
- package/package.json +2 -1
package/dist/index.cjs
CHANGED
|
@@ -2613,6 +2613,17 @@ function localStoragePlugin(config = {}) {
|
|
|
2613
2613
|
}
|
|
2614
2614
|
|
|
2615
2615
|
// src/plugins/llm/llm-service.ts
|
|
2616
|
+
function parseOpenAiEmbeddingJson(data) {
|
|
2617
|
+
let embedding = data?.data?.[0]?.embedding;
|
|
2618
|
+
if (!Array.isArray(embedding) && Array.isArray(data?.embedding)) {
|
|
2619
|
+
embedding = data.embedding;
|
|
2620
|
+
}
|
|
2621
|
+
if (!Array.isArray(embedding) && Array.isArray(data?.data?.[0])) {
|
|
2622
|
+
const first = data.data[0];
|
|
2623
|
+
embedding = Array.isArray(first) ? first : first?.embedding;
|
|
2624
|
+
}
|
|
2625
|
+
return Array.isArray(embedding) ? embedding : null;
|
|
2626
|
+
}
|
|
2616
2627
|
function parseHfInferenceEmbeddingBody(data) {
|
|
2617
2628
|
if (data === null || data === void 0) return null;
|
|
2618
2629
|
if (Array.isArray(data)) {
|
|
@@ -2634,28 +2645,59 @@ function parseHfInferenceEmbeddingBody(data) {
|
|
|
2634
2645
|
}
|
|
2635
2646
|
return null;
|
|
2636
2647
|
}
|
|
2648
|
+
var FALLBACK_CHAT_MODEL_OPENAI_SHAPE = "openai/gpt-4o-mini";
|
|
2649
|
+
var FALLBACK_CHAT_MODEL_HUGGINGFACE = "meta-llama/Meta-Llama-3.1-8B-Instruct";
|
|
2650
|
+
function defaultChatModelForGateway(baseURL) {
|
|
2651
|
+
const u = baseURL.toLowerCase();
|
|
2652
|
+
if (u.includes("huggingface.co") || u.includes("hf.co") || u.includes("hf-inference")) {
|
|
2653
|
+
return FALLBACK_CHAT_MODEL_HUGGINGFACE;
|
|
2654
|
+
}
|
|
2655
|
+
return FALLBACK_CHAT_MODEL_OPENAI_SHAPE;
|
|
2656
|
+
}
|
|
2657
|
+
function formatLlmGatewayError(status, bodyText, defaultChatModel) {
|
|
2658
|
+
let hint = "";
|
|
2659
|
+
try {
|
|
2660
|
+
const j = JSON.parse(bodyText);
|
|
2661
|
+
const code = j?.error?.code;
|
|
2662
|
+
if (code === "model_not_supported" || code === "model_not_found" || /not supported by any provider|does not exist/i.test(j?.error?.message ?? "")) {
|
|
2663
|
+
hint = ` Hint: set LLM_CHAT_MODEL (or CMS) to a model your gateway lists. HF router: Hub ids (e.g. "${FALLBACK_CHAT_MODEL_HUGGINGFACE}"); OpenRouter: e.g. "${FALLBACK_CHAT_MODEL_OPENAI_SHAPE}".`;
|
|
2664
|
+
}
|
|
2665
|
+
} catch {
|
|
2666
|
+
}
|
|
2667
|
+
return `LLM gateway error: ${status} ${bodyText}${hint}`;
|
|
2668
|
+
}
|
|
2637
2669
|
var LlmService = class _LlmService {
|
|
2638
2670
|
static embedHttpErrorLogged = false;
|
|
2639
2671
|
static embedShapeErrorLogged = false;
|
|
2640
2672
|
static embedSuccessLogged = false;
|
|
2641
2673
|
static hfDimensionMismatchWarned = false;
|
|
2674
|
+
static hfInferenceSdkErrorLogged = false;
|
|
2642
2675
|
baseURL;
|
|
2643
2676
|
apiKey;
|
|
2644
2677
|
defaultEmbeddingModel;
|
|
2645
2678
|
embeddingProvider;
|
|
2646
2679
|
hfInferenceBaseURL;
|
|
2647
2680
|
embeddingApiKey;
|
|
2681
|
+
defaultChatModel;
|
|
2648
2682
|
constructor(baseURL, apiKey, defaultEmbeddingModel, embedOpts) {
|
|
2649
2683
|
this.baseURL = baseURL.replace(/\/$/, "");
|
|
2650
2684
|
this.apiKey = apiKey;
|
|
2651
2685
|
this.defaultEmbeddingModel = defaultEmbeddingModel;
|
|
2652
2686
|
const p = (embedOpts?.embeddingProvider ?? "openai").toLowerCase();
|
|
2653
2687
|
this.embeddingProvider = p === "huggingface" ? "huggingface" : "openai";
|
|
2654
|
-
this.hfInferenceBaseURL = (embedOpts?.hfInferenceBaseUrl ?? "https://
|
|
2688
|
+
this.hfInferenceBaseURL = (embedOpts?.hfInferenceBaseUrl ?? "https://router.huggingface.co/v1").replace(
|
|
2655
2689
|
/\/$/,
|
|
2656
2690
|
""
|
|
2657
2691
|
);
|
|
2658
2692
|
this.embeddingApiKey = (embedOpts?.embeddingApiKey ?? apiKey).trim();
|
|
2693
|
+
const fromOpts = embedOpts?.defaultChatModel?.trim();
|
|
2694
|
+
const fromEnv = typeof process !== "undefined" ? process.env.LLM_CHAT_MODEL?.trim() : "";
|
|
2695
|
+
const auto = defaultChatModelForGateway(this.baseURL);
|
|
2696
|
+
this.defaultChatModel = (fromOpts || fromEnv || auto).trim() || auto;
|
|
2697
|
+
}
|
|
2698
|
+
resolveChatModel(explicit) {
|
|
2699
|
+
const m = (explicit?.trim() || this.defaultChatModel).trim();
|
|
2700
|
+
return m || defaultChatModelForGateway(this.baseURL);
|
|
2659
2701
|
}
|
|
2660
2702
|
get base() {
|
|
2661
2703
|
return this.baseURL.replace(/\/$/, "");
|
|
@@ -2695,7 +2737,7 @@ var LlmService = class _LlmService {
|
|
|
2695
2737
|
}
|
|
2696
2738
|
logChatCompletionsRequest(kind, messages, options, stream) {
|
|
2697
2739
|
if (!this.llmDebugEnabled()) return;
|
|
2698
|
-
const model = options.model
|
|
2740
|
+
const model = this.resolveChatModel(options.model);
|
|
2699
2741
|
console.log(`
|
|
2700
2742
|
[LLM ${kind}] POST`, this.url);
|
|
2701
2743
|
console.log("[Request options]", {
|
|
@@ -2713,7 +2755,12 @@ var LlmService = class _LlmService {
|
|
|
2713
2755
|
return this.embedHuggingFaceInference(input, model2);
|
|
2714
2756
|
}
|
|
2715
2757
|
const model = options.model ?? this.defaultEmbeddingModel ?? "text-embedding-3-small";
|
|
2716
|
-
|
|
2758
|
+
return this.fetchOpenAiShapeEmbeddings(`${this.base}/v1/embeddings`, model, input, this.apiKey);
|
|
2759
|
+
}
|
|
2760
|
+
/**
|
|
2761
|
+
* POST `{ url }` with OpenAI-style `{ model, input }` and Bearer token; parses `{ data:[{embedding}] }`.
|
|
2762
|
+
*/
|
|
2763
|
+
async fetchOpenAiShapeEmbeddings(embedUrl, model, input, bearerToken) {
|
|
2717
2764
|
if (this.embedDebugEnabled()) {
|
|
2718
2765
|
console.info("[LLM embed] request", { url: embedUrl, model, inputChars: input.length });
|
|
2719
2766
|
}
|
|
@@ -2721,7 +2768,7 @@ var LlmService = class _LlmService {
|
|
|
2721
2768
|
method: "POST",
|
|
2722
2769
|
headers: {
|
|
2723
2770
|
"Content-Type": "application/json",
|
|
2724
|
-
Authorization: `Bearer ${
|
|
2771
|
+
Authorization: `Bearer ${bearerToken}`
|
|
2725
2772
|
},
|
|
2726
2773
|
body: JSON.stringify({
|
|
2727
2774
|
model,
|
|
@@ -2730,7 +2777,7 @@ var LlmService = class _LlmService {
|
|
|
2730
2777
|
});
|
|
2731
2778
|
if (!res.ok) {
|
|
2732
2779
|
const body = await res.text();
|
|
2733
|
-
const hint404 = res.status === 404 ? "This base URL does not expose OpenAI-compatible POST /v1/embeddings
|
|
2780
|
+
const hint404 = res.status === 404 ? "This base URL does not expose OpenAI-compatible POST /v1/embeddings. For Hugging Face embeddings, set EMBEDDING_PROVIDER=huggingface and an HF token (EMBEDDING_API_KEY); the server uses @huggingface/inference (router /v1/embeddings is chat-only and is not used)." : "Embeddings HTTP error \u2014 knowledge_base_chunks.embedding will stay NULL for failed calls.";
|
|
2734
2781
|
if (!_LlmService.embedHttpErrorLogged) {
|
|
2735
2782
|
_LlmService.embedHttpErrorLogged = true;
|
|
2736
2783
|
console.error("[LLM embed]", hint404, {
|
|
@@ -2754,15 +2801,8 @@ var LlmService = class _LlmService {
|
|
|
2754
2801
|
}
|
|
2755
2802
|
return [];
|
|
2756
2803
|
}
|
|
2757
|
-
|
|
2758
|
-
if (!
|
|
2759
|
-
embedding = data.embedding;
|
|
2760
|
-
}
|
|
2761
|
-
if (!Array.isArray(embedding) && Array.isArray(data?.data?.[0])) {
|
|
2762
|
-
const first = data.data[0];
|
|
2763
|
-
embedding = Array.isArray(first) ? first : first?.embedding;
|
|
2764
|
-
}
|
|
2765
|
-
if (!Array.isArray(embedding)) {
|
|
2804
|
+
const embedding = parseOpenAiEmbeddingJson(data);
|
|
2805
|
+
if (!embedding) {
|
|
2766
2806
|
if (!_LlmService.embedShapeErrorLogged) {
|
|
2767
2807
|
_LlmService.embedShapeErrorLogged = true;
|
|
2768
2808
|
console.error(
|
|
@@ -2784,96 +2824,201 @@ var LlmService = class _LlmService {
|
|
|
2784
2824
|
}
|
|
2785
2825
|
return embedding;
|
|
2786
2826
|
}
|
|
2787
|
-
|
|
2788
|
-
|
|
2789
|
-
|
|
2790
|
-
|
|
2791
|
-
|
|
2792
|
-
|
|
2793
|
-
|
|
2794
|
-
|
|
2795
|
-
|
|
2796
|
-
|
|
2797
|
-
|
|
2798
|
-
|
|
2799
|
-
let embedUrl = pipelineUrl;
|
|
2800
|
-
let res = await fetch(embedUrl, {
|
|
2801
|
-
method: "POST",
|
|
2802
|
-
headers: {
|
|
2803
|
-
"Content-Type": "application/json",
|
|
2804
|
-
Authorization: `Bearer ${this.embeddingApiKey}`
|
|
2805
|
-
},
|
|
2806
|
-
body: requestBody
|
|
2807
|
-
});
|
|
2808
|
-
let bodyText = await res.text();
|
|
2809
|
-
if (res.status === 404) {
|
|
2810
|
-
embedUrl = modelUrl;
|
|
2811
|
-
if (this.embedDebugEnabled()) {
|
|
2812
|
-
console.warn("[LLM embed HF] /pipeline route returned 404; retrying with /models.", {
|
|
2813
|
-
model,
|
|
2814
|
-
fallbackUrl: embedUrl
|
|
2815
|
-
});
|
|
2816
|
-
}
|
|
2817
|
-
res = await fetch(embedUrl, {
|
|
2818
|
-
method: "POST",
|
|
2819
|
-
headers: {
|
|
2820
|
-
"Content-Type": "application/json",
|
|
2821
|
-
Authorization: `Bearer ${this.embeddingApiKey}`
|
|
2822
|
-
},
|
|
2823
|
-
body: requestBody
|
|
2827
|
+
/**
|
|
2828
|
+
* Hugging Face–recommended path: Hub resolves a provider that supports `feature-extraction` for the model.
|
|
2829
|
+
*/
|
|
2830
|
+
async embedViaHuggingfaceInferenceSdk(input, model) {
|
|
2831
|
+
const token = this.embeddingApiKey.trim();
|
|
2832
|
+
if (!token) return null;
|
|
2833
|
+
try {
|
|
2834
|
+
const { InferenceClient } = await import("@huggingface/inference");
|
|
2835
|
+
const hf = new InferenceClient(token);
|
|
2836
|
+
const raw = await hf.featureExtraction({
|
|
2837
|
+
model,
|
|
2838
|
+
inputs: input
|
|
2824
2839
|
});
|
|
2825
|
-
|
|
2826
|
-
|
|
2827
|
-
|
|
2828
|
-
|
|
2829
|
-
|
|
2830
|
-
|
|
2831
|
-
|
|
2832
|
-
|
|
2833
|
-
|
|
2834
|
-
|
|
2835
|
-
|
|
2836
|
-
|
|
2837
|
-
|
|
2838
|
-
|
|
2840
|
+
const embedding = parseHfInferenceEmbeddingBody(raw);
|
|
2841
|
+
if (!embedding?.length) {
|
|
2842
|
+
if (this.embedDebugEnabled()) {
|
|
2843
|
+
console.warn("[LLM embed HF] Inference SDK returned no parseable vector", {
|
|
2844
|
+
model,
|
|
2845
|
+
shape: Array.isArray(raw) ? `array(len=${raw.length})` : typeof raw
|
|
2846
|
+
});
|
|
2847
|
+
}
|
|
2848
|
+
return null;
|
|
2849
|
+
}
|
|
2850
|
+
return embedding;
|
|
2851
|
+
} catch (err) {
|
|
2852
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
2853
|
+
if (!_LlmService.hfInferenceSdkErrorLogged) {
|
|
2854
|
+
_LlmService.hfInferenceSdkErrorLogged = true;
|
|
2855
|
+
console.warn(
|
|
2856
|
+
"[LLM embed HF] @huggingface/inference featureExtraction failed \u2014 trying legacy HTTP URLs.",
|
|
2857
|
+
{ model, message: message.slice(0, 500) }
|
|
2839
2858
|
);
|
|
2840
2859
|
} else if (this.embedDebugEnabled()) {
|
|
2841
|
-
console.warn("[LLM embed HF] repeat
|
|
2860
|
+
console.warn("[LLM embed HF] repeat Inference SDK failure", { message: message.slice(0, 200) });
|
|
2842
2861
|
}
|
|
2843
|
-
return
|
|
2862
|
+
return null;
|
|
2844
2863
|
}
|
|
2845
|
-
|
|
2846
|
-
|
|
2847
|
-
|
|
2848
|
-
|
|
2849
|
-
|
|
2850
|
-
_LlmService.embedShapeErrorLogged = true;
|
|
2851
|
-
console.error("[LLM embed HF] Response is not JSON \u2014 treating as no embedding.");
|
|
2852
|
-
}
|
|
2853
|
-
return [];
|
|
2864
|
+
}
|
|
2865
|
+
/** Base URL for legacy `…/pipeline/…` and `…/models/{model}` tries (HF router only). */
|
|
2866
|
+
hfLegacyInferenceBase(normalizedBase) {
|
|
2867
|
+
if (/router\.huggingface\.co|api-inference\.huggingface\.co/i.test(normalizedBase)) {
|
|
2868
|
+
return "https://router.huggingface.co/hf-inference";
|
|
2854
2869
|
}
|
|
2855
|
-
|
|
2856
|
-
|
|
2857
|
-
|
|
2858
|
-
|
|
2859
|
-
|
|
2860
|
-
|
|
2870
|
+
return normalizedBase;
|
|
2871
|
+
}
|
|
2872
|
+
/**
|
|
2873
|
+
* Hugging Face embeddings: `@huggingface/inference` (Hub-routed feature extraction), then
|
|
2874
|
+
* legacy direct `hf-inference` HTTP tries. Note: `https://router.huggingface.co/v1/embeddings` is not
|
|
2875
|
+
* offered for embeddings (OpenAI-compatible `/v1` on HF is chat-only).
|
|
2876
|
+
*/
|
|
2877
|
+
async embedHuggingFaceInference(input, model) {
|
|
2878
|
+
const normalizedBase = this.hfInferenceBaseURL.replace(/\/$/, "");
|
|
2879
|
+
const fromSdk = await this.embedViaHuggingfaceInferenceSdk(input, model);
|
|
2880
|
+
if (fromSdk?.length) {
|
|
2881
|
+
if (!_LlmService.embedSuccessLogged) {
|
|
2882
|
+
_LlmService.embedSuccessLogged = true;
|
|
2883
|
+
console.info("[LLM embed HF] first success", { dimension: fromSdk.length, model, via: "@huggingface/inference" });
|
|
2884
|
+
if (!_LlmService.hfDimensionMismatchWarned && fromSdk.length !== 1536) {
|
|
2885
|
+
_LlmService.hfDimensionMismatchWarned = true;
|
|
2886
|
+
console.warn(
|
|
2887
|
+
`[LLM embed HF] Embedding dimensions=${fromSdk.length}; ensure knowledge_base_chunks.embedding uses vector(${fromSdk.length}) (see migrations).`
|
|
2888
|
+
);
|
|
2889
|
+
}
|
|
2890
|
+
} else if (this.embedDebugEnabled()) {
|
|
2891
|
+
console.info("[LLM embed HF] ok", { dimension: fromSdk.length, via: "@huggingface/inference" });
|
|
2861
2892
|
}
|
|
2862
|
-
return
|
|
2893
|
+
return fromSdk;
|
|
2863
2894
|
}
|
|
2864
|
-
|
|
2865
|
-
|
|
2866
|
-
|
|
2867
|
-
|
|
2868
|
-
|
|
2869
|
-
|
|
2870
|
-
|
|
2871
|
-
|
|
2895
|
+
const legacyBase = this.hfLegacyInferenceBase(normalizedBase);
|
|
2896
|
+
const baseWithProvider = /\/hf-inference$/i.test(legacyBase) || !/router\.huggingface\.co/i.test(legacyBase) ? legacyBase : `${legacyBase.replace(/\/$/, "")}/hf-inference`;
|
|
2897
|
+
const bodyVariants = [
|
|
2898
|
+
{
|
|
2899
|
+
label: "inputs_parameters_mean_normalize",
|
|
2900
|
+
body: JSON.stringify({
|
|
2901
|
+
inputs: input,
|
|
2902
|
+
parameters: { pooling: "mean", normalize: true },
|
|
2903
|
+
options: { wait_for_model: true }
|
|
2904
|
+
})
|
|
2905
|
+
},
|
|
2906
|
+
{
|
|
2907
|
+
label: "inputs_string",
|
|
2908
|
+
body: JSON.stringify({ inputs: input, options: { wait_for_model: true } })
|
|
2909
|
+
},
|
|
2910
|
+
{
|
|
2911
|
+
label: "inputs_string_array",
|
|
2912
|
+
body: JSON.stringify({ inputs: [input], options: { wait_for_model: true } })
|
|
2913
|
+
},
|
|
2914
|
+
{
|
|
2915
|
+
label: "inputs_wrapped_sentences",
|
|
2916
|
+
body: JSON.stringify({ inputs: { sentences: [input] }, options: { wait_for_model: true } })
|
|
2917
|
+
},
|
|
2918
|
+
{
|
|
2919
|
+
label: "top_level_sentences",
|
|
2920
|
+
body: JSON.stringify({ sentences: [input], options: { wait_for_model: true } })
|
|
2872
2921
|
}
|
|
2922
|
+
];
|
|
2923
|
+
const headers = {
|
|
2924
|
+
"Content-Type": "application/json",
|
|
2925
|
+
Authorization: `Bearer ${this.embeddingApiKey}`
|
|
2926
|
+
};
|
|
2927
|
+
const candidateUrls = [];
|
|
2928
|
+
const pushUrl = (u) => {
|
|
2929
|
+
if (!candidateUrls.includes(u)) candidateUrls.push(u);
|
|
2930
|
+
};
|
|
2931
|
+
pushUrl(`${baseWithProvider}/pipeline/feature-extraction/${model}`);
|
|
2932
|
+
pushUrl(`${baseWithProvider}/models/${model}`);
|
|
2933
|
+
let lastUrl = "";
|
|
2934
|
+
let lastStatus = 0;
|
|
2935
|
+
let lastBodyPreview = "";
|
|
2936
|
+
let lastAttempt = "";
|
|
2937
|
+
for (const embedUrl of candidateUrls) {
|
|
2938
|
+
for (const { label, body } of bodyVariants) {
|
|
2939
|
+
if (this.embedDebugEnabled()) {
|
|
2940
|
+
console.info("[LLM embed HF] attempt", { url: embedUrl, model, inputChars: input.length, body: label });
|
|
2941
|
+
}
|
|
2942
|
+
const res = await fetch(embedUrl, {
|
|
2943
|
+
method: "POST",
|
|
2944
|
+
headers,
|
|
2945
|
+
body
|
|
2946
|
+
});
|
|
2947
|
+
const bodyText = await res.text();
|
|
2948
|
+
lastUrl = embedUrl;
|
|
2949
|
+
lastStatus = res.status;
|
|
2950
|
+
lastBodyPreview = bodyText.slice(0, 800);
|
|
2951
|
+
lastAttempt = `${embedUrl} (${label})`;
|
|
2952
|
+
if (!res.ok) {
|
|
2953
|
+
if (this.embedDebugEnabled()) {
|
|
2954
|
+
console.warn("[LLM embed HF] attempt failed, trying next if any", {
|
|
2955
|
+
url: embedUrl,
|
|
2956
|
+
body: label,
|
|
2957
|
+
status: res.status,
|
|
2958
|
+
bodyPreview: bodyText.slice(0, 200)
|
|
2959
|
+
});
|
|
2960
|
+
}
|
|
2961
|
+
continue;
|
|
2962
|
+
}
|
|
2963
|
+
let data;
|
|
2964
|
+
try {
|
|
2965
|
+
data = JSON.parse(bodyText);
|
|
2966
|
+
} catch {
|
|
2967
|
+
if (this.embedDebugEnabled()) {
|
|
2968
|
+
console.warn("[LLM embed HF] response not JSON, trying next if any", { url: embedUrl, body: label });
|
|
2969
|
+
}
|
|
2970
|
+
continue;
|
|
2971
|
+
}
|
|
2972
|
+
const embedding = parseHfInferenceEmbeddingBody(data);
|
|
2973
|
+
if (!embedding?.length) {
|
|
2974
|
+
if (this.embedDebugEnabled()) {
|
|
2975
|
+
const preview = typeof data === "object" && data !== null && !Array.isArray(data) ? Object.keys(data) : typeof data;
|
|
2976
|
+
console.warn("[LLM embed HF] no parseable embedding, trying next if any", {
|
|
2977
|
+
url: embedUrl,
|
|
2978
|
+
body: label,
|
|
2979
|
+
preview
|
|
2980
|
+
});
|
|
2981
|
+
}
|
|
2982
|
+
continue;
|
|
2983
|
+
}
|
|
2984
|
+
if (!_LlmService.embedSuccessLogged) {
|
|
2985
|
+
_LlmService.embedSuccessLogged = true;
|
|
2986
|
+
console.info("[LLM embed HF] first success", { dimension: embedding.length, model, url: embedUrl, body: label });
|
|
2987
|
+
if (!_LlmService.hfDimensionMismatchWarned && embedding.length !== 1536) {
|
|
2988
|
+
_LlmService.hfDimensionMismatchWarned = true;
|
|
2989
|
+
console.warn(
|
|
2990
|
+
`[LLM embed HF] Embedding dimensions=${embedding.length}; ensure knowledge_base_chunks.embedding uses vector(${embedding.length}) (see migrations).`
|
|
2991
|
+
);
|
|
2992
|
+
}
|
|
2993
|
+
} else if (this.embedDebugEnabled()) {
|
|
2994
|
+
console.info("[LLM embed HF] ok", { dimension: embedding.length, url: embedUrl, body: label });
|
|
2995
|
+
}
|
|
2996
|
+
return embedding;
|
|
2997
|
+
}
|
|
2998
|
+
}
|
|
2999
|
+
if (!_LlmService.embedHttpErrorLogged) {
|
|
3000
|
+
_LlmService.embedHttpErrorLogged = true;
|
|
3001
|
+
console.error(
|
|
3002
|
+
"[LLM embed HF] All Hugging Face embedding attempts failed \u2014 knowledge chunks stay without vectors.",
|
|
3003
|
+
{
|
|
3004
|
+
triedInferenceSdk: true,
|
|
3005
|
+
triedLegacyHttp: candidateUrls,
|
|
3006
|
+
lastAttempt,
|
|
3007
|
+
lastUrl,
|
|
3008
|
+
lastStatus,
|
|
3009
|
+
lastBodyPreview,
|
|
3010
|
+
hint: "Use an HF token with permission to call Inference Providers / Hub API mapping. Install ships @huggingface/inference; EMBEDDING_MODEL defaults to sentence-transformers/all-MiniLM-L6-v2. Router OpenAI /v1/embeddings is not used (chat-only)."
|
|
3011
|
+
}
|
|
3012
|
+
);
|
|
2873
3013
|
} else if (this.embedDebugEnabled()) {
|
|
2874
|
-
console.
|
|
3014
|
+
console.warn("[LLM embed HF] repeat failure (all attempts)", {
|
|
3015
|
+
lastAttempt,
|
|
3016
|
+
lastUrl,
|
|
3017
|
+
lastStatus,
|
|
3018
|
+
bodyPreview: lastBodyPreview.slice(0, 200)
|
|
3019
|
+
});
|
|
2875
3020
|
}
|
|
2876
|
-
return
|
|
3021
|
+
return [];
|
|
2877
3022
|
}
|
|
2878
3023
|
buildAgentMessages(options) {
|
|
2879
3024
|
const { systemPrompt, userPrompt, history = [], context } = options;
|
|
@@ -2917,7 +3062,7 @@ ${context}`;
|
|
|
2917
3062
|
Authorization: `Bearer ${this.apiKey}`
|
|
2918
3063
|
},
|
|
2919
3064
|
body: JSON.stringify({
|
|
2920
|
-
model: options.model
|
|
3065
|
+
model: this.resolveChatModel(options.model),
|
|
2921
3066
|
messages,
|
|
2922
3067
|
stream: false,
|
|
2923
3068
|
temperature: options.temperature,
|
|
@@ -2926,7 +3071,7 @@ ${context}`;
|
|
|
2926
3071
|
});
|
|
2927
3072
|
if (!res.ok) {
|
|
2928
3073
|
const text = await res.text();
|
|
2929
|
-
throw new Error(
|
|
3074
|
+
throw new Error(formatLlmGatewayError(res.status, text, this.defaultChatModel));
|
|
2930
3075
|
}
|
|
2931
3076
|
const data = await res.json();
|
|
2932
3077
|
const content = data.choices?.[0]?.message?.content ?? "";
|
|
@@ -2944,7 +3089,7 @@ ${context}`;
|
|
|
2944
3089
|
Authorization: `Bearer ${this.apiKey}`
|
|
2945
3090
|
},
|
|
2946
3091
|
body: JSON.stringify({
|
|
2947
|
-
model: options.model
|
|
3092
|
+
model: this.resolveChatModel(options.model),
|
|
2948
3093
|
messages,
|
|
2949
3094
|
stream: true,
|
|
2950
3095
|
temperature: options.temperature,
|
|
@@ -2953,7 +3098,7 @@ ${context}`;
|
|
|
2953
3098
|
});
|
|
2954
3099
|
if (!res.ok) {
|
|
2955
3100
|
const text = await res.text();
|
|
2956
|
-
throw new Error(
|
|
3101
|
+
throw new Error(formatLlmGatewayError(res.status, text, this.defaultChatModel));
|
|
2957
3102
|
}
|
|
2958
3103
|
const reader = res.body?.getReader();
|
|
2959
3104
|
if (!reader) return;
|
|
@@ -3006,6 +3151,7 @@ function llmPlugin(config = {}) {
|
|
|
3006
3151
|
return void 0;
|
|
3007
3152
|
}
|
|
3008
3153
|
const embeddingModel = config.embeddingModel ?? context.config.EMBEDDING_MODEL;
|
|
3154
|
+
const defaultChatModel = config.defaultChatModel?.trim() || (typeof context.config.LLM_CHAT_MODEL === "string" ? context.config.LLM_CHAT_MODEL.trim() : "") || void 0;
|
|
3009
3155
|
const embeddingProvider = normalizeEmbeddingProvider(
|
|
3010
3156
|
config.embeddingProvider ?? context.config.EMBEDDING_PROVIDER
|
|
3011
3157
|
);
|
|
@@ -3014,7 +3160,8 @@ function llmPlugin(config = {}) {
|
|
|
3014
3160
|
return new LlmService(baseURL.replace(/\/$/, ""), apiKey, embeddingModel, {
|
|
3015
3161
|
embeddingProvider,
|
|
3016
3162
|
hfInferenceBaseUrl,
|
|
3017
|
-
embeddingApiKey
|
|
3163
|
+
embeddingApiKey,
|
|
3164
|
+
defaultChatModel
|
|
3018
3165
|
});
|
|
3019
3166
|
}
|
|
3020
3167
|
};
|
|
@@ -8535,6 +8682,13 @@ function getNextAuthOptions(config) {
|
|
|
8535
8682
|
|
|
8536
8683
|
// src/api/crud.ts
|
|
8537
8684
|
var import_typeorm46 = require("typeorm");
|
|
8685
|
+
var CRUD_LOG = "[cms-crud]";
|
|
8686
|
+
function logCrudClientError(op, detail) {
|
|
8687
|
+
console.warn(CRUD_LOG, op, detail);
|
|
8688
|
+
}
|
|
8689
|
+
function logCrudServerError(op, detail) {
|
|
8690
|
+
console.error(CRUD_LOG, op, detail);
|
|
8691
|
+
}
|
|
8538
8692
|
var DATE_COLUMN_TYPES = /* @__PURE__ */ new Set([
|
|
8539
8693
|
"date",
|
|
8540
8694
|
"datetime",
|
|
@@ -8646,6 +8800,13 @@ function createCrudHandler(dataSource, entityMap, options) {
|
|
|
8646
8800
|
if (authError) return authError;
|
|
8647
8801
|
const entity = entityMap[resource];
|
|
8648
8802
|
if (!resource || !entity) {
|
|
8803
|
+
logCrudClientError("GET list", {
|
|
8804
|
+
reason: "invalid_resource",
|
|
8805
|
+
resource,
|
|
8806
|
+
hasEntity: Boolean(entity),
|
|
8807
|
+
entityMapHasLlmAgents: Boolean(entityMap.llm_agents),
|
|
8808
|
+
entityMapKeyCount: Object.keys(entityMap).length
|
|
8809
|
+
});
|
|
8649
8810
|
return json({ error: "Invalid resource" }, { status: 400 });
|
|
8650
8811
|
}
|
|
8651
8812
|
const { searchParams } = new URL(req.url);
|
|
@@ -8853,12 +9014,22 @@ function createCrudHandler(dataSource, entityMap, options) {
|
|
|
8853
9014
|
}
|
|
8854
9015
|
}
|
|
8855
9016
|
where = mergeDeletedFalseWhere(repo, where);
|
|
8856
|
-
|
|
8857
|
-
|
|
8858
|
-
|
|
8859
|
-
|
|
8860
|
-
|
|
8861
|
-
|
|
9017
|
+
let data;
|
|
9018
|
+
let total;
|
|
9019
|
+
try {
|
|
9020
|
+
const r = await repo.findAndCount({
|
|
9021
|
+
skip,
|
|
9022
|
+
take: limit,
|
|
9023
|
+
order: { [sortField]: sortOrder },
|
|
9024
|
+
where
|
|
9025
|
+
});
|
|
9026
|
+
data = r[0];
|
|
9027
|
+
total = r[1];
|
|
9028
|
+
} catch (err) {
|
|
9029
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
9030
|
+
logCrudServerError("GET list query failed", { resource, sortField, sortOrder, message });
|
|
9031
|
+
throw err;
|
|
9032
|
+
}
|
|
8862
9033
|
return json({ total, page, limit, totalPages: Math.ceil(total / limit), data });
|
|
8863
9034
|
},
|
|
8864
9035
|
async POST(req, resource) {
|
|
@@ -8866,12 +9037,25 @@ function createCrudHandler(dataSource, entityMap, options) {
|
|
|
8866
9037
|
if (authError) return authError;
|
|
8867
9038
|
const entity = entityMap[resource];
|
|
8868
9039
|
if (!resource || !entity) {
|
|
9040
|
+
logCrudClientError("POST create", {
|
|
9041
|
+
reason: "invalid_resource",
|
|
9042
|
+
resource,
|
|
9043
|
+
hasEntity: Boolean(entity),
|
|
9044
|
+
entityMapHasLlmAgents: Boolean(entityMap.llm_agents)
|
|
9045
|
+
});
|
|
8869
9046
|
return json({ error: "Invalid resource" }, { status: 400 });
|
|
8870
9047
|
}
|
|
8871
|
-
const
|
|
8872
|
-
if (!
|
|
9048
|
+
const rawPostBody = await req.json();
|
|
9049
|
+
if (!rawPostBody || typeof rawPostBody !== "object" || Object.keys(rawPostBody).length === 0) {
|
|
9050
|
+
logCrudClientError("POST create", {
|
|
9051
|
+
reason: "invalid_request_payload",
|
|
9052
|
+
resource,
|
|
9053
|
+
rawType: rawPostBody == null ? "nullish" : typeof rawPostBody,
|
|
9054
|
+
keyCount: rawPostBody && typeof rawPostBody === "object" ? Object.keys(rawPostBody).length : 0
|
|
9055
|
+
});
|
|
8873
9056
|
return json({ error: "Invalid request payload" }, { status: 400 });
|
|
8874
9057
|
}
|
|
9058
|
+
const body = rawPostBody;
|
|
8875
9059
|
if (resource === "media") {
|
|
8876
9060
|
const b = body;
|
|
8877
9061
|
const kind = b.kind === "folder" ? "folder" : "file";
|
|
@@ -8905,8 +9089,24 @@ function createCrudHandler(dataSource, entityMap, options) {
|
|
|
8905
9089
|
}
|
|
8906
9090
|
}
|
|
8907
9091
|
const repo = dataSource.getRepository(entity);
|
|
8908
|
-
|
|
8909
|
-
|
|
9092
|
+
const persistBody = resource === "media" ? body : pickColumnUpdates(repo, body);
|
|
9093
|
+
if (resource !== "media" && Object.keys(persistBody).length === 0) {
|
|
9094
|
+
logCrudClientError("POST create", {
|
|
9095
|
+
reason: "no_scalar_columns_after_pick",
|
|
9096
|
+
resource,
|
|
9097
|
+
incomingKeys: Object.keys(body)
|
|
9098
|
+
});
|
|
9099
|
+
return json({ error: "Invalid request payload" }, { status: 400 });
|
|
9100
|
+
}
|
|
9101
|
+
sanitizeBodyForEntity(repo, persistBody);
|
|
9102
|
+
let created;
|
|
9103
|
+
try {
|
|
9104
|
+
created = await repo.save(repo.create(persistBody));
|
|
9105
|
+
} catch (err) {
|
|
9106
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
9107
|
+
logCrudServerError("POST create save failed", { resource, message, persistKeys: Object.keys(persistBody) });
|
|
9108
|
+
throw err;
|
|
9109
|
+
}
|
|
8910
9110
|
if (resource === "contacts") {
|
|
8911
9111
|
await syncContactRowToErp(created);
|
|
8912
9112
|
}
|
|
@@ -8921,6 +9121,7 @@ function createCrudHandler(dataSource, entityMap, options) {
|
|
|
8921
9121
|
if (authError) return authError;
|
|
8922
9122
|
const entity = entityMap[resource];
|
|
8923
9123
|
if (!resource || !entity) {
|
|
9124
|
+
logCrudClientError("GET_METADATA", { reason: "invalid_resource", resource });
|
|
8924
9125
|
return json({ error: "Invalid resource" }, { status: 400 });
|
|
8925
9126
|
}
|
|
8926
9127
|
const repo = dataSource.getRepository(entity);
|
|
@@ -8952,11 +9153,18 @@ function createCrudHandler(dataSource, entityMap, options) {
|
|
|
8952
9153
|
if (authError) return authError;
|
|
8953
9154
|
const entity = entityMap[resource];
|
|
8954
9155
|
if (!resource || !entity) {
|
|
9156
|
+
logCrudClientError("BULK_POST", { reason: "invalid_resource", resource });
|
|
8955
9157
|
return json({ error: "Invalid resource" }, { status: 400 });
|
|
8956
9158
|
}
|
|
8957
9159
|
const body = await req.json();
|
|
8958
9160
|
const { records, upsertKey = "id" } = body;
|
|
8959
9161
|
if (!Array.isArray(records) || records.length === 0) {
|
|
9162
|
+
logCrudClientError("BULK_POST", {
|
|
9163
|
+
reason: "records_required",
|
|
9164
|
+
resource,
|
|
9165
|
+
recordsIsArray: Array.isArray(records),
|
|
9166
|
+
recordCount: Array.isArray(records) ? records.length : 0
|
|
9167
|
+
});
|
|
8960
9168
|
return json({ error: "Records array is required" }, { status: 400 });
|
|
8961
9169
|
}
|
|
8962
9170
|
const repo = dataSource.getRepository(entity);
|
|
@@ -8975,6 +9183,7 @@ function createCrudHandler(dataSource, entityMap, options) {
|
|
|
8975
9183
|
});
|
|
8976
9184
|
} catch (error) {
|
|
8977
9185
|
const message = error instanceof Error ? error.message : "Bulk import failed";
|
|
9186
|
+
logCrudClientError("BULK_POST upsert failed", { resource, upsertKey, message });
|
|
8978
9187
|
return json({ error: message }, { status: 400 });
|
|
8979
9188
|
}
|
|
8980
9189
|
},
|
|
@@ -8983,6 +9192,7 @@ function createCrudHandler(dataSource, entityMap, options) {
|
|
|
8983
9192
|
if (authError) return authError;
|
|
8984
9193
|
const entity = entityMap[resource];
|
|
8985
9194
|
if (!resource || !entity) {
|
|
9195
|
+
logCrudClientError("GET_EXPORT", { reason: "invalid_resource", resource });
|
|
8986
9196
|
return json({ error: "Invalid resource" }, { status: 400 });
|
|
8987
9197
|
}
|
|
8988
9198
|
const { searchParams } = new URL(req.url);
|
|
@@ -9037,7 +9247,10 @@ function createCrudByIdHandler(dataSource, entityMap, options) {
|
|
|
9037
9247
|
const authError = await authz(req, resource, "read");
|
|
9038
9248
|
if (authError) return authError;
|
|
9039
9249
|
const entity = entityMap[resource];
|
|
9040
|
-
if (!entity)
|
|
9250
|
+
if (!entity) {
|
|
9251
|
+
logCrudClientError("GET by id", { reason: "invalid_resource", resource, id });
|
|
9252
|
+
return json({ error: "Invalid resource" }, { status: 400 });
|
|
9253
|
+
}
|
|
9041
9254
|
const repo = dataSource.getRepository(entity);
|
|
9042
9255
|
if (resource === "orders") {
|
|
9043
9256
|
const order = await repo.findOne({
|
|
@@ -9096,7 +9309,10 @@ function createCrudByIdHandler(dataSource, entityMap, options) {
|
|
|
9096
9309
|
const authError = await authz(req, resource, "update");
|
|
9097
9310
|
if (authError) return authError;
|
|
9098
9311
|
const entity = entityMap[resource];
|
|
9099
|
-
if (!entity)
|
|
9312
|
+
if (!entity) {
|
|
9313
|
+
logCrudClientError("PUT by id", { reason: "invalid_resource", resource, id });
|
|
9314
|
+
return json({ error: "Invalid resource" }, { status: 400 });
|
|
9315
|
+
}
|
|
9100
9316
|
const rawBody = await req.json();
|
|
9101
9317
|
const repo = dataSource.getRepository(entity);
|
|
9102
9318
|
const numericId = Number(id);
|
|
@@ -9209,7 +9425,10 @@ function createCrudByIdHandler(dataSource, entityMap, options) {
|
|
|
9209
9425
|
const authError = await authz(req, resource, "delete");
|
|
9210
9426
|
if (authError) return authError;
|
|
9211
9427
|
const entity = entityMap[resource];
|
|
9212
|
-
if (!entity)
|
|
9428
|
+
if (!entity) {
|
|
9429
|
+
logCrudClientError("DELETE by id", { reason: "invalid_resource", resource, id });
|
|
9430
|
+
return json({ error: "Invalid resource" }, { status: 400 });
|
|
9431
|
+
}
|
|
9213
9432
|
const repo = dataSource.getRepository(entity);
|
|
9214
9433
|
const numericId = Number(id);
|
|
9215
9434
|
if (entityHasSoftDelete(repo)) {
|
|
@@ -9379,6 +9598,9 @@ var EMBED_CONCURRENCY = 5;
|
|
|
9379
9598
|
var MAX_PDF_BYTES = 25 * 1024 * 1024;
|
|
9380
9599
|
var TEXT_FILE_TYPES = /* @__PURE__ */ new Set(["text/plain", "text/markdown", "application/json"]);
|
|
9381
9600
|
var KB_LOG = "[llm-agent-knowledge]";
|
|
9601
|
+
function logKbPipeline(step, meta) {
|
|
9602
|
+
console.info(`${KB_LOG} pipeline`, { step, ...meta });
|
|
9603
|
+
}
|
|
9382
9604
|
function llmEmbedDebug() {
|
|
9383
9605
|
const v = process.env.LLM_EMBED_DEBUG?.toLowerCase();
|
|
9384
9606
|
return v === "1" || v === "true" || v === "yes";
|
|
@@ -9393,11 +9615,29 @@ async function extractTextFromPdf(buffer) {
|
|
|
9393
9615
|
const data = await pdfParse(buffer);
|
|
9394
9616
|
return (data?.text ?? "").trim();
|
|
9395
9617
|
}
|
|
9396
|
-
async function
|
|
9618
|
+
async function loadChunksMissingEmbeddings(dataSource, documentId, slug) {
|
|
9619
|
+
const rows = await dataSource.query(
|
|
9620
|
+
`SELECT id, content FROM knowledge_base_chunks WHERE "documentId" = $1 AND embedding IS NULL ORDER BY "chunkIndex" ASC`,
|
|
9621
|
+
[documentId]
|
|
9622
|
+
);
|
|
9623
|
+
const list = rows ?? [];
|
|
9624
|
+
logKbPipeline("07_query_chunks_missing_embedding", {
|
|
9625
|
+
slug,
|
|
9626
|
+
documentId,
|
|
9627
|
+
missingEmbeddingCount: list.length
|
|
9628
|
+
});
|
|
9629
|
+
return list;
|
|
9630
|
+
}
|
|
9631
|
+
async function writeEmbeddingsConcurrent(dataSource, embed, chunks, concurrency, slug) {
|
|
9397
9632
|
let next = 0;
|
|
9398
9633
|
let written = 0;
|
|
9399
9634
|
let failed = 0;
|
|
9400
9635
|
let skippedEmpty = 0;
|
|
9636
|
+
logKbPipeline("09_embedding_workers_start", {
|
|
9637
|
+
slug,
|
|
9638
|
+
chunkCount: chunks.length,
|
|
9639
|
+
concurrency: Math.max(1, Math.min(concurrency, chunks.length))
|
|
9640
|
+
});
|
|
9401
9641
|
async function worker() {
|
|
9402
9642
|
for (; ; ) {
|
|
9403
9643
|
const i = next++;
|
|
@@ -9436,9 +9676,12 @@ async function writeEmbeddingsConcurrent(dataSource, embed, chunks, concurrency)
|
|
|
9436
9676
|
skippedEmpty
|
|
9437
9677
|
};
|
|
9438
9678
|
if (skippedEmpty > 0 && written === 0) {
|
|
9439
|
-
summary.hint = "embed() returned empty vectors \u2014 see [LLM embed] logs (
|
|
9679
|
+
summary.hint = "embed() returned empty vectors \u2014 see [LLM embed] / [LLM embed HF] logs (OpenAI gateway vs @huggingface/inference + legacy HTTP).";
|
|
9680
|
+
}
|
|
9681
|
+
logKbPipeline("10_embedding_workers_finished", { slug, ...summary });
|
|
9682
|
+
if (failed > 0 || skippedEmpty > 0 && written === 0) {
|
|
9683
|
+
console.error(`${KB_LOG} embedding pass finished with issues`, summary);
|
|
9440
9684
|
}
|
|
9441
|
-
console.error(`${KB_LOG} embedding pass finished`, summary);
|
|
9442
9685
|
return { written, failed };
|
|
9443
9686
|
}
|
|
9444
9687
|
function splitIntoChunks(text, maxLen) {
|
|
@@ -9474,6 +9717,58 @@ function createLlmAgentKnowledgeHandlers(config) {
|
|
|
9474
9717
|
}
|
|
9475
9718
|
return null;
|
|
9476
9719
|
}
|
|
9720
|
+
async function runEmbeddingPass(slug, savedChunks) {
|
|
9721
|
+
let embeddingsWritten = 0;
|
|
9722
|
+
let embeddingsFailed = 0;
|
|
9723
|
+
try {
|
|
9724
|
+
logKbPipeline("08_resolve_llm_plugin", { slug, chunkCount: savedChunks.length });
|
|
9725
|
+
const cms = await getCms();
|
|
9726
|
+
const llm = cms.getPlugin("llm");
|
|
9727
|
+
if (llm?.embed && savedChunks.length > 0) {
|
|
9728
|
+
logKbPipeline("08b_embed_fn_available", {
|
|
9729
|
+
slug,
|
|
9730
|
+
chunkCount: savedChunks.length,
|
|
9731
|
+
firstChunkId: savedChunks[0]?.id,
|
|
9732
|
+
lastChunkId: savedChunks[savedChunks.length - 1]?.id
|
|
9733
|
+
});
|
|
9734
|
+
const embedBound = (text) => llm.embed(text);
|
|
9735
|
+
const { written, failed } = await writeEmbeddingsConcurrent(
|
|
9736
|
+
dataSource,
|
|
9737
|
+
embedBound,
|
|
9738
|
+
savedChunks,
|
|
9739
|
+
EMBED_CONCURRENCY,
|
|
9740
|
+
slug
|
|
9741
|
+
);
|
|
9742
|
+
embeddingsWritten = written;
|
|
9743
|
+
embeddingsFailed = failed;
|
|
9744
|
+
} else {
|
|
9745
|
+
logKbPipeline("08c_embed_skipped", {
|
|
9746
|
+
slug,
|
|
9747
|
+
hasLlmPlugin: !!llm,
|
|
9748
|
+
hasEmbed: typeof llm?.embed === "function",
|
|
9749
|
+
chunkCount: savedChunks.length,
|
|
9750
|
+
reason: savedChunks.length === 0 ? "no_chunks" : !llm ? "no_llm_plugin" : "no_embed_method"
|
|
9751
|
+
});
|
|
9752
|
+
console.error(`${KB_LOG} embeddings skipped`, {
|
|
9753
|
+
slug,
|
|
9754
|
+
hasLlmPlugin: !!llm,
|
|
9755
|
+
hasEmbed: typeof llm?.embed === "function",
|
|
9756
|
+
chunkCount: savedChunks.length,
|
|
9757
|
+
hint: !llm || typeof llm.embed !== "function" ? "LLM plugin missing or no embed(); check LLM_GATEWAY_URL + LLM_API_KEY so llm plugin initializes." : void 0
|
|
9758
|
+
});
|
|
9759
|
+
}
|
|
9760
|
+
} catch (embErr) {
|
|
9761
|
+
const detail = embErr instanceof Error ? embErr.message : String(embErr);
|
|
9762
|
+
logKbPipeline("08d_embed_pass_exception", { slug, detail });
|
|
9763
|
+
console.error(`${KB_LOG} embedding step threw before/during batch`, { slug, detail, embErr });
|
|
9764
|
+
return {
|
|
9765
|
+
embeddingsWritten: 0,
|
|
9766
|
+
embeddingsFailed: savedChunks.length,
|
|
9767
|
+
embedError: detail
|
|
9768
|
+
};
|
|
9769
|
+
}
|
|
9770
|
+
return { embeddingsWritten, embeddingsFailed };
|
|
9771
|
+
}
|
|
9477
9772
|
return {
|
|
9478
9773
|
async list(req, slug) {
|
|
9479
9774
|
const denied = await gate(req, "read");
|
|
@@ -9502,8 +9797,11 @@ function createLlmAgentKnowledgeHandlers(config) {
|
|
|
9502
9797
|
const denied = await gate(req, "update");
|
|
9503
9798
|
if (denied) return denied;
|
|
9504
9799
|
try {
|
|
9800
|
+
const ct0 = req.headers.get("content-type") || "";
|
|
9801
|
+
logKbPipeline("01_request", { slug, contentType: ct0.slice(0, 80) });
|
|
9505
9802
|
const agent = await findAgentBySlug(dataSource, llmAgents, slug);
|
|
9506
9803
|
if (!agent) return json({ error: "Agent not found" }, { status: 404 });
|
|
9804
|
+
logKbPipeline("02_agent_loaded", { slug, agentId: agent.id });
|
|
9507
9805
|
let name = "";
|
|
9508
9806
|
let text = "";
|
|
9509
9807
|
let sourceUrl = null;
|
|
@@ -9515,6 +9813,14 @@ function createLlmAgentKnowledgeHandlers(config) {
|
|
|
9515
9813
|
name = (body?.name ?? "").trim();
|
|
9516
9814
|
text = (body?.text ?? "").trim();
|
|
9517
9815
|
sourceUrl = typeof body?.sourceUrl === "string" && body.sourceUrl.trim() ? body.sourceUrl.trim() : null;
|
|
9816
|
+
logKbPipeline("03_body_parsed", {
|
|
9817
|
+
slug,
|
|
9818
|
+
mode: "json",
|
|
9819
|
+
existingDocumentId,
|
|
9820
|
+
nameLen: name.length,
|
|
9821
|
+
textChars: text.length,
|
|
9822
|
+
hasSourceUrl: !!sourceUrl
|
|
9823
|
+
});
|
|
9518
9824
|
} else if (ct.includes("multipart/form-data")) {
|
|
9519
9825
|
const form = await req.formData();
|
|
9520
9826
|
name = form.get("name")?.trim() ?? "";
|
|
@@ -9524,9 +9830,17 @@ function createLlmAgentKnowledgeHandlers(config) {
|
|
|
9524
9830
|
const f = file;
|
|
9525
9831
|
const mime = (f.type || "").split(";")[0].trim().toLowerCase();
|
|
9526
9832
|
const buf = Buffer.from(await f.arrayBuffer());
|
|
9833
|
+
logKbPipeline("03_body_parsed", {
|
|
9834
|
+
slug,
|
|
9835
|
+
mode: "multipart",
|
|
9836
|
+
fileName: f.name || "(no name)",
|
|
9837
|
+
mime,
|
|
9838
|
+
fileBytes: buf.length
|
|
9839
|
+
});
|
|
9527
9840
|
if (TEXT_FILE_TYPES.has(mime)) {
|
|
9528
9841
|
const decoded = buf.toString("utf8");
|
|
9529
9842
|
if (!text) text = decoded;
|
|
9843
|
+
logKbPipeline("04_file_decoded_text", { slug, mime, textChars: text.length });
|
|
9530
9844
|
} else if (isPdfUpload(mime, f.name || "")) {
|
|
9531
9845
|
if (buf.length > MAX_PDF_BYTES) {
|
|
9532
9846
|
return json(
|
|
@@ -9535,8 +9849,10 @@ function createLlmAgentKnowledgeHandlers(config) {
|
|
|
9535
9849
|
);
|
|
9536
9850
|
}
|
|
9537
9851
|
try {
|
|
9852
|
+
logKbPipeline("04_pdf_extract_start", { slug, fileBytes: buf.length });
|
|
9538
9853
|
const extracted = await extractTextFromPdf(buf);
|
|
9539
9854
|
if (!text) text = extracted;
|
|
9855
|
+
logKbPipeline("04_pdf_extract_done", { slug, textChars: text.length });
|
|
9540
9856
|
} catch {
|
|
9541
9857
|
return json(
|
|
9542
9858
|
{ error: "Could not read PDF text (file may be encrypted, corrupt, or image-only)" },
|
|
@@ -9552,12 +9868,15 @@ function createLlmAgentKnowledgeHandlers(config) {
|
|
|
9552
9868
|
);
|
|
9553
9869
|
}
|
|
9554
9870
|
if (!name && f.name) name = f.name.replace(/\.[^/.]+$/, "") || f.name;
|
|
9871
|
+
} else {
|
|
9872
|
+
logKbPipeline("03_body_parsed", { slug, mode: "multipart", hasFile: false, textChars: text.length });
|
|
9555
9873
|
}
|
|
9556
9874
|
} else {
|
|
9557
9875
|
return json({ error: "Use application/json or multipart/form-data" }, { status: 400 });
|
|
9558
9876
|
}
|
|
9559
9877
|
const linkRepo = dataSource.getRepository(junction);
|
|
9560
9878
|
if (existingDocumentId != null) {
|
|
9879
|
+
logKbPipeline("05_branch", { slug, branch: "link_existing_document", documentId: existingDocumentId });
|
|
9561
9880
|
const docRepo2 = dataSource.getRepository(kbDoc);
|
|
9562
9881
|
const existing = await docRepo2.findOne({ where: { id: existingDocumentId } });
|
|
9563
9882
|
if (!existing) return json({ error: "documentId not found" }, { status: 404 });
|
|
@@ -9566,9 +9885,137 @@ function createLlmAgentKnowledgeHandlers(config) {
|
|
|
9566
9885
|
});
|
|
9567
9886
|
if (!dup2) {
|
|
9568
9887
|
await linkRepo.save(linkRepo.create({ agentId: agent.id, documentId: existingDocumentId }));
|
|
9888
|
+
logKbPipeline("06_junction_link_created", {
|
|
9889
|
+
slug,
|
|
9890
|
+
agentId: agent.id,
|
|
9891
|
+
documentId: existingDocumentId
|
|
9892
|
+
});
|
|
9893
|
+
} else {
|
|
9894
|
+
logKbPipeline("06_junction_link_exists", {
|
|
9895
|
+
slug,
|
|
9896
|
+
agentId: agent.id,
|
|
9897
|
+
documentId: existingDocumentId
|
|
9898
|
+
});
|
|
9569
9899
|
}
|
|
9570
|
-
|
|
9900
|
+
const chunkRepo2 = dataSource.getRepository(kbChunk);
|
|
9901
|
+
const docRow = existing;
|
|
9902
|
+
const chunkCount = await chunkRepo2.count({ where: { documentId: existingDocumentId } });
|
|
9903
|
+
logKbPipeline("06b_chunks_existing_count", {
|
|
9904
|
+
slug,
|
|
9905
|
+
documentId: existingDocumentId,
|
|
9906
|
+
chunkCount
|
|
9907
|
+
});
|
|
9908
|
+
let chunksToEmbed = [];
|
|
9909
|
+
let chunksCreated = 0;
|
|
9910
|
+
if (chunkCount === 0) {
|
|
9911
|
+
const text2 = (docRow.content ?? "").trim();
|
|
9912
|
+
if (!text2) {
|
|
9913
|
+
logKbPipeline("07_ingest_aborted_empty_document", { slug, documentId: existingDocumentId });
|
|
9914
|
+
return json({
|
|
9915
|
+
documentId: existingDocumentId,
|
|
9916
|
+
linked: true,
|
|
9917
|
+
created: false,
|
|
9918
|
+
chunkCount: 0,
|
|
9919
|
+
embeddingsWritten: 0,
|
|
9920
|
+
embeddingsFailed: 0,
|
|
9921
|
+
warning: "Document has no text content; add content then attach again to build chunks and embeddings."
|
|
9922
|
+
});
|
|
9923
|
+
}
|
|
9924
|
+
const parts2 = splitIntoChunks(text2, INGEST_CHUNK_CHARS);
|
|
9925
|
+
if (parts2.length > MAX_CHUNKS_PER_UPLOAD) {
|
|
9926
|
+
return json(
|
|
9927
|
+
{
|
|
9928
|
+
error: `Document is too large for one ingest (${parts2.length} chunks; max ${MAX_CHUNKS_PER_UPLOAD}). Split into smaller files.`
|
|
9929
|
+
},
|
|
9930
|
+
{ status: 413 }
|
|
9931
|
+
);
|
|
9932
|
+
}
|
|
9933
|
+
logKbPipeline("07_chunk_split", {
|
|
9934
|
+
slug,
|
|
9935
|
+
documentId: existingDocumentId,
|
|
9936
|
+
partCount: parts2.length,
|
|
9937
|
+
maxChunkChars: INGEST_CHUNK_CHARS,
|
|
9938
|
+
totalChars: text2.length
|
|
9939
|
+
});
|
|
9940
|
+
const now2 = /* @__PURE__ */ new Date();
|
|
9941
|
+
const chunkRows2 = parts2.map(
|
|
9942
|
+
(content, i) => chunkRepo2.create({
|
|
9943
|
+
documentId: existingDocumentId,
|
|
9944
|
+
content,
|
|
9945
|
+
chunkIndex: i,
|
|
9946
|
+
createdAt: now2
|
|
9947
|
+
})
|
|
9948
|
+
);
|
|
9949
|
+
const savedList2 = await chunkRepo2.save(chunkRows2);
|
|
9950
|
+
chunksCreated = savedList2.length;
|
|
9951
|
+
chunksToEmbed = savedList2.map((row, i) => ({
|
|
9952
|
+
id: row.id,
|
|
9953
|
+
content: parts2[i]
|
|
9954
|
+
}));
|
|
9955
|
+
logKbPipeline("07b_chunks_persisted", {
|
|
9956
|
+
slug,
|
|
9957
|
+
documentId: existingDocumentId,
|
|
9958
|
+
rowsInserted: chunksCreated
|
|
9959
|
+
});
|
|
9960
|
+
} else {
|
|
9961
|
+
chunksToEmbed = await loadChunksMissingEmbeddings(dataSource, existingDocumentId, slug);
|
|
9962
|
+
if (chunksToEmbed.length === 0) {
|
|
9963
|
+
logKbPipeline("08_skip_embed_all_chunks_have_vectors", {
|
|
9964
|
+
slug,
|
|
9965
|
+
documentId: existingDocumentId,
|
|
9966
|
+
existingChunkCount: chunkCount
|
|
9967
|
+
});
|
|
9968
|
+
}
|
|
9969
|
+
}
|
|
9970
|
+
const embedResult2 = chunksToEmbed.length > 0 ? await runEmbeddingPass(slug, chunksToEmbed) : { embeddingsWritten: 0, embeddingsFailed: 0 };
|
|
9971
|
+
if (embedResult2.embedError) {
|
|
9972
|
+
logKbPipeline("11_response", {
|
|
9973
|
+
slug,
|
|
9974
|
+
branch: "link",
|
|
9975
|
+
documentId: existingDocumentId,
|
|
9976
|
+
ok: false,
|
|
9977
|
+
embeddingError: true
|
|
9978
|
+
});
|
|
9979
|
+
return json(
|
|
9980
|
+
{
|
|
9981
|
+
documentId: existingDocumentId,
|
|
9982
|
+
linked: true,
|
|
9983
|
+
created: false,
|
|
9984
|
+
chunkCount: chunkCount + chunksCreated,
|
|
9985
|
+
chunksCreated,
|
|
9986
|
+
embeddingAttempted: true,
|
|
9987
|
+
embeddingsWritten: 0,
|
|
9988
|
+
embeddingsFailed: chunksToEmbed.length,
|
|
9989
|
+
warning: "Document linked; embedding step failed. Fix LLM/embed config and attach again (or unlink and re-attach) to retry NULL embeddings.",
|
|
9990
|
+
detail: embedResult2.embedError
|
|
9991
|
+
},
|
|
9992
|
+
{ status: 201 }
|
|
9993
|
+
);
|
|
9994
|
+
}
|
|
9995
|
+
logKbPipeline("11_response", {
|
|
9996
|
+
slug,
|
|
9997
|
+
branch: "link",
|
|
9998
|
+
documentId: existingDocumentId,
|
|
9999
|
+
ok: true,
|
|
10000
|
+
chunkCount: chunkCount + chunksCreated,
|
|
10001
|
+
chunksCreated,
|
|
10002
|
+
chunksQueuedForEmbedding: chunksToEmbed.length,
|
|
10003
|
+
embeddingsWritten: embedResult2.embeddingsWritten,
|
|
10004
|
+
embeddingsFailed: embedResult2.embeddingsFailed
|
|
10005
|
+
});
|
|
10006
|
+
return json({
|
|
10007
|
+
documentId: existingDocumentId,
|
|
10008
|
+
linked: true,
|
|
10009
|
+
created: false,
|
|
10010
|
+
chunkCount: chunkCount + chunksCreated,
|
|
10011
|
+
chunksCreated: chunksCreated || void 0,
|
|
10012
|
+
chunksQueuedForEmbedding: chunksToEmbed.length,
|
|
10013
|
+
embeddingAttempted: chunksToEmbed.length > 0,
|
|
10014
|
+
embeddingsWritten: embedResult2.embeddingsWritten,
|
|
10015
|
+
embeddingsFailed: embedResult2.embeddingsFailed
|
|
10016
|
+
});
|
|
9571
10017
|
}
|
|
10018
|
+
logKbPipeline("05_branch", { slug, branch: "new_upload_ingest" });
|
|
9572
10019
|
if (!text) return json({ error: "text or file with text content is required" }, { status: 400 });
|
|
9573
10020
|
if (!name) name = "Untitled";
|
|
9574
10021
|
const parts = splitIntoChunks(text, INGEST_CHUNK_CHARS);
|
|
@@ -9583,6 +10030,13 @@ function createLlmAgentKnowledgeHandlers(config) {
|
|
|
9583
10030
|
{ status: 413 }
|
|
9584
10031
|
);
|
|
9585
10032
|
}
|
|
10033
|
+
logKbPipeline("07_chunk_split", {
|
|
10034
|
+
slug,
|
|
10035
|
+
partCount: parts.length,
|
|
10036
|
+
maxChunkChars: INGEST_CHUNK_CHARS,
|
|
10037
|
+
totalChars: text.length,
|
|
10038
|
+
documentName: name
|
|
10039
|
+
});
|
|
9586
10040
|
const docRepo = dataSource.getRepository(kbDoc);
|
|
9587
10041
|
const chunkRepo = dataSource.getRepository(kbChunk);
|
|
9588
10042
|
const now = /* @__PURE__ */ new Date();
|
|
@@ -9590,6 +10044,13 @@ function createLlmAgentKnowledgeHandlers(config) {
|
|
|
9590
10044
|
docRepo.create({ name, content: text, sourceUrl, createdAt: now, updatedAt: now })
|
|
9591
10045
|
);
|
|
9592
10046
|
const docId = doc.id;
|
|
10047
|
+
logKbPipeline("06_knowledge_document_saved", {
|
|
10048
|
+
slug,
|
|
10049
|
+
documentId: docId,
|
|
10050
|
+
name,
|
|
10051
|
+
contentChars: text.length,
|
|
10052
|
+
hasSourceUrl: !!sourceUrl
|
|
10053
|
+
});
|
|
9593
10054
|
const chunkRows = parts.map(
|
|
9594
10055
|
(content, i) => chunkRepo.create({ documentId: docId, content, chunkIndex: i, createdAt: now })
|
|
9595
10056
|
);
|
|
@@ -9600,63 +10061,66 @@ function createLlmAgentKnowledgeHandlers(config) {
|
|
|
9600
10061
|
content: parts[i]
|
|
9601
10062
|
})
|
|
9602
10063
|
);
|
|
10064
|
+
logKbPipeline("07b_chunks_persisted", {
|
|
10065
|
+
slug,
|
|
10066
|
+
documentId: docId,
|
|
10067
|
+
rowsInserted: savedChunks.length,
|
|
10068
|
+
firstChunkId: savedChunks[0]?.id,
|
|
10069
|
+
lastChunkId: savedChunks[savedChunks.length - 1]?.id
|
|
10070
|
+
});
|
|
9603
10071
|
const dup = await linkRepo.findOne({ where: { agentId: agent.id, documentId: docId } });
|
|
9604
10072
|
if (!dup) {
|
|
9605
10073
|
await linkRepo.save(linkRepo.create({ agentId: agent.id, documentId: docId }));
|
|
10074
|
+
logKbPipeline("06c_junction_link_created", { slug, agentId: agent.id, documentId: docId });
|
|
10075
|
+
} else {
|
|
10076
|
+
logKbPipeline("06c_junction_link_exists", { slug, agentId: agent.id, documentId: docId });
|
|
9606
10077
|
}
|
|
9607
|
-
|
|
9608
|
-
|
|
9609
|
-
|
|
9610
|
-
|
|
9611
|
-
|
|
9612
|
-
|
|
9613
|
-
|
|
9614
|
-
|
|
9615
|
-
|
|
9616
|
-
dataSource,
|
|
9617
|
-
embedBound,
|
|
9618
|
-
savedChunks,
|
|
9619
|
-
EMBED_CONCURRENCY
|
|
9620
|
-
);
|
|
9621
|
-
embeddingsWritten = written;
|
|
9622
|
-
embeddingsFailed = failed;
|
|
9623
|
-
} else {
|
|
9624
|
-
console.error(`${KB_LOG} embeddings skipped`, {
|
|
9625
|
-
slug,
|
|
9626
|
-
hasLlmPlugin: !!llm,
|
|
9627
|
-
hasEmbed: typeof llm?.embed === "function",
|
|
9628
|
-
chunkCount: savedChunks.length,
|
|
9629
|
-
hint: !llm || typeof llm.embed !== "function" ? "LLM plugin missing or no embed(); check LLM_GATEWAY_URL + LLM_API_KEY so llm plugin initializes." : void 0
|
|
9630
|
-
});
|
|
9631
|
-
}
|
|
9632
|
-
} catch (embErr) {
|
|
9633
|
-
const detail = embErr instanceof Error ? embErr.message : String(embErr);
|
|
9634
|
-
console.error(`${KB_LOG} embedding step threw before/during batch`, { slug, detail, embErr });
|
|
10078
|
+
const embedResult = await runEmbeddingPass(slug, savedChunks);
|
|
10079
|
+
if (embedResult.embedError) {
|
|
10080
|
+
logKbPipeline("11_response", {
|
|
10081
|
+
slug,
|
|
10082
|
+
branch: "ingest",
|
|
10083
|
+
documentId: docId,
|
|
10084
|
+
ok: false,
|
|
10085
|
+
embeddingError: true
|
|
10086
|
+
});
|
|
9635
10087
|
return json(
|
|
9636
10088
|
{
|
|
9637
10089
|
documentId: docId,
|
|
9638
10090
|
chunkCount: savedChunks.length,
|
|
10091
|
+
embeddingAttempted: true,
|
|
9639
10092
|
created: true,
|
|
9640
10093
|
linked: true,
|
|
9641
10094
|
embeddingsWritten: 0,
|
|
9642
10095
|
embeddingsFailed: savedChunks.length,
|
|
9643
10096
|
warning: "Document saved and linked; embedding step failed.",
|
|
9644
|
-
detail
|
|
10097
|
+
detail: embedResult.embedError
|
|
9645
10098
|
},
|
|
9646
10099
|
{ status: 201 }
|
|
9647
10100
|
);
|
|
9648
10101
|
}
|
|
10102
|
+
logKbPipeline("11_response", {
|
|
10103
|
+
slug,
|
|
10104
|
+
branch: "ingest",
|
|
10105
|
+
documentId: docId,
|
|
10106
|
+
ok: true,
|
|
10107
|
+
chunkCount: savedChunks.length,
|
|
10108
|
+
embeddingsWritten: embedResult.embeddingsWritten,
|
|
10109
|
+
embeddingsFailed: embedResult.embeddingsFailed
|
|
10110
|
+
});
|
|
9649
10111
|
return json({
|
|
9650
10112
|
documentId: docId,
|
|
9651
10113
|
chunkCount: savedChunks.length,
|
|
10114
|
+
embeddingAttempted: savedChunks.length > 0,
|
|
9652
10115
|
created: true,
|
|
9653
10116
|
linked: true,
|
|
9654
|
-
embeddingsWritten,
|
|
9655
|
-
embeddingsFailed
|
|
10117
|
+
embeddingsWritten: embedResult.embeddingsWritten,
|
|
10118
|
+
embeddingsFailed: embedResult.embeddingsFailed
|
|
9656
10119
|
});
|
|
9657
10120
|
} catch (err) {
|
|
9658
10121
|
const msg = err instanceof Error ? err.message : "Failed to ingest knowledge";
|
|
9659
10122
|
const name = err instanceof Error ? err.name : "";
|
|
10123
|
+
logKbPipeline("99_pipeline_error", { slug, errorName: name, message: msg });
|
|
9660
10124
|
return json({ error: msg, errorName: name || void 0 }, { status: 500 });
|
|
9661
10125
|
}
|
|
9662
10126
|
},
|
|
@@ -9668,6 +10132,7 @@ function createLlmAgentKnowledgeHandlers(config) {
|
|
|
9668
10132
|
try {
|
|
9669
10133
|
const agent = await findAgentBySlug(dataSource, llmAgents, slug);
|
|
9670
10134
|
if (!agent) return json({ error: "Agent not found" }, { status: 404 });
|
|
10135
|
+
logKbPipeline("unlink_junction", { slug, agentId: agent.id, documentId });
|
|
9671
10136
|
const linkRepo = dataSource.getRepository(junction);
|
|
9672
10137
|
await linkRepo.delete({ agentId: agent.id, documentId });
|
|
9673
10138
|
return json({ ok: true });
|
|
@@ -9929,6 +10394,28 @@ function createAdminRolesHandlers(config) {
|
|
|
9929
10394
|
|
|
9930
10395
|
// src/api/cms-api-handler.ts
|
|
9931
10396
|
var KNOWLEDGE_SUFFIX = "knowledge";
|
|
10397
|
+
var CMS_API_LOG = "[cms-api]";
|
|
10398
|
+
function withLlmKnowledgeEntityFallbacks(base) {
|
|
10399
|
+
const keys = [
|
|
10400
|
+
"llm_agents",
|
|
10401
|
+
"llm_agent_knowledge_documents",
|
|
10402
|
+
"knowledge_base_documents",
|
|
10403
|
+
"knowledge_base_chunks"
|
|
10404
|
+
];
|
|
10405
|
+
const patch = {};
|
|
10406
|
+
for (const key of keys) {
|
|
10407
|
+
if (!(key in base) || base[key] == null) {
|
|
10408
|
+
const fallback = CMS_ENTITY_MAP[key];
|
|
10409
|
+
if (fallback) patch[key] = fallback;
|
|
10410
|
+
}
|
|
10411
|
+
}
|
|
10412
|
+
const merged = Object.keys(patch).length > 0 ? { ...base, ...patch } : base;
|
|
10413
|
+
if (!merged.llm_agents) {
|
|
10414
|
+
const agent = CMS_ENTITY_MAP.llm_agents ?? LlmAgent;
|
|
10415
|
+
return { ...merged, llm_agents: agent };
|
|
10416
|
+
}
|
|
10417
|
+
return merged;
|
|
10418
|
+
}
|
|
9932
10419
|
function matchLlmAgentKnowledgeRoute(path) {
|
|
9933
10420
|
const p = path[0] === "api" ? path.slice(1) : path;
|
|
9934
10421
|
if (p[0] !== "llm_agents" || p.length < 2) return null;
|
|
@@ -9966,9 +10453,9 @@ var DEFAULT_EXCLUDE = /* @__PURE__ */ new Set([
|
|
|
9966
10453
|
function createCmsApiHandler(config) {
|
|
9967
10454
|
const {
|
|
9968
10455
|
dataSource,
|
|
9969
|
-
entityMap,
|
|
10456
|
+
entityMap: rawEntityMap,
|
|
9970
10457
|
pathToModel = (s) => s,
|
|
9971
|
-
crudResources
|
|
10458
|
+
crudResources: crudResourcesOverride,
|
|
9972
10459
|
getCms,
|
|
9973
10460
|
userAuth: userAuthConfig,
|
|
9974
10461
|
dashboard,
|
|
@@ -9989,6 +10476,9 @@ function createCmsApiHandler(config) {
|
|
|
9989
10476
|
requireEntityPermission: userRequireEntityPermission,
|
|
9990
10477
|
getSessionUser
|
|
9991
10478
|
} = config;
|
|
10479
|
+
const entityMap = withLlmKnowledgeEntityFallbacks(rawEntityMap);
|
|
10480
|
+
const baseCrudResources = crudResourcesOverride ?? Object.keys(entityMap).filter((k) => !DEFAULT_EXCLUDE.has(k));
|
|
10481
|
+
const crudResources = entityMap.llm_agents && !baseCrudResources.includes("llm_agents") ? [...baseCrudResources, "llm_agents"] : baseCrudResources;
|
|
9992
10482
|
const requireEntityPermissionEffective = userRequireEntityPermission ?? (async (_req, entity, action) => config.json({ error: "Forbidden", reason: "entity_rbac_required", entity, action }, { status: 403 }));
|
|
9993
10483
|
const analytics = analyticsConfig ?? (getCms ? {
|
|
9994
10484
|
json: config.json,
|
|
@@ -10095,7 +10585,7 @@ function createCmsApiHandler(config) {
|
|
|
10095
10585
|
const chatHandlers = chatConfig ? createChatHandlers(chatConfig) : null;
|
|
10096
10586
|
const llmAgentKnowledgeMerged = llmAgentKnowledgeConfig ?? (chatConfig ? {
|
|
10097
10587
|
dataSource: chatConfig.dataSource,
|
|
10098
|
-
entityMap: chatConfig.entityMap,
|
|
10588
|
+
entityMap: withLlmKnowledgeEntityFallbacks(chatConfig.entityMap ?? rawEntityMap),
|
|
10099
10589
|
getCms: chatConfig.getCms,
|
|
10100
10590
|
json: chatConfig.json,
|
|
10101
10591
|
requireAuth: chatConfig.requireAuth
|
|
@@ -10109,7 +10599,9 @@ function createCmsApiHandler(config) {
|
|
|
10109
10599
|
return crudResources.includes(model) ? model : segment;
|
|
10110
10600
|
}
|
|
10111
10601
|
return {
|
|
10112
|
-
async handle(method,
|
|
10602
|
+
async handle(method, pathInput, req) {
|
|
10603
|
+
const m = typeof method === "string" ? method.toUpperCase() : "GET";
|
|
10604
|
+
const path = pathInput.length > 0 && pathInput[0] === "api" ? pathInput.slice(1) : pathInput;
|
|
10113
10605
|
async function analyticsGate() {
|
|
10114
10606
|
const a = await config.requireAuth(req);
|
|
10115
10607
|
if (a) return a;
|
|
@@ -10117,86 +10609,86 @@ function createCmsApiHandler(config) {
|
|
|
10117
10609
|
}
|
|
10118
10610
|
if (path[0] === "admin" && path[1] === "roles") {
|
|
10119
10611
|
if (!adminRoles) return config.json({ error: "Not found" }, { status: 404 });
|
|
10120
|
-
if (path.length === 2 &&
|
|
10121
|
-
if (path.length === 2 &&
|
|
10122
|
-
if (path.length === 3 &&
|
|
10123
|
-
if (path.length === 3 &&
|
|
10124
|
-
if (path.length === 4 && path[3] === "permissions" &&
|
|
10612
|
+
if (path.length === 2 && m === "GET") return adminRoles.list();
|
|
10613
|
+
if (path.length === 2 && m === "POST") return adminRoles.createGroup(req);
|
|
10614
|
+
if (path.length === 3 && m === "PATCH") return adminRoles.patchGroup(req, path[2]);
|
|
10615
|
+
if (path.length === 3 && m === "DELETE") return adminRoles.deleteGroup(path[2]);
|
|
10616
|
+
if (path.length === 4 && path[3] === "permissions" && m === "PUT") return adminRoles.putPermissions(req, path[2]);
|
|
10125
10617
|
return config.json({ error: "Not found" }, { status: 404 });
|
|
10126
10618
|
}
|
|
10127
|
-
if (path[0] === "dashboard" && path[1] === "stats" && path.length === 2 &&
|
|
10619
|
+
if (path[0] === "dashboard" && path[1] === "stats" && path.length === 2 && m === "GET" && dashboardGet) {
|
|
10128
10620
|
return dashboardGet(req);
|
|
10129
10621
|
}
|
|
10130
|
-
if (path[0] === "dashboard" && path[1] === "ecommerce" && path.length === 2 &&
|
|
10622
|
+
if (path[0] === "dashboard" && path[1] === "ecommerce" && path.length === 2 && m === "GET" && ecommerceAnalyticsGet) {
|
|
10131
10623
|
const g = await analyticsGate();
|
|
10132
10624
|
if (g) return g;
|
|
10133
10625
|
return ecommerceAnalyticsGet(req);
|
|
10134
10626
|
}
|
|
10135
10627
|
if (path[0] === "analytics" && analyticsHandlers) {
|
|
10136
|
-
if (path.length === 1 &&
|
|
10628
|
+
if (path.length === 1 && m === "GET") {
|
|
10137
10629
|
const g = await analyticsGate();
|
|
10138
10630
|
if (g) return g;
|
|
10139
10631
|
return analyticsHandlers.GET(req);
|
|
10140
10632
|
}
|
|
10141
|
-
if (path.length === 2 && path[1] === "property-id" &&
|
|
10633
|
+
if (path.length === 2 && path[1] === "property-id" && m === "GET") {
|
|
10142
10634
|
const g = await analyticsGate();
|
|
10143
10635
|
if (g) return g;
|
|
10144
10636
|
return analyticsHandlers.propertyId();
|
|
10145
10637
|
}
|
|
10146
|
-
if (path.length === 2 && path[1] === "permissions" &&
|
|
10638
|
+
if (path.length === 2 && path[1] === "permissions" && m === "GET") {
|
|
10147
10639
|
const g = await analyticsGate();
|
|
10148
10640
|
if (g) return g;
|
|
10149
10641
|
return analyticsHandlers.permissions();
|
|
10150
10642
|
}
|
|
10151
10643
|
}
|
|
10152
|
-
if (path[0] === "upload" && path.length === 1 &&
|
|
10153
|
-
if (path[0] === "media" && path[1] === "extract" && path.length === 3 &&
|
|
10644
|
+
if (path[0] === "upload" && path.length === 1 && m === "POST" && uploadPost) return uploadPost(req);
|
|
10645
|
+
if (path[0] === "media" && path[1] === "extract" && path.length === 3 && m === "POST" && zipExtractPost) {
|
|
10154
10646
|
return zipExtractPost(req, path[2]);
|
|
10155
10647
|
}
|
|
10156
|
-
if (path[0] === "blogs" && path[1] === "slug" && path.length === 3 &&
|
|
10648
|
+
if (path[0] === "blogs" && path[1] === "slug" && path.length === 3 && m === "GET" && blogBySlugGet) {
|
|
10157
10649
|
return blogBySlugGet(req, path[2]);
|
|
10158
10650
|
}
|
|
10159
|
-
if (path[0] === "forms" && path[1] === "slug" && path.length === 3 &&
|
|
10651
|
+
if (path[0] === "forms" && path[1] === "slug" && path.length === 3 && m === "GET" && formBySlugGet) {
|
|
10160
10652
|
return formBySlugGet(req, path[2]);
|
|
10161
10653
|
}
|
|
10162
10654
|
if (path[0] === "form-submissions") {
|
|
10163
10655
|
if (path.length === 1) {
|
|
10164
|
-
if (
|
|
10165
|
-
if (
|
|
10656
|
+
if (m === "GET" && formSubmissionList) return formSubmissionList(req);
|
|
10657
|
+
if (m === "POST" && formSubmissionPost) return formSubmissionPost(req);
|
|
10166
10658
|
}
|
|
10167
|
-
if (path.length === 2 &&
|
|
10659
|
+
if (path.length === 2 && m === "GET" && formSubmissionGetById) return formSubmissionGetById(req, path[1]);
|
|
10168
10660
|
}
|
|
10169
10661
|
if (path[0] === "forms" && formSaveHandlers) {
|
|
10170
|
-
if (path.length === 1 &&
|
|
10662
|
+
if (path.length === 1 && m === "POST") return formSaveHandlers.POST(req);
|
|
10171
10663
|
if (path.length === 2) {
|
|
10172
|
-
if (
|
|
10173
|
-
if (
|
|
10664
|
+
if (m === "GET") return formSaveHandlers.GET(req, path[1]);
|
|
10665
|
+
if (m === "PUT" || m === "PATCH") return formSaveHandlers.PUT(req, path[1]);
|
|
10174
10666
|
}
|
|
10175
10667
|
}
|
|
10176
10668
|
if (path[0] === "users" && usersHandlers) {
|
|
10177
10669
|
if (path.length === 1) {
|
|
10178
|
-
if (
|
|
10179
|
-
if (
|
|
10670
|
+
if (m === "GET") return usersHandlers.list(req);
|
|
10671
|
+
if (m === "POST") return usersHandlers.create(req);
|
|
10180
10672
|
}
|
|
10181
10673
|
if (path.length === 2) {
|
|
10182
|
-
if (path[1] === "avatar" &&
|
|
10183
|
-
if (path[1] === "profile" &&
|
|
10674
|
+
if (path[1] === "avatar" && m === "POST" && avatarPost) return avatarPost(req);
|
|
10675
|
+
if (path[1] === "profile" && m === "PUT" && profilePut) return profilePut(req);
|
|
10184
10676
|
const id = path[1];
|
|
10185
|
-
if (
|
|
10186
|
-
if (
|
|
10187
|
-
if (
|
|
10677
|
+
if (m === "GET") return usersHandlers.getById(req, id);
|
|
10678
|
+
if (m === "PUT" || m === "PATCH") return usersHandlers.update(req, id);
|
|
10679
|
+
if (m === "DELETE") return usersHandlers.delete(req, id);
|
|
10188
10680
|
}
|
|
10189
|
-
if (path.length === 3 && path[2] === "regenerate-invite" &&
|
|
10681
|
+
if (path.length === 3 && path[2] === "regenerate-invite" && m === "POST") {
|
|
10190
10682
|
return usersHandlers.regenerateInvite(req, path[1]);
|
|
10191
10683
|
}
|
|
10192
10684
|
}
|
|
10193
|
-
if (path[0] === "users" && path.length === 2 && userAuthRouter &&
|
|
10685
|
+
if (path[0] === "users" && path.length === 2 && userAuthRouter && m === "POST") {
|
|
10194
10686
|
return userAuthRouter.POST(req, path[1]);
|
|
10195
10687
|
}
|
|
10196
10688
|
if (path[0] === "settings" && path.length === 2 && settingsHandlers) {
|
|
10197
10689
|
const group = path[1];
|
|
10198
10690
|
const isPublic = settingsConfig?.publicGetGroups?.includes(group);
|
|
10199
|
-
if (
|
|
10691
|
+
if (m === "GET") {
|
|
10200
10692
|
if (!isPublic) {
|
|
10201
10693
|
const a = await config.requireAuth(req);
|
|
10202
10694
|
if (a) return a;
|
|
@@ -10205,15 +10697,38 @@ function createCmsApiHandler(config) {
|
|
|
10205
10697
|
}
|
|
10206
10698
|
return settingsHandlers.GET(req, group);
|
|
10207
10699
|
}
|
|
10208
|
-
if (
|
|
10700
|
+
if (m === "PUT") {
|
|
10209
10701
|
const pe = await requireEntityPermissionEffective(req, "settings", "update");
|
|
10210
10702
|
if (pe) return pe;
|
|
10211
10703
|
return settingsHandlers.PUT(req, group);
|
|
10212
10704
|
}
|
|
10213
10705
|
}
|
|
10214
10706
|
if (path[0] === "message-templates" && path[1] === "sms" && path.length === 2) {
|
|
10215
|
-
if (
|
|
10216
|
-
if (
|
|
10707
|
+
if (m === "GET") return smsMessageTemplateHandlers.GET(req);
|
|
10708
|
+
if (m === "PUT") return smsMessageTemplateHandlers.PUT(req);
|
|
10709
|
+
}
|
|
10710
|
+
if (path[0] === "llm_agents") {
|
|
10711
|
+
if (!entityMap.llm_agents) {
|
|
10712
|
+
console.error(CMS_API_LOG, "llm_agents route: entity missing after merge", {
|
|
10713
|
+
method: m,
|
|
10714
|
+
pathJoined: path.join("/"),
|
|
10715
|
+
entityMapKeyCount: Object.keys(entityMap).length
|
|
10716
|
+
});
|
|
10717
|
+
return config.json(
|
|
10718
|
+
{ error: "LlmAgent entity is not registered", hint: "Ensure migrations ran and entities include LlmAgent." },
|
|
10719
|
+
{ status: 503 }
|
|
10720
|
+
);
|
|
10721
|
+
}
|
|
10722
|
+
if (path.length === 1) {
|
|
10723
|
+
if (m === "GET") return crud.GET(req, "llm_agents");
|
|
10724
|
+
if (m === "POST") return crud.POST(req, "llm_agents");
|
|
10725
|
+
}
|
|
10726
|
+
if (path.length === 2 && !path[1]?.includes("knowledge")) {
|
|
10727
|
+
const id = path[1];
|
|
10728
|
+
if (m === "GET") return crudById.GET(req, "llm_agents", id);
|
|
10729
|
+
if (m === "PUT" || m === "PATCH") return crudById.PUT(req, "llm_agents", id);
|
|
10730
|
+
if (m === "DELETE") return crudById.DELETE(req, "llm_agents", id);
|
|
10731
|
+
}
|
|
10217
10732
|
}
|
|
10218
10733
|
{
|
|
10219
10734
|
const kbMatch = matchLlmAgentKnowledgeRoute(path);
|
|
@@ -10228,24 +10743,24 @@ function createCmsApiHandler(config) {
|
|
|
10228
10743
|
);
|
|
10229
10744
|
}
|
|
10230
10745
|
const { slug, documentId } = kbMatch;
|
|
10231
|
-
if (
|
|
10746
|
+
if (m === "DELETE" && documentId != null && documentId !== "") {
|
|
10232
10747
|
return llmAgentKnowledgeHandlers.unlink(req, slug, documentId);
|
|
10233
10748
|
}
|
|
10234
|
-
if (
|
|
10749
|
+
if (m === "GET" && (documentId == null || documentId === "")) {
|
|
10235
10750
|
return llmAgentKnowledgeHandlers.list(req, slug);
|
|
10236
10751
|
}
|
|
10237
|
-
if (
|
|
10752
|
+
if (m === "POST" && (documentId == null || documentId === "")) {
|
|
10238
10753
|
return llmAgentKnowledgeHandlers.post(req, slug);
|
|
10239
10754
|
}
|
|
10240
10755
|
}
|
|
10241
10756
|
}
|
|
10242
10757
|
if (path[0] === "chat" && chatHandlers) {
|
|
10243
|
-
if (path.length === 2 && path[1] === "config" &&
|
|
10244
|
-
if (path.length === 2 && path[1] === "identify" &&
|
|
10245
|
-
if (path.length === 4 && path[1] === "conversations" && path[3] === "messages" &&
|
|
10246
|
-
if (path.length === 2 && path[1] === "messages" &&
|
|
10758
|
+
if (path.length === 2 && path[1] === "config" && m === "GET") return chatHandlers.publicConfig(req);
|
|
10759
|
+
if (path.length === 2 && path[1] === "identify" && m === "POST") return chatHandlers.identify(req);
|
|
10760
|
+
if (path.length === 4 && path[1] === "conversations" && path[3] === "messages" && m === "GET") return chatHandlers.getMessages(req, path[2]);
|
|
10761
|
+
if (path.length === 2 && path[1] === "messages" && m === "POST") return chatHandlers.postMessage(req);
|
|
10247
10762
|
}
|
|
10248
|
-
if (path[0] === "orders" && path.length === 3 && path[2] === "invoice" &&
|
|
10763
|
+
if (path[0] === "orders" && path.length === 3 && path[2] === "invoice" && m === "GET" && getCms) {
|
|
10249
10764
|
const a = await config.requireAuth(req);
|
|
10250
10765
|
if (a) return a;
|
|
10251
10766
|
const pe = await requireEntityPermissionEffective(req, "orders", "read");
|
|
@@ -10259,17 +10774,17 @@ function createCmsApiHandler(config) {
|
|
|
10259
10774
|
if (path[0] === "orders" && path.length === 3 && path[2] === "repost-erp" && getCms) {
|
|
10260
10775
|
const a = await config.requireAuth(req);
|
|
10261
10776
|
if (a) return a;
|
|
10262
|
-
const pe = await requireEntityPermissionEffective(req, "orders",
|
|
10777
|
+
const pe = await requireEntityPermissionEffective(req, "orders", m === "GET" ? "read" : "update");
|
|
10263
10778
|
if (pe) return pe;
|
|
10264
10779
|
const oid = Number(path[1]);
|
|
10265
10780
|
if (!Number.isFinite(oid)) return config.json({ error: "Invalid id" }, { status: 400 });
|
|
10266
10781
|
const cms = await getCms();
|
|
10267
10782
|
const { isErpIntegrationEnabled: isErpIntegrationEnabled3 } = await Promise.resolve().then(() => (init_erp_config_enabled(), erp_config_enabled_exports));
|
|
10268
10783
|
const enabled = await isErpIntegrationEnabled3(cms, dataSource, entityMap);
|
|
10269
|
-
if (
|
|
10784
|
+
if (m === "GET") {
|
|
10270
10785
|
return config.json({ enabled });
|
|
10271
10786
|
}
|
|
10272
|
-
if (
|
|
10787
|
+
if (m === "POST") {
|
|
10273
10788
|
if (!enabled) return config.json({ error: "ERP integration is disabled" }, { status: 409 });
|
|
10274
10789
|
const { queueErpPaidOrderForOrderId: queueErpPaidOrderForOrderId2 } = await Promise.resolve().then(() => (init_paid_order_erp(), paid_order_erp_exports));
|
|
10275
10790
|
await queueErpPaidOrderForOrderId2(cms, dataSource, entityMap, oid);
|
|
@@ -10279,28 +10794,39 @@ function createCmsApiHandler(config) {
|
|
|
10279
10794
|
}
|
|
10280
10795
|
if (path.length === 0) return config.json({ error: "Not found" }, { status: 404 });
|
|
10281
10796
|
const resource = resolveResource(path[0]);
|
|
10282
|
-
if (!crudResources.includes(resource))
|
|
10797
|
+
if (!crudResources.includes(resource)) {
|
|
10798
|
+
console.warn(CMS_API_LOG, "generic CRUD gate: Invalid resource (not in crudResources)", {
|
|
10799
|
+
method: m,
|
|
10800
|
+
pathJoined: path.join("/"),
|
|
10801
|
+
pathSegments: path,
|
|
10802
|
+
resource,
|
|
10803
|
+
hasLlmAgentsEntity: Boolean(entityMap.llm_agents),
|
|
10804
|
+
crudResourcesHasResource: crudResources.includes(resource),
|
|
10805
|
+
crudResourcesLength: crudResources.length
|
|
10806
|
+
});
|
|
10807
|
+
return config.json({ error: "Invalid resource" }, { status: 400 });
|
|
10808
|
+
}
|
|
10283
10809
|
if (path.length === 2) {
|
|
10284
|
-
if (path[1] === "metadata" &&
|
|
10810
|
+
if (path[1] === "metadata" && m === "GET") {
|
|
10285
10811
|
return crud.GET_METADATA(req, resource);
|
|
10286
10812
|
}
|
|
10287
|
-
if (path[1] === "bulk" &&
|
|
10813
|
+
if (path[1] === "bulk" && m === "POST") {
|
|
10288
10814
|
return crud.BULK_POST(req, resource);
|
|
10289
10815
|
}
|
|
10290
|
-
if (path[1] === "export" &&
|
|
10816
|
+
if (path[1] === "export" && m === "GET") {
|
|
10291
10817
|
return crud.GET_EXPORT(req, resource);
|
|
10292
10818
|
}
|
|
10293
10819
|
}
|
|
10294
10820
|
if (path.length === 1) {
|
|
10295
|
-
if (
|
|
10296
|
-
if (
|
|
10821
|
+
if (m === "GET") return crud.GET(req, resource);
|
|
10822
|
+
if (m === "POST") return crud.POST(req, resource);
|
|
10297
10823
|
return config.json({ error: "Method not allowed" }, { status: 405 });
|
|
10298
10824
|
}
|
|
10299
10825
|
if (path.length === 2) {
|
|
10300
10826
|
const id = path[1];
|
|
10301
|
-
if (
|
|
10302
|
-
if (
|
|
10303
|
-
if (
|
|
10827
|
+
if (m === "GET") return crudById.GET(req, resource, id);
|
|
10828
|
+
if (m === "PUT" || m === "PATCH") return crudById.PUT(req, resource, id);
|
|
10829
|
+
if (m === "DELETE") return crudById.DELETE(req, resource, id);
|
|
10304
10830
|
return config.json({ error: "Method not allowed" }, { status: 405 });
|
|
10305
10831
|
}
|
|
10306
10832
|
return config.json({ error: "Not found" }, { status: 404 });
|