@infuro/cms-core 1.0.20 → 1.0.22
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/admin.cjs +194 -75
- package/dist/admin.cjs.map +1 -1
- package/dist/admin.js +188 -70
- package/dist/admin.js.map +1 -1
- package/dist/api.cjs +3230 -259
- package/dist/api.cjs.map +1 -1
- package/dist/api.d.cts +1 -1
- package/dist/api.d.ts +1 -1
- package/dist/api.js +3232 -252
- package/dist/api.js.map +1 -1
- package/dist/{index-DGtM2Gsk.d.cts → index-BGAh4fPQ.d.cts} +18 -1
- package/dist/{index--GBYw5JE.d.ts → index-Cnwh7B3r.d.ts} +18 -1
- package/dist/index.cjs +753 -209
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +34 -9
- package/dist/index.d.ts +34 -9
- package/dist/index.js +753 -209
- package/dist/index.js.map +1 -1
- package/dist/migrations/1775300000000-LlmAgents.ts +57 -57
- package/dist/migrations/1775300000001-LlmAgentsValidationRulesText.ts +43 -43
- package/dist/migrations/1775300000002-SeedLlmAgentsPermissions.ts +33 -33
- package/dist/migrations/1775300000003-LlmAgentKnowledgeDocuments.ts +50 -50
- package/package.json +2 -1
package/dist/index.js
CHANGED
|
@@ -2432,6 +2432,17 @@ function localStoragePlugin(config = {}) {
|
|
|
2432
2432
|
}
|
|
2433
2433
|
|
|
2434
2434
|
// src/plugins/llm/llm-service.ts
|
|
2435
|
+
function parseOpenAiEmbeddingJson(data) {
|
|
2436
|
+
let embedding = data?.data?.[0]?.embedding;
|
|
2437
|
+
if (!Array.isArray(embedding) && Array.isArray(data?.embedding)) {
|
|
2438
|
+
embedding = data.embedding;
|
|
2439
|
+
}
|
|
2440
|
+
if (!Array.isArray(embedding) && Array.isArray(data?.data?.[0])) {
|
|
2441
|
+
const first = data.data[0];
|
|
2442
|
+
embedding = Array.isArray(first) ? first : first?.embedding;
|
|
2443
|
+
}
|
|
2444
|
+
return Array.isArray(embedding) ? embedding : null;
|
|
2445
|
+
}
|
|
2435
2446
|
function parseHfInferenceEmbeddingBody(data) {
|
|
2436
2447
|
if (data === null || data === void 0) return null;
|
|
2437
2448
|
if (Array.isArray(data)) {
|
|
@@ -2453,28 +2464,59 @@ function parseHfInferenceEmbeddingBody(data) {
|
|
|
2453
2464
|
}
|
|
2454
2465
|
return null;
|
|
2455
2466
|
}
|
|
2467
|
+
var FALLBACK_CHAT_MODEL_OPENAI_SHAPE = "openai/gpt-4o-mini";
|
|
2468
|
+
var FALLBACK_CHAT_MODEL_HUGGINGFACE = "meta-llama/Meta-Llama-3.1-8B-Instruct";
|
|
2469
|
+
function defaultChatModelForGateway(baseURL) {
|
|
2470
|
+
const u = baseURL.toLowerCase();
|
|
2471
|
+
if (u.includes("huggingface.co") || u.includes("hf.co") || u.includes("hf-inference")) {
|
|
2472
|
+
return FALLBACK_CHAT_MODEL_HUGGINGFACE;
|
|
2473
|
+
}
|
|
2474
|
+
return FALLBACK_CHAT_MODEL_OPENAI_SHAPE;
|
|
2475
|
+
}
|
|
2476
|
+
function formatLlmGatewayError(status, bodyText, defaultChatModel) {
|
|
2477
|
+
let hint = "";
|
|
2478
|
+
try {
|
|
2479
|
+
const j = JSON.parse(bodyText);
|
|
2480
|
+
const code = j?.error?.code;
|
|
2481
|
+
if (code === "model_not_supported" || code === "model_not_found" || /not supported by any provider|does not exist/i.test(j?.error?.message ?? "")) {
|
|
2482
|
+
hint = ` Hint: set LLM_CHAT_MODEL (or CMS) to a model your gateway lists. HF router: Hub ids (e.g. "${FALLBACK_CHAT_MODEL_HUGGINGFACE}"); OpenRouter: e.g. "${FALLBACK_CHAT_MODEL_OPENAI_SHAPE}".`;
|
|
2483
|
+
}
|
|
2484
|
+
} catch {
|
|
2485
|
+
}
|
|
2486
|
+
return `LLM gateway error: ${status} ${bodyText}${hint}`;
|
|
2487
|
+
}
|
|
2456
2488
|
var LlmService = class _LlmService {
|
|
2457
2489
|
static embedHttpErrorLogged = false;
|
|
2458
2490
|
static embedShapeErrorLogged = false;
|
|
2459
2491
|
static embedSuccessLogged = false;
|
|
2460
2492
|
static hfDimensionMismatchWarned = false;
|
|
2493
|
+
static hfInferenceSdkErrorLogged = false;
|
|
2461
2494
|
baseURL;
|
|
2462
2495
|
apiKey;
|
|
2463
2496
|
defaultEmbeddingModel;
|
|
2464
2497
|
embeddingProvider;
|
|
2465
2498
|
hfInferenceBaseURL;
|
|
2466
2499
|
embeddingApiKey;
|
|
2500
|
+
defaultChatModel;
|
|
2467
2501
|
constructor(baseURL, apiKey, defaultEmbeddingModel, embedOpts) {
|
|
2468
2502
|
this.baseURL = baseURL.replace(/\/$/, "");
|
|
2469
2503
|
this.apiKey = apiKey;
|
|
2470
2504
|
this.defaultEmbeddingModel = defaultEmbeddingModel;
|
|
2471
2505
|
const p = (embedOpts?.embeddingProvider ?? "openai").toLowerCase();
|
|
2472
2506
|
this.embeddingProvider = p === "huggingface" ? "huggingface" : "openai";
|
|
2473
|
-
this.hfInferenceBaseURL = (embedOpts?.hfInferenceBaseUrl ?? "https://
|
|
2507
|
+
this.hfInferenceBaseURL = (embedOpts?.hfInferenceBaseUrl ?? "https://router.huggingface.co/v1").replace(
|
|
2474
2508
|
/\/$/,
|
|
2475
2509
|
""
|
|
2476
2510
|
);
|
|
2477
2511
|
this.embeddingApiKey = (embedOpts?.embeddingApiKey ?? apiKey).trim();
|
|
2512
|
+
const fromOpts = embedOpts?.defaultChatModel?.trim();
|
|
2513
|
+
const fromEnv = typeof process !== "undefined" ? process.env.LLM_CHAT_MODEL?.trim() : "";
|
|
2514
|
+
const auto = defaultChatModelForGateway(this.baseURL);
|
|
2515
|
+
this.defaultChatModel = (fromOpts || fromEnv || auto).trim() || auto;
|
|
2516
|
+
}
|
|
2517
|
+
resolveChatModel(explicit) {
|
|
2518
|
+
const m = (explicit?.trim() || this.defaultChatModel).trim();
|
|
2519
|
+
return m || defaultChatModelForGateway(this.baseURL);
|
|
2478
2520
|
}
|
|
2479
2521
|
get base() {
|
|
2480
2522
|
return this.baseURL.replace(/\/$/, "");
|
|
@@ -2514,7 +2556,7 @@ var LlmService = class _LlmService {
|
|
|
2514
2556
|
}
|
|
2515
2557
|
logChatCompletionsRequest(kind, messages, options, stream) {
|
|
2516
2558
|
if (!this.llmDebugEnabled()) return;
|
|
2517
|
-
const model = options.model
|
|
2559
|
+
const model = this.resolveChatModel(options.model);
|
|
2518
2560
|
console.log(`
|
|
2519
2561
|
[LLM ${kind}] POST`, this.url);
|
|
2520
2562
|
console.log("[Request options]", {
|
|
@@ -2532,7 +2574,12 @@ var LlmService = class _LlmService {
|
|
|
2532
2574
|
return this.embedHuggingFaceInference(input, model2);
|
|
2533
2575
|
}
|
|
2534
2576
|
const model = options.model ?? this.defaultEmbeddingModel ?? "text-embedding-3-small";
|
|
2535
|
-
|
|
2577
|
+
return this.fetchOpenAiShapeEmbeddings(`${this.base}/v1/embeddings`, model, input, this.apiKey);
|
|
2578
|
+
}
|
|
2579
|
+
/**
|
|
2580
|
+
* POST `{ url }` with OpenAI-style `{ model, input }` and Bearer token; parses `{ data:[{embedding}] }`.
|
|
2581
|
+
*/
|
|
2582
|
+
async fetchOpenAiShapeEmbeddings(embedUrl, model, input, bearerToken) {
|
|
2536
2583
|
if (this.embedDebugEnabled()) {
|
|
2537
2584
|
console.info("[LLM embed] request", { url: embedUrl, model, inputChars: input.length });
|
|
2538
2585
|
}
|
|
@@ -2540,7 +2587,7 @@ var LlmService = class _LlmService {
|
|
|
2540
2587
|
method: "POST",
|
|
2541
2588
|
headers: {
|
|
2542
2589
|
"Content-Type": "application/json",
|
|
2543
|
-
Authorization: `Bearer ${
|
|
2590
|
+
Authorization: `Bearer ${bearerToken}`
|
|
2544
2591
|
},
|
|
2545
2592
|
body: JSON.stringify({
|
|
2546
2593
|
model,
|
|
@@ -2549,7 +2596,7 @@ var LlmService = class _LlmService {
|
|
|
2549
2596
|
});
|
|
2550
2597
|
if (!res.ok) {
|
|
2551
2598
|
const body = await res.text();
|
|
2552
|
-
const hint404 = res.status === 404 ? "This base URL does not expose OpenAI-compatible POST /v1/embeddings
|
|
2599
|
+
const hint404 = res.status === 404 ? "This base URL does not expose OpenAI-compatible POST /v1/embeddings. For Hugging Face embeddings, set EMBEDDING_PROVIDER=huggingface and an HF token (EMBEDDING_API_KEY); the server uses @huggingface/inference (router /v1/embeddings is chat-only and is not used)." : "Embeddings HTTP error \u2014 knowledge_base_chunks.embedding will stay NULL for failed calls.";
|
|
2553
2600
|
if (!_LlmService.embedHttpErrorLogged) {
|
|
2554
2601
|
_LlmService.embedHttpErrorLogged = true;
|
|
2555
2602
|
console.error("[LLM embed]", hint404, {
|
|
@@ -2573,15 +2620,8 @@ var LlmService = class _LlmService {
|
|
|
2573
2620
|
}
|
|
2574
2621
|
return [];
|
|
2575
2622
|
}
|
|
2576
|
-
|
|
2577
|
-
if (!
|
|
2578
|
-
embedding = data.embedding;
|
|
2579
|
-
}
|
|
2580
|
-
if (!Array.isArray(embedding) && Array.isArray(data?.data?.[0])) {
|
|
2581
|
-
const first = data.data[0];
|
|
2582
|
-
embedding = Array.isArray(first) ? first : first?.embedding;
|
|
2583
|
-
}
|
|
2584
|
-
if (!Array.isArray(embedding)) {
|
|
2623
|
+
const embedding = parseOpenAiEmbeddingJson(data);
|
|
2624
|
+
if (!embedding) {
|
|
2585
2625
|
if (!_LlmService.embedShapeErrorLogged) {
|
|
2586
2626
|
_LlmService.embedShapeErrorLogged = true;
|
|
2587
2627
|
console.error(
|
|
@@ -2603,96 +2643,201 @@ var LlmService = class _LlmService {
|
|
|
2603
2643
|
}
|
|
2604
2644
|
return embedding;
|
|
2605
2645
|
}
|
|
2606
|
-
|
|
2607
|
-
|
|
2608
|
-
|
|
2609
|
-
|
|
2610
|
-
|
|
2611
|
-
|
|
2612
|
-
|
|
2613
|
-
|
|
2614
|
-
|
|
2615
|
-
|
|
2616
|
-
|
|
2617
|
-
|
|
2618
|
-
let embedUrl = pipelineUrl;
|
|
2619
|
-
let res = await fetch(embedUrl, {
|
|
2620
|
-
method: "POST",
|
|
2621
|
-
headers: {
|
|
2622
|
-
"Content-Type": "application/json",
|
|
2623
|
-
Authorization: `Bearer ${this.embeddingApiKey}`
|
|
2624
|
-
},
|
|
2625
|
-
body: requestBody
|
|
2626
|
-
});
|
|
2627
|
-
let bodyText = await res.text();
|
|
2628
|
-
if (res.status === 404) {
|
|
2629
|
-
embedUrl = modelUrl;
|
|
2630
|
-
if (this.embedDebugEnabled()) {
|
|
2631
|
-
console.warn("[LLM embed HF] /pipeline route returned 404; retrying with /models.", {
|
|
2632
|
-
model,
|
|
2633
|
-
fallbackUrl: embedUrl
|
|
2634
|
-
});
|
|
2635
|
-
}
|
|
2636
|
-
res = await fetch(embedUrl, {
|
|
2637
|
-
method: "POST",
|
|
2638
|
-
headers: {
|
|
2639
|
-
"Content-Type": "application/json",
|
|
2640
|
-
Authorization: `Bearer ${this.embeddingApiKey}`
|
|
2641
|
-
},
|
|
2642
|
-
body: requestBody
|
|
2646
|
+
/**
|
|
2647
|
+
* Hugging Face–recommended path: Hub resolves a provider that supports `feature-extraction` for the model.
|
|
2648
|
+
*/
|
|
2649
|
+
async embedViaHuggingfaceInferenceSdk(input, model) {
|
|
2650
|
+
const token = this.embeddingApiKey.trim();
|
|
2651
|
+
if (!token) return null;
|
|
2652
|
+
try {
|
|
2653
|
+
const { InferenceClient } = await import("@huggingface/inference");
|
|
2654
|
+
const hf = new InferenceClient(token);
|
|
2655
|
+
const raw = await hf.featureExtraction({
|
|
2656
|
+
model,
|
|
2657
|
+
inputs: input
|
|
2643
2658
|
});
|
|
2644
|
-
|
|
2645
|
-
|
|
2646
|
-
|
|
2647
|
-
|
|
2648
|
-
|
|
2649
|
-
|
|
2650
|
-
|
|
2651
|
-
|
|
2652
|
-
|
|
2653
|
-
|
|
2654
|
-
|
|
2655
|
-
|
|
2656
|
-
|
|
2657
|
-
|
|
2659
|
+
const embedding = parseHfInferenceEmbeddingBody(raw);
|
|
2660
|
+
if (!embedding?.length) {
|
|
2661
|
+
if (this.embedDebugEnabled()) {
|
|
2662
|
+
console.warn("[LLM embed HF] Inference SDK returned no parseable vector", {
|
|
2663
|
+
model,
|
|
2664
|
+
shape: Array.isArray(raw) ? `array(len=${raw.length})` : typeof raw
|
|
2665
|
+
});
|
|
2666
|
+
}
|
|
2667
|
+
return null;
|
|
2668
|
+
}
|
|
2669
|
+
return embedding;
|
|
2670
|
+
} catch (err) {
|
|
2671
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
2672
|
+
if (!_LlmService.hfInferenceSdkErrorLogged) {
|
|
2673
|
+
_LlmService.hfInferenceSdkErrorLogged = true;
|
|
2674
|
+
console.warn(
|
|
2675
|
+
"[LLM embed HF] @huggingface/inference featureExtraction failed \u2014 trying legacy HTTP URLs.",
|
|
2676
|
+
{ model, message: message.slice(0, 500) }
|
|
2658
2677
|
);
|
|
2659
2678
|
} else if (this.embedDebugEnabled()) {
|
|
2660
|
-
console.warn("[LLM embed HF] repeat
|
|
2679
|
+
console.warn("[LLM embed HF] repeat Inference SDK failure", { message: message.slice(0, 200) });
|
|
2661
2680
|
}
|
|
2662
|
-
return
|
|
2681
|
+
return null;
|
|
2663
2682
|
}
|
|
2664
|
-
|
|
2665
|
-
|
|
2666
|
-
|
|
2667
|
-
|
|
2668
|
-
|
|
2669
|
-
_LlmService.embedShapeErrorLogged = true;
|
|
2670
|
-
console.error("[LLM embed HF] Response is not JSON \u2014 treating as no embedding.");
|
|
2671
|
-
}
|
|
2672
|
-
return [];
|
|
2683
|
+
}
|
|
2684
|
+
/** Base URL for legacy `…/pipeline/…` and `…/models/{model}` tries (HF router only). */
|
|
2685
|
+
hfLegacyInferenceBase(normalizedBase) {
|
|
2686
|
+
if (/router\.huggingface\.co|api-inference\.huggingface\.co/i.test(normalizedBase)) {
|
|
2687
|
+
return "https://router.huggingface.co/hf-inference";
|
|
2673
2688
|
}
|
|
2674
|
-
|
|
2675
|
-
|
|
2676
|
-
|
|
2677
|
-
|
|
2678
|
-
|
|
2679
|
-
|
|
2689
|
+
return normalizedBase;
|
|
2690
|
+
}
|
|
2691
|
+
/**
|
|
2692
|
+
* Hugging Face embeddings: `@huggingface/inference` (Hub-routed feature extraction), then
|
|
2693
|
+
* legacy direct `hf-inference` HTTP tries. Note: `https://router.huggingface.co/v1/embeddings` is not
|
|
2694
|
+
* offered for embeddings (OpenAI-compatible `/v1` on HF is chat-only).
|
|
2695
|
+
*/
|
|
2696
|
+
async embedHuggingFaceInference(input, model) {
|
|
2697
|
+
const normalizedBase = this.hfInferenceBaseURL.replace(/\/$/, "");
|
|
2698
|
+
const fromSdk = await this.embedViaHuggingfaceInferenceSdk(input, model);
|
|
2699
|
+
if (fromSdk?.length) {
|
|
2700
|
+
if (!_LlmService.embedSuccessLogged) {
|
|
2701
|
+
_LlmService.embedSuccessLogged = true;
|
|
2702
|
+
console.info("[LLM embed HF] first success", { dimension: fromSdk.length, model, via: "@huggingface/inference" });
|
|
2703
|
+
if (!_LlmService.hfDimensionMismatchWarned && fromSdk.length !== 1536) {
|
|
2704
|
+
_LlmService.hfDimensionMismatchWarned = true;
|
|
2705
|
+
console.warn(
|
|
2706
|
+
`[LLM embed HF] Embedding dimensions=${fromSdk.length}; ensure knowledge_base_chunks.embedding uses vector(${fromSdk.length}) (see migrations).`
|
|
2707
|
+
);
|
|
2708
|
+
}
|
|
2709
|
+
} else if (this.embedDebugEnabled()) {
|
|
2710
|
+
console.info("[LLM embed HF] ok", { dimension: fromSdk.length, via: "@huggingface/inference" });
|
|
2680
2711
|
}
|
|
2681
|
-
return
|
|
2712
|
+
return fromSdk;
|
|
2682
2713
|
}
|
|
2683
|
-
|
|
2684
|
-
|
|
2685
|
-
|
|
2686
|
-
|
|
2687
|
-
|
|
2688
|
-
|
|
2689
|
-
|
|
2690
|
-
|
|
2714
|
+
const legacyBase = this.hfLegacyInferenceBase(normalizedBase);
|
|
2715
|
+
const baseWithProvider = /\/hf-inference$/i.test(legacyBase) || !/router\.huggingface\.co/i.test(legacyBase) ? legacyBase : `${legacyBase.replace(/\/$/, "")}/hf-inference`;
|
|
2716
|
+
const bodyVariants = [
|
|
2717
|
+
{
|
|
2718
|
+
label: "inputs_parameters_mean_normalize",
|
|
2719
|
+
body: JSON.stringify({
|
|
2720
|
+
inputs: input,
|
|
2721
|
+
parameters: { pooling: "mean", normalize: true },
|
|
2722
|
+
options: { wait_for_model: true }
|
|
2723
|
+
})
|
|
2724
|
+
},
|
|
2725
|
+
{
|
|
2726
|
+
label: "inputs_string",
|
|
2727
|
+
body: JSON.stringify({ inputs: input, options: { wait_for_model: true } })
|
|
2728
|
+
},
|
|
2729
|
+
{
|
|
2730
|
+
label: "inputs_string_array",
|
|
2731
|
+
body: JSON.stringify({ inputs: [input], options: { wait_for_model: true } })
|
|
2732
|
+
},
|
|
2733
|
+
{
|
|
2734
|
+
label: "inputs_wrapped_sentences",
|
|
2735
|
+
body: JSON.stringify({ inputs: { sentences: [input] }, options: { wait_for_model: true } })
|
|
2736
|
+
},
|
|
2737
|
+
{
|
|
2738
|
+
label: "top_level_sentences",
|
|
2739
|
+
body: JSON.stringify({ sentences: [input], options: { wait_for_model: true } })
|
|
2691
2740
|
}
|
|
2741
|
+
];
|
|
2742
|
+
const headers = {
|
|
2743
|
+
"Content-Type": "application/json",
|
|
2744
|
+
Authorization: `Bearer ${this.embeddingApiKey}`
|
|
2745
|
+
};
|
|
2746
|
+
const candidateUrls = [];
|
|
2747
|
+
const pushUrl = (u) => {
|
|
2748
|
+
if (!candidateUrls.includes(u)) candidateUrls.push(u);
|
|
2749
|
+
};
|
|
2750
|
+
pushUrl(`${baseWithProvider}/pipeline/feature-extraction/${model}`);
|
|
2751
|
+
pushUrl(`${baseWithProvider}/models/${model}`);
|
|
2752
|
+
let lastUrl = "";
|
|
2753
|
+
let lastStatus = 0;
|
|
2754
|
+
let lastBodyPreview = "";
|
|
2755
|
+
let lastAttempt = "";
|
|
2756
|
+
for (const embedUrl of candidateUrls) {
|
|
2757
|
+
for (const { label, body } of bodyVariants) {
|
|
2758
|
+
if (this.embedDebugEnabled()) {
|
|
2759
|
+
console.info("[LLM embed HF] attempt", { url: embedUrl, model, inputChars: input.length, body: label });
|
|
2760
|
+
}
|
|
2761
|
+
const res = await fetch(embedUrl, {
|
|
2762
|
+
method: "POST",
|
|
2763
|
+
headers,
|
|
2764
|
+
body
|
|
2765
|
+
});
|
|
2766
|
+
const bodyText = await res.text();
|
|
2767
|
+
lastUrl = embedUrl;
|
|
2768
|
+
lastStatus = res.status;
|
|
2769
|
+
lastBodyPreview = bodyText.slice(0, 800);
|
|
2770
|
+
lastAttempt = `${embedUrl} (${label})`;
|
|
2771
|
+
if (!res.ok) {
|
|
2772
|
+
if (this.embedDebugEnabled()) {
|
|
2773
|
+
console.warn("[LLM embed HF] attempt failed, trying next if any", {
|
|
2774
|
+
url: embedUrl,
|
|
2775
|
+
body: label,
|
|
2776
|
+
status: res.status,
|
|
2777
|
+
bodyPreview: bodyText.slice(0, 200)
|
|
2778
|
+
});
|
|
2779
|
+
}
|
|
2780
|
+
continue;
|
|
2781
|
+
}
|
|
2782
|
+
let data;
|
|
2783
|
+
try {
|
|
2784
|
+
data = JSON.parse(bodyText);
|
|
2785
|
+
} catch {
|
|
2786
|
+
if (this.embedDebugEnabled()) {
|
|
2787
|
+
console.warn("[LLM embed HF] response not JSON, trying next if any", { url: embedUrl, body: label });
|
|
2788
|
+
}
|
|
2789
|
+
continue;
|
|
2790
|
+
}
|
|
2791
|
+
const embedding = parseHfInferenceEmbeddingBody(data);
|
|
2792
|
+
if (!embedding?.length) {
|
|
2793
|
+
if (this.embedDebugEnabled()) {
|
|
2794
|
+
const preview = typeof data === "object" && data !== null && !Array.isArray(data) ? Object.keys(data) : typeof data;
|
|
2795
|
+
console.warn("[LLM embed HF] no parseable embedding, trying next if any", {
|
|
2796
|
+
url: embedUrl,
|
|
2797
|
+
body: label,
|
|
2798
|
+
preview
|
|
2799
|
+
});
|
|
2800
|
+
}
|
|
2801
|
+
continue;
|
|
2802
|
+
}
|
|
2803
|
+
if (!_LlmService.embedSuccessLogged) {
|
|
2804
|
+
_LlmService.embedSuccessLogged = true;
|
|
2805
|
+
console.info("[LLM embed HF] first success", { dimension: embedding.length, model, url: embedUrl, body: label });
|
|
2806
|
+
if (!_LlmService.hfDimensionMismatchWarned && embedding.length !== 1536) {
|
|
2807
|
+
_LlmService.hfDimensionMismatchWarned = true;
|
|
2808
|
+
console.warn(
|
|
2809
|
+
`[LLM embed HF] Embedding dimensions=${embedding.length}; ensure knowledge_base_chunks.embedding uses vector(${embedding.length}) (see migrations).`
|
|
2810
|
+
);
|
|
2811
|
+
}
|
|
2812
|
+
} else if (this.embedDebugEnabled()) {
|
|
2813
|
+
console.info("[LLM embed HF] ok", { dimension: embedding.length, url: embedUrl, body: label });
|
|
2814
|
+
}
|
|
2815
|
+
return embedding;
|
|
2816
|
+
}
|
|
2817
|
+
}
|
|
2818
|
+
if (!_LlmService.embedHttpErrorLogged) {
|
|
2819
|
+
_LlmService.embedHttpErrorLogged = true;
|
|
2820
|
+
console.error(
|
|
2821
|
+
"[LLM embed HF] All Hugging Face embedding attempts failed \u2014 knowledge chunks stay without vectors.",
|
|
2822
|
+
{
|
|
2823
|
+
triedInferenceSdk: true,
|
|
2824
|
+
triedLegacyHttp: candidateUrls,
|
|
2825
|
+
lastAttempt,
|
|
2826
|
+
lastUrl,
|
|
2827
|
+
lastStatus,
|
|
2828
|
+
lastBodyPreview,
|
|
2829
|
+
hint: "Use an HF token with permission to call Inference Providers / Hub API mapping. Install ships @huggingface/inference; EMBEDDING_MODEL defaults to sentence-transformers/all-MiniLM-L6-v2. Router OpenAI /v1/embeddings is not used (chat-only)."
|
|
2830
|
+
}
|
|
2831
|
+
);
|
|
2692
2832
|
} else if (this.embedDebugEnabled()) {
|
|
2693
|
-
console.
|
|
2833
|
+
console.warn("[LLM embed HF] repeat failure (all attempts)", {
|
|
2834
|
+
lastAttempt,
|
|
2835
|
+
lastUrl,
|
|
2836
|
+
lastStatus,
|
|
2837
|
+
bodyPreview: lastBodyPreview.slice(0, 200)
|
|
2838
|
+
});
|
|
2694
2839
|
}
|
|
2695
|
-
return
|
|
2840
|
+
return [];
|
|
2696
2841
|
}
|
|
2697
2842
|
buildAgentMessages(options) {
|
|
2698
2843
|
const { systemPrompt, userPrompt, history = [], context } = options;
|
|
@@ -2736,7 +2881,7 @@ ${context}`;
|
|
|
2736
2881
|
Authorization: `Bearer ${this.apiKey}`
|
|
2737
2882
|
},
|
|
2738
2883
|
body: JSON.stringify({
|
|
2739
|
-
model: options.model
|
|
2884
|
+
model: this.resolveChatModel(options.model),
|
|
2740
2885
|
messages,
|
|
2741
2886
|
stream: false,
|
|
2742
2887
|
temperature: options.temperature,
|
|
@@ -2745,7 +2890,7 @@ ${context}`;
|
|
|
2745
2890
|
});
|
|
2746
2891
|
if (!res.ok) {
|
|
2747
2892
|
const text = await res.text();
|
|
2748
|
-
throw new Error(
|
|
2893
|
+
throw new Error(formatLlmGatewayError(res.status, text, this.defaultChatModel));
|
|
2749
2894
|
}
|
|
2750
2895
|
const data = await res.json();
|
|
2751
2896
|
const content = data.choices?.[0]?.message?.content ?? "";
|
|
@@ -2763,7 +2908,7 @@ ${context}`;
|
|
|
2763
2908
|
Authorization: `Bearer ${this.apiKey}`
|
|
2764
2909
|
},
|
|
2765
2910
|
body: JSON.stringify({
|
|
2766
|
-
model: options.model
|
|
2911
|
+
model: this.resolveChatModel(options.model),
|
|
2767
2912
|
messages,
|
|
2768
2913
|
stream: true,
|
|
2769
2914
|
temperature: options.temperature,
|
|
@@ -2772,7 +2917,7 @@ ${context}`;
|
|
|
2772
2917
|
});
|
|
2773
2918
|
if (!res.ok) {
|
|
2774
2919
|
const text = await res.text();
|
|
2775
|
-
throw new Error(
|
|
2920
|
+
throw new Error(formatLlmGatewayError(res.status, text, this.defaultChatModel));
|
|
2776
2921
|
}
|
|
2777
2922
|
const reader = res.body?.getReader();
|
|
2778
2923
|
if (!reader) return;
|
|
@@ -2825,6 +2970,7 @@ function llmPlugin(config = {}) {
|
|
|
2825
2970
|
return void 0;
|
|
2826
2971
|
}
|
|
2827
2972
|
const embeddingModel = config.embeddingModel ?? context.config.EMBEDDING_MODEL;
|
|
2973
|
+
const defaultChatModel = config.defaultChatModel?.trim() || (typeof context.config.LLM_CHAT_MODEL === "string" ? context.config.LLM_CHAT_MODEL.trim() : "") || void 0;
|
|
2828
2974
|
const embeddingProvider = normalizeEmbeddingProvider(
|
|
2829
2975
|
config.embeddingProvider ?? context.config.EMBEDDING_PROVIDER
|
|
2830
2976
|
);
|
|
@@ -2833,7 +2979,8 @@ function llmPlugin(config = {}) {
|
|
|
2833
2979
|
return new LlmService(baseURL.replace(/\/$/, ""), apiKey, embeddingModel, {
|
|
2834
2980
|
embeddingProvider,
|
|
2835
2981
|
hfInferenceBaseUrl,
|
|
2836
|
-
embeddingApiKey
|
|
2982
|
+
embeddingApiKey,
|
|
2983
|
+
defaultChatModel
|
|
2837
2984
|
});
|
|
2838
2985
|
}
|
|
2839
2986
|
};
|
|
@@ -5176,15 +5323,33 @@ function createChatHandlers(config) {
|
|
|
5176
5323
|
const email = body?.email?.trim();
|
|
5177
5324
|
if (!name || !email) return json({ error: "name and email required" }, { status: 400 });
|
|
5178
5325
|
const repo = contactRepo();
|
|
5179
|
-
|
|
5180
|
-
|
|
5181
|
-
|
|
5182
|
-
|
|
5326
|
+
const phone = body.phone?.trim() || null;
|
|
5327
|
+
const existing = await repo.findOne({ where: { email } });
|
|
5328
|
+
let contact;
|
|
5329
|
+
if (!existing) {
|
|
5330
|
+
contact = await repo.save(repo.create({ name, email, phone }));
|
|
5331
|
+
} else {
|
|
5332
|
+
const row = existing;
|
|
5333
|
+
if (row.deleted) {
|
|
5334
|
+
await repo.update(row.id, {
|
|
5335
|
+
deleted: false,
|
|
5336
|
+
deletedAt: null,
|
|
5337
|
+
deletedBy: null,
|
|
5338
|
+
name,
|
|
5339
|
+
phone
|
|
5340
|
+
});
|
|
5341
|
+
const refreshed = await repo.findOne({ where: { id: row.id } });
|
|
5342
|
+
if (!refreshed) return json({ error: "Failed to identify", detail: "contact missing after reactivate" }, { status: 500 });
|
|
5343
|
+
contact = refreshed;
|
|
5344
|
+
} else {
|
|
5345
|
+
contact = existing;
|
|
5346
|
+
}
|
|
5183
5347
|
}
|
|
5184
5348
|
const convRepoInst = convRepo();
|
|
5185
|
-
const
|
|
5349
|
+
const contactId = contact.id;
|
|
5350
|
+
const conv = await convRepoInst.save(convRepoInst.create({ contactId }));
|
|
5186
5351
|
return json({
|
|
5187
|
-
contactId
|
|
5352
|
+
contactId,
|
|
5188
5353
|
conversationId: conv.id
|
|
5189
5354
|
});
|
|
5190
5355
|
} catch (err) {
|
|
@@ -8363,6 +8528,13 @@ function getNextAuthOptions(config) {
|
|
|
8363
8528
|
|
|
8364
8529
|
// src/api/crud.ts
|
|
8365
8530
|
import { Brackets, ILike as ILike2, MoreThan as MoreThan2 } from "typeorm";
|
|
8531
|
+
var CRUD_LOG = "[cms-crud]";
|
|
8532
|
+
function logCrudClientError(op, detail) {
|
|
8533
|
+
console.warn(CRUD_LOG, op, detail);
|
|
8534
|
+
}
|
|
8535
|
+
function logCrudServerError(op, detail) {
|
|
8536
|
+
console.error(CRUD_LOG, op, detail);
|
|
8537
|
+
}
|
|
8366
8538
|
var DATE_COLUMN_TYPES = /* @__PURE__ */ new Set([
|
|
8367
8539
|
"date",
|
|
8368
8540
|
"datetime",
|
|
@@ -8474,6 +8646,13 @@ function createCrudHandler(dataSource, entityMap, options) {
|
|
|
8474
8646
|
if (authError) return authError;
|
|
8475
8647
|
const entity = entityMap[resource];
|
|
8476
8648
|
if (!resource || !entity) {
|
|
8649
|
+
logCrudClientError("GET list", {
|
|
8650
|
+
reason: "invalid_resource",
|
|
8651
|
+
resource,
|
|
8652
|
+
hasEntity: Boolean(entity),
|
|
8653
|
+
entityMapHasLlmAgents: Boolean(entityMap.llm_agents),
|
|
8654
|
+
entityMapKeyCount: Object.keys(entityMap).length
|
|
8655
|
+
});
|
|
8477
8656
|
return json({ error: "Invalid resource" }, { status: 400 });
|
|
8478
8657
|
}
|
|
8479
8658
|
const { searchParams } = new URL(req.url);
|
|
@@ -8681,12 +8860,22 @@ function createCrudHandler(dataSource, entityMap, options) {
|
|
|
8681
8860
|
}
|
|
8682
8861
|
}
|
|
8683
8862
|
where = mergeDeletedFalseWhere(repo, where);
|
|
8684
|
-
|
|
8685
|
-
|
|
8686
|
-
|
|
8687
|
-
|
|
8688
|
-
|
|
8689
|
-
|
|
8863
|
+
let data;
|
|
8864
|
+
let total;
|
|
8865
|
+
try {
|
|
8866
|
+
const r = await repo.findAndCount({
|
|
8867
|
+
skip,
|
|
8868
|
+
take: limit,
|
|
8869
|
+
order: { [sortField]: sortOrder },
|
|
8870
|
+
where
|
|
8871
|
+
});
|
|
8872
|
+
data = r[0];
|
|
8873
|
+
total = r[1];
|
|
8874
|
+
} catch (err) {
|
|
8875
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
8876
|
+
logCrudServerError("GET list query failed", { resource, sortField, sortOrder, message });
|
|
8877
|
+
throw err;
|
|
8878
|
+
}
|
|
8690
8879
|
return json({ total, page, limit, totalPages: Math.ceil(total / limit), data });
|
|
8691
8880
|
},
|
|
8692
8881
|
async POST(req, resource) {
|
|
@@ -8694,12 +8883,25 @@ function createCrudHandler(dataSource, entityMap, options) {
|
|
|
8694
8883
|
if (authError) return authError;
|
|
8695
8884
|
const entity = entityMap[resource];
|
|
8696
8885
|
if (!resource || !entity) {
|
|
8886
|
+
logCrudClientError("POST create", {
|
|
8887
|
+
reason: "invalid_resource",
|
|
8888
|
+
resource,
|
|
8889
|
+
hasEntity: Boolean(entity),
|
|
8890
|
+
entityMapHasLlmAgents: Boolean(entityMap.llm_agents)
|
|
8891
|
+
});
|
|
8697
8892
|
return json({ error: "Invalid resource" }, { status: 400 });
|
|
8698
8893
|
}
|
|
8699
|
-
const
|
|
8700
|
-
if (!
|
|
8894
|
+
const rawPostBody = await req.json();
|
|
8895
|
+
if (!rawPostBody || typeof rawPostBody !== "object" || Object.keys(rawPostBody).length === 0) {
|
|
8896
|
+
logCrudClientError("POST create", {
|
|
8897
|
+
reason: "invalid_request_payload",
|
|
8898
|
+
resource,
|
|
8899
|
+
rawType: rawPostBody == null ? "nullish" : typeof rawPostBody,
|
|
8900
|
+
keyCount: rawPostBody && typeof rawPostBody === "object" ? Object.keys(rawPostBody).length : 0
|
|
8901
|
+
});
|
|
8701
8902
|
return json({ error: "Invalid request payload" }, { status: 400 });
|
|
8702
8903
|
}
|
|
8904
|
+
const body = rawPostBody;
|
|
8703
8905
|
if (resource === "media") {
|
|
8704
8906
|
const b = body;
|
|
8705
8907
|
const kind = b.kind === "folder" ? "folder" : "file";
|
|
@@ -8733,8 +8935,24 @@ function createCrudHandler(dataSource, entityMap, options) {
|
|
|
8733
8935
|
}
|
|
8734
8936
|
}
|
|
8735
8937
|
const repo = dataSource.getRepository(entity);
|
|
8736
|
-
|
|
8737
|
-
|
|
8938
|
+
const persistBody = resource === "media" ? body : pickColumnUpdates(repo, body);
|
|
8939
|
+
if (resource !== "media" && Object.keys(persistBody).length === 0) {
|
|
8940
|
+
logCrudClientError("POST create", {
|
|
8941
|
+
reason: "no_scalar_columns_after_pick",
|
|
8942
|
+
resource,
|
|
8943
|
+
incomingKeys: Object.keys(body)
|
|
8944
|
+
});
|
|
8945
|
+
return json({ error: "Invalid request payload" }, { status: 400 });
|
|
8946
|
+
}
|
|
8947
|
+
sanitizeBodyForEntity(repo, persistBody);
|
|
8948
|
+
let created;
|
|
8949
|
+
try {
|
|
8950
|
+
created = await repo.save(repo.create(persistBody));
|
|
8951
|
+
} catch (err) {
|
|
8952
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
8953
|
+
logCrudServerError("POST create save failed", { resource, message, persistKeys: Object.keys(persistBody) });
|
|
8954
|
+
throw err;
|
|
8955
|
+
}
|
|
8738
8956
|
if (resource === "contacts") {
|
|
8739
8957
|
await syncContactRowToErp(created);
|
|
8740
8958
|
}
|
|
@@ -8749,6 +8967,7 @@ function createCrudHandler(dataSource, entityMap, options) {
|
|
|
8749
8967
|
if (authError) return authError;
|
|
8750
8968
|
const entity = entityMap[resource];
|
|
8751
8969
|
if (!resource || !entity) {
|
|
8970
|
+
logCrudClientError("GET_METADATA", { reason: "invalid_resource", resource });
|
|
8752
8971
|
return json({ error: "Invalid resource" }, { status: 400 });
|
|
8753
8972
|
}
|
|
8754
8973
|
const repo = dataSource.getRepository(entity);
|
|
@@ -8780,11 +8999,18 @@ function createCrudHandler(dataSource, entityMap, options) {
|
|
|
8780
8999
|
if (authError) return authError;
|
|
8781
9000
|
const entity = entityMap[resource];
|
|
8782
9001
|
if (!resource || !entity) {
|
|
9002
|
+
logCrudClientError("BULK_POST", { reason: "invalid_resource", resource });
|
|
8783
9003
|
return json({ error: "Invalid resource" }, { status: 400 });
|
|
8784
9004
|
}
|
|
8785
9005
|
const body = await req.json();
|
|
8786
9006
|
const { records, upsertKey = "id" } = body;
|
|
8787
9007
|
if (!Array.isArray(records) || records.length === 0) {
|
|
9008
|
+
logCrudClientError("BULK_POST", {
|
|
9009
|
+
reason: "records_required",
|
|
9010
|
+
resource,
|
|
9011
|
+
recordsIsArray: Array.isArray(records),
|
|
9012
|
+
recordCount: Array.isArray(records) ? records.length : 0
|
|
9013
|
+
});
|
|
8788
9014
|
return json({ error: "Records array is required" }, { status: 400 });
|
|
8789
9015
|
}
|
|
8790
9016
|
const repo = dataSource.getRepository(entity);
|
|
@@ -8803,6 +9029,7 @@ function createCrudHandler(dataSource, entityMap, options) {
|
|
|
8803
9029
|
});
|
|
8804
9030
|
} catch (error) {
|
|
8805
9031
|
const message = error instanceof Error ? error.message : "Bulk import failed";
|
|
9032
|
+
logCrudClientError("BULK_POST upsert failed", { resource, upsertKey, message });
|
|
8806
9033
|
return json({ error: message }, { status: 400 });
|
|
8807
9034
|
}
|
|
8808
9035
|
},
|
|
@@ -8811,6 +9038,7 @@ function createCrudHandler(dataSource, entityMap, options) {
|
|
|
8811
9038
|
if (authError) return authError;
|
|
8812
9039
|
const entity = entityMap[resource];
|
|
8813
9040
|
if (!resource || !entity) {
|
|
9041
|
+
logCrudClientError("GET_EXPORT", { reason: "invalid_resource", resource });
|
|
8814
9042
|
return json({ error: "Invalid resource" }, { status: 400 });
|
|
8815
9043
|
}
|
|
8816
9044
|
const { searchParams } = new URL(req.url);
|
|
@@ -8865,7 +9093,10 @@ function createCrudByIdHandler(dataSource, entityMap, options) {
|
|
|
8865
9093
|
const authError = await authz(req, resource, "read");
|
|
8866
9094
|
if (authError) return authError;
|
|
8867
9095
|
const entity = entityMap[resource];
|
|
8868
|
-
if (!entity)
|
|
9096
|
+
if (!entity) {
|
|
9097
|
+
logCrudClientError("GET by id", { reason: "invalid_resource", resource, id });
|
|
9098
|
+
return json({ error: "Invalid resource" }, { status: 400 });
|
|
9099
|
+
}
|
|
8869
9100
|
const repo = dataSource.getRepository(entity);
|
|
8870
9101
|
if (resource === "orders") {
|
|
8871
9102
|
const order = await repo.findOne({
|
|
@@ -8924,7 +9155,10 @@ function createCrudByIdHandler(dataSource, entityMap, options) {
|
|
|
8924
9155
|
const authError = await authz(req, resource, "update");
|
|
8925
9156
|
if (authError) return authError;
|
|
8926
9157
|
const entity = entityMap[resource];
|
|
8927
|
-
if (!entity)
|
|
9158
|
+
if (!entity) {
|
|
9159
|
+
logCrudClientError("PUT by id", { reason: "invalid_resource", resource, id });
|
|
9160
|
+
return json({ error: "Invalid resource" }, { status: 400 });
|
|
9161
|
+
}
|
|
8928
9162
|
const rawBody = await req.json();
|
|
8929
9163
|
const repo = dataSource.getRepository(entity);
|
|
8930
9164
|
const numericId = Number(id);
|
|
@@ -9037,7 +9271,10 @@ function createCrudByIdHandler(dataSource, entityMap, options) {
|
|
|
9037
9271
|
const authError = await authz(req, resource, "delete");
|
|
9038
9272
|
if (authError) return authError;
|
|
9039
9273
|
const entity = entityMap[resource];
|
|
9040
|
-
if (!entity)
|
|
9274
|
+
if (!entity) {
|
|
9275
|
+
logCrudClientError("DELETE by id", { reason: "invalid_resource", resource, id });
|
|
9276
|
+
return json({ error: "Invalid resource" }, { status: 400 });
|
|
9277
|
+
}
|
|
9041
9278
|
const repo = dataSource.getRepository(entity);
|
|
9042
9279
|
const numericId = Number(id);
|
|
9043
9280
|
if (entityHasSoftDelete(repo)) {
|
|
@@ -9207,6 +9444,9 @@ var EMBED_CONCURRENCY = 5;
|
|
|
9207
9444
|
var MAX_PDF_BYTES = 25 * 1024 * 1024;
|
|
9208
9445
|
var TEXT_FILE_TYPES = /* @__PURE__ */ new Set(["text/plain", "text/markdown", "application/json"]);
|
|
9209
9446
|
var KB_LOG = "[llm-agent-knowledge]";
|
|
9447
|
+
function logKbPipeline(step, meta) {
|
|
9448
|
+
console.info(`${KB_LOG} pipeline`, { step, ...meta });
|
|
9449
|
+
}
|
|
9210
9450
|
function llmEmbedDebug() {
|
|
9211
9451
|
const v = process.env.LLM_EMBED_DEBUG?.toLowerCase();
|
|
9212
9452
|
return v === "1" || v === "true" || v === "yes";
|
|
@@ -9221,11 +9461,29 @@ async function extractTextFromPdf(buffer) {
|
|
|
9221
9461
|
const data = await pdfParse(buffer);
|
|
9222
9462
|
return (data?.text ?? "").trim();
|
|
9223
9463
|
}
|
|
9224
|
-
async function
|
|
9464
|
+
async function loadChunksMissingEmbeddings(dataSource, documentId, slug) {
|
|
9465
|
+
const rows = await dataSource.query(
|
|
9466
|
+
`SELECT id, content FROM knowledge_base_chunks WHERE "documentId" = $1 AND embedding IS NULL ORDER BY "chunkIndex" ASC`,
|
|
9467
|
+
[documentId]
|
|
9468
|
+
);
|
|
9469
|
+
const list = rows ?? [];
|
|
9470
|
+
logKbPipeline("07_query_chunks_missing_embedding", {
|
|
9471
|
+
slug,
|
|
9472
|
+
documentId,
|
|
9473
|
+
missingEmbeddingCount: list.length
|
|
9474
|
+
});
|
|
9475
|
+
return list;
|
|
9476
|
+
}
|
|
9477
|
+
async function writeEmbeddingsConcurrent(dataSource, embed, chunks, concurrency, slug) {
|
|
9225
9478
|
let next = 0;
|
|
9226
9479
|
let written = 0;
|
|
9227
9480
|
let failed = 0;
|
|
9228
9481
|
let skippedEmpty = 0;
|
|
9482
|
+
logKbPipeline("09_embedding_workers_start", {
|
|
9483
|
+
slug,
|
|
9484
|
+
chunkCount: chunks.length,
|
|
9485
|
+
concurrency: Math.max(1, Math.min(concurrency, chunks.length))
|
|
9486
|
+
});
|
|
9229
9487
|
async function worker() {
|
|
9230
9488
|
for (; ; ) {
|
|
9231
9489
|
const i = next++;
|
|
@@ -9264,9 +9522,12 @@ async function writeEmbeddingsConcurrent(dataSource, embed, chunks, concurrency)
|
|
|
9264
9522
|
skippedEmpty
|
|
9265
9523
|
};
|
|
9266
9524
|
if (skippedEmpty > 0 && written === 0) {
|
|
9267
|
-
summary.hint = "embed() returned empty vectors \u2014 see [LLM embed] logs (
|
|
9525
|
+
summary.hint = "embed() returned empty vectors \u2014 see [LLM embed] / [LLM embed HF] logs (OpenAI gateway vs @huggingface/inference + legacy HTTP).";
|
|
9526
|
+
}
|
|
9527
|
+
logKbPipeline("10_embedding_workers_finished", { slug, ...summary });
|
|
9528
|
+
if (failed > 0 || skippedEmpty > 0 && written === 0) {
|
|
9529
|
+
console.error(`${KB_LOG} embedding pass finished with issues`, summary);
|
|
9268
9530
|
}
|
|
9269
|
-
console.error(`${KB_LOG} embedding pass finished`, summary);
|
|
9270
9531
|
return { written, failed };
|
|
9271
9532
|
}
|
|
9272
9533
|
function splitIntoChunks(text, maxLen) {
|
|
@@ -9302,6 +9563,58 @@ function createLlmAgentKnowledgeHandlers(config) {
|
|
|
9302
9563
|
}
|
|
9303
9564
|
return null;
|
|
9304
9565
|
}
|
|
9566
|
+
async function runEmbeddingPass(slug, savedChunks) {
|
|
9567
|
+
let embeddingsWritten = 0;
|
|
9568
|
+
let embeddingsFailed = 0;
|
|
9569
|
+
try {
|
|
9570
|
+
logKbPipeline("08_resolve_llm_plugin", { slug, chunkCount: savedChunks.length });
|
|
9571
|
+
const cms = await getCms();
|
|
9572
|
+
const llm = cms.getPlugin("llm");
|
|
9573
|
+
if (llm?.embed && savedChunks.length > 0) {
|
|
9574
|
+
logKbPipeline("08b_embed_fn_available", {
|
|
9575
|
+
slug,
|
|
9576
|
+
chunkCount: savedChunks.length,
|
|
9577
|
+
firstChunkId: savedChunks[0]?.id,
|
|
9578
|
+
lastChunkId: savedChunks[savedChunks.length - 1]?.id
|
|
9579
|
+
});
|
|
9580
|
+
const embedBound = (text) => llm.embed(text);
|
|
9581
|
+
const { written, failed } = await writeEmbeddingsConcurrent(
|
|
9582
|
+
dataSource,
|
|
9583
|
+
embedBound,
|
|
9584
|
+
savedChunks,
|
|
9585
|
+
EMBED_CONCURRENCY,
|
|
9586
|
+
slug
|
|
9587
|
+
);
|
|
9588
|
+
embeddingsWritten = written;
|
|
9589
|
+
embeddingsFailed = failed;
|
|
9590
|
+
} else {
|
|
9591
|
+
logKbPipeline("08c_embed_skipped", {
|
|
9592
|
+
slug,
|
|
9593
|
+
hasLlmPlugin: !!llm,
|
|
9594
|
+
hasEmbed: typeof llm?.embed === "function",
|
|
9595
|
+
chunkCount: savedChunks.length,
|
|
9596
|
+
reason: savedChunks.length === 0 ? "no_chunks" : !llm ? "no_llm_plugin" : "no_embed_method"
|
|
9597
|
+
});
|
|
9598
|
+
console.error(`${KB_LOG} embeddings skipped`, {
|
|
9599
|
+
slug,
|
|
9600
|
+
hasLlmPlugin: !!llm,
|
|
9601
|
+
hasEmbed: typeof llm?.embed === "function",
|
|
9602
|
+
chunkCount: savedChunks.length,
|
|
9603
|
+
hint: !llm || typeof llm.embed !== "function" ? "LLM plugin missing or no embed(); check LLM_GATEWAY_URL + LLM_API_KEY so llm plugin initializes." : void 0
|
|
9604
|
+
});
|
|
9605
|
+
}
|
|
9606
|
+
} catch (embErr) {
|
|
9607
|
+
const detail = embErr instanceof Error ? embErr.message : String(embErr);
|
|
9608
|
+
logKbPipeline("08d_embed_pass_exception", { slug, detail });
|
|
9609
|
+
console.error(`${KB_LOG} embedding step threw before/during batch`, { slug, detail, embErr });
|
|
9610
|
+
return {
|
|
9611
|
+
embeddingsWritten: 0,
|
|
9612
|
+
embeddingsFailed: savedChunks.length,
|
|
9613
|
+
embedError: detail
|
|
9614
|
+
};
|
|
9615
|
+
}
|
|
9616
|
+
return { embeddingsWritten, embeddingsFailed };
|
|
9617
|
+
}
|
|
9305
9618
|
return {
|
|
9306
9619
|
async list(req, slug) {
|
|
9307
9620
|
const denied = await gate(req, "read");
|
|
@@ -9330,8 +9643,11 @@ function createLlmAgentKnowledgeHandlers(config) {
|
|
|
9330
9643
|
const denied = await gate(req, "update");
|
|
9331
9644
|
if (denied) return denied;
|
|
9332
9645
|
try {
|
|
9646
|
+
const ct0 = req.headers.get("content-type") || "";
|
|
9647
|
+
logKbPipeline("01_request", { slug, contentType: ct0.slice(0, 80) });
|
|
9333
9648
|
const agent = await findAgentBySlug(dataSource, llmAgents, slug);
|
|
9334
9649
|
if (!agent) return json({ error: "Agent not found" }, { status: 404 });
|
|
9650
|
+
logKbPipeline("02_agent_loaded", { slug, agentId: agent.id });
|
|
9335
9651
|
let name = "";
|
|
9336
9652
|
let text = "";
|
|
9337
9653
|
let sourceUrl = null;
|
|
@@ -9343,6 +9659,14 @@ function createLlmAgentKnowledgeHandlers(config) {
|
|
|
9343
9659
|
name = (body?.name ?? "").trim();
|
|
9344
9660
|
text = (body?.text ?? "").trim();
|
|
9345
9661
|
sourceUrl = typeof body?.sourceUrl === "string" && body.sourceUrl.trim() ? body.sourceUrl.trim() : null;
|
|
9662
|
+
logKbPipeline("03_body_parsed", {
|
|
9663
|
+
slug,
|
|
9664
|
+
mode: "json",
|
|
9665
|
+
existingDocumentId,
|
|
9666
|
+
nameLen: name.length,
|
|
9667
|
+
textChars: text.length,
|
|
9668
|
+
hasSourceUrl: !!sourceUrl
|
|
9669
|
+
});
|
|
9346
9670
|
} else if (ct.includes("multipart/form-data")) {
|
|
9347
9671
|
const form = await req.formData();
|
|
9348
9672
|
name = form.get("name")?.trim() ?? "";
|
|
@@ -9352,9 +9676,17 @@ function createLlmAgentKnowledgeHandlers(config) {
|
|
|
9352
9676
|
const f = file;
|
|
9353
9677
|
const mime = (f.type || "").split(";")[0].trim().toLowerCase();
|
|
9354
9678
|
const buf = Buffer.from(await f.arrayBuffer());
|
|
9679
|
+
logKbPipeline("03_body_parsed", {
|
|
9680
|
+
slug,
|
|
9681
|
+
mode: "multipart",
|
|
9682
|
+
fileName: f.name || "(no name)",
|
|
9683
|
+
mime,
|
|
9684
|
+
fileBytes: buf.length
|
|
9685
|
+
});
|
|
9355
9686
|
if (TEXT_FILE_TYPES.has(mime)) {
|
|
9356
9687
|
const decoded = buf.toString("utf8");
|
|
9357
9688
|
if (!text) text = decoded;
|
|
9689
|
+
logKbPipeline("04_file_decoded_text", { slug, mime, textChars: text.length });
|
|
9358
9690
|
} else if (isPdfUpload(mime, f.name || "")) {
|
|
9359
9691
|
if (buf.length > MAX_PDF_BYTES) {
|
|
9360
9692
|
return json(
|
|
@@ -9363,8 +9695,10 @@ function createLlmAgentKnowledgeHandlers(config) {
|
|
|
9363
9695
|
);
|
|
9364
9696
|
}
|
|
9365
9697
|
try {
|
|
9698
|
+
logKbPipeline("04_pdf_extract_start", { slug, fileBytes: buf.length });
|
|
9366
9699
|
const extracted = await extractTextFromPdf(buf);
|
|
9367
9700
|
if (!text) text = extracted;
|
|
9701
|
+
logKbPipeline("04_pdf_extract_done", { slug, textChars: text.length });
|
|
9368
9702
|
} catch {
|
|
9369
9703
|
return json(
|
|
9370
9704
|
{ error: "Could not read PDF text (file may be encrypted, corrupt, or image-only)" },
|
|
@@ -9380,12 +9714,15 @@ function createLlmAgentKnowledgeHandlers(config) {
|
|
|
9380
9714
|
);
|
|
9381
9715
|
}
|
|
9382
9716
|
if (!name && f.name) name = f.name.replace(/\.[^/.]+$/, "") || f.name;
|
|
9717
|
+
} else {
|
|
9718
|
+
logKbPipeline("03_body_parsed", { slug, mode: "multipart", hasFile: false, textChars: text.length });
|
|
9383
9719
|
}
|
|
9384
9720
|
} else {
|
|
9385
9721
|
return json({ error: "Use application/json or multipart/form-data" }, { status: 400 });
|
|
9386
9722
|
}
|
|
9387
9723
|
const linkRepo = dataSource.getRepository(junction);
|
|
9388
9724
|
if (existingDocumentId != null) {
|
|
9725
|
+
logKbPipeline("05_branch", { slug, branch: "link_existing_document", documentId: existingDocumentId });
|
|
9389
9726
|
const docRepo2 = dataSource.getRepository(kbDoc);
|
|
9390
9727
|
const existing = await docRepo2.findOne({ where: { id: existingDocumentId } });
|
|
9391
9728
|
if (!existing) return json({ error: "documentId not found" }, { status: 404 });
|
|
@@ -9394,9 +9731,137 @@ function createLlmAgentKnowledgeHandlers(config) {
|
|
|
9394
9731
|
});
|
|
9395
9732
|
if (!dup2) {
|
|
9396
9733
|
await linkRepo.save(linkRepo.create({ agentId: agent.id, documentId: existingDocumentId }));
|
|
9734
|
+
logKbPipeline("06_junction_link_created", {
|
|
9735
|
+
slug,
|
|
9736
|
+
agentId: agent.id,
|
|
9737
|
+
documentId: existingDocumentId
|
|
9738
|
+
});
|
|
9739
|
+
} else {
|
|
9740
|
+
logKbPipeline("06_junction_link_exists", {
|
|
9741
|
+
slug,
|
|
9742
|
+
agentId: agent.id,
|
|
9743
|
+
documentId: existingDocumentId
|
|
9744
|
+
});
|
|
9745
|
+
}
|
|
9746
|
+
const chunkRepo2 = dataSource.getRepository(kbChunk);
|
|
9747
|
+
const docRow = existing;
|
|
9748
|
+
const chunkCount = await chunkRepo2.count({ where: { documentId: existingDocumentId } });
|
|
9749
|
+
logKbPipeline("06b_chunks_existing_count", {
|
|
9750
|
+
slug,
|
|
9751
|
+
documentId: existingDocumentId,
|
|
9752
|
+
chunkCount
|
|
9753
|
+
});
|
|
9754
|
+
let chunksToEmbed = [];
|
|
9755
|
+
let chunksCreated = 0;
|
|
9756
|
+
if (chunkCount === 0) {
|
|
9757
|
+
const text2 = (docRow.content ?? "").trim();
|
|
9758
|
+
if (!text2) {
|
|
9759
|
+
logKbPipeline("07_ingest_aborted_empty_document", { slug, documentId: existingDocumentId });
|
|
9760
|
+
return json({
|
|
9761
|
+
documentId: existingDocumentId,
|
|
9762
|
+
linked: true,
|
|
9763
|
+
created: false,
|
|
9764
|
+
chunkCount: 0,
|
|
9765
|
+
embeddingsWritten: 0,
|
|
9766
|
+
embeddingsFailed: 0,
|
|
9767
|
+
warning: "Document has no text content; add content then attach again to build chunks and embeddings."
|
|
9768
|
+
});
|
|
9769
|
+
}
|
|
9770
|
+
const parts2 = splitIntoChunks(text2, INGEST_CHUNK_CHARS);
|
|
9771
|
+
if (parts2.length > MAX_CHUNKS_PER_UPLOAD) {
|
|
9772
|
+
return json(
|
|
9773
|
+
{
|
|
9774
|
+
error: `Document is too large for one ingest (${parts2.length} chunks; max ${MAX_CHUNKS_PER_UPLOAD}). Split into smaller files.`
|
|
9775
|
+
},
|
|
9776
|
+
{ status: 413 }
|
|
9777
|
+
);
|
|
9778
|
+
}
|
|
9779
|
+
logKbPipeline("07_chunk_split", {
|
|
9780
|
+
slug,
|
|
9781
|
+
documentId: existingDocumentId,
|
|
9782
|
+
partCount: parts2.length,
|
|
9783
|
+
maxChunkChars: INGEST_CHUNK_CHARS,
|
|
9784
|
+
totalChars: text2.length
|
|
9785
|
+
});
|
|
9786
|
+
const now2 = /* @__PURE__ */ new Date();
|
|
9787
|
+
const chunkRows2 = parts2.map(
|
|
9788
|
+
(content, i) => chunkRepo2.create({
|
|
9789
|
+
documentId: existingDocumentId,
|
|
9790
|
+
content,
|
|
9791
|
+
chunkIndex: i,
|
|
9792
|
+
createdAt: now2
|
|
9793
|
+
})
|
|
9794
|
+
);
|
|
9795
|
+
const savedList2 = await chunkRepo2.save(chunkRows2);
|
|
9796
|
+
chunksCreated = savedList2.length;
|
|
9797
|
+
chunksToEmbed = savedList2.map((row, i) => ({
|
|
9798
|
+
id: row.id,
|
|
9799
|
+
content: parts2[i]
|
|
9800
|
+
}));
|
|
9801
|
+
logKbPipeline("07b_chunks_persisted", {
|
|
9802
|
+
slug,
|
|
9803
|
+
documentId: existingDocumentId,
|
|
9804
|
+
rowsInserted: chunksCreated
|
|
9805
|
+
});
|
|
9806
|
+
} else {
|
|
9807
|
+
chunksToEmbed = await loadChunksMissingEmbeddings(dataSource, existingDocumentId, slug);
|
|
9808
|
+
if (chunksToEmbed.length === 0) {
|
|
9809
|
+
logKbPipeline("08_skip_embed_all_chunks_have_vectors", {
|
|
9810
|
+
slug,
|
|
9811
|
+
documentId: existingDocumentId,
|
|
9812
|
+
existingChunkCount: chunkCount
|
|
9813
|
+
});
|
|
9814
|
+
}
|
|
9397
9815
|
}
|
|
9398
|
-
|
|
9816
|
+
const embedResult2 = chunksToEmbed.length > 0 ? await runEmbeddingPass(slug, chunksToEmbed) : { embeddingsWritten: 0, embeddingsFailed: 0 };
|
|
9817
|
+
if (embedResult2.embedError) {
|
|
9818
|
+
logKbPipeline("11_response", {
|
|
9819
|
+
slug,
|
|
9820
|
+
branch: "link",
|
|
9821
|
+
documentId: existingDocumentId,
|
|
9822
|
+
ok: false,
|
|
9823
|
+
embeddingError: true
|
|
9824
|
+
});
|
|
9825
|
+
return json(
|
|
9826
|
+
{
|
|
9827
|
+
documentId: existingDocumentId,
|
|
9828
|
+
linked: true,
|
|
9829
|
+
created: false,
|
|
9830
|
+
chunkCount: chunkCount + chunksCreated,
|
|
9831
|
+
chunksCreated,
|
|
9832
|
+
embeddingAttempted: true,
|
|
9833
|
+
embeddingsWritten: 0,
|
|
9834
|
+
embeddingsFailed: chunksToEmbed.length,
|
|
9835
|
+
warning: "Document linked; embedding step failed. Fix LLM/embed config and attach again (or unlink and re-attach) to retry NULL embeddings.",
|
|
9836
|
+
detail: embedResult2.embedError
|
|
9837
|
+
},
|
|
9838
|
+
{ status: 201 }
|
|
9839
|
+
);
|
|
9840
|
+
}
|
|
9841
|
+
logKbPipeline("11_response", {
|
|
9842
|
+
slug,
|
|
9843
|
+
branch: "link",
|
|
9844
|
+
documentId: existingDocumentId,
|
|
9845
|
+
ok: true,
|
|
9846
|
+
chunkCount: chunkCount + chunksCreated,
|
|
9847
|
+
chunksCreated,
|
|
9848
|
+
chunksQueuedForEmbedding: chunksToEmbed.length,
|
|
9849
|
+
embeddingsWritten: embedResult2.embeddingsWritten,
|
|
9850
|
+
embeddingsFailed: embedResult2.embeddingsFailed
|
|
9851
|
+
});
|
|
9852
|
+
return json({
|
|
9853
|
+
documentId: existingDocumentId,
|
|
9854
|
+
linked: true,
|
|
9855
|
+
created: false,
|
|
9856
|
+
chunkCount: chunkCount + chunksCreated,
|
|
9857
|
+
chunksCreated: chunksCreated || void 0,
|
|
9858
|
+
chunksQueuedForEmbedding: chunksToEmbed.length,
|
|
9859
|
+
embeddingAttempted: chunksToEmbed.length > 0,
|
|
9860
|
+
embeddingsWritten: embedResult2.embeddingsWritten,
|
|
9861
|
+
embeddingsFailed: embedResult2.embeddingsFailed
|
|
9862
|
+
});
|
|
9399
9863
|
}
|
|
9864
|
+
logKbPipeline("05_branch", { slug, branch: "new_upload_ingest" });
|
|
9400
9865
|
if (!text) return json({ error: "text or file with text content is required" }, { status: 400 });
|
|
9401
9866
|
if (!name) name = "Untitled";
|
|
9402
9867
|
const parts = splitIntoChunks(text, INGEST_CHUNK_CHARS);
|
|
@@ -9411,6 +9876,13 @@ function createLlmAgentKnowledgeHandlers(config) {
|
|
|
9411
9876
|
{ status: 413 }
|
|
9412
9877
|
);
|
|
9413
9878
|
}
|
|
9879
|
+
logKbPipeline("07_chunk_split", {
|
|
9880
|
+
slug,
|
|
9881
|
+
partCount: parts.length,
|
|
9882
|
+
maxChunkChars: INGEST_CHUNK_CHARS,
|
|
9883
|
+
totalChars: text.length,
|
|
9884
|
+
documentName: name
|
|
9885
|
+
});
|
|
9414
9886
|
const docRepo = dataSource.getRepository(kbDoc);
|
|
9415
9887
|
const chunkRepo = dataSource.getRepository(kbChunk);
|
|
9416
9888
|
const now = /* @__PURE__ */ new Date();
|
|
@@ -9418,6 +9890,13 @@ function createLlmAgentKnowledgeHandlers(config) {
|
|
|
9418
9890
|
docRepo.create({ name, content: text, sourceUrl, createdAt: now, updatedAt: now })
|
|
9419
9891
|
);
|
|
9420
9892
|
const docId = doc.id;
|
|
9893
|
+
logKbPipeline("06_knowledge_document_saved", {
|
|
9894
|
+
slug,
|
|
9895
|
+
documentId: docId,
|
|
9896
|
+
name,
|
|
9897
|
+
contentChars: text.length,
|
|
9898
|
+
hasSourceUrl: !!sourceUrl
|
|
9899
|
+
});
|
|
9421
9900
|
const chunkRows = parts.map(
|
|
9422
9901
|
(content, i) => chunkRepo.create({ documentId: docId, content, chunkIndex: i, createdAt: now })
|
|
9423
9902
|
);
|
|
@@ -9428,63 +9907,66 @@ function createLlmAgentKnowledgeHandlers(config) {
|
|
|
9428
9907
|
content: parts[i]
|
|
9429
9908
|
})
|
|
9430
9909
|
);
|
|
9910
|
+
logKbPipeline("07b_chunks_persisted", {
|
|
9911
|
+
slug,
|
|
9912
|
+
documentId: docId,
|
|
9913
|
+
rowsInserted: savedChunks.length,
|
|
9914
|
+
firstChunkId: savedChunks[0]?.id,
|
|
9915
|
+
lastChunkId: savedChunks[savedChunks.length - 1]?.id
|
|
9916
|
+
});
|
|
9431
9917
|
const dup = await linkRepo.findOne({ where: { agentId: agent.id, documentId: docId } });
|
|
9432
9918
|
if (!dup) {
|
|
9433
9919
|
await linkRepo.save(linkRepo.create({ agentId: agent.id, documentId: docId }));
|
|
9920
|
+
logKbPipeline("06c_junction_link_created", { slug, agentId: agent.id, documentId: docId });
|
|
9921
|
+
} else {
|
|
9922
|
+
logKbPipeline("06c_junction_link_exists", { slug, agentId: agent.id, documentId: docId });
|
|
9434
9923
|
}
|
|
9435
|
-
|
|
9436
|
-
|
|
9437
|
-
|
|
9438
|
-
|
|
9439
|
-
|
|
9440
|
-
|
|
9441
|
-
|
|
9442
|
-
|
|
9443
|
-
|
|
9444
|
-
dataSource,
|
|
9445
|
-
embedBound,
|
|
9446
|
-
savedChunks,
|
|
9447
|
-
EMBED_CONCURRENCY
|
|
9448
|
-
);
|
|
9449
|
-
embeddingsWritten = written;
|
|
9450
|
-
embeddingsFailed = failed;
|
|
9451
|
-
} else {
|
|
9452
|
-
console.error(`${KB_LOG} embeddings skipped`, {
|
|
9453
|
-
slug,
|
|
9454
|
-
hasLlmPlugin: !!llm,
|
|
9455
|
-
hasEmbed: typeof llm?.embed === "function",
|
|
9456
|
-
chunkCount: savedChunks.length,
|
|
9457
|
-
hint: !llm || typeof llm.embed !== "function" ? "LLM plugin missing or no embed(); check LLM_GATEWAY_URL + LLM_API_KEY so llm plugin initializes." : void 0
|
|
9458
|
-
});
|
|
9459
|
-
}
|
|
9460
|
-
} catch (embErr) {
|
|
9461
|
-
const detail = embErr instanceof Error ? embErr.message : String(embErr);
|
|
9462
|
-
console.error(`${KB_LOG} embedding step threw before/during batch`, { slug, detail, embErr });
|
|
9924
|
+
const embedResult = await runEmbeddingPass(slug, savedChunks);
|
|
9925
|
+
if (embedResult.embedError) {
|
|
9926
|
+
logKbPipeline("11_response", {
|
|
9927
|
+
slug,
|
|
9928
|
+
branch: "ingest",
|
|
9929
|
+
documentId: docId,
|
|
9930
|
+
ok: false,
|
|
9931
|
+
embeddingError: true
|
|
9932
|
+
});
|
|
9463
9933
|
return json(
|
|
9464
9934
|
{
|
|
9465
9935
|
documentId: docId,
|
|
9466
9936
|
chunkCount: savedChunks.length,
|
|
9937
|
+
embeddingAttempted: true,
|
|
9467
9938
|
created: true,
|
|
9468
9939
|
linked: true,
|
|
9469
9940
|
embeddingsWritten: 0,
|
|
9470
9941
|
embeddingsFailed: savedChunks.length,
|
|
9471
9942
|
warning: "Document saved and linked; embedding step failed.",
|
|
9472
|
-
detail
|
|
9943
|
+
detail: embedResult.embedError
|
|
9473
9944
|
},
|
|
9474
9945
|
{ status: 201 }
|
|
9475
9946
|
);
|
|
9476
9947
|
}
|
|
9948
|
+
logKbPipeline("11_response", {
|
|
9949
|
+
slug,
|
|
9950
|
+
branch: "ingest",
|
|
9951
|
+
documentId: docId,
|
|
9952
|
+
ok: true,
|
|
9953
|
+
chunkCount: savedChunks.length,
|
|
9954
|
+
embeddingsWritten: embedResult.embeddingsWritten,
|
|
9955
|
+
embeddingsFailed: embedResult.embeddingsFailed
|
|
9956
|
+
});
|
|
9477
9957
|
return json({
|
|
9478
9958
|
documentId: docId,
|
|
9479
9959
|
chunkCount: savedChunks.length,
|
|
9960
|
+
embeddingAttempted: savedChunks.length > 0,
|
|
9480
9961
|
created: true,
|
|
9481
9962
|
linked: true,
|
|
9482
|
-
embeddingsWritten,
|
|
9483
|
-
embeddingsFailed
|
|
9963
|
+
embeddingsWritten: embedResult.embeddingsWritten,
|
|
9964
|
+
embeddingsFailed: embedResult.embeddingsFailed
|
|
9484
9965
|
});
|
|
9485
9966
|
} catch (err) {
|
|
9486
9967
|
const msg = err instanceof Error ? err.message : "Failed to ingest knowledge";
|
|
9487
9968
|
const name = err instanceof Error ? err.name : "";
|
|
9969
|
+
logKbPipeline("99_pipeline_error", { slug, errorName: name, message: msg });
|
|
9488
9970
|
return json({ error: msg, errorName: name || void 0 }, { status: 500 });
|
|
9489
9971
|
}
|
|
9490
9972
|
},
|
|
@@ -9496,6 +9978,7 @@ function createLlmAgentKnowledgeHandlers(config) {
|
|
|
9496
9978
|
try {
|
|
9497
9979
|
const agent = await findAgentBySlug(dataSource, llmAgents, slug);
|
|
9498
9980
|
if (!agent) return json({ error: "Agent not found" }, { status: 404 });
|
|
9981
|
+
logKbPipeline("unlink_junction", { slug, agentId: agent.id, documentId });
|
|
9499
9982
|
const linkRepo = dataSource.getRepository(junction);
|
|
9500
9983
|
await linkRepo.delete({ agentId: agent.id, documentId });
|
|
9501
9984
|
return json({ ok: true });
|
|
@@ -9757,6 +10240,28 @@ function createAdminRolesHandlers(config) {
|
|
|
9757
10240
|
|
|
9758
10241
|
// src/api/cms-api-handler.ts
|
|
9759
10242
|
var KNOWLEDGE_SUFFIX = "knowledge";
|
|
10243
|
+
var CMS_API_LOG = "[cms-api]";
|
|
10244
|
+
function withLlmKnowledgeEntityFallbacks(base) {
|
|
10245
|
+
const keys = [
|
|
10246
|
+
"llm_agents",
|
|
10247
|
+
"llm_agent_knowledge_documents",
|
|
10248
|
+
"knowledge_base_documents",
|
|
10249
|
+
"knowledge_base_chunks"
|
|
10250
|
+
];
|
|
10251
|
+
const patch = {};
|
|
10252
|
+
for (const key of keys) {
|
|
10253
|
+
if (!(key in base) || base[key] == null) {
|
|
10254
|
+
const fallback = CMS_ENTITY_MAP[key];
|
|
10255
|
+
if (fallback) patch[key] = fallback;
|
|
10256
|
+
}
|
|
10257
|
+
}
|
|
10258
|
+
const merged = Object.keys(patch).length > 0 ? { ...base, ...patch } : base;
|
|
10259
|
+
if (!merged.llm_agents) {
|
|
10260
|
+
const agent = CMS_ENTITY_MAP.llm_agents ?? LlmAgent;
|
|
10261
|
+
return { ...merged, llm_agents: agent };
|
|
10262
|
+
}
|
|
10263
|
+
return merged;
|
|
10264
|
+
}
|
|
9760
10265
|
function matchLlmAgentKnowledgeRoute(path) {
|
|
9761
10266
|
const p = path[0] === "api" ? path.slice(1) : path;
|
|
9762
10267
|
if (p[0] !== "llm_agents" || p.length < 2) return null;
|
|
@@ -9794,9 +10299,9 @@ var DEFAULT_EXCLUDE = /* @__PURE__ */ new Set([
|
|
|
9794
10299
|
function createCmsApiHandler(config) {
|
|
9795
10300
|
const {
|
|
9796
10301
|
dataSource,
|
|
9797
|
-
entityMap,
|
|
10302
|
+
entityMap: rawEntityMap,
|
|
9798
10303
|
pathToModel = (s) => s,
|
|
9799
|
-
crudResources
|
|
10304
|
+
crudResources: crudResourcesOverride,
|
|
9800
10305
|
getCms,
|
|
9801
10306
|
userAuth: userAuthConfig,
|
|
9802
10307
|
dashboard,
|
|
@@ -9817,6 +10322,9 @@ function createCmsApiHandler(config) {
|
|
|
9817
10322
|
requireEntityPermission: userRequireEntityPermission,
|
|
9818
10323
|
getSessionUser
|
|
9819
10324
|
} = config;
|
|
10325
|
+
const entityMap = withLlmKnowledgeEntityFallbacks(rawEntityMap);
|
|
10326
|
+
const baseCrudResources = crudResourcesOverride ?? Object.keys(entityMap).filter((k) => !DEFAULT_EXCLUDE.has(k));
|
|
10327
|
+
const crudResources = entityMap.llm_agents && !baseCrudResources.includes("llm_agents") ? [...baseCrudResources, "llm_agents"] : baseCrudResources;
|
|
9820
10328
|
const requireEntityPermissionEffective = userRequireEntityPermission ?? (async (_req, entity, action) => config.json({ error: "Forbidden", reason: "entity_rbac_required", entity, action }, { status: 403 }));
|
|
9821
10329
|
const analytics = analyticsConfig ?? (getCms ? {
|
|
9822
10330
|
json: config.json,
|
|
@@ -9923,7 +10431,7 @@ function createCmsApiHandler(config) {
|
|
|
9923
10431
|
const chatHandlers = chatConfig ? createChatHandlers(chatConfig) : null;
|
|
9924
10432
|
const llmAgentKnowledgeMerged = llmAgentKnowledgeConfig ?? (chatConfig ? {
|
|
9925
10433
|
dataSource: chatConfig.dataSource,
|
|
9926
|
-
entityMap: chatConfig.entityMap,
|
|
10434
|
+
entityMap: withLlmKnowledgeEntityFallbacks(chatConfig.entityMap ?? rawEntityMap),
|
|
9927
10435
|
getCms: chatConfig.getCms,
|
|
9928
10436
|
json: chatConfig.json,
|
|
9929
10437
|
requireAuth: chatConfig.requireAuth
|
|
@@ -9937,7 +10445,9 @@ function createCmsApiHandler(config) {
|
|
|
9937
10445
|
return crudResources.includes(model) ? model : segment;
|
|
9938
10446
|
}
|
|
9939
10447
|
return {
|
|
9940
|
-
async handle(method,
|
|
10448
|
+
async handle(method, pathInput, req) {
|
|
10449
|
+
const m = typeof method === "string" ? method.toUpperCase() : "GET";
|
|
10450
|
+
const path = pathInput.length > 0 && pathInput[0] === "api" ? pathInput.slice(1) : pathInput;
|
|
9941
10451
|
async function analyticsGate() {
|
|
9942
10452
|
const a = await config.requireAuth(req);
|
|
9943
10453
|
if (a) return a;
|
|
@@ -9945,86 +10455,86 @@ function createCmsApiHandler(config) {
|
|
|
9945
10455
|
}
|
|
9946
10456
|
if (path[0] === "admin" && path[1] === "roles") {
|
|
9947
10457
|
if (!adminRoles) return config.json({ error: "Not found" }, { status: 404 });
|
|
9948
|
-
if (path.length === 2 &&
|
|
9949
|
-
if (path.length === 2 &&
|
|
9950
|
-
if (path.length === 3 &&
|
|
9951
|
-
if (path.length === 3 &&
|
|
9952
|
-
if (path.length === 4 && path[3] === "permissions" &&
|
|
10458
|
+
if (path.length === 2 && m === "GET") return adminRoles.list();
|
|
10459
|
+
if (path.length === 2 && m === "POST") return adminRoles.createGroup(req);
|
|
10460
|
+
if (path.length === 3 && m === "PATCH") return adminRoles.patchGroup(req, path[2]);
|
|
10461
|
+
if (path.length === 3 && m === "DELETE") return adminRoles.deleteGroup(path[2]);
|
|
10462
|
+
if (path.length === 4 && path[3] === "permissions" && m === "PUT") return adminRoles.putPermissions(req, path[2]);
|
|
9953
10463
|
return config.json({ error: "Not found" }, { status: 404 });
|
|
9954
10464
|
}
|
|
9955
|
-
if (path[0] === "dashboard" && path[1] === "stats" && path.length === 2 &&
|
|
10465
|
+
if (path[0] === "dashboard" && path[1] === "stats" && path.length === 2 && m === "GET" && dashboardGet) {
|
|
9956
10466
|
return dashboardGet(req);
|
|
9957
10467
|
}
|
|
9958
|
-
if (path[0] === "dashboard" && path[1] === "ecommerce" && path.length === 2 &&
|
|
10468
|
+
if (path[0] === "dashboard" && path[1] === "ecommerce" && path.length === 2 && m === "GET" && ecommerceAnalyticsGet) {
|
|
9959
10469
|
const g = await analyticsGate();
|
|
9960
10470
|
if (g) return g;
|
|
9961
10471
|
return ecommerceAnalyticsGet(req);
|
|
9962
10472
|
}
|
|
9963
10473
|
if (path[0] === "analytics" && analyticsHandlers) {
|
|
9964
|
-
if (path.length === 1 &&
|
|
10474
|
+
if (path.length === 1 && m === "GET") {
|
|
9965
10475
|
const g = await analyticsGate();
|
|
9966
10476
|
if (g) return g;
|
|
9967
10477
|
return analyticsHandlers.GET(req);
|
|
9968
10478
|
}
|
|
9969
|
-
if (path.length === 2 && path[1] === "property-id" &&
|
|
10479
|
+
if (path.length === 2 && path[1] === "property-id" && m === "GET") {
|
|
9970
10480
|
const g = await analyticsGate();
|
|
9971
10481
|
if (g) return g;
|
|
9972
10482
|
return analyticsHandlers.propertyId();
|
|
9973
10483
|
}
|
|
9974
|
-
if (path.length === 2 && path[1] === "permissions" &&
|
|
10484
|
+
if (path.length === 2 && path[1] === "permissions" && m === "GET") {
|
|
9975
10485
|
const g = await analyticsGate();
|
|
9976
10486
|
if (g) return g;
|
|
9977
10487
|
return analyticsHandlers.permissions();
|
|
9978
10488
|
}
|
|
9979
10489
|
}
|
|
9980
|
-
if (path[0] === "upload" && path.length === 1 &&
|
|
9981
|
-
if (path[0] === "media" && path[1] === "extract" && path.length === 3 &&
|
|
10490
|
+
if (path[0] === "upload" && path.length === 1 && m === "POST" && uploadPost) return uploadPost(req);
|
|
10491
|
+
if (path[0] === "media" && path[1] === "extract" && path.length === 3 && m === "POST" && zipExtractPost) {
|
|
9982
10492
|
return zipExtractPost(req, path[2]);
|
|
9983
10493
|
}
|
|
9984
|
-
if (path[0] === "blogs" && path[1] === "slug" && path.length === 3 &&
|
|
10494
|
+
if (path[0] === "blogs" && path[1] === "slug" && path.length === 3 && m === "GET" && blogBySlugGet) {
|
|
9985
10495
|
return blogBySlugGet(req, path[2]);
|
|
9986
10496
|
}
|
|
9987
|
-
if (path[0] === "forms" && path[1] === "slug" && path.length === 3 &&
|
|
10497
|
+
if (path[0] === "forms" && path[1] === "slug" && path.length === 3 && m === "GET" && formBySlugGet) {
|
|
9988
10498
|
return formBySlugGet(req, path[2]);
|
|
9989
10499
|
}
|
|
9990
10500
|
if (path[0] === "form-submissions") {
|
|
9991
10501
|
if (path.length === 1) {
|
|
9992
|
-
if (
|
|
9993
|
-
if (
|
|
10502
|
+
if (m === "GET" && formSubmissionList) return formSubmissionList(req);
|
|
10503
|
+
if (m === "POST" && formSubmissionPost) return formSubmissionPost(req);
|
|
9994
10504
|
}
|
|
9995
|
-
if (path.length === 2 &&
|
|
10505
|
+
if (path.length === 2 && m === "GET" && formSubmissionGetById) return formSubmissionGetById(req, path[1]);
|
|
9996
10506
|
}
|
|
9997
10507
|
if (path[0] === "forms" && formSaveHandlers) {
|
|
9998
|
-
if (path.length === 1 &&
|
|
10508
|
+
if (path.length === 1 && m === "POST") return formSaveHandlers.POST(req);
|
|
9999
10509
|
if (path.length === 2) {
|
|
10000
|
-
if (
|
|
10001
|
-
if (
|
|
10510
|
+
if (m === "GET") return formSaveHandlers.GET(req, path[1]);
|
|
10511
|
+
if (m === "PUT" || m === "PATCH") return formSaveHandlers.PUT(req, path[1]);
|
|
10002
10512
|
}
|
|
10003
10513
|
}
|
|
10004
10514
|
if (path[0] === "users" && usersHandlers) {
|
|
10005
10515
|
if (path.length === 1) {
|
|
10006
|
-
if (
|
|
10007
|
-
if (
|
|
10516
|
+
if (m === "GET") return usersHandlers.list(req);
|
|
10517
|
+
if (m === "POST") return usersHandlers.create(req);
|
|
10008
10518
|
}
|
|
10009
10519
|
if (path.length === 2) {
|
|
10010
|
-
if (path[1] === "avatar" &&
|
|
10011
|
-
if (path[1] === "profile" &&
|
|
10520
|
+
if (path[1] === "avatar" && m === "POST" && avatarPost) return avatarPost(req);
|
|
10521
|
+
if (path[1] === "profile" && m === "PUT" && profilePut) return profilePut(req);
|
|
10012
10522
|
const id = path[1];
|
|
10013
|
-
if (
|
|
10014
|
-
if (
|
|
10015
|
-
if (
|
|
10523
|
+
if (m === "GET") return usersHandlers.getById(req, id);
|
|
10524
|
+
if (m === "PUT" || m === "PATCH") return usersHandlers.update(req, id);
|
|
10525
|
+
if (m === "DELETE") return usersHandlers.delete(req, id);
|
|
10016
10526
|
}
|
|
10017
|
-
if (path.length === 3 && path[2] === "regenerate-invite" &&
|
|
10527
|
+
if (path.length === 3 && path[2] === "regenerate-invite" && m === "POST") {
|
|
10018
10528
|
return usersHandlers.regenerateInvite(req, path[1]);
|
|
10019
10529
|
}
|
|
10020
10530
|
}
|
|
10021
|
-
if (path[0] === "users" && path.length === 2 && userAuthRouter &&
|
|
10531
|
+
if (path[0] === "users" && path.length === 2 && userAuthRouter && m === "POST") {
|
|
10022
10532
|
return userAuthRouter.POST(req, path[1]);
|
|
10023
10533
|
}
|
|
10024
10534
|
if (path[0] === "settings" && path.length === 2 && settingsHandlers) {
|
|
10025
10535
|
const group = path[1];
|
|
10026
10536
|
const isPublic = settingsConfig?.publicGetGroups?.includes(group);
|
|
10027
|
-
if (
|
|
10537
|
+
if (m === "GET") {
|
|
10028
10538
|
if (!isPublic) {
|
|
10029
10539
|
const a = await config.requireAuth(req);
|
|
10030
10540
|
if (a) return a;
|
|
@@ -10033,15 +10543,38 @@ function createCmsApiHandler(config) {
|
|
|
10033
10543
|
}
|
|
10034
10544
|
return settingsHandlers.GET(req, group);
|
|
10035
10545
|
}
|
|
10036
|
-
if (
|
|
10546
|
+
if (m === "PUT") {
|
|
10037
10547
|
const pe = await requireEntityPermissionEffective(req, "settings", "update");
|
|
10038
10548
|
if (pe) return pe;
|
|
10039
10549
|
return settingsHandlers.PUT(req, group);
|
|
10040
10550
|
}
|
|
10041
10551
|
}
|
|
10042
10552
|
if (path[0] === "message-templates" && path[1] === "sms" && path.length === 2) {
|
|
10043
|
-
if (
|
|
10044
|
-
if (
|
|
10553
|
+
if (m === "GET") return smsMessageTemplateHandlers.GET(req);
|
|
10554
|
+
if (m === "PUT") return smsMessageTemplateHandlers.PUT(req);
|
|
10555
|
+
}
|
|
10556
|
+
if (path[0] === "llm_agents") {
|
|
10557
|
+
if (!entityMap.llm_agents) {
|
|
10558
|
+
console.error(CMS_API_LOG, "llm_agents route: entity missing after merge", {
|
|
10559
|
+
method: m,
|
|
10560
|
+
pathJoined: path.join("/"),
|
|
10561
|
+
entityMapKeyCount: Object.keys(entityMap).length
|
|
10562
|
+
});
|
|
10563
|
+
return config.json(
|
|
10564
|
+
{ error: "LlmAgent entity is not registered", hint: "Ensure migrations ran and entities include LlmAgent." },
|
|
10565
|
+
{ status: 503 }
|
|
10566
|
+
);
|
|
10567
|
+
}
|
|
10568
|
+
if (path.length === 1) {
|
|
10569
|
+
if (m === "GET") return crud.GET(req, "llm_agents");
|
|
10570
|
+
if (m === "POST") return crud.POST(req, "llm_agents");
|
|
10571
|
+
}
|
|
10572
|
+
if (path.length === 2 && !path[1]?.includes("knowledge")) {
|
|
10573
|
+
const id = path[1];
|
|
10574
|
+
if (m === "GET") return crudById.GET(req, "llm_agents", id);
|
|
10575
|
+
if (m === "PUT" || m === "PATCH") return crudById.PUT(req, "llm_agents", id);
|
|
10576
|
+
if (m === "DELETE") return crudById.DELETE(req, "llm_agents", id);
|
|
10577
|
+
}
|
|
10045
10578
|
}
|
|
10046
10579
|
{
|
|
10047
10580
|
const kbMatch = matchLlmAgentKnowledgeRoute(path);
|
|
@@ -10056,24 +10589,24 @@ function createCmsApiHandler(config) {
|
|
|
10056
10589
|
);
|
|
10057
10590
|
}
|
|
10058
10591
|
const { slug, documentId } = kbMatch;
|
|
10059
|
-
if (
|
|
10592
|
+
if (m === "DELETE" && documentId != null && documentId !== "") {
|
|
10060
10593
|
return llmAgentKnowledgeHandlers.unlink(req, slug, documentId);
|
|
10061
10594
|
}
|
|
10062
|
-
if (
|
|
10595
|
+
if (m === "GET" && (documentId == null || documentId === "")) {
|
|
10063
10596
|
return llmAgentKnowledgeHandlers.list(req, slug);
|
|
10064
10597
|
}
|
|
10065
|
-
if (
|
|
10598
|
+
if (m === "POST" && (documentId == null || documentId === "")) {
|
|
10066
10599
|
return llmAgentKnowledgeHandlers.post(req, slug);
|
|
10067
10600
|
}
|
|
10068
10601
|
}
|
|
10069
10602
|
}
|
|
10070
10603
|
if (path[0] === "chat" && chatHandlers) {
|
|
10071
|
-
if (path.length === 2 && path[1] === "config" &&
|
|
10072
|
-
if (path.length === 2 && path[1] === "identify" &&
|
|
10073
|
-
if (path.length === 4 && path[1] === "conversations" && path[3] === "messages" &&
|
|
10074
|
-
if (path.length === 2 && path[1] === "messages" &&
|
|
10604
|
+
if (path.length === 2 && path[1] === "config" && m === "GET") return chatHandlers.publicConfig(req);
|
|
10605
|
+
if (path.length === 2 && path[1] === "identify" && m === "POST") return chatHandlers.identify(req);
|
|
10606
|
+
if (path.length === 4 && path[1] === "conversations" && path[3] === "messages" && m === "GET") return chatHandlers.getMessages(req, path[2]);
|
|
10607
|
+
if (path.length === 2 && path[1] === "messages" && m === "POST") return chatHandlers.postMessage(req);
|
|
10075
10608
|
}
|
|
10076
|
-
if (path[0] === "orders" && path.length === 3 && path[2] === "invoice" &&
|
|
10609
|
+
if (path[0] === "orders" && path.length === 3 && path[2] === "invoice" && m === "GET" && getCms) {
|
|
10077
10610
|
const a = await config.requireAuth(req);
|
|
10078
10611
|
if (a) return a;
|
|
10079
10612
|
const pe = await requireEntityPermissionEffective(req, "orders", "read");
|
|
@@ -10087,17 +10620,17 @@ function createCmsApiHandler(config) {
|
|
|
10087
10620
|
if (path[0] === "orders" && path.length === 3 && path[2] === "repost-erp" && getCms) {
|
|
10088
10621
|
const a = await config.requireAuth(req);
|
|
10089
10622
|
if (a) return a;
|
|
10090
|
-
const pe = await requireEntityPermissionEffective(req, "orders",
|
|
10623
|
+
const pe = await requireEntityPermissionEffective(req, "orders", m === "GET" ? "read" : "update");
|
|
10091
10624
|
if (pe) return pe;
|
|
10092
10625
|
const oid = Number(path[1]);
|
|
10093
10626
|
if (!Number.isFinite(oid)) return config.json({ error: "Invalid id" }, { status: 400 });
|
|
10094
10627
|
const cms = await getCms();
|
|
10095
10628
|
const { isErpIntegrationEnabled: isErpIntegrationEnabled3 } = await Promise.resolve().then(() => (init_erp_config_enabled(), erp_config_enabled_exports));
|
|
10096
10629
|
const enabled = await isErpIntegrationEnabled3(cms, dataSource, entityMap);
|
|
10097
|
-
if (
|
|
10630
|
+
if (m === "GET") {
|
|
10098
10631
|
return config.json({ enabled });
|
|
10099
10632
|
}
|
|
10100
|
-
if (
|
|
10633
|
+
if (m === "POST") {
|
|
10101
10634
|
if (!enabled) return config.json({ error: "ERP integration is disabled" }, { status: 409 });
|
|
10102
10635
|
const { queueErpPaidOrderForOrderId: queueErpPaidOrderForOrderId2 } = await Promise.resolve().then(() => (init_paid_order_erp(), paid_order_erp_exports));
|
|
10103
10636
|
await queueErpPaidOrderForOrderId2(cms, dataSource, entityMap, oid);
|
|
@@ -10107,28 +10640,39 @@ function createCmsApiHandler(config) {
|
|
|
10107
10640
|
}
|
|
10108
10641
|
if (path.length === 0) return config.json({ error: "Not found" }, { status: 404 });
|
|
10109
10642
|
const resource = resolveResource(path[0]);
|
|
10110
|
-
if (!crudResources.includes(resource))
|
|
10643
|
+
if (!crudResources.includes(resource)) {
|
|
10644
|
+
console.warn(CMS_API_LOG, "generic CRUD gate: Invalid resource (not in crudResources)", {
|
|
10645
|
+
method: m,
|
|
10646
|
+
pathJoined: path.join("/"),
|
|
10647
|
+
pathSegments: path,
|
|
10648
|
+
resource,
|
|
10649
|
+
hasLlmAgentsEntity: Boolean(entityMap.llm_agents),
|
|
10650
|
+
crudResourcesHasResource: crudResources.includes(resource),
|
|
10651
|
+
crudResourcesLength: crudResources.length
|
|
10652
|
+
});
|
|
10653
|
+
return config.json({ error: "Invalid resource" }, { status: 400 });
|
|
10654
|
+
}
|
|
10111
10655
|
if (path.length === 2) {
|
|
10112
|
-
if (path[1] === "metadata" &&
|
|
10656
|
+
if (path[1] === "metadata" && m === "GET") {
|
|
10113
10657
|
return crud.GET_METADATA(req, resource);
|
|
10114
10658
|
}
|
|
10115
|
-
if (path[1] === "bulk" &&
|
|
10659
|
+
if (path[1] === "bulk" && m === "POST") {
|
|
10116
10660
|
return crud.BULK_POST(req, resource);
|
|
10117
10661
|
}
|
|
10118
|
-
if (path[1] === "export" &&
|
|
10662
|
+
if (path[1] === "export" && m === "GET") {
|
|
10119
10663
|
return crud.GET_EXPORT(req, resource);
|
|
10120
10664
|
}
|
|
10121
10665
|
}
|
|
10122
10666
|
if (path.length === 1) {
|
|
10123
|
-
if (
|
|
10124
|
-
if (
|
|
10667
|
+
if (m === "GET") return crud.GET(req, resource);
|
|
10668
|
+
if (m === "POST") return crud.POST(req, resource);
|
|
10125
10669
|
return config.json({ error: "Method not allowed" }, { status: 405 });
|
|
10126
10670
|
}
|
|
10127
10671
|
if (path.length === 2) {
|
|
10128
10672
|
const id = path[1];
|
|
10129
|
-
if (
|
|
10130
|
-
if (
|
|
10131
|
-
if (
|
|
10673
|
+
if (m === "GET") return crudById.GET(req, resource, id);
|
|
10674
|
+
if (m === "PUT" || m === "PATCH") return crudById.PUT(req, resource, id);
|
|
10675
|
+
if (m === "DELETE") return crudById.DELETE(req, resource, id);
|
|
10132
10676
|
return config.json({ error: "Method not allowed" }, { status: 405 });
|
|
10133
10677
|
}
|
|
10134
10678
|
return config.json({ error: "Not found" }, { status: 404 });
|