lynkr 9.7.2 → 9.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +29 -19
- package/bin/cli.js +11 -0
- package/bin/lynkr-init.js +14 -1
- package/bin/lynkr-usage.js +78 -0
- package/bin/wrap.js +60 -35
- package/config/difficulty-anchors.json +22 -0
- package/package.json +24 -3
- package/scripts/audit-log-reader.js +399 -0
- package/scripts/calibrate-thresholds.js +38 -157
- package/scripts/compact-dictionary.js +204 -0
- package/scripts/test-deduplication.js +448 -0
- package/scripts/ws7-anchor-replay.js +108 -0
- package/skills/lynkr/SKILL.md +195 -0
- package/src/agents/context-manager.js +18 -2
- package/src/agents/definitions/loader.js +90 -0
- package/src/agents/executor.js +24 -2
- package/src/agents/index.js +9 -1
- package/src/agents/parallel-coordinator.js +2 -2
- package/src/agents/reflector.js +11 -1
- package/src/api/middleware/loop-guard.js +87 -0
- package/src/api/middleware/request-logging.js +5 -64
- package/src/api/middleware/session.js +0 -0
- package/src/api/openai-router.js +120 -101
- package/src/api/providers-handler.js +27 -2
- package/src/api/router.js +450 -125
- package/src/budget/index.js +2 -19
- package/src/cache/semantic.js +9 -0
- package/src/clients/databricks.js +459 -146
- package/src/clients/gpt-utils.js +11 -105
- package/src/clients/openai-format.js +10 -3
- package/src/clients/openrouter-utils.js +49 -24
- package/src/clients/prompt-cache-injection.js +1 -0
- package/src/clients/provider-capabilities.js +1 -1
- package/src/clients/responses-format.js +34 -3
- package/src/clients/routing.js +15 -0
- package/src/config/index.js +36 -2
- package/src/context/gcf.js +275 -0
- package/src/context/tool-result-compressor.js +51 -9
- package/src/dashboard/api.js +1 -0
- package/src/logger/index.js +14 -1
- package/src/memory/search.js +12 -40
- package/src/memory/tools.js +3 -24
- package/src/orchestrator/bypass.js +4 -2
- package/src/orchestrator/index.js +144 -88
- package/src/routing/affinity-store.js +194 -0
- package/src/routing/agentic-detector.js +36 -6
- package/src/routing/bandit.js +25 -6
- package/src/routing/calibration.js +212 -0
- package/src/routing/client-profiles.js +292 -0
- package/src/routing/complexity-analyzer.js +48 -11
- package/src/routing/deescalator.js +148 -0
- package/src/routing/degradation.js +109 -0
- package/src/routing/feedback.js +157 -0
- package/src/routing/index.js +897 -87
- package/src/routing/intent-score.js +339 -0
- package/src/routing/interaction.js +3 -0
- package/src/routing/knn-router.js +70 -21
- package/src/routing/model-registry.js +28 -7
- package/src/routing/model-tiers.js +25 -2
- package/src/routing/reward-pipeline.js +68 -2
- package/src/routing/risk-analyzer.js +30 -1
- package/src/routing/risk-classifier.js +6 -2
- package/src/routing/session-affinity.js +162 -34
- package/src/routing/telemetry.js +298 -10
- package/src/routing/verifier.js +267 -0
- package/src/server.js +66 -21
- package/src/sessions/cleanup.js +17 -0
- package/src/tools/index.js +1 -15
- package/src/tools/smart-selection.js +10 -0
- package/src/tools/web-client.js +3 -3
- package/.eslintrc.cjs +0 -12
- package/benchmark-configs/litellm_config.yaml +0 -86
- package/benchmark-configs/lynkr.env +0 -48
- package/benchmark-configs/portkey-config.json +0 -60
- package/benchmark-configs/portkey-docker.sh +0 -23
- package/benchmark-tier-routing.js +0 -449
- package/funding.json +0 -110
- package/src/api/middleware/validation.js +0 -261
- package/src/routing/drift-monitor.js +0 -113
- package/src/workers/helpers.js +0 -185
|
@@ -13,10 +13,12 @@ const { convertAnthropicToolsToOpenRouter } = require("./openrouter-utils");
|
|
|
13
13
|
const {
|
|
14
14
|
detectModelFamily
|
|
15
15
|
} = require("./bedrock-utils");
|
|
16
|
-
const { getGPTSystemPromptAddendum } = require("./gpt-utils");
|
|
17
16
|
const telemetry = require("../routing/telemetry");
|
|
18
17
|
const { scoreResponseQuality } = require("../routing/quality-scorer");
|
|
19
18
|
const { getLatencyTracker } = require("../routing/latency-tracker");
|
|
19
|
+
// WS5.4 — feedback loop. `recordOutcome` runs on setImmediate and never
|
|
20
|
+
// throws; every failure is captured into the degradation registry.
|
|
21
|
+
const { recordOutcome: recordFeedbackOutcome } = require("../routing/feedback");
|
|
20
22
|
|
|
21
23
|
|
|
22
24
|
|
|
@@ -51,8 +53,33 @@ const httpsAgent = new https.Agent({
|
|
|
51
53
|
keepAliveMsecs: 30000,
|
|
52
54
|
});
|
|
53
55
|
|
|
56
|
+
/**
|
|
57
|
+
* Strip Lynkr's internal underscore-prefixed fields from a body about to be
|
|
58
|
+
* sent upstream. Every provider validates its request body (Anthropic and
|
|
59
|
+
* Ollama Cloud both use Pydantic and reject unknown keys with "Extra inputs
|
|
60
|
+
* are not permitted"), so any leak of _sessionId / _forceProvider / _tierModel
|
|
61
|
+
* etc. into the outbound JSON hard-fails the request.
|
|
62
|
+
*
|
|
63
|
+
* This is a defense-in-depth strip at the last-hop chokepoint. Individual
|
|
64
|
+
* invoke functions already whitelist their outbound bodies, but doing this
|
|
65
|
+
* once here means no future codepath (or spread of `{...body}` in a new
|
|
66
|
+
* provider) can regress the invariant.
|
|
67
|
+
*/
|
|
68
|
+
function _stripInternalFields(body) {
|
|
69
|
+
if (!body || typeof body !== 'object') return body;
|
|
70
|
+
let cleaned = null;
|
|
71
|
+
for (const key of Object.keys(body)) {
|
|
72
|
+
if (key.startsWith('_')) {
|
|
73
|
+
if (!cleaned) cleaned = { ...body };
|
|
74
|
+
delete cleaned[key];
|
|
75
|
+
}
|
|
76
|
+
}
|
|
77
|
+
return cleaned || body;
|
|
78
|
+
}
|
|
79
|
+
|
|
54
80
|
async function performJsonRequest(url, { headers = {}, body, retryableStatusesOverride }, providerLabel) {
|
|
55
81
|
const agent = url.startsWith('https:') ? httpsAgent : httpAgent;
|
|
82
|
+
body = _stripInternalFields(body);
|
|
56
83
|
const isStreaming = body.stream === true;
|
|
57
84
|
|
|
58
85
|
// Streaming requests can't be retried, so handle them directly
|
|
@@ -399,6 +426,49 @@ function _liftLeakedThinkingBlocks(response) {
|
|
|
399
426
|
return response;
|
|
400
427
|
}
|
|
401
428
|
|
|
429
|
+
/**
|
|
430
|
+
* Ollama proxies cloud-provider errors as a JSON body that can arrive with
|
|
431
|
+
* HTTP 200 (e.g. quota exhaustion). Unthrown, that serves as an empty
|
|
432
|
+
* message and no fallback fires — so detect and throw, handing the request
|
|
433
|
+
* to invokeModel's tier-fallback catch. On streamed requests the daemon
|
|
434
|
+
* answers errors with JSON instead of SSE, so a json content-type on a
|
|
435
|
+
* streaming response IS the error case.
|
|
436
|
+
*/
|
|
437
|
+
async function _throwIfOllamaError(response, modelName) {
|
|
438
|
+
if (!response) return response;
|
|
439
|
+
|
|
440
|
+
// Streamed request answered with JSON → provider error, not a stream.
|
|
441
|
+
if (response.stream && /json/i.test(response.contentType || "")) {
|
|
442
|
+
let text = "";
|
|
443
|
+
try {
|
|
444
|
+
for await (const chunk of response.stream) text += chunk;
|
|
445
|
+
} catch { /* partial body is fine — we're throwing anyway */ }
|
|
446
|
+
let j = null;
|
|
447
|
+
try { j = JSON.parse(text); } catch { /* non-JSON error body */ }
|
|
448
|
+
// A JSON content-type that is actually a valid non-streamed Anthropic
|
|
449
|
+
// message would be unusual; treat anything without content blocks as error.
|
|
450
|
+
if (!Array.isArray(j?.content)) {
|
|
451
|
+
const msg = (typeof j?.error === "string" ? j.error : j?.error?.message) || text.slice(0, 200) || `HTTP ${response.status}`;
|
|
452
|
+
const err = new Error(`Ollama error for ${modelName}: ${msg}`);
|
|
453
|
+
err.status = Number(j?.StatusCode) >= 400 ? Number(j.StatusCode) : (response.ok ? 502 : response.status);
|
|
454
|
+
throw err;
|
|
455
|
+
}
|
|
456
|
+
// Valid message that happened to be JSON — reshape as non-streamed result.
|
|
457
|
+
return { ok: true, status: response.status, json: j, text, contentType: response.contentType, headers: response.headers };
|
|
458
|
+
}
|
|
459
|
+
|
|
460
|
+
const j = response.json;
|
|
461
|
+
const errMsg = typeof j?.error === "string" ? j.error
|
|
462
|
+
: typeof j?.error?.message === "string" ? j.error.message : null;
|
|
463
|
+
const embeddedStatus = Number(j?.StatusCode);
|
|
464
|
+
if (!response.ok || errMsg || (Number.isFinite(embeddedStatus) && embeddedStatus >= 400)) {
|
|
465
|
+
const err = new Error(`Ollama error for ${modelName}: ${errMsg || `HTTP ${response.status}`}`);
|
|
466
|
+
err.status = Number.isFinite(embeddedStatus) && embeddedStatus >= 400 ? embeddedStatus : response.status;
|
|
467
|
+
throw err;
|
|
468
|
+
}
|
|
469
|
+
return response;
|
|
470
|
+
}
|
|
471
|
+
|
|
402
472
|
async function invokeOllama(body, incomingHeaders = {}) {
|
|
403
473
|
if (!config.ollama?.endpoint) {
|
|
404
474
|
throw new Error("Ollama endpoint is not configured.");
|
|
@@ -415,7 +485,6 @@ async function invokeOllama(body, incomingHeaders = {}) {
|
|
|
415
485
|
const supportsTools = await checkOllamaToolSupport(modelName);
|
|
416
486
|
const injectToolsOllama = process.env.INJECT_TOOLS_OLLAMA !== "false";
|
|
417
487
|
|
|
418
|
-
// Determine tools to send
|
|
419
488
|
let toolsToSend = body.tools;
|
|
420
489
|
let toolsInjected = false;
|
|
421
490
|
|
|
@@ -447,7 +516,7 @@ async function invokeOllama(body, incomingHeaders = {}) {
|
|
|
447
516
|
toolCount,
|
|
448
517
|
toolsInjected,
|
|
449
518
|
supportsTools,
|
|
450
|
-
toolNames: (Array.isArray(toolsToSend) && toolsToSend.length > 0) ? toolsToSend.map(t => t.name) : []
|
|
519
|
+
toolNames: (Array.isArray(toolsToSend) && toolsToSend.length > 0) ? toolsToSend.map(t => t.name || t.function?.name) : []
|
|
451
520
|
}, `=== Ollama STANDARD TOOLS INJECTION for ${config.ollama.model} === ${logMessage}`);
|
|
452
521
|
|
|
453
522
|
// ---- Anthropic-native path (Ollama v0.14.0+) ----
|
|
@@ -458,12 +527,21 @@ async function invokeOllama(body, incomingHeaders = {}) {
|
|
|
458
527
|
"anthropic-version": "2023-06-01",
|
|
459
528
|
};
|
|
460
529
|
|
|
461
|
-
// Build body with only valid Anthropic Messages API fields
|
|
530
|
+
// Build body with only valid Anthropic Messages API fields.
|
|
531
|
+
// max_tokens floor: thinking models spend 200-400 tokens reasoning
|
|
532
|
+
// before any text; small client budgets die mid-think and return
|
|
533
|
+
// stop_reason=max_tokens with zero text.
|
|
534
|
+
const minMaxTokens = Number(process.env.LYNKR_OLLAMA_MIN_MAX_TOKENS) || 1536;
|
|
535
|
+
// Buffer ollama responses (opt out: LYNKR_OLLAMA_BUFFER_RESPONSES=false):
|
|
536
|
+
// thinking models stream reasoning deltas Claude Code renders as dead
|
|
537
|
+
// air. Buffered responses go through thinking-block cleanup + SSE
|
|
538
|
+
// synthesis, and become verifiable by the cascade (streams bypass it).
|
|
539
|
+
const bufferOllama = process.env.LYNKR_OLLAMA_BUFFER_RESPONSES !== "false";
|
|
462
540
|
const ollamaBody = {
|
|
463
541
|
model: modelName,
|
|
464
542
|
messages: body.messages,
|
|
465
|
-
max_tokens: body.max_tokens || 16384,
|
|
466
|
-
stream: body.stream ?? false,
|
|
543
|
+
max_tokens: Math.max(body.max_tokens || 16384, minMaxTokens),
|
|
544
|
+
stream: bufferOllama ? false : (body.stream ?? false),
|
|
467
545
|
};
|
|
468
546
|
|
|
469
547
|
if (body.system) ollamaBody.system = body.system;
|
|
@@ -487,7 +565,10 @@ async function invokeOllama(body, incomingHeaders = {}) {
|
|
|
487
565
|
logger.debug({ keepAlive: ollamaBody.keep_alive }, "Ollama keep_alive configured");
|
|
488
566
|
}
|
|
489
567
|
|
|
490
|
-
const response = await
|
|
568
|
+
const response = await _throwIfOllamaError(
|
|
569
|
+
await performJsonRequest(endpoint, { headers, body: ollamaBody }, "Ollama"),
|
|
570
|
+
modelName,
|
|
571
|
+
);
|
|
491
572
|
// Even on the Anthropic-native path, Ollama Cloud's MiniMax M2.5 adapter
|
|
492
573
|
// sometimes leaks <think>...</think> as raw text inside content blocks
|
|
493
574
|
// instead of emitting a thinking content block (ollama/ollama#14220 was
|
|
@@ -636,7 +717,10 @@ async function invokeOllama(body, incomingHeaders = {}) {
|
|
|
636
717
|
ollamaBody.tools = convertAnthropicToolsToOllama(toolsToSend);
|
|
637
718
|
}
|
|
638
719
|
|
|
639
|
-
return
|
|
720
|
+
return _throwIfOllamaError(
|
|
721
|
+
await performJsonRequest(endpoint, { headers, body: ollamaBody }, "Ollama"),
|
|
722
|
+
modelName,
|
|
723
|
+
);
|
|
640
724
|
}
|
|
641
725
|
|
|
642
726
|
async function invokeOpenRouter(body, incomingHeaders = {}) {
|
|
@@ -657,7 +741,6 @@ async function invokeOpenRouter(body, incomingHeaders = {}) {
|
|
|
657
741
|
"X-Title": "Claude-Ollama-Proxy"
|
|
658
742
|
};
|
|
659
743
|
|
|
660
|
-
// Convert messages and handle system message
|
|
661
744
|
const messages = convertAnthropicMessagesToOpenRouter(body.messages || []);
|
|
662
745
|
|
|
663
746
|
// Anthropic uses separate 'system' field, OpenAI needs it as first message
|
|
@@ -682,7 +765,6 @@ async function invokeOpenRouter(body, incomingHeaders = {}) {
|
|
|
682
765
|
let toolsInjected = false;
|
|
683
766
|
|
|
684
767
|
if (!Array.isArray(toolsToSend) || toolsToSend.length === 0) {
|
|
685
|
-
// Client didn't send tools (likely passthrough mode) - inject standard Claude Code tools
|
|
686
768
|
toolsToSend = STANDARD_TOOLS;
|
|
687
769
|
toolsInjected = true;
|
|
688
770
|
logger.debug({
|
|
@@ -696,7 +778,7 @@ async function invokeOpenRouter(body, incomingHeaders = {}) {
|
|
|
696
778
|
openRouterBody.tools = convertAnthropicToolsToOpenRouter(toolsToSend);
|
|
697
779
|
logger.debug({
|
|
698
780
|
toolCount: toolsToSend.length,
|
|
699
|
-
toolNames: toolsToSend.map(t => t.name),
|
|
781
|
+
toolNames: toolsToSend.map(t => t.name || t.function?.name),
|
|
700
782
|
toolsInjected
|
|
701
783
|
}, "Sending tools to OpenRouter");
|
|
702
784
|
}
|
|
@@ -704,6 +786,69 @@ async function invokeOpenRouter(body, incomingHeaders = {}) {
|
|
|
704
786
|
return performJsonRequest(endpoint, { headers, body: openRouterBody }, "OpenRouter");
|
|
705
787
|
}
|
|
706
788
|
|
|
789
|
+
// Eden AI is an OpenAI-compatible gateway (provider/model naming, EU/GDPR).
|
|
790
|
+
// It speaks the same wire format as OpenRouter, so this mirrors invokeOpenRouter
|
|
791
|
+
// and reuses the shared Anthropic<->OpenAI converters.
|
|
792
|
+
async function invokeEdenAI(body, incomingHeaders = {}) {
|
|
793
|
+
if (!config.edenai?.endpoint || !config.edenai?.apiKey) {
|
|
794
|
+
throw new Error("Eden AI endpoint or API key is not configured.");
|
|
795
|
+
}
|
|
796
|
+
|
|
797
|
+
const {
|
|
798
|
+
convertAnthropicToolsToOpenRouter,
|
|
799
|
+
convertAnthropicMessagesToOpenRouter
|
|
800
|
+
} = require("./openrouter-utils");
|
|
801
|
+
|
|
802
|
+
const endpoint = config.edenai.endpoint;
|
|
803
|
+
const headers = {
|
|
804
|
+
"Authorization": `Bearer ${config.edenai.apiKey}`,
|
|
805
|
+
"Content-Type": "application/json"
|
|
806
|
+
};
|
|
807
|
+
|
|
808
|
+
const messages = convertAnthropicMessagesToOpenRouter(body.messages || []);
|
|
809
|
+
|
|
810
|
+
if (body.system) {
|
|
811
|
+
messages.unshift({
|
|
812
|
+
role: "system",
|
|
813
|
+
content: body.system
|
|
814
|
+
});
|
|
815
|
+
}
|
|
816
|
+
|
|
817
|
+
const edenAIBody = {
|
|
818
|
+
model: body._suggestionModeModel || body._tierModel || config.edenai.model,
|
|
819
|
+
messages,
|
|
820
|
+
temperature: body.temperature ?? 0.7,
|
|
821
|
+
max_tokens: body.max_tokens ?? 16384,
|
|
822
|
+
top_p: body.top_p ?? 1.0,
|
|
823
|
+
stream: body.stream ?? false
|
|
824
|
+
};
|
|
825
|
+
|
|
826
|
+
// Add tools - inject standard tools if client didn't send any (passthrough mode)
|
|
827
|
+
let toolsToSend = body.tools;
|
|
828
|
+
let toolsInjected = false;
|
|
829
|
+
|
|
830
|
+
if (!Array.isArray(toolsToSend) || toolsToSend.length === 0) {
|
|
831
|
+
toolsToSend = STANDARD_TOOLS;
|
|
832
|
+
toolsInjected = true;
|
|
833
|
+
logger.debug({
|
|
834
|
+
injectedToolCount: STANDARD_TOOLS.length,
|
|
835
|
+
injectedToolNames: STANDARD_TOOL_NAMES,
|
|
836
|
+
reason: "Client did not send tools (passthrough mode)"
|
|
837
|
+
}, "=== INJECTING STANDARD TOOLS (Eden AI) ===");
|
|
838
|
+
}
|
|
839
|
+
|
|
840
|
+
if (Array.isArray(toolsToSend) && toolsToSend.length > 0) {
|
|
841
|
+
edenAIBody.tools = convertAnthropicToolsToOpenRouter(toolsToSend);
|
|
842
|
+
logger.debug({
|
|
843
|
+
toolCount: toolsToSend.length,
|
|
844
|
+
toolNames: toolsToSend.map(t => t.name || t.function?.name),
|
|
845
|
+
toolsInjected
|
|
846
|
+
}, "Sending tools to Eden AI");
|
|
847
|
+
}
|
|
848
|
+
|
|
849
|
+
return performJsonRequest(endpoint, { headers, body: edenAIBody }, "EdenAI");
|
|
850
|
+
}
|
|
851
|
+
|
|
707
852
|
function detectAzureFormat(url) {
|
|
708
853
|
if (url.includes("/openai/responses")) return "responses";
|
|
709
854
|
if (url.includes("/models/")) return "models";
|
|
@@ -722,7 +867,6 @@ async function invokeAzureOpenAI(body, incomingHeaders = {}) {
|
|
|
722
867
|
convertAnthropicMessagesToOpenRouter
|
|
723
868
|
} = require("./openrouter-utils");
|
|
724
869
|
|
|
725
|
-
// Azure OpenAI URL format
|
|
726
870
|
const endpoint = config.azureOpenAI.endpoint;
|
|
727
871
|
const format = detectAzureFormat(endpoint);
|
|
728
872
|
|
|
@@ -738,10 +882,8 @@ async function invokeAzureOpenAI(body, incomingHeaders = {}) {
|
|
|
738
882
|
headers["api-key"] = config.azureOpenAI.apiKey;
|
|
739
883
|
}
|
|
740
884
|
|
|
741
|
-
// Convert messages and handle system message
|
|
742
885
|
const messages = convertAnthropicMessagesToOpenRouter(body.messages || []);
|
|
743
886
|
|
|
744
|
-
// Anthropic uses separate 'system' field, OpenAI needs it as first message
|
|
745
887
|
if (body.system) {
|
|
746
888
|
messages.unshift({
|
|
747
889
|
role: "system",
|
|
@@ -749,9 +891,6 @@ async function invokeAzureOpenAI(body, incomingHeaders = {}) {
|
|
|
749
891
|
});
|
|
750
892
|
}
|
|
751
893
|
|
|
752
|
-
// System prompt injection disabled - breaks model response
|
|
753
|
-
// Tool guidance now provided via tool descriptions instead
|
|
754
|
-
|
|
755
894
|
const azureDeployment = body._suggestionModeModel || body._tierModel || config.azureOpenAI.deployment || "";
|
|
756
895
|
const isGpt5 = /gpt-5/i.test(azureDeployment);
|
|
757
896
|
const maxTokensKey = isGpt5 ? "max_completion_tokens" : "max_tokens";
|
|
@@ -777,7 +916,6 @@ async function invokeAzureOpenAI(body, incomingHeaders = {}) {
|
|
|
777
916
|
let toolsInjected = false;
|
|
778
917
|
|
|
779
918
|
if (!Array.isArray(toolsToSend) || toolsToSend.length === 0) {
|
|
780
|
-
// Client didn't send tools (likely passthrough mode) - inject standard Claude Code tools
|
|
781
919
|
toolsToSend = STANDARD_TOOLS;
|
|
782
920
|
toolsInjected = true;
|
|
783
921
|
logger.debug({
|
|
@@ -789,11 +927,11 @@ async function invokeAzureOpenAI(body, incomingHeaders = {}) {
|
|
|
789
927
|
|
|
790
928
|
if (Array.isArray(toolsToSend) && toolsToSend.length > 0) {
|
|
791
929
|
azureBody.tools = convertAnthropicToolsToOpenRouter(toolsToSend);
|
|
792
|
-
azureBody.parallel_tool_calls = true;
|
|
930
|
+
azureBody.parallel_tool_calls = true;
|
|
793
931
|
azureBody.tool_choice = "auto"; // Explicitly enable tool use (helps GPT models understand they should use tools)
|
|
794
932
|
logger.debug({
|
|
795
933
|
toolCount: toolsToSend.length,
|
|
796
|
-
toolNames: toolsToSend.map(t => t.name),
|
|
934
|
+
toolNames: toolsToSend.map(t => t.name || t.function?.name),
|
|
797
935
|
toolsInjected,
|
|
798
936
|
hasSystemMessage: !!body.system,
|
|
799
937
|
messageCount: messages.length,
|
|
@@ -891,7 +1029,6 @@ async function invokeAzureOpenAI(body, incomingHeaders = {}) {
|
|
|
891
1029
|
let cleaned = content.replace(/<system-reminder>[\s\S]*?<\/system-reminder>/gi, '');
|
|
892
1030
|
// Remove the continuation marker that orchestrator adds
|
|
893
1031
|
cleaned = cleaned.replace(/---\s*IMPORTANT:\s*Focus on and respond ONLY to my most recent request[^\n]*/gi, '');
|
|
894
|
-
// Trim whitespace
|
|
895
1032
|
return cleaned.trim();
|
|
896
1033
|
};
|
|
897
1034
|
|
|
@@ -918,8 +1055,6 @@ async function invokeAzureOpenAI(body, incomingHeaders = {}) {
|
|
|
918
1055
|
if (Array.isArray(msg.content)) {
|
|
919
1056
|
for (const block of msg.content) {
|
|
920
1057
|
if (block.type === "tool_result") {
|
|
921
|
-
// Convert tool_result to function_call_output
|
|
922
|
-
// Use tool_use_id if available, otherwise pop from pending call IDs
|
|
923
1058
|
const callId = block.tool_use_id || pendingCallIds.shift() || `call_${Date.now()}`;
|
|
924
1059
|
responsesInput.push({
|
|
925
1060
|
type: "function_call_output",
|
|
@@ -998,7 +1133,6 @@ async function invokeAzureOpenAI(body, incomingHeaders = {}) {
|
|
|
998
1133
|
}
|
|
999
1134
|
} else if (msg.role === "tool") {
|
|
1000
1135
|
// Tool results become function_call_output
|
|
1001
|
-
// Use tool_call_id if available, otherwise pop from pending call IDs
|
|
1002
1136
|
const callId = msg.tool_call_id || pendingCallIds.shift() || `call_${Date.now()}`;
|
|
1003
1137
|
responsesInput.push({
|
|
1004
1138
|
type: "function_call_output",
|
|
@@ -1029,9 +1163,25 @@ async function invokeAzureOpenAI(body, incomingHeaders = {}) {
|
|
|
1029
1163
|
if (result.ok && result.json?.output) {
|
|
1030
1164
|
const outputArray = result.json.output || [];
|
|
1031
1165
|
|
|
1032
|
-
//
|
|
1033
|
-
|
|
1034
|
-
|
|
1166
|
+
// Extract text across ALL message outputs and content-item shapes
|
|
1167
|
+
// ("output_text", "text", string content) — a single-shape find
|
|
1168
|
+
// silently returns "" and the whole answer vanishes.
|
|
1169
|
+
const messageOutputs = outputArray.filter(o => o.type === "message");
|
|
1170
|
+
const messageOutput = messageOutputs[0];
|
|
1171
|
+
const textContent = messageOutputs.map(mo => {
|
|
1172
|
+
if (typeof mo.content === "string") return mo.content;
|
|
1173
|
+
if (!Array.isArray(mo.content)) return "";
|
|
1174
|
+
return mo.content
|
|
1175
|
+
.filter(c => (c.type === "output_text" || c.type === "text") && typeof c.text === "string")
|
|
1176
|
+
.map(c => c.text)
|
|
1177
|
+
.join("");
|
|
1178
|
+
}).join("\n").trim() || "";
|
|
1179
|
+
if (!textContent && messageOutputs.length > 0) {
|
|
1180
|
+
logger.warn({
|
|
1181
|
+
contentShapes: messageOutputs.map(mo =>
|
|
1182
|
+
Array.isArray(mo.content) ? mo.content.map(c => c.type) : typeof mo.content),
|
|
1183
|
+
}, "[AzureResponses] Message output present but no text extracted — unknown content shape");
|
|
1184
|
+
}
|
|
1035
1185
|
|
|
1036
1186
|
// Find function_call outputs (tool calls are separate items in output array)
|
|
1037
1187
|
const rawToolCalls = outputArray
|
|
@@ -1075,7 +1225,6 @@ async function invokeAzureOpenAI(body, incomingHeaders = {}) {
|
|
|
1075
1225
|
toolCallNames: toolCalls.map(tc => tc.function.name)
|
|
1076
1226
|
}, "Parsing Responses API output");
|
|
1077
1227
|
|
|
1078
|
-
// Convert to Chat Completions format
|
|
1079
1228
|
result.json = {
|
|
1080
1229
|
id: result.json.id,
|
|
1081
1230
|
object: "chat.completion",
|
|
@@ -1090,7 +1239,14 @@ async function invokeAzureOpenAI(body, incomingHeaders = {}) {
|
|
|
1090
1239
|
},
|
|
1091
1240
|
finish_reason: toolCalls.length > 0 ? "tool_calls" : "stop"
|
|
1092
1241
|
}],
|
|
1093
|
-
usage
|
|
1242
|
+
// Responses API usage is {input,output}_tokens; downstream reads
|
|
1243
|
+
// Chat Completions' {prompt,completion}_tokens.
|
|
1244
|
+
usage: result.json.usage ? {
|
|
1245
|
+
prompt_tokens: result.json.usage.prompt_tokens ?? result.json.usage.input_tokens ?? 0,
|
|
1246
|
+
completion_tokens: result.json.usage.completion_tokens ?? result.json.usage.output_tokens ?? 0,
|
|
1247
|
+
total_tokens: result.json.usage.total_tokens
|
|
1248
|
+
?? ((result.json.usage.input_tokens ?? 0) + (result.json.usage.output_tokens ?? 0)),
|
|
1249
|
+
} : undefined
|
|
1094
1250
|
};
|
|
1095
1251
|
|
|
1096
1252
|
logger.debug({
|
|
@@ -1099,7 +1255,6 @@ async function invokeAzureOpenAI(body, incomingHeaders = {}) {
|
|
|
1099
1255
|
toolCallCount: toolCalls.length
|
|
1100
1256
|
}, "Converted Responses API to Chat Completions format");
|
|
1101
1257
|
|
|
1102
|
-
// Now convert from Chat Completions format to Anthropic format
|
|
1103
1258
|
const anthropicJson = convertOpenAIToAnthropic(result.json);
|
|
1104
1259
|
logger.debug({
|
|
1105
1260
|
anthropicContentTypes: anthropicJson.content?.map(c => c.type),
|
|
@@ -1140,15 +1295,12 @@ async function invokeOpenAI(body, incomingHeaders = {}) {
|
|
|
1140
1295
|
"Content-Type": "application/json",
|
|
1141
1296
|
};
|
|
1142
1297
|
|
|
1143
|
-
// Add organization header if configured
|
|
1144
1298
|
if (config.openai.organization) {
|
|
1145
1299
|
headers["OpenAI-Organization"] = config.openai.organization;
|
|
1146
1300
|
}
|
|
1147
1301
|
|
|
1148
|
-
// Convert messages and handle system message
|
|
1149
1302
|
const messages = convertAnthropicMessagesToOpenRouter(body.messages || []);
|
|
1150
1303
|
|
|
1151
|
-
// Anthropic uses separate 'system' field, OpenAI needs it as first message
|
|
1152
1304
|
if (body.system) {
|
|
1153
1305
|
messages.unshift({
|
|
1154
1306
|
role: "system",
|
|
@@ -1156,8 +1308,6 @@ async function invokeOpenAI(body, incomingHeaders = {}) {
|
|
|
1156
1308
|
});
|
|
1157
1309
|
}
|
|
1158
1310
|
|
|
1159
|
-
// System prompt injection disabled - breaks model response
|
|
1160
|
-
|
|
1161
1311
|
const openAIBody = {
|
|
1162
1312
|
model: body._suggestionModeModel || body._tierModel || config.openai.model || "gpt-4o",
|
|
1163
1313
|
messages,
|
|
@@ -1172,7 +1322,6 @@ async function invokeOpenAI(body, incomingHeaders = {}) {
|
|
|
1172
1322
|
let toolsInjected = false;
|
|
1173
1323
|
|
|
1174
1324
|
if (!Array.isArray(toolsToSend) || toolsToSend.length === 0) {
|
|
1175
|
-
// Client didn't send tools (likely passthrough mode) - inject standard Claude Code tools
|
|
1176
1325
|
toolsToSend = STANDARD_TOOLS;
|
|
1177
1326
|
toolsInjected = true;
|
|
1178
1327
|
logger.debug({
|
|
@@ -1188,7 +1337,7 @@ async function invokeOpenAI(body, incomingHeaders = {}) {
|
|
|
1188
1337
|
openAIBody.tool_choice = "auto"; // Let the model decide when to use tools
|
|
1189
1338
|
logger.debug({
|
|
1190
1339
|
toolCount: toolsToSend.length,
|
|
1191
|
-
toolNames: toolsToSend.map(t => t.name),
|
|
1340
|
+
toolNames: toolsToSend.map(t => t.name || t.function?.name),
|
|
1192
1341
|
toolsInjected
|
|
1193
1342
|
}, "=== SENDING TOOLS TO OPENAI ===");
|
|
1194
1343
|
}
|
|
@@ -1225,10 +1374,8 @@ async function invokeLlamaCpp(body, incomingHeaders = {}) {
|
|
|
1225
1374
|
headers["Authorization"] = `Bearer ${config.llamacpp.apiKey}`;
|
|
1226
1375
|
}
|
|
1227
1376
|
|
|
1228
|
-
// Convert messages to OpenAI format
|
|
1229
1377
|
const messages = convertAnthropicMessagesToOpenRouter(body.messages || []);
|
|
1230
1378
|
|
|
1231
|
-
// Handle system message
|
|
1232
1379
|
if (body.system) {
|
|
1233
1380
|
messages.unshift({ role: "system", content: body.system });
|
|
1234
1381
|
}
|
|
@@ -1290,7 +1437,7 @@ async function invokeLlamaCpp(body, incomingHeaders = {}) {
|
|
|
1290
1437
|
llamacppBody.tool_choice = "auto";
|
|
1291
1438
|
logger.debug({
|
|
1292
1439
|
toolCount: toolsToSend.length,
|
|
1293
|
-
toolNames: toolsToSend.map(t => t.name),
|
|
1440
|
+
toolNames: toolsToSend.map(t => t.name || t.function?.name),
|
|
1294
1441
|
toolsInjected
|
|
1295
1442
|
}, "=== SENDING TOOLS TO LLAMA.CPP ===");
|
|
1296
1443
|
}
|
|
@@ -1336,10 +1483,8 @@ async function invokeLMStudio(body, incomingHeaders = {}) {
|
|
|
1336
1483
|
headers["Authorization"] = `Bearer ${config.lmstudio.apiKey}`;
|
|
1337
1484
|
}
|
|
1338
1485
|
|
|
1339
|
-
// Convert messages to OpenAI format
|
|
1340
1486
|
const messages = convertAnthropicMessagesToOpenRouter(body.messages || []);
|
|
1341
1487
|
|
|
1342
|
-
// Handle system message
|
|
1343
1488
|
if (body.system) {
|
|
1344
1489
|
messages.unshift({ role: "system", content: body.system });
|
|
1345
1490
|
}
|
|
@@ -1371,7 +1516,7 @@ async function invokeLMStudio(body, incomingHeaders = {}) {
|
|
|
1371
1516
|
lmstudioBody.tool_choice = "auto";
|
|
1372
1517
|
logger.debug({
|
|
1373
1518
|
toolCount: toolsToSend.length,
|
|
1374
|
-
toolNames: toolsToSend.map(t => t.name),
|
|
1519
|
+
toolNames: toolsToSend.map(t => t.name || t.function?.name),
|
|
1375
1520
|
toolsInjected
|
|
1376
1521
|
}, "=== SENDING TOOLS TO LM STUDIO ===");
|
|
1377
1522
|
}
|
|
@@ -1446,7 +1591,6 @@ function normalizeBodyForConverse(body) {
|
|
|
1446
1591
|
}
|
|
1447
1592
|
|
|
1448
1593
|
async function invokeBedrock(body, incomingHeaders = {}) {
|
|
1449
|
-
// 1. Validate Bearer token
|
|
1450
1594
|
if (!config.bedrock?.apiKey) {
|
|
1451
1595
|
throw new Error(
|
|
1452
1596
|
"AWS Bedrock requires AWS_BEDROCK_API_KEY (Bearer token). " +
|
|
@@ -1457,7 +1601,7 @@ async function invokeBedrock(body, incomingHeaders = {}) {
|
|
|
1457
1601
|
const bearerToken = config.bedrock.apiKey;
|
|
1458
1602
|
logger.debug({ authMethod: "Bearer Token" }, "=== BEDROCK AUTH ===");
|
|
1459
1603
|
|
|
1460
|
-
//
|
|
1604
|
+
// Inject standard tools if needed
|
|
1461
1605
|
let toolsToSend = body.tools;
|
|
1462
1606
|
let toolsInjected = false;
|
|
1463
1607
|
|
|
@@ -1476,7 +1620,6 @@ async function invokeBedrock(body, incomingHeaders = {}) {
|
|
|
1476
1620
|
// message content, not Anthropic cache_control blocks.
|
|
1477
1621
|
const bedrockBody = { ...normalizeBodyForConverse(body), tools: toolsToSend };
|
|
1478
1622
|
|
|
1479
|
-
// 4. Detect model family and convert format
|
|
1480
1623
|
const modelId = body._tierModel || config.bedrock.modelId;
|
|
1481
1624
|
const modelFamily = detectModelFamily(modelId);
|
|
1482
1625
|
|
|
@@ -1488,7 +1631,7 @@ async function invokeBedrock(body, incomingHeaders = {}) {
|
|
|
1488
1631
|
streaming: body.stream || false,
|
|
1489
1632
|
}, "=== BEDROCK REQUEST (FETCH) ===");
|
|
1490
1633
|
|
|
1491
|
-
//
|
|
1634
|
+
// Convert to Bedrock Converse API format (simpler, more universal)
|
|
1492
1635
|
// Bedrock Converse API only allows 'user' and 'assistant' roles in messages array
|
|
1493
1636
|
|
|
1494
1637
|
// Extract system messages from messages array (if any)
|
|
@@ -1496,7 +1639,7 @@ async function invokeBedrock(body, incomingHeaders = {}) {
|
|
|
1496
1639
|
|
|
1497
1640
|
const converseBody = {
|
|
1498
1641
|
messages: bedrockBody.messages
|
|
1499
|
-
.filter(msg => msg.role !== 'system')
|
|
1642
|
+
.filter(msg => msg.role !== 'system')
|
|
1500
1643
|
.map(msg => ({
|
|
1501
1644
|
role: msg.role,
|
|
1502
1645
|
content: Array.isArray(msg.content)
|
|
@@ -1516,7 +1659,6 @@ async function invokeBedrock(body, incomingHeaders = {}) {
|
|
|
1516
1659
|
converseBody.system = [{ text: systemContent }];
|
|
1517
1660
|
}
|
|
1518
1661
|
|
|
1519
|
-
// Add inference config
|
|
1520
1662
|
if (bedrockBody.max_tokens) {
|
|
1521
1663
|
converseBody.inferenceConfig = {
|
|
1522
1664
|
maxTokens: bedrockBody.max_tokens,
|
|
@@ -1525,7 +1667,6 @@ async function invokeBedrock(body, incomingHeaders = {}) {
|
|
|
1525
1667
|
};
|
|
1526
1668
|
}
|
|
1527
1669
|
|
|
1528
|
-
// Add tools if present
|
|
1529
1670
|
if (bedrockBody.tools && bedrockBody.tools.length > 0) {
|
|
1530
1671
|
converseBody.toolConfig = {
|
|
1531
1672
|
tools: bedrockBody.tools.map(tool => ({
|
|
@@ -1540,7 +1681,6 @@ async function invokeBedrock(body, incomingHeaders = {}) {
|
|
|
1540
1681
|
};
|
|
1541
1682
|
}
|
|
1542
1683
|
|
|
1543
|
-
// 6. Construct Bedrock Converse API endpoint
|
|
1544
1684
|
const path = `/model/${modelId}/converse`;
|
|
1545
1685
|
const host = `bedrock-runtime.${config.bedrock.region}.amazonaws.com`;
|
|
1546
1686
|
const endpoint = `https://${host}${path}`;
|
|
@@ -1553,21 +1693,19 @@ async function invokeBedrock(body, incomingHeaders = {}) {
|
|
|
1553
1693
|
messageCount: converseBody.messages.length
|
|
1554
1694
|
}, "=== BEDROCK CONVERSE API REQUEST ===");
|
|
1555
1695
|
|
|
1556
|
-
// 7. Prepare request headers with Bearer token
|
|
1557
1696
|
const requestHeaders = {
|
|
1558
1697
|
"Content-Type": "application/json",
|
|
1559
1698
|
"Authorization": `Bearer ${bearerToken}`
|
|
1560
1699
|
};
|
|
1561
1700
|
|
|
1562
|
-
// 8. Make the Converse API request
|
|
1563
1701
|
try {
|
|
1564
1702
|
const response = await performJsonRequest(endpoint, {
|
|
1565
1703
|
headers: requestHeaders,
|
|
1566
|
-
body: converseBody
|
|
1567
|
-
}, "Bedrock");
|
|
1704
|
+
body: converseBody
|
|
1705
|
+
}, "Bedrock");
|
|
1568
1706
|
|
|
1569
1707
|
if (!response.ok) {
|
|
1570
|
-
const errorText = response.text;
|
|
1708
|
+
const errorText = response.text;
|
|
1571
1709
|
logger.error({
|
|
1572
1710
|
status: response.status,
|
|
1573
1711
|
error: errorText
|
|
@@ -1576,7 +1714,7 @@ async function invokeBedrock(body, incomingHeaders = {}) {
|
|
|
1576
1714
|
}
|
|
1577
1715
|
|
|
1578
1716
|
// Parse Converse API response (already parsed by performJsonRequest)
|
|
1579
|
-
const converseResponse = response.json;
|
|
1717
|
+
const converseResponse = response.json;
|
|
1580
1718
|
|
|
1581
1719
|
logger.debug({
|
|
1582
1720
|
stopReason: converseResponse.stopReason,
|
|
@@ -1585,7 +1723,6 @@ async function invokeBedrock(body, incomingHeaders = {}) {
|
|
|
1585
1723
|
hasToolUse: !!converseResponse.output?.message?.content?.some(c => c.toolUse)
|
|
1586
1724
|
}, "=== BEDROCK CONVERSE API RESPONSE ===");
|
|
1587
1725
|
|
|
1588
|
-
// Convert Converse API response to Anthropic format
|
|
1589
1726
|
const message = converseResponse.output.message;
|
|
1590
1727
|
const anthropicResponse = {
|
|
1591
1728
|
id: `bedrock-${Date.now()}`,
|
|
@@ -1671,7 +1808,6 @@ async function invokeZai(body, incomingHeaders = {}) {
|
|
|
1671
1808
|
convertAnthropicMessagesToOpenRouter
|
|
1672
1809
|
} = require("./openrouter-utils");
|
|
1673
1810
|
|
|
1674
|
-
// Convert messages using existing utility
|
|
1675
1811
|
let messages = convertAnthropicMessagesToOpenRouter(body.messages || []);
|
|
1676
1812
|
|
|
1677
1813
|
// Extract system content from body.system OR from system messages in the array
|
|
@@ -1687,7 +1823,6 @@ async function invokeZai(body, incomingHeaders = {}) {
|
|
|
1687
1823
|
const filteredMessages = [];
|
|
1688
1824
|
for (const msg of messages) {
|
|
1689
1825
|
if (msg.role === "system") {
|
|
1690
|
-
// Append system message content to systemContent
|
|
1691
1826
|
if (msg.content) {
|
|
1692
1827
|
systemContent = systemContent ? `${systemContent}\n${msg.content}` : msg.content;
|
|
1693
1828
|
}
|
|
@@ -1714,7 +1849,6 @@ async function invokeZai(body, incomingHeaders = {}) {
|
|
|
1714
1849
|
messages.push({ role: "user", content: systemContent });
|
|
1715
1850
|
}
|
|
1716
1851
|
|
|
1717
|
-
// Convert tools if present
|
|
1718
1852
|
let tools = undefined;
|
|
1719
1853
|
if (Array.isArray(body.tools) && body.tools.length > 0) {
|
|
1720
1854
|
tools = convertAnthropicToolsToOpenRouter(body.tools);
|
|
@@ -1728,14 +1862,12 @@ async function invokeZai(body, incomingHeaders = {}) {
|
|
|
1728
1862
|
stream: body.stream,
|
|
1729
1863
|
};
|
|
1730
1864
|
|
|
1731
|
-
// Only add tools if present
|
|
1732
1865
|
if (tools && tools.length > 0) {
|
|
1733
1866
|
zaiBody.tools = tools;
|
|
1734
1867
|
// Use "auto" to let the model decide when to use tools
|
|
1735
1868
|
// "required" was forcing tools even for simple greetings
|
|
1736
1869
|
zaiBody.tool_choice = "auto";
|
|
1737
|
-
|
|
1738
|
-
zaiBody.parallel_tool_calls = false; // Disable parallel tool calls - GPT often makes duplicate calls
|
|
1870
|
+
zaiBody.parallel_tool_calls = false; // Disabled: duplicate-call risk
|
|
1739
1871
|
}
|
|
1740
1872
|
|
|
1741
1873
|
headers = {
|
|
@@ -1800,7 +1932,6 @@ async function invokeZai(body, incomingHeaders = {}) {
|
|
|
1800
1932
|
isOpenAIFormat,
|
|
1801
1933
|
}, "=== Z.AI RAW RESPONSE ===");
|
|
1802
1934
|
|
|
1803
|
-
// Convert OpenAI response back to Anthropic format if needed
|
|
1804
1935
|
if (isOpenAIFormat && response?.ok && response?.json) {
|
|
1805
1936
|
const anthropicJson = convertOpenAIToAnthropic(response.json);
|
|
1806
1937
|
logger.debug({
|
|
@@ -1861,7 +1992,6 @@ async function invokeMoonshot(body, incomingHeaders = {}) {
|
|
|
1861
1992
|
// Guard against the deprecated auto model arriving via config too.
|
|
1862
1993
|
if (mappedModel === "moonshot-v1-auto") mappedModel = "moonshot-v1-128k";
|
|
1863
1994
|
|
|
1864
|
-
// Convert messages using existing utility
|
|
1865
1995
|
const messages = convertAnthropicMessagesToOpenRouter(body.messages || []);
|
|
1866
1996
|
|
|
1867
1997
|
// Moonshot natively supports system role — add as system message
|
|
@@ -1887,7 +2017,6 @@ async function invokeMoonshot(body, incomingHeaders = {}) {
|
|
|
1887
2017
|
stream: false, // Force non-streaming - OpenAI SSE to Anthropic SSE conversion not implemented
|
|
1888
2018
|
};
|
|
1889
2019
|
|
|
1890
|
-
// Convert and add tools if present
|
|
1891
2020
|
if (Array.isArray(body.tools) && body.tools.length > 0) {
|
|
1892
2021
|
moonshotBody.tools = convertAnthropicToolsToOpenRouter(body.tools);
|
|
1893
2022
|
moonshotBody.tool_choice = "auto";
|
|
@@ -1908,7 +2037,20 @@ async function invokeMoonshot(body, incomingHeaders = {}) {
|
|
|
1908
2037
|
toolCount: moonshotBody.tools?.length || 0,
|
|
1909
2038
|
}, "=== Moonshot REQUEST ===");
|
|
1910
2039
|
|
|
1911
|
-
|
|
2040
|
+
// 429 is NOT retryable here: Moonshot's org-level limits persist for
|
|
2041
|
+
// minutes-hours, so retry backoff holds requests hostage. Fail fast and
|
|
2042
|
+
// let tier-fallback climb instead.
|
|
2043
|
+
const response = await performJsonRequest(endpoint, {
|
|
2044
|
+
headers,
|
|
2045
|
+
body: moonshotBody,
|
|
2046
|
+
retryableStatusesOverride: [500, 502, 503, 504],
|
|
2047
|
+
}, "Moonshot");
|
|
2048
|
+
|
|
2049
|
+
if (!response.ok && response.status === 429) {
|
|
2050
|
+
const err = new Error(`Moonshot rate-limited (org quota): ${String(response.json?.error?.message || '').slice(0, 120)}`);
|
|
2051
|
+
err.status = 429;
|
|
2052
|
+
throw err;
|
|
2053
|
+
}
|
|
1912
2054
|
|
|
1913
2055
|
const rawMsg = response?.json?.choices?.[0]?.message;
|
|
1914
2056
|
logger.debug({
|
|
@@ -1925,7 +2067,6 @@ async function invokeMoonshot(body, incomingHeaders = {}) {
|
|
|
1925
2067
|
fullRawResponse: String(JSON.stringify(response?.json) || '').substring(0, 800),
|
|
1926
2068
|
}, "=== Moonshot RAW RESPONSE ===");
|
|
1927
2069
|
|
|
1928
|
-
// Convert OpenAI response back to Anthropic format
|
|
1929
2070
|
if (response?.ok && response?.json) {
|
|
1930
2071
|
const anthropicJson = convertOpenAIToAnthropic(response.json);
|
|
1931
2072
|
logger.debug({
|
|
@@ -1970,7 +2111,6 @@ function convertOpenAIToAnthropic(response) {
|
|
|
1970
2111
|
}
|
|
1971
2112
|
}
|
|
1972
2113
|
|
|
1973
|
-
// Add text content from message.content
|
|
1974
2114
|
// Don't add placeholder text if there are tool_calls - tools are the actual response
|
|
1975
2115
|
const hasToolCalls = Array.isArray(message.tool_calls) && message.tool_calls.length > 0;
|
|
1976
2116
|
|
|
@@ -1985,11 +2125,8 @@ function convertOpenAIToAnthropic(response) {
|
|
|
1985
2125
|
|
|
1986
2126
|
if (textContent) {
|
|
1987
2127
|
content.push({ type: "text", text: textContent });
|
|
1988
|
-
} else if (!reasoningContent) {
|
|
1989
|
-
// No content and no reasoning — will be handled by the empty check below
|
|
1990
2128
|
}
|
|
1991
2129
|
|
|
1992
|
-
// Convert tool calls
|
|
1993
2130
|
if (Array.isArray(message.tool_calls)) {
|
|
1994
2131
|
for (const toolCall of message.tool_calls) {
|
|
1995
2132
|
content.push({
|
|
@@ -2001,7 +2138,6 @@ function convertOpenAIToAnthropic(response) {
|
|
|
2001
2138
|
}
|
|
2002
2139
|
}
|
|
2003
2140
|
|
|
2004
|
-
// Ensure there's at least some content
|
|
2005
2141
|
if (content.length === 0) {
|
|
2006
2142
|
content.push({ type: "text", text: "" });
|
|
2007
2143
|
}
|
|
@@ -2043,13 +2179,11 @@ function sanitizeSchemaForGemini(schema) {
|
|
|
2043
2179
|
|
|
2044
2180
|
const sanitized = { ...schema };
|
|
2045
2181
|
|
|
2046
|
-
// Remove unsupported properties
|
|
2047
2182
|
delete sanitized.additionalProperties;
|
|
2048
2183
|
delete sanitized.$schema;
|
|
2049
2184
|
delete sanitized.definitions;
|
|
2050
2185
|
delete sanitized.$ref;
|
|
2051
2186
|
|
|
2052
|
-
// Recursively sanitize nested properties
|
|
2053
2187
|
if (sanitized.properties && typeof sanitized.properties === 'object') {
|
|
2054
2188
|
const cleanProps = {};
|
|
2055
2189
|
for (const [key, value] of Object.entries(sanitized.properties)) {
|
|
@@ -2058,12 +2192,10 @@ function sanitizeSchemaForGemini(schema) {
|
|
|
2058
2192
|
sanitized.properties = cleanProps;
|
|
2059
2193
|
}
|
|
2060
2194
|
|
|
2061
|
-
// Sanitize items in arrays
|
|
2062
2195
|
if (sanitized.items) {
|
|
2063
2196
|
sanitized.items = sanitizeSchemaForGemini(sanitized.items);
|
|
2064
2197
|
}
|
|
2065
2198
|
|
|
2066
|
-
// Sanitize anyOf, oneOf, allOf
|
|
2067
2199
|
for (const key of ['anyOf', 'oneOf', 'allOf']) {
|
|
2068
2200
|
if (Array.isArray(sanitized[key])) {
|
|
2069
2201
|
sanitized[key] = sanitized[key].map(item => sanitizeSchemaForGemini(item));
|
|
@@ -2074,9 +2206,10 @@ function sanitizeSchemaForGemini(schema) {
|
|
|
2074
2206
|
}
|
|
2075
2207
|
|
|
2076
2208
|
/**
|
|
2077
|
-
* Vertex AI
|
|
2209
|
+
* Vertex AI Provider - Gemini Models
|
|
2078
2210
|
*
|
|
2079
|
-
*
|
|
2211
|
+
* Despite the name, this calls the Gemini API directly
|
|
2212
|
+
* (generativelanguage.googleapis.com), not Google Cloud Vertex AI.
|
|
2080
2213
|
* Converts Anthropic format to Gemini format and back.
|
|
2081
2214
|
*/
|
|
2082
2215
|
async function invokeVertex(body, incomingHeaders = {}) {
|
|
@@ -2099,17 +2232,13 @@ async function invokeVertex(body, incomingHeaders = {}) {
|
|
|
2099
2232
|
"claude-opus-4-5": "gemini-2.5-pro",
|
|
2100
2233
|
};
|
|
2101
2234
|
|
|
2102
|
-
// Map model name
|
|
2103
2235
|
const requestedModel = body._tierModel || body.model || config.vertex.model;
|
|
2104
2236
|
const geminiModel = modelMap[requestedModel] || config.vertex.model || "gemini-2.0-flash";
|
|
2105
2237
|
|
|
2106
|
-
// Construct Gemini API endpoint
|
|
2107
2238
|
const endpoint = `https://generativelanguage.googleapis.com/v1beta/models/${geminiModel}:generateContent?key=${apiKey}`;
|
|
2108
2239
|
|
|
2109
|
-
// Convert Anthropic messages to Gemini format
|
|
2110
2240
|
const contents = convertAnthropicToGemini(body.messages || [], body.system);
|
|
2111
2241
|
|
|
2112
|
-
// Convert tools to Gemini format
|
|
2113
2242
|
let tools = undefined;
|
|
2114
2243
|
if (Array.isArray(body.tools) && body.tools.length > 0) {
|
|
2115
2244
|
tools = [{
|
|
@@ -2121,7 +2250,6 @@ async function invokeVertex(body, incomingHeaders = {}) {
|
|
|
2121
2250
|
}];
|
|
2122
2251
|
}
|
|
2123
2252
|
|
|
2124
|
-
// Build Gemini request body
|
|
2125
2253
|
const geminiBody = {
|
|
2126
2254
|
contents,
|
|
2127
2255
|
generationConfig: {
|
|
@@ -2131,10 +2259,8 @@ async function invokeVertex(body, incomingHeaders = {}) {
|
|
|
2131
2259
|
}
|
|
2132
2260
|
};
|
|
2133
2261
|
|
|
2134
|
-
// Add tools if present
|
|
2135
2262
|
if (tools) {
|
|
2136
2263
|
geminiBody.tools = tools;
|
|
2137
|
-
// Tell Gemini to use AUTO function calling mode
|
|
2138
2264
|
geminiBody.toolConfig = {
|
|
2139
2265
|
functionCallingConfig: {
|
|
2140
2266
|
mode: "AUTO"
|
|
@@ -2157,7 +2283,6 @@ async function invokeVertex(body, incomingHeaders = {}) {
|
|
|
2157
2283
|
|
|
2158
2284
|
const response = await performJsonRequest(endpoint, { headers, body: geminiBody }, "Vertex AI");
|
|
2159
2285
|
|
|
2160
|
-
// Log error details if request failed
|
|
2161
2286
|
if (!response?.ok) {
|
|
2162
2287
|
logger.error({
|
|
2163
2288
|
status: response?.status,
|
|
@@ -2172,7 +2297,6 @@ async function invokeVertex(body, incomingHeaders = {}) {
|
|
|
2172
2297
|
throw err;
|
|
2173
2298
|
}
|
|
2174
2299
|
|
|
2175
|
-
// Convert Gemini response to Anthropic format
|
|
2176
2300
|
if (response?.json) {
|
|
2177
2301
|
const anthropicJson = convertGeminiToAnthropic(response.json, requestedModel);
|
|
2178
2302
|
logger.debug({
|
|
@@ -2307,12 +2431,10 @@ function convertGeminiToAnthropic(response, requestedModel) {
|
|
|
2307
2431
|
}
|
|
2308
2432
|
}
|
|
2309
2433
|
|
|
2310
|
-
// Ensure at least empty text if no content
|
|
2311
2434
|
if (content.length === 0) {
|
|
2312
2435
|
content.push({ type: "text", text: "" });
|
|
2313
2436
|
}
|
|
2314
2437
|
|
|
2315
|
-
// Determine stop reason
|
|
2316
2438
|
let stopReason = "end_turn";
|
|
2317
2439
|
if (content.some(c => c.type === "tool_use")) {
|
|
2318
2440
|
stopReason = "tool_use";
|
|
@@ -2349,7 +2471,6 @@ async function invokeCodex(body, incomingHeaders = {}) {
|
|
|
2349
2471
|
throw new Error("Codex: no prompt content to send");
|
|
2350
2472
|
}
|
|
2351
2473
|
|
|
2352
|
-
// Start a new thread
|
|
2353
2474
|
const threadParams = { model };
|
|
2354
2475
|
if (systemContext) {
|
|
2355
2476
|
threadParams.instructions = systemContext;
|
|
@@ -2363,7 +2484,6 @@ async function invokeCodex(body, incomingHeaders = {}) {
|
|
|
2363
2484
|
|
|
2364
2485
|
logger.debug({ threadId, model, promptLength: prompt.length }, "[Codex] Thread started");
|
|
2365
2486
|
|
|
2366
|
-
// Send the turn and collect response
|
|
2367
2487
|
const turnResult = await codex.sendTurn(threadId, prompt, model);
|
|
2368
2488
|
|
|
2369
2489
|
logger.debug({
|
|
@@ -2371,7 +2491,6 @@ async function invokeCodex(body, incomingHeaders = {}) {
|
|
|
2371
2491
|
responseLength: turnResult.text?.length || 0,
|
|
2372
2492
|
}, "[Codex] Turn completed");
|
|
2373
2493
|
|
|
2374
|
-
// Convert to Anthropic format
|
|
2375
2494
|
const anthropicJson = convertCodexResponseToAnthropic(turnResult, model);
|
|
2376
2495
|
|
|
2377
2496
|
return {
|
|
@@ -2474,22 +2593,39 @@ function stripLynkrBadges(messages) {
|
|
|
2474
2593
|
const stripped = msg.content.replace(LYNKR_BADGE_PREFIX_RE, '');
|
|
2475
2594
|
mutated = true;
|
|
2476
2595
|
badgeCount++;
|
|
2477
|
-
|
|
2596
|
+
// Badge-only content must not become an empty string — Anthropic
|
|
2597
|
+
// rejects empty assistant content (this is the interrupted-response
|
|
2598
|
+
// continuation case, where the partial text WAS just the badge).
|
|
2599
|
+
return { ...msg, content: stripped.trim() ? stripped : '…' };
|
|
2478
2600
|
}
|
|
2479
2601
|
|
|
2480
2602
|
// Array content variant — Anthropic-format responses keep content as an
|
|
2481
|
-
// array of blocks.
|
|
2603
|
+
// array of blocks. Strip the badge PREFIX from matching blocks rather
|
|
2604
|
+
// than dropping whole blocks: clients that merge text blocks on replay
|
|
2605
|
+
// produce "badge + real answer" in ONE block, and dropping it would
|
|
2606
|
+
// silently delete the model's actual reply from history. A block that
|
|
2607
|
+
// was badge-only disappears entirely.
|
|
2482
2608
|
if (Array.isArray(msg.content)) {
|
|
2483
|
-
|
|
2484
|
-
const
|
|
2485
|
-
|
|
2486
|
-
|
|
2487
|
-
|
|
2609
|
+
let changed = false;
|
|
2610
|
+
const rebuilt = [];
|
|
2611
|
+
for (const b of msg.content) {
|
|
2612
|
+
if (b?.type === 'text' && typeof b.text === 'string' && LYNKR_BADGE_PREFIX_RE.test(b.text)) {
|
|
2613
|
+
changed = true;
|
|
2614
|
+
badgeCount++;
|
|
2615
|
+
const strippedText = b.text.replace(LYNKR_BADGE_PREFIX_RE, '');
|
|
2616
|
+
if (strippedText.trim()) rebuilt.push({ ...b, text: strippedText });
|
|
2617
|
+
// badge-only block → drop
|
|
2618
|
+
} else {
|
|
2619
|
+
rebuilt.push(b);
|
|
2620
|
+
}
|
|
2621
|
+
}
|
|
2622
|
+
if (!changed) return msg;
|
|
2488
2623
|
mutated = true;
|
|
2489
|
-
|
|
2490
|
-
//
|
|
2491
|
-
//
|
|
2492
|
-
|
|
2624
|
+
// Anthropic rejects BOTH empty content[] and empty-string text blocks
|
|
2625
|
+
// (min length 1) — the previous `text: ''` placeholder 400'd upstream
|
|
2626
|
+
// on interrupted-response continuations where the partial assistant
|
|
2627
|
+
// text was just the badge. Use a visible-but-benign ellipsis.
|
|
2628
|
+
return { ...msg, content: rebuilt.length ? rebuilt : [{ type: 'text', text: '…' }] };
|
|
2493
2629
|
}
|
|
2494
2630
|
|
|
2495
2631
|
return msg;
|
|
@@ -2518,7 +2654,32 @@ async function invokeModel(body, options = {}) {
|
|
|
2518
2654
|
const workspace = body._workspace || options.workspace || null;
|
|
2519
2655
|
const tenantPolicy = body._tenantPolicy || options.tenantPolicy || null;
|
|
2520
2656
|
const routingResult = options.forceProvider
|
|
2521
|
-
? {
|
|
2657
|
+
? {
|
|
2658
|
+
// Forced path (OAuth intent + subscription): the actual decision was
|
|
2659
|
+
// made upstream in api/router.js. Reconstitute the shape from
|
|
2660
|
+
// req.body so WS0 telemetry columns (tier, base_tier,
|
|
2661
|
+
// escalation_source, pinned, switch_reason) survive the hop.
|
|
2662
|
+
provider: options.forceProvider,
|
|
2663
|
+
model: body._tierModel ?? null,
|
|
2664
|
+
tier: body._tierName ?? null,
|
|
2665
|
+
method: body._forcedMethod || 'forced',
|
|
2666
|
+
base_tier: body._baseTier ?? null,
|
|
2667
|
+
escalation_source: body._escalationSource ?? null,
|
|
2668
|
+
pinned: body._pinnedRoute ? true : false,
|
|
2669
|
+
switch_reason: body._switchReason ?? null,
|
|
2670
|
+
// WS4 — off-policy evaluation from telemetry alone requires
|
|
2671
|
+
// propensity + candidates on every row. Deterministic default is
|
|
2672
|
+
// 1.0 with a single-entry candidate list matching the served pair.
|
|
2673
|
+
propensity: body._propensity ?? 1.0,
|
|
2674
|
+
candidates: body._candidates ?? [{ provider: options.forceProvider, model: body._tierModel ?? null }],
|
|
2675
|
+
// WS5 — feedback loop. Forward the bandit context vector so
|
|
2676
|
+
// bandit.update fires with the same features the arm was scored on,
|
|
2677
|
+
// and the query embedding so conclusive-quality outcomes grow the
|
|
2678
|
+
// kNN index without paying for a second embedding call.
|
|
2679
|
+
_banditContext: body._banditContext ?? null,
|
|
2680
|
+
_queryEmbedding: body._queryEmbedding ?? null,
|
|
2681
|
+
_queryText: body._queryText ?? null,
|
|
2682
|
+
}
|
|
2522
2683
|
: await determineProviderSmart(body, { workspace, tenantPolicy });
|
|
2523
2684
|
const initialProvider = routingResult.provider;
|
|
2524
2685
|
const tierSelectedModel = routingResult.model;
|
|
@@ -2609,7 +2770,6 @@ async function invokeModel(body, options = {}) {
|
|
|
2609
2770
|
|
|
2610
2771
|
metricsCollector.recordProviderRouting(initialProvider);
|
|
2611
2772
|
|
|
2612
|
-
// Get circuit breaker for initial provider
|
|
2613
2773
|
const breaker = registry.get(initialProvider, {
|
|
2614
2774
|
failureThreshold: 5,
|
|
2615
2775
|
successThreshold: 2,
|
|
@@ -2619,11 +2779,9 @@ async function invokeModel(body, options = {}) {
|
|
|
2619
2779
|
let retries = 0;
|
|
2620
2780
|
const startTime = Date.now();
|
|
2621
2781
|
|
|
2622
|
-
// Record request start for health tracking
|
|
2623
2782
|
healthTracker.recordRequestStart(initialProvider);
|
|
2624
2783
|
|
|
2625
2784
|
try {
|
|
2626
|
-
// Try initial provider with circuit breaker
|
|
2627
2785
|
const result = await breaker.execute(async () => {
|
|
2628
2786
|
if (initialProvider === "azure-openai") {
|
|
2629
2787
|
return await invokeAzureOpenAI(body, incomingHeaders);
|
|
@@ -2633,6 +2791,8 @@ async function invokeModel(body, options = {}) {
|
|
|
2633
2791
|
return await invokeOllama(body, incomingHeaders);
|
|
2634
2792
|
} else if (initialProvider === "openrouter") {
|
|
2635
2793
|
return await invokeOpenRouter(body, incomingHeaders);
|
|
2794
|
+
} else if (initialProvider === "edenai") {
|
|
2795
|
+
return await invokeEdenAI(body, incomingHeaders);
|
|
2636
2796
|
} else if (initialProvider === "openai") {
|
|
2637
2797
|
return await invokeOpenAI(body, incomingHeaders);
|
|
2638
2798
|
} else if (initialProvider === "llamacpp") {
|
|
@@ -2653,7 +2813,6 @@ async function invokeModel(body, options = {}) {
|
|
|
2653
2813
|
return await invokeDatabricks(body, incomingHeaders);
|
|
2654
2814
|
});
|
|
2655
2815
|
|
|
2656
|
-
// Record success metrics
|
|
2657
2816
|
const latency = Date.now() - startTime;
|
|
2658
2817
|
metricsCollector.recordProviderSuccess(initialProvider, latency);
|
|
2659
2818
|
metricsCollector.recordDatabricksRequest(true, retries);
|
|
@@ -2662,26 +2821,22 @@ async function invokeModel(body, options = {}) {
|
|
|
2662
2821
|
// Record latency for routing intelligence
|
|
2663
2822
|
getLatencyTracker().record(initialProvider, latency);
|
|
2664
2823
|
|
|
2665
|
-
// Record tokens and cost savings
|
|
2666
2824
|
const outputTokens = result.json?.usage?.output_tokens || result.json?.usage?.completion_tokens || 0;
|
|
2667
2825
|
const inputTokens = result.json?.usage?.input_tokens || result.json?.usage?.prompt_tokens || 0;
|
|
2668
2826
|
if (result.json?.usage) {
|
|
2669
2827
|
metricsCollector.recordTokens(inputTokens, outputTokens);
|
|
2670
2828
|
|
|
2671
|
-
// Estimate cost savings if Ollama was used
|
|
2672
2829
|
if (initialProvider === "ollama") {
|
|
2673
2830
|
const savings = estimateCostSavings(inputTokens, outputTokens);
|
|
2674
2831
|
metricsCollector.recordCostSavings(savings);
|
|
2675
2832
|
}
|
|
2676
2833
|
}
|
|
2677
2834
|
|
|
2678
|
-
// Count tool calls in response
|
|
2679
2835
|
const toolCallsMade = result.json?.content?.filter?.(
|
|
2680
2836
|
(b) => b.type === "tool_use"
|
|
2681
2837
|
)?.length || 0;
|
|
2682
2838
|
|
|
2683
|
-
|
|
2684
|
-
const qualityScore = scoreResponseQuality(
|
|
2839
|
+
let qualityScore = scoreResponseQuality(
|
|
2685
2840
|
{ tier: routingDecision.tier, hasTools: Array.isArray(body?.tools) && body.tools.length > 0 },
|
|
2686
2841
|
null,
|
|
2687
2842
|
{
|
|
@@ -2695,6 +2850,25 @@ async function invokeModel(body, options = {}) {
|
|
|
2695
2850
|
}
|
|
2696
2851
|
);
|
|
2697
2852
|
|
|
2853
|
+
// WS6 — cascade verification. Only cheap-tier answers are judged, only
|
|
2854
|
+
// on top-level calls (never re-verify an escalated attempt), only when
|
|
2855
|
+
// opted in. A verify-fail overrides the heuristic quality score so the
|
|
2856
|
+
// telemetry row and the WS5 feedback loop record the attempt as the
|
|
2857
|
+
// hard negative it is (kNN learns "questions like this outgrow the
|
|
2858
|
+
// cheap tier"), then the request escalates below.
|
|
2859
|
+
let cascadeVerify = null;
|
|
2860
|
+
if (
|
|
2861
|
+
process.env.LYNKR_CASCADE_VERIFY === 'true' &&
|
|
2862
|
+
!options._cascadeVerifyInner &&
|
|
2863
|
+
(routingDecision.tier === 'SIMPLE' || routingDecision.tier === 'MEDIUM')
|
|
2864
|
+
) {
|
|
2865
|
+
const { verify } = require('../routing/verifier');
|
|
2866
|
+
cascadeVerify = verify({ payload: body, responseBody: result.json });
|
|
2867
|
+
if (cascadeVerify.verdict === 'fail') {
|
|
2868
|
+
qualityScore = Math.min(qualityScore ?? 100, 20);
|
|
2869
|
+
}
|
|
2870
|
+
}
|
|
2871
|
+
|
|
2698
2872
|
// Record routing telemetry (non-blocking)
|
|
2699
2873
|
telemetry.record({
|
|
2700
2874
|
request_id: crypto.randomUUID(),
|
|
@@ -2708,7 +2882,7 @@ async function invokeModel(body, options = {}) {
|
|
|
2708
2882
|
message_count: Array.isArray(body?.messages) ? body.messages.length : 0,
|
|
2709
2883
|
request_type: routingResult.analysis?.requestType || null,
|
|
2710
2884
|
provider: initialProvider,
|
|
2711
|
-
model: routingDecision.model,
|
|
2885
|
+
model: routingDecision.model ?? body._tierModel ?? null,
|
|
2712
2886
|
routing_method: routingDecision.method,
|
|
2713
2887
|
was_fallback: false,
|
|
2714
2888
|
output_tokens: outputTokens || null,
|
|
@@ -2723,9 +2897,81 @@ async function invokeModel(body, options = {}) {
|
|
|
2723
2897
|
cost_usd: computeCostUsd(routingDecision.model || body._tierModel, inputTokens, outputTokens),
|
|
2724
2898
|
request_text: captureRequestText(body),
|
|
2725
2899
|
response_text: captureResponseText(result.json),
|
|
2900
|
+
base_tier: routingResult.base_tier ?? null,
|
|
2901
|
+
escalation_source: routingResult.escalation_source ?? null,
|
|
2902
|
+
propensity: routingResult.propensity ?? null,
|
|
2903
|
+
candidates: routingResult.candidates ?? null,
|
|
2904
|
+
pinned: routingResult.pinned ? 1 : 0,
|
|
2905
|
+
switch_reason: routingResult.switch_reason ?? null,
|
|
2726
2906
|
});
|
|
2727
2907
|
|
|
2728
|
-
//
|
|
2908
|
+
// WS5.4 — feedback loop (success path).
|
|
2909
|
+
recordFeedbackOutcome({
|
|
2910
|
+
routingResult,
|
|
2911
|
+
body,
|
|
2912
|
+
outcome: {
|
|
2913
|
+
qualityScore,
|
|
2914
|
+
costUsd: computeCostUsd(routingDecision.model || body._tierModel, inputTokens, outputTokens),
|
|
2915
|
+
latencyMs: latency,
|
|
2916
|
+
statusCode: 200,
|
|
2917
|
+
errorType: null,
|
|
2918
|
+
wasFallback: false,
|
|
2919
|
+
},
|
|
2920
|
+
});
|
|
2921
|
+
|
|
2922
|
+
// WS6 — cascade escalation: the cheap answer failed verification, so
|
|
2923
|
+
// discard it and climb the tier ladder (upward candidates only). If
|
|
2924
|
+
// every escalation attempt fails, serve the original cheap answer —
|
|
2925
|
+
// the cascade must never make things worse than no cascade.
|
|
2926
|
+
if (cascadeVerify?.verdict === 'fail') {
|
|
2927
|
+
logger.info({
|
|
2928
|
+
tier: routingDecision.tier,
|
|
2929
|
+
model: routingDecision.model,
|
|
2930
|
+
reasons: cascadeVerify.reasons,
|
|
2931
|
+
}, '[Cascade] Cheap-tier answer failed verification — escalating');
|
|
2932
|
+
try {
|
|
2933
|
+
const { getFallbackChain } = require('../routing/tier-fallback');
|
|
2934
|
+
const upward = getFallbackChain(routingDecision.tier).filter((c) => c.direction === 'up');
|
|
2935
|
+
for (const cand of upward) {
|
|
2936
|
+
try {
|
|
2937
|
+
const attempt = await invokeModel(
|
|
2938
|
+
{ ...body, _tierModel: cand.model },
|
|
2939
|
+
{
|
|
2940
|
+
forceProvider: cand.provider,
|
|
2941
|
+
_cascadeVerifyInner: true,
|
|
2942
|
+
_cascadeInner: true,
|
|
2943
|
+
disableFallback: true,
|
|
2944
|
+
workspace,
|
|
2945
|
+
tenantPolicy,
|
|
2946
|
+
headers: incomingHeaders,
|
|
2947
|
+
}
|
|
2948
|
+
);
|
|
2949
|
+
logger.info({
|
|
2950
|
+
from: `${routingDecision.tier}:${routingDecision.model}`,
|
|
2951
|
+
to: `${cand.tier}:${cand.model}`,
|
|
2952
|
+
}, '[Cascade] Escalated answer served');
|
|
2953
|
+
return {
|
|
2954
|
+
...attempt,
|
|
2955
|
+
actualProvider: cand.provider,
|
|
2956
|
+
routingDecision: {
|
|
2957
|
+
...routingDecision,
|
|
2958
|
+
provider: cand.provider,
|
|
2959
|
+
model: cand.model,
|
|
2960
|
+
servedTier: cand.tier,
|
|
2961
|
+
method: (routingDecision.method || '') + '+verify_escalated',
|
|
2962
|
+
cascade: { failedTier: routingDecision.tier, reasons: cascadeVerify.reasons },
|
|
2963
|
+
},
|
|
2964
|
+
};
|
|
2965
|
+
} catch (innerErr) {
|
|
2966
|
+
logger.warn({ to: cand.provider, error: innerErr.message }, '[Cascade] Escalation candidate failed, trying next');
|
|
2967
|
+
}
|
|
2968
|
+
}
|
|
2969
|
+
} catch (err) {
|
|
2970
|
+
logger.debug({ err: err.message }, '[Cascade] Escalation setup failed — serving original answer');
|
|
2971
|
+
}
|
|
2972
|
+
// All escalations failed → fall through to the original cheap answer.
|
|
2973
|
+
}
|
|
2974
|
+
|
|
2729
2975
|
return {
|
|
2730
2976
|
...result,
|
|
2731
2977
|
actualProvider: initialProvider,
|
|
@@ -2733,7 +2979,6 @@ async function invokeModel(body, options = {}) {
|
|
|
2733
2979
|
};
|
|
2734
2980
|
|
|
2735
2981
|
} catch (err) {
|
|
2736
|
-
// Record failure
|
|
2737
2982
|
const failLatency = Date.now() - startTime;
|
|
2738
2983
|
metricsCollector.recordProviderFailure(initialProvider);
|
|
2739
2984
|
healthTracker.recordFailure(initialProvider, err, err.status);
|
|
@@ -2813,7 +3058,12 @@ async function invokeModel(body, options = {}) {
|
|
|
2813
3058
|
if (!shouldFallback) {
|
|
2814
3059
|
metricsCollector.recordDatabricksRequest(false, retries);
|
|
2815
3060
|
|
|
2816
|
-
|
|
3061
|
+
const failQualityScore = scoreResponseQuality(
|
|
3062
|
+
{ tier: routingDecision.tier, hasTools: Array.isArray(body?.tools) && body.tools.length > 0 },
|
|
3063
|
+
null,
|
|
3064
|
+
{ error_type: err.code || err.name, was_fallback: false, retry_count: retries, latency_ms: failLatency }
|
|
3065
|
+
);
|
|
3066
|
+
|
|
2817
3067
|
telemetry.record({
|
|
2818
3068
|
request_id: crypto.randomUUID(),
|
|
2819
3069
|
session_id: body._sessionId || null,
|
|
@@ -2826,23 +3076,41 @@ async function invokeModel(body, options = {}) {
|
|
|
2826
3076
|
message_count: Array.isArray(body?.messages) ? body.messages.length : 0,
|
|
2827
3077
|
request_type: routingResult.analysis?.requestType || null,
|
|
2828
3078
|
provider: initialProvider,
|
|
2829
|
-
model: routingDecision.model,
|
|
3079
|
+
model: routingDecision.model ?? body._tierModel ?? null,
|
|
2830
3080
|
routing_method: routingDecision.method,
|
|
2831
3081
|
was_fallback: false,
|
|
2832
3082
|
latency_ms: failLatency,
|
|
2833
3083
|
status_code: err.status || null,
|
|
2834
3084
|
error_type: err.code || err.name || "unknown",
|
|
2835
|
-
quality_score:
|
|
2836
|
-
|
|
2837
|
-
|
|
2838
|
-
|
|
2839
|
-
|
|
3085
|
+
quality_score: failQualityScore,
|
|
3086
|
+
base_tier: routingResult.base_tier ?? null,
|
|
3087
|
+
escalation_source: routingResult.escalation_source ?? null,
|
|
3088
|
+
propensity: routingResult.propensity ?? null,
|
|
3089
|
+
candidates: routingResult.candidates ?? null,
|
|
3090
|
+
pinned: routingResult.pinned ? 1 : 0,
|
|
3091
|
+
switch_reason: routingResult.switch_reason ?? null,
|
|
3092
|
+
});
|
|
3093
|
+
|
|
3094
|
+
// WS5.4 — feedback loop (primary-failed, no fallback). Low quality
|
|
3095
|
+
// scores are signal too: the bandit's reward drops, and if the
|
|
3096
|
+
// outcome is conclusive-negative (≤40) the kNN index learns to
|
|
3097
|
+
// steer away from this (query → model) pairing.
|
|
3098
|
+
recordFeedbackOutcome({
|
|
3099
|
+
routingResult,
|
|
3100
|
+
body,
|
|
3101
|
+
outcome: {
|
|
3102
|
+
qualityScore: failQualityScore,
|
|
3103
|
+
costUsd: null,
|
|
3104
|
+
latencyMs: failLatency,
|
|
3105
|
+
statusCode: err.status || null,
|
|
3106
|
+
errorType: err.code || err.name || "unknown",
|
|
3107
|
+
wasFallback: false,
|
|
3108
|
+
},
|
|
2840
3109
|
});
|
|
2841
3110
|
|
|
2842
3111
|
throw err;
|
|
2843
3112
|
}
|
|
2844
3113
|
|
|
2845
|
-
// Determine failure reason
|
|
2846
3114
|
const reason = categorizeFailure(err);
|
|
2847
3115
|
const fallbackProvider = getFallbackProvider();
|
|
2848
3116
|
|
|
@@ -2855,11 +3123,9 @@ async function invokeModel(body, options = {}) {
|
|
|
2855
3123
|
|
|
2856
3124
|
metricsCollector.recordFallbackAttempt(initialProvider, fallbackProvider, reason);
|
|
2857
3125
|
|
|
2858
|
-
// Record fallback request start for health tracking
|
|
2859
3126
|
healthTracker.recordRequestStart(fallbackProvider);
|
|
2860
3127
|
|
|
2861
3128
|
try {
|
|
2862
|
-
// Get circuit breaker for fallback provider
|
|
2863
3129
|
const fallbackBreaker = registry.get(fallbackProvider, {
|
|
2864
3130
|
failureThreshold: 5,
|
|
2865
3131
|
successThreshold: 2,
|
|
@@ -2868,7 +3134,6 @@ async function invokeModel(body, options = {}) {
|
|
|
2868
3134
|
|
|
2869
3135
|
const fallbackStart = Date.now();
|
|
2870
3136
|
|
|
2871
|
-
// Execute fallback
|
|
2872
3137
|
const fallbackResult = await fallbackBreaker.execute(async () => {
|
|
2873
3138
|
if (fallbackProvider === "azure-openai") {
|
|
2874
3139
|
return await invokeAzureOpenAI(body, incomingHeaders);
|
|
@@ -2876,6 +3141,8 @@ async function invokeModel(body, options = {}) {
|
|
|
2876
3141
|
return await invokeAzureAnthropic(body, incomingHeaders);
|
|
2877
3142
|
} else if (fallbackProvider === "openrouter") {
|
|
2878
3143
|
return await invokeOpenRouter(body, incomingHeaders);
|
|
3144
|
+
} else if (fallbackProvider === "edenai") {
|
|
3145
|
+
return await invokeEdenAI(body, incomingHeaders);
|
|
2879
3146
|
} else if (fallbackProvider === "openai") {
|
|
2880
3147
|
return await invokeOpenAI(body, incomingHeaders);
|
|
2881
3148
|
} else if (fallbackProvider === "llamacpp") {
|
|
@@ -2892,12 +3159,10 @@ async function invokeModel(body, options = {}) {
|
|
|
2892
3159
|
|
|
2893
3160
|
const fallbackLatency = Date.now() - fallbackStart;
|
|
2894
3161
|
|
|
2895
|
-
// Record fallback success
|
|
2896
3162
|
metricsCollector.recordFallbackSuccess(fallbackLatency);
|
|
2897
3163
|
metricsCollector.recordDatabricksRequest(true, retries);
|
|
2898
3164
|
healthTracker.recordSuccess(fallbackProvider, fallbackLatency);
|
|
2899
3165
|
|
|
2900
|
-
// Record token usage
|
|
2901
3166
|
if (fallbackResult.json?.usage) {
|
|
2902
3167
|
metricsCollector.recordTokens(
|
|
2903
3168
|
fallbackResult.json.usage.input_tokens || fallbackResult.json.usage.prompt_tokens || 0,
|
|
@@ -2912,16 +3177,22 @@ async function invokeModel(body, options = {}) {
|
|
|
2912
3177
|
totalLatency: Date.now() - startTime,
|
|
2913
3178
|
}, "Fallback to cloud provider succeeded");
|
|
2914
3179
|
|
|
2915
|
-
// Record latency for fallback provider
|
|
2916
3180
|
getLatencyTracker().record(fallbackProvider, routingDecision?.model, fallbackLatency);
|
|
2917
3181
|
|
|
2918
|
-
// Capture fallback telemetry
|
|
2919
3182
|
const fbOutputTokens = fallbackResult.json?.usage?.output_tokens || fallbackResult.json?.usage?.completion_tokens || 0;
|
|
2920
3183
|
const fbInputTokens = fallbackResult.json?.usage?.input_tokens || fallbackResult.json?.usage?.prompt_tokens || 0;
|
|
2921
3184
|
const fbToolCalls = fallbackResult.json?.content?.filter?.(
|
|
2922
3185
|
(b) => b.type === "tool_use"
|
|
2923
3186
|
)?.length || 0;
|
|
2924
3187
|
|
|
3188
|
+
const fbTotalLatency = Date.now() - startTime;
|
|
3189
|
+
const fbQualityScore = scoreResponseQuality(
|
|
3190
|
+
{ tier: routingDecision.tier, hasTools: Array.isArray(body?.tools) && body.tools.length > 0 },
|
|
3191
|
+
null,
|
|
3192
|
+
{ status_code: 200, output_tokens: fbOutputTokens, tool_calls_made: fbToolCalls, was_fallback: true, retry_count: 0, latency_ms: fbTotalLatency }
|
|
3193
|
+
);
|
|
3194
|
+
const fbCostUsd = computeCostUsd(routingDecision.model || body._tierModel, fbInputTokens, fbOutputTokens);
|
|
3195
|
+
|
|
2925
3196
|
telemetry.record({
|
|
2926
3197
|
request_id: crypto.randomUUID(),
|
|
2927
3198
|
session_id: body._sessionId || null,
|
|
@@ -2934,27 +3205,45 @@ async function invokeModel(body, options = {}) {
|
|
|
2934
3205
|
message_count: Array.isArray(body?.messages) ? body.messages.length : 0,
|
|
2935
3206
|
request_type: routingResult.analysis?.requestType || null,
|
|
2936
3207
|
provider: fallbackProvider,
|
|
2937
|
-
model: routingDecision.model,
|
|
3208
|
+
model: routingDecision.model ?? body._tierModel ?? null,
|
|
2938
3209
|
routing_method: "fallback",
|
|
2939
3210
|
was_fallback: true,
|
|
2940
3211
|
output_tokens: fbOutputTokens || null,
|
|
2941
|
-
latency_ms:
|
|
3212
|
+
latency_ms: fbTotalLatency,
|
|
2942
3213
|
status_code: 200,
|
|
2943
3214
|
error_type: null,
|
|
2944
3215
|
tool_calls_made: fbToolCalls,
|
|
2945
3216
|
retry_count: 0,
|
|
2946
|
-
quality_score:
|
|
2947
|
-
{ tier: routingDecision.tier, hasTools: Array.isArray(body?.tools) && body.tools.length > 0 },
|
|
2948
|
-
null,
|
|
2949
|
-
{ status_code: 200, output_tokens: fbOutputTokens, tool_calls_made: fbToolCalls, was_fallback: true, retry_count: 0, latency_ms: Date.now() - startTime }
|
|
2950
|
-
),
|
|
3217
|
+
quality_score: fbQualityScore,
|
|
2951
3218
|
tokens_per_second: fbOutputTokens && fallbackLatency > 0 ? fbOutputTokens / (fallbackLatency / 1000) : null,
|
|
2952
|
-
cost_usd:
|
|
3219
|
+
cost_usd: fbCostUsd,
|
|
2953
3220
|
request_text: captureRequestText(body),
|
|
2954
3221
|
response_text: captureResponseText(fallbackResult.json),
|
|
3222
|
+
base_tier: routingResult.base_tier ?? null,
|
|
3223
|
+
escalation_source: routingResult.escalation_source ?? null,
|
|
3224
|
+
propensity: routingResult.propensity ?? null,
|
|
3225
|
+
candidates: routingResult.candidates ?? null,
|
|
3226
|
+
pinned: routingResult.pinned ? 1 : 0,
|
|
3227
|
+
switch_reason: routingResult.switch_reason ?? null,
|
|
3228
|
+
});
|
|
3229
|
+
|
|
3230
|
+
// WS5.4 — feedback loop (fallback success). The served provider
|
|
3231
|
+
// isn't the one the router picked, so the bandit reward records the
|
|
3232
|
+
// fallback outcome under the original arm — which is the honest
|
|
3233
|
+
// signal for future picks of that arm.
|
|
3234
|
+
recordFeedbackOutcome({
|
|
3235
|
+
routingResult,
|
|
3236
|
+
body,
|
|
3237
|
+
outcome: {
|
|
3238
|
+
qualityScore: fbQualityScore,
|
|
3239
|
+
costUsd: fbCostUsd,
|
|
3240
|
+
latencyMs: fbTotalLatency,
|
|
3241
|
+
statusCode: 200,
|
|
3242
|
+
errorType: null,
|
|
3243
|
+
wasFallback: true,
|
|
3244
|
+
},
|
|
2955
3245
|
});
|
|
2956
3246
|
|
|
2957
|
-
// Return result with actual provider used (fallback provider) and routing decision
|
|
2958
3247
|
return {
|
|
2959
3248
|
...fallbackResult,
|
|
2960
3249
|
actualProvider: fallbackProvider,
|
|
@@ -2967,12 +3256,12 @@ async function invokeModel(body, options = {}) {
|
|
|
2967
3256
|
};
|
|
2968
3257
|
|
|
2969
3258
|
} catch (fallbackErr) {
|
|
2970
|
-
// Both providers failed
|
|
2971
3259
|
metricsCollector.recordFallbackFailure();
|
|
2972
3260
|
metricsCollector.recordDatabricksRequest(false, retries);
|
|
2973
3261
|
healthTracker.recordFailure(fallbackProvider, fallbackErr, fallbackErr.status);
|
|
2974
3262
|
|
|
2975
|
-
|
|
3263
|
+
const dfLatencyMs = Date.now() - startTime;
|
|
3264
|
+
|
|
2976
3265
|
telemetry.record({
|
|
2977
3266
|
request_id: crypto.randomUUID(),
|
|
2978
3267
|
session_id: body._sessionId || null,
|
|
@@ -2980,13 +3269,35 @@ async function invokeModel(body, options = {}) {
|
|
|
2980
3269
|
complexity_score: routingResult.score ?? null,
|
|
2981
3270
|
tier: routingDecision.tier,
|
|
2982
3271
|
provider: fallbackProvider,
|
|
2983
|
-
model: routingDecision.model,
|
|
3272
|
+
model: routingDecision.model ?? body._tierModel ?? null,
|
|
2984
3273
|
routing_method: "fallback",
|
|
2985
3274
|
was_fallback: true,
|
|
2986
|
-
latency_ms:
|
|
3275
|
+
latency_ms: dfLatencyMs,
|
|
2987
3276
|
status_code: fallbackErr.status || null,
|
|
2988
3277
|
error_type: fallbackErr.code || fallbackErr.name || "double_failure",
|
|
2989
3278
|
quality_score: 0,
|
|
3279
|
+
base_tier: routingResult.base_tier ?? null,
|
|
3280
|
+
escalation_source: routingResult.escalation_source ?? null,
|
|
3281
|
+
propensity: routingResult.propensity ?? null,
|
|
3282
|
+
candidates: routingResult.candidates ?? null,
|
|
3283
|
+
pinned: routingResult.pinned ? 1 : 0,
|
|
3284
|
+
switch_reason: routingResult.switch_reason ?? null,
|
|
3285
|
+
});
|
|
3286
|
+
|
|
3287
|
+
// WS5.4 — feedback loop (double failure). quality=0 is a hard
|
|
3288
|
+
// negative exemplar; the kNN index will learn to steer away from
|
|
3289
|
+
// this arm for similar queries.
|
|
3290
|
+
recordFeedbackOutcome({
|
|
3291
|
+
routingResult,
|
|
3292
|
+
body,
|
|
3293
|
+
outcome: {
|
|
3294
|
+
qualityScore: 0,
|
|
3295
|
+
costUsd: null,
|
|
3296
|
+
latencyMs: dfLatencyMs,
|
|
3297
|
+
statusCode: fallbackErr.status || null,
|
|
3298
|
+
errorType: fallbackErr.code || fallbackErr.name || "double_failure",
|
|
3299
|
+
wasFallback: true,
|
|
3300
|
+
},
|
|
2990
3301
|
});
|
|
2991
3302
|
|
|
2992
3303
|
logger.error({
|
|
@@ -3062,4 +3373,6 @@ module.exports = {
|
|
|
3062
3373
|
stripLynkrBadges,
|
|
3063
3374
|
destroyHttpAgents,
|
|
3064
3375
|
normalizeBodyForConverse,
|
|
3376
|
+
_stripInternalFields,
|
|
3377
|
+
_throwIfOllamaError,
|
|
3065
3378
|
};
|