@agentproto/llm-endpoint 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +202 -0
- package/README.md +574 -0
- package/dist/cli.mjs +3738 -0
- package/dist/cli.mjs.map +1 -0
- package/dist/index.d.ts +427 -0
- package/dist/index.mjs +3714 -0
- package/dist/index.mjs.map +1 -0
- package/package.json +69 -0
package/dist/cli.mjs
ADDED
|
@@ -0,0 +1,3738 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
import { injectProviderKeysIntoEnv } from '@agentproto/providers-store';
|
|
3
|
+
import { createServer, request } from 'http';
|
|
4
|
+
import { request as request$1 } from 'https';
|
|
5
|
+
import { readFileSync } from 'fs';
|
|
6
|
+
import path, { resolve } from 'path';
|
|
7
|
+
import { fileURLToPath } from 'url';
|
|
8
|
+
import { randomBytes, createHash } from 'crypto';
|
|
9
|
+
import { getAuthProfile, KeychainStore } from '@agentproto/auth';
|
|
10
|
+
import { mkdir, writeFile, rename, rm, readFile, readdir } from 'fs/promises';
|
|
11
|
+
import { homedir } from 'os';
|
|
12
|
+
import { z } from 'zod';
|
|
13
|
+
import { batchHandleSchema, anthropicMessageSchema, messagesBodySchema, assertUniqueCustomIds, BatchValidationError, validateForBatch, anthropicBatchDriver, openrouterBatchDriver, newBatchId, BatchUnsupportedError, BatchStore, ulid, localQueueDriver, RetryableCompletionError } from '@agentproto/batch';
|
|
14
|
+
|
|
15
|
+
/**
|
|
16
|
+
* @agentproto/llm-endpoint v0.1.0-alpha
|
|
17
|
+
* Anthropic-Messages-compatible proxy gateway. One Claude API surface,
|
|
18
|
+
* many upstream providers (Moonshot / OpenRouter / ZAI / Groq) behind
|
|
19
|
+
* stable alias codenames. Schema translation, tool-cap trimming,
|
|
20
|
+
* orphaned-tool-call repair, thinking-block stripping.
|
|
21
|
+
*/
|
|
22
|
+
|
|
23
|
+
var defaultPack = {
|
|
24
|
+
id: "default",
|
|
25
|
+
label: "Default transparent routes",
|
|
26
|
+
description: "Provider-transparent model IDs routed directly to each backend",
|
|
27
|
+
models: {
|
|
28
|
+
"kimi-k3": { provider: "moonshot", model: "kimi-k3" },
|
|
29
|
+
"kimi-k2.7-code": { provider: "moonshot", model: "kimi-k2.7-code" },
|
|
30
|
+
"kimi-k2.6": { provider: "moonshot", model: "kimi-k2.6" },
|
|
31
|
+
"llama-3.3-70b-versatile": { provider: "groq", model: "llama-3.3-70b-versatile" },
|
|
32
|
+
"qwen/qwen3.6-27b": { provider: "groq", model: "qwen/qwen3.6-27b" },
|
|
33
|
+
"glm-5.2": { provider: "zai", model: "glm-5.2" },
|
|
34
|
+
"gpt-4.1": { provider: "openai", model: "gpt-4.1" },
|
|
35
|
+
"gpt-4o": { provider: "openai", model: "gpt-4o" },
|
|
36
|
+
"gpt-4o-mini": { provider: "openai", model: "gpt-4o-mini" },
|
|
37
|
+
// OpenRouter-hosted GLM. Also reachable via the transparent
|
|
38
|
+
// "openrouter/z-ai/glm-5.3-flash" reference, but a plain code is needed
|
|
39
|
+
// for the `@llm-endpoint` daemon route, which forwards a bare
|
|
40
|
+
// "<vendor>/<model>" string (one slash only) as the model field.
|
|
41
|
+
"z-ai/glm-5.3-flash": { provider: "openrouter", model: "z-ai/glm-5.3-flash" }
|
|
42
|
+
}
|
|
43
|
+
};
|
|
44
|
+
var xaiPack = {
|
|
45
|
+
id: "xai",
|
|
46
|
+
label: "xAI (Grok)",
|
|
47
|
+
description: "xAI Grok models via the OpenAI-compatible API",
|
|
48
|
+
models: {
|
|
49
|
+
"grok-4.5": { provider: "xai", model: "grok-4.5" },
|
|
50
|
+
"grok-4.3": { provider: "xai", model: "grok-4.3" },
|
|
51
|
+
"grok-4.20": { provider: "xai", model: "grok-4.20" },
|
|
52
|
+
"grok-build-0.1": { provider: "xai", model: "grok-build-0.1" }
|
|
53
|
+
}
|
|
54
|
+
};
|
|
55
|
+
var openaiPack = {
|
|
56
|
+
id: "openai",
|
|
57
|
+
label: "OpenAI",
|
|
58
|
+
description: "OpenAI models via the OpenAI-compatible API \u2014 synced from OpenRouter catalog",
|
|
59
|
+
models: {
|
|
60
|
+
// ── GPT-5.6 series ──
|
|
61
|
+
"gpt-5.6-luna": { provider: "openai", model: "gpt-5.6-luna" },
|
|
62
|
+
"gpt-5.6-luna-pro": { provider: "openai", model: "gpt-5.6-luna-pro" },
|
|
63
|
+
"gpt-5.6-terra": { provider: "openai", model: "gpt-5.6-terra" },
|
|
64
|
+
"gpt-5.6-terra-pro": { provider: "openai", model: "gpt-5.6-terra-pro" },
|
|
65
|
+
"gpt-5.6-sol": { provider: "openai", model: "gpt-5.6-sol" },
|
|
66
|
+
"gpt-5.6-sol-pro": { provider: "openai", model: "gpt-5.6-sol-pro" },
|
|
67
|
+
// ── GPT-5.5 ──
|
|
68
|
+
"gpt-5.5": { provider: "openai", model: "gpt-5.5" },
|
|
69
|
+
"gpt-5.5-pro": { provider: "openai", model: "gpt-5.5-pro" },
|
|
70
|
+
// ── GPT-5.4 ──
|
|
71
|
+
"gpt-5.4": { provider: "openai", model: "gpt-5.4" },
|
|
72
|
+
"gpt-5.4-mini": { provider: "openai", model: "gpt-5.4-mini" },
|
|
73
|
+
"gpt-5.4-nano": { provider: "openai", model: "gpt-5.4-nano" },
|
|
74
|
+
"gpt-5.4-pro": { provider: "openai", model: "gpt-5.4-pro" },
|
|
75
|
+
"gpt-5.4-image-2": { provider: "openai", model: "gpt-5.4-image-2" },
|
|
76
|
+
// ── GPT-5.3 ──
|
|
77
|
+
"gpt-5.3-chat": { provider: "openai", model: "gpt-5.3-chat" },
|
|
78
|
+
"gpt-5.3-codex": { provider: "openai", model: "gpt-5.3-codex" },
|
|
79
|
+
// ── GPT-5.2 ──
|
|
80
|
+
"gpt-5.2": { provider: "openai", model: "gpt-5.2" },
|
|
81
|
+
"gpt-5.2-pro": { provider: "openai", model: "gpt-5.2-pro" },
|
|
82
|
+
"gpt-5.2-chat": { provider: "openai", model: "gpt-5.2-chat" },
|
|
83
|
+
"gpt-5.2-codex": { provider: "openai", model: "gpt-5.2-codex" },
|
|
84
|
+
// ── GPT-5.1 ──
|
|
85
|
+
"gpt-5.1": { provider: "openai", model: "gpt-5.1" },
|
|
86
|
+
"gpt-5.1-chat": { provider: "openai", model: "gpt-5.1-chat" },
|
|
87
|
+
"gpt-5.1-codex": { provider: "openai", model: "gpt-5.1-codex" },
|
|
88
|
+
"gpt-5.1-codex-mini": { provider: "openai", model: "gpt-5.1-codex-mini" },
|
|
89
|
+
"gpt-5.1-codex-max": { provider: "openai", model: "gpt-5.1-codex-max" },
|
|
90
|
+
// ── GPT-5 ──
|
|
91
|
+
"gpt-5": { provider: "openai", model: "gpt-5" },
|
|
92
|
+
"gpt-5-pro": { provider: "openai", model: "gpt-5-pro" },
|
|
93
|
+
"gpt-5-mini": { provider: "openai", model: "gpt-5-mini" },
|
|
94
|
+
"gpt-5-nano": { provider: "openai", model: "gpt-5-nano" },
|
|
95
|
+
"gpt-5-chat": { provider: "openai", model: "gpt-5-chat" },
|
|
96
|
+
"gpt-5-codex": { provider: "openai", model: "gpt-5-codex" },
|
|
97
|
+
"gpt-chat-latest": { provider: "openai", model: "gpt-chat-latest" },
|
|
98
|
+
// ── GPT-5 image ──
|
|
99
|
+
"gpt-5-image": { provider: "openai", model: "gpt-5-image" },
|
|
100
|
+
"gpt-5-image-mini": { provider: "openai", model: "gpt-5-image-mini" },
|
|
101
|
+
// ── GPT-4.x ──
|
|
102
|
+
"gpt-4.1": { provider: "openai", model: "gpt-4.1" },
|
|
103
|
+
"gpt-4.1-mini": { provider: "openai", model: "gpt-4.1-mini" },
|
|
104
|
+
"gpt-4.1-nano": { provider: "openai", model: "gpt-4.1-nano" },
|
|
105
|
+
"gpt-4o": { provider: "openai", model: "gpt-4o" },
|
|
106
|
+
"gpt-4o-mini": { provider: "openai", model: "gpt-4o-mini" },
|
|
107
|
+
"gpt-4o-2024-05-13": { provider: "openai", model: "gpt-4o-2024-05-13" },
|
|
108
|
+
"gpt-4o-2024-08-06": { provider: "openai", model: "gpt-4o-2024-08-06" },
|
|
109
|
+
"gpt-4o-2024-11-20": { provider: "openai", model: "gpt-4o-2024-11-20" },
|
|
110
|
+
"gpt-4o-mini-2024-07-18": { provider: "openai", model: "gpt-4o-mini-2024-07-18" },
|
|
111
|
+
"gpt-4o-mini-search-preview": { provider: "openai", model: "gpt-4o-mini-search-preview" },
|
|
112
|
+
"gpt-4o-search-preview": { provider: "openai", model: "gpt-4o-search-preview" },
|
|
113
|
+
"gpt-4": { provider: "openai", model: "gpt-4" },
|
|
114
|
+
"gpt-4-turbo": { provider: "openai", model: "gpt-4-turbo" },
|
|
115
|
+
"gpt-4-turbo-preview": { provider: "openai", model: "gpt-4-turbo-preview" },
|
|
116
|
+
// ── GPT-3.5 (legacy) ──
|
|
117
|
+
"gpt-3.5-turbo": { provider: "openai", model: "gpt-3.5-turbo" },
|
|
118
|
+
"gpt-3.5-turbo-16k": { provider: "openai", model: "gpt-3.5-turbo-16k" },
|
|
119
|
+
"gpt-3.5-turbo-0613": { provider: "openai", model: "gpt-3.5-turbo-0613" },
|
|
120
|
+
"gpt-3.5-turbo-instruct": { provider: "openai", model: "gpt-3.5-turbo-instruct" },
|
|
121
|
+
// ── Audio ──
|
|
122
|
+
"gpt-audio": { provider: "openai", model: "gpt-audio" },
|
|
123
|
+
"gpt-audio-mini": { provider: "openai", model: "gpt-audio-mini" },
|
|
124
|
+
// ── o-series (reasoning) ──
|
|
125
|
+
o1: { provider: "openai", model: "o1" },
|
|
126
|
+
"o1-pro": { provider: "openai", model: "o1-pro" },
|
|
127
|
+
o3: { provider: "openai", model: "o3" },
|
|
128
|
+
"o3-pro": { provider: "openai", model: "o3-pro" },
|
|
129
|
+
"o3-mini": { provider: "openai", model: "o3-mini" },
|
|
130
|
+
"o3-mini-high": { provider: "openai", model: "o3-mini-high" },
|
|
131
|
+
"o3-deep-research": { provider: "openai", model: "o3-deep-research" },
|
|
132
|
+
"o4-mini": { provider: "openai", model: "o4-mini" },
|
|
133
|
+
"o4-mini-high": { provider: "openai", model: "o4-mini-high" },
|
|
134
|
+
"o4-mini-deep-research": { provider: "openai", model: "o4-mini-deep-research" },
|
|
135
|
+
// ── OSS ──
|
|
136
|
+
"gpt-oss-20b": { provider: "openai", model: "gpt-oss-20b" },
|
|
137
|
+
"gpt-oss-20b:free": { provider: "openai", model: "gpt-oss-20b:free" },
|
|
138
|
+
"gpt-oss-120b": { provider: "openai", model: "gpt-oss-120b" },
|
|
139
|
+
"gpt-oss-safeguard-20b": { provider: "openai", model: "gpt-oss-safeguard-20b" }
|
|
140
|
+
}
|
|
141
|
+
};
|
|
142
|
+
var anthropicPack = {
|
|
143
|
+
id: "anthropic",
|
|
144
|
+
label: "Anthropic (direct)",
|
|
145
|
+
description: "Claude models routed directly to api.anthropic.com via ANTHROPIC_API_KEY",
|
|
146
|
+
models: {
|
|
147
|
+
"claude-opus-4-8": { provider: "anthropic", model: "claude-opus-4-8" },
|
|
148
|
+
"claude-sonnet-5": { provider: "anthropic", model: "claude-sonnet-5" },
|
|
149
|
+
// Unlike the other entries here, Anthropic's live /v1/models does not
|
|
150
|
+
// expose a bare "claude-haiku-4-5" id — only the datestamped one
|
|
151
|
+
// resolves (verified against a live models-list fetch; see
|
|
152
|
+
// packages/model-catalog/src/llm/context-windows.generated.ts). Do not
|
|
153
|
+
// "fix" this to match the other entries — that would 404 upstream.
|
|
154
|
+
"claude-haiku-4-5": { provider: "anthropic", model: "claude-haiku-4-5-20251001" },
|
|
155
|
+
// claude-fable-5 is registered in the model catalog (catalog.ts) and
|
|
156
|
+
// context-windows.generated.ts with confirmed pricing. The model ID is
|
|
157
|
+
// used as-is because Anthropic exposes it without a datestamp alias.
|
|
158
|
+
"claude-fable-5": { provider: "anthropic", model: "claude-fable-5" }
|
|
159
|
+
}
|
|
160
|
+
};
|
|
161
|
+
var openrouterPack = {
|
|
162
|
+
id: "openrouter",
|
|
163
|
+
label: "OpenRouter",
|
|
164
|
+
description: "Source-backed models available through OpenRouter",
|
|
165
|
+
models: {
|
|
166
|
+
"anthropic/claude-3-5-sonnet-20241022": {
|
|
167
|
+
provider: "openrouter",
|
|
168
|
+
model: "anthropic/claude-3-5-sonnet-20241022",
|
|
169
|
+
equivalentClaudeName: "claude-3-5-sonnet-20241022"
|
|
170
|
+
},
|
|
171
|
+
"anthropic/claude-3-opus-20240229": {
|
|
172
|
+
provider: "openrouter",
|
|
173
|
+
model: "anthropic/claude-3-opus-20240229",
|
|
174
|
+
equivalentClaudeName: "claude-3-opus-20240229"
|
|
175
|
+
},
|
|
176
|
+
"openai/gpt-4o": { provider: "openrouter", model: "openai/gpt-4o" },
|
|
177
|
+
"openai/gpt-4o-mini": { provider: "openrouter", model: "openai/gpt-4o-mini" }
|
|
178
|
+
}
|
|
179
|
+
};
|
|
180
|
+
var requestyPack = {
|
|
181
|
+
id: "requesty",
|
|
182
|
+
label: "Requesty",
|
|
183
|
+
description: "Source-backed models available through the Requesty router",
|
|
184
|
+
models: {
|
|
185
|
+
// Requesty ids are already vendor-prefixed, so the pack code IS the
|
|
186
|
+
// upstream id — transparent, no aliasing. Claude-name compatibility for
|
|
187
|
+
// this router belongs in a local pack (see packs.local.example.ts).
|
|
188
|
+
"sference/thinkingcap-qwen3.6-27b": {
|
|
189
|
+
provider: "requesty",
|
|
190
|
+
model: "sference/thinkingcap-qwen3.6-27b"
|
|
191
|
+
},
|
|
192
|
+
"sference/glm-5.2": { provider: "requesty", model: "sference/glm-5.2" },
|
|
193
|
+
"openai/gpt-4.1": { provider: "requesty", model: "openai/gpt-4.1" },
|
|
194
|
+
"openai/gpt-4o": { provider: "requesty", model: "openai/gpt-4o" }
|
|
195
|
+
}
|
|
196
|
+
};
|
|
197
|
+
var codingPack = {
|
|
198
|
+
id: "coding",
|
|
199
|
+
label: "Coding (OpenRouter)",
|
|
200
|
+
description: "Curated production coding models routed through OpenRouter",
|
|
201
|
+
models: {
|
|
202
|
+
"openai/gpt-5.5": { provider: "openrouter", model: "openai/gpt-5.5", tier: "extra-high" },
|
|
203
|
+
"anthropic/claude-opus-4.8": { provider: "openrouter", model: "anthropic/claude-opus-4.8", tier: "high" },
|
|
204
|
+
"deepseek/deepseek-v4-pro": { provider: "openrouter", model: "deepseek/deepseek-v4-pro", tier: "high" },
|
|
205
|
+
"anthropic/claude-sonnet-5": { provider: "openrouter", model: "anthropic/claude-sonnet-5", tier: "medium" },
|
|
206
|
+
"z-ai/glm-5.2": { provider: "openrouter", model: "z-ai/glm-5.2", tier: "medium" },
|
|
207
|
+
"minimax/minimax-m3": { provider: "openrouter", model: "minimax/minimax-m3", tier: "small" }
|
|
208
|
+
}
|
|
209
|
+
};
|
|
210
|
+
var PACK_REGISTRY = {
|
|
211
|
+
[defaultPack.id]: defaultPack,
|
|
212
|
+
[xaiPack.id]: xaiPack,
|
|
213
|
+
[openaiPack.id]: openaiPack,
|
|
214
|
+
[openrouterPack.id]: openrouterPack,
|
|
215
|
+
[requestyPack.id]: requestyPack,
|
|
216
|
+
[anthropicPack.id]: anthropicPack,
|
|
217
|
+
[codingPack.id]: codingPack
|
|
218
|
+
};
|
|
219
|
+
var DEFAULT_PACK_ID = "default";
|
|
220
|
+
function resolvePack(packId) {
|
|
221
|
+
if (!packId) return PACK_REGISTRY[DEFAULT_PACK_ID];
|
|
222
|
+
const pack = PACK_REGISTRY[packId];
|
|
223
|
+
if (!pack) throw new RangeError(`Unknown pack id: "${packId}". Available: ${Object.keys(PACK_REGISTRY).join(", ")}`);
|
|
224
|
+
return pack;
|
|
225
|
+
}
|
|
226
|
+
function buildMappingFromPack(pack) {
|
|
227
|
+
return { ...pack.models };
|
|
228
|
+
}
|
|
229
|
+
function listPackIds() {
|
|
230
|
+
return Object.keys(PACK_REGISTRY);
|
|
231
|
+
}
|
|
232
|
+
function isRecord(value) {
|
|
233
|
+
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
234
|
+
}
|
|
235
|
+
function isModelTier(value) {
|
|
236
|
+
return value === "extra-high" || value === "high" || value === "medium" || value === "small";
|
|
237
|
+
}
|
|
238
|
+
function validateModelRoute(route, where, errors) {
|
|
239
|
+
if (!isRecord(route)) {
|
|
240
|
+
errors.push(`${where}: expected an object, got ${route === null ? "null" : typeof route}`);
|
|
241
|
+
return null;
|
|
242
|
+
}
|
|
243
|
+
const { provider, model, equivalentClaudeName, tier, contextWindow, maxOutputTokens } = route;
|
|
244
|
+
let ok = true;
|
|
245
|
+
if (typeof provider !== "string" || provider.length === 0) {
|
|
246
|
+
errors.push(`${where}.provider: required non-empty string`);
|
|
247
|
+
ok = false;
|
|
248
|
+
}
|
|
249
|
+
if (typeof model !== "string" || model.length === 0) {
|
|
250
|
+
errors.push(`${where}.model: required non-empty string`);
|
|
251
|
+
ok = false;
|
|
252
|
+
}
|
|
253
|
+
if (equivalentClaudeName !== void 0 && typeof equivalentClaudeName !== "string") {
|
|
254
|
+
errors.push(`${where}.equivalentClaudeName: must be a string when present`);
|
|
255
|
+
ok = false;
|
|
256
|
+
}
|
|
257
|
+
if (tier !== void 0 && !(typeof tier === "string" && isModelTier(tier))) {
|
|
258
|
+
errors.push(`${where}.tier: must be one of extra-high|high|medium|small when present`);
|
|
259
|
+
ok = false;
|
|
260
|
+
}
|
|
261
|
+
if (contextWindow !== void 0 && !(typeof contextWindow === "number" && Number.isFinite(contextWindow) && contextWindow > 0)) {
|
|
262
|
+
errors.push(`${where}.contextWindow: must be a positive finite number when present`);
|
|
263
|
+
ok = false;
|
|
264
|
+
}
|
|
265
|
+
if (maxOutputTokens !== void 0 && !(typeof maxOutputTokens === "number" && Number.isFinite(maxOutputTokens) && maxOutputTokens > 0)) {
|
|
266
|
+
errors.push(`${where}.maxOutputTokens: must be a positive finite number when present`);
|
|
267
|
+
ok = false;
|
|
268
|
+
}
|
|
269
|
+
if (!ok || typeof provider !== "string" || typeof model !== "string") return null;
|
|
270
|
+
const built = { provider, model };
|
|
271
|
+
if (typeof equivalentClaudeName === "string") built.equivalentClaudeName = equivalentClaudeName;
|
|
272
|
+
if (typeof tier === "string" && isModelTier(tier)) built.tier = tier;
|
|
273
|
+
if (typeof contextWindow === "number" && Number.isFinite(contextWindow) && contextWindow > 0) {
|
|
274
|
+
built.contextWindow = contextWindow;
|
|
275
|
+
}
|
|
276
|
+
if (typeof maxOutputTokens === "number" && Number.isFinite(maxOutputTokens) && maxOutputTokens > 0) {
|
|
277
|
+
built.maxOutputTokens = maxOutputTokens;
|
|
278
|
+
}
|
|
279
|
+
return built;
|
|
280
|
+
}
|
|
281
|
+
function validateModelPack(pack, label = "pack") {
|
|
282
|
+
const errors = [];
|
|
283
|
+
if (!isRecord(pack)) {
|
|
284
|
+
return { ok: false, errors: [`${label}: expected an object, got ${pack === null ? "null" : typeof pack}`] };
|
|
285
|
+
}
|
|
286
|
+
const { id, label: packLabel, description, models, toolsExclude, toolsAllow } = pack;
|
|
287
|
+
if (typeof id !== "string" || id.length === 0) errors.push(`${label}.id: required non-empty string`);
|
|
288
|
+
if (typeof packLabel !== "string") errors.push(`${label}.label: required string`);
|
|
289
|
+
if (typeof description !== "string") errors.push(`${label}.description: required string`);
|
|
290
|
+
const validPatternList = (v) => Array.isArray(v) && v.every((p) => typeof p === "string" && p.length > 0);
|
|
291
|
+
if (toolsExclude !== void 0 && !validPatternList(toolsExclude)) {
|
|
292
|
+
errors.push(`${label}.toolsExclude: must be an array of non-empty strings when present`);
|
|
293
|
+
}
|
|
294
|
+
if (toolsAllow !== void 0 && !validPatternList(toolsAllow)) {
|
|
295
|
+
errors.push(`${label}.toolsAllow: must be an array of non-empty strings when present`);
|
|
296
|
+
}
|
|
297
|
+
const builtModels = {};
|
|
298
|
+
if (!isRecord(models)) {
|
|
299
|
+
errors.push(`${label}.models: required object mapping code \u2192 route`);
|
|
300
|
+
} else {
|
|
301
|
+
for (const [code, route] of Object.entries(models)) {
|
|
302
|
+
const built = validateModelRoute(route, `${label}.models.${code}`, errors);
|
|
303
|
+
if (built) builtModels[code] = built;
|
|
304
|
+
}
|
|
305
|
+
}
|
|
306
|
+
if (errors.length > 0) return { ok: false, errors };
|
|
307
|
+
if (typeof id === "string" && typeof packLabel === "string" && typeof description === "string") {
|
|
308
|
+
return {
|
|
309
|
+
ok: true,
|
|
310
|
+
pack: {
|
|
311
|
+
id,
|
|
312
|
+
label: packLabel,
|
|
313
|
+
description,
|
|
314
|
+
models: builtModels,
|
|
315
|
+
...validPatternList(toolsExclude) ? { toolsExclude } : {},
|
|
316
|
+
...validPatternList(toolsAllow) ? { toolsAllow } : {}
|
|
317
|
+
}
|
|
318
|
+
};
|
|
319
|
+
}
|
|
320
|
+
return { ok: false, errors: [`${label}: failed final type narrowing`] };
|
|
321
|
+
}
|
|
322
|
+
function validateLocalPacks(envelope) {
|
|
323
|
+
if (!isRecord(envelope)) {
|
|
324
|
+
return { ok: false, errors: [`root: expected an object with a "packs" key, got ${envelope === null ? "null" : typeof envelope}`] };
|
|
325
|
+
}
|
|
326
|
+
if (!isRecord(envelope.packs)) {
|
|
327
|
+
return { ok: false, errors: ["packs: expected an object mapping id \u2192 pack"] };
|
|
328
|
+
}
|
|
329
|
+
const errors = [];
|
|
330
|
+
const packs = {};
|
|
331
|
+
for (const [id, pack] of Object.entries(envelope.packs)) {
|
|
332
|
+
const result = validateModelPack(pack, `packs.${id}`);
|
|
333
|
+
if (result.ok) packs[id] = result.pack;
|
|
334
|
+
else errors.push(...result.errors);
|
|
335
|
+
}
|
|
336
|
+
if (errors.length > 0) return { ok: false, errors };
|
|
337
|
+
return { ok: true, packs };
|
|
338
|
+
}
|
|
339
|
+
var KNOWN_TRANSPARENT_PROVIDERS = /* @__PURE__ */ new Set([
|
|
340
|
+
"anthropic",
|
|
341
|
+
"moonshot",
|
|
342
|
+
"openrouter",
|
|
343
|
+
"requesty",
|
|
344
|
+
"zai",
|
|
345
|
+
"groq",
|
|
346
|
+
"xai",
|
|
347
|
+
"openai",
|
|
348
|
+
// Self-hosted fine-tunes (vLLM --enable-lora) — see FORGE_BASE_URL in
|
|
349
|
+
// src/index.ts. Absent from every committed pack: LoRA adapters are
|
|
350
|
+
// registered on the forge server itself, not known ahead of time here.
|
|
351
|
+
"forge",
|
|
352
|
+
// Nebius AI Studio — see NEBIUS_BASE_URL/NEBIUS_API_KEY in src/index.ts.
|
|
353
|
+
// Same "configurable OpenAI-compatible upstream" mechanism as forge, but
|
|
354
|
+
// with a working default host (api.studio.nebius.com) and a required key.
|
|
355
|
+
"nebius"
|
|
356
|
+
]);
|
|
357
|
+
function parseTransparentModel(model) {
|
|
358
|
+
const slashIdx = model.indexOf("/");
|
|
359
|
+
if (slashIdx <= 0 || slashIdx === model.length - 1) return null;
|
|
360
|
+
const provider = model.slice(0, slashIdx);
|
|
361
|
+
if (!KNOWN_TRANSPARENT_PROVIDERS.has(provider)) return null;
|
|
362
|
+
return { provider, model: model.slice(slashIdx + 1) };
|
|
363
|
+
}
|
|
364
|
+
function matchesPattern(name, pattern) {
|
|
365
|
+
return pattern.endsWith("*") ? name.startsWith(pattern.slice(0, -1)) : name === pattern;
|
|
366
|
+
}
|
|
367
|
+
var TIER_TO_FAMILY = {
|
|
368
|
+
"extra-high": "fable",
|
|
369
|
+
high: "opus",
|
|
370
|
+
medium: "sonnet",
|
|
371
|
+
small: "haiku"
|
|
372
|
+
};
|
|
373
|
+
var DEFAULT_FAMILY = "sonnet";
|
|
374
|
+
function shaNumericId(model, digits = 7) {
|
|
375
|
+
const hex = createHash("sha256").update(model).digest("hex");
|
|
376
|
+
const n = BigInt("0x" + hex.slice(0, 15)) % 10n ** BigInt(digits);
|
|
377
|
+
return n.toString().padStart(digits, "0");
|
|
378
|
+
}
|
|
379
|
+
function toAnthropicStyle(pack, opts = {}) {
|
|
380
|
+
const { digits = 7, idPrefix = "claude-" } = opts;
|
|
381
|
+
const models = {};
|
|
382
|
+
for (const [code, route] of Object.entries(pack.models)) {
|
|
383
|
+
const family = route.tier ? TIER_TO_FAMILY[route.tier] : DEFAULT_FAMILY;
|
|
384
|
+
models[code] = {
|
|
385
|
+
...route,
|
|
386
|
+
equivalentClaudeName: `${idPrefix}${family}-${shaNumericId(route.model, digits)}`
|
|
387
|
+
};
|
|
388
|
+
}
|
|
389
|
+
return {
|
|
390
|
+
id: opts.id ?? pack.id,
|
|
391
|
+
label: opts.label ?? pack.label,
|
|
392
|
+
description: pack.description,
|
|
393
|
+
models
|
|
394
|
+
};
|
|
395
|
+
}
|
|
396
|
+
|
|
397
|
+
// src/responses.ts
|
|
398
|
+
function isPlainObject(value) {
|
|
399
|
+
return value !== null && typeof value === "object" && !Array.isArray(value);
|
|
400
|
+
}
|
|
401
|
+
function assertString(value, field) {
|
|
402
|
+
if (typeof value !== "string") {
|
|
403
|
+
throw new TypeError(`Expected ${field} to be a string`);
|
|
404
|
+
}
|
|
405
|
+
return value;
|
|
406
|
+
}
|
|
407
|
+
function assertOptionalString(value, field) {
|
|
408
|
+
if (value === void 0 || value === null) return void 0;
|
|
409
|
+
return assertString(value, field);
|
|
410
|
+
}
|
|
411
|
+
function assertOptionalNumber(value, field) {
|
|
412
|
+
if (value === void 0 || value === null) return void 0;
|
|
413
|
+
if (typeof value !== "number" || Number.isNaN(value)) {
|
|
414
|
+
throw new TypeError(`Expected ${field} to be a number`);
|
|
415
|
+
}
|
|
416
|
+
return value;
|
|
417
|
+
}
|
|
418
|
+
function assertOptionalBoolean(value, field) {
|
|
419
|
+
if (value === void 0 || value === null) return void 0;
|
|
420
|
+
if (typeof value !== "boolean") {
|
|
421
|
+
throw new TypeError(`Expected ${field} to be a boolean`);
|
|
422
|
+
}
|
|
423
|
+
return value;
|
|
424
|
+
}
|
|
425
|
+
function validateContentItem(item) {
|
|
426
|
+
if (!isPlainObject(item)) {
|
|
427
|
+
throw new TypeError("Input message content items must be objects");
|
|
428
|
+
}
|
|
429
|
+
const type = item.type;
|
|
430
|
+
if (type !== "input_text" && type !== "output_text") {
|
|
431
|
+
throw new TypeError(
|
|
432
|
+
`Unsupported input message content item type "${String(type)}"; only "input_text" and "output_text" are supported`
|
|
433
|
+
);
|
|
434
|
+
}
|
|
435
|
+
return { type, text: assertString(item.text, "content item text") };
|
|
436
|
+
}
|
|
437
|
+
function validateInputItem(item) {
|
|
438
|
+
if (!isPlainObject(item)) {
|
|
439
|
+
throw new TypeError("Input items must be objects");
|
|
440
|
+
}
|
|
441
|
+
const type = item.type;
|
|
442
|
+
if (type === "message") {
|
|
443
|
+
const role = assertString(item.role, "input message role");
|
|
444
|
+
if (role !== "user" && role !== "assistant" && role !== "system" && role !== "developer") {
|
|
445
|
+
throw new TypeError(
|
|
446
|
+
`Unsupported input message role "${role}"; only "user", "assistant", "system", and "developer" are supported`
|
|
447
|
+
);
|
|
448
|
+
}
|
|
449
|
+
let content;
|
|
450
|
+
if (typeof item.content === "string") {
|
|
451
|
+
content = item.content;
|
|
452
|
+
} else if (Array.isArray(item.content)) {
|
|
453
|
+
content = item.content.map(validateContentItem);
|
|
454
|
+
} else {
|
|
455
|
+
throw new TypeError("Input message content must be a string or an array of content items");
|
|
456
|
+
}
|
|
457
|
+
return { type: "message", role, content };
|
|
458
|
+
}
|
|
459
|
+
if (type === "function_call_output") {
|
|
460
|
+
return {
|
|
461
|
+
type: "function_call_output",
|
|
462
|
+
call_id: assertString(item.call_id, "function_call_output call_id"),
|
|
463
|
+
output: typeof item.output === "string" ? item.output : JSON.stringify(item.output ?? {})
|
|
464
|
+
};
|
|
465
|
+
}
|
|
466
|
+
throw new TypeError(
|
|
467
|
+
`Unsupported input item type "${String(type)}"; only "message" and "function_call_output" are supported`
|
|
468
|
+
);
|
|
469
|
+
}
|
|
470
|
+
function validateTool(tool) {
|
|
471
|
+
if (!isPlainObject(tool)) {
|
|
472
|
+
throw new TypeError("Tools must be objects");
|
|
473
|
+
}
|
|
474
|
+
if (tool.type !== "function") {
|
|
475
|
+
throw new TypeError(
|
|
476
|
+
`Unsupported tool type "${String(tool.type)}"; only "function" tools are supported`
|
|
477
|
+
);
|
|
478
|
+
}
|
|
479
|
+
return {
|
|
480
|
+
type: "function",
|
|
481
|
+
name: assertString(tool.name, "tool name"),
|
|
482
|
+
description: assertOptionalString(tool.description, "tool description"),
|
|
483
|
+
parameters: isPlainObject(tool.parameters) ? tool.parameters : void 0,
|
|
484
|
+
strict: assertOptionalBoolean(tool.strict, "tool strict")
|
|
485
|
+
};
|
|
486
|
+
}
|
|
487
|
+
function validateResponsesRequest(body) {
|
|
488
|
+
if (!isPlainObject(body)) {
|
|
489
|
+
throw new TypeError("Request body must be a JSON object");
|
|
490
|
+
}
|
|
491
|
+
const model = assertString(body.model, "model");
|
|
492
|
+
let input;
|
|
493
|
+
if (typeof body.input === "string") {
|
|
494
|
+
input = body.input;
|
|
495
|
+
} else if (Array.isArray(body.input)) {
|
|
496
|
+
input = body.input.map(validateInputItem);
|
|
497
|
+
} else {
|
|
498
|
+
throw new TypeError('"input" must be a string or an array of input items');
|
|
499
|
+
}
|
|
500
|
+
const instructions = assertOptionalString(body.instructions, "instructions");
|
|
501
|
+
let tools;
|
|
502
|
+
if (body.tools !== void 0 && body.tools !== null) {
|
|
503
|
+
if (!Array.isArray(body.tools)) {
|
|
504
|
+
throw new TypeError('"tools" must be an array');
|
|
505
|
+
}
|
|
506
|
+
tools = body.tools.map(validateTool);
|
|
507
|
+
}
|
|
508
|
+
let toolChoice;
|
|
509
|
+
if (body.tool_choice !== void 0 && body.tool_choice !== null) {
|
|
510
|
+
if (typeof body.tool_choice === "string") {
|
|
511
|
+
const tc = body.tool_choice;
|
|
512
|
+
if (tc !== "auto" && tc !== "none" && tc !== "required") {
|
|
513
|
+
throw new TypeError(
|
|
514
|
+
`Unsupported tool_choice "${tc}"; must be "auto", "none", "required", or a function object`
|
|
515
|
+
);
|
|
516
|
+
}
|
|
517
|
+
toolChoice = tc;
|
|
518
|
+
} else if (isPlainObject(body.tool_choice)) {
|
|
519
|
+
const tc = body.tool_choice;
|
|
520
|
+
if (tc.type !== "function") {
|
|
521
|
+
throw new TypeError(
|
|
522
|
+
`Unsupported tool_choice type "${String(tc.type)}"; only "function" is supported`
|
|
523
|
+
);
|
|
524
|
+
}
|
|
525
|
+
toolChoice = {
|
|
526
|
+
type: "function",
|
|
527
|
+
name: assertString(tc.name, "tool_choice function name")
|
|
528
|
+
};
|
|
529
|
+
} else {
|
|
530
|
+
throw new TypeError('"tool_choice" must be a string or an object');
|
|
531
|
+
}
|
|
532
|
+
}
|
|
533
|
+
if (body.previous_response_id !== void 0 && body.previous_response_id !== null) {
|
|
534
|
+
throw new TypeError(
|
|
535
|
+
'"previous_response_id" is not supported; the facade translates each request independently'
|
|
536
|
+
);
|
|
537
|
+
}
|
|
538
|
+
if (body.text !== void 0 && body.text !== null && isPlainObject(body.text) && body.text.format !== void 0 && body.text.format !== null) {
|
|
539
|
+
throw new TypeError(
|
|
540
|
+
'"text.format" / structured output is not supported by this facade'
|
|
541
|
+
);
|
|
542
|
+
}
|
|
543
|
+
const result = {
|
|
544
|
+
model,
|
|
545
|
+
input,
|
|
546
|
+
instructions,
|
|
547
|
+
tools,
|
|
548
|
+
tool_choice: toolChoice,
|
|
549
|
+
stream: assertOptionalBoolean(body.stream, "stream"),
|
|
550
|
+
parallel_tool_calls: assertOptionalBoolean(body.parallel_tool_calls, "parallel_tool_calls"),
|
|
551
|
+
max_output_tokens: assertOptionalNumber(body.max_output_tokens, "max_output_tokens"),
|
|
552
|
+
max_tokens: assertOptionalNumber(body.max_tokens, "max_tokens"),
|
|
553
|
+
temperature: assertOptionalNumber(body.temperature, "temperature"),
|
|
554
|
+
top_p: assertOptionalNumber(body.top_p, "top_p"),
|
|
555
|
+
// Harmless provider metadata: accepted to keep Codex happy, but dropped
|
|
556
|
+
// before the upstream call because chat/completions does not preserve them.
|
|
557
|
+
store: assertOptionalBoolean(body.store, "store"),
|
|
558
|
+
include: Array.isArray(body.include) ? body.include.map((v) => assertString(v, "include")) : void 0,
|
|
559
|
+
client_metadata: isPlainObject(body.client_metadata) ? body.client_metadata : void 0,
|
|
560
|
+
service_tier: assertOptionalString(body.service_tier, "service_tier"),
|
|
561
|
+
prompt_cache_key: assertOptionalString(body.prompt_cache_key, "prompt_cache_key"),
|
|
562
|
+
stream_options: body.stream_options,
|
|
563
|
+
reasoning: body.reasoning !== void 0 && body.reasoning !== null ? {
|
|
564
|
+
effort: ["low", "medium", "high"].includes(String(body.reasoning.effort)) ? String(body.reasoning.effort) : void 0,
|
|
565
|
+
summary: ["auto", "detailed", "concise", "auto_verbose"].includes(
|
|
566
|
+
String(body.reasoning.summary)
|
|
567
|
+
) ? String(body.reasoning.summary) : void 0
|
|
568
|
+
} : void 0,
|
|
569
|
+
text: body.text !== void 0 && body.text !== null ? { verbosity: ["low", "medium", "high"].includes(String(body.text.verbosity)) ? String(body.text.verbosity) : void 0 } : void 0
|
|
570
|
+
};
|
|
571
|
+
return result;
|
|
572
|
+
}
|
|
573
|
+
function flattenContent(content) {
|
|
574
|
+
if (typeof content === "string") return content;
|
|
575
|
+
return content.map((c) => c.text).join("");
|
|
576
|
+
}
|
|
577
|
+
function translateInputToMessages(input, instructions) {
|
|
578
|
+
const messages = [];
|
|
579
|
+
if (instructions) {
|
|
580
|
+
messages.push({ role: "system", content: instructions });
|
|
581
|
+
}
|
|
582
|
+
if (typeof input === "string") {
|
|
583
|
+
messages.push({ role: "user", content: input });
|
|
584
|
+
return messages;
|
|
585
|
+
}
|
|
586
|
+
for (const item of input) {
|
|
587
|
+
if (item.type === "message") {
|
|
588
|
+
messages.push({
|
|
589
|
+
role: item.role === "developer" ? "system" : item.role,
|
|
590
|
+
content: flattenContent(item.content)
|
|
591
|
+
});
|
|
592
|
+
} else if (item.type === "function_call_output") {
|
|
593
|
+
messages.push({
|
|
594
|
+
role: "tool",
|
|
595
|
+
tool_call_id: item.call_id,
|
|
596
|
+
content: item.output
|
|
597
|
+
});
|
|
598
|
+
}
|
|
599
|
+
}
|
|
600
|
+
return messages;
|
|
601
|
+
}
|
|
602
|
+
function translateToolToChatCompletions(tool) {
|
|
603
|
+
return {
|
|
604
|
+
type: "function",
|
|
605
|
+
function: {
|
|
606
|
+
name: tool.name,
|
|
607
|
+
description: tool.description,
|
|
608
|
+
parameters: tool.parameters
|
|
609
|
+
},
|
|
610
|
+
strict: tool.strict
|
|
611
|
+
};
|
|
612
|
+
}
|
|
613
|
+
function translateToolChoice(toolChoice) {
|
|
614
|
+
if (!toolChoice) return void 0;
|
|
615
|
+
if (typeof toolChoice === "string") return toolChoice;
|
|
616
|
+
return {
|
|
617
|
+
type: "function",
|
|
618
|
+
function: { name: toolChoice.name }
|
|
619
|
+
};
|
|
620
|
+
}
|
|
621
|
+
function responsesToChatCompletionsRequest(body, _resolvedTarget) {
|
|
622
|
+
const messages = translateInputToMessages(body.input, body.instructions);
|
|
623
|
+
const result = {
|
|
624
|
+
model: _resolvedTarget.model,
|
|
625
|
+
messages,
|
|
626
|
+
stream: body.stream,
|
|
627
|
+
parallel_tool_calls: body.parallel_tool_calls,
|
|
628
|
+
max_tokens: body.max_output_tokens ?? body.max_tokens,
|
|
629
|
+
temperature: body.temperature,
|
|
630
|
+
top_p: body.top_p
|
|
631
|
+
};
|
|
632
|
+
if (body.tools && body.tools.length > 0) {
|
|
633
|
+
result.tools = body.tools.map(translateToolToChatCompletions);
|
|
634
|
+
}
|
|
635
|
+
if (body.tool_choice) {
|
|
636
|
+
result.tool_choice = translateToolChoice(body.tool_choice);
|
|
637
|
+
}
|
|
638
|
+
if (body.reasoning?.effort) {
|
|
639
|
+
result.reasoning_effort = body.reasoning.effort;
|
|
640
|
+
}
|
|
641
|
+
return result;
|
|
642
|
+
}
|
|
643
|
+
function generateResponseId() {
|
|
644
|
+
return `resp_${Date.now()}_${Math.random().toString(36).slice(2, 8)}`;
|
|
645
|
+
}
|
|
646
|
+
function generateOutputItemId(prefix) {
|
|
647
|
+
return `${prefix}_${Date.now()}_${Math.random().toString(36).slice(2, 8)}`;
|
|
648
|
+
}
|
|
649
|
+
function chatChoiceToOutputItems(choice) {
|
|
650
|
+
const output = [];
|
|
651
|
+
const msg = choice && choice.message;
|
|
652
|
+
if (!msg) return output;
|
|
653
|
+
const text = typeof msg.content === "string" ? msg.content : "";
|
|
654
|
+
if (text) {
|
|
655
|
+
output.push({
|
|
656
|
+
type: "message",
|
|
657
|
+
id: generateOutputItemId("msg"),
|
|
658
|
+
role: "assistant",
|
|
659
|
+
content: [
|
|
660
|
+
{ type: "output_text", text, annotations: [] }
|
|
661
|
+
]
|
|
662
|
+
});
|
|
663
|
+
}
|
|
664
|
+
if (Array.isArray(msg.tool_calls)) {
|
|
665
|
+
for (const tc of msg.tool_calls) {
|
|
666
|
+
const fn = tc.function || {};
|
|
667
|
+
output.push({
|
|
668
|
+
type: "function_call",
|
|
669
|
+
id: tc.id || generateOutputItemId("fc"),
|
|
670
|
+
call_id: tc.id || generateOutputItemId("call"),
|
|
671
|
+
name: fn.name || "",
|
|
672
|
+
arguments: typeof fn.arguments === "string" ? fn.arguments : JSON.stringify(fn.arguments ?? {})
|
|
673
|
+
});
|
|
674
|
+
}
|
|
675
|
+
}
|
|
676
|
+
return output;
|
|
677
|
+
}
|
|
678
|
+
function chatUsageToResponsesUsage(usage) {
|
|
679
|
+
const prompt = usage?.prompt_tokens ?? 0;
|
|
680
|
+
const completion = usage?.completion_tokens ?? 0;
|
|
681
|
+
const cached = usage?.prompt_tokens_details?.cached_tokens ?? 0;
|
|
682
|
+
const reasoning = usage?.completion_tokens_details?.reasoning_tokens ?? 0;
|
|
683
|
+
return {
|
|
684
|
+
input_tokens: prompt,
|
|
685
|
+
input_tokens_details: { cached_tokens: cached },
|
|
686
|
+
output_tokens: completion,
|
|
687
|
+
output_tokens_details: { reasoning_tokens: reasoning },
|
|
688
|
+
total_tokens: usage?.total_tokens ?? prompt + completion
|
|
689
|
+
};
|
|
690
|
+
}
|
|
691
|
+
function chatCompletionsJsonToResponses(jsonStr, opts) {
|
|
692
|
+
try {
|
|
693
|
+
const upstream = JSON.parse(jsonStr);
|
|
694
|
+
if (upstream && typeof upstream === "object" && upstream.error) {
|
|
695
|
+
return JSON.stringify({
|
|
696
|
+
error: upstream.error
|
|
697
|
+
});
|
|
698
|
+
}
|
|
699
|
+
const choice = upstream.choices && upstream.choices[0];
|
|
700
|
+
const responseId = opts.responseId || upstream.id || generateResponseId();
|
|
701
|
+
const now = Math.floor(Date.now() / 1e3);
|
|
702
|
+
const response = {
|
|
703
|
+
id: responseId,
|
|
704
|
+
object: "response",
|
|
705
|
+
created_at: now,
|
|
706
|
+
status: "completed",
|
|
707
|
+
error: null,
|
|
708
|
+
incomplete_details: null,
|
|
709
|
+
instructions: null,
|
|
710
|
+
max_output_tokens: null,
|
|
711
|
+
model: opts.requestedModel,
|
|
712
|
+
output: chatChoiceToOutputItems(choice),
|
|
713
|
+
parallel_tool_calls: upstream.parallel_tool_calls ?? false,
|
|
714
|
+
tool_choice: "auto",
|
|
715
|
+
tools: [],
|
|
716
|
+
usage: chatUsageToResponsesUsage(upstream.usage)
|
|
717
|
+
};
|
|
718
|
+
return JSON.stringify(response);
|
|
719
|
+
} catch {
|
|
720
|
+
return jsonStr;
|
|
721
|
+
}
|
|
722
|
+
}
|
|
723
|
+
function sse(event, data) {
|
|
724
|
+
return `event: ${event}
|
|
725
|
+
data: ${JSON.stringify(data)}
|
|
726
|
+
|
|
727
|
+
`;
|
|
728
|
+
}
|
|
729
|
+
function createInitialResponse(responseId, model) {
|
|
730
|
+
return {
|
|
731
|
+
id: responseId,
|
|
732
|
+
object: "response",
|
|
733
|
+
created_at: Math.floor(Date.now() / 1e3),
|
|
734
|
+
status: "in_progress",
|
|
735
|
+
error: null,
|
|
736
|
+
incomplete_details: null,
|
|
737
|
+
instructions: null,
|
|
738
|
+
max_output_tokens: null,
|
|
739
|
+
model,
|
|
740
|
+
output: [],
|
|
741
|
+
parallel_tool_calls: false,
|
|
742
|
+
tool_choice: "auto",
|
|
743
|
+
tools: [],
|
|
744
|
+
usage: { input_tokens: 0, output_tokens: 0, total_tokens: 0 }
|
|
745
|
+
};
|
|
746
|
+
}
|
|
747
|
+
var OpenAIChatToResponsesStreamConverter = class {
|
|
748
|
+
state;
|
|
749
|
+
buffer = "";
|
|
750
|
+
constructor(opts) {
|
|
751
|
+
this.state = {
|
|
752
|
+
responseId: opts.responseId || generateResponseId(),
|
|
753
|
+
requestedModel: opts.requestedModel,
|
|
754
|
+
textItemId: null,
|
|
755
|
+
textStarted: false,
|
|
756
|
+
toolItems: /* @__PURE__ */ new Map(),
|
|
757
|
+
accUsage: {},
|
|
758
|
+
finished: false,
|
|
759
|
+
sentCreated: false
|
|
760
|
+
};
|
|
761
|
+
}
|
|
762
|
+
push(chunk) {
|
|
763
|
+
this.buffer += chunk;
|
|
764
|
+
const out = [];
|
|
765
|
+
const parts = this.buffer.split(/\r?\n\r?\n/);
|
|
766
|
+
this.buffer = parts.pop() ?? "";
|
|
767
|
+
for (const part of parts) {
|
|
768
|
+
const transformed = this.transformEvent(part);
|
|
769
|
+
if (transformed) out.push(...transformed);
|
|
770
|
+
}
|
|
771
|
+
return out;
|
|
772
|
+
}
|
|
773
|
+
flush() {
|
|
774
|
+
if (this.buffer.trim()) {
|
|
775
|
+
const transformed = this.transformEvent(this.buffer);
|
|
776
|
+
this.buffer = "";
|
|
777
|
+
return transformed || [];
|
|
778
|
+
}
|
|
779
|
+
return [];
|
|
780
|
+
}
|
|
781
|
+
ensureCreated() {
|
|
782
|
+
if (this.state.sentCreated) return [];
|
|
783
|
+
this.state.sentCreated = true;
|
|
784
|
+
return [sse("response.created", {
|
|
785
|
+
type: "response.created",
|
|
786
|
+
response: createInitialResponse(this.state.responseId, this.state.requestedModel)
|
|
787
|
+
})];
|
|
788
|
+
}
|
|
789
|
+
transformEvent(rawEvent) {
|
|
790
|
+
const lines = rawEvent.split(/\r?\n/);
|
|
791
|
+
const dataLines = [];
|
|
792
|
+
for (const line of lines) {
|
|
793
|
+
if (line.startsWith("data:")) dataLines.push(line.slice(5).trim());
|
|
794
|
+
}
|
|
795
|
+
if (!dataLines.length) return null;
|
|
796
|
+
const out = [];
|
|
797
|
+
for (const dl of dataLines) {
|
|
798
|
+
if (dl === "[DONE]") {
|
|
799
|
+
if (this.state.finished) continue;
|
|
800
|
+
out.push(...this.finalize());
|
|
801
|
+
this.state.finished = true;
|
|
802
|
+
continue;
|
|
803
|
+
}
|
|
804
|
+
let data;
|
|
805
|
+
try {
|
|
806
|
+
data = JSON.parse(dl);
|
|
807
|
+
} catch {
|
|
808
|
+
continue;
|
|
809
|
+
}
|
|
810
|
+
const choice = data.choices && data.choices[0];
|
|
811
|
+
const delta = choice && choice.delta;
|
|
812
|
+
const content = delta && delta.content;
|
|
813
|
+
const toolCalls = delta && delta.tool_calls;
|
|
814
|
+
out.push(...this.ensureCreated());
|
|
815
|
+
if (typeof content === "string") {
|
|
816
|
+
if (!this.state.textItemId) {
|
|
817
|
+
this.state.textItemId = generateOutputItemId("msg");
|
|
818
|
+
out.push(sse("response.output_item.added", {
|
|
819
|
+
type: "response.output_item.added",
|
|
820
|
+
item: {
|
|
821
|
+
type: "message",
|
|
822
|
+
id: this.state.textItemId,
|
|
823
|
+
role: "assistant",
|
|
824
|
+
content: []
|
|
825
|
+
},
|
|
826
|
+
output_index: 0
|
|
827
|
+
}));
|
|
828
|
+
}
|
|
829
|
+
if (content) {
|
|
830
|
+
if (!this.state.textStarted) {
|
|
831
|
+
out.push(sse("response.content_part.added", {
|
|
832
|
+
type: "response.content_part.added",
|
|
833
|
+
item_id: this.state.textItemId,
|
|
834
|
+
output_index: 0,
|
|
835
|
+
content_index: 0,
|
|
836
|
+
part: { type: "output_text", text: "" }
|
|
837
|
+
}));
|
|
838
|
+
this.state.textStarted = true;
|
|
839
|
+
}
|
|
840
|
+
out.push(sse("response.output_text.delta", {
|
|
841
|
+
type: "response.output_text.delta",
|
|
842
|
+
item_id: this.state.textItemId,
|
|
843
|
+
output_index: 0,
|
|
844
|
+
content_index: 0,
|
|
845
|
+
delta: content
|
|
846
|
+
}));
|
|
847
|
+
}
|
|
848
|
+
}
|
|
849
|
+
if (Array.isArray(toolCalls)) {
|
|
850
|
+
for (const tc of toolCalls) {
|
|
851
|
+
const idx = typeof tc.index === "number" ? tc.index : 0;
|
|
852
|
+
let tool = this.state.toolItems.get(idx);
|
|
853
|
+
if (tc.function && tc.function.name && !tool) {
|
|
854
|
+
tool = {
|
|
855
|
+
id: tc.id || generateOutputItemId("fc"),
|
|
856
|
+
callId: tc.id || generateOutputItemId("call"),
|
|
857
|
+
name: tc.function.name,
|
|
858
|
+
index: idx
|
|
859
|
+
};
|
|
860
|
+
this.state.toolItems.set(idx, tool);
|
|
861
|
+
out.push(sse("response.output_item.added", {
|
|
862
|
+
type: "response.output_item.added",
|
|
863
|
+
item: {
|
|
864
|
+
type: "function_call",
|
|
865
|
+
id: tool.id,
|
|
866
|
+
call_id: tool.callId,
|
|
867
|
+
name: tool.name,
|
|
868
|
+
arguments: ""
|
|
869
|
+
},
|
|
870
|
+
output_index: this.state.textItemId ? 1 : 0
|
|
871
|
+
}));
|
|
872
|
+
}
|
|
873
|
+
if (tool && tc.function && tc.function.arguments) {
|
|
874
|
+
out.push(sse("response.custom_tool_call_input.delta", {
|
|
875
|
+
type: "response.custom_tool_call_input.delta",
|
|
876
|
+
item_id: tool.id,
|
|
877
|
+
call_id: tool.callId,
|
|
878
|
+
delta: tc.function.arguments
|
|
879
|
+
}));
|
|
880
|
+
}
|
|
881
|
+
}
|
|
882
|
+
}
|
|
883
|
+
const u = data.x_groq && data.x_groq.usage ? data.x_groq.usage : data.usage;
|
|
884
|
+
if (u) {
|
|
885
|
+
if (u.prompt_tokens != null) this.state.accUsage.prompt_tokens = u.prompt_tokens;
|
|
886
|
+
if (u.completion_tokens != null) this.state.accUsage.completion_tokens = u.completion_tokens;
|
|
887
|
+
}
|
|
888
|
+
const fr = choice && choice.finish_reason;
|
|
889
|
+
if (fr && !this.state.finished) {
|
|
890
|
+
out.push(...this.finalize());
|
|
891
|
+
this.state.finished = true;
|
|
892
|
+
}
|
|
893
|
+
}
|
|
894
|
+
return out.length ? out : null;
|
|
895
|
+
}
|
|
896
|
+
finalize() {
|
|
897
|
+
const out = [];
|
|
898
|
+
if (this.state.textItemId) {
|
|
899
|
+
if (this.state.textStarted) {
|
|
900
|
+
out.push(sse("response.content_part.done", {
|
|
901
|
+
type: "response.content_part.done",
|
|
902
|
+
item_id: this.state.textItemId,
|
|
903
|
+
output_index: 0,
|
|
904
|
+
content_index: 0,
|
|
905
|
+
part: { type: "output_text", text: "" }
|
|
906
|
+
}));
|
|
907
|
+
}
|
|
908
|
+
out.push(sse("response.output_item.done", {
|
|
909
|
+
type: "response.output_item.done",
|
|
910
|
+
item: {
|
|
911
|
+
type: "message",
|
|
912
|
+
id: this.state.textItemId,
|
|
913
|
+
role: "assistant",
|
|
914
|
+
content: [{ type: "output_text", text: "" }]
|
|
915
|
+
},
|
|
916
|
+
output_index: 0
|
|
917
|
+
}));
|
|
918
|
+
}
|
|
919
|
+
this.state.toolItems.forEach((tool, idx) => {
|
|
920
|
+
out.push(sse("response.output_item.done", {
|
|
921
|
+
type: "response.output_item.done",
|
|
922
|
+
item: {
|
|
923
|
+
type: "function_call",
|
|
924
|
+
id: tool.id,
|
|
925
|
+
call_id: tool.callId,
|
|
926
|
+
name: tool.name,
|
|
927
|
+
arguments: ""
|
|
928
|
+
},
|
|
929
|
+
output_index: this.state.textItemId ? 1 + idx : idx
|
|
930
|
+
}));
|
|
931
|
+
});
|
|
932
|
+
const usage = {
|
|
933
|
+
input_tokens: this.state.accUsage.prompt_tokens || 0,
|
|
934
|
+
input_tokens_details: { cached_tokens: 0 },
|
|
935
|
+
output_tokens: this.state.accUsage.completion_tokens || 0,
|
|
936
|
+
output_tokens_details: { reasoning_tokens: 0 },
|
|
937
|
+
total_tokens: (this.state.accUsage.prompt_tokens || 0) + (this.state.accUsage.completion_tokens || 0)
|
|
938
|
+
};
|
|
939
|
+
out.push(sse("response.completed", {
|
|
940
|
+
type: "response.completed",
|
|
941
|
+
response: {
|
|
942
|
+
id: this.state.responseId,
|
|
943
|
+
object: "response",
|
|
944
|
+
status: "completed",
|
|
945
|
+
output: [],
|
|
946
|
+
usage
|
|
947
|
+
}
|
|
948
|
+
}));
|
|
949
|
+
return out;
|
|
950
|
+
}
|
|
951
|
+
};
|
|
952
|
+
var INTERNAL_TOKEN = randomBytes(32).toString("hex");
|
|
953
|
+
function isInternalLoopbackRequest(headers) {
|
|
954
|
+
const raw = headers["x-proxy-internal"];
|
|
955
|
+
const value = Array.isArray(raw) ? raw[0] : raw;
|
|
956
|
+
return value !== void 0 && value === INTERNAL_TOKEN;
|
|
957
|
+
}
|
|
958
|
+
var BATCH_TTL_MS = 24 * 60 * 60 * 1e3;
|
|
959
|
+
var PRUNE_AFTER_MS = 29 * 24 * 60 * 60 * 1e3;
|
|
960
|
+
var DEFAULT_LOCAL_QUEUE_CONCURRENCY = 4;
|
|
961
|
+
function stateDir() {
|
|
962
|
+
return process.env.LLM_ENDPOINT_STATE_DIR || path.join(homedir(), ".agentproto", "llm-endpoint");
|
|
963
|
+
}
|
|
964
|
+
function batchConcurrency() {
|
|
965
|
+
const raw = process.env.LLM_ENDPOINT_BATCH_CONCURRENCY;
|
|
966
|
+
const n = raw ? Number.parseInt(raw, 10) : NaN;
|
|
967
|
+
return Number.isFinite(n) && n > 0 ? n : DEFAULT_LOCAL_QUEUE_CONCURRENCY;
|
|
968
|
+
}
|
|
969
|
+
function defaultLoopbackPort() {
|
|
970
|
+
const raw = process.env.LLM_ENDPOINT_PORT ?? process.env.PORT;
|
|
971
|
+
const n = raw ? Number.parseInt(raw, 10) : NaN;
|
|
972
|
+
return Number.isFinite(n) ? n : 18090;
|
|
973
|
+
}
|
|
974
|
+
var proxySubBatchSchema = z.object({
|
|
975
|
+
provider: z.enum(["anthropic", "openrouter", "local-queue"]),
|
|
976
|
+
handle: batchHandleSchema,
|
|
977
|
+
customIds: z.array(z.string()),
|
|
978
|
+
cancelError: z.string().nullable()
|
|
979
|
+
});
|
|
980
|
+
var batchResultErrorShapeSchema = z.object({ type: z.string(), message: z.string() });
|
|
981
|
+
var resultLineOutcomeSchema = z.union([
|
|
982
|
+
z.object({ type: z.literal("succeeded"), message: anthropicMessageSchema }),
|
|
983
|
+
z.object({ type: z.literal("errored"), error: batchResultErrorShapeSchema }),
|
|
984
|
+
z.object({ type: z.literal("canceled") }),
|
|
985
|
+
z.object({ type: z.literal("expired") })
|
|
986
|
+
]);
|
|
987
|
+
var cachedResultLineSchema = z.object({
|
|
988
|
+
customId: z.string(),
|
|
989
|
+
result: resultLineOutcomeSchema
|
|
990
|
+
});
|
|
991
|
+
var proxyBatchRecordSchema = z.object({
|
|
992
|
+
id: z.string(),
|
|
993
|
+
createdAt: z.string(),
|
|
994
|
+
expiresAt: z.string(),
|
|
995
|
+
cancelInitiatedAt: z.string().nullable(),
|
|
996
|
+
endedAt: z.string().nullable(),
|
|
997
|
+
subBatches: z.array(proxySubBatchSchema),
|
|
998
|
+
cachedResults: z.array(cachedResultLineSchema).nullable()
|
|
999
|
+
});
|
|
1000
|
+
function newProxyBatchId() {
|
|
1001
|
+
return `msgbatch_${ulid()}`;
|
|
1002
|
+
}
|
|
1003
|
+
var ProxyBatchFileStore = class {
|
|
1004
|
+
constructor(dir) {
|
|
1005
|
+
this.dir = dir;
|
|
1006
|
+
}
|
|
1007
|
+
dir;
|
|
1008
|
+
file(id) {
|
|
1009
|
+
return path.join(this.dir, `${id}.json`);
|
|
1010
|
+
}
|
|
1011
|
+
async save(record) {
|
|
1012
|
+
await mkdir(this.dir, { recursive: true });
|
|
1013
|
+
const target = this.file(record.id);
|
|
1014
|
+
const tmp = `${target}.tmp-${randomBytes(8).toString("hex")}`;
|
|
1015
|
+
await writeFile(tmp, JSON.stringify(record, null, 2), "utf8");
|
|
1016
|
+
await rename(tmp, target);
|
|
1017
|
+
}
|
|
1018
|
+
async remove(id) {
|
|
1019
|
+
await rm(this.file(id), { force: true });
|
|
1020
|
+
}
|
|
1021
|
+
/** Loads a record, pruning (and returning `undefined` for) one older than 29 days. */
|
|
1022
|
+
async load(id) {
|
|
1023
|
+
const raw = await readFile(this.file(id), "utf8").catch(() => void 0);
|
|
1024
|
+
if (raw === void 0) return void 0;
|
|
1025
|
+
const record = proxyBatchRecordSchema.parse(JSON.parse(raw));
|
|
1026
|
+
if (Date.now() - Date.parse(record.createdAt) > PRUNE_AFTER_MS) {
|
|
1027
|
+
await this.remove(id);
|
|
1028
|
+
return void 0;
|
|
1029
|
+
}
|
|
1030
|
+
return record;
|
|
1031
|
+
}
|
|
1032
|
+
async list() {
|
|
1033
|
+
const entries = await readdir(this.dir).catch(() => []);
|
|
1034
|
+
const records = [];
|
|
1035
|
+
for (const entry of entries) {
|
|
1036
|
+
if (!entry.endsWith(".json") || entry.includes(".tmp-")) continue;
|
|
1037
|
+
const record = await this.load(entry.slice(0, -".json".length));
|
|
1038
|
+
if (record) records.push(record);
|
|
1039
|
+
}
|
|
1040
|
+
return records;
|
|
1041
|
+
}
|
|
1042
|
+
};
|
|
1043
|
+
var _proxyStore;
|
|
1044
|
+
function proxyStore() {
|
|
1045
|
+
if (!_proxyStore) _proxyStore = new ProxyBatchFileStore(path.join(stateDir(), "proxy-batches"));
|
|
1046
|
+
return _proxyStore;
|
|
1047
|
+
}
|
|
1048
|
+
var _localQueueStore;
|
|
1049
|
+
function localQueueStore() {
|
|
1050
|
+
if (!_localQueueStore) _localQueueStore = new BatchStore({ stateDir: stateDir() });
|
|
1051
|
+
return _localQueueStore;
|
|
1052
|
+
}
|
|
1053
|
+
var driverOverrides = {};
|
|
1054
|
+
function makeLoopbackComplete(port) {
|
|
1055
|
+
return (body) => new Promise((resolve, reject) => {
|
|
1056
|
+
const payload = JSON.stringify(body);
|
|
1057
|
+
const outgoing = request(
|
|
1058
|
+
{
|
|
1059
|
+
hostname: "127.0.0.1",
|
|
1060
|
+
port,
|
|
1061
|
+
path: "/v1/messages",
|
|
1062
|
+
method: "POST",
|
|
1063
|
+
headers: {
|
|
1064
|
+
"Content-Type": "application/json",
|
|
1065
|
+
"Content-Length": Buffer.byteLength(payload),
|
|
1066
|
+
"X-Proxy-Internal": INTERNAL_TOKEN
|
|
1067
|
+
}
|
|
1068
|
+
},
|
|
1069
|
+
(loopbackRes) => {
|
|
1070
|
+
let data = "";
|
|
1071
|
+
loopbackRes.setEncoding("utf8");
|
|
1072
|
+
loopbackRes.on("data", (chunk) => {
|
|
1073
|
+
data += chunk;
|
|
1074
|
+
});
|
|
1075
|
+
loopbackRes.on("end", () => {
|
|
1076
|
+
const status = loopbackRes.statusCode ?? 0;
|
|
1077
|
+
if (status === 429 || status >= 500) {
|
|
1078
|
+
reject(new RetryableCompletionError(`loopback /v1/messages returned ${status}`, status));
|
|
1079
|
+
return;
|
|
1080
|
+
}
|
|
1081
|
+
if (status >= 400) {
|
|
1082
|
+
reject(new Error(`loopback /v1/messages returned ${status}: ${data.slice(0, 200)}`));
|
|
1083
|
+
return;
|
|
1084
|
+
}
|
|
1085
|
+
let parsed;
|
|
1086
|
+
try {
|
|
1087
|
+
parsed = JSON.parse(data);
|
|
1088
|
+
} catch {
|
|
1089
|
+
reject(new Error("loopback /v1/messages returned invalid JSON"));
|
|
1090
|
+
return;
|
|
1091
|
+
}
|
|
1092
|
+
const result = anthropicMessageSchema.safeParse(parsed);
|
|
1093
|
+
if (!result.success) {
|
|
1094
|
+
reject(new Error("loopback /v1/messages returned an unexpected message shape"));
|
|
1095
|
+
return;
|
|
1096
|
+
}
|
|
1097
|
+
resolve(result.data);
|
|
1098
|
+
});
|
|
1099
|
+
}
|
|
1100
|
+
);
|
|
1101
|
+
outgoing.on("error", reject);
|
|
1102
|
+
outgoing.write(payload);
|
|
1103
|
+
outgoing.end();
|
|
1104
|
+
});
|
|
1105
|
+
}
|
|
1106
|
+
function defaultLocalQueueDriver(opts, port) {
|
|
1107
|
+
return localQueueDriver({ store: opts.store, concurrency: opts.concurrency, complete: makeLoopbackComplete(port) });
|
|
1108
|
+
}
|
|
1109
|
+
function localQueueDriverInstance(port) {
|
|
1110
|
+
const factory = driverOverrides.localQueue ?? ((o) => defaultLocalQueueDriver(o, port));
|
|
1111
|
+
return factory({ store: localQueueStore(), concurrency: batchConcurrency() });
|
|
1112
|
+
}
|
|
1113
|
+
var BatchCredentialError = class extends Error {
|
|
1114
|
+
};
|
|
1115
|
+
async function getDriverForSubBatch(sub, port) {
|
|
1116
|
+
if (sub.provider === "anthropic") {
|
|
1117
|
+
const cred = await resolveUpstreamCredential("anthropic");
|
|
1118
|
+
if (!cred || !cred.value) throw new BatchCredentialError('No API key for provider "anthropic"');
|
|
1119
|
+
if (cred.method !== "api-key") {
|
|
1120
|
+
throw new BatchCredentialError(
|
|
1121
|
+
"Anthropic batches require an API key credential; the resolved credential is a subscription OAuth token, which cannot be used with the Batches API."
|
|
1122
|
+
);
|
|
1123
|
+
}
|
|
1124
|
+
return (driverOverrides.anthropic ?? ((o) => anthropicBatchDriver(o)))({ apiKey: cred.value });
|
|
1125
|
+
}
|
|
1126
|
+
if (sub.provider === "openrouter") {
|
|
1127
|
+
const cred = await resolveUpstreamCredential("openrouter");
|
|
1128
|
+
if (!cred || !cred.value) throw new BatchCredentialError('No API key for provider "openrouter"');
|
|
1129
|
+
return (driverOverrides.openrouter ?? ((o) => openrouterBatchDriver(o)))({ apiKey: cred.value });
|
|
1130
|
+
}
|
|
1131
|
+
return localQueueDriverInstance(port);
|
|
1132
|
+
}
|
|
1133
|
+
function batchDriverKind(provider) {
|
|
1134
|
+
if (provider === "anthropic") return "anthropic";
|
|
1135
|
+
if (provider === "openrouter") return "openrouter";
|
|
1136
|
+
return "local-queue";
|
|
1137
|
+
}
|
|
1138
|
+
async function computeAggregateStatus(record, port) {
|
|
1139
|
+
const counts = { processing: 0, succeeded: 0, errored: 0, canceled: 0, expired: 0 };
|
|
1140
|
+
let allTerminal = true;
|
|
1141
|
+
for (const sub of record.subBatches) {
|
|
1142
|
+
const driver = await getDriverForSubBatch(sub, port);
|
|
1143
|
+
const status = await driver.status(sub.handle);
|
|
1144
|
+
counts.processing += status.counts.processing;
|
|
1145
|
+
counts.succeeded += status.counts.succeeded;
|
|
1146
|
+
counts.errored += status.counts.errored;
|
|
1147
|
+
counts.canceled += status.counts.canceled;
|
|
1148
|
+
counts.expired += status.counts.expired;
|
|
1149
|
+
if (status.state !== "ended" && status.state !== "failed") allTerminal = false;
|
|
1150
|
+
}
|
|
1151
|
+
if (!allTerminal) {
|
|
1152
|
+
return { state: record.cancelInitiatedAt ? "canceling" : "in_progress", counts, endedAt: null };
|
|
1153
|
+
}
|
|
1154
|
+
return { state: "ended", counts, endedAt: record.endedAt ?? (/* @__PURE__ */ new Date()).toISOString() };
|
|
1155
|
+
}
|
|
1156
|
+
async function getStatusAndPersist(record, port) {
|
|
1157
|
+
const status = await computeAggregateStatus(record, port);
|
|
1158
|
+
if (status.endedAt && !record.endedAt) {
|
|
1159
|
+
record.endedAt = status.endedAt;
|
|
1160
|
+
await proxyStore().save(record);
|
|
1161
|
+
}
|
|
1162
|
+
return status;
|
|
1163
|
+
}
|
|
1164
|
+
function toBatchObject(record, status) {
|
|
1165
|
+
return {
|
|
1166
|
+
id: record.id,
|
|
1167
|
+
type: "message_batch",
|
|
1168
|
+
processing_status: status.state,
|
|
1169
|
+
request_counts: status.counts,
|
|
1170
|
+
ended_at: status.endedAt,
|
|
1171
|
+
created_at: record.createdAt,
|
|
1172
|
+
expires_at: record.expiresAt,
|
|
1173
|
+
cancel_initiated_at: record.cancelInitiatedAt,
|
|
1174
|
+
results_url: status.state === "ended" ? `/v1/messages/batches/${record.id}/results` : null,
|
|
1175
|
+
archived_at: null
|
|
1176
|
+
};
|
|
1177
|
+
}
|
|
1178
|
+
function toResultShape(outcome, message, error) {
|
|
1179
|
+
switch (outcome) {
|
|
1180
|
+
case "succeeded":
|
|
1181
|
+
if (!message) return { type: "errored", error: { type: "api_error", message: "succeeded result missing a message" } };
|
|
1182
|
+
return { type: "succeeded", message };
|
|
1183
|
+
case "errored":
|
|
1184
|
+
return { type: "errored", error: error ?? { type: "api_error", message: "unknown error" } };
|
|
1185
|
+
case "canceled":
|
|
1186
|
+
return { type: "canceled" };
|
|
1187
|
+
case "expired":
|
|
1188
|
+
return { type: "expired" };
|
|
1189
|
+
}
|
|
1190
|
+
}
|
|
1191
|
+
var batchCreateItemSchema = z.object({
|
|
1192
|
+
custom_id: z.string().min(1),
|
|
1193
|
+
params: messagesBodySchema
|
|
1194
|
+
});
|
|
1195
|
+
var batchCreateBodySchema = z.object({
|
|
1196
|
+
requests: z.array(batchCreateItemSchema).min(1)
|
|
1197
|
+
});
|
|
1198
|
+
function describeZodError(error) {
|
|
1199
|
+
return error.issues.map((issue) => `${issue.path.join(".") || "(root)"}: ${issue.message}`).join("; ");
|
|
1200
|
+
}
|
|
1201
|
+
function respondJson(res, status, body) {
|
|
1202
|
+
res.writeHead(status, { "Content-Type": "application/json" });
|
|
1203
|
+
res.end(JSON.stringify(body));
|
|
1204
|
+
}
|
|
1205
|
+
function invalidRequest(message) {
|
|
1206
|
+
return { error: { type: "invalid_request_error", message } };
|
|
1207
|
+
}
|
|
1208
|
+
function notFoundBatch(id) {
|
|
1209
|
+
return { error: { type: "not_found_error", message: `Batch "${id}" not found.` } };
|
|
1210
|
+
}
|
|
1211
|
+
function readRequestBody(req) {
|
|
1212
|
+
return new Promise((resolve, reject) => {
|
|
1213
|
+
let data = "";
|
|
1214
|
+
req.on("data", (chunk) => {
|
|
1215
|
+
data += chunk.toString("utf8");
|
|
1216
|
+
});
|
|
1217
|
+
req.on("end", () => resolve(data));
|
|
1218
|
+
req.on("error", reject);
|
|
1219
|
+
});
|
|
1220
|
+
}
|
|
1221
|
+
var BATCHES_PATH_MARKER = "/messages/batches";
|
|
1222
|
+
function matchBatchesPath(urlPath) {
|
|
1223
|
+
const idx = urlPath.indexOf(BATCHES_PATH_MARKER);
|
|
1224
|
+
if (idx === -1) return null;
|
|
1225
|
+
const rest = urlPath.slice(idx + BATCHES_PATH_MARKER.length);
|
|
1226
|
+
if (rest === "") return { kind: "collection" };
|
|
1227
|
+
const segments = rest.split("/").filter((s) => s.length > 0);
|
|
1228
|
+
const id = segments[0];
|
|
1229
|
+
if (id === void 0) return null;
|
|
1230
|
+
if (segments.length === 1) return { kind: "item", id };
|
|
1231
|
+
const sub = segments[1];
|
|
1232
|
+
if (segments.length === 2 && sub === "results") return { kind: "results", id };
|
|
1233
|
+
if (segments.length === 2 && sub === "cancel") return { kind: "cancel", id };
|
|
1234
|
+
return null;
|
|
1235
|
+
}
|
|
1236
|
+
function handleBatchesRequest(req, res, ctx, urlPath) {
|
|
1237
|
+
const match = matchBatchesPath(urlPath);
|
|
1238
|
+
if (!match) return false;
|
|
1239
|
+
void dispatchBatchesRequest(req, res, ctx, match).catch((err) => {
|
|
1240
|
+
if (err instanceof BatchCredentialError) {
|
|
1241
|
+
respondJson(res, 401, { error: { type: "authentication_error", message: err.message } });
|
|
1242
|
+
return;
|
|
1243
|
+
}
|
|
1244
|
+
console.error("[Proxy][batches] unhandled error", err);
|
|
1245
|
+
if (res.headersSent) {
|
|
1246
|
+
res.end();
|
|
1247
|
+
return;
|
|
1248
|
+
}
|
|
1249
|
+
respondJson(res, 500, { error: { type: "api_error", message: err instanceof Error ? err.message : "internal error" } });
|
|
1250
|
+
});
|
|
1251
|
+
return true;
|
|
1252
|
+
}
|
|
1253
|
+
async function dispatchBatchesRequest(req, res, ctx, match) {
|
|
1254
|
+
const method = req.method ?? "GET";
|
|
1255
|
+
const port = req.socket.localPort ?? defaultLoopbackPort();
|
|
1256
|
+
if (match.kind === "collection") {
|
|
1257
|
+
if (method === "POST") return handleCreate(req, res, ctx, port);
|
|
1258
|
+
if (method === "GET") return handleList(res, ctx.parsedUrl, port);
|
|
1259
|
+
return respondJson(res, 404, { error: { type: "not_found", message: "Route not found" } });
|
|
1260
|
+
}
|
|
1261
|
+
if (match.kind === "item") {
|
|
1262
|
+
if (method === "GET") return handleRetrieve(res, match.id, port);
|
|
1263
|
+
if (method === "DELETE") return handleDeleteBatch(res, match.id, port);
|
|
1264
|
+
return respondJson(res, 404, { error: { type: "not_found", message: "Route not found" } });
|
|
1265
|
+
}
|
|
1266
|
+
if (match.kind === "results") {
|
|
1267
|
+
if (method === "GET") return handleResults(res, match.id, port);
|
|
1268
|
+
return respondJson(res, 404, { error: { type: "not_found", message: "Route not found" } });
|
|
1269
|
+
}
|
|
1270
|
+
if (method === "POST") return handleCancel(res, match.id, port);
|
|
1271
|
+
return respondJson(res, 404, { error: { type: "not_found", message: "Route not found" } });
|
|
1272
|
+
}
|
|
1273
|
+
async function handleCreate(req, res, ctx, port) {
|
|
1274
|
+
const raw = await readRequestBody(req);
|
|
1275
|
+
let json;
|
|
1276
|
+
try {
|
|
1277
|
+
json = JSON.parse(raw);
|
|
1278
|
+
} catch {
|
|
1279
|
+
respondJson(res, 400, invalidRequest("Invalid JSON body"));
|
|
1280
|
+
return;
|
|
1281
|
+
}
|
|
1282
|
+
const parsedBody = batchCreateBodySchema.safeParse(json);
|
|
1283
|
+
if (!parsedBody.success) {
|
|
1284
|
+
respondJson(res, 400, invalidRequest(describeZodError(parsedBody.error)));
|
|
1285
|
+
return;
|
|
1286
|
+
}
|
|
1287
|
+
const items = parsedBody.data.requests;
|
|
1288
|
+
try {
|
|
1289
|
+
assertUniqueCustomIds(items.map((item) => ({ customId: item.custom_id, body: item.params })));
|
|
1290
|
+
} catch (err) {
|
|
1291
|
+
if (!(err instanceof BatchValidationError)) throw err;
|
|
1292
|
+
respondJson(res, 400, invalidRequest(err.message));
|
|
1293
|
+
return;
|
|
1294
|
+
}
|
|
1295
|
+
const problems = [];
|
|
1296
|
+
for (const item of items) {
|
|
1297
|
+
try {
|
|
1298
|
+
validateForBatch({ customId: item.custom_id, body: item.params });
|
|
1299
|
+
} catch (err) {
|
|
1300
|
+
if (!(err instanceof BatchValidationError)) throw err;
|
|
1301
|
+
problems.push(err.message);
|
|
1302
|
+
}
|
|
1303
|
+
}
|
|
1304
|
+
if (problems.length > 0) {
|
|
1305
|
+
respondJson(res, 400, invalidRequest(problems.join("; ")));
|
|
1306
|
+
return;
|
|
1307
|
+
}
|
|
1308
|
+
const routeCtx = {
|
|
1309
|
+
activePack: ctx.activePack,
|
|
1310
|
+
queryModelCode: ctx.queryModelCode,
|
|
1311
|
+
queryProvider: ctx.queryProvider,
|
|
1312
|
+
forcedAliasCode: ctx.forcedAliasCode,
|
|
1313
|
+
allowAliases: true,
|
|
1314
|
+
anthropicFormat: ctx.anthropicFormat
|
|
1315
|
+
};
|
|
1316
|
+
const resolved = [];
|
|
1317
|
+
for (const item of items) {
|
|
1318
|
+
let target;
|
|
1319
|
+
try {
|
|
1320
|
+
target = resolveModelRoute(item.params, routeCtx);
|
|
1321
|
+
} catch (err) {
|
|
1322
|
+
problems.push(`"${item.custom_id}": ${err instanceof Error ? err.message : String(err)}`);
|
|
1323
|
+
continue;
|
|
1324
|
+
}
|
|
1325
|
+
const body = { ...item.params, model: target.model };
|
|
1326
|
+
trimTools(body, {
|
|
1327
|
+
provider: target.provider,
|
|
1328
|
+
queryTools: ctx.queryTools,
|
|
1329
|
+
queryNoTools: ctx.queryNoTools,
|
|
1330
|
+
headerTools: ctx.headerTools,
|
|
1331
|
+
headerNoTools: ctx.headerNoTools,
|
|
1332
|
+
headerExcludeTools: ctx.headerExcludeTools
|
|
1333
|
+
});
|
|
1334
|
+
resolved.push({ customId: item.custom_id, provider: target.provider, body });
|
|
1335
|
+
}
|
|
1336
|
+
if (problems.length > 0) {
|
|
1337
|
+
respondJson(res, 400, invalidRequest(problems.join("; ")));
|
|
1338
|
+
return;
|
|
1339
|
+
}
|
|
1340
|
+
const groups = /* @__PURE__ */ new Map();
|
|
1341
|
+
for (const item of resolved) {
|
|
1342
|
+
const kind = batchDriverKind(item.provider);
|
|
1343
|
+
const body = kind === "local-queue" ? { ...item.body, model: `${item.provider}/${item.body.model}` } : item.body;
|
|
1344
|
+
const list = groups.get(kind);
|
|
1345
|
+
if (list) list.push({ customId: item.customId, body });
|
|
1346
|
+
else groups.set(kind, [{ customId: item.customId, body }]);
|
|
1347
|
+
}
|
|
1348
|
+
const credentials = /* @__PURE__ */ new Map();
|
|
1349
|
+
for (const kind of groups.keys()) {
|
|
1350
|
+
if (kind !== "anthropic" && kind !== "openrouter") continue;
|
|
1351
|
+
const cred = await resolveUpstreamCredential(kind);
|
|
1352
|
+
if (!cred || !cred.value) {
|
|
1353
|
+
respondJson(res, 401, { error: { type: "authentication_error", message: `No API key for provider "${kind}"` } });
|
|
1354
|
+
return;
|
|
1355
|
+
}
|
|
1356
|
+
if (kind === "anthropic" && cred.method !== "api-key") {
|
|
1357
|
+
respondJson(res, 401, {
|
|
1358
|
+
error: {
|
|
1359
|
+
type: "authentication_error",
|
|
1360
|
+
message: "Anthropic batches require an API key credential; the resolved credential is a subscription OAuth token, which cannot be used with the Batches API."
|
|
1361
|
+
}
|
|
1362
|
+
});
|
|
1363
|
+
return;
|
|
1364
|
+
}
|
|
1365
|
+
credentials.set(kind, cred);
|
|
1366
|
+
}
|
|
1367
|
+
const subBatches = [];
|
|
1368
|
+
for (const [kind, requests] of groups) {
|
|
1369
|
+
if (kind === "anthropic") {
|
|
1370
|
+
const cred = credentials.get("anthropic");
|
|
1371
|
+
if (!cred) throw new Error("anthropic credential resolved but missing from cache");
|
|
1372
|
+
const driver = (driverOverrides.anthropic ?? ((o) => anthropicBatchDriver(o)))({ apiKey: cred.value });
|
|
1373
|
+
const handle = await driver.submit(requests);
|
|
1374
|
+
subBatches.push({ provider: "anthropic", handle, customIds: requests.map((r) => r.customId), cancelError: null });
|
|
1375
|
+
} else if (kind === "openrouter") {
|
|
1376
|
+
const cred = credentials.get("openrouter");
|
|
1377
|
+
if (!cred) throw new Error("openrouter credential resolved but missing from cache");
|
|
1378
|
+
const driver = (driverOverrides.openrouter ?? ((o) => openrouterBatchDriver(o)))({ apiKey: cred.value });
|
|
1379
|
+
const handle = await driver.submit(requests);
|
|
1380
|
+
subBatches.push({ provider: "openrouter", handle, customIds: requests.map((r) => r.customId), cancelError: null });
|
|
1381
|
+
} else {
|
|
1382
|
+
const handle = {
|
|
1383
|
+
id: newBatchId(),
|
|
1384
|
+
driver: "local-queue",
|
|
1385
|
+
provider: { batchIds: [] },
|
|
1386
|
+
createdAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
1387
|
+
requestCount: requests.length,
|
|
1388
|
+
models: Array.from(new Set(requests.map((r) => r.body.model)))
|
|
1389
|
+
};
|
|
1390
|
+
await localQueueStore().create(handle, requests);
|
|
1391
|
+
const driver = localQueueDriverInstance(port);
|
|
1392
|
+
void driver.resume(handle).catch((err) => {
|
|
1393
|
+
console.error(`[Proxy][batches] local-queue batch "${handle.id}" failed`, err);
|
|
1394
|
+
});
|
|
1395
|
+
subBatches.push({ provider: "local-queue", handle, customIds: requests.map((r) => r.customId), cancelError: null });
|
|
1396
|
+
}
|
|
1397
|
+
}
|
|
1398
|
+
const now = /* @__PURE__ */ new Date();
|
|
1399
|
+
const record = {
|
|
1400
|
+
id: newProxyBatchId(),
|
|
1401
|
+
createdAt: now.toISOString(),
|
|
1402
|
+
expiresAt: new Date(now.getTime() + BATCH_TTL_MS).toISOString(),
|
|
1403
|
+
cancelInitiatedAt: null,
|
|
1404
|
+
endedAt: null,
|
|
1405
|
+
subBatches,
|
|
1406
|
+
cachedResults: null
|
|
1407
|
+
};
|
|
1408
|
+
await proxyStore().save(record);
|
|
1409
|
+
const status = await getStatusAndPersist(record, port);
|
|
1410
|
+
respondJson(res, 200, toBatchObject(record, status));
|
|
1411
|
+
}
|
|
1412
|
+
async function handleRetrieve(res, id, port) {
|
|
1413
|
+
const record = await proxyStore().load(id);
|
|
1414
|
+
if (!record) {
|
|
1415
|
+
respondJson(res, 404, notFoundBatch(id));
|
|
1416
|
+
return;
|
|
1417
|
+
}
|
|
1418
|
+
const status = await getStatusAndPersist(record, port);
|
|
1419
|
+
respondJson(res, 200, toBatchObject(record, status));
|
|
1420
|
+
}
|
|
1421
|
+
async function handleList(res, parsedUrl, port) {
|
|
1422
|
+
const records = await proxyStore().list();
|
|
1423
|
+
records.sort((a, b) => b.createdAt.localeCompare(a.createdAt));
|
|
1424
|
+
const limitParam = parsedUrl.searchParams.get("limit");
|
|
1425
|
+
const parsedLimit = limitParam ? Number.parseInt(limitParam, 10) : NaN;
|
|
1426
|
+
const limit = Number.isFinite(parsedLimit) && parsedLimit > 0 ? parsedLimit : records.length;
|
|
1427
|
+
const page = records.slice(0, limit);
|
|
1428
|
+
const data = [];
|
|
1429
|
+
for (const record of page) {
|
|
1430
|
+
const status = await getStatusAndPersist(record, port);
|
|
1431
|
+
data.push(toBatchObject(record, status));
|
|
1432
|
+
}
|
|
1433
|
+
respondJson(res, 200, {
|
|
1434
|
+
data,
|
|
1435
|
+
has_more: false,
|
|
1436
|
+
first_id: data.length > 0 ? data[0].id : null,
|
|
1437
|
+
last_id: data.length > 0 ? data[data.length - 1].id : null
|
|
1438
|
+
});
|
|
1439
|
+
}
|
|
1440
|
+
async function handleResults(res, id, port) {
|
|
1441
|
+
const record = await proxyStore().load(id);
|
|
1442
|
+
if (!record) {
|
|
1443
|
+
respondJson(res, 404, notFoundBatch(id));
|
|
1444
|
+
return;
|
|
1445
|
+
}
|
|
1446
|
+
const status = await getStatusAndPersist(record, port);
|
|
1447
|
+
if (status.state !== "ended") {
|
|
1448
|
+
respondJson(res, 404, {
|
|
1449
|
+
error: { type: "not_found_error", message: `Batch "${id}" results are not available until processing_status is "ended".` }
|
|
1450
|
+
});
|
|
1451
|
+
return;
|
|
1452
|
+
}
|
|
1453
|
+
let lines = record.cachedResults;
|
|
1454
|
+
if (!lines) {
|
|
1455
|
+
lines = [];
|
|
1456
|
+
for (const sub of record.subBatches) {
|
|
1457
|
+
const driver = await getDriverForSubBatch(sub, port);
|
|
1458
|
+
for await (const r of driver.results(sub.handle)) {
|
|
1459
|
+
let message = r.message;
|
|
1460
|
+
if (sub.provider === "openrouter" && message) {
|
|
1461
|
+
const stripped = JSON.parse(stripThinkingFromAnthropicJson(JSON.stringify(message)));
|
|
1462
|
+
const strippedResult = anthropicMessageSchema.safeParse(stripped);
|
|
1463
|
+
if (strippedResult.success) message = strippedResult.data;
|
|
1464
|
+
}
|
|
1465
|
+
lines.push({ customId: r.customId, result: toResultShape(r.outcome, message, r.error) });
|
|
1466
|
+
}
|
|
1467
|
+
}
|
|
1468
|
+
record.cachedResults = lines;
|
|
1469
|
+
await proxyStore().save(record);
|
|
1470
|
+
}
|
|
1471
|
+
res.writeHead(200, { "Content-Type": "application/x-jsonl; charset=utf-8" });
|
|
1472
|
+
for (const line of lines) {
|
|
1473
|
+
res.write(`${JSON.stringify({ custom_id: line.customId, result: line.result })}
|
|
1474
|
+
`);
|
|
1475
|
+
}
|
|
1476
|
+
res.end();
|
|
1477
|
+
}
|
|
1478
|
+
async function handleCancel(res, id, port) {
|
|
1479
|
+
const record = await proxyStore().load(id);
|
|
1480
|
+
if (!record) {
|
|
1481
|
+
respondJson(res, 404, notFoundBatch(id));
|
|
1482
|
+
return;
|
|
1483
|
+
}
|
|
1484
|
+
if (!record.cancelInitiatedAt) record.cancelInitiatedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
1485
|
+
for (const sub of record.subBatches) {
|
|
1486
|
+
const driver = await getDriverForSubBatch(sub, port);
|
|
1487
|
+
try {
|
|
1488
|
+
await driver.cancel(sub.handle);
|
|
1489
|
+
} catch (err) {
|
|
1490
|
+
if (!(err instanceof BatchUnsupportedError)) throw err;
|
|
1491
|
+
sub.cancelError = err.message;
|
|
1492
|
+
}
|
|
1493
|
+
}
|
|
1494
|
+
await proxyStore().save(record);
|
|
1495
|
+
const status = await getStatusAndPersist(record, port);
|
|
1496
|
+
const forced = status.state === "ended" ? status : { ...status, state: "canceling" };
|
|
1497
|
+
respondJson(res, 200, toBatchObject(record, forced));
|
|
1498
|
+
}
|
|
1499
|
+
async function bestEffortDeleteAnthropicBatch(sub) {
|
|
1500
|
+
const cred = await resolveUpstreamCredential("anthropic");
|
|
1501
|
+
if (!cred || !cred.value || cred.method !== "api-key") return;
|
|
1502
|
+
const providerId = sub.handle.provider.batchIds[0];
|
|
1503
|
+
if (!providerId) return;
|
|
1504
|
+
await fetch(`https://api.anthropic.com/v1/messages/batches/${providerId}`, {
|
|
1505
|
+
method: "DELETE",
|
|
1506
|
+
headers: { "x-api-key": cred.value, "anthropic-version": "2023-06-01" }
|
|
1507
|
+
}).catch(() => void 0);
|
|
1508
|
+
}
|
|
1509
|
+
async function handleDeleteBatch(res, id, port) {
|
|
1510
|
+
const record = await proxyStore().load(id);
|
|
1511
|
+
if (!record) {
|
|
1512
|
+
respondJson(res, 404, notFoundBatch(id));
|
|
1513
|
+
return;
|
|
1514
|
+
}
|
|
1515
|
+
const status = await getStatusAndPersist(record, port);
|
|
1516
|
+
if (status.state !== "ended") {
|
|
1517
|
+
respondJson(res, 409, invalidRequest(`Batch "${id}" cannot be deleted before it has ended.`));
|
|
1518
|
+
return;
|
|
1519
|
+
}
|
|
1520
|
+
for (const sub of record.subBatches) {
|
|
1521
|
+
if (sub.provider === "anthropic") await bestEffortDeleteAnthropicBatch(sub);
|
|
1522
|
+
}
|
|
1523
|
+
await proxyStore().remove(id);
|
|
1524
|
+
respondJson(res, 200, { id, type: "message_batch_deleted" });
|
|
1525
|
+
}
|
|
1526
|
+
async function resumeIncompleteLocalQueueBatches() {
|
|
1527
|
+
const port = defaultLoopbackPort();
|
|
1528
|
+
const records = await proxyStore().list();
|
|
1529
|
+
for (const record of records) {
|
|
1530
|
+
for (const sub of record.subBatches) {
|
|
1531
|
+
if (sub.provider !== "local-queue") continue;
|
|
1532
|
+
const driver = localQueueDriverInstance(port);
|
|
1533
|
+
void driver.resume(sub.handle).catch((err) => {
|
|
1534
|
+
console.error(`[Proxy][batches] failed to resume local-queue batch "${sub.handle.id}" on boot`, err);
|
|
1535
|
+
});
|
|
1536
|
+
}
|
|
1537
|
+
}
|
|
1538
|
+
}
|
|
1539
|
+
var FORBIDDEN_DEFAULT_REQUEST_FIELD_KEYS = /* @__PURE__ */ new Set(["model", "messages", "stream", "tools", "input"]);
|
|
1540
|
+
var RESERVED_ENDPOINT_IDS = /* @__PURE__ */ new Set(["forge"]);
|
|
1541
|
+
function validateEndpointConfig(raw, where, errors) {
|
|
1542
|
+
if (!isRecord(raw)) {
|
|
1543
|
+
errors.push(`${where}: expected an object, got ${raw === null ? "null" : typeof raw}`);
|
|
1544
|
+
return null;
|
|
1545
|
+
}
|
|
1546
|
+
const { id, kind, baseUrl, apiKeyEnv, defaultRequestFields, timeoutMs } = raw;
|
|
1547
|
+
let ok = true;
|
|
1548
|
+
if (typeof id !== "string" || id.length === 0) {
|
|
1549
|
+
errors.push(`${where}.id: required non-empty string`);
|
|
1550
|
+
ok = false;
|
|
1551
|
+
}
|
|
1552
|
+
if (kind !== "openai") {
|
|
1553
|
+
errors.push(`${where}.kind: must be "openai" (got ${JSON.stringify(kind)})`);
|
|
1554
|
+
ok = false;
|
|
1555
|
+
}
|
|
1556
|
+
let validUrl = false;
|
|
1557
|
+
if (typeof baseUrl !== "string" || baseUrl.length === 0) {
|
|
1558
|
+
errors.push(`${where}.baseUrl: required non-empty string`);
|
|
1559
|
+
ok = false;
|
|
1560
|
+
} else {
|
|
1561
|
+
try {
|
|
1562
|
+
const parsed = new URL(baseUrl);
|
|
1563
|
+
if (parsed.protocol === "http:" || parsed.protocol === "https:") {
|
|
1564
|
+
validUrl = true;
|
|
1565
|
+
} else {
|
|
1566
|
+
errors.push(`${where}.baseUrl: must use http:// or https:// (got "${baseUrl}")`);
|
|
1567
|
+
ok = false;
|
|
1568
|
+
}
|
|
1569
|
+
} catch {
|
|
1570
|
+
errors.push(`${where}.baseUrl: "${baseUrl}" is not a valid URL`);
|
|
1571
|
+
ok = false;
|
|
1572
|
+
}
|
|
1573
|
+
}
|
|
1574
|
+
if (apiKeyEnv !== void 0 && (typeof apiKeyEnv !== "string" || apiKeyEnv.length === 0)) {
|
|
1575
|
+
errors.push(`${where}.apiKeyEnv: must be a non-empty string when present`);
|
|
1576
|
+
ok = false;
|
|
1577
|
+
}
|
|
1578
|
+
let builtFields;
|
|
1579
|
+
if (defaultRequestFields !== void 0) {
|
|
1580
|
+
if (!isRecord(defaultRequestFields)) {
|
|
1581
|
+
errors.push(`${where}.defaultRequestFields: must be an object when present`);
|
|
1582
|
+
ok = false;
|
|
1583
|
+
} else {
|
|
1584
|
+
const forbiddenKeys = Object.keys(defaultRequestFields).filter((k) => FORBIDDEN_DEFAULT_REQUEST_FIELD_KEYS.has(k));
|
|
1585
|
+
if (forbiddenKeys.length > 0) {
|
|
1586
|
+
errors.push(`${where}.defaultRequestFields: cannot set routing/auth field(s): ${forbiddenKeys.join(", ")}`);
|
|
1587
|
+
ok = false;
|
|
1588
|
+
} else if (defaultRequestFields.chat_template_kwargs !== void 0 && !isRecord(defaultRequestFields.chat_template_kwargs)) {
|
|
1589
|
+
errors.push(`${where}.defaultRequestFields.chat_template_kwargs: must be an object when present`);
|
|
1590
|
+
ok = false;
|
|
1591
|
+
} else {
|
|
1592
|
+
builtFields = defaultRequestFields;
|
|
1593
|
+
}
|
|
1594
|
+
}
|
|
1595
|
+
}
|
|
1596
|
+
let builtTimeout;
|
|
1597
|
+
if (timeoutMs !== void 0) {
|
|
1598
|
+
if (!isRecord(timeoutMs)) {
|
|
1599
|
+
errors.push(`${where}.timeoutMs: must be an object when present`);
|
|
1600
|
+
ok = false;
|
|
1601
|
+
} else if (timeoutMs.firstTokenMs !== void 0 && !(typeof timeoutMs.firstTokenMs === "number" && Number.isFinite(timeoutMs.firstTokenMs) && timeoutMs.firstTokenMs > 0)) {
|
|
1602
|
+
errors.push(`${where}.timeoutMs.firstTokenMs: must be a positive finite number when present`);
|
|
1603
|
+
ok = false;
|
|
1604
|
+
} else if (typeof timeoutMs.firstTokenMs === "number") {
|
|
1605
|
+
builtTimeout = { firstTokenMs: timeoutMs.firstTokenMs };
|
|
1606
|
+
}
|
|
1607
|
+
}
|
|
1608
|
+
if (!ok || typeof id !== "string" || typeof baseUrl !== "string" || !validUrl) return null;
|
|
1609
|
+
const built = { id, kind: "openai", baseUrl };
|
|
1610
|
+
if (typeof apiKeyEnv === "string") built.apiKeyEnv = apiKeyEnv;
|
|
1611
|
+
if (builtFields) built.defaultRequestFields = builtFields;
|
|
1612
|
+
if (builtTimeout) built.timeoutMs = builtTimeout;
|
|
1613
|
+
return built;
|
|
1614
|
+
}
|
|
1615
|
+
function parseEndpointsConfig(parsed) {
|
|
1616
|
+
if (!isRecord(parsed) || !Array.isArray(parsed.endpoints)) {
|
|
1617
|
+
return { endpoints: [], errors: ['root: expected an object with an "endpoints" array'] };
|
|
1618
|
+
}
|
|
1619
|
+
const errors = [];
|
|
1620
|
+
const endpoints = [];
|
|
1621
|
+
const seenIds = new Set(RESERVED_ENDPOINT_IDS);
|
|
1622
|
+
for (const [i, raw] of parsed.endpoints.entries()) {
|
|
1623
|
+
const built = validateEndpointConfig(raw, `endpoints[${i}]`, errors);
|
|
1624
|
+
if (!built) continue;
|
|
1625
|
+
if (seenIds.has(built.id)) {
|
|
1626
|
+
errors.push(
|
|
1627
|
+
`endpoints[${i}].id: duplicate endpoint id "${built.id}"` + (RESERVED_ENDPOINT_IDS.has(built.id) ? ' (reserved \u2014 "forge" is always the implicit FORGE_BASE_URL endpoint)' : "")
|
|
1628
|
+
);
|
|
1629
|
+
continue;
|
|
1630
|
+
}
|
|
1631
|
+
seenIds.add(built.id);
|
|
1632
|
+
endpoints.push(built);
|
|
1633
|
+
}
|
|
1634
|
+
if (errors.length > 0) return { endpoints: [], errors };
|
|
1635
|
+
return { endpoints, errors: [] };
|
|
1636
|
+
}
|
|
1637
|
+
function resolveEndpointsFilePath() {
|
|
1638
|
+
const override = process.env.LLM_ENDPOINT_ENDPOINTS_FILE?.trim();
|
|
1639
|
+
if (override) return override;
|
|
1640
|
+
return resolve(homedir(), ".agentproto", "llm-endpoints.json");
|
|
1641
|
+
}
|
|
1642
|
+
function readEndpointsFromDisk(path2 = resolveEndpointsFilePath()) {
|
|
1643
|
+
let raw;
|
|
1644
|
+
try {
|
|
1645
|
+
raw = readFileSync(path2, "utf-8");
|
|
1646
|
+
} catch {
|
|
1647
|
+
return { endpoints: [], errors: [], path: path2 };
|
|
1648
|
+
}
|
|
1649
|
+
let parsed;
|
|
1650
|
+
try {
|
|
1651
|
+
parsed = JSON.parse(raw);
|
|
1652
|
+
} catch (err) {
|
|
1653
|
+
return { endpoints: [], errors: [`${path2}: invalid JSON \u2014 ${err instanceof Error ? err.message : String(err)}`], path: path2 };
|
|
1654
|
+
}
|
|
1655
|
+
const result = parseEndpointsConfig(parsed);
|
|
1656
|
+
return { ...result, path: path2 };
|
|
1657
|
+
}
|
|
1658
|
+
var _endpointsCache = null;
|
|
1659
|
+
function getConfiguredEndpoints() {
|
|
1660
|
+
if (_endpointsCache !== null) return _endpointsCache;
|
|
1661
|
+
const { endpoints, errors, path: path2 } = readEndpointsFromDisk();
|
|
1662
|
+
for (const e of errors) {
|
|
1663
|
+
console.warn(`[llm-endpoint] Skipping invalid endpoints file ${path2} \u2014 ${e}`);
|
|
1664
|
+
}
|
|
1665
|
+
_endpointsCache = endpoints;
|
|
1666
|
+
return _endpointsCache;
|
|
1667
|
+
}
|
|
1668
|
+
function parseUpstreamUrl(value) {
|
|
1669
|
+
let parsed;
|
|
1670
|
+
try {
|
|
1671
|
+
parsed = new URL(value);
|
|
1672
|
+
} catch {
|
|
1673
|
+
return null;
|
|
1674
|
+
}
|
|
1675
|
+
if (parsed.protocol !== "http:" && parsed.protocol !== "https:") return null;
|
|
1676
|
+
const protocol = parsed.protocol === "http:" ? "http" : "https";
|
|
1677
|
+
const port = parsed.port ? Number(parsed.port) : protocol === "https" ? 443 : 80;
|
|
1678
|
+
const pathPrefix = parsed.pathname.replace(/\/+$/, "");
|
|
1679
|
+
return { hostname: parsed.hostname, port, protocol, pathPrefix };
|
|
1680
|
+
}
|
|
1681
|
+
|
|
1682
|
+
// src/index.ts
|
|
1683
|
+
var PORT = Number(process.env.LLM_ENDPOINT_PORT ?? process.env.PORT ?? 18090);
|
|
1684
|
+
var _mergedPackIdsCache = null;
|
|
1685
|
+
function getMergedPackIds() {
|
|
1686
|
+
if (_mergedPackIdsCache !== null) return _mergedPackIdsCache;
|
|
1687
|
+
_mergedPackIdsCache = [.../* @__PURE__ */ new Set([...listPackIds(), ...Object.keys(getLocalPacks())])];
|
|
1688
|
+
return _mergedPackIdsCache;
|
|
1689
|
+
}
|
|
1690
|
+
function resolvePackMerged(packId) {
|
|
1691
|
+
if (!packId) packId = DEFAULT_PACK_ID;
|
|
1692
|
+
const local = getLocalPacks()[packId];
|
|
1693
|
+
if (local) return local;
|
|
1694
|
+
return resolvePack(packId);
|
|
1695
|
+
}
|
|
1696
|
+
var _localPacksCache = null;
|
|
1697
|
+
function readLocalPacksFromDisk() {
|
|
1698
|
+
const moduleDir = fileURLToPath(new URL(".", import.meta.url));
|
|
1699
|
+
const candidates = [
|
|
1700
|
+
resolve(process.cwd(), "packs.local.json"),
|
|
1701
|
+
resolve(process.cwd(), "src", "packs.local.json"),
|
|
1702
|
+
resolve(moduleDir, "packs.local.json")
|
|
1703
|
+
];
|
|
1704
|
+
for (const localPath of candidates) {
|
|
1705
|
+
let raw;
|
|
1706
|
+
try {
|
|
1707
|
+
raw = readFileSync(localPath, "utf-8");
|
|
1708
|
+
} catch {
|
|
1709
|
+
continue;
|
|
1710
|
+
}
|
|
1711
|
+
let parsed;
|
|
1712
|
+
try {
|
|
1713
|
+
parsed = JSON.parse(raw);
|
|
1714
|
+
} catch (err) {
|
|
1715
|
+
return { packs: {}, errors: [`${localPath}: invalid JSON \u2014 ${err instanceof Error ? err.message : String(err)}`], path: localPath };
|
|
1716
|
+
}
|
|
1717
|
+
if (!isRecord(parsed) || !("packs" in parsed) || !parsed.packs) {
|
|
1718
|
+
continue;
|
|
1719
|
+
}
|
|
1720
|
+
const result = validateLocalPacks(parsed);
|
|
1721
|
+
if (!result.ok) {
|
|
1722
|
+
return { packs: {}, errors: result.errors.map((e) => `${localPath}: ${e}`), path: localPath };
|
|
1723
|
+
}
|
|
1724
|
+
const providerErrors = validateLocalPackProviders(result.packs);
|
|
1725
|
+
if (providerErrors.length > 0) {
|
|
1726
|
+
return { packs: {}, errors: providerErrors.map((e) => `${localPath}: ${e}`), path: localPath };
|
|
1727
|
+
}
|
|
1728
|
+
return { packs: result.packs, errors: [], path: localPath };
|
|
1729
|
+
}
|
|
1730
|
+
return { packs: {}, errors: [], path: null };
|
|
1731
|
+
}
|
|
1732
|
+
function validateLocalPackProviders(packs) {
|
|
1733
|
+
const errors = [];
|
|
1734
|
+
for (const [packId, pack] of Object.entries(packs)) {
|
|
1735
|
+
for (const [code, route] of Object.entries(pack.models)) {
|
|
1736
|
+
if (!isKnownProvider(route.provider)) {
|
|
1737
|
+
errors.push(
|
|
1738
|
+
`packs.${packId}.models.${code}.provider: unknown provider "${route.provider}" (known: ${[...KNOWN_PROVIDERS, ...getConfiguredEndpoints().map((e) => e.id)].join(", ")})`
|
|
1739
|
+
);
|
|
1740
|
+
}
|
|
1741
|
+
}
|
|
1742
|
+
}
|
|
1743
|
+
return errors;
|
|
1744
|
+
}
|
|
1745
|
+
function getLocalPacks() {
|
|
1746
|
+
if (_localPacksCache !== null) return _localPacksCache;
|
|
1747
|
+
const { packs, errors } = readLocalPacksFromDisk();
|
|
1748
|
+
for (const e of errors) {
|
|
1749
|
+
console.warn(`[llm-endpoint] Skipping invalid packs.local.json \u2014 ${e}`);
|
|
1750
|
+
}
|
|
1751
|
+
_localPacksCache = packs;
|
|
1752
|
+
return _localPacksCache;
|
|
1753
|
+
}
|
|
1754
|
+
function resetLocalPacksCache() {
|
|
1755
|
+
_localPacksCache = null;
|
|
1756
|
+
_mergedPackIdsCache = null;
|
|
1757
|
+
}
|
|
1758
|
+
var PROVIDER_MAX_TOOLS = {
|
|
1759
|
+
groq: 128,
|
|
1760
|
+
xai: 200
|
|
1761
|
+
};
|
|
1762
|
+
var KNOWN_PROVIDERS = /* @__PURE__ */ new Set([...KNOWN_TRANSPARENT_PROVIDERS]);
|
|
1763
|
+
function trimTools(payload, opts) {
|
|
1764
|
+
const { provider, queryTools, queryNoTools, headerTools, headerNoTools, headerExcludeTools, packToolsExclude, packToolsAllow } = opts;
|
|
1765
|
+
if (!payload || !Array.isArray(payload.tools) || payload.tools.length === 0) return;
|
|
1766
|
+
if (queryNoTools === "1" || queryTools === "none" || headerNoTools === "1") {
|
|
1767
|
+
console.log(`[Proxy][tools] strip total (${payload.tools.length} outils supprim\xE9s)`);
|
|
1768
|
+
delete payload.tools;
|
|
1769
|
+
delete payload.tool_choice;
|
|
1770
|
+
return;
|
|
1771
|
+
}
|
|
1772
|
+
const allowList = headerTools || queryTools;
|
|
1773
|
+
if (allowList) {
|
|
1774
|
+
const patterns = allowList.split(",").map((s) => s.trim()).filter(Boolean);
|
|
1775
|
+
const before = payload.tools.length;
|
|
1776
|
+
payload.tools = payload.tools.filter((t) => {
|
|
1777
|
+
const name = t && (t.name || t.function && t.function.name) || "";
|
|
1778
|
+
return patterns.some((p) => matchesPattern(name, p));
|
|
1779
|
+
});
|
|
1780
|
+
console.log(`[Proxy][tools] allow-list {${patterns.join(",")}} \u2192 ${before}\u2192${payload.tools.length}`);
|
|
1781
|
+
if (payload.tools.length === 0) {
|
|
1782
|
+
delete payload.tools;
|
|
1783
|
+
delete payload.tool_choice;
|
|
1784
|
+
}
|
|
1785
|
+
return;
|
|
1786
|
+
}
|
|
1787
|
+
if (headerExcludeTools) {
|
|
1788
|
+
const patterns = headerExcludeTools.split(",").map((s) => s.trim()).filter(Boolean);
|
|
1789
|
+
const before = payload.tools.length;
|
|
1790
|
+
payload.tools = payload.tools.filter((t) => {
|
|
1791
|
+
const name = t && (t.name || t.function && t.function.name) || "";
|
|
1792
|
+
return !patterns.some((p) => matchesPattern(name, p));
|
|
1793
|
+
});
|
|
1794
|
+
console.log(`[Proxy][tools] exclude-list {${patterns.join(",")}} \u2192 ${before}\u2192${payload.tools.length}`);
|
|
1795
|
+
if (payload.tools.length === 0) {
|
|
1796
|
+
delete payload.tools;
|
|
1797
|
+
delete payload.tool_choice;
|
|
1798
|
+
}
|
|
1799
|
+
return;
|
|
1800
|
+
}
|
|
1801
|
+
if (packToolsExclude && packToolsExclude.length > 0) {
|
|
1802
|
+
const before = payload.tools.length;
|
|
1803
|
+
payload.tools = payload.tools.filter((t) => {
|
|
1804
|
+
const name = t && (t.name || t.function && t.function.name) || "";
|
|
1805
|
+
return !packToolsExclude.some((p) => matchesPattern(name, p));
|
|
1806
|
+
});
|
|
1807
|
+
console.log(`[Proxy][tools] pack exclude-list {${packToolsExclude.join(",")}} \u2192 ${before}\u2192${payload.tools.length}`);
|
|
1808
|
+
} else if (packToolsAllow && packToolsAllow.length > 0) {
|
|
1809
|
+
const before = payload.tools.length;
|
|
1810
|
+
payload.tools = payload.tools.filter((t) => {
|
|
1811
|
+
const name = t && (t.name || t.function && t.function.name) || "";
|
|
1812
|
+
return packToolsAllow.some((p) => matchesPattern(name, p));
|
|
1813
|
+
});
|
|
1814
|
+
console.log(`[Proxy][tools] pack allow-list {${packToolsAllow.join(",")}} \u2192 ${before}\u2192${payload.tools.length}`);
|
|
1815
|
+
}
|
|
1816
|
+
if (payload.tools.length === 0) {
|
|
1817
|
+
delete payload.tools;
|
|
1818
|
+
delete payload.tool_choice;
|
|
1819
|
+
}
|
|
1820
|
+
const cap = PROVIDER_MAX_TOOLS[provider];
|
|
1821
|
+
if (cap && payload.tools.length > cap) {
|
|
1822
|
+
const before = payload.tools.length;
|
|
1823
|
+
payload.tools = payload.tools.slice(0, cap);
|
|
1824
|
+
console.log(`[Proxy][tools] troncation cap ${cap} pour ${provider} \u2192 ${before}\u2192${payload.tools.length}`);
|
|
1825
|
+
}
|
|
1826
|
+
}
|
|
1827
|
+
function resolveSecretKeys() {
|
|
1828
|
+
return {
|
|
1829
|
+
anthropic: process.env.ANTHROPIC_API_KEY || "",
|
|
1830
|
+
moonshot: process.env.MOONSHOT_API_KEY || "",
|
|
1831
|
+
openrouter: process.env.OPENROUTER_API_KEY || "",
|
|
1832
|
+
requesty: process.env.REQUESTY_API_KEY || "",
|
|
1833
|
+
zai: process.env.ZHIPUAI_API_KEY || process.env.ZAI_API_KEY || "",
|
|
1834
|
+
groq: process.env.GROQ_API_KEY || "",
|
|
1835
|
+
xai: process.env.XAI_API_KEY || "",
|
|
1836
|
+
openai: process.env.OPENAI_API_KEY || ""
|
|
1837
|
+
};
|
|
1838
|
+
}
|
|
1839
|
+
function getResolvedKeys() {
|
|
1840
|
+
return resolveSecretKeys();
|
|
1841
|
+
}
|
|
1842
|
+
var CONFIGURABLE_PROVIDERS = {
|
|
1843
|
+
forge: { baseUrlEnv: "FORGE_BASE_URL", apiKeyEnv: "FORGE_API_KEY", keyRequired: false },
|
|
1844
|
+
nebius: {
|
|
1845
|
+
baseUrlEnv: "NEBIUS_BASE_URL",
|
|
1846
|
+
apiKeyEnv: "NEBIUS_API_KEY",
|
|
1847
|
+
defaultBaseUrl: "https://api.studio.nebius.com/v1",
|
|
1848
|
+
keyRequired: true
|
|
1849
|
+
}
|
|
1850
|
+
};
|
|
1851
|
+
function isUpstreamKeyOptional(provider) {
|
|
1852
|
+
return getConfigurableProviderSpec(provider)?.keyRequired === false;
|
|
1853
|
+
}
|
|
1854
|
+
function configurableProviderUnavailableMessage(provider) {
|
|
1855
|
+
const spec = CONFIGURABLE_PROVIDERS[provider];
|
|
1856
|
+
if (!spec) return `Provider "${provider}" is not configured.`;
|
|
1857
|
+
if (spec.defaultBaseUrl) {
|
|
1858
|
+
return `"${provider}" upstream is misconfigured \u2014 ${spec.baseUrlEnv} is set but invalid (must be a valid http:// or https:// URL).`;
|
|
1859
|
+
}
|
|
1860
|
+
return `${provider} provider not configured \u2014 set ${spec.baseUrlEnv} to enable "${provider}/<model>" routing.`;
|
|
1861
|
+
}
|
|
1862
|
+
function parseConfigurableUpstreamUrl(value, provider) {
|
|
1863
|
+
let parsed;
|
|
1864
|
+
try {
|
|
1865
|
+
parsed = new URL(value);
|
|
1866
|
+
} catch {
|
|
1867
|
+
warnUpstreamOnce(`${provider}:bad-url`, `[Proxy][${provider}] "${value}" is not a valid URL; ${provider} provider disabled.`);
|
|
1868
|
+
return null;
|
|
1869
|
+
}
|
|
1870
|
+
if (parsed.protocol !== "http:" && parsed.protocol !== "https:") {
|
|
1871
|
+
warnUpstreamOnce(`${provider}:bad-scheme`, `[Proxy][${provider}] "${value}" must use http:// or https://; ${provider} provider disabled.`);
|
|
1872
|
+
return null;
|
|
1873
|
+
}
|
|
1874
|
+
return parseUpstreamUrl(value);
|
|
1875
|
+
}
|
|
1876
|
+
function resolveForgeBaseUrl(raw = process.env.FORGE_BASE_URL) {
|
|
1877
|
+
const value = raw?.trim();
|
|
1878
|
+
if (!value) return null;
|
|
1879
|
+
return parseConfigurableUpstreamUrl(value, "forge");
|
|
1880
|
+
}
|
|
1881
|
+
function resolveNebiusBaseUrl(raw = process.env.NEBIUS_BASE_URL) {
|
|
1882
|
+
const value = raw?.trim() || CONFIGURABLE_PROVIDERS.nebius.defaultBaseUrl;
|
|
1883
|
+
return parseConfigurableUpstreamUrl(value, "nebius");
|
|
1884
|
+
}
|
|
1885
|
+
function getConfigurableProviderSpec(provider) {
|
|
1886
|
+
const staticSpec = CONFIGURABLE_PROVIDERS[provider];
|
|
1887
|
+
if (staticSpec) {
|
|
1888
|
+
return {
|
|
1889
|
+
keyRequired: staticSpec.keyRequired,
|
|
1890
|
+
apiKeyEnv: staticSpec.apiKeyEnv,
|
|
1891
|
+
resolveUpstream: () => provider === "forge" ? resolveForgeBaseUrl() : resolveNebiusBaseUrl(),
|
|
1892
|
+
unavailableMessage: () => configurableProviderUnavailableMessage(provider)
|
|
1893
|
+
};
|
|
1894
|
+
}
|
|
1895
|
+
const fileEndpoint = getConfiguredEndpoints().find((e) => e.id === provider);
|
|
1896
|
+
if (fileEndpoint) {
|
|
1897
|
+
const upstream = parseUpstreamUrl(fileEndpoint.baseUrl);
|
|
1898
|
+
return {
|
|
1899
|
+
keyRequired: false,
|
|
1900
|
+
apiKeyEnv: fileEndpoint.apiKeyEnv,
|
|
1901
|
+
defaultRequestFields: fileEndpoint.defaultRequestFields,
|
|
1902
|
+
resolveUpstream: () => upstream,
|
|
1903
|
+
unavailableMessage: () => `"${provider}" endpoint is misconfigured (invalid baseUrl).`
|
|
1904
|
+
};
|
|
1905
|
+
}
|
|
1906
|
+
return void 0;
|
|
1907
|
+
}
|
|
1908
|
+
function applyDefaultRequestFields(payload, defaults, skipKeys) {
|
|
1909
|
+
if (!defaults) return;
|
|
1910
|
+
for (const [key, value] of Object.entries(defaults)) {
|
|
1911
|
+
if (skipKeys?.has(key)) continue;
|
|
1912
|
+
const existing = payload[key];
|
|
1913
|
+
if (existing === void 0) {
|
|
1914
|
+
payload[key] = value;
|
|
1915
|
+
} else if (isRecord(existing) && isRecord(value)) {
|
|
1916
|
+
payload[key] = { ...value, ...existing };
|
|
1917
|
+
}
|
|
1918
|
+
}
|
|
1919
|
+
}
|
|
1920
|
+
function sendUpstreamRequest(protocol, options, callback) {
|
|
1921
|
+
return protocol === "http" ? request(options, callback) : request$1(options, callback);
|
|
1922
|
+
}
|
|
1923
|
+
function getChatCompletionsEndpoint(provider) {
|
|
1924
|
+
const configurableSpec = getConfigurableProviderSpec(provider);
|
|
1925
|
+
if (configurableSpec) {
|
|
1926
|
+
const upstream = configurableSpec.resolveUpstream();
|
|
1927
|
+
if (!upstream) return null;
|
|
1928
|
+
return { hostname: upstream.hostname, path: `${upstream.pathPrefix}/chat/completions`, port: upstream.port, protocol: upstream.protocol };
|
|
1929
|
+
}
|
|
1930
|
+
switch (provider) {
|
|
1931
|
+
case "anthropic":
|
|
1932
|
+
return null;
|
|
1933
|
+
case "openrouter":
|
|
1934
|
+
return { hostname: "openrouter.ai", path: "/api/v1/chat/completions" };
|
|
1935
|
+
case "requesty":
|
|
1936
|
+
return { hostname: "router.requesty.ai", path: "/v1/chat/completions" };
|
|
1937
|
+
case "zai":
|
|
1938
|
+
return { hostname: "open.bigmodel.cn", path: "/api/paas/v4/chat/completions" };
|
|
1939
|
+
case "groq":
|
|
1940
|
+
return { hostname: "api.groq.com", path: "/openai/v1/chat/completions" };
|
|
1941
|
+
case "xai":
|
|
1942
|
+
return { hostname: "api.x.ai", path: "/v1/chat/completions" };
|
|
1943
|
+
case "openai":
|
|
1944
|
+
return { hostname: "api.openai.com", path: "/v1/chat/completions" };
|
|
1945
|
+
case "moonshot":
|
|
1946
|
+
default:
|
|
1947
|
+
return { hostname: "api.moonshot.ai", path: "/v1/chat/completions" };
|
|
1948
|
+
}
|
|
1949
|
+
}
|
|
1950
|
+
function probeOpenAiModels(upstream, cred, timeoutMs) {
|
|
1951
|
+
const headers = {};
|
|
1952
|
+
if (cred?.value) headers["Authorization"] = `Bearer ${cred.value}`;
|
|
1953
|
+
return new Promise((resolvePromise) => {
|
|
1954
|
+
const options = {
|
|
1955
|
+
hostname: upstream.hostname,
|
|
1956
|
+
port: upstream.port,
|
|
1957
|
+
path: `${upstream.pathPrefix}/models`,
|
|
1958
|
+
method: "GET",
|
|
1959
|
+
headers
|
|
1960
|
+
};
|
|
1961
|
+
const req = sendUpstreamRequest(upstream.protocol, options, (res) => {
|
|
1962
|
+
let body = "";
|
|
1963
|
+
res.setEncoding("utf8");
|
|
1964
|
+
res.on("data", (chunk) => {
|
|
1965
|
+
body += chunk;
|
|
1966
|
+
});
|
|
1967
|
+
res.on("end", () => {
|
|
1968
|
+
try {
|
|
1969
|
+
const parsed = JSON.parse(body);
|
|
1970
|
+
const data = isRecord(parsed) && Array.isArray(parsed.data) ? parsed.data : [];
|
|
1971
|
+
const ids = data.map((m) => isRecord(m) && typeof m.id === "string" ? m.id : null).filter((id) => id !== null);
|
|
1972
|
+
resolvePromise({ ok: true, ids });
|
|
1973
|
+
} catch {
|
|
1974
|
+
resolvePromise({ ok: false, ids: [] });
|
|
1975
|
+
}
|
|
1976
|
+
});
|
|
1977
|
+
});
|
|
1978
|
+
req.on("error", () => resolvePromise({ ok: false, ids: [] }));
|
|
1979
|
+
req.setTimeout(timeoutMs, () => {
|
|
1980
|
+
req.destroy();
|
|
1981
|
+
resolvePromise({ ok: false, ids: [] });
|
|
1982
|
+
});
|
|
1983
|
+
req.end();
|
|
1984
|
+
});
|
|
1985
|
+
}
|
|
1986
|
+
async function fetchForgeModelIds() {
|
|
1987
|
+
const forge = resolveForgeBaseUrl();
|
|
1988
|
+
if (!forge) return [];
|
|
1989
|
+
const cred = await resolveUpstreamCredential("forge");
|
|
1990
|
+
return (await probeOpenAiModels(forge, cred, 4e3)).ids;
|
|
1991
|
+
}
|
|
1992
|
+
var ENDPOINT_MODEL_PROBE_TIMEOUT_MS = 4e3;
|
|
1993
|
+
var ENDPOINT_MODEL_CACHE_TTL_MS = 3e4;
|
|
1994
|
+
var _endpointProbeCache = /* @__PURE__ */ new Map();
|
|
1995
|
+
async function probeFileEndpointModels(endpoint) {
|
|
1996
|
+
const now = Date.now();
|
|
1997
|
+
const cached = _endpointProbeCache.get(endpoint.id);
|
|
1998
|
+
if (cached && cached.expiresAt > now) return cached.result;
|
|
1999
|
+
const upstream = parseUpstreamUrl(endpoint.baseUrl);
|
|
2000
|
+
if (!upstream) return { ok: false, ids: [] };
|
|
2001
|
+
const cred = await resolveUpstreamCredential(endpoint.id);
|
|
2002
|
+
const result = await probeOpenAiModels(upstream, cred, ENDPOINT_MODEL_PROBE_TIMEOUT_MS);
|
|
2003
|
+
_endpointProbeCache.set(endpoint.id, { expiresAt: now + ENDPOINT_MODEL_CACHE_TTL_MS, result });
|
|
2004
|
+
return result;
|
|
2005
|
+
}
|
|
2006
|
+
function isKnownProvider(provider) {
|
|
2007
|
+
return KNOWN_PROVIDERS.has(provider) || getConfiguredEndpoints().some((e) => e.id === provider);
|
|
2008
|
+
}
|
|
2009
|
+
function applyProviderOverride(target, providerOverride) {
|
|
2010
|
+
const route = { provider: target.provider, model: target.model };
|
|
2011
|
+
if (!providerOverride) return route;
|
|
2012
|
+
if (!isKnownProvider(providerOverride)) {
|
|
2013
|
+
const allowed = [...KNOWN_PROVIDERS, ...getConfiguredEndpoints().map((e) => e.id)];
|
|
2014
|
+
throw new Error(`Unknown provider "${providerOverride}" in ?p= (allowed: ${allowed.join(", ")})`);
|
|
2015
|
+
}
|
|
2016
|
+
return { provider: providerOverride, model: route.model };
|
|
2017
|
+
}
|
|
2018
|
+
function parseAnyTransparentModel(model) {
|
|
2019
|
+
const known = parseTransparentModel(model);
|
|
2020
|
+
if (known) return known;
|
|
2021
|
+
const slashIdx = model.indexOf("/");
|
|
2022
|
+
if (slashIdx <= 0 || slashIdx === model.length - 1) return null;
|
|
2023
|
+
const provider = model.slice(0, slashIdx);
|
|
2024
|
+
if (!getConfiguredEndpoints().some((e) => e.id === provider)) return null;
|
|
2025
|
+
return { provider, model: model.slice(slashIdx + 1) };
|
|
2026
|
+
}
|
|
2027
|
+
function resolveModelRoute(payload, ctx, localPacks = getLocalPacks()) {
|
|
2028
|
+
const mapping = buildMappingFromPack(ctx.activePack);
|
|
2029
|
+
const isLocalPack = Boolean(localPacks[ctx.activePack.id]);
|
|
2030
|
+
const isAliasPack = isLocalPack || Boolean(ctx.anthropicFormat);
|
|
2031
|
+
if (ctx.forcedAliasCode) {
|
|
2032
|
+
const forcedCode = ctx.forcedAliasCode.toLowerCase();
|
|
2033
|
+
const target = Object.entries(mapping).find(([code]) => code.toLowerCase() === forcedCode)?.[1];
|
|
2034
|
+
if (!target) {
|
|
2035
|
+
throw new Error(`Unknown model alias "${ctx.forcedAliasCode}" in active pack "${ctx.activePack.id}"`);
|
|
2036
|
+
}
|
|
2037
|
+
console.log(`[Proxy] Forced model alias "${ctx.forcedAliasCode}" -> ${target.provider}:${target.model}`);
|
|
2038
|
+
return applyProviderOverride(target, ctx.queryProvider);
|
|
2039
|
+
}
|
|
2040
|
+
if (ctx.queryModelCode) {
|
|
2041
|
+
const queryCode = ctx.queryModelCode.toLowerCase();
|
|
2042
|
+
const target = Object.entries(mapping).find(([code]) => code.toLowerCase() === queryCode)?.[1];
|
|
2043
|
+
if (!target) {
|
|
2044
|
+
throw new Error(`Unknown model code "${ctx.queryModelCode}" in active pack "${ctx.activePack.id}"`);
|
|
2045
|
+
}
|
|
2046
|
+
console.log(`[Proxy] Detected URL model parameter code "${ctx.queryModelCode}" -> ${target.provider}:${target.model}`);
|
|
2047
|
+
return applyProviderOverride(target, ctx.queryProvider);
|
|
2048
|
+
}
|
|
2049
|
+
const incomingModel = (payload.model || "").trim();
|
|
2050
|
+
if (!incomingModel) {
|
|
2051
|
+
throw new Error('Missing "model" field in request body');
|
|
2052
|
+
}
|
|
2053
|
+
const incomingLower = incomingModel.toLowerCase();
|
|
2054
|
+
if (ctx.allowAliases && isAliasPack) {
|
|
2055
|
+
const aliasEntry = Object.entries(mapping).find(
|
|
2056
|
+
([code, target]) => target.equivalentClaudeName?.toLowerCase() === incomingLower || code.toLowerCase() === incomingLower
|
|
2057
|
+
);
|
|
2058
|
+
if (aliasEntry) {
|
|
2059
|
+
const target = aliasEntry[1];
|
|
2060
|
+
console.log(`[Proxy] Matched alias "${incomingModel}" -> ${target.provider}:${target.model}`);
|
|
2061
|
+
return applyProviderOverride(target, ctx.queryProvider);
|
|
2062
|
+
}
|
|
2063
|
+
}
|
|
2064
|
+
if (ctx.allowAliases) {
|
|
2065
|
+
const codeEntry = Object.entries(mapping).find(([code]) => code.toLowerCase() === incomingLower);
|
|
2066
|
+
if (codeEntry) {
|
|
2067
|
+
const target = codeEntry[1];
|
|
2068
|
+
console.log(`[Proxy] Matched pack code "${incomingModel}" -> ${target.provider}:${target.model}`);
|
|
2069
|
+
return applyProviderOverride(target, ctx.queryProvider);
|
|
2070
|
+
}
|
|
2071
|
+
}
|
|
2072
|
+
const transparent = parseAnyTransparentModel(incomingModel);
|
|
2073
|
+
if (transparent) {
|
|
2074
|
+
console.log(`[Proxy] Transparent model reference "${incomingModel}" -> ${transparent.provider}:${transparent.model}`);
|
|
2075
|
+
return applyProviderOverride(transparent, ctx.queryProvider);
|
|
2076
|
+
}
|
|
2077
|
+
if (ctx.queryProvider) {
|
|
2078
|
+
return applyProviderOverride({ provider: ctx.queryProvider, model: incomingModel }, ctx.queryProvider);
|
|
2079
|
+
}
|
|
2080
|
+
throw new Error(
|
|
2081
|
+
`Unable to resolve model "${incomingModel}". Use a transparent "provider/model" reference (e.g. "moonshot/kimi-k2.7-code"), select a pack, or provide ?p=<provider>.`
|
|
2082
|
+
);
|
|
2083
|
+
}
|
|
2084
|
+
function readQueryAndToolOptions(parsedUrl, req) {
|
|
2085
|
+
const queryProvider = parsedUrl.searchParams.get("p");
|
|
2086
|
+
const queryModelCode = parsedUrl.searchParams.get("m");
|
|
2087
|
+
const queryTools = parsedUrl.searchParams.get("tools");
|
|
2088
|
+
const queryNoTools = parsedUrl.searchParams.get("notools");
|
|
2089
|
+
const rawHeaderTools = req.headers["x-proxy-tools"];
|
|
2090
|
+
const headerTools = (Array.isArray(rawHeaderTools) ? rawHeaderTools[0] : rawHeaderTools) || null;
|
|
2091
|
+
const rawHeaderNoTools = req.headers["x-proxy-no-tools"];
|
|
2092
|
+
const headerNoTools = (Array.isArray(rawHeaderNoTools) ? rawHeaderNoTools[0] : rawHeaderNoTools) || null;
|
|
2093
|
+
const rawHeaderExcludeTools = req.headers["x-proxy-exclude-tools"];
|
|
2094
|
+
const headerExcludeTools = (Array.isArray(rawHeaderExcludeTools) ? rawHeaderExcludeTools[0] : rawHeaderExcludeTools) || null;
|
|
2095
|
+
const rawHeaderAlias = req.headers["x-proxy-model-alias"];
|
|
2096
|
+
const headerAlias = (Array.isArray(rawHeaderAlias) ? rawHeaderAlias[0] : rawHeaderAlias)?.toLowerCase().trim();
|
|
2097
|
+
const envAlias = process.env.PROXY_MODEL_ALIAS?.toLowerCase().trim();
|
|
2098
|
+
const forcedAliasCode = headerAlias || envAlias || null;
|
|
2099
|
+
const rawHeaderFormat = req.headers["x-proxy-format"];
|
|
2100
|
+
const headerFormat = (Array.isArray(rawHeaderFormat) ? rawHeaderFormat[0] : rawHeaderFormat)?.toLowerCase().trim();
|
|
2101
|
+
const queryFormat = parsedUrl.searchParams.get("format")?.toLowerCase().trim();
|
|
2102
|
+
const anthropicFormat = headerFormat === "anthropic" || queryFormat === "anthropic";
|
|
2103
|
+
return {
|
|
2104
|
+
queryProvider,
|
|
2105
|
+
queryModelCode,
|
|
2106
|
+
queryTools,
|
|
2107
|
+
queryNoTools,
|
|
2108
|
+
headerTools,
|
|
2109
|
+
headerNoTools,
|
|
2110
|
+
headerExcludeTools,
|
|
2111
|
+
forcedAliasCode,
|
|
2112
|
+
anthropicFormat
|
|
2113
|
+
};
|
|
2114
|
+
}
|
|
2115
|
+
function getApiKey(provider) {
|
|
2116
|
+
const configurable = getConfigurableProviderSpec(provider);
|
|
2117
|
+
if (configurable) return configurable.apiKeyEnv ? process.env[configurable.apiKeyEnv] || "" : "";
|
|
2118
|
+
return getResolvedKeys()[provider] || "";
|
|
2119
|
+
}
|
|
2120
|
+
var ANTHROPIC_VERSION = "2023-06-01";
|
|
2121
|
+
var ANTHROPIC_OAUTH_BETA = "oauth-2025-04-20";
|
|
2122
|
+
function upstreamProfileEnvVar(provider) {
|
|
2123
|
+
return `LLM_ENDPOINT_PROFILE_${provider.toUpperCase()}`;
|
|
2124
|
+
}
|
|
2125
|
+
function isAnthropicOAuthToken(provider, value) {
|
|
2126
|
+
return provider === "anthropic" && value.startsWith("sk-ant-oat");
|
|
2127
|
+
}
|
|
2128
|
+
var _warnedUpstream = /* @__PURE__ */ new Set();
|
|
2129
|
+
function warnUpstreamOnce(key, message) {
|
|
2130
|
+
if (_warnedUpstream.has(key)) return;
|
|
2131
|
+
_warnedUpstream.add(key);
|
|
2132
|
+
console.warn(message);
|
|
2133
|
+
}
|
|
2134
|
+
async function resolveUpstreamCredential(provider) {
|
|
2135
|
+
const profileId = process.env[upstreamProfileEnvVar(provider)]?.trim();
|
|
2136
|
+
if (profileId) {
|
|
2137
|
+
const profile = await getAuthProfile(profileId);
|
|
2138
|
+
if (!profile || profile.disabled) {
|
|
2139
|
+
warnUpstreamOnce(
|
|
2140
|
+
`profile:${provider}:${profileId}`,
|
|
2141
|
+
`[Proxy][auth] profile "${profileId}" mapped for provider "${provider}" is ${profile ? "disabled" : "missing"}; request will 401. Enable/create it or unset ${upstreamProfileEnvVar(provider)}.`
|
|
2142
|
+
);
|
|
2143
|
+
return void 0;
|
|
2144
|
+
}
|
|
2145
|
+
if (!profile.credentialRef) {
|
|
2146
|
+
warnUpstreamOnce(
|
|
2147
|
+
`source:${provider}:${profileId}`,
|
|
2148
|
+
`[Proxy][auth] profile "${profileId}" is source-backed (source="${profile.source ?? "?"}"); source-backed profiles are not yet supported by the proxy \u2014 use a credentialRef profile or a per-provider API-key env var instead. Request will 401.`
|
|
2149
|
+
);
|
|
2150
|
+
return void 0;
|
|
2151
|
+
}
|
|
2152
|
+
try {
|
|
2153
|
+
const stored = await new KeychainStore().read({ path: profile.credentialRef });
|
|
2154
|
+
if (!stored) {
|
|
2155
|
+
warnUpstreamOnce(
|
|
2156
|
+
`noref:${provider}:${profileId}`,
|
|
2157
|
+
`[Proxy][auth] profile "${profileId}" credentialRef resolved no stored credential; failing closed \u2014 request will 401 (no env-key fallback for a mapped profile).`
|
|
2158
|
+
);
|
|
2159
|
+
return void 0;
|
|
2160
|
+
}
|
|
2161
|
+
return { value: stored.value, method: profile.method };
|
|
2162
|
+
} catch (err) {
|
|
2163
|
+
warnUpstreamOnce(
|
|
2164
|
+
`keychain:${provider}:${profileId}`,
|
|
2165
|
+
`[Proxy][auth] keychain backend unavailable for profile "${profileId}" (${err.message}); platform-unsupported \u2014 falling back to the ${provider} env key.`
|
|
2166
|
+
);
|
|
2167
|
+
}
|
|
2168
|
+
}
|
|
2169
|
+
const value = getApiKey(provider);
|
|
2170
|
+
return { value, method: isAnthropicOAuthToken(provider, value) ? "oauth-bearer" : "api-key" };
|
|
2171
|
+
}
|
|
2172
|
+
function buildUpstreamAuthHeaders(provider, cred) {
|
|
2173
|
+
const { value, method } = cred;
|
|
2174
|
+
if (method === "oauth-bearer") {
|
|
2175
|
+
if (provider !== "anthropic") {
|
|
2176
|
+
warnUpstreamOnce(
|
|
2177
|
+
`oauth-misconfig:${provider}`,
|
|
2178
|
+
`[Proxy][auth] oauth-bearer credential resolved for non-anthropic provider "${provider}"; refusing to forward it (subscription/oauth credentials are only valid for the anthropic upstream). Request will 401.`
|
|
2179
|
+
);
|
|
2180
|
+
return null;
|
|
2181
|
+
}
|
|
2182
|
+
return {
|
|
2183
|
+
"Authorization": `Bearer ${value}`,
|
|
2184
|
+
"anthropic-version": ANTHROPIC_VERSION,
|
|
2185
|
+
"anthropic-beta": ANTHROPIC_OAUTH_BETA
|
|
2186
|
+
};
|
|
2187
|
+
}
|
|
2188
|
+
switch (provider) {
|
|
2189
|
+
case "anthropic":
|
|
2190
|
+
return { "x-api-key": value, "anthropic-version": ANTHROPIC_VERSION };
|
|
2191
|
+
case "moonshot":
|
|
2192
|
+
return { "X-API-Key": value };
|
|
2193
|
+
// openrouter/requesty also set `anthropic-version` at their call sites
|
|
2194
|
+
// (left inline — it is a request-shape header, not an auth header).
|
|
2195
|
+
default:
|
|
2196
|
+
return { "Authorization": `Bearer ${value}` };
|
|
2197
|
+
}
|
|
2198
|
+
}
|
|
2199
|
+
function isCredentialAllowedOnOpenAiSurface(cred) {
|
|
2200
|
+
return !cred || cred.method === "api-key";
|
|
2201
|
+
}
|
|
2202
|
+
var CANONICAL_UPSTREAM_ORDER = {
|
|
2203
|
+
anthropic: true,
|
|
2204
|
+
moonshot: true,
|
|
2205
|
+
openrouter: true,
|
|
2206
|
+
requesty: true,
|
|
2207
|
+
zai: true,
|
|
2208
|
+
groq: true,
|
|
2209
|
+
xai: true,
|
|
2210
|
+
openai: true
|
|
2211
|
+
};
|
|
2212
|
+
var CANONICAL_UPSTREAMS = Object.keys(CANONICAL_UPSTREAM_ORDER);
|
|
2213
|
+
function isCanonicalUpstream(provider) {
|
|
2214
|
+
return CANONICAL_UPSTREAMS.includes(provider);
|
|
2215
|
+
}
|
|
2216
|
+
async function upstreamCredentialPresent(provider) {
|
|
2217
|
+
const cred = await resolveUpstreamCredential(provider);
|
|
2218
|
+
return Boolean(cred?.value);
|
|
2219
|
+
}
|
|
2220
|
+
async function describeUpstreamStatus(provider, opts) {
|
|
2221
|
+
const linkedProfile = process.env[upstreamProfileEnvVar(provider)]?.trim() || null;
|
|
2222
|
+
if (linkedProfile) {
|
|
2223
|
+
const profile = await getAuthProfile(linkedProfile);
|
|
2224
|
+
const present = opts.probe ? await upstreamCredentialPresent(provider) : null;
|
|
2225
|
+
return { provider, linkedProfile, source: "profile", method: profile?.method ?? null, present };
|
|
2226
|
+
}
|
|
2227
|
+
if (getApiKey(provider)) {
|
|
2228
|
+
return { provider, linkedProfile: null, source: "env", method: "api-key", present: true };
|
|
2229
|
+
}
|
|
2230
|
+
return { provider, linkedProfile: null, source: "none", method: null, present: false };
|
|
2231
|
+
}
|
|
2232
|
+
async function collectUpstreamStatuses(opts) {
|
|
2233
|
+
return Promise.all(CANONICAL_UPSTREAMS.map((provider) => describeUpstreamStatus(provider, opts)));
|
|
2234
|
+
}
|
|
2235
|
+
var UPSTREAM_TEST_TIMEOUT_MS = 4e3;
|
|
2236
|
+
function getUpstreamProbe(provider) {
|
|
2237
|
+
switch (provider) {
|
|
2238
|
+
case "anthropic":
|
|
2239
|
+
return { hostname: "api.anthropic.com", path: "/v1/models?limit=1" };
|
|
2240
|
+
case "moonshot":
|
|
2241
|
+
return { hostname: "api.moonshot.ai", path: "/v1/models" };
|
|
2242
|
+
case "openrouter":
|
|
2243
|
+
return { hostname: "openrouter.ai", path: "/api/v1/key" };
|
|
2244
|
+
case "requesty":
|
|
2245
|
+
return { hostname: "router.requesty.ai", path: "/v1/models" };
|
|
2246
|
+
case "zai":
|
|
2247
|
+
return { hostname: "open.bigmodel.cn", path: "/api/paas/v4/models" };
|
|
2248
|
+
case "groq":
|
|
2249
|
+
return { hostname: "api.groq.com", path: "/openai/v1/models" };
|
|
2250
|
+
case "xai":
|
|
2251
|
+
return { hostname: "api.x.ai", path: "/v1/models" };
|
|
2252
|
+
case "openai":
|
|
2253
|
+
return { hostname: "api.openai.com", path: "/v1/models" };
|
|
2254
|
+
default:
|
|
2255
|
+
return null;
|
|
2256
|
+
}
|
|
2257
|
+
}
|
|
2258
|
+
function describeProbeStatus(status) {
|
|
2259
|
+
if (status >= 200 && status < 300) return "authenticated ok";
|
|
2260
|
+
if (status === 401 || status === 403) return "credential rejected";
|
|
2261
|
+
if (status === 404) return "probe endpoint not found (best-effort path)";
|
|
2262
|
+
if (status === 429) return "rate limited";
|
|
2263
|
+
return `unexpected status ${status}`;
|
|
2264
|
+
}
|
|
2265
|
+
function probeUpstreamHttp(probe, headers, timeoutMs) {
|
|
2266
|
+
return new Promise((resolvePromise) => {
|
|
2267
|
+
const proxyReq = request$1(
|
|
2268
|
+
{ hostname: probe.hostname, port: 443, path: probe.path, method: "GET", headers },
|
|
2269
|
+
(proxyRes) => {
|
|
2270
|
+
const status = proxyRes.statusCode ?? 0;
|
|
2271
|
+
proxyRes.on("data", () => {
|
|
2272
|
+
});
|
|
2273
|
+
proxyRes.on(
|
|
2274
|
+
"end",
|
|
2275
|
+
() => resolvePromise({ ok: status >= 200 && status < 300, status, detail: describeProbeStatus(status) })
|
|
2276
|
+
);
|
|
2277
|
+
}
|
|
2278
|
+
);
|
|
2279
|
+
proxyReq.on(
|
|
2280
|
+
"error",
|
|
2281
|
+
(err) => resolvePromise({ ok: false, status: 0, detail: `network error: ${err.message}` })
|
|
2282
|
+
);
|
|
2283
|
+
proxyReq.setTimeout(timeoutMs, () => {
|
|
2284
|
+
proxyReq.destroy();
|
|
2285
|
+
resolvePromise({ ok: false, status: 0, detail: `timed out after ${timeoutMs}ms` });
|
|
2286
|
+
});
|
|
2287
|
+
proxyReq.end();
|
|
2288
|
+
});
|
|
2289
|
+
}
|
|
2290
|
+
async function testUpstream(provider) {
|
|
2291
|
+
const probe = getUpstreamProbe(provider);
|
|
2292
|
+
if (!probe) return { ok: null, reason: "no-probe" };
|
|
2293
|
+
const cred = await resolveUpstreamCredential(provider);
|
|
2294
|
+
if (!cred?.value) {
|
|
2295
|
+
return { ok: false, status: 401, detail: "no credential resolved for this upstream" };
|
|
2296
|
+
}
|
|
2297
|
+
const authHeaders = buildUpstreamAuthHeaders(provider, cred);
|
|
2298
|
+
if (!authHeaders) {
|
|
2299
|
+
return { ok: false, status: 401, detail: "resolved credential is not forwardable to this upstream" };
|
|
2300
|
+
}
|
|
2301
|
+
return probeUpstreamHttp(probe, authHeaders, UPSTREAM_TEST_TIMEOUT_MS);
|
|
2302
|
+
}
|
|
2303
|
+
async function handleUpstreamsStatus(res, opts) {
|
|
2304
|
+
try {
|
|
2305
|
+
const data = await collectUpstreamStatuses(opts);
|
|
2306
|
+
res.writeHead(200, { "Content-Type": "application/json" });
|
|
2307
|
+
res.end(JSON.stringify({ object: "list", probe: opts.probe, data }));
|
|
2308
|
+
} catch (e) {
|
|
2309
|
+
console.error("[Proxy][upstreams] status error", e);
|
|
2310
|
+
res.writeHead(500, { "Content-Type": "application/json" });
|
|
2311
|
+
res.end(JSON.stringify({ error: { type: "api_error", message: e instanceof Error ? e.message : String(e) } }));
|
|
2312
|
+
}
|
|
2313
|
+
}
|
|
2314
|
+
async function handleUpstreamTest(res, provider) {
|
|
2315
|
+
try {
|
|
2316
|
+
if (!isCanonicalUpstream(provider)) {
|
|
2317
|
+
res.writeHead(404, { "Content-Type": "application/json" });
|
|
2318
|
+
res.end(JSON.stringify({ error: { type: "invalid_request_error", message: `Unknown upstream "${provider}". Known: ${CANONICAL_UPSTREAMS.join(", ")}` } }));
|
|
2319
|
+
return;
|
|
2320
|
+
}
|
|
2321
|
+
const result = await testUpstream(provider);
|
|
2322
|
+
res.writeHead(200, { "Content-Type": "application/json" });
|
|
2323
|
+
res.end(JSON.stringify({ provider, ...result }));
|
|
2324
|
+
} catch (e) {
|
|
2325
|
+
console.error("[Proxy][upstreams] test error", e);
|
|
2326
|
+
res.writeHead(500, { "Content-Type": "application/json" });
|
|
2327
|
+
res.end(JSON.stringify({ error: { type: "api_error", message: e instanceof Error ? e.message : String(e) } }));
|
|
2328
|
+
}
|
|
2329
|
+
}
|
|
2330
|
+
async function describeEndpointHealth(id, upstream) {
|
|
2331
|
+
const baseUrl = `${upstream.protocol}://${upstream.hostname}:${upstream.port}${upstream.pathPrefix}`;
|
|
2332
|
+
const cred = await resolveUpstreamCredential(id);
|
|
2333
|
+
const start2 = Date.now();
|
|
2334
|
+
const { ok, ids } = await probeOpenAiModels(upstream, cred, ENDPOINT_MODEL_PROBE_TIMEOUT_MS);
|
|
2335
|
+
return { id, baseUrl, reachable: ok, models: ids, latencyMs: Date.now() - start2 };
|
|
2336
|
+
}
|
|
2337
|
+
async function handleEndpointsStatus(res) {
|
|
2338
|
+
try {
|
|
2339
|
+
const entries = [];
|
|
2340
|
+
const forge = resolveForgeBaseUrl();
|
|
2341
|
+
if (forge) entries.push(await describeEndpointHealth("forge", forge));
|
|
2342
|
+
for (const endpoint of getConfiguredEndpoints()) {
|
|
2343
|
+
const upstream = parseUpstreamUrl(endpoint.baseUrl);
|
|
2344
|
+
if (!upstream) continue;
|
|
2345
|
+
entries.push(await describeEndpointHealth(endpoint.id, upstream));
|
|
2346
|
+
}
|
|
2347
|
+
res.writeHead(200, { "Content-Type": "application/json" });
|
|
2348
|
+
res.end(JSON.stringify({ object: "list", data: entries }));
|
|
2349
|
+
} catch (e) {
|
|
2350
|
+
console.error("[Proxy][endpoints] status error", e);
|
|
2351
|
+
res.writeHead(500, { "Content-Type": "application/json" });
|
|
2352
|
+
res.end(JSON.stringify({ error: { type: "api_error", message: e instanceof Error ? e.message : String(e) } }));
|
|
2353
|
+
}
|
|
2354
|
+
}
|
|
2355
|
+
function handleResponsesRequest(req, res, opts) {
|
|
2356
|
+
let body = "";
|
|
2357
|
+
req.on("data", (chunk) => {
|
|
2358
|
+
body += chunk;
|
|
2359
|
+
});
|
|
2360
|
+
req.on("end", async () => {
|
|
2361
|
+
try {
|
|
2362
|
+
const payload = JSON.parse(body);
|
|
2363
|
+
const validated = validateResponsesRequest(payload);
|
|
2364
|
+
let resolvedTarget;
|
|
2365
|
+
try {
|
|
2366
|
+
resolvedTarget = resolveModelRoute(validated, {
|
|
2367
|
+
activePack: opts.activePack,
|
|
2368
|
+
queryModelCode: null,
|
|
2369
|
+
// Responses facade uses transparent routing only
|
|
2370
|
+
queryProvider: opts.queryProvider,
|
|
2371
|
+
forcedAliasCode: null,
|
|
2372
|
+
allowAliases: false
|
|
2373
|
+
});
|
|
2374
|
+
} catch (e) {
|
|
2375
|
+
console.warn(`[Proxy][responses] ${e.message}`);
|
|
2376
|
+
res.writeHead(400, { "Content-Type": "application/json" });
|
|
2377
|
+
res.end(JSON.stringify({ error: { type: "invalid_request_error", message: e.message } }));
|
|
2378
|
+
return;
|
|
2379
|
+
}
|
|
2380
|
+
const chatPayload = responsesToChatCompletionsRequest(validated, resolvedTarget);
|
|
2381
|
+
trimTools(chatPayload, {
|
|
2382
|
+
provider: resolvedTarget.provider,
|
|
2383
|
+
queryTools: opts.queryTools,
|
|
2384
|
+
queryNoTools: opts.queryNoTools,
|
|
2385
|
+
headerTools: opts.headerTools,
|
|
2386
|
+
headerNoTools: opts.headerNoTools,
|
|
2387
|
+
headerExcludeTools: opts.headerExcludeTools
|
|
2388
|
+
});
|
|
2389
|
+
const responsesConfigurableSpec = getConfigurableProviderSpec(resolvedTarget.provider);
|
|
2390
|
+
const endpoint = getChatCompletionsEndpoint(resolvedTarget.provider);
|
|
2391
|
+
if (!endpoint) {
|
|
2392
|
+
const message = responsesConfigurableSpec ? responsesConfigurableSpec.unavailableMessage() : `Provider "${resolvedTarget.provider}" does not support the OpenAI-compatible /v1/responses surface.`;
|
|
2393
|
+
res.writeHead(400, { "Content-Type": "application/json" });
|
|
2394
|
+
res.end(JSON.stringify({ error: { type: "invalid_request_error", message } }));
|
|
2395
|
+
return;
|
|
2396
|
+
}
|
|
2397
|
+
if (responsesConfigurableSpec) applyDefaultRequestFields(chatPayload, responsesConfigurableSpec.defaultRequestFields);
|
|
2398
|
+
const { hostname, path: path2, port = 443, protocol = "https" } = endpoint;
|
|
2399
|
+
const cred = await resolveUpstreamCredential(resolvedTarget.provider);
|
|
2400
|
+
const targetApiKey = cred?.value ?? "";
|
|
2401
|
+
if (!isCredentialAllowedOnOpenAiSurface(cred)) {
|
|
2402
|
+
console.warn(`[Proxy][responses] refusing to forward a ${cred.method} credential to non-anthropic provider "${resolvedTarget.provider}"; returning 401.`);
|
|
2403
|
+
res.writeHead(401, { "Content-Type": "application/json" });
|
|
2404
|
+
res.end(JSON.stringify({ error: { type: "authentication_error", message: `Subscription/oauth credentials cannot be used on this OpenAI-compatible surface for provider "${resolvedTarget.provider}".` } }));
|
|
2405
|
+
return;
|
|
2406
|
+
}
|
|
2407
|
+
if (!targetApiKey && !isUpstreamKeyOptional(resolvedTarget.provider)) {
|
|
2408
|
+
res.writeHead(401, { "Content-Type": "application/json" });
|
|
2409
|
+
res.end(JSON.stringify({ error: { type: "authentication_error", message: `No API key for provider "${resolvedTarget.provider}"` } }));
|
|
2410
|
+
return;
|
|
2411
|
+
}
|
|
2412
|
+
console.log(`[Proxy Sortant][responses] Redirection vers ${resolvedTarget.provider} (${hostname}${path2}) avec le mod\xE8le "${chatPayload.model}"`);
|
|
2413
|
+
const options = {
|
|
2414
|
+
hostname,
|
|
2415
|
+
port,
|
|
2416
|
+
path: path2,
|
|
2417
|
+
method: "POST",
|
|
2418
|
+
headers: {
|
|
2419
|
+
"Content-Type": "application/json",
|
|
2420
|
+
...targetApiKey ? { "Authorization": `Bearer ${targetApiKey}` } : {}
|
|
2421
|
+
}
|
|
2422
|
+
};
|
|
2423
|
+
const proxyReq = sendUpstreamRequest(protocol, options, (proxyRes) => {
|
|
2424
|
+
const status = proxyRes.statusCode || 200;
|
|
2425
|
+
const contentType = proxyRes.headers["content-type"] || "";
|
|
2426
|
+
const isStreaming = validated.stream === true && /text\/event-stream/i.test(contentType);
|
|
2427
|
+
if (isStreaming) {
|
|
2428
|
+
const respHeaders = { ...proxyRes.headers };
|
|
2429
|
+
delete respHeaders["content-length"];
|
|
2430
|
+
delete respHeaders["transfer-encoding"];
|
|
2431
|
+
respHeaders["content-type"] = "text/event-stream";
|
|
2432
|
+
res.writeHead(status, respHeaders);
|
|
2433
|
+
const converter = new OpenAIChatToResponsesStreamConverter({ requestedModel: validated.model });
|
|
2434
|
+
proxyRes.setEncoding("utf8");
|
|
2435
|
+
proxyRes.on("data", (c) => {
|
|
2436
|
+
for (const out of converter.push(c)) res.write(out);
|
|
2437
|
+
});
|
|
2438
|
+
proxyRes.on("end", () => {
|
|
2439
|
+
for (const out of converter.flush()) res.write(out);
|
|
2440
|
+
res.end();
|
|
2441
|
+
});
|
|
2442
|
+
} else {
|
|
2443
|
+
const respHeaders = { ...proxyRes.headers };
|
|
2444
|
+
delete respHeaders["content-length"];
|
|
2445
|
+
delete respHeaders["transfer-encoding"];
|
|
2446
|
+
respHeaders["content-type"] = "application/json";
|
|
2447
|
+
res.writeHead(status, respHeaders);
|
|
2448
|
+
let upstreamBody = "";
|
|
2449
|
+
proxyRes.setEncoding("utf8");
|
|
2450
|
+
proxyRes.on("data", (c) => {
|
|
2451
|
+
upstreamBody += c;
|
|
2452
|
+
});
|
|
2453
|
+
proxyRes.on("end", () => {
|
|
2454
|
+
res.end(chatCompletionsJsonToResponses(upstreamBody || "{}", { requestedModel: validated.model }));
|
|
2455
|
+
});
|
|
2456
|
+
}
|
|
2457
|
+
});
|
|
2458
|
+
proxyReq.on("error", (err) => {
|
|
2459
|
+
console.error("[Proxy SORTANT error]", err);
|
|
2460
|
+
res.writeHead(500, { "Content-Type": "application/json" });
|
|
2461
|
+
res.end(JSON.stringify({ error: { type: "api_error", message: err.message } }));
|
|
2462
|
+
});
|
|
2463
|
+
proxyReq.write(JSON.stringify(chatPayload));
|
|
2464
|
+
proxyReq.end();
|
|
2465
|
+
} catch (e) {
|
|
2466
|
+
console.error("[Payload Error]", e);
|
|
2467
|
+
res.writeHead(400, { "Content-Type": "application/json" });
|
|
2468
|
+
res.end(JSON.stringify({ error: { type: "invalid_request_error", message: e.message } }));
|
|
2469
|
+
}
|
|
2470
|
+
});
|
|
2471
|
+
}
|
|
2472
|
+
function handleChatCompletionsRequest(req, res, opts) {
|
|
2473
|
+
let body = "";
|
|
2474
|
+
req.on("data", (chunk) => {
|
|
2475
|
+
body += chunk;
|
|
2476
|
+
});
|
|
2477
|
+
req.on("end", async () => {
|
|
2478
|
+
try {
|
|
2479
|
+
const payload = JSON.parse(body);
|
|
2480
|
+
let resolvedTarget;
|
|
2481
|
+
try {
|
|
2482
|
+
resolvedTarget = resolveModelRoute(payload, {
|
|
2483
|
+
activePack: opts.activePack,
|
|
2484
|
+
queryModelCode: null,
|
|
2485
|
+
queryProvider: opts.queryProvider,
|
|
2486
|
+
forcedAliasCode: null,
|
|
2487
|
+
allowAliases: false
|
|
2488
|
+
});
|
|
2489
|
+
} catch (e) {
|
|
2490
|
+
console.warn(`[Proxy][chat/completions] ${e.message}`);
|
|
2491
|
+
res.writeHead(400, { "Content-Type": "application/json" });
|
|
2492
|
+
res.end(JSON.stringify({ error: { type: "invalid_request_error", message: e.message } }));
|
|
2493
|
+
return;
|
|
2494
|
+
}
|
|
2495
|
+
payload.model = resolvedTarget.model;
|
|
2496
|
+
trimTools(payload, {
|
|
2497
|
+
provider: resolvedTarget.provider,
|
|
2498
|
+
queryTools: opts.queryTools,
|
|
2499
|
+
queryNoTools: opts.queryNoTools,
|
|
2500
|
+
headerTools: opts.headerTools,
|
|
2501
|
+
headerNoTools: opts.headerNoTools,
|
|
2502
|
+
headerExcludeTools: opts.headerExcludeTools
|
|
2503
|
+
});
|
|
2504
|
+
const chatConfigurableSpec = getConfigurableProviderSpec(resolvedTarget.provider);
|
|
2505
|
+
const endpoint = getChatCompletionsEndpoint(resolvedTarget.provider);
|
|
2506
|
+
if (!endpoint) {
|
|
2507
|
+
const message = chatConfigurableSpec ? chatConfigurableSpec.unavailableMessage() : `Provider "${resolvedTarget.provider}" does not support the OpenAI-compatible /v1/chat/completions surface.`;
|
|
2508
|
+
res.writeHead(400, { "Content-Type": "application/json" });
|
|
2509
|
+
res.end(JSON.stringify({ error: { type: "invalid_request_error", message } }));
|
|
2510
|
+
return;
|
|
2511
|
+
}
|
|
2512
|
+
if (chatConfigurableSpec) applyDefaultRequestFields(payload, chatConfigurableSpec.defaultRequestFields);
|
|
2513
|
+
const { hostname, path: path2, port = 443, protocol = "https" } = endpoint;
|
|
2514
|
+
const cred = await resolveUpstreamCredential(resolvedTarget.provider);
|
|
2515
|
+
const targetApiKey = cred?.value ?? "";
|
|
2516
|
+
if (!isCredentialAllowedOnOpenAiSurface(cred)) {
|
|
2517
|
+
console.warn(`[Proxy][chat/completions] refusing to forward a ${cred.method} credential to non-anthropic provider "${resolvedTarget.provider}"; returning 401.`);
|
|
2518
|
+
res.writeHead(401, { "Content-Type": "application/json" });
|
|
2519
|
+
res.end(JSON.stringify({ error: { type: "authentication_error", message: `Subscription/oauth credentials cannot be used on this OpenAI-compatible surface for provider "${resolvedTarget.provider}".` } }));
|
|
2520
|
+
return;
|
|
2521
|
+
}
|
|
2522
|
+
if (!targetApiKey && !isUpstreamKeyOptional(resolvedTarget.provider)) {
|
|
2523
|
+
res.writeHead(401, { "Content-Type": "application/json" });
|
|
2524
|
+
res.end(JSON.stringify({ error: { type: "authentication_error", message: `No API key for provider "${resolvedTarget.provider}"` } }));
|
|
2525
|
+
return;
|
|
2526
|
+
}
|
|
2527
|
+
console.log(`[Proxy Sortant][chat/completions] Redirection vers ${resolvedTarget.provider} (${hostname}${path2}) avec le mod\xE8le "${payload.model}"`);
|
|
2528
|
+
const options = {
|
|
2529
|
+
hostname,
|
|
2530
|
+
port,
|
|
2531
|
+
path: path2,
|
|
2532
|
+
method: "POST",
|
|
2533
|
+
headers: {
|
|
2534
|
+
"Content-Type": "application/json",
|
|
2535
|
+
...targetApiKey ? { "Authorization": `Bearer ${targetApiKey}` } : {}
|
|
2536
|
+
}
|
|
2537
|
+
};
|
|
2538
|
+
const proxyReq = sendUpstreamRequest(protocol, options, (proxyRes) => {
|
|
2539
|
+
const status = proxyRes.statusCode || 200;
|
|
2540
|
+
const respHeaders = { ...proxyRes.headers };
|
|
2541
|
+
delete respHeaders["content-length"];
|
|
2542
|
+
delete respHeaders["transfer-encoding"];
|
|
2543
|
+
res.writeHead(status, respHeaders);
|
|
2544
|
+
proxyRes.pipe(res);
|
|
2545
|
+
});
|
|
2546
|
+
proxyReq.on("error", (err) => {
|
|
2547
|
+
console.error("[Proxy SORTANT error]", err);
|
|
2548
|
+
res.writeHead(500, { "Content-Type": "application/json" });
|
|
2549
|
+
res.end(JSON.stringify({ error: { type: "api_error", message: err.message } }));
|
|
2550
|
+
});
|
|
2551
|
+
proxyReq.write(JSON.stringify(payload));
|
|
2552
|
+
proxyReq.end();
|
|
2553
|
+
} catch (e) {
|
|
2554
|
+
console.error("[Payload Error]", e);
|
|
2555
|
+
res.writeHead(400, { "Content-Type": "application/json" });
|
|
2556
|
+
res.end(JSON.stringify({ error: { type: "invalid_request_error", message: e.message } }));
|
|
2557
|
+
}
|
|
2558
|
+
});
|
|
2559
|
+
}
|
|
2560
|
+
function adaptAnthropicToOpenAI(payload) {
|
|
2561
|
+
if (payload.system != null) {
|
|
2562
|
+
let sysText = "";
|
|
2563
|
+
if (typeof payload.system === "string") {
|
|
2564
|
+
sysText = payload.system;
|
|
2565
|
+
} else if (Array.isArray(payload.system)) {
|
|
2566
|
+
sysText = payload.system.map((b) => typeof b === "string" ? b : b?.text ?? "").filter(Boolean).join("\n\n");
|
|
2567
|
+
}
|
|
2568
|
+
if (sysText) {
|
|
2569
|
+
if (!Array.isArray(payload.messages)) payload.messages = [];
|
|
2570
|
+
if (!payload.messages[0] || payload.messages[0].role !== "system") {
|
|
2571
|
+
payload.messages.unshift({ role: "system", content: sysText });
|
|
2572
|
+
}
|
|
2573
|
+
}
|
|
2574
|
+
delete payload.system;
|
|
2575
|
+
}
|
|
2576
|
+
if (payload.tool_choice && typeof payload.tool_choice === "object") {
|
|
2577
|
+
const tc = payload.tool_choice;
|
|
2578
|
+
if (tc.type === "any") {
|
|
2579
|
+
payload.tool_choice = "auto";
|
|
2580
|
+
} else if (tc.type === "tool" && tc.name) {
|
|
2581
|
+
payload.tool_choice = { type: "function", function: { name: tc.name } };
|
|
2582
|
+
}
|
|
2583
|
+
}
|
|
2584
|
+
if (Array.isArray(payload.stop_sequences) && payload.stop_sequences.length) {
|
|
2585
|
+
payload.stop = payload.stop_sequences;
|
|
2586
|
+
}
|
|
2587
|
+
if (Array.isArray(payload.messages)) {
|
|
2588
|
+
const declaredTools = new Set(
|
|
2589
|
+
Array.isArray(payload.tools) ? payload.tools.map((t) => t?.name).filter(Boolean) : []
|
|
2590
|
+
);
|
|
2591
|
+
const orphanedToolUseIds = /* @__PURE__ */ new Set();
|
|
2592
|
+
if (process.env.PROXY_DEBUG) {
|
|
2593
|
+
console.error(`[adapt] declaredTools=${declaredTools.size} names=${JSON.stringify([...declaredTools].slice(0, 20))}`);
|
|
2594
|
+
}
|
|
2595
|
+
const newMessages = [];
|
|
2596
|
+
for (const msg of payload.messages) {
|
|
2597
|
+
if (msg.content == null || typeof msg.content === "string") {
|
|
2598
|
+
newMessages.push(msg);
|
|
2599
|
+
continue;
|
|
2600
|
+
}
|
|
2601
|
+
if (!Array.isArray(msg.content)) {
|
|
2602
|
+
newMessages.push(msg);
|
|
2603
|
+
continue;
|
|
2604
|
+
}
|
|
2605
|
+
const blocks = msg.content;
|
|
2606
|
+
if (msg.role === "user" && blocks.some((b) => b && b.type === "tool_result")) {
|
|
2607
|
+
const userTextParts = [];
|
|
2608
|
+
for (const b of blocks) {
|
|
2609
|
+
if (!b) continue;
|
|
2610
|
+
if (b.type === "tool_result") {
|
|
2611
|
+
let trContent = "";
|
|
2612
|
+
if (typeof b.content === "string") trContent = b.content;
|
|
2613
|
+
else if (Array.isArray(b.content)) {
|
|
2614
|
+
trContent = b.content.map((c) => typeof c === "string" ? c : c?.text ?? "").join("\n");
|
|
2615
|
+
}
|
|
2616
|
+
if (orphanedToolUseIds.has(b.tool_use_id)) {
|
|
2617
|
+
if (trContent) userTextParts.push(`[Tool result: ${trContent}]`);
|
|
2618
|
+
} else {
|
|
2619
|
+
newMessages.push({ role: "tool", tool_call_id: b.tool_use_id, content: trContent || "" });
|
|
2620
|
+
}
|
|
2621
|
+
} else if (b.type === "text" && b.text) {
|
|
2622
|
+
userTextParts.push(b.text);
|
|
2623
|
+
}
|
|
2624
|
+
}
|
|
2625
|
+
if (userTextParts.length) {
|
|
2626
|
+
newMessages.push({ role: "user", content: userTextParts.join("\n") });
|
|
2627
|
+
}
|
|
2628
|
+
continue;
|
|
2629
|
+
}
|
|
2630
|
+
const textParts = [];
|
|
2631
|
+
const toolCalls = [];
|
|
2632
|
+
for (const b of blocks) {
|
|
2633
|
+
if (!b) continue;
|
|
2634
|
+
if (b.type === "text" && b.text) {
|
|
2635
|
+
textParts.push(b.text);
|
|
2636
|
+
} else if (b.type === "tool_use") {
|
|
2637
|
+
const isOrphan = !declaredTools.has(b.name);
|
|
2638
|
+
if (process.env.PROXY_DEBUG) {
|
|
2639
|
+
console.error(`[adapt] tool_use name=${b.name} isOrphan=${isOrphan} declaredHas=${declaredTools.has(b.name)} declaredSize=${declaredTools.size}`);
|
|
2640
|
+
}
|
|
2641
|
+
if (isOrphan) {
|
|
2642
|
+
const argStr = typeof b.input === "string" ? b.input : JSON.stringify(b.input ?? {});
|
|
2643
|
+
textParts.push(`[Used tool ${b.name} with args ${argStr}]`);
|
|
2644
|
+
if (b.id) orphanedToolUseIds.add(b.id);
|
|
2645
|
+
} else {
|
|
2646
|
+
toolCalls.push({
|
|
2647
|
+
id: b.id || `call_${Date.now()}_${Math.random().toString(36).slice(2, 8)}`,
|
|
2648
|
+
type: "function",
|
|
2649
|
+
function: {
|
|
2650
|
+
name: b.name,
|
|
2651
|
+
arguments: typeof b.input === "string" ? b.input : JSON.stringify(b.input ?? {})
|
|
2652
|
+
}
|
|
2653
|
+
});
|
|
2654
|
+
}
|
|
2655
|
+
}
|
|
2656
|
+
}
|
|
2657
|
+
const newMsg = { role: msg.role };
|
|
2658
|
+
newMsg.content = textParts.length ? textParts.join("\n") : toolCalls.length ? null : "";
|
|
2659
|
+
if (toolCalls.length) newMsg.tool_calls = toolCalls;
|
|
2660
|
+
if (msg.name) newMsg.name = msg.name;
|
|
2661
|
+
newMessages.push(newMsg);
|
|
2662
|
+
}
|
|
2663
|
+
payload.messages = newMessages;
|
|
2664
|
+
}
|
|
2665
|
+
delete payload.thinking;
|
|
2666
|
+
delete payload.context_management;
|
|
2667
|
+
delete payload.top_k;
|
|
2668
|
+
delete payload.metadata;
|
|
2669
|
+
delete payload.betas;
|
|
2670
|
+
delete payload.anthropic_beta;
|
|
2671
|
+
delete payload.stop_sequences;
|
|
2672
|
+
delete payload.cache_control;
|
|
2673
|
+
delete payload.output_config;
|
|
2674
|
+
delete payload.service_tier;
|
|
2675
|
+
delete payload.mcp_servers;
|
|
2676
|
+
}
|
|
2677
|
+
function stripThinkingFromAnthropicJson(jsonStr) {
|
|
2678
|
+
try {
|
|
2679
|
+
const obj = JSON.parse(jsonStr);
|
|
2680
|
+
if (Array.isArray(obj.content)) {
|
|
2681
|
+
obj.content = obj.content.filter(
|
|
2682
|
+
(b) => b && b.type !== "thinking" && b.type !== "redacted_thinking"
|
|
2683
|
+
);
|
|
2684
|
+
}
|
|
2685
|
+
return JSON.stringify(obj);
|
|
2686
|
+
} catch {
|
|
2687
|
+
return jsonStr;
|
|
2688
|
+
}
|
|
2689
|
+
}
|
|
2690
|
+
var DEFAULT_EMPTY_TURN_RETRIES = 1;
|
|
2691
|
+
function resolveEmptyTurnRetries() {
|
|
2692
|
+
const raw = process.env.LLM_ENDPOINT_EMPTY_TURN_RETRY;
|
|
2693
|
+
if (raw === void 0) return DEFAULT_EMPTY_TURN_RETRIES;
|
|
2694
|
+
const n = Number.parseInt(raw, 10);
|
|
2695
|
+
return Number.isFinite(n) && n >= 0 ? n : DEFAULT_EMPTY_TURN_RETRIES;
|
|
2696
|
+
}
|
|
2697
|
+
function isEmptyAnthropicTurn(jsonStr) {
|
|
2698
|
+
try {
|
|
2699
|
+
const obj = JSON.parse(jsonStr);
|
|
2700
|
+
if (!obj || obj.type === "error" || !Array.isArray(obj.content)) return false;
|
|
2701
|
+
if (obj.stop_reason !== "end_turn") return false;
|
|
2702
|
+
return !obj.content.some(
|
|
2703
|
+
(b) => b && b.type !== "thinking" && b.type !== "redacted_thinking"
|
|
2704
|
+
);
|
|
2705
|
+
} catch {
|
|
2706
|
+
return false;
|
|
2707
|
+
}
|
|
2708
|
+
}
|
|
2709
|
+
var AnthropicThinkingStripper = class {
|
|
2710
|
+
skipIndices = /* @__PURE__ */ new Set();
|
|
2711
|
+
indexMap = /* @__PURE__ */ new Map();
|
|
2712
|
+
nextOut = 0;
|
|
2713
|
+
buffer = "";
|
|
2714
|
+
/**
|
|
2715
|
+
* True once a NON-thinking content block has started — i.e. the turn is
|
|
2716
|
+
* producing something the client will actually see (text or tool_use).
|
|
2717
|
+
* Read by the empty-turn retry: a turn that ends without this produced
|
|
2718
|
+
* nothing but stripped thinking, so nothing has been written downstream yet
|
|
2719
|
+
* and the attempt is still safely discardable.
|
|
2720
|
+
*/
|
|
2721
|
+
hasMeaningfulContent = false;
|
|
2722
|
+
/** stop_reason seen on message_delta, if any. */
|
|
2723
|
+
stopReason = null;
|
|
2724
|
+
push(chunk) {
|
|
2725
|
+
this.buffer += chunk;
|
|
2726
|
+
const out = [];
|
|
2727
|
+
const parts = this.buffer.split(/\r?\n\r?\n/);
|
|
2728
|
+
this.buffer = parts.pop() ?? "";
|
|
2729
|
+
for (const part of parts) {
|
|
2730
|
+
const transformed = this.transformEvent(part);
|
|
2731
|
+
if (transformed) out.push(transformed);
|
|
2732
|
+
}
|
|
2733
|
+
return out;
|
|
2734
|
+
}
|
|
2735
|
+
flush() {
|
|
2736
|
+
if (this.buffer.trim()) {
|
|
2737
|
+
const transformed = this.transformEvent(this.buffer);
|
|
2738
|
+
this.buffer = "";
|
|
2739
|
+
return transformed ? [transformed] : [];
|
|
2740
|
+
}
|
|
2741
|
+
return [];
|
|
2742
|
+
}
|
|
2743
|
+
transformEvent(rawEvent) {
|
|
2744
|
+
const lines = rawEvent.split(/\r?\n/);
|
|
2745
|
+
let eventLine = null;
|
|
2746
|
+
const dataLines = [];
|
|
2747
|
+
for (const line of lines) {
|
|
2748
|
+
if (line.startsWith("event:")) eventLine = line.slice(6).trim();
|
|
2749
|
+
else if (line.startsWith("data:")) dataLines.push(line.slice(5).trim());
|
|
2750
|
+
}
|
|
2751
|
+
if (!dataLines.length) return rawEvent ? rawEvent + "\n\n" : null;
|
|
2752
|
+
let data;
|
|
2753
|
+
try {
|
|
2754
|
+
data = JSON.parse(dataLines.join("\n"));
|
|
2755
|
+
} catch {
|
|
2756
|
+
return rawEvent + "\n\n";
|
|
2757
|
+
}
|
|
2758
|
+
const type = data.type;
|
|
2759
|
+
if (type === "content_block_start") {
|
|
2760
|
+
const ct = data.content_block && data.content_block.type;
|
|
2761
|
+
if (ct === "thinking" || ct === "redacted_thinking") {
|
|
2762
|
+
this.skipIndices.add(data.index);
|
|
2763
|
+
return null;
|
|
2764
|
+
}
|
|
2765
|
+
const outIdx = this.nextOut++;
|
|
2766
|
+
this.indexMap.set(data.index, outIdx);
|
|
2767
|
+
data.index = outIdx;
|
|
2768
|
+
this.hasMeaningfulContent = true;
|
|
2769
|
+
} else if (type === "message_delta") {
|
|
2770
|
+
const sr = data.delta && data.delta.stop_reason;
|
|
2771
|
+
if (typeof sr === "string") this.stopReason = sr;
|
|
2772
|
+
} else if (type === "content_block_delta") {
|
|
2773
|
+
if (this.skipIndices.has(data.index)) return null;
|
|
2774
|
+
const dt = data.delta && data.delta.type;
|
|
2775
|
+
if (dt === "thinking_delta" || dt === "signature_delta" || dt === "redacted_thinking_delta") return null;
|
|
2776
|
+
data.index = this.indexMap.get(data.index) ?? data.index;
|
|
2777
|
+
} else if (type === "content_block_stop") {
|
|
2778
|
+
if (this.skipIndices.has(data.index)) return null;
|
|
2779
|
+
data.index = this.indexMap.get(data.index) ?? data.index;
|
|
2780
|
+
}
|
|
2781
|
+
const ev = eventLine ? `event: ${eventLine}
|
|
2782
|
+
` : "";
|
|
2783
|
+
return ev + `data: ${JSON.stringify(data)}
|
|
2784
|
+
|
|
2785
|
+
`;
|
|
2786
|
+
}
|
|
2787
|
+
};
|
|
2788
|
+
function openaiFinishToAnthropicStop(fr) {
|
|
2789
|
+
switch (fr) {
|
|
2790
|
+
case "stop":
|
|
2791
|
+
return "end_turn";
|
|
2792
|
+
case "length":
|
|
2793
|
+
return "max_tokens";
|
|
2794
|
+
case "tool_calls":
|
|
2795
|
+
return "tool_use";
|
|
2796
|
+
case "content_filter":
|
|
2797
|
+
return "end_turn";
|
|
2798
|
+
default:
|
|
2799
|
+
return "end_turn";
|
|
2800
|
+
}
|
|
2801
|
+
}
|
|
2802
|
+
function openaiJsonToAnthropic(jsonStr) {
|
|
2803
|
+
try {
|
|
2804
|
+
const o = JSON.parse(jsonStr);
|
|
2805
|
+
if (o && typeof o === "object" && o.error) {
|
|
2806
|
+
const e = o.error;
|
|
2807
|
+
return JSON.stringify({
|
|
2808
|
+
type: "error",
|
|
2809
|
+
error: {
|
|
2810
|
+
type: e.type || "api_error",
|
|
2811
|
+
message: e.message || (typeof e === "string" ? e : "Upstream error")
|
|
2812
|
+
}
|
|
2813
|
+
});
|
|
2814
|
+
}
|
|
2815
|
+
const choice = o.choices && o.choices[0];
|
|
2816
|
+
const msg = choice && choice.message;
|
|
2817
|
+
const content = [];
|
|
2818
|
+
if (msg && typeof msg.content === "string" && msg.content) {
|
|
2819
|
+
content.push({ type: "text", text: msg.content });
|
|
2820
|
+
}
|
|
2821
|
+
if (msg && Array.isArray(msg.tool_calls)) {
|
|
2822
|
+
for (const tc of msg.tool_calls) {
|
|
2823
|
+
let input = {};
|
|
2824
|
+
try {
|
|
2825
|
+
input = tc.function && tc.function.arguments ? JSON.parse(tc.function.arguments) : {};
|
|
2826
|
+
} catch {
|
|
2827
|
+
input = {};
|
|
2828
|
+
}
|
|
2829
|
+
content.push({
|
|
2830
|
+
type: "tool_use",
|
|
2831
|
+
id: tc.id || `toolu_${Date.now()}`,
|
|
2832
|
+
name: tc.function && tc.function.name,
|
|
2833
|
+
input
|
|
2834
|
+
});
|
|
2835
|
+
}
|
|
2836
|
+
}
|
|
2837
|
+
if (content.length === 0 && msg && typeof msg.reasoning_content === "string" && msg.reasoning_content && passthroughThinking()) {
|
|
2838
|
+
content.push({ type: "thinking", thinking: msg.reasoning_content, signature: "" });
|
|
2839
|
+
}
|
|
2840
|
+
const usage = o.usage || {};
|
|
2841
|
+
const out = {
|
|
2842
|
+
id: o.id || `msg_${Date.now()}`,
|
|
2843
|
+
type: "message",
|
|
2844
|
+
role: "assistant",
|
|
2845
|
+
model: o.model || "groq",
|
|
2846
|
+
content,
|
|
2847
|
+
stop_reason: openaiFinishToAnthropicStop(choice && choice.finish_reason),
|
|
2848
|
+
stop_sequence: null,
|
|
2849
|
+
usage: {
|
|
2850
|
+
input_tokens: usage.prompt_tokens || 0,
|
|
2851
|
+
output_tokens: usage.completion_tokens || 0
|
|
2852
|
+
}
|
|
2853
|
+
};
|
|
2854
|
+
return JSON.stringify(out);
|
|
2855
|
+
} catch {
|
|
2856
|
+
return jsonStr;
|
|
2857
|
+
}
|
|
2858
|
+
}
|
|
2859
|
+
var OpenAIToAnthropicStreamConverter = class {
|
|
2860
|
+
msgId = `msg_${Date.now()}`;
|
|
2861
|
+
sentStart = false;
|
|
2862
|
+
textBlockOpen = false;
|
|
2863
|
+
toolBlockIdx = -1;
|
|
2864
|
+
buffer = "";
|
|
2865
|
+
accUsage = {};
|
|
2866
|
+
finished = false;
|
|
2867
|
+
push(chunk) {
|
|
2868
|
+
this.buffer += chunk;
|
|
2869
|
+
const out = [];
|
|
2870
|
+
const parts = this.buffer.split(/\r?\n\r?\n/);
|
|
2871
|
+
this.buffer = parts.pop() ?? "";
|
|
2872
|
+
for (const part of parts) {
|
|
2873
|
+
const transformed = this.transformChunk(part);
|
|
2874
|
+
if (transformed) out.push(...transformed);
|
|
2875
|
+
}
|
|
2876
|
+
return out;
|
|
2877
|
+
}
|
|
2878
|
+
flush() {
|
|
2879
|
+
if (this.buffer.trim()) {
|
|
2880
|
+
const transformed = this.transformChunk(this.buffer);
|
|
2881
|
+
this.buffer = "";
|
|
2882
|
+
return transformed || [];
|
|
2883
|
+
}
|
|
2884
|
+
return [];
|
|
2885
|
+
}
|
|
2886
|
+
sse(event, data) {
|
|
2887
|
+
return `event: ${event}
|
|
2888
|
+
data: ${JSON.stringify(data)}
|
|
2889
|
+
|
|
2890
|
+
`;
|
|
2891
|
+
}
|
|
2892
|
+
transformChunk(rawEvent) {
|
|
2893
|
+
const lines = rawEvent.split(/\r?\n/);
|
|
2894
|
+
const dataLines = [];
|
|
2895
|
+
for (const line of lines) {
|
|
2896
|
+
if (line.startsWith("data:")) dataLines.push(line.slice(5).trim());
|
|
2897
|
+
}
|
|
2898
|
+
if (!dataLines.length) return null;
|
|
2899
|
+
const out = [];
|
|
2900
|
+
for (const dl of dataLines) {
|
|
2901
|
+
if (dl === "[DONE]") {
|
|
2902
|
+
if (this.finished) continue;
|
|
2903
|
+
if (this.textBlockOpen) {
|
|
2904
|
+
out.push(this.sse("content_block_stop", { type: "content_block_stop", index: 0 }));
|
|
2905
|
+
this.textBlockOpen = false;
|
|
2906
|
+
}
|
|
2907
|
+
out.push(this.sse("message_delta", { type: "message_delta", delta: { stop_reason: "end_turn", stop_sequence: null }, usage: { input_tokens: this.accUsage.prompt_tokens || 0, output_tokens: this.accUsage.completion_tokens || 0 } }));
|
|
2908
|
+
out.push(this.sse("message_stop", { type: "message_stop" }));
|
|
2909
|
+
this.finished = true;
|
|
2910
|
+
continue;
|
|
2911
|
+
}
|
|
2912
|
+
let data;
|
|
2913
|
+
try {
|
|
2914
|
+
data = JSON.parse(dl);
|
|
2915
|
+
} catch {
|
|
2916
|
+
continue;
|
|
2917
|
+
}
|
|
2918
|
+
const choice = data.choices && data.choices[0];
|
|
2919
|
+
const delta = choice && choice.delta;
|
|
2920
|
+
const content = delta && delta.content;
|
|
2921
|
+
if (!this.sentStart && (content !== void 0 || delta && delta.role)) {
|
|
2922
|
+
out.push(this.sse("message_start", {
|
|
2923
|
+
type: "message_start",
|
|
2924
|
+
message: {
|
|
2925
|
+
id: this.msgId,
|
|
2926
|
+
type: "message",
|
|
2927
|
+
role: "assistant",
|
|
2928
|
+
model: data.model || "groq",
|
|
2929
|
+
content: [],
|
|
2930
|
+
stop_reason: null,
|
|
2931
|
+
stop_sequence: null,
|
|
2932
|
+
usage: { input_tokens: 0, output_tokens: 0 }
|
|
2933
|
+
}
|
|
2934
|
+
}));
|
|
2935
|
+
this.sentStart = true;
|
|
2936
|
+
}
|
|
2937
|
+
if (typeof content === "string" && content) {
|
|
2938
|
+
if (!this.textBlockOpen) {
|
|
2939
|
+
out.push(this.sse("content_block_start", { type: "content_block_start", index: 0, content_block: { type: "text", text: "" } }));
|
|
2940
|
+
this.textBlockOpen = true;
|
|
2941
|
+
}
|
|
2942
|
+
out.push(this.sse("content_block_delta", { type: "content_block_delta", index: 0, delta: { type: "text_delta", text: content } }));
|
|
2943
|
+
}
|
|
2944
|
+
if (delta && Array.isArray(delta.tool_calls)) {
|
|
2945
|
+
if (this.textBlockOpen) {
|
|
2946
|
+
out.push(this.sse("content_block_stop", { type: "content_block_stop", index: 0 }));
|
|
2947
|
+
this.textBlockOpen = false;
|
|
2948
|
+
}
|
|
2949
|
+
for (const tc of delta.tool_calls) {
|
|
2950
|
+
if (tc.function && tc.function.name && tc.index !== void 0) {
|
|
2951
|
+
this.toolBlockIdx = tc.index + 1;
|
|
2952
|
+
out.push(this.sse("content_block_start", {
|
|
2953
|
+
type: "content_block_start",
|
|
2954
|
+
index: this.toolBlockIdx,
|
|
2955
|
+
content_block: { type: "tool_use", id: tc.id || `toolu_${Date.now()}`, name: tc.function.name, input: {} }
|
|
2956
|
+
}));
|
|
2957
|
+
}
|
|
2958
|
+
if (tc.function && tc.function.arguments) {
|
|
2959
|
+
out.push(this.sse("content_block_delta", {
|
|
2960
|
+
type: "content_block_delta",
|
|
2961
|
+
index: this.toolBlockIdx,
|
|
2962
|
+
delta: { type: "input_json_delta", partial_json: tc.function.arguments }
|
|
2963
|
+
}));
|
|
2964
|
+
}
|
|
2965
|
+
}
|
|
2966
|
+
}
|
|
2967
|
+
const u = data.x_groq && data.x_groq.usage ? data.x_groq.usage : data.usage;
|
|
2968
|
+
if (u) {
|
|
2969
|
+
if (u.prompt_tokens != null) this.accUsage.prompt_tokens = u.prompt_tokens;
|
|
2970
|
+
if (u.completion_tokens != null) this.accUsage.completion_tokens = u.completion_tokens;
|
|
2971
|
+
}
|
|
2972
|
+
const fr = choice && choice.finish_reason;
|
|
2973
|
+
if (fr && !this.finished) {
|
|
2974
|
+
if (this.textBlockOpen) {
|
|
2975
|
+
out.push(this.sse("content_block_stop", { type: "content_block_stop", index: 0 }));
|
|
2976
|
+
this.textBlockOpen = false;
|
|
2977
|
+
}
|
|
2978
|
+
if (this.toolBlockIdx >= 0) {
|
|
2979
|
+
out.push(this.sse("content_block_stop", { type: "content_block_stop", index: this.toolBlockIdx }));
|
|
2980
|
+
this.toolBlockIdx = -1;
|
|
2981
|
+
}
|
|
2982
|
+
out.push(this.sse("message_delta", { type: "message_delta", delta: { stop_reason: openaiFinishToAnthropicStop(fr), stop_sequence: null }, usage: { input_tokens: this.accUsage.prompt_tokens || 0, output_tokens: this.accUsage.completion_tokens || 0 } }));
|
|
2983
|
+
out.push(this.sse("message_stop", { type: "message_stop" }));
|
|
2984
|
+
this.finished = true;
|
|
2985
|
+
}
|
|
2986
|
+
}
|
|
2987
|
+
return out.length ? out : null;
|
|
2988
|
+
}
|
|
2989
|
+
};
|
|
2990
|
+
function parseAccessTokens(raw) {
|
|
2991
|
+
return new Set((raw ?? "").split(",").map((t) => t.trim()).filter(Boolean));
|
|
2992
|
+
}
|
|
2993
|
+
var _accessTokens = null;
|
|
2994
|
+
function accessTokens() {
|
|
2995
|
+
if (_accessTokens === null) _accessTokens = parseAccessTokens(process.env.LLM_ENDPOINT_ACCESS_TOKENS);
|
|
2996
|
+
return _accessTokens;
|
|
2997
|
+
}
|
|
2998
|
+
function extractInboundToken(headers) {
|
|
2999
|
+
const auth = headers["authorization"];
|
|
3000
|
+
const a = Array.isArray(auth) ? auth[0] : auth;
|
|
3001
|
+
if (a) {
|
|
3002
|
+
const m = /^Bearer\s+(.+)$/i.exec(a.trim());
|
|
3003
|
+
if (m && m[1].trim()) return m[1].trim();
|
|
3004
|
+
}
|
|
3005
|
+
const xp = headers["x-proxy-access"];
|
|
3006
|
+
const x = Array.isArray(xp) ? xp[0] : xp;
|
|
3007
|
+
if (x && x.trim()) return x.trim();
|
|
3008
|
+
return null;
|
|
3009
|
+
}
|
|
3010
|
+
function isAuthorized(headers, tokens) {
|
|
3011
|
+
if (tokens.size === 0) return true;
|
|
3012
|
+
const presented = extractInboundToken(headers);
|
|
3013
|
+
return presented !== null && tokens.has(presented);
|
|
3014
|
+
}
|
|
3015
|
+
var _edgeTokens = null;
|
|
3016
|
+
function edgeTokens() {
|
|
3017
|
+
if (_edgeTokens === null) _edgeTokens = parseAccessTokens(process.env.LLM_ENDPOINT_EDGE_TOKENS);
|
|
3018
|
+
return _edgeTokens;
|
|
3019
|
+
}
|
|
3020
|
+
function extractEdgeToken(headers) {
|
|
3021
|
+
const raw = headers["x-edge-auth"];
|
|
3022
|
+
const v = Array.isArray(raw) ? raw[0] : raw;
|
|
3023
|
+
if (v && v.trim()) return v.trim();
|
|
3024
|
+
return null;
|
|
3025
|
+
}
|
|
3026
|
+
function isEdgeAuthorized(headers, edgeTokensSet) {
|
|
3027
|
+
if (edgeTokensSet.size === 0) return true;
|
|
3028
|
+
const presented = extractEdgeToken(headers);
|
|
3029
|
+
return presented !== null && edgeTokensSet.has(presented);
|
|
3030
|
+
}
|
|
3031
|
+
function buildWafRuleExpression(opts) {
|
|
3032
|
+
if (opts.tokens.length === 0) {
|
|
3033
|
+
throw new Error("buildWafRuleExpression requires at least one token");
|
|
3034
|
+
}
|
|
3035
|
+
const headerName = opts.header.toLowerCase();
|
|
3036
|
+
const values = opts.tokens.map((t) => opts.header === "authorization" ? `Bearer ${t}` : t);
|
|
3037
|
+
const memberships = values.map((v) => `any(http.request.headers["${headerName}"][*] eq "${v}")`);
|
|
3038
|
+
const notValid = memberships.length === 1 ? `not ${memberships[0]}` : `not (${memberships.join(" or ")})`;
|
|
3039
|
+
const hostPrefix = opts.host ? `http.host eq "${opts.host}" and ` : "";
|
|
3040
|
+
return `(${hostPrefix}http.request.method ne "OPTIONS" and ${notValid})`;
|
|
3041
|
+
}
|
|
3042
|
+
function normalizeProxyPath(pathname) {
|
|
3043
|
+
return pathname.replace(/\/+$/, "").replace(/^\/v1\/v1(\/|$)/, "/v1$1").replace(/^(\/v1\/[^/]+)\/v1(\/|$)/, "$1$2");
|
|
3044
|
+
}
|
|
3045
|
+
var _publicModels = null;
|
|
3046
|
+
function publicModels() {
|
|
3047
|
+
if (_publicModels === null) {
|
|
3048
|
+
const v = (process.env.LLM_ENDPOINT_PUBLIC_MODELS ?? "").trim().toLowerCase();
|
|
3049
|
+
_publicModels = v === "1" || v === "true" || v === "yes" || v === "on";
|
|
3050
|
+
}
|
|
3051
|
+
return _publicModels;
|
|
3052
|
+
}
|
|
3053
|
+
var _passthroughThinking = null;
|
|
3054
|
+
function passthroughThinking() {
|
|
3055
|
+
if (_passthroughThinking === null) {
|
|
3056
|
+
const v = (process.env.LLM_ENDPOINT_PASSTHROUGH_THINKING ?? "").trim().toLowerCase();
|
|
3057
|
+
_passthroughThinking = v === "1" || v === "true" || v === "yes" || v === "on";
|
|
3058
|
+
}
|
|
3059
|
+
return _passthroughThinking;
|
|
3060
|
+
}
|
|
3061
|
+
function isPublicModelListPath(pathname) {
|
|
3062
|
+
const p = normalizeProxyPath(pathname);
|
|
3063
|
+
return p === "/v1/models" || p === "/models";
|
|
3064
|
+
}
|
|
3065
|
+
var server = createServer((req, res) => {
|
|
3066
|
+
res.setHeader("Access-Control-Allow-Origin", "*");
|
|
3067
|
+
res.setHeader("Access-Control-Allow-Methods", "GET, POST, DELETE, OPTIONS");
|
|
3068
|
+
res.setHeader("Access-Control-Allow-Headers", "Content-Type, x-api-key, Authorization, anthropic-version, X-Proxy-Access, X-Proxy-Pack, X-Proxy-Format, X-Edge-Auth");
|
|
3069
|
+
if (req.method === "OPTIONS") {
|
|
3070
|
+
res.writeHead(204);
|
|
3071
|
+
res.end();
|
|
3072
|
+
return;
|
|
3073
|
+
}
|
|
3074
|
+
const rawUrl = req.url || "";
|
|
3075
|
+
const parsedUrl = new URL(rawUrl, `http://localhost:${PORT}`);
|
|
3076
|
+
const urlPath = normalizeProxyPath(parsedUrl.pathname);
|
|
3077
|
+
const gateExempt = publicModels() && isPublicModelListPath(urlPath);
|
|
3078
|
+
const internalLoopback = isInternalLoopbackRequest(req.headers);
|
|
3079
|
+
if (!gateExempt && !internalLoopback && !isAuthorized(req.headers, accessTokens())) {
|
|
3080
|
+
res.writeHead(401, { "Content-Type": "application/json" });
|
|
3081
|
+
res.end(JSON.stringify({ error: { type: "authentication_error", message: "Missing or invalid proxy access token." } }));
|
|
3082
|
+
return;
|
|
3083
|
+
}
|
|
3084
|
+
if (!gateExempt && !internalLoopback && !isEdgeAuthorized(req.headers, edgeTokens())) {
|
|
3085
|
+
res.writeHead(401, { "Content-Type": "application/json" });
|
|
3086
|
+
res.end(JSON.stringify({ error: { type: "authentication_error", message: "Missing or invalid edge token." } }));
|
|
3087
|
+
return;
|
|
3088
|
+
}
|
|
3089
|
+
let packId = null;
|
|
3090
|
+
const proxyPackHeader = req.headers["x-proxy-pack"];
|
|
3091
|
+
const proxyPackValue = (Array.isArray(proxyPackHeader) ? proxyPackHeader[0] : proxyPackHeader) || null;
|
|
3092
|
+
if (proxyPackValue) {
|
|
3093
|
+
if (getMergedPackIds().includes(proxyPackValue)) {
|
|
3094
|
+
packId = proxyPackValue;
|
|
3095
|
+
} else {
|
|
3096
|
+
console.warn(`[Proxy] Unknown pack "${proxyPackValue}" via X-Proxy-Pack header \u2014 returning 400.`);
|
|
3097
|
+
res.writeHead(400, { "Content-Type": "application/json" });
|
|
3098
|
+
res.end(JSON.stringify({ error: { type: "invalid_request_error", message: `Unknown pack "${proxyPackValue}" via X-Proxy-Pack. Available: ${getMergedPackIds().join(", ")}` } }));
|
|
3099
|
+
return;
|
|
3100
|
+
}
|
|
3101
|
+
}
|
|
3102
|
+
if (!packId) {
|
|
3103
|
+
const packPathMatch = urlPath.match(/^\/v1\/([^\/]+)(?:\/messages(?:\/batches(?:\/[^/]+)?(?:\/(?:results|cancel))?)?|\/models|\/chat\/completions|\/responses)?$/);
|
|
3104
|
+
if (packPathMatch) {
|
|
3105
|
+
const potentialPack = packPathMatch[1];
|
|
3106
|
+
const RESERVED_SEGMENTS = /* @__PURE__ */ new Set(["v1", "messages", "models", "packs", "chat", "responses", "upstreams", "endpoints"]);
|
|
3107
|
+
if (potentialPack && !RESERVED_SEGMENTS.has(potentialPack)) {
|
|
3108
|
+
if (getMergedPackIds().includes(potentialPack)) {
|
|
3109
|
+
packId = potentialPack;
|
|
3110
|
+
} else {
|
|
3111
|
+
console.warn(`[Proxy] Unknown pack "${potentialPack}" via URL path \u2014 returning 400.`);
|
|
3112
|
+
res.writeHead(400, { "Content-Type": "application/json" });
|
|
3113
|
+
res.end(JSON.stringify({ error: { type: "invalid_request_error", message: `Unknown pack "${potentialPack}" in URL path. Available: ${getMergedPackIds().join(", ")}` } }));
|
|
3114
|
+
return;
|
|
3115
|
+
}
|
|
3116
|
+
}
|
|
3117
|
+
}
|
|
3118
|
+
}
|
|
3119
|
+
if (!packId) {
|
|
3120
|
+
const queryPack = parsedUrl.searchParams.get("pack");
|
|
3121
|
+
if (queryPack) {
|
|
3122
|
+
if (getMergedPackIds().includes(queryPack)) {
|
|
3123
|
+
packId = queryPack;
|
|
3124
|
+
} else {
|
|
3125
|
+
console.warn(`[Proxy] Unknown pack "${queryPack}" \u2014 returning 400.`);
|
|
3126
|
+
res.writeHead(400, { "Content-Type": "application/json" });
|
|
3127
|
+
res.end(JSON.stringify({ error: { type: "invalid_request_error", message: `Unknown pack "${queryPack}". Available: ${getMergedPackIds().join(", ")}` } }));
|
|
3128
|
+
return;
|
|
3129
|
+
}
|
|
3130
|
+
}
|
|
3131
|
+
}
|
|
3132
|
+
const { queryProvider, queryModelCode, queryTools, queryNoTools, headerTools, headerNoTools, headerExcludeTools, forcedAliasCode, anthropicFormat } = readQueryAndToolOptions(parsedUrl, req);
|
|
3133
|
+
const resolvedPack = resolvePackMerged(packId);
|
|
3134
|
+
const activePack = anthropicFormat ? toAnthropicStyle(resolvedPack) : resolvedPack;
|
|
3135
|
+
console.log(
|
|
3136
|
+
`[Proxy] Incoming request: ${req.method} ${rawUrl}` + (urlPath !== parsedUrl.pathname.replace(/\/+$/, "") ? ` (normalized: ${urlPath})` : "") + ` [pack: ${activePack.id}]`
|
|
3137
|
+
);
|
|
3138
|
+
if (req.method === "POST" && urlPath.endsWith("/responses")) {
|
|
3139
|
+
handleResponsesRequest(req, res, {
|
|
3140
|
+
activePack,
|
|
3141
|
+
queryModelCode,
|
|
3142
|
+
queryProvider,
|
|
3143
|
+
queryTools,
|
|
3144
|
+
queryNoTools,
|
|
3145
|
+
headerTools,
|
|
3146
|
+
headerNoTools,
|
|
3147
|
+
headerExcludeTools,
|
|
3148
|
+
forcedAliasCode
|
|
3149
|
+
});
|
|
3150
|
+
return;
|
|
3151
|
+
}
|
|
3152
|
+
if (req.method === "POST" && urlPath.endsWith("/chat/completions")) {
|
|
3153
|
+
handleChatCompletionsRequest(req, res, {
|
|
3154
|
+
activePack,
|
|
3155
|
+
queryProvider,
|
|
3156
|
+
queryTools,
|
|
3157
|
+
queryNoTools,
|
|
3158
|
+
headerTools,
|
|
3159
|
+
headerNoTools,
|
|
3160
|
+
headerExcludeTools
|
|
3161
|
+
});
|
|
3162
|
+
return;
|
|
3163
|
+
}
|
|
3164
|
+
if (req.method === "GET" && (urlPath === "/v1/packs" || urlPath === "/packs")) {
|
|
3165
|
+
const mergedRegistry = { ...PACK_REGISTRY, ...getLocalPacks() };
|
|
3166
|
+
const packsResponse = {
|
|
3167
|
+
object: "list",
|
|
3168
|
+
data: Object.values(mergedRegistry).map((pack) => ({
|
|
3169
|
+
id: pack.id,
|
|
3170
|
+
label: pack.label,
|
|
3171
|
+
description: pack.description,
|
|
3172
|
+
model_count: Object.keys(pack.models).length,
|
|
3173
|
+
models: Object.entries(pack.models).map(([code, target]) => ({
|
|
3174
|
+
code,
|
|
3175
|
+
provider: target.provider,
|
|
3176
|
+
model: target.model,
|
|
3177
|
+
equivalent_claude_name: target.equivalentClaudeName
|
|
3178
|
+
}))
|
|
3179
|
+
}))
|
|
3180
|
+
};
|
|
3181
|
+
console.log(`[Proxy] Returning pack list (${packsResponse.data.length} packs).`);
|
|
3182
|
+
res.writeHead(200, { "Content-Type": "application/json" });
|
|
3183
|
+
res.end(JSON.stringify(packsResponse));
|
|
3184
|
+
return;
|
|
3185
|
+
}
|
|
3186
|
+
if (req.method === "POST" && (urlPath === "/v1/packs/reload" || urlPath === "/packs/reload")) {
|
|
3187
|
+
const load = readLocalPacksFromDisk();
|
|
3188
|
+
if (load.errors.length > 0) {
|
|
3189
|
+
console.warn(`[Proxy] Pack reload rejected \u2014 ${load.errors.length} validation error(s).`);
|
|
3190
|
+
res.writeHead(400, { "Content-Type": "application/json" });
|
|
3191
|
+
res.end(JSON.stringify({ error: { type: "invalid_request_error", message: "Invalid packs.local.json", errors: load.errors } }));
|
|
3192
|
+
return;
|
|
3193
|
+
}
|
|
3194
|
+
resetLocalPacksCache();
|
|
3195
|
+
const localPacks = getLocalPacks();
|
|
3196
|
+
const packIds = getMergedPackIds();
|
|
3197
|
+
console.log(`[Proxy] Reloaded packs from ${load.path ?? "(no local file)"} \u2014 ${packIds.length} packs (${Object.keys(localPacks).length} local).`);
|
|
3198
|
+
res.writeHead(200, { "Content-Type": "application/json" });
|
|
3199
|
+
res.end(JSON.stringify({
|
|
3200
|
+
object: "packs.reload",
|
|
3201
|
+
reloaded: true,
|
|
3202
|
+
source: load.path,
|
|
3203
|
+
local_pack_ids: Object.keys(localPacks),
|
|
3204
|
+
pack_ids: packIds,
|
|
3205
|
+
count: packIds.length
|
|
3206
|
+
}));
|
|
3207
|
+
return;
|
|
3208
|
+
}
|
|
3209
|
+
if (req.method === "GET" && (urlPath === "/v1/upstreams" || urlPath === "/upstreams")) {
|
|
3210
|
+
const probe = parsedUrl.searchParams.get("probe") === "1";
|
|
3211
|
+
void handleUpstreamsStatus(res, { probe });
|
|
3212
|
+
return;
|
|
3213
|
+
}
|
|
3214
|
+
const upstreamTestMatch = urlPath.match(/^\/(?:v1\/)?upstreams\/([^/]+)\/test$/);
|
|
3215
|
+
if (req.method === "POST" && upstreamTestMatch) {
|
|
3216
|
+
void handleUpstreamTest(res, upstreamTestMatch[1]);
|
|
3217
|
+
return;
|
|
3218
|
+
}
|
|
3219
|
+
if (req.method === "GET" && (urlPath === "/v1/endpoints" || urlPath === "/endpoints")) {
|
|
3220
|
+
void handleEndpointsStatus(res);
|
|
3221
|
+
return;
|
|
3222
|
+
}
|
|
3223
|
+
if (req.method === "GET" && (urlPath === "/v1/models" || urlPath === "/models" || urlPath.endsWith("/models"))) {
|
|
3224
|
+
void (async () => {
|
|
3225
|
+
const isAnthropicStyle = req.headers["anthropic-version"] !== void 0 || req.headers["x-api-key"] !== void 0;
|
|
3226
|
+
const mapping = buildMappingFromPack(activePack);
|
|
3227
|
+
const isLocalPack = Boolean(getLocalPacks()[activePack.id]);
|
|
3228
|
+
const isAliasPack = isLocalPack || anthropicFormat;
|
|
3229
|
+
if (activePack.id === DEFAULT_PACK_ID) {
|
|
3230
|
+
const forgeIds = await fetchForgeModelIds();
|
|
3231
|
+
for (const id of forgeIds) {
|
|
3232
|
+
mapping[`forge/${id}`] = { provider: "forge", model: id };
|
|
3233
|
+
}
|
|
3234
|
+
for (const endpoint of getConfiguredEndpoints()) {
|
|
3235
|
+
const { ids } = await probeFileEndpointModels(endpoint);
|
|
3236
|
+
for (const id of ids) {
|
|
3237
|
+
mapping[`${endpoint.id}/${id}`] = { provider: endpoint.id, model: id };
|
|
3238
|
+
}
|
|
3239
|
+
}
|
|
3240
|
+
}
|
|
3241
|
+
if (isAnthropicStyle) {
|
|
3242
|
+
const anthropicModelsResponse = {
|
|
3243
|
+
data: Object.entries(mapping).map(([code, target]) => {
|
|
3244
|
+
const id = isAliasPack && target.equivalentClaudeName ? target.equivalentClaudeName : code;
|
|
3245
|
+
return {
|
|
3246
|
+
id,
|
|
3247
|
+
display_name: code,
|
|
3248
|
+
created_at: "2026-02-04T00:00:00Z",
|
|
3249
|
+
type: "model",
|
|
3250
|
+
capabilities: {},
|
|
3251
|
+
// Champs officiels du schéma Anthropic ModelInfo (docs
|
|
3252
|
+
// /en/api/models-list) : max_input_tokens = fenêtre de contexte,
|
|
3253
|
+
// max_tokens = budget de sortie max. Absents si non vérifiés.
|
|
3254
|
+
...target.contextWindow !== void 0 ? { max_input_tokens: target.contextWindow } : {},
|
|
3255
|
+
...target.maxOutputTokens !== void 0 ? { max_tokens: target.maxOutputTokens } : {}
|
|
3256
|
+
};
|
|
3257
|
+
}),
|
|
3258
|
+
has_more: false,
|
|
3259
|
+
first_id: null,
|
|
3260
|
+
last_id: null
|
|
3261
|
+
};
|
|
3262
|
+
console.log(`[Proxy] Returning native Anthropic-formatted model list.`);
|
|
3263
|
+
res.writeHead(200, { "Content-Type": "application/json" });
|
|
3264
|
+
res.end(JSON.stringify(anthropicModelsResponse));
|
|
3265
|
+
return;
|
|
3266
|
+
} else {
|
|
3267
|
+
const openaiModelsResponse = {
|
|
3268
|
+
object: "list",
|
|
3269
|
+
data: Object.entries(mapping).map(([code, target]) => ({
|
|
3270
|
+
id: code,
|
|
3271
|
+
object: "model",
|
|
3272
|
+
created: 1718841600,
|
|
3273
|
+
owned_by: target.provider,
|
|
3274
|
+
// Verified per-route limits only — fields are absent when unknown.
|
|
3275
|
+
...target.contextWindow !== void 0 ? { context_length: target.contextWindow } : {},
|
|
3276
|
+
...target.maxOutputTokens !== void 0 ? { max_completion_tokens: target.maxOutputTokens } : {}
|
|
3277
|
+
}))
|
|
3278
|
+
};
|
|
3279
|
+
console.log(`[Proxy] Returning standard OpenAI-formatted model list.`);
|
|
3280
|
+
res.writeHead(200, { "Content-Type": "application/json" });
|
|
3281
|
+
res.end(JSON.stringify(openaiModelsResponse));
|
|
3282
|
+
return;
|
|
3283
|
+
}
|
|
3284
|
+
})();
|
|
3285
|
+
return;
|
|
3286
|
+
}
|
|
3287
|
+
if (handleBatchesRequest(
|
|
3288
|
+
req,
|
|
3289
|
+
res,
|
|
3290
|
+
{
|
|
3291
|
+
activePack,
|
|
3292
|
+
parsedUrl,
|
|
3293
|
+
queryModelCode,
|
|
3294
|
+
queryProvider,
|
|
3295
|
+
forcedAliasCode,
|
|
3296
|
+
anthropicFormat,
|
|
3297
|
+
queryTools,
|
|
3298
|
+
queryNoTools,
|
|
3299
|
+
headerTools,
|
|
3300
|
+
headerNoTools,
|
|
3301
|
+
headerExcludeTools
|
|
3302
|
+
},
|
|
3303
|
+
urlPath
|
|
3304
|
+
)) {
|
|
3305
|
+
return;
|
|
3306
|
+
}
|
|
3307
|
+
if (req.method !== "POST" || !urlPath.endsWith("/messages")) {
|
|
3308
|
+
res.writeHead(404, { "Content-Type": "application/json" });
|
|
3309
|
+
res.end(JSON.stringify({ error: { type: "not_found", message: `Route not found` } }));
|
|
3310
|
+
return;
|
|
3311
|
+
}
|
|
3312
|
+
let body = "";
|
|
3313
|
+
req.on("data", (chunk) => {
|
|
3314
|
+
body += chunk;
|
|
3315
|
+
});
|
|
3316
|
+
req.on("end", async () => {
|
|
3317
|
+
try {
|
|
3318
|
+
const payload = JSON.parse(body);
|
|
3319
|
+
let resolvedTarget;
|
|
3320
|
+
try {
|
|
3321
|
+
resolvedTarget = resolveModelRoute(payload, {
|
|
3322
|
+
activePack,
|
|
3323
|
+
queryModelCode,
|
|
3324
|
+
queryProvider,
|
|
3325
|
+
forcedAliasCode,
|
|
3326
|
+
allowAliases: true,
|
|
3327
|
+
anthropicFormat
|
|
3328
|
+
});
|
|
3329
|
+
} catch (e) {
|
|
3330
|
+
console.warn(`[Proxy] ${e.message}`);
|
|
3331
|
+
res.writeHead(400, { "Content-Type": "application/json" });
|
|
3332
|
+
res.end(JSON.stringify({ error: { type: "invalid_request_error", message: e.message } }));
|
|
3333
|
+
return;
|
|
3334
|
+
}
|
|
3335
|
+
if (process.env.LLM_ENDPOINT_SHORTCIRCUIT_WARMUP === "1" && payload.stream !== true && typeof payload.max_tokens === "number" && payload.max_tokens <= 1 && Array.isArray(payload.messages) && payload.messages.length <= 2) {
|
|
3336
|
+
console.log(`[Proxy] warm-up short-circuit: model=${resolvedTarget.provider}:${resolvedTarget.model} max_tokens=${payload.max_tokens} (no upstream call)`);
|
|
3337
|
+
res.writeHead(200, { "Content-Type": "application/json" });
|
|
3338
|
+
res.end(JSON.stringify({
|
|
3339
|
+
id: `msg_warmup_${Date.now().toString(36)}`,
|
|
3340
|
+
type: "message",
|
|
3341
|
+
role: "assistant",
|
|
3342
|
+
model: payload.model,
|
|
3343
|
+
content: [{ type: "text", text: "ok" }],
|
|
3344
|
+
stop_reason: "end_turn",
|
|
3345
|
+
stop_sequence: null,
|
|
3346
|
+
usage: { input_tokens: 1, output_tokens: 1 }
|
|
3347
|
+
}));
|
|
3348
|
+
return;
|
|
3349
|
+
}
|
|
3350
|
+
{
|
|
3351
|
+
const mt = payload.max_tokens;
|
|
3352
|
+
const mtNote = typeof mt === "number" && mt <= 4 ? " <== warm-up/1-token" : "";
|
|
3353
|
+
console.log(
|
|
3354
|
+
`[Proxy] req: model=${resolvedTarget.provider}:${resolvedTarget.model} max_tokens=${typeof mt === "number" ? mt : "unset"} stream=${payload.stream === true ? "true" : "false"} msgs=${Array.isArray(payload.messages) ? payload.messages.length : "?"} tools=${Array.isArray(payload.tools) ? payload.tools.length : 0}` + mtNote
|
|
3355
|
+
);
|
|
3356
|
+
}
|
|
3357
|
+
let hostname = "";
|
|
3358
|
+
let path2 = "";
|
|
3359
|
+
let port = 443;
|
|
3360
|
+
let protocol = "https";
|
|
3361
|
+
let targetApiKey = "";
|
|
3362
|
+
let cred;
|
|
3363
|
+
let headers = { "Content-Type": "application/json" };
|
|
3364
|
+
payload.model = resolvedTarget.model;
|
|
3365
|
+
trimTools(payload, {
|
|
3366
|
+
provider: resolvedTarget.provider,
|
|
3367
|
+
queryTools,
|
|
3368
|
+
queryNoTools,
|
|
3369
|
+
headerTools,
|
|
3370
|
+
headerNoTools,
|
|
3371
|
+
headerExcludeTools,
|
|
3372
|
+
packToolsExclude: activePack.toolsExclude,
|
|
3373
|
+
packToolsAllow: activePack.toolsAllow
|
|
3374
|
+
});
|
|
3375
|
+
const messagesConfigurableSpec = getConfigurableProviderSpec(resolvedTarget.provider);
|
|
3376
|
+
if (messagesConfigurableSpec) {
|
|
3377
|
+
const upstream = messagesConfigurableSpec.resolveUpstream();
|
|
3378
|
+
if (!upstream) {
|
|
3379
|
+
res.writeHead(400, { "Content-Type": "application/json" });
|
|
3380
|
+
res.end(JSON.stringify({ error: { type: "invalid_request_error", message: messagesConfigurableSpec.unavailableMessage() } }));
|
|
3381
|
+
return;
|
|
3382
|
+
}
|
|
3383
|
+
hostname = upstream.hostname;
|
|
3384
|
+
port = upstream.port;
|
|
3385
|
+
protocol = upstream.protocol;
|
|
3386
|
+
path2 = `${upstream.pathPrefix}/chat/completions`;
|
|
3387
|
+
cred = await resolveUpstreamCredential(resolvedTarget.provider);
|
|
3388
|
+
targetApiKey = cred?.value ?? "";
|
|
3389
|
+
if (cred && cred.value) Object.assign(headers, buildUpstreamAuthHeaders(resolvedTarget.provider, cred));
|
|
3390
|
+
const clientThinkingEnabled = isRecord(payload.thinking) && payload.thinking.type === "enabled";
|
|
3391
|
+
adaptAnthropicToOpenAI(payload);
|
|
3392
|
+
applyDefaultRequestFields(
|
|
3393
|
+
payload,
|
|
3394
|
+
messagesConfigurableSpec.defaultRequestFields,
|
|
3395
|
+
clientThinkingEnabled ? /* @__PURE__ */ new Set(["reasoning_effort"]) : void 0
|
|
3396
|
+
);
|
|
3397
|
+
if (payload.tools && Array.isArray(payload.tools)) {
|
|
3398
|
+
payload.tools = payload.tools.map((t) => {
|
|
3399
|
+
if (t.input_schema) {
|
|
3400
|
+
return {
|
|
3401
|
+
type: "function",
|
|
3402
|
+
function: {
|
|
3403
|
+
name: t.name,
|
|
3404
|
+
description: t.description,
|
|
3405
|
+
parameters: t.input_schema
|
|
3406
|
+
}
|
|
3407
|
+
};
|
|
3408
|
+
}
|
|
3409
|
+
return t;
|
|
3410
|
+
});
|
|
3411
|
+
}
|
|
3412
|
+
} else {
|
|
3413
|
+
switch (resolvedTarget.provider) {
|
|
3414
|
+
case "anthropic":
|
|
3415
|
+
hostname = "api.anthropic.com";
|
|
3416
|
+
path2 = "/v1/messages";
|
|
3417
|
+
cred = await resolveUpstreamCredential("anthropic");
|
|
3418
|
+
targetApiKey = cred?.value ?? "";
|
|
3419
|
+
if (cred) Object.assign(headers, buildUpstreamAuthHeaders("anthropic", cred));
|
|
3420
|
+
break;
|
|
3421
|
+
case "openrouter":
|
|
3422
|
+
hostname = "openrouter.ai";
|
|
3423
|
+
path2 = "/api/v1/messages";
|
|
3424
|
+
cred = await resolveUpstreamCredential("openrouter");
|
|
3425
|
+
targetApiKey = cred?.value ?? "";
|
|
3426
|
+
if (cred) Object.assign(headers, buildUpstreamAuthHeaders("openrouter", cred));
|
|
3427
|
+
headers["anthropic-version"] = "2023-06-01";
|
|
3428
|
+
break;
|
|
3429
|
+
case "requesty":
|
|
3430
|
+
hostname = "router.requesty.ai";
|
|
3431
|
+
path2 = "/v1/messages";
|
|
3432
|
+
cred = await resolveUpstreamCredential("requesty");
|
|
3433
|
+
targetApiKey = cred?.value ?? "";
|
|
3434
|
+
if (cred) Object.assign(headers, buildUpstreamAuthHeaders("requesty", cred));
|
|
3435
|
+
headers["anthropic-version"] = "2023-06-01";
|
|
3436
|
+
break;
|
|
3437
|
+
case "zai":
|
|
3438
|
+
hostname = "open.bigmodel.cn";
|
|
3439
|
+
path2 = "/api/paas/v4/chat/completions";
|
|
3440
|
+
cred = await resolveUpstreamCredential("zai");
|
|
3441
|
+
targetApiKey = cred?.value ?? "";
|
|
3442
|
+
if (cred) Object.assign(headers, buildUpstreamAuthHeaders("zai", cred));
|
|
3443
|
+
adaptAnthropicToOpenAI(payload);
|
|
3444
|
+
if (payload.tools && Array.isArray(payload.tools)) {
|
|
3445
|
+
payload.tools = payload.tools.map((t) => {
|
|
3446
|
+
if (t.input_schema) {
|
|
3447
|
+
return {
|
|
3448
|
+
type: "function",
|
|
3449
|
+
function: {
|
|
3450
|
+
name: t.name,
|
|
3451
|
+
description: t.description,
|
|
3452
|
+
parameters: t.input_schema
|
|
3453
|
+
}
|
|
3454
|
+
};
|
|
3455
|
+
}
|
|
3456
|
+
return t;
|
|
3457
|
+
});
|
|
3458
|
+
}
|
|
3459
|
+
break;
|
|
3460
|
+
case "groq":
|
|
3461
|
+
hostname = "api.groq.com";
|
|
3462
|
+
path2 = "/openai/v1/chat/completions";
|
|
3463
|
+
cred = await resolveUpstreamCredential("groq");
|
|
3464
|
+
targetApiKey = cred?.value ?? "";
|
|
3465
|
+
if (cred) Object.assign(headers, buildUpstreamAuthHeaders("groq", cred));
|
|
3466
|
+
adaptAnthropicToOpenAI(payload);
|
|
3467
|
+
delete payload.reasoning_format;
|
|
3468
|
+
delete payload.reasoning_effort;
|
|
3469
|
+
delete payload.include_reasoning;
|
|
3470
|
+
if (resolvedTarget.model === "qwen/qwen3.6-27b") {
|
|
3471
|
+
payload.reasoning_effort = "none";
|
|
3472
|
+
} else if (resolvedTarget.model.startsWith("openai/gpt-oss")) {
|
|
3473
|
+
payload.include_reasoning = false;
|
|
3474
|
+
}
|
|
3475
|
+
if (payload.tools && Array.isArray(payload.tools)) {
|
|
3476
|
+
payload.tools = payload.tools.map((t) => {
|
|
3477
|
+
if (t.input_schema) {
|
|
3478
|
+
return {
|
|
3479
|
+
type: "function",
|
|
3480
|
+
function: {
|
|
3481
|
+
name: t.name,
|
|
3482
|
+
description: t.description,
|
|
3483
|
+
parameters: t.input_schema
|
|
3484
|
+
}
|
|
3485
|
+
};
|
|
3486
|
+
}
|
|
3487
|
+
return t;
|
|
3488
|
+
});
|
|
3489
|
+
}
|
|
3490
|
+
break;
|
|
3491
|
+
case "xai":
|
|
3492
|
+
hostname = "api.x.ai";
|
|
3493
|
+
path2 = "/v1/chat/completions";
|
|
3494
|
+
cred = await resolveUpstreamCredential("xai");
|
|
3495
|
+
targetApiKey = cred?.value ?? "";
|
|
3496
|
+
if (cred) Object.assign(headers, buildUpstreamAuthHeaders("xai", cred));
|
|
3497
|
+
adaptAnthropicToOpenAI(payload);
|
|
3498
|
+
if (payload.tools && Array.isArray(payload.tools)) {
|
|
3499
|
+
payload.tools = payload.tools.map((t) => {
|
|
3500
|
+
if (t.input_schema) {
|
|
3501
|
+
return {
|
|
3502
|
+
type: "function",
|
|
3503
|
+
function: {
|
|
3504
|
+
name: t.name,
|
|
3505
|
+
description: t.description,
|
|
3506
|
+
parameters: t.input_schema
|
|
3507
|
+
}
|
|
3508
|
+
};
|
|
3509
|
+
}
|
|
3510
|
+
return t;
|
|
3511
|
+
});
|
|
3512
|
+
}
|
|
3513
|
+
break;
|
|
3514
|
+
case "openai":
|
|
3515
|
+
hostname = "api.openai.com";
|
|
3516
|
+
path2 = "/v1/chat/completions";
|
|
3517
|
+
cred = await resolveUpstreamCredential("openai");
|
|
3518
|
+
targetApiKey = cred?.value ?? "";
|
|
3519
|
+
if (cred) Object.assign(headers, buildUpstreamAuthHeaders("openai", cred));
|
|
3520
|
+
adaptAnthropicToOpenAI(payload);
|
|
3521
|
+
if (payload.tools && Array.isArray(payload.tools)) {
|
|
3522
|
+
payload.tools = payload.tools.map((t) => {
|
|
3523
|
+
if (t.input_schema) {
|
|
3524
|
+
return {
|
|
3525
|
+
type: "function",
|
|
3526
|
+
function: {
|
|
3527
|
+
name: t.name,
|
|
3528
|
+
description: t.description,
|
|
3529
|
+
parameters: t.input_schema
|
|
3530
|
+
}
|
|
3531
|
+
};
|
|
3532
|
+
}
|
|
3533
|
+
return t;
|
|
3534
|
+
});
|
|
3535
|
+
}
|
|
3536
|
+
break;
|
|
3537
|
+
case "moonshot":
|
|
3538
|
+
default:
|
|
3539
|
+
hostname = "api.moonshot.ai";
|
|
3540
|
+
path2 = "/anthropic/v1/messages";
|
|
3541
|
+
cred = await resolveUpstreamCredential("moonshot");
|
|
3542
|
+
targetApiKey = cred?.value ?? "";
|
|
3543
|
+
if (cred) Object.assign(headers, buildUpstreamAuthHeaders("moonshot", cred));
|
|
3544
|
+
headers["Accept"] = "text/event-stream";
|
|
3545
|
+
if (!payload.max_tokens) {
|
|
3546
|
+
payload.max_tokens = 4096;
|
|
3547
|
+
}
|
|
3548
|
+
if (resolvedTarget.model === "kimi-k2.7-code" && !payload.thinking) {
|
|
3549
|
+
payload.thinking = { type: "enabled", budget_tokens: 4e3 };
|
|
3550
|
+
}
|
|
3551
|
+
break;
|
|
3552
|
+
}
|
|
3553
|
+
}
|
|
3554
|
+
if (cred && buildUpstreamAuthHeaders(resolvedTarget.provider, cred) === null) {
|
|
3555
|
+
res.writeHead(401, { "Content-Type": "application/json" });
|
|
3556
|
+
res.end(JSON.stringify({ error: { type: "authentication_error", message: `The resolved credential for provider "${resolvedTarget.provider}" cannot be used on this upstream (subscription/oauth credentials are only valid for the anthropic upstream).` } }));
|
|
3557
|
+
return;
|
|
3558
|
+
}
|
|
3559
|
+
if (!targetApiKey && !isUpstreamKeyOptional(resolvedTarget.provider)) {
|
|
3560
|
+
res.writeHead(401, { "Content-Type": "application/json" });
|
|
3561
|
+
res.end(JSON.stringify({ error: { type: "authentication_error", message: `No API key for provider "${resolvedTarget.provider}"` } }));
|
|
3562
|
+
return;
|
|
3563
|
+
}
|
|
3564
|
+
console.log(`[Proxy Sortant] Redirection vers ${resolvedTarget.provider} (${hostname}${path2}) avec le mod\xE8le "${payload.model}"`);
|
|
3565
|
+
const options = {
|
|
3566
|
+
hostname,
|
|
3567
|
+
port,
|
|
3568
|
+
path: path2,
|
|
3569
|
+
method: "POST",
|
|
3570
|
+
headers
|
|
3571
|
+
};
|
|
3572
|
+
let emptyTurnRetriesLeft = resolvedTarget.provider === "openrouter" || resolvedTarget.provider === "requesty" ? resolveEmptyTurnRetries() : 0;
|
|
3573
|
+
const sendUpstream = () => {
|
|
3574
|
+
const upstreamStart = Date.now();
|
|
3575
|
+
let upstreamTtfbMs = null;
|
|
3576
|
+
const proxyReq = sendUpstreamRequest(protocol, options, (proxyRes) => {
|
|
3577
|
+
const status = proxyRes.statusCode || 200;
|
|
3578
|
+
if (upstreamTtfbMs === null) upstreamTtfbMs = Date.now() - upstreamStart;
|
|
3579
|
+
proxyRes.on("end", () => {
|
|
3580
|
+
if (process.env.LLM_ENDPOINT_DEBUG_TIMING === "1") {
|
|
3581
|
+
console.log(
|
|
3582
|
+
`[Proxy] upstream done: ${resolvedTarget.provider}:${resolvedTarget.model} status=${status} ttfb=${upstreamTtfbMs}ms total=${Date.now() - upstreamStart}ms stream=${payload.stream === true ? "true" : "false"}`
|
|
3583
|
+
);
|
|
3584
|
+
}
|
|
3585
|
+
});
|
|
3586
|
+
const contentType = proxyRes.headers["content-type"] || "";
|
|
3587
|
+
const isStreaming = payload.stream === true && /text\/event-stream/i.test(contentType);
|
|
3588
|
+
const retryEmptyTurn = () => {
|
|
3589
|
+
if (status !== 200 || emptyTurnRetriesLeft <= 0) return false;
|
|
3590
|
+
emptyTurnRetriesLeft--;
|
|
3591
|
+
console.warn(
|
|
3592
|
+
`\x1B[33m[Proxy] tour vide (end_turn sans texte ni tool_use) de ${resolvedTarget.provider}:${resolvedTarget.model} \u2014 rejeu (${emptyTurnRetriesLeft} restant(s))\x1B[0m`
|
|
3593
|
+
);
|
|
3594
|
+
proxyRes.resume();
|
|
3595
|
+
sendUpstream();
|
|
3596
|
+
return true;
|
|
3597
|
+
};
|
|
3598
|
+
const needsStrip = resolvedTarget.provider === "openrouter" || resolvedTarget.provider === "requesty";
|
|
3599
|
+
const needsConvert = resolvedTarget.provider === "groq" || resolvedTarget.provider === "zai" || resolvedTarget.provider === "xai" || resolvedTarget.provider === "openai" || Boolean(getConfigurableProviderSpec(resolvedTarget.provider));
|
|
3600
|
+
if (needsConvert && !isStreaming) {
|
|
3601
|
+
const respHeaders = { ...proxyRes.headers };
|
|
3602
|
+
delete respHeaders["content-length"];
|
|
3603
|
+
delete respHeaders["transfer-encoding"];
|
|
3604
|
+
respHeaders["content-type"] = "application/json";
|
|
3605
|
+
res.writeHead(status, respHeaders);
|
|
3606
|
+
let body2 = "";
|
|
3607
|
+
proxyRes.setEncoding("utf8");
|
|
3608
|
+
proxyRes.on("data", (c) => {
|
|
3609
|
+
body2 += c;
|
|
3610
|
+
});
|
|
3611
|
+
proxyRes.on("end", () => {
|
|
3612
|
+
res.end(openaiJsonToAnthropic(body2 || "{}"));
|
|
3613
|
+
});
|
|
3614
|
+
} else if (needsConvert && isStreaming) {
|
|
3615
|
+
const respHeaders = { ...proxyRes.headers };
|
|
3616
|
+
delete respHeaders["content-length"];
|
|
3617
|
+
respHeaders["content-type"] = "text/event-stream";
|
|
3618
|
+
res.writeHead(status, respHeaders);
|
|
3619
|
+
const converter = new OpenAIToAnthropicStreamConverter();
|
|
3620
|
+
proxyRes.setEncoding("utf8");
|
|
3621
|
+
proxyRes.on("data", (c) => {
|
|
3622
|
+
for (const out of converter.push(c)) res.write(out);
|
|
3623
|
+
});
|
|
3624
|
+
proxyRes.on("end", () => {
|
|
3625
|
+
for (const out of converter.flush()) res.write(out);
|
|
3626
|
+
res.end();
|
|
3627
|
+
});
|
|
3628
|
+
} else if (needsStrip && !isStreaming) {
|
|
3629
|
+
let body2 = "";
|
|
3630
|
+
proxyRes.setEncoding("utf8");
|
|
3631
|
+
proxyRes.on("data", (c) => {
|
|
3632
|
+
body2 += c;
|
|
3633
|
+
});
|
|
3634
|
+
proxyRes.on("end", () => {
|
|
3635
|
+
if (isEmptyAnthropicTurn(body2) && retryEmptyTurn()) return;
|
|
3636
|
+
const respHeaders = { ...proxyRes.headers };
|
|
3637
|
+
delete respHeaders["content-length"];
|
|
3638
|
+
delete respHeaders["transfer-encoding"];
|
|
3639
|
+
res.writeHead(status, respHeaders);
|
|
3640
|
+
res.end(stripThinkingFromAnthropicJson(body2 || "{}"));
|
|
3641
|
+
});
|
|
3642
|
+
} else if (needsStrip && isStreaming) {
|
|
3643
|
+
const stripper = new AnthropicThinkingStripper();
|
|
3644
|
+
const pending = [];
|
|
3645
|
+
let committed = false;
|
|
3646
|
+
const commit = () => {
|
|
3647
|
+
if (committed) return;
|
|
3648
|
+
committed = true;
|
|
3649
|
+
const respHeaders = { ...proxyRes.headers };
|
|
3650
|
+
delete respHeaders["content-length"];
|
|
3651
|
+
res.writeHead(status, respHeaders);
|
|
3652
|
+
for (const out of pending) res.write(out);
|
|
3653
|
+
pending.length = 0;
|
|
3654
|
+
};
|
|
3655
|
+
proxyRes.setEncoding("utf8");
|
|
3656
|
+
proxyRes.on("data", (c) => {
|
|
3657
|
+
for (const out of stripper.push(c)) {
|
|
3658
|
+
if (committed) res.write(out);
|
|
3659
|
+
else pending.push(out);
|
|
3660
|
+
}
|
|
3661
|
+
if (!committed && stripper.hasMeaningfulContent) commit();
|
|
3662
|
+
});
|
|
3663
|
+
proxyRes.on("end", () => {
|
|
3664
|
+
for (const out of stripper.flush()) {
|
|
3665
|
+
if (committed) res.write(out);
|
|
3666
|
+
else pending.push(out);
|
|
3667
|
+
}
|
|
3668
|
+
if (!committed && stripper.hasMeaningfulContent) commit();
|
|
3669
|
+
if (!committed) {
|
|
3670
|
+
const emptyTurn = stripper.stopReason === "end_turn";
|
|
3671
|
+
if (emptyTurn && retryEmptyTurn()) return;
|
|
3672
|
+
commit();
|
|
3673
|
+
}
|
|
3674
|
+
res.end();
|
|
3675
|
+
});
|
|
3676
|
+
} else {
|
|
3677
|
+
res.writeHead(status, proxyRes.headers);
|
|
3678
|
+
proxyRes.pipe(res);
|
|
3679
|
+
}
|
|
3680
|
+
});
|
|
3681
|
+
proxyReq.on("error", (err) => {
|
|
3682
|
+
console.error("[Proxy SORTANT error]", err);
|
|
3683
|
+
if (res.headersSent) {
|
|
3684
|
+
res.end();
|
|
3685
|
+
return;
|
|
3686
|
+
}
|
|
3687
|
+
res.writeHead(500, { "Content-Type": "application/json" });
|
|
3688
|
+
res.end(JSON.stringify({ error: { type: "api_error", message: err.message } }));
|
|
3689
|
+
});
|
|
3690
|
+
proxyReq.write(JSON.stringify(payload));
|
|
3691
|
+
proxyReq.end();
|
|
3692
|
+
};
|
|
3693
|
+
sendUpstream();
|
|
3694
|
+
} catch (e) {
|
|
3695
|
+
console.error("[Payload Error]", e);
|
|
3696
|
+
res.writeHead(400, { "Content-Type": "application/json" });
|
|
3697
|
+
res.end(JSON.stringify({ error: { type: "invalid_request_error", message: e.message } }));
|
|
3698
|
+
}
|
|
3699
|
+
});
|
|
3700
|
+
});
|
|
3701
|
+
function start(port = PORT) {
|
|
3702
|
+
void resumeIncompleteLocalQueueBatches().catch((err) => {
|
|
3703
|
+
console.error("[Proxy][batches] boot reconciliation failed", err);
|
|
3704
|
+
});
|
|
3705
|
+
return server.listen(port, () => {
|
|
3706
|
+
console.log(`[Proxy Server] Live on http://localhost:${port}`);
|
|
3707
|
+
console.log(`[Proxy Server] Anthropic Messages, OpenAI Chat Completions, and OpenAI Responses surfaces configured.`);
|
|
3708
|
+
});
|
|
3709
|
+
}
|
|
3710
|
+
|
|
3711
|
+
// src/cli.ts
|
|
3712
|
+
if (process.argv[2] === "print-waf-rule") {
|
|
3713
|
+
const hostFlagIdx = process.argv.indexOf("--host");
|
|
3714
|
+
const host = hostFlagIdx !== -1 ? process.argv[hostFlagIdx + 1] : process.env.LLM_ENDPOINT_PUBLIC_HOST || void 0;
|
|
3715
|
+
const edgeTokens2 = [...parseAccessTokens(process.env.LLM_ENDPOINT_EDGE_TOKENS)];
|
|
3716
|
+
const accessTokensList = [...parseAccessTokens(process.env.LLM_ENDPOINT_ACCESS_TOKENS)];
|
|
3717
|
+
const useEdge = edgeTokens2.length > 0;
|
|
3718
|
+
const tokens = useEdge ? edgeTokens2 : accessTokensList;
|
|
3719
|
+
const header = useEdge ? "x-edge-auth" : "authorization";
|
|
3720
|
+
if (tokens.length === 0) {
|
|
3721
|
+
console.error(
|
|
3722
|
+
"[llm-endpoint] print-waf-rule: no tokens found. Set LLM_ENDPOINT_EDGE_TOKENS or LLM_ENDPOINT_ACCESS_TOKENS first."
|
|
3723
|
+
);
|
|
3724
|
+
process.exit(1);
|
|
3725
|
+
}
|
|
3726
|
+
console.log(buildWafRuleExpression({ host, tokens, header }));
|
|
3727
|
+
process.exit(0);
|
|
3728
|
+
}
|
|
3729
|
+
try {
|
|
3730
|
+
const injected = await injectProviderKeysIntoEnv(process.env);
|
|
3731
|
+
if (injected.length > 0) {
|
|
3732
|
+
console.error(`[llm-endpoint] loaded ${injected.length} provider key(s) from store: ${injected.join(", ")}`);
|
|
3733
|
+
}
|
|
3734
|
+
} catch {
|
|
3735
|
+
}
|
|
3736
|
+
start();
|
|
3737
|
+
//# sourceMappingURL=cli.mjs.map
|
|
3738
|
+
//# sourceMappingURL=cli.mjs.map
|