@a5c-ai/transport-adapter 5.1.1-staging.00ceebd28cf2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +54 -0
- package/dist/__tests__/codecs.test.d.ts +1 -0
- package/dist/__tests__/codecs.test.d.ts.map +1 -0
- package/dist/__tests__/codecs.test.js +698 -0
- package/dist/__tests__/codecs.test.js.map +1 -0
- package/dist/__tests__/openai-engine.test.d.ts +1 -0
- package/dist/__tests__/openai-engine.test.d.ts.map +1 -0
- package/dist/__tests__/openai-engine.test.js +89 -0
- package/dist/__tests__/openai-engine.test.js.map +1 -0
- package/dist/bin/adapters-proxy.d.ts +2 -0
- package/dist/bin/adapters-proxy.d.ts.map +1 -0
- package/dist/bin/adapters-proxy.js +5 -0
- package/dist/bin/adapters-proxy.js.map +1 -0
- package/dist/bin/adapters-transport-proxy.d.ts +2 -0
- package/dist/bin/adapters-transport-proxy.d.ts.map +1 -0
- package/dist/bin/adapters-transport-proxy.js +57 -0
- package/dist/bin/adapters-transport-proxy.js.map +1 -0
- package/dist/codec.d.ts +36 -0
- package/dist/codec.d.ts.map +1 -0
- package/dist/codec.js +2 -0
- package/dist/codec.js.map +1 -0
- package/dist/codecs/anthropic.d.ts +12 -0
- package/dist/codecs/anthropic.d.ts.map +1 -0
- package/dist/codecs/anthropic.js +185 -0
- package/dist/codecs/anthropic.js.map +1 -0
- package/dist/codecs/bedrock.d.ts +12 -0
- package/dist/codecs/bedrock.d.ts.map +1 -0
- package/dist/codecs/bedrock.js +175 -0
- package/dist/codecs/bedrock.js.map +1 -0
- package/dist/codecs/google.d.ts +12 -0
- package/dist/codecs/google.d.ts.map +1 -0
- package/dist/codecs/google.js +176 -0
- package/dist/codecs/google.js.map +1 -0
- package/dist/codecs/index.d.ts +30 -0
- package/dist/codecs/index.d.ts.map +1 -0
- package/dist/codecs/index.js +115 -0
- package/dist/codecs/index.js.map +1 -0
- package/dist/codecs/openai-chat.d.ts +12 -0
- package/dist/codecs/openai-chat.d.ts.map +1 -0
- package/dist/codecs/openai-chat.js +191 -0
- package/dist/codecs/openai-chat.js.map +1 -0
- package/dist/codecs/openai-responses.d.ts +12 -0
- package/dist/codecs/openai-responses.d.ts.map +1 -0
- package/dist/codecs/openai-responses.js +229 -0
- package/dist/codecs/openai-responses.js.map +1 -0
- package/dist/config.d.ts +8 -0
- package/dist/config.d.ts.map +1 -0
- package/dist/config.js +86 -0
- package/dist/config.js.map +1 -0
- package/dist/engines/anthropic.d.ts +7 -0
- package/dist/engines/anthropic.d.ts.map +1 -0
- package/dist/engines/anthropic.js +232 -0
- package/dist/engines/anthropic.js.map +1 -0
- package/dist/engines/google.d.ts +17 -0
- package/dist/engines/google.d.ts.map +1 -0
- package/dist/engines/google.js +232 -0
- package/dist/engines/google.js.map +1 -0
- package/dist/engines/index.d.ts +3 -0
- package/dist/engines/index.d.ts.map +1 -0
- package/dist/engines/index.js +3 -0
- package/dist/engines/index.js.map +1 -0
- package/dist/engines/openai.d.ts +12 -0
- package/dist/engines/openai.d.ts.map +1 -0
- package/dist/engines/openai.js +253 -0
- package/dist/engines/openai.js.map +1 -0
- package/dist/index.d.ts +22 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +19 -0
- package/dist/index.js.map +1 -0
- package/dist/runtime.d.ts +19 -0
- package/dist/runtime.d.ts.map +1 -0
- package/dist/runtime.js +56 -0
- package/dist/runtime.js.map +1 -0
- package/dist/server.d.ts +4 -0
- package/dist/server.d.ts.map +1 -0
- package/dist/server.js +1428 -0
- package/dist/server.js.map +1 -0
- package/dist/types.d.ts +109 -0
- package/dist/types.d.ts.map +1 -0
- package/dist/types.js +11 -0
- package/dist/types.js.map +1 -0
- package/package.json +99 -0
package/dist/server.js
ADDED
|
@@ -0,0 +1,1428 @@
|
|
|
1
|
+
import * as http from 'node:http';
|
|
2
|
+
import { randomUUID } from 'node:crypto';
|
|
3
|
+
import { Readable } from 'node:stream';
|
|
4
|
+
import { pipeline } from 'node:stream/promises';
|
|
5
|
+
import { WebSocketServer } from 'ws';
|
|
6
|
+
import { Hono } from 'hono';
|
|
7
|
+
import { getCodec } from './codecs/index.js';
|
|
8
|
+
const STREAMING_TRANSPORTS = new Set([
|
|
9
|
+
'anthropic',
|
|
10
|
+
'openai-chat',
|
|
11
|
+
'openai-responses',
|
|
12
|
+
'google',
|
|
13
|
+
'bedrock-converse',
|
|
14
|
+
'passthrough',
|
|
15
|
+
]);
|
|
16
|
+
function headerValue(req, name) {
|
|
17
|
+
const value = req.headers.get(name);
|
|
18
|
+
return value && value.length > 0 ? value : undefined;
|
|
19
|
+
}
|
|
20
|
+
function costFeedbackMetadataFromRequest(req, defaults) {
|
|
21
|
+
return {
|
|
22
|
+
...defaults,
|
|
23
|
+
runId: headerValue(req, 'x-babysitter-run-id') ?? defaults?.runId,
|
|
24
|
+
sessionId: headerValue(req, 'x-babysitter-session-id') ?? defaults?.sessionId,
|
|
25
|
+
effectId: headerValue(req, 'x-babysitter-effect-id') ?? defaults?.effectId,
|
|
26
|
+
taskId: headerValue(req, 'x-babysitter-task-id') ?? defaults?.taskId,
|
|
27
|
+
taskKind: headerValue(req, 'x-babysitter-task-kind') ?? defaults?.taskKind,
|
|
28
|
+
source: headerValue(req, 'x-babysitter-cost-source') ?? defaults?.source,
|
|
29
|
+
idempotencyKey: headerValue(req, 'x-babysitter-cost-idempotency-key') ?? defaults?.idempotencyKey,
|
|
30
|
+
};
|
|
31
|
+
}
|
|
32
|
+
function shouldEmitCostFeedback(metadata) {
|
|
33
|
+
return Boolean(metadata?.runId || metadata?.sessionId || metadata?.effectId || metadata?.idempotencyKey);
|
|
34
|
+
}
|
|
35
|
+
function buildCostFeedbackRecord(result, config, metadata) {
|
|
36
|
+
const inputTokens = result.costRecord?.inputTokens ?? result.usage.promptTokens;
|
|
37
|
+
const outputTokens = result.costRecord?.outputTokens ?? result.usage.completionTokens;
|
|
38
|
+
return {
|
|
39
|
+
...metadata,
|
|
40
|
+
provider: config.targetProvider,
|
|
41
|
+
model: result.costRecord ? (result.model || config.targetModel) : (result.model || config.targetModel),
|
|
42
|
+
inputTokens,
|
|
43
|
+
outputTokens,
|
|
44
|
+
cacheReadTokens: result.costRecord?.cacheReadTokens,
|
|
45
|
+
cacheCreationTokens: result.costRecord?.cacheWriteTokens,
|
|
46
|
+
};
|
|
47
|
+
}
|
|
48
|
+
async function emitCostFeedback(sink, result, config, metadata) {
|
|
49
|
+
if (!sink || !shouldEmitCostFeedback(metadata))
|
|
50
|
+
return;
|
|
51
|
+
await sink(buildCostFeedbackRecord(result, config, metadata));
|
|
52
|
+
}
|
|
53
|
+
async function emitUsageCostFeedback(sink, usage, config, metadata) {
|
|
54
|
+
if (!sink || !usage || !shouldEmitCostFeedback(metadata))
|
|
55
|
+
return;
|
|
56
|
+
await sink({
|
|
57
|
+
...metadata,
|
|
58
|
+
provider: config.targetProvider,
|
|
59
|
+
model: config.targetModel,
|
|
60
|
+
inputTokens: usage.promptTokens,
|
|
61
|
+
outputTokens: usage.completionTokens,
|
|
62
|
+
});
|
|
63
|
+
}
|
|
64
|
+
class MetricsTracker {
|
|
65
|
+
totalInputTokens = 0;
|
|
66
|
+
totalOutputTokens = 0;
|
|
67
|
+
totalRequests = 0;
|
|
68
|
+
totalErrors = 0;
|
|
69
|
+
startedAt = Date.now();
|
|
70
|
+
record(inputTokens, outputTokens) {
|
|
71
|
+
this.totalInputTokens += inputTokens;
|
|
72
|
+
this.totalOutputTokens += outputTokens;
|
|
73
|
+
this.totalRequests += 1;
|
|
74
|
+
}
|
|
75
|
+
recordRequest() {
|
|
76
|
+
this.totalRequests += 1;
|
|
77
|
+
}
|
|
78
|
+
recordError() {
|
|
79
|
+
this.totalErrors += 1;
|
|
80
|
+
}
|
|
81
|
+
toJSON() {
|
|
82
|
+
const uptimeSeconds = Math.max(0, (Date.now() - this.startedAt) / 1000);
|
|
83
|
+
return {
|
|
84
|
+
total_input_tokens: this.totalInputTokens,
|
|
85
|
+
total_output_tokens: this.totalOutputTokens,
|
|
86
|
+
total_requests: this.totalRequests,
|
|
87
|
+
total_errors: this.totalErrors,
|
|
88
|
+
uptime_seconds: Math.round(uptimeSeconds * 10) / 10,
|
|
89
|
+
avg_tokens_per_request: Math.round(((this.totalInputTokens + this.totalOutputTokens) / Math.max(this.totalRequests, 1)) * 10) / 10,
|
|
90
|
+
};
|
|
91
|
+
}
|
|
92
|
+
}
|
|
93
|
+
function firstHeaderValue(value) {
|
|
94
|
+
return Array.isArray(value) ? value[0] : value;
|
|
95
|
+
}
|
|
96
|
+
function isAuthorizedHeaderValues(apiKey, authorization, authToken) {
|
|
97
|
+
if (!authToken) {
|
|
98
|
+
return true;
|
|
99
|
+
}
|
|
100
|
+
if (apiKey === authToken) {
|
|
101
|
+
return true;
|
|
102
|
+
}
|
|
103
|
+
if (!authorization) {
|
|
104
|
+
return false;
|
|
105
|
+
}
|
|
106
|
+
const [scheme, value] = authorization.split(/\s+/, 2);
|
|
107
|
+
return scheme?.toLowerCase() === 'bearer' && value === authToken;
|
|
108
|
+
}
|
|
109
|
+
function isAuthorized(req, authToken) {
|
|
110
|
+
return isAuthorizedHeaderValues(req.headers.get('x-api-key'), req.headers.get('authorization'), authToken);
|
|
111
|
+
}
|
|
112
|
+
function isAuthorizedUpgrade(req, authToken) {
|
|
113
|
+
return isAuthorizedHeaderValues(firstHeaderValue(req.headers['x-api-key']), firstHeaderValue(req.headers['authorization']), authToken);
|
|
114
|
+
}
|
|
115
|
+
function parseMessageContent(content) {
|
|
116
|
+
if (typeof content === 'string') {
|
|
117
|
+
return content;
|
|
118
|
+
}
|
|
119
|
+
if (Array.isArray(content)) {
|
|
120
|
+
return content
|
|
121
|
+
.map((part) => {
|
|
122
|
+
if (typeof part === 'string') {
|
|
123
|
+
return part;
|
|
124
|
+
}
|
|
125
|
+
if (part && typeof part === 'object' && 'text' in part && typeof part.text === 'string') {
|
|
126
|
+
return part.text;
|
|
127
|
+
}
|
|
128
|
+
return '';
|
|
129
|
+
})
|
|
130
|
+
.filter(Boolean)
|
|
131
|
+
.join(' ');
|
|
132
|
+
}
|
|
133
|
+
return '';
|
|
134
|
+
}
|
|
135
|
+
function normalizeMessages(raw) {
|
|
136
|
+
if (!Array.isArray(raw)) {
|
|
137
|
+
return [];
|
|
138
|
+
}
|
|
139
|
+
return raw.map((entry) => {
|
|
140
|
+
const record = entry;
|
|
141
|
+
return {
|
|
142
|
+
role: typeof record.role === 'string' ? record.role : 'user',
|
|
143
|
+
content: parseMessageContent(record.content ?? record.parts),
|
|
144
|
+
rawContent: record.content,
|
|
145
|
+
};
|
|
146
|
+
});
|
|
147
|
+
}
|
|
148
|
+
function normalizeInput(input) {
|
|
149
|
+
if (typeof input === 'string') {
|
|
150
|
+
return input;
|
|
151
|
+
}
|
|
152
|
+
if (Array.isArray(input)) {
|
|
153
|
+
return input
|
|
154
|
+
.map((entry) => {
|
|
155
|
+
if (typeof entry === 'string') {
|
|
156
|
+
return entry;
|
|
157
|
+
}
|
|
158
|
+
if (entry && typeof entry === 'object' && 'text' in entry && typeof entry.text === 'string') {
|
|
159
|
+
return entry.text;
|
|
160
|
+
}
|
|
161
|
+
return '';
|
|
162
|
+
})
|
|
163
|
+
.filter(Boolean)
|
|
164
|
+
.join(' ');
|
|
165
|
+
}
|
|
166
|
+
return '';
|
|
167
|
+
}
|
|
168
|
+
function toRecord(value) {
|
|
169
|
+
return value && typeof value === 'object' && !Array.isArray(value)
|
|
170
|
+
? value
|
|
171
|
+
: undefined;
|
|
172
|
+
}
|
|
173
|
+
function usageShape(result) {
|
|
174
|
+
return {
|
|
175
|
+
prompt_tokens: result.usage.promptTokens,
|
|
176
|
+
completion_tokens: result.usage.completionTokens,
|
|
177
|
+
total_tokens: result.usage.totalTokens,
|
|
178
|
+
};
|
|
179
|
+
}
|
|
180
|
+
async function readJsonBody(req) {
|
|
181
|
+
try {
|
|
182
|
+
const body = await req.json();
|
|
183
|
+
if (!body || typeof body !== 'object') {
|
|
184
|
+
return {};
|
|
185
|
+
}
|
|
186
|
+
return body;
|
|
187
|
+
}
|
|
188
|
+
catch {
|
|
189
|
+
return {};
|
|
190
|
+
}
|
|
191
|
+
}
|
|
192
|
+
async function readStrictJsonBody(req) {
|
|
193
|
+
try {
|
|
194
|
+
const body = await req.json();
|
|
195
|
+
if (!body || typeof body !== 'object' || Array.isArray(body)) {
|
|
196
|
+
return createErrorResponse('Request body must be a JSON object.');
|
|
197
|
+
}
|
|
198
|
+
return body;
|
|
199
|
+
}
|
|
200
|
+
catch {
|
|
201
|
+
return createErrorResponse('Request body must be valid JSON.');
|
|
202
|
+
}
|
|
203
|
+
}
|
|
204
|
+
function isStreamingRequestedByBody(body) {
|
|
205
|
+
return body.stream === true;
|
|
206
|
+
}
|
|
207
|
+
function createErrorResponse(message, status = 400) {
|
|
208
|
+
return new Response(JSON.stringify({ error: { message } }), {
|
|
209
|
+
status,
|
|
210
|
+
headers: { 'content-type': 'application/json' },
|
|
211
|
+
});
|
|
212
|
+
}
|
|
213
|
+
function supportsStreaming(transport) {
|
|
214
|
+
return STREAMING_TRANSPORTS.has(transport);
|
|
215
|
+
}
|
|
216
|
+
function buildCompletionRequest(body, transport, stream, thoughtSignatureStore) {
|
|
217
|
+
const codec = getCodec(transport);
|
|
218
|
+
if (codec) {
|
|
219
|
+
const decoded = codec.decodeRequest(body);
|
|
220
|
+
const rawMessages = Array.isArray(body.messages)
|
|
221
|
+
? body.messages
|
|
222
|
+
: Array.isArray(body.contents)
|
|
223
|
+
? body.contents
|
|
224
|
+
: undefined;
|
|
225
|
+
const messages = rawMessages
|
|
226
|
+
? decoded.messages.map((message, index) => {
|
|
227
|
+
const raw = rawMessages[index];
|
|
228
|
+
return {
|
|
229
|
+
...message,
|
|
230
|
+
rawContent: message.rawContent ?? raw?.content ?? raw?.parts,
|
|
231
|
+
};
|
|
232
|
+
})
|
|
233
|
+
: decoded.messages;
|
|
234
|
+
return {
|
|
235
|
+
...decoded,
|
|
236
|
+
transport,
|
|
237
|
+
messages,
|
|
238
|
+
model: typeof body.model === 'string'
|
|
239
|
+
? body.model
|
|
240
|
+
: typeof body.modelId === 'string'
|
|
241
|
+
? body.modelId
|
|
242
|
+
: decoded.model,
|
|
243
|
+
stream,
|
|
244
|
+
input: transport === 'openai-responses'
|
|
245
|
+
? decoded.input ?? normalizeInput(body.input)
|
|
246
|
+
: decoded.input,
|
|
247
|
+
thoughtSignatureStore,
|
|
248
|
+
};
|
|
249
|
+
}
|
|
250
|
+
const messages = transport === 'google' || transport === 'vertex-native'
|
|
251
|
+
? normalizeMessages(body.contents)
|
|
252
|
+
: transport === 'openai-responses'
|
|
253
|
+
? [{ role: 'user', content: normalizeInput(body.input) }]
|
|
254
|
+
: transport === 'bedrock-converse'
|
|
255
|
+
? normalizeMessages(Array.isArray(body.messages)
|
|
256
|
+
? body.messages.map((message) => ({
|
|
257
|
+
role: message.role,
|
|
258
|
+
content: Array.isArray(message.content) ? message.content : message.content,
|
|
259
|
+
}))
|
|
260
|
+
: [])
|
|
261
|
+
: normalizeMessages(body.messages);
|
|
262
|
+
return {
|
|
263
|
+
model: typeof body.model === 'string'
|
|
264
|
+
? body.model
|
|
265
|
+
: typeof body.modelId === 'string'
|
|
266
|
+
? body.modelId
|
|
267
|
+
: 'mock-model',
|
|
268
|
+
transport,
|
|
269
|
+
messages,
|
|
270
|
+
tools: Array.isArray(body.tools) ? body.tools : undefined,
|
|
271
|
+
toolChoice: body.tool_choice ?? body.toolChoice ?? undefined,
|
|
272
|
+
stream,
|
|
273
|
+
input: transport === 'openai-responses' ? normalizeInput(body.input) : undefined,
|
|
274
|
+
raw: body,
|
|
275
|
+
thoughtSignatureStore,
|
|
276
|
+
};
|
|
277
|
+
}
|
|
278
|
+
async function createExecutionPlan(req, transport, options = {}) {
|
|
279
|
+
const body = await readJsonBody(req.clone());
|
|
280
|
+
const bodyRequestedStream = isStreamingRequestedByBody(body);
|
|
281
|
+
if (transport === 'google' && bodyRequestedStream && !options.forceStreaming) {
|
|
282
|
+
return createErrorResponse('Google streaming requires the dedicated :streamGenerateContent route.');
|
|
283
|
+
}
|
|
284
|
+
const streamRequested = options.forceStreaming === true || bodyRequestedStream;
|
|
285
|
+
return {
|
|
286
|
+
body,
|
|
287
|
+
request: buildCompletionRequest(body, transport, streamRequested, options.thoughtSignatureStore),
|
|
288
|
+
streamRequested,
|
|
289
|
+
};
|
|
290
|
+
}
|
|
291
|
+
function normalizeProviderId(provider) {
|
|
292
|
+
return provider.trim().toLowerCase().replace(/[_\s]+/g, '-');
|
|
293
|
+
}
|
|
294
|
+
function resolvePassthroughApiBase(config, env = process.env) {
|
|
295
|
+
if (config.apiBase) {
|
|
296
|
+
return config.apiBase;
|
|
297
|
+
}
|
|
298
|
+
const legacyOverride = env.AGENT_MUX_PROXY_TARGET_API_BASE?.trim();
|
|
299
|
+
if (legacyOverride) {
|
|
300
|
+
return legacyOverride;
|
|
301
|
+
}
|
|
302
|
+
const provider = normalizeProviderId(config.targetProvider);
|
|
303
|
+
switch (provider) {
|
|
304
|
+
case 'anthropic':
|
|
305
|
+
return 'https://api.anthropic.com';
|
|
306
|
+
case 'openai':
|
|
307
|
+
return 'https://api.openai.com';
|
|
308
|
+
case 'google':
|
|
309
|
+
return 'https://generativelanguage.googleapis.com';
|
|
310
|
+
case 'openrouter':
|
|
311
|
+
return 'https://openrouter.ai/api';
|
|
312
|
+
case 'groq':
|
|
313
|
+
return 'https://api.groq.com/openai';
|
|
314
|
+
case 'fireworks':
|
|
315
|
+
case 'fireworks-ai':
|
|
316
|
+
return 'https://api.fireworks.ai/inference';
|
|
317
|
+
case 'together':
|
|
318
|
+
case 'together-ai':
|
|
319
|
+
return 'https://api.together.xyz';
|
|
320
|
+
case 'deepseek':
|
|
321
|
+
return 'https://api.deepseek.com';
|
|
322
|
+
case 'mistral':
|
|
323
|
+
return 'https://api.mistral.ai';
|
|
324
|
+
case 'cerebras':
|
|
325
|
+
return 'https://api.cerebras.ai';
|
|
326
|
+
case 'sambanova':
|
|
327
|
+
return 'https://api.sambanova.ai';
|
|
328
|
+
case 'nvidia-nim':
|
|
329
|
+
return 'https://integrate.api.nvidia.com';
|
|
330
|
+
case 'perplexity':
|
|
331
|
+
return 'https://api.perplexity.ai';
|
|
332
|
+
case 'cohere':
|
|
333
|
+
return 'https://api.cohere.com';
|
|
334
|
+
case 'ollama':
|
|
335
|
+
return 'http://localhost:11434';
|
|
336
|
+
case 'local':
|
|
337
|
+
return 'http://localhost:8080';
|
|
338
|
+
case 'lmstudio':
|
|
339
|
+
return 'http://localhost:1234';
|
|
340
|
+
case 'vllm':
|
|
341
|
+
return 'http://localhost:8000';
|
|
342
|
+
default:
|
|
343
|
+
return undefined;
|
|
344
|
+
}
|
|
345
|
+
}
|
|
346
|
+
function injectPassthroughProviderAuth(config, headers, env = process.env) {
|
|
347
|
+
const provider = normalizeProviderId(config.targetProvider);
|
|
348
|
+
switch (provider) {
|
|
349
|
+
case 'anthropic': {
|
|
350
|
+
const apiKey = env.ANTHROPIC_API_KEY?.trim();
|
|
351
|
+
if (apiKey) {
|
|
352
|
+
headers.set('x-api-key', apiKey);
|
|
353
|
+
headers.set('anthropic-version', '2023-06-01');
|
|
354
|
+
}
|
|
355
|
+
return;
|
|
356
|
+
}
|
|
357
|
+
case 'google': {
|
|
358
|
+
const apiKey = env.GOOGLE_API_KEY?.trim() || env.GEMINI_API_KEY?.trim();
|
|
359
|
+
if (apiKey) {
|
|
360
|
+
headers.set('x-goog-api-key', apiKey);
|
|
361
|
+
}
|
|
362
|
+
return;
|
|
363
|
+
}
|
|
364
|
+
case 'openai':
|
|
365
|
+
case 'openrouter':
|
|
366
|
+
case 'groq':
|
|
367
|
+
case 'fireworks':
|
|
368
|
+
case 'fireworks-ai':
|
|
369
|
+
case 'together':
|
|
370
|
+
case 'together-ai':
|
|
371
|
+
case 'deepseek':
|
|
372
|
+
case 'mistral':
|
|
373
|
+
case 'cerebras':
|
|
374
|
+
case 'sambanova':
|
|
375
|
+
case 'nvidia-nim':
|
|
376
|
+
case 'perplexity':
|
|
377
|
+
case 'cohere':
|
|
378
|
+
case 'custom': {
|
|
379
|
+
const envKeysByProvider = {
|
|
380
|
+
openai: ['OPENAI_API_KEY'],
|
|
381
|
+
openrouter: ['OPENROUTER_API_KEY', 'OPENAI_API_KEY'],
|
|
382
|
+
groq: ['GROQ_API_KEY', 'OPENAI_API_KEY'],
|
|
383
|
+
fireworks: ['FIREWORKS_API_KEY', 'OPENAI_API_KEY'],
|
|
384
|
+
'fireworks-ai': ['FIREWORKS_API_KEY', 'OPENAI_API_KEY'],
|
|
385
|
+
together: ['TOGETHER_API_KEY', 'OPENAI_API_KEY'],
|
|
386
|
+
'together-ai': ['TOGETHER_API_KEY', 'OPENAI_API_KEY'],
|
|
387
|
+
deepseek: ['DEEPSEEK_API_KEY', 'OPENAI_API_KEY'],
|
|
388
|
+
mistral: ['MISTRAL_API_KEY', 'OPENAI_API_KEY'],
|
|
389
|
+
cerebras: ['CEREBRAS_API_KEY', 'OPENAI_API_KEY'],
|
|
390
|
+
sambanova: ['SAMBANOVA_API_KEY', 'OPENAI_API_KEY'],
|
|
391
|
+
'nvidia-nim': ['NVIDIA_API_KEY', 'OPENAI_API_KEY'],
|
|
392
|
+
perplexity: ['PERPLEXITY_API_KEY', 'OPENAI_API_KEY'],
|
|
393
|
+
cohere: ['COHERE_API_KEY', 'OPENAI_API_KEY'],
|
|
394
|
+
custom: ['OPENAI_API_KEY'],
|
|
395
|
+
};
|
|
396
|
+
const apiKey = envKeysByProvider[provider]
|
|
397
|
+
?.map((envKey) => env[envKey]?.trim())
|
|
398
|
+
.find(Boolean);
|
|
399
|
+
if (apiKey) {
|
|
400
|
+
headers.set('authorization', `Bearer ${apiKey}`);
|
|
401
|
+
}
|
|
402
|
+
return;
|
|
403
|
+
}
|
|
404
|
+
}
|
|
405
|
+
}
|
|
406
|
+
function redactPassthroughInternalHeaders(headers) {
|
|
407
|
+
for (const key of Array.from(headers.keys())) {
|
|
408
|
+
if (key.toLowerCase().startsWith('x-babysitter-')) {
|
|
409
|
+
headers.delete(key);
|
|
410
|
+
}
|
|
411
|
+
}
|
|
412
|
+
}
|
|
413
|
+
async function proxyUpstream(req, config, forwardedPath) {
|
|
414
|
+
const apiBase = resolvePassthroughApiBase(config);
|
|
415
|
+
if (!apiBase) {
|
|
416
|
+
return new Response(JSON.stringify({ error: 'No completion engine or apiBase configured.' }), {
|
|
417
|
+
status: 501,
|
|
418
|
+
headers: { 'content-type': 'application/json' },
|
|
419
|
+
});
|
|
420
|
+
}
|
|
421
|
+
const requestUrl = new URL(req.url);
|
|
422
|
+
const upstreamUrl = new URL(forwardedPath ?? requestUrl.pathname, apiBase);
|
|
423
|
+
upstreamUrl.search = requestUrl.search;
|
|
424
|
+
const headers = new Headers(req.headers);
|
|
425
|
+
headers.delete('host');
|
|
426
|
+
headers.delete('x-api-key');
|
|
427
|
+
headers.delete('authorization');
|
|
428
|
+
redactPassthroughInternalHeaders(headers);
|
|
429
|
+
injectPassthroughProviderAuth(config, headers);
|
|
430
|
+
const init = {
|
|
431
|
+
method: req.method,
|
|
432
|
+
headers,
|
|
433
|
+
};
|
|
434
|
+
if (req.method !== 'GET' && req.method !== 'HEAD') {
|
|
435
|
+
init.body = await req.arrayBuffer();
|
|
436
|
+
}
|
|
437
|
+
return fetch(upstreamUrl, init);
|
|
438
|
+
}
|
|
439
|
+
function encodeSseChunk(prefix, payload) {
|
|
440
|
+
return `${prefix}${JSON.stringify(payload)}\n\n`;
|
|
441
|
+
}
|
|
442
|
+
function parseJsonObject(value) {
|
|
443
|
+
try {
|
|
444
|
+
const parsed = JSON.parse(value || '{}');
|
|
445
|
+
return toRecord(parsed) ?? {};
|
|
446
|
+
}
|
|
447
|
+
catch {
|
|
448
|
+
return {};
|
|
449
|
+
}
|
|
450
|
+
}
|
|
451
|
+
function openAiResponsesUsage(result) {
|
|
452
|
+
if (!result)
|
|
453
|
+
return null;
|
|
454
|
+
return {
|
|
455
|
+
input_tokens: result.usage.promptTokens,
|
|
456
|
+
output_tokens: result.usage.completionTokens,
|
|
457
|
+
total_tokens: result.usage.totalTokens,
|
|
458
|
+
};
|
|
459
|
+
}
|
|
460
|
+
function openAiResponsesMessageItem(itemId, text, status) {
|
|
461
|
+
return {
|
|
462
|
+
id: itemId,
|
|
463
|
+
type: 'message',
|
|
464
|
+
status,
|
|
465
|
+
role: 'assistant',
|
|
466
|
+
content: status === 'completed'
|
|
467
|
+
? [{ type: 'output_text', text, annotations: [] }]
|
|
468
|
+
: [],
|
|
469
|
+
};
|
|
470
|
+
}
|
|
471
|
+
function openAiResponsesFunctionCallItem(input) {
|
|
472
|
+
const itemId = input.itemId ?? `fc_${randomUUID()}`;
|
|
473
|
+
return {
|
|
474
|
+
type: 'function_call',
|
|
475
|
+
id: itemId,
|
|
476
|
+
call_id: input.id || itemId,
|
|
477
|
+
name: input.name,
|
|
478
|
+
arguments: input.arguments,
|
|
479
|
+
status: input.status ?? 'completed',
|
|
480
|
+
...input.metadata,
|
|
481
|
+
};
|
|
482
|
+
}
|
|
483
|
+
function openAiResponsesShape(input) {
|
|
484
|
+
return {
|
|
485
|
+
id: input.id,
|
|
486
|
+
object: 'response',
|
|
487
|
+
created_at: input.createdAt,
|
|
488
|
+
status: input.status,
|
|
489
|
+
model: input.config.targetModel,
|
|
490
|
+
output: input.output ?? [],
|
|
491
|
+
parallel_tool_calls: true,
|
|
492
|
+
tool_choice: 'auto',
|
|
493
|
+
tools: [],
|
|
494
|
+
usage: input.usage ?? null,
|
|
495
|
+
};
|
|
496
|
+
}
|
|
497
|
+
function anthropicStreamResponse(stream, config) {
|
|
498
|
+
const encoder = new TextEncoder();
|
|
499
|
+
const messageId = `msg_${randomUUID()}`;
|
|
500
|
+
return new Response(new ReadableStream({
|
|
501
|
+
async start(controller) {
|
|
502
|
+
controller.enqueue(encoder.encode(encodeSseChunk('event: message_start\ndata: ', {
|
|
503
|
+
type: 'message_start',
|
|
504
|
+
message: {
|
|
505
|
+
id: messageId,
|
|
506
|
+
type: 'message',
|
|
507
|
+
role: 'assistant',
|
|
508
|
+
content: [],
|
|
509
|
+
model: config.targetModel,
|
|
510
|
+
},
|
|
511
|
+
})));
|
|
512
|
+
controller.enqueue(encoder.encode(encodeSseChunk('event: content_block_start\ndata: ', {
|
|
513
|
+
type: 'content_block_start',
|
|
514
|
+
index: 0,
|
|
515
|
+
content_block: { type: 'text', text: '' },
|
|
516
|
+
})));
|
|
517
|
+
let contentIndex = 0;
|
|
518
|
+
let hasToolUse = false;
|
|
519
|
+
for await (const event of stream) {
|
|
520
|
+
if (event.type === 'text-delta' && event.text) {
|
|
521
|
+
controller.enqueue(encoder.encode(encodeSseChunk('event: content_block_delta\ndata: ', {
|
|
522
|
+
type: 'content_block_delta',
|
|
523
|
+
index: contentIndex,
|
|
524
|
+
delta: { type: 'text_delta', text: event.text },
|
|
525
|
+
})));
|
|
526
|
+
}
|
|
527
|
+
else if (event.type === 'tool-call') {
|
|
528
|
+
// Close text block before first tool_use
|
|
529
|
+
if (!hasToolUse) {
|
|
530
|
+
controller.enqueue(encoder.encode(encodeSseChunk('event: content_block_stop\ndata: ', { type: 'content_block_stop', index: contentIndex })));
|
|
531
|
+
contentIndex++;
|
|
532
|
+
hasToolUse = true;
|
|
533
|
+
}
|
|
534
|
+
// Anthropic streaming format: content_block_start with empty input,
|
|
535
|
+
// then input_json_delta with the full JSON, then content_block_stop.
|
|
536
|
+
const toolUseBlock = { type: 'tool_use', id: event.id, name: event.name, input: {} };
|
|
537
|
+
if (event.metadata)
|
|
538
|
+
Object.assign(toolUseBlock, event.metadata);
|
|
539
|
+
controller.enqueue(encoder.encode(encodeSseChunk('event: content_block_start\ndata: ', {
|
|
540
|
+
type: 'content_block_start',
|
|
541
|
+
index: contentIndex,
|
|
542
|
+
content_block: toolUseBlock,
|
|
543
|
+
})));
|
|
544
|
+
controller.enqueue(encoder.encode(encodeSseChunk('event: content_block_delta\ndata: ', {
|
|
545
|
+
type: 'content_block_delta',
|
|
546
|
+
index: contentIndex,
|
|
547
|
+
delta: { type: 'input_json_delta', partial_json: event.arguments },
|
|
548
|
+
})));
|
|
549
|
+
controller.enqueue(encoder.encode(encodeSseChunk('event: content_block_stop\ndata: ', { type: 'content_block_stop', index: contentIndex })));
|
|
550
|
+
contentIndex++;
|
|
551
|
+
}
|
|
552
|
+
}
|
|
553
|
+
if (!hasToolUse) {
|
|
554
|
+
controller.enqueue(encoder.encode(encodeSseChunk('event: content_block_stop\ndata: ', { type: 'content_block_stop', index: contentIndex })));
|
|
555
|
+
}
|
|
556
|
+
controller.enqueue(encoder.encode(encodeSseChunk('event: message_delta\ndata: ', {
|
|
557
|
+
type: 'message_delta',
|
|
558
|
+
delta: { stop_reason: hasToolUse ? 'tool_use' : 'end_turn' },
|
|
559
|
+
})));
|
|
560
|
+
controller.enqueue(encoder.encode(encodeSseChunk('event: message_stop\ndata: ', { type: 'message_stop' })));
|
|
561
|
+
controller.close();
|
|
562
|
+
},
|
|
563
|
+
}), { headers: { 'content-type': 'text/event-stream; charset=utf-8', 'cache-control': 'no-cache' } });
|
|
564
|
+
}
|
|
565
|
+
function openAiChatStreamResponse(stream, config) {
|
|
566
|
+
const encoder = new TextEncoder();
|
|
567
|
+
const responseId = `chatcmpl_${randomUUID()}`;
|
|
568
|
+
const created = Math.floor(Date.now() / 1000);
|
|
569
|
+
return new Response(new ReadableStream({
|
|
570
|
+
async start(controller) {
|
|
571
|
+
controller.enqueue(encoder.encode(encodeSseChunk('data: ', {
|
|
572
|
+
id: responseId,
|
|
573
|
+
object: 'chat.completion.chunk',
|
|
574
|
+
created,
|
|
575
|
+
model: config.targetModel,
|
|
576
|
+
choices: [{ index: 0, delta: { role: 'assistant' }, finish_reason: null }],
|
|
577
|
+
})));
|
|
578
|
+
let finishReason = 'stop';
|
|
579
|
+
let toolCallIndex = 0;
|
|
580
|
+
for await (const event of stream) {
|
|
581
|
+
if (event.type === 'text-delta' && event.text) {
|
|
582
|
+
controller.enqueue(encoder.encode(encodeSseChunk('data: ', {
|
|
583
|
+
id: responseId,
|
|
584
|
+
object: 'chat.completion.chunk',
|
|
585
|
+
created,
|
|
586
|
+
model: config.targetModel,
|
|
587
|
+
choices: [{ index: 0, delta: { content: event.text }, finish_reason: null }],
|
|
588
|
+
})));
|
|
589
|
+
continue;
|
|
590
|
+
}
|
|
591
|
+
if (event.type === 'tool-call') {
|
|
592
|
+
controller.enqueue(encoder.encode(encodeSseChunk('data: ', {
|
|
593
|
+
id: responseId,
|
|
594
|
+
object: 'chat.completion.chunk',
|
|
595
|
+
created,
|
|
596
|
+
model: config.targetModel,
|
|
597
|
+
choices: [{
|
|
598
|
+
index: 0,
|
|
599
|
+
delta: {
|
|
600
|
+
tool_calls: [{
|
|
601
|
+
index: toolCallIndex,
|
|
602
|
+
id: event.id,
|
|
603
|
+
type: 'function',
|
|
604
|
+
function: { name: event.name, arguments: event.arguments },
|
|
605
|
+
}],
|
|
606
|
+
},
|
|
607
|
+
finish_reason: null,
|
|
608
|
+
}],
|
|
609
|
+
})));
|
|
610
|
+
toolCallIndex++;
|
|
611
|
+
continue;
|
|
612
|
+
}
|
|
613
|
+
if (event.type === 'done' && event.finishReason) {
|
|
614
|
+
finishReason = event.finishReason;
|
|
615
|
+
}
|
|
616
|
+
}
|
|
617
|
+
controller.enqueue(encoder.encode(encodeSseChunk('data: ', {
|
|
618
|
+
id: responseId,
|
|
619
|
+
object: 'chat.completion.chunk',
|
|
620
|
+
created,
|
|
621
|
+
model: config.targetModel,
|
|
622
|
+
choices: [{ index: 0, delta: {}, finish_reason: finishReason }],
|
|
623
|
+
})));
|
|
624
|
+
controller.enqueue(encoder.encode('data: [DONE]\n\n'));
|
|
625
|
+
controller.close();
|
|
626
|
+
},
|
|
627
|
+
}), { headers: { 'content-type': 'text/event-stream; charset=utf-8', 'cache-control': 'no-cache' } });
|
|
628
|
+
}
|
|
629
|
+
function openAiResponsesStreamResponse(stream, config) {
|
|
630
|
+
const encoder = new TextEncoder();
|
|
631
|
+
const responseId = `resp_${randomUUID()}`;
|
|
632
|
+
const itemId = `msg_${randomUUID()}`;
|
|
633
|
+
const createdAt = Math.floor(Date.now() / 1000);
|
|
634
|
+
return new Response(new ReadableStream({
|
|
635
|
+
async start(controller) {
|
|
636
|
+
let outputText = '';
|
|
637
|
+
controller.enqueue(encoder.encode(encodeSseChunk('event: response.created\ndata: ', {
|
|
638
|
+
type: 'response.created',
|
|
639
|
+
response: openAiResponsesShape({ id: responseId, createdAt, config, status: 'in_progress' }),
|
|
640
|
+
})));
|
|
641
|
+
controller.enqueue(encoder.encode(encodeSseChunk('event: response.output_item.added\ndata: ', {
|
|
642
|
+
type: 'response.output_item.added',
|
|
643
|
+
output_index: 0,
|
|
644
|
+
item: openAiResponsesMessageItem(itemId, '', 'in_progress'),
|
|
645
|
+
})));
|
|
646
|
+
controller.enqueue(encoder.encode(encodeSseChunk('event: response.content_part.added\ndata: ', {
|
|
647
|
+
type: 'response.content_part.added',
|
|
648
|
+
item_id: itemId,
|
|
649
|
+
output_index: 0,
|
|
650
|
+
content_index: 0,
|
|
651
|
+
part: { type: 'output_text', text: '', annotations: [] },
|
|
652
|
+
})));
|
|
653
|
+
let toolCallIndex = 0;
|
|
654
|
+
const toolItems = [];
|
|
655
|
+
try {
|
|
656
|
+
for await (const event of stream) {
|
|
657
|
+
if (event.type === 'text-delta' && event.text) {
|
|
658
|
+
outputText += event.text;
|
|
659
|
+
controller.enqueue(encoder.encode(encodeSseChunk('event: response.output_text.delta\ndata: ', {
|
|
660
|
+
type: 'response.output_text.delta',
|
|
661
|
+
item_id: itemId,
|
|
662
|
+
output_index: 0,
|
|
663
|
+
content_index: 0,
|
|
664
|
+
delta: event.text,
|
|
665
|
+
})));
|
|
666
|
+
}
|
|
667
|
+
else if (event.type === 'tool-call') {
|
|
668
|
+
const toolItemId = `fc_${randomUUID()}`;
|
|
669
|
+
const outputIndex = toolCallIndex + 1;
|
|
670
|
+
const toolItem = openAiResponsesFunctionCallItem({
|
|
671
|
+
...event,
|
|
672
|
+
itemId: toolItemId,
|
|
673
|
+
status: 'completed',
|
|
674
|
+
});
|
|
675
|
+
toolItems.push(toolItem);
|
|
676
|
+
controller.enqueue(encoder.encode(encodeSseChunk('event: response.output_item.added\ndata: ', {
|
|
677
|
+
type: 'response.output_item.added',
|
|
678
|
+
output_index: outputIndex,
|
|
679
|
+
item: openAiResponsesFunctionCallItem({
|
|
680
|
+
...event,
|
|
681
|
+
itemId: toolItemId,
|
|
682
|
+
arguments: '',
|
|
683
|
+
status: 'in_progress',
|
|
684
|
+
}),
|
|
685
|
+
})));
|
|
686
|
+
if (event.arguments) {
|
|
687
|
+
controller.enqueue(encoder.encode(encodeSseChunk('event: response.function_call_arguments.delta\ndata: ', {
|
|
688
|
+
type: 'response.function_call_arguments.delta',
|
|
689
|
+
item_id: toolItemId,
|
|
690
|
+
output_index: outputIndex,
|
|
691
|
+
delta: event.arguments,
|
|
692
|
+
})));
|
|
693
|
+
}
|
|
694
|
+
controller.enqueue(encoder.encode(encodeSseChunk('event: response.function_call_arguments.done\ndata: ', {
|
|
695
|
+
type: 'response.function_call_arguments.done',
|
|
696
|
+
item_id: toolItemId,
|
|
697
|
+
output_index: outputIndex,
|
|
698
|
+
arguments: event.arguments,
|
|
699
|
+
})));
|
|
700
|
+
controller.enqueue(encoder.encode(encodeSseChunk('event: response.output_item.done\ndata: ', {
|
|
701
|
+
type: 'response.output_item.done',
|
|
702
|
+
output_index: outputIndex,
|
|
703
|
+
item: toolItem,
|
|
704
|
+
})));
|
|
705
|
+
toolCallIndex++;
|
|
706
|
+
}
|
|
707
|
+
}
|
|
708
|
+
}
|
|
709
|
+
catch (streamError) {
|
|
710
|
+
const errorMsg = streamError instanceof Error ? streamError.message : String(streamError);
|
|
711
|
+
console.error(`[transport-adapter] SSE stream error: ${errorMsg}`);
|
|
712
|
+
controller.enqueue(encoder.encode(encodeSseChunk('event: error\ndata: ', {
|
|
713
|
+
type: 'error',
|
|
714
|
+
error: { type: 'server_error', message: `Upstream stream error: ${errorMsg}` },
|
|
715
|
+
})));
|
|
716
|
+
}
|
|
717
|
+
const messageItem = openAiResponsesMessageItem(itemId, outputText, 'completed');
|
|
718
|
+
controller.enqueue(encoder.encode(encodeSseChunk('event: response.output_text.done\ndata: ', {
|
|
719
|
+
type: 'response.output_text.done',
|
|
720
|
+
item_id: itemId,
|
|
721
|
+
output_index: 0,
|
|
722
|
+
content_index: 0,
|
|
723
|
+
text: outputText,
|
|
724
|
+
})));
|
|
725
|
+
controller.enqueue(encoder.encode(encodeSseChunk('event: response.content_part.done\ndata: ', {
|
|
726
|
+
type: 'response.content_part.done',
|
|
727
|
+
item_id: itemId,
|
|
728
|
+
output_index: 0,
|
|
729
|
+
content_index: 0,
|
|
730
|
+
part: { type: 'output_text', text: outputText, annotations: [] },
|
|
731
|
+
})));
|
|
732
|
+
controller.enqueue(encoder.encode(encodeSseChunk('event: response.output_item.done\ndata: ', {
|
|
733
|
+
type: 'response.output_item.done',
|
|
734
|
+
output_index: 0,
|
|
735
|
+
item: messageItem,
|
|
736
|
+
})));
|
|
737
|
+
controller.enqueue(encoder.encode(encodeSseChunk('event: response.completed\ndata: ', {
|
|
738
|
+
type: 'response.completed',
|
|
739
|
+
response: openAiResponsesShape({ id: responseId, createdAt, config, status: 'completed', output: [messageItem, ...toolItems] }),
|
|
740
|
+
})));
|
|
741
|
+
controller.enqueue(encoder.encode('data: [DONE]\n\n'));
|
|
742
|
+
controller.close();
|
|
743
|
+
},
|
|
744
|
+
}), { headers: { 'content-type': 'text/event-stream; charset=utf-8', 'cache-control': 'no-cache' } });
|
|
745
|
+
}
|
|
746
|
+
function completionResultToResponseEvent(result, config) {
|
|
747
|
+
return {
|
|
748
|
+
type: 'response.completed',
|
|
749
|
+
response: openAiResponsesResponse(result, config),
|
|
750
|
+
};
|
|
751
|
+
}
|
|
752
|
+
async function sendOpenAiResponsesWebSocketStream(ws, stream, config) {
|
|
753
|
+
const responseId = `resp_${randomUUID()}`;
|
|
754
|
+
const itemId = `msg_${randomUUID()}`;
|
|
755
|
+
const createdAt = Math.floor(Date.now() / 1000);
|
|
756
|
+
let outputText = '';
|
|
757
|
+
const toolItems = [];
|
|
758
|
+
ws.send(JSON.stringify({
|
|
759
|
+
type: 'response.created',
|
|
760
|
+
response: openAiResponsesShape({ id: responseId, createdAt, config, status: 'in_progress' }),
|
|
761
|
+
}));
|
|
762
|
+
ws.send(JSON.stringify({
|
|
763
|
+
type: 'response.output_item.added',
|
|
764
|
+
output_index: 0,
|
|
765
|
+
item: openAiResponsesMessageItem(itemId, '', 'in_progress'),
|
|
766
|
+
}));
|
|
767
|
+
ws.send(JSON.stringify({
|
|
768
|
+
type: 'response.content_part.added',
|
|
769
|
+
item_id: itemId,
|
|
770
|
+
output_index: 0,
|
|
771
|
+
content_index: 0,
|
|
772
|
+
part: { type: 'output_text', text: '', annotations: [] },
|
|
773
|
+
}));
|
|
774
|
+
for await (const event of stream) {
|
|
775
|
+
if (event.type === 'text-delta' && event.text) {
|
|
776
|
+
outputText += event.text;
|
|
777
|
+
ws.send(JSON.stringify({
|
|
778
|
+
type: 'response.output_text.delta',
|
|
779
|
+
item_id: itemId,
|
|
780
|
+
output_index: 0,
|
|
781
|
+
content_index: 0,
|
|
782
|
+
delta: event.text,
|
|
783
|
+
}));
|
|
784
|
+
}
|
|
785
|
+
else if (event.type === 'tool-call') {
|
|
786
|
+
const toolItemId = `fc_${randomUUID()}`;
|
|
787
|
+
const outputIndex = toolItems.length + 1;
|
|
788
|
+
const toolItem = openAiResponsesFunctionCallItem({
|
|
789
|
+
...event,
|
|
790
|
+
itemId: toolItemId,
|
|
791
|
+
status: 'completed',
|
|
792
|
+
});
|
|
793
|
+
toolItems.push(toolItem);
|
|
794
|
+
ws.send(JSON.stringify({
|
|
795
|
+
type: 'response.output_item.added',
|
|
796
|
+
output_index: outputIndex,
|
|
797
|
+
item: openAiResponsesFunctionCallItem({
|
|
798
|
+
...event,
|
|
799
|
+
itemId: toolItemId,
|
|
800
|
+
arguments: '',
|
|
801
|
+
status: 'in_progress',
|
|
802
|
+
}),
|
|
803
|
+
}));
|
|
804
|
+
if (event.arguments) {
|
|
805
|
+
ws.send(JSON.stringify({
|
|
806
|
+
type: 'response.function_call_arguments.delta',
|
|
807
|
+
item_id: toolItemId,
|
|
808
|
+
output_index: outputIndex,
|
|
809
|
+
delta: event.arguments,
|
|
810
|
+
}));
|
|
811
|
+
}
|
|
812
|
+
ws.send(JSON.stringify({
|
|
813
|
+
type: 'response.function_call_arguments.done',
|
|
814
|
+
item_id: toolItemId,
|
|
815
|
+
output_index: outputIndex,
|
|
816
|
+
arguments: event.arguments,
|
|
817
|
+
}));
|
|
818
|
+
ws.send(JSON.stringify({
|
|
819
|
+
type: 'response.output_item.done',
|
|
820
|
+
output_index: outputIndex,
|
|
821
|
+
item: toolItem,
|
|
822
|
+
}));
|
|
823
|
+
}
|
|
824
|
+
}
|
|
825
|
+
const messageItem = openAiResponsesMessageItem(itemId, outputText, 'completed');
|
|
826
|
+
ws.send(JSON.stringify({
|
|
827
|
+
type: 'response.output_text.done',
|
|
828
|
+
item_id: itemId,
|
|
829
|
+
output_index: 0,
|
|
830
|
+
content_index: 0,
|
|
831
|
+
text: outputText,
|
|
832
|
+
}));
|
|
833
|
+
ws.send(JSON.stringify({
|
|
834
|
+
type: 'response.content_part.done',
|
|
835
|
+
item_id: itemId,
|
|
836
|
+
output_index: 0,
|
|
837
|
+
content_index: 0,
|
|
838
|
+
part: { type: 'output_text', text: outputText, annotations: [] },
|
|
839
|
+
}));
|
|
840
|
+
ws.send(JSON.stringify({
|
|
841
|
+
type: 'response.output_item.done',
|
|
842
|
+
output_index: 0,
|
|
843
|
+
item: messageItem,
|
|
844
|
+
}));
|
|
845
|
+
ws.send(JSON.stringify({
|
|
846
|
+
type: 'response.completed',
|
|
847
|
+
response: openAiResponsesShape({ id: responseId, createdAt, config, status: 'completed', output: [messageItem, ...toolItems] }),
|
|
848
|
+
}));
|
|
849
|
+
}
|
|
850
|
+
async function handleOpenAiResponsesWebSocketMessage(ws, data, config, completionEngine, metrics, thoughtSignatureStore) {
|
|
851
|
+
let envelope;
|
|
852
|
+
try {
|
|
853
|
+
envelope = JSON.parse(data.toString('utf8'));
|
|
854
|
+
}
|
|
855
|
+
catch {
|
|
856
|
+
ws.send(JSON.stringify({ type: 'error', error: { message: 'WebSocket message must be valid JSON.' } }));
|
|
857
|
+
return;
|
|
858
|
+
}
|
|
859
|
+
if (!envelope || typeof envelope !== 'object') {
|
|
860
|
+
ws.send(JSON.stringify({ type: 'error', error: { message: 'WebSocket message must be a JSON object.' } }));
|
|
861
|
+
return;
|
|
862
|
+
}
|
|
863
|
+
const record = envelope;
|
|
864
|
+
if (record['type'] !== 'response.create') {
|
|
865
|
+
ws.send(JSON.stringify({ type: 'error', error: { message: `Unsupported WebSocket event type: ${String(record['type'])}` } }));
|
|
866
|
+
return;
|
|
867
|
+
}
|
|
868
|
+
const body = toRecord(record['response']) ?? toRecord(record['payload']) ?? record;
|
|
869
|
+
const plan = {
|
|
870
|
+
body,
|
|
871
|
+
request: buildCompletionRequest(body, 'openai-responses', true, thoughtSignatureStore),
|
|
872
|
+
streamRequested: true,
|
|
873
|
+
};
|
|
874
|
+
plan.request.model = config.targetModel;
|
|
875
|
+
try {
|
|
876
|
+
if (!completionEngine) {
|
|
877
|
+
ws.send(JSON.stringify({ type: 'error', error: { message: 'Responses WebSocket requires a configured completion engine.' } }));
|
|
878
|
+
metrics.recordError();
|
|
879
|
+
return;
|
|
880
|
+
}
|
|
881
|
+
if (completionEngine.stream) {
|
|
882
|
+
await sendOpenAiResponsesWebSocketStream(ws, trackCompletionStream(completionEngine.stream(plan.request), metrics, config), config);
|
|
883
|
+
return;
|
|
884
|
+
}
|
|
885
|
+
const result = await completionEngine.complete(plan.request);
|
|
886
|
+
metrics.record(result.usage.promptTokens, result.usage.completionTokens);
|
|
887
|
+
ws.send(JSON.stringify(completionResultToResponseEvent(result, config)));
|
|
888
|
+
}
|
|
889
|
+
catch (error) {
|
|
890
|
+
metrics.recordError();
|
|
891
|
+
const errMsg = error instanceof Error ? error.message : String(error);
|
|
892
|
+
console.error(`[transport-adapter] WebSocket completion error: ${errMsg}`);
|
|
893
|
+
ws.send(JSON.stringify({
|
|
894
|
+
type: 'error',
|
|
895
|
+
error: { message: error instanceof Error ? error.message : String(error) },
|
|
896
|
+
}));
|
|
897
|
+
}
|
|
898
|
+
}
|
|
899
|
+
function googleStreamResponse(stream) {
|
|
900
|
+
const encoder = new TextEncoder();
|
|
901
|
+
return new Response(new ReadableStream({
|
|
902
|
+
async start(controller) {
|
|
903
|
+
for await (const event of stream) {
|
|
904
|
+
if (event.type === 'text-delta' && event.text) {
|
|
905
|
+
controller.enqueue(encoder.encode(`data: ${JSON.stringify({
|
|
906
|
+
candidates: [{ content: { parts: [{ text: event.text }], role: 'model' } }],
|
|
907
|
+
})}\n\n`));
|
|
908
|
+
}
|
|
909
|
+
else if (event.type === 'tool-call') {
|
|
910
|
+
controller.enqueue(encoder.encode(`data: ${JSON.stringify({
|
|
911
|
+
candidates: [{ content: { parts: [{ functionCall: { name: event.name, args: parseJsonObject(event.arguments) } }], role: 'model' } }],
|
|
912
|
+
})}\n\n`));
|
|
913
|
+
}
|
|
914
|
+
else if (event.type === 'done') {
|
|
915
|
+
controller.enqueue(encoder.encode(`data: ${JSON.stringify({
|
|
916
|
+
candidates: [{ content: { parts: [], role: 'model' }, finishReason: event.finishReason?.toUpperCase() || 'STOP' }],
|
|
917
|
+
usageMetadata: event.usage ? { promptTokenCount: event.usage.promptTokens, candidatesTokenCount: event.usage.completionTokens, totalTokenCount: event.usage.totalTokens } : undefined,
|
|
918
|
+
})}\n\n`));
|
|
919
|
+
}
|
|
920
|
+
}
|
|
921
|
+
controller.close();
|
|
922
|
+
},
|
|
923
|
+
}), { headers: { 'content-type': 'text/event-stream; charset=utf-8', 'cache-control': 'no-cache' } });
|
|
924
|
+
}
|
|
925
|
+
function renderStreamResponse(transport, stream, config) {
|
|
926
|
+
switch (transport) {
|
|
927
|
+
case 'anthropic':
|
|
928
|
+
return anthropicStreamResponse(stream, config);
|
|
929
|
+
case 'openai-chat':
|
|
930
|
+
return openAiChatStreamResponse(stream, config);
|
|
931
|
+
case 'openai-responses':
|
|
932
|
+
return openAiResponsesStreamResponse(stream, config);
|
|
933
|
+
case 'google':
|
|
934
|
+
return googleStreamResponse(stream);
|
|
935
|
+
case 'bedrock-converse':
|
|
936
|
+
return codecStreamResponse(transport, stream, 'application/x-ndjson; charset=utf-8');
|
|
937
|
+
default:
|
|
938
|
+
return createErrorResponse(`Streaming is not supported for ${transport}.`);
|
|
939
|
+
}
|
|
940
|
+
}
|
|
941
|
+
function codecStreamResponse(transport, stream, contentType) {
|
|
942
|
+
const codec = getCodec(transport);
|
|
943
|
+
if (!codec) {
|
|
944
|
+
return createErrorResponse(`Streaming is not supported for ${transport}.`);
|
|
945
|
+
}
|
|
946
|
+
const encoder = new TextEncoder();
|
|
947
|
+
return new Response(new ReadableStream({
|
|
948
|
+
async start(controller) {
|
|
949
|
+
for await (const event of stream) {
|
|
950
|
+
const chunk = codec.encodeStreamChunk(event);
|
|
951
|
+
if (chunk) {
|
|
952
|
+
controller.enqueue(encoder.encode(chunk));
|
|
953
|
+
}
|
|
954
|
+
}
|
|
955
|
+
controller.close();
|
|
956
|
+
},
|
|
957
|
+
}), { headers: { 'content-type': contentType, 'cache-control': 'no-cache' } });
|
|
958
|
+
}
|
|
959
|
+
function trackCompletionStream(stream, metrics, config, costFeedbackSink, costFeedbackMetadata) {
|
|
960
|
+
return {
|
|
961
|
+
async *[Symbol.asyncIterator]() {
|
|
962
|
+
let recorded = false;
|
|
963
|
+
try {
|
|
964
|
+
for await (const event of stream) {
|
|
965
|
+
if (event.type === 'done') {
|
|
966
|
+
if (event.usage) {
|
|
967
|
+
metrics.record(event.usage.promptTokens, event.usage.completionTokens);
|
|
968
|
+
await emitUsageCostFeedback(costFeedbackSink, event.usage, config, costFeedbackMetadata);
|
|
969
|
+
}
|
|
970
|
+
else {
|
|
971
|
+
metrics.recordRequest();
|
|
972
|
+
}
|
|
973
|
+
recorded = true;
|
|
974
|
+
}
|
|
975
|
+
yield event;
|
|
976
|
+
}
|
|
977
|
+
if (!recorded) {
|
|
978
|
+
metrics.recordRequest();
|
|
979
|
+
}
|
|
980
|
+
}
|
|
981
|
+
catch (error) {
|
|
982
|
+
metrics.recordError();
|
|
983
|
+
throw error;
|
|
984
|
+
}
|
|
985
|
+
},
|
|
986
|
+
};
|
|
987
|
+
}
|
|
988
|
+
async function trackCompletionOutcome(result, metrics, options = {}) {
|
|
989
|
+
if (result instanceof Response) {
|
|
990
|
+
if (result.status >= 400) {
|
|
991
|
+
metrics.recordError();
|
|
992
|
+
}
|
|
993
|
+
else if (options.countSuccessResponse) {
|
|
994
|
+
metrics.recordRequest();
|
|
995
|
+
}
|
|
996
|
+
return result;
|
|
997
|
+
}
|
|
998
|
+
metrics.record(result.usage.promptTokens, result.usage.completionTokens);
|
|
999
|
+
if (options.config) {
|
|
1000
|
+
await emitCostFeedback(options.costFeedbackSink, result, options.config, options.costFeedbackMetadata);
|
|
1001
|
+
}
|
|
1002
|
+
return result;
|
|
1003
|
+
}
|
|
1004
|
+
async function resolveCompletion(req, config, completionEngine, plan, metrics, options = {}) {
|
|
1005
|
+
if (plan.streamRequested) {
|
|
1006
|
+
if (!config.stream) {
|
|
1007
|
+
return createErrorResponse('Streaming was requested but is disabled by proxy configuration.');
|
|
1008
|
+
}
|
|
1009
|
+
if (!supportsStreaming(config.exposedTransport)) {
|
|
1010
|
+
return createErrorResponse(`Streaming is not supported for ${config.exposedTransport}.`);
|
|
1011
|
+
}
|
|
1012
|
+
if (!completionEngine) {
|
|
1013
|
+
return proxyUpstream(req, config, options.forceStreaming ? new URL(req.url).pathname : undefined);
|
|
1014
|
+
}
|
|
1015
|
+
if (!completionEngine.stream) {
|
|
1016
|
+
return createErrorResponse(`Streaming was requested for ${config.exposedTransport}, but the configured completion engine cannot stream.`, 501);
|
|
1017
|
+
}
|
|
1018
|
+
return renderStreamResponse(config.exposedTransport, trackCompletionStream(completionEngine.stream(plan.request), metrics, config, options.costFeedbackSink, options.costFeedbackMetadata), config);
|
|
1019
|
+
}
|
|
1020
|
+
if (!completionEngine) {
|
|
1021
|
+
return proxyUpstream(req, config);
|
|
1022
|
+
}
|
|
1023
|
+
plan.request.model = config.targetModel;
|
|
1024
|
+
return completionEngine.complete(plan.request);
|
|
1025
|
+
}
|
|
1026
|
+
function anthropicResponse(result, config) {
|
|
1027
|
+
const content = [];
|
|
1028
|
+
if (result.text) {
|
|
1029
|
+
content.push({ type: 'text', text: result.text });
|
|
1030
|
+
}
|
|
1031
|
+
if (result.toolCalls) {
|
|
1032
|
+
for (const tc of result.toolCalls) {
|
|
1033
|
+
let input;
|
|
1034
|
+
try {
|
|
1035
|
+
input = JSON.parse(tc.arguments);
|
|
1036
|
+
}
|
|
1037
|
+
catch (e) {
|
|
1038
|
+
process.stderr.write(`[transport-adapter] tool call arguments parse failed for ${tc.name}: ${e instanceof Error ? e.message : String(e)}\n`);
|
|
1039
|
+
input = { _parseError: true, _rawArguments: tc.arguments };
|
|
1040
|
+
}
|
|
1041
|
+
const block = { type: 'tool_use', id: tc.id, name: tc.name, input };
|
|
1042
|
+
if (tc.metadata)
|
|
1043
|
+
Object.assign(block, tc.metadata);
|
|
1044
|
+
content.push(block);
|
|
1045
|
+
}
|
|
1046
|
+
}
|
|
1047
|
+
if (content.length === 0) {
|
|
1048
|
+
content.push({ type: 'text', text: '' });
|
|
1049
|
+
}
|
|
1050
|
+
return {
|
|
1051
|
+
id: result.id,
|
|
1052
|
+
type: 'message',
|
|
1053
|
+
role: 'assistant',
|
|
1054
|
+
model: config.targetModel,
|
|
1055
|
+
stop_reason: result.toolCalls?.length ? 'tool_use' : result.finishReason,
|
|
1056
|
+
content,
|
|
1057
|
+
usage: {
|
|
1058
|
+
input_tokens: result.usage.promptTokens,
|
|
1059
|
+
output_tokens: result.usage.completionTokens,
|
|
1060
|
+
},
|
|
1061
|
+
};
|
|
1062
|
+
}
|
|
1063
|
+
function openAiChatResponse(result, config) {
|
|
1064
|
+
const message = { role: 'assistant', content: result.text || null };
|
|
1065
|
+
if (result.toolCalls?.length) {
|
|
1066
|
+
message.tool_calls = result.toolCalls.map(tc => ({
|
|
1067
|
+
id: tc.id,
|
|
1068
|
+
type: 'function',
|
|
1069
|
+
function: { name: tc.name, arguments: tc.arguments },
|
|
1070
|
+
}));
|
|
1071
|
+
}
|
|
1072
|
+
return {
|
|
1073
|
+
id: result.id,
|
|
1074
|
+
object: 'chat.completion',
|
|
1075
|
+
created: Math.floor(Date.now() / 1000),
|
|
1076
|
+
model: config.targetModel,
|
|
1077
|
+
choices: [
|
|
1078
|
+
{
|
|
1079
|
+
index: 0,
|
|
1080
|
+
message,
|
|
1081
|
+
finish_reason: result.toolCalls?.length ? 'tool_calls' : result.finishReason,
|
|
1082
|
+
},
|
|
1083
|
+
],
|
|
1084
|
+
usage: usageShape(result),
|
|
1085
|
+
};
|
|
1086
|
+
}
|
|
1087
|
+
function openAiResponsesResponse(result, config) {
|
|
1088
|
+
const createdAt = Math.floor(Date.now() / 1000);
|
|
1089
|
+
const item = openAiResponsesMessageItem(`msg_${randomUUID()}`, result.text, 'completed');
|
|
1090
|
+
const toolItems = result.toolCalls?.map(openAiResponsesFunctionCallItem) ?? [];
|
|
1091
|
+
return openAiResponsesShape({
|
|
1092
|
+
id: result.id,
|
|
1093
|
+
createdAt,
|
|
1094
|
+
config,
|
|
1095
|
+
status: 'completed',
|
|
1096
|
+
output: [item, ...toolItems],
|
|
1097
|
+
usage: openAiResponsesUsage(result),
|
|
1098
|
+
});
|
|
1099
|
+
}
|
|
1100
|
+
function googleResponse(result) {
|
|
1101
|
+
const parts = [];
|
|
1102
|
+
if (result.text)
|
|
1103
|
+
parts.push({ text: result.text });
|
|
1104
|
+
if (result.toolCalls && result.toolCalls.length > 0) {
|
|
1105
|
+
for (const tc of result.toolCalls) {
|
|
1106
|
+
try {
|
|
1107
|
+
parts.push({ functionCall: { name: tc.name, args: JSON.parse(tc.arguments || '{}') } });
|
|
1108
|
+
}
|
|
1109
|
+
catch {
|
|
1110
|
+
parts.push({ functionCall: { name: tc.name, args: {} } });
|
|
1111
|
+
}
|
|
1112
|
+
}
|
|
1113
|
+
}
|
|
1114
|
+
if (parts.length === 0)
|
|
1115
|
+
parts.push({ text: result.text ?? '' });
|
|
1116
|
+
return {
|
|
1117
|
+
candidates: [
|
|
1118
|
+
{
|
|
1119
|
+
content: {
|
|
1120
|
+
role: 'model',
|
|
1121
|
+
parts,
|
|
1122
|
+
},
|
|
1123
|
+
finishReason: result.toolCalls?.length ? 'STOP' : 'STOP',
|
|
1124
|
+
},
|
|
1125
|
+
],
|
|
1126
|
+
usageMetadata: {
|
|
1127
|
+
promptTokenCount: result.usage.promptTokens,
|
|
1128
|
+
candidatesTokenCount: result.usage.completionTokens,
|
|
1129
|
+
totalTokenCount: result.usage.totalTokens,
|
|
1130
|
+
},
|
|
1131
|
+
};
|
|
1132
|
+
}
|
|
1133
|
+
function bedrockResponse(result) {
|
|
1134
|
+
return {
|
|
1135
|
+
output: {
|
|
1136
|
+
message: {
|
|
1137
|
+
role: 'assistant',
|
|
1138
|
+
content: [{ text: result.text }],
|
|
1139
|
+
},
|
|
1140
|
+
},
|
|
1141
|
+
stopReason: 'end_turn',
|
|
1142
|
+
usage: {
|
|
1143
|
+
inputTokens: result.usage.promptTokens,
|
|
1144
|
+
outputTokens: result.usage.completionTokens,
|
|
1145
|
+
totalTokens: result.usage.totalTokens,
|
|
1146
|
+
},
|
|
1147
|
+
};
|
|
1148
|
+
}
|
|
1149
|
+
async function resolveTokenCount(body, config, completionEngine) {
|
|
1150
|
+
if (!completionEngine?.countTokens) {
|
|
1151
|
+
return createErrorResponse(`Token counting is not supported for provider ${config.targetProvider}.`, 501);
|
|
1152
|
+
}
|
|
1153
|
+
const request = buildCompletionRequest(body, config.exposedTransport, false);
|
|
1154
|
+
request.model = config.targetModel;
|
|
1155
|
+
try {
|
|
1156
|
+
return await completionEngine.countTokens(request);
|
|
1157
|
+
}
|
|
1158
|
+
catch (error) {
|
|
1159
|
+
return createErrorResponse(error instanceof Error ? error.message : String(error));
|
|
1160
|
+
}
|
|
1161
|
+
}
|
|
1162
|
+
export function createTransportMuxApp({ config, completionEngine, costFeedbackSink, costFeedbackMetadata }) {
|
|
1163
|
+
const app = new Hono();
|
|
1164
|
+
const metrics = new MetricsTracker();
|
|
1165
|
+
const thoughtSignatureStore = new Map();
|
|
1166
|
+
app.onError((error, c) => {
|
|
1167
|
+
metrics.recordError();
|
|
1168
|
+
return c.json({ error: { message: error instanceof Error ? error.message : String(error) } }, 500);
|
|
1169
|
+
});
|
|
1170
|
+
app.use('*', async (c, next) => {
|
|
1171
|
+
if (c.req.path === '/'
|
|
1172
|
+
|| c.req.path === '/health'
|
|
1173
|
+
|| c.req.path === '/v1/models'
|
|
1174
|
+
|| c.req.path === '/metrics'
|
|
1175
|
+
|| c.req.path === '/cache/stats') {
|
|
1176
|
+
await next();
|
|
1177
|
+
return;
|
|
1178
|
+
}
|
|
1179
|
+
if (!isAuthorized(c.req.raw, config.authToken)) {
|
|
1180
|
+
// Gemini CLI sends the API key as a ?key= query parameter
|
|
1181
|
+
const queryKey = c.req.query('key');
|
|
1182
|
+
if (!queryKey || queryKey !== config.authToken) {
|
|
1183
|
+
console.error(`[transport-adapter] AUTH REJECT: path=${c.req.path} queryKey=${queryKey ? queryKey.slice(0, 8) + '...' : 'null'} expectedToken=${config.authToken?.slice(0, 8)}... headers: x-api-key=${c.req.header('x-api-key')?.slice(0, 8) ?? 'null'} auth=${c.req.header('authorization')?.slice(0, 20) ?? 'null'}`);
|
|
1184
|
+
return c.json({ error: { message: 'Unauthorized' } }, 401);
|
|
1185
|
+
}
|
|
1186
|
+
}
|
|
1187
|
+
await next();
|
|
1188
|
+
});
|
|
1189
|
+
app.get('/health', (c) => c.json({ ok: true, transport: config.exposedTransport }));
|
|
1190
|
+
app.get('/v1/models', (c) => c.json({
|
|
1191
|
+
object: 'list',
|
|
1192
|
+
data: [{ id: config.targetModel, object: 'model', owned_by: config.targetProvider }],
|
|
1193
|
+
}));
|
|
1194
|
+
app.get('/metrics', (c) => c.json(metrics.toJSON()));
|
|
1195
|
+
app.get('/cache/stats', (c) => c.json({ enabled: false }));
|
|
1196
|
+
app.post('/v1/count_tokens', async (c) => {
|
|
1197
|
+
const body = await readStrictJsonBody(c.req.raw);
|
|
1198
|
+
if (body instanceof Response) {
|
|
1199
|
+
return body;
|
|
1200
|
+
}
|
|
1201
|
+
const result = await resolveTokenCount(body, config, completionEngine);
|
|
1202
|
+
if (result instanceof Response) {
|
|
1203
|
+
return result;
|
|
1204
|
+
}
|
|
1205
|
+
return c.json(result);
|
|
1206
|
+
});
|
|
1207
|
+
app.post('/v1/messages', async (c) => {
|
|
1208
|
+
const plan = await createExecutionPlan(c.req.raw, 'anthropic', { thoughtSignatureStore });
|
|
1209
|
+
if (plan instanceof Response) {
|
|
1210
|
+
return plan;
|
|
1211
|
+
}
|
|
1212
|
+
plan.request.model = config.targetModel;
|
|
1213
|
+
const feedbackMetadata = costFeedbackMetadataFromRequest(c.req.raw, costFeedbackMetadata);
|
|
1214
|
+
const result = await trackCompletionOutcome(await resolveCompletion(c.req.raw, config, completionEngine, plan, metrics, { costFeedbackSink, costFeedbackMetadata: feedbackMetadata }), metrics, { countSuccessResponse: !plan.streamRequested || !completionEngine, config, costFeedbackSink, costFeedbackMetadata: feedbackMetadata });
|
|
1215
|
+
if (result instanceof Response) {
|
|
1216
|
+
return result;
|
|
1217
|
+
}
|
|
1218
|
+
return c.json(anthropicResponse(result, config));
|
|
1219
|
+
});
|
|
1220
|
+
app.post('/v1/chat/completions', async (c) => {
|
|
1221
|
+
const plan = await createExecutionPlan(c.req.raw, 'openai-chat');
|
|
1222
|
+
if (plan instanceof Response) {
|
|
1223
|
+
return plan;
|
|
1224
|
+
}
|
|
1225
|
+
plan.request.model = config.targetModel;
|
|
1226
|
+
const feedbackMetadata = costFeedbackMetadataFromRequest(c.req.raw, costFeedbackMetadata);
|
|
1227
|
+
const result = await trackCompletionOutcome(await resolveCompletion(c.req.raw, config, completionEngine, plan, metrics, { costFeedbackSink, costFeedbackMetadata: feedbackMetadata }), metrics, { countSuccessResponse: !plan.streamRequested || !completionEngine, config, costFeedbackSink, costFeedbackMetadata: feedbackMetadata });
|
|
1228
|
+
if (result instanceof Response) {
|
|
1229
|
+
return result;
|
|
1230
|
+
}
|
|
1231
|
+
return c.json(openAiChatResponse(result, config));
|
|
1232
|
+
});
|
|
1233
|
+
app.post('/v1/responses', async (c) => {
|
|
1234
|
+
console.error(`[transport-adapter] POST /v1/responses (HTTP SSE)`);
|
|
1235
|
+
const plan = await createExecutionPlan(c.req.raw, 'openai-responses', { thoughtSignatureStore });
|
|
1236
|
+
if (plan instanceof Response) {
|
|
1237
|
+
return plan;
|
|
1238
|
+
}
|
|
1239
|
+
plan.request.model = config.targetModel;
|
|
1240
|
+
const feedbackMetadata = costFeedbackMetadataFromRequest(c.req.raw, costFeedbackMetadata);
|
|
1241
|
+
const result = await trackCompletionOutcome(await resolveCompletion(c.req.raw, config, completionEngine, plan, metrics, { costFeedbackSink, costFeedbackMetadata: feedbackMetadata }), metrics, { countSuccessResponse: !plan.streamRequested || !completionEngine, config, costFeedbackSink, costFeedbackMetadata: feedbackMetadata });
|
|
1242
|
+
if (result instanceof Response) {
|
|
1243
|
+
return result;
|
|
1244
|
+
}
|
|
1245
|
+
return c.json(openAiResponsesResponse(result, config));
|
|
1246
|
+
});
|
|
1247
|
+
app.post('/v1beta/models/*', async (c) => {
|
|
1248
|
+
const forceStreaming = c.req.path.endsWith(':streamGenerateContent');
|
|
1249
|
+
const plan = await createExecutionPlan(c.req.raw, 'google', { forceStreaming });
|
|
1250
|
+
if (plan instanceof Response) {
|
|
1251
|
+
return plan;
|
|
1252
|
+
}
|
|
1253
|
+
plan.request.model = config.targetModel;
|
|
1254
|
+
const feedbackMetadata = costFeedbackMetadataFromRequest(c.req.raw, costFeedbackMetadata);
|
|
1255
|
+
const result = await trackCompletionOutcome(await resolveCompletion(c.req.raw, config, completionEngine, plan, metrics, { forceStreaming, costFeedbackSink, costFeedbackMetadata: feedbackMetadata }), metrics, { countSuccessResponse: !plan.streamRequested || !completionEngine, config, costFeedbackSink, costFeedbackMetadata: feedbackMetadata });
|
|
1256
|
+
if (result instanceof Response) {
|
|
1257
|
+
return result;
|
|
1258
|
+
}
|
|
1259
|
+
return c.json(googleResponse(result));
|
|
1260
|
+
});
|
|
1261
|
+
// Vertex AI paths: gemini-cli in Vertex AI mode sends to /v1beta1/projects/*/...
|
|
1262
|
+
for (const prefix of ['/v1/projects/*', '/v1beta/projects/*', '/v1beta1/projects/*']) {
|
|
1263
|
+
app.post(prefix, vertexNativeHandler);
|
|
1264
|
+
}
|
|
1265
|
+
async function vertexNativeHandler(c) {
|
|
1266
|
+
const forceStreaming = c.req.path.endsWith(':streamGenerateContent');
|
|
1267
|
+
const plan = await createExecutionPlan(c.req.raw, 'google', { forceStreaming });
|
|
1268
|
+
if (plan instanceof Response) {
|
|
1269
|
+
return plan;
|
|
1270
|
+
}
|
|
1271
|
+
plan.request.model = config.targetModel;
|
|
1272
|
+
const feedbackMetadata = costFeedbackMetadataFromRequest(c.req.raw, costFeedbackMetadata);
|
|
1273
|
+
const result = await trackCompletionOutcome(await resolveCompletion(c.req.raw, config, completionEngine, plan, metrics, { costFeedbackSink, costFeedbackMetadata: feedbackMetadata }), metrics, { countSuccessResponse: !plan.streamRequested || !completionEngine, config, costFeedbackSink, costFeedbackMetadata: feedbackMetadata });
|
|
1274
|
+
if (result instanceof Response) {
|
|
1275
|
+
return result;
|
|
1276
|
+
}
|
|
1277
|
+
return forceStreaming
|
|
1278
|
+
? renderStreamResponse('google', result, config)
|
|
1279
|
+
: c.json(googleResponse(result));
|
|
1280
|
+
}
|
|
1281
|
+
app.post('/converse', async (c) => {
|
|
1282
|
+
const plan = await createExecutionPlan(c.req.raw, 'bedrock-converse');
|
|
1283
|
+
if (plan instanceof Response) {
|
|
1284
|
+
return plan;
|
|
1285
|
+
}
|
|
1286
|
+
plan.request.model = config.targetModel;
|
|
1287
|
+
const feedbackMetadata = costFeedbackMetadataFromRequest(c.req.raw, costFeedbackMetadata);
|
|
1288
|
+
const result = await trackCompletionOutcome(await resolveCompletion(c.req.raw, config, completionEngine, plan, metrics, { costFeedbackSink, costFeedbackMetadata: feedbackMetadata }), metrics, { countSuccessResponse: !plan.streamRequested || !completionEngine, config, costFeedbackSink, costFeedbackMetadata: feedbackMetadata });
|
|
1289
|
+
if (result instanceof Response) {
|
|
1290
|
+
return result;
|
|
1291
|
+
}
|
|
1292
|
+
return c.json(bedrockResponse(result));
|
|
1293
|
+
});
|
|
1294
|
+
app.post('/models/chat/completions', async (c) => {
|
|
1295
|
+
const plan = await createExecutionPlan(c.req.raw, 'azure-foundry');
|
|
1296
|
+
if (plan instanceof Response) {
|
|
1297
|
+
return plan;
|
|
1298
|
+
}
|
|
1299
|
+
plan.request.model = config.targetModel;
|
|
1300
|
+
const feedbackMetadata = costFeedbackMetadataFromRequest(c.req.raw, costFeedbackMetadata);
|
|
1301
|
+
const result = await trackCompletionOutcome(await resolveCompletion(c.req.raw, config, completionEngine, plan, metrics, { costFeedbackSink, costFeedbackMetadata: feedbackMetadata }), metrics, { countSuccessResponse: !plan.streamRequested || !completionEngine, config, costFeedbackSink, costFeedbackMetadata: feedbackMetadata });
|
|
1302
|
+
if (result instanceof Response) {
|
|
1303
|
+
return result;
|
|
1304
|
+
}
|
|
1305
|
+
return c.json(openAiChatResponse(result, config));
|
|
1306
|
+
});
|
|
1307
|
+
app.all('/passthrough/*', async (c) => {
|
|
1308
|
+
const forwardedPath = c.req.path.replace(/^\/passthrough/, '') || '/';
|
|
1309
|
+
return trackCompletionOutcome(await proxyUpstream(c.req.raw, config, forwardedPath), metrics, {
|
|
1310
|
+
countSuccessResponse: true,
|
|
1311
|
+
});
|
|
1312
|
+
});
|
|
1313
|
+
return app;
|
|
1314
|
+
}
|
|
1315
|
+
async function nodeRequestBody(req) {
|
|
1316
|
+
if (req.method === 'GET' || req.method === 'HEAD') {
|
|
1317
|
+
return undefined;
|
|
1318
|
+
}
|
|
1319
|
+
const chunks = [];
|
|
1320
|
+
for await (const chunk of req) {
|
|
1321
|
+
if (Buffer.isBuffer(chunk)) {
|
|
1322
|
+
chunks.push(chunk);
|
|
1323
|
+
continue;
|
|
1324
|
+
}
|
|
1325
|
+
if (chunk instanceof Uint8Array) {
|
|
1326
|
+
chunks.push(Buffer.from(chunk));
|
|
1327
|
+
continue;
|
|
1328
|
+
}
|
|
1329
|
+
if (typeof chunk === 'string') {
|
|
1330
|
+
chunks.push(Buffer.from(chunk));
|
|
1331
|
+
}
|
|
1332
|
+
}
|
|
1333
|
+
return Buffer.concat(chunks);
|
|
1334
|
+
}
|
|
1335
|
+
async function writeNodeResponse(res, response) {
|
|
1336
|
+
res.statusCode = response.status;
|
|
1337
|
+
response.headers.forEach((value, key) => {
|
|
1338
|
+
res.setHeader(key, value);
|
|
1339
|
+
});
|
|
1340
|
+
if (!response.body) {
|
|
1341
|
+
res.end();
|
|
1342
|
+
return;
|
|
1343
|
+
}
|
|
1344
|
+
try {
|
|
1345
|
+
await pipeline(Readable.fromWeb(response.body), res);
|
|
1346
|
+
}
|
|
1347
|
+
catch (err) {
|
|
1348
|
+
if (!res.writableEnded) {
|
|
1349
|
+
res.end();
|
|
1350
|
+
}
|
|
1351
|
+
}
|
|
1352
|
+
}
|
|
1353
|
+
export async function startProxyServer(config, completionEngine) {
|
|
1354
|
+
const webSocketMetrics = new MetricsTracker();
|
|
1355
|
+
const webSocketThoughtSignatureStore = new Map();
|
|
1356
|
+
const app = createTransportMuxApp({ config, completionEngine });
|
|
1357
|
+
const server = http.createServer((req, res) => {
|
|
1358
|
+
void (async () => {
|
|
1359
|
+
const url = new URL(req.url ?? '/', `http://${config.host}:${config.port}`);
|
|
1360
|
+
const body = await nodeRequestBody(req);
|
|
1361
|
+
const request = new Request(url, {
|
|
1362
|
+
method: req.method,
|
|
1363
|
+
headers: new Headers(req.headers),
|
|
1364
|
+
body: body == null ? undefined : new Blob([new Uint8Array(body)]),
|
|
1365
|
+
});
|
|
1366
|
+
const response = await app.fetch(request);
|
|
1367
|
+
await writeNodeResponse(res, response);
|
|
1368
|
+
})().catch((error) => {
|
|
1369
|
+
if (!res.headersSent) {
|
|
1370
|
+
res.statusCode = 500;
|
|
1371
|
+
res.setHeader('content-type', 'application/json');
|
|
1372
|
+
res.end(JSON.stringify({ error: error instanceof Error ? error.message : String(error) }));
|
|
1373
|
+
}
|
|
1374
|
+
else if (!res.writableEnded) {
|
|
1375
|
+
res.end();
|
|
1376
|
+
}
|
|
1377
|
+
});
|
|
1378
|
+
});
|
|
1379
|
+
const webSocketServer = new WebSocketServer({ noServer: true });
|
|
1380
|
+
server.on('upgrade', (req, socket, head) => {
|
|
1381
|
+
const url = new URL(req.url ?? '/', `http://${config.host}:${config.port}`);
|
|
1382
|
+
console.error(`[transport-adapter] WebSocket upgrade: ${url.pathname} transport=${config.exposedTransport}`);
|
|
1383
|
+
if (url.pathname !== '/v1/responses' || config.exposedTransport !== 'openai-responses') {
|
|
1384
|
+
console.error(`[transport-adapter] WebSocket rejected: wrong path or transport`);
|
|
1385
|
+
socket.write('HTTP/1.1 404 Not Found\r\n\r\n');
|
|
1386
|
+
socket.destroy();
|
|
1387
|
+
return;
|
|
1388
|
+
}
|
|
1389
|
+
if (!isAuthorizedUpgrade(req, config.authToken)) {
|
|
1390
|
+
console.error(`[transport-adapter] WebSocket rejected: unauthorized`);
|
|
1391
|
+
socket.write('HTTP/1.1 401 Unauthorized\r\n\r\n');
|
|
1392
|
+
socket.destroy();
|
|
1393
|
+
return;
|
|
1394
|
+
}
|
|
1395
|
+
webSocketServer.handleUpgrade(req, socket, head, (ws) => {
|
|
1396
|
+
webSocketServer.emit('connection', ws, req);
|
|
1397
|
+
});
|
|
1398
|
+
});
|
|
1399
|
+
webSocketServer.on('connection', (ws) => {
|
|
1400
|
+
ws.on('message', (data) => {
|
|
1401
|
+
void handleOpenAiResponsesWebSocketMessage(ws, data, config, completionEngine, webSocketMetrics, webSocketThoughtSignatureStore);
|
|
1402
|
+
});
|
|
1403
|
+
});
|
|
1404
|
+
await new Promise((resolve, reject) => {
|
|
1405
|
+
server.once('error', reject);
|
|
1406
|
+
server.listen(config.port, config.host, () => resolve());
|
|
1407
|
+
});
|
|
1408
|
+
const address = server.address();
|
|
1409
|
+
if (!address || typeof address === 'string') {
|
|
1410
|
+
throw new Error('Failed to resolve bound proxy server address.');
|
|
1411
|
+
}
|
|
1412
|
+
return {
|
|
1413
|
+
url: `http://${config.host}:${address.port}`,
|
|
1414
|
+
port: address.port,
|
|
1415
|
+
async stop() {
|
|
1416
|
+
await new Promise((resolve, reject) => {
|
|
1417
|
+
webSocketServer.close((wsError) => {
|
|
1418
|
+
if (wsError) {
|
|
1419
|
+
reject(wsError);
|
|
1420
|
+
return;
|
|
1421
|
+
}
|
|
1422
|
+
server.close((error) => (error ? reject(error) : resolve()));
|
|
1423
|
+
});
|
|
1424
|
+
});
|
|
1425
|
+
},
|
|
1426
|
+
};
|
|
1427
|
+
}
|
|
1428
|
+
//# sourceMappingURL=server.js.map
|