@nvae/llmswitch 1.0.0 → 1.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +52 -9
- package/dist/adapters/claude.js +10 -10
- package/dist/adapters/codex.js +68 -16
- package/dist/adapters/opencode.js +45 -25
- package/dist/bridge/manager.js +8 -1
- package/dist/bridge/responses-to-chat-response.js +1 -1
- package/dist/bridge/runtime.js +40 -2
- package/dist/bridge/server.js +71 -26
- package/dist/bridge/state.js +59 -27
- package/dist/bridge/translate-response.js +98 -0
- package/dist/bridge/transport.js +39 -8
- package/dist/cli.js +18 -6
- package/dist/commands/bridge-cmd.js +15 -8
- package/dist/commands/gateway-cmd.js +112 -31
- package/dist/commands/home-cmd.js +20 -2
- package/dist/commands/launch-cmd.js +4 -0
- package/dist/commands/launch.js +36 -13
- package/dist/commands/prompts.js +96 -13
- package/dist/commands/tool.js +298 -134
- package/dist/gateway/keys.js +11 -2
- package/dist/gateway/rate-limit.js +8 -104
- package/dist/gateway/router.js +7 -1
- package/dist/gateway/server.js +1 -22
- package/dist/gateway/store.js +37 -4
- package/dist/gateway/usage.js +24 -4
- package/dist/index.js +0 -0
- package/dist/store/profiles.js +54 -14
- package/dist/types.js +12 -0
- package/dist/utils/display.js +71 -0
- package/dist/utils/fetch-models.js +26 -12
- package/dist/utils/file-lock.js +149 -0
- package/dist/utils/fs.js +187 -9
- package/dist/utils/model-metadata.js +54 -1
- package/package.json +7 -2
package/dist/bridge/server.js
CHANGED
|
@@ -1,11 +1,12 @@
|
|
|
1
1
|
import { createServer, } from "node:http";
|
|
2
|
-
import { parseBridgeRuntimeLimits } from "./runtime.js";
|
|
2
|
+
import { ConcurrencyGate, parseBridgeRuntimeLimits } from "./runtime.js";
|
|
3
3
|
import { requestWithNodeTransport, } from "./transport.js";
|
|
4
4
|
import { constantTimeTokenEqual, readBridgeState, readBridgeUpstreams, } from "./state.js";
|
|
5
5
|
import { anthropicToChatRequest } from "./anthropic-translate-request.js";
|
|
6
6
|
import { chatChunkToAnthropicEvents, chatCompletionToAnthropicMessage, createAnthropicStreamState, forceCompleteAnthropicStream, parseChatSseLine, } from "./anthropic-translate-response.js";
|
|
7
7
|
import { collectCustomToolNames, responsesToChatRequest, responsesToCompletionsRequest, } from "./translate-request.js";
|
|
8
8
|
import { chatChunkToResponsesEvents, chatCompletionToResponse, createStreamState, forceCompleteStream, parseChatSseLine as parseChatSseLineResponses, } from "./translate-response.js";
|
|
9
|
+
import { modelItemId, modelsFromPayload } from "../utils/fetch-models.js";
|
|
9
10
|
function headerValue(value) {
|
|
10
11
|
return Array.isArray(value) ? value[0] : value;
|
|
11
12
|
}
|
|
@@ -141,22 +142,21 @@ function requestUpstream(upstream, url, method, body, signal) {
|
|
|
141
142
|
maxResponseBytes: limits.maxResponseBytes,
|
|
142
143
|
});
|
|
143
144
|
}
|
|
144
|
-
async function fetchModelsJson(upstream) {
|
|
145
|
+
async function fetchModelsJson(upstream, signal) {
|
|
145
146
|
const url = joinUrl(upstream.baseUrl, "/models");
|
|
146
|
-
const response = await requestUpstream(upstream, url, "GET");
|
|
147
|
+
const response = await requestUpstream(upstream, url, "GET", undefined, signal);
|
|
147
148
|
if (!response.ok) {
|
|
148
149
|
return { ok: false, status: response.status, data: [] };
|
|
149
150
|
}
|
|
150
151
|
try {
|
|
151
152
|
const json = (await response.json());
|
|
152
|
-
|
|
153
|
-
return { ok: true, status: 200, data: data };
|
|
153
|
+
return { ok: true, status: 200, data: modelsFromPayload(json) };
|
|
154
154
|
}
|
|
155
155
|
catch {
|
|
156
156
|
return { ok: false, status: 502, data: [] };
|
|
157
157
|
}
|
|
158
158
|
}
|
|
159
|
-
async function proxyModelsMerged(_req, res, upstreams) {
|
|
159
|
+
async function proxyModelsMerged(_req, res, upstreams, signal) {
|
|
160
160
|
const sides = [upstreams.codex, upstreams.claude, upstreams.opencode].filter((u) => Boolean(u?.baseUrl));
|
|
161
161
|
if (!sides.length) {
|
|
162
162
|
sendJson(res, 503, {
|
|
@@ -164,7 +164,7 @@ async function proxyModelsMerged(_req, res, upstreams) {
|
|
|
164
164
|
});
|
|
165
165
|
return;
|
|
166
166
|
}
|
|
167
|
-
const results = await Promise.all(sides.map((u) => fetchModelsJson(u).catch(() => ({
|
|
167
|
+
const results = await Promise.all(sides.map((u) => fetchModelsJson(u, signal).catch(() => ({
|
|
168
168
|
ok: false,
|
|
169
169
|
status: 502,
|
|
170
170
|
data: [],
|
|
@@ -175,13 +175,14 @@ async function proxyModelsMerged(_req, res, upstreams) {
|
|
|
175
175
|
if (!result.ok)
|
|
176
176
|
continue;
|
|
177
177
|
for (const item of result.data) {
|
|
178
|
-
const id = item
|
|
179
|
-
? String(item.id)
|
|
180
|
-
: "";
|
|
178
|
+
const id = modelItemId(item);
|
|
181
179
|
if (!id || seen.has(id))
|
|
182
180
|
continue;
|
|
183
181
|
seen.add(id);
|
|
184
|
-
|
|
182
|
+
const row = item && typeof item === "object"
|
|
183
|
+
? item
|
|
184
|
+
: {};
|
|
185
|
+
merged.push(row.id ? item : { ...row, id });
|
|
185
186
|
}
|
|
186
187
|
}
|
|
187
188
|
if (!merged.length && results.every((r) => !r.ok)) {
|
|
@@ -192,7 +193,7 @@ async function proxyModelsMerged(_req, res, upstreams) {
|
|
|
192
193
|
}
|
|
193
194
|
sendJson(res, 200, { object: "list", data: merged });
|
|
194
195
|
}
|
|
195
|
-
async function handleResponses(req, res, upstream, bodyBuf) {
|
|
196
|
+
async function handleResponses(req, res, upstream, bodyBuf, signal) {
|
|
196
197
|
let body;
|
|
197
198
|
try {
|
|
198
199
|
body = JSON.parse(bodyBuf.toString("utf8"));
|
|
@@ -204,12 +205,12 @@ async function handleResponses(req, res, upstream, bodyBuf) {
|
|
|
204
205
|
const mode = upstream.mode || "chat";
|
|
205
206
|
const wantStream = Boolean(body.stream);
|
|
206
207
|
if (mode === "completions") {
|
|
207
|
-
await forwardCompletions(req, res, upstream, body, wantStream);
|
|
208
|
+
await forwardCompletions(req, res, upstream, body, wantStream, signal);
|
|
208
209
|
return;
|
|
209
210
|
}
|
|
210
|
-
await forwardChatResponses(req, res, upstream, body, wantStream);
|
|
211
|
+
await forwardChatResponses(req, res, upstream, body, wantStream, signal);
|
|
211
212
|
}
|
|
212
|
-
async function handleMessages(_req, res, upstream, bodyBuf) {
|
|
213
|
+
async function handleMessages(_req, res, upstream, bodyBuf, signal) {
|
|
213
214
|
let body;
|
|
214
215
|
try {
|
|
215
216
|
body = JSON.parse(bodyBuf.toString("utf8"));
|
|
@@ -226,7 +227,7 @@ async function handleMessages(_req, res, upstream, bodyBuf) {
|
|
|
226
227
|
const url = joinUrl(upstream.baseUrl, "/chat/completions");
|
|
227
228
|
let response;
|
|
228
229
|
try {
|
|
229
|
-
response = await requestUpstream(upstream, url, "POST", JSON.stringify(chatReq));
|
|
230
|
+
response = await requestUpstream(upstream, url, "POST", JSON.stringify(chatReq), signal);
|
|
230
231
|
}
|
|
231
232
|
catch (err) {
|
|
232
233
|
sendJson(res, 502, {
|
|
@@ -259,13 +260,13 @@ async function handleMessages(_req, res, upstream, bodyBuf) {
|
|
|
259
260
|
}
|
|
260
261
|
await pipeChatStreamToAnthropic(response, res, String(body.model || ""));
|
|
261
262
|
}
|
|
262
|
-
async function forwardChatResponses(req, res, upstream, body, wantStream) {
|
|
263
|
+
async function forwardChatResponses(req, res, upstream, body, wantStream, signal) {
|
|
263
264
|
const chatReq = responsesToChatRequest(body);
|
|
264
265
|
const customTools = collectCustomToolNames(body.tools);
|
|
265
266
|
const url = joinUrl(upstream.baseUrl, "/chat/completions");
|
|
266
267
|
let response;
|
|
267
268
|
try {
|
|
268
|
-
response = await requestUpstream(upstream, url, "POST", JSON.stringify(chatReq));
|
|
269
|
+
response = await requestUpstream(upstream, url, "POST", JSON.stringify(chatReq), signal);
|
|
269
270
|
}
|
|
270
271
|
catch (err) {
|
|
271
272
|
sendJson(res, 502, {
|
|
@@ -299,7 +300,7 @@ async function forwardChatResponses(req, res, upstream, body, wantStream) {
|
|
|
299
300
|
* upstream `/chat/completions` (with llm-switch transport applying the proxy)
|
|
300
301
|
* and relay the raw response, preserving streaming for SSE.
|
|
301
302
|
*/
|
|
302
|
-
async function forwardOpenCodeChat(_req, res, upstream, bodyBuf) {
|
|
303
|
+
async function forwardOpenCodeChat(_req, res, upstream, bodyBuf, signal) {
|
|
303
304
|
let body;
|
|
304
305
|
try {
|
|
305
306
|
body = JSON.parse(bodyBuf.toString("utf8"));
|
|
@@ -312,7 +313,7 @@ async function forwardOpenCodeChat(_req, res, upstream, bodyBuf) {
|
|
|
312
313
|
const url = joinUrl(upstream.baseUrl, "/chat/completions");
|
|
313
314
|
let response;
|
|
314
315
|
try {
|
|
315
|
-
response = await requestUpstream(upstream, url, "POST", bodyBuf.toString("utf8"));
|
|
316
|
+
response = await requestUpstream(upstream, url, "POST", bodyBuf.toString("utf8"), signal);
|
|
316
317
|
}
|
|
317
318
|
catch (err) {
|
|
318
319
|
sendJson(res, 502, {
|
|
@@ -340,13 +341,13 @@ async function forwardOpenCodeChat(_req, res, upstream, bodyBuf) {
|
|
|
340
341
|
}
|
|
341
342
|
await pipeRawStream(response, res);
|
|
342
343
|
}
|
|
343
|
-
async function forwardCompletions(_req, res, upstream, body, wantStream) {
|
|
344
|
+
async function forwardCompletions(_req, res, upstream, body, wantStream, signal) {
|
|
344
345
|
const completionReq = responsesToCompletionsRequest(body);
|
|
345
346
|
const customTools = collectCustomToolNames(body.tools);
|
|
346
347
|
const url = joinUrl(upstream.baseUrl, "/completions");
|
|
347
348
|
let response;
|
|
348
349
|
try {
|
|
349
|
-
response = await requestUpstream(upstream, url, "POST", JSON.stringify(completionReq));
|
|
350
|
+
response = await requestUpstream(upstream, url, "POST", JSON.stringify(completionReq), signal);
|
|
350
351
|
}
|
|
351
352
|
catch (err) {
|
|
352
353
|
sendJson(res, 502, {
|
|
@@ -520,7 +521,41 @@ async function pipeRawStream(upstream, res) {
|
|
|
520
521
|
}
|
|
521
522
|
}
|
|
522
523
|
export function createBridgeServer(options = {}) {
|
|
524
|
+
const limits = parseBridgeRuntimeLimits();
|
|
525
|
+
// LLM_SWITCH_MAX_CONCURRENCY 与 gateway 共用;默认 0 表示不限,
|
|
526
|
+
// 此时 gate 只统计在途请求、不做拒绝。
|
|
527
|
+
const gate = new ConcurrencyGate(limits.maxConcurrency);
|
|
523
528
|
return createServer(async (req, res) => {
|
|
529
|
+
// 客户端断开(Ctrl-C)后必须把上游请求也取消掉,否则 bridge 会一直把流读到
|
|
530
|
+
// idle/total 超时,白白消耗上游 token 与连接。
|
|
531
|
+
//
|
|
532
|
+
// 注意运行时差异:Node 会在客户端掉线时给 ServerResponse 发 "close",
|
|
533
|
+
// 但 Bun 的 node:http 不发(只有 req 的 "aborted" 与 socket 的 "close")。
|
|
534
|
+
// bridge 守护进程两种运行时都可能跑,所以三个信号都监听,并用
|
|
535
|
+
// writableFinished 兜底避免正常收尾时误取消。
|
|
536
|
+
const controller = new AbortController();
|
|
537
|
+
const abort = () => {
|
|
538
|
+
if (res.writableFinished)
|
|
539
|
+
return;
|
|
540
|
+
controller.abort();
|
|
541
|
+
};
|
|
542
|
+
const socket = res.socket;
|
|
543
|
+
res.on("close", abort);
|
|
544
|
+
req.on("aborted", abort);
|
|
545
|
+
socket?.on("close", abort);
|
|
546
|
+
const signal = controller.signal;
|
|
547
|
+
const isDataPlane = req.method === "POST" &&
|
|
548
|
+
/^\/(v1\/)?(responses|messages|chat\/completions|completions)$/.test((req.url || "/").split("?")[0].replace(/\/+$/, "") || "/");
|
|
549
|
+
if (isDataPlane && !gate.tryAcquire()) {
|
|
550
|
+
res.setHeader("Retry-After", "1");
|
|
551
|
+
sendJson(res, 503, {
|
|
552
|
+
error: {
|
|
553
|
+
code: "too_many_concurrent_requests",
|
|
554
|
+
message: `并发请求数已达上限 ${limits.maxConcurrency}(LLM_SWITCH_MAX_CONCURRENCY 可调整)`,
|
|
555
|
+
},
|
|
556
|
+
});
|
|
557
|
+
return;
|
|
558
|
+
}
|
|
524
559
|
try {
|
|
525
560
|
const state = readBridgeState();
|
|
526
561
|
const expectedControlToken = options.controlToken ?? state.instance?.controlToken;
|
|
@@ -614,7 +649,7 @@ export function createBridgeServer(options = {}) {
|
|
|
614
649
|
});
|
|
615
650
|
return;
|
|
616
651
|
}
|
|
617
|
-
await proxyModelsMerged(req, res, merged);
|
|
652
|
+
await proxyModelsMerged(req, res, merged, signal);
|
|
618
653
|
return;
|
|
619
654
|
}
|
|
620
655
|
if (req.method === "POST" &&
|
|
@@ -637,7 +672,7 @@ export function createBridgeServer(options = {}) {
|
|
|
637
672
|
return;
|
|
638
673
|
}
|
|
639
674
|
const body = await readBody(req);
|
|
640
|
-
await handleResponses(req, res, merged.codex, body);
|
|
675
|
+
await handleResponses(req, res, merged.codex, body, signal);
|
|
641
676
|
return;
|
|
642
677
|
}
|
|
643
678
|
if (req.method === "POST" &&
|
|
@@ -663,7 +698,7 @@ export function createBridgeServer(options = {}) {
|
|
|
663
698
|
return;
|
|
664
699
|
}
|
|
665
700
|
const body = await readBody(req);
|
|
666
|
-
await handleMessages(req, res, merged.claude, body);
|
|
701
|
+
await handleMessages(req, res, merged.claude, body, signal);
|
|
667
702
|
return;
|
|
668
703
|
}
|
|
669
704
|
if (req.method === "POST" &&
|
|
@@ -686,7 +721,7 @@ export function createBridgeServer(options = {}) {
|
|
|
686
721
|
return;
|
|
687
722
|
}
|
|
688
723
|
const body = await readBody(req);
|
|
689
|
-
await forwardOpenCodeChat(req, res, merged.opencode, body);
|
|
724
|
+
await forwardOpenCodeChat(req, res, merged.opencode, body, signal);
|
|
690
725
|
return;
|
|
691
726
|
}
|
|
692
727
|
sendJson(res, 404, {
|
|
@@ -696,6 +731,9 @@ export function createBridgeServer(options = {}) {
|
|
|
696
731
|
});
|
|
697
732
|
}
|
|
698
733
|
catch (err) {
|
|
734
|
+
// 客户端已经走了就没人读响应了,不必再写。
|
|
735
|
+
if (signal.aborted || res.writableEnded)
|
|
736
|
+
return;
|
|
699
737
|
if (err instanceof RequestBodyTooLargeError) {
|
|
700
738
|
sendJson(res, 413, {
|
|
701
739
|
error: { code: "request_too_large", message: err.message },
|
|
@@ -708,6 +746,13 @@ export function createBridgeServer(options = {}) {
|
|
|
708
746
|
},
|
|
709
747
|
});
|
|
710
748
|
}
|
|
749
|
+
finally {
|
|
750
|
+
res.off("close", abort);
|
|
751
|
+
req.off("aborted", abort);
|
|
752
|
+
socket?.off("close", abort);
|
|
753
|
+
if (isDataPlane)
|
|
754
|
+
gate.release();
|
|
755
|
+
}
|
|
711
756
|
});
|
|
712
757
|
}
|
|
713
758
|
export function listenBridge(port, host, options = {}) {
|
package/dist/bridge/state.js
CHANGED
|
@@ -240,41 +240,73 @@ function persistState(next) {
|
|
|
240
240
|
* revision must equal the on-disk revision (compare-and-set).
|
|
241
241
|
*/
|
|
242
242
|
export function writeBridgeState(state) {
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
243
|
+
return withBridgeLock(() => {
|
|
244
|
+
const current = readBridgeState();
|
|
245
|
+
if (existsSync(getBridgeStatePath()) &&
|
|
246
|
+
state.revision !== current.revision) {
|
|
247
|
+
throw new BridgeStateConflictError(current.revision, state.revision);
|
|
248
|
+
}
|
|
249
|
+
const next = withFlatAliases({
|
|
250
|
+
version: STATE_VERSION,
|
|
251
|
+
revision: current.revision + 1,
|
|
252
|
+
listener: state.listener,
|
|
253
|
+
instance: state.instance,
|
|
254
|
+
upstreams: normalizeBridgeUpstreams(state.upstreams),
|
|
255
|
+
pending: state.pending,
|
|
256
|
+
});
|
|
257
|
+
persistState(next);
|
|
258
|
+
return next;
|
|
254
259
|
});
|
|
255
|
-
persistState(next);
|
|
256
|
-
return next;
|
|
257
260
|
}
|
|
258
261
|
/**
|
|
259
262
|
* Atomically mutate state under compare-and-set. `expectedRevision` defaults to
|
|
260
263
|
* the current on-disk revision.
|
|
264
|
+
*
|
|
265
|
+
* The read-modify-write runs inside the on-disk bridge lock. Without it two
|
|
266
|
+
* concurrent `llms <tool> use` runs can interleave: both read the same revision,
|
|
267
|
+
* both write, and one side's upstream (or the daemon's instance identity) is
|
|
268
|
+
* lost — which then makes `bridge stop` unable to recognise the live daemon.
|
|
261
269
|
*/
|
|
262
270
|
export function updateBridgeState(mutate, expectedRevision) {
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
271
|
+
return withBridgeLock(() => {
|
|
272
|
+
const current = readBridgeState();
|
|
273
|
+
if (expectedRevision !== undefined &&
|
|
274
|
+
expectedRevision !== current.revision) {
|
|
275
|
+
throw new BridgeStateConflictError(current.revision, expectedRevision);
|
|
276
|
+
}
|
|
277
|
+
const mutated = mutate(current);
|
|
278
|
+
const next = withFlatAliases({
|
|
279
|
+
version: STATE_VERSION,
|
|
280
|
+
revision: current.revision + 1,
|
|
281
|
+
listener: mutated.listener,
|
|
282
|
+
instance: mutated.instance,
|
|
283
|
+
upstreams: normalizeBridgeUpstreams(mutated.upstreams),
|
|
284
|
+
pending: mutated.pending,
|
|
285
|
+
});
|
|
286
|
+
persistState(next);
|
|
287
|
+
return next;
|
|
275
288
|
});
|
|
276
|
-
|
|
277
|
-
|
|
289
|
+
}
|
|
290
|
+
/**
|
|
291
|
+
* Run `fn` while holding the exclusive bridge lock.
|
|
292
|
+
*
|
|
293
|
+
* Reentrant within a process: the file lock uses `flag: "wx"`, so a nested
|
|
294
|
+
* acquire from the same process would block until it timed out. A depth counter
|
|
295
|
+
* keeps nested writes (e.g. clearBridgeUpstream → stopBridge) working.
|
|
296
|
+
*/
|
|
297
|
+
let bridgeLockDepth = 0;
|
|
298
|
+
export function withBridgeLock(fn) {
|
|
299
|
+
if (bridgeLockDepth > 0)
|
|
300
|
+
return fn();
|
|
301
|
+
const lock = acquireBridgeLock();
|
|
302
|
+
bridgeLockDepth += 1;
|
|
303
|
+
try {
|
|
304
|
+
return fn();
|
|
305
|
+
}
|
|
306
|
+
finally {
|
|
307
|
+
bridgeLockDepth -= 1;
|
|
308
|
+
lock.release();
|
|
309
|
+
}
|
|
278
310
|
}
|
|
279
311
|
// --- upstream facade (compat) ----------------------------------------------
|
|
280
312
|
export function readBridgeUpstreams() {
|
|
@@ -37,6 +37,9 @@ export function createStreamState(model, responseId, customTools, webSearchEnabl
|
|
|
37
37
|
textItemId: null,
|
|
38
38
|
textStarted: false,
|
|
39
39
|
textContentIndex: 0,
|
|
40
|
+
reasoningItemId: null,
|
|
41
|
+
reasoningStarted: false,
|
|
42
|
+
reasoningText: "",
|
|
40
43
|
outputIndex: 0,
|
|
41
44
|
currentText: "",
|
|
42
45
|
completedItems: [],
|
|
@@ -147,6 +150,8 @@ function ensureCreated(state, out) {
|
|
|
147
150
|
function ensureTextItem(state, out) {
|
|
148
151
|
if (state.textStarted)
|
|
149
152
|
return;
|
|
153
|
+
// reasoning 必须排在正文之前,先把它收尾再开正文项。
|
|
154
|
+
closeReasoningItem(state, out);
|
|
150
155
|
state.textStarted = true;
|
|
151
156
|
state.textItemId = newId("msg");
|
|
152
157
|
state.currentText = "";
|
|
@@ -168,6 +173,74 @@ function ensureTextItem(state, out) {
|
|
|
168
173
|
part: { type: "output_text", text: "", annotations: [] },
|
|
169
174
|
}));
|
|
170
175
|
}
|
|
176
|
+
/**
|
|
177
|
+
* Reasoning models on Chat Completions upstreams stream their chain of thought
|
|
178
|
+
* as `delta.reasoning_content`. Responses clients (Codex) expect it as a
|
|
179
|
+
* `reasoning` output item with summary text, so surface it instead of dropping it.
|
|
180
|
+
*/
|
|
181
|
+
function ensureReasoningItem(state, out) {
|
|
182
|
+
if (state.reasoningStarted)
|
|
183
|
+
return;
|
|
184
|
+
state.reasoningStarted = true;
|
|
185
|
+
state.reasoningItemId = newId("rs");
|
|
186
|
+
state.reasoningText = "";
|
|
187
|
+
out.push(sseEvent("response.output_item.added", {
|
|
188
|
+
output_index: state.outputIndex,
|
|
189
|
+
item: {
|
|
190
|
+
id: state.reasoningItemId,
|
|
191
|
+
type: "reasoning",
|
|
192
|
+
status: "in_progress",
|
|
193
|
+
summary: [],
|
|
194
|
+
},
|
|
195
|
+
}));
|
|
196
|
+
out.push(sseEvent("response.reasoning_summary_part.added", {
|
|
197
|
+
item_id: state.reasoningItemId,
|
|
198
|
+
output_index: state.outputIndex,
|
|
199
|
+
summary_index: 0,
|
|
200
|
+
part: { type: "summary_text", text: "" },
|
|
201
|
+
}));
|
|
202
|
+
}
|
|
203
|
+
function emitReasoningDelta(state, out, content) {
|
|
204
|
+
if (!content)
|
|
205
|
+
return;
|
|
206
|
+
ensureReasoningItem(state, out);
|
|
207
|
+
state.reasoningText += content;
|
|
208
|
+
out.push(sseEvent("response.reasoning_summary_text.delta", {
|
|
209
|
+
item_id: state.reasoningItemId,
|
|
210
|
+
output_index: state.outputIndex,
|
|
211
|
+
summary_index: 0,
|
|
212
|
+
delta: content,
|
|
213
|
+
}));
|
|
214
|
+
}
|
|
215
|
+
function closeReasoningItem(state, out) {
|
|
216
|
+
if (!state.reasoningStarted || !state.reasoningItemId)
|
|
217
|
+
return;
|
|
218
|
+
const outputIndex = state.outputIndex;
|
|
219
|
+
const item = {
|
|
220
|
+
id: state.reasoningItemId,
|
|
221
|
+
type: "reasoning",
|
|
222
|
+
status: "completed",
|
|
223
|
+
summary: [{ type: "summary_text", text: state.reasoningText }],
|
|
224
|
+
};
|
|
225
|
+
out.push(sseEvent("response.reasoning_summary_text.done", {
|
|
226
|
+
item_id: state.reasoningItemId,
|
|
227
|
+
output_index: outputIndex,
|
|
228
|
+
summary_index: 0,
|
|
229
|
+
text: state.reasoningText,
|
|
230
|
+
}));
|
|
231
|
+
out.push(sseEvent("response.reasoning_summary_part.done", {
|
|
232
|
+
item_id: state.reasoningItemId,
|
|
233
|
+
output_index: outputIndex,
|
|
234
|
+
summary_index: 0,
|
|
235
|
+
part: { type: "summary_text", text: state.reasoningText },
|
|
236
|
+
}));
|
|
237
|
+
out.push(sseEvent("response.output_item.done", { output_index: outputIndex, item }));
|
|
238
|
+
state.completedItems.push({ outputIndex, item });
|
|
239
|
+
state.outputIndex += 1;
|
|
240
|
+
state.reasoningStarted = false;
|
|
241
|
+
state.reasoningItemId = null;
|
|
242
|
+
state.reasoningText = "";
|
|
243
|
+
}
|
|
171
244
|
function emitTextDelta(state, out, content) {
|
|
172
245
|
if (!content)
|
|
173
246
|
return;
|
|
@@ -279,6 +352,14 @@ export function chatChunkToResponsesEvents(chunk, state) {
|
|
|
279
352
|
const choice = choiceRaw;
|
|
280
353
|
const delta = (choice.delta || choice.message || {});
|
|
281
354
|
const finish = choice.finish_reason;
|
|
355
|
+
const reasoningDelta = typeof delta.reasoning_content === "string"
|
|
356
|
+
? delta.reasoning_content
|
|
357
|
+
: typeof delta.reasoning === "string"
|
|
358
|
+
? delta.reasoning
|
|
359
|
+
: "";
|
|
360
|
+
if (reasoningDelta) {
|
|
361
|
+
emitReasoningDelta(state, out, reasoningDelta);
|
|
362
|
+
}
|
|
282
363
|
if (typeof delta.content === "string" && delta.content.length > 0) {
|
|
283
364
|
consumeTextContent(state, out, delta.content);
|
|
284
365
|
}
|
|
@@ -448,6 +529,8 @@ function finalizeStream(state, out, finishReason) {
|
|
|
448
529
|
if (state.textStarted && state.textItemId) {
|
|
449
530
|
closeTextItem(state, out);
|
|
450
531
|
}
|
|
532
|
+
// 只有 reasoning 没有正文的回复(或工具调用前只推理)也要把 reasoning 收尾。
|
|
533
|
+
closeReasoningItem(state, out);
|
|
451
534
|
for (const entry of state.toolCalls.values()) {
|
|
452
535
|
if (!entry.started)
|
|
453
536
|
continue;
|
|
@@ -549,6 +632,21 @@ export function chatCompletionToResponse(chat, modelFallback, customTools, webSe
|
|
|
549
632
|
const content = typeof message.content === "string"
|
|
550
633
|
? message.content
|
|
551
634
|
: textFromCompletions;
|
|
635
|
+
// 推理模型在 Chat 上游用 reasoning_content 返回思维链;Responses 客户端
|
|
636
|
+
// (Codex)读的是 reasoning 输出项,之前这段内容被整体丢弃。
|
|
637
|
+
const reasoningText = typeof message.reasoning_content === "string"
|
|
638
|
+
? message.reasoning_content
|
|
639
|
+
: typeof message.reasoning === "string"
|
|
640
|
+
? message.reasoning
|
|
641
|
+
: "";
|
|
642
|
+
if (reasoningText) {
|
|
643
|
+
output.push({
|
|
644
|
+
id: newId("rs"),
|
|
645
|
+
type: "reasoning",
|
|
646
|
+
status: "completed",
|
|
647
|
+
summary: [{ type: "summary_text", text: reasoningText }],
|
|
648
|
+
});
|
|
649
|
+
}
|
|
552
650
|
for (const segment of splitWebSearchContent(content, webSearchEnabled)) {
|
|
553
651
|
if (segment.type === "web_search") {
|
|
554
652
|
output.push({
|
package/dist/bridge/transport.js
CHANGED
|
@@ -66,6 +66,39 @@ export function selectProxyUrl(_target, proxy) {
|
|
|
66
66
|
* Build a fresh per-request Agent for the given proxy URL. Callers own its
|
|
67
67
|
* lifecycle and must call `.destroy()` once the response is drained.
|
|
68
68
|
*/
|
|
69
|
+
/**
|
|
70
|
+
* Proxy agents are cached and keep-alive.
|
|
71
|
+
*
|
|
72
|
+
* A fresh Agent per request meant a new TCP+TLS (and CONNECT) handshake for
|
|
73
|
+
* every upstream call, which is significant on chatty streaming workloads.
|
|
74
|
+
* Cached agents are shared, so they must never be destroyed on a single
|
|
75
|
+
* request's failure — Node's Agent already evicts broken sockets itself.
|
|
76
|
+
* Keyed by proxy URL because socks5 vs socks5h changes DNS resolution.
|
|
77
|
+
*/
|
|
78
|
+
const proxyAgentCache = new Map();
|
|
79
|
+
const PROXY_AGENT_CACHE_MAX = 16;
|
|
80
|
+
export function getProxyAgent(target, proxyUrl) {
|
|
81
|
+
const key = `${proxyUrl}|${target.protocol}`;
|
|
82
|
+
const cached = proxyAgentCache.get(key);
|
|
83
|
+
if (cached)
|
|
84
|
+
return cached;
|
|
85
|
+
const agent = createTransportAgent(target, proxyUrl);
|
|
86
|
+
if (proxyAgentCache.size >= PROXY_AGENT_CACHE_MAX) {
|
|
87
|
+
// 简单淘汰:清掉最早插入的一个,避免无界增长。
|
|
88
|
+
const oldest = proxyAgentCache.keys().next().value;
|
|
89
|
+
if (oldest !== undefined)
|
|
90
|
+
proxyAgentCache.delete(oldest);
|
|
91
|
+
}
|
|
92
|
+
proxyAgentCache.set(key, agent);
|
|
93
|
+
return agent;
|
|
94
|
+
}
|
|
95
|
+
/** Test seam: drop cached agents. */
|
|
96
|
+
export function clearProxyAgentCache() {
|
|
97
|
+
for (const agent of proxyAgentCache.values()) {
|
|
98
|
+
agent.destroy?.();
|
|
99
|
+
}
|
|
100
|
+
proxyAgentCache.clear();
|
|
101
|
+
}
|
|
69
102
|
export function createTransportAgent(_target, proxyUrl) {
|
|
70
103
|
let proxy;
|
|
71
104
|
try {
|
|
@@ -78,10 +111,10 @@ export function createTransportAgent(_target, proxyUrl) {
|
|
|
78
111
|
throw new TransportProtocolError(`不支持的 proxy protocol: ${proxy.protocol}`);
|
|
79
112
|
}
|
|
80
113
|
if (proxy.protocol === "http:" || proxy.protocol === "https:") {
|
|
81
|
-
return new HttpsProxyAgent(proxy);
|
|
114
|
+
return new HttpsProxyAgent(proxy, { keepAlive: true });
|
|
82
115
|
}
|
|
83
116
|
// socks5 resolves DNS locally; socks5h/socks4a defer to the proxy.
|
|
84
|
-
return new SocksProxyAgent(proxy);
|
|
117
|
+
return new SocksProxyAgent(proxy, { keepAlive: true });
|
|
85
118
|
}
|
|
86
119
|
function sanitizeHeaders(headers) {
|
|
87
120
|
const out = {};
|
|
@@ -274,7 +307,8 @@ function parseTarget(url) {
|
|
|
274
307
|
}
|
|
275
308
|
function performRequest(target, options) {
|
|
276
309
|
const proxyUrl = selectProxyUrl(target, options.proxy);
|
|
277
|
-
|
|
310
|
+
// 复用同一个代理 Agent(keep-alive);无代理时走 Node 全局 agent,本身已 keep-alive。
|
|
311
|
+
const agent = proxyUrl ? getProxyAgent(target, proxyUrl) : undefined;
|
|
278
312
|
const isHttps = target.protocol === "https:";
|
|
279
313
|
const requestImpl = isHttps ? httpsRequest : httpRequest;
|
|
280
314
|
const connectTimeoutMs = options.connectTimeoutMs ?? DEFAULT_CONNECT_TIMEOUT_MS;
|
|
@@ -299,11 +333,8 @@ function performRequest(target, options) {
|
|
|
299
333
|
timers.clear();
|
|
300
334
|
},
|
|
301
335
|
};
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
agent.destroy();
|
|
305
|
-
}
|
|
306
|
-
};
|
|
336
|
+
// Agent 是共享的,单个请求失败不能销毁它,否则会连带打断其他在途请求。
|
|
337
|
+
const cleanupAgent = () => { };
|
|
307
338
|
const fail = (err) => {
|
|
308
339
|
if (settled)
|
|
309
340
|
return;
|
package/dist/cli.js
CHANGED
|
@@ -14,7 +14,10 @@ export function createProgram() {
|
|
|
14
14
|
.name("llms")
|
|
15
15
|
.description("为 Claude Code / Codex / OpenCode 切换供应商、模型与上游代理")
|
|
16
16
|
.version(getVersion())
|
|
17
|
-
|
|
17
|
+
// 注意:不要在根命令上声明 --json。commander 会把它当作根命令的选项,
|
|
18
|
+
// 从而吞掉所有子命令自己的 --json(子命令 opts.json 恒为 undefined)。
|
|
19
|
+
// JSON 输出一律由各子命令自行声明。
|
|
20
|
+
.showSuggestionAfterError();
|
|
18
21
|
program
|
|
19
22
|
.command("path")
|
|
20
23
|
.description("显示 llm-switch 本地配置目录")
|
|
@@ -42,11 +45,20 @@ export async function run(argv = process.argv) {
|
|
|
42
45
|
await program.parseAsync(argv);
|
|
43
46
|
}
|
|
44
47
|
catch (err) {
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
"
|
|
48
|
-
|
|
49
|
-
|
|
48
|
+
const code = err && typeof err === "object" && "code" in err
|
|
49
|
+
? String(err.code)
|
|
50
|
+
: "";
|
|
51
|
+
if (code === "commander.helpDisplayed" ||
|
|
52
|
+
code === "commander.help" ||
|
|
53
|
+
code === "commander.version") {
|
|
54
|
+
return;
|
|
55
|
+
}
|
|
56
|
+
// commander 已经把用法错误写到 stderr 了,不要再加「错误:」重复打印一遍。
|
|
57
|
+
if (code.startsWith("commander.")) {
|
|
58
|
+
process.exitCode =
|
|
59
|
+
typeof err.exitCode === "number"
|
|
60
|
+
? err.exitCode
|
|
61
|
+
: 1;
|
|
50
62
|
return;
|
|
51
63
|
}
|
|
52
64
|
const message = err instanceof Error ? err.message : String(err);
|
|
@@ -1,9 +1,9 @@
|
|
|
1
|
-
import { ensureBridgeForProfile, isBridgeAlive, isPidRunning, readPid, runBridgeForeground, startBridgeDaemon, stopBridge, } from "../bridge/manager.js";
|
|
1
|
+
import { bridgeToolForCliTool, ensureBridgeForProfile, isBridgeAlive, isPidRunning, profileNeedsBridge, readPid, runBridgeForeground, startBridgeDaemon, stopBridge, } from "../bridge/manager.js";
|
|
2
2
|
import { bridgeBaseUrl, bridgeRootUrl, readBridgeState, } from "../bridge/state.js";
|
|
3
3
|
import { parseBridgePort } from "../bridge/runtime.js";
|
|
4
4
|
import { DEFAULT_BRIDGE_HOST, DEFAULT_BRIDGE_PORT } from "../bridge/types.js";
|
|
5
5
|
import { getActiveProfile, resolveProfileOrThrow } from "../store/profiles.js";
|
|
6
|
-
import { isTool } from "../types.js";
|
|
6
|
+
import { TOOLS, isTool } from "../types.js";
|
|
7
7
|
export function registerBridgeCommand(program) {
|
|
8
8
|
const bridge = program
|
|
9
9
|
.command("bridge")
|
|
@@ -81,7 +81,7 @@ export function registerBridgeCommand(program) {
|
|
|
81
81
|
}
|
|
82
82
|
console.log(`状态:${alive ? "运行中" : "未运行"}`);
|
|
83
83
|
console.log(`根地址:${data.rootUrl}`);
|
|
84
|
-
console.log(`
|
|
84
|
+
console.log(`OpenAI 兼容 base:${data.codexBaseUrl}`);
|
|
85
85
|
console.log(`PID:${pid ?? "-"}`);
|
|
86
86
|
for (const tool of ["codex", "claude", "opencode"]) {
|
|
87
87
|
const u = data.upstreams[tool];
|
|
@@ -96,19 +96,26 @@ export function registerBridgeCommand(program) {
|
|
|
96
96
|
bridge
|
|
97
97
|
.command("reload")
|
|
98
98
|
.description("用当前启用的 profile 刷新某一侧上游(不重启进程)")
|
|
99
|
-
.argument("[tool]",
|
|
99
|
+
.argument("[tool]", `claude | codex | opencode(默认 codex)`, "codex")
|
|
100
100
|
.option("--profile <name>", "指定 profile")
|
|
101
101
|
.action(async (toolArg, opts) => {
|
|
102
102
|
const toolName = toolArg || "codex";
|
|
103
|
-
if (!isTool(toolName)
|
|
104
|
-
throw new Error("
|
|
103
|
+
if (!isTool(toolName)) {
|
|
104
|
+
throw new Error(`未知工具「${toolName}」。可选:${TOOLS.join("、")}`);
|
|
105
|
+
}
|
|
106
|
+
// 三个工具都可能走 bridge(openai-chat 上游),reload 必须全部支持。
|
|
107
|
+
const tool = bridgeToolForCliTool(toolName);
|
|
108
|
+
if (!tool) {
|
|
109
|
+
throw new Error(`${toolName} 不使用本地 bridge`);
|
|
105
110
|
}
|
|
106
|
-
const tool = toolName;
|
|
107
111
|
const profile = opts.profile
|
|
108
112
|
? resolveProfileOrThrow(tool, opts.profile)
|
|
109
113
|
: getActiveProfile(tool);
|
|
110
114
|
if (!profile) {
|
|
111
|
-
throw new Error(
|
|
115
|
+
throw new Error(`没有已启用的 ${tool} profile。可用:llms ${tool} use,或 llms bridge reload ${tool} --profile <name>`);
|
|
116
|
+
}
|
|
117
|
+
if (!profileNeedsBridge(profile)) {
|
|
118
|
+
throw new Error(`${tool}/${profile.name} 是 ${profile.apiFormat} 原生直连,不经过 bridge,无需 reload。`);
|
|
112
119
|
}
|
|
113
120
|
const connection = await ensureBridgeForProfile(profile, tool);
|
|
114
121
|
console.log(`已刷新 ${tool} 上游 ${profile.name} → bridge ${connection.baseUrl}`);
|