mslxdff 0.1.83 → 0.1.84
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/package.json
CHANGED
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
import { clineHeaders } from "../providers/cline/headers.js";
|
|
2
|
+
import { computeMetrics } from "../metrics.js";
|
|
3
|
+
|
|
4
|
+
export async function clineBenchOne({ baseUrl, model, accessToken, prompt, maxTokens, timeoutMs, fetchImpl }) {
|
|
5
|
+
const controller = new AbortController();
|
|
6
|
+
const timer = setTimeout(() => controller.abort(new Error(`timeout ${timeoutMs}ms`)), timeoutMs);
|
|
7
|
+
const t0 = performance.now();
|
|
8
|
+
let ttfbMs = null;
|
|
9
|
+
let content = "";
|
|
10
|
+
try {
|
|
11
|
+
const res = await fetchImpl(`${baseUrl}/api/v1/chat/completions`, {
|
|
12
|
+
method: "POST",
|
|
13
|
+
headers: { ...clineHeaders(`sess_bench_${Date.now()}`, accessToken), Accept: "text/event-stream" },
|
|
14
|
+
body: JSON.stringify({ model, messages: [{ role: "user", content: prompt }], stream: true, max_tokens: maxTokens, session_id: `sess_bench_${Date.now()}`, reasoning_effort: "high" }),
|
|
15
|
+
signal: controller.signal,
|
|
16
|
+
});
|
|
17
|
+
if (res instanceof Error) throw res;
|
|
18
|
+
if (!res.ok) {
|
|
19
|
+
let txt = "";
|
|
20
|
+
try { txt = await res.text(); } catch {}
|
|
21
|
+
const label = res.status === 401 ? "鉴权失败" : res.status === 429 ? "限流" : res.status >= 500 ? `上游错误 ${res.status}` : `HTTP ${res.status}`;
|
|
22
|
+
return { id: model, ok: false, status: res.status, label, error: txt.slice(0, 300), ttfbMs, totalMs: Math.round(performance.now() - t0), tps: null, charsPerSec: null, tokens: null };
|
|
23
|
+
}
|
|
24
|
+
const reader = res.body.getReader();
|
|
25
|
+
const decoder = new TextDecoder();
|
|
26
|
+
let buf = "";
|
|
27
|
+
for (;;) {
|
|
28
|
+
const { done, value } = await reader.read();
|
|
29
|
+
if (done) break;
|
|
30
|
+
if (ttfbMs === null) ttfbMs = Math.round(performance.now() - t0);
|
|
31
|
+
buf += decoder.decode(value, { stream: true });
|
|
32
|
+
let idx;
|
|
33
|
+
while ((idx = buf.indexOf("\n")) >= 0) {
|
|
34
|
+
const line = buf.slice(0, idx);
|
|
35
|
+
buf = buf.slice(idx + 1);
|
|
36
|
+
if (!line.startsWith("data:")) continue;
|
|
37
|
+
const payload = line.slice(5).trim();
|
|
38
|
+
if (!payload || payload === "[DONE]") continue;
|
|
39
|
+
try {
|
|
40
|
+
const j = JSON.parse(payload);
|
|
41
|
+
const c = j?.choices?.[0]?.delta?.content || j?.choices?.[0]?.message?.content || "";
|
|
42
|
+
if (typeof c === "string") content += c;
|
|
43
|
+
} catch {}
|
|
44
|
+
}
|
|
45
|
+
}
|
|
46
|
+
const totalMs = Math.round(performance.now() - t0);
|
|
47
|
+
const chars = content.length;
|
|
48
|
+
const { tps, charsPerSec } = computeMetrics({ ttfbMs, totalMs, promptTokens: null, completionTokens: null, chars });
|
|
49
|
+
return { id: model, ok: true, status: 200, label: "成功", ttfbMs, totalMs, tps, charsPerSec, tokens: null, chars };
|
|
50
|
+
} catch (e) {
|
|
51
|
+
const msg = e?.message || String(e);
|
|
52
|
+
return { id: model, ok: false, label: /timeout|abort/i.test(msg) ? "超时" : "网络错误", error: msg.slice(0, 300), ttfbMs, totalMs: Math.round(performance.now() - t0), tps: null, charsPerSec: null, tokens: null };
|
|
53
|
+
} finally { clearTimeout(timer); }
|
|
54
|
+
}
|
package/src/bench/via.js
CHANGED
|
@@ -72,6 +72,8 @@ export async function orchestrateVia({
|
|
|
72
72
|
token,
|
|
73
73
|
timeoutMs = 30000,
|
|
74
74
|
clock = Date.now,
|
|
75
|
+
onProgress = null,
|
|
76
|
+
delayMs = 0,
|
|
75
77
|
} = {}) {
|
|
76
78
|
const filtered = includeOpencode ? models : models.filter((m) => {
|
|
77
79
|
const s = typeof m === "string" ? m : (m.id || m.model || m.provider || "");
|
|
@@ -87,17 +89,23 @@ export async function orchestrateVia({
|
|
|
87
89
|
return { provider, model: id, id };
|
|
88
90
|
});
|
|
89
91
|
const results = [];
|
|
90
|
-
|
|
92
|
+
const total = normModels.length;
|
|
93
|
+
for (let idx = 0; idx < normModels.length; idx++) {
|
|
94
|
+
const entry = normModels[idx];
|
|
91
95
|
const provider = entry.provider;
|
|
92
96
|
const model = entry.model;
|
|
93
|
-
|
|
97
|
+
const seq = idx + 1;
|
|
98
|
+
// direct —— 串行,完成后立刻回调供实时进度
|
|
94
99
|
let direct = null;
|
|
95
100
|
if (directRunner) {
|
|
96
101
|
try { direct = await directRunner({ provider, model, timeoutMs, clock }); } catch (e) { direct = { ok: false, label: "网络错误", error: String(e), ttfbMs: null, totalMs: 0 }; }
|
|
97
102
|
} else {
|
|
98
103
|
direct = { ok: false, label: "未配置 directRunner", error: "missing directRunner", ttfbMs: null, totalMs: 0 };
|
|
99
104
|
}
|
|
105
|
+
if (typeof onProgress === "function") try { await onProgress({ phase: "direct", provider, model, seq, total, result: direct }); } catch {}
|
|
106
|
+
if (delayMs) await new Promise((r) => setTimeout(r, delayMs));
|
|
100
107
|
const via = {};
|
|
108
|
+
// 串行逐个 peer,避免 A/B/C 同时打同一上游并发限流
|
|
101
109
|
for (const peer of peers) {
|
|
102
110
|
const raw = String(peer.name || peer.id || peer.url || "peer");
|
|
103
111
|
let peerId = raw;
|
|
@@ -106,12 +114,16 @@ export async function orchestrateVia({
|
|
|
106
114
|
try { const u = new URL(raw); const host = u.hostname; const port = u.port ? `:${u.port}` : ""; if (host) peerId = `${host}${port}`; else peerId = raw.slice(-16); } catch { peerId = raw.slice(-16); }
|
|
107
115
|
}
|
|
108
116
|
const peerToken = peer.token || token || "";
|
|
117
|
+
let r;
|
|
109
118
|
try {
|
|
110
|
-
|
|
119
|
+
r = await viaProbeFn({ peerUrl: peer.url, token: peerToken, providerId: provider, model, prompt: "hi", maxTokens: 5, timeoutMs, clock });
|
|
111
120
|
via[peerId] = r;
|
|
112
121
|
} catch (e) {
|
|
113
|
-
|
|
122
|
+
r = { ok: false, label: "网络错误", error: String(e?.message || e), ttfbMs: null, totalMs: 0 };
|
|
123
|
+
via[peerId] = r;
|
|
114
124
|
}
|
|
125
|
+
if (typeof onProgress === "function") try { await onProgress({ phase: "via", provider, model, seq, total, peerId, result: r }); } catch {}
|
|
126
|
+
if (delayMs) await new Promise((rr) => setTimeout(rr, delayMs));
|
|
115
127
|
}
|
|
116
128
|
// best
|
|
117
129
|
let best = "direct";
|
|
@@ -0,0 +1,160 @@
|
|
|
1
|
+
import { defaultChatPath } from "../../../state/provider-config.js";
|
|
2
|
+
import { isRefreshToken, clineHeaders } from "../../../providers/cline/headers.js";
|
|
3
|
+
import { refreshTokenForBase } from "../../../providers/cline/auth.js";
|
|
4
|
+
import { runOne } from "../../../bench/runner.js";
|
|
5
|
+
import { formatViaReport } from "../../../bench/report.js";
|
|
6
|
+
import { clineBenchOne } from "../../../bench/cline-bench.js";
|
|
7
|
+
|
|
8
|
+
export function buildHeadersForProvider(providerId, apiKey, auth) {
|
|
9
|
+
const h = {};
|
|
10
|
+
if (String(providerId).toLowerCase() === "workbuddy") {
|
|
11
|
+
h["Content-Type"] = "application/json";
|
|
12
|
+
h["Accept"] = "text/event-stream";
|
|
13
|
+
h["User-Agent"] = "CLI/2.115.0 WorkBuddy/2.115.0";
|
|
14
|
+
h["Origin"] = "https://www.codebuddy.cn";
|
|
15
|
+
h["Referer"] = "https://www.codebuddy.cn/";
|
|
16
|
+
h["X-Product"] = "SaaS";
|
|
17
|
+
if (apiKey) h["Authorization"] = `Bearer ${apiKey}`;
|
|
18
|
+
if (auth?.uid) h["X-User-Id"] = auth.uid;
|
|
19
|
+
h["X-Domain"] = auth?.domain || "www.codebuddy.cn";
|
|
20
|
+
if (auth?.enterpriseId) { h["X-Enterprise-Id"] = auth.enterpriseId; h["X-Tenant-Id"] = auth.enterpriseId; }
|
|
21
|
+
return h;
|
|
22
|
+
}
|
|
23
|
+
if (apiKey) h["Authorization"] = `Bearer ${apiKey}`;
|
|
24
|
+
return h;
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
export async function handleVia({ providerId, opts, fetchImpl, loadConfigs, loadKeys, loadAllowed, loadBaseUrl }) {
|
|
28
|
+
const { getOnlinePeers, orchestrateVia, resolveIncludeOpencode } = await import("../../../bench/via.js");
|
|
29
|
+
const peers = await getOnlinePeers();
|
|
30
|
+
if (!peers.length) {
|
|
31
|
+
const msg = "未加入组或无在线 peer,--via 无意义。先 mslxdff -group list / -addtogroup";
|
|
32
|
+
if (opts.json) console.log(JSON.stringify({ meta: { at: new Date().toISOString(), samples: opts.samples, timeout: opts.timeoutMs, includeOpencode: false, peers: [], opencodeSkipped: true }, results: [], advice: msg }, null, 2));
|
|
33
|
+
else console.log(msg);
|
|
34
|
+
process.exit(0);
|
|
35
|
+
}
|
|
36
|
+
const isTTY = Boolean(process.stdin.isTTY && process.stdout.isTTY);
|
|
37
|
+
let includeOpencode = opts.includeOpencode;
|
|
38
|
+
if (includeOpencode) {
|
|
39
|
+
const confirmFn = async () => {
|
|
40
|
+
const readline = await import("node:readline");
|
|
41
|
+
return new Promise((res) => {
|
|
42
|
+
const rl = readline.createInterface({ input: process.stdin, output: process.stdout });
|
|
43
|
+
rl.question("将消耗 B/C/D 的 opencode 额度,确认测 opencode via?y/N ", (ans) => { rl.close(); res(ans.trim().toLowerCase() === "y"); });
|
|
44
|
+
});
|
|
45
|
+
};
|
|
46
|
+
const log = (s) => (opts.json ? console.error(s) : console.log(s));
|
|
47
|
+
includeOpencode = await resolveIncludeOpencode({ includeOpencode, isTTY, confirmFn, log });
|
|
48
|
+
}
|
|
49
|
+
const { loadToken } = await import("../../../state.js");
|
|
50
|
+
let token = "";
|
|
51
|
+
try { token = (await loadToken()).token || ""; } catch {}
|
|
52
|
+
let targetIds = [];
|
|
53
|
+
const isAll = providerId === "bench" || providerId === "all";
|
|
54
|
+
if (isAll) {
|
|
55
|
+
const configs = loadConfigs();
|
|
56
|
+
const ids = new Set(Object.keys(configs));
|
|
57
|
+
ids.add("opencode"); ids.add("openrouter");
|
|
58
|
+
for (const pid of [...ids]) {
|
|
59
|
+
const allowed = loadAllowed(pid) || [];
|
|
60
|
+
if (allowed.length) targetIds.push(pid);
|
|
61
|
+
}
|
|
62
|
+
try {
|
|
63
|
+
const { readFileSync } = await import("node:fs");
|
|
64
|
+
const { defaultStateFile } = await import("../../../state.js");
|
|
65
|
+
const raw = JSON.parse(readFileSync(defaultStateFile(), "utf8"));
|
|
66
|
+
for (const k of Object.keys(raw.providerConfigs || {})) if (!targetIds.includes(k) && (loadAllowed(k) || []).length) targetIds.push(k);
|
|
67
|
+
for (const k of Object.keys(raw.providerKeys || {})) if (!targetIds.includes(k) && (loadAllowed(k) || []).length) targetIds.push(k);
|
|
68
|
+
} catch {}
|
|
69
|
+
if (!targetIds.length) targetIds = ["openrouter", "workbuddy", "clinebot"].filter((p) => (loadAllowed(p) || []).length);
|
|
70
|
+
} else {
|
|
71
|
+
targetIds = [providerId];
|
|
72
|
+
}
|
|
73
|
+
const allResults = [];
|
|
74
|
+
const viaLog = (s) => (opts.json ? console.error(s) : console.log(s));
|
|
75
|
+
viaLog(`bench-via: direct vs via peers (samples=${opts.samples}, timeout=${opts.timeoutMs}ms, ${includeOpencode ? "opencode=included" : "opencode=skipped"}) peers=${peers.map((p) => p.id).join(",")} 模式: 串行(直连→逐 peer)`);
|
|
76
|
+
const delayMs = Number(process.env.MSLXDFF_BENCH_DELAY_MS || 120) || 0;
|
|
77
|
+
for (const pid of targetIds) {
|
|
78
|
+
const cfg = (loadConfigs()[pid] || {});
|
|
79
|
+
const keys = loadKeys(pid) || [];
|
|
80
|
+
const allowed = loadAllowed(pid) || [];
|
|
81
|
+
const baseUrl = (loadBaseUrl(pid) || cfg.baseUrl || "").trim();
|
|
82
|
+
if (!allowed.length) { viaLog(`provider ${pid}: 无勾选模型,跳过`); continue; }
|
|
83
|
+
if (!baseUrl && pid !== "opencode") { viaLog(`provider ${pid}: missing baseUrl 跳过`); continue; }
|
|
84
|
+
if (!keys.length && pid !== "opencode") { viaLog(`provider ${pid}: 未配置 Key 跳过`); continue; }
|
|
85
|
+
const chatPath = cfg.chatPath || defaultChatPath(pid);
|
|
86
|
+
let auths = [];
|
|
87
|
+
try { const m = await import("../../../state.js"); auths = m.loadProviderAuths ? m.loadProviderAuths(pid) : []; } catch {}
|
|
88
|
+
const models = allowed.map((id) => ({ provider: pid, model: String(id), id: String(id) }));
|
|
89
|
+
const directRunner = async ({ provider, model }) => {
|
|
90
|
+
const p = provider || pid;
|
|
91
|
+
const idx = models.findIndex((x) => x.model === model);
|
|
92
|
+
const kIdx = idx >= 0 ? idx % (keys.length || 1) : 0;
|
|
93
|
+
const aIdx = Math.min(kIdx, Math.max(auths.length - 1, 0));
|
|
94
|
+
const key = keys[kIdx] || keys[0] || "";
|
|
95
|
+
const auth = auths[aIdx] || auths[0] || null;
|
|
96
|
+
const cKeys = keys.filter((k) => isRefreshToken(k, p));
|
|
97
|
+
if (cKeys.length) {
|
|
98
|
+
const normBase = String(baseUrl).replace(/\/+$/, "");
|
|
99
|
+
const chatBase = normBase.endsWith("/api/v1") ? normBase.slice(0, -7) : normBase;
|
|
100
|
+
const rt = cKeys[kIdx % cKeys.length];
|
|
101
|
+
const at = await refreshTokenForBase({ refreshToken: rt, baseUrl: normBase, fetchImpl });
|
|
102
|
+
if (!at) return { ok: false, label: "鉴权失败", error: "refresh failed", ttfbMs: null, totalMs: 0 };
|
|
103
|
+
return clineBenchOne({ baseUrl: chatBase, model, accessToken: at, prompt: "hi", maxTokens: 5, timeoutMs: opts.timeoutMs, fetchImpl });
|
|
104
|
+
}
|
|
105
|
+
const headers = buildHeadersForProvider(p, key, auth);
|
|
106
|
+
return runOne({ baseUrl: p === "opencode" ? (process.env.UPSTREAM_BASE_URL || "https://opencode.ai") : baseUrl, chatPath, model, providerId: p, apiKey: key || "public", headers, prompt: "hi", maxTokens: 5, timeoutMs: opts.timeoutMs, fetchImpl });
|
|
107
|
+
};
|
|
108
|
+
const { viaProbe } = await import("../../../bench/via-probe.js");
|
|
109
|
+
const viaProbeFn = async (args) => {
|
|
110
|
+
const { peerUrl, providerId, model } = args;
|
|
111
|
+
const p = providerId || pid;
|
|
112
|
+
const idx = models.findIndex((x) => x.model === model);
|
|
113
|
+
const kIdx = idx >= 0 ? idx % (keys.length || 1) : 0;
|
|
114
|
+
const aIdx = Math.min(kIdx, Math.max(auths.length - 1, 0));
|
|
115
|
+
const key = keys[kIdx] || keys[0] || "";
|
|
116
|
+
const auth = auths[aIdx] || auths[0] || null;
|
|
117
|
+
const cKeys = keys.filter((k) => isRefreshToken(k, p));
|
|
118
|
+
if (cKeys.length) {
|
|
119
|
+
const normBase = String(baseUrl).replace(/\/+$/, "");
|
|
120
|
+
const chatBase = normBase.endsWith("/api/v1") ? normBase.slice(0, -7) : normBase;
|
|
121
|
+
const rt = cKeys[0];
|
|
122
|
+
const at = await refreshTokenForBase({ refreshToken: rt, baseUrl: normBase, fetchImpl });
|
|
123
|
+
if (!at) return { ok: false, label: "鉴权失败", error: "refresh failed", ttfbMs: null, totalMs: 0 };
|
|
124
|
+
const targetUrl = `${chatBase}/api/v1/chat/completions`;
|
|
125
|
+
const headers = clineHeaders(`sess_bench_via_${Date.now()}`, at);
|
|
126
|
+
const rawModel = String(model).startsWith(`${p}/`) ? String(model).slice(p.length + 1) : String(model);
|
|
127
|
+
const body = { model: rawModel, messages: [{ role: "user", content: "hi" }], stream: true, max_tokens: 5, session_id: `sess_bench_via_${Date.now()}`, reasoning_effort: "high" };
|
|
128
|
+
const peer = peers.find((pe) => pe.url === peerUrl || (pe.name || pe.id) === peerUrl);
|
|
129
|
+
const peerToken = peer?.token || token;
|
|
130
|
+
return viaProbe({ peerUrl, token: peerToken, relayTarget: targetUrl, relayHeaders: headers, relayBody: body, timeoutMs: opts.timeoutMs, fetchImpl });
|
|
131
|
+
}
|
|
132
|
+
const rawModel = String(model).startsWith(`${p}/`) ? String(model).slice(p.length + 1) : String(model);
|
|
133
|
+
const targetUrl = p === "opencode" ? `${process.env.UPSTREAM_BASE_URL || "https://opencode.ai"}/zen/v1/chat/completions` : `${String(baseUrl).replace(/\/+$/, "")}${chatPath}`;
|
|
134
|
+
const headers = buildHeadersForProvider(p, key, auth);
|
|
135
|
+
const body = { model: rawModel, stream: false, messages: [{ role: "user", content: "hi" }], max_tokens: 5 };
|
|
136
|
+
const peer = peers.find((pe) => pe.url === peerUrl || (pe.name || pe.id) === peerUrl);
|
|
137
|
+
const peerToken = peer?.token || token;
|
|
138
|
+
return viaProbe({ peerUrl, token: peerToken, relayTarget: targetUrl, relayHeaders: headers, relayBody: body, timeoutMs: opts.timeoutMs, fetchImpl });
|
|
139
|
+
};
|
|
140
|
+
if (!opts.json) viaLog(`\n[${pid}] 共 ${models.length} 个模型,串行测试中...`);
|
|
141
|
+
else viaLog(`[${pid}] ${models.length} models sequential`);
|
|
142
|
+
const onProgress = async ({ phase, provider, model, seq, total, peerId, result }) => {
|
|
143
|
+
const short = String(model).length > 28 ? String(model).slice(0, 28) : String(model);
|
|
144
|
+
const ms = result?.ttfbMs != null ? `${result.ttfbMs}ms` : result?.totalMs != null ? `${result.totalMs}ms` : "—";
|
|
145
|
+
const okTag = result?.ok ? "成功" : (result?.label || "失败");
|
|
146
|
+
const extra = result?.ok ? "" : result?.error ? ` (${String(result.error).slice(0, 40)})` : "";
|
|
147
|
+
if (phase === "direct") viaLog(` [${seq}/${total}] ${provider}/${short} 直连 ${ms} ${okTag}${extra} 完成`);
|
|
148
|
+
else viaLog(` ↳ via ${peerId} ${ms} ${okTag}${extra}`);
|
|
149
|
+
};
|
|
150
|
+
const { orchestrateVia } = await import("../../../bench/via.js");
|
|
151
|
+
const part = await orchestrateVia({ models, peers, directRunner, viaProbeFn, includeOpencode, token, timeoutMs: opts.timeoutMs, onProgress, delayMs });
|
|
152
|
+
allResults.push(...part);
|
|
153
|
+
if (!opts.json) viaLog(` ${pid}: ${part.length} 模型完成 ✓`);
|
|
154
|
+
}
|
|
155
|
+
const meta = { at: new Date().toISOString(), samples: opts.samples, timeout: opts.timeoutMs, includeOpencode, peers: peers.map((p) => p.id), opencodeSkipped: !includeOpencode };
|
|
156
|
+
const report = formatViaReport(allResults, { peers, meta, json: opts.json });
|
|
157
|
+
if (opts.json) console.log(report.text);
|
|
158
|
+
else console.log("\n" + report.text);
|
|
159
|
+
process.exit(0);
|
|
160
|
+
}
|
|
@@ -1,62 +1,11 @@
|
|
|
1
1
|
import { probeModels } from "../../../bench/probe.js";
|
|
2
2
|
import { runOne } from "../../../bench/runner.js";
|
|
3
|
-
import { formatReport
|
|
3
|
+
import { formatReport } from "../../../bench/report.js";
|
|
4
4
|
import { defaultModelsPath, defaultChatPath } from "../../../state/provider-config.js";
|
|
5
|
-
import { isRefreshToken
|
|
5
|
+
import { isRefreshToken } from "../../../providers/cline/headers.js";
|
|
6
6
|
import { refreshTokenForBase } from "../../../providers/cline/auth.js";
|
|
7
|
-
import {
|
|
8
|
-
|
|
9
|
-
async function clineBenchOne({ baseUrl, model, accessToken, prompt, maxTokens, timeoutMs, fetchImpl }) {
|
|
10
|
-
const controller = new AbortController();
|
|
11
|
-
const timer = setTimeout(() => controller.abort(new Error(`timeout ${timeoutMs}ms`)), timeoutMs);
|
|
12
|
-
const t0 = performance.now();
|
|
13
|
-
let ttfbMs = null;
|
|
14
|
-
let content = "";
|
|
15
|
-
try {
|
|
16
|
-
const res = await fetchImpl(`${baseUrl}/api/v1/chat/completions`, {
|
|
17
|
-
method: "POST",
|
|
18
|
-
headers: { ...clineHeaders(`sess_bench_${Date.now()}`, accessToken), Accept: "text/event-stream" },
|
|
19
|
-
body: JSON.stringify({ model, messages: [{ role: "user", content: prompt }], stream: true, max_tokens: maxTokens, session_id: `sess_bench_${Date.now()}`, reasoning_effort: "high" }),
|
|
20
|
-
signal: controller.signal,
|
|
21
|
-
});
|
|
22
|
-
if (res instanceof Error) throw res;
|
|
23
|
-
if (!res.ok) {
|
|
24
|
-
let txt = "";
|
|
25
|
-
try { txt = await res.text(); } catch {}
|
|
26
|
-
const label = res.status === 401 ? "鉴权失败" : res.status === 429 ? "限流" : res.status >= 500 ? `上游错误 ${res.status}` : `HTTP ${res.status}`;
|
|
27
|
-
return { id: model, ok: false, status: res.status, label, error: txt.slice(0, 300), ttfbMs, totalMs: Math.round(performance.now() - t0), tps: null, charsPerSec: null, tokens: null };
|
|
28
|
-
}
|
|
29
|
-
const reader = res.body.getReader();
|
|
30
|
-
const decoder = new TextDecoder();
|
|
31
|
-
let buf = "";
|
|
32
|
-
for (;;) {
|
|
33
|
-
const { done, value } = await reader.read();
|
|
34
|
-
if (done) break;
|
|
35
|
-
if (ttfbMs === null) ttfbMs = Math.round(performance.now() - t0);
|
|
36
|
-
buf += decoder.decode(value, { stream: true });
|
|
37
|
-
let idx;
|
|
38
|
-
while ((idx = buf.indexOf("\n")) >= 0) {
|
|
39
|
-
const line = buf.slice(0, idx);
|
|
40
|
-
buf = buf.slice(idx + 1);
|
|
41
|
-
if (!line.startsWith("data:")) continue;
|
|
42
|
-
const payload = line.slice(5).trim();
|
|
43
|
-
if (!payload || payload === "[DONE]") continue;
|
|
44
|
-
try {
|
|
45
|
-
const j = JSON.parse(payload);
|
|
46
|
-
const c = j?.choices?.[0]?.delta?.content || j?.choices?.[0]?.message?.content || "";
|
|
47
|
-
if (typeof c === "string") content += c;
|
|
48
|
-
} catch {}
|
|
49
|
-
}
|
|
50
|
-
}
|
|
51
|
-
const totalMs = Math.round(performance.now() - t0);
|
|
52
|
-
const chars = content.length;
|
|
53
|
-
const { tps, charsPerSec } = computeMetrics({ ttfbMs, totalMs, promptTokens: null, completionTokens: null, chars });
|
|
54
|
-
return { id: model, ok: true, status: 200, label: "成功", ttfbMs, totalMs, tps, charsPerSec, tokens: null, chars };
|
|
55
|
-
} catch (e) {
|
|
56
|
-
const msg = e?.message || String(e);
|
|
57
|
-
return { id: model, ok: false, label: /timeout|abort/i.test(msg) ? "超时" : "网络错误", error: msg.slice(0, 300), ttfbMs, totalMs: Math.round(performance.now() - t0), tps: null, charsPerSec: null, tokens: null };
|
|
58
|
-
} finally { clearTimeout(timer); }
|
|
59
|
-
}
|
|
7
|
+
import { clineBenchOne } from "../../../bench/cline-bench.js";
|
|
8
|
+
import { buildHeadersForProvider, handleVia } from "./bench-via.js";
|
|
60
9
|
|
|
61
10
|
function parseBenchArgs(rest) {
|
|
62
11
|
const opts = { json: false, prompt: "hi", maxTokens: 32, timeoutMs: 30000, via: false, includeOpencode: false, samples: 1 };
|
|
@@ -77,160 +26,6 @@ function parseBenchArgs(rest) {
|
|
|
77
26
|
return opts;
|
|
78
27
|
}
|
|
79
28
|
|
|
80
|
-
function buildHeadersForProvider(providerId, apiKey, auth) {
|
|
81
|
-
const h = {};
|
|
82
|
-
if (String(providerId).toLowerCase() === "workbuddy") {
|
|
83
|
-
h["Content-Type"] = "application/json";
|
|
84
|
-
h["Accept"] = "text/event-stream";
|
|
85
|
-
h["User-Agent"] = "CLI/2.115.0 WorkBuddy/2.115.0";
|
|
86
|
-
h["Origin"] = "https://www.codebuddy.cn";
|
|
87
|
-
h["Referer"] = "https://www.codebuddy.cn/";
|
|
88
|
-
h["X-Product"] = "SaaS";
|
|
89
|
-
if (apiKey) h["Authorization"] = `Bearer ${apiKey}`;
|
|
90
|
-
if (auth?.uid) h["X-User-Id"] = auth.uid;
|
|
91
|
-
h["X-Domain"] = auth?.domain || "www.codebuddy.cn";
|
|
92
|
-
if (auth?.enterpriseId) { h["X-Enterprise-Id"] = auth.enterpriseId; h["X-Tenant-Id"] = auth.enterpriseId; }
|
|
93
|
-
return h;
|
|
94
|
-
}
|
|
95
|
-
if (apiKey) h["Authorization"] = `Bearer ${apiKey}`;
|
|
96
|
-
return h;
|
|
97
|
-
}
|
|
98
|
-
|
|
99
|
-
async function handleVia({ providerId, opts, fetchImpl, loadConfigs, loadKeys, loadAllowed, loadBaseUrl }) {
|
|
100
|
-
const { getOnlinePeers, orchestrateVia, resolveIncludeOpencode } = await import("../../../bench/via.js");
|
|
101
|
-
const peers = await getOnlinePeers();
|
|
102
|
-
if (!peers.length) {
|
|
103
|
-
const msg = "未加入组或无在线 peer,--via 无意义。先 mslxdff -group list / -addtogroup";
|
|
104
|
-
if (opts.json) console.log(JSON.stringify({ meta: { at: new Date().toISOString(), samples: opts.samples, timeout: opts.timeoutMs, includeOpencode: false, peers: [], opencodeSkipped: true }, results: [], advice: msg }, null, 2));
|
|
105
|
-
else console.log(msg);
|
|
106
|
-
process.exit(0);
|
|
107
|
-
}
|
|
108
|
-
const isTTY = Boolean(process.stdin.isTTY && process.stdout.isTTY);
|
|
109
|
-
let includeOpencode = opts.includeOpencode;
|
|
110
|
-
if (includeOpencode) {
|
|
111
|
-
const confirmFn = async () => {
|
|
112
|
-
const readline = await import("node:readline");
|
|
113
|
-
return new Promise((res) => {
|
|
114
|
-
const rl = readline.createInterface({ input: process.stdin, output: process.stdout });
|
|
115
|
-
rl.question("将消耗 B/C/D 的 opencode 额度,确认测 opencode via?y/N ", (ans) => { rl.close(); res(ans.trim().toLowerCase() === "y"); });
|
|
116
|
-
});
|
|
117
|
-
};
|
|
118
|
-
const log = (s) => (opts.json ? console.error(s) : console.log(s));
|
|
119
|
-
includeOpencode = await resolveIncludeOpencode({ includeOpencode, isTTY, confirmFn, log });
|
|
120
|
-
}
|
|
121
|
-
const { loadToken } = await import("../../../state.js");
|
|
122
|
-
let token = "";
|
|
123
|
-
try { token = (await loadToken()).token || ""; } catch {}
|
|
124
|
-
// target providers
|
|
125
|
-
let targetIds = [];
|
|
126
|
-
const isAll = providerId === "bench" || providerId === "all";
|
|
127
|
-
if (isAll) {
|
|
128
|
-
const configs = loadConfigs();
|
|
129
|
-
const ids = new Set(Object.keys(configs));
|
|
130
|
-
ids.add("opencode"); ids.add("openrouter");
|
|
131
|
-
try { const m = await import("../../../state.js"); const raw = m.loadProviderKeys ? null : null; } catch {}
|
|
132
|
-
// collect from providerKeys via loadKeys probing? simple: iterate ids and keep those with allowed
|
|
133
|
-
for (const pid of [...ids]) {
|
|
134
|
-
const allowed = loadAllowed(pid) || [];
|
|
135
|
-
if (allowed.length) targetIds.push(pid);
|
|
136
|
-
else if (pid === "opencode" && !includeOpencode) continue;
|
|
137
|
-
}
|
|
138
|
-
// also check generic ids from state raw
|
|
139
|
-
try {
|
|
140
|
-
const { readFileSync } = await import("node:fs");
|
|
141
|
-
const { defaultStateFile } = await import("../../../state.js");
|
|
142
|
-
const raw = JSON.parse(readFileSync(defaultStateFile(), "utf8"));
|
|
143
|
-
for (const k of Object.keys(raw.providerConfigs || {})) if (!targetIds.includes(k) && (loadAllowed(k) || []).length) targetIds.push(k);
|
|
144
|
-
for (const k of Object.keys(raw.providerKeys || {})) if (!targetIds.includes(k) && (loadAllowed(k) || []).length) targetIds.push(k);
|
|
145
|
-
} catch {}
|
|
146
|
-
if (!targetIds.length) targetIds = ["openrouter", "workbuddy", "clinebot"].filter((p) => (loadAllowed(p) || []).length);
|
|
147
|
-
} else {
|
|
148
|
-
targetIds = [providerId];
|
|
149
|
-
}
|
|
150
|
-
const allResults = [];
|
|
151
|
-
const viaLog = (s) => (opts.json ? console.error(s) : console.log(s));
|
|
152
|
-
viaLog(`bench-via: direct vs via peers (samples=${opts.samples}, timeout=${opts.timeoutMs}ms, ${includeOpencode ? "opencode=included" : "opencode=skipped"}) peers=${peers.map((p) => p.id).join(",")}`);
|
|
153
|
-
for (const pid of targetIds) {
|
|
154
|
-
const cfg = (loadConfigs()[pid] || {});
|
|
155
|
-
const keys = loadKeys(pid) || [];
|
|
156
|
-
const allowed = loadAllowed(pid) || [];
|
|
157
|
-
const baseUrl = (loadBaseUrl(pid) || cfg.baseUrl || "").trim();
|
|
158
|
-
if (!allowed.length) {
|
|
159
|
-
viaLog(`provider ${pid}: 无勾选模型,跳过`);
|
|
160
|
-
continue;
|
|
161
|
-
}
|
|
162
|
-
if (!baseUrl && pid !== "opencode") { viaLog(`provider ${pid}: missing baseUrl 跳过`); continue; }
|
|
163
|
-
if (!keys.length && pid !== "opencode") { viaLog(`provider ${pid}: 未配置 Key 跳过`); continue; }
|
|
164
|
-
const chatPath = cfg.chatPath || defaultChatPath(pid);
|
|
165
|
-
let auths = [];
|
|
166
|
-
try { const m = await import("../../../state.js"); auths = m.loadProviderAuths ? m.loadProviderAuths(pid) : []; } catch {}
|
|
167
|
-
const models = allowed.map((id) => ({ provider: pid, model: String(id), id: String(id) }));
|
|
168
|
-
// directRunner per provider
|
|
169
|
-
const directRunner = async ({ provider, model }) => {
|
|
170
|
-
const p = provider || pid;
|
|
171
|
-
const idx = models.findIndex((x) => x.model === model);
|
|
172
|
-
const kIdx = idx >= 0 ? idx % (keys.length || 1) : 0;
|
|
173
|
-
const aIdx = Math.min(kIdx, Math.max(auths.length - 1, 0));
|
|
174
|
-
const key = keys[kIdx] || keys[0] || "";
|
|
175
|
-
const auth = auths[aIdx] || auths[0] || null;
|
|
176
|
-
const cKeys = keys.filter((k) => isRefreshToken(k, p));
|
|
177
|
-
if (cKeys.length) {
|
|
178
|
-
const normBase = String(baseUrl).replace(/\/+$/, "");
|
|
179
|
-
const chatBase = normBase.endsWith("/api/v1") ? normBase.slice(0, -7) : normBase;
|
|
180
|
-
const rt = cKeys[kIdx % cKeys.length];
|
|
181
|
-
const at = await refreshTokenForBase({ refreshToken: rt, baseUrl: normBase, fetchImpl });
|
|
182
|
-
if (!at) return { ok: false, label: "鉴权失败", error: "refresh failed", ttfbMs: null, totalMs: 0 };
|
|
183
|
-
return clineBenchOne({ baseUrl: chatBase, model, accessToken: at, prompt: "hi", maxTokens: 5, timeoutMs: opts.timeoutMs, fetchImpl });
|
|
184
|
-
}
|
|
185
|
-
const headers = buildHeadersForProvider(p, key, auth);
|
|
186
|
-
return runOne({ baseUrl: p === "opencode" ? (process.env.UPSTREAM_BASE_URL || "https://opencode.ai") : baseUrl, chatPath, model, providerId: p, apiKey: key || "public", headers, prompt: "hi", maxTokens: 5, timeoutMs: opts.timeoutMs, fetchImpl });
|
|
187
|
-
};
|
|
188
|
-
const { viaProbe } = await import("../../../bench/via-probe.js");
|
|
189
|
-
const { joinUrl: _joinUrl } = await import("../../../providers/base.js");
|
|
190
|
-
const viaProbeFn = async (args) => {
|
|
191
|
-
// 纯中继:A 把 targetUrl+headers+body 发给 B,B 原样 fetch 到上游(不查 B 本地 providerConfigs)
|
|
192
|
-
const { peerUrl, providerId, model } = args;
|
|
193
|
-
const p = providerId || pid;
|
|
194
|
-
const idx = models.findIndex((x) => x.model === model);
|
|
195
|
-
const kIdx = idx >= 0 ? idx % (keys.length || 1) : 0;
|
|
196
|
-
const aIdx = Math.min(kIdx, Math.max(auths.length - 1, 0));
|
|
197
|
-
const key = keys[kIdx] || keys[0] || "";
|
|
198
|
-
const auth = auths[aIdx] || auths[0] || null;
|
|
199
|
-
const cKeys = keys.filter((k) => isRefreshToken(k, p));
|
|
200
|
-
if (cKeys.length) {
|
|
201
|
-
const normBase = String(baseUrl).replace(/\/+$/, "");
|
|
202
|
-
const chatBase = normBase.endsWith("/api/v1") ? normBase.slice(0, -7) : normBase;
|
|
203
|
-
const rt = cKeys[0];
|
|
204
|
-
const at = await refreshTokenForBase({ refreshToken: rt, baseUrl: normBase, fetchImpl });
|
|
205
|
-
if (!at) return { ok: false, label: "鉴权失败", error: "refresh failed", ttfbMs: null, totalMs: 0 };
|
|
206
|
-
const targetUrl = `${chatBase}/api/v1/chat/completions`;
|
|
207
|
-
const headers = clineHeaders(`sess_bench_via_${Date.now()}`, at);
|
|
208
|
-
const rawModel = String(model).startsWith(`${p}/`) ? String(model).slice(p.length + 1) : String(model);
|
|
209
|
-
const body = { model: rawModel, messages: [{ role: "user", content: "hi" }], stream: true, max_tokens: 5, session_id: `sess_bench_via_${Date.now()}`, reasoning_effort: "high" };
|
|
210
|
-
const peer = peers.find((pe) => pe.url === peerUrl || (pe.name || pe.id) === peerUrl);
|
|
211
|
-
const peerToken = peer?.token || token;
|
|
212
|
-
return viaProbe({ peerUrl, token: peerToken, relayTarget: targetUrl, relayHeaders: headers, relayBody: body, timeoutMs: opts.timeoutMs, fetchImpl });
|
|
213
|
-
}
|
|
214
|
-
const rawModel = String(model).startsWith(`${p}/`) ? String(model).slice(p.length + 1) : String(model);
|
|
215
|
-
const targetUrl = p === "opencode" ? `${process.env.UPSTREAM_BASE_URL || "https://opencode.ai"}/zen/v1/chat/completions` : `${String(baseUrl).replace(/\/+$/, "")}${chatPath}`;
|
|
216
|
-
const headers = buildHeadersForProvider(p, key, auth);
|
|
217
|
-
const body = { model: rawModel, stream: false, messages: [{ role: "user", content: "hi" }], max_tokens: 5 };
|
|
218
|
-
// 用对端 peer.token 做 relay 鉴权(B 只认自己的 Bearer)
|
|
219
|
-
const peer = peers.find((pe) => pe.url === peerUrl || (pe.name || pe.id) === peerUrl);
|
|
220
|
-
const peerToken = peer?.token || token;
|
|
221
|
-
return viaProbe({ peerUrl, token: peerToken, relayTarget: targetUrl, relayHeaders: headers, relayBody: body, timeoutMs: opts.timeoutMs, fetchImpl });
|
|
222
|
-
};
|
|
223
|
-
const part = await orchestrateVia({ models, peers, directRunner, viaProbeFn, includeOpencode, token, timeoutMs: opts.timeoutMs });
|
|
224
|
-
allResults.push(...part);
|
|
225
|
-
if (!opts.json) viaLog(` ${pid}: ${part.length} 模型完成`);
|
|
226
|
-
}
|
|
227
|
-
const meta = { at: new Date().toISOString(), samples: opts.samples, timeout: opts.timeoutMs, includeOpencode, peers: peers.map((p) => p.id), opencodeSkipped: !includeOpencode };
|
|
228
|
-
const report = formatViaReport(allResults, { peers, meta, json: opts.json });
|
|
229
|
-
if (opts.json) console.log(report.text);
|
|
230
|
-
else console.log("\n" + report.text);
|
|
231
|
-
process.exit(0);
|
|
232
|
-
}
|
|
233
|
-
|
|
234
29
|
export async function handleProviderBench(id, sub, rest, args, deps = {}) {
|
|
235
30
|
const _restArr = rest || [];
|
|
236
31
|
const _hasVia = _restArr.includes("--via") || _restArr.includes("--bench-via");
|
|
@@ -238,7 +33,6 @@ export async function handleProviderBench(id, sub, rest, args, deps = {}) {
|
|
|
238
33
|
const isBench = sub === "bench" || sub === "benchmark" || sub === "eval" || sub === "test" || _isBenchViaAll;
|
|
239
34
|
if (!isBench) return false;
|
|
240
35
|
const opts = parseBenchArgs(_restArr);
|
|
241
|
-
// via branch
|
|
242
36
|
if (opts.via) {
|
|
243
37
|
const loadConfigs = deps.loadProviderConfigs || (await import("../../../state.js")).loadProviderConfigs;
|
|
244
38
|
const loadKeys = deps.loadProviderKeys || (await import("../../../state.js")).loadProviderKeys;
|