dsh-livebench-panel 0.2.12 → 0.2.14
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +6 -3
- package/lib/client.js +179 -77
- package/lib/index.js +32 -8
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -44,11 +44,11 @@ cd livebench
|
|
|
44
44
|
- **标签页位置**:`conversation.view` 槽位 `id: "livebench"`、`order: 20`(对话 = 0,轨迹 = 10,LiveBench 排最右)。
|
|
45
45
|
- **模型下拉框**:读取 `$DSH_HOME/settings.yaml` 的 `llm-pi-ai.providers` **并合并内置 `deepseek-official` 路由**(`@deepseek-ai/dsh-llm-deepseek` 注册的 DeepSeek-V4-Flash / V4-Pro / V4-Flash-Vision-Exp,默认 `https://api.deepseek.com` + `DEEPSEEK_API_KEY`,用户 `llm-deepseek` 设置节可覆盖),与 harness 模型选择同源,展示**全部 provider / 全部模型**;`openai-completions` 且配置了 `baseURL` 的 provider 会被标记为可直连 LiveBench(`--api-base` 路由)。
|
|
46
46
|
- **推理强度下拉框**(模型右侧):来自模型配置的 `reasoningEfforts` 映射(如 gpt-5.6 系 off/low/medium/high/xhigh/max,DeepSeek off/low/high/max)。选定后:
|
|
47
|
-
- 插件向 `livebench/model/model_configs/
|
|
47
|
+
- 插件向 `livebench/model/model_configs/dsh_panel_generated__<display-name>.yaml` 写入一条模型配置(每个模型一个文件,避免并发写同名文件互相覆盖),经 LiveBench 的 `api_kwargs.default.reasoning_effort` 透传给 API(`off` 表示不透传、由后端走默认);
|
|
48
48
|
- 强度编码进 display-name(如 `code-gpt__gpt-5.6-sol@high`),**不同强度在成绩表中是独立条目**,可直接对比。
|
|
49
49
|
- **参数下拉框**:题集 release(LiveBench 全部 releases)、分类(coding/math/reasoning/language/data_analysis/instruction_following)、任务(随分类联动)、题目序号范围、max-tokens。
|
|
50
|
-
- **运行控制**:开始 / 停止 /
|
|
51
|
-
- **成绩表**:直接读取 `data/live_bench/**/model_judgment/ground_truth_judgment.jsonl` 计算 模型 × 任务 平均分(分数 = 正确率 ×100
|
|
50
|
+
- **运行控制**:开始 / 停止 / 刷新;最多 9 个模型并发评测(每个模型同时只跑 1 次);实时滚动日志(每 2.5s 轮询);「刷新」会清掉已结束的运行日志。
|
|
51
|
+
- **成绩表**:直接读取 `data/live_bench/**/model_judgment/ground_truth_judgment.jsonl` 计算 模型 × 任务 平均分(分数 = 正确率 ×100,单元格下方括号标注「本次没做或没做完的题数 / 本次设定的题数」),无需等 LiveBench 自己出榜。支持按模型名/时间排序、拖 ⠿ 自定义行序、勾选或单行按钮删除成绩(删除会同时清掉该模型的答案与判分行;只有空壳文件的“幽灵行”不会再被扫描出来,删除结果稳定持久)。
|
|
52
52
|
|
|
53
53
|
## 依赖
|
|
54
54
|
|
|
@@ -65,6 +65,9 @@ cd livebench
|
|
|
65
65
|
| `/dsh-livebench-panel/api/status` | GET | 运行状态 + 日志尾部 |
|
|
66
66
|
| `/dsh-livebench-panel/api/stop` | POST | 终止当前评测 |
|
|
67
67
|
| `/dsh-livebench-panel/api/results` | GET | 汇总成绩行 |
|
|
68
|
+
| `/dsh-livebench-panel/api/delete` | POST | 删除指定 模型×分类×任务 的答案与判分记录(正文 `{rows:[{model,category,task}]}`;正在评测的模型会被跳过并在 `skipped` 中返回) |
|
|
69
|
+
| `/dsh-livebench-panel/api/clear` | POST | 清掉已结束的运行日志/记录 |
|
|
70
|
+
| `/dsh-livebench-panel/api/home` | POST | 保存 LiveBench 检出路径 |
|
|
68
71
|
|
|
69
72
|
## 安装(本地开发,遵循 ~/.dsh/AGENTS.md)
|
|
70
73
|
|
package/lib/client.js
CHANGED
|
@@ -8,7 +8,7 @@ window.__ModuleLoader__.load({
|
|
|
8
8
|
const { createElement: h, useCallback, useEffect, useMemo, useRef, useState } = react;
|
|
9
9
|
|
|
10
10
|
//#region dsh-css:livebench-panel.css
|
|
11
|
-
const css = `.dlb_root{width:100%;max-width:
|
|
11
|
+
const css = `.dlb_root{width:100%;max-width:1400px;margin:0 auto;padding:16px 20px 32px;color:var(--dsw-alias-label-primary);display:flex;flex-direction:column;gap:14px}
|
|
12
12
|
.dlb_head{display:flex;align-items:baseline;gap:10px}
|
|
13
13
|
.dlb_head h2{margin:0;font-size:16px;font-weight:600}
|
|
14
14
|
.dlb_sub{color:var(--dsw-alias-label-tertiary);font-size:12px}
|
|
@@ -21,6 +21,8 @@ window.__ModuleLoader__.load({
|
|
|
21
21
|
.dlb_select,.dlb_input{height:34px;border:1px solid var(--dsw-alias-border-l2);background:var(--dsw-alias-bg-layer-1);color:var(--dsw-alias-label-primary);font:inherit;font-size:13px;border-radius:8px;padding:0 8px;outline:none;min-width:0}
|
|
22
22
|
.dlb_select:focus-visible,.dlb_input:focus-visible{border-color:var(--dsw-alias-state-business-primary)}
|
|
23
23
|
.dlb_row{display:flex;align-items:center;gap:8px;flex-wrap:wrap}
|
|
24
|
+
.dlb_row{align-items:flex-start}
|
|
25
|
+
.dlb_cardHead{display:flex;flex-direction:column;gap:2px;min-width:0;flex:1 1 auto}
|
|
24
26
|
.dlb_btn{appearance:none;font:inherit;cursor:pointer;border-radius:8px;padding:6px 14px;font-size:13px;border:1px solid transparent;background:var(--dsw-alias-state-business-primary);color:var(--dsw-alias-label-primary)}
|
|
25
27
|
.dlb_btn:disabled{cursor:default;opacity:.55}
|
|
26
28
|
.dlb_btnGhost{background:transparent;border-color:var(--dsw-alias-border-l2);color:var(--dsw-alias-label-secondary)}
|
|
@@ -29,9 +31,31 @@ window.__ModuleLoader__.load({
|
|
|
29
31
|
.dlb_badge[data-ok="1"]{color:var(--dsw-alias-state-success-primary);border-color:color-mix(in srgb,var(--dsw-alias-state-success-primary) 40%,transparent)}
|
|
30
32
|
.dlb_badge[data-ok="0"]{color:var(--dsw-alias-state-error-primary);border-color:color-mix(in srgb,var(--dsw-alias-state-error-primary) 40%,transparent)}
|
|
31
33
|
.dlb_log{background:var(--dsw-alias-bg-layer-1);border:1px solid var(--dsw-alias-border-l2);border-radius:8px;padding:10px;font:12px/1.5 ui-monospace,SFMono-Regular,Menlo,Consolas,monospace;max-height:260px;overflow:auto;white-space:pre-wrap;word-break:break-all;color:var(--dsw-alias-label-secondary)}
|
|
32
|
-
.
|
|
33
|
-
.dlb_table
|
|
34
|
-
.dlb_table th{
|
|
34
|
+
.dlb_tableWrap{overflow:auto;max-height:min(64vh,620px);border:1px solid var(--dsw-alias-border-l2);border-radius:10px;background:var(--dsw-alias-bg-layer-2)}
|
|
35
|
+
.dlb_table{table-layout:fixed;border-collapse:separate;border-spacing:0;font-size:11.5px}
|
|
36
|
+
.dlb_table th,.dlb_table td{padding:4px 5px;border-bottom:1px solid var(--dsw-alias-border-l2);white-space:nowrap;vertical-align:middle;text-align:left;overflow:hidden;text-overflow:ellipsis}
|
|
37
|
+
.dlb_table td{height:34px}
|
|
38
|
+
.dlb_table thead th{color:var(--dsw-alias-label-tertiary);font-weight:500;background:var(--dsw-alias-bg-layer-2);position:sticky;top:0;z-index:3}
|
|
39
|
+
.dlb_table thead tr.dlb_catRow th{top:0;height:20px;padding:0 6px;font-size:9.5px;letter-spacing:.04em;text-transform:uppercase;background:var(--dsw-alias-bg-layer-1);cursor:default}
|
|
40
|
+
.dlb_table thead tr.dlb_thRow th{top:20px;height:26px;font-size:11px;box-shadow:inset 0 -1px 0 var(--dsw-alias-border-l2)}
|
|
41
|
+
.dlb_table tbody tr:nth-child(even) td{background:color-mix(in srgb,var(--dsw-alias-fill-l1,rgba(255,255,255,.04)) 55%,transparent)}
|
|
42
|
+
.dlb_table tbody tr:hover td{background:color-mix(in srgb,var(--dsw-alias-state-business-primary) 9%,transparent)}
|
|
43
|
+
.dlb_stick{position:sticky;background:var(--dsw-alias-bg-layer-2);z-index:2}
|
|
44
|
+
.dlb_table thead tr.dlb_catRow th.dlb_stick{z-index:5}
|
|
45
|
+
.dlb_table thead tr.dlb_thRow th.dlb_stick{z-index:5}
|
|
46
|
+
.dlb_table tbody tr:nth-child(even) .dlb_stick{background:color-mix(in srgb,var(--dsw-alias-bg-layer-2) 90%,var(--dsw-alias-fill-l1,rgba(255,255,255,.05)))}
|
|
47
|
+
.dlb_table tbody tr:hover .dlb_stick{background:color-mix(in srgb,var(--dsw-alias-bg-layer-2) 88%,var(--dsw-alias-state-business-primary))}
|
|
48
|
+
.dlb_grpStart{border-left:1px solid var(--dsw-alias-border-l2) !important}
|
|
49
|
+
.dlb_cellScore{text-align:center !important;font-variant-numeric:tabular-nums;font-weight:600;padding-left:2px !important;padding-right:2px !important}
|
|
50
|
+
.dlb_cellSub{font-size:9px;font-weight:400;color:var(--dsw-alias-label-tertiary);font-variant-numeric:tabular-nums}
|
|
51
|
+
.dlb_thTask{font-size:10.5px !important;text-align:center !important;cursor:default}
|
|
52
|
+
.dlb_modelCell{padding-left:8px !important}
|
|
53
|
+
.dlb_modelName{font-weight:500;overflow:hidden;text-overflow:ellipsis}
|
|
54
|
+
.dlb_dim{opacity:.35;font-weight:400}
|
|
55
|
+
.dlb_timeCell{font-size:10.5px;color:var(--dsw-alias-label-secondary);font-variant-numeric:tabular-nums;text-align:center !important}
|
|
56
|
+
.dlb_sHigh{color:var(--dsw-alias-state-success-primary)}
|
|
57
|
+
.dlb_sMid{color:var(--dsw-alias-state-warning-primary,#d08700)}
|
|
58
|
+
.dlb_sLow{color:var(--dsw-alias-state-error-primary)}
|
|
35
59
|
.dlb_score{font-variant-numeric:tabular-nums;font-weight:600}
|
|
36
60
|
.dlb_error{color:var(--dsw-alias-state-error-primary);font-size:12px;margin:0}
|
|
37
61
|
.dlb_hint{color:var(--dsw-alias-label-tertiary);font-size:11.5px;margin:0;line-height:1.5}
|
|
@@ -309,20 +333,41 @@ window.__ModuleLoader__.load({
|
|
|
309
333
|
};
|
|
310
334
|
|
|
311
335
|
// ---- 成绩矩阵:model 为行标识、category/task 为列标识 ----
|
|
312
|
-
// time 列 =
|
|
336
|
+
// time 列 = 该模型这次评测的开始时间:
|
|
337
|
+
// 1) 评测元数据 startedAt;2) displayName 尾部的运行戳 __rYYYYMMDD-HHMMSS;3) 最早的判分时间
|
|
338
|
+
const runStampSeconds = (modelName) => {
|
|
339
|
+
const m = /__r(\d{4})(\d{2})(\d{2})-(\d{2})(\d{2})(\d{2})$/.exec(String(modelName));
|
|
340
|
+
if (!m) return null;
|
|
341
|
+
const t = new Date(Number(m[1]), Number(m[2]) - 1, Number(m[3]), Number(m[4]), Number(m[5]), Number(m[6])).getTime();
|
|
342
|
+
return Number.isFinite(t) ? Math.floor(t / 1000) : null;
|
|
343
|
+
};
|
|
313
344
|
const taskKey = (category, task) => `${category}/${task}`;
|
|
314
345
|
const matrix = useMemo(() => {
|
|
315
346
|
if (!results) return null;
|
|
316
347
|
const models = [...new Set(results.rows.map((r) => r.model))].sort();
|
|
317
|
-
const
|
|
348
|
+
const taskSet = new Set(results.rows.map((r) => taskKey(r.category, r.task)));
|
|
349
|
+
const tasks = [...taskSet].sort();
|
|
350
|
+
// 按 category 分组(同一 category 的 task 相邻),列上以分隔线分组
|
|
351
|
+
const groups = [];
|
|
352
|
+
const byCat = new Map();
|
|
353
|
+
for (const key of tasks) {
|
|
354
|
+
const category = key.split("/")[0];
|
|
355
|
+
if (!byCat.has(category)) { byCat.set(category, []); groups.push({ category, tasks: byCat.get(category) }); }
|
|
356
|
+
byCat.get(category).push(key);
|
|
357
|
+
}
|
|
358
|
+
groups.sort((a, b) => a.category.localeCompare(b.category));
|
|
359
|
+
const groupStart = new Set(groups.map((g) => g.tasks[0]));
|
|
318
360
|
const cells = new Map();
|
|
319
361
|
const startTimes = new Map();
|
|
320
362
|
for (const row of results.rows) {
|
|
321
363
|
cells.set(`${row.model}__@__${taskKey(row.category, row.task)}`, row);
|
|
364
|
+
const fromMeta = row.runStartedAt ? Math.floor(Date.parse(row.runStartedAt) / 1000) : null;
|
|
365
|
+
const candidate = (Number.isFinite(fromMeta) ? fromMeta : null) ?? runStampSeconds(row.model) ?? row.time ?? null;
|
|
322
366
|
const prev = startTimes.get(row.model);
|
|
323
|
-
if (
|
|
367
|
+
if (candidate && (prev === undefined || candidate < prev)) startTimes.set(row.model, candidate);
|
|
324
368
|
}
|
|
325
|
-
|
|
369
|
+
const taskTotals = (results && results.taskTotals) || {};
|
|
370
|
+
return { models, tasks, groups, groupStart, cells, startTimes, taskTotals };
|
|
326
371
|
}, [results]);
|
|
327
372
|
const cellOf = (model, key) => matrix.cells.get(`${model}__@__${key}`);
|
|
328
373
|
// 排序:sortBy=null 时用拖拽自定义顺序;点击表头在 正序/倒序 间切换
|
|
@@ -392,17 +437,29 @@ window.__ModuleLoader__.load({
|
|
|
392
437
|
}
|
|
393
438
|
if (serverRows.length === 0) return;
|
|
394
439
|
if (!window.confirm(`确认删除所选 ${modelsToDelete.length} 个模型的全部评测成绩(共 ${serverRows.length} 条记录,含答案)?不可恢复。`)) return;
|
|
395
|
-
await api("/delete", {
|
|
440
|
+
const delRes = await api("/delete", {
|
|
396
441
|
method: "POST",
|
|
397
442
|
headers: { "content-type": "application/json" },
|
|
398
443
|
body: JSON.stringify({ rows: serverRows }),
|
|
399
444
|
});
|
|
400
445
|
setPickedModels(new Set());
|
|
446
|
+
if (delRes && delRes.skipped > 0) {
|
|
447
|
+
window.alert(`${delRes.skipped} 个模型正在评测中,已跳过未删除;请等评测结束后再删。`);
|
|
448
|
+
}
|
|
401
449
|
persistModelOrder(modelOrder.filter((model) => !modelsToDelete.includes(model)));
|
|
402
450
|
await loadResults();
|
|
403
451
|
};
|
|
404
452
|
|
|
405
|
-
|
|
453
|
+
// 显示用短名:去掉运行时间戳后缀(运行时间单列显示在模型名下方)
|
|
454
|
+
const shortModelName = (m) => String(m).replace(/__r[0-9]{8}-[0-9]{6}$/, "");
|
|
455
|
+
const fmtTimeShort = (t) => {
|
|
456
|
+
if (!t) return "—";
|
|
457
|
+
const dt = new Date(t * 1000);
|
|
458
|
+
const pad = (n) => String(n).padStart(2, "0");
|
|
459
|
+
return `${pad(dt.getMonth() + 1)}-${pad(dt.getDate())} ${pad(dt.getHours())}:${pad(dt.getMinutes())}`;
|
|
460
|
+
};
|
|
461
|
+
|
|
462
|
+
const fmtTime = (t) => {
|
|
406
463
|
if (!t) return "—";
|
|
407
464
|
const dt = new Date(t * 1000);
|
|
408
465
|
const pad = (n) => String(n).padStart(2, "0");
|
|
@@ -542,81 +599,126 @@ window.__ModuleLoader__.load({
|
|
|
542
599
|
),
|
|
543
600
|
sortedModels.length > 0 && h("div", { className: c("card") },
|
|
544
601
|
h("div", { className: c("row") },
|
|
545
|
-
h("
|
|
602
|
+
h("div", { className: c("cardHead") },
|
|
603
|
+
h("span", { className: c("label") }, `评测成绩(行 = 模型,列 = 任务)· ${sortedModels.length} 个模型 × ${matrix.tasks.length} 个任务`),
|
|
604
|
+
h("span", { className: c("hint") }, "分数 = 该任务平均分 ×100;括号 (-本次没做或没做完的题数/本次设定的题数);× = 未执行该任务;— = 无有效判分(如全部访问失败,不计入正确率);颜色 ≥75 绿 / 40–75 橙 / <40 红;拖 ⠿ 换行序(存在本机),点表头按模型名或时间排序。"),
|
|
605
|
+
),
|
|
546
606
|
h("button", {
|
|
547
607
|
className: c("btn") + " " + c("btnDanger"),
|
|
548
608
|
disabled: pickedModels.size === 0,
|
|
549
609
|
onClick: () => deleteModels([...pickedModels]),
|
|
550
610
|
}, `删除所选(${pickedModels.size})`),
|
|
551
611
|
),
|
|
552
|
-
h("
|
|
553
|
-
h("
|
|
554
|
-
|
|
555
|
-
|
|
556
|
-
|
|
557
|
-
|
|
558
|
-
|
|
559
|
-
|
|
560
|
-
|
|
561
|
-
|
|
562
|
-
|
|
563
|
-
|
|
564
|
-
|
|
565
|
-
|
|
566
|
-
|
|
567
|
-
|
|
568
|
-
|
|
569
|
-
|
|
570
|
-
|
|
571
|
-
|
|
572
|
-
|
|
573
|
-
|
|
574
|
-
|
|
575
|
-
|
|
576
|
-
|
|
577
|
-
|
|
578
|
-
|
|
579
|
-
|
|
580
|
-
|
|
581
|
-
|
|
582
|
-
|
|
583
|
-
|
|
584
|
-
|
|
585
|
-
h("
|
|
586
|
-
|
|
587
|
-
|
|
612
|
+
h("div", { className: c("tableWrap") },
|
|
613
|
+
h("table", {
|
|
614
|
+
className: c("table"),
|
|
615
|
+
style: { width: `${20 + 26 + 190 + matrix.tasks.length * 74 + 30 + 86}px` },
|
|
616
|
+
},
|
|
617
|
+
h("colgroup", null,
|
|
618
|
+
h("col", { style: { width: "20px" } }),
|
|
619
|
+
h("col", { style: { width: "26px" } }),
|
|
620
|
+
h("col", { style: { width: "190px" } }),
|
|
621
|
+
matrix.tasks.map((taskKeyCol) => h("col", { key: taskKeyCol, style: { width: "74px" } })),
|
|
622
|
+
h("col", { style: { width: "30px" } }),
|
|
623
|
+
h("col", { style: { width: "86px" } }),
|
|
624
|
+
),
|
|
625
|
+
h("thead", null,
|
|
626
|
+
// 第一行:category 分组(同一 category 的 task 列相邻,组间有分隔线)
|
|
627
|
+
h("tr", { className: c("catRow") },
|
|
628
|
+
h("th", { className: c("stick"), style: { left: 0 }, colSpan: 3 }, "分类 ↓"),
|
|
629
|
+
matrix.groups.map((group) => h("th", {
|
|
630
|
+
key: group.category,
|
|
631
|
+
className: c("grpStart"),
|
|
632
|
+
colSpan: group.tasks.length,
|
|
633
|
+
style: { textAlign: "center" },
|
|
634
|
+
title: `${group.category}(${group.tasks.length} 个任务)`,
|
|
635
|
+
}, `${group.category} · ${group.tasks.length}`)),
|
|
636
|
+
h("th", null, ""),
|
|
637
|
+
h("th", null, ""),
|
|
638
|
+
),
|
|
639
|
+
// 第二行:task 列头 + model / time 排序
|
|
640
|
+
h("tr", { className: c("thRow") },
|
|
641
|
+
h("th", { className: c("stick"), style: { left: 0 } }, "⠿"),
|
|
642
|
+
h("th", { className: c("stick"), style: { left: "20px" } },
|
|
643
|
+
h("input", { type: "checkbox", checked: pickedModels.size > 0 && pickedModels.size === sortedModels.length,
|
|
644
|
+
onChange: (event) => setPickedModels(event.target.checked ? new Set(sortedModels) : new Set()) })),
|
|
645
|
+
h("th", {
|
|
646
|
+
className: c("stick"),
|
|
647
|
+
style: { left: "46px", cursor: "pointer", userSelect: "none", paddingLeft: "8px" },
|
|
648
|
+
title: "点击按模型名排序(再次点击切换正序/倒序)",
|
|
649
|
+
onClick: () => toggleSort("model"),
|
|
650
|
+
}, "模型 / 运行时间" + sortMark("model")),
|
|
588
651
|
matrix.tasks.map((taskKeyCol) => {
|
|
589
|
-
const
|
|
590
|
-
|
|
591
|
-
|
|
592
|
-
|
|
593
|
-
|
|
594
|
-
|
|
595
|
-
return h("td", { key: taskKeyCol, className: c("score"), title: "该模型未执行此任务(无任何记录)" },
|
|
596
|
-
h("div", null, "×"),
|
|
597
|
-
h("div", { style: subStyle }, sub));
|
|
598
|
-
}
|
|
599
|
-
const allFailed = row.answered > 0 && row.errors >= row.answered;
|
|
600
|
-
const title = allFailed
|
|
601
|
-
? `该任务 ${row.errors} 题全部访问失败(未做完),不计入正确率`
|
|
602
|
-
: `做完 ${row.done ?? 0} 题,没做/失败 ${row.notDone ?? 0} 题` +
|
|
603
|
-
(row.errors > 0 ? `(其中 ${row.errors} 题 API 失败)` : "");
|
|
604
|
-
if (allFailed || row.judged === 0) {
|
|
605
|
-
return h("td", { key: taskKeyCol, className: c("score"), title },
|
|
606
|
-
h("div", null, "—"),
|
|
607
|
-
h("div", { style: subStyle }, sub));
|
|
608
|
-
}
|
|
609
|
-
return h("td", { key: taskKeyCol, className: c("score"), title },
|
|
610
|
-
h("div", null, row.score.toFixed(1)),
|
|
611
|
-
h("div", { style: subStyle }, sub));
|
|
652
|
+
const [cat, task] = taskKeyCol.split("/");
|
|
653
|
+
return h("th", {
|
|
654
|
+
key: taskKeyCol,
|
|
655
|
+
className: c("thTask") + (matrix.groupStart.has(taskKeyCol) ? " " + c("grpStart") : ""),
|
|
656
|
+
title: `${cat} / ${task}`,
|
|
657
|
+
}, task);
|
|
612
658
|
}),
|
|
613
|
-
h("
|
|
614
|
-
|
|
615
|
-
|
|
616
|
-
|
|
617
|
-
|
|
618
|
-
|
|
619
|
-
|
|
659
|
+
h("th", { title: "删除该模型的这一行成绩" }, "✕"),
|
|
660
|
+
h("th", {
|
|
661
|
+
style: { cursor: "pointer", userSelect: "none" },
|
|
662
|
+
title: "点击按运行时间排序(再次点击切换正序/倒序)",
|
|
663
|
+
onClick: () => toggleSort("time"),
|
|
664
|
+
}, "time" + sortMark("time")),
|
|
665
|
+
),
|
|
666
|
+
),
|
|
667
|
+
h("tbody", null,
|
|
668
|
+
sortedModels.map((model) => {
|
|
669
|
+
const checked = pickedModels.has(model);
|
|
670
|
+
const startTime = matrix.startTimes.get(model);
|
|
671
|
+
return h("tr", {
|
|
672
|
+
key: model,
|
|
673
|
+
draggable: true,
|
|
674
|
+
onDragStart: () => { dragKey.current = model; },
|
|
675
|
+
onDragOver: (event) => event.preventDefault(),
|
|
676
|
+
onDrop: () => onRowDrop(model),
|
|
677
|
+
},
|
|
678
|
+
h("td", { className: c("stick") + " " + c("drag"), style: { left: 0 }, title: "按住拖动排序" }, "⠿"),
|
|
679
|
+
h("td", { className: c("stick"), style: { left: "20px" } },
|
|
680
|
+
h("input", { type: "checkbox", checked, onChange: () => togglePicked(model) })),
|
|
681
|
+
h("td", { className: c("stick") + " " + c("modelCell"), style: { left: "46px" }, title: model },
|
|
682
|
+
h("div", { className: c("modelName") }, shortModelName(model)),
|
|
683
|
+
h("div", { className: c("cellSub") }, fmtTimeShort(startTime))),
|
|
684
|
+
matrix.tasks.map((taskKeyCol) => {
|
|
685
|
+
const row = matrix.cells.get(`${model}__@__${taskKeyCol}`);
|
|
686
|
+
// 单元格:上方正确率,下方 (-没做或没做完/本次设定题数)。
|
|
687
|
+
// 没做/没做完(访问失败、网络中断)不计入正确率分母。
|
|
688
|
+
// 该任务总题数不参与正确率计算,故不展示。
|
|
689
|
+
const configuredCol = row ? (row.configured ?? 0) : (matrix.taskTotals[taskKeyCol] ?? 0);
|
|
690
|
+
const sub = `(-${row ? (row.notDone ?? 0) : configuredCol}/${configuredCol})`;
|
|
691
|
+
const cls = c("cellScore") + (matrix.groupStart.has(taskKeyCol) ? " " + c("grpStart") : "");
|
|
692
|
+
if (!row) {
|
|
693
|
+
return h("td", { key: taskKeyCol, className: cls, title: `未执行该任务:本次设定 ${configuredCol} 题` },
|
|
694
|
+
h("div", { className: c("dim") }, "×"),
|
|
695
|
+
h("div", { className: c("cellSub") }, sub));
|
|
696
|
+
}
|
|
697
|
+
const allFailed = row.answered > 0 && row.errors >= row.answered;
|
|
698
|
+
const title = allFailed
|
|
699
|
+
? `本次设定 ${row.configured ?? 0} 题,其中 ${row.errors} 题全部访问失败(未做完),不计入正确率 ${sub}`
|
|
700
|
+
: `本次设定 ${row.configured ?? 0} 题,做完 ${row.done ?? 0} 题,没做/没做完 ${row.notDone ?? 0} 题 ${sub}` +
|
|
701
|
+
(row.errors > 0 ? `(其中 ${row.errors} 题 API 失败)` : "");
|
|
702
|
+
if (allFailed || row.judged === 0) {
|
|
703
|
+
return h("td", { key: taskKeyCol, className: cls, title },
|
|
704
|
+
h("div", { className: c("dim") }, "—"),
|
|
705
|
+
h("div", { className: c("cellSub") }, sub));
|
|
706
|
+
}
|
|
707
|
+
const scoreCls = row.score >= 75 ? c("sHigh") : row.score >= 40 ? c("sMid") : c("sLow");
|
|
708
|
+
return h("td", { key: taskKeyCol, className: cls, title },
|
|
709
|
+
h("div", { className: scoreCls }, row.score.toFixed(1)),
|
|
710
|
+
h("div", { className: c("cellSub") }, sub));
|
|
711
|
+
}),
|
|
712
|
+
h("td", { style: { textAlign: "center" } }, h("button", {
|
|
713
|
+
className: c("btnGhost") + " " + c("btn"), style: { padding: "1px 7px", fontSize: "11px", lineHeight: "16px" },
|
|
714
|
+
title: "删除该模型的所有评测成绩",
|
|
715
|
+
onClick: () => deleteModels([model]),
|
|
716
|
+
}, "✕")),
|
|
717
|
+
h("td", { className: c("timeCell") }, fmtTimeShort(startTime)),
|
|
718
|
+
);
|
|
719
|
+
}),
|
|
720
|
+
),
|
|
721
|
+
),
|
|
620
722
|
),
|
|
621
723
|
),
|
|
622
724
|
config !== null && config.available === false && h("div", { className: c("card") + " " + c("notice") },
|
package/lib/index.js
CHANGED
|
@@ -25,7 +25,7 @@
|
|
|
25
25
|
* @module dsh-livebench-panel
|
|
26
26
|
*/
|
|
27
27
|
import { spawn } from "node:child_process";
|
|
28
|
-
import { readdirSync, readFileSync, existsSync, statSync, writeFileSync, mkdirSync } from "node:fs";
|
|
28
|
+
import { readdirSync, readFileSync, existsSync, statSync, writeFileSync, mkdirSync, unlinkSync } from "node:fs";
|
|
29
29
|
import { createRequire } from "node:module";
|
|
30
30
|
import { delimiter, dirname, join } from "node:path";
|
|
31
31
|
import { homedir } from "node:os";
|
|
@@ -557,7 +557,9 @@ async function computeResults(dataDir) {
|
|
|
557
557
|
stat.answered += 1;
|
|
558
558
|
if (line.includes('"$ERROR$"')) stat.errors += 1;
|
|
559
559
|
}
|
|
560
|
-
|
|
560
|
+
// 空文件(尚未产出答案,或成绩已被删除只留下 0 字节壳)不计入结果:
|
|
561
|
+
// 否则删掉的记录会在下一次扫描时凭“文件名仍在”重新冒出来。
|
|
562
|
+
if (stat.answered > 0) answers.set(key, stat);
|
|
561
563
|
}
|
|
562
564
|
const jText = await readTextFileAsync(join(taskDir, "model_judgment", "ground_truth_judgment.jsonl"));
|
|
563
565
|
if (jText !== null) {
|
|
@@ -591,8 +593,8 @@ async function computeResults(dataDir) {
|
|
|
591
593
|
const judgedCount = entry?.n ?? 0;
|
|
592
594
|
// 做完的题 = 实际产出的有效答案(尝试数 - 访问失败数)
|
|
593
595
|
const done = Math.max(0, stat.answered - stat.errors);
|
|
594
|
-
//
|
|
595
|
-
let configured =
|
|
596
|
+
// 设定题数 = 本次评测设定的范围(end 含端点);没有元数据(历史数据)时用该任务的题目总数兜底
|
|
597
|
+
let configured = total;
|
|
596
598
|
const meta = runMeta.get(model);
|
|
597
599
|
if (meta !== undefined) {
|
|
598
600
|
const inScope = (meta.benchNames || []).some((bn) => {
|
|
@@ -603,9 +605,10 @@ async function computeResults(dataDir) {
|
|
|
603
605
|
});
|
|
604
606
|
if (inScope) {
|
|
605
607
|
const hasRange = meta.begin !== null && meta.begin !== undefined && meta.end !== null && meta.end !== undefined;
|
|
606
|
-
configured =
|
|
608
|
+
if (hasRange) configured = Math.min(total, Number(meta.end) - Number(meta.begin) + 1);
|
|
607
609
|
}
|
|
608
610
|
}
|
|
611
|
+
// 实际尝试数比设定还多(断点重跑等)时,以实际为准
|
|
609
612
|
if (configured < stat.answered) configured = stat.answered;
|
|
610
613
|
const notDone = Math.max(0, configured - done);
|
|
611
614
|
// 正确率 = 做对的题 / 已判分的做完题(访问失败/未做的题不计入分母)
|
|
@@ -624,9 +627,14 @@ async function computeResults(dataDir) {
|
|
|
624
627
|
done,
|
|
625
628
|
configured,
|
|
626
629
|
notDone,
|
|
630
|
+
// 本次运行的开始时间(ISO);旧数据没有元数据时为 null,前端退回解析 displayName 里的运行戳
|
|
631
|
+
runStartedAt: meta !== undefined && typeof meta.startedAt === "string" ? meta.startedAt : null,
|
|
627
632
|
};
|
|
628
633
|
});
|
|
629
|
-
|
|
634
|
+
// taskTotals 同时给出裸任务名与 `category/task` 两种键,便于前端按列取题目总数
|
|
635
|
+
const taskTotals = Object.fromEntries(totals);
|
|
636
|
+
for (const [t, n] of totals) taskTotals[`${catByTask.get(t) ?? ""}/${t}`] = n;
|
|
637
|
+
const result = { rows, taskTotals };
|
|
630
638
|
resultsCache = { at: Date.now(), data: result };
|
|
631
639
|
return result;
|
|
632
640
|
}
|
|
@@ -678,7 +686,18 @@ function filterJsonlByModel(path, field, model) {
|
|
|
678
686
|
if (match) removed += 1;
|
|
679
687
|
else kept.push(line);
|
|
680
688
|
}
|
|
681
|
-
if (removed > 0)
|
|
689
|
+
if (removed > 0) {
|
|
690
|
+
if (kept.length > 0) writeFileSync(path, kept.join("\n") + "\n", "utf8");
|
|
691
|
+
else {
|
|
692
|
+
// 全部清空时直接删掉文件,别留 0 字节壳让扫描把它当成一行成绩。
|
|
693
|
+
// (若文件被评测进程占用导致 unlink 失败,退化为写空文件。)
|
|
694
|
+
try { unlinkSync(path); }
|
|
695
|
+
catch { writeFileSync(path, "", "utf8"); }
|
|
696
|
+
}
|
|
697
|
+
} else if (kept.length === 0) {
|
|
698
|
+
// 本来就是 0 字节空壳(早期删除留下的):一并清掉
|
|
699
|
+
try { unlinkSync(path); } catch { /* 被占用则保留 */ }
|
|
700
|
+
}
|
|
682
701
|
return removed;
|
|
683
702
|
}
|
|
684
703
|
|
|
@@ -1171,7 +1190,11 @@ function apply(ctx) {
|
|
|
1171
1190
|
const rows = Array.isArray(body.rows) ? body.rows.slice(0, 200) : [];
|
|
1172
1191
|
const layout = livebenchLayout();
|
|
1173
1192
|
const tasks = scanCategoryTasks(layout.dataDir);
|
|
1193
|
+
// 正在跑的模型不删:文件还被评测进程持有,删了会损坏本次结果
|
|
1194
|
+
const activeModels = new Set();
|
|
1195
|
+
for (const r of runs.values()) if (r.exitCode === null) activeModels.add(r.displayName);
|
|
1174
1196
|
let removed = 0;
|
|
1197
|
+
let skipped = 0;
|
|
1175
1198
|
for (const row of rows) {
|
|
1176
1199
|
if (!row || typeof row !== "object") continue;
|
|
1177
1200
|
const model = typeof row.model === "string" ? row.model : "";
|
|
@@ -1181,12 +1204,13 @@ function apply(ctx) {
|
|
|
1181
1204
|
if (!/^[A-Za-z0-9._@-]{1,160}$/.test(model)) continue;
|
|
1182
1205
|
if (!/^[A-Za-z0-9_]{1,80}$/.test(category) || !/^[A-Za-z0-9_]{1,80}$/.test(task)) continue;
|
|
1183
1206
|
if (!(tasks[category] && tasks[category][task])) continue;
|
|
1207
|
+
if (activeModels.has(model)) { skipped += 1; continue; }
|
|
1184
1208
|
const taskDir = join(layout.dataDir, category, task);
|
|
1185
1209
|
removed += filterJsonlByModel(join(taskDir, "model_answer", `${model}.jsonl`), "model_id", model);
|
|
1186
1210
|
removed += filterJsonlByModel(join(taskDir, "model_judgment", "ground_truth_judgment.jsonl"), "model", model);
|
|
1187
1211
|
}
|
|
1188
1212
|
resultsCache = { at: 0, data: null };
|
|
1189
|
-
sendJson(res, 200, { ok: true, removed });
|
|
1213
|
+
sendJson(res, 200, { ok: true, removed, skipped });
|
|
1190
1214
|
},
|
|
1191
1215
|
}), `${name}: delete route`);
|
|
1192
1216
|
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "dsh-livebench-panel",
|
|
3
|
-
"version": "0.2.
|
|
3
|
+
"version": "0.2.14",
|
|
4
4
|
"description": "DSH web plugin: a LiveBench tab in the Trajectory view (right of 对话/轨迹). Run LiveBench evaluations against every model configured in the DeepSeek Harness — pick provider/model, category, task, release and question range from dropdowns, watch progress, and read scores in place.",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"type": "module",
|