dsh-skill-eval 1.20.2 → 1.20.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +43 -0
- package/README.md +13 -6
- package/README.zh.md +11 -4
- package/client.js +230 -76
- package/docs/ACCEPTANCE.md +1 -1
- package/docs/AUDIT.md +18 -0
- package/docs/DETAIL.md +11 -6
- package/docs/ENTRYPOINTS.md +1 -1
- package/docs/RELEASE.md +2 -2
- package/docs/UI-REVIEW.md +8 -8
- package/docs/WORKFLOW.md +1 -1
- package/index.js +98 -36
- package/lib/accept.js +1 -1
- package/lib/advice.js +12 -2
- package/lib/dev.js +1 -1
- package/lib/history.js +3 -2
- package/lib/html.js +9 -0
- package/lib/recorder.js +16 -6
- package/lib/render.js +5 -7
- package/lib/report.js +49 -6
- package/lib/score.js +40 -0
- package/lib/settings.js +26 -7
- package/lib/snapshot.js +111 -27
- package/lib/store.js +55 -1
- package/lib/validity.js +12 -2
- package/package.json +1 -1
- package/scripts/accept.mjs +7 -6
- package/scripts/doctor.mjs +46 -11
- package/scripts/lib.mjs +35 -0
- package/scripts/verify-live.mjs +16 -6
package/docs/UI-REVIEW.md
CHANGED
|
@@ -81,16 +81,16 @@ headless Chrome 按 520px 面板宽 / 900px 配置页截图比对;不是真实
|
|
|
81
81
|
方法:真实宿主路由(`curl`)+ 用 jsdom 挂载**线上真实载荷** `/skill-eval/data/1.20.1`
|
|
82
82
|
后逐个点击、切标签;宿主侧再用 `scripts/verify-live.mjs` 对照。不是截图,也不是只看代码。
|
|
83
83
|
|
|
84
|
-
| 控件
|
|
85
|
-
|
|
|
86
|
-
| 配置
|
|
87
|
-
| 刷新
|
|
88
|
-
| 复制给 agent | 写入完整 report JSON(实测 249,630 字符),失败回退到「复制失败」
|
|
89
|
-
| MCP
|
|
90
|
-
| 开发者
|
|
84
|
+
| 控件 | 实测结果 | 结论 |
|
|
85
|
+
| ------------ | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------- |
|
|
86
|
+
| 配置 | 旧实现 `window.open('/skill-eval/board/<v>/config')`。DSH 桌面壳的 `setWindowOpenHandler` 只把 **https** URL 交给系统浏览器,其余一律 `deny`,而看板跑在 `http://127.0.0.1:<port>` → 点击**完全没有反应** | **已修**:改为面板内设置弹窗(复用 `SettingsForm`),Esc/点背景关闭,Tab 焦点限制在弹窗内;桌面壳里隐藏无法兑现的「在网页里打开」 |
|
|
87
|
+
| 刷新 | 点击后重新 GET `/skill-eval/data/<v>`,loading 期间禁用 | 正常 |
|
|
88
|
+
| 复制给 agent | 写入完整 report JSON(实测 249,630 字符),失败回退到「复制失败」 | 正常;**数据未到达前按钮禁用**(原先会复制 `{}` 并提示已复制) |
|
|
89
|
+
| MCP | 图表 + 表格 + 建议/出错工具;线上 context7:1 次调用、2606ms、0% 错误、88.9 B | 数值正确;阈值线原硬编码 3000ms,**改为跟随 `mcpP95Ms` 配置**(旧载荷仍回退 3000) |
|
|
90
|
+
| 开发者 | 原先只回显评分配置,且「数据接口」硬编码 `/skill-eval/data`,看不到真正的诊断 | **已修**:data 增加 `dev` 块(已加载模块/磁盘版本 + skew、路由与工具数、settings 文件、warning 数),标签显示版本化 URL 与「需要重启」 |
|
|
91
91
|
|
|
92
92
|
验证:`node scripts/verify-live.mjs http://127.0.0.1:55265` 对 1.20.1 全 PASS;
|
|
93
|
-
`node --test test/` **
|
|
93
|
+
`node --test test/` **258 项全绿**(新增「配置按钮不再调用 window.open,改为面板内表单」用例)。
|
|
94
94
|
|
|
95
95
|
已知取舍(未改):**「复制给 agent」复制的是完整 JSON**(≈250KB,含 400 条步进记录)。
|
|
96
96
|
如果目标是让 agent 直接读,`buildMarkdown()` 的摘要更省上下文;但完整 JSON 便于排障,
|
package/docs/WORKFLOW.md
CHANGED
|
@@ -64,7 +64,7 @@
|
|
|
64
64
|
| 想看整体汇报 | 侧边栏面板图标「技能评测」→ 主面板(或浏览器打开 `/skill-eval/board/<版本>`) | 技能/运行/MCP/开发者标签,含下一步与效度 |
|
|
65
65
|
| 想改评分标准 | 设置 → 插件 → `dsh-skill-eval` 包页面 → 配置 | 权重/阈值/预算/开关,保存即生效 |
|
|
66
66
|
| 入口速查 | [ENTRYPOINTS.md](ENTRYPOINTS.md) | 三个日常入口 + 网页/命令/终端 |
|
|
67
|
-
| 发版前自检 | `dsh-skill-eval-doctor` + `accept` + `verify` | 19 / 17 /
|
|
67
|
+
| 发版前自检 | `dsh-skill-eval-doctor` + `accept` + `verify` | 19 / 17 / 7+2 项:打包、代次、产品面 |
|
|
68
68
|
|
|
69
69
|
## 判定规则(避免自欺)
|
|
70
70
|
|
package/index.js
CHANGED
|
@@ -36,7 +36,7 @@ import {
|
|
|
36
36
|
normalizeSettings,
|
|
37
37
|
parseFormSettings,
|
|
38
38
|
renderConfigHtml,
|
|
39
|
-
|
|
39
|
+
updateSettings,
|
|
40
40
|
SETTINGS_FIELDS,
|
|
41
41
|
SETTINGS_FILE,
|
|
42
42
|
} from './lib/settings.js'
|
|
@@ -276,11 +276,11 @@ function log(ctx, level, message) {
|
|
|
276
276
|
const VIEWS = ['summary', 'mcp', 'runs', 'standard', 'loop', 'validity', 'history', 'impact', 'dev', 'turns']
|
|
277
277
|
|
|
278
278
|
/** Read the files the acceptance check needs, best effort. */
|
|
279
|
-
async function acceptanceFiles(store, readFile) {
|
|
279
|
+
async function acceptanceFiles(store, readFile, snapshotDir = path.join(store.dir, 'reports')) {
|
|
280
280
|
const dir = store.dir
|
|
281
281
|
const eventsPath = path.join(dir, 'events.jsonl')
|
|
282
|
-
const snapshotMarkdownPath = path.join(
|
|
283
|
-
const snapshotJsonPath = path.join(
|
|
282
|
+
const snapshotMarkdownPath = path.join(snapshotDir, 'latest.md')
|
|
283
|
+
const snapshotJsonPath = path.join(snapshotDir, 'latest.json')
|
|
284
284
|
let events = []
|
|
285
285
|
try {
|
|
286
286
|
const raw = await readFile(eventsPath, 'utf8')
|
|
@@ -362,7 +362,7 @@ async function runReport(analyzer, args, config) {
|
|
|
362
362
|
config: config,
|
|
363
363
|
routes: DIAGNOSTICS.routes,
|
|
364
364
|
records,
|
|
365
|
-
store: { dir: analyzer.store.dir, failed: analyzer.store.failed,
|
|
365
|
+
store: { dir: analyzer.store.dir, failed: analyzer.store.failed, queued: analyzer.store.pending },
|
|
366
366
|
tools: DIAGNOSTICS.tools,
|
|
367
367
|
inspectProvider: DIAGNOSTICS.inspectProvider ? 'skill-eval' : null,
|
|
368
368
|
warnings: DIAGNOSTICS.warnings,
|
|
@@ -489,6 +489,16 @@ export function apply(ctx, config) {
|
|
|
489
489
|
}
|
|
490
490
|
|
|
491
491
|
const analyzer = createAnalyzer(ctx, cfg)
|
|
492
|
+
ctx.effect(
|
|
493
|
+
() => () => {
|
|
494
|
+
try {
|
|
495
|
+
analyzer.dispose()
|
|
496
|
+
} catch {
|
|
497
|
+
// best effort on shutdown
|
|
498
|
+
}
|
|
499
|
+
},
|
|
500
|
+
'skill-eval: analyzer',
|
|
501
|
+
)
|
|
492
502
|
analyzer.store.onError = (message) => {
|
|
493
503
|
DIAGNOSTICS.warnings.push(`store write failed: ${message}`)
|
|
494
504
|
log(ctx, 'warn', `skill-eval store write failed: ${message}`)
|
|
@@ -513,27 +523,38 @@ export function apply(ctx, config) {
|
|
|
513
523
|
// A settled turn writes the aggregated report to disk. This is the "result
|
|
514
524
|
// after every run" surface: it costs zero tokens because it never touches the
|
|
515
525
|
// model context.
|
|
516
|
-
|
|
517
|
-
|
|
518
|
-
|
|
519
|
-
|
|
520
|
-
|
|
521
|
-
|
|
522
|
-
|
|
523
|
-
|
|
524
|
-
|
|
525
|
-
|
|
526
|
+
// Always create the snapshotter and read the switches live: reportOnTurnEnd
|
|
527
|
+
// only controls latest.md/json, turnReports controls the per-turn boards, and
|
|
528
|
+
// a settings save must not need a restart (the UI promises immediate apply).
|
|
529
|
+
const snapshots = createSnapshots({
|
|
530
|
+
analyzer,
|
|
531
|
+
dir: cfg.reportDir ? String(cfg.reportDir) : path.join(analyzer.store.dir, 'reports'),
|
|
532
|
+
log: (message) => log(ctx, 'warn', message),
|
|
533
|
+
enabled: () => cfg.reportOnTurnEnd !== false,
|
|
534
|
+
turnReports: () => cfg.turnReports !== false,
|
|
535
|
+
turnReportsKeep: cfg.turnReportsKeep ?? 50,
|
|
536
|
+
})
|
|
526
537
|
|
|
527
538
|
if (snapshots) {
|
|
528
|
-
ctx.on('agent/turn-stopping', () => {
|
|
539
|
+
const offTurnStopping = ctx.on('agent/turn-stopping', (payload) => {
|
|
529
540
|
try {
|
|
530
|
-
|
|
541
|
+
// Label the board with the turn that actually settled. Without this a
|
|
542
|
+
// concurrent subagent turn (or a new load inside the debounce) would
|
|
543
|
+
// make the writer pick whatever turnLoads is globally newest.
|
|
544
|
+
const sessionId = payload?.agent ? String(payload.agent.id) : null
|
|
545
|
+
const turn = typeof payload?.turn === 'number' ? payload.turn : null
|
|
546
|
+
snapshots.request(sessionId ? { sessionId, turn } : null)
|
|
531
547
|
} catch {
|
|
532
548
|
// intentionally ignored: snapshot request is best effort on turn stop
|
|
533
549
|
}
|
|
534
550
|
})
|
|
535
551
|
ctx.effect(
|
|
536
552
|
() => () => {
|
|
553
|
+
try {
|
|
554
|
+
if (typeof offTurnStopping === 'function') offTurnStopping()
|
|
555
|
+
} catch {
|
|
556
|
+
// best effort
|
|
557
|
+
}
|
|
537
558
|
void snapshots.dispose()
|
|
538
559
|
},
|
|
539
560
|
'skill-eval: snapshots',
|
|
@@ -639,8 +660,10 @@ export function apply(ctx, config) {
|
|
|
639
660
|
const keyword = target.toLowerCase()
|
|
640
661
|
if (keyword === 'accept' || keyword === 'doctor') {
|
|
641
662
|
const report = await analyzer.build()
|
|
642
|
-
const files = await acceptanceFiles(
|
|
643
|
-
|
|
663
|
+
const files = await acceptanceFiles(
|
|
664
|
+
analyzer.store,
|
|
665
|
+
(file) => import('node:fs/promises').then((fs) => fs.readFile(file, 'utf8')),
|
|
666
|
+
cfg.reportDir ? String(cfg.reportDir) : path.join(analyzer.store.dir, 'reports'),
|
|
644
667
|
)
|
|
645
668
|
const acceptance = checkInProcessReport(report, { version: PLUGIN_VERSION ?? 'unknown', files })
|
|
646
669
|
return { kind: 'success', text: '\n' + renderAcceptance(acceptance) + '\n' }
|
|
@@ -712,6 +735,38 @@ export function apply(ctx, config) {
|
|
|
712
735
|
const clientBase = '/skill-eval'
|
|
713
736
|
const dataPath = `${clientBase}/data`
|
|
714
737
|
|
|
738
|
+
// Writes are same-origin fenced. Comparing Origin to Host alone does not
|
|
739
|
+
// stop DNS rebinding (both are the attacker's domain after rebinding), so
|
|
740
|
+
// the Host itself must be the server's own origin or a loopback name.
|
|
741
|
+
let serverOrigin = null
|
|
742
|
+
const hostnameOf = (value) => {
|
|
743
|
+
try {
|
|
744
|
+
return new URL(String(value).includes('://') ? String(value) : 'http://' + String(value)).hostname.toLowerCase()
|
|
745
|
+
} catch {
|
|
746
|
+
return ''
|
|
747
|
+
}
|
|
748
|
+
}
|
|
749
|
+
const allowedWriteHosts = () => {
|
|
750
|
+
const hosts = new Set(['localhost', '127.0.0.1', '[::1]', '::1'])
|
|
751
|
+
const own = serverOrigin ? hostnameOf(serverOrigin) : ''
|
|
752
|
+
if (own) hosts.add(own)
|
|
753
|
+
return hosts
|
|
754
|
+
}
|
|
755
|
+
const rememberOrigin = (webServer) => {
|
|
756
|
+
try {
|
|
757
|
+
const value = webServer.getOrigin()
|
|
758
|
+
if (typeof value === 'string' && value) serverOrigin = value
|
|
759
|
+
} catch {
|
|
760
|
+
// keep the loopback fallback
|
|
761
|
+
}
|
|
762
|
+
}
|
|
763
|
+
ctx.inject(['webServer'], (c) => {
|
|
764
|
+
c.effect(() => {
|
|
765
|
+
rememberOrigin(c.webServer)
|
|
766
|
+
return () => {}
|
|
767
|
+
}, 'skill-eval: origin')
|
|
768
|
+
})
|
|
769
|
+
|
|
715
770
|
// The board's dev tab shows what the agent's `view:"dev"` report shows in
|
|
716
771
|
// short form: which generation is loaded, whether the package on disk moved
|
|
717
772
|
// ahead of it (the ESM-cache restart trap), how many routes this generation
|
|
@@ -736,7 +791,12 @@ export function apply(ctx, config) {
|
|
|
736
791
|
'content-type': 'application/json; charset=utf-8',
|
|
737
792
|
'cache-control': 'no-store',
|
|
738
793
|
})
|
|
739
|
-
|
|
794
|
+
// The pages may live under a custom dashboardPath; ship their exact
|
|
795
|
+
// paths so the client never hardcodes a route the host did not register.
|
|
796
|
+
const pages = PLUGIN_VERSION
|
|
797
|
+
? { board: `${base}/board/${PLUGIN_VERSION}`, config: `${base}/board/${PLUGIN_VERSION}/config` }
|
|
798
|
+
: { board: base, config: `${base}/config` }
|
|
799
|
+
res.end(JSON.stringify({ ...report, dev: diagnosticsPayload(), pages }, null, 2))
|
|
740
800
|
}
|
|
741
801
|
|
|
742
802
|
const bodyTooLarge = () => Object.assign(new Error('request body too large'), { statusCode: 413 })
|
|
@@ -781,11 +841,13 @@ export function apply(ctx, config) {
|
|
|
781
841
|
const trustedWrite = (req, { requireHeader = false } = {}) => {
|
|
782
842
|
const site = String(req.headers?.['sec-fetch-site'] ?? '').toLowerCase()
|
|
783
843
|
if (site === 'cross-site') return false
|
|
844
|
+
const host = String(req.headers?.['host'] ?? '')
|
|
845
|
+
if (!allowedWriteHosts().has(hostnameOf(host))) return false
|
|
784
846
|
const origin = req.headers?.origin
|
|
785
847
|
if (origin) {
|
|
786
848
|
try {
|
|
787
849
|
const parsed = new URL(origin)
|
|
788
|
-
if (
|
|
850
|
+
if (parsed.host !== host) return false
|
|
789
851
|
} catch {
|
|
790
852
|
return false
|
|
791
853
|
}
|
|
@@ -827,7 +889,7 @@ export function apply(ctx, config) {
|
|
|
827
889
|
send(400, renderConfigHtml({ config: cfg, base: pageBase, errors, lang }))
|
|
828
890
|
return
|
|
829
891
|
}
|
|
830
|
-
if (settingsDir) await
|
|
892
|
+
if (settingsDir) await updateSettings(settingsDir, values)
|
|
831
893
|
applySettings(cfg, values)
|
|
832
894
|
analyzer.invalidate()
|
|
833
895
|
DIAGNOSTICS.settings = loadSettingsSync(settingsDir)
|
|
@@ -908,7 +970,7 @@ export function apply(ctx, config) {
|
|
|
908
970
|
send(400, { saved: false, errors, values: settingsPayload().values })
|
|
909
971
|
return
|
|
910
972
|
}
|
|
911
|
-
if (settingsDir) await
|
|
973
|
+
if (settingsDir) await updateSettings(settingsDir, values)
|
|
912
974
|
applySettings(cfg, values)
|
|
913
975
|
analyzer.invalidate()
|
|
914
976
|
DIAGNOSTICS.settings = loadSettingsSync(settingsDir)
|
|
@@ -1093,19 +1155,19 @@ export function apply(ctx, config) {
|
|
|
1093
1155
|
|
|
1094
1156
|
const SETTINGS_KEYS = SETTINGS_FIELDS.map((field) => field.key)
|
|
1095
1157
|
|
|
1096
|
-
|
|
1097
|
-
|
|
1098
|
-
c.
|
|
1099
|
-
|
|
1100
|
-
|
|
1101
|
-
|
|
1102
|
-
|
|
1103
|
-
|
|
1104
|
-
|
|
1105
|
-
|
|
1106
|
-
|
|
1107
|
-
})
|
|
1158
|
+
ctx.inject(['systemPrompt'], (c) => {
|
|
1159
|
+
c.effect(() => {
|
|
1160
|
+
const stop = c.systemPrompt.section({
|
|
1161
|
+
name: 'skill-eval:usage',
|
|
1162
|
+
order: 118,
|
|
1163
|
+
text: () =>
|
|
1164
|
+
cfg.promptHint === false
|
|
1165
|
+
? ''
|
|
1166
|
+
: 'Skill telemetry is recorded for every `skill` load. When asked how a skill performed or how to improve it, call `skill_eval_report` (pass {"skill":"<name>"} for one skill) and act on the cited fixes instead of guessing. Before editing a skill named in the report, screen the change with `skill_eval_review` (leakage critic) and record the hypothesis; ask `skill_eval_report` with view:"loop" for the next step and view:"validity" for what the evidence can and cannot support.',
|
|
1167
|
+
})
|
|
1168
|
+
return () => safe(stop)
|
|
1169
|
+
}, 'skill-eval: prompt section')
|
|
1170
|
+
})
|
|
1108
1171
|
|
|
1109
1172
|
log(ctx, 'info', `skill-eval: recording enabled, data dir ${analyzer.store.dir}`)
|
|
1110
1173
|
}
|
|
1111
|
-
|
package/lib/accept.js
CHANGED
|
@@ -325,7 +325,7 @@ export function checkVersionedPayload(versioned, canonical) {
|
|
|
325
325
|
results.push(
|
|
326
326
|
result(
|
|
327
327
|
'canonical-route',
|
|
328
|
-
|
|
328
|
+
canonicalKeys === versionedKeys,
|
|
329
329
|
canonicalKeys === versionedKeys
|
|
330
330
|
? 'canonical and versioned routes agree'
|
|
331
331
|
: 'canonical route still answers from an older handler (the versioned route is current; a restart fixes the canonical one)',
|
package/lib/advice.js
CHANGED
|
@@ -6,7 +6,7 @@
|
|
|
6
6
|
* severity: 'critical' | 'warning' | 'info'
|
|
7
7
|
*/
|
|
8
8
|
|
|
9
|
-
import { bootstrapRate } from './score.js'
|
|
9
|
+
import { bootstrapCallRate, bootstrapRate } from './score.js'
|
|
10
10
|
|
|
11
11
|
const SEVERITY_ORDER = { critical: 0, warning: 1, info: 2 }
|
|
12
12
|
|
|
@@ -190,6 +190,15 @@ function gated(windows, isHit, threshold, resamples) {
|
|
|
190
190
|
return { level: 'none', ci }
|
|
191
191
|
}
|
|
192
192
|
|
|
193
|
+
function gatedCalls(windows, threshold, resamples) {
|
|
194
|
+
if (!Array.isArray(windows) || windows.length === 0) return { level: 'strong', ci: null }
|
|
195
|
+
const ci = bootstrapCallRate(windows, { resamples, seed: 17 })
|
|
196
|
+
if (!ci) return { level: 'mean', ci: null }
|
|
197
|
+
if (ci.low > threshold) return { level: 'strong', ci }
|
|
198
|
+
if (ci.mean > threshold) return { level: 'weak', ci }
|
|
199
|
+
return { level: 'none', ci }
|
|
200
|
+
}
|
|
201
|
+
|
|
193
202
|
function noiseNote(ci, threshold, level) {
|
|
194
203
|
if (!ci) return ''
|
|
195
204
|
const base = ` (95% interval ${Math.round(ci.low * 100)}-${Math.round(ci.high * 100)}%, threshold ${Math.round(
|
|
@@ -227,7 +236,8 @@ export function buildAdvice({
|
|
|
227
236
|
})
|
|
228
237
|
}
|
|
229
238
|
|
|
230
|
-
|
|
239
|
+
// Gate on the same call-weighted rate the evidence reports.
|
|
240
|
+
const errGate = gating ? gatedCalls(windows, 0.25, 400) : { level: 'strong', ci: null }
|
|
231
241
|
if (stats.errRate !== null && stats.errRate > 0.25 && errGate.level !== 'none') {
|
|
232
242
|
out.push({
|
|
233
243
|
id: 'post-load-errors',
|
package/lib/dev.js
CHANGED
|
@@ -103,7 +103,7 @@ export function buildDevReport(input = {}) {
|
|
|
103
103
|
store: {
|
|
104
104
|
dir: store.dir ?? dataDir,
|
|
105
105
|
failed: store.failed ?? null,
|
|
106
|
-
queued:
|
|
106
|
+
queued: Number.isFinite(store.queued) ? store.queued : null,
|
|
107
107
|
records: records.length,
|
|
108
108
|
counts,
|
|
109
109
|
types,
|
package/lib/history.js
CHANGED
|
@@ -6,6 +6,7 @@
|
|
|
6
6
|
* reported as inconclusive rather than as an improvement.
|
|
7
7
|
*/
|
|
8
8
|
|
|
9
|
+
import { formatTimestamp } from './html.js'
|
|
9
10
|
import { foldSamples } from './store.js'
|
|
10
11
|
|
|
11
12
|
const rate = (list, predicate) => (list.length === 0 ? null : list.filter(predicate).length / list.length)
|
|
@@ -109,7 +110,7 @@ export function renderHistoryMarkdown(history) {
|
|
|
109
110
|
'|---|---|---|---|---|---|---|',
|
|
110
111
|
]
|
|
111
112
|
for (const load of history.loads.slice(-15)) {
|
|
112
|
-
const when =
|
|
113
|
+
const when = formatTimestamp(load.at)
|
|
113
114
|
lines.push(
|
|
114
115
|
`| ${when} | ${load.ok ? 'yes' : 'no'} | ${load.calls ?? '—'} | ${load.errors ?? '—'} | ${load.corrections} | ${
|
|
115
116
|
load.loadMs ?? '—'
|
|
@@ -124,7 +125,7 @@ export function renderImpactMarkdown(impact) {
|
|
|
124
125
|
return `No screened edit recorded for \`${impact?.skill ?? '?'}\` yet. Use skill_eval_review before changing it.`
|
|
125
126
|
const lines = [`## Edit impact — \`${impact.skill}\``, '', `${impact.edits} screened edit(s)`, '']
|
|
126
127
|
for (const comparison of impact.comparisons) {
|
|
127
|
-
const when =
|
|
128
|
+
const when = formatTimestamp(comparison.at)
|
|
128
129
|
lines.push(`### ${when} · ${comparison.dimension ?? 'unspecified'} · ${comparison.verdict}`)
|
|
129
130
|
if (comparison.hypothesis) lines.push(`hypothesis: ${comparison.hypothesis}`)
|
|
130
131
|
lines.push(`- ${comparison.note}`)
|
package/lib/html.js
CHANGED
|
@@ -1,4 +1,13 @@
|
|
|
1
1
|
/** Escape text for the few HTML strings built outside React. */
|
|
2
|
+
/** Format a finite epoch-ms timestamp, or an em dash when it is not a valid date. */
|
|
3
|
+
export function formatTimestamp(value) {
|
|
4
|
+
const n = Number(value)
|
|
5
|
+
if (!Number.isFinite(n)) return '—'
|
|
6
|
+
const date = new Date(n)
|
|
7
|
+
if (Number.isNaN(date.getTime())) return '—'
|
|
8
|
+
return date.toISOString().replace('T', ' ').slice(0, 19)
|
|
9
|
+
}
|
|
10
|
+
|
|
2
11
|
export function escapeHtml(value) {
|
|
3
12
|
return String(value ?? '')
|
|
4
13
|
.replace(/&/g, '&')
|
package/lib/recorder.js
CHANGED
|
@@ -88,7 +88,8 @@ export function createRecorder(ctx, config, { store, resolveSkill }) {
|
|
|
88
88
|
const sessions = new Map() // sessionId -> { windows: [] }
|
|
89
89
|
|
|
90
90
|
const windowLimit = Math.max(1, Number(config.windowToolCalls ?? 12))
|
|
91
|
-
|
|
91
|
+
// Read live: the settings form advertises windowTtlMs as applying immediately.
|
|
92
|
+
const ttlMs = () => Math.max(60_000, Number(config.windowTtlMs ?? 30 * 60_000))
|
|
92
93
|
|
|
93
94
|
function stateFor(sessionId) {
|
|
94
95
|
const existing = sessions.get(sessionId)
|
|
@@ -313,7 +314,15 @@ export function createRecorder(ctx, config, { store, resolveSkill }) {
|
|
|
313
314
|
const seenKey = `${sessionId}|${id}`
|
|
314
315
|
if (seenInvocations.has(seenKey)) continue
|
|
315
316
|
seenInvocations.add(seenKey)
|
|
316
|
-
|
|
317
|
+
// Evict the oldest keys instead of clearing everything: a wholesale
|
|
318
|
+
// reset would let a repeated invocation message be recorded twice.
|
|
319
|
+
if (seenInvocations.size > 4000) {
|
|
320
|
+
for (let i = 0; i < 500; i += 1) {
|
|
321
|
+
const oldest = seenInvocations.values().next().value
|
|
322
|
+
if (oldest === undefined) break
|
|
323
|
+
seenInvocations.delete(oldest)
|
|
324
|
+
}
|
|
325
|
+
}
|
|
317
326
|
const state = stateFor(sessionId)
|
|
318
327
|
const windowId = `${sessionId}:${id}`
|
|
319
328
|
void store.append({
|
|
@@ -439,7 +448,7 @@ export function createRecorder(ctx, config, { store, resolveSkill }) {
|
|
|
439
448
|
}
|
|
440
449
|
}
|
|
441
450
|
const newest = state.windows[state.windows.length - 1]
|
|
442
|
-
if (newest && nowMs() - newest.lastActivityAt > ttlMs) closeWindows(state, 'stale')
|
|
451
|
+
if (newest && nowMs() - newest.lastActivityAt > ttlMs()) closeWindows(state, 'stale')
|
|
443
452
|
|
|
444
453
|
if (exec.name === 'skill') {
|
|
445
454
|
const skill = skillNameOf(exec.arguments)
|
|
@@ -462,14 +471,15 @@ export function createRecorder(ctx, config, { store, resolveSkill }) {
|
|
|
462
471
|
turn: turns.get(sessionId),
|
|
463
472
|
windowId,
|
|
464
473
|
})
|
|
465
|
-
|
|
466
|
-
|
|
474
|
+
if (ok) {
|
|
475
|
+
openWindow(state, skill, sessionId, exec.callId, steps.get(sessionId), originOf(exec.agent))
|
|
467
476
|
void captureStatic(
|
|
468
477
|
skill,
|
|
469
478
|
exec.agent,
|
|
470
479
|
exec.signal,
|
|
471
480
|
result?.value && typeof result.value.content === 'string' ? result.value.content : undefined,
|
|
472
481
|
)
|
|
482
|
+
}
|
|
473
483
|
return
|
|
474
484
|
}
|
|
475
485
|
|
|
@@ -540,7 +550,7 @@ export function createRecorder(ctx, config, { store, resolveSkill }) {
|
|
|
540
550
|
const state = sessions.get(sessionId)
|
|
541
551
|
if (!state || state.windows.length === 0) return
|
|
542
552
|
const newest = state.windows[state.windows.length - 1]
|
|
543
|
-
if (nowMs() - newest.lastActivityAt > ttlMs) {
|
|
553
|
+
if (nowMs() - newest.lastActivityAt > ttlMs()) {
|
|
544
554
|
closeWindows(state, 'stale')
|
|
545
555
|
return
|
|
546
556
|
}
|
package/lib/render.js
CHANGED
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
import { localizeReport } from './advice.js'
|
|
4
4
|
import { buildLoopPlans } from './loop.js'
|
|
5
|
-
import { escapeHtml as esc } from './html.js'
|
|
5
|
+
import { escapeHtml as esc, formatTimestamp } from './html.js'
|
|
6
6
|
|
|
7
7
|
function bar(value, width = 10) {
|
|
8
8
|
if (!Number.isFinite(value)) return '·'.repeat(width)
|
|
@@ -358,7 +358,7 @@ const BOARD = {
|
|
|
358
358
|
evidenceTitle: 'What the evidence supports',
|
|
359
359
|
coverage: (v) =>
|
|
360
360
|
`coverage: ${v.coverage.skills} skill(s) · ${v.coverage.withCI} with 95% intervals · ${v.coverage.provisional} provisional · ranking ${
|
|
361
|
-
v.ranking.significant ? 'significant' : 'not significant'
|
|
361
|
+
v.ranking.unknown ? 'not enough evidence' : v.ranking.significant ? 'significant' : 'not significant'
|
|
362
362
|
}`,
|
|
363
363
|
canSay: 'Can say',
|
|
364
364
|
cannotSay: 'Cannot say yet',
|
|
@@ -392,7 +392,7 @@ const BOARD = {
|
|
|
392
392
|
evidenceTitle: '证据能支持什么',
|
|
393
393
|
coverage: (v) =>
|
|
394
394
|
`覆盖:${v.coverage.skills} 个技能 · ${v.coverage.withCI} 个有 95% 区间 · ${v.coverage.provisional} 个临时分 · 排名${
|
|
395
|
-
v.ranking.significant ? '显著' : '不显著'
|
|
395
|
+
v.ranking.unknown ? '证据不足' : v.ranking.significant ? '显著' : '不显著'
|
|
396
396
|
}`,
|
|
397
397
|
canSay: '可以断言',
|
|
398
398
|
cannotSay: '还不能断言',
|
|
@@ -466,13 +466,11 @@ export function renderHtml(input, options = {}) {
|
|
|
466
466
|
const editRows = (report.edits ?? [])
|
|
467
467
|
.slice(0, 8)
|
|
468
468
|
.map((edit) => [
|
|
469
|
-
esc(
|
|
470
|
-
Number.isFinite(Number(edit.at)) ? new Date(Number(edit.at)).toISOString().replace('T', ' ').slice(0, 19) : '—',
|
|
471
|
-
),
|
|
469
|
+
esc(formatTimestamp(edit.at)),
|
|
472
470
|
esc(edit.skill ?? ''),
|
|
473
471
|
esc(edit.dimension ?? '—'),
|
|
474
472
|
`<b>${esc(edit.verdict ?? '')}</b>`,
|
|
475
|
-
esc((edit.hypothesis ?? '').slice(0, 70)),
|
|
473
|
+
esc(String(edit.hypothesis ?? '').slice(0, 70)),
|
|
476
474
|
])
|
|
477
475
|
const ledgerBlock = tableSection(t.ledgerTitle, t.ledgerHead, editRows)
|
|
478
476
|
const otherLang = lang === 'zh' ? 'en' : 'zh'
|
package/lib/report.js
CHANGED
|
@@ -44,7 +44,10 @@ export function rankingOf(entries) {
|
|
|
44
44
|
const scored = (entries ?? []).filter(
|
|
45
45
|
(entry) => entry && entry.score && entry.score.ci && typeof entry.score.total === 'number',
|
|
46
46
|
)
|
|
47
|
-
|
|
47
|
+
// Fewer than two intervals means nothing was compared; claiming a
|
|
48
|
+
// significant ranking here would be a fabricated statistic.
|
|
49
|
+
if (scored.length < 2)
|
|
50
|
+
return { significant: null, note: 'not enough scored skills with 95% intervals to rank them', ties: [] }
|
|
48
51
|
const ties = []
|
|
49
52
|
for (let i = 1; i < scored.length; i += 1) {
|
|
50
53
|
const a = scored[i - 1]
|
|
@@ -118,6 +121,34 @@ export function normalizeWeights(input) {
|
|
|
118
121
|
return out
|
|
119
122
|
}
|
|
120
123
|
|
|
124
|
+
/**
|
|
125
|
+
* Display weights as integers that always sum to 100. Independent rounding can
|
|
126
|
+
* show 29+29+29+14=101 for a legal 2/2/2/1 configuration, which then fails the
|
|
127
|
+
* release check; largest-remainder keeps the displayed total honest.
|
|
128
|
+
*/
|
|
129
|
+
export function percentWeights(weights) {
|
|
130
|
+
const keys = Object.keys(weights ?? {})
|
|
131
|
+
if (keys.length === 0) return {}
|
|
132
|
+
const exact = keys.map((key) => Math.max(0, Number(weights[key]) || 0) * 100)
|
|
133
|
+
const base = exact.map((value) => Math.floor(value))
|
|
134
|
+
let remainder = 100 - base.reduce((sum, value) => sum + value, 0)
|
|
135
|
+
const order = exact
|
|
136
|
+
.map((value, index) => ({ index, frac: value - Math.floor(value) }))
|
|
137
|
+
.sort((a, b) => b.frac - a.frac)
|
|
138
|
+
for (const item of order) {
|
|
139
|
+
if (remainder <= 0) break
|
|
140
|
+
base[item.index] += 1
|
|
141
|
+
remainder -= 1
|
|
142
|
+
}
|
|
143
|
+
while (remainder > 0) {
|
|
144
|
+
let largest = 0
|
|
145
|
+
for (let i = 1; i < base.length; i += 1) if (base[i] > base[largest]) largest = i
|
|
146
|
+
base[largest] += 1
|
|
147
|
+
remainder -= 1
|
|
148
|
+
}
|
|
149
|
+
return Object.fromEntries(keys.map((key, index) => [key, base[index]]))
|
|
150
|
+
}
|
|
151
|
+
|
|
121
152
|
export function resolveThresholds(input) {
|
|
122
153
|
const raw = { ...DEFAULT_THRESHOLDS, ...(input && typeof input === 'object' ? input : {}) }
|
|
123
154
|
const pick = (key, fallback) => {
|
|
@@ -153,7 +184,7 @@ export function createAnalyzer(ctx, config) {
|
|
|
153
184
|
staticCache.clear()
|
|
154
185
|
reportCache = null
|
|
155
186
|
}
|
|
156
|
-
ctx.on('skills/change', invalidate)
|
|
187
|
+
const offSkillsChange = ctx.on('skills/change', invalidate)
|
|
157
188
|
|
|
158
189
|
/**
|
|
159
190
|
* Prefer the live catalog (fresh after an edit) and fall back to the static
|
|
@@ -370,9 +401,7 @@ export function createAnalyzer(ctx, config) {
|
|
|
370
401
|
ranking,
|
|
371
402
|
scoring: {
|
|
372
403
|
weights,
|
|
373
|
-
weightsPercent:
|
|
374
|
-
Object.entries(weights).map(([key, value]) => [key, Math.round(value * 100)]),
|
|
375
|
-
),
|
|
404
|
+
weightsPercent: percentWeights(weights),
|
|
376
405
|
thresholds,
|
|
377
406
|
minSamples,
|
|
378
407
|
editBudgetMax: config.editBudgetMax ?? 4,
|
|
@@ -413,6 +442,7 @@ export function createAnalyzer(ctx, config) {
|
|
|
413
442
|
}
|
|
414
443
|
}
|
|
415
444
|
for (const [tool, stats] of Object.entries(bucket.tools ?? {})) {
|
|
445
|
+
if (!stats || typeof stats !== 'object' || Array.isArray(stats)) continue
|
|
416
446
|
const t = Object.hasOwn(row.tools, tool) ? row.tools[tool] : { calls: 0, errors: 0, ms: 0, samples: [] }
|
|
417
447
|
t.calls += Number(stats.calls) || 0
|
|
418
448
|
t.errors += Number(stats.errors) || 0
|
|
@@ -642,5 +672,18 @@ export function createAnalyzer(ctx, config) {
|
|
|
642
672
|
return report.skills[0] ?? null
|
|
643
673
|
}
|
|
644
674
|
|
|
645
|
-
return {
|
|
675
|
+
return {
|
|
676
|
+
build,
|
|
677
|
+
buildOne,
|
|
678
|
+
store,
|
|
679
|
+
invalidate,
|
|
680
|
+
resolveStatic,
|
|
681
|
+
dispose() {
|
|
682
|
+
try {
|
|
683
|
+
if (typeof offSkillsChange === 'function') offSkillsChange()
|
|
684
|
+
} catch {
|
|
685
|
+
// best effort teardown
|
|
686
|
+
}
|
|
687
|
+
},
|
|
688
|
+
}
|
|
646
689
|
}
|
package/lib/score.js
CHANGED
|
@@ -334,6 +334,7 @@ export function aggregate(samples, options = {}) {
|
|
|
334
334
|
}
|
|
335
335
|
}
|
|
336
336
|
for (const [tool, stats] of Object.entries(bucket.tools ?? {})) {
|
|
337
|
+
if (!stats || typeof stats !== 'object' || Array.isArray(stats)) continue
|
|
337
338
|
const t = Object.hasOwn(entry.tools, tool) ? entry.tools[tool] : { calls: 0, errors: 0, ms: 0, samples: [] }
|
|
338
339
|
t.calls += count(stats.calls)
|
|
339
340
|
t.errors += count(stats.errors)
|
|
@@ -496,6 +497,7 @@ export function scoreSkill({
|
|
|
496
497
|
provisional: stats.loads < minSamples,
|
|
497
498
|
confidence,
|
|
498
499
|
samples: stats.loads,
|
|
500
|
+
minSamples,
|
|
499
501
|
dimensions: dims,
|
|
500
502
|
effectiveWeights: used,
|
|
501
503
|
coverage: round(weightSum, 2),
|
|
@@ -507,6 +509,44 @@ export function scoreSkill({
|
|
|
507
509
|
* rate, ...). Advice uses it as a noise floor: a rule only keeps its severity
|
|
508
510
|
* when the interval's low end is already past the threshold.
|
|
509
511
|
*/
|
|
512
|
+
/**
|
|
513
|
+
* Bootstrap interval for a call-weighted rate. Advice reports "X% of the tool
|
|
514
|
+
* calls inside the window failed", so its noise floor must resample calls, not
|
|
515
|
+
* the fraction of windows that contain any error: one severe window can
|
|
516
|
+
* otherwise be gated away by nine clean one-call windows.
|
|
517
|
+
*/
|
|
518
|
+
export function bootstrapCallRate(windows, options = {}) {
|
|
519
|
+
const rows = Array.isArray(windows) ? windows.filter(Boolean) : []
|
|
520
|
+
const resamples = Number.isFinite(options.resamples) ? options.resamples : 400
|
|
521
|
+
if (rows.length < 2 || resamples <= 0) return null
|
|
522
|
+
const totalCalls = rows.reduce((sum, w) => sum + (Number(w.toolCalls) || 0), 0)
|
|
523
|
+
const totalErrors = rows.reduce((sum, w) => sum + (Number(w.errors) || 0), 0)
|
|
524
|
+
if (!(totalCalls > 0)) return null
|
|
525
|
+
const random = mulberry32(Number.isFinite(options.seed) ? options.seed : 1)
|
|
526
|
+
const rates = []
|
|
527
|
+
for (let i = 0; i < resamples; i += 1) {
|
|
528
|
+
let calls = 0
|
|
529
|
+
let errors = 0
|
|
530
|
+
for (let j = 0; j < rows.length; j += 1) {
|
|
531
|
+
const w = rows[Math.floor(random() * rows.length)]
|
|
532
|
+
calls += Number(w.toolCalls) || 0
|
|
533
|
+
errors += Number(w.errors) || 0
|
|
534
|
+
}
|
|
535
|
+
if (calls > 0) rates.push(errors / calls)
|
|
536
|
+
}
|
|
537
|
+
if (rates.length === 0) return null
|
|
538
|
+
rates.sort((a, b) => a - b)
|
|
539
|
+
const at = (p) => rates[Math.min(rates.length - 1, Math.max(0, Math.round(p * (rates.length - 1))))]
|
|
540
|
+
return {
|
|
541
|
+
mean: Math.min(1, totalErrors / totalCalls),
|
|
542
|
+
low: at(0.025),
|
|
543
|
+
high: at(0.975),
|
|
544
|
+
resamples: rates.length,
|
|
545
|
+
windows: rows.length,
|
|
546
|
+
calls: totalCalls,
|
|
547
|
+
}
|
|
548
|
+
}
|
|
549
|
+
|
|
510
550
|
export function bootstrapRate(windows, isHit, options = {}) {
|
|
511
551
|
const rows = Array.isArray(windows) ? windows.filter(Boolean) : []
|
|
512
552
|
const resamples = Number.isFinite(options.resamples) ? options.resamples : 400
|