dsh-skill-eval 1.20.2 → 1.20.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/docs/UI-REVIEW.md CHANGED
@@ -81,16 +81,16 @@ headless Chrome 按 520px 面板宽 / 900px 配置页截图比对;不是真实
81
81
  方法:真实宿主路由(`curl`)+ 用 jsdom 挂载**线上真实载荷** `/skill-eval/data/1.20.1`
82
82
  后逐个点击、切标签;宿主侧再用 `scripts/verify-live.mjs` 对照。不是截图,也不是只看代码。
83
83
 
84
- | 控件 | 实测结果 | 结论 |
85
- | --- | --- | --- |
86
- | 配置 | 旧实现 `window.open('/skill-eval/board/<v>/config')`。DSH 桌面壳的 `setWindowOpenHandler` 只把 **https** URL 交给系统浏览器,其余一律 `deny`,而看板跑在 `http://127.0.0.1:<port>` → 点击**完全没有反应** | **已修**:改为面板内设置弹窗(复用 `SettingsForm`),Esc/点背景关闭,Tab 焦点限制在弹窗内;桌面壳里隐藏无法兑现的「在网页里打开」 |
87
- | 刷新 | 点击后重新 GET `/skill-eval/data/<v>`,loading 期间禁用 | 正常 |
88
- | 复制给 agent | 写入完整 report JSON(实测 249,630 字符),失败回退到「复制失败」 | 正常;**数据未到达前按钮禁用**(原先会复制 `{}` 并提示已复制) |
89
- | MCP | 图表 + 表格 + 建议/出错工具;线上 context7:1 次调用、2606ms、0% 错误、88.9 B | 数值正确;阈值线原硬编码 3000ms,**改为跟随 `mcpP95Ms` 配置**(旧载荷仍回退 3000) |
90
- | 开发者 | 原先只回显评分配置,且「数据接口」硬编码 `/skill-eval/data`,看不到真正的诊断 | **已修**:data 增加 `dev` 块(已加载模块/磁盘版本 + skew、路由与工具数、settings 文件、warning 数),标签显示版本化 URL 与「需要重启」 |
84
+ | 控件 | 实测结果 | 结论 |
85
+ | ------------ | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------- |
86
+ | 配置 | 旧实现 `window.open('/skill-eval/board/<v>/config')`。DSH 桌面壳的 `setWindowOpenHandler` 只把 **https** URL 交给系统浏览器,其余一律 `deny`,而看板跑在 `http://127.0.0.1:<port>` → 点击**完全没有反应** | **已修**:改为面板内设置弹窗(复用 `SettingsForm`),Esc/点背景关闭,Tab 焦点限制在弹窗内;桌面壳里隐藏无法兑现的「在网页里打开」 |
87
+ | 刷新 | 点击后重新 GET `/skill-eval/data/<v>`,loading 期间禁用 | 正常 |
88
+ | 复制给 agent | 写入完整 report JSON(实测 249,630 字符),失败回退到「复制失败」 | 正常;**数据未到达前按钮禁用**(原先会复制 `{}` 并提示已复制) |
89
+ | MCP | 图表 + 表格 + 建议/出错工具;线上 context7:1 次调用、2606ms、0% 错误、88.9 B | 数值正确;阈值线原硬编码 3000ms,**改为跟随 `mcpP95Ms` 配置**(旧载荷仍回退 3000) |
90
+ | 开发者 | 原先只回显评分配置,且「数据接口」硬编码 `/skill-eval/data`,看不到真正的诊断 | **已修**:data 增加 `dev` 块(已加载模块/磁盘版本 + skew、路由与工具数、settings 文件、warning 数),标签显示版本化 URL 与「需要重启」 |
91
91
 
92
92
  验证:`node scripts/verify-live.mjs http://127.0.0.1:55265` 对 1.20.1 全 PASS;
93
- `node --test test/` **241 项全绿**(新增「配置按钮不再调用 window.open,改为面板内表单」用例)。
93
+ `node --test test/` **258 项全绿**(新增「配置按钮不再调用 window.open,改为面板内表单」用例)。
94
94
 
95
95
  已知取舍(未改):**「复制给 agent」复制的是完整 JSON**(≈250KB,含 400 条步进记录)。
96
96
  如果目标是让 agent 直接读,`buildMarkdown()` 的摘要更省上下文;但完整 JSON 便于排障,
package/docs/WORKFLOW.md CHANGED
@@ -64,7 +64,7 @@
64
64
  | 想看整体汇报 | 侧边栏面板图标「技能评测」→ 主面板(或浏览器打开 `/skill-eval/board/<版本>`) | 技能/运行/MCP/开发者标签,含下一步与效度 |
65
65
  | 想改评分标准 | 设置 → 插件 → `dsh-skill-eval` 包页面 → 配置 | 权重/阈值/预算/开关,保存即生效 |
66
66
  | 入口速查 | [ENTRYPOINTS.md](ENTRYPOINTS.md) | 三个日常入口 + 网页/命令/终端 |
67
- | 发版前自检 | `dsh-skill-eval-doctor` + `accept` + `verify` | 19 / 17 / 9 项:打包、代次、产品面 |
67
+ | 发版前自检 | `dsh-skill-eval-doctor` + `accept` + `verify` | 19 / 17 / 7+2 项:打包、代次、产品面 |
68
68
 
69
69
  ## 判定规则(避免自欺)
70
70
 
package/index.js CHANGED
@@ -36,7 +36,7 @@ import {
36
36
  normalizeSettings,
37
37
  parseFormSettings,
38
38
  renderConfigHtml,
39
- saveSettings,
39
+ updateSettings,
40
40
  SETTINGS_FIELDS,
41
41
  SETTINGS_FILE,
42
42
  } from './lib/settings.js'
@@ -276,11 +276,11 @@ function log(ctx, level, message) {
276
276
  const VIEWS = ['summary', 'mcp', 'runs', 'standard', 'loop', 'validity', 'history', 'impact', 'dev', 'turns']
277
277
 
278
278
  /** Read the files the acceptance check needs, best effort. */
279
- async function acceptanceFiles(store, readFile) {
279
+ async function acceptanceFiles(store, readFile, snapshotDir = path.join(store.dir, 'reports')) {
280
280
  const dir = store.dir
281
281
  const eventsPath = path.join(dir, 'events.jsonl')
282
- const snapshotMarkdownPath = path.join(dir, 'reports', 'latest.md')
283
- const snapshotJsonPath = path.join(dir, 'reports', 'latest.json')
282
+ const snapshotMarkdownPath = path.join(snapshotDir, 'latest.md')
283
+ const snapshotJsonPath = path.join(snapshotDir, 'latest.json')
284
284
  let events = []
285
285
  try {
286
286
  const raw = await readFile(eventsPath, 'utf8')
@@ -362,7 +362,7 @@ async function runReport(analyzer, args, config) {
362
362
  config: config,
363
363
  routes: DIAGNOSTICS.routes,
364
364
  records,
365
- store: { dir: analyzer.store.dir, failed: analyzer.store.failed, queue: analyzer.store.queue },
365
+ store: { dir: analyzer.store.dir, failed: analyzer.store.failed, queued: analyzer.store.pending },
366
366
  tools: DIAGNOSTICS.tools,
367
367
  inspectProvider: DIAGNOSTICS.inspectProvider ? 'skill-eval' : null,
368
368
  warnings: DIAGNOSTICS.warnings,
@@ -489,6 +489,16 @@ export function apply(ctx, config) {
489
489
  }
490
490
 
491
491
  const analyzer = createAnalyzer(ctx, cfg)
492
+ ctx.effect(
493
+ () => () => {
494
+ try {
495
+ analyzer.dispose()
496
+ } catch {
497
+ // best effort on shutdown
498
+ }
499
+ },
500
+ 'skill-eval: analyzer',
501
+ )
492
502
  analyzer.store.onError = (message) => {
493
503
  DIAGNOSTICS.warnings.push(`store write failed: ${message}`)
494
504
  log(ctx, 'warn', `skill-eval store write failed: ${message}`)
@@ -513,27 +523,38 @@ export function apply(ctx, config) {
513
523
  // A settled turn writes the aggregated report to disk. This is the "result
514
524
  // after every run" surface: it costs zero tokens because it never touches the
515
525
  // model context.
516
- const snapshots =
517
- cfg.reportOnTurnEnd === false
518
- ? null
519
- : createSnapshots({
520
- analyzer,
521
- dir: cfg.reportDir ? String(cfg.reportDir) : path.join(analyzer.store.dir, 'reports'),
522
- log: (message) => log(ctx, 'warn', message),
523
- turnReports: cfg.turnReports !== false,
524
- turnReportsKeep: cfg.turnReportsKeep ?? 50,
525
- })
526
+ // Always create the snapshotter and read the switches live: reportOnTurnEnd
527
+ // only controls latest.md/json, turnReports controls the per-turn boards, and
528
+ // a settings save must not need a restart (the UI promises immediate apply).
529
+ const snapshots = createSnapshots({
530
+ analyzer,
531
+ dir: cfg.reportDir ? String(cfg.reportDir) : path.join(analyzer.store.dir, 'reports'),
532
+ log: (message) => log(ctx, 'warn', message),
533
+ enabled: () => cfg.reportOnTurnEnd !== false,
534
+ turnReports: () => cfg.turnReports !== false,
535
+ turnReportsKeep: cfg.turnReportsKeep ?? 50,
536
+ })
526
537
 
527
538
  if (snapshots) {
528
- ctx.on('agent/turn-stopping', () => {
539
+ const offTurnStopping = ctx.on('agent/turn-stopping', (payload) => {
529
540
  try {
530
- snapshots.request()
541
+ // Label the board with the turn that actually settled. Without this a
542
+ // concurrent subagent turn (or a new load inside the debounce) would
543
+ // make the writer pick whatever turnLoads is globally newest.
544
+ const sessionId = payload?.agent ? String(payload.agent.id) : null
545
+ const turn = typeof payload?.turn === 'number' ? payload.turn : null
546
+ snapshots.request(sessionId ? { sessionId, turn } : null)
531
547
  } catch {
532
548
  // intentionally ignored: snapshot request is best effort on turn stop
533
549
  }
534
550
  })
535
551
  ctx.effect(
536
552
  () => () => {
553
+ try {
554
+ if (typeof offTurnStopping === 'function') offTurnStopping()
555
+ } catch {
556
+ // best effort
557
+ }
537
558
  void snapshots.dispose()
538
559
  },
539
560
  'skill-eval: snapshots',
@@ -639,8 +660,10 @@ export function apply(ctx, config) {
639
660
  const keyword = target.toLowerCase()
640
661
  if (keyword === 'accept' || keyword === 'doctor') {
641
662
  const report = await analyzer.build()
642
- const files = await acceptanceFiles(analyzer.store, (file) =>
643
- import('node:fs/promises').then((fs) => fs.readFile(file, 'utf8')),
663
+ const files = await acceptanceFiles(
664
+ analyzer.store,
665
+ (file) => import('node:fs/promises').then((fs) => fs.readFile(file, 'utf8')),
666
+ cfg.reportDir ? String(cfg.reportDir) : path.join(analyzer.store.dir, 'reports'),
644
667
  )
645
668
  const acceptance = checkInProcessReport(report, { version: PLUGIN_VERSION ?? 'unknown', files })
646
669
  return { kind: 'success', text: '\n' + renderAcceptance(acceptance) + '\n' }
@@ -712,6 +735,38 @@ export function apply(ctx, config) {
712
735
  const clientBase = '/skill-eval'
713
736
  const dataPath = `${clientBase}/data`
714
737
 
738
+ // Writes are same-origin fenced. Comparing Origin to Host alone does not
739
+ // stop DNS rebinding (both are the attacker's domain after rebinding), so
740
+ // the Host itself must be the server's own origin or a loopback name.
741
+ let serverOrigin = null
742
+ const hostnameOf = (value) => {
743
+ try {
744
+ return new URL(String(value).includes('://') ? String(value) : 'http://' + String(value)).hostname.toLowerCase()
745
+ } catch {
746
+ return ''
747
+ }
748
+ }
749
+ const allowedWriteHosts = () => {
750
+ const hosts = new Set(['localhost', '127.0.0.1', '[::1]', '::1'])
751
+ const own = serverOrigin ? hostnameOf(serverOrigin) : ''
752
+ if (own) hosts.add(own)
753
+ return hosts
754
+ }
755
+ const rememberOrigin = (webServer) => {
756
+ try {
757
+ const value = webServer.getOrigin()
758
+ if (typeof value === 'string' && value) serverOrigin = value
759
+ } catch {
760
+ // keep the loopback fallback
761
+ }
762
+ }
763
+ ctx.inject(['webServer'], (c) => {
764
+ c.effect(() => {
765
+ rememberOrigin(c.webServer)
766
+ return () => {}
767
+ }, 'skill-eval: origin')
768
+ })
769
+
715
770
  // The board's dev tab shows what the agent's `view:"dev"` report shows in
716
771
  // short form: which generation is loaded, whether the package on disk moved
717
772
  // ahead of it (the ESM-cache restart trap), how many routes this generation
@@ -736,7 +791,12 @@ export function apply(ctx, config) {
736
791
  'content-type': 'application/json; charset=utf-8',
737
792
  'cache-control': 'no-store',
738
793
  })
739
- res.end(JSON.stringify({ ...report, dev: diagnosticsPayload() }, null, 2))
794
+ // The pages may live under a custom dashboardPath; ship their exact
795
+ // paths so the client never hardcodes a route the host did not register.
796
+ const pages = PLUGIN_VERSION
797
+ ? { board: `${base}/board/${PLUGIN_VERSION}`, config: `${base}/board/${PLUGIN_VERSION}/config` }
798
+ : { board: base, config: `${base}/config` }
799
+ res.end(JSON.stringify({ ...report, dev: diagnosticsPayload(), pages }, null, 2))
740
800
  }
741
801
 
742
802
  const bodyTooLarge = () => Object.assign(new Error('request body too large'), { statusCode: 413 })
@@ -781,11 +841,13 @@ export function apply(ctx, config) {
781
841
  const trustedWrite = (req, { requireHeader = false } = {}) => {
782
842
  const site = String(req.headers?.['sec-fetch-site'] ?? '').toLowerCase()
783
843
  if (site === 'cross-site') return false
844
+ const host = String(req.headers?.['host'] ?? '')
845
+ if (!allowedWriteHosts().has(hostnameOf(host))) return false
784
846
  const origin = req.headers?.origin
785
847
  if (origin) {
786
848
  try {
787
849
  const parsed = new URL(origin)
788
- if (!req.headers.host || parsed.host !== req.headers.host) return false
850
+ if (parsed.host !== host) return false
789
851
  } catch {
790
852
  return false
791
853
  }
@@ -827,7 +889,7 @@ export function apply(ctx, config) {
827
889
  send(400, renderConfigHtml({ config: cfg, base: pageBase, errors, lang }))
828
890
  return
829
891
  }
830
- if (settingsDir) await saveSettings(settingsDir, { ...loadSettingsSync(settingsDir), ...values })
892
+ if (settingsDir) await updateSettings(settingsDir, values)
831
893
  applySettings(cfg, values)
832
894
  analyzer.invalidate()
833
895
  DIAGNOSTICS.settings = loadSettingsSync(settingsDir)
@@ -908,7 +970,7 @@ export function apply(ctx, config) {
908
970
  send(400, { saved: false, errors, values: settingsPayload().values })
909
971
  return
910
972
  }
911
- if (settingsDir) await saveSettings(settingsDir, { ...loadSettingsSync(settingsDir), ...values })
973
+ if (settingsDir) await updateSettings(settingsDir, values)
912
974
  applySettings(cfg, values)
913
975
  analyzer.invalidate()
914
976
  DIAGNOSTICS.settings = loadSettingsSync(settingsDir)
@@ -1093,19 +1155,19 @@ export function apply(ctx, config) {
1093
1155
 
1094
1156
  const SETTINGS_KEYS = SETTINGS_FIELDS.map((field) => field.key)
1095
1157
 
1096
- if (cfg.promptHint !== false)
1097
- ctx.inject(['systemPrompt'], (c) => {
1098
- c.effect(() => {
1099
- const stop = c.systemPrompt.section({
1100
- name: 'skill-eval:usage',
1101
- order: 118,
1102
- text: () =>
1103
- 'Skill telemetry is recorded for every `skill` load. When asked how a skill performed or how to improve it, call `skill_eval_report` (pass {"skill":"<name>"} for one skill) and act on the cited fixes instead of guessing. Before editing a skill named in the report, screen the change with `skill_eval_review` (leakage critic) and record the hypothesis; ask `skill_eval_report` with view:"loop" for the next step and view:"validity" for what the evidence can and cannot support.',
1104
- })
1105
- return () => safe(stop)
1106
- }, 'skill-eval: prompt section')
1107
- })
1158
+ ctx.inject(['systemPrompt'], (c) => {
1159
+ c.effect(() => {
1160
+ const stop = c.systemPrompt.section({
1161
+ name: 'skill-eval:usage',
1162
+ order: 118,
1163
+ text: () =>
1164
+ cfg.promptHint === false
1165
+ ? ''
1166
+ : 'Skill telemetry is recorded for every `skill` load. When asked how a skill performed or how to improve it, call `skill_eval_report` (pass {"skill":"<name>"} for one skill) and act on the cited fixes instead of guessing. Before editing a skill named in the report, screen the change with `skill_eval_review` (leakage critic) and record the hypothesis; ask `skill_eval_report` with view:"loop" for the next step and view:"validity" for what the evidence can and cannot support.',
1167
+ })
1168
+ return () => safe(stop)
1169
+ }, 'skill-eval: prompt section')
1170
+ })
1108
1171
 
1109
1172
  log(ctx, 'info', `skill-eval: recording enabled, data dir ${analyzer.store.dir}`)
1110
1173
  }
1111
-
package/lib/accept.js CHANGED
@@ -325,7 +325,7 @@ export function checkVersionedPayload(versioned, canonical) {
325
325
  results.push(
326
326
  result(
327
327
  'canonical-route',
328
- true,
328
+ canonicalKeys === versionedKeys,
329
329
  canonicalKeys === versionedKeys
330
330
  ? 'canonical and versioned routes agree'
331
331
  : 'canonical route still answers from an older handler (the versioned route is current; a restart fixes the canonical one)',
package/lib/advice.js CHANGED
@@ -6,7 +6,7 @@
6
6
  * severity: 'critical' | 'warning' | 'info'
7
7
  */
8
8
 
9
- import { bootstrapRate } from './score.js'
9
+ import { bootstrapCallRate, bootstrapRate } from './score.js'
10
10
 
11
11
  const SEVERITY_ORDER = { critical: 0, warning: 1, info: 2 }
12
12
 
@@ -190,6 +190,15 @@ function gated(windows, isHit, threshold, resamples) {
190
190
  return { level: 'none', ci }
191
191
  }
192
192
 
193
+ function gatedCalls(windows, threshold, resamples) {
194
+ if (!Array.isArray(windows) || windows.length === 0) return { level: 'strong', ci: null }
195
+ const ci = bootstrapCallRate(windows, { resamples, seed: 17 })
196
+ if (!ci) return { level: 'mean', ci: null }
197
+ if (ci.low > threshold) return { level: 'strong', ci }
198
+ if (ci.mean > threshold) return { level: 'weak', ci }
199
+ return { level: 'none', ci }
200
+ }
201
+
193
202
  function noiseNote(ci, threshold, level) {
194
203
  if (!ci) return ''
195
204
  const base = ` (95% interval ${Math.round(ci.low * 100)}-${Math.round(ci.high * 100)}%, threshold ${Math.round(
@@ -227,7 +236,8 @@ export function buildAdvice({
227
236
  })
228
237
  }
229
238
 
230
- const errGate = gate((w) => (w.errors || 0) > 0, 0.25)
239
+ // Gate on the same call-weighted rate the evidence reports.
240
+ const errGate = gating ? gatedCalls(windows, 0.25, 400) : { level: 'strong', ci: null }
231
241
  if (stats.errRate !== null && stats.errRate > 0.25 && errGate.level !== 'none') {
232
242
  out.push({
233
243
  id: 'post-load-errors',
package/lib/dev.js CHANGED
@@ -103,7 +103,7 @@ export function buildDevReport(input = {}) {
103
103
  store: {
104
104
  dir: store.dir ?? dataDir,
105
105
  failed: store.failed ?? null,
106
- queued: Array.isArray(store.queue) ? store.queue.length : null,
106
+ queued: Number.isFinite(store.queued) ? store.queued : null,
107
107
  records: records.length,
108
108
  counts,
109
109
  types,
package/lib/history.js CHANGED
@@ -6,6 +6,7 @@
6
6
  * reported as inconclusive rather than as an improvement.
7
7
  */
8
8
 
9
+ import { formatTimestamp } from './html.js'
9
10
  import { foldSamples } from './store.js'
10
11
 
11
12
  const rate = (list, predicate) => (list.length === 0 ? null : list.filter(predicate).length / list.length)
@@ -109,7 +110,7 @@ export function renderHistoryMarkdown(history) {
109
110
  '|---|---|---|---|---|---|---|',
110
111
  ]
111
112
  for (const load of history.loads.slice(-15)) {
112
- const when = new Date(load.at).toISOString().replace('T', ' ').slice(0, 19)
113
+ const when = formatTimestamp(load.at)
113
114
  lines.push(
114
115
  `| ${when} | ${load.ok ? 'yes' : 'no'} | ${load.calls ?? '—'} | ${load.errors ?? '—'} | ${load.corrections} | ${
115
116
  load.loadMs ?? '—'
@@ -124,7 +125,7 @@ export function renderImpactMarkdown(impact) {
124
125
  return `No screened edit recorded for \`${impact?.skill ?? '?'}\` yet. Use skill_eval_review before changing it.`
125
126
  const lines = [`## Edit impact — \`${impact.skill}\``, '', `${impact.edits} screened edit(s)`, '']
126
127
  for (const comparison of impact.comparisons) {
127
- const when = new Date(comparison.at).toISOString().replace('T', ' ').slice(0, 19)
128
+ const when = formatTimestamp(comparison.at)
128
129
  lines.push(`### ${when} · ${comparison.dimension ?? 'unspecified'} · ${comparison.verdict}`)
129
130
  if (comparison.hypothesis) lines.push(`hypothesis: ${comparison.hypothesis}`)
130
131
  lines.push(`- ${comparison.note}`)
package/lib/html.js CHANGED
@@ -1,4 +1,13 @@
1
1
  /** Escape text for the few HTML strings built outside React. */
2
+ /** Format a finite epoch-ms timestamp, or an em dash when it is not a valid date. */
3
+ export function formatTimestamp(value) {
4
+ const n = Number(value)
5
+ if (!Number.isFinite(n)) return '—'
6
+ const date = new Date(n)
7
+ if (Number.isNaN(date.getTime())) return '—'
8
+ return date.toISOString().replace('T', ' ').slice(0, 19)
9
+ }
10
+
2
11
  export function escapeHtml(value) {
3
12
  return String(value ?? '')
4
13
  .replace(/&/g, '&amp;')
package/lib/recorder.js CHANGED
@@ -88,7 +88,8 @@ export function createRecorder(ctx, config, { store, resolveSkill }) {
88
88
  const sessions = new Map() // sessionId -> { windows: [] }
89
89
 
90
90
  const windowLimit = Math.max(1, Number(config.windowToolCalls ?? 12))
91
- const ttlMs = Math.max(60_000, Number(config.windowTtlMs ?? 30 * 60_000))
91
+ // Read live: the settings form advertises windowTtlMs as applying immediately.
92
+ const ttlMs = () => Math.max(60_000, Number(config.windowTtlMs ?? 30 * 60_000))
92
93
 
93
94
  function stateFor(sessionId) {
94
95
  const existing = sessions.get(sessionId)
@@ -313,7 +314,15 @@ export function createRecorder(ctx, config, { store, resolveSkill }) {
313
314
  const seenKey = `${sessionId}|${id}`
314
315
  if (seenInvocations.has(seenKey)) continue
315
316
  seenInvocations.add(seenKey)
316
- if (seenInvocations.size > 4000) seenInvocations.clear()
317
+ // Evict the oldest keys instead of clearing everything: a wholesale
318
+ // reset would let a repeated invocation message be recorded twice.
319
+ if (seenInvocations.size > 4000) {
320
+ for (let i = 0; i < 500; i += 1) {
321
+ const oldest = seenInvocations.values().next().value
322
+ if (oldest === undefined) break
323
+ seenInvocations.delete(oldest)
324
+ }
325
+ }
317
326
  const state = stateFor(sessionId)
318
327
  const windowId = `${sessionId}:${id}`
319
328
  void store.append({
@@ -439,7 +448,7 @@ export function createRecorder(ctx, config, { store, resolveSkill }) {
439
448
  }
440
449
  }
441
450
  const newest = state.windows[state.windows.length - 1]
442
- if (newest && nowMs() - newest.lastActivityAt > ttlMs) closeWindows(state, 'stale')
451
+ if (newest && nowMs() - newest.lastActivityAt > ttlMs()) closeWindows(state, 'stale')
443
452
 
444
453
  if (exec.name === 'skill') {
445
454
  const skill = skillNameOf(exec.arguments)
@@ -462,14 +471,15 @@ export function createRecorder(ctx, config, { store, resolveSkill }) {
462
471
  turn: turns.get(sessionId),
463
472
  windowId,
464
473
  })
465
- openWindow(state, skill, sessionId, exec.callId, steps.get(sessionId), originOf(exec.agent))
466
- if (ok)
474
+ if (ok) {
475
+ openWindow(state, skill, sessionId, exec.callId, steps.get(sessionId), originOf(exec.agent))
467
476
  void captureStatic(
468
477
  skill,
469
478
  exec.agent,
470
479
  exec.signal,
471
480
  result?.value && typeof result.value.content === 'string' ? result.value.content : undefined,
472
481
  )
482
+ }
473
483
  return
474
484
  }
475
485
 
@@ -540,7 +550,7 @@ export function createRecorder(ctx, config, { store, resolveSkill }) {
540
550
  const state = sessions.get(sessionId)
541
551
  if (!state || state.windows.length === 0) return
542
552
  const newest = state.windows[state.windows.length - 1]
543
- if (nowMs() - newest.lastActivityAt > ttlMs) {
553
+ if (nowMs() - newest.lastActivityAt > ttlMs()) {
544
554
  closeWindows(state, 'stale')
545
555
  return
546
556
  }
package/lib/render.js CHANGED
@@ -2,7 +2,7 @@
2
2
 
3
3
  import { localizeReport } from './advice.js'
4
4
  import { buildLoopPlans } from './loop.js'
5
- import { escapeHtml as esc } from './html.js'
5
+ import { escapeHtml as esc, formatTimestamp } from './html.js'
6
6
 
7
7
  function bar(value, width = 10) {
8
8
  if (!Number.isFinite(value)) return '·'.repeat(width)
@@ -358,7 +358,7 @@ const BOARD = {
358
358
  evidenceTitle: 'What the evidence supports',
359
359
  coverage: (v) =>
360
360
  `coverage: ${v.coverage.skills} skill(s) · ${v.coverage.withCI} with 95% intervals · ${v.coverage.provisional} provisional · ranking ${
361
- v.ranking.significant ? 'significant' : 'not significant'
361
+ v.ranking.unknown ? 'not enough evidence' : v.ranking.significant ? 'significant' : 'not significant'
362
362
  }`,
363
363
  canSay: 'Can say',
364
364
  cannotSay: 'Cannot say yet',
@@ -392,7 +392,7 @@ const BOARD = {
392
392
  evidenceTitle: '证据能支持什么',
393
393
  coverage: (v) =>
394
394
  `覆盖:${v.coverage.skills} 个技能 · ${v.coverage.withCI} 个有 95% 区间 · ${v.coverage.provisional} 个临时分 · 排名${
395
- v.ranking.significant ? '显著' : '不显著'
395
+ v.ranking.unknown ? '证据不足' : v.ranking.significant ? '显著' : '不显著'
396
396
  }`,
397
397
  canSay: '可以断言',
398
398
  cannotSay: '还不能断言',
@@ -466,13 +466,11 @@ export function renderHtml(input, options = {}) {
466
466
  const editRows = (report.edits ?? [])
467
467
  .slice(0, 8)
468
468
  .map((edit) => [
469
- esc(
470
- Number.isFinite(Number(edit.at)) ? new Date(Number(edit.at)).toISOString().replace('T', ' ').slice(0, 19) : '—',
471
- ),
469
+ esc(formatTimestamp(edit.at)),
472
470
  esc(edit.skill ?? ''),
473
471
  esc(edit.dimension ?? '—'),
474
472
  `<b>${esc(edit.verdict ?? '')}</b>`,
475
- esc((edit.hypothesis ?? '').slice(0, 70)),
473
+ esc(String(edit.hypothesis ?? '').slice(0, 70)),
476
474
  ])
477
475
  const ledgerBlock = tableSection(t.ledgerTitle, t.ledgerHead, editRows)
478
476
  const otherLang = lang === 'zh' ? 'en' : 'zh'
package/lib/report.js CHANGED
@@ -44,7 +44,10 @@ export function rankingOf(entries) {
44
44
  const scored = (entries ?? []).filter(
45
45
  (entry) => entry && entry.score && entry.score.ci && typeof entry.score.total === 'number',
46
46
  )
47
- if (scored.length < 2) return { significant: true, note: null, ties: [] }
47
+ // Fewer than two intervals means nothing was compared; claiming a
48
+ // significant ranking here would be a fabricated statistic.
49
+ if (scored.length < 2)
50
+ return { significant: null, note: 'not enough scored skills with 95% intervals to rank them', ties: [] }
48
51
  const ties = []
49
52
  for (let i = 1; i < scored.length; i += 1) {
50
53
  const a = scored[i - 1]
@@ -118,6 +121,34 @@ export function normalizeWeights(input) {
118
121
  return out
119
122
  }
120
123
 
124
+ /**
125
+ * Display weights as integers that always sum to 100. Independent rounding can
126
+ * show 29+29+29+14=101 for a legal 2/2/2/1 configuration, which then fails the
127
+ * release check; largest-remainder keeps the displayed total honest.
128
+ */
129
+ export function percentWeights(weights) {
130
+ const keys = Object.keys(weights ?? {})
131
+ if (keys.length === 0) return {}
132
+ const exact = keys.map((key) => Math.max(0, Number(weights[key]) || 0) * 100)
133
+ const base = exact.map((value) => Math.floor(value))
134
+ let remainder = 100 - base.reduce((sum, value) => sum + value, 0)
135
+ const order = exact
136
+ .map((value, index) => ({ index, frac: value - Math.floor(value) }))
137
+ .sort((a, b) => b.frac - a.frac)
138
+ for (const item of order) {
139
+ if (remainder <= 0) break
140
+ base[item.index] += 1
141
+ remainder -= 1
142
+ }
143
+ while (remainder > 0) {
144
+ let largest = 0
145
+ for (let i = 1; i < base.length; i += 1) if (base[i] > base[largest]) largest = i
146
+ base[largest] += 1
147
+ remainder -= 1
148
+ }
149
+ return Object.fromEntries(keys.map((key, index) => [key, base[index]]))
150
+ }
151
+
121
152
  export function resolveThresholds(input) {
122
153
  const raw = { ...DEFAULT_THRESHOLDS, ...(input && typeof input === 'object' ? input : {}) }
123
154
  const pick = (key, fallback) => {
@@ -153,7 +184,7 @@ export function createAnalyzer(ctx, config) {
153
184
  staticCache.clear()
154
185
  reportCache = null
155
186
  }
156
- ctx.on('skills/change', invalidate)
187
+ const offSkillsChange = ctx.on('skills/change', invalidate)
157
188
 
158
189
  /**
159
190
  * Prefer the live catalog (fresh after an edit) and fall back to the static
@@ -370,9 +401,7 @@ export function createAnalyzer(ctx, config) {
370
401
  ranking,
371
402
  scoring: {
372
403
  weights,
373
- weightsPercent: Object.fromEntries(
374
- Object.entries(weights).map(([key, value]) => [key, Math.round(value * 100)]),
375
- ),
404
+ weightsPercent: percentWeights(weights),
376
405
  thresholds,
377
406
  minSamples,
378
407
  editBudgetMax: config.editBudgetMax ?? 4,
@@ -413,6 +442,7 @@ export function createAnalyzer(ctx, config) {
413
442
  }
414
443
  }
415
444
  for (const [tool, stats] of Object.entries(bucket.tools ?? {})) {
445
+ if (!stats || typeof stats !== 'object' || Array.isArray(stats)) continue
416
446
  const t = Object.hasOwn(row.tools, tool) ? row.tools[tool] : { calls: 0, errors: 0, ms: 0, samples: [] }
417
447
  t.calls += Number(stats.calls) || 0
418
448
  t.errors += Number(stats.errors) || 0
@@ -642,5 +672,18 @@ export function createAnalyzer(ctx, config) {
642
672
  return report.skills[0] ?? null
643
673
  }
644
674
 
645
- return { build, buildOne, store, invalidate, resolveStatic }
675
+ return {
676
+ build,
677
+ buildOne,
678
+ store,
679
+ invalidate,
680
+ resolveStatic,
681
+ dispose() {
682
+ try {
683
+ if (typeof offSkillsChange === 'function') offSkillsChange()
684
+ } catch {
685
+ // best effort teardown
686
+ }
687
+ },
688
+ }
646
689
  }
package/lib/score.js CHANGED
@@ -334,6 +334,7 @@ export function aggregate(samples, options = {}) {
334
334
  }
335
335
  }
336
336
  for (const [tool, stats] of Object.entries(bucket.tools ?? {})) {
337
+ if (!stats || typeof stats !== 'object' || Array.isArray(stats)) continue
337
338
  const t = Object.hasOwn(entry.tools, tool) ? entry.tools[tool] : { calls: 0, errors: 0, ms: 0, samples: [] }
338
339
  t.calls += count(stats.calls)
339
340
  t.errors += count(stats.errors)
@@ -496,6 +497,7 @@ export function scoreSkill({
496
497
  provisional: stats.loads < minSamples,
497
498
  confidence,
498
499
  samples: stats.loads,
500
+ minSamples,
499
501
  dimensions: dims,
500
502
  effectiveWeights: used,
501
503
  coverage: round(weightSum, 2),
@@ -507,6 +509,44 @@ export function scoreSkill({
507
509
  * rate, ...). Advice uses it as a noise floor: a rule only keeps its severity
508
510
  * when the interval's low end is already past the threshold.
509
511
  */
512
+ /**
513
+ * Bootstrap interval for a call-weighted rate. Advice reports "X% of the tool
514
+ * calls inside the window failed", so its noise floor must resample calls, not
515
+ * the fraction of windows that contain any error: one severe window can
516
+ * otherwise be gated away by nine clean one-call windows.
517
+ */
518
+ export function bootstrapCallRate(windows, options = {}) {
519
+ const rows = Array.isArray(windows) ? windows.filter(Boolean) : []
520
+ const resamples = Number.isFinite(options.resamples) ? options.resamples : 400
521
+ if (rows.length < 2 || resamples <= 0) return null
522
+ const totalCalls = rows.reduce((sum, w) => sum + (Number(w.toolCalls) || 0), 0)
523
+ const totalErrors = rows.reduce((sum, w) => sum + (Number(w.errors) || 0), 0)
524
+ if (!(totalCalls > 0)) return null
525
+ const random = mulberry32(Number.isFinite(options.seed) ? options.seed : 1)
526
+ const rates = []
527
+ for (let i = 0; i < resamples; i += 1) {
528
+ let calls = 0
529
+ let errors = 0
530
+ for (let j = 0; j < rows.length; j += 1) {
531
+ const w = rows[Math.floor(random() * rows.length)]
532
+ calls += Number(w.toolCalls) || 0
533
+ errors += Number(w.errors) || 0
534
+ }
535
+ if (calls > 0) rates.push(errors / calls)
536
+ }
537
+ if (rates.length === 0) return null
538
+ rates.sort((a, b) => a - b)
539
+ const at = (p) => rates[Math.min(rates.length - 1, Math.max(0, Math.round(p * (rates.length - 1))))]
540
+ return {
541
+ mean: Math.min(1, totalErrors / totalCalls),
542
+ low: at(0.025),
543
+ high: at(0.975),
544
+ resamples: rates.length,
545
+ windows: rows.length,
546
+ calls: totalCalls,
547
+ }
548
+ }
549
+
510
550
  export function bootstrapRate(windows, isHit, options = {}) {
511
551
  const rows = Array.isArray(windows) ? windows.filter(Boolean) : []
512
552
  const resamples = Number.isFinite(options.resamples) ? options.resamples : 400