dsh-skill-eval 1.20.2 → 1.20.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +43 -0
- package/README.md +13 -6
- package/README.zh.md +11 -4
- package/client.js +230 -76
- package/docs/ACCEPTANCE.md +1 -1
- package/docs/AUDIT.md +18 -0
- package/docs/DETAIL.md +11 -6
- package/docs/ENTRYPOINTS.md +1 -1
- package/docs/RELEASE.md +2 -2
- package/docs/UI-REVIEW.md +8 -8
- package/docs/WORKFLOW.md +1 -1
- package/index.js +98 -36
- package/lib/accept.js +1 -1
- package/lib/advice.js +12 -2
- package/lib/dev.js +1 -1
- package/lib/history.js +3 -2
- package/lib/html.js +9 -0
- package/lib/recorder.js +16 -6
- package/lib/render.js +5 -7
- package/lib/report.js +49 -6
- package/lib/score.js +40 -0
- package/lib/settings.js +26 -7
- package/lib/snapshot.js +111 -27
- package/lib/store.js +55 -1
- package/lib/validity.js +12 -2
- package/package.json +1 -1
- package/scripts/accept.mjs +7 -6
- package/scripts/doctor.mjs +46 -11
- package/scripts/lib.mjs +35 -0
- package/scripts/verify-live.mjs +16 -6
package/client.js
CHANGED
|
@@ -58,7 +58,14 @@ window.__ModuleLoader__.load({
|
|
|
58
58
|
corr: '纠正率',
|
|
59
59
|
err: '错误率',
|
|
60
60
|
latency: '加载耗时',
|
|
61
|
-
dims: {
|
|
61
|
+
dims: {
|
|
62
|
+
static: '静态质量',
|
|
63
|
+
reliability: '可靠性',
|
|
64
|
+
effectiveness: '有效性',
|
|
65
|
+
efficiency: '效率',
|
|
66
|
+
lifecycle: '生命周期',
|
|
67
|
+
mcp: 'MCP',
|
|
68
|
+
},
|
|
62
69
|
provisional: '样本不足,分数为临时值',
|
|
63
70
|
noAdvice: '没有需要修的问题',
|
|
64
71
|
footerHint: '也可以直接让 agent 调 skill_eval_report',
|
|
@@ -157,6 +164,10 @@ window.__ModuleLoader__.load({
|
|
|
157
164
|
skew: '需要重启',
|
|
158
165
|
totals: '记录量',
|
|
159
166
|
store: '存储错误',
|
|
167
|
+
settingsFile: '配置文件',
|
|
168
|
+
warnings: '告警',
|
|
169
|
+
inspect: 'Inspect 提供者',
|
|
170
|
+
ok: '正常',
|
|
160
171
|
},
|
|
161
172
|
},
|
|
162
173
|
en: {
|
|
@@ -177,7 +188,14 @@ window.__ModuleLoader__.load({
|
|
|
177
188
|
corr: 'corr',
|
|
178
189
|
err: 'err',
|
|
179
190
|
latency: 'load',
|
|
180
|
-
dims: {
|
|
191
|
+
dims: {
|
|
192
|
+
static: 'static',
|
|
193
|
+
reliability: 'reliability',
|
|
194
|
+
effectiveness: 'effectiveness',
|
|
195
|
+
efficiency: 'efficiency',
|
|
196
|
+
lifecycle: 'lifecycle',
|
|
197
|
+
mcp: 'mcp',
|
|
198
|
+
},
|
|
181
199
|
provisional: 'provisional score: not enough loads yet',
|
|
182
200
|
noAdvice: 'nothing to fix from recorded evidence',
|
|
183
201
|
footerHint: 'The agent can read the same report through skill_eval_report',
|
|
@@ -277,6 +295,10 @@ window.__ModuleLoader__.load({
|
|
|
277
295
|
skew: 'restart',
|
|
278
296
|
totals: 'records',
|
|
279
297
|
store: 'store error',
|
|
298
|
+
settingsFile: 'settings file',
|
|
299
|
+
warnings: 'warnings',
|
|
300
|
+
inspect: 'inspect provider',
|
|
301
|
+
ok: 'ok',
|
|
280
302
|
},
|
|
281
303
|
},
|
|
282
304
|
}
|
|
@@ -301,7 +323,7 @@ window.__ModuleLoader__.load({
|
|
|
301
323
|
// Keep in sync with package.json `version` (a test enforces it). The host
|
|
302
324
|
// registers this exact path per release, so a plugin that was hot-reloaded
|
|
303
325
|
// without reclaiming the canonical route still answers here.
|
|
304
|
-
const DATA_VERSION = '1.20.
|
|
326
|
+
const DATA_VERSION = '1.20.3'
|
|
305
327
|
const DATA_URL = '/skill-eval/data/' + DATA_VERSION
|
|
306
328
|
const DATA_URL_FALLBACK = '/skill-eval/data'
|
|
307
329
|
const SETTINGS_URL = '/skill-eval/settings/' + DATA_VERSION
|
|
@@ -317,8 +339,13 @@ window.__ModuleLoader__.load({
|
|
|
317
339
|
|
|
318
340
|
function scoreColor(value) {
|
|
319
341
|
if (typeof value !== 'number' || !Number.isFinite(value)) return 'var(--dsw-alias-state-idle-primary, #a1a1aa)'
|
|
320
|
-
|
|
321
|
-
|
|
342
|
+
// Colour bands follow the configured grade floors, so a 92-point skill can
|
|
343
|
+
// never render green while its letter says B.
|
|
344
|
+
const thresholds = (store.get().data && store.get().data.scoring && store.get().data.scoring.thresholds) || {}
|
|
345
|
+
const excellent = Number(thresholds.excellent) || 90
|
|
346
|
+
const pass = Number(thresholds.pass) || 70
|
|
347
|
+
if (value >= excellent) return 'var(--dsw-alias-state-success-primary, #2f9e44)'
|
|
348
|
+
if (value >= pass) return 'var(--dsw-alias-state-warn-primary, #d97706)'
|
|
322
349
|
return 'var(--dsw-alias-state-error-primary, #c92a2a)'
|
|
323
350
|
}
|
|
324
351
|
|
|
@@ -335,6 +362,16 @@ window.__ModuleLoader__.load({
|
|
|
335
362
|
error.status = response.status
|
|
336
363
|
throw error
|
|
337
364
|
}
|
|
365
|
+
const type =
|
|
366
|
+
(response.headers && typeof response.headers.get === 'function' && response.headers.get('content-type')) || ''
|
|
367
|
+
if (type.indexOf('json') === -1) {
|
|
368
|
+
// DSH's prefix route answers 200 HTML for an unclaimed versioned subpath,
|
|
369
|
+
// so a status-only fallback would never fire during a restart-skew.
|
|
370
|
+
const error = new Error(url + ' returned ' + (type || 'no content-type') + ' instead of JSON')
|
|
371
|
+
error.status = response.status
|
|
372
|
+
error.notJson = true
|
|
373
|
+
throw error
|
|
374
|
+
}
|
|
338
375
|
return response.json()
|
|
339
376
|
}
|
|
340
377
|
|
|
@@ -356,7 +393,8 @@ window.__ModuleLoader__.load({
|
|
|
356
393
|
.catch((error) => {
|
|
357
394
|
// a host that predates the versioned alias (or a hot reload that kept
|
|
358
395
|
// the canonical route) still answers on the unversioned path
|
|
359
|
-
if (error && error.status === 404
|
|
396
|
+
if (error && (error.status === 404 || error.notJson || error instanceof SyntaxError))
|
|
397
|
+
return fetchReport(DATA_URL_FALLBACK)
|
|
360
398
|
throw error
|
|
361
399
|
})
|
|
362
400
|
.then((data) => {
|
|
@@ -384,11 +422,30 @@ window.__ModuleLoader__.load({
|
|
|
384
422
|
const BOARD_URL = '/skill-eval/board/' + DATA_VERSION
|
|
385
423
|
const CONFIG_URL = BOARD_URL + '/config'
|
|
386
424
|
|
|
425
|
+
function pagePath(kind) {
|
|
426
|
+
try {
|
|
427
|
+
const pages = store.get().data && store.get().data.pages
|
|
428
|
+
const value = pages && pages[kind]
|
|
429
|
+
if (typeof value === 'string' && value) return value
|
|
430
|
+
} catch {
|
|
431
|
+
// fall back to the default route
|
|
432
|
+
}
|
|
433
|
+
return kind === 'board' ? BOARD_URL : CONFIG_URL
|
|
434
|
+
}
|
|
435
|
+
|
|
387
436
|
function boardUrl() {
|
|
388
437
|
try {
|
|
389
|
-
return window.location.origin +
|
|
438
|
+
return window.location.origin + pagePath('board')
|
|
439
|
+
} catch {
|
|
440
|
+
return pagePath('board')
|
|
441
|
+
}
|
|
442
|
+
}
|
|
443
|
+
|
|
444
|
+
function configUrl() {
|
|
445
|
+
try {
|
|
446
|
+
return window.location.origin + pagePath('config')
|
|
390
447
|
} catch {
|
|
391
|
-
return
|
|
448
|
+
return pagePath('config')
|
|
392
449
|
}
|
|
393
450
|
}
|
|
394
451
|
|
|
@@ -484,26 +541,53 @@ window.__ModuleLoader__.load({
|
|
|
484
541
|
const settingsStore = createStore({ loading: false, saving: false, data: null, error: null, savedAt: 0 })
|
|
485
542
|
const useSettings = () => useSyncExternalStore(settingsStore.subscribe, settingsStore.get, settingsStore.get)
|
|
486
543
|
|
|
487
|
-
|
|
488
|
-
|
|
489
|
-
|
|
490
|
-
|
|
491
|
-
|
|
492
|
-
|
|
544
|
+
// One generation counter for reads and writes: a slow GET must not overwrite
|
|
545
|
+
// the values a newer save just persisted.
|
|
546
|
+
let settingsGeneration = 0
|
|
547
|
+
|
|
548
|
+
async function fetchSettingsPayload(url) {
|
|
549
|
+
const response = await fetch(url, { headers: { accept: 'application/json' } })
|
|
493
550
|
const type =
|
|
494
551
|
(response.headers && typeof response.headers.get === 'function' && response.headers.get('content-type')) || ''
|
|
495
|
-
if (type.indexOf('json') === -1) {
|
|
496
|
-
//
|
|
497
|
-
//
|
|
498
|
-
|
|
552
|
+
if (!response.ok || type.indexOf('json') === -1) {
|
|
553
|
+
// DSH's stale prefix route answers 200 HTML for an unclaimed subpath, so
|
|
554
|
+
// treat "not JSON" the same as 404 and try the canonical route.
|
|
555
|
+
const error = new Error(!response.ok ? 'HTTP ' + response.status : 'host-too-old')
|
|
556
|
+
error.status = response.status
|
|
557
|
+
error.notJson = type.indexOf('json') === -1
|
|
558
|
+
throw error
|
|
559
|
+
}
|
|
560
|
+
return response.json()
|
|
561
|
+
}
|
|
562
|
+
|
|
563
|
+
async function fetchSettings() {
|
|
564
|
+
const generation = (settingsGeneration += 1)
|
|
565
|
+
const update = (patch) => {
|
|
566
|
+
if (generation === settingsGeneration) settingsStore.set((state) => ({ ...state, ...patch }))
|
|
499
567
|
}
|
|
500
|
-
|
|
501
|
-
|
|
568
|
+
update({ loading: true, error: null })
|
|
569
|
+
let data
|
|
570
|
+
try {
|
|
571
|
+
data = await fetchSettingsPayload(SETTINGS_URL)
|
|
572
|
+
} catch (error) {
|
|
573
|
+
const retry = error && (error.status === 404 || error.notJson)
|
|
574
|
+
if (!retry) throw error
|
|
575
|
+
try {
|
|
576
|
+
data = await fetchSettingsPayload(SETTINGS_URL_FALLBACK)
|
|
577
|
+
} catch (fallbackError) {
|
|
578
|
+
if (fallbackError && (fallbackError.status === 404 || fallbackError.notJson))
|
|
579
|
+
throw new Error('host-too-old', { cause: fallbackError })
|
|
580
|
+
throw fallbackError
|
|
581
|
+
}
|
|
582
|
+
}
|
|
583
|
+
update({ loading: false, data })
|
|
502
584
|
return data
|
|
503
585
|
}
|
|
504
586
|
|
|
505
587
|
async function saveSettings(values) {
|
|
506
|
-
|
|
588
|
+
// Invalidate any in-flight GET; its snapshot predates this save.
|
|
589
|
+
settingsGeneration += 1
|
|
590
|
+
settingsStore.set((state) => ({ ...state, saving: true, error: null, savedAt: 0 }))
|
|
507
591
|
try {
|
|
508
592
|
const post = (url) =>
|
|
509
593
|
fetch(url, {
|
|
@@ -518,6 +602,7 @@ window.__ModuleLoader__.load({
|
|
|
518
602
|
settingsStore.set((state) => ({
|
|
519
603
|
...state,
|
|
520
604
|
saving: false,
|
|
605
|
+
savedAt: 0,
|
|
521
606
|
error: payload.errors?.[0]?.message ?? 'HTTP ' + response.status,
|
|
522
607
|
}))
|
|
523
608
|
return false
|
|
@@ -533,6 +618,7 @@ window.__ModuleLoader__.load({
|
|
|
533
618
|
settingsStore.set((state) => ({
|
|
534
619
|
...state,
|
|
535
620
|
saving: false,
|
|
621
|
+
savedAt: 0,
|
|
536
622
|
error: error instanceof Error ? error.message : String(error),
|
|
537
623
|
}))
|
|
538
624
|
return false
|
|
@@ -548,6 +634,8 @@ window.__ModuleLoader__.load({
|
|
|
548
634
|
const [draft, setDraft] = useState(null)
|
|
549
635
|
useEffect(() => {
|
|
550
636
|
let alive = true
|
|
637
|
+
// A success banner from a previous mount must not reappear on reopen.
|
|
638
|
+
settingsStore.set((current) => (current.savedAt ? { ...current, savedAt: 0 } : current))
|
|
551
639
|
fetchSettings().catch((error) => {
|
|
552
640
|
if (alive) settingsStore.set((current) => ({ ...current, loading: false, error: error.message }))
|
|
553
641
|
})
|
|
@@ -560,8 +648,10 @@ window.__ModuleLoader__.load({
|
|
|
560
648
|
const groups = data && data.fields ? groupSettingsFields(data.fields, lang) : []
|
|
561
649
|
const set = (key, value) => setDraft({ ...values, [key]: value })
|
|
562
650
|
const save = async () => {
|
|
563
|
-
const
|
|
564
|
-
|
|
651
|
+
const submitted = values
|
|
652
|
+
const ok = await saveSettings(submitted)
|
|
653
|
+
// Keep edits typed while the POST was in flight instead of discarding them.
|
|
654
|
+
if (ok) setDraft((current) => (current === null || current === submitted ? null : current))
|
|
565
655
|
}
|
|
566
656
|
if (state.loading && !data) return h('div', { className: 'sev-settings' }, t.settingsLoading)
|
|
567
657
|
if (state.error && !data) {
|
|
@@ -576,7 +666,7 @@ window.__ModuleLoader__.load({
|
|
|
576
666
|
{
|
|
577
667
|
type: 'button',
|
|
578
668
|
className: 'sev-btn',
|
|
579
|
-
onClick: () => window.open(
|
|
669
|
+
onClick: () => window.open(configUrl(), '_blank', 'noopener'),
|
|
580
670
|
},
|
|
581
671
|
t.settingsOpenWeb,
|
|
582
672
|
)
|
|
@@ -623,6 +713,8 @@ window.__ModuleLoader__.load({
|
|
|
623
713
|
h('input', {
|
|
624
714
|
type: 'checkbox',
|
|
625
715
|
checked: values[field.key] === true,
|
|
716
|
+
disabled: state.saving,
|
|
717
|
+
'aria-label': lang === 'zh' ? field.labelZh || field.label : field.label,
|
|
626
718
|
onChange: (event) => set(field.key, event.currentTarget.checked),
|
|
627
719
|
}),
|
|
628
720
|
h('i', { 'aria-hidden': 'true' }),
|
|
@@ -636,6 +728,8 @@ window.__ModuleLoader__.load({
|
|
|
636
728
|
max: field.max,
|
|
637
729
|
step: field.step,
|
|
638
730
|
value: values[field.key] ?? '',
|
|
731
|
+
disabled: state.saving,
|
|
732
|
+
'aria-label': lang === 'zh' ? field.labelZh || field.label : field.label,
|
|
639
733
|
onChange: (event) => set(field.key, event.currentTarget.value),
|
|
640
734
|
}),
|
|
641
735
|
field.unit ? h('em', null, unitLabel(field.unit, lang)) : null,
|
|
@@ -658,7 +752,7 @@ window.__ModuleLoader__.load({
|
|
|
658
752
|
{
|
|
659
753
|
type: 'button',
|
|
660
754
|
className: 'sev-btn sev-btn--ghost',
|
|
661
|
-
onClick: () => window.open(
|
|
755
|
+
onClick: () => window.open(configUrl(), '_blank', 'noopener'),
|
|
662
756
|
},
|
|
663
757
|
t.settingsOpenWeb,
|
|
664
758
|
)
|
|
@@ -1402,11 +1496,15 @@ window.__ModuleLoader__.load({
|
|
|
1402
1496
|
// denied there), so the dialog is the entry point that always works.
|
|
1403
1497
|
useEffect(() => {
|
|
1404
1498
|
if (!configOpen) return undefined
|
|
1499
|
+
const opener = typeof document !== 'undefined' ? document.activeElement : null
|
|
1405
1500
|
const onKey = (event) => {
|
|
1406
1501
|
if (event.key === 'Escape') setConfigOpen(false)
|
|
1407
1502
|
}
|
|
1408
1503
|
window.addEventListener('keydown', onKey)
|
|
1409
|
-
return () =>
|
|
1504
|
+
return () => {
|
|
1505
|
+
window.removeEventListener('keydown', onKey)
|
|
1506
|
+
if (opener && typeof opener.focus === 'function') opener.focus()
|
|
1507
|
+
}
|
|
1410
1508
|
}, [configOpen])
|
|
1411
1509
|
const data = state.data
|
|
1412
1510
|
const skills = (data && data.skills) || []
|
|
@@ -1427,9 +1525,7 @@ window.__ModuleLoader__.load({
|
|
|
1427
1525
|
const trapFocus = (event) => {
|
|
1428
1526
|
if (event.key !== 'Tab') return
|
|
1429
1527
|
const focusable = [
|
|
1430
|
-
...event.currentTarget.querySelectorAll(
|
|
1431
|
-
'button,[href],input,select,textarea,[tabindex]:not([tabindex="-1"])',
|
|
1432
|
-
),
|
|
1528
|
+
...event.currentTarget.querySelectorAll('button,[href],input,select,textarea,[tabindex]:not([tabindex="-1"])'),
|
|
1433
1529
|
].filter((node) => !node.disabled && node.offsetParent !== null)
|
|
1434
1530
|
if (focusable.length === 0) return
|
|
1435
1531
|
const first = focusable[0]
|
|
@@ -1452,7 +1548,11 @@ window.__ModuleLoader__.load({
|
|
|
1452
1548
|
const needle = query.trim().toLowerCase()
|
|
1453
1549
|
const visibleSkills = skills.filter(
|
|
1454
1550
|
(entry) =>
|
|
1455
|
-
(!onlyActionable || hasFix(entry)) &&
|
|
1551
|
+
(!onlyActionable || hasFix(entry)) &&
|
|
1552
|
+
(needle.length === 0 ||
|
|
1553
|
+
String(entry.skill ?? '')
|
|
1554
|
+
.toLowerCase()
|
|
1555
|
+
.includes(needle)),
|
|
1456
1556
|
)
|
|
1457
1557
|
const updated =
|
|
1458
1558
|
data && data.generatedAt && Number.isFinite(Date.parse(data.generatedAt))
|
|
@@ -1493,9 +1593,23 @@ window.__ModuleLoader__.load({
|
|
|
1493
1593
|
key: view,
|
|
1494
1594
|
type: 'button',
|
|
1495
1595
|
role: 'tab',
|
|
1596
|
+
id: 'sev-board-tab-' + view,
|
|
1597
|
+
'aria-controls': 'sev-board-view',
|
|
1496
1598
|
'aria-selected': state.view === view,
|
|
1599
|
+
tabIndex: state.view === view ? 0 : -1,
|
|
1497
1600
|
className: 'sev-tab' + (state.view === view ? ' sev-tab--on' : ''),
|
|
1498
1601
|
onClick: () => setView(view),
|
|
1602
|
+
onKeyDown: (event) => {
|
|
1603
|
+
if (event.key !== 'ArrowRight' && event.key !== 'ArrowLeft') return
|
|
1604
|
+
event.preventDefault()
|
|
1605
|
+
const views = ['skills', 'steps', 'runs', 'mcp', 'dev']
|
|
1606
|
+
const index = views.indexOf(view)
|
|
1607
|
+
const next = views[(index + (event.key === 'ArrowRight' ? 1 : views.length - 1)) % views.length]
|
|
1608
|
+
setView(next)
|
|
1609
|
+
const target =
|
|
1610
|
+
typeof document !== 'undefined' ? document.getElementById('sev-board-tab-' + next) : null
|
|
1611
|
+
if (target) target.focus()
|
|
1612
|
+
},
|
|
1499
1613
|
},
|
|
1500
1614
|
t.tabs[view],
|
|
1501
1615
|
),
|
|
@@ -1573,32 +1687,36 @@ window.__ModuleLoader__.load({
|
|
|
1573
1687
|
),
|
|
1574
1688
|
)
|
|
1575
1689
|
: null,
|
|
1576
|
-
|
|
1577
|
-
|
|
1578
|
-
|
|
1579
|
-
|
|
1580
|
-
|
|
1581
|
-
:
|
|
1582
|
-
|
|
1583
|
-
|
|
1584
|
-
|
|
1585
|
-
|
|
1586
|
-
|
|
1587
|
-
|
|
1588
|
-
|
|
1589
|
-
|
|
1590
|
-
|
|
1591
|
-
|
|
1592
|
-
|
|
1593
|
-
|
|
1594
|
-
|
|
1595
|
-
|
|
1596
|
-
|
|
1597
|
-
|
|
1598
|
-
|
|
1599
|
-
|
|
1600
|
-
|
|
1601
|
-
|
|
1690
|
+
h(
|
|
1691
|
+
'div',
|
|
1692
|
+
{ role: 'tabpanel', id: 'sev-board-view', 'aria-labelledby': 'sev-board-tab-' + state.view },
|
|
1693
|
+
state.view === 'skills'
|
|
1694
|
+
? !state.loading && !state.error && skills.length === 0
|
|
1695
|
+
? h('div', { className: 'sev-empty' }, t.empty)
|
|
1696
|
+
: visibleSkills.length === 0
|
|
1697
|
+
? h('div', { className: 'sev-empty' }, t.noMatch)
|
|
1698
|
+
: visibleSkills.map((skill) =>
|
|
1699
|
+
h(SkillCard, {
|
|
1700
|
+
key: skill.skill,
|
|
1701
|
+
skill,
|
|
1702
|
+
t,
|
|
1703
|
+
lang,
|
|
1704
|
+
onTrace: (name) => store.set((current) => ({ ...current, view: 'steps', stepFilter: name })),
|
|
1705
|
+
}),
|
|
1706
|
+
)
|
|
1707
|
+
: state.view === 'steps'
|
|
1708
|
+
? h(StepsView, {
|
|
1709
|
+
data,
|
|
1710
|
+
t,
|
|
1711
|
+
filter: state.stepFilter || '',
|
|
1712
|
+
onFilter: (name) => store.set((current) => ({ ...current, stepFilter: name })),
|
|
1713
|
+
})
|
|
1714
|
+
: state.view === 'runs'
|
|
1715
|
+
? h(RunsView, { data, t })
|
|
1716
|
+
: state.view === 'mcp'
|
|
1717
|
+
? h(McpView, { data, t, lang })
|
|
1718
|
+
: h(DevView, { data, t }),
|
|
1719
|
+
),
|
|
1602
1720
|
configOpen
|
|
1603
1721
|
? h(
|
|
1604
1722
|
'div',
|
|
@@ -1940,14 +2058,16 @@ window.__ModuleLoader__.load({
|
|
|
1940
2058
|
h(
|
|
1941
2059
|
'div',
|
|
1942
2060
|
{ className: 'sev-advice-list' },
|
|
1943
|
-
failing
|
|
1944
|
-
|
|
1945
|
-
|
|
1946
|
-
|
|
1947
|
-
|
|
1948
|
-
|
|
2061
|
+
failing
|
|
2062
|
+
.slice(0, 8)
|
|
2063
|
+
.map((tool) =>
|
|
2064
|
+
h(
|
|
2065
|
+
'div',
|
|
2066
|
+
{ className: 'sev-sub-row', key: tool.server + '/' + tool.tool },
|
|
2067
|
+
h('span', null, tool.server + '/' + tool.tool),
|
|
2068
|
+
h('b', null, (tool.errors || 0) + '/' + (tool.calls || 0)),
|
|
2069
|
+
),
|
|
1949
2070
|
),
|
|
1950
|
-
),
|
|
1951
2071
|
),
|
|
1952
2072
|
)
|
|
1953
2073
|
: null,
|
|
@@ -1992,13 +2112,13 @@ window.__ModuleLoader__.load({
|
|
|
1992
2112
|
[t.devLabels.api, DATA_URL],
|
|
1993
2113
|
[
|
|
1994
2114
|
t.devLabels.module,
|
|
1995
|
-
(dev.moduleVersion ?? '—') +
|
|
1996
|
-
' / ' +
|
|
1997
|
-
(dev.diskVersion ?? '—') +
|
|
1998
|
-
(dev.skew ? ' ⚠ ' + t.devLabels.skew : ''),
|
|
2115
|
+
(dev.moduleVersion ?? '—') + ' / ' + (dev.diskVersion ?? '—') + (dev.skew ? ' ⚠ ' + t.devLabels.skew : ''),
|
|
1999
2116
|
],
|
|
2000
2117
|
[t.devLabels.routes, String(dev.routes ?? '—') + ' · ' + t.devLabels.tools + ' ' + String(dev.tools ?? '—')],
|
|
2001
|
-
[t.devLabels.
|
|
2118
|
+
[t.devLabels.settingsFile, dev.settingsFile || '—'],
|
|
2119
|
+
[t.devLabels.warnings, String(dev.warnings ?? 0)],
|
|
2120
|
+
[t.devLabels.inspect, dev.inspectProvider ? t.devLabels.ok : '—'],
|
|
2121
|
+
[t.devLabels.store, totals.storeError ? String(totals.storeError) : t.devLabels.ok],
|
|
2002
2122
|
[t.version, DATA_VERSION],
|
|
2003
2123
|
]
|
|
2004
2124
|
return h(
|
|
@@ -2042,9 +2162,26 @@ window.__ModuleLoader__.load({
|
|
|
2042
2162
|
const { lang, t } = useLang()
|
|
2043
2163
|
const state = useStore()
|
|
2044
2164
|
|
|
2165
|
+
const retryRef = useRef({ key: '', count: 0 })
|
|
2045
2166
|
useEffect(() => {
|
|
2046
2167
|
ensureReport()
|
|
2047
2168
|
}, [sessionId, turn && turn.turn])
|
|
2169
|
+
// ensureReport() has a 10 s warm-cache window; a turn that settles inside it
|
|
2170
|
+
// has no turnLoads row yet and would render nothing forever. Retry twice.
|
|
2171
|
+
useEffect(() => {
|
|
2172
|
+
const turnNumber =
|
|
2173
|
+
typeof turn?.turn === 'number' ? turn.turn : typeof turn?.number === 'number' ? turn.number : null
|
|
2174
|
+
if (turnNumber === null) return undefined
|
|
2175
|
+
const data = state.data
|
|
2176
|
+
if (!data || !Array.isArray(data.turnLoads)) return undefined
|
|
2177
|
+
if (data.turnLoads.some((row) => row.sessionId === sessionId && row.turn === turnNumber)) return undefined
|
|
2178
|
+
const key = sessionId + ':' + turnNumber
|
|
2179
|
+
if (retryRef.current.key !== key) retryRef.current = { key, count: 0 }
|
|
2180
|
+
if (retryRef.current.count >= 2) return undefined
|
|
2181
|
+
retryRef.current.count += 1
|
|
2182
|
+
const timer = setTimeout(() => load(), 1200)
|
|
2183
|
+
return () => clearTimeout(timer)
|
|
2184
|
+
}, [sessionId, turn, state.data, state.fetchedAt])
|
|
2048
2185
|
|
|
2049
2186
|
if (!turn) return null
|
|
2050
2187
|
// `status` is 'closed' on current hosts; older/other builds may omit it or
|
|
@@ -2071,12 +2208,9 @@ window.__ModuleLoader__.load({
|
|
|
2071
2208
|
(node) => node && node.location && node.location.turn && node.location.turn.turn === turnNumber,
|
|
2072
2209
|
)
|
|
2073
2210
|
toolCalls = inTurn.filter((node) => node.kind === 'tool-call').length
|
|
2074
|
-
|
|
2075
|
-
|
|
2076
|
-
|
|
2077
|
-
node.data &&
|
|
2078
|
-
(node.data.isError === true || (node.data.result && node.data.result.isError === true)),
|
|
2079
|
-
).length
|
|
2211
|
+
// DSH's tool-call node payload is ToolChatData { root: ToolCallBlock }; the
|
|
2212
|
+
// error flag lives on the settled root block, not on data.isError.
|
|
2213
|
+
errors = inTurn.filter((node) => node?.kind === 'tool-call' && node.data?.root?.isError === true).length
|
|
2080
2214
|
}
|
|
2081
2215
|
|
|
2082
2216
|
const skills = [...new Set(rows.flatMap((row) => row.skills))]
|
|
@@ -2161,7 +2295,7 @@ window.__ModuleLoader__.load({
|
|
|
2161
2295
|
{ key: (edit.at || '') + '-' + index },
|
|
2162
2296
|
(edit.verdict === 'accept' ? '✅ ' : edit.verdict === 'reject' ? '🚫 ' : '⚠️ ') +
|
|
2163
2297
|
(edit.dimension || 'static') +
|
|
2164
|
-
(edit.hypothesis ? ' · ' + edit.hypothesis.slice(0, 60) : ''),
|
|
2298
|
+
(edit.hypothesis ? ' · ' + String(edit.hypothesis).slice(0, 60) : ''),
|
|
2165
2299
|
),
|
|
2166
2300
|
),
|
|
2167
2301
|
)
|
|
@@ -2341,9 +2475,24 @@ window.__ModuleLoader__.load({
|
|
|
2341
2475
|
key: view,
|
|
2342
2476
|
type: 'button',
|
|
2343
2477
|
role: 'tab',
|
|
2478
|
+
id: 'sev-overlay-tab-' + view,
|
|
2479
|
+
'aria-controls': 'sev-overlay-view',
|
|
2344
2480
|
'aria-selected': state.view === view,
|
|
2481
|
+
tabIndex: state.view === view ? 0 : -1,
|
|
2345
2482
|
className: 'sev-tab' + (state.view === view ? ' sev-tab--on' : ''),
|
|
2346
2483
|
onClick: () => setView(view),
|
|
2484
|
+
onKeyDown: (event) => {
|
|
2485
|
+
if (event.key !== 'ArrowRight' && event.key !== 'ArrowLeft') return
|
|
2486
|
+
event.preventDefault()
|
|
2487
|
+
const views = ['skills', 'steps', 'runs', 'mcp', 'dev']
|
|
2488
|
+
const index = views.indexOf(view)
|
|
2489
|
+
const next =
|
|
2490
|
+
views[(index + (event.key === 'ArrowRight' ? 1 : views.length - 1)) % views.length]
|
|
2491
|
+
setView(next)
|
|
2492
|
+
const target =
|
|
2493
|
+
typeof document !== 'undefined' ? document.getElementById('sev-overlay-tab-' + next) : null
|
|
2494
|
+
if (target) target.focus()
|
|
2495
|
+
},
|
|
2347
2496
|
},
|
|
2348
2497
|
t.tabs[view],
|
|
2349
2498
|
),
|
|
@@ -2383,7 +2532,12 @@ window.__ModuleLoader__.load({
|
|
|
2383
2532
|
),
|
|
2384
2533
|
h(
|
|
2385
2534
|
'div',
|
|
2386
|
-
{
|
|
2535
|
+
{
|
|
2536
|
+
className: 'sev-body',
|
|
2537
|
+
role: 'tabpanel',
|
|
2538
|
+
id: 'sev-overlay-view',
|
|
2539
|
+
'aria-labelledby': 'sev-overlay-tab-' + state.view,
|
|
2540
|
+
},
|
|
2387
2541
|
state.loading && !data ? h('div', { className: 'sev-empty' }, t.loading) : null,
|
|
2388
2542
|
data && !data.scoring
|
|
2389
2543
|
? h('div', { className: 'sev-stale' }, h('b', null, t.staleTitle), h('span', null, t.staleHint))
|
package/docs/ACCEPTANCE.md
CHANGED
|
@@ -66,6 +66,6 @@ The 1.20.0 four-layer audit and its fixes are recorded in [AUDIT.md](AUDIT.md).
|
|
|
66
66
|
node --test test/ # 239 passing (npm test builds the client first)
|
|
67
67
|
node scripts/doctor.mjs --release # release gate incl. dist artifact
|
|
68
68
|
dsh-skill-eval-accept # 17 checks against the running host
|
|
69
|
-
dsh-skill-eval-verify #
|
|
69
|
+
dsh-skill-eval-verify # 7 checks + 2 manual rows against the running host
|
|
70
70
|
npm publish --dry-run # packaging/publishability preview
|
|
71
71
|
```
|
package/docs/AUDIT.md
CHANGED
|
@@ -62,3 +62,21 @@ the fixes have regression tests (`test/audit-regressions.test.js`, `test/store.t
|
|
|
62
62
|
- Threshold ordering (`excellent ≥ good ≥ pass`) is repaired at scoring time but
|
|
63
63
|
not rejected at write time; settings `mcpErrorRatePercent` allows 0 while the
|
|
64
64
|
Config schema requires ≥1.
|
|
65
|
+
|
|
66
|
+
## 1.20.3 复核修正(Review corrections)
|
|
67
|
+
|
|
68
|
+
1.20.3 的独立复核(宿主、客户端、测试/打包、另一模型)发现以下**原审计声明不成立**,现已修复并有回归测试:
|
|
69
|
+
|
|
70
|
+
| 原声明 | 实际情况(1.20.2) | 1.20.3 |
|
|
71
|
+
| ---------------------------------------------- | --------------------------------------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------- |
|
|
72
|
+
| #9 写请求已做同源围栏,rebinding 只剩读风险 | `Origin.host === Host` 在 rebinding 下两者都是攻击者域名;配置页 POST 不需要自定义头 | Host 也必须命中 `getOrigin()`/loopback 允许表;新增 DNS rebinding、缺失头、超大 body 的 403/413 测试;文档改为"写已围栏,读仍依赖宿主" |
|
|
73
|
+
| #11 并发保存不再丢字段 | 只串行化了写;读-改-写在队列外,两个并发 partial POST 会互相覆盖 | 新增 `updateSettings()`:读+合并+临时文件+rename 全在同一队列 job;并发 partial 测试 |
|
|
74
|
+
| #15 accept 从 served report 读 dataDir | `JSON.parse(await fetchReport())` 对已解析对象抛错并被吞掉,实际仍按 mtime 扫描;`--data-dir` 的值还会被当成 base URL | 直接使用返回对象;参数解析移到 `scripts/lib.mjs` 并测试 |
|
|
75
|
+
| #6 坏记录不会炸报告 | `window.mcp` 的嵌套 tool 条目未归一化,一条 null 就让 `/data`、看板、工具全 500 | 读路径逐层归一化 + score/report 双保险;回归测试 |
|
|
76
|
+
| 1.19 起 `dsh-skill-eval-doctor` 可做安装包预检 | 打出的 tarball 不含 `client/`,doctor 的 client-build 检查永远失败且给出的修复命令不可执行 | 缺 `client/parts.json` 视为已打包安装;`--release` 实际打开 tarball 校验条目 |
|
|
77
|
+
| 设置"保存后立即生效" | `windowTtlMs`、`reportOnTurnEnd`、`turnReports`、`promptHint` 在 apply 时固化 | 全部运行时读取;快照与提示词开关无需重启 |
|
|
78
|
+
| 每回合看板 | 写入时取全局最新 `turnLoads`,并发会话/去抖窗口会写错或覆盖 | `agent/turn-stopping` 传 sessionId+turn;一次去抖写入可为多个回合各写一份 |
|
|
79
|
+
| 回合结果条显示 calls/errors | 读 `data.isError`,但 DSH 契约是 `ToolChatData { root }`,错误数恒为 0;测试锁了错误形状 | 改读 `root.isError`;缓存 10s 内没有本回合行时重试两次 |
|
|
80
|
+
| 失败加载不污染证据 | 失败 `skill` 仍会 `openWindow`,turn-end 落盘并可能顶掉上一个技能窗口 | 仅成功加载开窗;turn-end/dispose 后测试断言无 window |
|
|
81
|
+
|
|
82
|
+
仍未消除的残余风险:DNS rebinding 的**读**(`/data` 可被同源 rebinding 页面读取)依赖 DSH 宿主级围栏;插件侧只围栏写。
|
package/docs/DETAIL.md
CHANGED
|
@@ -20,9 +20,11 @@ pass line) are borrowed from those projects. No code is copied.
|
|
|
20
20
|
|
|
21
21
|
## What is recorded
|
|
22
22
|
|
|
23
|
-
|
|
24
|
-
(override with `dataDir` or `DSH_SKILL_EVAL_DIR`)
|
|
25
|
-
|
|
23
|
+
Five append-only JSONL record types live in `$DSH_HOME/skill-eval/$DSH_PROFILE/events.jsonl`
|
|
24
|
+
(override with `dataDir` or `DSH_SKILL_EVAL_DIR`): `skill`, `window`, `static`
|
|
25
|
+
(one authoring profile per resolved skill), `step` (one row per agent step) and
|
|
26
|
+
`edit` (the review ledger). The session log is **never** written to — plugins
|
|
27
|
+
must not append foreign event types to it.
|
|
26
28
|
|
|
27
29
|
```jsonc
|
|
28
30
|
// written when a `skill` tool call completes
|
|
@@ -428,8 +430,11 @@ Known validity limits, stated as limits rather than fixed:
|
|
|
428
430
|
|
|
429
431
|
## Configuration keys
|
|
430
432
|
|
|
431
|
-
|
|
432
|
-
|
|
433
|
+
The config keys below are the ones worth documenting; the native form renders
|
|
434
|
+
the 25 tunable settings (the authoritative list is `SETTINGS_FIELDS`). Several
|
|
435
|
+
keys here are config-only (`dataDir`, `dashboardPath`, `registerTool`,
|
|
436
|
+
`inspectProvider`). The form persists to `settings.json`;
|
|
437
|
+
`dataDir`/`DSH_SKILL_EVAL_DIR` still wins for the data root.
|
|
433
438
|
|
|
434
439
|
```yaml
|
|
435
440
|
enabled: true
|
|
@@ -440,7 +445,7 @@ weights: { static: 30, reliability: 25, effectiveness: 30, efficiency: 15 } # no
|
|
|
440
445
|
thresholds: { excellent: 90, good: 80, pass: 70 } # A/B/C floors
|
|
441
446
|
windowTtlMs: 1800000 # a run older than this is not blamed on a later message
|
|
442
447
|
dataDir: ~/.dsh/skill-eval # optional override
|
|
443
|
-
turnReports: true # one
|
|
448
|
+
turnReports: true # one aggregate snapshot per settled turn under reports/turns/
|
|
444
449
|
turnReportsKeep: 50 # how many per-turn boards to retain
|
|
445
450
|
reportOnTurnEnd: true # write reports/latest.md + latest.json after each turn (no tokens)
|
|
446
451
|
tokenBudgetPerLoad: 20000 # billable tokens per load before efficiency/advice react
|
package/docs/ENTRYPOINTS.md
CHANGED
|
@@ -48,7 +48,7 @@ The board and settings page are bilingual: `Accept-Language` picks the language,
|
|
|
48
48
|
```bash
|
|
49
49
|
dsh-skill-eval-doctor # 打包预检 19 项(--release 加产物检查)· packaging pre-flight
|
|
50
50
|
dsh-skill-eval-accept # 代次/数据契约 17 项 · generation & data contract
|
|
51
|
-
dsh-skill-eval-verify # 产品面可达
|
|
51
|
+
dsh-skill-eval-verify # 产品面可达 7 项 + 2 项手动 · product surfaces (after a reload)
|
|
52
52
|
```
|
|
53
53
|
|
|
54
54
|
## 调试(可选) · Debug (opt-in)
|
package/docs/RELEASE.md
CHANGED
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
## 1. 代码与文案
|
|
4
4
|
|
|
5
|
-
- [ ] `npm run build` 全绿(format check、lint、client build、doctor、**
|
|
5
|
+
- [ ] `npm run build` 全绿(format check、lint、client build、doctor、**258 tests**)
|
|
6
6
|
- [ ] `npm run build:pack` 产出 tarball 并通过 `doctor --release`
|
|
7
7
|
- [ ] `package.json` version 与 `client.js` 的 `DATA_VERSION` 一致(doctor 检查)
|
|
8
8
|
- [ ] CHANGELOG 有本次版本条目
|
|
@@ -11,7 +11,7 @@
|
|
|
11
11
|
## 2. 真实宿主验证
|
|
12
12
|
|
|
13
13
|
- [ ] `dsh-skill-eval-accept` PASS(17 项,含看板、配置页、回合快照、设置读写)
|
|
14
|
-
- [ ] `dsh-skill-eval-verify` PASS(
|
|
14
|
+
- [ ] `dsh-skill-eval-verify` PASS(7 项 + 2 项手动,确认宿主真的服务这一代代码)
|
|
15
15
|
- [ ] 打开侧边栏「技能评测」面板:五个标签(技能/轨迹/运行/MCP/开发者)、刷新、复制
|
|
16
16
|
- [ ] 设置 → 插件 → `dsh-skill-eval` → 配置:改一个值 → 保存 → 看到「已保存并立即生效」
|
|
17
17
|
- [ ] 在同一轮加载多个 skill:回合条显示每个 skill 的 chip 与各自 next fix
|