dsh-skill-eval 0.0.0-stage → 1.20.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +675 -0
- package/LICENSE +21 -0
- package/README.md +200 -2
- package/README.zh.md +157 -0
- package/client.js +2637 -0
- package/cordis.patch.yml +9 -0
- package/docs/ACCEPTANCE.md +71 -0
- package/docs/AUDIT.md +64 -0
- package/docs/CODE-REVIEW.md +49 -0
- package/docs/COMPARISON.md +33 -0
- package/docs/DETAIL.md +463 -0
- package/docs/DEV-RELOAD.md +114 -0
- package/docs/ENTRYPOINTS.md +65 -0
- package/docs/RELEASE.md +44 -0
- package/docs/RRSI-NOTES.md +129 -0
- package/docs/SCENARIOS.md +91 -0
- package/docs/UI-REVIEW.md +115 -0
- package/docs/WORKFLOW.md +108 -0
- package/icon.svg +9 -0
- package/index.js +1111 -0
- package/lib/accept.js +350 -0
- package/lib/advice.js +547 -0
- package/lib/critic.js +101 -0
- package/lib/dev.js +148 -0
- package/lib/history.js +140 -0
- package/lib/html.js +8 -0
- package/lib/inspect.js +142 -0
- package/lib/integrity.js +46 -0
- package/lib/loop.js +195 -0
- package/lib/random.js +29 -0
- package/lib/recorder.js +634 -0
- package/lib/render.js +582 -0
- package/lib/report.js +646 -0
- package/lib/score.js +525 -0
- package/lib/settings.js +396 -0
- package/lib/snapshot.js +166 -0
- package/lib/steps.js +141 -0
- package/lib/store.js +186 -0
- package/lib/validity.js +115 -0
- package/locale/en.json +6 -0
- package/locale/zh.json +6 -0
- package/package.json +96 -4
- package/scripts/accept.mjs +226 -0
- package/scripts/build-client.mjs +59 -0
- package/scripts/build.mjs +81 -0
- package/scripts/doctor.mjs +199 -0
- package/scripts/lib.mjs +52 -0
- package/scripts/verify-live.mjs +168 -0
|
@@ -0,0 +1,168 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
/**
|
|
3
|
+
* One-command live verification after a restart.
|
|
4
|
+
*
|
|
5
|
+
* The acceptance script answers "is this generation live"; this script answers
|
|
6
|
+
* "can the user actually see and change the product": versioned JSON, settings
|
|
7
|
+
* API, board sections, per-turn boards, and the two native entry points that
|
|
8
|
+
* only the client half can provide (those are reported as manual checks because
|
|
9
|
+
* a CLI cannot inspect the browser's slot tree).
|
|
10
|
+
*/
|
|
11
|
+
import { readFile, readdir } from 'node:fs/promises'
|
|
12
|
+
import path from 'node:path'
|
|
13
|
+
import { dashboardBase, expectedVersion, httpGet, resolveBaseUrl } from './lib.mjs'
|
|
14
|
+
|
|
15
|
+
const manifest = JSON.parse(await readFile(new URL('../package.json', import.meta.url), 'utf8'))
|
|
16
|
+
const version = expectedVersion(manifest.version)
|
|
17
|
+
|
|
18
|
+
const resolved = await resolveBaseUrl(process.argv[2])
|
|
19
|
+
if (!resolved.baseUrl) {
|
|
20
|
+
console.log('dsh-skill-eval verify: no base URL; pass http://127.0.0.1:<port> or export DSH_WEB_URL')
|
|
21
|
+
process.exit(1)
|
|
22
|
+
}
|
|
23
|
+
const base = resolved.baseUrl
|
|
24
|
+
const dashboard = dashboardBase()
|
|
25
|
+
// Hot reload keeps the canonical page route in older hands; the versioned
|
|
26
|
+
// alias always belongs to the generation that just started.
|
|
27
|
+
const boardPath = `${dashboard}/board/${version}`
|
|
28
|
+
const configPath = `${boardPath}/config`
|
|
29
|
+
const rows = []
|
|
30
|
+
const add = (id, ok, detail) => rows.push({ id, ok, detail })
|
|
31
|
+
|
|
32
|
+
const get = (pathname, expectType) => httpGet(base, pathname, { expectType })
|
|
33
|
+
|
|
34
|
+
// 1. the generation alias and the JSON contract
|
|
35
|
+
let report = null
|
|
36
|
+
try {
|
|
37
|
+
const { body } = await get(`${dashboard}/data/${version}`, 'application/json')
|
|
38
|
+
report = JSON.parse(body)
|
|
39
|
+
const keys = ['scoring', 'skills', 'totals', 'turnLoads', 'edits', 'validity', 'mcpServers', 'runs']
|
|
40
|
+
const missing = keys.filter((key) => !(key in report))
|
|
41
|
+
add(
|
|
42
|
+
'versioned-json',
|
|
43
|
+
missing.length === 0,
|
|
44
|
+
missing.length === 0 ? `data/${version} → JSON with ${keys.length} keys` : `missing ${missing.join(', ')}`,
|
|
45
|
+
)
|
|
46
|
+
} catch (error) {
|
|
47
|
+
add('versioned-json', false, error instanceof Error ? error.message : String(error))
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
// 2. settings API: fields + effective values
|
|
51
|
+
try {
|
|
52
|
+
const { body } = await get(`${dashboard}/settings/${version}`, 'application/json')
|
|
53
|
+
const payload = JSON.parse(body)
|
|
54
|
+
const fields = Array.isArray(payload.fields) ? payload.fields.length : 0
|
|
55
|
+
const hasWeights = payload.values && typeof payload.values['weights.static'] !== 'undefined'
|
|
56
|
+
add(
|
|
57
|
+
'settings-api',
|
|
58
|
+
fields > 0 && Boolean(hasWeights),
|
|
59
|
+
`${fields} field(s), weights.static=${payload.values?.['weights.static']}`,
|
|
60
|
+
)
|
|
61
|
+
} catch (error) {
|
|
62
|
+
add('settings-api', false, error instanceof Error ? error.message : String(error))
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
// 3. the board HTML carries the reporting sections
|
|
66
|
+
try {
|
|
67
|
+
let body
|
|
68
|
+
try {
|
|
69
|
+
body = (await get(boardPath, 'text/html')).body
|
|
70
|
+
} catch {
|
|
71
|
+
body = (await get(dashboard, 'text/html')).body
|
|
72
|
+
}
|
|
73
|
+
const sections = ['Next steps', 'What the evidence supports', 'Screened edits', 'config'].filter((needle) =>
|
|
74
|
+
body.includes(needle),
|
|
75
|
+
)
|
|
76
|
+
add('board-sections', sections.length >= 3, `found ${sections.length}/4: ${sections.join(', ')}`)
|
|
77
|
+
} catch (error) {
|
|
78
|
+
add('board-sections', false, error instanceof Error ? error.message : String(error))
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
// 4. the settings page renders the fields
|
|
82
|
+
try {
|
|
83
|
+
let body
|
|
84
|
+
try {
|
|
85
|
+
body = (await get(configPath, 'text/html')).body
|
|
86
|
+
} catch {
|
|
87
|
+
body = (await get(`${dashboard}/config`, 'text/html')).body
|
|
88
|
+
}
|
|
89
|
+
const fields = ['weights.static', 'thresholds.excellent', 'tokenBudgetPerLoad', 'adviceNoiseFloor'].filter((key) =>
|
|
90
|
+
body.includes(`name="${key}"`),
|
|
91
|
+
)
|
|
92
|
+
add('config-page', fields.length === 4, `found ${fields.length}/4 field(s)`)
|
|
93
|
+
} catch (error) {
|
|
94
|
+
add('config-page', false, error instanceof Error ? error.message : String(error))
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
// 5. per-turn boards on disk (only meaningful when the generation is live)
|
|
98
|
+
try {
|
|
99
|
+
const dir = report?.dataDir
|
|
100
|
+
if (!dir) throw new Error('report has no dataDir (versioned route is not live yet)')
|
|
101
|
+
const reportsDir = path.join(dir, 'reports')
|
|
102
|
+
let turns = []
|
|
103
|
+
try {
|
|
104
|
+
turns = (await readdir(path.join(reportsDir, 'turns'))).filter((name) => name.endsWith('.json'))
|
|
105
|
+
} catch {
|
|
106
|
+
turns = []
|
|
107
|
+
}
|
|
108
|
+
add(
|
|
109
|
+
'turn-boards',
|
|
110
|
+
true,
|
|
111
|
+
turns.length > 0
|
|
112
|
+
? `${turns.length} board(s), newest ${turns.sort().at(-1)}`
|
|
113
|
+
: 'none yet — load a skill, finish the turn, then re-run',
|
|
114
|
+
)
|
|
115
|
+
} catch (error) {
|
|
116
|
+
add('turn-boards', false, error instanceof Error ? error.message : String(error))
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
// 6. the report contract itself: both aliases agree and the numbers are sane
|
|
120
|
+
try {
|
|
121
|
+
const canonical = JSON.parse((await get(`${dashboard}/data`, 'application/json')).body)
|
|
122
|
+
const versioned = JSON.parse((await get(`${dashboard}/data/${version}`, 'application/json')).body)
|
|
123
|
+
const sameLoads = canonical.totals?.invocations === versioned.totals?.invocations
|
|
124
|
+
const percent = Object.values(versioned.scoring?.weightsPercent ?? {})
|
|
125
|
+
const sum = percent.reduce((a, b) => a + b, 0)
|
|
126
|
+
const withCi = (versioned.skills ?? []).filter((skill) => skill.score?.ci).length
|
|
127
|
+
const scored = (versioned.skills ?? []).length
|
|
128
|
+
const ok = sameLoads && percent.length === 4 && sum === 100
|
|
129
|
+
add(
|
|
130
|
+
'report-contract',
|
|
131
|
+
ok,
|
|
132
|
+
`canonical=versioned:${sameLoads}, weights sum ${sum}, ${withCi}/${scored} skill(s) with a 95% interval`,
|
|
133
|
+
)
|
|
134
|
+
} catch (error) {
|
|
135
|
+
add('report-contract', false, error instanceof Error ? error.message : String(error))
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
// 7. snapshots on disk for the data dir the host reports
|
|
139
|
+
try {
|
|
140
|
+
const snapshot = JSON.parse(
|
|
141
|
+
await readFile(path.join(report?.dataDir ?? '/nonexistent', 'reports', 'latest.json'), 'utf8'),
|
|
142
|
+
)
|
|
143
|
+
add(
|
|
144
|
+
'latest-snapshot',
|
|
145
|
+
true,
|
|
146
|
+
`${report?.dataDir}/reports/latest.json: ${(snapshot.skills ?? []).length} skill(s), ${(snapshot.runs ?? []).length} run(s)`,
|
|
147
|
+
)
|
|
148
|
+
} catch (error) {
|
|
149
|
+
add('latest-snapshot', false, error instanceof Error ? error.message : String(error))
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
// manual checks the CLI cannot perform
|
|
153
|
+
add(
|
|
154
|
+
'manual:native-seats',
|
|
155
|
+
true,
|
|
156
|
+
'open the sidebar Skill eval panel icon (board) and Settings → Plugins → dsh-skill-eval (bundle config)',
|
|
157
|
+
)
|
|
158
|
+
add('manual:turn-strip', true, 'load a skill in a turn; its result strip shows score, next fix and edit history')
|
|
159
|
+
|
|
160
|
+
const failed = rows.filter((row) => !row.ok)
|
|
161
|
+
console.log(`dsh-skill-eval verify — ${base} (version ${version})`)
|
|
162
|
+
for (const row of rows) console.log(` [${row.ok ? 'PASS' : 'FAIL'}] ${row.id.padEnd(18)} ${row.detail}`)
|
|
163
|
+
console.log('')
|
|
164
|
+
if (failed.length > 0) {
|
|
165
|
+
console.log(`FAIL (${failed.length}) — restart DeepSeek Harness if versioned-json reports text/html.`)
|
|
166
|
+
process.exit(1)
|
|
167
|
+
}
|
|
168
|
+
console.log('PASS — the running host serves this generation and the product surfaces are reachable.')
|