dsh-skill-eval 0.0.0-stage → 1.20.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (48) hide show
  1. package/CHANGELOG.md +718 -0
  2. package/LICENSE +21 -0
  3. package/README.md +207 -2
  4. package/README.zh.md +164 -0
  5. package/client.js +2791 -0
  6. package/cordis.patch.yml +9 -0
  7. package/docs/ACCEPTANCE.md +71 -0
  8. package/docs/AUDIT.md +82 -0
  9. package/docs/CODE-REVIEW.md +49 -0
  10. package/docs/COMPARISON.md +33 -0
  11. package/docs/DETAIL.md +468 -0
  12. package/docs/DEV-RELOAD.md +114 -0
  13. package/docs/ENTRYPOINTS.md +65 -0
  14. package/docs/RELEASE.md +44 -0
  15. package/docs/RRSI-NOTES.md +129 -0
  16. package/docs/SCENARIOS.md +91 -0
  17. package/docs/UI-REVIEW.md +115 -0
  18. package/docs/WORKFLOW.md +108 -0
  19. package/icon.svg +9 -0
  20. package/index.js +1173 -0
  21. package/lib/accept.js +350 -0
  22. package/lib/advice.js +557 -0
  23. package/lib/critic.js +101 -0
  24. package/lib/dev.js +148 -0
  25. package/lib/history.js +141 -0
  26. package/lib/html.js +17 -0
  27. package/lib/inspect.js +142 -0
  28. package/lib/integrity.js +46 -0
  29. package/lib/loop.js +195 -0
  30. package/lib/random.js +29 -0
  31. package/lib/recorder.js +644 -0
  32. package/lib/render.js +580 -0
  33. package/lib/report.js +689 -0
  34. package/lib/score.js +565 -0
  35. package/lib/settings.js +415 -0
  36. package/lib/snapshot.js +250 -0
  37. package/lib/steps.js +141 -0
  38. package/lib/store.js +240 -0
  39. package/lib/validity.js +125 -0
  40. package/locale/en.json +6 -0
  41. package/locale/zh.json +6 -0
  42. package/package.json +96 -4
  43. package/scripts/accept.mjs +227 -0
  44. package/scripts/build-client.mjs +59 -0
  45. package/scripts/build.mjs +81 -0
  46. package/scripts/doctor.mjs +234 -0
  47. package/scripts/lib.mjs +87 -0
  48. package/scripts/verify-live.mjs +178 -0
@@ -0,0 +1,178 @@
1
+ #!/usr/bin/env node
2
+ /**
3
+ * One-command live verification after a restart.
4
+ *
5
+ * The acceptance script answers "is this generation live"; this script answers
6
+ * "can the user actually see and change the product": versioned JSON, settings
7
+ * API, board sections, per-turn boards, and the two native entry points that
8
+ * only the client half can provide (those are reported as manual checks because
9
+ * a CLI cannot inspect the browser's slot tree).
10
+ */
11
+ import { readFile, readdir } from 'node:fs/promises'
12
+ import path from 'node:path'
13
+ import { dashboardBase, expectedVersion, httpGet, resolveBaseUrl } from './lib.mjs'
14
+
15
+ const manifest = JSON.parse(await readFile(new URL('../package.json', import.meta.url), 'utf8'))
16
+ const version = expectedVersion(manifest.version)
17
+
18
+ const resolved = await resolveBaseUrl(process.argv[2])
19
+ if (!resolved.baseUrl) {
20
+ console.log('dsh-skill-eval verify: no base URL; pass http://127.0.0.1:<port> or export DSH_WEB_URL')
21
+ process.exit(1)
22
+ }
23
+ const base = resolved.baseUrl
24
+ const dashboard = dashboardBase()
25
+ // Hot reload keeps the canonical page route in older hands; the versioned
26
+ // alias always belongs to the generation that just started.
27
+ const boardPath = `${dashboard}/board/${version}`
28
+ const configPath = `${boardPath}/config`
29
+ const rows = []
30
+ const add = (id, ok, detail, options = {}) => rows.push({ id, ok, detail, manual: options.manual === true })
31
+
32
+ const get = (pathname, expectType) => httpGet(base, pathname, { expectType })
33
+
34
+ // 1. the generation alias and the JSON contract
35
+ let report = null
36
+ try {
37
+ const { body } = await get(`${dashboard}/data/${version}`, 'application/json')
38
+ report = JSON.parse(body)
39
+ const keys = ['scoring', 'skills', 'totals', 'turnLoads', 'edits', 'validity', 'mcpServers', 'runs']
40
+ const missing = keys.filter((key) => !(key in report))
41
+ add(
42
+ 'versioned-json',
43
+ missing.length === 0,
44
+ missing.length === 0 ? `data/${version} → JSON with ${keys.length} keys` : `missing ${missing.join(', ')}`,
45
+ )
46
+ } catch (error) {
47
+ add('versioned-json', false, error instanceof Error ? error.message : String(error))
48
+ }
49
+
50
+ // 2. settings API: fields + effective values
51
+ try {
52
+ const { body } = await get(`${dashboard}/settings/${version}`, 'application/json')
53
+ const payload = JSON.parse(body)
54
+ const fields = Array.isArray(payload.fields) ? payload.fields.length : 0
55
+ const hasWeights = payload.values && typeof payload.values['weights.static'] !== 'undefined'
56
+ add(
57
+ 'settings-api',
58
+ fields > 0 && Boolean(hasWeights),
59
+ `${fields} field(s), weights.static=${payload.values?.['weights.static']}`,
60
+ )
61
+ } catch (error) {
62
+ add('settings-api', false, error instanceof Error ? error.message : String(error))
63
+ }
64
+
65
+ // 3. the board HTML carries the reporting sections
66
+ try {
67
+ let body
68
+ try {
69
+ body = (await get(boardPath, 'text/html')).body
70
+ } catch {
71
+ body = (await get(dashboard, 'text/html')).body
72
+ }
73
+ const sections = ['Next steps', 'What the evidence supports', 'Screened edits', 'config'].filter((needle) =>
74
+ body.includes(needle),
75
+ )
76
+ add('board-sections', sections.length >= 3, `found ${sections.length}/4: ${sections.join(', ')}`)
77
+ } catch (error) {
78
+ add('board-sections', false, error instanceof Error ? error.message : String(error))
79
+ }
80
+
81
+ // 4. the settings page renders the fields
82
+ try {
83
+ let body
84
+ try {
85
+ body = (await get(configPath, 'text/html')).body
86
+ } catch {
87
+ body = (await get(`${dashboard}/config`, 'text/html')).body
88
+ }
89
+ const fields = ['weights.static', 'thresholds.excellent', 'tokenBudgetPerLoad', 'adviceNoiseFloor'].filter((key) =>
90
+ body.includes(`name="${key}"`),
91
+ )
92
+ add('config-page', fields.length === 4, `found ${fields.length}/4 field(s)`)
93
+ } catch (error) {
94
+ add('config-page', false, error instanceof Error ? error.message : String(error))
95
+ }
96
+
97
+ // 5. per-turn boards on disk (only meaningful when the generation is live)
98
+ try {
99
+ const dir = report?.dataDir
100
+ if (!dir) throw new Error('report has no dataDir (versioned route is not live yet)')
101
+ const reportsDir = path.join(dir, 'reports')
102
+ let turns = []
103
+ try {
104
+ turns = (await readdir(path.join(reportsDir, 'turns'))).filter((name) => name.endsWith('.json'))
105
+ } catch {
106
+ turns = []
107
+ }
108
+ add(
109
+ 'turn-boards',
110
+ turns.length > 0,
111
+ turns.length > 0
112
+ ? `${turns.length} board(s), newest ${turns.sort().at(-1)}`
113
+ : 'none yet — load a skill, finish the turn, then re-run',
114
+ )
115
+ } catch (error) {
116
+ add('turn-boards', false, error instanceof Error ? error.message : String(error))
117
+ }
118
+
119
+ // 6. the report contract itself: both aliases agree and the numbers are sane
120
+ try {
121
+ const canonical = JSON.parse((await get(`${dashboard}/data`, 'application/json')).body)
122
+ const versioned = JSON.parse((await get(`${dashboard}/data/${version}`, 'application/json')).body)
123
+ const sameLoads = canonical.totals?.invocations === versioned.totals?.invocations
124
+ const percent = Object.values(versioned.scoring?.weightsPercent ?? {})
125
+ const sum = percent.reduce((a, b) => a + b, 0)
126
+ const withCi = (versioned.skills ?? []).filter((skill) => skill.score?.ci).length
127
+ const scored = (versioned.skills ?? []).length
128
+ const ok = sameLoads && percent.length === 4 && sum === 100
129
+ add(
130
+ 'report-contract',
131
+ ok,
132
+ `canonical=versioned:${sameLoads}, weights sum ${sum}, ${withCi}/${scored} skill(s) with a 95% interval`,
133
+ )
134
+ } catch (error) {
135
+ add('report-contract', false, error instanceof Error ? error.message : String(error))
136
+ }
137
+
138
+ // 7. snapshots on disk for the data dir the host reports
139
+ try {
140
+ const snapshot = JSON.parse(
141
+ await readFile(path.join(report?.dataDir ?? '/nonexistent', 'reports', 'latest.json'), 'utf8'),
142
+ )
143
+ add(
144
+ 'latest-snapshot',
145
+ true,
146
+ `${report?.dataDir}/reports/latest.json: ${(snapshot.skills ?? []).length} skill(s), ${(snapshot.runs ?? []).length} run(s)`,
147
+ )
148
+ } catch (error) {
149
+ add('latest-snapshot', false, error instanceof Error ? error.message : String(error))
150
+ }
151
+
152
+ // manual checks the CLI cannot perform
153
+ add(
154
+ 'manual:native-seats',
155
+ true,
156
+ 'open the sidebar Skill eval panel icon (board) and Settings → Plugins → dsh-skill-eval (bundle config)',
157
+ { manual: true },
158
+ )
159
+ add('manual:turn-strip', true, 'load a skill in a turn; its result strip shows score, next fix and edit history', {
160
+ manual: true,
161
+ })
162
+
163
+ const failed = rows.filter((row) => !row.ok && !row.manual)
164
+ console.log(`dsh-skill-eval verify — ${base} (version ${version})`)
165
+ for (const row of rows) {
166
+ const state = row.manual ? 'INFO' : row.ok ? 'PASS' : 'FAIL'
167
+ console.log(` [${state}] ${row.id.padEnd(18)} ${row.detail}`)
168
+ }
169
+ console.log('')
170
+ if (failed.length > 0) {
171
+ console.log(`FAIL (${failed.length}) — restart DeepSeek Harness if versioned-json reports text/html.`)
172
+ process.exit(1)
173
+ }
174
+ const checks = rows.filter((row) => !row.manual).length
175
+ const manual = rows.filter((row) => row.manual).length
176
+ console.log(
177
+ `PASS — the running host serves this generation and the product surfaces are reachable (${checks} check(s), ${manual} manual).`,
178
+ )