dsh-skill-eval 0.0.0-stage → 1.20.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +718 -0
- package/LICENSE +21 -0
- package/README.md +207 -2
- package/README.zh.md +164 -0
- package/client.js +2791 -0
- package/cordis.patch.yml +9 -0
- package/docs/ACCEPTANCE.md +71 -0
- package/docs/AUDIT.md +82 -0
- package/docs/CODE-REVIEW.md +49 -0
- package/docs/COMPARISON.md +33 -0
- package/docs/DETAIL.md +468 -0
- package/docs/DEV-RELOAD.md +114 -0
- package/docs/ENTRYPOINTS.md +65 -0
- package/docs/RELEASE.md +44 -0
- package/docs/RRSI-NOTES.md +129 -0
- package/docs/SCENARIOS.md +91 -0
- package/docs/UI-REVIEW.md +115 -0
- package/docs/WORKFLOW.md +108 -0
- package/icon.svg +9 -0
- package/index.js +1173 -0
- package/lib/accept.js +350 -0
- package/lib/advice.js +557 -0
- package/lib/critic.js +101 -0
- package/lib/dev.js +148 -0
- package/lib/history.js +141 -0
- package/lib/html.js +17 -0
- package/lib/inspect.js +142 -0
- package/lib/integrity.js +46 -0
- package/lib/loop.js +195 -0
- package/lib/random.js +29 -0
- package/lib/recorder.js +644 -0
- package/lib/render.js +580 -0
- package/lib/report.js +689 -0
- package/lib/score.js +565 -0
- package/lib/settings.js +415 -0
- package/lib/snapshot.js +250 -0
- package/lib/steps.js +141 -0
- package/lib/store.js +240 -0
- package/lib/validity.js +125 -0
- package/locale/en.json +6 -0
- package/locale/zh.json +6 -0
- package/package.json +96 -4
- package/scripts/accept.mjs +227 -0
- package/scripts/build-client.mjs +59 -0
- package/scripts/build.mjs +81 -0
- package/scripts/doctor.mjs +234 -0
- package/scripts/lib.mjs +87 -0
- package/scripts/verify-live.mjs +178 -0
package/lib/accept.js
ADDED
|
@@ -0,0 +1,350 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Post-restart acceptance checks.
|
|
3
|
+
*
|
|
4
|
+
* A profile plugin only loads at host boot, so after installing or upgrading
|
|
5
|
+
* `dsh-skill-eval` the running page can keep serving a previous generation.
|
|
6
|
+
* This module answers "did the current generation actually load?" from outside
|
|
7
|
+
* the process: the JSON route, the snapshot files and the recorded telemetry.
|
|
8
|
+
*/
|
|
9
|
+
|
|
10
|
+
export const REQUIRED_REPORT_KEYS = ['scoring', 'mcpServers', 'runs', 'skill', 'skills', 'totals'].filter(
|
|
11
|
+
(key) => key !== 'skill',
|
|
12
|
+
)
|
|
13
|
+
export const REQUIRED_SCORING_KEYS = ['weights', 'weightsPercent', 'thresholds', 'minSamples']
|
|
14
|
+
export const STALE_HINT =
|
|
15
|
+
'the running host still serves an older plugin generation: quit and reopen DeepSeek Harness once'
|
|
16
|
+
export const UNREACHABLE_HINT =
|
|
17
|
+
'cannot reach the harness host: pass the current base URL as an argument or export DSH_WEB_URL (the GUI port changes on every launch)'
|
|
18
|
+
|
|
19
|
+
function result(id, ok, detail) {
|
|
20
|
+
return { id, ok, detail }
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
/** Pure shape check over a `/skill-eval/data` payload. */
|
|
24
|
+
export function checkReportPayload(payload) {
|
|
25
|
+
const results = []
|
|
26
|
+
if (!payload || typeof payload !== 'object') {
|
|
27
|
+
return [result('report-shape', false, 'response is not a JSON object')]
|
|
28
|
+
}
|
|
29
|
+
const missing = REQUIRED_REPORT_KEYS.filter((key) => !(key in payload))
|
|
30
|
+
results.push(
|
|
31
|
+
result(
|
|
32
|
+
'report-shape',
|
|
33
|
+
missing.length === 0,
|
|
34
|
+
missing.length === 0 ? 'has scoring, mcpServers, runs, totals' : `missing ${missing.join(', ')} — ${STALE_HINT}`,
|
|
35
|
+
),
|
|
36
|
+
)
|
|
37
|
+
const scoring = payload.scoring && typeof payload.scoring === 'object' ? payload.scoring : null
|
|
38
|
+
const scoringMissing = scoring ? REQUIRED_SCORING_KEYS.filter((key) => !(key in scoring)) : REQUIRED_SCORING_KEYS
|
|
39
|
+
results.push(
|
|
40
|
+
result(
|
|
41
|
+
'scoring-block',
|
|
42
|
+
scoringMissing.length === 0,
|
|
43
|
+
scoringMissing.length === 0
|
|
44
|
+
? `weights ${JSON.stringify(scoring.weightsPercent)} · A≥${scoring.thresholds.excellent}`
|
|
45
|
+
: `missing ${scoringMissing.join(', ')}`,
|
|
46
|
+
),
|
|
47
|
+
)
|
|
48
|
+
if (scoring && scoring.weightsPercent) {
|
|
49
|
+
const sum = Object.values(scoring.weightsPercent).reduce((a, b) => a + Number(b || 0), 0)
|
|
50
|
+
results.push(result('weights-normalised', Math.abs(sum - 100) <= 1, `weights sum to ${sum}`))
|
|
51
|
+
}
|
|
52
|
+
results.push(
|
|
53
|
+
result(
|
|
54
|
+
'mcp-board',
|
|
55
|
+
Array.isArray(payload.mcpServers),
|
|
56
|
+
`mcpServers is an array (${(payload.mcpServers || []).length} row(s))`,
|
|
57
|
+
),
|
|
58
|
+
)
|
|
59
|
+
results.push(
|
|
60
|
+
result('runs-board', Array.isArray(payload.runs), `runs is an array (${(payload.runs || []).length} row(s))`),
|
|
61
|
+
)
|
|
62
|
+
results.push(
|
|
63
|
+
result(
|
|
64
|
+
'ranking',
|
|
65
|
+
'ranking' in payload,
|
|
66
|
+
'ranking' in payload ? `significant=${payload.ranking?.significant}` : 'missing ranking block',
|
|
67
|
+
),
|
|
68
|
+
)
|
|
69
|
+
const scored = (payload.skills || []).filter((entry) => entry.score && typeof entry.score.total === 'number')
|
|
70
|
+
const withCi = scored.filter((entry) => entry.score.ci)
|
|
71
|
+
results.push(
|
|
72
|
+
result(
|
|
73
|
+
'score-ci',
|
|
74
|
+
scored.length === 0 || withCi.length > 0,
|
|
75
|
+
scored.length === 0
|
|
76
|
+
? 'no scored skills yet'
|
|
77
|
+
: `${withCi.length}/${scored.length} scored skill(s) carry a 95% interval`,
|
|
78
|
+
),
|
|
79
|
+
)
|
|
80
|
+
return results
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
/** File-side checks: snapshot and telemetry live under the profile data dir. */
|
|
84
|
+
export function checkFileState({
|
|
85
|
+
eventsPath,
|
|
86
|
+
snapshotMarkdownPath,
|
|
87
|
+
snapshotJsonPath,
|
|
88
|
+
events,
|
|
89
|
+
snapshotText,
|
|
90
|
+
snapshotJsonText,
|
|
91
|
+
}) {
|
|
92
|
+
const results = []
|
|
93
|
+
const lines = Array.isArray(events) ? events : []
|
|
94
|
+
const counts = {}
|
|
95
|
+
for (const record of lines) {
|
|
96
|
+
const type = record && record.t ? record.t : 'unknown'
|
|
97
|
+
counts[type] = (counts[type] || 0) + 1
|
|
98
|
+
}
|
|
99
|
+
results.push(
|
|
100
|
+
result(
|
|
101
|
+
'telemetry-file',
|
|
102
|
+
lines.length > 0,
|
|
103
|
+
lines.length > 0
|
|
104
|
+
? `${eventsPath}: ${lines.length} record(s) ${JSON.stringify(counts)}`
|
|
105
|
+
: `${eventsPath} is empty`,
|
|
106
|
+
),
|
|
107
|
+
)
|
|
108
|
+
results.push(
|
|
109
|
+
result(
|
|
110
|
+
'snapshot-markdown',
|
|
111
|
+
typeof snapshotText === 'string' && snapshotText.includes('Skill execution report'),
|
|
112
|
+
typeof snapshotText === 'string'
|
|
113
|
+
? `${snapshotMarkdownPath} (${snapshotText.length} bytes)`
|
|
114
|
+
: `${snapshotMarkdownPath} missing`,
|
|
115
|
+
),
|
|
116
|
+
)
|
|
117
|
+
let parsed = null
|
|
118
|
+
try {
|
|
119
|
+
parsed = typeof snapshotJsonText === 'string' ? JSON.parse(snapshotJsonText) : null
|
|
120
|
+
} catch {
|
|
121
|
+
parsed = null
|
|
122
|
+
}
|
|
123
|
+
results.push(
|
|
124
|
+
result(
|
|
125
|
+
'snapshot-json',
|
|
126
|
+
parsed !== null,
|
|
127
|
+
parsed === null
|
|
128
|
+
? `${snapshotJsonPath} missing or invalid`
|
|
129
|
+
: `${snapshotJsonPath} has ${(parsed.skills || []).length} skill(s)`,
|
|
130
|
+
),
|
|
131
|
+
)
|
|
132
|
+
if (parsed) {
|
|
133
|
+
const modern = ['scoring', 'ranking', 'runs', 'mcpServers'].filter((key) => key in parsed)
|
|
134
|
+
results.push(
|
|
135
|
+
result(
|
|
136
|
+
'snapshot-generation',
|
|
137
|
+
modern.length >= 3,
|
|
138
|
+
modern.length >= 3
|
|
139
|
+
? `snapshot carries ${modern.join(', ')}`
|
|
140
|
+
: `snapshot lacks ${['scoring', 'ranking', 'runs', 'mcpServers'].filter((key) => !(key in parsed)).join(', ')} — this generation predates the current boards`,
|
|
141
|
+
),
|
|
142
|
+
)
|
|
143
|
+
}
|
|
144
|
+
return results
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
/** Compose the whole acceptance run over injected IO so it stays testable. */
|
|
148
|
+
export async function runAcceptance({
|
|
149
|
+
baseUrl,
|
|
150
|
+
fetchReport,
|
|
151
|
+
fetchVersioned,
|
|
152
|
+
fetchConfigPage,
|
|
153
|
+
fetchSettingsApi,
|
|
154
|
+
readFiles,
|
|
155
|
+
}) {
|
|
156
|
+
const results = []
|
|
157
|
+
let canonical = null
|
|
158
|
+
if (typeof fetchReport === 'function') {
|
|
159
|
+
try {
|
|
160
|
+
canonical = await fetchReport()
|
|
161
|
+
results.push(...checkReportPayload(canonical))
|
|
162
|
+
} catch (error) {
|
|
163
|
+
const message = error instanceof Error ? error.message : String(error)
|
|
164
|
+
const entry = result('report-fetch', false, message)
|
|
165
|
+
if (error && (error.unreachable === true || /fetch failed|ECONNREFUSED|ENOTFOUND|EHOSTUNREACH/i.test(message))) {
|
|
166
|
+
entry.unreachable = true
|
|
167
|
+
}
|
|
168
|
+
results.push(entry)
|
|
169
|
+
}
|
|
170
|
+
}
|
|
171
|
+
if (typeof fetchVersioned === 'function') {
|
|
172
|
+
try {
|
|
173
|
+
const versioned = await fetchVersioned()
|
|
174
|
+
const same =
|
|
175
|
+
canonical && versioned && canonical.totals
|
|
176
|
+
? canonical.totals.invocations === versioned.totals?.invocations
|
|
177
|
+
: Boolean(versioned && 'scoring' in versioned)
|
|
178
|
+
results.push(
|
|
179
|
+
result(
|
|
180
|
+
'versioned-route',
|
|
181
|
+
same,
|
|
182
|
+
same
|
|
183
|
+
? 'the versioned data route answers with the same report'
|
|
184
|
+
: 'versioned data route answers with a different generation',
|
|
185
|
+
),
|
|
186
|
+
)
|
|
187
|
+
results.push(...checkVersionedPayload(versioned, canonical))
|
|
188
|
+
if (typeof fetchSettingsApi === 'function') {
|
|
189
|
+
try {
|
|
190
|
+
results.push(checkSettingsApi(await fetchSettingsApi()))
|
|
191
|
+
} catch (error) {
|
|
192
|
+
const message = error instanceof Error ? error.message : String(error)
|
|
193
|
+
const entry = result('settings-api', false, message)
|
|
194
|
+
if (/fetch failed|ECONNREFUSED|ENOTFOUND/i.test(message)) entry.unreachable = true
|
|
195
|
+
results.push(entry)
|
|
196
|
+
}
|
|
197
|
+
}
|
|
198
|
+
if (typeof fetchConfigPage === 'function') {
|
|
199
|
+
try {
|
|
200
|
+
results.push(checkConfigPage(await fetchConfigPage()))
|
|
201
|
+
} catch (error) {
|
|
202
|
+
const message = error instanceof Error ? error.message : String(error)
|
|
203
|
+
const entry = result('config-page', false, message)
|
|
204
|
+
if (/fetch failed|ECONNREFUSED|ENOTFOUND/i.test(message)) entry.unreachable = true
|
|
205
|
+
results.push(entry)
|
|
206
|
+
}
|
|
207
|
+
}
|
|
208
|
+
} catch (error) {
|
|
209
|
+
const message = error instanceof Error ? error.message : String(error)
|
|
210
|
+
const entry = result('versioned-route', false, `${message} — the running host predates the versioned alias`)
|
|
211
|
+
if (error && (error.unreachable === true || /fetch failed|ECONNREFUSED|ENOTFOUND|EHOSTUNREACH/i.test(message))) {
|
|
212
|
+
entry.unreachable = true
|
|
213
|
+
}
|
|
214
|
+
results.push(entry)
|
|
215
|
+
}
|
|
216
|
+
}
|
|
217
|
+
if (typeof readFiles === 'function') {
|
|
218
|
+
try {
|
|
219
|
+
const files = await readFiles()
|
|
220
|
+
results.push(...checkFileState(files))
|
|
221
|
+
} catch (error) {
|
|
222
|
+
results.push(result('files', false, error instanceof Error ? error.message : String(error)))
|
|
223
|
+
}
|
|
224
|
+
}
|
|
225
|
+
results.push(
|
|
226
|
+
result(
|
|
227
|
+
'client-surface',
|
|
228
|
+
true,
|
|
229
|
+
'manual: load a skill in a turn — a result strip should appear under that completed turn (scores, calls/errors, expandable fixes). The popup is opt-in via window.__SEV_ENABLE_POPUP__.',
|
|
230
|
+
),
|
|
231
|
+
)
|
|
232
|
+
const failed = results.filter((item) => !item.ok)
|
|
233
|
+
const unreachable = results.filter((item) => item.unreachable)
|
|
234
|
+
const onlyUnreachable = failed.length > 0 && failed.every((item) => item.unreachable)
|
|
235
|
+
return {
|
|
236
|
+
baseUrl,
|
|
237
|
+
ok: failed.length === 0,
|
|
238
|
+
results,
|
|
239
|
+
summary:
|
|
240
|
+
failed.length === 0
|
|
241
|
+
? 'PASS'
|
|
242
|
+
: onlyUnreachable || unreachable.length === failed.length
|
|
243
|
+
? `UNREACHABLE (${failed.length}) — ${UNREACHABLE_HINT}`
|
|
244
|
+
: `FAIL (${failed.length}) — ${STALE_HINT}`,
|
|
245
|
+
}
|
|
246
|
+
}
|
|
247
|
+
|
|
248
|
+
/**
|
|
249
|
+
* In-process acceptance: runs from inside the host (a slash command), so it
|
|
250
|
+
* needs no HTTP hop. It reuses the report shape checks and adds the pieces only
|
|
251
|
+
* the host can read directly: plugin version, run intervals, file generation.
|
|
252
|
+
*/
|
|
253
|
+
export function checkInProcessReport(report, options = {}) {
|
|
254
|
+
const results = []
|
|
255
|
+
results.push(result('plugin-version', true, String(options.version ?? 'unknown')))
|
|
256
|
+
results.push(...checkReportPayload(report))
|
|
257
|
+
const runs = Array.isArray(report?.runs) ? report.runs : []
|
|
258
|
+
const runsWithCi = runs.filter((run) => run && run.score && run.score.ci)
|
|
259
|
+
results.push(
|
|
260
|
+
result(
|
|
261
|
+
'run-ci',
|
|
262
|
+
runs.length === 0 || runsWithCi.length > 0,
|
|
263
|
+
runs.length === 0 ? 'no runs yet' : `${runsWithCi.length}/${runs.length} run(s) carry an interval`,
|
|
264
|
+
),
|
|
265
|
+
)
|
|
266
|
+
const files = options.files
|
|
267
|
+
if (files) results.push(...checkFileState(files))
|
|
268
|
+
results.push(
|
|
269
|
+
result(
|
|
270
|
+
'client-surface',
|
|
271
|
+
true,
|
|
272
|
+
'manual: load a skill in a turn — the result strip renders under that completed turn',
|
|
273
|
+
),
|
|
274
|
+
)
|
|
275
|
+
const failed = results.filter((item) => !item.ok)
|
|
276
|
+
return { ok: failed.length === 0, results, summary: failed.length === 0 ? 'PASS' : `FAIL (${failed.length})` }
|
|
277
|
+
}
|
|
278
|
+
|
|
279
|
+
/** The settings API backs the native plugin-config page. */
|
|
280
|
+
export function checkSettingsApi(payload) {
|
|
281
|
+
const fields = Array.isArray(payload?.fields) ? payload.fields : []
|
|
282
|
+
const values = payload && typeof payload.values === 'object' ? payload.values : null
|
|
283
|
+
const ok = fields.length > 0 && values !== null && fields.some((field) => field.key === 'weights.static')
|
|
284
|
+
return result(
|
|
285
|
+
'settings-api',
|
|
286
|
+
ok,
|
|
287
|
+
ok
|
|
288
|
+
? `${fields.length} field(s) served, weights.static=${values['weights.static']}`
|
|
289
|
+
: 'settings API did not return fields/values',
|
|
290
|
+
)
|
|
291
|
+
}
|
|
292
|
+
|
|
293
|
+
/** The settings page must exist, or the board cannot be configured from the browser. */
|
|
294
|
+
export function checkConfigPage(html) {
|
|
295
|
+
const required = ['name="weights.static"', 'name="minSamples"', 'name="adviceNoiseFloor"']
|
|
296
|
+
const missing = required.filter((needle) => !String(html ?? '').includes(needle))
|
|
297
|
+
return result(
|
|
298
|
+
'config-page',
|
|
299
|
+
missing.length === 0,
|
|
300
|
+
missing.length === 0 ? 'settings page serves every core field' : `settings page is missing: ${missing.join(', ')}`,
|
|
301
|
+
)
|
|
302
|
+
}
|
|
303
|
+
|
|
304
|
+
/**
|
|
305
|
+
* Checks that can only run against the versioned route: it is the one a hot
|
|
306
|
+
* reload can always reclaim, so it is the honest signal of "current generation".
|
|
307
|
+
*/
|
|
308
|
+
export function checkVersionedPayload(versioned, canonical) {
|
|
309
|
+
const results = []
|
|
310
|
+
const skills = Array.isArray(versioned?.skills) ? versioned.skills : []
|
|
311
|
+
const tokenAware = skills.some((entry) => entry && entry.stats && 'tokensPerLoad' in entry.stats)
|
|
312
|
+
results.push(
|
|
313
|
+
result(
|
|
314
|
+
'token-accounting',
|
|
315
|
+
skills.length === 0 || tokenAware,
|
|
316
|
+
skills.length === 0
|
|
317
|
+
? 'no scored skills to inspect'
|
|
318
|
+
: tokenAware
|
|
319
|
+
? 'skill stats carry tokensPerLoad (1.2.0+)'
|
|
320
|
+
: 'skill stats have no token fields — older generation',
|
|
321
|
+
),
|
|
322
|
+
)
|
|
323
|
+
const canonicalKeys = canonical && typeof canonical === 'object' ? Object.keys(canonical).sort().join(',') : ''
|
|
324
|
+
const versionedKeys = versioned && typeof versioned === 'object' ? Object.keys(versioned).sort().join(',') : ''
|
|
325
|
+
results.push(
|
|
326
|
+
result(
|
|
327
|
+
'canonical-route',
|
|
328
|
+
canonicalKeys === versionedKeys,
|
|
329
|
+
canonicalKeys === versionedKeys
|
|
330
|
+
? 'canonical and versioned routes agree'
|
|
331
|
+
: 'canonical route still answers from an older handler (the versioned route is current; a restart fixes the canonical one)',
|
|
332
|
+
),
|
|
333
|
+
)
|
|
334
|
+
return results
|
|
335
|
+
}
|
|
336
|
+
|
|
337
|
+
/** Pick the base URL: explicit argument, environment, then the relay bridge record. */
|
|
338
|
+
export function pickBaseUrl(explicit, envUrl, bridge) {
|
|
339
|
+
const clean = (value) =>
|
|
340
|
+
typeof value === 'string' && value.trim().length > 0 ? value.trim().replace(/\/$/, '') : null
|
|
341
|
+
const fromArg = clean(explicit)
|
|
342
|
+
if (fromArg) return { baseUrl: fromArg, source: 'argument' }
|
|
343
|
+
const fromEnv = clean(envUrl)
|
|
344
|
+
if (fromEnv) return { baseUrl: fromEnv, source: 'DSH_WEB_URL' }
|
|
345
|
+
if (bridge && typeof bridge === 'object') {
|
|
346
|
+
const upstream = clean(bridge.upstream)
|
|
347
|
+
if (upstream) return { baseUrl: upstream, source: 'relay-bridge' }
|
|
348
|
+
}
|
|
349
|
+
return { baseUrl: null, source: 'none' }
|
|
350
|
+
}
|