dsh-skill-eval 0.0.0-stage → 1.20.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (48) hide show
  1. package/CHANGELOG.md +718 -0
  2. package/LICENSE +21 -0
  3. package/README.md +207 -2
  4. package/README.zh.md +164 -0
  5. package/client.js +2791 -0
  6. package/cordis.patch.yml +9 -0
  7. package/docs/ACCEPTANCE.md +71 -0
  8. package/docs/AUDIT.md +82 -0
  9. package/docs/CODE-REVIEW.md +49 -0
  10. package/docs/COMPARISON.md +33 -0
  11. package/docs/DETAIL.md +468 -0
  12. package/docs/DEV-RELOAD.md +114 -0
  13. package/docs/ENTRYPOINTS.md +65 -0
  14. package/docs/RELEASE.md +44 -0
  15. package/docs/RRSI-NOTES.md +129 -0
  16. package/docs/SCENARIOS.md +91 -0
  17. package/docs/UI-REVIEW.md +115 -0
  18. package/docs/WORKFLOW.md +108 -0
  19. package/icon.svg +9 -0
  20. package/index.js +1173 -0
  21. package/lib/accept.js +350 -0
  22. package/lib/advice.js +557 -0
  23. package/lib/critic.js +101 -0
  24. package/lib/dev.js +148 -0
  25. package/lib/history.js +141 -0
  26. package/lib/html.js +17 -0
  27. package/lib/inspect.js +142 -0
  28. package/lib/integrity.js +46 -0
  29. package/lib/loop.js +195 -0
  30. package/lib/random.js +29 -0
  31. package/lib/recorder.js +644 -0
  32. package/lib/render.js +580 -0
  33. package/lib/report.js +689 -0
  34. package/lib/score.js +565 -0
  35. package/lib/settings.js +415 -0
  36. package/lib/snapshot.js +250 -0
  37. package/lib/steps.js +141 -0
  38. package/lib/store.js +240 -0
  39. package/lib/validity.js +125 -0
  40. package/locale/en.json +6 -0
  41. package/locale/zh.json +6 -0
  42. package/package.json +96 -4
  43. package/scripts/accept.mjs +227 -0
  44. package/scripts/build-client.mjs +59 -0
  45. package/scripts/build.mjs +81 -0
  46. package/scripts/doctor.mjs +234 -0
  47. package/scripts/lib.mjs +87 -0
  48. package/scripts/verify-live.mjs +178 -0
package/lib/accept.js ADDED
@@ -0,0 +1,350 @@
1
+ /**
2
+ * Post-restart acceptance checks.
3
+ *
4
+ * A profile plugin only loads at host boot, so after installing or upgrading
5
+ * `dsh-skill-eval` the running page can keep serving a previous generation.
6
+ * This module answers "did the current generation actually load?" from outside
7
+ * the process: the JSON route, the snapshot files and the recorded telemetry.
8
+ */
9
+
10
+ export const REQUIRED_REPORT_KEYS = ['scoring', 'mcpServers', 'runs', 'skill', 'skills', 'totals'].filter(
11
+ (key) => key !== 'skill',
12
+ )
13
+ export const REQUIRED_SCORING_KEYS = ['weights', 'weightsPercent', 'thresholds', 'minSamples']
14
+ export const STALE_HINT =
15
+ 'the running host still serves an older plugin generation: quit and reopen DeepSeek Harness once'
16
+ export const UNREACHABLE_HINT =
17
+ 'cannot reach the harness host: pass the current base URL as an argument or export DSH_WEB_URL (the GUI port changes on every launch)'
18
+
19
+ function result(id, ok, detail) {
20
+ return { id, ok, detail }
21
+ }
22
+
23
+ /** Pure shape check over a `/skill-eval/data` payload. */
24
+ export function checkReportPayload(payload) {
25
+ const results = []
26
+ if (!payload || typeof payload !== 'object') {
27
+ return [result('report-shape', false, 'response is not a JSON object')]
28
+ }
29
+ const missing = REQUIRED_REPORT_KEYS.filter((key) => !(key in payload))
30
+ results.push(
31
+ result(
32
+ 'report-shape',
33
+ missing.length === 0,
34
+ missing.length === 0 ? 'has scoring, mcpServers, runs, totals' : `missing ${missing.join(', ')} — ${STALE_HINT}`,
35
+ ),
36
+ )
37
+ const scoring = payload.scoring && typeof payload.scoring === 'object' ? payload.scoring : null
38
+ const scoringMissing = scoring ? REQUIRED_SCORING_KEYS.filter((key) => !(key in scoring)) : REQUIRED_SCORING_KEYS
39
+ results.push(
40
+ result(
41
+ 'scoring-block',
42
+ scoringMissing.length === 0,
43
+ scoringMissing.length === 0
44
+ ? `weights ${JSON.stringify(scoring.weightsPercent)} · A≥${scoring.thresholds.excellent}`
45
+ : `missing ${scoringMissing.join(', ')}`,
46
+ ),
47
+ )
48
+ if (scoring && scoring.weightsPercent) {
49
+ const sum = Object.values(scoring.weightsPercent).reduce((a, b) => a + Number(b || 0), 0)
50
+ results.push(result('weights-normalised', Math.abs(sum - 100) <= 1, `weights sum to ${sum}`))
51
+ }
52
+ results.push(
53
+ result(
54
+ 'mcp-board',
55
+ Array.isArray(payload.mcpServers),
56
+ `mcpServers is an array (${(payload.mcpServers || []).length} row(s))`,
57
+ ),
58
+ )
59
+ results.push(
60
+ result('runs-board', Array.isArray(payload.runs), `runs is an array (${(payload.runs || []).length} row(s))`),
61
+ )
62
+ results.push(
63
+ result(
64
+ 'ranking',
65
+ 'ranking' in payload,
66
+ 'ranking' in payload ? `significant=${payload.ranking?.significant}` : 'missing ranking block',
67
+ ),
68
+ )
69
+ const scored = (payload.skills || []).filter((entry) => entry.score && typeof entry.score.total === 'number')
70
+ const withCi = scored.filter((entry) => entry.score.ci)
71
+ results.push(
72
+ result(
73
+ 'score-ci',
74
+ scored.length === 0 || withCi.length > 0,
75
+ scored.length === 0
76
+ ? 'no scored skills yet'
77
+ : `${withCi.length}/${scored.length} scored skill(s) carry a 95% interval`,
78
+ ),
79
+ )
80
+ return results
81
+ }
82
+
83
+ /** File-side checks: snapshot and telemetry live under the profile data dir. */
84
+ export function checkFileState({
85
+ eventsPath,
86
+ snapshotMarkdownPath,
87
+ snapshotJsonPath,
88
+ events,
89
+ snapshotText,
90
+ snapshotJsonText,
91
+ }) {
92
+ const results = []
93
+ const lines = Array.isArray(events) ? events : []
94
+ const counts = {}
95
+ for (const record of lines) {
96
+ const type = record && record.t ? record.t : 'unknown'
97
+ counts[type] = (counts[type] || 0) + 1
98
+ }
99
+ results.push(
100
+ result(
101
+ 'telemetry-file',
102
+ lines.length > 0,
103
+ lines.length > 0
104
+ ? `${eventsPath}: ${lines.length} record(s) ${JSON.stringify(counts)}`
105
+ : `${eventsPath} is empty`,
106
+ ),
107
+ )
108
+ results.push(
109
+ result(
110
+ 'snapshot-markdown',
111
+ typeof snapshotText === 'string' && snapshotText.includes('Skill execution report'),
112
+ typeof snapshotText === 'string'
113
+ ? `${snapshotMarkdownPath} (${snapshotText.length} bytes)`
114
+ : `${snapshotMarkdownPath} missing`,
115
+ ),
116
+ )
117
+ let parsed = null
118
+ try {
119
+ parsed = typeof snapshotJsonText === 'string' ? JSON.parse(snapshotJsonText) : null
120
+ } catch {
121
+ parsed = null
122
+ }
123
+ results.push(
124
+ result(
125
+ 'snapshot-json',
126
+ parsed !== null,
127
+ parsed === null
128
+ ? `${snapshotJsonPath} missing or invalid`
129
+ : `${snapshotJsonPath} has ${(parsed.skills || []).length} skill(s)`,
130
+ ),
131
+ )
132
+ if (parsed) {
133
+ const modern = ['scoring', 'ranking', 'runs', 'mcpServers'].filter((key) => key in parsed)
134
+ results.push(
135
+ result(
136
+ 'snapshot-generation',
137
+ modern.length >= 3,
138
+ modern.length >= 3
139
+ ? `snapshot carries ${modern.join(', ')}`
140
+ : `snapshot lacks ${['scoring', 'ranking', 'runs', 'mcpServers'].filter((key) => !(key in parsed)).join(', ')} — this generation predates the current boards`,
141
+ ),
142
+ )
143
+ }
144
+ return results
145
+ }
146
+
147
+ /** Compose the whole acceptance run over injected IO so it stays testable. */
148
+ export async function runAcceptance({
149
+ baseUrl,
150
+ fetchReport,
151
+ fetchVersioned,
152
+ fetchConfigPage,
153
+ fetchSettingsApi,
154
+ readFiles,
155
+ }) {
156
+ const results = []
157
+ let canonical = null
158
+ if (typeof fetchReport === 'function') {
159
+ try {
160
+ canonical = await fetchReport()
161
+ results.push(...checkReportPayload(canonical))
162
+ } catch (error) {
163
+ const message = error instanceof Error ? error.message : String(error)
164
+ const entry = result('report-fetch', false, message)
165
+ if (error && (error.unreachable === true || /fetch failed|ECONNREFUSED|ENOTFOUND|EHOSTUNREACH/i.test(message))) {
166
+ entry.unreachable = true
167
+ }
168
+ results.push(entry)
169
+ }
170
+ }
171
+ if (typeof fetchVersioned === 'function') {
172
+ try {
173
+ const versioned = await fetchVersioned()
174
+ const same =
175
+ canonical && versioned && canonical.totals
176
+ ? canonical.totals.invocations === versioned.totals?.invocations
177
+ : Boolean(versioned && 'scoring' in versioned)
178
+ results.push(
179
+ result(
180
+ 'versioned-route',
181
+ same,
182
+ same
183
+ ? 'the versioned data route answers with the same report'
184
+ : 'versioned data route answers with a different generation',
185
+ ),
186
+ )
187
+ results.push(...checkVersionedPayload(versioned, canonical))
188
+ if (typeof fetchSettingsApi === 'function') {
189
+ try {
190
+ results.push(checkSettingsApi(await fetchSettingsApi()))
191
+ } catch (error) {
192
+ const message = error instanceof Error ? error.message : String(error)
193
+ const entry = result('settings-api', false, message)
194
+ if (/fetch failed|ECONNREFUSED|ENOTFOUND/i.test(message)) entry.unreachable = true
195
+ results.push(entry)
196
+ }
197
+ }
198
+ if (typeof fetchConfigPage === 'function') {
199
+ try {
200
+ results.push(checkConfigPage(await fetchConfigPage()))
201
+ } catch (error) {
202
+ const message = error instanceof Error ? error.message : String(error)
203
+ const entry = result('config-page', false, message)
204
+ if (/fetch failed|ECONNREFUSED|ENOTFOUND/i.test(message)) entry.unreachable = true
205
+ results.push(entry)
206
+ }
207
+ }
208
+ } catch (error) {
209
+ const message = error instanceof Error ? error.message : String(error)
210
+ const entry = result('versioned-route', false, `${message} — the running host predates the versioned alias`)
211
+ if (error && (error.unreachable === true || /fetch failed|ECONNREFUSED|ENOTFOUND|EHOSTUNREACH/i.test(message))) {
212
+ entry.unreachable = true
213
+ }
214
+ results.push(entry)
215
+ }
216
+ }
217
+ if (typeof readFiles === 'function') {
218
+ try {
219
+ const files = await readFiles()
220
+ results.push(...checkFileState(files))
221
+ } catch (error) {
222
+ results.push(result('files', false, error instanceof Error ? error.message : String(error)))
223
+ }
224
+ }
225
+ results.push(
226
+ result(
227
+ 'client-surface',
228
+ true,
229
+ 'manual: load a skill in a turn — a result strip should appear under that completed turn (scores, calls/errors, expandable fixes). The popup is opt-in via window.__SEV_ENABLE_POPUP__.',
230
+ ),
231
+ )
232
+ const failed = results.filter((item) => !item.ok)
233
+ const unreachable = results.filter((item) => item.unreachable)
234
+ const onlyUnreachable = failed.length > 0 && failed.every((item) => item.unreachable)
235
+ return {
236
+ baseUrl,
237
+ ok: failed.length === 0,
238
+ results,
239
+ summary:
240
+ failed.length === 0
241
+ ? 'PASS'
242
+ : onlyUnreachable || unreachable.length === failed.length
243
+ ? `UNREACHABLE (${failed.length}) — ${UNREACHABLE_HINT}`
244
+ : `FAIL (${failed.length}) — ${STALE_HINT}`,
245
+ }
246
+ }
247
+
248
+ /**
249
+ * In-process acceptance: runs from inside the host (a slash command), so it
250
+ * needs no HTTP hop. It reuses the report shape checks and adds the pieces only
251
+ * the host can read directly: plugin version, run intervals, file generation.
252
+ */
253
+ export function checkInProcessReport(report, options = {}) {
254
+ const results = []
255
+ results.push(result('plugin-version', true, String(options.version ?? 'unknown')))
256
+ results.push(...checkReportPayload(report))
257
+ const runs = Array.isArray(report?.runs) ? report.runs : []
258
+ const runsWithCi = runs.filter((run) => run && run.score && run.score.ci)
259
+ results.push(
260
+ result(
261
+ 'run-ci',
262
+ runs.length === 0 || runsWithCi.length > 0,
263
+ runs.length === 0 ? 'no runs yet' : `${runsWithCi.length}/${runs.length} run(s) carry an interval`,
264
+ ),
265
+ )
266
+ const files = options.files
267
+ if (files) results.push(...checkFileState(files))
268
+ results.push(
269
+ result(
270
+ 'client-surface',
271
+ true,
272
+ 'manual: load a skill in a turn — the result strip renders under that completed turn',
273
+ ),
274
+ )
275
+ const failed = results.filter((item) => !item.ok)
276
+ return { ok: failed.length === 0, results, summary: failed.length === 0 ? 'PASS' : `FAIL (${failed.length})` }
277
+ }
278
+
279
+ /** The settings API backs the native plugin-config page. */
280
+ export function checkSettingsApi(payload) {
281
+ const fields = Array.isArray(payload?.fields) ? payload.fields : []
282
+ const values = payload && typeof payload.values === 'object' ? payload.values : null
283
+ const ok = fields.length > 0 && values !== null && fields.some((field) => field.key === 'weights.static')
284
+ return result(
285
+ 'settings-api',
286
+ ok,
287
+ ok
288
+ ? `${fields.length} field(s) served, weights.static=${values['weights.static']}`
289
+ : 'settings API did not return fields/values',
290
+ )
291
+ }
292
+
293
+ /** The settings page must exist, or the board cannot be configured from the browser. */
294
+ export function checkConfigPage(html) {
295
+ const required = ['name="weights.static"', 'name="minSamples"', 'name="adviceNoiseFloor"']
296
+ const missing = required.filter((needle) => !String(html ?? '').includes(needle))
297
+ return result(
298
+ 'config-page',
299
+ missing.length === 0,
300
+ missing.length === 0 ? 'settings page serves every core field' : `settings page is missing: ${missing.join(', ')}`,
301
+ )
302
+ }
303
+
304
+ /**
305
+ * Checks that can only run against the versioned route: it is the one a hot
306
+ * reload can always reclaim, so it is the honest signal of "current generation".
307
+ */
308
+ export function checkVersionedPayload(versioned, canonical) {
309
+ const results = []
310
+ const skills = Array.isArray(versioned?.skills) ? versioned.skills : []
311
+ const tokenAware = skills.some((entry) => entry && entry.stats && 'tokensPerLoad' in entry.stats)
312
+ results.push(
313
+ result(
314
+ 'token-accounting',
315
+ skills.length === 0 || tokenAware,
316
+ skills.length === 0
317
+ ? 'no scored skills to inspect'
318
+ : tokenAware
319
+ ? 'skill stats carry tokensPerLoad (1.2.0+)'
320
+ : 'skill stats have no token fields — older generation',
321
+ ),
322
+ )
323
+ const canonicalKeys = canonical && typeof canonical === 'object' ? Object.keys(canonical).sort().join(',') : ''
324
+ const versionedKeys = versioned && typeof versioned === 'object' ? Object.keys(versioned).sort().join(',') : ''
325
+ results.push(
326
+ result(
327
+ 'canonical-route',
328
+ canonicalKeys === versionedKeys,
329
+ canonicalKeys === versionedKeys
330
+ ? 'canonical and versioned routes agree'
331
+ : 'canonical route still answers from an older handler (the versioned route is current; a restart fixes the canonical one)',
332
+ ),
333
+ )
334
+ return results
335
+ }
336
+
337
+ /** Pick the base URL: explicit argument, environment, then the relay bridge record. */
338
+ export function pickBaseUrl(explicit, envUrl, bridge) {
339
+ const clean = (value) =>
340
+ typeof value === 'string' && value.trim().length > 0 ? value.trim().replace(/\/$/, '') : null
341
+ const fromArg = clean(explicit)
342
+ if (fromArg) return { baseUrl: fromArg, source: 'argument' }
343
+ const fromEnv = clean(envUrl)
344
+ if (fromEnv) return { baseUrl: fromEnv, source: 'DSH_WEB_URL' }
345
+ if (bridge && typeof bridge === 'object') {
346
+ const upstream = clean(bridge.upstream)
347
+ if (upstream) return { baseUrl: upstream, source: 'relay-bridge' }
348
+ }
349
+ return { baseUrl: null, source: 'none' }
350
+ }