dsh-skill-eval 0.0.0-stage → 1.20.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +675 -0
- package/LICENSE +21 -0
- package/README.md +200 -2
- package/README.zh.md +157 -0
- package/client.js +2637 -0
- package/cordis.patch.yml +9 -0
- package/docs/ACCEPTANCE.md +71 -0
- package/docs/AUDIT.md +64 -0
- package/docs/CODE-REVIEW.md +49 -0
- package/docs/COMPARISON.md +33 -0
- package/docs/DETAIL.md +463 -0
- package/docs/DEV-RELOAD.md +114 -0
- package/docs/ENTRYPOINTS.md +65 -0
- package/docs/RELEASE.md +44 -0
- package/docs/RRSI-NOTES.md +129 -0
- package/docs/SCENARIOS.md +91 -0
- package/docs/UI-REVIEW.md +115 -0
- package/docs/WORKFLOW.md +108 -0
- package/icon.svg +9 -0
- package/index.js +1111 -0
- package/lib/accept.js +350 -0
- package/lib/advice.js +547 -0
- package/lib/critic.js +101 -0
- package/lib/dev.js +148 -0
- package/lib/history.js +140 -0
- package/lib/html.js +8 -0
- package/lib/inspect.js +142 -0
- package/lib/integrity.js +46 -0
- package/lib/loop.js +195 -0
- package/lib/random.js +29 -0
- package/lib/recorder.js +634 -0
- package/lib/render.js +582 -0
- package/lib/report.js +646 -0
- package/lib/score.js +525 -0
- package/lib/settings.js +396 -0
- package/lib/snapshot.js +166 -0
- package/lib/steps.js +141 -0
- package/lib/store.js +186 -0
- package/lib/validity.js +115 -0
- package/locale/en.json +6 -0
- package/locale/zh.json +6 -0
- package/package.json +96 -4
- package/scripts/accept.mjs +226 -0
- package/scripts/build-client.mjs +59 -0
- package/scripts/build.mjs +81 -0
- package/scripts/doctor.mjs +199 -0
- package/scripts/lib.mjs +52 -0
- package/scripts/verify-live.mjs +168 -0
package/lib/steps.js
ADDED
|
@@ -0,0 +1,141 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Per-step loop telemetry: one record per agent loop step, scored so the panel
|
|
3
|
+
* can show how a turn actually went instead of only its final aggregate.
|
|
4
|
+
*
|
|
5
|
+
* Records are written by the recorder as `{t:'step', sessionId, turn, step,
|
|
6
|
+
* toolCalls, errors, loopErrors, tokens, skills, ...}`. This module sanitises
|
|
7
|
+
* them, derives a 0-100 step score and groups them by turn for display.
|
|
8
|
+
*/
|
|
9
|
+
import { DEFAULT_THRESHOLDS, gradeOf } from './score.js'
|
|
10
|
+
|
|
11
|
+
const clamp = (value, low, high) => Math.min(high, Math.max(low, value))
|
|
12
|
+
const round = (value) => Math.round(value * 10) / 10
|
|
13
|
+
const count = (value) => {
|
|
14
|
+
const n = Number(value)
|
|
15
|
+
return Number.isFinite(n) && n > 0 ? n : 0
|
|
16
|
+
}
|
|
17
|
+
const text = (value) => (typeof value === 'string' ? value : '')
|
|
18
|
+
|
|
19
|
+
/** Score one loop step: errors and loop failures dominate, call volume is a hint. */
|
|
20
|
+
export function scoreStep(step, thresholds = DEFAULT_THRESHOLDS) {
|
|
21
|
+
if (!step || typeof step !== 'object') return null
|
|
22
|
+
const toolCalls = count(step.toolCalls)
|
|
23
|
+
const errors = count(step.errors)
|
|
24
|
+
const loopErrors = count(step.loopErrors)
|
|
25
|
+
const penalties = []
|
|
26
|
+
let score = 100
|
|
27
|
+
if (errors > 0) {
|
|
28
|
+
const penalty = Math.min(50, errors * 20)
|
|
29
|
+
score -= penalty
|
|
30
|
+
penalties.push({ id: 'tool-errors', penalty })
|
|
31
|
+
}
|
|
32
|
+
if (loopErrors > 0) {
|
|
33
|
+
const penalty = Math.min(60, loopErrors * 35)
|
|
34
|
+
score -= penalty
|
|
35
|
+
penalties.push({ id: 'loop-errors', penalty })
|
|
36
|
+
}
|
|
37
|
+
if (toolCalls > 6) {
|
|
38
|
+
const penalty = Math.min(18, Math.round((toolCalls - 6) * 3))
|
|
39
|
+
score -= penalty
|
|
40
|
+
penalties.push({ id: 'call-volume', penalty })
|
|
41
|
+
}
|
|
42
|
+
const total = round(clamp(score, 0, 100))
|
|
43
|
+
return { total, grade: gradeOf(total, thresholds), penalties }
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
/** Sanitise + score the raw `step` records, oldest first. */
|
|
47
|
+
export function buildSteps(records, { thresholds = DEFAULT_THRESHOLDS, limit = 400 } = {}) {
|
|
48
|
+
const steps = []
|
|
49
|
+
for (const record of Array.isArray(records) ? records : []) {
|
|
50
|
+
if (!record || record.t !== 'step' || typeof record.sessionId !== 'string') continue
|
|
51
|
+
const scored = scoreStep(record, thresholds)
|
|
52
|
+
if (!scored) continue
|
|
53
|
+
steps.push({
|
|
54
|
+
at: Number.isFinite(Number(record.at)) ? Number(record.at) : null,
|
|
55
|
+
sessionId: record.sessionId,
|
|
56
|
+
turn: Number.isFinite(Number(record.turn)) ? Number(record.turn) : null,
|
|
57
|
+
step: Number.isFinite(Number(record.step)) ? Number(record.step) : null,
|
|
58
|
+
durationMs: count(record.durationMs),
|
|
59
|
+
toolCalls: count(record.toolCalls),
|
|
60
|
+
errors: count(record.errors),
|
|
61
|
+
mcpCalls: count(record.mcpCalls),
|
|
62
|
+
mcpErrors: count(record.mcpErrors),
|
|
63
|
+
loopErrors: count(record.loopErrors),
|
|
64
|
+
skills: Array.isArray(record.skills) ? record.skills.filter((name) => typeof name === 'string') : [],
|
|
65
|
+
tokens: {
|
|
66
|
+
input: count(record.tokens?.input),
|
|
67
|
+
output: count(record.tokens?.output),
|
|
68
|
+
cacheRead: count(record.tokens?.cacheRead),
|
|
69
|
+
calls: count(record.tokens?.calls),
|
|
70
|
+
},
|
|
71
|
+
errorNames: Array.isArray(record.errorNames) ? record.errorNames.filter((name) => typeof name === 'string') : [],
|
|
72
|
+
closeReason: text(record.closeReason) || null,
|
|
73
|
+
score: scored,
|
|
74
|
+
})
|
|
75
|
+
}
|
|
76
|
+
steps.sort((a, b) => (a.at ?? 0) - (b.at ?? 0))
|
|
77
|
+
return limit > 0 ? steps.slice(-limit) : steps
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
/** Group steps into turns and add turn-level averages plus overall comparison data. */
|
|
81
|
+
export function summarizeSteps(steps, { limit = 12 } = {}) {
|
|
82
|
+
const byTurn = new Map()
|
|
83
|
+
for (const step of Array.isArray(steps) ? steps : []) {
|
|
84
|
+
if (!step) continue
|
|
85
|
+
const key = `${step.sessionId}#${step.turn ?? '?'}`
|
|
86
|
+
let turn = byTurn.get(key)
|
|
87
|
+
if (!turn) {
|
|
88
|
+
turn = {
|
|
89
|
+
sessionId: step.sessionId,
|
|
90
|
+
turn: step.turn,
|
|
91
|
+
firstAt: step.at,
|
|
92
|
+
lastAt: step.at,
|
|
93
|
+
steps: [],
|
|
94
|
+
skills: [],
|
|
95
|
+
}
|
|
96
|
+
byTurn.set(key, turn)
|
|
97
|
+
}
|
|
98
|
+
turn.steps.push(step)
|
|
99
|
+
turn.lastAt = Math.max(turn.lastAt ?? 0, step.at ?? 0)
|
|
100
|
+
for (const skill of step.skills ?? []) if (!turn.skills.includes(skill)) turn.skills.push(skill)
|
|
101
|
+
}
|
|
102
|
+
const turns = [...byTurn.values()]
|
|
103
|
+
.map((turn) => {
|
|
104
|
+
const scores = turn.steps.map((step) => step.score?.total).filter((value) => Number.isFinite(value))
|
|
105
|
+
const average = scores.length > 0 ? scores.reduce((sum, value) => sum + value, 0) / scores.length : null
|
|
106
|
+
return {
|
|
107
|
+
...turn,
|
|
108
|
+
steps: turn.steps.sort((a, b) => (a.step ?? 0) - (b.step ?? 0)),
|
|
109
|
+
average: average === null ? null : round(average),
|
|
110
|
+
best: scores.length > 0 ? Math.max(...scores) : null,
|
|
111
|
+
worst: scores.length > 0 ? Math.min(...scores) : null,
|
|
112
|
+
}
|
|
113
|
+
})
|
|
114
|
+
.sort((a, b) => (b.lastAt ?? 0) - (a.lastAt ?? 0))
|
|
115
|
+
const recent = turns.slice(0, limit)
|
|
116
|
+
const latest = recent.length > 0 ? recent[0] : null
|
|
117
|
+
const previous = recent.length > 1 ? recent[1] : null
|
|
118
|
+
const delta =
|
|
119
|
+
latest && previous && Number.isFinite(latest.average) && Number.isFinite(previous.average)
|
|
120
|
+
? round(latest.average - previous.average)
|
|
121
|
+
: null
|
|
122
|
+
const allScores = steps.map((step) => step.score?.total).filter((value) => Number.isFinite(value))
|
|
123
|
+
const overall = {
|
|
124
|
+
steps: allScores.length,
|
|
125
|
+
turns: turns.length,
|
|
126
|
+
average: allScores.length > 0 ? round(allScores.reduce((sum, value) => sum + value, 0) / allScores.length) : null,
|
|
127
|
+
latestAverage: latest?.average ?? null,
|
|
128
|
+
previousAverage: previous?.average ?? null,
|
|
129
|
+
delta,
|
|
130
|
+
}
|
|
131
|
+
return { turns: recent, totalTurns: turns.length, overall }
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
/**
|
|
135
|
+
* Steps that mention a skill, for the per-skill drill-down. `skill` empty means
|
|
136
|
+
* all steps; the caller can keep its own filter chip in sync with this list.
|
|
137
|
+
*/
|
|
138
|
+
export function stepsForSkill(steps, skill) {
|
|
139
|
+
if (!skill) return Array.isArray(steps) ? steps : []
|
|
140
|
+
return (Array.isArray(steps) ? steps : []).filter((step) => (step.skills ?? []).includes(skill))
|
|
141
|
+
}
|
package/lib/store.js
ADDED
|
@@ -0,0 +1,186 @@
|
|
|
1
|
+
import { promises as fs } from 'node:fs'
|
|
2
|
+
import path from 'node:path'
|
|
3
|
+
import os from 'node:os'
|
|
4
|
+
|
|
5
|
+
function expandHome(p) {
|
|
6
|
+
if (typeof p !== 'string' || p.length === 0) return null
|
|
7
|
+
if (p === '~') return os.homedir()
|
|
8
|
+
if (p.startsWith('~/')) return path.join(os.homedir(), p.slice(2))
|
|
9
|
+
return path.resolve(p)
|
|
10
|
+
}
|
|
11
|
+
|
|
12
|
+
/**
|
|
13
|
+
* Where telemetry lives. Kept OUTSIDE the session log on purpose: the session
|
|
14
|
+
* log is the harness's source of truth and plugins must not append foreign
|
|
15
|
+
* event types to it.
|
|
16
|
+
*/
|
|
17
|
+
export function resolveDataDir(config = {}) {
|
|
18
|
+
const explicit = expandHome(config.dataDir) || expandHome(process.env.DSH_SKILL_EVAL_DIR)
|
|
19
|
+
if (explicit) return explicit
|
|
20
|
+
const home = expandHome(process.env.DSH_HOME) || path.join(os.homedir(), '.dsh')
|
|
21
|
+
const profile = process.env.DSH_PROFILE || 'default'
|
|
22
|
+
return path.join(home, 'skill-eval', profile)
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
/**
|
|
26
|
+
* Append-only JSONL store. Writes are serialized through one promise chain so
|
|
27
|
+
* concurrent tool results cannot interleave partial lines.
|
|
28
|
+
*/
|
|
29
|
+
export class EventStore {
|
|
30
|
+
constructor(dir) {
|
|
31
|
+
this.dir = dir
|
|
32
|
+
this.file = path.join(dir, 'events.jsonl')
|
|
33
|
+
this.queue = Promise.resolve()
|
|
34
|
+
this.failed = null
|
|
35
|
+
// Bumped on every successful append; the analyzer uses it to invalidate a
|
|
36
|
+
// cached report without re-reading the whole store on every request.
|
|
37
|
+
this.version = 0
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
append(record) {
|
|
41
|
+
let line
|
|
42
|
+
try {
|
|
43
|
+
line = JSON.stringify(record)
|
|
44
|
+
} catch {
|
|
45
|
+
return Promise.resolve()
|
|
46
|
+
}
|
|
47
|
+
if (typeof line !== 'string') return Promise.resolve()
|
|
48
|
+
this.queue = this.queue
|
|
49
|
+
.then(async () => {
|
|
50
|
+
await fs.mkdir(this.dir, { recursive: true })
|
|
51
|
+
// A crash can leave a record without its trailing newline. Appending on
|
|
52
|
+
// top of that fragment would glue two records into one unparseable line
|
|
53
|
+
// and silently lose the new record too, so close the torn tail first.
|
|
54
|
+
try {
|
|
55
|
+
const info = await fs.stat(this.file)
|
|
56
|
+
if (info.size > 0) {
|
|
57
|
+
const handle = await fs.open(this.file, 'r')
|
|
58
|
+
try {
|
|
59
|
+
const last = Buffer.alloc(1)
|
|
60
|
+
await handle.read(last, 0, 1, info.size - 1)
|
|
61
|
+
if (last[0] !== 0x0a) await fs.appendFile(this.file, '\n', 'utf8')
|
|
62
|
+
} finally {
|
|
63
|
+
await handle.close()
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
} catch (error) {
|
|
67
|
+
if (!error || error.code !== 'ENOENT') throw error
|
|
68
|
+
}
|
|
69
|
+
await fs.appendFile(this.file, line + '\n', 'utf8')
|
|
70
|
+
this.failed = null
|
|
71
|
+
this.version += 1
|
|
72
|
+
})
|
|
73
|
+
.catch((error) => {
|
|
74
|
+
const message = error instanceof Error ? error.message : String(error)
|
|
75
|
+
const first = this.failed === null
|
|
76
|
+
this.failed = message
|
|
77
|
+
if (first && typeof this.onError === 'function') {
|
|
78
|
+
try {
|
|
79
|
+
this.onError(message)
|
|
80
|
+
} catch {
|
|
81
|
+
// the logger itself must never break recording
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
})
|
|
85
|
+
return this.queue
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
/** Read every parseable record; malformed lines from a crash are skipped. */
|
|
89
|
+
async readAll() {
|
|
90
|
+
let raw
|
|
91
|
+
try {
|
|
92
|
+
raw = await fs.readFile(this.file, 'utf8')
|
|
93
|
+
} catch (error) {
|
|
94
|
+
if (error && error.code === 'ENOENT') return []
|
|
95
|
+
throw error
|
|
96
|
+
}
|
|
97
|
+
const out = []
|
|
98
|
+
for (const line of raw.split('\n')) {
|
|
99
|
+
const trimmed = line.trim()
|
|
100
|
+
if (trimmed.length === 0) continue
|
|
101
|
+
try {
|
|
102
|
+
out.push(JSON.parse(trimmed))
|
|
103
|
+
} catch {
|
|
104
|
+
// torn write — ignore
|
|
105
|
+
}
|
|
106
|
+
}
|
|
107
|
+
return out
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
async size() {
|
|
111
|
+
try {
|
|
112
|
+
const st = await fs.stat(this.file)
|
|
113
|
+
return st.size
|
|
114
|
+
} catch {
|
|
115
|
+
return 0
|
|
116
|
+
}
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
/** Cheap identity of the current file contents for report caching. */
|
|
120
|
+
async stamp() {
|
|
121
|
+
try {
|
|
122
|
+
const st = await fs.stat(this.file)
|
|
123
|
+
return `${st.mtimeMs}:${st.size}`
|
|
124
|
+
} catch {
|
|
125
|
+
return 'missing'
|
|
126
|
+
}
|
|
127
|
+
}
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
/**
|
|
131
|
+
* Fold append-only events into per-skill invocation samples.
|
|
132
|
+
*
|
|
133
|
+
* Record shapes written by the plugin:
|
|
134
|
+
* { t:'skill', v:1, at, sessionId, callId, skill, ok, loadMs, error?, windowId }
|
|
135
|
+
* { t:'window', v:1, at, windowId, sessionId, skill, toolCalls, errors,
|
|
136
|
+
* errorNames[], corrections[], otherSkills[], turnEnded, truncated }
|
|
137
|
+
*/
|
|
138
|
+
export function foldSamples(records) {
|
|
139
|
+
const byWindow = new Map()
|
|
140
|
+
for (const r of records) {
|
|
141
|
+
if (r && r.t === 'window' && typeof r.windowId === 'string') byWindow.set(r.windowId, r)
|
|
142
|
+
}
|
|
143
|
+
const bySkill = new Map()
|
|
144
|
+
for (const r of records) {
|
|
145
|
+
if (!r || r.t !== 'skill' || typeof r.skill !== 'string') continue
|
|
146
|
+
const entry = bySkill.get(r.skill) ?? { skill: r.skill, samples: [], firstAt: r.at, lastAt: r.at }
|
|
147
|
+
const window = r.windowId ? byWindow.get(r.windowId) : undefined
|
|
148
|
+
entry.samples.push({
|
|
149
|
+
at: r.at,
|
|
150
|
+
sessionId: r.sessionId,
|
|
151
|
+
callId: r.callId,
|
|
152
|
+
ok: r.ok === true,
|
|
153
|
+
turn: typeof r.turn === 'number' ? r.turn : null,
|
|
154
|
+
via: r.via === 'invocation' ? 'invocation' : 'tool',
|
|
155
|
+
loadMs: typeof r.loadMs === 'number' ? r.loadMs : undefined,
|
|
156
|
+
error: r.error,
|
|
157
|
+
window: window
|
|
158
|
+
? {
|
|
159
|
+
toolCalls: Number(window.toolCalls) || 0,
|
|
160
|
+
errors: Number(window.errors) || 0,
|
|
161
|
+
errorNames: Array.isArray(window.errorNames)
|
|
162
|
+
? window.errorNames.filter((name) => typeof name === 'string')
|
|
163
|
+
: [],
|
|
164
|
+
mcp: window.mcp && typeof window.mcp === 'object' && !Array.isArray(window.mcp) ? window.mcp : {},
|
|
165
|
+
origin: typeof window.origin === 'string' ? window.origin : null,
|
|
166
|
+
tokens:
|
|
167
|
+
window.tokens && typeof window.tokens === 'object'
|
|
168
|
+
? window.tokens
|
|
169
|
+
: { input: 0, output: 0, cacheRead: 0, calls: 0 },
|
|
170
|
+
loopErrors: Number(window.loopErrors) || 0,
|
|
171
|
+
llmRetries: Number(window.llmRetries) || 0,
|
|
172
|
+
closeReason: typeof window.closeReason === 'string' ? window.closeReason : null,
|
|
173
|
+
lastLoopError: typeof window.lastLoopError === 'string' ? window.lastLoopError : null,
|
|
174
|
+
corrections: Array.isArray(window.corrections) ? window.corrections : [],
|
|
175
|
+
otherSkills: Array.isArray(window.otherSkills) ? window.otherSkills : [],
|
|
176
|
+
turnEnded: window.turnEnded === true,
|
|
177
|
+
truncated: window.truncated === true,
|
|
178
|
+
}
|
|
179
|
+
: null,
|
|
180
|
+
})
|
|
181
|
+
entry.firstAt = Math.min(entry.firstAt ?? r.at, r.at)
|
|
182
|
+
entry.lastAt = Math.max(entry.lastAt ?? r.at, r.at)
|
|
183
|
+
bySkill.set(r.skill, entry)
|
|
184
|
+
}
|
|
185
|
+
return [...bySkill.values()].sort((a, b) => (b.lastAt ?? 0) - (a.lastAt ?? 0))
|
|
186
|
+
}
|
package/lib/validity.js
ADDED
|
@@ -0,0 +1,115 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Validity dashboard: what this evidence can and cannot support.
|
|
3
|
+
*
|
|
4
|
+
* The honest part of an evaluator is knowing which claims are measured and
|
|
5
|
+
* which are still guesses. This module derives only statistics that the stored
|
|
6
|
+
* report and ledger actually contain — no extrapolation, no invented metrics.
|
|
7
|
+
*/
|
|
8
|
+
|
|
9
|
+
export function buildValidity(report, options = {}) {
|
|
10
|
+
const skills = Array.isArray(report?.skills) ? report.skills : []
|
|
11
|
+
const edits = Array.isArray(report?.edits) ? report.edits : []
|
|
12
|
+
const minSamples = options.minSamples ?? report?.scoring?.minSamples ?? 3
|
|
13
|
+
const resamples = report?.scoring?.resamples ?? null
|
|
14
|
+
|
|
15
|
+
const withCI = skills.filter((entry) => entry?.score?.ci).length
|
|
16
|
+
const provisional = skills.filter((entry) => entry?.score?.provisional).length
|
|
17
|
+
const ranked = report?.ranking?.significant
|
|
18
|
+
const adviceCounts = { critical: 0, warning: 0, info: 0 }
|
|
19
|
+
for (const entry of skills) {
|
|
20
|
+
for (const item of entry?.advice ?? []) {
|
|
21
|
+
if (adviceCounts[item.severity] !== undefined) adviceCounts[item.severity] += 1
|
|
22
|
+
}
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
const ledger = {
|
|
26
|
+
total: edits.length,
|
|
27
|
+
accept: 0,
|
|
28
|
+
warn: 0,
|
|
29
|
+
reject: 0,
|
|
30
|
+
dimensions: Object.create(null),
|
|
31
|
+
withHypothesis: 0,
|
|
32
|
+
}
|
|
33
|
+
const VERDICTS = new Set(['accept', 'warn', 'reject'])
|
|
34
|
+
for (const entry of edits) {
|
|
35
|
+
if (entry && VERDICTS.has(entry.verdict)) ledger[entry.verdict] += 1
|
|
36
|
+
if (entry && typeof entry.dimension === 'string' && entry.dimension) {
|
|
37
|
+
ledger.dimensions[entry.dimension] = (ledger.dimensions[entry.dimension] ?? 0) + 1
|
|
38
|
+
}
|
|
39
|
+
if (entry && entry.hypothesis) ledger.withHypothesis += 1
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
const explorable = skills.filter((entry) => entry?.exploration).length
|
|
43
|
+
const actionable = skills.filter((entry) =>
|
|
44
|
+
(entry?.advice ?? []).some((item) => item.severity === 'critical' || item.severity === 'warning'),
|
|
45
|
+
).length
|
|
46
|
+
|
|
47
|
+
const canSay = []
|
|
48
|
+
const cannotSay = []
|
|
49
|
+
if (skills.length > 0) canSay.push(`scores for ${skills.length} skill(s) from recorded loads`)
|
|
50
|
+
if (withCI > 0) canSay.push(`95% intervals for ${withCI} skill(s) (bootstrap resamples: ${resamples ?? 'unknown'})`)
|
|
51
|
+
if (ranked === false) canSay.push('at least two skills are a statistical tie')
|
|
52
|
+
if (ledger.accept > 0) canSay.push(`${ledger.accept} edit(s) screened and accepted before applying`)
|
|
53
|
+
if (withCI < skills.length) {
|
|
54
|
+
cannotSay.push(
|
|
55
|
+
`${skills.length - withCI} skill(s) have no interval yet (n < ${minSamples}) — their score is provisional`,
|
|
56
|
+
)
|
|
57
|
+
}
|
|
58
|
+
if (skills.length < 3) cannotSay.push('too few skills to estimate a ranking or a cross-skill baseline')
|
|
59
|
+
cannotSay.push('before/after effect of an edit: needs the same skill measured across the edit boundary')
|
|
60
|
+
cannotSay.push('token cost of a skill the model did not actually load')
|
|
61
|
+
|
|
62
|
+
const canSayZh = []
|
|
63
|
+
const cannotSayZh = []
|
|
64
|
+
if (skills.length > 0) canSayZh.push(`基于已记录加载的 ${skills.length} 个技能分数`)
|
|
65
|
+
if (withCI > 0) canSayZh.push(`${withCI} 个技能的 95% 区间(bootstrap 重采样:${resamples ?? '未知'})`)
|
|
66
|
+
if (ranked === false) canSayZh.push('至少两个技能在统计上打平')
|
|
67
|
+
if (ledger.accept > 0) canSayZh.push(`${ledger.accept} 项改动在应用前经过审查并被接受`)
|
|
68
|
+
if (withCI < skills.length)
|
|
69
|
+
cannotSayZh.push(`${skills.length - withCI} 个技能还没有区间(样本 < ${minSamples}),分数是临时值`)
|
|
70
|
+
if (skills.length < 3) cannotSayZh.push('技能太少,无法估计排名或跨技能基线')
|
|
71
|
+
cannotSayZh.push('改动前后的效果:需要在改动边界两侧对同一技能测量')
|
|
72
|
+
cannotSayZh.push('模型并未真正加载的技能其 token 成本')
|
|
73
|
+
|
|
74
|
+
return {
|
|
75
|
+
generatedAt: report?.generatedAt ?? null,
|
|
76
|
+
dataDir: report?.dataDir ?? null,
|
|
77
|
+
coverage: { skills: skills.length, withCI, provisional, actionable, explorable },
|
|
78
|
+
ranking: { significant: ranked === true, significantReported: ranked !== undefined },
|
|
79
|
+
advice: adviceCounts,
|
|
80
|
+
ledger,
|
|
81
|
+
canSay,
|
|
82
|
+
cannotSay,
|
|
83
|
+
canSayZh,
|
|
84
|
+
cannotSayZh,
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
export function renderValidityMarkdown(validity) {
|
|
89
|
+
if (!validity) return '## Validity\n\nno report yet'
|
|
90
|
+
const c = validity.coverage
|
|
91
|
+
const l = validity.ledger
|
|
92
|
+
const lines = [
|
|
93
|
+
'## Validity — what the evidence supports',
|
|
94
|
+
'',
|
|
95
|
+
`coverage: **${c.skills}** skill(s) · **${c.withCI}** with 95% intervals · **${c.provisional}** provisional · **${c.actionable}** with an actionable fix`,
|
|
96
|
+
`ranking: ${validity.ranking.significant ? '**significant**' : 'not significant (intervals overlap)'}`,
|
|
97
|
+
`advice: ${validity.advice.critical} critical · ${validity.advice.warning} warning · ${validity.advice.info} info`,
|
|
98
|
+
`ledger: ${l.total} edit(s) screened — ${l.accept} accept · ${l.warn} warn · ${l.reject} reject` +
|
|
99
|
+
(Object.keys(l.dimensions).length > 0
|
|
100
|
+
? ` · dimensions: ${Object.entries(l.dimensions)
|
|
101
|
+
.map(([key, count]) => `${key}×${count}`)
|
|
102
|
+
.join(', ')}`
|
|
103
|
+
: ''),
|
|
104
|
+
'',
|
|
105
|
+
'### Can say',
|
|
106
|
+
...validity.canSay.map((item) => `- ${item}`),
|
|
107
|
+
'',
|
|
108
|
+
'### Cannot say yet',
|
|
109
|
+
...validity.cannotSay.map((item) => `- ${item}`),
|
|
110
|
+
]
|
|
111
|
+
if (validity.coverage.explorable > 0) {
|
|
112
|
+
lines.push('', `exploration: ${c.explorable} skill(s) stuck on one dimension — see \`/skill-eval loop all\``)
|
|
113
|
+
}
|
|
114
|
+
return lines.join('\n')
|
|
115
|
+
}
|
package/locale/en.json
ADDED
package/locale/zh.json
ADDED
package/package.json
CHANGED
|
@@ -1,6 +1,98 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "dsh-skill-eval",
|
|
3
|
-
"version": "
|
|
4
|
-
"
|
|
5
|
-
"
|
|
6
|
-
|
|
3
|
+
"version": "1.20.2",
|
|
4
|
+
"description": "Skill execution telemetry & effectiveness scoring for DeepSeek Harness: records every skill load, folds the follow-up window into evidence, scores each skill and hands the agent concrete optimization advice.",
|
|
5
|
+
"type": "module",
|
|
6
|
+
"main": "index.js",
|
|
7
|
+
"exports": {
|
|
8
|
+
".": "./index.js",
|
|
9
|
+
"./client": "./client.js",
|
|
10
|
+
"./locale/en.json": "./locale/en.json",
|
|
11
|
+
"./locale/zh.json": "./locale/zh.json",
|
|
12
|
+
"./icon": "./icon.svg",
|
|
13
|
+
"./package.json": "./package.json"
|
|
14
|
+
},
|
|
15
|
+
"icon": "icon.svg",
|
|
16
|
+
"scripts": {
|
|
17
|
+
"build": "node scripts/build.mjs",
|
|
18
|
+
"build:fast": "node scripts/build.mjs --fast",
|
|
19
|
+
"build:pack": "node scripts/build.mjs --pack",
|
|
20
|
+
"build:install": "node scripts/build.mjs --install",
|
|
21
|
+
"build:client": "node scripts/build-client.mjs",
|
|
22
|
+
"prepack": "node scripts/build-client.mjs",
|
|
23
|
+
"pretest": "node scripts/build-client.mjs",
|
|
24
|
+
"test": "node --test test/",
|
|
25
|
+
"accept": "node scripts/accept.mjs",
|
|
26
|
+
"verify": "node scripts/verify-live.mjs",
|
|
27
|
+
"doctor": "node scripts/doctor.mjs",
|
|
28
|
+
"release:check": "npm run build:client && npm run format:check && npm run lint && node scripts/doctor.mjs --release && node --test test/ && node scripts/accept.mjs",
|
|
29
|
+
"release:pack": "npm pack --pack-destination dist",
|
|
30
|
+
"format": "prettier --write .",
|
|
31
|
+
"format:check": "prettier --check .",
|
|
32
|
+
"lint": "eslint .",
|
|
33
|
+
"lint:strict": "eslint . --max-warnings=0"
|
|
34
|
+
},
|
|
35
|
+
"peerDependencies": {
|
|
36
|
+
"@deepseek-ai/schemastery": "^3.18.2"
|
|
37
|
+
},
|
|
38
|
+
"dsh": {
|
|
39
|
+
"engines": {
|
|
40
|
+
"dsh": ">=0.2.0"
|
|
41
|
+
},
|
|
42
|
+
"bundle": {
|
|
43
|
+
"patch": "./cordis.patch.yml"
|
|
44
|
+
},
|
|
45
|
+
"client": {
|
|
46
|
+
"platform": "web",
|
|
47
|
+
"immediately": true
|
|
48
|
+
}
|
|
49
|
+
},
|
|
50
|
+
"files": [
|
|
51
|
+
"index.js",
|
|
52
|
+
"client.js",
|
|
53
|
+
"lib/**/*.js",
|
|
54
|
+
"cordis.patch.yml",
|
|
55
|
+
"locale",
|
|
56
|
+
"icon.svg",
|
|
57
|
+
"docs/*.md",
|
|
58
|
+
"README.md",
|
|
59
|
+
"README.zh.md",
|
|
60
|
+
"scripts",
|
|
61
|
+
"CHANGELOG.md",
|
|
62
|
+
"LICENSE"
|
|
63
|
+
],
|
|
64
|
+
"license": "MIT",
|
|
65
|
+
"repository": {
|
|
66
|
+
"type": "git",
|
|
67
|
+
"url": "git+https://github.com/carlslin/dsh-skill-eval.git"
|
|
68
|
+
},
|
|
69
|
+
"homepage": "https://github.com/carlslin/dsh-skill-eval#readme",
|
|
70
|
+
"bugs": {
|
|
71
|
+
"url": "https://github.com/carlslin/dsh-skill-eval/issues"
|
|
72
|
+
},
|
|
73
|
+
"devDependencies": {
|
|
74
|
+
"@deepseek-ai/schemastery": "^3.19.0",
|
|
75
|
+
"@eslint/js": "^10.0.1",
|
|
76
|
+
"eslint": "^10.15.0",
|
|
77
|
+
"globals": "^17.14.0",
|
|
78
|
+
"jsdom": "^25.0.1",
|
|
79
|
+
"prettier": "^3.10.1",
|
|
80
|
+
"react": "^18.3.1",
|
|
81
|
+
"react-dom": "^18.3.1"
|
|
82
|
+
},
|
|
83
|
+
"bin": {
|
|
84
|
+
"dsh-skill-eval-verify": "scripts/verify-live.mjs",
|
|
85
|
+
"dsh-skill-eval-doctor": "scripts/doctor.mjs",
|
|
86
|
+
"dsh-skill-eval-accept": "scripts/accept.mjs"
|
|
87
|
+
},
|
|
88
|
+
"author": "dsh-skill-eval contributors",
|
|
89
|
+
"keywords": [
|
|
90
|
+
"deepseek",
|
|
91
|
+
"dsh",
|
|
92
|
+
"plugin",
|
|
93
|
+
"skills",
|
|
94
|
+
"telemetry",
|
|
95
|
+
"evaluation",
|
|
96
|
+
"scoring"
|
|
97
|
+
]
|
|
98
|
+
}
|