dsh-skill-eval 1.20.2 → 1.20.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +43 -0
- package/README.md +13 -6
- package/README.zh.md +11 -4
- package/client.js +230 -76
- package/docs/ACCEPTANCE.md +1 -1
- package/docs/AUDIT.md +18 -0
- package/docs/DETAIL.md +11 -6
- package/docs/ENTRYPOINTS.md +1 -1
- package/docs/RELEASE.md +2 -2
- package/docs/UI-REVIEW.md +8 -8
- package/docs/WORKFLOW.md +1 -1
- package/index.js +98 -36
- package/lib/accept.js +1 -1
- package/lib/advice.js +12 -2
- package/lib/dev.js +1 -1
- package/lib/history.js +3 -2
- package/lib/html.js +9 -0
- package/lib/recorder.js +16 -6
- package/lib/render.js +5 -7
- package/lib/report.js +49 -6
- package/lib/score.js +40 -0
- package/lib/settings.js +26 -7
- package/lib/snapshot.js +111 -27
- package/lib/store.js +55 -1
- package/lib/validity.js +12 -2
- package/package.json +1 -1
- package/scripts/accept.mjs +7 -6
- package/scripts/doctor.mjs +46 -11
- package/scripts/lib.mjs +35 -0
- package/scripts/verify-live.mjs +16 -6
package/lib/settings.js
CHANGED
|
@@ -127,7 +127,7 @@ export const UNIT_ZH = { loads: '次加载', tokens: 'token', edits: '次修改'
|
|
|
127
127
|
|
|
128
128
|
export function unitLabel(unit, zh) {
|
|
129
129
|
if (!unit) return ''
|
|
130
|
-
return zh ? UNIT_ZH[unit] ?? unit : unit
|
|
130
|
+
return zh ? (UNIT_ZH[unit] ?? unit) : unit
|
|
131
131
|
}
|
|
132
132
|
|
|
133
133
|
export const SETTINGS_GROUPS = [
|
|
@@ -276,14 +276,12 @@ export async function loadSettings(dataDir) {
|
|
|
276
276
|
|
|
277
277
|
let saveQueue = Promise.resolve()
|
|
278
278
|
|
|
279
|
-
|
|
280
|
-
* Serialize writes and replace the file atomically: concurrent saves cannot
|
|
281
|
-
* lose each other's fields, and a crash mid-write leaves the previous file
|
|
282
|
-
* intact instead of truncated JSON that would silently drop every override.
|
|
283
|
-
*/
|
|
284
|
-
export function saveSettings(dataDir, values) {
|
|
279
|
+
function enqueueSettingsWrite(dataDir, resolveValues) {
|
|
285
280
|
const file = path.join(dataDir, 'settings.json')
|
|
286
281
|
const job = saveQueue.then(async () => {
|
|
282
|
+
// Resolve inside the queue: a read-merge-write (updateSettings) must see
|
|
283
|
+
// the result of the previous job, not the stale file from before it.
|
|
284
|
+
const values = await resolveValues()
|
|
287
285
|
const temp = `${file}.tmp`
|
|
288
286
|
await mkdir(dataDir, { recursive: true })
|
|
289
287
|
await writeFile(temp, `${JSON.stringify(values, null, 2)}\n`, 'utf8')
|
|
@@ -294,6 +292,27 @@ export function saveSettings(dataDir, values) {
|
|
|
294
292
|
return job
|
|
295
293
|
}
|
|
296
294
|
|
|
295
|
+
/**
|
|
296
|
+
* Serialize writes and replace the file atomically: a crash mid-write leaves
|
|
297
|
+
* the previous file intact instead of truncated JSON that would silently drop
|
|
298
|
+
* every override.
|
|
299
|
+
*/
|
|
300
|
+
export function saveSettings(dataDir, values) {
|
|
301
|
+
return enqueueSettingsWrite(dataDir, () => values)
|
|
302
|
+
}
|
|
303
|
+
|
|
304
|
+
/**
|
|
305
|
+
* Merge a partial patch into the settings file inside the write queue. Two
|
|
306
|
+
* overlapping HTTP saves must not both read the pre-update file and drop each
|
|
307
|
+
* other's field; the whole read-merge-write therefore runs as one queued job.
|
|
308
|
+
*/
|
|
309
|
+
export function updateSettings(dataDir, patch) {
|
|
310
|
+
return enqueueSettingsWrite(dataDir, async () => ({
|
|
311
|
+
...(await loadSettings(dataDir)),
|
|
312
|
+
...(patch && typeof patch === 'object' ? patch : {}),
|
|
313
|
+
}))
|
|
314
|
+
}
|
|
315
|
+
|
|
297
316
|
/** Synchronous variant used at plugin apply time (the config must exist first). */
|
|
298
317
|
export function loadSettingsSync(dataDir) {
|
|
299
318
|
if (typeof dataDir !== 'string' || dataDir.length === 0) return {}
|
package/lib/snapshot.js
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { promises as fs } from 'node:fs'
|
|
2
2
|
import path from 'node:path'
|
|
3
|
+
import { formatTimestamp } from './html.js'
|
|
3
4
|
import { renderSummaryMarkdown } from './render.js'
|
|
4
5
|
|
|
5
6
|
/**
|
|
@@ -33,13 +34,17 @@ export async function listTurnReports(dir) {
|
|
|
33
34
|
try {
|
|
34
35
|
const info = await fs.stat(file)
|
|
35
36
|
const report = JSON.parse(await fs.readFile(file, 'utf8'))
|
|
37
|
+
const slug = name.replace(/\.json$/, '')
|
|
38
|
+
// The slug carries the turn the board was written for; turnLoads[0] is
|
|
39
|
+
// the global newest load and can belong to another session.
|
|
40
|
+
const parsed = slug.match(/^(.*)-t(\d+|unspecified)$/)
|
|
36
41
|
rows.push({
|
|
37
42
|
file: name,
|
|
38
|
-
slug
|
|
43
|
+
slug,
|
|
39
44
|
bytes: info.size,
|
|
40
45
|
at: info.mtimeMs,
|
|
41
|
-
sessionId: report?.turnLoads?.[0]?.sessionId ?? null,
|
|
42
|
-
turn: report?.turnLoads?.[0]?.turn ?? null,
|
|
46
|
+
sessionId: parsed ? parsed[1] : (report?.turnLoads?.[0]?.sessionId ?? null),
|
|
47
|
+
turn: parsed && parsed[2] !== 'unspecified' ? Number(parsed[2]) : (report?.turnLoads?.[0]?.turn ?? null),
|
|
43
48
|
skills: (report?.skills ?? []).slice(0, 4).map((skill) => skill.skill),
|
|
44
49
|
totals: report?.totals ?? null,
|
|
45
50
|
})
|
|
@@ -59,7 +64,7 @@ export function renderTurnReportsMarkdown(rows, { limit = 20 } = {}) {
|
|
|
59
64
|
lines.push('| written | session | turn | skills | size | file |')
|
|
60
65
|
lines.push('|---|---|---|---|---|---|')
|
|
61
66
|
for (const row of rows.slice(0, limit)) {
|
|
62
|
-
const when =
|
|
67
|
+
const when = formatTimestamp(row.at)
|
|
63
68
|
lines.push(
|
|
64
69
|
`| ${when} | \`${String(row.sessionId ?? '?').slice(-8)}\` | ${row.turn ?? '—'} | ${(row.skills ?? []).join(', ') || '—'} | ${row.bytes}b | \`${row.file}\` |`,
|
|
65
70
|
)
|
|
@@ -67,38 +72,105 @@ export function renderTurnReportsMarkdown(rows, { limit = 20 } = {}) {
|
|
|
67
72
|
return lines.join('\n')
|
|
68
73
|
}
|
|
69
74
|
|
|
70
|
-
export function createSnapshots({
|
|
75
|
+
export function createSnapshots({
|
|
76
|
+
analyzer,
|
|
77
|
+
dir,
|
|
78
|
+
debounceMs = 1500,
|
|
79
|
+
log,
|
|
80
|
+
enabled = true,
|
|
81
|
+
turnReports = true,
|
|
82
|
+
turnReportsKeep = 50,
|
|
83
|
+
}) {
|
|
84
|
+
// Getters so a settings change applies without recreating the snapshotter.
|
|
85
|
+
const latestEnabled = () => (typeof enabled === 'function' ? enabled() !== false : enabled !== false)
|
|
86
|
+
const turnsEnabled = () => (typeof turnReports === 'function' ? turnReports() !== false : turnReports !== false)
|
|
71
87
|
let timer = null
|
|
72
88
|
let running = null
|
|
73
89
|
let pending = false
|
|
90
|
+
// Contexts of the turns that settled while the debounce was open. Keeping
|
|
91
|
+
// them lets the writer label each board with its own turn instead of the
|
|
92
|
+
// globally newest turn in the store (wrong under concurrent sessions).
|
|
93
|
+
let pendingTurns = []
|
|
94
|
+
let disposed = false
|
|
74
95
|
|
|
75
96
|
const markdownPath = path.join(dir, 'latest.md')
|
|
76
97
|
const jsonPath = path.join(dir, 'latest.json')
|
|
77
98
|
|
|
78
|
-
|
|
99
|
+
// Same crash rule as settings: a torn snapshot would be read as missing.
|
|
100
|
+
async function writeAtomic(file, content) {
|
|
101
|
+
const temp = `${file}.tmp`
|
|
102
|
+
await fs.writeFile(temp, content, 'utf8')
|
|
103
|
+
await fs.rename(temp, file)
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
function pickTurnEntries(report, contexts) {
|
|
107
|
+
const entries = Array.isArray(report.turnLoads) ? report.turnLoads : []
|
|
108
|
+
const newest = [...entries].sort((a, b) => (b.lastAt ?? 0) - (a.lastAt ?? 0))[0]
|
|
109
|
+
const picked = []
|
|
110
|
+
const seen = new Set()
|
|
111
|
+
const add = (entry) => {
|
|
112
|
+
if (!entry) return
|
|
113
|
+
const slug = turnSlug(entry)
|
|
114
|
+
if (seen.has(slug)) return
|
|
115
|
+
seen.add(slug)
|
|
116
|
+
picked.push(entry)
|
|
117
|
+
}
|
|
118
|
+
if (!Array.isArray(contexts) || contexts.length === 0) {
|
|
119
|
+
add(newest)
|
|
120
|
+
return picked
|
|
121
|
+
}
|
|
122
|
+
for (const context of contexts) {
|
|
123
|
+
const hasKey = context && (context.sessionId || Number.isFinite(context.turn))
|
|
124
|
+
if (!hasKey) {
|
|
125
|
+
add(newest)
|
|
126
|
+
continue
|
|
127
|
+
}
|
|
128
|
+
const rows = context.sessionId ? entries.filter((entry) => entry.sessionId === context.sessionId) : entries
|
|
129
|
+
const match = Number.isFinite(context.turn)
|
|
130
|
+
? rows.find((entry) => entry.turn === context.turn)
|
|
131
|
+
: [...rows].sort((a, b) => (b.lastAt ?? 0) - (a.lastAt ?? 0))[0]
|
|
132
|
+
// No matching load means this turn loaded no skill: do not overwrite a
|
|
133
|
+
// different turn's board with it.
|
|
134
|
+
add(match ?? null)
|
|
135
|
+
}
|
|
136
|
+
return picked
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
const renderMarkdown = (report) =>
|
|
140
|
+
renderSummaryMarkdown({
|
|
141
|
+
...report,
|
|
142
|
+
skills: report.skills.map((skill) => ({
|
|
143
|
+
...skill,
|
|
144
|
+
topAdvice: skill.topAdvice.slice(0, 3),
|
|
145
|
+
})),
|
|
146
|
+
}) + '\n'
|
|
147
|
+
|
|
148
|
+
async function write(contexts = []) {
|
|
149
|
+
const writeLatest = latestEnabled()
|
|
150
|
+
const writeTurns = turnsEnabled()
|
|
151
|
+
if (!writeLatest && !writeTurns) return null
|
|
79
152
|
const report = await analyzer.build()
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
topAdvice: skill.topAdvice.slice(0, 3),
|
|
88
|
-
})),
|
|
89
|
-
}) + '\n'
|
|
90
|
-
await fs.writeFile(markdownPath, markdown, 'utf8')
|
|
153
|
+
let markdown = null
|
|
154
|
+
if (writeLatest) {
|
|
155
|
+
await fs.mkdir(dir, { recursive: true })
|
|
156
|
+
await writeAtomic(jsonPath, JSON.stringify(report, null, 2))
|
|
157
|
+
markdown = renderMarkdown(report)
|
|
158
|
+
await writeAtomic(markdownPath, markdown)
|
|
159
|
+
}
|
|
91
160
|
|
|
92
161
|
// One board per settled turn: the "每次执行后结果在单独的看板" surface.
|
|
93
|
-
if (
|
|
94
|
-
|
|
95
|
-
|
|
162
|
+
if (writeTurns) {
|
|
163
|
+
markdown = markdown ?? renderMarkdown(report)
|
|
164
|
+
const turnEntries = pickTurnEntries(report, contexts)
|
|
165
|
+
if (turnEntries.length > 0) {
|
|
96
166
|
try {
|
|
97
167
|
const turnsDir = path.join(dir, 'turns')
|
|
98
168
|
await fs.mkdir(turnsDir, { recursive: true })
|
|
99
|
-
const
|
|
100
|
-
|
|
101
|
-
|
|
169
|
+
for (const turnEntry of turnEntries) {
|
|
170
|
+
const slug = turnSlug(turnEntry)
|
|
171
|
+
await writeAtomic(path.join(turnsDir, `${slug}.json`), JSON.stringify(report, null, 2))
|
|
172
|
+
await writeAtomic(path.join(turnsDir, `${slug}.md`), markdown)
|
|
173
|
+
}
|
|
102
174
|
if (turnReportsKeep > 0) {
|
|
103
175
|
const files = (await fs.readdir(turnsDir)).filter((name) => name.endsWith('.json'))
|
|
104
176
|
if (files.length > turnReportsKeep) {
|
|
@@ -121,18 +193,21 @@ export function createSnapshots({ analyzer, dir, debounceMs = 1500, log, turnRep
|
|
|
121
193
|
}
|
|
122
194
|
|
|
123
195
|
async function run() {
|
|
196
|
+
if (disposed) return Promise.resolve()
|
|
124
197
|
if (running) {
|
|
125
198
|
pending = true
|
|
126
199
|
return running
|
|
127
200
|
}
|
|
201
|
+
const contexts = pendingTurns
|
|
202
|
+
pendingTurns = []
|
|
128
203
|
running = (async () => {
|
|
129
204
|
try {
|
|
130
|
-
await write()
|
|
205
|
+
await write(contexts)
|
|
131
206
|
} catch (error) {
|
|
132
207
|
log?.(`snapshot failed: ${error instanceof Error ? error.message : String(error)}`)
|
|
133
208
|
} finally {
|
|
134
209
|
running = null
|
|
135
|
-
if (pending) {
|
|
210
|
+
if (pending || pendingTurns.length > 0) {
|
|
136
211
|
pending = false
|
|
137
212
|
void request()
|
|
138
213
|
}
|
|
@@ -141,7 +216,9 @@ export function createSnapshots({ analyzer, dir, debounceMs = 1500, log, turnRep
|
|
|
141
216
|
return running
|
|
142
217
|
}
|
|
143
218
|
|
|
144
|
-
function request() {
|
|
219
|
+
function request(context) {
|
|
220
|
+
if (disposed) return
|
|
221
|
+
if (turnsEnabled()) pendingTurns.push(context ?? null)
|
|
145
222
|
if (timer) return
|
|
146
223
|
timer = setTimeout(() => {
|
|
147
224
|
timer = null
|
|
@@ -160,7 +237,14 @@ export function createSnapshots({ analyzer, dir, debounceMs = 1500, log, turnRep
|
|
|
160
237
|
timer = null
|
|
161
238
|
}
|
|
162
239
|
if (running) await running
|
|
163
|
-
|
|
240
|
+
const contexts = pendingTurns
|
|
241
|
+
pendingTurns = []
|
|
242
|
+
disposed = true
|
|
243
|
+
try {
|
|
244
|
+
await write(contexts)
|
|
245
|
+
} catch (error) {
|
|
246
|
+
log?.(`snapshot failed: ${error instanceof Error ? error.message : String(error)}`)
|
|
247
|
+
}
|
|
164
248
|
},
|
|
165
249
|
}
|
|
166
250
|
}
|
package/lib/store.js
CHANGED
|
@@ -31,6 +31,7 @@ export class EventStore {
|
|
|
31
31
|
this.dir = dir
|
|
32
32
|
this.file = path.join(dir, 'events.jsonl')
|
|
33
33
|
this.queue = Promise.resolve()
|
|
34
|
+
this.pending = 0
|
|
34
35
|
this.failed = null
|
|
35
36
|
// Bumped on every successful append; the analyzer uses it to invalidate a
|
|
36
37
|
// cached report without re-reading the whole store on every request.
|
|
@@ -45,6 +46,7 @@ export class EventStore {
|
|
|
45
46
|
return Promise.resolve()
|
|
46
47
|
}
|
|
47
48
|
if (typeof line !== 'string') return Promise.resolve()
|
|
49
|
+
this.pending += 1
|
|
48
50
|
this.queue = this.queue
|
|
49
51
|
.then(async () => {
|
|
50
52
|
await fs.mkdir(this.dir, { recursive: true })
|
|
@@ -82,6 +84,9 @@ export class EventStore {
|
|
|
82
84
|
}
|
|
83
85
|
}
|
|
84
86
|
})
|
|
87
|
+
.finally(() => {
|
|
88
|
+
this.pending = Math.max(0, this.pending - 1)
|
|
89
|
+
})
|
|
85
90
|
return this.queue
|
|
86
91
|
}
|
|
87
92
|
|
|
@@ -127,6 +132,46 @@ export class EventStore {
|
|
|
127
132
|
}
|
|
128
133
|
}
|
|
129
134
|
|
|
135
|
+
function finiteNumbers(value, limit) {
|
|
136
|
+
if (!Array.isArray(value)) return []
|
|
137
|
+
const out = []
|
|
138
|
+
for (const item of value) {
|
|
139
|
+
if (typeof item === 'number' && Number.isFinite(item) && out.length < limit) out.push(item)
|
|
140
|
+
}
|
|
141
|
+
return out
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
/**
|
|
145
|
+
* Normalise a persisted MCP bucket tree. A hand-edited or corrupted window
|
|
146
|
+
* record (a null tool entry, a string where an object belongs) must never be
|
|
147
|
+
* able to crash the report surfaces, so every level is type-checked here.
|
|
148
|
+
*/
|
|
149
|
+
export function sanitizeMcp(value) {
|
|
150
|
+
if (!value || typeof value !== 'object' || Array.isArray(value)) return {}
|
|
151
|
+
const out = Object.create(null)
|
|
152
|
+
for (const [server, bucket] of Object.entries(value)) {
|
|
153
|
+
if (!bucket || typeof bucket !== 'object' || Array.isArray(bucket)) continue
|
|
154
|
+
const tools = Object.create(null)
|
|
155
|
+
for (const [tool, stats] of Object.entries(bucket.tools ?? {})) {
|
|
156
|
+
if (!stats || typeof stats !== 'object' || Array.isArray(stats)) continue
|
|
157
|
+
tools[tool] = {
|
|
158
|
+
calls: Number(stats.calls) || 0,
|
|
159
|
+
errors: Number(stats.errors) || 0,
|
|
160
|
+
ms: Number(stats.ms) || 0,
|
|
161
|
+
samples: finiteNumbers(stats.samples, 20),
|
|
162
|
+
}
|
|
163
|
+
}
|
|
164
|
+
out[server] = {
|
|
165
|
+
calls: Number(bucket.calls) || 0,
|
|
166
|
+
errors: Number(bucket.errors) || 0,
|
|
167
|
+
ms: Number(bucket.ms) || 0,
|
|
168
|
+
samples: finiteNumbers(bucket.samples, 40),
|
|
169
|
+
tools,
|
|
170
|
+
}
|
|
171
|
+
}
|
|
172
|
+
return out
|
|
173
|
+
}
|
|
174
|
+
|
|
130
175
|
/**
|
|
131
176
|
* Fold append-only events into per-skill invocation samples.
|
|
132
177
|
*
|
|
@@ -141,8 +186,17 @@ export function foldSamples(records) {
|
|
|
141
186
|
if (r && r.t === 'window' && typeof r.windowId === 'string') byWindow.set(r.windowId, r)
|
|
142
187
|
}
|
|
143
188
|
const bySkill = new Map()
|
|
189
|
+
// A hot reload drops the recorder's in-memory invocation dedupe, so the same
|
|
190
|
+
// injected /skill-name message can be recorded twice. Collapse it here on
|
|
191
|
+
// the durable read path (same session + message id + invocation origin).
|
|
192
|
+
const seenInvocations = new Set()
|
|
144
193
|
for (const r of records) {
|
|
145
194
|
if (!r || r.t !== 'skill' || typeof r.skill !== 'string') continue
|
|
195
|
+
if (r.via === 'invocation') {
|
|
196
|
+
const key = `${r.sessionId ?? ''}|${r.callId ?? ''}`
|
|
197
|
+
if (seenInvocations.has(key)) continue
|
|
198
|
+
seenInvocations.add(key)
|
|
199
|
+
}
|
|
146
200
|
const entry = bySkill.get(r.skill) ?? { skill: r.skill, samples: [], firstAt: r.at, lastAt: r.at }
|
|
147
201
|
const window = r.windowId ? byWindow.get(r.windowId) : undefined
|
|
148
202
|
entry.samples.push({
|
|
@@ -161,7 +215,7 @@ export function foldSamples(records) {
|
|
|
161
215
|
errorNames: Array.isArray(window.errorNames)
|
|
162
216
|
? window.errorNames.filter((name) => typeof name === 'string')
|
|
163
217
|
: [],
|
|
164
|
-
mcp:
|
|
218
|
+
mcp: sanitizeMcp(window.mcp),
|
|
165
219
|
origin: typeof window.origin === 'string' ? window.origin : null,
|
|
166
220
|
tokens:
|
|
167
221
|
window.tokens && typeof window.tokens === 'object'
|
package/lib/validity.js
CHANGED
|
@@ -75,7 +75,11 @@ export function buildValidity(report, options = {}) {
|
|
|
75
75
|
generatedAt: report?.generatedAt ?? null,
|
|
76
76
|
dataDir: report?.dataDir ?? null,
|
|
77
77
|
coverage: { skills: skills.length, withCI, provisional, actionable, explorable },
|
|
78
|
-
ranking: {
|
|
78
|
+
ranking: {
|
|
79
|
+
significant: ranked === true,
|
|
80
|
+
unknown: ranked === null || ranked === undefined,
|
|
81
|
+
significantReported: ranked !== undefined,
|
|
82
|
+
},
|
|
79
83
|
advice: adviceCounts,
|
|
80
84
|
ledger,
|
|
81
85
|
canSay,
|
|
@@ -93,7 +97,13 @@ export function renderValidityMarkdown(validity) {
|
|
|
93
97
|
'## Validity — what the evidence supports',
|
|
94
98
|
'',
|
|
95
99
|
`coverage: **${c.skills}** skill(s) · **${c.withCI}** with 95% intervals · **${c.provisional}** provisional · **${c.actionable}** with an actionable fix`,
|
|
96
|
-
`ranking: ${
|
|
100
|
+
`ranking: ${
|
|
101
|
+
validity.ranking.unknown
|
|
102
|
+
? 'not enough evidence (fewer than two 95% intervals)'
|
|
103
|
+
: validity.ranking.significant
|
|
104
|
+
? '**significant**'
|
|
105
|
+
: 'not significant (intervals overlap)'
|
|
106
|
+
}`,
|
|
97
107
|
`advice: ${validity.advice.critical} critical · ${validity.advice.warning} warning · ${validity.advice.info} info`,
|
|
98
108
|
`ledger: ${l.total} edit(s) screened — ${l.accept} accept · ${l.warn} warn · ${l.reject} reject` +
|
|
99
109
|
(Object.keys(l.dimensions).length > 0
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "dsh-skill-eval",
|
|
3
|
-
"version": "1.20.
|
|
3
|
+
"version": "1.20.3",
|
|
4
4
|
"description": "Skill execution telemetry & effectiveness scoring for DeepSeek Harness: records every skill load, folds the follow-up window into evidence, scores each skill and hands the agent concrete optimization advice.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "index.js",
|
package/scripts/accept.mjs
CHANGED
|
@@ -13,11 +13,12 @@ import { homedir } from 'node:os'
|
|
|
13
13
|
import path from 'node:path'
|
|
14
14
|
import process from 'node:process'
|
|
15
15
|
import { runAcceptance } from '../lib/accept.js'
|
|
16
|
-
import { dashboardBase, expectedVersion, resolveBaseUrl } from './lib.mjs'
|
|
16
|
+
import { dashboardBase, expectedVersion, parseCliArgs, resolveBaseUrl } from './lib.mjs'
|
|
17
17
|
|
|
18
18
|
const args = process.argv.slice(2)
|
|
19
|
-
const
|
|
20
|
-
const
|
|
19
|
+
const parsedArgs = parseCliArgs(args)
|
|
20
|
+
const baseArg = parsedArgs.baseArg
|
|
21
|
+
const dataDirOverride = parsedArgs.values['--data-dir'] ? path.resolve(parsedArgs.values['--data-dir']) : null
|
|
21
22
|
const resolved = await resolveBaseUrl(baseArg)
|
|
22
23
|
const baseUrl = resolved.baseUrl
|
|
23
24
|
const manifest = JSON.parse(await readFile(new URL('../package.json', import.meta.url), 'utf8'))
|
|
@@ -25,7 +26,7 @@ const currentVersion = expectedVersion(manifest.version)
|
|
|
25
26
|
const version = typeof manifest.version === 'string' ? manifest.version : null
|
|
26
27
|
|
|
27
28
|
async function findDataDir() {
|
|
28
|
-
if (
|
|
29
|
+
if (dataDirOverride) return dataDirOverride
|
|
29
30
|
const home = process.env.DSH_HOME || path.join(homedir(), '.dsh')
|
|
30
31
|
const root = path.join(home, 'skill-eval')
|
|
31
32
|
let candidates
|
|
@@ -158,10 +159,10 @@ function manifestVersion() {
|
|
|
158
159
|
async function readFiles() {
|
|
159
160
|
// The served report names the data directory it actually read; prefer it over
|
|
160
161
|
// scanning profiles, which can pick a different profile with a newer mtime.
|
|
161
|
-
let dataDir =
|
|
162
|
+
let dataDir = dataDirOverride
|
|
162
163
|
if (!dataDir) {
|
|
163
164
|
try {
|
|
164
|
-
const served =
|
|
165
|
+
const served = await fetchReport()
|
|
165
166
|
if (served && typeof served.dataDir === 'string' && served.dataDir.length > 0) dataDir = served.dataDir
|
|
166
167
|
} catch {
|
|
167
168
|
// fall back to the profile scan below
|
package/scripts/doctor.mjs
CHANGED
|
@@ -116,12 +116,18 @@ try {
|
|
|
116
116
|
}
|
|
117
117
|
add('locale-meta', localeOk, localeDetail)
|
|
118
118
|
|
|
119
|
-
// the shipped client.js must match its source parts
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
119
|
+
// the shipped client.js must match its source parts. A published tarball does
|
|
120
|
+
// not ship client/ (only the built client.js), so a missing parts.json means
|
|
121
|
+
// "packaged install" and there is nothing to compare against.
|
|
122
|
+
if (await exists('client/parts.json')) {
|
|
123
|
+
try {
|
|
124
|
+
await run(process.execPath, [path.join(root, 'scripts/build-client.mjs'), '--check'], { cwd: root })
|
|
125
|
+
add('client-build', true, 'client.js matches client/*.js')
|
|
126
|
+
} catch {
|
|
127
|
+
add('client-build', false, 'client.js is stale: run `node scripts/build-client.mjs`')
|
|
128
|
+
}
|
|
129
|
+
} else {
|
|
130
|
+
add('client-build', true, 'packaged install: client.js is prebuilt (client/ sources are not shipped)')
|
|
125
131
|
}
|
|
126
132
|
|
|
127
133
|
// the client half must reference the versioned data route it will fetch
|
|
@@ -160,6 +166,11 @@ const filesOk =
|
|
|
160
166
|
declaredFiles.includes('cordis.patch.yml') &&
|
|
161
167
|
declaredFiles.includes('icon.svg') &&
|
|
162
168
|
declaredFiles.includes('docs/*.md') &&
|
|
169
|
+
declaredFiles.includes('locale') &&
|
|
170
|
+
declaredFiles.includes('scripts') &&
|
|
171
|
+
declaredFiles.includes('README.md') &&
|
|
172
|
+
declaredFiles.includes('CHANGELOG.md') &&
|
|
173
|
+
declaredFiles.includes('LICENSE') &&
|
|
163
174
|
!declaredFiles.includes('docs')
|
|
164
175
|
add(
|
|
165
176
|
'package-files',
|
|
@@ -181,11 +192,35 @@ add(
|
|
|
181
192
|
)
|
|
182
193
|
if (process.argv.includes('--release')) {
|
|
183
194
|
const artifact = `dist/${manifest.name}-${manifest.version}.tgz`
|
|
184
|
-
|
|
185
|
-
'dist-artifact',
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
195
|
+
if (!(await exists(artifact))) {
|
|
196
|
+
add('dist-artifact', false, `run "npm run release:pack" to build ${artifact}`)
|
|
197
|
+
} else {
|
|
198
|
+
try {
|
|
199
|
+
// Opening the tarball is the only way to catch a files-list edit that
|
|
200
|
+
// ships a manifest but drops a runtime file (locale, scripts, lib).
|
|
201
|
+
const { stdout } = await run('tar', ['-tzf', path.join(root, artifact)])
|
|
202
|
+
const entries = stdout.split('\n').filter(Boolean)
|
|
203
|
+
const required = [
|
|
204
|
+
'package/index.js',
|
|
205
|
+
'package/client.js',
|
|
206
|
+
'package/cordis.patch.yml',
|
|
207
|
+
'package/locale/en.json',
|
|
208
|
+
'package/locale/zh.json',
|
|
209
|
+
'package/scripts/doctor.mjs',
|
|
210
|
+
]
|
|
211
|
+
const missing = required.filter((entry) => !entries.includes(entry))
|
|
212
|
+
const hasLib = entries.some((entry) => entry.startsWith('package/lib/') && entry.endsWith('.js'))
|
|
213
|
+
add(
|
|
214
|
+
'dist-artifact',
|
|
215
|
+
missing.length === 0 && hasLib,
|
|
216
|
+
missing.length === 0 && hasLib
|
|
217
|
+
? `${artifact} (${entries.length} entries)`
|
|
218
|
+
: `missing ${missing.join(', ') || 'lib/*.js'}`,
|
|
219
|
+
)
|
|
220
|
+
} catch (error) {
|
|
221
|
+
add('dist-artifact', false, error instanceof Error ? error.message : String(error))
|
|
222
|
+
}
|
|
223
|
+
}
|
|
189
224
|
}
|
|
190
225
|
|
|
191
226
|
const failed = rows.filter((row) => !row.ok)
|
package/scripts/lib.mjs
CHANGED
|
@@ -14,6 +14,41 @@ export async function readBridge() {
|
|
|
14
14
|
}
|
|
15
15
|
}
|
|
16
16
|
|
|
17
|
+
/**
|
|
18
|
+
* Parse CLI arguments for the terminal checks: flags that take a value must
|
|
19
|
+
* consume their value instead of leaking it into the positional base URL.
|
|
20
|
+
* Returns { baseArg, values } where values is keyed by the flag name.
|
|
21
|
+
*/
|
|
22
|
+
export function parseCliArgs(argv, flagsWithValues = ['--data-dir']) {
|
|
23
|
+
const args = Array.isArray(argv) ? argv : []
|
|
24
|
+
let baseArg = null
|
|
25
|
+
const values = {}
|
|
26
|
+
for (let i = 0; i < args.length; i += 1) {
|
|
27
|
+
const arg = String(args[i])
|
|
28
|
+
if (arg === '--') {
|
|
29
|
+
const next = args[i + 1]
|
|
30
|
+
if (next !== undefined && baseArg === null) baseArg = String(next)
|
|
31
|
+
break
|
|
32
|
+
}
|
|
33
|
+
if (arg.startsWith('--') && arg.includes('=')) {
|
|
34
|
+
const name = arg.slice(0, arg.indexOf('='))
|
|
35
|
+
if (flagsWithValues.includes(name)) values[name] = arg.slice(name.length + 1)
|
|
36
|
+
continue
|
|
37
|
+
}
|
|
38
|
+
if (flagsWithValues.includes(arg)) {
|
|
39
|
+
const value = args[i + 1]
|
|
40
|
+
if (value !== undefined) {
|
|
41
|
+
values[arg] = String(value)
|
|
42
|
+
i += 1
|
|
43
|
+
}
|
|
44
|
+
continue
|
|
45
|
+
}
|
|
46
|
+
if (arg.startsWith('--')) continue
|
|
47
|
+
if (baseArg === null) baseArg = arg
|
|
48
|
+
}
|
|
49
|
+
return { baseArg, values }
|
|
50
|
+
}
|
|
51
|
+
|
|
17
52
|
/** Base URL precedence: explicit argument > DSH_WEB_URL > relay bridge. */
|
|
18
53
|
export async function resolveBaseUrl(explicit) {
|
|
19
54
|
return pickBaseUrl(explicit, process.env.DSH_WEB_URL, await readBridge())
|
package/scripts/verify-live.mjs
CHANGED
|
@@ -27,7 +27,7 @@ const dashboard = dashboardBase()
|
|
|
27
27
|
const boardPath = `${dashboard}/board/${version}`
|
|
28
28
|
const configPath = `${boardPath}/config`
|
|
29
29
|
const rows = []
|
|
30
|
-
const add = (id, ok, detail) => rows.push({ id, ok, detail })
|
|
30
|
+
const add = (id, ok, detail, options = {}) => rows.push({ id, ok, detail, manual: options.manual === true })
|
|
31
31
|
|
|
32
32
|
const get = (pathname, expectType) => httpGet(base, pathname, { expectType })
|
|
33
33
|
|
|
@@ -107,7 +107,7 @@ try {
|
|
|
107
107
|
}
|
|
108
108
|
add(
|
|
109
109
|
'turn-boards',
|
|
110
|
-
|
|
110
|
+
turns.length > 0,
|
|
111
111
|
turns.length > 0
|
|
112
112
|
? `${turns.length} board(s), newest ${turns.sort().at(-1)}`
|
|
113
113
|
: 'none yet — load a skill, finish the turn, then re-run',
|
|
@@ -154,15 +154,25 @@ add(
|
|
|
154
154
|
'manual:native-seats',
|
|
155
155
|
true,
|
|
156
156
|
'open the sidebar Skill eval panel icon (board) and Settings → Plugins → dsh-skill-eval (bundle config)',
|
|
157
|
+
{ manual: true },
|
|
157
158
|
)
|
|
158
|
-
add('manual:turn-strip', true, 'load a skill in a turn; its result strip shows score, next fix and edit history'
|
|
159
|
+
add('manual:turn-strip', true, 'load a skill in a turn; its result strip shows score, next fix and edit history', {
|
|
160
|
+
manual: true,
|
|
161
|
+
})
|
|
159
162
|
|
|
160
|
-
const failed = rows.filter((row) => !row.ok)
|
|
163
|
+
const failed = rows.filter((row) => !row.ok && !row.manual)
|
|
161
164
|
console.log(`dsh-skill-eval verify — ${base} (version ${version})`)
|
|
162
|
-
for (const row of rows)
|
|
165
|
+
for (const row of rows) {
|
|
166
|
+
const state = row.manual ? 'INFO' : row.ok ? 'PASS' : 'FAIL'
|
|
167
|
+
console.log(` [${state}] ${row.id.padEnd(18)} ${row.detail}`)
|
|
168
|
+
}
|
|
163
169
|
console.log('')
|
|
164
170
|
if (failed.length > 0) {
|
|
165
171
|
console.log(`FAIL (${failed.length}) — restart DeepSeek Harness if versioned-json reports text/html.`)
|
|
166
172
|
process.exit(1)
|
|
167
173
|
}
|
|
168
|
-
|
|
174
|
+
const checks = rows.filter((row) => !row.manual).length
|
|
175
|
+
const manual = rows.filter((row) => row.manual).length
|
|
176
|
+
console.log(
|
|
177
|
+
`PASS — the running host serves this generation and the product surfaces are reachable (${checks} check(s), ${manual} manual).`,
|
|
178
|
+
)
|