dsh-skill-eval 0.0.0-stage → 1.20.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +675 -0
- package/LICENSE +21 -0
- package/README.md +200 -2
- package/README.zh.md +157 -0
- package/client.js +2637 -0
- package/cordis.patch.yml +9 -0
- package/docs/ACCEPTANCE.md +71 -0
- package/docs/AUDIT.md +64 -0
- package/docs/CODE-REVIEW.md +49 -0
- package/docs/COMPARISON.md +33 -0
- package/docs/DETAIL.md +463 -0
- package/docs/DEV-RELOAD.md +114 -0
- package/docs/ENTRYPOINTS.md +65 -0
- package/docs/RELEASE.md +44 -0
- package/docs/RRSI-NOTES.md +129 -0
- package/docs/SCENARIOS.md +91 -0
- package/docs/UI-REVIEW.md +115 -0
- package/docs/WORKFLOW.md +108 -0
- package/icon.svg +9 -0
- package/index.js +1111 -0
- package/lib/accept.js +350 -0
- package/lib/advice.js +547 -0
- package/lib/critic.js +101 -0
- package/lib/dev.js +148 -0
- package/lib/history.js +140 -0
- package/lib/html.js +8 -0
- package/lib/inspect.js +142 -0
- package/lib/integrity.js +46 -0
- package/lib/loop.js +195 -0
- package/lib/random.js +29 -0
- package/lib/recorder.js +634 -0
- package/lib/render.js +582 -0
- package/lib/report.js +646 -0
- package/lib/score.js +525 -0
- package/lib/settings.js +396 -0
- package/lib/snapshot.js +166 -0
- package/lib/steps.js +141 -0
- package/lib/store.js +186 -0
- package/lib/validity.js +115 -0
- package/locale/en.json +6 -0
- package/locale/zh.json +6 -0
- package/package.json +96 -4
- package/scripts/accept.mjs +226 -0
- package/scripts/build-client.mjs +59 -0
- package/scripts/build.mjs +81 -0
- package/scripts/doctor.mjs +199 -0
- package/scripts/lib.mjs +52 -0
- package/scripts/verify-live.mjs +168 -0
package/index.js
ADDED
|
@@ -0,0 +1,1111 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* dsh-skill-eval — skill execution telemetry, scoring and optimization advice.
|
|
3
|
+
*
|
|
4
|
+
* Host-plane plugin. It records every `skill` tool load plus the stretch of work
|
|
5
|
+
* that follows it, then scores each skill 0-100 on two axes:
|
|
6
|
+
*
|
|
7
|
+
* - static authoring quality (description, trigger cues, progressive
|
|
8
|
+
* disclosure, size, examples) — computed from the live skill catalog
|
|
9
|
+
* - live execution evidence (load failures, post-load tool errors, user
|
|
10
|
+
* corrections, ignored loads, sibling collisions, cost) — folded from the
|
|
11
|
+
* append-only event store
|
|
12
|
+
*
|
|
13
|
+
* The result is exposed three ways: the `skill_eval_report` tool (so the agent
|
|
14
|
+
* can read its own numbers and act on the cited fixes), a `/skill-eval` slash
|
|
15
|
+
* command, and an HTML dashboard served by the host web server.
|
|
16
|
+
*
|
|
17
|
+
* @module dsh-skill-eval
|
|
18
|
+
*/
|
|
19
|
+
import { readFileSync } from 'node:fs'
|
|
20
|
+
import path from 'node:path'
|
|
21
|
+
import z from '@deepseek-ai/schemastery'
|
|
22
|
+
import { createAnalyzer } from './lib/report.js'
|
|
23
|
+
import { DEFAULT_THRESHOLDS, DEFAULT_WEIGHTS } from './lib/score.js'
|
|
24
|
+
import { createRecorder } from './lib/recorder.js'
|
|
25
|
+
import { checkInProcessReport } from './lib/accept.js'
|
|
26
|
+
import { reviewSkillEdit } from './lib/critic.js'
|
|
27
|
+
import { buildLoopPlan, buildLoopPlans, renderLoopBatchMarkdown, renderLoopMarkdown } from './lib/loop.js'
|
|
28
|
+
import { buildDevReport, renderDevMarkdown } from './lib/dev.js'
|
|
29
|
+
import { buildHistory, buildImpact, renderHistoryMarkdown, renderImpactMarkdown } from './lib/history.js'
|
|
30
|
+
import { resolveDataDir } from './lib/store.js'
|
|
31
|
+
import { hashHex } from './lib/random.js'
|
|
32
|
+
import {
|
|
33
|
+
applySettings,
|
|
34
|
+
FIELD_ZH,
|
|
35
|
+
loadSettingsSync,
|
|
36
|
+
normalizeSettings,
|
|
37
|
+
parseFormSettings,
|
|
38
|
+
renderConfigHtml,
|
|
39
|
+
saveSettings,
|
|
40
|
+
SETTINGS_FIELDS,
|
|
41
|
+
SETTINGS_FILE,
|
|
42
|
+
} from './lib/settings.js'
|
|
43
|
+
import { renderValidityMarkdown } from './lib/validity.js'
|
|
44
|
+
import { createInspectProvider } from './lib/inspect.js'
|
|
45
|
+
import { createSnapshots, listTurnReports, renderTurnReportsMarkdown } from './lib/snapshot.js'
|
|
46
|
+
import {
|
|
47
|
+
renderHtml,
|
|
48
|
+
resolveLang,
|
|
49
|
+
renderMcpMarkdown,
|
|
50
|
+
renderRunsMarkdown,
|
|
51
|
+
renderSkillMarkdown,
|
|
52
|
+
renderStandardMarkdown,
|
|
53
|
+
renderSummaryMarkdown,
|
|
54
|
+
} from './lib/render.js'
|
|
55
|
+
|
|
56
|
+
export const name = 'skill-eval'
|
|
57
|
+
|
|
58
|
+
/**
|
|
59
|
+
* The running package version. The data route is registered both canonically
|
|
60
|
+
* and under this version, because a plugin reload cannot reclaim a route the
|
|
61
|
+
* previous generation already registered: the versioned path is new on every
|
|
62
|
+
* release, so a hot-reloaded plugin can still serve current data.
|
|
63
|
+
*/
|
|
64
|
+
function packageVersion() {
|
|
65
|
+
try {
|
|
66
|
+
const manifest = new URL('./package.json', import.meta.url)
|
|
67
|
+
const parsed = JSON.parse(readFileSync(manifest, 'utf8'))
|
|
68
|
+
return typeof parsed.version === 'string' ? parsed.version : null
|
|
69
|
+
} catch {
|
|
70
|
+
return null
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
const PLUGIN_VERSION = packageVersion()
|
|
75
|
+
|
|
76
|
+
/** Per-apply diagnostics surfaced by the `dev` view (the restart-skew detector). */
|
|
77
|
+
const DIAGNOSTICS = { routes: [], tools: [], inspectProvider: false, warnings: [], settingsFile: null, settings: {} }
|
|
78
|
+
|
|
79
|
+
// cordis 4 resolves `inject` as an array or a name -> config map, with no
|
|
80
|
+
// required/optional split. Optional services are attached with `ctx.inject(...)`
|
|
81
|
+
// below, which starts the callback only once that service is available.
|
|
82
|
+
export const inject = ['tools', 'skills']
|
|
83
|
+
|
|
84
|
+
export const Config = z.object({
|
|
85
|
+
enabled: z.boolean().default(true).description('record skill telemetry and expose the report tool'),
|
|
86
|
+
dataDir: z.string().description('override the telemetry directory (default: $DSH_HOME/skill-eval/$DSH_PROFILE)'),
|
|
87
|
+
windowToolCalls: z
|
|
88
|
+
.natural()
|
|
89
|
+
.min(1)
|
|
90
|
+
.max(200)
|
|
91
|
+
.default(12)
|
|
92
|
+
.description('tool calls after a skill load that still count as evidence for that load'),
|
|
93
|
+
windowTtlMs: z
|
|
94
|
+
.natural()
|
|
95
|
+
.min(60_000)
|
|
96
|
+
.max(86_400_000)
|
|
97
|
+
.default(30 * 60_000)
|
|
98
|
+
.description('how long an open skill window keeps waiting for follow-up work'),
|
|
99
|
+
minSamples: z.natural().min(1).max(100).default(3).description('loads needed before a score stops being provisional'),
|
|
100
|
+
weights: z
|
|
101
|
+
.object({
|
|
102
|
+
static: z.natural().min(0).max(100).default(30).description('authoring quality weight'),
|
|
103
|
+
reliability: z.natural().min(0).max(100).default(25).description('load success and error weight'),
|
|
104
|
+
effectiveness: z.natural().min(0).max(100).default(30).description('user correction weight'),
|
|
105
|
+
efficiency: z.natural().min(0).max(100).default(15).description('cost weight'),
|
|
106
|
+
})
|
|
107
|
+
.description('relative weights; normalised to 100% automatically, set one to 0 to drop a dimension'),
|
|
108
|
+
thresholds: z
|
|
109
|
+
.object({
|
|
110
|
+
excellent: z.natural().min(1).max(100).default(90).description('A grade floor'),
|
|
111
|
+
good: z.natural().min(1).max(100).default(80).description('B grade floor'),
|
|
112
|
+
pass: z.natural().min(1).max(100).default(70).description('C grade floor'),
|
|
113
|
+
})
|
|
114
|
+
.description('grade bands; D stays above 60 and F below it'),
|
|
115
|
+
maxAdvice: z.natural().min(1).max(10).default(3).description('advice items returned per skill in the summary tool'),
|
|
116
|
+
tokenBudgetPerLoad: z
|
|
117
|
+
.natural()
|
|
118
|
+
.min(1000)
|
|
119
|
+
.default(20000)
|
|
120
|
+
.description('billable policy tokens a single skill load may cost before efficiency and advice react'),
|
|
121
|
+
cacheReadWeight: z
|
|
122
|
+
.number()
|
|
123
|
+
.min(0)
|
|
124
|
+
.max(1)
|
|
125
|
+
.default(0)
|
|
126
|
+
.description(
|
|
127
|
+
'how much a cached input token counts against the token budget (cache reads are near-free by default)',
|
|
128
|
+
),
|
|
129
|
+
countTokens: z.boolean().default(true).description('attribute real provider usage to the open skill windows'),
|
|
130
|
+
editBudgetMax: z.natural().min(1).max(10).default(4).description('edits a first-round skill may change at once'),
|
|
131
|
+
editBudgetMin: z.natural().min(1).max(10).default(1).description('edits allowed after the budget has annealed'),
|
|
132
|
+
editBudgetHorizon: z
|
|
133
|
+
.natural()
|
|
134
|
+
.min(1)
|
|
135
|
+
.default(6)
|
|
136
|
+
.description('accepted edits over which the budget anneals to its minimum'),
|
|
137
|
+
noveltyWindow: z
|
|
138
|
+
.natural()
|
|
139
|
+
.min(2)
|
|
140
|
+
.max(10)
|
|
141
|
+
.default(2)
|
|
142
|
+
.description('consecutive same-dimension edits that trigger an exploration directive'),
|
|
143
|
+
pruneMinLoads: z
|
|
144
|
+
.natural()
|
|
145
|
+
.min(1)
|
|
146
|
+
.default(6)
|
|
147
|
+
.description('loads required before a consistently low score becomes a prune recommendation'),
|
|
148
|
+
pruneMaxScore: z
|
|
149
|
+
.natural()
|
|
150
|
+
.min(0)
|
|
151
|
+
.max(100)
|
|
152
|
+
.default(60)
|
|
153
|
+
.description('score below which a well-sampled skill is a prune candidate'),
|
|
154
|
+
registerReviewTool: z
|
|
155
|
+
.boolean()
|
|
156
|
+
.default(true)
|
|
157
|
+
.description('register skill_eval_review (leakage critic + edit ledger)'),
|
|
158
|
+
adviceNoiseFloor: z
|
|
159
|
+
.boolean()
|
|
160
|
+
.default(true)
|
|
161
|
+
.description('only keep a rate-based advice severity when its bootstrap interval clears the threshold'),
|
|
162
|
+
turnReports: z.boolean().default(true).description('write one board per settled turn under reports/turns/'),
|
|
163
|
+
turnReportsKeep: z.natural().min(1).default(50).description('how many per-turn boards to keep'),
|
|
164
|
+
reportOnTurnEnd: z
|
|
165
|
+
.boolean()
|
|
166
|
+
.default(true)
|
|
167
|
+
.description('write reports/latest.md + latest.json after a turn settles (never enters the model context)'),
|
|
168
|
+
reportDir: z.string().description('override the snapshot directory (default: <dataDir>/reports)'),
|
|
169
|
+
ciResamples: z
|
|
170
|
+
.natural()
|
|
171
|
+
.min(0)
|
|
172
|
+
.max(2000)
|
|
173
|
+
.default(200)
|
|
174
|
+
.description('bootstrap resamples for the 95% score interval (0 disables it)'),
|
|
175
|
+
mcpErrorRatePercent: z
|
|
176
|
+
.natural()
|
|
177
|
+
.min(1)
|
|
178
|
+
.max(100)
|
|
179
|
+
.default(20)
|
|
180
|
+
.description('MCP server error-rate percent that raises a critical MCP fix'),
|
|
181
|
+
mcpToolErrorRatePercent: z
|
|
182
|
+
.natural()
|
|
183
|
+
.min(1)
|
|
184
|
+
.max(100)
|
|
185
|
+
.default(30)
|
|
186
|
+
.description('single MCP tool error-rate percent that raises a warning'),
|
|
187
|
+
mcpToolP95Ms: z
|
|
188
|
+
.natural()
|
|
189
|
+
.min(100)
|
|
190
|
+
.default(5000)
|
|
191
|
+
.description('per-tool MCP p95 latency that raises the mcp-tool-slow warning'),
|
|
192
|
+
mcpP95Ms: z.natural().min(100).default(3000).description('MCP p95 latency in ms that raises a slow-server warning'),
|
|
193
|
+
inspectProvider: z
|
|
194
|
+
.boolean()
|
|
195
|
+
.default(true)
|
|
196
|
+
.description('expose the eval data as a Cordis Inspect provider for developer mode'),
|
|
197
|
+
promptHint: z
|
|
198
|
+
.boolean()
|
|
199
|
+
.default(true)
|
|
200
|
+
.description('add the one-line usage hint to the system prompt (the only per-turn token cost, about 60 tokens)'),
|
|
201
|
+
registerTool: z.boolean().default(true).description('register the skill_eval_report tool'),
|
|
202
|
+
registerCommand: z.boolean().default(true).description('register the /skill-eval command'),
|
|
203
|
+
webDashboard: z
|
|
204
|
+
.boolean()
|
|
205
|
+
.default(true)
|
|
206
|
+
.description('serve the HTML dashboard and the JSON route the in-app popup reads'),
|
|
207
|
+
dashboardPath: z
|
|
208
|
+
.string()
|
|
209
|
+
.default('/skill-eval')
|
|
210
|
+
.description('path the dashboard is served on (change it when a reload could not reclaim the old route)'),
|
|
211
|
+
})
|
|
212
|
+
|
|
213
|
+
const REPORT_PARAMETERS = {
|
|
214
|
+
type: 'object',
|
|
215
|
+
properties: {
|
|
216
|
+
skill: {
|
|
217
|
+
type: 'string',
|
|
218
|
+
description: 'report a single skill by name; omit for the cross-skill summary',
|
|
219
|
+
},
|
|
220
|
+
format: {
|
|
221
|
+
type: 'string',
|
|
222
|
+
enum: ['markdown', 'json'],
|
|
223
|
+
description: 'markdown (default) for humans and models, json for post-processing',
|
|
224
|
+
},
|
|
225
|
+
all: {
|
|
226
|
+
type: 'boolean',
|
|
227
|
+
description: 'with view:"loop": return the plan for every skill that has an actionable fix',
|
|
228
|
+
},
|
|
229
|
+
view: {
|
|
230
|
+
type: 'string',
|
|
231
|
+
enum: ['summary', 'mcp', 'runs', 'standard', 'loop', 'validity', 'history', 'impact', 'dev', 'turns'],
|
|
232
|
+
description:
|
|
233
|
+
'summary (default) scores every skill; mcp debugs MCP servers; runs lists recent sessions; standard shows the grading configuration; loop shows the next guided optimization step (add all:true for every actionable skill)',
|
|
234
|
+
},
|
|
235
|
+
},
|
|
236
|
+
additionalProperties: false,
|
|
237
|
+
}
|
|
238
|
+
|
|
239
|
+
const REPORT_OUTPUT = {
|
|
240
|
+
schema: { type: 'string', description: 'markdown or JSON report text' },
|
|
241
|
+
render(_args, value) {
|
|
242
|
+
return [{ type: 'text', text: typeof value === 'string' ? value : JSON.stringify(value, null, 2) }]
|
|
243
|
+
},
|
|
244
|
+
}
|
|
245
|
+
|
|
246
|
+
/** Run a disposer without letting teardown failures escape into the host. */
|
|
247
|
+
function safe(dispose) {
|
|
248
|
+
return () => {
|
|
249
|
+
try {
|
|
250
|
+
dispose()
|
|
251
|
+
} catch {
|
|
252
|
+
// intentionally ignored: teardown failures must not escape into the host
|
|
253
|
+
}
|
|
254
|
+
}
|
|
255
|
+
}
|
|
256
|
+
|
|
257
|
+
function log(ctx, level, message) {
|
|
258
|
+
if (DIAGNOSTICS.warnings.length < 50) DIAGNOSTICS.warnings.push(`${level}: ${message}`)
|
|
259
|
+
try {
|
|
260
|
+
const logger = typeof ctx.logger === 'function' ? ctx.logger('skill-eval') : ctx.logger
|
|
261
|
+
logger?.[level]?.(message)
|
|
262
|
+
} catch {
|
|
263
|
+
// logging is best effort
|
|
264
|
+
}
|
|
265
|
+
// `DSH_SKILL_EVAL_DEBUG=1` mirrors diagnostics to stderr, which is where a
|
|
266
|
+
// headless host (and a terminal launch) actually shows them.
|
|
267
|
+
try {
|
|
268
|
+
if (level === 'warn' || process.env.DSH_SKILL_EVAL_DEBUG === '1') {
|
|
269
|
+
console.warn('[skill-eval] ' + message)
|
|
270
|
+
}
|
|
271
|
+
} catch {
|
|
272
|
+
// intentionally ignored: stderr mirror is best effort
|
|
273
|
+
}
|
|
274
|
+
}
|
|
275
|
+
|
|
276
|
+
const VIEWS = ['summary', 'mcp', 'runs', 'standard', 'loop', 'validity', 'history', 'impact', 'dev', 'turns']
|
|
277
|
+
|
|
278
|
+
/** Read the files the acceptance check needs, best effort. */
|
|
279
|
+
async function acceptanceFiles(store, readFile) {
|
|
280
|
+
const dir = store.dir
|
|
281
|
+
const eventsPath = path.join(dir, 'events.jsonl')
|
|
282
|
+
const snapshotMarkdownPath = path.join(dir, 'reports', 'latest.md')
|
|
283
|
+
const snapshotJsonPath = path.join(dir, 'reports', 'latest.json')
|
|
284
|
+
let events = []
|
|
285
|
+
try {
|
|
286
|
+
const raw = await readFile(eventsPath, 'utf8')
|
|
287
|
+
for (const line of raw.split('\n')) {
|
|
288
|
+
const trimmed = line.trim()
|
|
289
|
+
if (!trimmed) continue
|
|
290
|
+
try {
|
|
291
|
+
events.push(JSON.parse(trimmed))
|
|
292
|
+
} catch {
|
|
293
|
+
// torn write
|
|
294
|
+
}
|
|
295
|
+
}
|
|
296
|
+
} catch {
|
|
297
|
+
events = []
|
|
298
|
+
}
|
|
299
|
+
const optional = async (file) => {
|
|
300
|
+
try {
|
|
301
|
+
return await readFile(file, 'utf8')
|
|
302
|
+
} catch {
|
|
303
|
+
return null
|
|
304
|
+
}
|
|
305
|
+
}
|
|
306
|
+
return {
|
|
307
|
+
eventsPath,
|
|
308
|
+
snapshotMarkdownPath,
|
|
309
|
+
snapshotJsonPath,
|
|
310
|
+
events,
|
|
311
|
+
snapshotText: await optional(snapshotMarkdownPath),
|
|
312
|
+
snapshotJsonText: await optional(snapshotJsonPath),
|
|
313
|
+
}
|
|
314
|
+
}
|
|
315
|
+
|
|
316
|
+
/** Read `?lang=` from a raw request URL without needing an origin. */
|
|
317
|
+
function requestLang(req) {
|
|
318
|
+
const raw = typeof req?.url === 'string' ? req.url : ''
|
|
319
|
+
const index = raw.indexOf('?')
|
|
320
|
+
if (index === -1) return null
|
|
321
|
+
try {
|
|
322
|
+
return new URLSearchParams(raw.slice(index + 1)).get('lang')
|
|
323
|
+
} catch {
|
|
324
|
+
return null
|
|
325
|
+
}
|
|
326
|
+
}
|
|
327
|
+
|
|
328
|
+
function renderAcceptance(acceptance) {
|
|
329
|
+
const lines = ['## In-process acceptance', '']
|
|
330
|
+
for (const item of acceptance.results) {
|
|
331
|
+
lines.push(`- **${item.ok ? 'PASS' : 'FAIL'}** \`${item.id}\` — ${item.detail}`)
|
|
332
|
+
}
|
|
333
|
+
lines.push('', acceptance.summary)
|
|
334
|
+
return lines.join('\n')
|
|
335
|
+
}
|
|
336
|
+
|
|
337
|
+
async function runReport(analyzer, args, config) {
|
|
338
|
+
const format = args?.format === 'json' ? 'json' : 'markdown'
|
|
339
|
+
const view = VIEWS.includes(args?.view) ? args.view : 'summary'
|
|
340
|
+
if (view === 'turns') {
|
|
341
|
+
const reportsDir = config.reportDir ? String(config.reportDir) : path.join(analyzer.store.dir, 'reports')
|
|
342
|
+
const rows = await listTurnReports(reportsDir)
|
|
343
|
+
return format === 'json' ? JSON.stringify({ turns: rows }, null, 2) : renderTurnReportsMarkdown(rows)
|
|
344
|
+
}
|
|
345
|
+
if (view === 'dev') {
|
|
346
|
+
const records = await analyzer.store.readAll()
|
|
347
|
+
const diskVersion = await import('node:fs/promises').then((fs) =>
|
|
348
|
+
fs.readFile(new URL('./package.json', import.meta.url), 'utf8'),
|
|
349
|
+
)
|
|
350
|
+
let parsedDiskVersion
|
|
351
|
+
try {
|
|
352
|
+
parsedDiskVersion = JSON.parse(diskVersion).version ?? null
|
|
353
|
+
} catch {
|
|
354
|
+
parsedDiskVersion = null
|
|
355
|
+
}
|
|
356
|
+
const dev = buildDevReport({
|
|
357
|
+
moduleVersion: PLUGIN_VERSION ?? null,
|
|
358
|
+
diskVersion: parsedDiskVersion,
|
|
359
|
+
dataDir: analyzer.store.dir,
|
|
360
|
+
settingsFile: DIAGNOSTICS.settingsFile,
|
|
361
|
+
settings: DIAGNOSTICS.settings,
|
|
362
|
+
config: config,
|
|
363
|
+
routes: DIAGNOSTICS.routes,
|
|
364
|
+
records,
|
|
365
|
+
store: { dir: analyzer.store.dir, failed: analyzer.store.failed, queue: analyzer.store.queue },
|
|
366
|
+
tools: DIAGNOSTICS.tools,
|
|
367
|
+
inspectProvider: DIAGNOSTICS.inspectProvider ? 'skill-eval' : null,
|
|
368
|
+
warnings: DIAGNOSTICS.warnings,
|
|
369
|
+
})
|
|
370
|
+
return format === 'json' ? JSON.stringify(dev, null, 2) : renderDevMarkdown(dev)
|
|
371
|
+
}
|
|
372
|
+
if (view === 'history' || view === 'impact') {
|
|
373
|
+
const skill = typeof args?.skill === 'string' ? args.skill : ''
|
|
374
|
+
if (!skill) return `view:"${view}" needs a skill name, e.g. {"view":"${view}","skill":"pdf-tools"}`
|
|
375
|
+
const records = await analyzer.store.readAll()
|
|
376
|
+
const payload = view === 'history' ? buildHistory(records, skill) : buildImpact(records, skill)
|
|
377
|
+
if (format === 'json') return JSON.stringify(payload, null, 2)
|
|
378
|
+
return view === 'history' ? renderHistoryMarkdown(payload) : renderImpactMarkdown(payload)
|
|
379
|
+
}
|
|
380
|
+
if (view === 'validity') {
|
|
381
|
+
const report = await analyzer.build()
|
|
382
|
+
return format === 'json' ? JSON.stringify(report.validity, null, 2) : renderValidityMarkdown(report.validity)
|
|
383
|
+
}
|
|
384
|
+
if (view === 'loop') {
|
|
385
|
+
const report = await analyzer.build()
|
|
386
|
+
if (args?.all === true || args?.skill === 'all') {
|
|
387
|
+
const plans = buildLoopPlans(report, {
|
|
388
|
+
editBudgetMax: config.editBudgetMax ?? 4,
|
|
389
|
+
reloads: report.scoring?.minSamples ?? 3,
|
|
390
|
+
})
|
|
391
|
+
return format === 'json' ? JSON.stringify({ plans }, null, 2) : renderLoopBatchMarkdown(plans)
|
|
392
|
+
}
|
|
393
|
+
const plan = buildLoopPlan(report, {
|
|
394
|
+
skill: typeof args?.skill === 'string' ? args.skill : undefined,
|
|
395
|
+
editBudgetMax: config.editBudgetMax ?? 4,
|
|
396
|
+
reloads: report.scoring?.minSamples ?? 3,
|
|
397
|
+
})
|
|
398
|
+
return format === 'json' ? JSON.stringify(plan, null, 2) : renderLoopMarkdown(plan)
|
|
399
|
+
}
|
|
400
|
+
if (view === 'mcp' || view === 'runs' || view === 'standard') {
|
|
401
|
+
const report = await analyzer.build()
|
|
402
|
+
if (format === 'json') {
|
|
403
|
+
return JSON.stringify(
|
|
404
|
+
view === 'mcp'
|
|
405
|
+
? { mcpServers: report.mcpServers }
|
|
406
|
+
: view === 'runs'
|
|
407
|
+
? { runs: report.runs }
|
|
408
|
+
: { scoring: report.scoring },
|
|
409
|
+
null,
|
|
410
|
+
2,
|
|
411
|
+
)
|
|
412
|
+
}
|
|
413
|
+
return view === 'mcp'
|
|
414
|
+
? renderMcpMarkdown(report)
|
|
415
|
+
: view === 'runs'
|
|
416
|
+
? renderRunsMarkdown(report)
|
|
417
|
+
: renderStandardMarkdown(report)
|
|
418
|
+
}
|
|
419
|
+
if (args?.skill) {
|
|
420
|
+
const entry = await analyzer.buildOne(String(args.skill))
|
|
421
|
+
if (!entry) {
|
|
422
|
+
const known = (await analyzer.build()).skills.map((s) => s.skill)
|
|
423
|
+
return `No evidence recorded for skill \`${args.skill}\`.\n\nSkills with recorded loads: ${
|
|
424
|
+
known.length > 0 ? known.map((n) => `\`${n}\``).join(', ') : '(none)'
|
|
425
|
+
}`
|
|
426
|
+
}
|
|
427
|
+
if (format === 'json') return JSON.stringify(entry, null, 2)
|
|
428
|
+
return renderSkillMarkdown({ ...entry, advice: entry.advice })
|
|
429
|
+
}
|
|
430
|
+
const report = await analyzer.build()
|
|
431
|
+
if (format === 'json') return JSON.stringify(report, null, 2)
|
|
432
|
+
return renderSummaryMarkdown({
|
|
433
|
+
...report,
|
|
434
|
+
skills: report.skills.map((s) => ({ ...s, topAdvice: s.topAdvice.slice(0, config.maxAdvice ?? 3) })),
|
|
435
|
+
})
|
|
436
|
+
}
|
|
437
|
+
|
|
438
|
+
/**
|
|
439
|
+
* Resolve one skill inside an agent scope.
|
|
440
|
+
*
|
|
441
|
+
* The catalog is layered: reads only include the user and project layers when
|
|
442
|
+
* the agent is passed as `scope`, so a plain `skills.list()` misses exactly the
|
|
443
|
+
* skills this plugin is measuring. `cwd` widens project-level discovery.
|
|
444
|
+
*/
|
|
445
|
+
function makeResolveSkill(ctx) {
|
|
446
|
+
return async (skillName, agent, signal) => {
|
|
447
|
+
const attempts = [
|
|
448
|
+
{ scope: agent, signal },
|
|
449
|
+
{ scope: agent, signal, cwd: process.cwd() },
|
|
450
|
+
]
|
|
451
|
+
for (const options of attempts) {
|
|
452
|
+
try {
|
|
453
|
+
const summary = (await ctx.skills.list(options)).find((s) => s.name === skillName)
|
|
454
|
+
if (!summary) continue
|
|
455
|
+
const definition = await ctx.skills.get(skillName, options)
|
|
456
|
+
return { summary, definition }
|
|
457
|
+
} catch {
|
|
458
|
+
// try the next lookup shape
|
|
459
|
+
}
|
|
460
|
+
}
|
|
461
|
+
return null
|
|
462
|
+
}
|
|
463
|
+
}
|
|
464
|
+
|
|
465
|
+
export function apply(ctx, config) {
|
|
466
|
+
const cfg = { ...config }
|
|
467
|
+
if (cfg.enabled === false) return
|
|
468
|
+
|
|
469
|
+
// Fill the same defaults the schema would, so the settings API can report
|
|
470
|
+
// effective values even when it is called from a bare config object.
|
|
471
|
+
cfg.weights = { ...DEFAULT_WEIGHTS, ...(cfg.weights ?? {}) }
|
|
472
|
+
cfg.thresholds = { ...DEFAULT_THRESHOLDS, ...(cfg.thresholds ?? {}) }
|
|
473
|
+
// Settings saved from the browser page win over the profile config; they are
|
|
474
|
+
// applied before the analyzer reads the object so a reload and a live edit
|
|
475
|
+
// behave the same way.
|
|
476
|
+
DIAGNOSTICS.routes = []
|
|
477
|
+
DIAGNOSTICS.tools = []
|
|
478
|
+
DIAGNOSTICS.warnings = []
|
|
479
|
+
DIAGNOSTICS.inspectProvider = false
|
|
480
|
+
let settingsDir = null
|
|
481
|
+
try {
|
|
482
|
+
settingsDir = resolveDataDir(cfg)
|
|
483
|
+
const saved = loadSettingsSync(settingsDir)
|
|
484
|
+
applySettings(cfg, saved)
|
|
485
|
+
DIAGNOSTICS.settingsFile = settingsDir ? path.join(settingsDir, SETTINGS_FILE) : null
|
|
486
|
+
DIAGNOSTICS.settings = saved ?? {}
|
|
487
|
+
} catch (error) {
|
|
488
|
+
log(ctx, 'warn', `skill-eval settings load failed: ${error instanceof Error ? error.message : String(error)}`)
|
|
489
|
+
}
|
|
490
|
+
|
|
491
|
+
const analyzer = createAnalyzer(ctx, cfg)
|
|
492
|
+
analyzer.store.onError = (message) => {
|
|
493
|
+
DIAGNOSTICS.warnings.push(`store write failed: ${message}`)
|
|
494
|
+
log(ctx, 'warn', `skill-eval store write failed: ${message}`)
|
|
495
|
+
}
|
|
496
|
+
|
|
497
|
+
const recorder = createRecorder(ctx, cfg, {
|
|
498
|
+
store: analyzer.store,
|
|
499
|
+
resolveSkill: makeResolveSkill(ctx),
|
|
500
|
+
})
|
|
501
|
+
|
|
502
|
+
ctx.effect(
|
|
503
|
+
() => () => {
|
|
504
|
+
try {
|
|
505
|
+
void recorder.dispose()
|
|
506
|
+
} catch {
|
|
507
|
+
// best effort on shutdown
|
|
508
|
+
}
|
|
509
|
+
},
|
|
510
|
+
'skill-eval: recorder',
|
|
511
|
+
)
|
|
512
|
+
|
|
513
|
+
// A settled turn writes the aggregated report to disk. This is the "result
|
|
514
|
+
// after every run" surface: it costs zero tokens because it never touches the
|
|
515
|
+
// model context.
|
|
516
|
+
const snapshots =
|
|
517
|
+
cfg.reportOnTurnEnd === false
|
|
518
|
+
? null
|
|
519
|
+
: createSnapshots({
|
|
520
|
+
analyzer,
|
|
521
|
+
dir: cfg.reportDir ? String(cfg.reportDir) : path.join(analyzer.store.dir, 'reports'),
|
|
522
|
+
log: (message) => log(ctx, 'warn', message),
|
|
523
|
+
turnReports: cfg.turnReports !== false,
|
|
524
|
+
turnReportsKeep: cfg.turnReportsKeep ?? 50,
|
|
525
|
+
})
|
|
526
|
+
|
|
527
|
+
if (snapshots) {
|
|
528
|
+
ctx.on('agent/turn-stopping', () => {
|
|
529
|
+
try {
|
|
530
|
+
snapshots.request()
|
|
531
|
+
} catch {
|
|
532
|
+
// intentionally ignored: snapshot request is best effort on turn stop
|
|
533
|
+
}
|
|
534
|
+
})
|
|
535
|
+
ctx.effect(
|
|
536
|
+
() => () => {
|
|
537
|
+
void snapshots.dispose()
|
|
538
|
+
},
|
|
539
|
+
'skill-eval: snapshots',
|
|
540
|
+
)
|
|
541
|
+
}
|
|
542
|
+
|
|
543
|
+
if (cfg.registerTool !== false) {
|
|
544
|
+
ctx.effect(() => {
|
|
545
|
+
const disposers = []
|
|
546
|
+
DIAGNOSTICS.tools.push('skill_eval_report')
|
|
547
|
+
disposers.push(
|
|
548
|
+
ctx.tools.register({
|
|
549
|
+
name: 'skill_eval_report',
|
|
550
|
+
description:
|
|
551
|
+
'Report recorded skill execution telemetry with a 0-100 score, an A-F grade and evidence-cited fixes. Use it when the user asks how a skill performed, why it is unreliable, or how to improve a skill. Every fix is backed by a recorded signal (failed loads, post-load tool errors, user corrections, ignored loads, sibling collisions, load latency) or a static authoring check.',
|
|
552
|
+
parameters: REPORT_PARAMETERS,
|
|
553
|
+
output: REPORT_OUTPUT,
|
|
554
|
+
async execute(args) {
|
|
555
|
+
try {
|
|
556
|
+
return await runReport(analyzer, args ?? {}, cfg)
|
|
557
|
+
} catch (error) {
|
|
558
|
+
return `skill_eval_report failed: ${error instanceof Error ? error.message : String(error)}`
|
|
559
|
+
}
|
|
560
|
+
},
|
|
561
|
+
}),
|
|
562
|
+
)
|
|
563
|
+
return () => {
|
|
564
|
+
for (const dispose of disposers) safe(dispose)()
|
|
565
|
+
}
|
|
566
|
+
}, 'skill-eval: report tool')
|
|
567
|
+
}
|
|
568
|
+
|
|
569
|
+
if (cfg.registerReviewTool !== false) {
|
|
570
|
+
ctx.effect(() => {
|
|
571
|
+
DIAGNOSTICS.tools.push('skill_eval_review')
|
|
572
|
+
const dispose = ctx.tools.register({
|
|
573
|
+
name: 'skill_eval_review',
|
|
574
|
+
description:
|
|
575
|
+
'Screen a proposed skill edit for leakage before applying it (benchmark names, hardcoded answers/dates/paths, task-specific logic) and record the hypothesis + verdict in the audit ledger. Use it before rewriting a skill because of eval feedback.',
|
|
576
|
+
parameters: {
|
|
577
|
+
type: 'object',
|
|
578
|
+
properties: {
|
|
579
|
+
skill: { type: 'string', description: 'skill being edited' },
|
|
580
|
+
dimension: {
|
|
581
|
+
type: 'string',
|
|
582
|
+
enum: ['static', 'reliability', 'effectiveness', 'efficiency', 'mcp', 'lifecycle'],
|
|
583
|
+
description: 'which dimension this edit targets (drives the exploration directive)',
|
|
584
|
+
},
|
|
585
|
+
hypothesis: { type: 'string', description: 'what should improve and how it will be measured' },
|
|
586
|
+
diff: { type: 'string', description: 'the proposed change, as a diff or the new body text' },
|
|
587
|
+
},
|
|
588
|
+
required: ['skill'],
|
|
589
|
+
additionalProperties: false,
|
|
590
|
+
},
|
|
591
|
+
output: {
|
|
592
|
+
schema: { type: 'string', description: 'verdict and reasons' },
|
|
593
|
+
render(_args, value) {
|
|
594
|
+
return [{ type: 'text', text: typeof value === 'string' ? value : JSON.stringify(value) }]
|
|
595
|
+
},
|
|
596
|
+
},
|
|
597
|
+
async execute(args) {
|
|
598
|
+
try {
|
|
599
|
+
const verdict = reviewSkillEdit({
|
|
600
|
+
skill: String(args?.skill ?? ''),
|
|
601
|
+
hypothesis: typeof args?.hypothesis === 'string' ? args.hypothesis : undefined,
|
|
602
|
+
diff: typeof args?.diff === 'string' ? args.diff : undefined,
|
|
603
|
+
})
|
|
604
|
+
await analyzer.store.append({
|
|
605
|
+
t: 'edit',
|
|
606
|
+
v: 1,
|
|
607
|
+
at: Date.now(),
|
|
608
|
+
skill: String(args?.skill ?? ''),
|
|
609
|
+
hypothesis: typeof args?.hypothesis === 'string' ? args.hypothesis.slice(0, 500) : '',
|
|
610
|
+
verdict: verdict.verdict,
|
|
611
|
+
dimension: typeof args?.dimension === 'string' ? args.dimension : null,
|
|
612
|
+
reasons: verdict.reasons.map((reason) => reason.code),
|
|
613
|
+
diffHash: typeof args?.diff === 'string' ? hashHex(args.diff) : null,
|
|
614
|
+
})
|
|
615
|
+
const lines = [`skill_eval_review: **${verdict.verdict}** for \`${args?.skill ?? ''}\``]
|
|
616
|
+
if (verdict.reasons.length === 0) lines.push('no leakage signals found')
|
|
617
|
+
for (const reason of verdict.reasons)
|
|
618
|
+
lines.push(`- [${reason.severity}] ${reason.code}: ${reason.label} — ${reason.detail}`)
|
|
619
|
+
return lines.join('\n')
|
|
620
|
+
} catch (error) {
|
|
621
|
+
return `skill_eval_review failed: ${error instanceof Error ? error.message : String(error)}`
|
|
622
|
+
}
|
|
623
|
+
},
|
|
624
|
+
})
|
|
625
|
+
return () => safe(dispose)
|
|
626
|
+
}, 'skill-eval: review tool')
|
|
627
|
+
}
|
|
628
|
+
|
|
629
|
+
if (cfg.registerCommand !== false) {
|
|
630
|
+
ctx.inject(['commands'], (c) => {
|
|
631
|
+
c.effect(() => {
|
|
632
|
+
const dispose = c.commands.register({
|
|
633
|
+
name: 'skill-eval',
|
|
634
|
+
description: 'show the skill execution report (scores, grades and cited fixes)',
|
|
635
|
+
input: { hint: '[skill-name]' },
|
|
636
|
+
async handler(invocation) {
|
|
637
|
+
const target = String(invocation.rawInput ?? '').trim()
|
|
638
|
+
try {
|
|
639
|
+
const keyword = target.toLowerCase()
|
|
640
|
+
if (keyword === 'accept' || keyword === 'doctor') {
|
|
641
|
+
const report = await analyzer.build()
|
|
642
|
+
const files = await acceptanceFiles(analyzer.store, (file) =>
|
|
643
|
+
import('node:fs/promises').then((fs) => fs.readFile(file, 'utf8')),
|
|
644
|
+
)
|
|
645
|
+
const acceptance = checkInProcessReport(report, { version: PLUGIN_VERSION ?? 'unknown', files })
|
|
646
|
+
return { kind: 'success', text: '\n' + renderAcceptance(acceptance) + '\n' }
|
|
647
|
+
}
|
|
648
|
+
if (keyword.startsWith('history ') || keyword.startsWith('impact ')) {
|
|
649
|
+
const view = keyword.split(' ')[0]
|
|
650
|
+
const skill = target.slice(view.length + 1).trim()
|
|
651
|
+
const text = await runReport(analyzer, { view, skill }, cfg)
|
|
652
|
+
return { kind: 'success', text: `\n${text}\n` }
|
|
653
|
+
}
|
|
654
|
+
if (keyword === 'config') {
|
|
655
|
+
const lines = [
|
|
656
|
+
'## 配置 / effective settings',
|
|
657
|
+
'',
|
|
658
|
+
`override file: \`${path.join(settingsDir ?? '<unknown>', SETTINGS_FILE)}\``,
|
|
659
|
+
'',
|
|
660
|
+
]
|
|
661
|
+
for (const field of SETTINGS_KEYS) {
|
|
662
|
+
const value = field
|
|
663
|
+
.split('.')
|
|
664
|
+
.reduce((target, key) => (target == null ? undefined : target[key]), cfg)
|
|
665
|
+
lines.push(`- \`${field}\`: **${value === undefined ? '—' : String(value)}**`)
|
|
666
|
+
}
|
|
667
|
+
lines.push('', '浏览器里打开 `<dashboardPath>/config` 可直接修改(保存后立即生效)。')
|
|
668
|
+
return { kind: 'success', text: `\n${lines.join('\n')}\n` }
|
|
669
|
+
}
|
|
670
|
+
if (keyword === 'turns' || keyword === 'boards') {
|
|
671
|
+
const text = await runReport(analyzer, { view: 'turns' }, cfg)
|
|
672
|
+
return { kind: 'success', text: `\n${text}\n` }
|
|
673
|
+
}
|
|
674
|
+
if (keyword === 'dev' || keyword === 'diagnostics') {
|
|
675
|
+
const text = await runReport(analyzer, { view: 'dev' }, cfg)
|
|
676
|
+
return { kind: 'success', text: `\n${text}\n` }
|
|
677
|
+
}
|
|
678
|
+
if (keyword === 'validity' || keyword === 'valid') {
|
|
679
|
+
const text = await runReport(analyzer, { view: 'validity' }, cfg)
|
|
680
|
+
return { kind: 'success', text: `\n${text}\n` }
|
|
681
|
+
}
|
|
682
|
+
if (keyword === 'loop' || keyword.startsWith('loop ')) {
|
|
683
|
+
const rest = keyword.startsWith('loop ') ? target.slice(5).trim() : ''
|
|
684
|
+
if (rest === 'all') {
|
|
685
|
+
const text = await runReport(analyzer, { view: 'loop', all: true }, cfg)
|
|
686
|
+
return { kind: 'success', text: `\n${text}\n` }
|
|
687
|
+
}
|
|
688
|
+
const text = await runReport(analyzer, { view: 'loop', skill: rest || undefined }, cfg)
|
|
689
|
+
return { kind: 'success', text: `\n${text}\n` }
|
|
690
|
+
}
|
|
691
|
+
const view = ['mcp', 'runs', 'standard'].includes(keyword) ? keyword : undefined
|
|
692
|
+
const text = await runReport(analyzer, view ? { view } : { skill: target || undefined }, cfg)
|
|
693
|
+
return { kind: 'success', text: `\n${text}\n` }
|
|
694
|
+
} catch (error) {
|
|
695
|
+
return {
|
|
696
|
+
kind: 'error',
|
|
697
|
+
text: `skill-eval failed: ${error instanceof Error ? error.message : String(error)}`,
|
|
698
|
+
}
|
|
699
|
+
}
|
|
700
|
+
},
|
|
701
|
+
})
|
|
702
|
+
return () => safe(dispose)
|
|
703
|
+
}, 'skill-eval: command')
|
|
704
|
+
})
|
|
705
|
+
}
|
|
706
|
+
|
|
707
|
+
if (cfg.webDashboard !== false) {
|
|
708
|
+
const base = String(cfg.dashboardPath || '/skill-eval').replace(/\/$/, '')
|
|
709
|
+
// The client bundle ships a fixed route contract: data/settings stay under
|
|
710
|
+
// /skill-eval even when the *pages* move, otherwise a custom dashboardPath
|
|
711
|
+
// silently breaks the panel and the native config form.
|
|
712
|
+
const clientBase = '/skill-eval'
|
|
713
|
+
const dataPath = `${clientBase}/data`
|
|
714
|
+
|
|
715
|
+
// The board's dev tab shows what the agent's `view:"dev"` report shows in
|
|
716
|
+
// short form: which generation is loaded, whether the package on disk moved
|
|
717
|
+
// ahead of it (the ESM-cache restart trap), how many routes this generation
|
|
718
|
+
// claimed, and whether the event store is healthy.
|
|
719
|
+
const diagnosticsPayload = () => {
|
|
720
|
+
const diskVersion = packageVersion()
|
|
721
|
+
return {
|
|
722
|
+
moduleVersion: PLUGIN_VERSION ?? null,
|
|
723
|
+
diskVersion,
|
|
724
|
+
skew: Boolean(PLUGIN_VERSION && diskVersion && PLUGIN_VERSION !== diskVersion),
|
|
725
|
+
routes: DIAGNOSTICS.routes.length,
|
|
726
|
+
tools: DIAGNOSTICS.tools.length,
|
|
727
|
+
inspectProvider: Boolean(DIAGNOSTICS.inspectProvider),
|
|
728
|
+
settingsFile: DIAGNOSTICS.settingsFile ?? null,
|
|
729
|
+
warnings: DIAGNOSTICS.warnings.length,
|
|
730
|
+
}
|
|
731
|
+
}
|
|
732
|
+
|
|
733
|
+
const writeReport = async (res) => {
|
|
734
|
+
const report = await analyzer.build()
|
|
735
|
+
res.writeHead(200, {
|
|
736
|
+
'content-type': 'application/json; charset=utf-8',
|
|
737
|
+
'cache-control': 'no-store',
|
|
738
|
+
})
|
|
739
|
+
res.end(JSON.stringify({ ...report, dev: diagnosticsPayload() }, null, 2))
|
|
740
|
+
}
|
|
741
|
+
|
|
742
|
+
const bodyTooLarge = () => Object.assign(new Error('request body too large'), { statusCode: 413 })
|
|
743
|
+
const readBody = (req, limit = 64 * 1024) =>
|
|
744
|
+
new Promise((resolve, reject) => {
|
|
745
|
+
const declared = Number(req.headers?.['content-length'])
|
|
746
|
+
if (Number.isFinite(declared) && declared > limit) {
|
|
747
|
+
reject(bodyTooLarge())
|
|
748
|
+
return
|
|
749
|
+
}
|
|
750
|
+
let body = ''
|
|
751
|
+
let size = 0
|
|
752
|
+
let settled = false
|
|
753
|
+
const fail = (error) => {
|
|
754
|
+
if (settled) return
|
|
755
|
+
settled = true
|
|
756
|
+
reject(error)
|
|
757
|
+
if (typeof req.resume === 'function') req.resume()
|
|
758
|
+
}
|
|
759
|
+
if (typeof req.setEncoding === 'function') req.setEncoding('utf8')
|
|
760
|
+
req.on('data', (chunk) => {
|
|
761
|
+
if (settled) return
|
|
762
|
+
size += Buffer.byteLength(chunk)
|
|
763
|
+
if (size > limit) {
|
|
764
|
+
fail(bodyTooLarge())
|
|
765
|
+
return
|
|
766
|
+
}
|
|
767
|
+
body += chunk
|
|
768
|
+
})
|
|
769
|
+
req.on('end', () => {
|
|
770
|
+
if (settled) return
|
|
771
|
+
settled = true
|
|
772
|
+
resolve(body)
|
|
773
|
+
})
|
|
774
|
+
req.on('error', (error) => fail(error))
|
|
775
|
+
})
|
|
776
|
+
|
|
777
|
+
// Plugin routes bypass the host's own trust fence, so settings writes get a
|
|
778
|
+
// minimal same-origin gate here: reject cross-site fetch metadata, reject a
|
|
779
|
+
// mismatched Origin, and require a custom header on the JSON API (which a
|
|
780
|
+
// cross-site simple request cannot send without a preflight it would fail).
|
|
781
|
+
const trustedWrite = (req, { requireHeader = false } = {}) => {
|
|
782
|
+
const site = String(req.headers?.['sec-fetch-site'] ?? '').toLowerCase()
|
|
783
|
+
if (site === 'cross-site') return false
|
|
784
|
+
const origin = req.headers?.origin
|
|
785
|
+
if (origin) {
|
|
786
|
+
try {
|
|
787
|
+
const parsed = new URL(origin)
|
|
788
|
+
if (!req.headers.host || parsed.host !== req.headers.host) return false
|
|
789
|
+
} catch {
|
|
790
|
+
return false
|
|
791
|
+
}
|
|
792
|
+
}
|
|
793
|
+
if (requireHeader && req.headers?.['x-skill-eval-config'] !== '1') return false
|
|
794
|
+
return true
|
|
795
|
+
}
|
|
796
|
+
|
|
797
|
+
const servePageAt = (pageBase) => async (req, res) => {
|
|
798
|
+
try {
|
|
799
|
+
const report = await analyzer.build()
|
|
800
|
+
const lang = resolveLang(req.headers?.['accept-language'], requestLang(req))
|
|
801
|
+
res.writeHead(200, { 'content-type': 'text/html; charset=utf-8', 'cache-control': 'no-store' })
|
|
802
|
+
res.end(renderHtml(report, { base: pageBase, lang }))
|
|
803
|
+
} catch (error) {
|
|
804
|
+
res.writeHead(500, { 'content-type': 'text/plain; charset=utf-8' })
|
|
805
|
+
res.end(`skill-eval dashboard failed: ${error instanceof Error ? error.message : String(error)}`)
|
|
806
|
+
}
|
|
807
|
+
}
|
|
808
|
+
|
|
809
|
+
const handleConfig = async (req, res, url, pageBase = base) => {
|
|
810
|
+
const lang = resolveLang(req.headers?.['accept-language'], url.searchParams.get('lang'))
|
|
811
|
+
const send = (status, html) => {
|
|
812
|
+
res.writeHead(status, { 'content-type': 'text/html; charset=utf-8', 'cache-control': 'no-store' })
|
|
813
|
+
res.end(html)
|
|
814
|
+
}
|
|
815
|
+
try {
|
|
816
|
+
if (req.method === 'POST') {
|
|
817
|
+
if (!trustedWrite(req)) {
|
|
818
|
+
send(403, 'cross-site settings write rejected')
|
|
819
|
+
return
|
|
820
|
+
}
|
|
821
|
+
const body = await readBody(req)
|
|
822
|
+
const submitted = parseFormSettings(body)
|
|
823
|
+
const { values, errors } = normalizeSettings(submitted)
|
|
824
|
+
const weightError = effectiveWeightError(values)
|
|
825
|
+
if (weightError) errors.push(weightError)
|
|
826
|
+
if (errors.length > 0) {
|
|
827
|
+
send(400, renderConfigHtml({ config: cfg, base: pageBase, errors, lang }))
|
|
828
|
+
return
|
|
829
|
+
}
|
|
830
|
+
if (settingsDir) await saveSettings(settingsDir, { ...loadSettingsSync(settingsDir), ...values })
|
|
831
|
+
applySettings(cfg, values)
|
|
832
|
+
analyzer.invalidate()
|
|
833
|
+
DIAGNOSTICS.settings = loadSettingsSync(settingsDir)
|
|
834
|
+
send(200, renderConfigHtml({ config: cfg, base: pageBase, saved: true, lang }))
|
|
835
|
+
return
|
|
836
|
+
}
|
|
837
|
+
send(
|
|
838
|
+
200,
|
|
839
|
+
renderConfigHtml({
|
|
840
|
+
config: cfg,
|
|
841
|
+
base: pageBase,
|
|
842
|
+
saved: url.searchParams.get('saved') === '1',
|
|
843
|
+
errors: [],
|
|
844
|
+
lang,
|
|
845
|
+
}),
|
|
846
|
+
)
|
|
847
|
+
} catch (error) {
|
|
848
|
+
const status = error && error.statusCode === 413 ? 413 : 500
|
|
849
|
+
send(status, status === 413 ? 'request body too large' : 'config failed')
|
|
850
|
+
}
|
|
851
|
+
}
|
|
852
|
+
|
|
853
|
+
// The JSON API is its own exact route so the popup keeps working even when
|
|
854
|
+
// the page route below cannot be reclaimed after a reload (the host keeps
|
|
855
|
+
// routes registered by a previous instance). The versioned alias is new on
|
|
856
|
+
// every release, so even a hot reload that cannot reclaim `dataPath` gets a
|
|
857
|
+
// working URL for the current generation.
|
|
858
|
+
const effectiveWeightError = (values) => {
|
|
859
|
+
const weights = { ...(cfg.weights ?? {}) }
|
|
860
|
+
for (const [key, value] of Object.entries(values ?? {})) {
|
|
861
|
+
if (key.startsWith('weights.')) weights[key.slice('weights.'.length)] = value
|
|
862
|
+
}
|
|
863
|
+
const sum = ['static', 'reliability', 'effectiveness', 'efficiency'].reduce(
|
|
864
|
+
(total, dimension) => total + (Number(weights[dimension]) || 0),
|
|
865
|
+
0,
|
|
866
|
+
)
|
|
867
|
+
return sum > 0 ? null : { key: 'weights.total', message: 'weights sum must be positive' }
|
|
868
|
+
}
|
|
869
|
+
const settingsValue = (key) => {
|
|
870
|
+
const [head, leaf] = key.split('.')
|
|
871
|
+
const raw = leaf ? cfg?.[head]?.[leaf] : cfg?.[head]
|
|
872
|
+
return raw === undefined ? null : raw
|
|
873
|
+
}
|
|
874
|
+
const settingsPayload = () => ({
|
|
875
|
+
fields: SETTINGS_FIELDS.map((field) => ({
|
|
876
|
+
...field,
|
|
877
|
+
labelZh: FIELD_ZH[field.key] ?? field.label,
|
|
878
|
+
value: settingsValue(field.key),
|
|
879
|
+
})),
|
|
880
|
+
values: Object.fromEntries(SETTINGS_FIELDS.map((field) => [field.key, settingsValue(field.key)])),
|
|
881
|
+
version: PLUGIN_VERSION ?? null,
|
|
882
|
+
})
|
|
883
|
+
const readJsonBody = async (req) => {
|
|
884
|
+
const raw = await readBody(req)
|
|
885
|
+
if (!raw) return {}
|
|
886
|
+
try {
|
|
887
|
+
return JSON.parse(raw)
|
|
888
|
+
} catch {
|
|
889
|
+
return Object.fromEntries(new URLSearchParams(raw))
|
|
890
|
+
}
|
|
891
|
+
}
|
|
892
|
+
const handleSettingsApi = async (req, res) => {
|
|
893
|
+
const send = (status, payload) => {
|
|
894
|
+
res.writeHead(status, { 'content-type': 'application/json; charset=utf-8', 'cache-control': 'no-store' })
|
|
895
|
+
res.end(JSON.stringify(payload, null, 2))
|
|
896
|
+
}
|
|
897
|
+
try {
|
|
898
|
+
if (req.method === 'POST') {
|
|
899
|
+
if (!trustedWrite(req, { requireHeader: true })) {
|
|
900
|
+
send(403, { saved: false, errors: [{ key: 'origin', message: 'cross-site settings write rejected' }] })
|
|
901
|
+
return
|
|
902
|
+
}
|
|
903
|
+
const body = await readJsonBody(req)
|
|
904
|
+
const { values, errors } = normalizeSettings(body)
|
|
905
|
+
const weightError = effectiveWeightError(values)
|
|
906
|
+
if (weightError) errors.push(weightError)
|
|
907
|
+
if (errors.length > 0) {
|
|
908
|
+
send(400, { saved: false, errors, values: settingsPayload().values })
|
|
909
|
+
return
|
|
910
|
+
}
|
|
911
|
+
if (settingsDir) await saveSettings(settingsDir, { ...loadSettingsSync(settingsDir), ...values })
|
|
912
|
+
applySettings(cfg, values)
|
|
913
|
+
analyzer.invalidate()
|
|
914
|
+
DIAGNOSTICS.settings = loadSettingsSync(settingsDir)
|
|
915
|
+
send(200, { saved: true, errors: [], values: settingsPayload().values })
|
|
916
|
+
return
|
|
917
|
+
}
|
|
918
|
+
send(200, settingsPayload())
|
|
919
|
+
} catch (error) {
|
|
920
|
+
const status = error && error.statusCode === 413 ? 413 : 500
|
|
921
|
+
send(status, {
|
|
922
|
+
saved: false,
|
|
923
|
+
errors: [{ key: 'server', message: status === 413 ? 'request body too large' : 'settings failed' }],
|
|
924
|
+
})
|
|
925
|
+
}
|
|
926
|
+
}
|
|
927
|
+
|
|
928
|
+
const dataPaths = PLUGIN_VERSION ? [`${dataPath}/${PLUGIN_VERSION}`, dataPath] : [dataPath]
|
|
929
|
+
|
|
930
|
+
// Settings API: the native plugin-config page renders this instead of the
|
|
931
|
+
// standalone HTML form. Versioned alias for the same hot-reload reason.
|
|
932
|
+
const settingsPaths = PLUGIN_VERSION
|
|
933
|
+
? [`${clientBase}/settings/${PLUGIN_VERSION}`, `${clientBase}/settings`]
|
|
934
|
+
: [`${clientBase}/settings`]
|
|
935
|
+
for (const routePath of settingsPaths)
|
|
936
|
+
ctx.inject(['webServer'], (c) => {
|
|
937
|
+
c.effect(() => {
|
|
938
|
+
let dispose
|
|
939
|
+
try {
|
|
940
|
+
dispose = c.webServer.register({ kind: 'exact', path: routePath, handler: handleSettingsApi })
|
|
941
|
+
DIAGNOSTICS.routes.push(routePath)
|
|
942
|
+
} catch (error) {
|
|
943
|
+
log(
|
|
944
|
+
ctx,
|
|
945
|
+
'warn',
|
|
946
|
+
`settings route ${routePath} not registered: ${error instanceof Error ? error.message : String(error)}`,
|
|
947
|
+
)
|
|
948
|
+
return () => {}
|
|
949
|
+
}
|
|
950
|
+
return () => safe(dispose)
|
|
951
|
+
}, 'skill-eval: settings route')
|
|
952
|
+
})
|
|
953
|
+
for (const routePath of dataPaths)
|
|
954
|
+
ctx.inject(['webServer'], (c) => {
|
|
955
|
+
c.effect(() => {
|
|
956
|
+
let dispose
|
|
957
|
+
try {
|
|
958
|
+
dispose = c.webServer.register({
|
|
959
|
+
kind: 'exact',
|
|
960
|
+
path: routePath,
|
|
961
|
+
async handler(req, res) {
|
|
962
|
+
try {
|
|
963
|
+
await writeReport(res)
|
|
964
|
+
} catch (error) {
|
|
965
|
+
res.writeHead(500, { 'content-type': 'text/plain; charset=utf-8' })
|
|
966
|
+
res.end(`skill-eval data failed: ${error instanceof Error ? error.message : String(error)}`)
|
|
967
|
+
}
|
|
968
|
+
},
|
|
969
|
+
})
|
|
970
|
+
DIAGNOSTICS.routes.push(routePath)
|
|
971
|
+
} catch (error) {
|
|
972
|
+
log(
|
|
973
|
+
ctx,
|
|
974
|
+
'warn',
|
|
975
|
+
`data route ${routePath} not registered: ${error instanceof Error ? error.message : String(error)}`,
|
|
976
|
+
)
|
|
977
|
+
return () => {}
|
|
978
|
+
}
|
|
979
|
+
return () => safe(dispose)
|
|
980
|
+
}, 'skill-eval: data route')
|
|
981
|
+
})
|
|
982
|
+
|
|
983
|
+
// Exact routes are matched before any prefix route, so registering the page
|
|
984
|
+
// and the settings page exactly keeps them working even when a previous
|
|
985
|
+
// generation still owns the `${base}` prefix (routes cannot be reclaimed).
|
|
986
|
+
// Exact page routes cannot be reclaimed once an older generation owns them
|
|
987
|
+
// (hot reload replaces the plugin but the host keeps the old route). The
|
|
988
|
+
// versioned alias is therefore the reliable URL for the current generation,
|
|
989
|
+
// and the canonical paths stay as best-effort for a clean start.
|
|
990
|
+
const boardVersion = PLUGIN_VERSION || null
|
|
991
|
+
const versionedBase = boardVersion ? `${base}/board/${boardVersion}` : null
|
|
992
|
+
const configAt = (pageBase) => async (req, res) => {
|
|
993
|
+
const origin = `http://${req.headers?.host ?? 'localhost'}`
|
|
994
|
+
await handleConfig(req, res, new URL(req.url ?? `${pageBase}/config`, origin), pageBase)
|
|
995
|
+
}
|
|
996
|
+
// Each route renders links/actions against its OWN base: the versioned page
|
|
997
|
+
// must post to the versioned config route, otherwise a hot reload sends the
|
|
998
|
+
// save back to a route the previous generation still owns.
|
|
999
|
+
const pageRoutes = [
|
|
1000
|
+
{ path: base, handler: servePageAt(base) },
|
|
1001
|
+
{ path: `${base}/config`, handler: configAt(base) },
|
|
1002
|
+
...(versionedBase
|
|
1003
|
+
? [
|
|
1004
|
+
{ path: versionedBase, handler: servePageAt(versionedBase) },
|
|
1005
|
+
{ path: `${versionedBase}/config`, handler: configAt(versionedBase) },
|
|
1006
|
+
]
|
|
1007
|
+
: []),
|
|
1008
|
+
]
|
|
1009
|
+
for (const pageRoute of pageRoutes)
|
|
1010
|
+
ctx.inject(['webServer'], (c) => {
|
|
1011
|
+
c.effect(() => {
|
|
1012
|
+
let dispose
|
|
1013
|
+
try {
|
|
1014
|
+
dispose = c.webServer.register({ kind: 'exact', path: pageRoute.path, handler: pageRoute.handler })
|
|
1015
|
+
DIAGNOSTICS.routes.push(pageRoute.path)
|
|
1016
|
+
} catch (error) {
|
|
1017
|
+
log(
|
|
1018
|
+
ctx,
|
|
1019
|
+
'warn',
|
|
1020
|
+
`page route ${pageRoute.path} not registered: ${error instanceof Error ? error.message : String(error)}`,
|
|
1021
|
+
)
|
|
1022
|
+
return () => {}
|
|
1023
|
+
}
|
|
1024
|
+
return () => safe(dispose)
|
|
1025
|
+
}, 'skill-eval: page route')
|
|
1026
|
+
})
|
|
1027
|
+
|
|
1028
|
+
ctx.inject(['webServer'], (c) => {
|
|
1029
|
+
c.effect(() => {
|
|
1030
|
+
let dispose
|
|
1031
|
+
try {
|
|
1032
|
+
dispose = c.webServer.register({
|
|
1033
|
+
kind: 'prefix',
|
|
1034
|
+
path: base,
|
|
1035
|
+
async handler(req, res) {
|
|
1036
|
+
let origin = 'http://localhost'
|
|
1037
|
+
try {
|
|
1038
|
+
origin = c.webServer.getOrigin()
|
|
1039
|
+
} catch {
|
|
1040
|
+
// keep the fallback origin
|
|
1041
|
+
}
|
|
1042
|
+
const url = new URL(req.url ?? base, origin)
|
|
1043
|
+
try {
|
|
1044
|
+
if (url.pathname.replace(/\/$/, '').endsWith('/data')) {
|
|
1045
|
+
await writeReport(res)
|
|
1046
|
+
return
|
|
1047
|
+
}
|
|
1048
|
+
if (url.pathname.replace(/\/$/, '').endsWith('/config')) {
|
|
1049
|
+
await handleConfig(req, res, url)
|
|
1050
|
+
return
|
|
1051
|
+
}
|
|
1052
|
+
const report = await analyzer.build()
|
|
1053
|
+
const lang = resolveLang(req.headers?.['accept-language'], url.searchParams.get('lang'))
|
|
1054
|
+
res.writeHead(200, { 'content-type': 'text/html; charset=utf-8', 'cache-control': 'no-store' })
|
|
1055
|
+
res.end(renderHtml(report, { base, lang }))
|
|
1056
|
+
} catch (error) {
|
|
1057
|
+
res.writeHead(500, { 'content-type': 'text/plain; charset=utf-8' })
|
|
1058
|
+
res.end(`skill-eval dashboard failed: ${error instanceof Error ? error.message : String(error)}`)
|
|
1059
|
+
}
|
|
1060
|
+
},
|
|
1061
|
+
})
|
|
1062
|
+
DIAGNOSTICS.routes.push(base)
|
|
1063
|
+
} catch (error) {
|
|
1064
|
+
log(
|
|
1065
|
+
ctx,
|
|
1066
|
+
'warn',
|
|
1067
|
+
`dashboard route ${base} not registered: ${error instanceof Error ? error.message : String(error)}`,
|
|
1068
|
+
)
|
|
1069
|
+
return () => {}
|
|
1070
|
+
}
|
|
1071
|
+
return () => safe(dispose)
|
|
1072
|
+
}, 'skill-eval: dashboard')
|
|
1073
|
+
})
|
|
1074
|
+
}
|
|
1075
|
+
|
|
1076
|
+
// Developer-mode surface: the eval data registers next to the harness's own
|
|
1077
|
+
// Service/Event/Config providers, so `cordis_inspect_query` can read it.
|
|
1078
|
+
if (cfg.inspectProvider !== false) {
|
|
1079
|
+
ctx.inject(['cordisInspect'], (c) => {
|
|
1080
|
+
c.effect(() => {
|
|
1081
|
+
try {
|
|
1082
|
+
const dispose = c.cordisInspect.register(createInspectProvider(analyzer))
|
|
1083
|
+
DIAGNOSTICS.inspectProvider = true
|
|
1084
|
+
log(ctx, 'info', 'inspect provider "skill-eval" registered')
|
|
1085
|
+
return dispose
|
|
1086
|
+
} catch (error) {
|
|
1087
|
+
log(ctx, 'warn', `inspect provider not registered: ${error instanceof Error ? error.message : String(error)}`)
|
|
1088
|
+
return () => {}
|
|
1089
|
+
}
|
|
1090
|
+
}, 'skill-eval: inspect provider')
|
|
1091
|
+
})
|
|
1092
|
+
}
|
|
1093
|
+
|
|
1094
|
+
const SETTINGS_KEYS = SETTINGS_FIELDS.map((field) => field.key)
|
|
1095
|
+
|
|
1096
|
+
if (cfg.promptHint !== false)
|
|
1097
|
+
ctx.inject(['systemPrompt'], (c) => {
|
|
1098
|
+
c.effect(() => {
|
|
1099
|
+
const stop = c.systemPrompt.section({
|
|
1100
|
+
name: 'skill-eval:usage',
|
|
1101
|
+
order: 118,
|
|
1102
|
+
text: () =>
|
|
1103
|
+
'Skill telemetry is recorded for every `skill` load. When asked how a skill performed or how to improve it, call `skill_eval_report` (pass {"skill":"<name>"} for one skill) and act on the cited fixes instead of guessing. Before editing a skill named in the report, screen the change with `skill_eval_review` (leakage critic) and record the hypothesis; ask `skill_eval_report` with view:"loop" for the next step and view:"validity" for what the evidence can and cannot support.',
|
|
1104
|
+
})
|
|
1105
|
+
return () => safe(stop)
|
|
1106
|
+
}, 'skill-eval: prompt section')
|
|
1107
|
+
})
|
|
1108
|
+
|
|
1109
|
+
log(ctx, 'info', `skill-eval: recording enabled, data dir ${analyzer.store.dir}`)
|
|
1110
|
+
}
|
|
1111
|
+
|