dsh-skill-eval 0.0.0-stage → 1.20.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (48) hide show
  1. package/CHANGELOG.md +675 -0
  2. package/LICENSE +21 -0
  3. package/README.md +200 -2
  4. package/README.zh.md +157 -0
  5. package/client.js +2637 -0
  6. package/cordis.patch.yml +9 -0
  7. package/docs/ACCEPTANCE.md +71 -0
  8. package/docs/AUDIT.md +64 -0
  9. package/docs/CODE-REVIEW.md +49 -0
  10. package/docs/COMPARISON.md +33 -0
  11. package/docs/DETAIL.md +463 -0
  12. package/docs/DEV-RELOAD.md +114 -0
  13. package/docs/ENTRYPOINTS.md +65 -0
  14. package/docs/RELEASE.md +44 -0
  15. package/docs/RRSI-NOTES.md +129 -0
  16. package/docs/SCENARIOS.md +91 -0
  17. package/docs/UI-REVIEW.md +115 -0
  18. package/docs/WORKFLOW.md +108 -0
  19. package/icon.svg +9 -0
  20. package/index.js +1111 -0
  21. package/lib/accept.js +350 -0
  22. package/lib/advice.js +547 -0
  23. package/lib/critic.js +101 -0
  24. package/lib/dev.js +148 -0
  25. package/lib/history.js +140 -0
  26. package/lib/html.js +8 -0
  27. package/lib/inspect.js +142 -0
  28. package/lib/integrity.js +46 -0
  29. package/lib/loop.js +195 -0
  30. package/lib/random.js +29 -0
  31. package/lib/recorder.js +634 -0
  32. package/lib/render.js +582 -0
  33. package/lib/report.js +646 -0
  34. package/lib/score.js +525 -0
  35. package/lib/settings.js +396 -0
  36. package/lib/snapshot.js +166 -0
  37. package/lib/steps.js +141 -0
  38. package/lib/store.js +186 -0
  39. package/lib/validity.js +115 -0
  40. package/locale/en.json +6 -0
  41. package/locale/zh.json +6 -0
  42. package/package.json +96 -4
  43. package/scripts/accept.mjs +226 -0
  44. package/scripts/build-client.mjs +59 -0
  45. package/scripts/build.mjs +81 -0
  46. package/scripts/doctor.mjs +199 -0
  47. package/scripts/lib.mjs +52 -0
  48. package/scripts/verify-live.mjs +168 -0
package/index.js ADDED
@@ -0,0 +1,1111 @@
1
+ /**
2
+ * dsh-skill-eval — skill execution telemetry, scoring and optimization advice.
3
+ *
4
+ * Host-plane plugin. It records every `skill` tool load plus the stretch of work
5
+ * that follows it, then scores each skill 0-100 on two axes:
6
+ *
7
+ * - static authoring quality (description, trigger cues, progressive
8
+ * disclosure, size, examples) — computed from the live skill catalog
9
+ * - live execution evidence (load failures, post-load tool errors, user
10
+ * corrections, ignored loads, sibling collisions, cost) — folded from the
11
+ * append-only event store
12
+ *
13
+ * The result is exposed three ways: the `skill_eval_report` tool (so the agent
14
+ * can read its own numbers and act on the cited fixes), a `/skill-eval` slash
15
+ * command, and an HTML dashboard served by the host web server.
16
+ *
17
+ * @module dsh-skill-eval
18
+ */
19
+ import { readFileSync } from 'node:fs'
20
+ import path from 'node:path'
21
+ import z from '@deepseek-ai/schemastery'
22
+ import { createAnalyzer } from './lib/report.js'
23
+ import { DEFAULT_THRESHOLDS, DEFAULT_WEIGHTS } from './lib/score.js'
24
+ import { createRecorder } from './lib/recorder.js'
25
+ import { checkInProcessReport } from './lib/accept.js'
26
+ import { reviewSkillEdit } from './lib/critic.js'
27
+ import { buildLoopPlan, buildLoopPlans, renderLoopBatchMarkdown, renderLoopMarkdown } from './lib/loop.js'
28
+ import { buildDevReport, renderDevMarkdown } from './lib/dev.js'
29
+ import { buildHistory, buildImpact, renderHistoryMarkdown, renderImpactMarkdown } from './lib/history.js'
30
+ import { resolveDataDir } from './lib/store.js'
31
+ import { hashHex } from './lib/random.js'
32
+ import {
33
+ applySettings,
34
+ FIELD_ZH,
35
+ loadSettingsSync,
36
+ normalizeSettings,
37
+ parseFormSettings,
38
+ renderConfigHtml,
39
+ saveSettings,
40
+ SETTINGS_FIELDS,
41
+ SETTINGS_FILE,
42
+ } from './lib/settings.js'
43
+ import { renderValidityMarkdown } from './lib/validity.js'
44
+ import { createInspectProvider } from './lib/inspect.js'
45
+ import { createSnapshots, listTurnReports, renderTurnReportsMarkdown } from './lib/snapshot.js'
46
+ import {
47
+ renderHtml,
48
+ resolveLang,
49
+ renderMcpMarkdown,
50
+ renderRunsMarkdown,
51
+ renderSkillMarkdown,
52
+ renderStandardMarkdown,
53
+ renderSummaryMarkdown,
54
+ } from './lib/render.js'
55
+
56
+ export const name = 'skill-eval'
57
+
58
+ /**
59
+ * The running package version. The data route is registered both canonically
60
+ * and under this version, because a plugin reload cannot reclaim a route the
61
+ * previous generation already registered: the versioned path is new on every
62
+ * release, so a hot-reloaded plugin can still serve current data.
63
+ */
64
+ function packageVersion() {
65
+ try {
66
+ const manifest = new URL('./package.json', import.meta.url)
67
+ const parsed = JSON.parse(readFileSync(manifest, 'utf8'))
68
+ return typeof parsed.version === 'string' ? parsed.version : null
69
+ } catch {
70
+ return null
71
+ }
72
+ }
73
+
74
+ const PLUGIN_VERSION = packageVersion()
75
+
76
+ /** Per-apply diagnostics surfaced by the `dev` view (the restart-skew detector). */
77
+ const DIAGNOSTICS = { routes: [], tools: [], inspectProvider: false, warnings: [], settingsFile: null, settings: {} }
78
+
79
+ // cordis 4 resolves `inject` as an array or a name -> config map, with no
80
+ // required/optional split. Optional services are attached with `ctx.inject(...)`
81
+ // below, which starts the callback only once that service is available.
82
+ export const inject = ['tools', 'skills']
83
+
84
+ export const Config = z.object({
85
+ enabled: z.boolean().default(true).description('record skill telemetry and expose the report tool'),
86
+ dataDir: z.string().description('override the telemetry directory (default: $DSH_HOME/skill-eval/$DSH_PROFILE)'),
87
+ windowToolCalls: z
88
+ .natural()
89
+ .min(1)
90
+ .max(200)
91
+ .default(12)
92
+ .description('tool calls after a skill load that still count as evidence for that load'),
93
+ windowTtlMs: z
94
+ .natural()
95
+ .min(60_000)
96
+ .max(86_400_000)
97
+ .default(30 * 60_000)
98
+ .description('how long an open skill window keeps waiting for follow-up work'),
99
+ minSamples: z.natural().min(1).max(100).default(3).description('loads needed before a score stops being provisional'),
100
+ weights: z
101
+ .object({
102
+ static: z.natural().min(0).max(100).default(30).description('authoring quality weight'),
103
+ reliability: z.natural().min(0).max(100).default(25).description('load success and error weight'),
104
+ effectiveness: z.natural().min(0).max(100).default(30).description('user correction weight'),
105
+ efficiency: z.natural().min(0).max(100).default(15).description('cost weight'),
106
+ })
107
+ .description('relative weights; normalised to 100% automatically, set one to 0 to drop a dimension'),
108
+ thresholds: z
109
+ .object({
110
+ excellent: z.natural().min(1).max(100).default(90).description('A grade floor'),
111
+ good: z.natural().min(1).max(100).default(80).description('B grade floor'),
112
+ pass: z.natural().min(1).max(100).default(70).description('C grade floor'),
113
+ })
114
+ .description('grade bands; D stays above 60 and F below it'),
115
+ maxAdvice: z.natural().min(1).max(10).default(3).description('advice items returned per skill in the summary tool'),
116
+ tokenBudgetPerLoad: z
117
+ .natural()
118
+ .min(1000)
119
+ .default(20000)
120
+ .description('billable policy tokens a single skill load may cost before efficiency and advice react'),
121
+ cacheReadWeight: z
122
+ .number()
123
+ .min(0)
124
+ .max(1)
125
+ .default(0)
126
+ .description(
127
+ 'how much a cached input token counts against the token budget (cache reads are near-free by default)',
128
+ ),
129
+ countTokens: z.boolean().default(true).description('attribute real provider usage to the open skill windows'),
130
+ editBudgetMax: z.natural().min(1).max(10).default(4).description('edits a first-round skill may change at once'),
131
+ editBudgetMin: z.natural().min(1).max(10).default(1).description('edits allowed after the budget has annealed'),
132
+ editBudgetHorizon: z
133
+ .natural()
134
+ .min(1)
135
+ .default(6)
136
+ .description('accepted edits over which the budget anneals to its minimum'),
137
+ noveltyWindow: z
138
+ .natural()
139
+ .min(2)
140
+ .max(10)
141
+ .default(2)
142
+ .description('consecutive same-dimension edits that trigger an exploration directive'),
143
+ pruneMinLoads: z
144
+ .natural()
145
+ .min(1)
146
+ .default(6)
147
+ .description('loads required before a consistently low score becomes a prune recommendation'),
148
+ pruneMaxScore: z
149
+ .natural()
150
+ .min(0)
151
+ .max(100)
152
+ .default(60)
153
+ .description('score below which a well-sampled skill is a prune candidate'),
154
+ registerReviewTool: z
155
+ .boolean()
156
+ .default(true)
157
+ .description('register skill_eval_review (leakage critic + edit ledger)'),
158
+ adviceNoiseFloor: z
159
+ .boolean()
160
+ .default(true)
161
+ .description('only keep a rate-based advice severity when its bootstrap interval clears the threshold'),
162
+ turnReports: z.boolean().default(true).description('write one board per settled turn under reports/turns/'),
163
+ turnReportsKeep: z.natural().min(1).default(50).description('how many per-turn boards to keep'),
164
+ reportOnTurnEnd: z
165
+ .boolean()
166
+ .default(true)
167
+ .description('write reports/latest.md + latest.json after a turn settles (never enters the model context)'),
168
+ reportDir: z.string().description('override the snapshot directory (default: <dataDir>/reports)'),
169
+ ciResamples: z
170
+ .natural()
171
+ .min(0)
172
+ .max(2000)
173
+ .default(200)
174
+ .description('bootstrap resamples for the 95% score interval (0 disables it)'),
175
+ mcpErrorRatePercent: z
176
+ .natural()
177
+ .min(1)
178
+ .max(100)
179
+ .default(20)
180
+ .description('MCP server error-rate percent that raises a critical MCP fix'),
181
+ mcpToolErrorRatePercent: z
182
+ .natural()
183
+ .min(1)
184
+ .max(100)
185
+ .default(30)
186
+ .description('single MCP tool error-rate percent that raises a warning'),
187
+ mcpToolP95Ms: z
188
+ .natural()
189
+ .min(100)
190
+ .default(5000)
191
+ .description('per-tool MCP p95 latency that raises the mcp-tool-slow warning'),
192
+ mcpP95Ms: z.natural().min(100).default(3000).description('MCP p95 latency in ms that raises a slow-server warning'),
193
+ inspectProvider: z
194
+ .boolean()
195
+ .default(true)
196
+ .description('expose the eval data as a Cordis Inspect provider for developer mode'),
197
+ promptHint: z
198
+ .boolean()
199
+ .default(true)
200
+ .description('add the one-line usage hint to the system prompt (the only per-turn token cost, about 60 tokens)'),
201
+ registerTool: z.boolean().default(true).description('register the skill_eval_report tool'),
202
+ registerCommand: z.boolean().default(true).description('register the /skill-eval command'),
203
+ webDashboard: z
204
+ .boolean()
205
+ .default(true)
206
+ .description('serve the HTML dashboard and the JSON route the in-app popup reads'),
207
+ dashboardPath: z
208
+ .string()
209
+ .default('/skill-eval')
210
+ .description('path the dashboard is served on (change it when a reload could not reclaim the old route)'),
211
+ })
212
+
213
+ const REPORT_PARAMETERS = {
214
+ type: 'object',
215
+ properties: {
216
+ skill: {
217
+ type: 'string',
218
+ description: 'report a single skill by name; omit for the cross-skill summary',
219
+ },
220
+ format: {
221
+ type: 'string',
222
+ enum: ['markdown', 'json'],
223
+ description: 'markdown (default) for humans and models, json for post-processing',
224
+ },
225
+ all: {
226
+ type: 'boolean',
227
+ description: 'with view:"loop": return the plan for every skill that has an actionable fix',
228
+ },
229
+ view: {
230
+ type: 'string',
231
+ enum: ['summary', 'mcp', 'runs', 'standard', 'loop', 'validity', 'history', 'impact', 'dev', 'turns'],
232
+ description:
233
+ 'summary (default) scores every skill; mcp debugs MCP servers; runs lists recent sessions; standard shows the grading configuration; loop shows the next guided optimization step (add all:true for every actionable skill)',
234
+ },
235
+ },
236
+ additionalProperties: false,
237
+ }
238
+
239
+ const REPORT_OUTPUT = {
240
+ schema: { type: 'string', description: 'markdown or JSON report text' },
241
+ render(_args, value) {
242
+ return [{ type: 'text', text: typeof value === 'string' ? value : JSON.stringify(value, null, 2) }]
243
+ },
244
+ }
245
+
246
+ /** Run a disposer without letting teardown failures escape into the host. */
247
+ function safe(dispose) {
248
+ return () => {
249
+ try {
250
+ dispose()
251
+ } catch {
252
+ // intentionally ignored: teardown failures must not escape into the host
253
+ }
254
+ }
255
+ }
256
+
257
+ function log(ctx, level, message) {
258
+ if (DIAGNOSTICS.warnings.length < 50) DIAGNOSTICS.warnings.push(`${level}: ${message}`)
259
+ try {
260
+ const logger = typeof ctx.logger === 'function' ? ctx.logger('skill-eval') : ctx.logger
261
+ logger?.[level]?.(message)
262
+ } catch {
263
+ // logging is best effort
264
+ }
265
+ // `DSH_SKILL_EVAL_DEBUG=1` mirrors diagnostics to stderr, which is where a
266
+ // headless host (and a terminal launch) actually shows them.
267
+ try {
268
+ if (level === 'warn' || process.env.DSH_SKILL_EVAL_DEBUG === '1') {
269
+ console.warn('[skill-eval] ' + message)
270
+ }
271
+ } catch {
272
+ // intentionally ignored: stderr mirror is best effort
273
+ }
274
+ }
275
+
276
+ const VIEWS = ['summary', 'mcp', 'runs', 'standard', 'loop', 'validity', 'history', 'impact', 'dev', 'turns']
277
+
278
+ /** Read the files the acceptance check needs, best effort. */
279
+ async function acceptanceFiles(store, readFile) {
280
+ const dir = store.dir
281
+ const eventsPath = path.join(dir, 'events.jsonl')
282
+ const snapshotMarkdownPath = path.join(dir, 'reports', 'latest.md')
283
+ const snapshotJsonPath = path.join(dir, 'reports', 'latest.json')
284
+ let events = []
285
+ try {
286
+ const raw = await readFile(eventsPath, 'utf8')
287
+ for (const line of raw.split('\n')) {
288
+ const trimmed = line.trim()
289
+ if (!trimmed) continue
290
+ try {
291
+ events.push(JSON.parse(trimmed))
292
+ } catch {
293
+ // torn write
294
+ }
295
+ }
296
+ } catch {
297
+ events = []
298
+ }
299
+ const optional = async (file) => {
300
+ try {
301
+ return await readFile(file, 'utf8')
302
+ } catch {
303
+ return null
304
+ }
305
+ }
306
+ return {
307
+ eventsPath,
308
+ snapshotMarkdownPath,
309
+ snapshotJsonPath,
310
+ events,
311
+ snapshotText: await optional(snapshotMarkdownPath),
312
+ snapshotJsonText: await optional(snapshotJsonPath),
313
+ }
314
+ }
315
+
316
+ /** Read `?lang=` from a raw request URL without needing an origin. */
317
+ function requestLang(req) {
318
+ const raw = typeof req?.url === 'string' ? req.url : ''
319
+ const index = raw.indexOf('?')
320
+ if (index === -1) return null
321
+ try {
322
+ return new URLSearchParams(raw.slice(index + 1)).get('lang')
323
+ } catch {
324
+ return null
325
+ }
326
+ }
327
+
328
+ function renderAcceptance(acceptance) {
329
+ const lines = ['## In-process acceptance', '']
330
+ for (const item of acceptance.results) {
331
+ lines.push(`- **${item.ok ? 'PASS' : 'FAIL'}** \`${item.id}\` — ${item.detail}`)
332
+ }
333
+ lines.push('', acceptance.summary)
334
+ return lines.join('\n')
335
+ }
336
+
337
+ async function runReport(analyzer, args, config) {
338
+ const format = args?.format === 'json' ? 'json' : 'markdown'
339
+ const view = VIEWS.includes(args?.view) ? args.view : 'summary'
340
+ if (view === 'turns') {
341
+ const reportsDir = config.reportDir ? String(config.reportDir) : path.join(analyzer.store.dir, 'reports')
342
+ const rows = await listTurnReports(reportsDir)
343
+ return format === 'json' ? JSON.stringify({ turns: rows }, null, 2) : renderTurnReportsMarkdown(rows)
344
+ }
345
+ if (view === 'dev') {
346
+ const records = await analyzer.store.readAll()
347
+ const diskVersion = await import('node:fs/promises').then((fs) =>
348
+ fs.readFile(new URL('./package.json', import.meta.url), 'utf8'),
349
+ )
350
+ let parsedDiskVersion
351
+ try {
352
+ parsedDiskVersion = JSON.parse(diskVersion).version ?? null
353
+ } catch {
354
+ parsedDiskVersion = null
355
+ }
356
+ const dev = buildDevReport({
357
+ moduleVersion: PLUGIN_VERSION ?? null,
358
+ diskVersion: parsedDiskVersion,
359
+ dataDir: analyzer.store.dir,
360
+ settingsFile: DIAGNOSTICS.settingsFile,
361
+ settings: DIAGNOSTICS.settings,
362
+ config: config,
363
+ routes: DIAGNOSTICS.routes,
364
+ records,
365
+ store: { dir: analyzer.store.dir, failed: analyzer.store.failed, queue: analyzer.store.queue },
366
+ tools: DIAGNOSTICS.tools,
367
+ inspectProvider: DIAGNOSTICS.inspectProvider ? 'skill-eval' : null,
368
+ warnings: DIAGNOSTICS.warnings,
369
+ })
370
+ return format === 'json' ? JSON.stringify(dev, null, 2) : renderDevMarkdown(dev)
371
+ }
372
+ if (view === 'history' || view === 'impact') {
373
+ const skill = typeof args?.skill === 'string' ? args.skill : ''
374
+ if (!skill) return `view:"${view}" needs a skill name, e.g. {"view":"${view}","skill":"pdf-tools"}`
375
+ const records = await analyzer.store.readAll()
376
+ const payload = view === 'history' ? buildHistory(records, skill) : buildImpact(records, skill)
377
+ if (format === 'json') return JSON.stringify(payload, null, 2)
378
+ return view === 'history' ? renderHistoryMarkdown(payload) : renderImpactMarkdown(payload)
379
+ }
380
+ if (view === 'validity') {
381
+ const report = await analyzer.build()
382
+ return format === 'json' ? JSON.stringify(report.validity, null, 2) : renderValidityMarkdown(report.validity)
383
+ }
384
+ if (view === 'loop') {
385
+ const report = await analyzer.build()
386
+ if (args?.all === true || args?.skill === 'all') {
387
+ const plans = buildLoopPlans(report, {
388
+ editBudgetMax: config.editBudgetMax ?? 4,
389
+ reloads: report.scoring?.minSamples ?? 3,
390
+ })
391
+ return format === 'json' ? JSON.stringify({ plans }, null, 2) : renderLoopBatchMarkdown(plans)
392
+ }
393
+ const plan = buildLoopPlan(report, {
394
+ skill: typeof args?.skill === 'string' ? args.skill : undefined,
395
+ editBudgetMax: config.editBudgetMax ?? 4,
396
+ reloads: report.scoring?.minSamples ?? 3,
397
+ })
398
+ return format === 'json' ? JSON.stringify(plan, null, 2) : renderLoopMarkdown(plan)
399
+ }
400
+ if (view === 'mcp' || view === 'runs' || view === 'standard') {
401
+ const report = await analyzer.build()
402
+ if (format === 'json') {
403
+ return JSON.stringify(
404
+ view === 'mcp'
405
+ ? { mcpServers: report.mcpServers }
406
+ : view === 'runs'
407
+ ? { runs: report.runs }
408
+ : { scoring: report.scoring },
409
+ null,
410
+ 2,
411
+ )
412
+ }
413
+ return view === 'mcp'
414
+ ? renderMcpMarkdown(report)
415
+ : view === 'runs'
416
+ ? renderRunsMarkdown(report)
417
+ : renderStandardMarkdown(report)
418
+ }
419
+ if (args?.skill) {
420
+ const entry = await analyzer.buildOne(String(args.skill))
421
+ if (!entry) {
422
+ const known = (await analyzer.build()).skills.map((s) => s.skill)
423
+ return `No evidence recorded for skill \`${args.skill}\`.\n\nSkills with recorded loads: ${
424
+ known.length > 0 ? known.map((n) => `\`${n}\``).join(', ') : '(none)'
425
+ }`
426
+ }
427
+ if (format === 'json') return JSON.stringify(entry, null, 2)
428
+ return renderSkillMarkdown({ ...entry, advice: entry.advice })
429
+ }
430
+ const report = await analyzer.build()
431
+ if (format === 'json') return JSON.stringify(report, null, 2)
432
+ return renderSummaryMarkdown({
433
+ ...report,
434
+ skills: report.skills.map((s) => ({ ...s, topAdvice: s.topAdvice.slice(0, config.maxAdvice ?? 3) })),
435
+ })
436
+ }
437
+
438
+ /**
439
+ * Resolve one skill inside an agent scope.
440
+ *
441
+ * The catalog is layered: reads only include the user and project layers when
442
+ * the agent is passed as `scope`, so a plain `skills.list()` misses exactly the
443
+ * skills this plugin is measuring. `cwd` widens project-level discovery.
444
+ */
445
+ function makeResolveSkill(ctx) {
446
+ return async (skillName, agent, signal) => {
447
+ const attempts = [
448
+ { scope: agent, signal },
449
+ { scope: agent, signal, cwd: process.cwd() },
450
+ ]
451
+ for (const options of attempts) {
452
+ try {
453
+ const summary = (await ctx.skills.list(options)).find((s) => s.name === skillName)
454
+ if (!summary) continue
455
+ const definition = await ctx.skills.get(skillName, options)
456
+ return { summary, definition }
457
+ } catch {
458
+ // try the next lookup shape
459
+ }
460
+ }
461
+ return null
462
+ }
463
+ }
464
+
465
+ export function apply(ctx, config) {
466
+ const cfg = { ...config }
467
+ if (cfg.enabled === false) return
468
+
469
+ // Fill the same defaults the schema would, so the settings API can report
470
+ // effective values even when it is called from a bare config object.
471
+ cfg.weights = { ...DEFAULT_WEIGHTS, ...(cfg.weights ?? {}) }
472
+ cfg.thresholds = { ...DEFAULT_THRESHOLDS, ...(cfg.thresholds ?? {}) }
473
+ // Settings saved from the browser page win over the profile config; they are
474
+ // applied before the analyzer reads the object so a reload and a live edit
475
+ // behave the same way.
476
+ DIAGNOSTICS.routes = []
477
+ DIAGNOSTICS.tools = []
478
+ DIAGNOSTICS.warnings = []
479
+ DIAGNOSTICS.inspectProvider = false
480
+ let settingsDir = null
481
+ try {
482
+ settingsDir = resolveDataDir(cfg)
483
+ const saved = loadSettingsSync(settingsDir)
484
+ applySettings(cfg, saved)
485
+ DIAGNOSTICS.settingsFile = settingsDir ? path.join(settingsDir, SETTINGS_FILE) : null
486
+ DIAGNOSTICS.settings = saved ?? {}
487
+ } catch (error) {
488
+ log(ctx, 'warn', `skill-eval settings load failed: ${error instanceof Error ? error.message : String(error)}`)
489
+ }
490
+
491
+ const analyzer = createAnalyzer(ctx, cfg)
492
+ analyzer.store.onError = (message) => {
493
+ DIAGNOSTICS.warnings.push(`store write failed: ${message}`)
494
+ log(ctx, 'warn', `skill-eval store write failed: ${message}`)
495
+ }
496
+
497
+ const recorder = createRecorder(ctx, cfg, {
498
+ store: analyzer.store,
499
+ resolveSkill: makeResolveSkill(ctx),
500
+ })
501
+
502
+ ctx.effect(
503
+ () => () => {
504
+ try {
505
+ void recorder.dispose()
506
+ } catch {
507
+ // best effort on shutdown
508
+ }
509
+ },
510
+ 'skill-eval: recorder',
511
+ )
512
+
513
+ // A settled turn writes the aggregated report to disk. This is the "result
514
+ // after every run" surface: it costs zero tokens because it never touches the
515
+ // model context.
516
+ const snapshots =
517
+ cfg.reportOnTurnEnd === false
518
+ ? null
519
+ : createSnapshots({
520
+ analyzer,
521
+ dir: cfg.reportDir ? String(cfg.reportDir) : path.join(analyzer.store.dir, 'reports'),
522
+ log: (message) => log(ctx, 'warn', message),
523
+ turnReports: cfg.turnReports !== false,
524
+ turnReportsKeep: cfg.turnReportsKeep ?? 50,
525
+ })
526
+
527
+ if (snapshots) {
528
+ ctx.on('agent/turn-stopping', () => {
529
+ try {
530
+ snapshots.request()
531
+ } catch {
532
+ // intentionally ignored: snapshot request is best effort on turn stop
533
+ }
534
+ })
535
+ ctx.effect(
536
+ () => () => {
537
+ void snapshots.dispose()
538
+ },
539
+ 'skill-eval: snapshots',
540
+ )
541
+ }
542
+
543
+ if (cfg.registerTool !== false) {
544
+ ctx.effect(() => {
545
+ const disposers = []
546
+ DIAGNOSTICS.tools.push('skill_eval_report')
547
+ disposers.push(
548
+ ctx.tools.register({
549
+ name: 'skill_eval_report',
550
+ description:
551
+ 'Report recorded skill execution telemetry with a 0-100 score, an A-F grade and evidence-cited fixes. Use it when the user asks how a skill performed, why it is unreliable, or how to improve a skill. Every fix is backed by a recorded signal (failed loads, post-load tool errors, user corrections, ignored loads, sibling collisions, load latency) or a static authoring check.',
552
+ parameters: REPORT_PARAMETERS,
553
+ output: REPORT_OUTPUT,
554
+ async execute(args) {
555
+ try {
556
+ return await runReport(analyzer, args ?? {}, cfg)
557
+ } catch (error) {
558
+ return `skill_eval_report failed: ${error instanceof Error ? error.message : String(error)}`
559
+ }
560
+ },
561
+ }),
562
+ )
563
+ return () => {
564
+ for (const dispose of disposers) safe(dispose)()
565
+ }
566
+ }, 'skill-eval: report tool')
567
+ }
568
+
569
+ if (cfg.registerReviewTool !== false) {
570
+ ctx.effect(() => {
571
+ DIAGNOSTICS.tools.push('skill_eval_review')
572
+ const dispose = ctx.tools.register({
573
+ name: 'skill_eval_review',
574
+ description:
575
+ 'Screen a proposed skill edit for leakage before applying it (benchmark names, hardcoded answers/dates/paths, task-specific logic) and record the hypothesis + verdict in the audit ledger. Use it before rewriting a skill because of eval feedback.',
576
+ parameters: {
577
+ type: 'object',
578
+ properties: {
579
+ skill: { type: 'string', description: 'skill being edited' },
580
+ dimension: {
581
+ type: 'string',
582
+ enum: ['static', 'reliability', 'effectiveness', 'efficiency', 'mcp', 'lifecycle'],
583
+ description: 'which dimension this edit targets (drives the exploration directive)',
584
+ },
585
+ hypothesis: { type: 'string', description: 'what should improve and how it will be measured' },
586
+ diff: { type: 'string', description: 'the proposed change, as a diff or the new body text' },
587
+ },
588
+ required: ['skill'],
589
+ additionalProperties: false,
590
+ },
591
+ output: {
592
+ schema: { type: 'string', description: 'verdict and reasons' },
593
+ render(_args, value) {
594
+ return [{ type: 'text', text: typeof value === 'string' ? value : JSON.stringify(value) }]
595
+ },
596
+ },
597
+ async execute(args) {
598
+ try {
599
+ const verdict = reviewSkillEdit({
600
+ skill: String(args?.skill ?? ''),
601
+ hypothesis: typeof args?.hypothesis === 'string' ? args.hypothesis : undefined,
602
+ diff: typeof args?.diff === 'string' ? args.diff : undefined,
603
+ })
604
+ await analyzer.store.append({
605
+ t: 'edit',
606
+ v: 1,
607
+ at: Date.now(),
608
+ skill: String(args?.skill ?? ''),
609
+ hypothesis: typeof args?.hypothesis === 'string' ? args.hypothesis.slice(0, 500) : '',
610
+ verdict: verdict.verdict,
611
+ dimension: typeof args?.dimension === 'string' ? args.dimension : null,
612
+ reasons: verdict.reasons.map((reason) => reason.code),
613
+ diffHash: typeof args?.diff === 'string' ? hashHex(args.diff) : null,
614
+ })
615
+ const lines = [`skill_eval_review: **${verdict.verdict}** for \`${args?.skill ?? ''}\``]
616
+ if (verdict.reasons.length === 0) lines.push('no leakage signals found')
617
+ for (const reason of verdict.reasons)
618
+ lines.push(`- [${reason.severity}] ${reason.code}: ${reason.label} — ${reason.detail}`)
619
+ return lines.join('\n')
620
+ } catch (error) {
621
+ return `skill_eval_review failed: ${error instanceof Error ? error.message : String(error)}`
622
+ }
623
+ },
624
+ })
625
+ return () => safe(dispose)
626
+ }, 'skill-eval: review tool')
627
+ }
628
+
629
+ if (cfg.registerCommand !== false) {
630
+ ctx.inject(['commands'], (c) => {
631
+ c.effect(() => {
632
+ const dispose = c.commands.register({
633
+ name: 'skill-eval',
634
+ description: 'show the skill execution report (scores, grades and cited fixes)',
635
+ input: { hint: '[skill-name]' },
636
+ async handler(invocation) {
637
+ const target = String(invocation.rawInput ?? '').trim()
638
+ try {
639
+ const keyword = target.toLowerCase()
640
+ if (keyword === 'accept' || keyword === 'doctor') {
641
+ const report = await analyzer.build()
642
+ const files = await acceptanceFiles(analyzer.store, (file) =>
643
+ import('node:fs/promises').then((fs) => fs.readFile(file, 'utf8')),
644
+ )
645
+ const acceptance = checkInProcessReport(report, { version: PLUGIN_VERSION ?? 'unknown', files })
646
+ return { kind: 'success', text: '\n' + renderAcceptance(acceptance) + '\n' }
647
+ }
648
+ if (keyword.startsWith('history ') || keyword.startsWith('impact ')) {
649
+ const view = keyword.split(' ')[0]
650
+ const skill = target.slice(view.length + 1).trim()
651
+ const text = await runReport(analyzer, { view, skill }, cfg)
652
+ return { kind: 'success', text: `\n${text}\n` }
653
+ }
654
+ if (keyword === 'config') {
655
+ const lines = [
656
+ '## 配置 / effective settings',
657
+ '',
658
+ `override file: \`${path.join(settingsDir ?? '<unknown>', SETTINGS_FILE)}\``,
659
+ '',
660
+ ]
661
+ for (const field of SETTINGS_KEYS) {
662
+ const value = field
663
+ .split('.')
664
+ .reduce((target, key) => (target == null ? undefined : target[key]), cfg)
665
+ lines.push(`- \`${field}\`: **${value === undefined ? '—' : String(value)}**`)
666
+ }
667
+ lines.push('', '浏览器里打开 `<dashboardPath>/config` 可直接修改(保存后立即生效)。')
668
+ return { kind: 'success', text: `\n${lines.join('\n')}\n` }
669
+ }
670
+ if (keyword === 'turns' || keyword === 'boards') {
671
+ const text = await runReport(analyzer, { view: 'turns' }, cfg)
672
+ return { kind: 'success', text: `\n${text}\n` }
673
+ }
674
+ if (keyword === 'dev' || keyword === 'diagnostics') {
675
+ const text = await runReport(analyzer, { view: 'dev' }, cfg)
676
+ return { kind: 'success', text: `\n${text}\n` }
677
+ }
678
+ if (keyword === 'validity' || keyword === 'valid') {
679
+ const text = await runReport(analyzer, { view: 'validity' }, cfg)
680
+ return { kind: 'success', text: `\n${text}\n` }
681
+ }
682
+ if (keyword === 'loop' || keyword.startsWith('loop ')) {
683
+ const rest = keyword.startsWith('loop ') ? target.slice(5).trim() : ''
684
+ if (rest === 'all') {
685
+ const text = await runReport(analyzer, { view: 'loop', all: true }, cfg)
686
+ return { kind: 'success', text: `\n${text}\n` }
687
+ }
688
+ const text = await runReport(analyzer, { view: 'loop', skill: rest || undefined }, cfg)
689
+ return { kind: 'success', text: `\n${text}\n` }
690
+ }
691
+ const view = ['mcp', 'runs', 'standard'].includes(keyword) ? keyword : undefined
692
+ const text = await runReport(analyzer, view ? { view } : { skill: target || undefined }, cfg)
693
+ return { kind: 'success', text: `\n${text}\n` }
694
+ } catch (error) {
695
+ return {
696
+ kind: 'error',
697
+ text: `skill-eval failed: ${error instanceof Error ? error.message : String(error)}`,
698
+ }
699
+ }
700
+ },
701
+ })
702
+ return () => safe(dispose)
703
+ }, 'skill-eval: command')
704
+ })
705
+ }
706
+
707
+ if (cfg.webDashboard !== false) {
708
+ const base = String(cfg.dashboardPath || '/skill-eval').replace(/\/$/, '')
709
+ // The client bundle ships a fixed route contract: data/settings stay under
710
+ // /skill-eval even when the *pages* move, otherwise a custom dashboardPath
711
+ // silently breaks the panel and the native config form.
712
+ const clientBase = '/skill-eval'
713
+ const dataPath = `${clientBase}/data`
714
+
715
+ // The board's dev tab shows what the agent's `view:"dev"` report shows in
716
+ // short form: which generation is loaded, whether the package on disk moved
717
+ // ahead of it (the ESM-cache restart trap), how many routes this generation
718
+ // claimed, and whether the event store is healthy.
719
+ const diagnosticsPayload = () => {
720
+ const diskVersion = packageVersion()
721
+ return {
722
+ moduleVersion: PLUGIN_VERSION ?? null,
723
+ diskVersion,
724
+ skew: Boolean(PLUGIN_VERSION && diskVersion && PLUGIN_VERSION !== diskVersion),
725
+ routes: DIAGNOSTICS.routes.length,
726
+ tools: DIAGNOSTICS.tools.length,
727
+ inspectProvider: Boolean(DIAGNOSTICS.inspectProvider),
728
+ settingsFile: DIAGNOSTICS.settingsFile ?? null,
729
+ warnings: DIAGNOSTICS.warnings.length,
730
+ }
731
+ }
732
+
733
+ const writeReport = async (res) => {
734
+ const report = await analyzer.build()
735
+ res.writeHead(200, {
736
+ 'content-type': 'application/json; charset=utf-8',
737
+ 'cache-control': 'no-store',
738
+ })
739
+ res.end(JSON.stringify({ ...report, dev: diagnosticsPayload() }, null, 2))
740
+ }
741
+
742
+ const bodyTooLarge = () => Object.assign(new Error('request body too large'), { statusCode: 413 })
743
+ const readBody = (req, limit = 64 * 1024) =>
744
+ new Promise((resolve, reject) => {
745
+ const declared = Number(req.headers?.['content-length'])
746
+ if (Number.isFinite(declared) && declared > limit) {
747
+ reject(bodyTooLarge())
748
+ return
749
+ }
750
+ let body = ''
751
+ let size = 0
752
+ let settled = false
753
+ const fail = (error) => {
754
+ if (settled) return
755
+ settled = true
756
+ reject(error)
757
+ if (typeof req.resume === 'function') req.resume()
758
+ }
759
+ if (typeof req.setEncoding === 'function') req.setEncoding('utf8')
760
+ req.on('data', (chunk) => {
761
+ if (settled) return
762
+ size += Buffer.byteLength(chunk)
763
+ if (size > limit) {
764
+ fail(bodyTooLarge())
765
+ return
766
+ }
767
+ body += chunk
768
+ })
769
+ req.on('end', () => {
770
+ if (settled) return
771
+ settled = true
772
+ resolve(body)
773
+ })
774
+ req.on('error', (error) => fail(error))
775
+ })
776
+
777
+ // Plugin routes bypass the host's own trust fence, so settings writes get a
778
+ // minimal same-origin gate here: reject cross-site fetch metadata, reject a
779
+ // mismatched Origin, and require a custom header on the JSON API (which a
780
+ // cross-site simple request cannot send without a preflight it would fail).
781
+ const trustedWrite = (req, { requireHeader = false } = {}) => {
782
+ const site = String(req.headers?.['sec-fetch-site'] ?? '').toLowerCase()
783
+ if (site === 'cross-site') return false
784
+ const origin = req.headers?.origin
785
+ if (origin) {
786
+ try {
787
+ const parsed = new URL(origin)
788
+ if (!req.headers.host || parsed.host !== req.headers.host) return false
789
+ } catch {
790
+ return false
791
+ }
792
+ }
793
+ if (requireHeader && req.headers?.['x-skill-eval-config'] !== '1') return false
794
+ return true
795
+ }
796
+
797
+ const servePageAt = (pageBase) => async (req, res) => {
798
+ try {
799
+ const report = await analyzer.build()
800
+ const lang = resolveLang(req.headers?.['accept-language'], requestLang(req))
801
+ res.writeHead(200, { 'content-type': 'text/html; charset=utf-8', 'cache-control': 'no-store' })
802
+ res.end(renderHtml(report, { base: pageBase, lang }))
803
+ } catch (error) {
804
+ res.writeHead(500, { 'content-type': 'text/plain; charset=utf-8' })
805
+ res.end(`skill-eval dashboard failed: ${error instanceof Error ? error.message : String(error)}`)
806
+ }
807
+ }
808
+
809
+ const handleConfig = async (req, res, url, pageBase = base) => {
810
+ const lang = resolveLang(req.headers?.['accept-language'], url.searchParams.get('lang'))
811
+ const send = (status, html) => {
812
+ res.writeHead(status, { 'content-type': 'text/html; charset=utf-8', 'cache-control': 'no-store' })
813
+ res.end(html)
814
+ }
815
+ try {
816
+ if (req.method === 'POST') {
817
+ if (!trustedWrite(req)) {
818
+ send(403, 'cross-site settings write rejected')
819
+ return
820
+ }
821
+ const body = await readBody(req)
822
+ const submitted = parseFormSettings(body)
823
+ const { values, errors } = normalizeSettings(submitted)
824
+ const weightError = effectiveWeightError(values)
825
+ if (weightError) errors.push(weightError)
826
+ if (errors.length > 0) {
827
+ send(400, renderConfigHtml({ config: cfg, base: pageBase, errors, lang }))
828
+ return
829
+ }
830
+ if (settingsDir) await saveSettings(settingsDir, { ...loadSettingsSync(settingsDir), ...values })
831
+ applySettings(cfg, values)
832
+ analyzer.invalidate()
833
+ DIAGNOSTICS.settings = loadSettingsSync(settingsDir)
834
+ send(200, renderConfigHtml({ config: cfg, base: pageBase, saved: true, lang }))
835
+ return
836
+ }
837
+ send(
838
+ 200,
839
+ renderConfigHtml({
840
+ config: cfg,
841
+ base: pageBase,
842
+ saved: url.searchParams.get('saved') === '1',
843
+ errors: [],
844
+ lang,
845
+ }),
846
+ )
847
+ } catch (error) {
848
+ const status = error && error.statusCode === 413 ? 413 : 500
849
+ send(status, status === 413 ? 'request body too large' : 'config failed')
850
+ }
851
+ }
852
+
853
+ // The JSON API is its own exact route so the popup keeps working even when
854
+ // the page route below cannot be reclaimed after a reload (the host keeps
855
+ // routes registered by a previous instance). The versioned alias is new on
856
+ // every release, so even a hot reload that cannot reclaim `dataPath` gets a
857
+ // working URL for the current generation.
858
+ const effectiveWeightError = (values) => {
859
+ const weights = { ...(cfg.weights ?? {}) }
860
+ for (const [key, value] of Object.entries(values ?? {})) {
861
+ if (key.startsWith('weights.')) weights[key.slice('weights.'.length)] = value
862
+ }
863
+ const sum = ['static', 'reliability', 'effectiveness', 'efficiency'].reduce(
864
+ (total, dimension) => total + (Number(weights[dimension]) || 0),
865
+ 0,
866
+ )
867
+ return sum > 0 ? null : { key: 'weights.total', message: 'weights sum must be positive' }
868
+ }
869
+ const settingsValue = (key) => {
870
+ const [head, leaf] = key.split('.')
871
+ const raw = leaf ? cfg?.[head]?.[leaf] : cfg?.[head]
872
+ return raw === undefined ? null : raw
873
+ }
874
+ const settingsPayload = () => ({
875
+ fields: SETTINGS_FIELDS.map((field) => ({
876
+ ...field,
877
+ labelZh: FIELD_ZH[field.key] ?? field.label,
878
+ value: settingsValue(field.key),
879
+ })),
880
+ values: Object.fromEntries(SETTINGS_FIELDS.map((field) => [field.key, settingsValue(field.key)])),
881
+ version: PLUGIN_VERSION ?? null,
882
+ })
883
+ const readJsonBody = async (req) => {
884
+ const raw = await readBody(req)
885
+ if (!raw) return {}
886
+ try {
887
+ return JSON.parse(raw)
888
+ } catch {
889
+ return Object.fromEntries(new URLSearchParams(raw))
890
+ }
891
+ }
892
+ const handleSettingsApi = async (req, res) => {
893
+ const send = (status, payload) => {
894
+ res.writeHead(status, { 'content-type': 'application/json; charset=utf-8', 'cache-control': 'no-store' })
895
+ res.end(JSON.stringify(payload, null, 2))
896
+ }
897
+ try {
898
+ if (req.method === 'POST') {
899
+ if (!trustedWrite(req, { requireHeader: true })) {
900
+ send(403, { saved: false, errors: [{ key: 'origin', message: 'cross-site settings write rejected' }] })
901
+ return
902
+ }
903
+ const body = await readJsonBody(req)
904
+ const { values, errors } = normalizeSettings(body)
905
+ const weightError = effectiveWeightError(values)
906
+ if (weightError) errors.push(weightError)
907
+ if (errors.length > 0) {
908
+ send(400, { saved: false, errors, values: settingsPayload().values })
909
+ return
910
+ }
911
+ if (settingsDir) await saveSettings(settingsDir, { ...loadSettingsSync(settingsDir), ...values })
912
+ applySettings(cfg, values)
913
+ analyzer.invalidate()
914
+ DIAGNOSTICS.settings = loadSettingsSync(settingsDir)
915
+ send(200, { saved: true, errors: [], values: settingsPayload().values })
916
+ return
917
+ }
918
+ send(200, settingsPayload())
919
+ } catch (error) {
920
+ const status = error && error.statusCode === 413 ? 413 : 500
921
+ send(status, {
922
+ saved: false,
923
+ errors: [{ key: 'server', message: status === 413 ? 'request body too large' : 'settings failed' }],
924
+ })
925
+ }
926
+ }
927
+
928
+ const dataPaths = PLUGIN_VERSION ? [`${dataPath}/${PLUGIN_VERSION}`, dataPath] : [dataPath]
929
+
930
+ // Settings API: the native plugin-config page renders this instead of the
931
+ // standalone HTML form. Versioned alias for the same hot-reload reason.
932
+ const settingsPaths = PLUGIN_VERSION
933
+ ? [`${clientBase}/settings/${PLUGIN_VERSION}`, `${clientBase}/settings`]
934
+ : [`${clientBase}/settings`]
935
+ for (const routePath of settingsPaths)
936
+ ctx.inject(['webServer'], (c) => {
937
+ c.effect(() => {
938
+ let dispose
939
+ try {
940
+ dispose = c.webServer.register({ kind: 'exact', path: routePath, handler: handleSettingsApi })
941
+ DIAGNOSTICS.routes.push(routePath)
942
+ } catch (error) {
943
+ log(
944
+ ctx,
945
+ 'warn',
946
+ `settings route ${routePath} not registered: ${error instanceof Error ? error.message : String(error)}`,
947
+ )
948
+ return () => {}
949
+ }
950
+ return () => safe(dispose)
951
+ }, 'skill-eval: settings route')
952
+ })
953
+ for (const routePath of dataPaths)
954
+ ctx.inject(['webServer'], (c) => {
955
+ c.effect(() => {
956
+ let dispose
957
+ try {
958
+ dispose = c.webServer.register({
959
+ kind: 'exact',
960
+ path: routePath,
961
+ async handler(req, res) {
962
+ try {
963
+ await writeReport(res)
964
+ } catch (error) {
965
+ res.writeHead(500, { 'content-type': 'text/plain; charset=utf-8' })
966
+ res.end(`skill-eval data failed: ${error instanceof Error ? error.message : String(error)}`)
967
+ }
968
+ },
969
+ })
970
+ DIAGNOSTICS.routes.push(routePath)
971
+ } catch (error) {
972
+ log(
973
+ ctx,
974
+ 'warn',
975
+ `data route ${routePath} not registered: ${error instanceof Error ? error.message : String(error)}`,
976
+ )
977
+ return () => {}
978
+ }
979
+ return () => safe(dispose)
980
+ }, 'skill-eval: data route')
981
+ })
982
+
983
+ // Exact routes are matched before any prefix route, so registering the page
984
+ // and the settings page exactly keeps them working even when a previous
985
+ // generation still owns the `${base}` prefix (routes cannot be reclaimed).
986
+ // Exact page routes cannot be reclaimed once an older generation owns them
987
+ // (hot reload replaces the plugin but the host keeps the old route). The
988
+ // versioned alias is therefore the reliable URL for the current generation,
989
+ // and the canonical paths stay as best-effort for a clean start.
990
+ const boardVersion = PLUGIN_VERSION || null
991
+ const versionedBase = boardVersion ? `${base}/board/${boardVersion}` : null
992
+ const configAt = (pageBase) => async (req, res) => {
993
+ const origin = `http://${req.headers?.host ?? 'localhost'}`
994
+ await handleConfig(req, res, new URL(req.url ?? `${pageBase}/config`, origin), pageBase)
995
+ }
996
+ // Each route renders links/actions against its OWN base: the versioned page
997
+ // must post to the versioned config route, otherwise a hot reload sends the
998
+ // save back to a route the previous generation still owns.
999
+ const pageRoutes = [
1000
+ { path: base, handler: servePageAt(base) },
1001
+ { path: `${base}/config`, handler: configAt(base) },
1002
+ ...(versionedBase
1003
+ ? [
1004
+ { path: versionedBase, handler: servePageAt(versionedBase) },
1005
+ { path: `${versionedBase}/config`, handler: configAt(versionedBase) },
1006
+ ]
1007
+ : []),
1008
+ ]
1009
+ for (const pageRoute of pageRoutes)
1010
+ ctx.inject(['webServer'], (c) => {
1011
+ c.effect(() => {
1012
+ let dispose
1013
+ try {
1014
+ dispose = c.webServer.register({ kind: 'exact', path: pageRoute.path, handler: pageRoute.handler })
1015
+ DIAGNOSTICS.routes.push(pageRoute.path)
1016
+ } catch (error) {
1017
+ log(
1018
+ ctx,
1019
+ 'warn',
1020
+ `page route ${pageRoute.path} not registered: ${error instanceof Error ? error.message : String(error)}`,
1021
+ )
1022
+ return () => {}
1023
+ }
1024
+ return () => safe(dispose)
1025
+ }, 'skill-eval: page route')
1026
+ })
1027
+
1028
+ ctx.inject(['webServer'], (c) => {
1029
+ c.effect(() => {
1030
+ let dispose
1031
+ try {
1032
+ dispose = c.webServer.register({
1033
+ kind: 'prefix',
1034
+ path: base,
1035
+ async handler(req, res) {
1036
+ let origin = 'http://localhost'
1037
+ try {
1038
+ origin = c.webServer.getOrigin()
1039
+ } catch {
1040
+ // keep the fallback origin
1041
+ }
1042
+ const url = new URL(req.url ?? base, origin)
1043
+ try {
1044
+ if (url.pathname.replace(/\/$/, '').endsWith('/data')) {
1045
+ await writeReport(res)
1046
+ return
1047
+ }
1048
+ if (url.pathname.replace(/\/$/, '').endsWith('/config')) {
1049
+ await handleConfig(req, res, url)
1050
+ return
1051
+ }
1052
+ const report = await analyzer.build()
1053
+ const lang = resolveLang(req.headers?.['accept-language'], url.searchParams.get('lang'))
1054
+ res.writeHead(200, { 'content-type': 'text/html; charset=utf-8', 'cache-control': 'no-store' })
1055
+ res.end(renderHtml(report, { base, lang }))
1056
+ } catch (error) {
1057
+ res.writeHead(500, { 'content-type': 'text/plain; charset=utf-8' })
1058
+ res.end(`skill-eval dashboard failed: ${error instanceof Error ? error.message : String(error)}`)
1059
+ }
1060
+ },
1061
+ })
1062
+ DIAGNOSTICS.routes.push(base)
1063
+ } catch (error) {
1064
+ log(
1065
+ ctx,
1066
+ 'warn',
1067
+ `dashboard route ${base} not registered: ${error instanceof Error ? error.message : String(error)}`,
1068
+ )
1069
+ return () => {}
1070
+ }
1071
+ return () => safe(dispose)
1072
+ }, 'skill-eval: dashboard')
1073
+ })
1074
+ }
1075
+
1076
+ // Developer-mode surface: the eval data registers next to the harness's own
1077
+ // Service/Event/Config providers, so `cordis_inspect_query` can read it.
1078
+ if (cfg.inspectProvider !== false) {
1079
+ ctx.inject(['cordisInspect'], (c) => {
1080
+ c.effect(() => {
1081
+ try {
1082
+ const dispose = c.cordisInspect.register(createInspectProvider(analyzer))
1083
+ DIAGNOSTICS.inspectProvider = true
1084
+ log(ctx, 'info', 'inspect provider "skill-eval" registered')
1085
+ return dispose
1086
+ } catch (error) {
1087
+ log(ctx, 'warn', `inspect provider not registered: ${error instanceof Error ? error.message : String(error)}`)
1088
+ return () => {}
1089
+ }
1090
+ }, 'skill-eval: inspect provider')
1091
+ })
1092
+ }
1093
+
1094
+ const SETTINGS_KEYS = SETTINGS_FIELDS.map((field) => field.key)
1095
+
1096
+ if (cfg.promptHint !== false)
1097
+ ctx.inject(['systemPrompt'], (c) => {
1098
+ c.effect(() => {
1099
+ const stop = c.systemPrompt.section({
1100
+ name: 'skill-eval:usage',
1101
+ order: 118,
1102
+ text: () =>
1103
+ 'Skill telemetry is recorded for every `skill` load. When asked how a skill performed or how to improve it, call `skill_eval_report` (pass {"skill":"<name>"} for one skill) and act on the cited fixes instead of guessing. Before editing a skill named in the report, screen the change with `skill_eval_review` (leakage critic) and record the hypothesis; ask `skill_eval_report` with view:"loop" for the next step and view:"validity" for what the evidence can and cannot support.',
1104
+ })
1105
+ return () => safe(stop)
1106
+ }, 'skill-eval: prompt section')
1107
+ })
1108
+
1109
+ log(ctx, 'info', `skill-eval: recording enabled, data dir ${analyzer.store.dir}`)
1110
+ }
1111
+