dsh-skill-eval 0.0.0-stage → 1.20.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +718 -0
- package/LICENSE +21 -0
- package/README.md +207 -2
- package/README.zh.md +164 -0
- package/client.js +2791 -0
- package/cordis.patch.yml +9 -0
- package/docs/ACCEPTANCE.md +71 -0
- package/docs/AUDIT.md +82 -0
- package/docs/CODE-REVIEW.md +49 -0
- package/docs/COMPARISON.md +33 -0
- package/docs/DETAIL.md +468 -0
- package/docs/DEV-RELOAD.md +114 -0
- package/docs/ENTRYPOINTS.md +65 -0
- package/docs/RELEASE.md +44 -0
- package/docs/RRSI-NOTES.md +129 -0
- package/docs/SCENARIOS.md +91 -0
- package/docs/UI-REVIEW.md +115 -0
- package/docs/WORKFLOW.md +108 -0
- package/icon.svg +9 -0
- package/index.js +1173 -0
- package/lib/accept.js +350 -0
- package/lib/advice.js +557 -0
- package/lib/critic.js +101 -0
- package/lib/dev.js +148 -0
- package/lib/history.js +141 -0
- package/lib/html.js +17 -0
- package/lib/inspect.js +142 -0
- package/lib/integrity.js +46 -0
- package/lib/loop.js +195 -0
- package/lib/random.js +29 -0
- package/lib/recorder.js +644 -0
- package/lib/render.js +580 -0
- package/lib/report.js +689 -0
- package/lib/score.js +565 -0
- package/lib/settings.js +415 -0
- package/lib/snapshot.js +250 -0
- package/lib/steps.js +141 -0
- package/lib/store.js +240 -0
- package/lib/validity.js +125 -0
- package/locale/en.json +6 -0
- package/locale/zh.json +6 -0
- package/package.json +96 -4
- package/scripts/accept.mjs +227 -0
- package/scripts/build-client.mjs +59 -0
- package/scripts/build.mjs +81 -0
- package/scripts/doctor.mjs +234 -0
- package/scripts/lib.mjs +87 -0
- package/scripts/verify-live.mjs +178 -0
package/lib/dev.js
ADDED
|
@@ -0,0 +1,148 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Developer diagnostics: the "is this thing actually working" view.
|
|
3
|
+
*
|
|
4
|
+
* It answers the questions that cost the most time while developing the plugin:
|
|
5
|
+
* which module generation is actually loaded, whether the on-disk package moved
|
|
6
|
+
* ahead of it (the ESM-cache restart trap), where the data lives, which routes
|
|
7
|
+
* this generation managed to claim, and whether the event store is healthy.
|
|
8
|
+
*
|
|
9
|
+
* Pure: everything is passed in.
|
|
10
|
+
*/
|
|
11
|
+
|
|
12
|
+
const TYPE_ORDER = ['skill', 'window', 'static', 'edit']
|
|
13
|
+
|
|
14
|
+
export function buildDevReport(input = {}) {
|
|
15
|
+
const {
|
|
16
|
+
moduleVersion = null,
|
|
17
|
+
diskVersion = null,
|
|
18
|
+
dataDir = null,
|
|
19
|
+
settingsFile = null,
|
|
20
|
+
settings = {},
|
|
21
|
+
config = {},
|
|
22
|
+
routes = [],
|
|
23
|
+
records = [],
|
|
24
|
+
store = {},
|
|
25
|
+
tools = [],
|
|
26
|
+
inspectProvider = null,
|
|
27
|
+
warnings = [],
|
|
28
|
+
now = Date.now(),
|
|
29
|
+
} = input
|
|
30
|
+
|
|
31
|
+
const counts = {}
|
|
32
|
+
let lastAt = null
|
|
33
|
+
for (const record of records) {
|
|
34
|
+
if (!record || typeof record.t !== 'string') continue
|
|
35
|
+
counts[record.t] = (counts[record.t] ?? 0) + 1
|
|
36
|
+
const at = Number(record.at)
|
|
37
|
+
if (Number.isFinite(at) && (lastAt === null || at > lastAt)) lastAt = at
|
|
38
|
+
}
|
|
39
|
+
const types = Object.keys(counts).sort((a, b) => {
|
|
40
|
+
const ai = TYPE_ORDER.indexOf(a)
|
|
41
|
+
const bi = TYPE_ORDER.indexOf(b)
|
|
42
|
+
if (ai === -1 && bi === -1) return a.localeCompare(b)
|
|
43
|
+
if (ai === -1) return 1
|
|
44
|
+
if (bi === -1) return -1
|
|
45
|
+
return ai - bi
|
|
46
|
+
})
|
|
47
|
+
|
|
48
|
+
const skew = typeof moduleVersion === 'string' && typeof diskVersion === 'string' && moduleVersion !== diskVersion
|
|
49
|
+
const findings = []
|
|
50
|
+
if (skew) {
|
|
51
|
+
findings.push({
|
|
52
|
+
code: 'module-skew',
|
|
53
|
+
severity: 'critical',
|
|
54
|
+
detail: `loaded module is ${moduleVersion} but package.json is ${diskVersion} — restart DeepSeek Harness to load the new code (disable/enable only re-runs apply on the cached module)`,
|
|
55
|
+
})
|
|
56
|
+
}
|
|
57
|
+
if (store.failed) {
|
|
58
|
+
findings.push({
|
|
59
|
+
code: 'store-write-failed',
|
|
60
|
+
severity: 'critical',
|
|
61
|
+
detail: `event store reported a failed write: ${String(store.failed)}`,
|
|
62
|
+
})
|
|
63
|
+
}
|
|
64
|
+
if (records.length === 0) {
|
|
65
|
+
findings.push({ code: 'no-telemetry', severity: 'warning', detail: 'no events recorded yet; load a skill once' })
|
|
66
|
+
}
|
|
67
|
+
if (tools.length === 0) {
|
|
68
|
+
findings.push({
|
|
69
|
+
code: 'no-tools',
|
|
70
|
+
severity: 'warning',
|
|
71
|
+
detail: 'no tools registered (registerTool disabled or registration failed)',
|
|
72
|
+
})
|
|
73
|
+
}
|
|
74
|
+
if (!inspectProvider) {
|
|
75
|
+
findings.push({
|
|
76
|
+
code: 'no-inspect-provider',
|
|
77
|
+
severity: 'info',
|
|
78
|
+
detail: 'Cordis Inspect provider not registered (registry unavailable?)',
|
|
79
|
+
})
|
|
80
|
+
}
|
|
81
|
+
if (!routes.some((route) => route.indexOf('/settings') !== -1)) {
|
|
82
|
+
findings.push({
|
|
83
|
+
code: 'no-settings-route',
|
|
84
|
+
severity: 'warning',
|
|
85
|
+
detail: 'settings API route is not registered on this generation',
|
|
86
|
+
})
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
return {
|
|
90
|
+
generatedAt: new Date(now).toISOString(),
|
|
91
|
+
versions: { module: moduleVersion, disk: diskVersion, skew },
|
|
92
|
+
dataDir,
|
|
93
|
+
settingsFile,
|
|
94
|
+
settingsKeys: Object.keys(settings ?? {}).length,
|
|
95
|
+
config: {
|
|
96
|
+
weights: config.weights ?? null,
|
|
97
|
+
thresholds: config.thresholds ?? null,
|
|
98
|
+
minSamples: config.minSamples ?? null,
|
|
99
|
+
tokenBudgetPerLoad: config.tokenBudgetPerLoad ?? null,
|
|
100
|
+
adviceNoiseFloor: config.adviceNoiseFloor ?? null,
|
|
101
|
+
enablePopup: null,
|
|
102
|
+
},
|
|
103
|
+
store: {
|
|
104
|
+
dir: store.dir ?? dataDir,
|
|
105
|
+
failed: store.failed ?? null,
|
|
106
|
+
queued: Number.isFinite(store.queued) ? store.queued : null,
|
|
107
|
+
records: records.length,
|
|
108
|
+
counts,
|
|
109
|
+
types,
|
|
110
|
+
lastEventAt: lastAt,
|
|
111
|
+
lastEventAgeMs: lastAt === null ? null : Math.max(0, now - lastAt),
|
|
112
|
+
},
|
|
113
|
+
routes: [...routes].sort(),
|
|
114
|
+
tools: [...tools].sort(),
|
|
115
|
+
inspectProvider,
|
|
116
|
+
warnings: warnings.slice(-8),
|
|
117
|
+
findings,
|
|
118
|
+
}
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
export function renderDevMarkdown(dev) {
|
|
122
|
+
if (!dev) return '## Developer diagnostics\n\nno data'
|
|
123
|
+
const lines = ['## Developer diagnostics', '']
|
|
124
|
+
lines.push(
|
|
125
|
+
`module: **${dev.versions.module ?? '?'}** · on disk: **${dev.versions.disk ?? '?'}** · skew: **${
|
|
126
|
+
dev.versions.skew ? 'YES — restart needed' : 'no'
|
|
127
|
+
}**`,
|
|
128
|
+
)
|
|
129
|
+
lines.push(`data: \`${dev.dataDir ?? '?'}\``)
|
|
130
|
+
if (dev.settingsFile) lines.push(`settings override: \`${dev.settingsFile}\` (${dev.settingsKeys} key(s))`)
|
|
131
|
+
lines.push(
|
|
132
|
+
`store: ${dev.store.records} record(s) · types: ${dev.store.types.map((type) => `${type}×${dev.store.counts[type]}`).join(', ') || 'none'}`,
|
|
133
|
+
)
|
|
134
|
+
if (dev.store.lastEventAgeMs !== null) lines.push(`last event: ${Math.round(dev.store.lastEventAgeMs / 1000)}s ago`)
|
|
135
|
+
lines.push(`routes: ${dev.routes.length} exact/prefix path(s)`)
|
|
136
|
+
lines.push(`tools: ${dev.tools.join(', ') || 'none'}`)
|
|
137
|
+
if (dev.findings.length > 0) {
|
|
138
|
+
lines.push('', '### Findings')
|
|
139
|
+
for (const finding of dev.findings) lines.push(`- [${finding.severity}] **${finding.code}** — ${finding.detail}`)
|
|
140
|
+
} else {
|
|
141
|
+
lines.push('', '### Findings', '- nothing suspicious')
|
|
142
|
+
}
|
|
143
|
+
if (dev.warnings.length > 0) {
|
|
144
|
+
lines.push('', '### Recent warnings')
|
|
145
|
+
for (const warning of dev.warnings) lines.push(`- ${warning}`)
|
|
146
|
+
}
|
|
147
|
+
return lines.join('\n')
|
|
148
|
+
}
|
package/lib/history.js
ADDED
|
@@ -0,0 +1,141 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Time-aware answers: "when did this skill go bad" and "did my last edit help".
|
|
3
|
+
*
|
|
4
|
+
* Both work from the durable window records, so they need no new collection.
|
|
5
|
+
* Judgements stay conservative: a delta smaller than one window of spread is
|
|
6
|
+
* reported as inconclusive rather than as an improvement.
|
|
7
|
+
*/
|
|
8
|
+
|
|
9
|
+
import { formatTimestamp } from './html.js'
|
|
10
|
+
import { foldSamples } from './store.js'
|
|
11
|
+
|
|
12
|
+
const rate = (list, predicate) => (list.length === 0 ? null : list.filter(predicate).length / list.length)
|
|
13
|
+
|
|
14
|
+
function shapeLoad(sample) {
|
|
15
|
+
const w = sample.window ?? {}
|
|
16
|
+
return {
|
|
17
|
+
at: sample.at,
|
|
18
|
+
ok: sample.ok === true,
|
|
19
|
+
loadMs: sample.loadMs ?? null,
|
|
20
|
+
calls: w.toolCalls ?? null,
|
|
21
|
+
errors: w.errors ?? null,
|
|
22
|
+
corrections: Array.isArray(w.corrections) ? w.corrections.length : 0,
|
|
23
|
+
otherSkills: Array.isArray(w.otherSkills) ? w.otherSkills.length : 0,
|
|
24
|
+
loopErrors: w.loopErrors ?? 0,
|
|
25
|
+
turn: sample.turn ?? null,
|
|
26
|
+
}
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
/** Chronological load history for one skill, with a half-over-half trend. */
|
|
30
|
+
export function buildHistory(records, skill) {
|
|
31
|
+
const group = foldSamples(records).find((item) => item.skill === skill)
|
|
32
|
+
const samples = group ? group.samples : []
|
|
33
|
+
const loads = samples.map(shapeLoad)
|
|
34
|
+
const withWindow = loads.filter((load) => load.calls !== null)
|
|
35
|
+
const split = Math.floor(withWindow.length / 2)
|
|
36
|
+
const before = withWindow.slice(0, split)
|
|
37
|
+
const after = withWindow.slice(split)
|
|
38
|
+
const beforeRate = rate(before, (load) => load.errors > 0 || load.corrections > 0)
|
|
39
|
+
const afterRate = rate(after, (load) => load.errors > 0 || load.corrections > 0)
|
|
40
|
+
let trend = 'not-enough-data'
|
|
41
|
+
let note = `need at least 4 closed loads to compare halves (have ${withWindow.length})`
|
|
42
|
+
if (before.length >= 2 && after.length >= 2 && beforeRate !== null && afterRate !== null) {
|
|
43
|
+
const delta = beforeRate - afterRate
|
|
44
|
+
trend = delta > 0.2 ? 'improving' : delta < -0.2 ? 'degrading' : 'flat'
|
|
45
|
+
note = `problem rate ${Math.round(beforeRate * 100)}% → ${Math.round(afterRate * 100)}% (${before.length} → ${after.length} loads)`
|
|
46
|
+
}
|
|
47
|
+
return {
|
|
48
|
+
skill,
|
|
49
|
+
loads,
|
|
50
|
+
closed: withWindow.length,
|
|
51
|
+
trend,
|
|
52
|
+
note,
|
|
53
|
+
firstAt: loads[0]?.at ?? null,
|
|
54
|
+
lastAt: loads[loads.length - 1]?.at ?? null,
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
/**
|
|
59
|
+
* Pair each screened edit with the loads recorded before and after it.
|
|
60
|
+
* Edits without a timestamp or without two closed loads on a side are
|
|
61
|
+
* inconclusive rather than silently "improved".
|
|
62
|
+
*/
|
|
63
|
+
export function buildImpact(records, skill) {
|
|
64
|
+
const group = foldSamples(records).find((item) => item.skill === skill)
|
|
65
|
+
const samples = group ? group.samples : []
|
|
66
|
+
const edits = records
|
|
67
|
+
.filter((record) => record && record.t === 'edit' && record.skill === skill && Number.isFinite(record.at))
|
|
68
|
+
.sort((a, b) => a.at - b.at)
|
|
69
|
+
const comparisons = edits.map((edit) => {
|
|
70
|
+
const before = samples.filter((sample) => sample.at < edit.at)
|
|
71
|
+
const after = samples.filter((sample) => sample.at > edit.at)
|
|
72
|
+
const rateOf = (list) => rate(list, (sample) => (sample.window?.errors ?? 0) > 0)
|
|
73
|
+
const beforeRate = rateOf(before)
|
|
74
|
+
const afterRate = rateOf(after)
|
|
75
|
+
const enough = before.length >= 2 && after.length >= 2
|
|
76
|
+
let verdict = 'inconclusive'
|
|
77
|
+
let delta = null
|
|
78
|
+
if (enough && beforeRate !== null && afterRate !== null) {
|
|
79
|
+
delta = Math.round((beforeRate - afterRate) * 1000) / 1000
|
|
80
|
+
verdict = delta > 0.15 ? 'improved' : delta < -0.15 ? 'degraded' : 'flat'
|
|
81
|
+
}
|
|
82
|
+
return {
|
|
83
|
+
at: edit.at,
|
|
84
|
+
dimension: edit.dimension ?? null,
|
|
85
|
+
hypothesis: edit.hypothesis ?? '',
|
|
86
|
+
screened: edit.verdict ?? null,
|
|
87
|
+
before: { loads: before.length, errorRate: beforeRate },
|
|
88
|
+
after: { loads: after.length, errorRate: afterRate },
|
|
89
|
+
delta,
|
|
90
|
+
verdict,
|
|
91
|
+
note: enough
|
|
92
|
+
? `error rate ${beforeRate === null ? '—' : Math.round(beforeRate * 100) + '%'} → ${
|
|
93
|
+
afterRate === null ? '—' : Math.round(afterRate * 100) + '%'
|
|
94
|
+
} (${before.length} / ${after.length} loads)`
|
|
95
|
+
: `needs 2 closed loads on each side (have ${before.length} / ${after.length})`,
|
|
96
|
+
}
|
|
97
|
+
})
|
|
98
|
+
return { skill, edits: edits.length, comparisons }
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
export function renderHistoryMarkdown(history) {
|
|
102
|
+
if (!history || history.loads.length === 0) return `No closed window recorded for \`${history?.skill ?? '?'}\` yet.`
|
|
103
|
+
const lines = [
|
|
104
|
+
`## History — \`${history.skill}\``,
|
|
105
|
+
'',
|
|
106
|
+
`loads: **${history.loads.length}** (${history.closed} closed) · trend: **${history.trend}**`,
|
|
107
|
+
history.note,
|
|
108
|
+
'',
|
|
109
|
+
'| at | ok | calls | errors | corrections | ms | turn |',
|
|
110
|
+
'|---|---|---|---|---|---|---|',
|
|
111
|
+
]
|
|
112
|
+
for (const load of history.loads.slice(-15)) {
|
|
113
|
+
const when = formatTimestamp(load.at)
|
|
114
|
+
lines.push(
|
|
115
|
+
`| ${when} | ${load.ok ? 'yes' : 'no'} | ${load.calls ?? '—'} | ${load.errors ?? '—'} | ${load.corrections} | ${
|
|
116
|
+
load.loadMs ?? '—'
|
|
117
|
+
} | ${load.turn ?? '—'} |`,
|
|
118
|
+
)
|
|
119
|
+
}
|
|
120
|
+
return lines.join('\n')
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
export function renderImpactMarkdown(impact) {
|
|
124
|
+
if (!impact || impact.edits === 0)
|
|
125
|
+
return `No screened edit recorded for \`${impact?.skill ?? '?'}\` yet. Use skill_eval_review before changing it.`
|
|
126
|
+
const lines = [`## Edit impact — \`${impact.skill}\``, '', `${impact.edits} screened edit(s)`, '']
|
|
127
|
+
for (const comparison of impact.comparisons) {
|
|
128
|
+
const when = formatTimestamp(comparison.at)
|
|
129
|
+
lines.push(`### ${when} · ${comparison.dimension ?? 'unspecified'} · ${comparison.verdict}`)
|
|
130
|
+
if (comparison.hypothesis) lines.push(`hypothesis: ${comparison.hypothesis}`)
|
|
131
|
+
lines.push(`- ${comparison.note}`)
|
|
132
|
+
if (comparison.delta !== null) {
|
|
133
|
+
lines.push(
|
|
134
|
+
`- error-rate delta: ${comparison.delta > 0 ? '-' : '+'}${Math.abs(Math.round(comparison.delta * 100))}pp`,
|
|
135
|
+
)
|
|
136
|
+
}
|
|
137
|
+
lines.push('')
|
|
138
|
+
}
|
|
139
|
+
lines.push('Judgement rule: only ≥2 closed loads on each side and a >15pp delta counts as improved/degraded.')
|
|
140
|
+
return lines.join('\n')
|
|
141
|
+
}
|
package/lib/html.js
ADDED
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
/** Escape text for the few HTML strings built outside React. */
|
|
2
|
+
/** Format a finite epoch-ms timestamp, or an em dash when it is not a valid date. */
|
|
3
|
+
export function formatTimestamp(value) {
|
|
4
|
+
const n = Number(value)
|
|
5
|
+
if (!Number.isFinite(n)) return '—'
|
|
6
|
+
const date = new Date(n)
|
|
7
|
+
if (Number.isNaN(date.getTime())) return '—'
|
|
8
|
+
return date.toISOString().replace('T', ' ').slice(0, 19)
|
|
9
|
+
}
|
|
10
|
+
|
|
11
|
+
export function escapeHtml(value) {
|
|
12
|
+
return String(value ?? '')
|
|
13
|
+
.replace(/&/g, '&')
|
|
14
|
+
.replace(/</g, '<')
|
|
15
|
+
.replace(/>/g, '>')
|
|
16
|
+
.replace(/"/g, '"')
|
|
17
|
+
}
|
package/lib/inspect.js
ADDED
|
@@ -0,0 +1,142 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Cordis Inspect provider: the developer-mode surface of this plugin.
|
|
3
|
+
*
|
|
4
|
+
* `cordis_inspect_list` / `cordis_inspect_query` are the read-only capabilities
|
|
5
|
+
* the harness exposes to a developer (and to the model when the inspector
|
|
6
|
+
* bundle is on). Registering here means the eval data is queryable next to the
|
|
7
|
+
* harness's own Service/Event/Config providers instead of only through HTTP.
|
|
8
|
+
*/
|
|
9
|
+
|
|
10
|
+
const EMPTY_INPUT = { type: 'object', properties: {}, additionalProperties: false }
|
|
11
|
+
const ANY_OUTPUT = { description: 'JSON data owned by the skill-eval inspect provider.' }
|
|
12
|
+
|
|
13
|
+
const SKILL_INPUT = {
|
|
14
|
+
type: 'object',
|
|
15
|
+
properties: {
|
|
16
|
+
skill: { type: 'string', description: 'Exact skill name.' },
|
|
17
|
+
},
|
|
18
|
+
required: ['skill'],
|
|
19
|
+
additionalProperties: false,
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
function adviceIds(advice) {
|
|
23
|
+
return (advice ?? []).map((item) => ({
|
|
24
|
+
id: item.id,
|
|
25
|
+
severity: item.severity,
|
|
26
|
+
dimension: item.dimension,
|
|
27
|
+
}))
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
/** Bounded summary row: numbers plus advice identity, never the full prose. */
|
|
31
|
+
function summarize(entry) {
|
|
32
|
+
const score = entry.score ?? {}
|
|
33
|
+
const dims = score.dimensions ?? {}
|
|
34
|
+
return {
|
|
35
|
+
skill: entry.skill,
|
|
36
|
+
score: score.total,
|
|
37
|
+
grade: score.grade,
|
|
38
|
+
confidence: score.confidence,
|
|
39
|
+
provisional: score.provisional,
|
|
40
|
+
loads: entry.stats?.loads ?? 0,
|
|
41
|
+
closed: entry.stats?.closed ?? 0,
|
|
42
|
+
ci: score.ci ?? null,
|
|
43
|
+
dimensions: {
|
|
44
|
+
static: dims.static,
|
|
45
|
+
reliability: dims.reliability,
|
|
46
|
+
effectiveness: dims.effectiveness,
|
|
47
|
+
efficiency: dims.efficiency,
|
|
48
|
+
},
|
|
49
|
+
advice: adviceIds(entry.advice),
|
|
50
|
+
adviceBudget: entry.adviceBudget ?? null,
|
|
51
|
+
exploration: entry.exploration ?? null,
|
|
52
|
+
}
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
export function createInspectProvider(analyzer) {
|
|
56
|
+
return {
|
|
57
|
+
manifest: {
|
|
58
|
+
id: 'skill-eval',
|
|
59
|
+
description:
|
|
60
|
+
'dsh-skill-eval: recorded skill and MCP execution telemetry with 0-100 scores, A-F grades, confidence and cited fixes.',
|
|
61
|
+
methods: [
|
|
62
|
+
{
|
|
63
|
+
name: 'summary',
|
|
64
|
+
description: 'Every skill with score, grade, confidence, four dimensions and advice identities.',
|
|
65
|
+
inputSchema: EMPTY_INPUT,
|
|
66
|
+
outputSchema: ANY_OUTPUT,
|
|
67
|
+
},
|
|
68
|
+
{
|
|
69
|
+
name: 'skill',
|
|
70
|
+
description: 'One skill entry in full, including its advice text.',
|
|
71
|
+
inputSchema: SKILL_INPUT,
|
|
72
|
+
outputSchema: ANY_OUTPUT,
|
|
73
|
+
},
|
|
74
|
+
{
|
|
75
|
+
name: 'mcp',
|
|
76
|
+
description: 'Per-MCP-server calls, error rate, average latency and health score.',
|
|
77
|
+
inputSchema: EMPTY_INPUT,
|
|
78
|
+
outputSchema: ANY_OUTPUT,
|
|
79
|
+
},
|
|
80
|
+
{
|
|
81
|
+
name: 'runs',
|
|
82
|
+
description: 'Recent per-session runs: loads, tool calls, errors, corrections and skills.',
|
|
83
|
+
inputSchema: EMPTY_INPUT,
|
|
84
|
+
outputSchema: ANY_OUTPUT,
|
|
85
|
+
},
|
|
86
|
+
{
|
|
87
|
+
name: 'edits',
|
|
88
|
+
description: 'Audit ledger of reviewed skill edits: hypothesis, leakage verdict and reasons.',
|
|
89
|
+
inputSchema: EMPTY_INPUT,
|
|
90
|
+
outputSchema: ANY_OUTPUT,
|
|
91
|
+
},
|
|
92
|
+
{
|
|
93
|
+
name: 'standard',
|
|
94
|
+
description: 'The active grading standard: normalised weights, grade floors and minimum samples.',
|
|
95
|
+
inputSchema: EMPTY_INPUT,
|
|
96
|
+
outputSchema: ANY_OUTPUT,
|
|
97
|
+
},
|
|
98
|
+
],
|
|
99
|
+
},
|
|
100
|
+
|
|
101
|
+
async query(method, input) {
|
|
102
|
+
if (method === 'summary') {
|
|
103
|
+
const report = await analyzer.build()
|
|
104
|
+
return {
|
|
105
|
+
generatedAt: report.generatedAt,
|
|
106
|
+
dataDir: report.dataDir,
|
|
107
|
+
totals: report.totals,
|
|
108
|
+
scoring: report.scoring ?? null,
|
|
109
|
+
skills: report.skills.map(summarize),
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
if (method === 'skill') {
|
|
113
|
+
const name = input && typeof input.skill === 'string' ? input.skill : ''
|
|
114
|
+
if (name.length === 0) throw new Error('skill-eval inspect: "skill" is required')
|
|
115
|
+
const entry = await analyzer.buildOne(name)
|
|
116
|
+
if (!entry) throw new Error(`skill-eval inspect: no evidence recorded for "${name}"`)
|
|
117
|
+
return entry
|
|
118
|
+
}
|
|
119
|
+
if (method === 'mcp') {
|
|
120
|
+
const report = await analyzer.build()
|
|
121
|
+
return {
|
|
122
|
+
generatedAt: report.generatedAt,
|
|
123
|
+
mcpServers: report.mcpServers ?? [],
|
|
124
|
+
advice: report.mcpAdvice ?? [],
|
|
125
|
+
}
|
|
126
|
+
}
|
|
127
|
+
if (method === 'edits') {
|
|
128
|
+
const report = await analyzer.build()
|
|
129
|
+
return { generatedAt: report.generatedAt, edits: report.edits ?? [] }
|
|
130
|
+
}
|
|
131
|
+
if (method === 'runs') {
|
|
132
|
+
const report = await analyzer.build()
|
|
133
|
+
return { generatedAt: report.generatedAt, runs: report.runs ?? [] }
|
|
134
|
+
}
|
|
135
|
+
if (method === 'standard') {
|
|
136
|
+
const report = await analyzer.build()
|
|
137
|
+
return { scoring: report.scoring ?? null }
|
|
138
|
+
}
|
|
139
|
+
throw new Error(`skill-eval inspect: unknown method "${method}"`)
|
|
140
|
+
},
|
|
141
|
+
}
|
|
142
|
+
}
|
package/lib/integrity.js
ADDED
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Reference integrity: a skill that promises `references/forms.md` or
|
|
3
|
+
* `scripts/build.py` must ship those files. Broken promises are the most
|
|
4
|
+
* common "the skill told me to read a file that does not exist" failure, and
|
|
5
|
+
* they are cheap to check at load time (we already know the base directory).
|
|
6
|
+
*/
|
|
7
|
+
|
|
8
|
+
const REFERENCE_PATTERN = /(?:references\/[A-Za-z0-9._\-/]+|scripts\/[A-Za-z0-9._\-/]+|assets\/[A-Za-z0-9._\-/]+)/g
|
|
9
|
+
|
|
10
|
+
/** All repo-relative paths the body points at, deduplicated and bounded. */
|
|
11
|
+
export function referencedPaths(content, limit = 40) {
|
|
12
|
+
if (typeof content !== 'string' || content.length === 0) return []
|
|
13
|
+
const found = new Set()
|
|
14
|
+
for (const match of content.matchAll(REFERENCE_PATTERN)) {
|
|
15
|
+
const path = match[0].replace(/[.,;:))】"'`]+$/, '')
|
|
16
|
+
if (path.includes('..')) continue
|
|
17
|
+
found.add(path)
|
|
18
|
+
if (found.size >= limit) break
|
|
19
|
+
}
|
|
20
|
+
return [...found]
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
/**
|
|
24
|
+
* Check the referenced paths against the skill directory.
|
|
25
|
+
*
|
|
26
|
+
* @param {{baseDir?: string|null, content?: string, stat?: Function}} input
|
|
27
|
+
* `stat` is injected so the check stays testable (and the plugin can keep it
|
|
28
|
+
* best-effort in a host without fs access).
|
|
29
|
+
* @returns {Promise<{checked: number, missing: string[], ok: boolean}|null>}
|
|
30
|
+
*/
|
|
31
|
+
export async function checkReferences({ baseDir, content, stat } = {}) {
|
|
32
|
+
const paths = referencedPaths(content)
|
|
33
|
+
if (paths.length === 0) return { checked: 0, missing: [], ok: true }
|
|
34
|
+
if (typeof baseDir !== 'string' || baseDir.length === 0) return null
|
|
35
|
+
if (typeof stat !== 'function') return null
|
|
36
|
+
const missing = []
|
|
37
|
+
for (const relative of paths) {
|
|
38
|
+
const absolute = `${baseDir.replace(/\/$/, '')}/${relative}`
|
|
39
|
+
try {
|
|
40
|
+
await stat(absolute)
|
|
41
|
+
} catch {
|
|
42
|
+
missing.push(relative)
|
|
43
|
+
}
|
|
44
|
+
}
|
|
45
|
+
return { checked: paths.length, missing: missing.slice(0, 20), ok: missing.length === 0 }
|
|
46
|
+
}
|
package/lib/loop.js
ADDED
|
@@ -0,0 +1,195 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The guided optimization loop: turn the current report into the next concrete
|
|
3
|
+
* step, RRSI style — one attributable edit inside the annealed budget, screened
|
|
4
|
+
* by the leakage critic, measured again afterwards.
|
|
5
|
+
*
|
|
6
|
+
* Pure: it reads a report and returns a plan; nothing is applied here.
|
|
7
|
+
*/
|
|
8
|
+
|
|
9
|
+
import { EDIT_DIMENSIONS } from './report.js'
|
|
10
|
+
|
|
11
|
+
const SEVERITY_ORDER = { critical: 0, warning: 1, info: 2 }
|
|
12
|
+
|
|
13
|
+
function scoreLine(entry) {
|
|
14
|
+
const score = entry.score ?? {}
|
|
15
|
+
const ci = score.ci ? ` (95% CI ${score.ci.low}-${score.ci.high})` : ''
|
|
16
|
+
return `${score.total ?? '—'}/100 ${score.grade ?? '?'}${ci}`
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
/** Pick the skill the user should work on next. */
|
|
20
|
+
export function pickLoopTarget(report, skillName) {
|
|
21
|
+
const entries = Array.isArray(report?.skills) ? report.skills : []
|
|
22
|
+
if (skillName) return entries.find((entry) => entry.skill === skillName) ?? null
|
|
23
|
+
const withActionable = entries.filter((entry) =>
|
|
24
|
+
(entry.advice ?? []).some((item) => item.severity === 'critical' || item.severity === 'warning'),
|
|
25
|
+
)
|
|
26
|
+
if (withActionable.length > 0) {
|
|
27
|
+
// worst first: a low score with an actionable fix is the best place to spend a round
|
|
28
|
+
return [...withActionable].sort((a, b) => (a.score?.total ?? 101) - (b.score?.total ?? 101))[0]
|
|
29
|
+
}
|
|
30
|
+
return entries[0] ?? null
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
/**
|
|
34
|
+
* @param {object} report - the analyzer report
|
|
35
|
+
* @param {{skill?: string, editBudgetMax?: number, minSamples?: number, reloads?: number}} options
|
|
36
|
+
*/
|
|
37
|
+
export function buildLoopPlan(report, options = {}) {
|
|
38
|
+
const budgetMax = Number.isFinite(options.editBudgetMax) ? options.editBudgetMax : 4
|
|
39
|
+
const reloads = Number.isFinite(options.reloads) ? options.reloads : (report?.scoring?.minSamples ?? 3)
|
|
40
|
+
const target = pickLoopTarget(report, options.skill)
|
|
41
|
+
if (!target) {
|
|
42
|
+
return {
|
|
43
|
+
empty: true,
|
|
44
|
+
reason:
|
|
45
|
+
(report?.skills ?? []).length === 0
|
|
46
|
+
? 'no skill loads recorded yet — load a skill once, then run the loop'
|
|
47
|
+
: `no evidence for skill "${options.skill}"`,
|
|
48
|
+
steps: [],
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
const budget = Math.max(1, Math.min(Number(target.adviceBudget) || budgetMax, budgetMax))
|
|
53
|
+
const actionable = [...(target.advice ?? [])]
|
|
54
|
+
.filter((item) => item.severity === 'critical' || item.severity === 'warning')
|
|
55
|
+
.sort((a, b) => (SEVERITY_ORDER[a.severity] ?? 3) - (SEVERITY_ORDER[b.severity] ?? 3))
|
|
56
|
+
const chosen = actionable.slice(0, budget)
|
|
57
|
+
const exploration = target.exploration ?? null
|
|
58
|
+
|
|
59
|
+
const steps = []
|
|
60
|
+
if (chosen.length === 0) {
|
|
61
|
+
steps.push({
|
|
62
|
+
kind: 'measure',
|
|
63
|
+
title: `No actionable fix within the budget for ${target.skill}`,
|
|
64
|
+
detail: `The evidence is ${scoreLine(target)}${target.score?.provisional ? ' (provisional)' : ''}. Keep using the skill and re-run the loop once ${reloads} more load(s) are recorded, or prune it if it stays low.`,
|
|
65
|
+
})
|
|
66
|
+
} else {
|
|
67
|
+
steps.push({
|
|
68
|
+
kind: 'edit',
|
|
69
|
+
title: `Apply ${chosen.length === 1 ? 'one attributable edit' : `${chosen.length} edits`} (budget ${budget})`,
|
|
70
|
+
detail: chosen.map((item) => `[${item.severity}] ${item.title} — ${item.fix}`).join('\n'),
|
|
71
|
+
})
|
|
72
|
+
const dimension = exploration ? exploration.suggestion : (chosen[0]?.dimension ?? null)
|
|
73
|
+
steps.push({
|
|
74
|
+
kind: 'review',
|
|
75
|
+
title: 'Screen the edit before applying it',
|
|
76
|
+
detail: `Call skill_eval_review {skill:"${target.skill}", dimension:"${dimension ?? 'static'}", hypothesis:"<what should improve and how you will measure it>", diff:"<the change>"}. A reject verdict means the edit leaks task-specific knowledge; revise instead of applying.`,
|
|
77
|
+
})
|
|
78
|
+
if (exploration) {
|
|
79
|
+
steps.push({
|
|
80
|
+
kind: 'explore',
|
|
81
|
+
title: `Break the ${exploration.stuck} loop`,
|
|
82
|
+
detail: `${exploration.reason}; make this edit about "${exploration.suggestion}" instead.`,
|
|
83
|
+
})
|
|
84
|
+
}
|
|
85
|
+
steps.push({
|
|
86
|
+
kind: 'measure',
|
|
87
|
+
title: `Apply, then re-measure after ~${reloads} load(s)`,
|
|
88
|
+
detail: `Current state: ${scoreLine(target)}. Compare the next report's score/CI for this skill; only keep the change when the interval moves, not when the point estimate does.`,
|
|
89
|
+
})
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
return {
|
|
93
|
+
skill: target.skill,
|
|
94
|
+
score: target.score ?? null,
|
|
95
|
+
budget,
|
|
96
|
+
chosen,
|
|
97
|
+
exploration,
|
|
98
|
+
dimension: exploration ? exploration.suggestion : (chosen[0]?.dimension ?? null),
|
|
99
|
+
steps,
|
|
100
|
+
empty: false,
|
|
101
|
+
}
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
export function renderLoopMarkdown(plan) {
|
|
105
|
+
if (!plan || plan.empty) {
|
|
106
|
+
return `## Optimization loop\n\n${plan?.reason ?? 'nothing to do'}`
|
|
107
|
+
}
|
|
108
|
+
const lines = [`## Optimization loop — \`${plan.skill}\``, '']
|
|
109
|
+
lines.push(`current: **${scoreLine(plan)}** · edit budget this round: **${plan.budget}**`)
|
|
110
|
+
if (plan.exploration) {
|
|
111
|
+
lines.push(`exploration: ${plan.exploration.reason} → target \`${plan.exploration.suggestion}\``)
|
|
112
|
+
}
|
|
113
|
+
lines.push('')
|
|
114
|
+
plan.steps.forEach((step, index) => {
|
|
115
|
+
lines.push(`${index + 1}. **${step.title}**`)
|
|
116
|
+
for (const line of String(step.detail).split('\n')) lines.push(` ${line}`)
|
|
117
|
+
})
|
|
118
|
+
lines.push('')
|
|
119
|
+
lines.push('Run `/skill-eval loop` again after the re-measurement round; the budget anneals as edits are accepted.')
|
|
120
|
+
return lines.join('\n')
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
export const LOOP_HINT_DIMENSIONS = EDIT_DIMENSIONS
|
|
124
|
+
|
|
125
|
+
const DIMENSION_COST = { static: 1, lifecycle: 1, mcp: 1.5, efficiency: 1.5, reliability: 2.5, effectiveness: 2.5 }
|
|
126
|
+
const SEVERITY_WEIGHT = { critical: 1, warning: 0.5 }
|
|
127
|
+
|
|
128
|
+
/**
|
|
129
|
+
* Expected-gain / cost ranking so the batch view answers "what should I fix
|
|
130
|
+
* first", not just "who scores lowest".
|
|
131
|
+
*
|
|
132
|
+
* gain = headroom (100 - score) plus a severity bonus, times evidence weight
|
|
133
|
+
* cost = how invasive the dimension usually is (a static body edit beats a
|
|
134
|
+
* behaviour change), divided so cheaper edits rank higher
|
|
135
|
+
*/
|
|
136
|
+
export function impactOf(plan) {
|
|
137
|
+
const total = typeof plan?.score?.total === 'number' ? plan.score.total : null
|
|
138
|
+
const chosen = plan?.chosen ?? []
|
|
139
|
+
if (total === null || chosen.length === 0) return null
|
|
140
|
+
const severity = Math.max(...chosen.map((item) => SEVERITY_WEIGHT[item.severity] ?? 0.25))
|
|
141
|
+
const samples = plan.score?.samples ?? 0
|
|
142
|
+
const minSamples = plan.score?.minSamples ?? 3
|
|
143
|
+
const evidence = Math.max(0.35, Math.min(1, samples / Math.max(2, minSamples)))
|
|
144
|
+
const gain = (100 - total) * severity * evidence
|
|
145
|
+
const dimension = plan.dimension ?? chosen[0]?.dimension ?? 'static'
|
|
146
|
+
const cost = DIMENSION_COST[dimension] ?? 2
|
|
147
|
+
// Effort penalises sublinearly: a critical break still outranks a cosmetic
|
|
148
|
+
// one-line fix, but a cheap static edit beats an equally sizeable behaviour
|
|
149
|
+
// change.
|
|
150
|
+
const score = gain / Math.sqrt(cost)
|
|
151
|
+
return {
|
|
152
|
+
score: Math.round(score * 10) / 10,
|
|
153
|
+
expectedGain: Math.round(gain * 10) / 10,
|
|
154
|
+
cost,
|
|
155
|
+
severity,
|
|
156
|
+
evidence: Math.round(evidence * 100) / 100,
|
|
157
|
+
reasons: [
|
|
158
|
+
`${Math.round(100 - total)} points of headroom`,
|
|
159
|
+
severity === 1 ? 'critical fix' : 'warning-level fix',
|
|
160
|
+
`~${cost}x effort (${dimension})`,
|
|
161
|
+
evidence < 1 ? 'provisional evidence' : 'measured evidence',
|
|
162
|
+
],
|
|
163
|
+
}
|
|
164
|
+
}
|
|
165
|
+
|
|
166
|
+
/** Every skill that currently has an actionable fix, worst score first. */
|
|
167
|
+
export function buildLoopPlans(report, options = {}) {
|
|
168
|
+
const entries = (Array.isArray(report?.skills) ? report.skills : []).filter((entry) =>
|
|
169
|
+
(entry.advice ?? []).some((item) => item.severity === 'critical' || item.severity === 'warning'),
|
|
170
|
+
)
|
|
171
|
+
const plans = entries.map((entry) => buildLoopPlan(report, { ...options, skill: entry.skill }))
|
|
172
|
+
for (const plan of plans) plan.priority = impactOf(plan)
|
|
173
|
+
return plans.sort((a, b) => (b.priority?.score ?? 0) - (a.priority?.score ?? 0))
|
|
174
|
+
}
|
|
175
|
+
|
|
176
|
+
export function renderLoopBatchMarkdown(plans) {
|
|
177
|
+
if (!plans || plans.length === 0) {
|
|
178
|
+
return '## Optimization loop\n\nNo skill currently has an actionable fix within budget. Keep using the skills and re-run once more loads are recorded.'
|
|
179
|
+
}
|
|
180
|
+
const lines = [`## Optimization loop — ${plans.length} skill(s) with an actionable fix`, '']
|
|
181
|
+
for (const plan of plans) {
|
|
182
|
+
const top = plan.chosen?.[0]
|
|
183
|
+
const priority = plan.priority ? ` · priority ${plan.priority.score} (${plan.priority.reasons.join('; ')})` : ''
|
|
184
|
+
lines.push(`### \`${plan.skill}\` — ${scoreLine(plan)} · budget ${plan.budget}${priority}`)
|
|
185
|
+
if (top) lines.push(`- [${top.severity}] ${top.title}`)
|
|
186
|
+
lines.push(`- fix: ${top ? top.fix : 'none'}`)
|
|
187
|
+
const review = plan.steps?.find((step) => step.kind === 'review')
|
|
188
|
+
if (review) lines.push(`- screen: \`skill_eval_review {skill:"${plan.skill}", dimension:"${plan.dimension}"}\``)
|
|
189
|
+
if (plan.exploration)
|
|
190
|
+
lines.push(`- explore: ${plan.exploration.reason} → target \`${plan.exploration.suggestion}\``)
|
|
191
|
+
lines.push('')
|
|
192
|
+
}
|
|
193
|
+
lines.push('Run `/skill-eval loop <skill>` for the full step-by-step plan on one skill.')
|
|
194
|
+
return lines.join('\n')
|
|
195
|
+
}
|