dsh-skill-eval 0.0.0-stage → 1.20.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +675 -0
- package/LICENSE +21 -0
- package/README.md +200 -2
- package/README.zh.md +157 -0
- package/client.js +2637 -0
- package/cordis.patch.yml +9 -0
- package/docs/ACCEPTANCE.md +71 -0
- package/docs/AUDIT.md +64 -0
- package/docs/CODE-REVIEW.md +49 -0
- package/docs/COMPARISON.md +33 -0
- package/docs/DETAIL.md +463 -0
- package/docs/DEV-RELOAD.md +114 -0
- package/docs/ENTRYPOINTS.md +65 -0
- package/docs/RELEASE.md +44 -0
- package/docs/RRSI-NOTES.md +129 -0
- package/docs/SCENARIOS.md +91 -0
- package/docs/UI-REVIEW.md +115 -0
- package/docs/WORKFLOW.md +108 -0
- package/icon.svg +9 -0
- package/index.js +1111 -0
- package/lib/accept.js +350 -0
- package/lib/advice.js +547 -0
- package/lib/critic.js +101 -0
- package/lib/dev.js +148 -0
- package/lib/history.js +140 -0
- package/lib/html.js +8 -0
- package/lib/inspect.js +142 -0
- package/lib/integrity.js +46 -0
- package/lib/loop.js +195 -0
- package/lib/random.js +29 -0
- package/lib/recorder.js +634 -0
- package/lib/render.js +582 -0
- package/lib/report.js +646 -0
- package/lib/score.js +525 -0
- package/lib/settings.js +396 -0
- package/lib/snapshot.js +166 -0
- package/lib/steps.js +141 -0
- package/lib/store.js +186 -0
- package/lib/validity.js +115 -0
- package/locale/en.json +6 -0
- package/locale/zh.json +6 -0
- package/package.json +96 -4
- package/scripts/accept.mjs +226 -0
- package/scripts/build-client.mjs +59 -0
- package/scripts/build.mjs +81 -0
- package/scripts/doctor.mjs +199 -0
- package/scripts/lib.mjs +52 -0
- package/scripts/verify-live.mjs +168 -0
package/lib/settings.js
ADDED
|
@@ -0,0 +1,396 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Runtime settings page support.
|
|
3
|
+
*
|
|
4
|
+
* The plugin config lives in the profile, which is not editable from a browser.
|
|
5
|
+
* This module keeps a small, validated override file inside the data directory
|
|
6
|
+
* (`settings.json`) so the reporting board can expose a configuration page
|
|
7
|
+
* without touching profile files. Overrides win over profile values and are
|
|
8
|
+
* applied on the next load as well as immediately in memory.
|
|
9
|
+
*/
|
|
10
|
+
|
|
11
|
+
import { readFileSync } from 'node:fs'
|
|
12
|
+
import { mkdir, readFile, rename, writeFile } from 'node:fs/promises'
|
|
13
|
+
import path from 'node:path'
|
|
14
|
+
import { escapeHtml as escape } from './html.js'
|
|
15
|
+
|
|
16
|
+
const NUMBER = (min, max, step = 1) => ({ type: 'number', min, max, step })
|
|
17
|
+
const BOOL = { type: 'boolean' }
|
|
18
|
+
|
|
19
|
+
/** Every field the settings page may change, with bounds and labels. */
|
|
20
|
+
export const SETTINGS_FIELDS = [
|
|
21
|
+
{ key: 'weights.static', label: 'weight: static', ...NUMBER(0, 100, 5) },
|
|
22
|
+
{ key: 'weights.reliability', label: 'weight: reliability', ...NUMBER(0, 100, 5) },
|
|
23
|
+
{ key: 'weights.effectiveness', label: 'weight: effectiveness', ...NUMBER(0, 100, 5) },
|
|
24
|
+
{ key: 'weights.efficiency', label: 'weight: efficiency', ...NUMBER(0, 100, 5) },
|
|
25
|
+
{ key: 'thresholds.excellent', label: 'grade A ≥', ...NUMBER(1, 100) },
|
|
26
|
+
{ key: 'thresholds.good', label: 'grade B ≥', ...NUMBER(1, 100) },
|
|
27
|
+
{ key: 'thresholds.pass', label: 'grade C ≥', ...NUMBER(1, 100) },
|
|
28
|
+
{ key: 'minSamples', label: 'min samples for an interval', ...NUMBER(1, 100) },
|
|
29
|
+
{ key: 'windowTtlMs', label: 'window TTL (ms)', ...NUMBER(60000, 86400000, 60000) },
|
|
30
|
+
{ key: 'maxAdvice', label: 'max actionable fixes per report', ...NUMBER(1, 10) },
|
|
31
|
+
{ key: 'ciResamples', label: 'bootstrap resamples', ...NUMBER(0, 2000) },
|
|
32
|
+
{ key: 'tokenBudgetPerLoad', label: 'token budget per load', ...NUMBER(1000, 1000000, 1000) },
|
|
33
|
+
{ key: 'cacheReadWeight', label: 'cached input weight', ...NUMBER(0, 1, 0.05) },
|
|
34
|
+
{ key: 'pruneMinLoads', label: 'prune after N loads', ...NUMBER(1, 100) },
|
|
35
|
+
{ key: 'pruneMaxScore', label: 'prune below score', ...NUMBER(0, 100) },
|
|
36
|
+
{ key: 'editBudgetMax', label: 'edit budget max', ...NUMBER(1, 10) },
|
|
37
|
+
{ key: 'editBudgetMin', label: 'edit budget min', ...NUMBER(1, 10) },
|
|
38
|
+
{ key: 'editBudgetHorizon', label: 'budget anneal horizon', ...NUMBER(1, 30) },
|
|
39
|
+
{ key: 'noveltyWindow', label: 'exploration window', ...NUMBER(2, 10) },
|
|
40
|
+
{ key: 'mcpErrorRatePercent', label: 'MCP error-rate alert %', ...NUMBER(1, 100) },
|
|
41
|
+
{ key: 'mcpP95Ms', label: 'MCP p95 alert (ms)', ...NUMBER(100, 600000) },
|
|
42
|
+
{ key: 'adviceNoiseFloor', label: 'advice noise floor', ...BOOL },
|
|
43
|
+
{ key: 'countTokens', label: 'capture token usage', ...BOOL },
|
|
44
|
+
{ key: 'promptHint', label: 'inject prompt hint', ...BOOL },
|
|
45
|
+
{ key: 'reportOnTurnEnd', label: 'write snapshot every turn', ...BOOL },
|
|
46
|
+
]
|
|
47
|
+
|
|
48
|
+
// Short help lines mirror the generic settings layout (label + description +
|
|
49
|
+
// control); the same text is shipped to the client form.
|
|
50
|
+
const FIELD_HELP = {
|
|
51
|
+
'weights.static': { en: 'Weight of the authoring-quality dimension.', zh: '静态质量维度的权重。' },
|
|
52
|
+
'weights.reliability': { en: 'Weight of load success and follow-up errors.', zh: '加载成功率与后续错误的权重。' },
|
|
53
|
+
'weights.effectiveness': { en: 'Weight of user corrections and ignored loads.', zh: '用户纠正与加载后忽略的权重。' },
|
|
54
|
+
'weights.efficiency': { en: 'Weight of tool calls, latency and tokens.', zh: '工具调用、延迟与 token 的权重。' },
|
|
55
|
+
'thresholds.excellent': { en: 'Total score needed to reach grade A.', zh: '总分达到多少才算 A 级。' },
|
|
56
|
+
'thresholds.good': { en: 'Total score needed to reach grade B.', zh: '总分达到多少才算 B 级。' },
|
|
57
|
+
'thresholds.pass': { en: 'Total score needed to reach grade C.', zh: '总分达到多少才算 C 级。' },
|
|
58
|
+
minSamples: { en: 'Loads before an interval and grade become stable.', zh: '达到多少加载后区间与等级才算稳定。' },
|
|
59
|
+
windowTtlMs: { en: 'How long an open window keeps waiting for follow-up work.', zh: '打开的窗口等待后续工作多久。' },
|
|
60
|
+
tokenBudgetPerLoad: {
|
|
61
|
+
en: 'Billable tokens one load may cost before efficiency reacts.',
|
|
62
|
+
zh: '单次加载可计费 token 预算,超出会拉低效率分。',
|
|
63
|
+
},
|
|
64
|
+
cacheReadWeight: {
|
|
65
|
+
en: 'How much a cached input token counts against the budget.',
|
|
66
|
+
zh: '缓存输入 token 折算进预算的比例。',
|
|
67
|
+
},
|
|
68
|
+
countTokens: {
|
|
69
|
+
en: 'Attribute real provider usage to the open windows.',
|
|
70
|
+
zh: '把真实 provider 用量归因到打开的窗口。',
|
|
71
|
+
},
|
|
72
|
+
adviceNoiseFloor: {
|
|
73
|
+
en: 'Only keep a rate-based severity when it clears its 95% interval.',
|
|
74
|
+
zh: '只有越过 95% 区间的比例类问题才保持原严重度。',
|
|
75
|
+
},
|
|
76
|
+
pruneMinLoads: { en: 'Loads required before a prune recommendation appears.', zh: '出现剪枝建议前所需的加载次数。' },
|
|
77
|
+
pruneMaxScore: {
|
|
78
|
+
en: 'Score below which a well-sampled skill is a prune candidate.',
|
|
79
|
+
zh: '样本充足时,低于该分数会成为剪枝候选。',
|
|
80
|
+
},
|
|
81
|
+
editBudgetMax: { en: 'Concurrent edits a first-round skill may carry.', zh: '首轮技能允许同时改动的数量。' },
|
|
82
|
+
editBudgetMin: { en: 'Floor after the budget anneals.', zh: '预算退火后的下限。' },
|
|
83
|
+
mcpErrorRatePercent: {
|
|
84
|
+
en: 'MCP server error rate that raises a critical fix.',
|
|
85
|
+
zh: 'MCP 服务器错误率达到多少时给出 critical 修复。',
|
|
86
|
+
},
|
|
87
|
+
mcpP95Ms: {
|
|
88
|
+
en: 'MCP server p95 latency that raises a slow-server warning.',
|
|
89
|
+
zh: 'MCP 服务器 p95 延迟达到多少时告警。',
|
|
90
|
+
},
|
|
91
|
+
promptHint: {
|
|
92
|
+
en: 'One-line system-prompt hint; off means zero fixed token cost.',
|
|
93
|
+
zh: '系统提示里的单行提示;关闭即零固定 token 成本。',
|
|
94
|
+
},
|
|
95
|
+
reportOnTurnEnd: {
|
|
96
|
+
en: 'Write latest.md/latest.json after each settled turn.',
|
|
97
|
+
zh: '每个结算回合后写 latest.md/latest.json。',
|
|
98
|
+
},
|
|
99
|
+
}
|
|
100
|
+
const FIELD_UNIT = {
|
|
101
|
+
'weights.static': '%',
|
|
102
|
+
'weights.reliability': '%',
|
|
103
|
+
'weights.effectiveness': '%',
|
|
104
|
+
'weights.efficiency': '%',
|
|
105
|
+
minSamples: 'loads',
|
|
106
|
+
windowTtlMs: 'ms',
|
|
107
|
+
tokenBudgetPerLoad: 'tokens',
|
|
108
|
+
pruneMinLoads: 'loads',
|
|
109
|
+
editBudgetMax: 'edits',
|
|
110
|
+
editBudgetMin: 'edits',
|
|
111
|
+
editBudgetHorizon: 'edits',
|
|
112
|
+
mcpP95Ms: 'ms',
|
|
113
|
+
}
|
|
114
|
+
for (const field of SETTINGS_FIELDS) {
|
|
115
|
+
const help = FIELD_HELP[field.key]
|
|
116
|
+
if (help) {
|
|
117
|
+
field.help = help.en
|
|
118
|
+
field.helpZh = help.zh
|
|
119
|
+
}
|
|
120
|
+
if (FIELD_UNIT[field.key]) field.unit = FIELD_UNIT[field.key]
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
const FIELD_BY_KEY = new Map(SETTINGS_FIELDS.map((field) => [field.key, field]))
|
|
124
|
+
|
|
125
|
+
/** Units are stored in English; the Chinese page and the in-panel form use these. */
|
|
126
|
+
export const UNIT_ZH = { loads: '次加载', tokens: 'token', edits: '次修改' }
|
|
127
|
+
|
|
128
|
+
export function unitLabel(unit, zh) {
|
|
129
|
+
if (!unit) return ''
|
|
130
|
+
return zh ? UNIT_ZH[unit] ?? unit : unit
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
export const SETTINGS_GROUPS = [
|
|
134
|
+
{
|
|
135
|
+
title: 'Scoring',
|
|
136
|
+
titleZh: '评分权重与阈值',
|
|
137
|
+
keys: SETTINGS_FIELDS.filter(
|
|
138
|
+
(field) =>
|
|
139
|
+
/^(weights|thresholds|minSamples|ciResamples|maxAdvice)\./.test(field.key) ||
|
|
140
|
+
['minSamples', 'maxAdvice', 'ciResamples'].includes(field.key),
|
|
141
|
+
).map((field) => field.key),
|
|
142
|
+
},
|
|
143
|
+
{ title: 'Cost', titleZh: '成本', keys: ['tokenBudgetPerLoad', 'cacheReadWeight', 'countTokens'] },
|
|
144
|
+
{
|
|
145
|
+
title: 'Advice & lifecycle',
|
|
146
|
+
titleZh: '建议与生命周期',
|
|
147
|
+
keys: [
|
|
148
|
+
'adviceNoiseFloor',
|
|
149
|
+
'windowTtlMs',
|
|
150
|
+
'pruneMinLoads',
|
|
151
|
+
'pruneMaxScore',
|
|
152
|
+
'editBudgetMax',
|
|
153
|
+
'editBudgetMin',
|
|
154
|
+
'editBudgetHorizon',
|
|
155
|
+
'noveltyWindow',
|
|
156
|
+
],
|
|
157
|
+
},
|
|
158
|
+
{ title: 'MCP', titleZh: 'MCP', keys: ['mcpErrorRatePercent', 'mcpP95Ms'] },
|
|
159
|
+
{ title: 'Output', titleZh: '输出', keys: ['promptHint', 'reportOnTurnEnd'] },
|
|
160
|
+
]
|
|
161
|
+
|
|
162
|
+
const GROUP_BY_KEY = new Map()
|
|
163
|
+
for (const group of SETTINGS_GROUPS) for (const key of group.keys) GROUP_BY_KEY.set(key, group)
|
|
164
|
+
for (const field of SETTINGS_FIELDS) {
|
|
165
|
+
const group = GROUP_BY_KEY.get(field.key)
|
|
166
|
+
if (group) {
|
|
167
|
+
field.group = group.title
|
|
168
|
+
field.groupZh = group.titleZh
|
|
169
|
+
}
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
export const FIELD_ZH = {
|
|
173
|
+
'weights.static': '权重:静态质量',
|
|
174
|
+
'weights.reliability': '权重:可靠性',
|
|
175
|
+
'weights.effectiveness': '权重:有效性',
|
|
176
|
+
'weights.efficiency': '权重:效率',
|
|
177
|
+
'thresholds.excellent': 'A 级下限',
|
|
178
|
+
'thresholds.good': 'B 级下限',
|
|
179
|
+
'thresholds.pass': 'C 级下限',
|
|
180
|
+
minSamples: '给区间所需最小样本',
|
|
181
|
+
maxAdvice: '每轮最多可行动建议',
|
|
182
|
+
ciResamples: 'bootstrap 重采样次数',
|
|
183
|
+
tokenBudgetPerLoad: '单次加载 token 预算',
|
|
184
|
+
cacheReadWeight: '缓存命中折算权重',
|
|
185
|
+
windowTtlMs: '窗口有效期 (ms)',
|
|
186
|
+
pruneMinLoads: '剪枝所需加载次数',
|
|
187
|
+
pruneMaxScore: '低于该分数进入剪枝候选',
|
|
188
|
+
editBudgetMax: '编辑预算上限',
|
|
189
|
+
editBudgetMin: '编辑预算下限',
|
|
190
|
+
editBudgetHorizon: '预算退火跨度',
|
|
191
|
+
noveltyWindow: '探索窗口',
|
|
192
|
+
mcpErrorRatePercent: 'MCP 错误率告警 %',
|
|
193
|
+
mcpP95Ms: 'MCP p95 告警 (ms)',
|
|
194
|
+
adviceNoiseFloor: '建议噪声地板',
|
|
195
|
+
countTokens: '采集 token 用量',
|
|
196
|
+
promptHint: '注入提示词',
|
|
197
|
+
reportOnTurnEnd: '每回合写快照',
|
|
198
|
+
}
|
|
199
|
+
|
|
200
|
+
const SET = (target, keyPath, value) => {
|
|
201
|
+
const [head, leaf] = keyPath.split('.')
|
|
202
|
+
if (leaf) {
|
|
203
|
+
target[head] = { ...(target[head] ?? {}), [leaf]: value }
|
|
204
|
+
} else {
|
|
205
|
+
target[head] = value
|
|
206
|
+
}
|
|
207
|
+
}
|
|
208
|
+
|
|
209
|
+
/**
|
|
210
|
+
* Parse and validate submitted values.
|
|
211
|
+
*
|
|
212
|
+
* @param {Record<string, string|boolean|number>} input flat `key → value` map
|
|
213
|
+
* @returns {{values: Record<string, unknown>, errors: Array<{key: string, message: string}>}}
|
|
214
|
+
*/
|
|
215
|
+
export function normalizeSettings(input) {
|
|
216
|
+
const values = {}
|
|
217
|
+
const errors = []
|
|
218
|
+
for (const [key, raw] of Object.entries(input ?? {})) {
|
|
219
|
+
const field = FIELD_BY_KEY.get(key)
|
|
220
|
+
if (!field) continue
|
|
221
|
+
if (field.type === 'boolean') {
|
|
222
|
+
values[key] = raw === true || raw === 'on' || raw === 'true' || raw === '1'
|
|
223
|
+
continue
|
|
224
|
+
}
|
|
225
|
+
const num = Number(typeof raw === 'string' ? raw.trim() : raw)
|
|
226
|
+
if (!Number.isFinite(num)) {
|
|
227
|
+
errors.push({ key, message: `not a number: ${String(raw)}` })
|
|
228
|
+
continue
|
|
229
|
+
}
|
|
230
|
+
if (num < field.min || num > field.max) {
|
|
231
|
+
errors.push({ key, message: `out of range ${field.min}..${field.max}` })
|
|
232
|
+
continue
|
|
233
|
+
}
|
|
234
|
+
values[key] = num
|
|
235
|
+
}
|
|
236
|
+
const weights = ['static', 'reliability', 'effectiveness', 'efficiency'].filter(
|
|
237
|
+
(name) => typeof values[`weights.${name}`] === 'number',
|
|
238
|
+
)
|
|
239
|
+
if (weights.length === 4) {
|
|
240
|
+
const sum = weights.reduce((total, name) => total + values[`weights.${name}`], 0)
|
|
241
|
+
if (sum <= 0) errors.push({ key: 'weights.total', message: 'weights sum must be positive' })
|
|
242
|
+
}
|
|
243
|
+
return { values, errors }
|
|
244
|
+
}
|
|
245
|
+
|
|
246
|
+
/** Read the override file; missing or unreadable means "no overrides". */
|
|
247
|
+
/**
|
|
248
|
+
* Parse an HTML form body. Unchecked checkboxes are simply absent from a form
|
|
249
|
+
* POST, so booleans must default to false — otherwise "turn this off" can
|
|
250
|
+
* never be saved from the plain web settings page.
|
|
251
|
+
*/
|
|
252
|
+
export function parseFormSettings(body) {
|
|
253
|
+
const submitted = Object.fromEntries(new URLSearchParams(body ?? ''))
|
|
254
|
+
// Only a submission from our own full form may turn absent checkboxes into
|
|
255
|
+
// false. A partial POST (other clients, CSRF) must leave prior values alone.
|
|
256
|
+
const fullForm = submitted.__full_form === '1'
|
|
257
|
+
delete submitted.__full_form
|
|
258
|
+
if (fullForm) {
|
|
259
|
+
for (const field of SETTINGS_FIELDS) {
|
|
260
|
+
if (field.type === 'boolean' && !(field.key in submitted)) submitted[field.key] = false
|
|
261
|
+
}
|
|
262
|
+
}
|
|
263
|
+
return submitted
|
|
264
|
+
}
|
|
265
|
+
|
|
266
|
+
export async function loadSettings(dataDir) {
|
|
267
|
+
if (typeof dataDir !== 'string' || dataDir.length === 0) return {}
|
|
268
|
+
try {
|
|
269
|
+
const raw = await readFile(path.join(dataDir, 'settings.json'), 'utf8')
|
|
270
|
+
const parsed = JSON.parse(raw)
|
|
271
|
+
return parsed && typeof parsed === 'object' && !Array.isArray(parsed) ? parsed : {}
|
|
272
|
+
} catch {
|
|
273
|
+
return {}
|
|
274
|
+
}
|
|
275
|
+
}
|
|
276
|
+
|
|
277
|
+
let saveQueue = Promise.resolve()
|
|
278
|
+
|
|
279
|
+
/**
|
|
280
|
+
* Serialize writes and replace the file atomically: concurrent saves cannot
|
|
281
|
+
* lose each other's fields, and a crash mid-write leaves the previous file
|
|
282
|
+
* intact instead of truncated JSON that would silently drop every override.
|
|
283
|
+
*/
|
|
284
|
+
export function saveSettings(dataDir, values) {
|
|
285
|
+
const file = path.join(dataDir, 'settings.json')
|
|
286
|
+
const job = saveQueue.then(async () => {
|
|
287
|
+
const temp = `${file}.tmp`
|
|
288
|
+
await mkdir(dataDir, { recursive: true })
|
|
289
|
+
await writeFile(temp, `${JSON.stringify(values, null, 2)}\n`, 'utf8')
|
|
290
|
+
await rename(temp, file)
|
|
291
|
+
return file
|
|
292
|
+
})
|
|
293
|
+
saveQueue = job.catch(() => {})
|
|
294
|
+
return job
|
|
295
|
+
}
|
|
296
|
+
|
|
297
|
+
/** Synchronous variant used at plugin apply time (the config must exist first). */
|
|
298
|
+
export function loadSettingsSync(dataDir) {
|
|
299
|
+
if (typeof dataDir !== 'string' || dataDir.length === 0) return {}
|
|
300
|
+
try {
|
|
301
|
+
const parsed = JSON.parse(readFileSync(path.join(dataDir, 'settings.json'), 'utf8'))
|
|
302
|
+
return parsed && typeof parsed === 'object' && !Array.isArray(parsed) ? parsed : {}
|
|
303
|
+
} catch {
|
|
304
|
+
return {}
|
|
305
|
+
}
|
|
306
|
+
}
|
|
307
|
+
|
|
308
|
+
/** Merge flat override keys into a config-shaped object (mutates and returns it). */
|
|
309
|
+
export function applySettings(target, values) {
|
|
310
|
+
for (const [key, value] of Object.entries(values ?? {})) {
|
|
311
|
+
if (!FIELD_BY_KEY.has(key)) continue
|
|
312
|
+
SET(target, key, value)
|
|
313
|
+
}
|
|
314
|
+
return target
|
|
315
|
+
}
|
|
316
|
+
|
|
317
|
+
/** Current effective value for one field, from a config-shaped object. */
|
|
318
|
+
export function readSetting(config, key) {
|
|
319
|
+
const [head, leaf] = key.split('.')
|
|
320
|
+
if (leaf) return config?.[head]?.[leaf]
|
|
321
|
+
return config?.[head]
|
|
322
|
+
}
|
|
323
|
+
|
|
324
|
+
/** Render the configuration page. */
|
|
325
|
+
export function renderConfigHtml({ config, base, saved = false, errors = [], lang = 'en' }) {
|
|
326
|
+
const zh = lang === 'zh'
|
|
327
|
+
const label = (def) => (zh && FIELD_ZH[def.key] ? FIELD_ZH[def.key] : def.label)
|
|
328
|
+
const groupTitle = (group) => (zh && group.titleZh ? group.titleZh : group.title)
|
|
329
|
+
const field = (def) => {
|
|
330
|
+
const value = readSetting(config, def.key)
|
|
331
|
+
if (def.type === 'boolean') {
|
|
332
|
+
return `<label class="row check"><input type="checkbox" name="${def.key}" ${
|
|
333
|
+
value === true ? 'checked' : ''
|
|
334
|
+
}><span>${escape(label(def))}</span></label>`
|
|
335
|
+
}
|
|
336
|
+
const help = zh ? def.helpZh : def.help
|
|
337
|
+
return `<label class="row"><span class="label">${escape(label(def))}${
|
|
338
|
+
help ? `<small>${escape(help)}</small>` : ''
|
|
339
|
+
}</span><span class="control"><input type="number" name="${def.key}" value="${escape(value ?? '')}" min="${
|
|
340
|
+
def.min
|
|
341
|
+
}" max="${def.max}" step="${def.step}">${def.unit ? `<em>${escape(unitLabel(def.unit, zh))}</em>` : ''}</span></label>`
|
|
342
|
+
}
|
|
343
|
+
const groups = SETTINGS_GROUPS.map(
|
|
344
|
+
(group) =>
|
|
345
|
+
`<section><h2>${escape(groupTitle(group))}</h2>${group.keys
|
|
346
|
+
.map((key) => FIELD_BY_KEY.get(key))
|
|
347
|
+
.filter(Boolean)
|
|
348
|
+
.map(field)
|
|
349
|
+
.join('')}</section>`,
|
|
350
|
+
).join('')
|
|
351
|
+
return `<!doctype html>
|
|
352
|
+
<html lang="${zh ? 'zh' : 'en'}"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1">
|
|
353
|
+
<title>${zh ? '技能评测 · 配置' : 'Skill eval · Config'}</title>
|
|
354
|
+
<style>
|
|
355
|
+
:root{color-scheme:light dark}
|
|
356
|
+
body{margin:0;padding:28px;font:14px/1.55 ui-sans-serif,-apple-system,"Segoe UI",Roboto,sans-serif;background:#fafafa;color:#18181b;max-width:760px}
|
|
357
|
+
@media (prefers-color-scheme:dark){body{background:#111113;color:#e7e7ea}section{background:#1a1a1e!important;border-color:#2a2a30!important}input{background:#111113!important;color:#e7e7ea!important}h2{color:#d4d4d8!important}.row small,.sub{color:#a1a1aa!important}}
|
|
358
|
+
h1{font-size:18px;margin:0 0 4px}
|
|
359
|
+
.sub{color:#71717a;font-size:12px;margin-bottom:18px}
|
|
360
|
+
section{background:#fff;border:1px solid #e5e5e9;border-radius:12px;padding:14px 16px;margin-bottom:14px}
|
|
361
|
+
h2{font-size:13px;margin:0 0 10px;color:#52525b}
|
|
362
|
+
.row{display:grid;grid-template-columns:minmax(180px,1fr) 200px;gap:12px;align-items:center;font-size:12px;padding:9px 0;border-bottom:1px solid #f0f0f2}.row .label{display:flex;flex-direction:column;gap:2px}.row small{color:#71717a;font-size:11px;font-weight:400}.row .control{display:flex;align-items:center;gap:6px;justify-content:flex-end}.row .control em{font-style:normal;color:#71717a;font-size:11px}
|
|
363
|
+
.row input[type=number]{padding:5px 7px;border:1px solid #d4d4d8;border-radius:7px;font:inherit}
|
|
364
|
+
.row.check{grid-template-columns:24px 1fr}
|
|
365
|
+
button{padding:8px 16px;border-radius:9px;border:1px solid #18181b;background:#18181b;color:#fff;font:inherit;cursor:pointer}
|
|
366
|
+
a{color:#3b82f6;text-decoration:none;font-size:12px}
|
|
367
|
+
.banner{padding:8px 12px;border-radius:9px;font-size:12px;margin-bottom:14px}
|
|
368
|
+
.banner.ok{background:#dcfce7;color:#166534}
|
|
369
|
+
.banner.err{background:#fee2e2;color:#991b1b}
|
|
370
|
+
</style></head>
|
|
371
|
+
<body>
|
|
372
|
+
<h1>${zh ? '技能评测 · 配置' : 'Skill eval · Config'}</h1>
|
|
373
|
+
<div class="sub">${
|
|
374
|
+
zh
|
|
375
|
+
? '保存到数据目录的 <code>settings.json</code>,优先于 profile 配置,立即生效并覆盖下次加载。'
|
|
376
|
+
: 'Saved to <code>settings.json</code> in the data directory; it wins over the profile config and applies immediately.'
|
|
377
|
+
} <a href="${escape(base)}">${zh ? '← 返回看板' : '← Board'}</a> · <a href="?lang=${zh ? 'en' : 'zh'}">${
|
|
378
|
+
zh ? 'English' : '中文'
|
|
379
|
+
}</a></div>
|
|
380
|
+
${saved ? `<div class="banner ok">${zh ? '已保存并应用。' : 'Saved and applied.'}</div>` : ''}
|
|
381
|
+
${
|
|
382
|
+
errors.length
|
|
383
|
+
? `<div class="banner err">${
|
|
384
|
+
zh ? `有 ${errors.length} 个字段未保存:` : `${errors.length} field(s) rejected: `
|
|
385
|
+
}${errors.map((error) => `${escape(error.key)} (${escape(error.message)})`).join(zh ? ';' : '; ')}</div>`
|
|
386
|
+
: ''
|
|
387
|
+
}
|
|
388
|
+
<form method="post" action="${escape(base)}/config">
|
|
389
|
+
<input type="hidden" name="__full_form" value="1">
|
|
390
|
+
${groups}
|
|
391
|
+
<button type="submit">${zh ? '保存' : 'Save'}</button>
|
|
392
|
+
</form>
|
|
393
|
+
</body></html>`
|
|
394
|
+
}
|
|
395
|
+
|
|
396
|
+
export const SETTINGS_FILE = 'settings.json'
|
package/lib/snapshot.js
ADDED
|
@@ -0,0 +1,166 @@
|
|
|
1
|
+
import { promises as fs } from 'node:fs'
|
|
2
|
+
import path from 'node:path'
|
|
3
|
+
import { renderSummaryMarkdown } from './render.js'
|
|
4
|
+
|
|
5
|
+
/**
|
|
6
|
+
* Writes an aggregated snapshot of the report to disk after a turn settles.
|
|
7
|
+
*
|
|
8
|
+
* This is the "result after every run" surface that costs nothing: it never
|
|
9
|
+
* touches the model context, so it adds zero tokens to the user's conversation.
|
|
10
|
+
* The snapshot is debounced because one turn can settle several windows.
|
|
11
|
+
*/
|
|
12
|
+
export function turnSlug(entry) {
|
|
13
|
+
const session = String(entry?.sessionId ?? 'session')
|
|
14
|
+
.replace(/[^A-Za-z0-9_-]+/g, '-')
|
|
15
|
+
.slice(-40)
|
|
16
|
+
const turn = Number.isFinite(entry?.turn) ? `t${entry.turn}` : 'tunspecified'
|
|
17
|
+
return `${session}-${turn}`
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
/** List the per-turn boards newest first (best effort; a missing dir is empty). */
|
|
21
|
+
export async function listTurnReports(dir) {
|
|
22
|
+
const turnsDir = path.join(dir, 'turns')
|
|
23
|
+
let names
|
|
24
|
+
try {
|
|
25
|
+
names = await fs.readdir(turnsDir)
|
|
26
|
+
} catch {
|
|
27
|
+
return []
|
|
28
|
+
}
|
|
29
|
+
const rows = []
|
|
30
|
+
for (const name of names) {
|
|
31
|
+
if (!name.endsWith('.json')) continue
|
|
32
|
+
const file = path.join(turnsDir, name)
|
|
33
|
+
try {
|
|
34
|
+
const info = await fs.stat(file)
|
|
35
|
+
const report = JSON.parse(await fs.readFile(file, 'utf8'))
|
|
36
|
+
rows.push({
|
|
37
|
+
file: name,
|
|
38
|
+
slug: name.replace(/\.json$/, ''),
|
|
39
|
+
bytes: info.size,
|
|
40
|
+
at: info.mtimeMs,
|
|
41
|
+
sessionId: report?.turnLoads?.[0]?.sessionId ?? null,
|
|
42
|
+
turn: report?.turnLoads?.[0]?.turn ?? null,
|
|
43
|
+
skills: (report?.skills ?? []).slice(0, 4).map((skill) => skill.skill),
|
|
44
|
+
totals: report?.totals ?? null,
|
|
45
|
+
})
|
|
46
|
+
} catch {
|
|
47
|
+
// skip unreadable or partial files
|
|
48
|
+
}
|
|
49
|
+
}
|
|
50
|
+
return rows.sort((a, b) => b.at - a.at)
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
export function renderTurnReportsMarkdown(rows, { limit = 20 } = {}) {
|
|
54
|
+
const lines = ['## Per-turn boards', '']
|
|
55
|
+
if (!rows || rows.length === 0) {
|
|
56
|
+
lines.push('No per-turn board written yet. One is written after each settled turn that loaded a skill.')
|
|
57
|
+
return lines.join('\n')
|
|
58
|
+
}
|
|
59
|
+
lines.push('| written | session | turn | skills | size | file |')
|
|
60
|
+
lines.push('|---|---|---|---|---|---|')
|
|
61
|
+
for (const row of rows.slice(0, limit)) {
|
|
62
|
+
const when = new Date(row.at).toISOString().replace('T', ' ').slice(0, 19)
|
|
63
|
+
lines.push(
|
|
64
|
+
`| ${when} | \`${String(row.sessionId ?? '?').slice(-8)}\` | ${row.turn ?? '—'} | ${(row.skills ?? []).join(', ') || '—'} | ${row.bytes}b | \`${row.file}\` |`,
|
|
65
|
+
)
|
|
66
|
+
}
|
|
67
|
+
return lines.join('\n')
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
export function createSnapshots({ analyzer, dir, debounceMs = 1500, log, turnReports = true, turnReportsKeep = 50 }) {
|
|
71
|
+
let timer = null
|
|
72
|
+
let running = null
|
|
73
|
+
let pending = false
|
|
74
|
+
|
|
75
|
+
const markdownPath = path.join(dir, 'latest.md')
|
|
76
|
+
const jsonPath = path.join(dir, 'latest.json')
|
|
77
|
+
|
|
78
|
+
async function write() {
|
|
79
|
+
const report = await analyzer.build()
|
|
80
|
+
await fs.mkdir(dir, { recursive: true })
|
|
81
|
+
await fs.writeFile(jsonPath, JSON.stringify(report, null, 2), 'utf8')
|
|
82
|
+
const markdown =
|
|
83
|
+
renderSummaryMarkdown({
|
|
84
|
+
...report,
|
|
85
|
+
skills: report.skills.map((skill) => ({
|
|
86
|
+
...skill,
|
|
87
|
+
topAdvice: skill.topAdvice.slice(0, 3),
|
|
88
|
+
})),
|
|
89
|
+
}) + '\n'
|
|
90
|
+
await fs.writeFile(markdownPath, markdown, 'utf8')
|
|
91
|
+
|
|
92
|
+
// One board per settled turn: the "每次执行后结果在单独的看板" surface.
|
|
93
|
+
if (turnReports) {
|
|
94
|
+
const turnEntry = [...(report.turnLoads ?? [])].sort((a, b) => (b.lastAt ?? 0) - (a.lastAt ?? 0))[0]
|
|
95
|
+
if (turnEntry) {
|
|
96
|
+
try {
|
|
97
|
+
const turnsDir = path.join(dir, 'turns')
|
|
98
|
+
await fs.mkdir(turnsDir, { recursive: true })
|
|
99
|
+
const slug = turnSlug(turnEntry)
|
|
100
|
+
await fs.writeFile(path.join(turnsDir, `${slug}.json`), JSON.stringify(report, null, 2), 'utf8')
|
|
101
|
+
await fs.writeFile(path.join(turnsDir, `${slug}.md`), markdown, 'utf8')
|
|
102
|
+
if (turnReportsKeep > 0) {
|
|
103
|
+
const files = (await fs.readdir(turnsDir)).filter((name) => name.endsWith('.json'))
|
|
104
|
+
if (files.length > turnReportsKeep) {
|
|
105
|
+
const withStats = await Promise.all(
|
|
106
|
+
files.map(async (name) => ({ name, at: (await fs.stat(path.join(turnsDir, name))).mtimeMs })),
|
|
107
|
+
)
|
|
108
|
+
withStats.sort((a, b) => b.at - a.at)
|
|
109
|
+
for (const stale of withStats.slice(turnReportsKeep)) {
|
|
110
|
+
await fs.rm(path.join(turnsDir, stale.name), { force: true })
|
|
111
|
+
await fs.rm(path.join(turnsDir, stale.name.replace(/\.json$/, '.md')), { force: true })
|
|
112
|
+
}
|
|
113
|
+
}
|
|
114
|
+
}
|
|
115
|
+
} catch (error) {
|
|
116
|
+
log?.(`turn snapshot failed: ${error instanceof Error ? error.message : String(error)}`)
|
|
117
|
+
}
|
|
118
|
+
}
|
|
119
|
+
}
|
|
120
|
+
return report
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
async function run() {
|
|
124
|
+
if (running) {
|
|
125
|
+
pending = true
|
|
126
|
+
return running
|
|
127
|
+
}
|
|
128
|
+
running = (async () => {
|
|
129
|
+
try {
|
|
130
|
+
await write()
|
|
131
|
+
} catch (error) {
|
|
132
|
+
log?.(`snapshot failed: ${error instanceof Error ? error.message : String(error)}`)
|
|
133
|
+
} finally {
|
|
134
|
+
running = null
|
|
135
|
+
if (pending) {
|
|
136
|
+
pending = false
|
|
137
|
+
void request()
|
|
138
|
+
}
|
|
139
|
+
}
|
|
140
|
+
})()
|
|
141
|
+
return running
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
function request() {
|
|
145
|
+
if (timer) return
|
|
146
|
+
timer = setTimeout(() => {
|
|
147
|
+
timer = null
|
|
148
|
+
void run()
|
|
149
|
+
}, debounceMs)
|
|
150
|
+
if (typeof timer.unref === 'function') timer.unref()
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
return {
|
|
154
|
+
request,
|
|
155
|
+
writeNow: run,
|
|
156
|
+
paths: { markdown: markdownPath, json: jsonPath },
|
|
157
|
+
async dispose() {
|
|
158
|
+
if (timer) {
|
|
159
|
+
clearTimeout(timer)
|
|
160
|
+
timer = null
|
|
161
|
+
}
|
|
162
|
+
if (running) await running
|
|
163
|
+
await run()
|
|
164
|
+
},
|
|
165
|
+
}
|
|
166
|
+
}
|