oh-my-knowledge 0.21.0 → 0.23.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -0
- package/README.zh.md +2 -0
- package/dist/src/analysis/report-diagnostics.d.ts +26 -2
- package/dist/src/analysis/report-diagnostics.d.ts.map +1 -1
- package/dist/src/analysis/report-diagnostics.js +96 -3
- package/dist/src/analysis/report-diagnostics.js.map +1 -1
- package/dist/src/analysis/sample-diagnostics.d.ts +5 -1
- package/dist/src/analysis/sample-diagnostics.d.ts.map +1 -1
- package/dist/src/analysis/sample-diagnostics.js +105 -0
- package/dist/src/analysis/sample-diagnostics.js.map +1 -1
- package/dist/src/authoring/evolver.d.ts +2 -2
- package/dist/src/authoring/evolver.d.ts.map +1 -1
- package/dist/src/authoring/evolver.js +24 -7
- package/dist/src/authoring/evolver.js.map +1 -1
- package/dist/src/authoring/generator.d.ts +24 -0
- package/dist/src/authoring/generator.d.ts.map +1 -1
- package/dist/src/authoring/generator.js +67 -5
- package/dist/src/authoring/generator.js.map +1 -1
- package/dist/src/cli/coverage-renderer.d.ts +15 -0
- package/dist/src/cli/coverage-renderer.d.ts.map +1 -0
- package/dist/src/cli/coverage-renderer.js +74 -0
- package/dist/src/cli/coverage-renderer.js.map +1 -0
- package/dist/src/cli/i18n-dict.d.ts +1 -1
- package/dist/src/cli/i18n-dict.d.ts.map +1 -1
- package/dist/src/cli/i18n-dict.js +42 -2
- package/dist/src/cli/i18n-dict.js.map +1 -1
- package/dist/src/cli/index.d.ts +3 -0
- package/dist/src/cli/index.d.ts.map +1 -0
- package/dist/src/{cli.js → cli/index.js} +54 -293
- package/dist/src/cli/index.js.map +1 -0
- package/dist/src/cli/parse-run-config.d.ts +56 -0
- package/dist/src/cli/parse-run-config.d.ts.map +1 -0
- package/dist/src/cli/parse-run-config.js +195 -0
- package/dist/src/cli/parse-run-config.js.map +1 -0
- package/dist/src/cli/progress.d.ts +24 -0
- package/dist/src/cli/progress.d.ts.map +1 -0
- package/dist/src/cli/progress.js +55 -0
- package/dist/src/cli/progress.js.map +1 -0
- package/dist/src/cli/update-check.d.ts +3 -0
- package/dist/src/cli/update-check.d.ts.map +1 -0
- package/dist/src/cli/update-check.js +37 -0
- package/dist/src/cli/update-check.js.map +1 -0
- package/dist/src/eval-core/cache.d.ts +6 -2
- package/dist/src/eval-core/cache.d.ts.map +1 -1
- package/dist/src/eval-core/cache.js +14 -4
- package/dist/src/eval-core/cache.js.map +1 -1
- package/dist/src/eval-core/dependency-checker.d.ts +1 -1
- package/dist/src/eval-core/dependency-checker.js +1 -1
- package/dist/src/eval-core/evaluation-execution.d.ts +1 -1
- package/dist/src/eval-core/evaluation-execution.d.ts.map +1 -1
- package/dist/src/eval-core/evaluation-execution.js +6 -4
- package/dist/src/eval-core/evaluation-execution.js.map +1 -1
- package/dist/src/eval-core/evaluation-reporting.d.ts.map +1 -1
- package/dist/src/eval-core/evaluation-reporting.js +4 -0
- package/dist/src/eval-core/evaluation-reporting.js.map +1 -1
- package/dist/src/eval-core/execution-strategy.d.ts.map +1 -1
- package/dist/src/eval-core/execution-strategy.js +35 -2
- package/dist/src/eval-core/execution-strategy.js.map +1 -1
- package/dist/src/eval-core/schema.d.ts.map +1 -1
- package/dist/src/eval-core/schema.js +16 -2
- package/dist/src/eval-core/schema.js.map +1 -1
- package/dist/src/eval-core/verdict.d.ts +3 -3
- package/dist/src/eval-core/verdict.js +3 -3
- package/dist/src/eval-workflows/each-evaluation-workflow.d.ts +8 -1
- package/dist/src/eval-workflows/each-evaluation-workflow.d.ts.map +1 -1
- package/dist/src/eval-workflows/each-evaluation-workflow.js +3 -2
- package/dist/src/eval-workflows/each-evaluation-workflow.js.map +1 -1
- package/dist/src/eval-workflows/evaluation-pipeline.d.ts +16 -2
- package/dist/src/eval-workflows/evaluation-pipeline.d.ts.map +1 -1
- package/dist/src/eval-workflows/evaluation-pipeline.js +50 -4
- package/dist/src/eval-workflows/evaluation-pipeline.js.map +1 -1
- package/dist/src/eval-workflows/evaluation-preparation.d.ts +5 -1
- package/dist/src/eval-workflows/evaluation-preparation.d.ts.map +1 -1
- package/dist/src/eval-workflows/evaluation-preparation.js +2 -2
- package/dist/src/eval-workflows/evaluation-preparation.js.map +1 -1
- package/dist/src/eval-workflows/run-evaluation.d.ts +14 -3
- package/dist/src/eval-workflows/run-evaluation.d.ts.map +1 -1
- package/dist/src/eval-workflows/run-evaluation.js +25 -3
- package/dist/src/eval-workflows/run-evaluation.js.map +1 -1
- package/dist/src/executors/claude-cli.d.ts +1 -1
- package/dist/src/executors/claude-cli.d.ts.map +1 -1
- package/dist/src/executors/claude-cli.js +21 -1
- package/dist/src/executors/claude-cli.js.map +1 -1
- package/dist/src/executors/claude-sdk.d.ts +14 -1
- package/dist/src/executors/claude-sdk.d.ts.map +1 -1
- package/dist/src/executors/claude-sdk.js +19 -1
- package/dist/src/executors/claude-sdk.js.map +1 -1
- package/dist/src/executors/script.d.ts.map +1 -1
- package/dist/src/executors/script.js +10 -1
- package/dist/src/executors/script.js.map +1 -1
- package/dist/src/grading/assertions.js +3 -3
- package/dist/src/grading/assertions.js.map +1 -1
- package/dist/src/grading/gold-dataset.d.ts +1 -1
- package/dist/src/grading/gold-dataset.js +1 -1
- package/dist/src/inputs/eval-config.js +20 -1
- package/dist/src/inputs/eval-config.js.map +1 -1
- package/dist/src/inputs/load-samples.d.ts.map +1 -1
- package/dist/src/inputs/load-samples.js +30 -4
- package/dist/src/inputs/load-samples.js.map +1 -1
- package/dist/src/inputs/skill-loader.d.ts +10 -1
- package/dist/src/inputs/skill-loader.d.ts.map +1 -1
- package/dist/src/inputs/skill-loader.js +18 -1
- package/dist/src/inputs/skill-loader.js.map +1 -1
- package/dist/src/renderer/html-renderer.js +2 -2
- package/dist/src/renderer/html-renderer.js.map +1 -1
- package/dist/src/renderer/summary.d.ts.map +1 -1
- package/dist/src/renderer/summary.js +0 -13
- package/dist/src/renderer/summary.js.map +1 -1
- package/dist/src/renderer/trends.js +1 -1
- package/dist/src/renderer/trends.js.map +1 -1
- package/dist/src/server/report-server.js +15 -15
- package/dist/src/server/report-server.js.map +1 -1
- package/dist/src/types/eval.d.ts +30 -2
- package/dist/src/types/eval.d.ts.map +1 -1
- package/dist/src/types/executor.d.ts +1 -0
- package/dist/src/types/executor.d.ts.map +1 -1
- package/dist/src/types/report.d.ts +38 -2
- package/dist/src/types/report.d.ts.map +1 -1
- package/package.json +14 -6
- package/dist/src/cli.d.ts +0 -3
- package/dist/src/cli.d.ts.map +0 -1
- package/dist/src/cli.js.map +0 -1
|
@@ -1,208 +1,12 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
2
|
import { parseArgs } from 'node:util';
|
|
3
3
|
import { resolve } from 'node:path';
|
|
4
|
-
import { homedir } from 'node:os';
|
|
5
4
|
import { join } from 'node:path';
|
|
6
5
|
import { existsSync } from 'node:fs';
|
|
7
|
-
import { tCli, getCliLang, parseLangFromArgv, langFromArgv } from './
|
|
8
|
-
import {
|
|
9
|
-
import {
|
|
10
|
-
|
|
11
|
-
// Constants
|
|
12
|
-
// ---------------------------------------------------------------------------
|
|
13
|
-
const DEFAULT_REPORTS_DIR = join(homedir(), '.oh-my-knowledge', 'reports');
|
|
14
|
-
// Shared CLI options for run/ci commands.
|
|
15
|
-
// Defaults are applied inside parseRunConfig (after config-file merge) so that
|
|
16
|
-
// CLI `undefined` can be reliably distinguished from "user passed the default value".
|
|
17
|
-
// Priority order resolved in parseRunConfig: CLI arg > --config file > hard-coded default.
|
|
18
|
-
/**
|
|
19
|
-
* 所有子命令都接受的通用 flag。新增 --lang 让 parseArgs strict:false 模式下
|
|
20
|
-
* 仍能把值类型化到 values.lang 上(否则未声明的 flag 会被丢弃)。
|
|
21
|
-
*/
|
|
22
|
-
const COMMON_OPTIONS = {
|
|
23
|
-
lang: { type: 'string' },
|
|
24
|
-
};
|
|
25
|
-
const RUN_OPTIONS = {
|
|
26
|
-
...COMMON_OPTIONS,
|
|
27
|
-
samples: { type: 'string' },
|
|
28
|
-
'skill-dir': { type: 'string' },
|
|
29
|
-
control: { type: 'string' },
|
|
30
|
-
treatment: { type: 'string' },
|
|
31
|
-
config: { type: 'string' },
|
|
32
|
-
model: { type: 'string' },
|
|
33
|
-
'judge-model': { type: 'string' },
|
|
34
|
-
'output-dir': { type: 'string' },
|
|
35
|
-
'no-judge': { type: 'boolean' },
|
|
36
|
-
'no-cache': { type: 'boolean' },
|
|
37
|
-
'dry-run': { type: 'boolean' },
|
|
38
|
-
concurrency: { type: 'string' },
|
|
39
|
-
timeout: { type: 'string' },
|
|
40
|
-
executor: { type: 'string' },
|
|
41
|
-
'judge-executor': { type: 'string' },
|
|
42
|
-
each: { type: 'boolean' },
|
|
43
|
-
'skip-preflight': { type: 'boolean' },
|
|
44
|
-
'mcp-config': { type: 'string' },
|
|
45
|
-
'no-serve': { type: 'boolean' },
|
|
46
|
-
verbose: { type: 'boolean' },
|
|
47
|
-
retry: { type: 'string' },
|
|
48
|
-
resume: { type: 'string' },
|
|
49
|
-
'layered-stats': { type: 'boolean' },
|
|
50
|
-
};
|
|
51
|
-
// ---------------------------------------------------------------------------
|
|
52
|
-
// parseRunConfig
|
|
53
|
-
// ---------------------------------------------------------------------------
|
|
54
|
-
function parseRunConfig(argv, extraOptions = {}) {
|
|
55
|
-
const { values } = parseArgs({
|
|
56
|
-
args: argv,
|
|
57
|
-
options: { ...RUN_OPTIONS, ...extraOptions },
|
|
58
|
-
strict: false,
|
|
59
|
-
});
|
|
60
|
-
if (values.variants !== undefined) {
|
|
61
|
-
throw new Error(`--variants 已在 v0.16 废除,请改用 --control <expr> 与 --treatment <v1,v2,...>\n`
|
|
62
|
-
+ ` 迁移示例:--variants baseline,my-skill → --control baseline --treatment my-skill\n`
|
|
63
|
-
+ ` 复杂场景可用 --config eval.yaml(参见 docs/terminology-spec.md)`);
|
|
64
|
-
}
|
|
65
|
-
// 1) Load --config (if provided). All subsequent fields fall back to it when CLI is silent.
|
|
66
|
-
const evalConfig = values.config
|
|
67
|
-
? loadEvalConfig(values.config)
|
|
68
|
-
: null;
|
|
69
|
-
// 2) Resolve samples path: CLI > config > auto-detect .json/.yaml/.yml in cwd.
|
|
70
|
-
const cliSamples = values.samples;
|
|
71
|
-
let samplesFile;
|
|
72
|
-
if (cliSamples) {
|
|
73
|
-
samplesFile = cliSamples;
|
|
74
|
-
}
|
|
75
|
-
else if (evalConfig?.samples) {
|
|
76
|
-
samplesFile = evalConfig.samples; // already resolved against config file dir
|
|
77
|
-
}
|
|
78
|
-
else {
|
|
79
|
-
samplesFile = 'eval-samples.json';
|
|
80
|
-
if (!existsSync(resolve(samplesFile))) {
|
|
81
|
-
if (existsSync(resolve('eval-samples.yaml')))
|
|
82
|
-
samplesFile = 'eval-samples.yaml';
|
|
83
|
-
else if (existsSync(resolve('eval-samples.yml')))
|
|
84
|
-
samplesFile = 'eval-samples.yml';
|
|
85
|
-
}
|
|
86
|
-
}
|
|
87
|
-
const skillDir = resolve(values['skill-dir'] ?? 'skills');
|
|
88
|
-
// 3) Resolve variantSpecs: CLI > config. If neither, error with a helpful hint.
|
|
89
|
-
const controlExpr = values.control;
|
|
90
|
-
const treatmentExprs = values.treatment
|
|
91
|
-
? values.treatment.split(',').map((v) => v.trim()).filter(Boolean)
|
|
92
|
-
: [];
|
|
93
|
-
let variantSpecs;
|
|
94
|
-
if (controlExpr || treatmentExprs.length > 0) {
|
|
95
|
-
// CLI roles present → CLI entirely replaces config.variants (no merging).
|
|
96
|
-
variantSpecs = [];
|
|
97
|
-
if (controlExpr) {
|
|
98
|
-
variantSpecs.push({ name: parseVariantCwd(controlExpr).name, role: 'control', expr: controlExpr });
|
|
99
|
-
}
|
|
100
|
-
for (const expr of treatmentExprs) {
|
|
101
|
-
variantSpecs.push({ name: parseVariantCwd(expr).name, role: 'treatment', expr });
|
|
102
|
-
}
|
|
103
|
-
}
|
|
104
|
-
else if (evalConfig) {
|
|
105
|
-
variantSpecs = configVariantsToSpecs(evalConfig.variants);
|
|
106
|
-
}
|
|
107
|
-
else if (values.each) {
|
|
108
|
-
// --each 模式自动用 baseline (control) vs 每个 skill (treatment),
|
|
109
|
-
// 不需要用户显式传 --control / --treatment,校验跳过。
|
|
110
|
-
variantSpecs = [];
|
|
111
|
-
}
|
|
112
|
-
else {
|
|
113
|
-
const discovered = discoverVariants(skillDir);
|
|
114
|
-
const hint = discovered.length > 0 ? `\n skill-dir (${skillDir}) 下发现的候选:${discovered.join(', ')}` : '';
|
|
115
|
-
throw new Error(`请通过 --control / --treatment 或 --config eval.yaml 声明 variant 角色。\n`
|
|
116
|
-
+ ` 示例:omk bench run --control baseline --treatment my-skill${hint}\n`
|
|
117
|
-
+ ` --each 模式下自动用 baseline vs 每个 skill,无需显式声明\n`
|
|
118
|
-
+ ` 术语见 docs/terminology-spec.md(v0.16 起废除 --variants,改用 experiment role 显式声明)`);
|
|
119
|
-
}
|
|
120
|
-
const seenNames = new Set();
|
|
121
|
-
for (const spec of variantSpecs) {
|
|
122
|
-
if (seenNames.has(spec.name)) {
|
|
123
|
-
throw new Error(`variant "${spec.name}" 重复出现——同一 variant 不能同时属于 --control 与 --treatment,也不能在 --treatment 中重复。`);
|
|
124
|
-
}
|
|
125
|
-
seenNames.add(spec.name);
|
|
126
|
-
}
|
|
127
|
-
// 4) Apply CLI > config > hard-coded default for all other fields.
|
|
128
|
-
const executorName = values.executor ?? evalConfig?.executor ?? 'claude';
|
|
129
|
-
const judgeExecutorName = values['judge-executor'] ?? evalConfig?.judgeExecutor ?? executorName;
|
|
130
|
-
const model = values.model ?? evalConfig?.model ?? 'sonnet';
|
|
131
|
-
const judgeModelRaw = values['judge-model'] !== undefined
|
|
132
|
-
? values['judge-model']
|
|
133
|
-
: evalConfig?.judgeModel ?? 'haiku';
|
|
134
|
-
const judgeModel = judgeModelRaw ?? 'haiku';
|
|
135
|
-
const outputDir = resolve(values['output-dir'] ?? DEFAULT_REPORTS_DIR);
|
|
136
|
-
const concurrencyRaw = values.concurrency !== undefined
|
|
137
|
-
? Number(values.concurrency)
|
|
138
|
-
: evalConfig?.concurrency ?? 1;
|
|
139
|
-
const concurrency = Math.max(1, Number(concurrencyRaw) || 1);
|
|
140
|
-
const timeoutSec = values.timeout !== undefined
|
|
141
|
-
? Number(values.timeout)
|
|
142
|
-
: evalConfig?.timeoutMs
|
|
143
|
-
? evalConfig.timeoutMs / 1000
|
|
144
|
-
: 120;
|
|
145
|
-
const timeoutMs = Math.max(1, Number(timeoutSec) || 120) * 1000;
|
|
146
|
-
const noJudge = values['no-judge'] ?? false;
|
|
147
|
-
const noCache = values['no-cache'] ?? evalConfig?.noCache ?? false;
|
|
148
|
-
const dryRun = values['dry-run'] ?? false;
|
|
149
|
-
const skipPreflight = values['skip-preflight'] ?? false;
|
|
150
|
-
const mcpConfig = values['mcp-config'] ?? evalConfig?.mcpConfig;
|
|
151
|
-
const verbose = values.verbose ?? false;
|
|
152
|
-
const retry = Math.max(0, Number(values.retry ?? 0) || 0);
|
|
153
|
-
const resume = values.resume;
|
|
154
|
-
const blind = values.blind ?? evalConfig?.blind ?? false;
|
|
155
|
-
const layeredStats = values['layered-stats'] ?? false;
|
|
156
|
-
return {
|
|
157
|
-
values,
|
|
158
|
-
config: {
|
|
159
|
-
samplesPath: resolve(samplesFile),
|
|
160
|
-
skillDir,
|
|
161
|
-
variantSpecs,
|
|
162
|
-
model,
|
|
163
|
-
judgeModel,
|
|
164
|
-
outputDir,
|
|
165
|
-
noJudge,
|
|
166
|
-
noCache,
|
|
167
|
-
dryRun,
|
|
168
|
-
concurrency,
|
|
169
|
-
timeoutMs,
|
|
170
|
-
executorName,
|
|
171
|
-
judgeExecutorName,
|
|
172
|
-
skipPreflight,
|
|
173
|
-
mcpConfig,
|
|
174
|
-
verbose,
|
|
175
|
-
retry,
|
|
176
|
-
resume,
|
|
177
|
-
blind,
|
|
178
|
-
layeredStats,
|
|
179
|
-
budget: evalConfig?.budget,
|
|
180
|
-
},
|
|
181
|
-
};
|
|
182
|
-
}
|
|
183
|
-
// ---------------------------------------------------------------------------
|
|
184
|
-
// Update check
|
|
185
|
-
// ---------------------------------------------------------------------------
|
|
186
|
-
async function checkUpdate(lang) {
|
|
187
|
-
try {
|
|
188
|
-
const { readFileSync } = await import('node:fs');
|
|
189
|
-
const { fileURLToPath } = await import('node:url');
|
|
190
|
-
const { dirname, join } = await import('node:path');
|
|
191
|
-
const __dirname = dirname(fileURLToPath(import.meta.url));
|
|
192
|
-
const pkg = JSON.parse(readFileSync(join(__dirname, 'package.json'), 'utf-8'));
|
|
193
|
-
const registry = pkg.publishConfig?.registry || 'https://registry.npmjs.org';
|
|
194
|
-
const res = await fetch(`${registry}/${pkg.name}/latest`, { signal: AbortSignal.timeout(3000) });
|
|
195
|
-
if (!res.ok)
|
|
196
|
-
return;
|
|
197
|
-
const data = await res.json();
|
|
198
|
-
if (data.version && data.version !== pkg.version) {
|
|
199
|
-
process.stderr.write(tCli('cli.update.new_version_available', lang, {
|
|
200
|
-
old: pkg.version, new: data.version, pkg: pkg.name,
|
|
201
|
-
}));
|
|
202
|
-
}
|
|
203
|
-
}
|
|
204
|
-
catch { /* 静默失败,不影响正常使用 */ }
|
|
205
|
-
}
|
|
6
|
+
import { tCli, getCliLang, parseLangFromArgv, langFromArgv } from './i18n.js';
|
|
7
|
+
import { parseRunConfig, DEFAULT_REPORTS_DIR, COMMON_OPTIONS, } from './parse-run-config.js';
|
|
8
|
+
import { makeOnProgress } from './progress.js';
|
|
9
|
+
import { checkUpdate } from './update-check.js';
|
|
206
10
|
// ---------------------------------------------------------------------------
|
|
207
11
|
// Main
|
|
208
12
|
// ---------------------------------------------------------------------------
|
|
@@ -279,66 +83,13 @@ async function main() {
|
|
|
279
83
|
* Factory: 闭住 lang, 返回 onProgress callback。evaluation engine 回调时不传
|
|
280
84
|
* 上下文, 所以 lang 必须在 handler 入口处通过 closure 传进来。
|
|
281
85
|
*/
|
|
282
|
-
function makeOnProgress(lang) {
|
|
283
|
-
return ({ phase, completed, total, sample_id, variant, durationMs, inputTokens, outputTokens, costUSD, score, outputPreview, judgePhase: _judgePhase, judgeDim, skipped, attempt, maxAttempts, error, }) => {
|
|
284
|
-
const ctx = { i: completed ?? '', n: total ?? '', sample: sample_id ?? '', variant: variant ?? '' };
|
|
285
|
-
if (phase === 'preflight') {
|
|
286
|
-
process.stderr.write(tCli('cli.progress.preflight_starting', lang));
|
|
287
|
-
return;
|
|
288
|
-
}
|
|
289
|
-
if (phase === 'retry') {
|
|
290
|
-
process.stderr.write(tCli('cli.progress.sample_retry', lang, {
|
|
291
|
-
...ctx, attempt: attempt ?? '', max: maxAttempts ?? '',
|
|
292
|
-
}));
|
|
293
|
-
return;
|
|
294
|
-
}
|
|
295
|
-
if (phase === 'error') {
|
|
296
|
-
process.stderr.write(tCli('cli.progress.sample_error', lang, { ...ctx, error: error ?? '' }));
|
|
297
|
-
return;
|
|
298
|
-
}
|
|
299
|
-
if (phase === 'start') {
|
|
300
|
-
process.stderr.write(tCli('cli.progress.sample_executing', lang, ctx));
|
|
301
|
-
}
|
|
302
|
-
else if (phase === 'exec_done') {
|
|
303
|
-
const cost = costUSD != null && costUSD > 0 ? ` $${costUSD.toFixed(4)}` : '';
|
|
304
|
-
process.stderr.write(tCli('cli.progress.sample_exec_done', lang, {
|
|
305
|
-
...ctx, ms: durationMs ?? '', input: inputTokens ?? '', output: outputTokens ?? '', cost,
|
|
306
|
-
}));
|
|
307
|
-
if (outputPreview) {
|
|
308
|
-
process.stderr.write(tCli('cli.progress.output_preview', lang, {
|
|
309
|
-
preview: outputPreview.slice(0, 150).replace(/\n/g, ' '),
|
|
310
|
-
}));
|
|
311
|
-
}
|
|
312
|
-
}
|
|
313
|
-
else if (phase === 'grading') {
|
|
314
|
-
const dim = judgeDim ? ` [${judgeDim}]` : '';
|
|
315
|
-
process.stderr.write(tCli('cli.progress.judging', lang, { ...ctx, dim }));
|
|
316
|
-
}
|
|
317
|
-
else if (phase === 'judge_done') {
|
|
318
|
-
const dim = judgeDim ? ` [${judgeDim}]` : '';
|
|
319
|
-
process.stderr.write(tCli('cli.progress.judged', lang, { ...ctx, dim, score: score ?? '' }));
|
|
320
|
-
}
|
|
321
|
-
else if (phase === 'done' && skipped) {
|
|
322
|
-
if (sample_id)
|
|
323
|
-
process.stderr.write(tCli('cli.progress.skipped', lang, ctx));
|
|
324
|
-
}
|
|
325
|
-
else {
|
|
326
|
-
const cost = costUSD != null && costUSD > 0 ? ` $${costUSD.toFixed(4)}` : '';
|
|
327
|
-
const scoreInfo = typeof score === 'number' ? ` score=${score}` : '';
|
|
328
|
-
process.stderr.write(tCli('cli.progress.sample_done', lang, {
|
|
329
|
-
...ctx, ms: durationMs ?? '', input: inputTokens ?? '', output: outputTokens ?? '',
|
|
330
|
-
cost, score: scoreInfo,
|
|
331
|
-
}));
|
|
332
|
-
}
|
|
333
|
-
};
|
|
334
|
-
}
|
|
335
86
|
// ---------------------------------------------------------------------------
|
|
336
87
|
// handleRun
|
|
337
88
|
// ---------------------------------------------------------------------------
|
|
338
89
|
async function handleRun(argv) {
|
|
339
90
|
const lang = langFromArgv(argv);
|
|
340
91
|
const { values, config } = parseRunConfig(argv, {
|
|
341
|
-
blind: { type: 'boolean'
|
|
92
|
+
blind: { type: 'boolean' },
|
|
342
93
|
repeat: { type: 'string', default: '1' },
|
|
343
94
|
'judge-repeat': { type: 'string', default: '1' },
|
|
344
95
|
'judge-models': { type: 'string' },
|
|
@@ -350,8 +101,10 @@ async function handleRun(argv) {
|
|
|
350
101
|
'budget-per-sample-usd': { type: 'string' },
|
|
351
102
|
'budget-per-sample-ms': { type: 'string' },
|
|
352
103
|
});
|
|
353
|
-
const { runEvaluation, runMultiple, runEachEvaluation } = await import('
|
|
354
|
-
|
|
104
|
+
const { runEvaluation, runMultiple, runEachEvaluation } = await import('../eval-workflows/run-evaluation.js');
|
|
105
|
+
if (values.blind !== undefined) {
|
|
106
|
+
config.blind = values.blind;
|
|
107
|
+
}
|
|
355
108
|
config.onProgress = makeOnProgress(lang);
|
|
356
109
|
// --repeat 输入校验: 非 ≥1 整数时提示并钳到 1, 不静默掩盖用户错字 / 极端输入。
|
|
357
110
|
// 提前到 --each 分支之前, 保证 each 模式也能读到 repeat (曾经 bug: --each 吞 --repeat)。
|
|
@@ -394,7 +147,7 @@ async function handleRun(argv) {
|
|
|
394
147
|
}
|
|
395
148
|
}
|
|
396
149
|
// --budget-usd / --budget-per-sample-usd / --budget-per-sample-ms:
|
|
397
|
-
//
|
|
150
|
+
// hard budget caps. CLI flags override config-file values. When the
|
|
398
151
|
// total-USD cap is exceeded mid-run, remaining tasks are skipped and a
|
|
399
152
|
// partial report is persisted with meta.budgetExhausted=true.
|
|
400
153
|
const budgetUSD = values['budget-usd'] != null ? Number(values['budget-usd']) : undefined;
|
|
@@ -447,10 +200,10 @@ async function handleRun(argv) {
|
|
|
447
200
|
process.stderr.write(tCli('cli.run.batch_complete', lang));
|
|
448
201
|
process.stderr.write(tCli('cli.run.report_saved', lang, { path: filePath }));
|
|
449
202
|
if (!values['no-serve'] && process.stdout.isTTY) {
|
|
450
|
-
const { createReportServer } = await import('
|
|
203
|
+
const { createReportServer } = await import('../server/report-server.js');
|
|
451
204
|
const server = createReportServer({ reportsDir: config.outputDir });
|
|
452
205
|
const serverUrl = await server.start();
|
|
453
|
-
const reportUrl = `${serverUrl}/
|
|
206
|
+
const reportUrl = `${serverUrl}/reports/${report.id}`;
|
|
454
207
|
process.stderr.write(tCli('cli.run.report_server_running', lang, { url: serverUrl }));
|
|
455
208
|
process.stderr.write(tCli('cli.run.report_server_view', lang, { url: reportUrl }));
|
|
456
209
|
process.stderr.write(tCli('cli.run.report_server_stop', lang));
|
|
@@ -487,7 +240,7 @@ async function handleRun(argv) {
|
|
|
487
240
|
// --gold-dir: compute α/κ/Pearson against gold annotations and re-persist.
|
|
488
241
|
const goldDir = values['gold-dir'];
|
|
489
242
|
if (goldDir && filePath) {
|
|
490
|
-
const { attachGoldAgreementToReport, formatGoldCompare } = await import('
|
|
243
|
+
const { attachGoldAgreementToReport, formatGoldCompare } = await import('../grading/gold-cli.js');
|
|
491
244
|
const out = attachGoldAgreementToReport({
|
|
492
245
|
report,
|
|
493
246
|
goldDir,
|
|
@@ -515,12 +268,12 @@ async function handleRun(argv) {
|
|
|
515
268
|
process.stderr.write(tCli('cli.run.report_saved', lang, { path: filePath }));
|
|
516
269
|
if (!values['no-serve'] && process.stdout.isTTY) {
|
|
517
270
|
// Auto-start report server
|
|
518
|
-
const { createReportServer } = await import('
|
|
271
|
+
const { createReportServer } = await import('../server/report-server.js');
|
|
519
272
|
const server = createReportServer({
|
|
520
273
|
reportsDir: config.outputDir,
|
|
521
274
|
});
|
|
522
275
|
const serverUrl = await server.start();
|
|
523
|
-
const reportUrl = `${serverUrl}/
|
|
276
|
+
const reportUrl = `${serverUrl}/reports/${report.id}`;
|
|
524
277
|
process.stderr.write(tCli('cli.run.report_server_running', lang, { url: serverUrl }));
|
|
525
278
|
process.stderr.write(tCli('cli.run.report_server_view', lang, { url: reportUrl }));
|
|
526
279
|
process.stderr.write(tCli('cli.run.report_server_stop', lang));
|
|
@@ -576,8 +329,8 @@ async function handleReport(argv) {
|
|
|
576
329
|
return;
|
|
577
330
|
}
|
|
578
331
|
if (values.export) {
|
|
579
|
-
const { createFileStore } = await import('
|
|
580
|
-
const { renderRunDetail, renderEachRunDetail } = await import('
|
|
332
|
+
const { createFileStore } = await import('../server/report-store.js');
|
|
333
|
+
const { renderRunDetail, renderEachRunDetail } = await import('../renderer/html-renderer.js');
|
|
581
334
|
const { writeFileSync } = await import('node:fs');
|
|
582
335
|
const store = createFileStore(resolve(values['reports-dir']));
|
|
583
336
|
const report = await store.get(values.export);
|
|
@@ -592,7 +345,7 @@ async function handleReport(argv) {
|
|
|
592
345
|
console.log('Open in browser, or Ctrl+P to save as PDF');
|
|
593
346
|
return;
|
|
594
347
|
}
|
|
595
|
-
const { createReportServer } = await import('
|
|
348
|
+
const { createReportServer } = await import('../server/report-server.js');
|
|
596
349
|
const server = createReportServer({
|
|
597
350
|
port: Number(values.port),
|
|
598
351
|
reportsDir: resolve(values['reports-dir']),
|
|
@@ -715,7 +468,7 @@ async function handleAnalyze(argv) {
|
|
|
715
468
|
const to = values.to;
|
|
716
469
|
const skills = values.skills ? values.skills.split(',').map((s) => s.trim()).filter(Boolean) : undefined;
|
|
717
470
|
console.log(`[omk] analyzing ${tracePath}...`);
|
|
718
|
-
const { computeSkillHealthReport } = await import('
|
|
471
|
+
const { computeSkillHealthReport } = await import('../observability/skill-health-analyzer.js');
|
|
719
472
|
const report = computeSkillHealthReport(tracePath, {
|
|
720
473
|
kbRoot: values.kb ? resolve(values.kb) : undefined,
|
|
721
474
|
from,
|
|
@@ -776,7 +529,7 @@ async function handleGenSamples(argv) {
|
|
|
776
529
|
strict: false,
|
|
777
530
|
allowPositionals: true,
|
|
778
531
|
});
|
|
779
|
-
const { generateSamples } = await import('
|
|
532
|
+
const { generateSamples } = await import('../authoring/generator.js');
|
|
780
533
|
const { readFileSync, writeFileSync } = await import('node:fs');
|
|
781
534
|
const count = Math.max(1, Number(values.count) || 5);
|
|
782
535
|
const model = values.model;
|
|
@@ -908,7 +661,7 @@ async function handleEvolve(argv) {
|
|
|
908
661
|
else if (existsSync(resolve('eval-samples.yml')))
|
|
909
662
|
samplesFile = 'eval-samples.yml';
|
|
910
663
|
}
|
|
911
|
-
const { evolveSkill } = await import('
|
|
664
|
+
const { evolveSkill } = await import('../authoring/evolver.js');
|
|
912
665
|
process.stderr.write(tCli('cli.evolve.section_header', lang, { path: skillPath }));
|
|
913
666
|
try {
|
|
914
667
|
const result = await evolveSkill({
|
|
@@ -982,7 +735,7 @@ async function handleGate(argv) {
|
|
|
982
735
|
threshold: { type: 'string', default: '3.5' },
|
|
983
736
|
'trivial-diff': { type: 'string' },
|
|
984
737
|
});
|
|
985
|
-
const { runEvaluation } = await import('
|
|
738
|
+
const { runEvaluation } = await import('../eval-workflows/run-evaluation.js');
|
|
986
739
|
config.onProgress = makeOnProgress(lang);
|
|
987
740
|
try {
|
|
988
741
|
const { report } = (await runEvaluation(config));
|
|
@@ -994,7 +747,7 @@ async function handleGate(argv) {
|
|
|
994
747
|
// bootstrap diff CI / saturation / Krippendorff α)。computeVerdict 是单一
|
|
995
748
|
// 决策源, exit code 跟 verdict.level 走 — 数据 underpowered 直接 FAIL,
|
|
996
749
|
// 堵住"过 PASS 就 deploy"的漏洞。
|
|
997
|
-
const { computeVerdict, formatVerdictText } = await import('
|
|
750
|
+
const { computeVerdict, formatVerdictText } = await import('../eval-core/verdict.js');
|
|
998
751
|
const result = computeVerdict(report, {
|
|
999
752
|
gateThreshold: Number(values.threshold),
|
|
1000
753
|
triviallySmallDiff: values['trivial-diff'] != null ? Number(values['trivial-diff']) : undefined,
|
|
@@ -1022,7 +775,7 @@ async function handleGate(argv) {
|
|
|
1022
775
|
async function handleDiff(argv) {
|
|
1023
776
|
const lang = langFromArgv(argv);
|
|
1024
777
|
// Flag-aware split: separate positional report IDs from flags so we can support
|
|
1025
|
-
// omk bench diff <id> — within-report sample-level
|
|
778
|
+
// omk bench diff <id> — within-report sample-level
|
|
1026
779
|
// omk bench diff <id1> <id2> — cross-report variant-level (legacy)
|
|
1027
780
|
// both with optional --regressions-only / --threshold / --variant flags.
|
|
1028
781
|
const positional = [];
|
|
@@ -1056,7 +809,7 @@ async function handleDiff(argv) {
|
|
|
1056
809
|
},
|
|
1057
810
|
strict: false,
|
|
1058
811
|
});
|
|
1059
|
-
const { createFileStore } = await import('
|
|
812
|
+
const { createFileStore } = await import('../server/report-store.js');
|
|
1060
813
|
const store = createFileStore(resolve(DEFAULT_REPORTS_DIR));
|
|
1061
814
|
if (positional.length === 1) {
|
|
1062
815
|
await runSampleLevelDiff(positional[0], store, values, lang);
|
|
@@ -1120,7 +873,7 @@ async function handleDiff(argv) {
|
|
|
1120
873
|
console.log('');
|
|
1121
874
|
}
|
|
1122
875
|
/**
|
|
1123
|
-
* Within-report sample-level diff
|
|
876
|
+
* Within-report sample-level diff. Compares two variants' scores on
|
|
1124
877
|
* each shared sample and surfaces the worst regressions / biggest wins.
|
|
1125
878
|
*
|
|
1126
879
|
* Default focus is variants[0] (control) vs variants[1] (treatment), but
|
|
@@ -1223,7 +976,7 @@ async function handleGold(argv) {
|
|
|
1223
976
|
},
|
|
1224
977
|
strict: false,
|
|
1225
978
|
});
|
|
1226
|
-
const { initGoldDataset } = await import('
|
|
979
|
+
const { initGoldDataset } = await import('../grading/gold-cli.js');
|
|
1227
980
|
try {
|
|
1228
981
|
const written = initGoldDataset(values.out, {
|
|
1229
982
|
annotator: values.annotator,
|
|
@@ -1247,7 +1000,7 @@ async function handleGold(argv) {
|
|
|
1247
1000
|
console.error(tCli('cli.common.usage_gold_validate', lang));
|
|
1248
1001
|
process.exit(1);
|
|
1249
1002
|
}
|
|
1250
|
-
const { validateGoldDataset } = await import('
|
|
1003
|
+
const { validateGoldDataset } = await import('../grading/gold-cli.js');
|
|
1251
1004
|
const result = validateGoldDataset(dir);
|
|
1252
1005
|
if (result.ok) {
|
|
1253
1006
|
console.log(tCli('cli.gold.validate_ok', lang, { n: result.sampleCount }));
|
|
@@ -1281,9 +1034,9 @@ async function handleGold(argv) {
|
|
|
1281
1034
|
console.error('--gold-dir is required');
|
|
1282
1035
|
process.exit(1);
|
|
1283
1036
|
}
|
|
1284
|
-
const { loadGoldDataset } = await import('
|
|
1285
|
-
const { compareGoldToReport, formatGoldCompare } = await import('
|
|
1286
|
-
const { createFileStore } = await import('
|
|
1037
|
+
const { loadGoldDataset } = await import('../grading/gold-dataset.js');
|
|
1038
|
+
const { compareGoldToReport, formatGoldCompare } = await import('../grading/gold-cli.js');
|
|
1039
|
+
const { createFileStore } = await import('../server/report-store.js');
|
|
1287
1040
|
const { dataset, issues } = loadGoldDataset(goldDir);
|
|
1288
1041
|
if (!dataset) {
|
|
1289
1042
|
console.error('Cannot load gold dataset:');
|
|
@@ -1351,7 +1104,7 @@ async function handleDebiasValidate(argv) {
|
|
|
1351
1104
|
},
|
|
1352
1105
|
strict: false,
|
|
1353
1106
|
});
|
|
1354
|
-
const { createFileStore } = await import('
|
|
1107
|
+
const { createFileStore } = await import('../server/report-store.js');
|
|
1355
1108
|
const store = createFileStore(resolve(values['reports-dir']));
|
|
1356
1109
|
const report = await store.get(reportId);
|
|
1357
1110
|
if (!report) {
|
|
@@ -1365,7 +1118,7 @@ async function handleDebiasValidate(argv) {
|
|
|
1365
1118
|
console.error('Cannot find samples path. Pass --samples <path> or ensure report has request.samplesPath.');
|
|
1366
1119
|
process.exit(1);
|
|
1367
1120
|
}
|
|
1368
|
-
const { loadSamples } = await import('
|
|
1121
|
+
const { loadSamples } = await import('../inputs/load-samples.js');
|
|
1369
1122
|
const { samples } = loadSamples(samplesPath);
|
|
1370
1123
|
const judgeModel = values['judge-model']
|
|
1371
1124
|
?? report.meta?.judgeModel;
|
|
@@ -1374,9 +1127,9 @@ async function handleDebiasValidate(argv) {
|
|
|
1374
1127
|
process.exit(1);
|
|
1375
1128
|
}
|
|
1376
1129
|
process.stderr.write(tCli('cli.debias.warn_cost_doubles', lang));
|
|
1377
|
-
const { createExecutor } = await import('
|
|
1130
|
+
const { createExecutor } = await import('../executors/index.js');
|
|
1378
1131
|
const judgeExecutor = createExecutor(values['judge-executor']);
|
|
1379
|
-
const { validateLengthDebias, formatDebiasValidate } = await import('
|
|
1132
|
+
const { validateLengthDebias, formatDebiasValidate } = await import('../grading/debias-validate.js');
|
|
1380
1133
|
const seedVal = values.seed != null ? Number(values.seed) : undefined;
|
|
1381
1134
|
const bsRaw = Number(values['bootstrap-samples']) || 1000;
|
|
1382
1135
|
const result = await validateLengthDebias({
|
|
@@ -1412,7 +1165,7 @@ async function handleSaturation(argv) {
|
|
|
1412
1165
|
},
|
|
1413
1166
|
strict: false,
|
|
1414
1167
|
});
|
|
1415
|
-
const { createFileStore } = await import('
|
|
1168
|
+
const { createFileStore } = await import('../server/report-store.js');
|
|
1416
1169
|
const store = createFileStore(resolve(values['reports-dir']));
|
|
1417
1170
|
const report = await store.get(reportId);
|
|
1418
1171
|
if (!report) {
|
|
@@ -1463,7 +1216,7 @@ async function handleSaturation(argv) {
|
|
|
1463
1216
|
console.log('');
|
|
1464
1217
|
}
|
|
1465
1218
|
// ---------------------------------------------------------------------------
|
|
1466
|
-
// handleVerdict — one-line ship/no-ship verdict
|
|
1219
|
+
// handleVerdict — one-line ship/no-ship verdict
|
|
1467
1220
|
// ---------------------------------------------------------------------------
|
|
1468
1221
|
async function handleVerdict(argv) {
|
|
1469
1222
|
const lang = langFromArgv(argv);
|
|
@@ -1483,14 +1236,14 @@ async function handleVerdict(argv) {
|
|
|
1483
1236
|
},
|
|
1484
1237
|
strict: false,
|
|
1485
1238
|
});
|
|
1486
|
-
const { createFileStore } = await import('
|
|
1239
|
+
const { createFileStore } = await import('../server/report-store.js');
|
|
1487
1240
|
const store = createFileStore(resolve(values['reports-dir']));
|
|
1488
1241
|
const report = await store.get(reportId);
|
|
1489
1242
|
if (!report) {
|
|
1490
1243
|
console.error(tCli('cli.common.report_not_found', lang, { id: reportId }));
|
|
1491
1244
|
process.exit(1);
|
|
1492
1245
|
}
|
|
1493
|
-
const { computeVerdict, formatVerdictText } = await import('
|
|
1246
|
+
const { computeVerdict, formatVerdictText } = await import('../eval-core/verdict.js');
|
|
1494
1247
|
const result = computeVerdict(report, {
|
|
1495
1248
|
gateThreshold: values.threshold != null ? Number(values.threshold) : undefined,
|
|
1496
1249
|
triviallySmallDiff: values['trivial-diff'] != null ? Number(values['trivial-diff']) : undefined,
|
|
@@ -1532,7 +1285,7 @@ async function handleDiagnose(argv) {
|
|
|
1532
1285
|
},
|
|
1533
1286
|
strict: false,
|
|
1534
1287
|
});
|
|
1535
|
-
const { createFileStore } = await import('
|
|
1288
|
+
const { createFileStore } = await import('../server/report-store.js');
|
|
1536
1289
|
const store = createFileStore(resolve(values['reports-dir']));
|
|
1537
1290
|
const report = await store.get(reportId);
|
|
1538
1291
|
if (!report) {
|
|
@@ -1547,7 +1300,7 @@ async function handleDiagnose(argv) {
|
|
|
1547
1300
|
const samplesPath = values.samples ?? report.meta?.request?.samplesPath;
|
|
1548
1301
|
if (samplesPath && existsSync(samplesPath)) {
|
|
1549
1302
|
try {
|
|
1550
|
-
const { loadSamples } = await import('
|
|
1303
|
+
const { loadSamples } = await import('../inputs/load-samples.js');
|
|
1551
1304
|
samples = loadSamples(samplesPath).samples;
|
|
1552
1305
|
}
|
|
1553
1306
|
catch (err) {
|
|
@@ -1558,7 +1311,7 @@ async function handleDiagnose(argv) {
|
|
|
1558
1311
|
}
|
|
1559
1312
|
const topRaw = Number(values.top);
|
|
1560
1313
|
const topN = Number.isFinite(topRaw) && topRaw > 0 ? topRaw : undefined;
|
|
1561
|
-
const { diagnoseSamples, formatSampleDiagnostics } = await import('
|
|
1314
|
+
const { diagnoseSamples, formatSampleDiagnostics } = await import('../analysis/sample-diagnostics.js');
|
|
1562
1315
|
const diag = diagnoseSamples(report, {
|
|
1563
1316
|
samples,
|
|
1564
1317
|
duplicateRouge: values['duplicate-rouge'] != null ? Number(values['duplicate-rouge']) : undefined,
|
|
@@ -1568,6 +1321,14 @@ async function handleDiagnose(argv) {
|
|
|
1568
1321
|
flatThreshold: values.flat != null ? Number(values.flat) : undefined,
|
|
1569
1322
|
});
|
|
1570
1323
|
console.log(formatSampleDiagnostics(diag, { topN }));
|
|
1324
|
+
// Sample design science coverage block. Render after diagnose 主体,因为
|
|
1325
|
+
// coverage 是声明式元数据(capability/difficulty/construct/provenance)的整体分布,
|
|
1326
|
+
// 跟 issue list 是不同视角的两件事。优先从 samples (现场加载) 算,fallback 到
|
|
1327
|
+
// report.analysis.sampleQuality(报告里持久化的数据)。
|
|
1328
|
+
const { renderSampleDesignCoverage } = await import('./coverage-renderer.js');
|
|
1329
|
+
const coverageBlock = renderSampleDesignCoverage(samples, report.analysis?.sampleQuality, lang);
|
|
1330
|
+
if (coverageBlock)
|
|
1331
|
+
console.log(coverageBlock);
|
|
1571
1332
|
// Exit code: 0 if health ≥ 70 and no errors; 1 otherwise. CI-friendly.
|
|
1572
1333
|
if (diag.totals.errors === 0 && diag.healthScore >= 70) {
|
|
1573
1334
|
process.exit(0);
|
|
@@ -1597,7 +1358,7 @@ async function handleFailures(argv) {
|
|
|
1597
1358
|
},
|
|
1598
1359
|
strict: false,
|
|
1599
1360
|
});
|
|
1600
|
-
const { createFileStore } = await import('
|
|
1361
|
+
const { createFileStore } = await import('../server/report-store.js');
|
|
1601
1362
|
const store = createFileStore(resolve(values['reports-dir']));
|
|
1602
1363
|
const report = await store.get(reportId);
|
|
1603
1364
|
if (!report) {
|
|
@@ -1609,9 +1370,9 @@ async function handleFailures(argv) {
|
|
|
1609
1370
|
console.error(tCli('cli.common.no_judge_model', lang));
|
|
1610
1371
|
process.exit(1);
|
|
1611
1372
|
}
|
|
1612
|
-
const { createExecutor } = await import('
|
|
1373
|
+
const { createExecutor } = await import('../executors/index.js');
|
|
1613
1374
|
const executor = createExecutor(values['judge-executor']);
|
|
1614
|
-
const { clusterFailures, formatFailureClusterReport } = await import('
|
|
1375
|
+
const { clusterFailures, formatFailureClusterReport } = await import('../analysis/failure-clusterer.js');
|
|
1615
1376
|
const out = await clusterFailures({
|
|
1616
1377
|
report: report,
|
|
1617
1378
|
executor,
|
|
@@ -1626,4 +1387,4 @@ async function handleFailures(argv) {
|
|
|
1626
1387
|
// Entry
|
|
1627
1388
|
// ---------------------------------------------------------------------------
|
|
1628
1389
|
main();
|
|
1629
|
-
//# sourceMappingURL=
|
|
1390
|
+
//# sourceMappingURL=index.js.map
|