oh-my-knowledge 0.21.0 → 0.23.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (122) hide show
  1. package/README.md +3 -0
  2. package/README.zh.md +2 -0
  3. package/dist/src/analysis/report-diagnostics.d.ts +26 -2
  4. package/dist/src/analysis/report-diagnostics.d.ts.map +1 -1
  5. package/dist/src/analysis/report-diagnostics.js +96 -3
  6. package/dist/src/analysis/report-diagnostics.js.map +1 -1
  7. package/dist/src/analysis/sample-diagnostics.d.ts +5 -1
  8. package/dist/src/analysis/sample-diagnostics.d.ts.map +1 -1
  9. package/dist/src/analysis/sample-diagnostics.js +105 -0
  10. package/dist/src/analysis/sample-diagnostics.js.map +1 -1
  11. package/dist/src/authoring/evolver.d.ts +2 -2
  12. package/dist/src/authoring/evolver.d.ts.map +1 -1
  13. package/dist/src/authoring/evolver.js +24 -7
  14. package/dist/src/authoring/evolver.js.map +1 -1
  15. package/dist/src/authoring/generator.d.ts +24 -0
  16. package/dist/src/authoring/generator.d.ts.map +1 -1
  17. package/dist/src/authoring/generator.js +67 -5
  18. package/dist/src/authoring/generator.js.map +1 -1
  19. package/dist/src/cli/coverage-renderer.d.ts +15 -0
  20. package/dist/src/cli/coverage-renderer.d.ts.map +1 -0
  21. package/dist/src/cli/coverage-renderer.js +74 -0
  22. package/dist/src/cli/coverage-renderer.js.map +1 -0
  23. package/dist/src/cli/i18n-dict.d.ts +1 -1
  24. package/dist/src/cli/i18n-dict.d.ts.map +1 -1
  25. package/dist/src/cli/i18n-dict.js +42 -2
  26. package/dist/src/cli/i18n-dict.js.map +1 -1
  27. package/dist/src/cli/index.d.ts +3 -0
  28. package/dist/src/cli/index.d.ts.map +1 -0
  29. package/dist/src/{cli.js → cli/index.js} +54 -293
  30. package/dist/src/cli/index.js.map +1 -0
  31. package/dist/src/cli/parse-run-config.d.ts +56 -0
  32. package/dist/src/cli/parse-run-config.d.ts.map +1 -0
  33. package/dist/src/cli/parse-run-config.js +195 -0
  34. package/dist/src/cli/parse-run-config.js.map +1 -0
  35. package/dist/src/cli/progress.d.ts +24 -0
  36. package/dist/src/cli/progress.d.ts.map +1 -0
  37. package/dist/src/cli/progress.js +55 -0
  38. package/dist/src/cli/progress.js.map +1 -0
  39. package/dist/src/cli/update-check.d.ts +3 -0
  40. package/dist/src/cli/update-check.d.ts.map +1 -0
  41. package/dist/src/cli/update-check.js +37 -0
  42. package/dist/src/cli/update-check.js.map +1 -0
  43. package/dist/src/eval-core/cache.d.ts +6 -2
  44. package/dist/src/eval-core/cache.d.ts.map +1 -1
  45. package/dist/src/eval-core/cache.js +14 -4
  46. package/dist/src/eval-core/cache.js.map +1 -1
  47. package/dist/src/eval-core/dependency-checker.d.ts +1 -1
  48. package/dist/src/eval-core/dependency-checker.js +1 -1
  49. package/dist/src/eval-core/evaluation-execution.d.ts +1 -1
  50. package/dist/src/eval-core/evaluation-execution.d.ts.map +1 -1
  51. package/dist/src/eval-core/evaluation-execution.js +6 -4
  52. package/dist/src/eval-core/evaluation-execution.js.map +1 -1
  53. package/dist/src/eval-core/evaluation-reporting.d.ts.map +1 -1
  54. package/dist/src/eval-core/evaluation-reporting.js +4 -0
  55. package/dist/src/eval-core/evaluation-reporting.js.map +1 -1
  56. package/dist/src/eval-core/execution-strategy.d.ts.map +1 -1
  57. package/dist/src/eval-core/execution-strategy.js +35 -2
  58. package/dist/src/eval-core/execution-strategy.js.map +1 -1
  59. package/dist/src/eval-core/schema.d.ts.map +1 -1
  60. package/dist/src/eval-core/schema.js +16 -2
  61. package/dist/src/eval-core/schema.js.map +1 -1
  62. package/dist/src/eval-core/verdict.d.ts +3 -3
  63. package/dist/src/eval-core/verdict.js +3 -3
  64. package/dist/src/eval-workflows/each-evaluation-workflow.d.ts +8 -1
  65. package/dist/src/eval-workflows/each-evaluation-workflow.d.ts.map +1 -1
  66. package/dist/src/eval-workflows/each-evaluation-workflow.js +3 -2
  67. package/dist/src/eval-workflows/each-evaluation-workflow.js.map +1 -1
  68. package/dist/src/eval-workflows/evaluation-pipeline.d.ts +16 -2
  69. package/dist/src/eval-workflows/evaluation-pipeline.d.ts.map +1 -1
  70. package/dist/src/eval-workflows/evaluation-pipeline.js +50 -4
  71. package/dist/src/eval-workflows/evaluation-pipeline.js.map +1 -1
  72. package/dist/src/eval-workflows/evaluation-preparation.d.ts +5 -1
  73. package/dist/src/eval-workflows/evaluation-preparation.d.ts.map +1 -1
  74. package/dist/src/eval-workflows/evaluation-preparation.js +2 -2
  75. package/dist/src/eval-workflows/evaluation-preparation.js.map +1 -1
  76. package/dist/src/eval-workflows/run-evaluation.d.ts +14 -3
  77. package/dist/src/eval-workflows/run-evaluation.d.ts.map +1 -1
  78. package/dist/src/eval-workflows/run-evaluation.js +25 -3
  79. package/dist/src/eval-workflows/run-evaluation.js.map +1 -1
  80. package/dist/src/executors/claude-cli.d.ts +1 -1
  81. package/dist/src/executors/claude-cli.d.ts.map +1 -1
  82. package/dist/src/executors/claude-cli.js +21 -1
  83. package/dist/src/executors/claude-cli.js.map +1 -1
  84. package/dist/src/executors/claude-sdk.d.ts +14 -1
  85. package/dist/src/executors/claude-sdk.d.ts.map +1 -1
  86. package/dist/src/executors/claude-sdk.js +19 -1
  87. package/dist/src/executors/claude-sdk.js.map +1 -1
  88. package/dist/src/executors/script.d.ts.map +1 -1
  89. package/dist/src/executors/script.js +10 -1
  90. package/dist/src/executors/script.js.map +1 -1
  91. package/dist/src/grading/assertions.js +3 -3
  92. package/dist/src/grading/assertions.js.map +1 -1
  93. package/dist/src/grading/gold-dataset.d.ts +1 -1
  94. package/dist/src/grading/gold-dataset.js +1 -1
  95. package/dist/src/inputs/eval-config.js +20 -1
  96. package/dist/src/inputs/eval-config.js.map +1 -1
  97. package/dist/src/inputs/load-samples.d.ts.map +1 -1
  98. package/dist/src/inputs/load-samples.js +30 -4
  99. package/dist/src/inputs/load-samples.js.map +1 -1
  100. package/dist/src/inputs/skill-loader.d.ts +10 -1
  101. package/dist/src/inputs/skill-loader.d.ts.map +1 -1
  102. package/dist/src/inputs/skill-loader.js +18 -1
  103. package/dist/src/inputs/skill-loader.js.map +1 -1
  104. package/dist/src/renderer/html-renderer.js +2 -2
  105. package/dist/src/renderer/html-renderer.js.map +1 -1
  106. package/dist/src/renderer/summary.d.ts.map +1 -1
  107. package/dist/src/renderer/summary.js +0 -13
  108. package/dist/src/renderer/summary.js.map +1 -1
  109. package/dist/src/renderer/trends.js +1 -1
  110. package/dist/src/renderer/trends.js.map +1 -1
  111. package/dist/src/server/report-server.js +15 -15
  112. package/dist/src/server/report-server.js.map +1 -1
  113. package/dist/src/types/eval.d.ts +30 -2
  114. package/dist/src/types/eval.d.ts.map +1 -1
  115. package/dist/src/types/executor.d.ts +1 -0
  116. package/dist/src/types/executor.d.ts.map +1 -1
  117. package/dist/src/types/report.d.ts +38 -2
  118. package/dist/src/types/report.d.ts.map +1 -1
  119. package/package.json +14 -6
  120. package/dist/src/cli.d.ts +0 -3
  121. package/dist/src/cli.d.ts.map +0 -1
  122. package/dist/src/cli.js.map +0 -1
@@ -1,208 +1,12 @@
1
1
  #!/usr/bin/env node
2
2
  import { parseArgs } from 'node:util';
3
3
  import { resolve } from 'node:path';
4
- import { homedir } from 'node:os';
5
4
  import { join } from 'node:path';
6
5
  import { existsSync } from 'node:fs';
7
- import { tCli, getCliLang, parseLangFromArgv, langFromArgv } from './cli/i18n.js';
8
- import { discoverVariants, parseVariantCwd } from './inputs/skill-loader.js';
9
- import { loadEvalConfig, configVariantsToSpecs } from './inputs/eval-config.js';
10
- // ---------------------------------------------------------------------------
11
- // Constants
12
- // ---------------------------------------------------------------------------
13
- const DEFAULT_REPORTS_DIR = join(homedir(), '.oh-my-knowledge', 'reports');
14
- // Shared CLI options for run/ci commands.
15
- // Defaults are applied inside parseRunConfig (after config-file merge) so that
16
- // CLI `undefined` can be reliably distinguished from "user passed the default value".
17
- // Priority order resolved in parseRunConfig: CLI arg > --config file > hard-coded default.
18
- /**
19
- * 所有子命令都接受的通用 flag。新增 --lang 让 parseArgs strict:false 模式下
20
- * 仍能把值类型化到 values.lang 上(否则未声明的 flag 会被丢弃)。
21
- */
22
- const COMMON_OPTIONS = {
23
- lang: { type: 'string' },
24
- };
25
- const RUN_OPTIONS = {
26
- ...COMMON_OPTIONS,
27
- samples: { type: 'string' },
28
- 'skill-dir': { type: 'string' },
29
- control: { type: 'string' },
30
- treatment: { type: 'string' },
31
- config: { type: 'string' },
32
- model: { type: 'string' },
33
- 'judge-model': { type: 'string' },
34
- 'output-dir': { type: 'string' },
35
- 'no-judge': { type: 'boolean' },
36
- 'no-cache': { type: 'boolean' },
37
- 'dry-run': { type: 'boolean' },
38
- concurrency: { type: 'string' },
39
- timeout: { type: 'string' },
40
- executor: { type: 'string' },
41
- 'judge-executor': { type: 'string' },
42
- each: { type: 'boolean' },
43
- 'skip-preflight': { type: 'boolean' },
44
- 'mcp-config': { type: 'string' },
45
- 'no-serve': { type: 'boolean' },
46
- verbose: { type: 'boolean' },
47
- retry: { type: 'string' },
48
- resume: { type: 'string' },
49
- 'layered-stats': { type: 'boolean' },
50
- };
51
- // ---------------------------------------------------------------------------
52
- // parseRunConfig
53
- // ---------------------------------------------------------------------------
54
- function parseRunConfig(argv, extraOptions = {}) {
55
- const { values } = parseArgs({
56
- args: argv,
57
- options: { ...RUN_OPTIONS, ...extraOptions },
58
- strict: false,
59
- });
60
- if (values.variants !== undefined) {
61
- throw new Error(`--variants 已在 v0.16 废除,请改用 --control <expr> 与 --treatment <v1,v2,...>\n`
62
- + ` 迁移示例:--variants baseline,my-skill → --control baseline --treatment my-skill\n`
63
- + ` 复杂场景可用 --config eval.yaml(参见 docs/terminology-spec.md)`);
64
- }
65
- // 1) Load --config (if provided). All subsequent fields fall back to it when CLI is silent.
66
- const evalConfig = values.config
67
- ? loadEvalConfig(values.config)
68
- : null;
69
- // 2) Resolve samples path: CLI > config > auto-detect .json/.yaml/.yml in cwd.
70
- const cliSamples = values.samples;
71
- let samplesFile;
72
- if (cliSamples) {
73
- samplesFile = cliSamples;
74
- }
75
- else if (evalConfig?.samples) {
76
- samplesFile = evalConfig.samples; // already resolved against config file dir
77
- }
78
- else {
79
- samplesFile = 'eval-samples.json';
80
- if (!existsSync(resolve(samplesFile))) {
81
- if (existsSync(resolve('eval-samples.yaml')))
82
- samplesFile = 'eval-samples.yaml';
83
- else if (existsSync(resolve('eval-samples.yml')))
84
- samplesFile = 'eval-samples.yml';
85
- }
86
- }
87
- const skillDir = resolve(values['skill-dir'] ?? 'skills');
88
- // 3) Resolve variantSpecs: CLI > config. If neither, error with a helpful hint.
89
- const controlExpr = values.control;
90
- const treatmentExprs = values.treatment
91
- ? values.treatment.split(',').map((v) => v.trim()).filter(Boolean)
92
- : [];
93
- let variantSpecs;
94
- if (controlExpr || treatmentExprs.length > 0) {
95
- // CLI roles present → CLI entirely replaces config.variants (no merging).
96
- variantSpecs = [];
97
- if (controlExpr) {
98
- variantSpecs.push({ name: parseVariantCwd(controlExpr).name, role: 'control', expr: controlExpr });
99
- }
100
- for (const expr of treatmentExprs) {
101
- variantSpecs.push({ name: parseVariantCwd(expr).name, role: 'treatment', expr });
102
- }
103
- }
104
- else if (evalConfig) {
105
- variantSpecs = configVariantsToSpecs(evalConfig.variants);
106
- }
107
- else if (values.each) {
108
- // --each 模式自动用 baseline (control) vs 每个 skill (treatment),
109
- // 不需要用户显式传 --control / --treatment,校验跳过。
110
- variantSpecs = [];
111
- }
112
- else {
113
- const discovered = discoverVariants(skillDir);
114
- const hint = discovered.length > 0 ? `\n skill-dir (${skillDir}) 下发现的候选:${discovered.join(', ')}` : '';
115
- throw new Error(`请通过 --control / --treatment 或 --config eval.yaml 声明 variant 角色。\n`
116
- + ` 示例:omk bench run --control baseline --treatment my-skill${hint}\n`
117
- + ` --each 模式下自动用 baseline vs 每个 skill,无需显式声明\n`
118
- + ` 术语见 docs/terminology-spec.md(v0.16 起废除 --variants,改用 experiment role 显式声明)`);
119
- }
120
- const seenNames = new Set();
121
- for (const spec of variantSpecs) {
122
- if (seenNames.has(spec.name)) {
123
- throw new Error(`variant "${spec.name}" 重复出现——同一 variant 不能同时属于 --control 与 --treatment,也不能在 --treatment 中重复。`);
124
- }
125
- seenNames.add(spec.name);
126
- }
127
- // 4) Apply CLI > config > hard-coded default for all other fields.
128
- const executorName = values.executor ?? evalConfig?.executor ?? 'claude';
129
- const judgeExecutorName = values['judge-executor'] ?? evalConfig?.judgeExecutor ?? executorName;
130
- const model = values.model ?? evalConfig?.model ?? 'sonnet';
131
- const judgeModelRaw = values['judge-model'] !== undefined
132
- ? values['judge-model']
133
- : evalConfig?.judgeModel ?? 'haiku';
134
- const judgeModel = judgeModelRaw ?? 'haiku';
135
- const outputDir = resolve(values['output-dir'] ?? DEFAULT_REPORTS_DIR);
136
- const concurrencyRaw = values.concurrency !== undefined
137
- ? Number(values.concurrency)
138
- : evalConfig?.concurrency ?? 1;
139
- const concurrency = Math.max(1, Number(concurrencyRaw) || 1);
140
- const timeoutSec = values.timeout !== undefined
141
- ? Number(values.timeout)
142
- : evalConfig?.timeoutMs
143
- ? evalConfig.timeoutMs / 1000
144
- : 120;
145
- const timeoutMs = Math.max(1, Number(timeoutSec) || 120) * 1000;
146
- const noJudge = values['no-judge'] ?? false;
147
- const noCache = values['no-cache'] ?? evalConfig?.noCache ?? false;
148
- const dryRun = values['dry-run'] ?? false;
149
- const skipPreflight = values['skip-preflight'] ?? false;
150
- const mcpConfig = values['mcp-config'] ?? evalConfig?.mcpConfig;
151
- const verbose = values.verbose ?? false;
152
- const retry = Math.max(0, Number(values.retry ?? 0) || 0);
153
- const resume = values.resume;
154
- const blind = values.blind ?? evalConfig?.blind ?? false;
155
- const layeredStats = values['layered-stats'] ?? false;
156
- return {
157
- values,
158
- config: {
159
- samplesPath: resolve(samplesFile),
160
- skillDir,
161
- variantSpecs,
162
- model,
163
- judgeModel,
164
- outputDir,
165
- noJudge,
166
- noCache,
167
- dryRun,
168
- concurrency,
169
- timeoutMs,
170
- executorName,
171
- judgeExecutorName,
172
- skipPreflight,
173
- mcpConfig,
174
- verbose,
175
- retry,
176
- resume,
177
- blind,
178
- layeredStats,
179
- budget: evalConfig?.budget,
180
- },
181
- };
182
- }
183
- // ---------------------------------------------------------------------------
184
- // Update check
185
- // ---------------------------------------------------------------------------
186
- async function checkUpdate(lang) {
187
- try {
188
- const { readFileSync } = await import('node:fs');
189
- const { fileURLToPath } = await import('node:url');
190
- const { dirname, join } = await import('node:path');
191
- const __dirname = dirname(fileURLToPath(import.meta.url));
192
- const pkg = JSON.parse(readFileSync(join(__dirname, 'package.json'), 'utf-8'));
193
- const registry = pkg.publishConfig?.registry || 'https://registry.npmjs.org';
194
- const res = await fetch(`${registry}/${pkg.name}/latest`, { signal: AbortSignal.timeout(3000) });
195
- if (!res.ok)
196
- return;
197
- const data = await res.json();
198
- if (data.version && data.version !== pkg.version) {
199
- process.stderr.write(tCli('cli.update.new_version_available', lang, {
200
- old: pkg.version, new: data.version, pkg: pkg.name,
201
- }));
202
- }
203
- }
204
- catch { /* 静默失败,不影响正常使用 */ }
205
- }
6
+ import { tCli, getCliLang, parseLangFromArgv, langFromArgv } from './i18n.js';
7
+ import { parseRunConfig, DEFAULT_REPORTS_DIR, COMMON_OPTIONS, } from './parse-run-config.js';
8
+ import { makeOnProgress } from './progress.js';
9
+ import { checkUpdate } from './update-check.js';
206
10
  // ---------------------------------------------------------------------------
207
11
  // Main
208
12
  // ---------------------------------------------------------------------------
@@ -279,66 +83,13 @@ async function main() {
279
83
  * Factory: 闭住 lang, 返回 onProgress callback。evaluation engine 回调时不传
280
84
  * 上下文, 所以 lang 必须在 handler 入口处通过 closure 传进来。
281
85
  */
282
- function makeOnProgress(lang) {
283
- return ({ phase, completed, total, sample_id, variant, durationMs, inputTokens, outputTokens, costUSD, score, outputPreview, judgePhase: _judgePhase, judgeDim, skipped, attempt, maxAttempts, error, }) => {
284
- const ctx = { i: completed ?? '', n: total ?? '', sample: sample_id ?? '', variant: variant ?? '' };
285
- if (phase === 'preflight') {
286
- process.stderr.write(tCli('cli.progress.preflight_starting', lang));
287
- return;
288
- }
289
- if (phase === 'retry') {
290
- process.stderr.write(tCli('cli.progress.sample_retry', lang, {
291
- ...ctx, attempt: attempt ?? '', max: maxAttempts ?? '',
292
- }));
293
- return;
294
- }
295
- if (phase === 'error') {
296
- process.stderr.write(tCli('cli.progress.sample_error', lang, { ...ctx, error: error ?? '' }));
297
- return;
298
- }
299
- if (phase === 'start') {
300
- process.stderr.write(tCli('cli.progress.sample_executing', lang, ctx));
301
- }
302
- else if (phase === 'exec_done') {
303
- const cost = costUSD != null && costUSD > 0 ? ` $${costUSD.toFixed(4)}` : '';
304
- process.stderr.write(tCli('cli.progress.sample_exec_done', lang, {
305
- ...ctx, ms: durationMs ?? '', input: inputTokens ?? '', output: outputTokens ?? '', cost,
306
- }));
307
- if (outputPreview) {
308
- process.stderr.write(tCli('cli.progress.output_preview', lang, {
309
- preview: outputPreview.slice(0, 150).replace(/\n/g, ' '),
310
- }));
311
- }
312
- }
313
- else if (phase === 'grading') {
314
- const dim = judgeDim ? ` [${judgeDim}]` : '';
315
- process.stderr.write(tCli('cli.progress.judging', lang, { ...ctx, dim }));
316
- }
317
- else if (phase === 'judge_done') {
318
- const dim = judgeDim ? ` [${judgeDim}]` : '';
319
- process.stderr.write(tCli('cli.progress.judged', lang, { ...ctx, dim, score: score ?? '' }));
320
- }
321
- else if (phase === 'done' && skipped) {
322
- if (sample_id)
323
- process.stderr.write(tCli('cli.progress.skipped', lang, ctx));
324
- }
325
- else {
326
- const cost = costUSD != null && costUSD > 0 ? ` $${costUSD.toFixed(4)}` : '';
327
- const scoreInfo = typeof score === 'number' ? ` score=${score}` : '';
328
- process.stderr.write(tCli('cli.progress.sample_done', lang, {
329
- ...ctx, ms: durationMs ?? '', input: inputTokens ?? '', output: outputTokens ?? '',
330
- cost, score: scoreInfo,
331
- }));
332
- }
333
- };
334
- }
335
86
  // ---------------------------------------------------------------------------
336
87
  // handleRun
337
88
  // ---------------------------------------------------------------------------
338
89
  async function handleRun(argv) {
339
90
  const lang = langFromArgv(argv);
340
91
  const { values, config } = parseRunConfig(argv, {
341
- blind: { type: 'boolean', default: false },
92
+ blind: { type: 'boolean' },
342
93
  repeat: { type: 'string', default: '1' },
343
94
  'judge-repeat': { type: 'string', default: '1' },
344
95
  'judge-models': { type: 'string' },
@@ -350,8 +101,10 @@ async function handleRun(argv) {
350
101
  'budget-per-sample-usd': { type: 'string' },
351
102
  'budget-per-sample-ms': { type: 'string' },
352
103
  });
353
- const { runEvaluation, runMultiple, runEachEvaluation } = await import('./eval-workflows/run-evaluation.js');
354
- config.blind = values.blind;
104
+ const { runEvaluation, runMultiple, runEachEvaluation } = await import('../eval-workflows/run-evaluation.js');
105
+ if (values.blind !== undefined) {
106
+ config.blind = values.blind;
107
+ }
355
108
  config.onProgress = makeOnProgress(lang);
356
109
  // --repeat 输入校验: 非 ≥1 整数时提示并钳到 1, 不静默掩盖用户错字 / 极端输入。
357
110
  // 提前到 --each 分支之前, 保证 each 模式也能读到 repeat (曾经 bug: --each 吞 --repeat)。
@@ -394,7 +147,7 @@ async function handleRun(argv) {
394
147
  }
395
148
  }
396
149
  // --budget-usd / --budget-per-sample-usd / --budget-per-sample-ms:
397
- // v0.22 hard budget caps. CLI flags override config-file values. When the
150
+ // hard budget caps. CLI flags override config-file values. When the
398
151
  // total-USD cap is exceeded mid-run, remaining tasks are skipped and a
399
152
  // partial report is persisted with meta.budgetExhausted=true.
400
153
  const budgetUSD = values['budget-usd'] != null ? Number(values['budget-usd']) : undefined;
@@ -447,10 +200,10 @@ async function handleRun(argv) {
447
200
  process.stderr.write(tCli('cli.run.batch_complete', lang));
448
201
  process.stderr.write(tCli('cli.run.report_saved', lang, { path: filePath }));
449
202
  if (!values['no-serve'] && process.stdout.isTTY) {
450
- const { createReportServer } = await import('./server/report-server.js');
203
+ const { createReportServer } = await import('../server/report-server.js');
451
204
  const server = createReportServer({ reportsDir: config.outputDir });
452
205
  const serverUrl = await server.start();
453
- const reportUrl = `${serverUrl}/run/${report.id}`;
206
+ const reportUrl = `${serverUrl}/reports/${report.id}`;
454
207
  process.stderr.write(tCli('cli.run.report_server_running', lang, { url: serverUrl }));
455
208
  process.stderr.write(tCli('cli.run.report_server_view', lang, { url: reportUrl }));
456
209
  process.stderr.write(tCli('cli.run.report_server_stop', lang));
@@ -487,7 +240,7 @@ async function handleRun(argv) {
487
240
  // --gold-dir: compute α/κ/Pearson against gold annotations and re-persist.
488
241
  const goldDir = values['gold-dir'];
489
242
  if (goldDir && filePath) {
490
- const { attachGoldAgreementToReport, formatGoldCompare } = await import('./grading/gold-cli.js');
243
+ const { attachGoldAgreementToReport, formatGoldCompare } = await import('../grading/gold-cli.js');
491
244
  const out = attachGoldAgreementToReport({
492
245
  report,
493
246
  goldDir,
@@ -515,12 +268,12 @@ async function handleRun(argv) {
515
268
  process.stderr.write(tCli('cli.run.report_saved', lang, { path: filePath }));
516
269
  if (!values['no-serve'] && process.stdout.isTTY) {
517
270
  // Auto-start report server
518
- const { createReportServer } = await import('./server/report-server.js');
271
+ const { createReportServer } = await import('../server/report-server.js');
519
272
  const server = createReportServer({
520
273
  reportsDir: config.outputDir,
521
274
  });
522
275
  const serverUrl = await server.start();
523
- const reportUrl = `${serverUrl}/run/${report.id}`;
276
+ const reportUrl = `${serverUrl}/reports/${report.id}`;
524
277
  process.stderr.write(tCli('cli.run.report_server_running', lang, { url: serverUrl }));
525
278
  process.stderr.write(tCli('cli.run.report_server_view', lang, { url: reportUrl }));
526
279
  process.stderr.write(tCli('cli.run.report_server_stop', lang));
@@ -576,8 +329,8 @@ async function handleReport(argv) {
576
329
  return;
577
330
  }
578
331
  if (values.export) {
579
- const { createFileStore } = await import('./server/report-store.js');
580
- const { renderRunDetail, renderEachRunDetail } = await import('./renderer/html-renderer.js');
332
+ const { createFileStore } = await import('../server/report-store.js');
333
+ const { renderRunDetail, renderEachRunDetail } = await import('../renderer/html-renderer.js');
581
334
  const { writeFileSync } = await import('node:fs');
582
335
  const store = createFileStore(resolve(values['reports-dir']));
583
336
  const report = await store.get(values.export);
@@ -592,7 +345,7 @@ async function handleReport(argv) {
592
345
  console.log('Open in browser, or Ctrl+P to save as PDF');
593
346
  return;
594
347
  }
595
- const { createReportServer } = await import('./server/report-server.js');
348
+ const { createReportServer } = await import('../server/report-server.js');
596
349
  const server = createReportServer({
597
350
  port: Number(values.port),
598
351
  reportsDir: resolve(values['reports-dir']),
@@ -715,7 +468,7 @@ async function handleAnalyze(argv) {
715
468
  const to = values.to;
716
469
  const skills = values.skills ? values.skills.split(',').map((s) => s.trim()).filter(Boolean) : undefined;
717
470
  console.log(`[omk] analyzing ${tracePath}...`);
718
- const { computeSkillHealthReport } = await import('./observability/skill-health-analyzer.js');
471
+ const { computeSkillHealthReport } = await import('../observability/skill-health-analyzer.js');
719
472
  const report = computeSkillHealthReport(tracePath, {
720
473
  kbRoot: values.kb ? resolve(values.kb) : undefined,
721
474
  from,
@@ -776,7 +529,7 @@ async function handleGenSamples(argv) {
776
529
  strict: false,
777
530
  allowPositionals: true,
778
531
  });
779
- const { generateSamples } = await import('./authoring/generator.js');
532
+ const { generateSamples } = await import('../authoring/generator.js');
780
533
  const { readFileSync, writeFileSync } = await import('node:fs');
781
534
  const count = Math.max(1, Number(values.count) || 5);
782
535
  const model = values.model;
@@ -908,7 +661,7 @@ async function handleEvolve(argv) {
908
661
  else if (existsSync(resolve('eval-samples.yml')))
909
662
  samplesFile = 'eval-samples.yml';
910
663
  }
911
- const { evolveSkill } = await import('./authoring/evolver.js');
664
+ const { evolveSkill } = await import('../authoring/evolver.js');
912
665
  process.stderr.write(tCli('cli.evolve.section_header', lang, { path: skillPath }));
913
666
  try {
914
667
  const result = await evolveSkill({
@@ -982,7 +735,7 @@ async function handleGate(argv) {
982
735
  threshold: { type: 'string', default: '3.5' },
983
736
  'trivial-diff': { type: 'string' },
984
737
  });
985
- const { runEvaluation } = await import('./eval-workflows/run-evaluation.js');
738
+ const { runEvaluation } = await import('../eval-workflows/run-evaluation.js');
986
739
  config.onProgress = makeOnProgress(lang);
987
740
  try {
988
741
  const { report } = (await runEvaluation(config));
@@ -994,7 +747,7 @@ async function handleGate(argv) {
994
747
  // bootstrap diff CI / saturation / Krippendorff α)。computeVerdict 是单一
995
748
  // 决策源, exit code 跟 verdict.level 走 — 数据 underpowered 直接 FAIL,
996
749
  // 堵住"过 PASS 就 deploy"的漏洞。
997
- const { computeVerdict, formatVerdictText } = await import('./eval-core/verdict.js');
750
+ const { computeVerdict, formatVerdictText } = await import('../eval-core/verdict.js');
998
751
  const result = computeVerdict(report, {
999
752
  gateThreshold: Number(values.threshold),
1000
753
  triviallySmallDiff: values['trivial-diff'] != null ? Number(values['trivial-diff']) : undefined,
@@ -1022,7 +775,7 @@ async function handleGate(argv) {
1022
775
  async function handleDiff(argv) {
1023
776
  const lang = langFromArgv(argv);
1024
777
  // Flag-aware split: separate positional report IDs from flags so we can support
1025
- // omk bench diff <id> — within-report sample-level (v0.22)
778
+ // omk bench diff <id> — within-report sample-level
1026
779
  // omk bench diff <id1> <id2> — cross-report variant-level (legacy)
1027
780
  // both with optional --regressions-only / --threshold / --variant flags.
1028
781
  const positional = [];
@@ -1056,7 +809,7 @@ async function handleDiff(argv) {
1056
809
  },
1057
810
  strict: false,
1058
811
  });
1059
- const { createFileStore } = await import('./server/report-store.js');
812
+ const { createFileStore } = await import('../server/report-store.js');
1060
813
  const store = createFileStore(resolve(DEFAULT_REPORTS_DIR));
1061
814
  if (positional.length === 1) {
1062
815
  await runSampleLevelDiff(positional[0], store, values, lang);
@@ -1120,7 +873,7 @@ async function handleDiff(argv) {
1120
873
  console.log('');
1121
874
  }
1122
875
  /**
1123
- * Within-report sample-level diff (v0.22). Compares two variants' scores on
876
+ * Within-report sample-level diff. Compares two variants' scores on
1124
877
  * each shared sample and surfaces the worst regressions / biggest wins.
1125
878
  *
1126
879
  * Default focus is variants[0] (control) vs variants[1] (treatment), but
@@ -1223,7 +976,7 @@ async function handleGold(argv) {
1223
976
  },
1224
977
  strict: false,
1225
978
  });
1226
- const { initGoldDataset } = await import('./grading/gold-cli.js');
979
+ const { initGoldDataset } = await import('../grading/gold-cli.js');
1227
980
  try {
1228
981
  const written = initGoldDataset(values.out, {
1229
982
  annotator: values.annotator,
@@ -1247,7 +1000,7 @@ async function handleGold(argv) {
1247
1000
  console.error(tCli('cli.common.usage_gold_validate', lang));
1248
1001
  process.exit(1);
1249
1002
  }
1250
- const { validateGoldDataset } = await import('./grading/gold-cli.js');
1003
+ const { validateGoldDataset } = await import('../grading/gold-cli.js');
1251
1004
  const result = validateGoldDataset(dir);
1252
1005
  if (result.ok) {
1253
1006
  console.log(tCli('cli.gold.validate_ok', lang, { n: result.sampleCount }));
@@ -1281,9 +1034,9 @@ async function handleGold(argv) {
1281
1034
  console.error('--gold-dir is required');
1282
1035
  process.exit(1);
1283
1036
  }
1284
- const { loadGoldDataset } = await import('./grading/gold-dataset.js');
1285
- const { compareGoldToReport, formatGoldCompare } = await import('./grading/gold-cli.js');
1286
- const { createFileStore } = await import('./server/report-store.js');
1037
+ const { loadGoldDataset } = await import('../grading/gold-dataset.js');
1038
+ const { compareGoldToReport, formatGoldCompare } = await import('../grading/gold-cli.js');
1039
+ const { createFileStore } = await import('../server/report-store.js');
1287
1040
  const { dataset, issues } = loadGoldDataset(goldDir);
1288
1041
  if (!dataset) {
1289
1042
  console.error('Cannot load gold dataset:');
@@ -1351,7 +1104,7 @@ async function handleDebiasValidate(argv) {
1351
1104
  },
1352
1105
  strict: false,
1353
1106
  });
1354
- const { createFileStore } = await import('./server/report-store.js');
1107
+ const { createFileStore } = await import('../server/report-store.js');
1355
1108
  const store = createFileStore(resolve(values['reports-dir']));
1356
1109
  const report = await store.get(reportId);
1357
1110
  if (!report) {
@@ -1365,7 +1118,7 @@ async function handleDebiasValidate(argv) {
1365
1118
  console.error('Cannot find samples path. Pass --samples <path> or ensure report has request.samplesPath.');
1366
1119
  process.exit(1);
1367
1120
  }
1368
- const { loadSamples } = await import('./inputs/load-samples.js');
1121
+ const { loadSamples } = await import('../inputs/load-samples.js');
1369
1122
  const { samples } = loadSamples(samplesPath);
1370
1123
  const judgeModel = values['judge-model']
1371
1124
  ?? report.meta?.judgeModel;
@@ -1374,9 +1127,9 @@ async function handleDebiasValidate(argv) {
1374
1127
  process.exit(1);
1375
1128
  }
1376
1129
  process.stderr.write(tCli('cli.debias.warn_cost_doubles', lang));
1377
- const { createExecutor } = await import('./executors/index.js');
1130
+ const { createExecutor } = await import('../executors/index.js');
1378
1131
  const judgeExecutor = createExecutor(values['judge-executor']);
1379
- const { validateLengthDebias, formatDebiasValidate } = await import('./grading/debias-validate.js');
1132
+ const { validateLengthDebias, formatDebiasValidate } = await import('../grading/debias-validate.js');
1380
1133
  const seedVal = values.seed != null ? Number(values.seed) : undefined;
1381
1134
  const bsRaw = Number(values['bootstrap-samples']) || 1000;
1382
1135
  const result = await validateLengthDebias({
@@ -1412,7 +1165,7 @@ async function handleSaturation(argv) {
1412
1165
  },
1413
1166
  strict: false,
1414
1167
  });
1415
- const { createFileStore } = await import('./server/report-store.js');
1168
+ const { createFileStore } = await import('../server/report-store.js');
1416
1169
  const store = createFileStore(resolve(values['reports-dir']));
1417
1170
  const report = await store.get(reportId);
1418
1171
  if (!report) {
@@ -1463,7 +1216,7 @@ async function handleSaturation(argv) {
1463
1216
  console.log('');
1464
1217
  }
1465
1218
  // ---------------------------------------------------------------------------
1466
- // handleVerdict — one-line ship/no-ship verdict (v0.22)
1219
+ // handleVerdict — one-line ship/no-ship verdict
1467
1220
  // ---------------------------------------------------------------------------
1468
1221
  async function handleVerdict(argv) {
1469
1222
  const lang = langFromArgv(argv);
@@ -1483,14 +1236,14 @@ async function handleVerdict(argv) {
1483
1236
  },
1484
1237
  strict: false,
1485
1238
  });
1486
- const { createFileStore } = await import('./server/report-store.js');
1239
+ const { createFileStore } = await import('../server/report-store.js');
1487
1240
  const store = createFileStore(resolve(values['reports-dir']));
1488
1241
  const report = await store.get(reportId);
1489
1242
  if (!report) {
1490
1243
  console.error(tCli('cli.common.report_not_found', lang, { id: reportId }));
1491
1244
  process.exit(1);
1492
1245
  }
1493
- const { computeVerdict, formatVerdictText } = await import('./eval-core/verdict.js');
1246
+ const { computeVerdict, formatVerdictText } = await import('../eval-core/verdict.js');
1494
1247
  const result = computeVerdict(report, {
1495
1248
  gateThreshold: values.threshold != null ? Number(values.threshold) : undefined,
1496
1249
  triviallySmallDiff: values['trivial-diff'] != null ? Number(values['trivial-diff']) : undefined,
@@ -1532,7 +1285,7 @@ async function handleDiagnose(argv) {
1532
1285
  },
1533
1286
  strict: false,
1534
1287
  });
1535
- const { createFileStore } = await import('./server/report-store.js');
1288
+ const { createFileStore } = await import('../server/report-store.js');
1536
1289
  const store = createFileStore(resolve(values['reports-dir']));
1537
1290
  const report = await store.get(reportId);
1538
1291
  if (!report) {
@@ -1547,7 +1300,7 @@ async function handleDiagnose(argv) {
1547
1300
  const samplesPath = values.samples ?? report.meta?.request?.samplesPath;
1548
1301
  if (samplesPath && existsSync(samplesPath)) {
1549
1302
  try {
1550
- const { loadSamples } = await import('./inputs/load-samples.js');
1303
+ const { loadSamples } = await import('../inputs/load-samples.js');
1551
1304
  samples = loadSamples(samplesPath).samples;
1552
1305
  }
1553
1306
  catch (err) {
@@ -1558,7 +1311,7 @@ async function handleDiagnose(argv) {
1558
1311
  }
1559
1312
  const topRaw = Number(values.top);
1560
1313
  const topN = Number.isFinite(topRaw) && topRaw > 0 ? topRaw : undefined;
1561
- const { diagnoseSamples, formatSampleDiagnostics } = await import('./analysis/sample-diagnostics.js');
1314
+ const { diagnoseSamples, formatSampleDiagnostics } = await import('../analysis/sample-diagnostics.js');
1562
1315
  const diag = diagnoseSamples(report, {
1563
1316
  samples,
1564
1317
  duplicateRouge: values['duplicate-rouge'] != null ? Number(values['duplicate-rouge']) : undefined,
@@ -1568,6 +1321,14 @@ async function handleDiagnose(argv) {
1568
1321
  flatThreshold: values.flat != null ? Number(values.flat) : undefined,
1569
1322
  });
1570
1323
  console.log(formatSampleDiagnostics(diag, { topN }));
1324
+ // Sample design science coverage block. Render after diagnose 主体,因为
1325
+ // coverage 是声明式元数据(capability/difficulty/construct/provenance)的整体分布,
1326
+ // 跟 issue list 是不同视角的两件事。优先从 samples (现场加载) 算,fallback 到
1327
+ // report.analysis.sampleQuality(报告里持久化的数据)。
1328
+ const { renderSampleDesignCoverage } = await import('./coverage-renderer.js');
1329
+ const coverageBlock = renderSampleDesignCoverage(samples, report.analysis?.sampleQuality, lang);
1330
+ if (coverageBlock)
1331
+ console.log(coverageBlock);
1571
1332
  // Exit code: 0 if health ≥ 70 and no errors; 1 otherwise. CI-friendly.
1572
1333
  if (diag.totals.errors === 0 && diag.healthScore >= 70) {
1573
1334
  process.exit(0);
@@ -1597,7 +1358,7 @@ async function handleFailures(argv) {
1597
1358
  },
1598
1359
  strict: false,
1599
1360
  });
1600
- const { createFileStore } = await import('./server/report-store.js');
1361
+ const { createFileStore } = await import('../server/report-store.js');
1601
1362
  const store = createFileStore(resolve(values['reports-dir']));
1602
1363
  const report = await store.get(reportId);
1603
1364
  if (!report) {
@@ -1609,9 +1370,9 @@ async function handleFailures(argv) {
1609
1370
  console.error(tCli('cli.common.no_judge_model', lang));
1610
1371
  process.exit(1);
1611
1372
  }
1612
- const { createExecutor } = await import('./executors/index.js');
1373
+ const { createExecutor } = await import('../executors/index.js');
1613
1374
  const executor = createExecutor(values['judge-executor']);
1614
- const { clusterFailures, formatFailureClusterReport } = await import('./analysis/failure-clusterer.js');
1375
+ const { clusterFailures, formatFailureClusterReport } = await import('../analysis/failure-clusterer.js');
1615
1376
  const out = await clusterFailures({
1616
1377
  report: report,
1617
1378
  executor,
@@ -1626,4 +1387,4 @@ async function handleFailures(argv) {
1626
1387
  // Entry
1627
1388
  // ---------------------------------------------------------------------------
1628
1389
  main();
1629
- //# sourceMappingURL=cli.js.map
1390
+ //# sourceMappingURL=index.js.map